+ }
+ }
/*
* Find the free block within the range.
*/
@@ -405,25 +452,58 @@ alloc_range_bias(struct drm_buddy *mm,
}
static struct drm_buddy_block *
-get_maxblock(struct drm_buddy *mm, unsigned int order)
+__drm_buddy_alloc_range_bias(struct drm_buddy *mm,
+ u64 start, u64 end,
+ unsigned int order,
+ unsigned long flags)
+{
+ struct drm_buddy_block *block;
+ bool fallback = 0;
+
+ block = __alloc_range_bias(mm, start, end, order,
+ flags, fallback);
+ if (IS_ERR(block))
+ return __alloc_range_bias(mm, start, end, order,
+ flags, !fallback);
+
+ return block;
+}
+
+static struct drm_buddy_block *
+get_maxblock(struct drm_buddy *mm, unsigned int order,
+ unsigned long flags)
{
- struct drm_buddy_block *max_block = NULL, *node;
+ struct drm_buddy_block *max_block = NULL, *block = NULL;
unsigned int i;
for (i = order; i <= mm->max_order; ++i) {
- if (!list_empty(&mm->free_list[i])) {
- node = list_last_entry(&mm->free_list[i],
- struct drm_buddy_block,
- link);
- if (!max_block) {
- max_block = node;
- continue;
+ struct drm_buddy_block *tmp_block;
+
+ list_for_each_entry_reverse(tmp_block, &mm->free_list[i],
link) {
+ if (flags & DRM_BUDDY_CLEAR_ALLOCATION) {
+ /* Find a cleared block */
+ if (!drm_buddy_block_is_clear(tmp_block))
+ continue;
+ } else {
+ if (drm_buddy_block_is_clear(tmp_block))
+ continue;
}
- if (drm_buddy_block_offset(node) >
- drm_buddy_block_offset(max_block)) {
- max_block = node;
- }
+ block = tmp_block;
+ break;
+ }
+
+ if (!block)
+ continue;
+
+ if (!max_block) {
+ max_block = block;
+ continue;
+ }
+
+ if (drm_buddy_block_offset(block) >
+ drm_buddy_block_offset(max_block)) {
+ max_block = block;
}
}
@@ -440,11 +520,35 @@ alloc_from_freelist(struct drm_buddy *mm,
int err;
if (flags & DRM_BUDDY_TOPDOWN_ALLOCATION) {
- block = get_maxblock(mm, order);
+ block = get_maxblock(mm, order, flags);
if (block)
/* Store the obtained block order */
tmp = drm_buddy_block_order(block);
} else {
+ for (tmp = order; tmp <= mm->max_order; ++tmp) {
+ struct drm_buddy_block *tmp_block;
+
+ list_for_each_entry_reverse(tmp_block,
&mm->free_list[tmp], link) {
+ if (flags & DRM_BUDDY_CLEAR_ALLOCATION) {
+ /* Find a cleared block */
+ if (!drm_buddy_block_is_clear(tmp_block))
+ continue;
+ } else {
+ if (drm_buddy_block_is_clear(tmp_block))
+ continue;
+ }
+
+ block = tmp_block;
+ break;
+ }
+
+ if (block)
+ break;
+ }
+ }
+
+ if (!block) {
+ /* Fallback method */
for (tmp = order; tmp <= mm->max_order; ++tmp) {
if (!list_empty(&mm->free_list[tmp])) {
block = list_last_entry(&mm->free_list[tmp],
@@ -454,10 +558,10 @@ alloc_from_freelist(struct drm_buddy *mm,
break;
}
}
- }
- if (!block)
- return ERR_PTR(-ENOSPC);
+ if (!block)
+ return ERR_PTR(-ENOSPC);
+ }
BUG_ON(!drm_buddy_block_is_free(block));
@@ -524,6 +628,8 @@ static int __alloc_range(struct drm_buddy *mm,
mark_allocated(block);
total_allocated += drm_buddy_block_size(mm, block);
mm->avail -= drm_buddy_block_size(mm, block);
+ if (drm_buddy_block_is_clear(block))
+ mm->clear_avail -= drm_buddy_block_size(mm, block);
list_add_tail(&block->link, &allocated);
continue;
}
@@ -558,7 +664,7 @@ static int __alloc_range(struct drm_buddy *mm,
list_splice_tail(&allocated, blocks);
*total_allocated_on_err = total_allocated;
} else {
- drm_buddy_free_list(mm, &allocated);
+ drm_buddy_free_list(mm, &allocated, 0);
}
return err;
@@ -624,11 +730,11 @@ static int __alloc_contig_try_harder(struct
drm_buddy *mm,
list_splice(&blocks_lhs, blocks);
return 0;
} else if (err != -ENOSPC) {
- drm_buddy_free_list(mm, blocks);
+ drm_buddy_free_list(mm, blocks, 0);
return err;
}
/* Free blocks for the next iteration */
- drm_buddy_free_list(mm, blocks);
+ drm_buddy_free_list(mm, blocks, 0);
}
return -ENOSPC;
@@ -684,6 +790,8 @@ int drm_buddy_block_trim(struct drm_buddy *mm,
list_del(&block->link);
mark_free(mm, block);
mm->avail += drm_buddy_block_size(mm, block);
+ if (drm_buddy_block_is_clear(block))
+ mm->clear_avail += drm_buddy_block_size(mm, block);
/* Prevent recursively freeing this node */
parent = block->parent;
@@ -695,6 +803,8 @@ int drm_buddy_block_trim(struct drm_buddy *mm,
if (err) {
mark_allocated(block);
mm->avail -= drm_buddy_block_size(mm, block);
+ if (drm_buddy_block_is_clear(block))
+ mm->clear_avail -= drm_buddy_block_size(mm, block);
list_add(&block->link, blocks);
}
@@ -782,7 +892,8 @@ int drm_buddy_alloc_blocks(struct drm_buddy *mm,
do {
if (flags & DRM_BUDDY_RANGE_ALLOCATION)
/* Allocate traversing within the range */
- block = alloc_range_bias(mm, start, end, order);
+ block = __drm_buddy_alloc_range_bias(mm, start, end,
+ order, flags);
else
/* Allocate from freelist */
block = alloc_from_freelist(mm, order, flags);
@@ -808,6 +919,8 @@ int drm_buddy_alloc_blocks(struct drm_buddy *mm,
mark_allocated(block);
mm->avail -= drm_buddy_block_size(mm, block);
+ if (drm_buddy_block_is_clear(block))
+ mm->clear_avail -= drm_buddy_block_size(mm, block);
kmemleak_update_trace(block);
list_add_tail(&block->link, &allocated);
@@ -846,7 +959,7 @@ int drm_buddy_alloc_blocks(struct drm_buddy *mm,
return 0;
err_free:
- drm_buddy_free_list(mm, &allocated);
+ drm_buddy_free_list(mm, &allocated, 0);
return err;
}
EXPORT_SYMBOL(drm_buddy_alloc_blocks);
@@ -879,8 +992,8 @@ void drm_buddy_print(struct drm_buddy *mm, struct
drm_printer *p)
{
int order;
- drm_printf(p, "chunk_size: %lluKiB, total: %lluMiB, free:
%lluMiB\n",
- mm->chunk_size >> 10, mm->size >> 20, mm->avail >> 20);
+ drm_printf(p, "chunk_size: %lluKiB, total: %lluMiB, free:
%lluMiB, clear_free: %lluMiB\n",
+ mm->chunk_size >> 10, mm->size >> 20, mm->avail >> 20,
mm->clear_avail >> 20);
for (order = mm->max_order; order >= 0; order--) {
struct drm_buddy_block *block;
diff --git a/drivers/gpu/drm/i915/i915_ttm_buddy_manager.c
b/drivers/gpu/drm/i915/i915_ttm_buddy_manager.c
index 0d735d5c2b35..942345548bc3 100644
--- a/drivers/gpu/drm/i915/i915_ttm_buddy_manager.c
+++ b/drivers/gpu/drm/i915/i915_ttm_buddy_manager.c
@@ -126,7 +126,7 @@ static int i915_ttm_buddy_man_alloc(struct
ttm_resource_manager *man,
return 0;
err_free_blocks:
- drm_buddy_free_list(mm, &bman_res->blocks);
+ drm_buddy_free_list(mm, &bman_res->blocks, 0);
mutex_unlock(&bman->lock);
err_free_res:
ttm_resource_fini(man, &bman_res->base);
@@ -141,7 +141,7 @@ static void i915_ttm_buddy_man_free(struct
ttm_resource_manager *man,
struct i915_ttm_buddy_manager *bman = to_buddy_manager(man);
mutex_lock(&bman->lock);
- drm_buddy_free_list(&bman->mm, &bman_res->blocks);
+ drm_buddy_free_list(&bman->mm, &bman_res->blocks, 0);
bman->visible_avail += bman_res->used_visible_size;
mutex_unlock(&bman->lock);
@@ -345,7 +345,7 @@ int i915_ttm_buddy_man_fini(struct ttm_device
*bdev, unsigned int type)
ttm_set_driver_manager(bdev, type, NULL);
mutex_lock(&bman->lock);
- drm_buddy_free_list(mm, &bman->reserved);
+ drm_buddy_free_list(mm, &bman->reserved, 0);
drm_buddy_fini(mm);
bman->visible_avail += bman->visible_reserved;
WARN_ON_ONCE(bman->visible_avail != bman->visible_size);
diff --git a/drivers/gpu/drm/tests/drm_buddy_test.c
b/drivers/gpu/drm/tests/drm_buddy_test.c
index ea2af6bd9abe..e0860fce9ebd 100644
--- a/drivers/gpu/drm/tests/drm_buddy_test.c
+++ b/drivers/gpu/drm/tests/drm_buddy_test.c
@@ -83,7 +83,7 @@ static void
drm_test_buddy_alloc_pathological(struct kunit *test)
top, max_order);
}
- drm_buddy_free_list(&mm, &holes);
+ drm_buddy_free_list(&mm, &holes, 0);
/* Nothing larger than blocks of chunk_size now available */
for (order = 1; order <= max_order; order++) {
@@ -95,7 +95,7 @@ static void
drm_test_buddy_alloc_pathological(struct kunit *test)
}
list_splice_tail(&holes, &blocks);
- drm_buddy_free_list(&mm, &blocks);
+ drm_buddy_free_list(&mm, &blocks, 0);
drm_buddy_fini(&mm);
}
@@ -190,7 +190,7 @@ static void
drm_test_buddy_alloc_pessimistic(struct kunit *test)
list_del(&block->link);
drm_buddy_free_block(&mm, block);
- drm_buddy_free_list(&mm, &blocks);
+ drm_buddy_free_list(&mm, &blocks, 0);
drm_buddy_fini(&mm);
}
@@ -236,7 +236,7 @@ static void
drm_test_buddy_alloc_optimistic(struct kunit *test)
size, size, &tmp, flags),
"buddy_alloc unexpectedly succeeded, it
should be full!");
- drm_buddy_free_list(&mm, &blocks);
+ drm_buddy_free_list(&mm, &blocks, 0);
drm_buddy_fini(&mm);
}
@@ -271,7 +271,7 @@ static void drm_test_buddy_alloc_limit(struct
kunit *test)
drm_buddy_block_size(&mm, block),
BIT_ULL(mm.max_order) * PAGE_SIZE);
- drm_buddy_free_list(&mm, &allocated);
+ drm_buddy_free_list(&mm, &allocated, 0);
drm_buddy_fini(&mm);
}
diff --git a/include/drm/drm_buddy.h b/include/drm/drm_buddy.h
index a5b39fc01003..f7311b59f2b0 100644
--- a/include/drm/drm_buddy.h
+++ b/include/drm/drm_buddy.h
@@ -6,6 +6,7 @@
#ifndef __DRM_BUDDY_H__
#define __DRM_BUDDY_H__
+#include <linux/bitfield.h>
#include <linux/bitops.h>
#include <linux/list.h>
#include <linux/slab.h>
@@ -25,15 +26,19 @@
#define DRM_BUDDY_RANGE_ALLOCATION BIT(0)
#define DRM_BUDDY_TOPDOWN_ALLOCATION BIT(1)
#define DRM_BUDDY_CONTIGUOUS_ALLOCATION BIT(2)
+#define DRM_BUDDY_CLEAR_ALLOCATION BIT(3)
+#define DRM_BUDDY_CLEARED BIT(4)
struct drm_buddy_block {
#define DRM_BUDDY_HEADER_OFFSET GENMASK_ULL(63, 12)
#define DRM_BUDDY_HEADER_STATE GENMASK_ULL(11, 10)
+#define DRM_BUDDY_HEADER_CLEAR GENMASK_ULL(9, 9)
+
#define DRM_BUDDY_ALLOCATED (1 << 10)
#define DRM_BUDDY_FREE (2 << 10)
#define DRM_BUDDY_SPLIT (3 << 10)
/* Free to be used, if needed in the future */
-#define DRM_BUDDY_HEADER_UNUSED GENMASK_ULL(9, 6)
+#define DRM_BUDDY_HEADER_UNUSED GENMASK_ULL(8, 6)
#define DRM_BUDDY_HEADER_ORDER GENMASK_ULL(5, 0)
u64 header;
@@ -86,6 +91,7 @@ struct drm_buddy {
u64 chunk_size;
u64 size;
u64 avail;
+ u64 clear_avail;
};
static inline u64
@@ -112,6 +118,12 @@ drm_buddy_block_is_allocated(struct
drm_buddy_block *block)
return drm_buddy_block_state(block) == DRM_BUDDY_ALLOCATED;
}
+static inline bool
+drm_buddy_block_is_clear(struct drm_buddy_block *block)
+{
+ return block->header & DRM_BUDDY_HEADER_CLEAR;
+}
+
static inline bool
drm_buddy_block_is_free(struct drm_buddy_block *block)
{
@@ -150,7 +162,9 @@ int drm_buddy_block_trim(struct drm_buddy *mm,
void drm_buddy_free_block(struct drm_buddy *mm, struct
drm_buddy_block *block);
-void drm_buddy_free_list(struct drm_buddy *mm, struct list_head
*objects);
+void drm_buddy_free_list(struct drm_buddy *mm,
+ struct list_head *objects,
+ unsigned long flags);
void drm_buddy_print(struct drm_buddy *mm, struct drm_printer *p);
void drm_buddy_block_print(struct drm_buddy *mm,