general ttm lru cannot statisfy amdgpu per-vm-bo requirement,
we have to adapt it in amdgpu driver at least.
Change-Id: I92b2286ef507c2e055ad9101cf31279d5f8db475
Signed-off-by: Chunming Zhou <david1.zhou at amd.com>
---
drivers/gpu/drm/ttm/ttm_bo.c | 54 ++++++++++++++++++++++++++++++-----------
include/drm/ttm/ttm_bo_driver.h | 49 +++++++++++++++++++++++++++++++++++++
2 files changed, 89 insertions(+), 14 deletions(-)
diff --git a/drivers/gpu/drm/ttm/ttm_bo.c b/drivers/gpu/drm/ttm/ttm_bo.c
index 15506682a0be..98da2cf63c9b 100644
--- a/drivers/gpu/drm/ttm/ttm_bo.c
+++ b/drivers/gpu/drm/ttm/ttm_bo.c
@@ -164,6 +164,8 @@ void ttm_bo_add_to_lru(struct ttm_buffer_object *bo)
reservation_object_assert_held(bo->resv);
+ if (bdev->driver->add_to_lru)
+ return bdev->driver->add_to_lru(bo);
if (!(bo->mem.placement & TTM_PL_FLAG_NO_EVICT)) {
BUG_ON(!list_empty(&bo->lru));
@@ -188,6 +190,8 @@ static void ttm_bo_ref_bug(struct kref *list_kref)
void ttm_bo_del_from_lru(struct ttm_buffer_object *bo)
{
+ struct ttm_bo_device *bdev = bo->bdev;
+
if (!list_empty(&bo->swap)) {
list_del_init(&bo->swap);
kref_put(&bo->list_kref, ttm_bo_ref_bug);
@@ -201,6 +205,8 @@ void ttm_bo_del_from_lru(struct ttm_buffer_object *bo)
* TODO: Add a driver hook to delete from
* driver-specific LRU's here.
*/
+ if (bdev->driver->del_from_lru)
+ return bdev->driver->del_from_lru(bo);
}
void ttm_bo_del_sub_from_lru(struct ttm_buffer_object *bo)
@@ -215,10 +221,14 @@ EXPORT_SYMBOL(ttm_bo_del_sub_from_lru);
void ttm_bo_move_to_lru_tail(struct ttm_buffer_object *bo)
{
+ struct ttm_bo_device *bdev = bo->bdev;
+
reservation_object_assert_held(bo->resv);
ttm_bo_del_from_lru(bo);
ttm_bo_add_to_lru(bo);
+ if (bdev->driver->move_to_lru_tail)
+ return bdev->driver->move_to_lru_tail(bo);
}
EXPORT_SYMBOL(ttm_bo_move_to_lru_tail);
@@ -685,8 +695,8 @@ EXPORT_SYMBOL(ttm_bo_eviction_valuable);
*
* b. Otherwise, trylock it.
*/
-static bool ttm_bo_evict_swapout_allowable(struct ttm_buffer_object *bo,
- struct ttm_operation_ctx *ctx, bool *locked)
+bool ttm_bo_evict_swapout_allowable(struct ttm_buffer_object *bo,
+ struct ttm_operation_ctx *ctx, bool *locked)
{
bool ret = false;
@@ -703,6 +713,7 @@ static bool ttm_bo_evict_swapout_allowable(struct ttm_buffer_object *bo,
return ret;
}
+EXPORT_SYMBOL(ttm_bo_evict_swapout_allowable);
static struct ttm_buffer_object *
ttm_mem_get_evictable_bo(struct ttm_bo_device *bdev,
@@ -736,6 +747,9 @@ ttm_mem_get_evictable_bo(struct ttm_bo_device *bdev,
bo = NULL;
}
+ if (!bo && bdev->driver->get_evictable_bo)
+ bo= bdev->driver->get_evictable_bo(bdev, mem_type, place,
+ ctx, locked);
return bo;
}
@@ -1311,6 +1325,21 @@ int ttm_bo_create(struct ttm_bo_device *bdev,
}
EXPORT_SYMBOL(ttm_bo_create);
+bool ttm_lru_empty(struct ttm_bo_device *bdev, unsigned mem_type)
+{
+ struct ttm_mem_type_manager *man = &bdev->man[mem_type];
+ int i;
+
+ for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) {
+ if (!list_empty(&man->lru[i]))
+ return false;
+ }
+ if (bdev->driver->lru_empty)
+ return bdev->driver->lru_empty(bdev, mem_type);
+
+ return true;
+}
+
static int ttm_bo_force_list_clean(struct ttm_bo_device *bdev,
unsigned mem_type)
{
@@ -1323,21 +1352,18 @@ static int ttm_bo_force_list_clean(struct ttm_bo_device *bdev,
struct ttm_bo_global *glob = bdev->glob;
struct dma_fence *fence;
int ret;
- unsigned i;
/*
* Can't use standard list traversal since we're unlocking.
*/
spin_lock(&glob->lru_lock);
- for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) {
- while (!list_empty(&man->lru[i])) {
- spin_unlock(&glob->lru_lock);
- ret = ttm_mem_evict_first(bdev, mem_type, NULL, &ctx);
- if (ret)
- return ret;
- spin_lock(&glob->lru_lock);
- }
+ while (!ttm_lru_empty(bdev, mem_type)) {
+ spin_unlock(&glob->lru_lock);
+ ret = ttm_mem_evict_first(bdev, mem_type, NULL, &ctx);
+ if (ret)
+ return ret;
+ spin_lock(&glob->lru_lock);
}
spin_unlock(&glob->lru_lock);
@@ -1533,9 +1559,9 @@ int ttm_bo_device_release(struct ttm_bo_device *bdev)
pr_debug("Delayed destroy list was clean\n");
spin_lock(&glob->lru_lock);
- for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i)
- if (list_empty(&bdev->man[0].lru[0]))
- pr_debug("Swap list %d was clean\n", i);
+ for (i = 0; i < TTM_NUM_MEM_TYPES; ++i)
+ if (ttm_lru_empty(bdev, i))
+ pr_debug("lru list %d was clean\n", i);
spin_unlock(&glob->lru_lock);
drm_vma_offset_manager_destroy(&bdev->vma_manager);
diff --git a/include/drm/ttm/ttm_bo_driver.h b/include/drm/ttm/ttm_bo_driver.h
index 3234cc322e70..29339b0a2fd6 100644
--- a/include/drm/ttm/ttm_bo_driver.h
+++ b/include/drm/ttm/ttm_bo_driver.h
@@ -284,6 +284,52 @@ struct ttm_bo_driver {
*/
bool (*eviction_valuable)(struct ttm_buffer_object *bo,
const struct ttm_place *place);
+
+ /**
+ * struct ttm_bo_driver member get_evictable_bo
+ *
+ * @bdev: the buffer object device.
+ * @mem_type: memory type
+ * @place: placement we need room for
+ * @ctx: context for this evict with parameters
+ * @locked: return if the evictable bo is already locked.
+ *
+ * return an evictable bo for evicting.
+ */
+ struct ttm_buffer_object *(*get_evictable_bo)(struct ttm_bo_device *bdev,
+ uint32_t mem_type,
+ const struct ttm_place *place,
+ struct ttm_operation_ctx *ctx,
+ bool *locked);
+
+ /**
+ * struct ttm_bo_driver member add_to_lru
+ *
+ * @bo: the buffer object to be add
+ *
+ * add bo to driver specific lru
+ */
+ void (*add_to_lru)(struct ttm_buffer_object *bo);
+
+ /**
+ * struct ttm_bo_driver member del_from_lru
+ *
+ * @bo: the buffer object to be add
+ *
+ * delete bo from driver specific lru
+ */
+ void (*del_from_lru)(struct ttm_buffer_object *bo);
+
+ /**
+ * struct ttm_bo_driver member move_to_lru_tail
+ *
+ * @bo: the buffer object to be add
+ *
+ * move to driver specific lru tail
+ */
+ void (*move_to_lru_tail)(struct ttm_buffer_object *bo);
+
+ bool (*lru_empty)(struct ttm_bo_device *bdev, unsigned mem_type);
/**
* struct ttm_bo_driver member evict_flags:
*
@@ -760,6 +806,9 @@ int ttm_mem_io_reserve(struct ttm_bo_device *bdev,
struct ttm_mem_reg *mem);
void ttm_mem_io_free(struct ttm_bo_device *bdev,
struct ttm_mem_reg *mem);
+bool ttm_bo_evict_swapout_allowable(struct ttm_buffer_object *bo,
+ struct ttm_operation_ctx *ctx,
+ bool *locked);
/**
* ttm_bo_move_ttm
*
--
2.14.1
