We are calling the eviction_valuable driver callback at eviction time to
determine whether we actually can evict a buffer object.
The upcoming i915 TTM backend needs the same functionality for swapout,
and that might actually be beneficial to other drivers as well.
Add an eviction_valuable call also in the swapout path. Try to keep the
current behaviour for all drivers by returning true if the buffer object
is already in the TTM_PL_SYSTEM placement. We change behaviour for the
case where a buffer object is in a TT backed placement when swapped out,
in which case the drivers normal eviction_valuable path is run.
Finally make sure we don't try to swapout a bo that was recently purged
and therefore unpopulated.
Cc: Christian König <christian.koenig@xxxxxxx>
Signed-off-by: Thomas Hellström <thomas.hellstrom@xxxxxxxxxxxxxxx>
---
drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c | 4 +++
drivers/gpu/drm/ttm/ttm_bo.c | 41 +++++++++++++++----------
drivers/gpu/drm/ttm/ttm_tt.c | 4 +++
3 files changed, 33 insertions(+), 16 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
index 8c7ec09eb1a4..d5a9d7a88315 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
@@ -1399,6 +1399,10 @@ static bool amdgpu_ttm_bo_eviction_valuable(struct ttm_buffer_object *bo,
struct dma_fence *f;
int i;
+ /* Swapout? */
+ if (bo->mem.mem_type == TTM_PL_SYSTEM)
+ return true;
+
if (bo->type == ttm_bo_type_kernel &&
!amdgpu_vm_evictable(ttm_to_amdgpu_bo(bo)))
return false;
diff --git a/drivers/gpu/drm/ttm/ttm_bo.c b/drivers/gpu/drm/ttm/ttm_bo.c
index a8fa3375b8aa..3f7c64a1cda1 100644
--- a/drivers/gpu/drm/ttm/ttm_bo.c
+++ b/drivers/gpu/drm/ttm/ttm_bo.c
@@ -536,6 +536,10 @@ static int ttm_bo_evict(struct ttm_buffer_object *bo,
bool ttm_bo_eviction_valuable(struct ttm_buffer_object *bo,
const struct ttm_place *place)
{
+ dma_resv_assert_held(bo->base.resv);
+ if (bo->mem.mem_type == TTM_PL_SYSTEM)
+ return true;
+
/* Don't evict this BO if it's outside of the
* requested placement range
*/
@@ -558,7 +562,9 @@ EXPORT_SYMBOL(ttm_bo_eviction_valuable);
* b. Otherwise, trylock it.
*/
static bool ttm_bo_evict_swapout_allowable(struct ttm_buffer_object *bo,
- struct ttm_operation_ctx *ctx, bool *locked, bool *busy)
+ struct ttm_operation_ctx *ctx,
+ const struct ttm_place *place,
+ bool *locked, bool *busy)
{
bool ret = false;
@@ -576,6 +582,12 @@ static bool ttm_bo_evict_swapout_allowable(struct ttm_buffer_object *bo,
*busy = !ret;
}
+ if (ret && place && !bo->bdev->funcs->eviction_valuable(bo, place)) {
+ ret = false;
+ if (locked)
+ dma_resv_unlock(bo->base.resv);
+ }
+
return ret;
}
@@ -630,20 +642,14 @@ int ttm_mem_evict_first(struct ttm_device *bdev,
list_for_each_entry(bo, &man->lru[i], lru) {
bool busy;
- if (!ttm_bo_evict_swapout_allowable(bo, ctx, &locked,
- &busy)) {
+ if (!ttm_bo_evict_swapout_allowable(bo, ctx, place,
+ &locked, &busy)) {
if (busy && !busy_bo && ticket !=
dma_resv_locking_ctx(bo->base.resv))
busy_bo = bo;
continue;
}
- if (place && !bdev->funcs->eviction_valuable(bo,
- place)) {
- if (locked)
- dma_resv_unlock(bo->base.resv);
- continue;
- }
if (!ttm_bo_get_unless_zero(bo)) {
if (locked)
dma_resv_unlock(bo->base.resv);
@@ -1138,10 +1144,18 @@ EXPORT_SYMBOL(ttm_bo_wait);
int ttm_bo_swapout(struct ttm_buffer_object *bo, struct ttm_operation_ctx *ctx,
gfp_t gfp_flags)
{
+ struct ttm_place place = {};
bool locked;
int ret;
- if (!ttm_bo_evict_swapout_allowable(bo, ctx, &locked, NULL))
+ /*
+ * While the bo may already reside in SYSTEM placement, set
+ * SYSTEM as new placement to cover also the move further below.
+ * The driver may use the fact that we're moving from SYSTEM
+ * as an indication that we're about to swap out.
+ */
+ place.mem_type = TTM_PL_SYSTEM;
+ if (!ttm_bo_evict_swapout_allowable(bo, ctx, &place, &locked, NULL))
return -EBUSY;
if (!ttm_bo_get_unless_zero(bo)) {
@@ -1166,12 +1180,7 @@ int ttm_bo_swapout(struct ttm_buffer_object *bo, struct ttm_operation_ctx *ctx,
if (bo->mem.mem_type != TTM_PL_SYSTEM) {
struct ttm_operation_ctx ctx = { false, false };
struct ttm_resource evict_mem;
- struct ttm_place place, hop;
-
- memset(&place, 0, sizeof(place));
- memset(&hop, 0, sizeof(hop));
-
- place.mem_type = TTM_PL_SYSTEM;
+ struct ttm_place hop = {};
ret = ttm_resource_alloc(bo, &place, &evict_mem);
if (unlikely(ret))
diff --git a/drivers/gpu/drm/ttm/ttm_tt.c b/drivers/gpu/drm/ttm/ttm_tt.c
index 0b1053e93db2..d46c702ce0ca 100644
--- a/drivers/gpu/drm/ttm/ttm_tt.c
+++ b/drivers/gpu/drm/ttm/ttm_tt.c
@@ -263,6 +263,9 @@ int ttm_tt_swapout(struct ttm_device *bdev, struct ttm_tt *ttm,
struct page *to_page;
int i, ret;
+ if (!ttm_tt_is_populated(ttm))
+ return 0;
+
swap_storage = shmem_file_setup("ttm swap", size, 0);
if (IS_ERR(swap_storage)) {
pr_err("Failed allocating swap storage\n");
@@ -404,6 +407,7 @@ void ttm_tt_unpopulate(struct ttm_device *bdev, struct ttm_tt *ttm)
ttm->page_flags &= ~TTM_PAGE_FLAG_PRIV_POPULATED;
}
+EXPORT_SYMBOL(ttm_tt_unpopulate);
#ifdef CONFIG_DEBUG_FS
--
2.31.1
