Note if the added fence is a write by using the lsb in the fenc pointer.
Signed-off-by: Christian König <christian.koenig at amd.com>
---
drivers/dma-buf/dma-buf.c | 8 +++-
drivers/dma-buf/reservation.c | 59 +++++++++++++++++-----------
drivers/gpu/drm/amd/amdgpu/amdgpu_object.c | 2 +-
drivers/gpu/drm/etnaviv/etnaviv_gem_submit.c | 3 +-
drivers/gpu/drm/i915/i915_gem.c | 6 ++-
drivers/gpu/drm/i915/i915_gem_execbuffer.c | 2 +-
drivers/gpu/drm/msm/msm_gem.c | 3 +-
drivers/gpu/drm/nouveau/nouveau_bo.c | 2 +-
drivers/gpu/drm/qxl/qxl_release.c | 3 +-
drivers/gpu/drm/radeon/radeon_object.c | 2 +-
drivers/gpu/drm/ttm/ttm_bo.c | 2 +-
drivers/gpu/drm/ttm/ttm_execbuf_util.c | 3 +-
drivers/gpu/drm/vc4/vc4_gem.c | 3 +-
drivers/gpu/drm/vgem/vgem_fence.c | 2 +-
include/linux/reservation.h | 20 +++++++---
15 files changed, 76 insertions(+), 44 deletions(-)
diff --git a/drivers/dma-buf/dma-buf.c b/drivers/dma-buf/dma-buf.c
index 13884474d158..6b816cd505d6 100644
--- a/drivers/dma-buf/dma-buf.c
+++ b/drivers/dma-buf/dma-buf.c
@@ -244,7 +244,10 @@ static __poll_t dma_buf_poll(struct file *file, poll_table *poll)
goto out;
for (i = 0; i < shared_count; ++i) {
- struct dma_fence *fence = rcu_dereference(fobj->shared[i]);
+ struct dma_fence *fence;
+
+ fence = reservation_object_shared_fence(
+ rcu_dereference(fobj->shared[i]));
if (!dma_fence_get_rcu(fence)) {
/*
@@ -1062,7 +1065,8 @@ static int dma_buf_debug_show(struct seq_file *s, void *unused)
fence->ops->get_timeline_name(fence),
dma_fence_is_signaled(fence) ? "" : "un");
for (i = 0; i < shared_count; i++) {
- fence = rcu_dereference(fobj->shared[i]);
+ fence = reservation_object_shared_fence(
+ rcu_dereference(fobj->shared[i]));
if (!dma_fence_get_rcu(fence))
continue;
seq_printf(s, "\tShared fence: %s %s %ssignalled\n",
diff --git a/drivers/dma-buf/reservation.c b/drivers/dma-buf/reservation.c
index 1f0c61b540ba..0f98384b86d4 100644
--- a/drivers/dma-buf/reservation.c
+++ b/drivers/dma-buf/reservation.c
@@ -93,14 +93,14 @@ int reservation_object_reserve_shared(struct reservation_object *obj)
* the new.
*/
for (i = 0, j = 0, k = max; i < (old ? old->shared_count : 0); ++i) {
- struct dma_fence *fence;
+ void *e;
- fence = rcu_dereference_protected(old->shared[i],
- reservation_object_held(obj));
- if (dma_fence_is_signaled(fence))
- RCU_INIT_POINTER(new->shared[--k], fence);
+ e = rcu_dereference_protected(old->shared[i],
+ reservation_object_held(obj));
+ if (dma_fence_is_signaled(reservation_object_shared_fence(e)))
+ RCU_INIT_POINTER(new->shared[--k], e);
else
- RCU_INIT_POINTER(new->shared[j++], fence);
+ RCU_INIT_POINTER(new->shared[j++], e);
}
new->shared_count = j;
new->shared_max = max;
@@ -120,11 +120,11 @@ int reservation_object_reserve_shared(struct reservation_object *obj)
/* Drop the references to the signaled fences */
for (i = k; i < new->shared_max; ++i) {
- struct dma_fence *fence;
+ void *e;
- fence = rcu_dereference_protected(new->shared[i],
- reservation_object_held(obj));
- dma_fence_put(fence);
+ e = rcu_dereference_protected(new->shared[i],
+ reservation_object_held(obj));
+ dma_fence_put(reservation_object_shared_fence(e));
}
kfree_rcu(old, rcu);
@@ -141,7 +141,8 @@ EXPORT_SYMBOL(reservation_object_reserve_shared);
* reservation_object_reserve_shared() has been called.
*/
void reservation_object_add_shared_fence(struct reservation_object *obj,
- struct dma_fence *fence)
+ struct dma_fence *fence,
+ bool is_write)
{
struct reservation_object_list *fobj;
unsigned int i;
@@ -155,13 +156,17 @@ void reservation_object_add_shared_fence(struct reservation_object *obj,
for (i = 0; i < fobj->shared_count; ++i) {
struct dma_fence *old_fence;
+ void *e;
- old_fence = rcu_dereference_protected(fobj->shared[i],
- reservation_object_held(obj));
- if (old_fence->context == fence->context ||
+ e = rcu_dereference_protected(fobj->shared[i],
+ reservation_object_held(obj));
+ old_fence = reservation_object_shared_fence(e);
+ if ((old_fence->context == fence->context &&
+ reservation_object_shared_is_write(e) == is_write) ||
dma_fence_is_signaled(old_fence)) {
/* memory barrier is added by write_seqcount_begin */
- RCU_INIT_POINTER(fobj->shared[i], fence);
+ RCU_INIT_POINTER(fobj->shared[i],
+ (void *)(is_write | (long)fence));
write_seqcount_end(&obj->seq);
preempt_enable();
dma_fence_put(old_fence);
@@ -173,7 +178,8 @@ void reservation_object_add_shared_fence(struct reservation_object *obj,
* memory barrier is added by write_seqcount_begin,
* fobj->shared_count is protected by this lock too
*/
- RCU_INIT_POINTER(fobj->shared[fobj->shared_count], fence);
+ RCU_INIT_POINTER(fobj->shared[fobj->shared_count],
+ (void *)(is_write | (long)fence));
fobj->shared_count++;
write_seqcount_end(&obj->seq);
@@ -213,8 +219,7 @@ void reservation_object_add_excl_fence(struct reservation_object *obj,
/* inplace update, no shared fences */
while (i--)
- dma_fence_put(rcu_dereference_protected(old->shared[i],
- reservation_object_held(obj)));
+ dma_fence_put(reservation_object_get_shared_fence(obj, old, i));
dma_fence_put(old_fence);
}
@@ -260,8 +265,10 @@ int reservation_object_copy_fences(struct reservation_object *dst,
dst_list->shared_max = shared_count;
for (i = 0; i < src_list->shared_count; ++i) {
struct dma_fence *fence;
+ void *e;
- fence = rcu_dereference(src_list->shared[i]);
+ e = rcu_dereference(src_list->shared[i]);
+ fence = reservation_object_shared_fence(e);
if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT,
&fence->flags))
continue;
@@ -277,7 +284,7 @@ int reservation_object_copy_fences(struct reservation_object *dst,
continue;
}
- rcu_assign_pointer(dst_list->shared[dst_list->shared_count++], fence);
+ rcu_assign_pointer(dst_list->shared[dst_list->shared_count++], e);
}
} else {
dst_list = NULL;
@@ -368,7 +375,9 @@ int reservation_object_get_fences_rcu(struct reservation_object *obj,
shared = nshared;
shared_count = fobj ? fobj->shared_count : 0;
for (i = 0; i < shared_count; ++i) {
- shared[i] = rcu_dereference(fobj->shared[i]);
+ void *e = rcu_dereference(fobj->shared[i]);
+
+ shared[i] = reservation_object_shared_fence(e);
if (!dma_fence_get_rcu(shared[i]))
break;
}
@@ -456,8 +465,10 @@ long reservation_object_wait_timeout_rcu(struct reservation_object *obj,
shared_count = fobj->shared_count;
for (i = 0; !fence && i < shared_count; ++i) {
- struct dma_fence *lfence = rcu_dereference(fobj->shared[i]);
+ void *e = rcu_dereference(fobj->shared[i]);
+ struct dma_fence *lfence;
+ lfence = reservation_object_shared_fence(e);
if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT,
&lfence->flags))
continue;
@@ -545,8 +556,10 @@ bool reservation_object_test_signaled_rcu(struct reservation_object *obj,
shared_count = fobj->shared_count;
for (i = 0; i < shared_count; ++i) {
- struct dma_fence *fence = rcu_dereference(fobj->shared[i]);
+ void *e = rcu_dereference(fobj->shared[i]);
+ struct dma_fence *fence;
+ fence = reservation_object_shared_fence(e);
ret = reservation_object_test_signaled_single(fence);
if (ret < 0)
goto retry;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c
index b0e14a3d54ef..303143b89275 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c
@@ -1342,7 +1342,7 @@ void amdgpu_bo_fence(struct amdgpu_bo *bo, struct dma_fence *fence,
struct reservation_object *resv = bo->tbo.resv;
if (shared)
- reservation_object_add_shared_fence(resv, fence);
+ reservation_object_add_shared_fence(resv, fence, true);
else
reservation_object_add_excl_fence(resv, fence);
}
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gem_submit.c b/drivers/gpu/drm/etnaviv/etnaviv_gem_submit.c
index 46ecd3e66ac9..5f4a872a88dd 100644
--- a/drivers/gpu/drm/etnaviv/etnaviv_gem_submit.c
+++ b/drivers/gpu/drm/etnaviv/etnaviv_gem_submit.c
@@ -214,7 +214,8 @@ static void submit_attach_object_fences(struct etnaviv_gem_submit *submit)
submit->out_fence);
else
reservation_object_add_shared_fence(etnaviv_obj->resv,
- submit->out_fence);
+ submit->out_fence,
+ false);
submit_unlock_object(submit, i);
}
diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c
index 86f1f9aaa119..0415420e1cfd 100644
--- a/drivers/gpu/drm/i915/i915_gem.c
+++ b/drivers/gpu/drm/i915/i915_gem.c
@@ -4552,8 +4552,10 @@ i915_gem_busy_ioctl(struct drm_device *dev, void *data,
unsigned int shared_count = list->shared_count, i;
for (i = 0; i < shared_count; ++i) {
- struct dma_fence *fence =
- rcu_dereference(list->shared[i]);
+ struct dma_fence *fence;
+
+ fence = reservation_object_shared_fence(
+ rcu_dereference(list->shared[i]));
args->busy |= busy_check_reader(fence);
}
diff --git a/drivers/gpu/drm/i915/i915_gem_execbuffer.c b/drivers/gpu/drm/i915/i915_gem_execbuffer.c
index eefd449502e2..85e3f92e87f8 100644
--- a/drivers/gpu/drm/i915/i915_gem_execbuffer.c
+++ b/drivers/gpu/drm/i915/i915_gem_execbuffer.c
@@ -1771,7 +1771,7 @@ static void eb_export_fence(struct i915_vma *vma,
if (flags & EXEC_OBJECT_WRITE)
reservation_object_add_excl_fence(resv, &rq->fence);
else if (reservation_object_reserve_shared(resv) == 0)
- reservation_object_add_shared_fence(resv, &rq->fence);
+ reservation_object_add_shared_fence(resv, &rq->fence, false);
reservation_object_unlock(resv);
}
diff --git a/drivers/gpu/drm/msm/msm_gem.c b/drivers/gpu/drm/msm/msm_gem.c
index 95d25dbfde2b..fdee77627a0f 100644
--- a/drivers/gpu/drm/msm/msm_gem.c
+++ b/drivers/gpu/drm/msm/msm_gem.c
@@ -672,7 +672,8 @@ void msm_gem_move_to_active(struct drm_gem_object *obj,
if (exclusive)
reservation_object_add_excl_fence(msm_obj->resv, fence);
else
- reservation_object_add_shared_fence(msm_obj->resv, fence);
+ reservation_object_add_shared_fence(msm_obj->resv, fence,
+ false);
list_del_init(&msm_obj->mm_list);
list_add_tail(&msm_obj->mm_list, &gpu->active_list);
}
diff --git a/drivers/gpu/drm/nouveau/nouveau_bo.c b/drivers/gpu/drm/nouveau/nouveau_bo.c
index 7214022dfb91..741faead4c7f 100644
--- a/drivers/gpu/drm/nouveau/nouveau_bo.c
+++ b/drivers/gpu/drm/nouveau/nouveau_bo.c
@@ -1657,7 +1657,7 @@ nouveau_bo_fence(struct nouveau_bo *nvbo, struct nouveau_fence *fence, bool excl
if (exclusive)
reservation_object_add_excl_fence(resv, &fence->base);
else if (fence)
- reservation_object_add_shared_fence(resv, &fence->base);
+ reservation_object_add_shared_fence(resv, &fence->base, false);
}
struct ttm_bo_driver nouveau_bo_driver = {
diff --git a/drivers/gpu/drm/qxl/qxl_release.c b/drivers/gpu/drm/qxl/qxl_release.c
index 7cb214577275..e68ab1efd809 100644
--- a/drivers/gpu/drm/qxl/qxl_release.c
+++ b/drivers/gpu/drm/qxl/qxl_release.c
@@ -466,7 +466,8 @@ void qxl_release_fence_buffer_objects(struct qxl_release *release)
bo = entry->bo;
qbo = to_qxl_bo(bo);
- reservation_object_add_shared_fence(bo->resv, &release->base);
+ reservation_object_add_shared_fence(bo->resv, &release->base,
+ false);
ttm_bo_add_to_lru(bo);
reservation_object_unlock(bo->resv);
}
diff --git a/drivers/gpu/drm/radeon/radeon_object.c b/drivers/gpu/drm/radeon/radeon_object.c
index ba2fd295697f..11114ffb7495 100644
--- a/drivers/gpu/drm/radeon/radeon_object.c
+++ b/drivers/gpu/drm/radeon/radeon_object.c
@@ -870,7 +870,7 @@ void radeon_bo_fence(struct radeon_bo *bo, struct radeon_fence *fence,
struct reservation_object *resv = bo->tbo.resv;
if (shared)
- reservation_object_add_shared_fence(resv, &fence->base);
+ reservation_object_add_shared_fence(resv, &fence->base, false);
else
reservation_object_add_excl_fence(resv, &fence->base);
}
diff --git a/drivers/gpu/drm/ttm/ttm_bo.c b/drivers/gpu/drm/ttm/ttm_bo.c
index 820d97d3e8b9..9e98d8977cd0 100644
--- a/drivers/gpu/drm/ttm/ttm_bo.c
+++ b/drivers/gpu/drm/ttm/ttm_bo.c
@@ -794,7 +794,7 @@ static int ttm_bo_add_move_fence(struct ttm_buffer_object *bo,
spin_unlock(&man->move_lock);
if (fence) {
- reservation_object_add_shared_fence(bo->resv, fence);
+ reservation_object_add_shared_fence(bo->resv, fence, true);
ret = reservation_object_reserve_shared(bo->resv);
if (unlikely(ret))
diff --git a/drivers/gpu/drm/ttm/ttm_execbuf_util.c b/drivers/gpu/drm/ttm/ttm_execbuf_util.c
index e73ae0d22897..f82a460d106b 100644
--- a/drivers/gpu/drm/ttm/ttm_execbuf_util.c
+++ b/drivers/gpu/drm/ttm/ttm_execbuf_util.c
@@ -202,7 +202,8 @@ void ttm_eu_fence_buffer_objects(struct ww_acquire_ctx *ticket,
list_for_each_entry(entry, list, head) {
bo = entry->bo;
if (entry->shared)
- reservation_object_add_shared_fence(bo->resv, fence);
+ reservation_object_add_shared_fence(bo->resv, fence,
+ true);
else
reservation_object_add_excl_fence(bo->resv, fence);
ttm_bo_add_to_lru(bo);
diff --git a/drivers/gpu/drm/vc4/vc4_gem.c b/drivers/gpu/drm/vc4/vc4_gem.c
index 7910b9acedd6..9db15a76c25f 100644
--- a/drivers/gpu/drm/vc4/vc4_gem.c
+++ b/drivers/gpu/drm/vc4/vc4_gem.c
@@ -536,7 +536,8 @@ vc4_update_bo_seqnos(struct vc4_exec_info *exec, uint64_t seqno)
bo = to_vc4_bo(&exec->bo[i]->base);
bo->seqno = seqno;
- reservation_object_add_shared_fence(bo->resv, exec->fence);
+ reservation_object_add_shared_fence(bo->resv, exec->fence,
+ false);
}
list_for_each_entry(bo, &exec->unref_list, unref_head) {
diff --git a/drivers/gpu/drm/vgem/vgem_fence.c b/drivers/gpu/drm/vgem/vgem_fence.c
index b28876c222b4..5d8b47056be6 100644
--- a/drivers/gpu/drm/vgem/vgem_fence.c
+++ b/drivers/gpu/drm/vgem/vgem_fence.c
@@ -194,7 +194,7 @@ int vgem_fence_attach_ioctl(struct drm_device *dev,
if (arg->flags & VGEM_FENCE_WRITE)
reservation_object_add_excl_fence(resv, fence);
else if ((ret = reservation_object_reserve_shared(resv)) == 0)
- reservation_object_add_shared_fence(resv, fence);
+ reservation_object_add_shared_fence(resv, fence, false);
reservation_object_unlock(resv);
/* Record the fence in our idr for later signaling */
diff --git a/include/linux/reservation.h b/include/linux/reservation.h
index 8a3298574bf5..d73bf025df4b 100644
--- a/include/linux/reservation.h
+++ b/include/linux/reservation.h
@@ -59,7 +59,7 @@ extern const char reservation_seqcount_string[];
struct reservation_object_list {
struct rcu_head rcu;
u32 shared_count, shared_max;
- struct dma_fence __rcu *shared[];
+ void * __rcu *shared[];
};
/**
@@ -80,6 +80,8 @@ struct reservation_object {
#define reservation_object_held(obj) lockdep_is_held(&(obj)->lock.base)
#define reservation_object_assert_held(obj) \
lockdep_assert_held(&(obj)->lock.base)
+#define reservation_object_shared_fence(e) ((struct dma_fence *)((long)e & ~1ul))
+#define reservation_object_shared_is_write(e) ((long)e & 1)
/**
* reservation_object_init - initialize a reservation object
@@ -116,8 +118,11 @@ reservation_object_fini(struct reservation_object *obj)
fobj = rcu_dereference_protected(obj->fence, 1);
if (fobj) {
- for (i = 0; i < fobj->shared_count; ++i)
- dma_fence_put(rcu_dereference_protected(fobj->shared[i], 1));
+ for (i = 0; i < fobj->shared_count; ++i) {
+ void *e = rcu_dereference_protected(fobj->shared[i], 1);
+
+ dma_fence_put(reservation_object_shared_fence(e));
+ }
kfree(fobj);
}
@@ -155,8 +160,10 @@ reservation_object_get_shared_fence(struct reservation_object *obj,
struct reservation_object_list *list,
unsigned int idx)
{
- return rcu_dereference_protected(list->shared[idx],
- reservation_object_held(obj));
+ void *e = rcu_dereference_protected(list->shared[idx],
+ reservation_object_held(obj));
+
+ return reservation_object_shared_fence(e);
}
/**
@@ -282,7 +289,8 @@ reservation_object_get_excl_rcu(struct reservation_object *obj)
int reservation_object_reserve_shared(struct reservation_object *obj);
void reservation_object_add_shared_fence(struct reservation_object *obj,
- struct dma_fence *fence);
+ struct dma_fence *fence,
+ bool as_write);
void reservation_object_add_excl_fence(struct reservation_object *obj,
struct dma_fence *fence);
--
2.14.1
