From: Ketil Johnsen <ketil.johnsen@xxxxxxx>
The code is needed both for the existing kernel submission path and for
implementing cross-group sync (XGS) queues which link between
drm_syncobj and the HW syncobj primitives.
Signed-off-by: Ketil Johnsen <ketil.johnsen@xxxxxxx>
Signed-off-by: Mihail Atanassov <mihail.atanassov@xxxxxxx>
---
drivers/gpu/drm/panthor/Makefile | 3 +-
drivers/gpu/drm/panthor/panthor_sched.c | 154 +++++---------------
drivers/gpu/drm/panthor/panthor_syncobj.c | 167 ++++++++++++++++++++++
drivers/gpu/drm/panthor/panthor_syncobj.h | 27 ++++
4 files changed, 234 insertions(+), 117 deletions(-)
create mode 100644 drivers/gpu/drm/panthor/panthor_syncobj.c
create mode 100644 drivers/gpu/drm/panthor/panthor_syncobj.h
diff --git a/drivers/gpu/drm/panthor/Makefile b/drivers/gpu/drm/panthor/Makefile
index 15294719b09c..0af27f33bfe2 100644
--- a/drivers/gpu/drm/panthor/Makefile
+++ b/drivers/gpu/drm/panthor/Makefile
@@ -9,6 +9,7 @@ panthor-y := \
panthor_gpu.o \
panthor_heap.o \
panthor_mmu.o \
- panthor_sched.o
+ panthor_sched.o \
+ panthor_syncobj.o
obj-$(CONFIG_DRM_PANTHOR) += panthor.o
diff --git a/drivers/gpu/drm/panthor/panthor_sched.c b/drivers/gpu/drm/panthor/panthor_sched.c
index 3b56526a4b97..f272aeee8a8f 100644
--- a/drivers/gpu/drm/panthor/panthor_sched.c
+++ b/drivers/gpu/drm/panthor/panthor_sched.c
@@ -31,6 +31,7 @@
#include "panthor_mmu.h"
#include "panthor_regs.h"
#include "panthor_sched.h"
+#include "panthor_syncobj.h"
/**
* DOC: Scheduler
@@ -318,39 +319,6 @@ struct panthor_scheduler {
} reset;
};
-/**
- * struct panthor_syncobj_32b - 32-bit FW synchronization object
- */
-struct panthor_syncobj_32b {
- /** @seqno: Sequence number. */
- u32 seqno;
-
- /**
- * @status: Status.
- *
- * Not zero on failure.
- */
- u32 status;
-};
-
-/**
- * struct panthor_syncobj_64b - 64-bit FW synchronization object
- */
-struct panthor_syncobj_64b {
- /** @seqno: Sequence number. */
- u64 seqno;
-
- /**
- * @status: Status.
- *
- * Not zero on failure.
- */
- u32 status;
-
- /** @pad: MBZ. */
- u32 pad;
-};
-
/**
* struct panthor_queue - Execution queue
*/
@@ -445,17 +413,8 @@ struct panthor_queue {
/** @sync64: True if this is a 64-bit sync object. */
bool sync64;
- /** @bo: Buffer object holding the synchronization object. */
- struct drm_gem_object *obj;
-
- /** @offset: Offset of the synchronization object inside @bo. */
- u64 offset;
-
- /**
- * @kmap: Kernel mapping of the buffer object holding the
- * synchronization object.
- */
- void *kmap;
+ /** @syncobj: Wrapper for the syncobj in memory */
+ struct panthor_syncobj *syncobj;
} syncwait;
/** @fence_ctx: Fence context fields. */
@@ -794,53 +753,6 @@ struct panthor_job {
struct dma_fence *done_fence;
};
-static void
-panthor_queue_put_syncwait_obj(struct panthor_queue *queue)
-{
- if (queue->syncwait.kmap) {
- struct iosys_map map = IOSYS_MAP_INIT_VADDR(queue->syncwait.kmap);
-
- drm_gem_vunmap_unlocked(queue->syncwait.obj, &map);
- queue->syncwait.kmap = NULL;
- }
-
- drm_gem_object_put(queue->syncwait.obj);
- queue->syncwait.obj = NULL;
-}
-
-static void *
-panthor_queue_get_syncwait_obj(struct panthor_group *group, struct panthor_queue *queue)
-{
- struct panthor_device *ptdev = group->ptdev;
- struct panthor_gem_object *bo;
- struct iosys_map map;
- int ret;
-
- if (queue->syncwait.kmap)
- return queue->syncwait.kmap + queue->syncwait.offset;
-
- bo = panthor_vm_get_bo_for_va(group->vm,
- queue->syncwait.gpu_va,
- &queue->syncwait.offset);
- if (drm_WARN_ON(&ptdev->base, IS_ERR_OR_NULL(bo)))
- goto err_put_syncwait_obj;
-
- queue->syncwait.obj = &bo->base.base;
- ret = drm_gem_vmap_unlocked(queue->syncwait.obj, &map);
- if (drm_WARN_ON(&ptdev->base, ret))
- goto err_put_syncwait_obj;
-
- queue->syncwait.kmap = map.vaddr;
- if (drm_WARN_ON(&ptdev->base, !queue->syncwait.kmap))
- goto err_put_syncwait_obj;
-
- return queue->syncwait.kmap + queue->syncwait.offset;
-
-err_put_syncwait_obj:
- panthor_queue_put_syncwait_obj(queue);
- return NULL;
-}
-
static void group_free_queue(struct panthor_group *group, struct panthor_queue *queue)
{
if (IS_ERR_OR_NULL(queue))
@@ -852,7 +764,7 @@ static void group_free_queue(struct panthor_group *group, struct panthor_queue *
if (queue->scheduler.ops)
drm_sched_fini(&queue->scheduler);
- panthor_queue_put_syncwait_obj(queue);
+ panthor_syncobj_release(queue->syncwait.syncobj);
if (queue->ringbuf_offset)
drm_vma_node_revoke(&queue->ringbuf->obj->vma_node, group->pfile->drm_file);
@@ -2065,7 +1977,6 @@ group_term_post_processing(struct panthor_group *group)
cookie = dma_fence_begin_signalling();
for (i = 0; i < group->queue_count; i++) {
struct panthor_queue *queue = group->queues[i];
- struct panthor_syncobj_64b *syncobj;
int err;
if (group->fatal_queues & BIT(i))
@@ -2086,12 +1997,13 @@ group_term_post_processing(struct panthor_group *group)
}
spin_unlock(&queue->fence_ctx.lock);
- if (!group->user_submit) {
+ if (!group->user_submit)
/* Manually update the syncobj seqno to unblock waiters. */
- syncobj = group->syncobjs->kmap + (i * sizeof(*syncobj));
- syncobj->status = ~0;
- syncobj->seqno = atomic64_read(&queue->fence_ctx.seqno);
- }
+ panthor_syncobj_ptr64_signal_with_error(
+ group->syncobjs->kmap + (i * PANTHOR_SYNCOBJ64_SIZE),
+ atomic64_read(&queue->fence_ctx.seqno),
+ ~0);
+
sched_queue_work(group->ptdev->scheduler, sync_upd);
}
dma_fence_end_signalling(cookie);
@@ -2461,28 +2373,32 @@ static void tick_work(struct work_struct *work)
static int panthor_queue_eval_syncwait(struct panthor_group *group, u8 queue_idx)
{
struct panthor_queue *queue = group->queues[queue_idx];
- union {
- struct panthor_syncobj_64b sync64;
- struct panthor_syncobj_32b sync32;
- } *syncobj;
+ struct panthor_syncobj *syncobj;
bool result;
u64 value;
- syncobj = panthor_queue_get_syncwait_obj(group, queue);
- if (!syncobj)
- return -EINVAL;
+ if (!queue->syncwait.syncobj) {
+ syncobj = panthor_syncobj_create(group->ptdev,
+ group->vm,
+ queue->syncwait.gpu_va,
+ queue->syncwait.sync64);
+ if (IS_ERR_OR_NULL(syncobj))
+ return PTR_ERR(syncobj);
- value = queue->syncwait.sync64 ?
- syncobj->sync64.seqno :
- syncobj->sync32.seqno;
+ queue->syncwait.syncobj = syncobj;
+ }
+
+ value = panthor_syncobj_get_value(queue->syncwait.syncobj);
if (queue->syncwait.gt)
result = value > queue->syncwait.ref;
else
result = value <= queue->syncwait.ref;
- if (result)
- panthor_queue_put_syncwait_obj(queue);
+ if (result) {
+ panthor_syncobj_release(queue->syncwait.syncobj);
+ queue->syncwait.syncobj = NULL;
+ }
return result;
}
@@ -2887,16 +2803,22 @@ static void group_sync_upd_work(struct work_struct *work)
cookie = dma_fence_begin_signalling();
for (queue_idx = 0; queue_idx < group->queue_count; queue_idx++) {
struct panthor_queue *queue = group->queues[queue_idx];
- struct panthor_syncobj_64b *syncobj;
+ void *syncobj;
if (!queue)
continue;
- syncobj = group->syncobjs->kmap + (queue_idx * sizeof(*syncobj));
+ syncobj = group->syncobjs->kmap + (queue_idx * PANTHOR_SYNCOBJ64_SIZE);
spin_lock(&queue->fence_ctx.lock);
list_for_each_entry_safe(job, job_tmp, &queue->fence_ctx.in_flight_jobs, node) {
- if (syncobj->seqno < job->done_fence->seqno)
+ u64 value;
+
+ if (!job->call_info.size)
+ continue;
+
+ value = panthor_syncobj_ptr64_get_value(syncobj);
+ if (value < job->done_fence->seqno)
break;
list_move_tail(&job->node, &done_jobs);
@@ -2928,7 +2850,7 @@ queue_run_job(struct drm_sched_job *sched_job)
ptdev->csif_info.unpreserved_cs_reg_count;
u64 val_reg = addr_reg + 2;
u64 sync_addr = panthor_kernel_bo_gpuva(group->syncobjs) +
- job->queue_idx * sizeof(struct panthor_syncobj_64b);
+ job->queue_idx * PANTHOR_SYNCOBJ64_SIZE;
u32 waitall_mask = GENMASK(sched->sb_slot_count - 1, 0);
struct dma_fence *done_fence;
int ret;
@@ -3289,7 +3211,7 @@ int panthor_group_create(struct panthor_file *pfile,
if (!group->user_submit) {
group->syncobjs = panthor_kernel_bo_create(ptdev, group->vm,
group_args->queues.count *
- sizeof(struct panthor_syncobj_64b),
+ PANTHOR_SYNCOBJ64_SIZE,
DRM_PANTHOR_BO_NO_MMAP,
DRM_PANTHOR_VM_BIND_OP_MAP_NOEXEC |
DRM_PANTHOR_VM_BIND_OP_MAP_UNCACHED,
@@ -3304,7 +3226,7 @@ int panthor_group_create(struct panthor_file *pfile,
goto err_put_group;
memset(group->syncobjs->kmap, 0,
- group_args->queues.count * sizeof(struct panthor_syncobj_64b));
+ group_args->queues.count * PANTHOR_SYNCOBJ64_SIZE);
}
for (i = 0; i < group_args->queues.count; i++) {
diff --git a/drivers/gpu/drm/panthor/panthor_syncobj.c b/drivers/gpu/drm/panthor/panthor_syncobj.c
new file mode 100644
index 000000000000..337f75bfa648
--- /dev/null
+++ b/drivers/gpu/drm/panthor/panthor_syncobj.c
@@ -0,0 +1,167 @@
+// SPDX-License-Identifier: GPL-2.0 or MIT
+/* Copyright 2024 ARM Limited. All rights reserved. */
+
+#include <linux/iosys-map.h>
+
+#include "panthor_device.h"
+#include "panthor_gem.h"
+#include "panthor_mmu.h"
+#include "panthor_syncobj.h"
+
+/**
+ * struct panthor_syncobj_32b - 32-bit FW synchronization object
+ */
+struct panthor_syncobj_32b {
+ /** @value: Value field. */
+ u32 value;
+
+ /**
+ * @error: Error status.
+ *
+ * Not zero on failure.
+ */
+ u32 error;
+};
+
+/**
+ * struct panthor_syncobj_64b - 64-bit FW synchronization object
+ */
+struct panthor_syncobj_64b {
+ /** @value: Value field. */
+ u64 value;
+
+ /**
+ * @error: Error status.
+ *
+ * Not zero on failure.
+ */
+ u32 error;
+
+ /** @pad: MBZ. */
+ u32 pad;
+};
+
+struct panthor_syncobj {
+ /** @bo: Buffer object holding the synchronization object. */
+ struct drm_gem_object *bo;
+
+ /** @offset: Offset of the synchronization object inside @bo. */
+ u64 offset;
+
+ /**
+ * @kmap: Kernel mapping of the buffer object holding the
+ * synchronization object.
+ */
+ void *kmap;
+
+ /** @ptr: CPU ptr to synchronization object */
+ union {
+ struct panthor_syncobj_64b sync64;
+ struct panthor_syncobj_32b sync32;
+ } *ptr;
+
+ /** @sync64: true for 64-bit synchronization object, otherwise 32-bit */
+ bool sync64;
+};
+
+
+
+struct panthor_syncobj *panthor_syncobj_create(struct panthor_device *ptdev,
+ struct panthor_vm *vm, u64 gpu_va,
+ bool sync64)
+{
+ struct panthor_gem_object *bo;
+ struct iosys_map map;
+ struct panthor_syncobj *syncobj;
+ int err;
+
+ syncobj = kzalloc(sizeof(*syncobj), GFP_KERNEL);
+ if (!syncobj) {
+ err = -ENOMEM;
+ goto err;
+ }
+
+ bo = panthor_vm_get_bo_for_va(vm, gpu_va, &syncobj->offset);
+ if (drm_WARN_ON(&ptdev->base, IS_ERR_OR_NULL(bo))) {
+ err = -EINVAL;
+ goto err_free_syncobj;
+ }
+
+ syncobj->bo = &bo->base.base;
+
+ err = drm_gem_vmap_unlocked(syncobj->bo, &map);
+ if (drm_WARN_ON(&ptdev->base, err))
+ goto err_put_gem_object;
+
+ syncobj->kmap = map.vaddr;
+ syncobj->ptr = syncobj->kmap + syncobj->offset;
+ syncobj->sync64 = sync64;
+
+ return syncobj;
+
+err_put_gem_object:
+ drm_gem_object_put(syncobj->bo);
+err_free_syncobj:
+ kfree(syncobj);
+err:
+ return ERR_PTR(err);
+}
+
+void panthor_syncobj_release(struct panthor_syncobj *syncobj)
+{
+ if (syncobj) {
+ struct iosys_map map = IOSYS_MAP_INIT_VADDR(syncobj->kmap);
+
+ drm_gem_vunmap_unlocked(syncobj->bo, &map);
+ drm_gem_object_put(syncobj->bo);
+ kfree(syncobj);
+ }
+}
+
+u64 panthor_syncobj_get_value(struct panthor_syncobj *syncobj)
+{
+ return syncobj->sync64 ?
+ syncobj->ptr->sync64.value :
+ syncobj->ptr->sync32.value;
+}
+
+u32 panthor_syncobj_get_error(struct panthor_syncobj *syncobj)
+{
+ return syncobj->sync64 ?
+ syncobj->ptr->sync64.error :
+ syncobj->ptr->sync32.error;
+}
+
+void panthor_syncobj_signal(struct panthor_syncobj *syncobj, u64 value)
+{
+ if (syncobj->sync64)
+ syncobj->ptr->sync64.value = value;
+ else
+ syncobj->ptr->sync32.value = (u32)value;
+}
+
+void panthor_syncobj_signal_with_error(struct panthor_syncobj *syncobj, u64 value, u32 error)
+{
+ if (syncobj->sync64) {
+ syncobj->ptr->sync64.value = value;
+ syncobj->ptr->sync64.error = error;
+ } else {
+ syncobj->ptr->sync32.value = (u32)value;
+ syncobj->ptr->sync32.error = error;
+ }
+}
+
+u64 panthor_syncobj_ptr64_get_value(void *syncobj_ptr)
+{
+ struct panthor_syncobj_64b *syncobj = syncobj_ptr;
+
+ return syncobj->value;
+}
+
+void panthor_syncobj_ptr64_signal_with_error(void *syncobj_ptr, u64 value, u32 error)
+{
+ struct panthor_syncobj_64b *syncobj = syncobj_ptr;
+
+ syncobj->value = value;
+ syncobj->error = error;
+}
diff --git a/drivers/gpu/drm/panthor/panthor_syncobj.h b/drivers/gpu/drm/panthor/panthor_syncobj.h
new file mode 100644
index 000000000000..018cfc87cdaa
--- /dev/null
+++ b/drivers/gpu/drm/panthor/panthor_syncobj.h
@@ -0,0 +1,27 @@
+/* SPDX-License-Identifier: GPL-2.0 or MIT */
+/* Copyright 2024 ARM Limited. All rights reserved. */
+
+#ifndef __PANTHOR_SYNCOBJ_H__
+#define __PANTHOR_SYNCOBJ_H__
+
+#define PANTHOR_SYNCOBJ32_SIZE 8
+#define PANTHOR_SYNCOBJ64_SIZE 16
+
+struct panthor_syncobj;
+struct panthor_vm;
+
+struct panthor_syncobj *panthor_syncobj_create(struct panthor_device *ptdev,
+ struct panthor_vm *vm, u64 gpu_va,
+ bool sync64);
+void panthor_syncobj_release(struct panthor_syncobj *syncobj);
+
+u64 panthor_syncobj_get_value(struct panthor_syncobj *syncobj);
+u32 panthor_syncobj_get_error(struct panthor_syncobj *syncobj);
+
+void panthor_syncobj_signal(struct panthor_syncobj *syncobj, u64 value);
+void panthor_syncobj_signal_with_error(struct panthor_syncobj *syncobj, u64 value, u32 error);
+
+u64 panthor_syncobj_ptr64_get_value(void *syncobj_ptr);
+void panthor_syncobj_ptr64_signal_with_error(void *syncobj_ptr, u64 value, u32 error);
+
+#endif
--
2.45.0
