I think at some point ROCr could start using libdrm APIs for memory
management and user mode queues on kernels and GPUs that support this. I
think more work is required on the memory management side, though. ROCr
would fallback to libhsakmt on older kernels and older GPUs (pre-GFX11).
Regards,
Felix
Am 2023-01-03 um 14:52 schrieb Alex Deucher:
On Tue, Jan 3, 2023 at 2:25 PM Liu, Shaoyun <Shaoyun.Liu@xxxxxxx> wrote:
[AMD Official Use Only - General]
What about the existing rocm apps that already use the hsakmt APIs for user queue ?
We'd have to keep both APIs around for existing chips for backwards
compatibility.
Alex
Shaoyun.liu
-----Original Message-----
From: Alex Deucher <alexdeucher@xxxxxxxxx>
Sent: Tuesday, January 3, 2023 2:22 PM
To: Liu, Shaoyun <Shaoyun.Liu@xxxxxxx>
Cc: Kuehling, Felix <Felix.Kuehling@xxxxxxx>; Sharma, Shashank <Shashank.Sharma@xxxxxxx>; amd-gfx@xxxxxxxxxxxxxxxxxxxxx; Deucher, Alexander <Alexander.Deucher@xxxxxxx>; Koenig, Christian <Christian.Koenig@xxxxxxx>; Yadav, Arvind <Arvind.Yadav@xxxxxxx>; Paneer Selvam, Arunpravin <Arunpravin.PaneerSelvam@xxxxxxx>
Subject: Re: [RFC 1/7] drm/amdgpu: UAPI for user queue management
On Tue, Jan 3, 2023 at 2:17 PM Liu, Shaoyun <Shaoyun.Liu@xxxxxxx> wrote:
[AMD Official Use Only - General]
Hsakmt has the interfaces for compute user queue. Do we want a unify API for both graphic and compute ?
Yeah, that is the eventual goal, hence the flag for AQL vs PM4.
Alex
Regards
Shaoyun.liu
-----Original Message-----
From: amd-gfx <amd-gfx-bounces@xxxxxxxxxxxxxxxxxxxxx> On Behalf Of
Felix Kuehling
Sent: Tuesday, January 3, 2023 1:30 PM
To: Sharma, Shashank <Shashank.Sharma@xxxxxxx>;
amd-gfx@xxxxxxxxxxxxxxxxxxxxx
Cc: Deucher, Alexander <Alexander.Deucher@xxxxxxx>; Koenig, Christian
<Christian.Koenig@xxxxxxx>; Yadav, Arvind <Arvind.Yadav@xxxxxxx>;
Paneer Selvam, Arunpravin <Arunpravin.PaneerSelvam@xxxxxxx>
Subject: Re: [RFC 1/7] drm/amdgpu: UAPI for user queue management
Am 2022-12-23 um 14:36 schrieb Shashank Sharma:
From: Alex Deucher <alexander.deucher@xxxxxxx>
This patch intorduces new UAPI/IOCTL for usermode graphics queue.
The userspace app will fill this structure and request the graphics
driver to add a graphics work queue for it. The output of this UAPI
is a queue id.
This UAPI maps the queue into GPU, so the graphics app can start
submitting work to the queue as soon as the call returns.
Cc: Alex Deucher <alexander.deucher@xxxxxxx>
Cc: Christian Koenig <christian.koenig@xxxxxxx>
Signed-off-by: Alex Deucher <alexander.deucher@xxxxxxx>
Signed-off-by: Shashank Sharma <shashank.sharma@xxxxxxx>
---
include/uapi/drm/amdgpu_drm.h | 52 +++++++++++++++++++++++++++++++++++
1 file changed, 52 insertions(+)
diff --git a/include/uapi/drm/amdgpu_drm.h
b/include/uapi/drm/amdgpu_drm.h index 0d93ec132ebb..a3d0dd6f62c5
100644
--- a/include/uapi/drm/amdgpu_drm.h
+++ b/include/uapi/drm/amdgpu_drm.h
@@ -54,6 +54,7 @@ extern "C" {
#define DRM_AMDGPU_VM 0x13
#define DRM_AMDGPU_FENCE_TO_HANDLE 0x14
#define DRM_AMDGPU_SCHED 0x15
+#define DRM_AMDGPU_USERQ 0x16
#define DRM_IOCTL_AMDGPU_GEM_CREATE DRM_IOWR(DRM_COMMAND_BASE + DRM_AMDGPU_GEM_CREATE, union drm_amdgpu_gem_create)
#define DRM_IOCTL_AMDGPU_GEM_MMAP DRM_IOWR(DRM_COMMAND_BASE + DRM_AMDGPU_GEM_MMAP, union drm_amdgpu_gem_mmap)
@@ -71,6 +72,7 @@ extern "C" {
#define DRM_IOCTL_AMDGPU_VM DRM_IOWR(DRM_COMMAND_BASE + DRM_AMDGPU_VM, union drm_amdgpu_vm)
#define DRM_IOCTL_AMDGPU_FENCE_TO_HANDLE DRM_IOWR(DRM_COMMAND_BASE + DRM_AMDGPU_FENCE_TO_HANDLE, union drm_amdgpu_fence_to_handle)
#define DRM_IOCTL_AMDGPU_SCHED DRM_IOW(DRM_COMMAND_BASE + DRM_AMDGPU_SCHED, union drm_amdgpu_sched)
+#define DRM_IOCTL_AMDGPU_USERQ DRM_IOW(DRM_COMMAND_BASE + DRM_AMDGPU_USERQ, union drm_amdgpu_userq)
/**
* DOC: memory domains
@@ -288,6 +290,56 @@ union drm_amdgpu_ctx {
union drm_amdgpu_ctx_out out;
};
+/* user queue IOCTL */
+#define AMDGPU_USERQ_OP_CREATE 1
+#define AMDGPU_USERQ_OP_FREE 2
+
+#define AMDGPU_USERQ_MQD_FLAGS_SECURE (1 << 0)
What does "secure" mean here? I don't see this flag referenced anywhere in the rest of the patch series.
Regards,
Felix
+#define AMDGPU_USERQ_MQD_FLAGS_AQL (1 << 1)
+
+struct drm_amdgpu_userq_mqd {
+ /** Flags: AMDGPU_USERQ_MQD_FLAGS_* */
+ __u32 flags;
+ /** IP type: AMDGPU_HW_IP_* */
+ __u32 ip_type;
+ /** GEM object handle */
+ __u32 doorbell_handle;
+ /** Doorbell offset in dwords */
+ __u32 doorbell_offset;
+ /** GPU virtual address of the queue */
+ __u64 queue_va;
+ /** Size of the queue in bytes */
+ __u64 queue_size;
+ /** GPU virtual address of the rptr */
+ __u64 rptr_va;
+ /** GPU virtual address of the wptr */
+ __u64 wptr_va;
+};
+
+struct drm_amdgpu_userq_in {
+ /** AMDGPU_USERQ_OP_* */
+ __u32 op;
+ /** Flags */
+ __u32 flags;
+ /** Context handle to associate the queue with */
+ __u32 ctx_id;
+ __u32 pad;
+ /** Queue descriptor */
+ struct drm_amdgpu_userq_mqd mqd; };
+
+struct drm_amdgpu_userq_out {
+ /** Queue handle */
+ __u32 q_id;
+ /** Flags */
+ __u32 flags;
+};
+
+union drm_amdgpu_userq {
+ struct drm_amdgpu_userq_in in;
+ struct drm_amdgpu_userq_out out; };
+
/* vm ioctl */
#define AMDGPU_VM_OP_RESERVE_VMID 1
#define AMDGPU_VM_OP_UNRESERVE_VMID 2
