The flags have been refactored--good. Acked-by: Luben Tuikov <luben.tuikov@xxxxxxx> On 2022-11-29 02:10, jiadong.zhu@xxxxxxx wrote: > From: "Jiadong.Zhu" <Jiadong.Zhu@xxxxxxx> > > 1. Modify the unmap_queue package on gfx9. Add trailing fence to track the > preemption done. > 2. Modify emit_ce_meta emit_de_meta functions for the resumed ibs. > > v2: Restyle code not to use ternary operator. > v3: Modify code format. > v4: Enable Mid-Command Buffer Preemption for gfx9 by default. > v5: Optimize the flag bit set for emit_fence. > v6: Modify log message for preemption timeout. > > Cc: Christian Koenig <Christian.Koenig@xxxxxxx> > Cc: Michel Dänzer <michel@xxxxxxxxxxx> > Cc: Luben Tuikov <Luben.Tuikov@xxxxxxx> > Signed-off-by: Jiadong.Zhu <Jiadong.Zhu@xxxxxxx> > Acked-by: Christian König <christian.koenig@xxxxxxx> > Acked-by: Huang Rui <ray.huang@xxxxxxx> > --- > drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h | 1 + > drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c | 182 +++++++++++++++++++---- > drivers/gpu/drm/amd/amdgpu/soc15d.h | 2 + > 3 files changed, 156 insertions(+), 29 deletions(-) > > diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h > index a744aa9bac95..073e767c057f 100644 > --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h > +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h > @@ -60,6 +60,7 @@ enum amdgpu_ring_priority_level { > #define AMDGPU_FENCE_FLAG_64BIT (1 << 0) > #define AMDGPU_FENCE_FLAG_INT (1 << 1) > #define AMDGPU_FENCE_FLAG_TC_WB_ONLY (1 << 2) > +#define AMDGPU_FENCE_FLAG_EXEC (1 << 3) > > #define to_amdgpu_ring(s) container_of((s), struct amdgpu_ring, sched) > > diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c > index ec80047b1b41..ec528b1d82e5 100644 > --- a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c > +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c > @@ -755,7 +755,7 @@ static void gfx_v9_0_set_rlc_funcs(struct amdgpu_device *adev); > static int gfx_v9_0_get_cu_info(struct amdgpu_device *adev, > struct amdgpu_cu_info *cu_info); > static uint64_t gfx_v9_0_get_gpu_clock_counter(struct amdgpu_device *adev); > -static void gfx_v9_0_ring_emit_de_meta(struct amdgpu_ring *ring); > +static void gfx_v9_0_ring_emit_de_meta(struct amdgpu_ring *ring, bool resume); > static u64 gfx_v9_0_ring_get_rptr_compute(struct amdgpu_ring *ring); > static void gfx_v9_0_query_ras_error_count(struct amdgpu_device *adev, > void *ras_error_status); > @@ -828,9 +828,10 @@ static void gfx_v9_0_kiq_unmap_queues(struct amdgpu_ring *kiq_ring, > PACKET3_UNMAP_QUEUES_DOORBELL_OFFSET0(ring->doorbell_index)); > > if (action == PREEMPT_QUEUES_NO_UNMAP) { > - amdgpu_ring_write(kiq_ring, lower_32_bits(gpu_addr)); > - amdgpu_ring_write(kiq_ring, upper_32_bits(gpu_addr)); > - amdgpu_ring_write(kiq_ring, seq); > + amdgpu_ring_write(kiq_ring, lower_32_bits(ring->wptr & ring->buf_mask)); > + amdgpu_ring_write(kiq_ring, 0); > + amdgpu_ring_write(kiq_ring, 0); > + > } else { > amdgpu_ring_write(kiq_ring, 0); > amdgpu_ring_write(kiq_ring, 0); > @@ -5326,11 +5327,17 @@ static void gfx_v9_0_ring_emit_ib_gfx(struct amdgpu_ring *ring, > > control |= ib->length_dw | (vmid << 24); > > - if (amdgpu_sriov_vf(ring->adev) && (ib->flags & AMDGPU_IB_FLAG_PREEMPT)) { > + if (ib->flags & AMDGPU_IB_FLAG_PREEMPT) { > control |= INDIRECT_BUFFER_PRE_ENB(1); > > + if (flags & AMDGPU_IB_PREEMPTED) > + control |= INDIRECT_BUFFER_PRE_RESUME(1); > + > if (!(ib->flags & AMDGPU_IB_FLAG_CE) && vmid) > - gfx_v9_0_ring_emit_de_meta(ring); > + gfx_v9_0_ring_emit_de_meta(ring, > + (!amdgpu_sriov_vf(ring->adev) && > + flags & AMDGPU_IB_PREEMPTED) ? > + true : false); > } > > amdgpu_ring_write(ring, header); > @@ -5385,17 +5392,24 @@ static void gfx_v9_0_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, > bool write64bit = flags & AMDGPU_FENCE_FLAG_64BIT; > bool int_sel = flags & AMDGPU_FENCE_FLAG_INT; > bool writeback = flags & AMDGPU_FENCE_FLAG_TC_WB_ONLY; > + bool exec = flags & AMDGPU_FENCE_FLAG_EXEC; > + uint32_t dw2 = 0; > > /* RELEASE_MEM - flush caches, send int */ > amdgpu_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 6)); > - amdgpu_ring_write(ring, ((writeback ? (EOP_TC_WB_ACTION_EN | > - EOP_TC_NC_ACTION_EN) : > - (EOP_TCL1_ACTION_EN | > - EOP_TC_ACTION_EN | > - EOP_TC_WB_ACTION_EN | > - EOP_TC_MD_ACTION_EN)) | > - EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) | > - EVENT_INDEX(5))); > + > + if (writeback) { > + dw2 = EOP_TC_NC_ACTION_EN; > + } else { > + dw2 = EOP_TCL1_ACTION_EN | EOP_TC_ACTION_EN | > + EOP_TC_MD_ACTION_EN; > + } > + dw2 |= EOP_TC_WB_ACTION_EN | EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) | > + EVENT_INDEX(5); > + if (exec) > + dw2 |= EOP_EXEC; > + > + amdgpu_ring_write(ring, dw2); > amdgpu_ring_write(ring, DATA_SEL(write64bit ? 2 : 1) | INT_SEL(int_sel ? 2 : 0)); > > /* > @@ -5500,33 +5514,135 @@ static void gfx_v9_ring_emit_sb(struct amdgpu_ring *ring) > amdgpu_ring_write(ring, 0); > } > > -static void gfx_v9_0_ring_emit_ce_meta(struct amdgpu_ring *ring) > +static void gfx_v9_0_ring_emit_ce_meta(struct amdgpu_ring *ring, bool resume) > { > + struct amdgpu_device *adev = ring->adev; > struct v9_ce_ib_state ce_payload = {0}; > - uint64_t csa_addr; > + uint64_t offset, ce_payload_gpu_addr; > + void *ce_payload_cpu_addr; > int cnt; > > cnt = (sizeof(ce_payload) >> 2) + 4 - 2; > - csa_addr = amdgpu_csa_vaddr(ring->adev); > + > + if (ring->is_mes_queue) { > + offset = offsetof(struct amdgpu_mes_ctx_meta_data, > + gfx[0].gfx_meta_data) + > + offsetof(struct v9_gfx_meta_data, ce_payload); > + ce_payload_gpu_addr = > + amdgpu_mes_ctx_get_offs_gpu_addr(ring, offset); > + ce_payload_cpu_addr = > + amdgpu_mes_ctx_get_offs_cpu_addr(ring, offset); > + } else { > + offset = offsetof(struct v9_gfx_meta_data, ce_payload); > + ce_payload_gpu_addr = amdgpu_csa_vaddr(ring->adev) + offset; > + ce_payload_cpu_addr = adev->virt.csa_cpu_addr + offset; > + } > > amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, cnt)); > amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(2) | > WRITE_DATA_DST_SEL(8) | > WR_CONFIRM) | > WRITE_DATA_CACHE_POLICY(0)); > - amdgpu_ring_write(ring, lower_32_bits(csa_addr + offsetof(struct v9_gfx_meta_data, ce_payload))); > - amdgpu_ring_write(ring, upper_32_bits(csa_addr + offsetof(struct v9_gfx_meta_data, ce_payload))); > - amdgpu_ring_write_multiple(ring, (void *)&ce_payload, sizeof(ce_payload) >> 2); > + amdgpu_ring_write(ring, lower_32_bits(ce_payload_gpu_addr)); > + amdgpu_ring_write(ring, upper_32_bits(ce_payload_gpu_addr)); > + > + if (resume) > + amdgpu_ring_write_multiple(ring, ce_payload_cpu_addr, > + sizeof(ce_payload) >> 2); > + else > + amdgpu_ring_write_multiple(ring, (void *)&ce_payload, > + sizeof(ce_payload) >> 2); > +} > + > +static int gfx_v9_0_ring_preempt_ib(struct amdgpu_ring *ring) > +{ > + int i, r = 0; > + struct amdgpu_device *adev = ring->adev; > + struct amdgpu_kiq *kiq = &adev->gfx.kiq; > + struct amdgpu_ring *kiq_ring = &kiq->ring; > + unsigned long flags; > + > + if (!kiq->pmf || !kiq->pmf->kiq_unmap_queues) > + return -EINVAL; > + > + spin_lock_irqsave(&kiq->ring_lock, flags); > + > + if (amdgpu_ring_alloc(kiq_ring, kiq->pmf->unmap_queues_size)) { > + spin_unlock_irqrestore(&kiq->ring_lock, flags); > + return -ENOMEM; > + } > + > + /* assert preemption condition */ > + amdgpu_ring_set_preempt_cond_exec(ring, false); > + > + ring->trail_seq += 1; > + amdgpu_ring_alloc(ring, 13); > + gfx_v9_0_ring_emit_fence(ring, ring->trail_fence_gpu_addr, > + ring->trail_seq, AMDGPU_FENCE_FLAG_EXEC); > + /*reset the CP_VMID_PREEMPT after trailing fence*/ > + amdgpu_ring_emit_wreg(ring, > + SOC15_REG_OFFSET(GC, 0, mmCP_VMID_PREEMPT), > + 0x0); > + > + /* assert IB preemption, emit the trailing fence */ > + kiq->pmf->kiq_unmap_queues(kiq_ring, ring, PREEMPT_QUEUES_NO_UNMAP, > + ring->trail_fence_gpu_addr, > + ring->trail_seq); > + > + amdgpu_ring_commit(kiq_ring); > + spin_unlock_irqrestore(&kiq->ring_lock, flags); > + > + /* poll the trailing fence */ > + for (i = 0; i < adev->usec_timeout; i++) { > + if (ring->trail_seq == > + le32_to_cpu(*ring->trail_fence_cpu_addr)) > + break; > + udelay(1); > + } > + > + if (i >= adev->usec_timeout) { > + r = -EINVAL; > + DRM_WARN("ring %d timeout to preempt ib\n", ring->idx); > + } > + > + amdgpu_ring_commit(ring); > + > + /* deassert preemption condition */ > + amdgpu_ring_set_preempt_cond_exec(ring, true); > + return r; > } > > -static void gfx_v9_0_ring_emit_de_meta(struct amdgpu_ring *ring) > +static void gfx_v9_0_ring_emit_de_meta(struct amdgpu_ring *ring, bool resume) > { > + struct amdgpu_device *adev = ring->adev; > struct v9_de_ib_state de_payload = {0}; > - uint64_t csa_addr, gds_addr; > + uint64_t offset, gds_addr, de_payload_gpu_addr; > + void *de_payload_cpu_addr; > int cnt; > > - csa_addr = amdgpu_csa_vaddr(ring->adev); > - gds_addr = csa_addr + 4096; > + if (ring->is_mes_queue) { > + offset = offsetof(struct amdgpu_mes_ctx_meta_data, > + gfx[0].gfx_meta_data) + > + offsetof(struct v9_gfx_meta_data, de_payload); > + de_payload_gpu_addr = > + amdgpu_mes_ctx_get_offs_gpu_addr(ring, offset); > + de_payload_cpu_addr = > + amdgpu_mes_ctx_get_offs_cpu_addr(ring, offset); > + > + offset = offsetof(struct amdgpu_mes_ctx_meta_data, > + gfx[0].gds_backup) + > + offsetof(struct v9_gfx_meta_data, de_payload); > + gds_addr = amdgpu_mes_ctx_get_offs_gpu_addr(ring, offset); > + } else { > + offset = offsetof(struct v9_gfx_meta_data, de_payload); > + de_payload_gpu_addr = amdgpu_csa_vaddr(ring->adev) + offset; > + de_payload_cpu_addr = adev->virt.csa_cpu_addr + offset; > + > + gds_addr = ALIGN(amdgpu_csa_vaddr(ring->adev) + > + AMDGPU_CSA_SIZE - adev->gds.gds_size, > + PAGE_SIZE); > + } > + > de_payload.gds_backup_addrlo = lower_32_bits(gds_addr); > de_payload.gds_backup_addrhi = upper_32_bits(gds_addr); > > @@ -5536,9 +5652,15 @@ static void gfx_v9_0_ring_emit_de_meta(struct amdgpu_ring *ring) > WRITE_DATA_DST_SEL(8) | > WR_CONFIRM) | > WRITE_DATA_CACHE_POLICY(0)); > - amdgpu_ring_write(ring, lower_32_bits(csa_addr + offsetof(struct v9_gfx_meta_data, de_payload))); > - amdgpu_ring_write(ring, upper_32_bits(csa_addr + offsetof(struct v9_gfx_meta_data, de_payload))); > - amdgpu_ring_write_multiple(ring, (void *)&de_payload, sizeof(de_payload) >> 2); > + amdgpu_ring_write(ring, lower_32_bits(de_payload_gpu_addr)); > + amdgpu_ring_write(ring, upper_32_bits(de_payload_gpu_addr)); > + > + if (resume) > + amdgpu_ring_write_multiple(ring, de_payload_cpu_addr, > + sizeof(de_payload) >> 2); > + else > + amdgpu_ring_write_multiple(ring, (void *)&de_payload, > + sizeof(de_payload) >> 2); > } > > static void gfx_v9_0_ring_emit_frame_cntl(struct amdgpu_ring *ring, bool start, > @@ -5554,8 +5676,9 @@ static void gfx_v9_ring_emit_cntxcntl(struct amdgpu_ring *ring, uint32_t flags) > { > uint32_t dw2 = 0; > > - if (amdgpu_sriov_vf(ring->adev)) > - gfx_v9_0_ring_emit_ce_meta(ring); > + gfx_v9_0_ring_emit_ce_meta(ring, > + (!amdgpu_sriov_vf(ring->adev) && > + flags & AMDGPU_IB_PREEMPTED) ? true : false); > > dw2 |= 0x80000000; /* set load_enable otherwise this package is just NOPs */ > if (flags & AMDGPU_HAVE_CTX_SWITCH) { > @@ -6908,6 +7031,7 @@ static const struct amdgpu_ring_funcs gfx_v9_0_ring_funcs_gfx = { > .emit_cntxcntl = gfx_v9_ring_emit_cntxcntl, > .init_cond_exec = gfx_v9_0_ring_emit_init_cond_exec, > .patch_cond_exec = gfx_v9_0_ring_emit_patch_cond_exec, > + .preempt_ib = gfx_v9_0_ring_preempt_ib, > .emit_frame_cntl = gfx_v9_0_ring_emit_frame_cntl, > .emit_wreg = gfx_v9_0_ring_emit_wreg, > .emit_reg_wait = gfx_v9_0_ring_emit_reg_wait, > diff --git a/drivers/gpu/drm/amd/amdgpu/soc15d.h b/drivers/gpu/drm/amd/amdgpu/soc15d.h > index 799925d22fc8..2357ff39323f 100644 > --- a/drivers/gpu/drm/amd/amdgpu/soc15d.h > +++ b/drivers/gpu/drm/amd/amdgpu/soc15d.h > @@ -162,6 +162,7 @@ > * 2 - Bypass > */ > #define INDIRECT_BUFFER_PRE_ENB(x) ((x) << 21) > +#define INDIRECT_BUFFER_PRE_RESUME(x) ((x) << 30) > #define PACKET3_COPY_DATA 0x40 > #define PACKET3_PFP_SYNC_ME 0x42 > #define PACKET3_COND_WRITE 0x45 > @@ -184,6 +185,7 @@ > #define EOP_TC_ACTION_EN (1 << 17) /* L2 */ > #define EOP_TC_NC_ACTION_EN (1 << 19) > #define EOP_TC_MD_ACTION_EN (1 << 21) /* L2 metadata */ > +#define EOP_EXEC (1 << 28) /* For Trailing Fence */ > > #define DATA_SEL(x) ((x) << 29) > /* 0 - discard