Re: [PATCH v2] drm/amdgpu: refine amdgpu_bo_create_kernel_at()

[Christian König <christian.koenig@xxxxxxx>]

Am 28.07.23 um 09:00 schrieb Lazar, Lijo:
> On 7/28/2023 11:34 AM, Lang Yu wrote:
> > Use amdgpu_bo_create_reserved() to create a BO in VRAM
> > domain would fail if requested VRAM size is large(>128MB)
> > on APU which usually has a default 512MB VRAM.
> >
> > That's because VRAM is framgented after several allocations.
> >
> > The approach is using amdgpu_bo_create_reserved() to
> > create a BO in CPU domain first, it will always succeed.
> >
> > v2: Don't overwrite the contents at specific offset.
> >
> > Signed-off-by: Lang Yu <Lang.Yu@xxxxxxx>
> > ---
> >   drivers/gpu/drm/amd/amdgpu/amdgpu_object.c | 24 ++++++++++++++++------
> >   drivers/gpu/drm/amd/amdgpu/amdgpu_object.h |  3 ++-
> >   drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c    | 15 +++++++-------
> >   drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c   |  2 +-
> >   4 files changed, 28 insertions(+), 16 deletions(-)
> >
> > diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c 
> > b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c
> > index ff73cc11d47e..fa30e96f27d0 100644
> > --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c
> > +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c
> > @@ -358,6 +358,7 @@ int amdgpu_bo_create_kernel(struct amdgpu_device 
> > *adev,
> >    * @offset: offset of the BO
> >    * @size: size of the BO
> >    * @bo_ptr:  used to initialize BOs in structures
> > + * @gpu_addr: GPU addr of the pinned BO
> >    * @cpu_addr: optional CPU address mapping
> >    *
> >    * Creates a kernel BO at a specific offset in VRAM.
> > @@ -367,7 +368,8 @@ int amdgpu_bo_create_kernel(struct amdgpu_device 
> > *adev,
> >    */
> >   int amdgpu_bo_create_kernel_at(struct amdgpu_device *adev,
> >                      uint64_t offset, uint64_t size,
> > -                   struct amdgpu_bo **bo_ptr, void **cpu_addr)
> > +                   struct amdgpu_bo **bo_ptr,
> > +                   u64 *gpu_addr, void **cpu_addr)
>
> A generic question (not considering other details in this patch) - 
> this API is literally asking to create BO at a particular GPU VRAM 
> offset. The offset goes in as the input, so why does it need to return 
> a GPU offset?

Yeah, that's exactly why I said that the intention of the function was 
misunderstood.

This change here doesn't seem to make much sense.

Regards,
Christian.

> Thanks,
> Lijo
>
> >   {
> >       struct ttm_operation_ctx ctx = { false, false };
> >       unsigned int i;
> > @@ -377,32 +379,42 @@ int amdgpu_bo_create_kernel_at(struct 
> > amdgpu_device *adev,
> >       size = ALIGN(size, PAGE_SIZE);
> >         r = amdgpu_bo_create_reserved(adev, size, PAGE_SIZE,
> > -                      AMDGPU_GEM_DOMAIN_VRAM, bo_ptr, NULL,
> > -                      cpu_addr);
> > +                      AMDGPU_GEM_DOMAIN_CPU,
> > +                      bo_ptr, NULL, NULL);
> >       if (r)
> >           return r;
> >         if ((*bo_ptr) == NULL)
> >           return 0;
> >   +    (*bo_ptr)->flags = AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS;
> > +    (*bo_ptr)->flags |= cpu_addr ? 
> > AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED
> > +        : AMDGPU_GEM_CREATE_NO_CPU_ACCESS;
> > +
> >       /*
> >        * Remove the original mem node and create a new one at the 
> > request
> >        * position.
> >        */
> > -    if (cpu_addr)
> > -        amdgpu_bo_kunmap(*bo_ptr);
> > -
> >       ttm_resource_free(&(*bo_ptr)->tbo, &(*bo_ptr)->tbo.resource);
> >         for (i = 0; i < (*bo_ptr)->placement.num_placement; ++i) {
> >           (*bo_ptr)->placements[i].fpfn = offset >> PAGE_SHIFT;
> >           (*bo_ptr)->placements[i].lpfn = (offset + size) >> PAGE_SHIFT;
> > +        (*bo_ptr)->placements[i].mem_type = TTM_PL_VRAM;
> > +        (*bo_ptr)->placements[i].flags = TTM_PL_FLAG_CONTIGUOUS;
> > +
> > +        if (!((*bo_ptr)->flags & 
> > AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED))
> > +            (*bo_ptr)->placements[i].flags |= TTM_PL_FLAG_TOPDOWN;
> >       }
> > +
> >       r = ttm_bo_mem_space(&(*bo_ptr)->tbo, &(*bo_ptr)->placement,
> >                    &(*bo_ptr)->tbo.resource, &ctx);
> >       if (r)
> >           goto error;
> >   +    if (gpu_addr)
> > +        *gpu_addr = amdgpu_bo_gpu_offset(*bo_ptr);
> > +
> >       if (cpu_addr) {
> >           r = amdgpu_bo_kmap(*bo_ptr, cpu_addr);
> >           if (r)
> > diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h 
> > b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h
> > index 5d3440d719e4..8f5b5664a1b6 100644
> > --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h
> > +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h
> > @@ -315,7 +315,8 @@ int amdgpu_bo_create_kernel(struct amdgpu_device 
> > *adev,
> >                   u64 *gpu_addr, void **cpu_addr);
> >   int amdgpu_bo_create_kernel_at(struct amdgpu_device *adev,
> >                      uint64_t offset, uint64_t size,
> > -                   struct amdgpu_bo **bo_ptr, void **cpu_addr);
> > +                   struct amdgpu_bo **bo_ptr,
> > +                   u64 *gpu_addr, void **cpu_addr);
> >   int amdgpu_bo_create_user(struct amdgpu_device *adev,
> >                 struct amdgpu_bo_param *bp,
> >                 struct amdgpu_bo_user **ubo_ptr);
> > diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c 
> > b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
> > index 7c6dd3de1867..a210c243dac0 100644
> > --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
> > +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
> > @@ -1619,7 +1619,7 @@ static int 
> > amdgpu_ttm_fw_reserve_vram_init(struct amdgpu_device *adev)
> > adev->mman.fw_vram_usage_start_offset,
> >                         adev->mman.fw_vram_usage_size,
> > &adev->mman.fw_vram_usage_reserved_bo,
> > -                      &adev->mman.fw_vram_usage_va);
> > +                      NULL, &adev->mman.fw_vram_usage_va);
> >   }
> >     /**
> > @@ -1644,7 +1644,7 @@ static int 
> > amdgpu_ttm_drv_reserve_vram_init(struct amdgpu_device *adev)
> > adev->mman.drv_vram_usage_start_offset,
> >                         adev->mman.drv_vram_usage_size,
> > &adev->mman.drv_vram_usage_reserved_bo,
> > -                      &adev->mman.drv_vram_usage_va);
> > +                      NULL, &adev->mman.drv_vram_usage_va);
> >   }
> >     /*
> > @@ -1729,8 +1729,7 @@ static int amdgpu_ttm_reserve_tmr(struct 
> > amdgpu_device *adev)
> >           ret = amdgpu_bo_create_kernel_at(adev,
> >                            ctx->c2p_train_data_offset,
> >                            ctx->train_data_size,
> > -                         &ctx->c2p_bo,
> > -                         NULL);
> > +                         &ctx->c2p_bo, NULL, NULL);
> >           if (ret) {
> >               DRM_ERROR("alloc c2p_bo failed(%d)!\n", ret);
> >               amdgpu_ttm_training_reserve_vram_fini(adev);
> > @@ -1742,7 +1741,7 @@ static int amdgpu_ttm_reserve_tmr(struct 
> > amdgpu_device *adev)
> >       if (!adev->gmc.is_app_apu) {
> >           ret = amdgpu_bo_create_kernel_at(
> >               adev, adev->gmc.real_vram_size - reserve_size,
> > -            reserve_size, &adev->mman.fw_reserved_memory, NULL);
> > +            reserve_size, &adev->mman.fw_reserved_memory, NULL, NULL);
> >           if (ret) {
> >               DRM_ERROR("alloc tmr failed(%d)!\n", ret);
> > amdgpu_bo_free_kernel(&adev->mman.fw_reserved_memory,
> > @@ -1885,14 +1884,14 @@ int amdgpu_ttm_init(struct amdgpu_device *adev)
> >           r = amdgpu_bo_create_kernel_at(adev, 0,
> >                              adev->mman.stolen_vga_size,
> > &adev->mman.stolen_vga_memory,
> > -                           NULL);
> > +                           NULL, NULL);
> >           if (r)
> >               return r;
> >             r = amdgpu_bo_create_kernel_at(adev, 
> > adev->mman.stolen_vga_size,
> >                              adev->mman.stolen_extended_size,
> > &adev->mman.stolen_extended_memory,
> > -                           NULL);
> > +                           NULL, NULL);
> >             if (r)
> >               return r;
> > @@ -1901,7 +1900,7 @@ int amdgpu_ttm_init(struct amdgpu_device *adev)
> > adev->mman.stolen_reserved_offset,
> >                              adev->mman.stolen_reserved_size,
> > &adev->mman.stolen_reserved_memory,
> > -                           NULL);
> > +                           NULL, NULL);
> >           if (r)
> >               return r;
> >       } else {
> > diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c 
> > b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
> > index 41aa853a07d2..b93b42b916ce 100644
> > --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
> > +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
> > @@ -397,7 +397,7 @@ static void amdgpu_virt_ras_reserve_bps(struct 
> > amdgpu_device *adev)
> >            */
> >           if (amdgpu_bo_create_kernel_at(adev, bp << 
> > AMDGPU_GPU_PAGE_SHIFT,
> >                              AMDGPU_GPU_PAGE_SIZE,
> > -                           &bo, NULL))
> > +                           &bo, NULL, NULL))
> >               DRM_DEBUG("RAS WARN: reserve vram for retired page %llx 
> > fail\n", bp);
> >             data->bps_bo[i] = bo;