Re: [PATCH 09/16] drm/amdgpu: create kernel doorbell page

[Shashank Sharma <shashank.sharma@xxxxxxx>]

On 30/03/2023 13:30, Christian König wrote:
> Am 29.03.23 um 17:47 schrieb Shashank Sharma:
> > From: Shashank Sharma <contactshashanksharma@xxxxxxxxx>
> >
> > This patch:
> > - creates a doorbell page for graphics driver usages.
> > - removes the adev->doorbell.ptr variable, replaces it with
> >    kernel-doorbell-bo's cpu address.
> >
> > Cc: Alex Deucher <alexander.deucher@xxxxxxx>
> > Cc: Christian Koenig <christian.koenig@xxxxxxx>
> > Signed-off-by: Shashank Sharma <shashank.sharma@xxxxxxx>
> > ---
> >   drivers/gpu/drm/amd/amdgpu/amdgpu_doorbell.h  | 16 ++++++-
> >   .../gpu/drm/amd/amdgpu/amdgpu_doorbell_mgr.c  | 44 +++++++++++++++----
> >   drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c       |  7 +++
> >   3 files changed, 57 insertions(+), 10 deletions(-)
> >
> > diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_doorbell.h 
> > b/drivers/gpu/drm/amd/amdgpu/amdgpu_doorbell.h
> > index 6581b78fe438..10a9bb10e974 100644
> > --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_doorbell.h
> > +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_doorbell.h
> > @@ -49,10 +49,13 @@ struct amdgpu_doorbell {
> >       /* doorbell mmio */
> >       resource_size_t        base;
> >       resource_size_t        size;
> > -    u32 __iomem        *ptr;
> > +    u32    __iomem        *ptr;
>
> This one can probably go away if we use the pointer from 
> amdgpu_bo_create_kernel().

We started like that, but later realized that the cpu_addr from 
create_kernel() will just limit us

to that object only, whereas we are keeping this ptr to ioremap the 
whole doorbell space in one shot.

- Shashank

> >         /* Number of doorbells reserved for amdgpu kernel driver */
> >       u32 num_kernel_doorbells;
> > +
> > +    /* For kernel doorbell pages */
> > +    struct amdgpu_doorbell_obj kernel_doorbells;
> >   };
> >     /* Reserved doorbells for amdgpu (including multimedia).
> > @@ -369,6 +372,17 @@ void amdgpu_doorbell_free_page(struct 
> > amdgpu_device *adev,
> >   int amdgpu_doorbell_alloc_page(struct amdgpu_device *adev,
> >                   struct amdgpu_doorbell_obj *db_obj);
> >   +/**
> > + * amdgpu_doorbell_create_kernel_doorbells - Create kernel doorbells 
> > for graphics
> > + *
> > + * @adev: amdgpu_device pointer
> > + *
> > + * Creates doorbells for graphics driver
> > + *
> > + * returns 0 on success, error otherwise.
> > + */
> > +int amdgpu_doorbell_create_kernel_doorbells(struct amdgpu_device 
> > *adev);
> > +
> >   #define RDOORBELL32(index) amdgpu_mm_rdoorbell(adev, (index))
> >   #define WDOORBELL32(index, v) amdgpu_mm_wdoorbell(adev, (index), (v))
> >   #define RDOORBELL64(index) amdgpu_mm_rdoorbell64(adev, (index))
> > diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_doorbell_mgr.c 
> > b/drivers/gpu/drm/amd/amdgpu/amdgpu_doorbell_mgr.c
> > index 8be15b82b545..b46fe8b1378d 100644
> > --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_doorbell_mgr.c
> > +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_doorbell_mgr.c
> > @@ -160,6 +160,38 @@ int amdgpu_doorbell_alloc_page(struct 
> > amdgpu_device *adev,
> >       return 0;
> >   }
> >   +/**
> > + * amdgpu_doorbell_create_kernel_doorbells - Create kernel doorbells 
> > for graphics
> > + *
> > + * @adev: amdgpu_device pointer
> > + *
> > + * Creates doorbells for graphics driver
> > + *
> > + * returns 0 on success, error otherwise.
> > + */
> > +int amdgpu_doorbell_create_kernel_doorbells(struct amdgpu_device *adev)
> > +{
> > +    int r;
> > +    struct amdgpu_doorbell_obj *kernel_doorbells = 
> > &adev->doorbell.kernel_doorbells;
> > +
> > +    kernel_doorbells->doorbell_bitmap = 
> > bitmap_zalloc(adev->doorbell.num_kernel_doorbells,
> > +                              GFP_KERNEL);
> > +    if (!kernel_doorbells->doorbell_bitmap) {
> > +        DRM_ERROR("Failed to create kernel doorbell bitmap\n");
> > +        return -ENOMEM;
> > +    }
> > +
> > +    kernel_doorbells->size = adev->doorbell.num_kernel_doorbells * 
> > sizeof(u32);
> > +    r = amdgpu_doorbell_alloc_page(adev, kernel_doorbells);
> > +    if (r) {
> > +        bitmap_free(kernel_doorbells->doorbell_bitmap);
> > +        DRM_ERROR("Failed to allocate kernel doorbells, err=%d\n", r);
> > +        return r;
> > +    }
> > +
> > +    return 0;
> > +}
> > +
> >   /*
> >    * GPU doorbell aperture helpers function.
> >    */
> > @@ -179,7 +211,6 @@ int amdgpu_device_doorbell_init(struct 
> > amdgpu_device *adev)
> >           adev->doorbell.base = 0;
> >           adev->doorbell.size = 0;
> >           adev->doorbell.num_kernel_doorbells = 0;
> > -        adev->doorbell.ptr = NULL;
> >           return 0;
> >       }
> >   @@ -208,12 +239,7 @@ int amdgpu_device_doorbell_init(struct 
> > amdgpu_device *adev)
> >       if (adev->asic_type >= CHIP_VEGA10)
> >           adev->doorbell.num_kernel_doorbells += 0x400;
> >   -    adev->doorbell.ptr = ioremap(adev->doorbell.base,
> > -                     adev->doorbell.num_kernel_doorbells *
> > -                     sizeof(u32));
> > -    if (adev->doorbell.ptr == NULL)
> > -        return -ENOMEM;
> > -
> > +    adev->doorbell.ptr = ioremap(adev->doorbell.base, 
> > adev->doorbell.size);
> >       return 0;
> >   }
> >   @@ -226,6 +252,6 @@ int amdgpu_device_doorbell_init(struct 
> > amdgpu_device *adev)
> >    */
> >   void amdgpu_device_doorbell_fini(struct amdgpu_device *adev)
> >   {
> > -    iounmap(adev->doorbell.ptr);
> > -    adev->doorbell.ptr = NULL;
> > + bitmap_free(adev->doorbell.kernel_doorbells.doorbell_bitmap);
> > +    amdgpu_doorbell_free_page(adev, &adev->doorbell.kernel_doorbells);
> >   }
> > diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c 
> > b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
> > index 203d77a20507..75c6852845c4 100644
> > --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
> > +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
> > @@ -1866,6 +1866,13 @@ int amdgpu_ttm_init(struct amdgpu_device *adev)
> >           return r;
> >       }
> >   +    /* Create a boorbell page for kernel usages */
> > +    r = amdgpu_doorbell_create_kernel_doorbells(adev);
> > +    if (r) {
> > +        DRM_ERROR("Failed to initialize kernel doorbells. \n");
> > +        return r;
> > +    }
> > +
> >       /* Initialize preemptible memory pool */
> >       r = amdgpu_preempt_mgr_init(adev);
> >       if (r) {