Re: [PATCH 2/3] drm/amdkfd: Set svm range max pages

[philip yang <yangp@xxxxxxx>]

On 2022-07-25 10:34, Felix Kuehling wrote:

> Am
>       2022-07-25 um 08:23 schrieb Philip Yang:
>       
> > This will be used to split giant svm range
> >         into smaller ranges, to
> >         
> >         support VRAM overcommitment by giant range and improve GPU retry
> >         fault
> >         
> >         recover on giant range.
> >         
> >         
> >         Signed-off-by: Philip Yang <Philip.Yang@xxxxxxx>
> >         
> >         ---
> >         
> >           drivers/gpu/drm/amd/amdkfd/kfd_migrate.c |  2 ++
> >         
> >           drivers/gpu/drm/amd/amdkfd/kfd_svm.c     | 15 +++++++++++++++
> >         
> >           drivers/gpu/drm/amd/amdkfd/kfd_svm.h     |  3 +++
> >         
> >           3 files changed, 20 insertions(+)
> >         
> >         
> >         diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
> >         b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
> >         
> >         index 9667015a6cbc..b1f87aa6138b 100644
> >         
> >         --- a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
> >         
> >         +++ b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
> >         
> >         @@ -1019,6 +1019,8 @@ int svm_migrate_init(struct amdgpu_device
> >         *adev)
> >         
> >                
> >         amdgpu_amdkfd_reserve_system_mem(SVM_HMM_PAGE_STRUCT_SIZE(size));
> >         
> >           +    svm_range_set_max_pages(adev);
> >         
> >         +
> >         
> >               pr_info("HMM registered %ldMB device memory\n", size
> >         >> 20);
> >         
> >                 return 0;
> >         
> >         diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
> >         b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
> >         
> >         index b592aee6d9d6..cf9565ddddf8 100644
> >         
> >         --- a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
> >         
> >         +++ b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
> >         
> >         @@ -46,6 +46,11 @@
> >         
> >            */
> >         
> >           #define AMDGPU_SVM_RANGE_RETRY_FAULT_PENDING    (2UL *
> >         NSEC_PER_MSEC)
> >         
> >           +/* Giant svm range split into smaller ranges based on this,
> >         it is decided using
> >         
> >         + * minimum of all dGPU/APU 1/32 VRAM size, between 2MB to 1GB
> >         and align to 2MB.
> >         
> >         + */
> >         
> >         +uint64_t max_svm_range_pages;
> >         
> >         +
> >         
> >           struct criu_svm_metadata {
> >         
> >               struct list_head list;
> >         
> >               struct kfd_criu_svm_range_priv_data data;
> >         
> >         @@ -1869,6 +1874,16 @@ static struct svm_range
> >         *svm_range_clone(struct svm_range *old)
> >         
> >                 return new;
> >         
> >           }
> >         
> >         +__init void svm_range_set_max_pages(struct amdgpu_device *adev)
> >         
> >       
>       
>       Why is this marked as __init? This can run much later than module
>       init.
>       
>       
>

Thanks, this was called in amdgpu_amdkfd_init from module init, forget to remove __init after calling from svm_migrate_init, which is called from pci_probe.

Regards,

Philip

>       
> > +{
> >         
> >         +    uint64_t pages;
> >         
> >         +
> >         
> >         +    /* 1/32 VRAM size in pages */
> >         
> >         +    pages = adev->gmc.real_vram_size >> 17;
> >         
> >         +    pages = clamp(pages, 1ULL << 9, 1ULL << 18);
> >         
> >         +    max_svm_range_pages = min_not_zero(max_svm_range_pages,
> >         pages);
> >         
> >         +    max_svm_range_pages = ALIGN(max_svm_range_pages, 1ULL
> >         << 9);
> >         
> >       
>       
>       I'd recommend updating max_svm_range_pages with a single
>       WRITE_ONCE to avoid race conditions with GPU hot-plug.
>       
>       
>       Regards,
>       
>         Felix
>       
>       
>       
> > +}
> >         
> >             /**
> >         
> >            * svm_range_add - add svm range and handle overlap
> >         
> >         diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_svm.h
> >         b/drivers/gpu/drm/amd/amdkfd/kfd_svm.h
> >         
> >         index eab7f6d3b13c..346a41bf8dbf 100644
> >         
> >         --- a/drivers/gpu/drm/amd/amdkfd/kfd_svm.h
> >         
> >         +++ b/drivers/gpu/drm/amd/amdkfd/kfd_svm.h
> >         
> >         @@ -204,6 +204,9 @@ void
> >         svm_range_list_lock_and_flush_work(struct svm_range_list *svms,
> >         struct mm_s
> >         
> >           #define KFD_IS_SVM_API_SUPPORTED(dev) ((dev)->pgmap.type !=
> >         0)
> >         
> >             void svm_range_bo_unref_async(struct svm_range_bo *svm_bo);
> >         
> >         +
> >         
> >         +__init void svm_range_set_max_pages(struct amdgpu_device
> >         *adev);
> >         
> >         +
> >         
> >           #else
> >         
> >             struct kfd_process;
> >         
> >       
>