Am 2021-04-15 um 5:43 p.m. schrieb Philip Yang:
> HMM migration alloc sizeof(struct page) on system memory for each VRAM
> page, it is 1GB system memory reserved for 64GB VRAM. To avoid
> application OOM, increase system memory used size based on VRAM size of
> all GPUs, then application alloc memory will fail if system memory usage
> reach the limit.
>
> Signed-off-by: Philip Yang <Philip.Yang@xxxxxxx>
> Reviewed-by: Oak Zeng <Oak.Zeng@xxxxxxx>
Reviewed-by: Felix Kuehling <Felix.Kuehling@xxxxxxx>
Thanks. I'll apply this to amd-staging-drm-next together with the big
HMM patch series.
Regards,
Felix
> ---
> drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h | 1 +
> drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c | 5 +++++
> drivers/gpu/drm/amd/amdkfd/kfd_migrate.c | 8 ++++++++
> 3 files changed, 14 insertions(+)
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
> index 025e8bade8c8..2cb7f8c30b9f 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
> @@ -272,6 +272,7 @@ void amdgpu_amdkfd_gpuvm_init_mem_limits(void);
> void amdgpu_amdkfd_gpuvm_destroy_cb(struct amdgpu_device *adev,
> struct amdgpu_vm *vm);
> void amdgpu_amdkfd_unreserve_memory_limit(struct amdgpu_bo *bo);
> +void amdgpu_amdkfd_reserve_system_mem(uint64_t size);
> #else
> static inline
> void amdgpu_amdkfd_gpuvm_init_mem_limits(void)
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
> index d70a21ea576d..6ea1039b08a0 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
> @@ -105,6 +105,11 @@ void amdgpu_amdkfd_gpuvm_init_mem_limits(void)
> (kfd_mem_limit.max_ttm_mem_limit >> 20));
> }
>
> +void amdgpu_amdkfd_reserve_system_mem(uint64_t size)
> +{
> + kfd_mem_limit.system_mem_used += size;
> +}
> +
> /* Estimate page table size needed to represent a given memory size
> *
> * With 4KB pages, we need one 8 byte PTE for each 4KB of memory
> diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
> index 7d8659517447..1373ce9af890 100644
> --- a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
> +++ b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
> @@ -876,6 +876,9 @@ static const struct dev_pagemap_ops svm_migrate_pgmap_ops = {
> .migrate_to_ram = svm_migrate_to_ram,
> };
>
> +/* Each VRAM page uses sizeof(struct page) on system memory */
> +#define SVM_HMM_PAGE_STRUCT_SIZE(size) ((size)/PAGE_SIZE * sizeof(struct page))
> +
> int svm_migrate_init(struct amdgpu_device *adev)
> {
> struct kfd_dev *kfddev = adev->kfd.dev;
> @@ -912,6 +915,11 @@ int svm_migrate_init(struct amdgpu_device *adev)
> return PTR_ERR(r);
> }
>
> + pr_debug("reserve %ldMB system memory for VRAM pages struct\n",
> + SVM_HMM_PAGE_STRUCT_SIZE(size) >> 20);
> +
> + amdgpu_amdkfd_reserve_system_mem(SVM_HMM_PAGE_STRUCT_SIZE(size));
> +
> pr_info("HMM registered %ldMB device memory\n", size >> 20);
>
> return 0;
_______________________________________________
amd-gfx mailing list
amd-gfx@xxxxxxxxxxxxxxxxxxxxx
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
