On 2018-01-26 03:13 PM, Christian König wrote:
> Move amd_iommu_v2 initialization into amdgpu when it is enabled.
>
> This is WIP and really ugly since amdgpu should not depend directly on
> amd_iommu_v2.
>
> Signed-off-by: Christian König <christian.koenig at amd.com>
> ---
> drivers/gpu/drm/amd/amdgpu/Kconfig | 8 ++++++++
> drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 12 ++++++++++++
> drivers/gpu/drm/amd/amdkfd/kfd_device.c | 11 ++++++++++-
> 3 files changed, 30 insertions(+), 1 deletion(-)
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/Kconfig b/drivers/gpu/drm/amd/amdgpu/Kconfig
> index e8af1f5e8a79..7d3fdcbf0acb 100644
> --- a/drivers/gpu/drm/amd/amdgpu/Kconfig
> +++ b/drivers/gpu/drm/amd/amdgpu/Kconfig
> @@ -40,5 +40,13 @@ config DRM_AMDGPU_GART_DEBUGFS
> Selecting this option creates a debugfs file to inspect the mapped
> pages. Uses more memory for housekeeping, enable only for debugging.
>
> +config DRM_AMDGPU_ATC
> + bool "Enable the ATC to provide SVM support"
> + depends on DRM_AMDGPU
> + depends on AMD_IOMMU_V2
> + default y
> + help
> + This enables support for the ATC to provide a shared virtual memory implementation.
> +
> source "drivers/gpu/drm/amd/acp/Kconfig"
> source "drivers/gpu/drm/amd/display/Kconfig"
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
> index 84281ee34a25..04205236cc5e 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
> @@ -35,6 +35,7 @@
> #include <linux/vgaarb.h>
> #include <linux/vga_switcheroo.h>
> #include <linux/efi.h>
> +#include <linux/amd-iommu.h>
> #include "amdgpu.h"
> #include "amdgpu_trace.h"
> #include "amdgpu_i2c.h"
> @@ -1355,6 +1356,12 @@ static int amdgpu_device_ip_init(struct amdgpu_device *adev)
> adev->ip_blocks[i].status.hw = true;
> }
>
> +#ifdef CONFIG_DRM_AMDGPU_ATC
> + r = amd_iommu_init_device(adev->pdev, 0x10000);
KFD queries how many PASIDs the IOMMU can support with
amd_iommu_device_info. KFD only assigns PASIDs within that range. It can
be much smaller than the 16-bits supported by the GPU.
For a VM that uses ATC, you need to make sure it gets a PASID in the
range supported by the IOMMU. The PASID manager already supports that
and keeps smaller PASIDs for users that really need them.
Regards,
 Felix
> + if (r)
> + DRM_ERROR("amd_iommu_init_device error %d\n", r);
> +#endif
> +
> amdgpu_amdkfd_device_init(adev);
>
> if (amdgpu_sriov_vf(adev))
> @@ -1428,6 +1435,11 @@ static int amdgpu_device_ip_fini(struct amdgpu_device *adev)
> int i, r;
>
> amdgpu_amdkfd_device_fini(adev);
> +
> +#ifdef CONFIG_DRM_AMDGPU_ATC
> + amd_iommu_free_device(adev->pdev);
> +#endif
> +
> /* need to disable SMC first */
> for (i = 0; i < adev->num_ip_blocks; i++) {
> if (!adev->ip_blocks[i].status.hw)
> diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device.c b/drivers/gpu/drm/amd/amdkfd/kfd_device.c
> index a8fa33a08de3..7ed090f7c5f0 100644
> --- a/drivers/gpu/drm/amd/amdkfd/kfd_device.c
> +++ b/drivers/gpu/drm/amd/amdkfd/kfd_device.c
> @@ -373,7 +373,10 @@ void kgd2kfd_suspend(struct kfd_dev *kfd)
>
> amd_iommu_set_invalidate_ctx_cb(kfd->pdev, NULL);
> amd_iommu_set_invalid_ppr_cb(kfd->pdev, NULL);
> +
> +#ifndef CONFIG_DRM_AMDGPU_ATC
> amd_iommu_free_device(kfd->pdev);
> +#endif
> }
>
> int kgd2kfd_resume(struct kfd_dev *kfd)
> @@ -388,11 +391,15 @@ int kgd2kfd_resume(struct kfd_dev *kfd)
> static int kfd_resume(struct kfd_dev *kfd)
> {
> int err = 0;
> +
> +#ifndef CONFIG_DRM_AMDGPU_ATC
> unsigned int pasid_limit = kfd_get_pasid_limit();
>
> err = amd_iommu_init_device(kfd->pdev, pasid_limit);
> if (err)
> return -ENXIO;
> +#endif
> +
> amd_iommu_set_invalidate_ctx_cb(kfd->pdev,
> iommu_pasid_shutdown_callback);
> amd_iommu_set_invalid_ppr_cb(kfd->pdev,
> @@ -414,8 +421,10 @@ static int kfd_resume(struct kfd_dev *kfd)
>
> dqm_start_error:
> processes_bind_error:
> - amd_iommu_free_device(kfd->pdev);
>
> +#ifndef CONFIG_DRM_AMDGPU_ATC
> + amd_iommu_free_device(kfd->pdev);
> +#endif
> return err;
> }
>
