Am 07.02.2018 um 21:51 schrieb Felix Kuehling:
> On 2018-02-07 06:20 AM, Christian König wrote:
>> Am 07.02.2018 um 02:32 schrieb Felix Kuehling:
>>> dGPUs work without IOMMUv2. Make IOMMUv2 initialization dependent on
>>> ASIC information. Also allow building KFD without IOMMUv2 support.
>>> This is still useful for dGPUs and prepares for enabling KFD on
>>> architectures that don't support AMD IOMMUv2.
>>>
>>> v2:
>>> * Centralize IOMMUv2 code to avoid #ifdefs in too many places
>>>
>>> Signed-off-by: Felix Kuehling <Felix.Kuehling at amd.com>
>>> ---
>>>  drivers/gpu/drm/amd/amdkfd/Kconfig       |  2 +-
>>>  drivers/gpu/drm/amd/amdkfd/Makefile      |  4 +
>>>  drivers/gpu/drm/amd/amdkfd/kfd_crat.c    | 14 +-
>>>  drivers/gpu/drm/amd/amdkfd/kfd_device.c  | 127 +++--------
>>>  drivers/gpu/drm/amd/amdkfd/kfd_events.c  |  3 +
>>>  drivers/gpu/drm/amd/amdkfd/kfd_iommu.c   | 356
>>> ++++++++++++++++++++++++++++++
>>>  drivers/gpu/drm/amd/amdkfd/kfd_iommu.h   | 78 +++++++
>>>  drivers/gpu/drm/amd/amdkfd/kfd_priv.h    | 14 +-
>>>  drivers/gpu/drm/amd/amdkfd/kfd_process.c | 138 +-----------
>>> Â drivers/gpu/drm/amd/amdkfd/kfd_topology.c |Â 16 +-
>>> Â drivers/gpu/drm/amd/amdkfd/kfd_topology.h |Â Â 6 +-
>>> Â 11 files changed, 493 insertions(+), 265 deletions(-)
>>> Â create mode 100644 drivers/gpu/drm/amd/amdkfd/kfd_iommu.c
>>> Â create mode 100644 drivers/gpu/drm/amd/amdkfd/kfd_iommu.h
>>>
>>> diff --git a/drivers/gpu/drm/amd/amdkfd/Kconfig
>>> b/drivers/gpu/drm/amd/amdkfd/Kconfig
>>> index bc5a294..5bbeb95 100644
>>> --- a/drivers/gpu/drm/amd/amdkfd/Kconfig
>>> +++ b/drivers/gpu/drm/amd/amdkfd/Kconfig
>>> @@ -4,6 +4,6 @@
>>> Â Â config HSA_AMD
>>> Â Â Â Â Â tristate "HSA kernel driver for AMD GPU devices"
>>> -Â Â Â depends on DRM_AMDGPU && AMD_IOMMU_V2 && X86_64
>>> +Â Â Â depends on DRM_AMDGPU && X86_64
>> You still need a weak dependency on AMD_IOMMU_V2 here, in other words
>> add "imply AMD_IOMMU_V2".
>>
>> This prevents illegal combinations like linking amdkfd into the kernel
>> while amd_iommu_v2 is a module.
>>
>> But it should still allow to completely disable amd_iommu_v2 and
>> compile amdkfd without support for it.
> Thanks, that's good to know. An updated patch is attached (to avoid
> resending the whole series).
Patch is Acked-by: Christian König <christian.koenig at amd.com>.
Regards,
Christian.
>
> Regards,
> Â Felix
>
>> Christian.
>>
>>> Â Â Â Â Â help
>>> Â Â Â Â Â Â Â Enable this if you want to use HSA features on AMD GPU devices.
>>> diff --git a/drivers/gpu/drm/amd/amdkfd/Makefile
>>> b/drivers/gpu/drm/amd/amdkfd/Makefile
>>> index a317e76..0d02422 100644
>>> --- a/drivers/gpu/drm/amd/amdkfd/Makefile
>>> +++ b/drivers/gpu/drm/amd/amdkfd/Makefile
>>> @@ -37,6 +37,10 @@ amdkfd-y   := kfd_module.o kfd_device.o
>>> kfd_chardev.o kfd_topology.o \
>>> Â Â Â Â Â Â Â Â Â kfd_interrupt.o kfd_events.o cik_event_interrupt.o \
>>> Â Â Â Â Â Â Â Â Â kfd_dbgdev.o kfd_dbgmgr.o kfd_crat.o
>>> Â +ifneq ($(CONFIG_AMD_IOMMU_V2),)
>>> +amdkfd-y += kfd_iommu.o
>>> +endif
>>> +
>>> Â amdkfd-$(CONFIG_DEBUG_FS) += kfd_debugfs.o
>>> Â Â obj-$(CONFIG_HSA_AMD)Â Â Â += amdkfd.o
>>> diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_crat.c
>>> b/drivers/gpu/drm/amd/amdkfd/kfd_crat.c
>>> index 2bc2816..7493f47 100644
>>> --- a/drivers/gpu/drm/amd/amdkfd/kfd_crat.c
>>> +++ b/drivers/gpu/drm/amd/amdkfd/kfd_crat.c
>>> @@ -22,10 +22,10 @@
>>> Â Â #include <linux/pci.h>
>>> Â #include <linux/acpi.h>
>>> -#include <linux/amd-iommu.h>
>>> Â #include "kfd_crat.h"
>>> Â #include "kfd_priv.h"
>>> Â #include "kfd_topology.h"
>>> +#include "kfd_iommu.h"
>>> Â Â /* GPU Processor ID base for dGPUs for which VCRAT needs to be
>>> created.
>>> Â Â * GPU processor ID are expressed with Bit[31]=1.
>>> @@ -1037,15 +1037,11 @@ static int kfd_create_vcrat_image_gpu(void
>>> *pcrat_image,
>>> Â Â Â Â Â struct crat_subtype_generic *sub_type_hdr;
>>> Â Â Â Â Â struct crat_subtype_computeunit *cu;
>>> Â Â Â Â Â struct kfd_cu_info cu_info;
>>> -Â Â Â struct amd_iommu_device_info iommu_info;
>>> Â Â Â Â Â int avail_size = *size;
>>> Â Â Â Â Â uint32_t total_num_of_cu;
>>> Â Â Â Â Â int num_of_cache_entries = 0;
>>> Â Â Â Â Â int cache_mem_filled = 0;
>>> Â Â Â Â Â int ret = 0;
>>> -Â Â Â const u32 required_iommu_flags = AMD_IOMMU_DEVICE_FLAG_ATS_SUP |
>>> -Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â AMD_IOMMU_DEVICE_FLAG_PRI_SUP |
>>> -Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â AMD_IOMMU_DEVICE_FLAG_PASID_SUP;
>>> Â Â Â Â Â struct kfd_local_mem_info local_mem_info;
>>> Â Â Â Â Â Â if (!pcrat_image || avail_size < VCRAT_SIZE_FOR_GPU)
>>> @@ -1106,12 +1102,8 @@ static int kfd_create_vcrat_image_gpu(void
>>> *pcrat_image,
>>> Â Â Â Â Â /* Check if this node supports IOMMU. During parsing this flag
>>> will
>>> Â Â Â Â Â Â * translate to HSA_CAP_ATS_PRESENT
>>> Â Â Â Â Â Â */
>>> -Â Â Â iommu_info.flags = 0;
>>> -Â Â Â if (amd_iommu_device_info(kdev->pdev, &iommu_info) == 0) {
>>> -Â Â Â Â Â Â Â if ((iommu_info.flags & required_iommu_flags) ==
>>> -Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â required_iommu_flags)
>>> -Â Â Â Â Â Â Â Â Â Â Â cu->hsa_capability |= CRAT_CU_FLAGS_IOMMU_PRESENT;
>>> -Â Â Â }
>>> +Â Â Â if (!kfd_iommu_check_device(kdev))
>>> +Â Â Â Â Â Â Â cu->hsa_capability |= CRAT_CU_FLAGS_IOMMU_PRESENT;
>>> Â Â Â Â Â Â crat_table->length += sub_type_hdr->length;
>>> Â Â Â Â Â crat_table->total_entries++;
>>> diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device.c
>>> b/drivers/gpu/drm/amd/amdkfd/kfd_device.c
>>> index 83d6f41..4ac2d61 100644
>>> --- a/drivers/gpu/drm/amd/amdkfd/kfd_device.c
>>> +++ b/drivers/gpu/drm/amd/amdkfd/kfd_device.c
>>> @@ -20,7 +20,9 @@
>>> Â Â * OTHER DEALINGS IN THE SOFTWARE.
>>> Â Â */
>>> Â +#if defined(CONFIG_AMD_IOMMU_V2_MODULE) ||
>>> defined(CONFIG_AMD_IOMMU_V2)
>>> Â #include <linux/amd-iommu.h>
>>> +#endif
>>> Â #include <linux/bsearch.h>
>>> Â #include <linux/pci.h>
>>> Â #include <linux/slab.h>
>>> @@ -28,9 +30,11 @@
>>> Â #include "kfd_device_queue_manager.h"
>>> Â #include "kfd_pm4_headers_vi.h"
>>> Â #include "cwsr_trap_handler_gfx8.asm"
>>> +#include "kfd_iommu.h"
>>> Â Â #define MQD_SIZE_ALIGNED 768
>>> Â +#ifdef KFD_SUPPORT_IOMMU_V2
>>> Â static const struct kfd_device_info kaveri_device_info = {
>>> Â Â Â Â Â .asic_family = CHIP_KAVERI,
>>> Â Â Â Â Â .max_pasid_bits = 16,
>>> @@ -41,6 +45,7 @@ static const struct kfd_device_info
>>> kaveri_device_info = {
>>> Â Â Â Â Â .num_of_watch_points = 4,
>>> Â Â Â Â Â .mqd_size_aligned = MQD_SIZE_ALIGNED,
>>> Â Â Â Â Â .supports_cwsr = false,
>>> +Â Â Â .needs_iommu_device = true,
>>> Â Â Â Â Â .needs_pci_atomics = false,
>>> Â };
>>> Â @@ -54,8 +59,10 @@ static const struct kfd_device_info
>>> carrizo_device_info = {
>>> Â Â Â Â Â .num_of_watch_points = 4,
>>> Â Â Â Â Â .mqd_size_aligned = MQD_SIZE_ALIGNED,
>>> Â Â Â Â Â .supports_cwsr = true,
>>> +Â Â Â .needs_iommu_device = true,
>>> Â Â Â Â Â .needs_pci_atomics = false,
>>> Â };
>>> +#endif
>>> Â Â static const struct kfd_device_info hawaii_device_info = {
>>> Â Â Â Â Â .asic_family = CHIP_HAWAII,
>>> @@ -67,6 +74,7 @@ static const struct kfd_device_info
>>> hawaii_device_info = {
>>> Â Â Â Â Â .num_of_watch_points = 4,
>>> Â Â Â Â Â .mqd_size_aligned = MQD_SIZE_ALIGNED,
>>> Â Â Â Â Â .supports_cwsr = false,
>>> +Â Â Â .needs_iommu_device = false,
>>> Â Â Â Â Â .needs_pci_atomics = false,
>>> Â };
>>> Â @@ -79,6 +87,7 @@ static const struct kfd_device_info
>>> tonga_device_info = {
>>> Â Â Â Â Â .num_of_watch_points = 4,
>>> Â Â Â Â Â .mqd_size_aligned = MQD_SIZE_ALIGNED,
>>> Â Â Â Â Â .supports_cwsr = false,
>>> +Â Â Â .needs_iommu_device = false,
>>> Â Â Â Â Â .needs_pci_atomics = true,
>>> Â };
>>> Â @@ -91,6 +100,7 @@ static const struct kfd_device_info
>>> tonga_vf_device_info = {
>>> Â Â Â Â Â .num_of_watch_points = 4,
>>> Â Â Â Â Â .mqd_size_aligned = MQD_SIZE_ALIGNED,
>>> Â Â Â Â Â .supports_cwsr = false,
>>> +Â Â Â .needs_iommu_device = false,
>>> Â Â Â Â Â .needs_pci_atomics = false,
>>> Â };
>>> Â @@ -103,6 +113,7 @@ static const struct kfd_device_info
>>> fiji_device_info = {
>>> Â Â Â Â Â .num_of_watch_points = 4,
>>> Â Â Â Â Â .mqd_size_aligned = MQD_SIZE_ALIGNED,
>>> Â Â Â Â Â .supports_cwsr = true,
>>> +Â Â Â .needs_iommu_device = false,
>>> Â Â Â Â Â .needs_pci_atomics = true,
>>> Â };
>>> Â @@ -115,6 +126,7 @@ static const struct kfd_device_info
>>> fiji_vf_device_info = {
>>> Â Â Â Â Â .num_of_watch_points = 4,
>>> Â Â Â Â Â .mqd_size_aligned = MQD_SIZE_ALIGNED,
>>> Â Â Â Â Â .supports_cwsr = true,
>>> +Â Â Â .needs_iommu_device = false,
>>> Â Â Â Â Â .needs_pci_atomics = false,
>>> Â };
>>> Â @@ -128,6 +140,7 @@ static const struct kfd_device_info
>>> polaris10_device_info = {
>>> Â Â Â Â Â .num_of_watch_points = 4,
>>> Â Â Â Â Â .mqd_size_aligned = MQD_SIZE_ALIGNED,
>>> Â Â Â Â Â .supports_cwsr = true,
>>> +Â Â Â .needs_iommu_device = false,
>>> Â Â Â Â Â .needs_pci_atomics = true,
>>> Â };
>>> Â @@ -140,6 +153,7 @@ static const struct kfd_device_info
>>> polaris10_vf_device_info = {
>>> Â Â Â Â Â .num_of_watch_points = 4,
>>> Â Â Â Â Â .mqd_size_aligned = MQD_SIZE_ALIGNED,
>>> Â Â Â Â Â .supports_cwsr = true,
>>> +Â Â Â .needs_iommu_device = false,
>>> Â Â Â Â Â .needs_pci_atomics = false,
>>> Â };
>>> Â @@ -152,6 +166,7 @@ static const struct kfd_device_info
>>> polaris11_device_info = {
>>> Â Â Â Â Â .num_of_watch_points = 4,
>>> Â Â Â Â Â .mqd_size_aligned = MQD_SIZE_ALIGNED,
>>> Â Â Â Â Â .supports_cwsr = true,
>>> +Â Â Â .needs_iommu_device = false,
>>> Â Â Â Â Â .needs_pci_atomics = true,
>>> Â };
>>> Â @@ -162,6 +177,7 @@ struct kfd_deviceid {
>>> Â };
>>> Â Â static const struct kfd_deviceid supported_devices[] = {
>>> +#ifdef KFD_SUPPORT_IOMMU_V2
>>>      { 0x1304, &kaveri_device_info },   /* Kaveri */
>>>      { 0x1305, &kaveri_device_info },   /* Kaveri */
>>>      { 0x1306, &kaveri_device_info },   /* Kaveri */
>>> @@ -189,6 +205,7 @@ static const struct kfd_deviceid
>>> supported_devices[] = {
>>>      { 0x9875, &carrizo_device_info },   /* Carrizo */
>>>      { 0x9876, &carrizo_device_info },   /* Carrizo */
>>>      { 0x9877, &carrizo_device_info },   /* Carrizo */
>>> +#endif
>>>      { 0x67A0, &hawaii_device_info },   /* Hawaii */
>>>      { 0x67A1, &hawaii_device_info },   /* Hawaii */
>>>      { 0x67A2, &hawaii_device_info },   /* Hawaii */
>>> @@ -302,77 +319,6 @@ struct kfd_dev *kgd2kfd_probe(struct kgd_dev *kgd,
>>> Â Â Â Â Â return kfd;
>>> Â }
>>> Â -static bool device_iommu_pasid_init(struct kfd_dev *kfd)
>>> -{
>>> -Â Â Â const u32 required_iommu_flags = AMD_IOMMU_DEVICE_FLAG_ATS_SUP |
>>> -Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â AMD_IOMMU_DEVICE_FLAG_PRI_SUP |
>>> -Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â AMD_IOMMU_DEVICE_FLAG_PASID_SUP;
>>> -
>>> -Â Â Â struct amd_iommu_device_info iommu_info;
>>> -Â Â Â unsigned int pasid_limit;
>>> -Â Â Â int err;
>>> -
>>> -Â Â Â err = amd_iommu_device_info(kfd->pdev, &iommu_info);
>>> -Â Â Â if (err < 0) {
>>> -Â Â Â Â Â Â Â dev_err(kfd_device,
>>> -Â Â Â Â Â Â Â Â Â Â Â "error getting iommu info. is the iommu enabled?\n");
>>> -Â Â Â Â Â Â Â return false;
>>> -Â Â Â }
>>> -
>>> -Â Â Â if ((iommu_info.flags & required_iommu_flags) !=
>>> required_iommu_flags) {
>>> -Â Â Â Â Â Â Â dev_err(kfd_device, "error required iommu flags ats %i, pri
>>> %i, pasid %i\n",
>>> -Â Â Â Â Â Â Â Â Â Â Â Â Â Â (iommu_info.flags & AMD_IOMMU_DEVICE_FLAG_ATS_SUP) != 0,
>>> -Â Â Â Â Â Â Â Â Â Â Â Â Â Â (iommu_info.flags & AMD_IOMMU_DEVICE_FLAG_PRI_SUP) != 0,
>>> -Â Â Â Â Â Â Â Â Â Â Â Â Â Â (iommu_info.flags & AMD_IOMMU_DEVICE_FLAG_PASID_SUP)
>>> -Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â != 0);
>>> -Â Â Â Â Â Â Â return false;
>>> -Â Â Â }
>>> -
>>> -Â Â Â pasid_limit = min_t(unsigned int,
>>> -Â Â Â Â Â Â Â Â Â Â Â (unsigned int)(1 << kfd->device_info->max_pasid_bits),
>>> -Â Â Â Â Â Â Â Â Â Â Â iommu_info.max_pasids);
>>> -
>>> -Â Â Â if (!kfd_set_pasid_limit(pasid_limit)) {
>>> -Â Â Â Â Â Â Â dev_err(kfd_device, "error setting pasid limit\n");
>>> -Â Â Â Â Â Â Â return false;
>>> -Â Â Â }
>>> -
>>> -Â Â Â return true;
>>> -}
>>> -
>>> -static void iommu_pasid_shutdown_callback(struct pci_dev *pdev, int
>>> pasid)
>>> -{
>>> -Â Â Â struct kfd_dev *dev = kfd_device_by_pci_dev(pdev);
>>> -
>>> -Â Â Â if (dev)
>>> -Â Â Â Â Â Â Â kfd_process_iommu_unbind_callback(dev, pasid);
>>> -}
>>> -
>>> -/*
>>> - * This function called by IOMMU driver on PPR failure
>>> - */
>>> -static int iommu_invalid_ppr_cb(struct pci_dev *pdev, int pasid,
>>> -Â Â Â Â Â Â Â unsigned long address, u16 flags)
>>> -{
>>> -Â Â Â struct kfd_dev *dev;
>>> -
>>> -Â Â Â dev_warn(kfd_device,
>>> -Â Â Â Â Â Â Â Â Â Â Â "Invalid PPR device %x:%x.%x pasid %d address 0x%lX
>>> flags 0x%X",
>>> -Â Â Â Â Â Â Â Â Â Â Â PCI_BUS_NUM(pdev->devfn),
>>> -Â Â Â Â Â Â Â Â Â Â Â PCI_SLOT(pdev->devfn),
>>> -Â Â Â Â Â Â Â Â Â Â Â PCI_FUNC(pdev->devfn),
>>> -Â Â Â Â Â Â Â Â Â Â Â pasid,
>>> -Â Â Â Â Â Â Â Â Â Â Â address,
>>> -Â Â Â Â Â Â Â Â Â Â Â flags);
>>> -
>>> -Â Â Â dev = kfd_device_by_pci_dev(pdev);
>>> -Â Â Â if (!WARN_ON(!dev))
>>> -Â Â Â Â Â Â Â kfd_signal_iommu_event(dev, pasid, address,
>>> -Â Â Â Â Â Â Â Â Â Â Â flags & PPR_FAULT_WRITE, flags & PPR_FAULT_EXEC);
>>> -
>>> -Â Â Â return AMD_IOMMU_INV_PRI_RSP_INVALID;
>>> -}
>>> -
>>> Â static void kfd_cwsr_init(struct kfd_dev *kfd)
>>> Â {
>>> Â Â Â Â Â if (cwsr_enable && kfd->device_info->supports_cwsr) {
>>> @@ -462,11 +408,9 @@ bool kgd2kfd_device_init(struct kfd_dev *kfd,
>>> Â Â Â Â Â Â Â Â Â goto device_queue_manager_error;
>>> Â Â Â Â Â }
>>> Â -Â Â Â if (!device_iommu_pasid_init(kfd)) {
>>> -Â Â Â Â Â Â Â dev_err(kfd_device,
>>> -Â Â Â Â Â Â Â Â Â Â Â "Error initializing iommuv2 for device %x:%x\n",
>>> -Â Â Â Â Â Â Â Â Â Â Â kfd->pdev->vendor, kfd->pdev->device);
>>> -Â Â Â Â Â Â Â goto device_iommu_pasid_error;
>>> +Â Â Â if (kfd_iommu_device_init(kfd)) {
>>> +Â Â Â Â Â Â Â dev_err(kfd_device, "Error initializing iommuv2\n");
>>> +Â Â Â Â Â Â Â goto device_iommu_error;
>>> Â Â Â Â Â }
>>> Â Â Â Â Â Â kfd_cwsr_init(kfd);
>>> @@ -486,7 +430,7 @@ bool kgd2kfd_device_init(struct kfd_dev *kfd,
>>> Â Â Â Â Â goto out;
>>> Â Â kfd_resume_error:
>>> -device_iommu_pasid_error:
>>> +device_iommu_error:
>>> Â Â Â Â Â device_queue_manager_uninit(kfd->dqm);
>>> Â device_queue_manager_error:
>>> Â Â Â Â Â kfd_interrupt_exit(kfd);
>>> @@ -527,11 +471,7 @@ void kgd2kfd_suspend(struct kfd_dev *kfd)
>>> Â Â Â Â Â Â kfd->dqm->ops.stop(kfd->dqm);
>>> Â -Â Â Â kfd_unbind_processes_from_device(kfd);
>>> -
>>> -Â Â Â amd_iommu_set_invalidate_ctx_cb(kfd->pdev, NULL);
>>> -Â Â Â amd_iommu_set_invalid_ppr_cb(kfd->pdev, NULL);
>>> -Â Â Â amd_iommu_free_device(kfd->pdev);
>>> +Â Â Â kfd_iommu_suspend(kfd);
>>> Â }
>>> Â Â int kgd2kfd_resume(struct kfd_dev *kfd)
>>> @@ -546,19 +486,14 @@ int kgd2kfd_resume(struct kfd_dev *kfd)
>>> Â static int kfd_resume(struct kfd_dev *kfd)
>>> Â {
>>> Â Â Â Â Â int err = 0;
>>> -Â Â Â unsigned int pasid_limit = kfd_get_pasid_limit();
>>> -
>>> -Â Â Â err = amd_iommu_init_device(kfd->pdev, pasid_limit);
>>> -Â Â Â if (err)
>>> -Â Â Â Â Â Â Â return -ENXIO;
>>> -Â Â Â amd_iommu_set_invalidate_ctx_cb(kfd->pdev,
>>> -Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â iommu_pasid_shutdown_callback);
>>> -Â Â Â amd_iommu_set_invalid_ppr_cb(kfd->pdev,
>>> -Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â iommu_invalid_ppr_cb);
>>> Â -Â Â Â err = kfd_bind_processes_to_device(kfd);
>>> -Â Â Â if (err)
>>> -Â Â Â Â Â Â Â goto processes_bind_error;
>>> +Â Â Â err = kfd_iommu_resume(kfd);
>>> +Â Â Â if (err) {
>>> +Â Â Â Â Â Â Â dev_err(kfd_device,
>>> +Â Â Â Â Â Â Â Â Â Â Â "Failed to resume IOMMU for device %x:%x\n",
>>> +Â Â Â Â Â Â Â Â Â Â Â kfd->pdev->vendor, kfd->pdev->device);
>>> +Â Â Â Â Â Â Â return err;
>>> +Â Â Â }
>>> Â Â Â Â Â Â err = kfd->dqm->ops.start(kfd->dqm);
>>> Â Â Â Â Â if (err) {
>>> @@ -571,9 +506,7 @@ static int kfd_resume(struct kfd_dev *kfd)
>>> Â Â Â Â Â return err;
>>> Â Â dqm_start_error:
>>> -processes_bind_error:
>>> -Â Â Â amd_iommu_free_device(kfd->pdev);
>>> -
>>> +Â Â Â kfd_iommu_suspend(kfd);
>>> Â Â Â Â Â return err;
>>> Â }
>>> Â diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_events.c
>>> b/drivers/gpu/drm/amd/amdkfd/kfd_events.c
>>> index 93aae5c..6fb9c0d 100644
>>> --- a/drivers/gpu/drm/amd/amdkfd/kfd_events.c
>>> +++ b/drivers/gpu/drm/amd/amdkfd/kfd_events.c
>>> @@ -30,6 +30,7 @@
>>> Â #include <linux/memory.h>
>>> Â #include "kfd_priv.h"
>>> Â #include "kfd_events.h"
>>> +#include "kfd_iommu.h"
>>> Â #include <linux/device.h>
>>> Â Â /*
>>> @@ -837,6 +838,7 @@ static void
>>> lookup_events_by_type_and_signal(struct kfd_process *p,
>>> Â Â Â Â Â }
>>> Â }
>>> Â +#ifdef KFD_SUPPORT_IOMMU_V2
>>> Â void kfd_signal_iommu_event(struct kfd_dev *dev, unsigned int pasid,
>>> Â Â Â Â Â Â Â Â Â unsigned long address, bool is_write_requested,
>>> Â Â Â Â Â Â Â Â Â bool is_execute_requested)
>>> @@ -905,6 +907,7 @@ void kfd_signal_iommu_event(struct kfd_dev *dev,
>>> unsigned int pasid,
>>> Â Â Â Â Â mutex_unlock(&p->event_mutex);
>>> Â Â Â Â Â kfd_unref_process(p);
>>> Â }
>>> +#endif /* KFD_SUPPORT_IOMMU_V2 */
>>> Â Â void kfd_signal_hw_exception_event(unsigned int pasid)
>>> Â {
>>> diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_iommu.c
>>> b/drivers/gpu/drm/amd/amdkfd/kfd_iommu.c
>>> new file mode 100644
>>> index 0000000..81dee34
>>> --- /dev/null
>>> +++ b/drivers/gpu/drm/amd/amdkfd/kfd_iommu.c
>>> @@ -0,0 +1,356 @@
>>> +/*
>>> + * Copyright 2018 Advanced Micro Devices, Inc.
>>> + *
>>> + * Permission is hereby granted, free of charge, to any person
>>> obtaining a
>>> + * copy of this software and associated documentation files (the
>>> "Software"),
>>> + * to deal in the Software without restriction, including without
>>> limitation
>>> + * the rights to use, copy, modify, merge, publish, distribute,
>>> sublicense,
>>> + * and/or sell copies of the Software, and to permit persons to whom
>>> the
>>> + * Software is furnished to do so, subject to the following conditions:
>>> + *
>>> + * The above copyright notice and this permission notice shall be
>>> included in
>>> + * all copies or substantial portions of the Software.
>>> + *
>>> + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
>>> EXPRESS OR
>>> + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
>>> MERCHANTABILITY,
>>> + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO
>>> EVENT SHALL
>>> + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM,
>>> DAMAGES OR
>>> + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
>>> OTHERWISE,
>>> + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
>>> USE OR
>>> + * OTHER DEALINGS IN THE SOFTWARE.
>>> + */
>>> +
>>> +#include <linux/printk.h>
>>> +#include <linux/device.h>
>>> +#include <linux/slab.h>
>>> +#include <linux/pci.h>
>>> +#include <linux/amd-iommu.h>
>>> +#include "kfd_priv.h"
>>> +#include "kfd_dbgmgr.h"
>>> +#include "kfd_topology.h"
>>> +#include "kfd_iommu.h"
>>> +
>>> +static const u32 required_iommu_flags = AMD_IOMMU_DEVICE_FLAG_ATS_SUP |
>>> +Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â AMD_IOMMU_DEVICE_FLAG_PRI_SUP |
>>> +Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â AMD_IOMMU_DEVICE_FLAG_PASID_SUP;
>>> +
>>> +/** kfd_iommu_check_device - Check whether IOMMU is available for
>>> device
>>> + */
>>> +int kfd_iommu_check_device(struct kfd_dev *kfd)
>>> +{
>>> +Â Â Â struct amd_iommu_device_info iommu_info;
>>> +Â Â Â int err;
>>> +
>>> +Â Â Â if (!kfd->device_info->needs_iommu_device)
>>> +Â Â Â Â Â Â Â return -ENODEV;
>>> +
>>> +Â Â Â iommu_info.flags = 0;
>>> +Â Â Â err = amd_iommu_device_info(kfd->pdev, &iommu_info);
>>> +Â Â Â if (err)
>>> +Â Â Â Â Â Â Â return err;
>>> +
>>> +Â Â Â if ((iommu_info.flags & required_iommu_flags) !=
>>> required_iommu_flags)
>>> +Â Â Â Â Â Â Â return -ENODEV;
>>> +
>>> +Â Â Â return 0;
>>> +}
>>> +
>>> +/** kfd_iommu_device_init - Initialize IOMMU for device
>>> + */
>>> +int kfd_iommu_device_init(struct kfd_dev *kfd)
>>> +{
>>> +Â Â Â struct amd_iommu_device_info iommu_info;
>>> +Â Â Â unsigned int pasid_limit;
>>> +Â Â Â int err;
>>> +
>>> +Â Â Â if (!kfd->device_info->needs_iommu_device)
>>> +Â Â Â Â Â Â Â return 0;
>>> +
>>> +Â Â Â iommu_info.flags = 0;
>>> +Â Â Â err = amd_iommu_device_info(kfd->pdev, &iommu_info);
>>> +Â Â Â if (err < 0) {
>>> +Â Â Â Â Â Â Â dev_err(kfd_device,
>>> +Â Â Â Â Â Â Â Â Â Â Â "error getting iommu info. is the iommu enabled?\n");
>>> +Â Â Â Â Â Â Â return -ENODEV;
>>> +Â Â Â }
>>> +
>>> +Â Â Â if ((iommu_info.flags & required_iommu_flags) !=
>>> required_iommu_flags) {
>>> +Â Â Â Â Â Â Â dev_err(kfd_device, "error required iommu flags ats %i, pri
>>> %i, pasid %i\n",
>>> +Â Â Â Â Â Â Â Â Â Â Â Â Â Â (iommu_info.flags & AMD_IOMMU_DEVICE_FLAG_ATS_SUP) != 0,
>>> +Â Â Â Â Â Â Â Â Â Â Â Â Â Â (iommu_info.flags & AMD_IOMMU_DEVICE_FLAG_PRI_SUP) != 0,
>>> +Â Â Â Â Â Â Â Â Â Â Â Â Â Â (iommu_info.flags & AMD_IOMMU_DEVICE_FLAG_PASID_SUP)
>>> +Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â != 0);
>>> +Â Â Â Â Â Â Â return -ENODEV;
>>> +Â Â Â }
>>> +
>>> +Â Â Â pasid_limit = min_t(unsigned int,
>>> +Â Â Â Â Â Â Â Â Â Â Â (unsigned int)(1 << kfd->device_info->max_pasid_bits),
>>> +Â Â Â Â Â Â Â Â Â Â Â iommu_info.max_pasids);
>>> +
>>> +Â Â Â if (!kfd_set_pasid_limit(pasid_limit)) {
>>> +Â Â Â Â Â Â Â dev_err(kfd_device, "error setting pasid limit\n");
>>> +Â Â Â Â Â Â Â return -EBUSY;
>>> +Â Â Â }
>>> +
>>> +Â Â Â return 0;
>>> +}
>>> +
>>> +/** kfd_iommu_bind_process_to_device - Have the IOMMU bind a process
>>> + *
>>> + * Binds the given process to the given device using its PASID. This
>>> + * enables IOMMUv2 address translation for the process on the device.
>>> + *
>>> + * This function assumes that the process mutex is held.
>>> + */
>>> +int kfd_iommu_bind_process_to_device(struct kfd_process_device *pdd)
>>> +{
>>> +Â Â Â struct kfd_dev *dev = pdd->dev;
>>> +Â Â Â struct kfd_process *p = pdd->process;
>>> +Â Â Â int err;
>>> +
>>> +Â Â Â if (!dev->device_info->needs_iommu_device || pdd->bound ==
>>> PDD_BOUND)
>>> +Â Â Â Â Â Â Â return 0;
>>> +
>>> +Â Â Â if (unlikely(pdd->bound == PDD_BOUND_SUSPENDED)) {
>>> +Â Â Â Â Â Â Â pr_err("Binding PDD_BOUND_SUSPENDED pdd is unexpected!\n");
>>> +Â Â Â Â Â Â Â return -EINVAL;
>>> +Â Â Â }
>>> +
>>> +Â Â Â err = amd_iommu_bind_pasid(dev->pdev, p->pasid, p->lead_thread);
>>> +Â Â Â if (!err)
>>> +Â Â Â Â Â Â Â pdd->bound = PDD_BOUND;
>>> +
>>> +Â Â Â return err;
>>> +}
>>> +
>>> +/** kfd_iommu_unbind_process - Unbind process from all devices
>>> + *
>>> + * This removes all IOMMU device bindings of the process. To be used
>>> + * before process termination.
>>> + */
>>> +void kfd_iommu_unbind_process(struct kfd_process *p)
>>> +{
>>> +Â Â Â struct kfd_process_device *pdd;
>>> +
>>> +Â Â Â list_for_each_entry(pdd, &p->per_device_data, per_device_list)
>>> +Â Â Â Â Â Â Â if (pdd->bound == PDD_BOUND)
>>> +Â Â Â Â Â Â Â Â Â Â Â amd_iommu_unbind_pasid(pdd->dev->pdev, p->pasid);
>>> +}
>>> +
>>> +/* Callback for process shutdown invoked by the IOMMU driver */
>>> +static void iommu_pasid_shutdown_callback(struct pci_dev *pdev, int
>>> pasid)
>>> +{
>>> +Â Â Â struct kfd_dev *dev = kfd_device_by_pci_dev(pdev);
>>> +Â Â Â struct kfd_process *p;
>>> +Â Â Â struct kfd_process_device *pdd;
>>> +
>>> +Â Â Â if (!dev)
>>> +Â Â Â Â Â Â Â return;
>>> +
>>> +Â Â Â /*
>>> +Â Â Â Â * Look for the process that matches the pasid. If there is no such
>>> +Â Â Â Â * process, we either released it in amdkfd's own notifier, or
>>> there
>>> +Â Â Â Â * is a bug. Unfortunately, there is no way to tell...
>>> +Â Â Â Â */
>>> +Â Â Â p = kfd_lookup_process_by_pasid(pasid);
>>> +Â Â Â if (!p)
>>> +Â Â Â Â Â Â Â return;
>>> +
>>> +Â Â Â pr_debug("Unbinding process %d from IOMMU\n", pasid);
>>> +
>>> +Â Â Â mutex_lock(kfd_get_dbgmgr_mutex());
>>> +
>>> +Â Â Â if (dev->dbgmgr && dev->dbgmgr->pasid == p->pasid) {
>>> +Â Â Â Â Â Â Â if (!kfd_dbgmgr_unregister(dev->dbgmgr, p)) {
>>> +Â Â Â Â Â Â Â Â Â Â Â kfd_dbgmgr_destroy(dev->dbgmgr);
>>> +Â Â Â Â Â Â Â Â Â Â Â dev->dbgmgr = NULL;
>>> +Â Â Â Â Â Â Â }
>>> +Â Â Â }
>>> +
>>> +Â Â Â mutex_unlock(kfd_get_dbgmgr_mutex());
>>> +
>>> +Â Â Â mutex_lock(&p->mutex);
>>> +
>>> +Â Â Â pdd = kfd_get_process_device_data(dev, p);
>>> +Â Â Â if (pdd)
>>> +Â Â Â Â Â Â Â /* For GPU relying on IOMMU, we need to dequeue here
>>> +Â Â Â Â Â Â Â Â * when PASID is still bound.
>>> +Â Â Â Â Â Â Â Â */
>>> +Â Â Â Â Â Â Â kfd_process_dequeue_from_device(pdd);
>>> +
>>> +Â Â Â mutex_unlock(&p->mutex);
>>> +
>>> +Â Â Â kfd_unref_process(p);
>>> +}
>>> +
>>> +/* This function called by IOMMU driver on PPR failure */
>>> +static int iommu_invalid_ppr_cb(struct pci_dev *pdev, int pasid,
>>> +Â Â Â Â Â Â Â unsigned long address, u16 flags)
>>> +{
>>> +Â Â Â struct kfd_dev *dev;
>>> +
>>> +Â Â Â dev_warn(kfd_device,
>>> +Â Â Â Â Â Â Â Â Â Â Â "Invalid PPR device %x:%x.%x pasid %d address 0x%lX
>>> flags 0x%X",
>>> +Â Â Â Â Â Â Â Â Â Â Â PCI_BUS_NUM(pdev->devfn),
>>> +Â Â Â Â Â Â Â Â Â Â Â PCI_SLOT(pdev->devfn),
>>> +Â Â Â Â Â Â Â Â Â Â Â PCI_FUNC(pdev->devfn),
>>> +Â Â Â Â Â Â Â Â Â Â Â pasid,
>>> +Â Â Â Â Â Â Â Â Â Â Â address,
>>> +Â Â Â Â Â Â Â Â Â Â Â flags);
>>> +
>>> +Â Â Â dev = kfd_device_by_pci_dev(pdev);
>>> +Â Â Â if (!WARN_ON(!dev))
>>> +Â Â Â Â Â Â Â kfd_signal_iommu_event(dev, pasid, address,
>>> +Â Â Â Â Â Â Â Â Â Â Â flags & PPR_FAULT_WRITE, flags & PPR_FAULT_EXEC);
>>> +
>>> +Â Â Â return AMD_IOMMU_INV_PRI_RSP_INVALID;
>>> +}
>>> +
>>> +/*
>>> + * Bind processes do the device that have been temporarily unbound
>>> + * (PDD_BOUND_SUSPENDED) in kfd_unbind_processes_from_device.
>>> + */
>>> +static int kfd_bind_processes_to_device(struct kfd_dev *kfd)
>>> +{
>>> +Â Â Â struct kfd_process_device *pdd;
>>> +Â Â Â struct kfd_process *p;
>>> +Â Â Â unsigned int temp;
>>> +Â Â Â int err = 0;
>>> +
>>> +Â Â Â int idx = srcu_read_lock(&kfd_processes_srcu);
>>> +
>>> +Â Â Â hash_for_each_rcu(kfd_processes_table, temp, p, kfd_processes) {
>>> +Â Â Â Â Â Â Â mutex_lock(&p->mutex);
>>> +Â Â Â Â Â Â Â pdd = kfd_get_process_device_data(kfd, p);
>>> +
>>> +Â Â Â Â Â Â Â if (WARN_ON(!pdd) || pdd->bound != PDD_BOUND_SUSPENDED) {
>>> +Â Â Â Â Â Â Â Â Â Â Â mutex_unlock(&p->mutex);
>>> +Â Â Â Â Â Â Â Â Â Â Â continue;
>>> +Â Â Â Â Â Â Â }
>>> +
>>> +Â Â Â Â Â Â Â err = amd_iommu_bind_pasid(kfd->pdev, p->pasid,
>>> +Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â p->lead_thread);
>>> +Â Â Â Â Â Â Â if (err < 0) {
>>> +Â Â Â Â Â Â Â Â Â Â Â pr_err("Unexpected pasid %d binding failure\n",
>>> +Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â p->pasid);
>>> +Â Â Â Â Â Â Â Â Â Â Â mutex_unlock(&p->mutex);
>>> +Â Â Â Â Â Â Â Â Â Â Â break;
>>> +Â Â Â Â Â Â Â }
>>> +
>>> +Â Â Â Â Â Â Â pdd->bound = PDD_BOUND;
>>> +Â Â Â Â Â Â Â mutex_unlock(&p->mutex);
>>> +Â Â Â }
>>> +
>>> +Â Â Â srcu_read_unlock(&kfd_processes_srcu, idx);
>>> +
>>> +Â Â Â return err;
>>> +}
>>> +
>>> +/*
>>> + * Mark currently bound processes as PDD_BOUND_SUSPENDED. These
>>> + * processes will be restored to PDD_BOUND state in
>>> + * kfd_bind_processes_to_device.
>>> + */
>>> +static void kfd_unbind_processes_from_device(struct kfd_dev *kfd)
>>> +{
>>> +Â Â Â struct kfd_process_device *pdd;
>>> +Â Â Â struct kfd_process *p;
>>> +Â Â Â unsigned int temp;
>>> +
>>> +Â Â Â int idx = srcu_read_lock(&kfd_processes_srcu);
>>> +
>>> +Â Â Â hash_for_each_rcu(kfd_processes_table, temp, p, kfd_processes) {
>>> +Â Â Â Â Â Â Â mutex_lock(&p->mutex);
>>> +Â Â Â Â Â Â Â pdd = kfd_get_process_device_data(kfd, p);
>>> +
>>> +Â Â Â Â Â Â Â if (WARN_ON(!pdd)) {
>>> +Â Â Â Â Â Â Â Â Â Â Â mutex_unlock(&p->mutex);
>>> +Â Â Â Â Â Â Â Â Â Â Â continue;
>>> +Â Â Â Â Â Â Â }
>>> +
>>> +Â Â Â Â Â Â Â if (pdd->bound == PDD_BOUND)
>>> +Â Â Â Â Â Â Â Â Â Â Â pdd->bound = PDD_BOUND_SUSPENDED;
>>> +Â Â Â Â Â Â Â mutex_unlock(&p->mutex);
>>> +Â Â Â }
>>> +
>>> +Â Â Â srcu_read_unlock(&kfd_processes_srcu, idx);
>>> +}
>>> +
>>> +/** kfd_iommu_suspend - Prepare IOMMU for suspend
>>> + *
>>> + * This unbinds processes from the device and disables the IOMMU for
>>> + * the device.
>>> + */
>>> +void kfd_iommu_suspend(struct kfd_dev *kfd)
>>> +{
>>> +Â Â Â if (!kfd->device_info->needs_iommu_device)
>>> +Â Â Â Â Â Â Â return;
>>> +
>>> +Â Â Â kfd_unbind_processes_from_device(kfd);
>>> +
>>> +Â Â Â amd_iommu_set_invalidate_ctx_cb(kfd->pdev, NULL);
>>> +Â Â Â amd_iommu_set_invalid_ppr_cb(kfd->pdev, NULL);
>>> +Â Â Â amd_iommu_free_device(kfd->pdev);
>>> +}
>>> +
>>> +/** kfd_iommu_resume - Restore IOMMU after resume
>>> + *
>>> + * This reinitializes the IOMMU for the device and re-binds previously
>>> + * suspended processes to the device.
>>> + */
>>> +int kfd_iommu_resume(struct kfd_dev *kfd)
>>> +{
>>> +Â Â Â unsigned int pasid_limit;
>>> +Â Â Â int err;
>>> +
>>> +Â Â Â if (!kfd->device_info->needs_iommu_device)
>>> +Â Â Â Â Â Â Â return 0;
>>> +
>>> +Â Â Â pasid_limit = kfd_get_pasid_limit();
>>> +
>>> +Â Â Â err = amd_iommu_init_device(kfd->pdev, pasid_limit);
>>> +Â Â Â if (err)
>>> +Â Â Â Â Â Â Â return -ENXIO;
>>> +
>>> +Â Â Â amd_iommu_set_invalidate_ctx_cb(kfd->pdev,
>>> +Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â iommu_pasid_shutdown_callback);
>>> +Â Â Â amd_iommu_set_invalid_ppr_cb(kfd->pdev,
>>> +Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â iommu_invalid_ppr_cb);
>>> +
>>> +Â Â Â err = kfd_bind_processes_to_device(kfd);
>>> +Â Â Â if (err) {
>>> +Â Â Â Â Â Â Â amd_iommu_set_invalidate_ctx_cb(kfd->pdev, NULL);
>>> +Â Â Â Â Â Â Â amd_iommu_set_invalid_ppr_cb(kfd->pdev, NULL);
>>> +Â Â Â Â Â Â Â amd_iommu_free_device(kfd->pdev);
>>> +Â Â Â Â Â Â Â return err;
>>> +Â Â Â }
>>> +
>>> +Â Â Â return 0;
>>> +}
>>> +
>>> +extern bool amd_iommu_pc_supported(void);
>>> +extern u8 amd_iommu_pc_get_max_banks(u16 devid);
>>> +extern u8 amd_iommu_pc_get_max_counters(u16 devid);
>>> +
>>> +/** kfd_iommu_add_perf_counters - Add IOMMU performance counters to
>>> topology
>>> + */
>>> +int kfd_iommu_add_perf_counters(struct kfd_topology_device *kdev)
>>> +{
>>> +Â Â Â struct kfd_perf_properties *props;
>>> +
>>> +Â Â Â if (!(kdev->node_props.capability & HSA_CAP_ATS_PRESENT))
>>> +Â Â Â Â Â Â Â return 0;
>>> +
>>> +Â Â Â if (!amd_iommu_pc_supported())
>>> +Â Â Â Â Â Â Â return 0;
>>> +
>>> +Â Â Â props = kfd_alloc_struct(props);
>>> +Â Â Â if (!props)
>>> +Â Â Â Â Â Â Â return -ENOMEM;
>>> +Â Â Â strcpy(props->block_name, "iommu");
>>> +Â Â Â props->max_concurrent = amd_iommu_pc_get_max_banks(0) *
>>> +Â Â Â Â Â Â Â amd_iommu_pc_get_max_counters(0); /* assume one iommu */
>>> +Â Â Â list_add_tail(&props->list, &kdev->perf_props);
>>> +
>>> +Â Â Â return 0;
>>> +}
>>> diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_iommu.h
>>> b/drivers/gpu/drm/amd/amdkfd/kfd_iommu.h
>>> new file mode 100644
>>> index 0000000..dd23d9f
>>> --- /dev/null
>>> +++ b/drivers/gpu/drm/amd/amdkfd/kfd_iommu.h
>>> @@ -0,0 +1,78 @@
>>> +/*
>>> + * Copyright 2018 Advanced Micro Devices, Inc.
>>> + *
>>> + * Permission is hereby granted, free of charge, to any person
>>> obtaining a
>>> + * copy of this software and associated documentation files (the
>>> "Software"),
>>> + * to deal in the Software without restriction, including without
>>> limitation
>>> + * the rights to use, copy, modify, merge, publish, distribute,
>>> sublicense,
>>> + * and/or sell copies of the Software, and to permit persons to whom
>>> the
>>> + * Software is furnished to do so, subject to the following conditions:
>>> + *
>>> + * The above copyright notice and this permission notice shall be
>>> included in
>>> + * all copies or substantial portions of the Software.
>>> + *
>>> + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
>>> EXPRESS OR
>>> + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
>>> MERCHANTABILITY,
>>> + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO
>>> EVENT SHALL
>>> + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM,
>>> DAMAGES OR
>>> + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
>>> OTHERWISE,
>>> + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
>>> USE OR
>>> + * OTHER DEALINGS IN THE SOFTWARE.
>>> + */
>>> +
>>> +#ifndef __KFD_IOMMU_H__
>>> +#define __KFD_IOMMU_H__
>>> +
>>> +#if defined(CONFIG_AMD_IOMMU_V2_MODULE) || defined(CONFIG_AMD_IOMMU_V2)
>>> +
>>> +#define KFD_SUPPORT_IOMMU_V2
>>> +
>>> +int kfd_iommu_check_device(struct kfd_dev *kfd);
>>> +int kfd_iommu_device_init(struct kfd_dev *kfd);
>>> +
>>> +int kfd_iommu_bind_process_to_device(struct kfd_process_device *pdd);
>>> +void kfd_iommu_unbind_process(struct kfd_process *p);
>>> +
>>> +void kfd_iommu_suspend(struct kfd_dev *kfd);
>>> +int kfd_iommu_resume(struct kfd_dev *kfd);
>>> +
>>> +int kfd_iommu_add_perf_counters(struct kfd_topology_device *kdev);
>>> +
>>> +#else
>>> +
>>> +static inline int kfd_iommu_check_device(struct kfd_dev *kfd)
>>> +{
>>> +Â Â Â return -ENODEV;
>>> +}
>>> +static inline int kfd_iommu_device_init(struct kfd_dev *kfd)
>>> +{
>>> +Â Â Â return 0;
>>> +}
>>> +
>>> +static inline int kfd_iommu_bind_process_to_device(
>>> +Â Â Â struct kfd_process_device *pdd)
>>> +{
>>> +Â Â Â return 0;
>>> +}
>>> +static inline void kfd_iommu_unbind_process(struct kfd_process *p)
>>> +{
>>> +Â Â Â /* empty */
>>> +}
>>> +
>>> +static inline void kfd_iommu_suspend(struct kfd_dev *kfd)
>>> +{
>>> +Â Â Â /* empty */
>>> +}
>>> +static inline int kfd_iommu_resume(struct kfd_dev *kfd)
>>> +{
>>> +Â Â Â return 0;
>>> +}
>>> +
>>> +static inline int kfd_iommu_add_perf_counters(struct
>>> kfd_topology_device *kdev)
>>> +{
>>> +Â Â Â return 0;
>>> +}
>>> +
>>> +#endif /* defined(CONFIG_AMD_IOMMU_V2) */
>>> +
>>> +#endif /* __KFD_IOMMU_H__ */
>>> diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
>>> b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
>>> index 594f853..f12eb5d 100644
>>> --- a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
>>> +++ b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
>>> @@ -158,6 +158,7 @@ struct kfd_device_info {
>>> Â Â Â Â Â uint8_t num_of_watch_points;
>>> Â Â Â Â Â uint16_t mqd_size_aligned;
>>> Â Â Â Â Â bool supports_cwsr;
>>> +Â Â Â bool needs_iommu_device;
>>> Â Â Â Â Â bool needs_pci_atomics;
>>> Â };
>>> Â @@ -517,15 +518,15 @@ struct kfd_process_device {
>>> Â Â Â Â Â uint64_t scratch_base;
>>> Â Â Â Â Â uint64_t scratch_limit;
>>> Â -Â Â Â /* Is this process/pasid bound to this device?
>>> (amd_iommu_bind_pasid) */
>>> -Â Â Â enum kfd_pdd_bound bound;
>>> -
>>> Â Â Â Â Â /* Flag used to tell the pdd has dequeued from the dqm.
>>> Â Â Â Â Â Â * This is used to prevent dev->dqm->ops.process_termination()
>>> from
>>> Â Â Â Â Â Â * being called twice when it is already called in IOMMU callback
>>> Â Â Â Â Â Â * function.
>>> Â Â Â Â Â Â */
>>> Â Â Â Â Â bool already_dequeued;
>>> +
>>> +Â Â Â /* Is this process/pasid bound to this device?
>>> (amd_iommu_bind_pasid) */
>>> +Â Â Â enum kfd_pdd_bound bound;
>>> Â };
>>> Â Â #define qpd_to_pdd(x) container_of(x, struct kfd_process_device,
>>> qpd)
>>> @@ -590,6 +591,10 @@ struct kfd_process {
>>> Â Â Â Â Â bool signal_event_limit_reached;
>>> Â };
>>> Â +#define KFD_PROCESS_TABLE_SIZE 5 /* bits: 32 entries */
>>> +extern DECLARE_HASHTABLE(kfd_processes_table, KFD_PROCESS_TABLE_SIZE);
>>> +extern struct srcu_struct kfd_processes_srcu;
>>> +
>>> Â /**
>>> Â Â * Ioctl function type.
>>> Â Â *
>>> @@ -617,9 +622,6 @@ void kfd_unref_process(struct kfd_process *p);
>>> Â Â struct kfd_process_device *kfd_bind_process_to_device(struct
>>> kfd_dev *dev,
>>> Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â struct kfd_process *p);
>>> -int kfd_bind_processes_to_device(struct kfd_dev *dev);
>>> -void kfd_unbind_processes_from_device(struct kfd_dev *dev);
>>> -void kfd_process_iommu_unbind_callback(struct kfd_dev *dev, unsigned
>>> int pasid);
>>> Â struct kfd_process_device *kfd_get_process_device_data(struct
>>> kfd_dev *dev,
>>> Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â struct kfd_process *p);
>>> Â struct kfd_process_device *kfd_create_process_device_data(struct
>>> kfd_dev *dev,
>>> diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_process.c
>>> b/drivers/gpu/drm/amd/amdkfd/kfd_process.c
>>> index 4ff5f0f..e9aee76 100644
>>> --- a/drivers/gpu/drm/amd/amdkfd/kfd_process.c
>>> +++ b/drivers/gpu/drm/amd/amdkfd/kfd_process.c
>>> @@ -35,16 +35,16 @@ struct mm_struct;
>>> Â Â #include "kfd_priv.h"
>>> Â #include "kfd_dbgmgr.h"
>>> +#include "kfd_iommu.h"
>>> Â Â /*
>>> Â Â * List of struct kfd_process (field kfd_process).
>>> Â Â * Unique/indexed by mm_struct*
>>> Â Â */
>>> -#define KFD_PROCESS_TABLE_SIZE 5 /* bits: 32 entries */
>>> -static DEFINE_HASHTABLE(kfd_processes_table, KFD_PROCESS_TABLE_SIZE);
>>> +DEFINE_HASHTABLE(kfd_processes_table, KFD_PROCESS_TABLE_SIZE);
>>> Â static DEFINE_MUTEX(kfd_processes_mutex);
>>> Â -DEFINE_STATIC_SRCU(kfd_processes_srcu);
>>> +DEFINE_SRCU(kfd_processes_srcu);
>>> Â Â static struct workqueue_struct *kfd_process_wq;
>>> Â @@ -173,14 +173,8 @@ static void kfd_process_wq_release(struct
>>> work_struct *work)
>>> Â {
>>> Â Â Â Â Â struct kfd_process *p = container_of(work, struct kfd_process,
>>> Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â release_work);
>>> -Â Â Â struct kfd_process_device *pdd;
>>> Â -Â Â Â pr_debug("Releasing process (pasid %d) in workqueue\n",
>>> p->pasid);
>>> -
>>> -Â Â Â list_for_each_entry(pdd, &p->per_device_data, per_device_list) {
>>> -Â Â Â Â Â Â Â if (pdd->bound == PDD_BOUND)
>>> -Â Â Â Â Â Â Â Â Â Â Â amd_iommu_unbind_pasid(pdd->dev->pdev, p->pasid);
>>> -Â Â Â }
>>> +Â Â Â kfd_iommu_unbind_process(p);
>>> Â Â Â Â Â Â kfd_process_destroy_pdds(p);
>>> Â @@ -429,133 +423,13 @@ struct kfd_process_device
>>> *kfd_bind_process_to_device(struct kfd_dev *dev,
>>> Â Â Â Â Â Â Â Â Â return ERR_PTR(-ENOMEM);
>>> Â Â Â Â Â }
>>> Â -Â Â Â if (pdd->bound == PDD_BOUND) {
>>> -Â Â Â Â Â Â Â return pdd;
>>> -Â Â Â } else if (unlikely(pdd->bound == PDD_BOUND_SUSPENDED)) {
>>> -Â Â Â Â Â Â Â pr_err("Binding PDD_BOUND_SUSPENDED pdd is unexpected!\n");
>>> -Â Â Â Â Â Â Â return ERR_PTR(-EINVAL);
>>> -Â Â Â }
>>> -
>>> -Â Â Â err = amd_iommu_bind_pasid(dev->pdev, p->pasid, p->lead_thread);
>>> -Â Â Â if (err < 0)
>>> +Â Â Â err = kfd_iommu_bind_process_to_device(pdd);
>>> +Â Â Â if (err)
>>> Â Â Â Â Â Â Â Â Â return ERR_PTR(err);
>>> Â -Â Â Â pdd->bound = PDD_BOUND;
>>> -
>>> Â Â Â Â Â return pdd;
>>> Â }
>>> Â -/*
>>> - * Bind processes do the device that have been temporarily unbound
>>> - * (PDD_BOUND_SUSPENDED) in kfd_unbind_processes_from_device.
>>> - */
>>> -int kfd_bind_processes_to_device(struct kfd_dev *dev)
>>> -{
>>> -Â Â Â struct kfd_process_device *pdd;
>>> -Â Â Â struct kfd_process *p;
>>> -Â Â Â unsigned int temp;
>>> -Â Â Â int err = 0;
>>> -
>>> -Â Â Â int idx = srcu_read_lock(&kfd_processes_srcu);
>>> -
>>> -Â Â Â hash_for_each_rcu(kfd_processes_table, temp, p, kfd_processes) {
>>> -Â Â Â Â Â Â Â mutex_lock(&p->mutex);
>>> -Â Â Â Â Â Â Â pdd = kfd_get_process_device_data(dev, p);
>>> -
>>> -Â Â Â Â Â Â Â if (WARN_ON(!pdd) || pdd->bound != PDD_BOUND_SUSPENDED) {
>>> -Â Â Â Â Â Â Â Â Â Â Â mutex_unlock(&p->mutex);
>>> -Â Â Â Â Â Â Â Â Â Â Â continue;
>>> -Â Â Â Â Â Â Â }
>>> -
>>> -Â Â Â Â Â Â Â err = amd_iommu_bind_pasid(dev->pdev, p->pasid,
>>> -Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â p->lead_thread);
>>> -Â Â Â Â Â Â Â if (err < 0) {
>>> -Â Â Â Â Â Â Â Â Â Â Â pr_err("Unexpected pasid %d binding failure\n",
>>> -Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â p->pasid);
>>> -Â Â Â Â Â Â Â Â Â Â Â mutex_unlock(&p->mutex);
>>> -Â Â Â Â Â Â Â Â Â Â Â break;
>>> -Â Â Â Â Â Â Â }
>>> -
>>> -Â Â Â Â Â Â Â pdd->bound = PDD_BOUND;
>>> -Â Â Â Â Â Â Â mutex_unlock(&p->mutex);
>>> -Â Â Â }
>>> -
>>> -Â Â Â srcu_read_unlock(&kfd_processes_srcu, idx);
>>> -
>>> -Â Â Â return err;
>>> -}
>>> -
>>> -/*
>>> - * Mark currently bound processes as PDD_BOUND_SUSPENDED. These
>>> - * processes will be restored to PDD_BOUND state in
>>> - * kfd_bind_processes_to_device.
>>> - */
>>> -void kfd_unbind_processes_from_device(struct kfd_dev *dev)
>>> -{
>>> -Â Â Â struct kfd_process_device *pdd;
>>> -Â Â Â struct kfd_process *p;
>>> -Â Â Â unsigned int temp;
>>> -
>>> -Â Â Â int idx = srcu_read_lock(&kfd_processes_srcu);
>>> -
>>> -Â Â Â hash_for_each_rcu(kfd_processes_table, temp, p, kfd_processes) {
>>> -Â Â Â Â Â Â Â mutex_lock(&p->mutex);
>>> -Â Â Â Â Â Â Â pdd = kfd_get_process_device_data(dev, p);
>>> -
>>> -Â Â Â Â Â Â Â if (WARN_ON(!pdd)) {
>>> -Â Â Â Â Â Â Â Â Â Â Â mutex_unlock(&p->mutex);
>>> -Â Â Â Â Â Â Â Â Â Â Â continue;
>>> -Â Â Â Â Â Â Â }
>>> -
>>> -Â Â Â Â Â Â Â if (pdd->bound == PDD_BOUND)
>>> -Â Â Â Â Â Â Â Â Â Â Â pdd->bound = PDD_BOUND_SUSPENDED;
>>> -Â Â Â Â Â Â Â mutex_unlock(&p->mutex);
>>> -Â Â Â }
>>> -
>>> -Â Â Â srcu_read_unlock(&kfd_processes_srcu, idx);
>>> -}
>>> -
>>> -void kfd_process_iommu_unbind_callback(struct kfd_dev *dev, unsigned
>>> int pasid)
>>> -{
>>> -Â Â Â struct kfd_process *p;
>>> -Â Â Â struct kfd_process_device *pdd;
>>> -
>>> -Â Â Â /*
>>> -Â Â Â Â * Look for the process that matches the pasid. If there is no such
>>> -Â Â Â Â * process, we either released it in amdkfd's own notifier, or
>>> there
>>> -Â Â Â Â * is a bug. Unfortunately, there is no way to tell...
>>> -Â Â Â Â */
>>> -Â Â Â p = kfd_lookup_process_by_pasid(pasid);
>>> -Â Â Â if (!p)
>>> -Â Â Â Â Â Â Â return;
>>> -
>>> -Â Â Â pr_debug("Unbinding process %d from IOMMU\n", pasid);
>>> -
>>> -Â Â Â mutex_lock(kfd_get_dbgmgr_mutex());
>>> -
>>> -Â Â Â if (dev->dbgmgr && dev->dbgmgr->pasid == p->pasid) {
>>> -Â Â Â Â Â Â Â if (!kfd_dbgmgr_unregister(dev->dbgmgr, p)) {
>>> -Â Â Â Â Â Â Â Â Â Â Â kfd_dbgmgr_destroy(dev->dbgmgr);
>>> -Â Â Â Â Â Â Â Â Â Â Â dev->dbgmgr = NULL;
>>> -Â Â Â Â Â Â Â }
>>> -Â Â Â }
>>> -
>>> -Â Â Â mutex_unlock(kfd_get_dbgmgr_mutex());
>>> -
>>> -Â Â Â mutex_lock(&p->mutex);
>>> -
>>> -Â Â Â pdd = kfd_get_process_device_data(dev, p);
>>> -Â Â Â if (pdd)
>>> -Â Â Â Â Â Â Â /* For GPU relying on IOMMU, we need to dequeue here
>>> -Â Â Â Â Â Â Â Â * when PASID is still bound.
>>> -Â Â Â Â Â Â Â Â */
>>> -Â Â Â Â Â Â Â kfd_process_dequeue_from_device(pdd);
>>> -
>>> -Â Â Â mutex_unlock(&p->mutex);
>>> -
>>> -Â Â Â kfd_unref_process(p);
>>> -}
>>> -
>>> Â struct kfd_process_device *kfd_get_first_process_device_data(
>>> Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â struct kfd_process *p)
>>> Â {
>>> diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_topology.c
>>> b/drivers/gpu/drm/amd/amdkfd/kfd_topology.c
>>> index 7783250..2506155 100644
>>> --- a/drivers/gpu/drm/amd/amdkfd/kfd_topology.c
>>> +++ b/drivers/gpu/drm/amd/amdkfd/kfd_topology.c
>>> @@ -35,6 +35,7 @@
>>> Â #include "kfd_crat.h"
>>> Â #include "kfd_topology.h"
>>> Â #include "kfd_device_queue_manager.h"
>>> +#include "kfd_iommu.h"
>>> Â Â /* topology_device_list - Master list of all topology devices */
>>> Â static struct list_head topology_device_list;
>>> @@ -875,19 +876,8 @@ static void find_system_memory(const struct
>>> dmi_header *dm,
>>> Â Â */
>>> Â static int kfd_add_perf_to_topology(struct kfd_topology_device *kdev)
>>> Â {
>>> -Â Â Â struct kfd_perf_properties *props;
>>> -
>>> -Â Â Â if (amd_iommu_pc_supported()) {
>>> -Â Â Â Â Â Â Â props = kfd_alloc_struct(props);
>>> -Â Â Â Â Â Â Â if (!props)
>>> -Â Â Â Â Â Â Â Â Â Â Â return -ENOMEM;
>>> -Â Â Â Â Â Â Â strcpy(props->block_name, "iommu");
>>> -Â Â Â Â Â Â Â props->max_concurrent = amd_iommu_pc_get_max_banks(0) *
>>> -Â Â Â Â Â Â Â Â Â Â Â amd_iommu_pc_get_max_counters(0); /* assume one iommu */
>>> -Â Â Â Â Â Â Â list_add_tail(&props->list, &kdev->perf_props);
>>> -Â Â Â }
>>> -
>>> -Â Â Â return 0;
>>> +Â Â Â /* These are the only counters supported so far */
>>> +Â Â Â return kfd_iommu_add_perf_counters(kdev);
>>> Â }
>>> Â Â /* kfd_add_non_crat_information - Add information that is not
>>> currently
>>> diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_topology.h
>>> b/drivers/gpu/drm/amd/amdkfd/kfd_topology.h
>>> index 53fca1f..c0be2be 100644
>>> --- a/drivers/gpu/drm/amd/amdkfd/kfd_topology.h
>>> +++ b/drivers/gpu/drm/amd/amdkfd/kfd_topology.h
>>> @@ -25,7 +25,7 @@
>>> Â Â #include <linux/types.h>
>>> Â #include <linux/list.h>
>>> -#include "kfd_priv.h"
>>> +#include "kfd_crat.h"
>>> Â Â #define KFD_TOPOLOGY_PUBLIC_NAME_SIZE 128
>>> Â @@ -183,8 +183,4 @@ struct kfd_topology_device
>>> *kfd_create_topology_device(
>>> Â Â Â Â Â Â Â Â Â struct list_head *device_list);
>>> Â void kfd_release_topology_device_list(struct list_head *device_list);
>>> Â -extern bool amd_iommu_pc_supported(void);
>>> -extern u8 amd_iommu_pc_get_max_banks(u16 devid);
>>> -extern u8 amd_iommu_pc_get_max_counters(u16 devid);
>>> -
>>> Â #endif /* __KFD_TOPOLOGY_H__ */
>
>
> _______________________________________________
> amd-gfx mailing list
> amd-gfx at lists.freedesktop.org
> https://lists.freedesktop.org/mailman/listinfo/amd-gfx
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <https://lists.freedesktop.org/archives/amd-gfx/attachments/20180208/08e99c95/attachment-0001.html>
