Replace our MMU notifier with hmm_mirror_ops.sync_cpu_device_pagetables callback. Enable CONFIG_HMM and CONFIG_HMM_MIRROR as a dependency in DRM_AMDGPU_USERPTR Kconfig. It supports both KFD userptr and gfx userptr paths. This depends on several HMM patchset from Jérôme Glisse queued for upstream. See http://172.27.226.38/root/kernel_amd/commits/hmm-dev-v01 (for AMD intranet) Change-Id: Ie62c3c5e3c5b8521ab3b438d1eff2aa2a003835e Signed-off-by: Philip Yang <Philip.Yang at amd.com> --- drivers/gpu/drm/amd/amdgpu/Kconfig | 6 +-- drivers/gpu/drm/amd/amdgpu/Makefile | 1 + drivers/gpu/drm/amd/amdgpu/amdgpu_hmm.c | 78 +++++++++++++++++++++++++++++++++ drivers/gpu/drm/amd/amdgpu/amdgpu_hmm.h | 41 +++++++++++++++++ drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c | 50 ++++++++++++++++++++- drivers/gpu/drm/amd/amdgpu/amdgpu_mn.h | 7 +++ 6 files changed, 178 insertions(+), 5 deletions(-) create mode 100644 drivers/gpu/drm/amd/amdgpu/amdgpu_hmm.c create mode 100644 drivers/gpu/drm/amd/amdgpu/amdgpu_hmm.h diff --git a/drivers/gpu/drm/amd/amdgpu/Kconfig b/drivers/gpu/drm/amd/amdgpu/Kconfig index 9221e54..960a633 100644 --- a/drivers/gpu/drm/amd/amdgpu/Kconfig +++ b/drivers/gpu/drm/amd/amdgpu/Kconfig @@ -26,10 +26,10 @@ config DRM_AMDGPU_CIK config DRM_AMDGPU_USERPTR bool "Always enable userptr write support" depends on DRM_AMDGPU - select MMU_NOTIFIER + select HMM_MIRROR help - This option selects CONFIG_MMU_NOTIFIER if it isn't already - selected to enabled full userptr support. + This option selects CONFIG_HMM and CONFIG_HMM_MIRROR if it + isn't already selected to enabled full userptr support. config DRM_AMDGPU_GART_DEBUGFS bool "Allow GART access through debugfs" diff --git a/drivers/gpu/drm/amd/amdgpu/Makefile b/drivers/gpu/drm/amd/amdgpu/Makefile index 138cb78..ee691e8 100644 --- a/drivers/gpu/drm/amd/amdgpu/Makefile +++ b/drivers/gpu/drm/amd/amdgpu/Makefile @@ -172,6 +172,7 @@ amdgpu-$(CONFIG_COMPAT) += amdgpu_ioc32.o amdgpu-$(CONFIG_VGA_SWITCHEROO) += amdgpu_atpx_handler.o amdgpu-$(CONFIG_ACPI) += amdgpu_acpi.o amdgpu-$(CONFIG_MMU_NOTIFIER) += amdgpu_mn.o +amdgpu-$(CONFIG_HMM) += amdgpu_hmm.o include $(FULL_AMD_PATH)/powerplay/Makefile diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_hmm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_hmm.c new file mode 100644 index 0000000..6c506f6 --- /dev/null +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_hmm.c @@ -0,0 +1,78 @@ +/* + * Copyright 2018 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#include <linux/mutex.h> +#include <linux/slab.h> +#include <linux/notifier.h> +#include <linux/compat.h> +#include <linux/mman.h> +#include <linux/hmm.h> +#include <asm/page.h> +#include "amdgpu.h" +#include "amdgpu_mn.h" + +static void amdgpu_hmm_release(struct hmm_mirror *mirror) +{ + pr_debug("mirror=%p\n", mirror); + amdgpu_hmm_mn_release(mirror); +} + +static int amdgpu_hmm_sync_cpu_device_pagetables(struct hmm_mirror *mirror, + const struct hmm_update *update) +{ + struct hmm *hmm; + struct mm_struct *mm; + unsigned long start; + unsigned long end; + + start = update->start; + end = update->end; + + pr_debug("mirror %p start %lx end %lx\n", mirror, start, end); + + hmm = mirror->hmm; + mm = *(struct mm_struct **)hmm; + + return amdgpu_mn_invalidate_range(mirror, mm, start, end, + update->blockable); +} + +static struct hmm_mirror_ops amdgpu_hmm_mirror_ops = { + .sync_cpu_device_pagetables = amdgpu_hmm_sync_cpu_device_pagetables, + .release = amdgpu_hmm_release +}; + +int amdgpu_hmm_register(struct hmm_mirror *mirror, struct mm_struct *mm) +{ + pr_debug("mirror=%p\n", mirror); + + mirror->ops = &amdgpu_hmm_mirror_ops; + + return hmm_mirror_register(mirror, mm); +} + +void amdgpu_hmm_unregister(struct hmm_mirror *mirror) +{ + pr_debug("mirror=%p\n", mirror); + + hmm_mirror_unregister(mirror); +} diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_hmm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_hmm.h new file mode 100644 index 0000000..a21a5f6 --- /dev/null +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_hmm.h @@ -0,0 +1,41 @@ +/* + * Copyright 2018 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#ifndef __AMDGPU_HMM_H__ +#define __AMDGPU_HMM_H__ + +#include <linux/mm.h> +#include <linux/hmm.h> + +#if IS_ENABLED(CONFIG_HMM) + +int amdgpu_hmm_register(struct hmm_mirror *mirror, struct mm_struct *mm); +void amdgpu_hmm_unregister(struct hmm_mirror *mirror); + +#else + +#define amdgpu_hmm_register(x, y) (0) +#define amdgpu_hmm_unregister(x) + +#endif + +#endif diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c index e55508b..2f5df5b 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c @@ -47,11 +47,13 @@ #include <linux/module.h> #include <linux/mmu_notifier.h> #include <linux/interval_tree.h> +#include <linux/hmm.h> #include <drm/drmP.h> #include <drm/drm.h> #include "amdgpu.h" #include "amdgpu_amdkfd.h" +#include "amdgpu_hmm.h" /** * struct amdgpu_mn @@ -66,6 +68,7 @@ * @objects: interval tree containing amdgpu_mn_nodes * @read_lock: mutex for recursive locking of @lock * @recursion: depth of recursion + * @hmm_mirror: HMM mirror function support * * Data for each amdgpu device and process address space. */ @@ -87,6 +90,9 @@ struct amdgpu_mn { struct rb_root_cached objects; struct mutex read_lock; atomic_t recursion; + + /* HMM mirror */ + struct hmm_mirror hmm_mirror; }; /** @@ -129,7 +135,9 @@ static void amdgpu_mn_destroy(struct work_struct *work) } up_write(&amn->lock); mutex_unlock(&adev->mn_lock); - mmu_notifier_unregister_no_release(&amn->mn, amn->mm); + + amdgpu_hmm_unregister(&amn->hmm_mirror); + kfree(amn); } @@ -372,6 +380,44 @@ static const struct mmu_notifier_ops amdgpu_mn_ops[] = { */ #define AMDGPU_MN_KEY(mm, type) ((unsigned long)(mm) + (type)) +void amdgpu_hmm_mn_release(struct hmm_mirror *mirror) +{ + struct amdgpu_mn *amn; + + amn = container_of(mirror, struct amdgpu_mn, hmm_mirror); + + amdgpu_mn_release(&amn->mn, amn->mm); +} + +int amdgpu_mn_invalidate_range(struct hmm_mirror *mirror, + struct mm_struct *mm, + unsigned long start, + unsigned long end, + bool blockable) + +{ + unsigned long key; + struct amdgpu_mn *amn; + int r = 0; + + amn = container_of(mirror, struct amdgpu_mn, hmm_mirror); + key = AMDGPU_MN_KEY(mm, amn->type); + + hash_for_each_possible(amn->adev->mn_hash, amn, node, key) + if (AMDGPU_MN_KEY(amn->mm, amn->type) == key) { + r = amn->mn.ops->invalidate_range_start(&amn->mn, mm, + start, end, blockable); + amn->mn.ops->invalidate_range_end(&amn->mn, mm, + start, end); + if (r) { + DRM_ERROR("failed to invalidate %lx\n", start); + break; + } + } + + return r; +} + /** * amdgpu_mn_get - create notifier context * @@ -413,7 +459,7 @@ struct amdgpu_mn *amdgpu_mn_get(struct amdgpu_device *adev, mutex_init(&amn->read_lock); atomic_set(&amn->recursion, 0); - r = __mmu_notifier_register(&amn->mn, mm); + r = amdgpu_hmm_register(&amn->hmm_mirror, mm); if (r) goto free_amn; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_mn.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_mn.h index eb0f432..e7781db 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_mn.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_mn.h @@ -28,6 +28,7 @@ * MMU Notifier */ struct amdgpu_mn; +struct hmm_mirror; enum amdgpu_mn_type { AMDGPU_MN_TYPE_GFX, @@ -41,6 +42,12 @@ struct amdgpu_mn *amdgpu_mn_get(struct amdgpu_device *adev, enum amdgpu_mn_type type); int amdgpu_mn_register(struct amdgpu_bo *bo, unsigned long addr); void amdgpu_mn_unregister(struct amdgpu_bo *bo); +int amdgpu_mn_invalidate_range(struct hmm_mirror *mirror, + struct mm_struct *mm, + unsigned long start, + unsigned long end, + bool blockable); +void amdgpu_hmm_mn_release(struct hmm_mirror *mirror); #else static inline void amdgpu_mn_lock(struct amdgpu_mn *mn) {} static inline void amdgpu_mn_unlock(struct amdgpu_mn *mn) {} -- 2.7.4
