From: Mike Day <michael.day@xxxxxxx> DMA drivers should implement following functions to enable folio migration offloading: migrate_dma() - This function takes src and dst folios list undergoing migration. It is responsible for transfer of page content between the src and dst folios. can_migrate_dma() - It performs necessary checks if DMA-migration is supported for the give src and dst folios. DMA driver should include a mechanism to call start_offloading and stop_offloading for enabling and disabling migration offload respectively. Signed-off-by: Mike Day <michael.day@xxxxxxx> Signed-off-by: Shivank Garg <shivankg@xxxxxxx> --- include/linux/migrate_dma.h | 36 ++++++++++++++++++++++++++ mm/Kconfig | 8 ++++++ mm/Makefile | 1 + mm/migrate.c | 40 +++++++++++++++++++++++++++-- mm/migrate_dma.c | 51 +++++++++++++++++++++++++++++++++++++ 5 files changed, 134 insertions(+), 2 deletions(-) create mode 100644 include/linux/migrate_dma.h create mode 100644 mm/migrate_dma.c diff --git a/include/linux/migrate_dma.h b/include/linux/migrate_dma.h new file mode 100644 index 000000000000..307b234450c3 --- /dev/null +++ b/include/linux/migrate_dma.h @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef _MIGRATE_DMA_H +#define _MIGRATE_DMA_H +#include <linux/migrate_mode.h> + +#define MIGRATOR_NAME_LEN 32 +struct migrator { + char name[MIGRATOR_NAME_LEN]; + void (*migrate_dma)(struct list_head *dst_list, struct list_head *src_list); + bool (*can_migrate_dma)(struct folio *dst, struct folio *src); + struct rcu_head srcu_head; + struct module *owner; +}; + +extern struct migrator migrator; +extern struct mutex migrator_mut; +extern struct srcu_struct mig_srcu; + +#ifdef CONFIG_DMA_MIGRATION +void srcu_mig_cb(struct rcu_head *head); +void dma_update_migrator(struct migrator *mig); +unsigned char *get_active_migrator_name(void); +bool can_dma_migrate(struct folio *dst, struct folio *src); +void start_offloading(struct migrator *migrator); +void stop_offloading(void); +#else +static inline void srcu_mig_cb(struct rcu_head *head) { }; +static inline void dma_update_migrator(struct migrator *mig) { }; +static inline unsigned char *get_active_migrator_name(void) { return NULL; }; +static inline bool can_dma_migrate(struct folio *dst, struct folio *src) {return true; }; +static inline void start_offloading(struct migrator *migrator) { }; +static inline void stop_offloading(void) { }; +#endif /* CONFIG_DMA_MIGRATION */ + +#endif /* _MIGRATE_DMA_H */ diff --git a/mm/Kconfig b/mm/Kconfig index ffc3a2ba3a8c..e3ff6583fedb 100644 --- a/mm/Kconfig +++ b/mm/Kconfig @@ -662,6 +662,14 @@ config MIGRATION config DEVICE_MIGRATION def_bool MIGRATION && ZONE_DEVICE +config DMA_MIGRATION + bool "Migrate Pages offloading copy to DMA" + def_bool n + depends on MIGRATION + help + An interface allowing external modules or driver to offload + page copying in page migration. + config ARCH_ENABLE_HUGEPAGE_MIGRATION bool diff --git a/mm/Makefile b/mm/Makefile index e4b5b75aaec9..1e31fb79d700 100644 --- a/mm/Makefile +++ b/mm/Makefile @@ -87,6 +87,7 @@ obj-$(CONFIG_FAILSLAB) += failslab.o obj-$(CONFIG_FAIL_PAGE_ALLOC) += fail_page_alloc.o obj-$(CONFIG_MEMTEST) += memtest.o obj-$(CONFIG_MIGRATION) += migrate.o +obj-$(CONFIG_DMA_MIGRATION) += migrate_dma.o obj-$(CONFIG_NUMA) += memory-tiers.o obj-$(CONFIG_DEVICE_MIGRATION) += migrate_device.o obj-$(CONFIG_TRANSPARENT_HUGEPAGE) += huge_memory.o khugepaged.o diff --git a/mm/migrate.c b/mm/migrate.c index fce69a494742..db826e3862a1 100644 --- a/mm/migrate.c +++ b/mm/migrate.c @@ -50,6 +50,7 @@ #include <linux/random.h> #include <linux/sched/sysctl.h> #include <linux/memory-tiers.h> +#include <linux/migrate_dma.h> #include <asm/tlbflush.h> @@ -656,6 +657,37 @@ void folio_migrate_copy(struct folio *newfolio, struct folio *folio) } EXPORT_SYMBOL(folio_migrate_copy); +DEFINE_STATIC_CALL(_folios_copy, folios_copy); +DEFINE_STATIC_CALL(_can_dma_migrate, can_dma_migrate); + +#ifdef CONFIG_DMA_MIGRATION +void srcu_mig_cb(struct rcu_head *head) +{ + static_call_query(_folios_copy); +} + +void dma_update_migrator(struct migrator *mig) +{ + int index; + + mutex_lock(&migrator_mut); + index = srcu_read_lock(&mig_srcu); + strscpy(migrator.name, mig ? mig->name : "kernel", MIGRATOR_NAME_LEN); + static_call_update(_folios_copy, mig ? mig->migrate_dma : folios_copy); + static_call_update(_can_dma_migrate, mig ? mig->can_migrate_dma : can_dma_migrate); + if (READ_ONCE(migrator.owner)) + module_put(migrator.owner); + xchg(&migrator.owner, mig ? mig->owner : NULL); + if (READ_ONCE(migrator.owner)) + try_module_get(migrator.owner); + srcu_read_unlock(&mig_srcu, index); + mutex_unlock(&migrator_mut); + call_srcu(&mig_srcu, &migrator.srcu_head, srcu_mig_cb); + srcu_barrier(&mig_srcu); +} + +#endif /* CONFIG_DMA_MIGRATION */ + /************************************************************ * Migration functions ***********************************************************/ @@ -1686,6 +1718,7 @@ static void migrate_folios_batch_move(struct list_head *src_folios, struct anon_vma *anon_vma = NULL; bool is_lru; int is_thp = 0; + bool can_migrate = true; struct migrate_folio_info *mig_info, *mig_info2; LIST_HEAD(temp_src_folios); LIST_HEAD(temp_dst_folios); @@ -1720,7 +1753,10 @@ static void migrate_folios_batch_move(struct list_head *src_folios, * This does everything except the page copy. The actual page copy * is handled later in a batch manner. */ - if (likely(is_lru)) { + can_migrate = static_call(_can_dma_migrate)(dst, folio); + if (unlikely(!can_migrate)) + rc = -EAGAIN; + else if (likely(is_lru)) { struct address_space *mapping = folio_mapping(folio); if (!mapping) @@ -1786,7 +1822,7 @@ static void migrate_folios_batch_move(struct list_head *src_folios, goto out; /* Batch copy the folios */ - folios_copy(dst_folios, src_folios); + static_call(_folios_copy)(dst_folios, src_folios); /* * Iterate the folio lists to remove migration pte and restore them diff --git a/mm/migrate_dma.c b/mm/migrate_dma.c new file mode 100644 index 000000000000..c8b078fdff17 --- /dev/null +++ b/mm/migrate_dma.c @@ -0,0 +1,51 @@ +// SPDX-License-Identifier: GPL-2.0 +#include <linux/migrate.h> +#include <linux/migrate_dma.h> +#include <linux/rculist.h> +#include <linux/static_call.h> + +atomic_t dispatch_to_dma = ATOMIC_INIT(0); +EXPORT_SYMBOL_GPL(dispatch_to_dma); + +DEFINE_MUTEX(migrator_mut); +DEFINE_SRCU(mig_srcu); + +struct migrator migrator = { + .name = "kernel", + .migrate_dma = folios_copy, + .can_migrate_dma = can_dma_migrate, + .srcu_head.func = srcu_mig_cb, + .owner = NULL, +}; + +bool can_dma_migrate(struct folio *dst, struct folio *src) +{ + return true; +} +EXPORT_SYMBOL_GPL(can_dma_migrate); + +void start_offloading(struct migrator *m) +{ + int offloading = 0; + + pr_info("starting migration offload by %s\n", m->name); + dma_update_migrator(m); + atomic_try_cmpxchg(&dispatch_to_dma, &offloading, 1); +} +EXPORT_SYMBOL_GPL(start_offloading); + +void stop_offloading(void) +{ + int offloading = 1; + + pr_info("stopping migration offload by %s\n", migrator.name); + dma_update_migrator(NULL); + atomic_try_cmpxchg(&dispatch_to_dma, &offloading, 0); +} +EXPORT_SYMBOL_GPL(stop_offloading); + +unsigned char *get_active_migrator_name(void) +{ + return migrator.name; +} +EXPORT_SYMBOL_GPL(get_active_migrator_name); -- 2.34.1