1. enable multi-threaded copy
2. specify how many CPU threads to use
3. push from local CPUs or pull from remote CPUs
4. change NR_MAX_BATCHED_MIGRATION to HPAGE_PUD_NR to allow batching THP
copies.
These are for testing purpose only.
Signed-off-by: Zi Yan <ziy@xxxxxxxxxx>
---
include/linux/mm.h | 4 ++++
include/linux/sysctl.h | 1 +
kernel/sysctl.c | 29 ++++++++++++++++++++++++++++-
mm/copy_pages.c | 10 +++++++---
mm/migrate.c | 6 ++++--
5 files changed, 44 insertions(+), 6 deletions(-)
diff --git a/include/linux/mm.h b/include/linux/mm.h
index 1a11f9df5c2d..277b12b9ef0d 100644
--- a/include/linux/mm.h
+++ b/include/linux/mm.h
@@ -42,6 +42,10 @@ struct pt_regs;
struct folio_batch;
extern int sysctl_page_lock_unfairness;
+extern int sysctl_use_mt_copy;
+extern unsigned int sysctl_limit_mt_num;
+extern unsigned int sysctl_push_0_pull_1;
+
void mm_core_init(void);
void init_mm_internals(void);
diff --git a/include/linux/sysctl.h b/include/linux/sysctl.h
index 40a6ac6c9713..f33dafea2533 100644
--- a/include/linux/sysctl.h
+++ b/include/linux/sysctl.h
@@ -52,6 +52,7 @@ struct ctl_dir;
/* this is needed for the proc_dointvec_minmax for [fs_]overflow UID and GID */
#define SYSCTL_MAXOLDUID ((void *)&sysctl_vals[10])
#define SYSCTL_NEG_ONE ((void *)&sysctl_vals[11])
+#define SYSCTL_32 ((void *)&sysctl_vals[12])
extern const int sysctl_vals[];
diff --git a/kernel/sysctl.c b/kernel/sysctl.c
index 5c9202cb8f59..f9ba48cd6e09 100644
--- a/kernel/sysctl.c
+++ b/kernel/sysctl.c
@@ -82,7 +82,7 @@
#endif
/* shared constants to be used in various sysctls */
-const int sysctl_vals[] = { 0, 1, 2, 3, 4, 100, 200, 1000, 3000, INT_MAX, 65535, -1 };
+const int sysctl_vals[] = { 0, 1, 2, 3, 4, 100, 200, 1000, 3000, INT_MAX, 65535, -1, 32 };
EXPORT_SYMBOL(sysctl_vals);
const unsigned long sysctl_long_vals[] = { 0, 1, LONG_MAX };
@@ -2091,6 +2091,33 @@ static struct ctl_table vm_table[] = {
.extra2 = SYSCTL_ONE,
},
#endif
+ {
+ .procname = "use_mt_copy",
+ .data = &use_mt_copy,
+ .maxlen = sizeof(use_mt_copy),
+ .mode = 0644,
+ .proc_handler = proc_dointvec_minmax,
+ .extra1 = SYSCTL_ZERO,
+ .extra2 = SYSCTL_ONE,
+ },
+ {
+ .procname = "limit_mt_num",
+ .data = &limit_mt_num,
+ .maxlen = sizeof(limit_mt_num),
+ .mode = 0644,
+ .proc_handler = proc_dointvec_minmax,
+ .extra1 = SYSCTL_ONE,
+ .extra2 = SYSCTL_32,
+ },
+ {
+ .procname = "push_0_pull_1",
+ .data = &push_0_pull_1,
+ .maxlen = sizeof(push_0_pull_1),
+ .mode = 0644,
+ .proc_handler = proc_dointvec_minmax,
+ .extra1 = SYSCTL_ZERO,
+ .extra2 = SYSCTL_ONE,
+ },
{
.procname = "drop_caches",
.data = &sysctl_drop_caches,
diff --git a/mm/copy_pages.c b/mm/copy_pages.c
index 0e2231199f66..257034550c86 100644
--- a/mm/copy_pages.c
+++ b/mm/copy_pages.c
@@ -10,7 +10,9 @@
#include <linux/migrate.h>
-unsigned int limit_mt_num = 4;
+unsigned int sysctl_limit_mt_num = 4;
+/* push by default */
+unsigned int sysctl_push_0_pull_1;
struct copy_item {
char *to;
@@ -45,11 +47,13 @@ int copy_page_lists_mt(struct list_head *dst_folios,
struct list_head *src_folios, int nr_items)
{
int err = 0;
- unsigned int total_mt_num = limit_mt_num;
+ unsigned int total_mt_num = sysctl_limit_mt_num;
int to_node = folio_nid(list_first_entry(dst_folios, struct folio, lru));
+ int from_node = folio_nid(list_first_entry(src_folios, struct folio, lru));
int i;
struct copy_page_info *work_items[32] = {0};
- const struct cpumask *per_node_cpumask = cpumask_of_node(to_node);
+ const struct cpumask *per_node_cpumask =
+ cpumask_of_node(sysctl_push_0_pull_1 ? to_node : from_node);
int cpu_id_list[32] = {0};
int cpu;
int max_items_per_thread;
diff --git a/mm/migrate.c b/mm/migrate.c
index 18440180d747..0f7a4b09acda 100644
--- a/mm/migrate.c
+++ b/mm/migrate.c
@@ -51,6 +51,7 @@
#include "internal.h"
+int sysctl_use_mt_copy;
bool isolate_movable_page(struct page *page, isolate_mode_t mode)
{
@@ -1621,7 +1622,7 @@ static inline int try_split_folio(struct folio *folio, struct list_head *split_f
}
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
-#define NR_MAX_BATCHED_MIGRATION HPAGE_PMD_NR
+#define NR_MAX_BATCHED_MIGRATION HPAGE_PUD_NR
#else
#define NR_MAX_BATCHED_MIGRATION 512
#endif
@@ -1868,7 +1869,8 @@ static void migrate_folios_batch_move(struct list_head *src_folios,
goto out;
/* Batch copy the folios */
- if (total_nr_pages > 32) {
+ /* if (total_nr_pages > 32) { */
+ if (sysctl_use_mt_copy) {
copy_page_lists_mt(dst_folios, src_folios, total_nr_folios);
} else {
dst = list_first_entry(dst_folios, struct folio, lru);
--
2.45.2
