From: Huang Ying <ying.huang@xxxxxxxxx>
It isn't easy for the administrator to determine the hot threshold.
So in this patch, a method to adjust the hot threshold automatically
is implemented. The basic idea is to control the number of the
candidate promotion pages to match the promotion rate limit. If the
hint page fault latency of a page is less than the hot threshold, we
will try to promote the page, that is, the page is the candidate
promotion page.
If the number of the candidate promotion pages in the statistics
interval is much higher than the promotion rate limit, the hot
threshold will be lowered to reduce the number of the candidate
promotion pages. Otherwise, the hot threshold will be raised to
increase the number of the candidate promotion pages.
To make the above method works, in each statistics interval, the total
number of the pages to check (on which the hint page faults occur) and
the hot/cold distribution need to be stable. Because the page tables
are scanned linearly in autonuma, but the hot/cold distribution isn't
uniform along the address. The statistics interval should be larger
than the autonuma scan period. So in the patch, the max scan period
is used as statistics interval and it works well in our tests.
The sysctl knob kernel.numa_balancing_hot_threshold_ms becomes the
initial value and max value of the hot threshold.
The patch improves the score of pmbench memory accessing benchmark
with 80:20 read/write ratio and normal access address distribution by
5.5% with 24.6% fewer NUMA page migrations on a 2 socket Intel server
with Optance DC Persistent Memory. Because it improves the accuracy
of the hot page selection.
Signed-off-by: "Huang, Ying" <ying.huang@xxxxxxxxx>
Cc: Andrew Morton <akpm@xxxxxxxxxxxxxxxxxxxx>
Cc: Michal Hocko <mhocko@xxxxxxxx>
Cc: Rik van Riel <riel@xxxxxxxxxx>
Cc: Mel Gorman <mgorman@xxxxxxx>
Cc: Peter Zijlstra <peterz@xxxxxxxxxxxxx>
Cc: Ingo Molnar <mingo@xxxxxxxxxx>
Cc: Dave Hansen <dave.hansen@xxxxxxxxxxxxxxx>
Cc: Dan Williams <dan.j.williams@xxxxxxxxx>
Cc: linux-kernel@xxxxxxxxxxxxxxx
Cc: linux-mm@xxxxxxxxx
---
include/linux/mmzone.h | 3 +++
kernel/sched/fair.c | 40 ++++++++++++++++++++++++++++++++++++----
2 files changed, 39 insertions(+), 4 deletions(-)
diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h
index 6e7a28becdc2..4ed82eb5c8b5 100644
--- a/include/linux/mmzone.h
+++ b/include/linux/mmzone.h
@@ -792,6 +792,9 @@ typedef struct pglist_data {
#ifdef CONFIG_NUMA_BALANCING
unsigned long numa_ts;
unsigned long numa_try;
+ unsigned long numa_threshold_ts;
+ unsigned long numa_threshold_try;
+ unsigned long numa_threshold;
#endif
/* Fields commonly accessed by the page reclaim scanner */
diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index e5f7f4139c82..90098c35d336 100644
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -1487,6 +1487,35 @@ static bool numa_migration_check_rate_limit(struct pglist_data *pgdat,
return true;
}
+#define NUMA_MIGRATION_ADJUST_STEPS 16
+
+static void numa_migration_adjust_threshold(struct pglist_data *pgdat,
+ unsigned long rate_limit,
+ unsigned long ref_th)
+{
+ unsigned long now = jiffies, last_th_ts, th_period;
+ unsigned long unit_th, th;
+ unsigned long try, ref_try, tdiff;
+
+ th_period = msecs_to_jiffies(sysctl_numa_balancing_scan_period_max);
+ last_th_ts = pgdat->numa_threshold_ts;
+ if (now > last_th_ts + th_period &&
+ cmpxchg(&pgdat->numa_threshold_ts, last_th_ts, now) == last_th_ts) {
+ ref_try = rate_limit *
+ sysctl_numa_balancing_scan_period_max / 1000;
+ try = node_page_state(pgdat, NUMA_TRY_MIGRATE);
+ tdiff = try - pgdat->numa_threshold_try;
+ unit_th = ref_th / NUMA_MIGRATION_ADJUST_STEPS;
+ th = pgdat->numa_threshold ? : ref_th;
+ if (tdiff > ref_try * 11 / 10)
+ th = max(th - unit_th, unit_th);
+ else if (tdiff < ref_try * 9 / 10)
+ th = min(th + unit_th, ref_th);
+ pgdat->numa_threshold_try = try;
+ pgdat->numa_threshold = th;
+ }
+}
+
bool should_numa_migrate_memory(struct task_struct *p, struct page * page,
int src_nid, int dst_cpu, unsigned long addr,
int flags)
@@ -1502,7 +1531,7 @@ bool should_numa_migrate_memory(struct task_struct *p, struct page * page,
if (sysctl_numa_balancing_mode & NUMA_BALANCING_MEMORY_TIERING &&
next_promotion_node(src_nid) != -1) {
struct pglist_data *pgdat;
- unsigned long rate_limit, latency, th;
+ unsigned long rate_limit, latency, th, def_th;
pgdat = NODE_DATA(dst_nid);
if (pgdat_free_space_enough(pgdat))
@@ -1512,15 +1541,18 @@ bool should_numa_migrate_memory(struct task_struct *p, struct page * page,
if (!(flags & TNF_YOUNG))
return false;
- th = msecs_to_jiffies(sysctl_numa_balancing_hot_threshold);
+ def_th = msecs_to_jiffies(sysctl_numa_balancing_hot_threshold);
+ rate_limit =
+ sysctl_numa_balancing_rate_limit << (20 - PAGE_SHIFT);
+ numa_migration_adjust_threshold(pgdat, rate_limit, def_th);
+
+ th = pgdat->numa_threshold ? : def_th;
if (flags & TNF_WRITE)
th *= 2;
latency = numa_hint_fault_latency(p, addr);
if (latency > th)
return false;
- rate_limit =
- sysctl_numa_balancing_rate_limit << (20 - PAGE_SHIFT);
return numa_migration_check_rate_limit(pgdat, rate_limit,
hpage_nr_pages(page));
}
--
2.24.1
