Kemi Wang <kemi.wang@xxxxxxxxx> writes: > This is the second step which introduces a tunable interface that allow VM > stats configurable for optimizing zone_statistics(), as suggested by Dave > Hansen and Ying Huang. > > ======================================= > When performance becomes a bottleneck and you can tolerate some possible > tool breakage and some decreased counter precision (e.g. numa counter), you > can do: > echo [C|c]oarse > /proc/sys/vm/vmstat_mode > In this case, numa counter update is ignored. We can see about > *4.8%*(185->176) drop of cpu cycles per single page allocation and reclaim > on Jesper's page_bench01 (single thread) and *8.1%*(343->315) drop of cpu > cycles per single page allocation and reclaim on Jesper's page_bench03 (88 > threads) running on a 2-Socket Broadwell-based server (88 threads, 126G > memory). > > Benchmark link provided by Jesper D Brouer(increase loop times to > 10000000): > https://github.com/netoptimizer/prototype-kernel/tree/master/kernel/mm/ > bench > > ======================================= > When performance is not a bottleneck and you want all tooling to work, > you When page allocation performance isn't a bottleneck ... > can do: > echo [S|s]trict > /proc/sys/vm/vmstat_mode > > ======================================= > We recommend automatic detection of virtual memory statistics by system, > this is also system default configuration, you can do: > echo [A|a]uto > /proc/sys/vm/vmstat_mode > In this case, automatic detection of VM statistics, numa counter update > is skipped unless it has been read by users at least once, e.g. cat > /proc/zoneinfo. > > Therefore, with different VM stats mode, numa counters update can operate > differently so that everybody can benefit. > > Many thanks to Michal Hocko and Dave Hansen for comments to help improve > the original patch. > > ChangeLog: > Since V1->V2: > a) Merge to one patch; > b) Use jump label to eliminate the overhead of branch selection; > c) Add a single-time log message at boot time to help tell users what > happened. > > Reported-by: Jesper Dangaard Brouer <brouer@xxxxxxxxxx> > Suggested-by: Dave Hansen <dave.hansen@xxxxxxxxx> > Suggested-by: Ying Huang <ying.huang@xxxxxxxxx> > Signed-off-by: Kemi Wang <kemi.wang@xxxxxxxxx> > --- > Documentation/sysctl/vm.txt | 26 +++++++++ > drivers/base/node.c | 2 + > include/linux/vmstat.h | 22 ++++++++ > init/main.c | 2 + > kernel/sysctl.c | 7 +++ > mm/page_alloc.c | 14 +++++ > mm/vmstat.c | 126 ++++++++++++++++++++++++++++++++++++++++++++ > 7 files changed, 199 insertions(+) > > diff --git a/Documentation/sysctl/vm.txt b/Documentation/sysctl/vm.txt > index 9baf66a..6ab2843 100644 > --- a/Documentation/sysctl/vm.txt > +++ b/Documentation/sysctl/vm.txt > @@ -61,6 +61,7 @@ Currently, these files are in /proc/sys/vm: > - swappiness > - user_reserve_kbytes > - vfs_cache_pressure > +- vmstat_mode > - watermark_scale_factor > - zone_reclaim_mode > > @@ -843,6 +844,31 @@ ten times more freeable objects than there are. > > ============================================================= > > +vmstat_mode > + > +This interface allows virtual memory statistics configurable. > + > +When performance becomes a bottleneck and you can tolerate some possible > +tool breakage and some decreased counter precision (e.g. numa counter), you > +can do: > + echo [C|c]oarse > /proc/sys/vm/vmstat_mode > +ignorable statistics list: > +- numa counters > + > +When performance is not a bottleneck and you want all tooling to work, you > +can do: > + echo [S|s]trict > /proc/sys/vm/vmstat_mode > + > +We recommend automatic detection of virtual memory statistics by system, > +this is also system default configuration, you can do: > + echo [A|a]uto > /proc/sys/vm/vmstat_mode > + > +E.g. numa statistics does not affect system's decision and it is very > +rarely consumed. If set vmstat_mode = auto, numa counters update is skipped > +unless the counter is *read* by users at least once. > + > +============================================================== > + > watermark_scale_factor: > > This factor controls the aggressiveness of kswapd. It defines the > diff --git a/drivers/base/node.c b/drivers/base/node.c > index 3855902..033c0c3 100644 > --- a/drivers/base/node.c > +++ b/drivers/base/node.c > @@ -153,6 +153,7 @@ static DEVICE_ATTR(meminfo, S_IRUGO, node_read_meminfo, NULL); > static ssize_t node_read_numastat(struct device *dev, > struct device_attribute *attr, char *buf) > { > + disable_zone_statistics = false; > return sprintf(buf, > "numa_hit %lu\n" > "numa_miss %lu\n" > @@ -194,6 +195,7 @@ static ssize_t node_read_vmstat(struct device *dev, > NR_VM_NUMA_STAT_ITEMS], > node_page_state(pgdat, i)); > > + disable_zone_statistics = false; > return n; > } > static DEVICE_ATTR(vmstat, S_IRUGO, node_read_vmstat, NULL); > diff --git a/include/linux/vmstat.h b/include/linux/vmstat.h > index ade7cb5..22670cf 100644 > --- a/include/linux/vmstat.h > +++ b/include/linux/vmstat.h > @@ -6,9 +6,27 @@ > #include <linux/mmzone.h> > #include <linux/vm_event_item.h> > #include <linux/atomic.h> > +#include <linux/static_key.h> > > extern int sysctl_stat_interval; > > +DECLARE_STATIC_KEY_FALSE(vmstat_mode_key); > +extern bool disable_zone_statistics; > +/* > + * vmstat_mode: > + * 0 = auto mode of vmstat, automatic detection of VM statistics. > + * 1 = strict mode of vmstat, keep all VM statistics. > + * 2 = coarse mode of vmstat, ignore unimportant VM statistics. > + */ > +#define VMSTAT_AUTO_MODE 0 > +#define VMSTAT_STRICT_MODE 1 > +#define VMSTAT_COARSE_MODE 2 > +#define VMSTAT_MODE_LEN 16 > +extern int vmstat_mode; > +extern char sysctl_vmstat_mode[]; > +extern int sysctl_vmstat_mode_handler(struct ctl_table *table, int write, > + void __user *buffer, size_t *length, loff_t *ppos); > + > #ifdef CONFIG_VM_EVENT_COUNTERS > /* > * Light weight per cpu counter implementation. > @@ -229,6 +247,10 @@ extern unsigned long sum_zone_node_page_state(int node, > extern unsigned long sum_zone_numa_state(int node, enum numa_stat_item item); > extern unsigned long node_page_state(struct pglist_data *pgdat, > enum node_stat_item item); > +extern void zero_zone_numa_counters(struct zone *zone); > +extern void zero_zones_numa_counters(void); > +extern void zero_global_numa_counters(void); > +extern void invalid_numa_statistics(void); > #else > #define sum_zone_node_page_state(node, item) global_zone_page_state(item) > #define node_page_state(node, item) global_node_page_state(item) > diff --git a/init/main.c b/init/main.c > index 0ee9c686..940b71c 100644 > --- a/init/main.c > +++ b/init/main.c > @@ -567,6 +567,8 @@ asmlinkage __visible void __init start_kernel(void) > sort_main_extable(); > trap_init(); > mm_init(); > + pr_info("vmstat: System detection of virtual memory statistics, NUMA\n" > +"counters update is skipped unless they are read by users at least once\n"); > > ftrace_init(); > > diff --git a/kernel/sysctl.c b/kernel/sysctl.c > index 6648fbb..f5b813b 100644 > --- a/kernel/sysctl.c > +++ b/kernel/sysctl.c > @@ -1234,6 +1234,13 @@ static struct ctl_table kern_table[] = { > > static struct ctl_table vm_table[] = { > { > + .procname = "vmstat_mode", > + .data = &sysctl_vmstat_mode, > + .maxlen = VMSTAT_MODE_LEN, > + .mode = 0644, > + .proc_handler = sysctl_vmstat_mode_handler, > + }, > + { > .procname = "overcommit_memory", > .data = &sysctl_overcommit_memory, > .maxlen = sizeof(sysctl_overcommit_memory), > diff --git a/mm/page_alloc.c b/mm/page_alloc.c > index c841af8..46afc8a 100644 > --- a/mm/page_alloc.c > +++ b/mm/page_alloc.c > @@ -83,6 +83,9 @@ DEFINE_PER_CPU(int, numa_node); > EXPORT_PER_CPU_SYMBOL(numa_node); > #endif > > +DEFINE_STATIC_KEY_FALSE(vmstat_mode_key); > +bool disable_zone_statistics = true; > + > #ifdef CONFIG_HAVE_MEMORYLESS_NODES > /* > * N.B., Do NOT reference the '_numa_mem_' per cpu variable directly. > @@ -2743,6 +2746,17 @@ static inline void zone_statistics(struct zone *preferred_zone, struct zone *z) > #ifdef CONFIG_NUMA > enum numa_stat_item local_stat = NUMA_LOCAL; > > + /* > + * skip zone_statistics() if vmstat mode is coarse or zone statistics > + * is inactive in auto mode > + */ > + > + if (static_branch_unlikely(&vmstat_mode_key)) { > + if (vmstat_mode == VMSTAT_COARSE_MODE) > + return; > + } else if (disable_zone_statistics) > + return; I suspect this will not help performance. I suggest to make the most common case (auto mode + disable_zone_statistics (true)?) unconditionally with jump label, and other combination conditionally. Best Regards, Huang, Ying > + > if (z->node != numa_node_id()) > local_stat = NUMA_OTHER; > > diff --git a/mm/vmstat.c b/mm/vmstat.c > index 4bb13e7..d4ab53e 100644 > --- a/mm/vmstat.c > +++ b/mm/vmstat.c > @@ -32,6 +32,94 @@ > > #define NUMA_STATS_THRESHOLD (U16_MAX - 2) > > +int vmstat_mode = VMSTAT_AUTO_MODE; > +char sysctl_vmstat_mode[VMSTAT_MODE_LEN] = "auto"; > +static const char *vmstat_mode_name[3] = {"auto", "strict", "coarse"}; > +static DEFINE_MUTEX(vmstat_mode_lock); > + > + > +static int __parse_vmstat_mode(char *s) > +{ > + const char *str = s; > + > + if (strcmp(str, "auto") == 0 || strcmp(str, "Auto") == 0) { > + vmstat_mode = VMSTAT_AUTO_MODE; > + static_branch_disable(&vmstat_mode_key); > + } else if (strcmp(str, "strict") == 0 || strcmp(str, "Strict") == 0) { > + vmstat_mode = VMSTAT_STRICT_MODE; > + static_branch_enable(&vmstat_mode_key); > + } else if (strcmp(str, "coarse") == 0 || strcmp(str, "Coarse") == 0) { > + vmstat_mode = VMSTAT_COARSE_MODE; > + static_branch_enable(&vmstat_mode_key); > + } else { > + pr_warn("Ignoring invalid vmstat_mode value: %s\n", s); > + return -EINVAL; > + } > + return 0; > +} > + > +int sysctl_vmstat_mode_handler(struct ctl_table *table, int write, > + void __user *buffer, size_t *length, loff_t *ppos) > +{ > + char old_string[VMSTAT_MODE_LEN]; > + int ret, oldval; > + > + mutex_lock(&vmstat_mode_lock); > + if (write) > + strncpy(old_string, (char *)table->data, VMSTAT_MODE_LEN); > + ret = proc_dostring(table, write, buffer, length, ppos); > + if (ret || !write) { > + mutex_unlock(&vmstat_mode_lock); > + return ret; > + } > + > + oldval = vmstat_mode; > + if (__parse_vmstat_mode((char *)table->data)) { > + /* > + * invalid sysctl_vmstat_mode value, restore saved string > + */ > + strncpy((char *)table->data, old_string, VMSTAT_MODE_LEN); > + vmstat_mode = oldval; > + } else { > + /* > + * check whether vmstat mode changes or not > + */ > + if (vmstat_mode == oldval) { > + /* no change */ > + mutex_unlock(&vmstat_mode_lock); > + return 0; > + } else if (vmstat_mode == VMSTAT_AUTO_MODE) { > + pr_info("vmstat mode changes from %s to auto mode\n", > + vmstat_mode_name[oldval]); > + /* > + * Set default numa stats action when vmstat mode changes > + * from coarse to auto > + */ > + if (oldval == VMSTAT_COARSE_MODE) > + disable_zone_statistics = true; > + } else if (vmstat_mode == VMSTAT_STRICT_MODE) > + pr_info("vmstat mode changes from %s to strict mode\n", > + vmstat_mode_name[oldval]); > + else if (vmstat_mode == VMSTAT_COARSE_MODE) { > + pr_info("vmstat mode changes from %s to coarse mode\n", > + vmstat_mode_name[oldval]); > +#ifdef CONFIG_NUMA > + /* > + * Invalidate numa counters when vmstat mode is set to coarse > + * mode, because users can't tell the difference between the > + * dead state and when allocator activity is quiet once > + * zone_statistics() is turned off. > + */ > + invalid_numa_statistics(); > +#endif > + } else > + pr_warn("invalid vmstat_mode:%d\n", vmstat_mode); > + } > + > + mutex_unlock(&vmstat_mode_lock); > + return 0; > +} > + > #ifdef CONFIG_VM_EVENT_COUNTERS > DEFINE_PER_CPU(struct vm_event_state, vm_event_states) = {{0}}; > EXPORT_PER_CPU_SYMBOL(vm_event_states); > @@ -914,6 +1002,42 @@ unsigned long sum_zone_numa_state(int node, > return count; > } > > +/* zero numa counters within a zone */ > +void zero_zone_numa_counters(struct zone *zone) > +{ > + int item, cpu; > + > + for (item = 0; item < NR_VM_NUMA_STAT_ITEMS; item++) { > + atomic_long_set(&zone->vm_numa_stat[item], 0); > + for_each_online_cpu(cpu) > + per_cpu_ptr(zone->pageset, cpu)->vm_numa_stat_diff[item] = 0; > + } > +} > + > +/* zero numa counters of all the populated zones */ > +void zero_zones_numa_counters(void) > +{ > + struct zone *zone; > + > + for_each_populated_zone(zone) > + zero_zone_numa_counters(zone); > +} > + > +/* zero global numa counters */ > +void zero_global_numa_counters(void) > +{ > + int item; > + > + for (item = 0; item < NR_VM_NUMA_STAT_ITEMS; item++) > + atomic_long_set(&vm_numa_stat[item], 0); > +} > + > +void invalid_numa_statistics(void) > +{ > + zero_zones_numa_counters(); > + zero_global_numa_counters(); > +} > + > /* > * Determine the per node value of a stat item. > */ > @@ -1582,6 +1706,7 @@ static int zoneinfo_show(struct seq_file *m, void *arg) > { > pg_data_t *pgdat = (pg_data_t *)arg; > walk_zones_in_node(m, pgdat, false, false, zoneinfo_show_print); > + disable_zone_statistics = false; > return 0; > } > > @@ -1678,6 +1803,7 @@ static int vmstat_show(struct seq_file *m, void *arg) > > static void vmstat_stop(struct seq_file *m, void *arg) > { > + disable_zone_statistics = false; > kfree(m->private); > m->private = NULL; > }
