The patch titled
Subject: mm: add per-order mTHP anon_alloc and anon_alloc_fallback counters
has been added to the -mm mm-unstable branch. Its filename is
mm-add-per-order-mthp-anon_alloc-and-anon_alloc_fallback-counters.patch
This patch will shortly appear at
https://git.kernel.org/pub/scm/linux/kernel/git/akpm/25-new.git/tree/patches/mm-add-per-order-mthp-anon_alloc-and-anon_alloc_fallback-counters.patch
This patch will later appear in the mm-unstable branch at
git://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm
Before you just go and hit "reply", please:
a) Consider who else should be cc'ed
b) Prefer to cc a suitable mailing list as well
c) Ideally: find the original patch on the mailing list and do a
reply-to-all to that, adding suitable additional cc's
*** Remember to use Documentation/process/submit-checklist.rst when testing your code ***
The -mm tree is included into linux-next via the mm-everything
branch at git://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm
and is updated there every 2-3 working days
------------------------------------------------------
From: Barry Song <v-songbaohua@xxxxxxxx>
Subject: mm: add per-order mTHP anon_alloc and anon_alloc_fallback counters
Date: Fri, 5 Apr 2024 23:27:03 +1300
Patch series "mm: add per-order mTHP alloc and swpout counters", v4.
The patchset introduces a framework to facilitate mTHP counters, starting
with the allocation and swap-out counters. Currently, only four new nodes
are appended to the stats directory for each mTHP size.
/sys/kernel/mm/transparent_hugepage/hugepages-<size>/stats
anon_alloc
anon_alloc_fallback
anon_swpout
anon_swpout_fallback
These four nodes are crucial for us to monitor the fragmentation levels of
both the buddy system and the swapfile. In the future, we may consider
adding additional nodes for further insights.
This patch (of 2):
Profiling a system blindly with mTHP has become challenging due to the
lack of visibility into its operations. Presenting the success rate of
mTHP allocations appears to be pressing need.
Recently, I've been experiencing significant difficulty debugging
performance improvements and regressions without these figures. It's
crucial for us to understand the true effectiveness of mTHP in real-world
scenarios, especially in systems with fragmented memory.
This patch sets up the framework for per-order mTHP counters, starting
with the introduction of anon_alloc and anon_alloc_fallback counters.
Incorporating additional counters should now be straightforward as well.
Link: https://lkml.kernel.org/r/20240405102704.77559-1-21cnbao@xxxxxxxxx
Link: https://lkml.kernel.org/r/20240405102704.77559-2-21cnbao@xxxxxxxxx
Signed-off-by: Barry Song <v-songbaohua@xxxxxxxx>
Cc: Chris Li <chrisl@xxxxxxxxxx>
Cc: David Hildenbrand <david@xxxxxxxxxx>
Cc: Domenico Cerasuolo <cerasuolodomenico@xxxxxxxxx>
Cc: Kairui Song <kasong@xxxxxxxxxxx>
Cc: Matthew Wilcox (Oracle) <willy@xxxxxxxxxxxxx>
Cc: Peter Xu <peterx@xxxxxxxxxx>
Cc: Ryan Roberts <ryan.roberts@xxxxxxx>
Cc: Suren Baghdasaryan <surenb@xxxxxxxxxx>
Cc: Yosry Ahmed <yosryahmed@xxxxxxxxxx>
Cc: Yu Zhao <yuzhao@xxxxxxxxxx>
Signed-off-by: Andrew Morton <akpm@xxxxxxxxxxxxxxxxxxxx>
---
include/linux/huge_mm.h | 19 +++++++++++++++
mm/huge_memory.c | 48 ++++++++++++++++++++++++++++++++++++++
mm/memory.c | 2 +
3 files changed, 69 insertions(+)
--- a/include/linux/huge_mm.h~mm-add-per-order-mthp-anon_alloc-and-anon_alloc_fallback-counters
+++ a/include/linux/huge_mm.h
@@ -264,6 +264,25 @@ unsigned long thp_vma_allowable_orders(s
enforce_sysfs, orders);
}
+enum mthp_stat_item {
+ MTHP_STAT_ANON_ALLOC,
+ MTHP_STAT_ANON_ALLOC_FALLBACK,
+ __MTHP_STAT_COUNT
+};
+
+struct mthp_stat {
+ unsigned long stats[PMD_ORDER + 1][__MTHP_STAT_COUNT];
+};
+
+DECLARE_PER_CPU(struct mthp_stat, mthp_stats);
+
+static inline void count_mthp_stat(int order, enum mthp_stat_item item)
+{
+ if (unlikely(order > PMD_ORDER))
+ return;
+ this_cpu_inc(mthp_stats.stats[order][item]);
+}
+
#define transparent_hugepage_use_zero_page() \
(transparent_hugepage_flags & \
(1<<TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG))
--- a/mm/huge_memory.c~mm-add-per-order-mthp-anon_alloc-and-anon_alloc_fallback-counters
+++ a/mm/huge_memory.c
@@ -526,6 +526,46 @@ static const struct kobj_type thpsize_kt
.sysfs_ops = &kobj_sysfs_ops,
};
+DEFINE_PER_CPU(struct mthp_stat, mthp_stats) = {{{0}}};
+
+static unsigned long sum_mthp_stat(int order, enum mthp_stat_item item)
+{
+ unsigned long sum = 0;
+ int cpu;
+
+ for_each_online_cpu(cpu) {
+ struct mthp_stat *this = &per_cpu(mthp_stats, cpu);
+
+ sum += this->stats[order][item];
+ }
+
+ return sum;
+}
+
+#define DEFINE_MTHP_STAT_ATTR(_name, _index) \
+static ssize_t _name##_show(struct kobject *kobj, \
+ struct kobj_attribute *attr, char *buf) \
+{ \
+ int order = to_thpsize(kobj)->order; \
+ \
+ return sysfs_emit(buf, "%lu\n", sum_mthp_stat(order, _index)); \
+} \
+static struct kobj_attribute _name##_attr = __ATTR_RO(_name)
+
+DEFINE_MTHP_STAT_ATTR(anon_alloc, MTHP_STAT_ANON_ALLOC);
+DEFINE_MTHP_STAT_ATTR(anon_alloc_fallback, MTHP_STAT_ANON_ALLOC_FALLBACK);
+
+static struct attribute *stats_attrs[] = {
+ &anon_alloc_attr.attr,
+ &anon_alloc_fallback_attr.attr,
+ NULL,
+};
+
+static struct attribute_group stats_attr_group = {
+ .name = "stats",
+ .attrs = stats_attrs,
+};
+
static struct thpsize *thpsize_create(int order, struct kobject *parent)
{
unsigned long size = (PAGE_SIZE << order) / SZ_1K;
@@ -549,6 +589,12 @@ static struct thpsize *thpsize_create(in
return ERR_PTR(ret);
}
+ ret = sysfs_create_group(&thpsize->kobj, &stats_attr_group);
+ if (ret) {
+ kobject_put(&thpsize->kobj);
+ return ERR_PTR(ret);
+ }
+
thpsize->order = order;
return thpsize;
}
@@ -1050,8 +1096,10 @@ vm_fault_t do_huge_pmd_anonymous_page(st
folio = vma_alloc_folio(gfp, HPAGE_PMD_ORDER, vma, haddr, true);
if (unlikely(!folio)) {
count_vm_event(THP_FAULT_FALLBACK);
+ count_mthp_stat(HPAGE_PMD_ORDER, MTHP_STAT_ANON_ALLOC_FALLBACK);
return VM_FAULT_FALLBACK;
}
+ count_mthp_stat(HPAGE_PMD_ORDER, MTHP_STAT_ANON_ALLOC);
return __do_huge_pmd_anonymous_page(vmf, &folio->page, gfp);
}
--- a/mm/memory.c~mm-add-per-order-mthp-anon_alloc-and-anon_alloc_fallback-counters
+++ a/mm/memory.c
@@ -4374,8 +4374,10 @@ static struct folio *alloc_anon_folio(st
}
folio_throttle_swaprate(folio, gfp);
clear_huge_page(&folio->page, vmf->address, 1 << order);
+ count_mthp_stat(order, MTHP_STAT_ANON_ALLOC);
return folio;
}
+ count_mthp_stat(order, MTHP_STAT_ANON_ALLOC_FALLBACK);
next:
order = next_order(&orders, order);
}
_
Patches currently in -mm which might be from v-songbaohua@xxxxxxxx are
arm64-mm-swap-support-thp_swap-on-hardware-with-mte.patch
mm-hold-ptl-from-the-first-pte-while-reclaiming-a-large-folio.patch
mm-alloc_anon_folio-avoid-doing-vma_thp_gfp_mask-in-fallback-cases.patch
mm-add-per-order-mthp-anon_alloc-and-anon_alloc_fallback-counters.patch
mm-add-per-order-mthp-anon_swpout-and-anon_swpout_fallback-counters.patch
