From: Barry Song <v-songbaohua@xxxxxxxx>
Profiling a system blindly with mTHP has become challenging due
to the lack of visibility into its operations. Presenting the
success rate of mTHP allocations appears to be pressing need.
Recently, I've been experiencing significant difficulty debugging
performance improvements and regressions without these figures.
It's crucial for us to understand the true effectiveness of
mTHP in real-world scenarios, especially in systems with
fragmented memory.
This patch sets up the framework for per-order mTHP counters,
starting with the introduction of anon_alloc_success and
anon_alloc_fail counters. Incorporating additional counters
should now be straightforward as well.
Signed-off-by: Barry Song <v-songbaohua@xxxxxxxx>
---
-v3:
* save some memory as order-0 and order-1 can't be THP, Ryan;
* rename to anon_alloc as right now we only support anon to address
David's comment;
* drop a redundant "else", Ryan
include/linux/huge_mm.h | 18 ++++++++++++++
mm/huge_memory.c | 54 +++++++++++++++++++++++++++++++++++++++++
mm/memory.c | 2 ++
3 files changed, 74 insertions(+)
diff --git a/include/linux/huge_mm.h b/include/linux/huge_mm.h
index e896ca4760f6..5e9af6be9537 100644
--- a/include/linux/huge_mm.h
+++ b/include/linux/huge_mm.h
@@ -70,6 +70,7 @@ extern struct kobj_attribute shmem_enabled_attr;
* (which is a limitation of the THP implementation).
*/
#define THP_ORDERS_ALL_ANON ((BIT(PMD_ORDER + 1) - 1) & ~(BIT(0) | BIT(1)))
+#define THP_MIN_ORDER 2
/*
* Mask of all large folio orders supported for file THP.
@@ -264,6 +265,23 @@ unsigned long thp_vma_allowable_orders(struct vm_area_struct *vma,
enforce_sysfs, orders);
}
+enum thp_event_item {
+ THP_ANON_ALLOC_SUCCESS,
+ THP_ANON_ALLOC_FAIL,
+ NR_THP_EVENT_ITEMS
+};
+
+struct thp_event_state {
+ unsigned long event[PMD_ORDER + 1 - THP_MIN_ORDER][NR_THP_EVENT_ITEMS];
+};
+
+DECLARE_PER_CPU(struct thp_event_state, thp_event_states);
+
+static inline void count_thp_event(int order, enum thp_event_item item)
+{
+ this_cpu_inc(thp_event_states.event[order - THP_MIN_ORDER][item]);
+}
+
#define transparent_hugepage_use_zero_page() \
(transparent_hugepage_flags & \
(1<<TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG))
diff --git a/mm/huge_memory.c b/mm/huge_memory.c
index 3ca9282a0dc9..e283039719f2 100644
--- a/mm/huge_memory.c
+++ b/mm/huge_memory.c
@@ -526,6 +526,52 @@ static const struct kobj_type thpsize_ktype = {
.sysfs_ops = &kobj_sysfs_ops,
};
+DEFINE_PER_CPU(struct thp_event_state, thp_event_states) = {{{0}}};
+
+static unsigned long sum_thp_events(int order, enum thp_event_item item)
+{
+ unsigned long sum = 0;
+ int cpu;
+
+ for_each_online_cpu(cpu) {
+ struct thp_event_state *this = &per_cpu(thp_event_states, cpu);
+
+ sum += this->event[order - THP_MIN_ORDER][item];
+ }
+
+ return sum;
+}
+
+static ssize_t anon_alloc_success_show(struct kobject *kobj,
+ struct kobj_attribute *attr, char *buf)
+{
+ int order = to_thpsize(kobj)->order;
+
+ return sysfs_emit(buf, "%lu\n", sum_thp_events(order, THP_ANON_ALLOC_SUCCESS));
+}
+
+static ssize_t anon_alloc_fail_show(struct kobject *kobj,
+ struct kobj_attribute *attr, char *buf)
+{
+ int order = to_thpsize(kobj)->order;
+
+ return sysfs_emit(buf, "%lu\n", sum_thp_events(order, THP_ANON_ALLOC_FAIL));
+}
+
+static struct kobj_attribute anon_alloc_success_attr = __ATTR_RO(anon_alloc_success);
+static struct kobj_attribute anon_alloc_fail_attr = __ATTR_RO(anon_alloc_fail);
+
+static struct attribute *stats_attrs[] = {
+ &anon_alloc_success_attr.attr,
+ &anon_alloc_fail_attr.attr,
+ NULL,
+};
+
+static struct attribute_group stats_attr_group = {
+ .name = "stats",
+ .attrs = stats_attrs,
+};
+
static struct thpsize *thpsize_create(int order, struct kobject *parent)
{
unsigned long size = (PAGE_SIZE << order) / SZ_1K;
@@ -549,6 +595,12 @@ static struct thpsize *thpsize_create(int order, struct kobject *parent)
return ERR_PTR(ret);
}
+ ret = sysfs_create_group(&thpsize->kobj, &stats_attr_group);
+ if (ret) {
+ kobject_put(&thpsize->kobj);
+ return ERR_PTR(ret);
+ }
+
thpsize->order = order;
return thpsize;
}
@@ -1050,8 +1102,10 @@ vm_fault_t do_huge_pmd_anonymous_page(struct vm_fault *vmf)
folio = vma_alloc_folio(gfp, HPAGE_PMD_ORDER, vma, haddr, true);
if (unlikely(!folio)) {
count_vm_event(THP_FAULT_FALLBACK);
+ count_thp_event(HPAGE_PMD_ORDER, THP_ANON_ALLOC_FAIL);
return VM_FAULT_FALLBACK;
}
+ count_thp_event(HPAGE_PMD_ORDER, THP_ANON_ALLOC_SUCCESS);
return __do_huge_pmd_anonymous_page(vmf, &folio->page, gfp);
}
diff --git a/mm/memory.c b/mm/memory.c
index 912cd738ec03..40de2b7a73ab 100644
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -4374,8 +4374,10 @@ static struct folio *alloc_anon_folio(struct vm_fault *vmf)
}
folio_throttle_swaprate(folio, gfp);
clear_huge_page(&folio->page, vmf->address, 1 << order);
+ count_thp_event(order, THP_ANON_ALLOC_SUCCESS);
return folio;
}
+ count_thp_event(order, THP_ANON_ALLOC_FAIL);
next:
order = next_order(&orders, order);
}
--
2.34.1
