The patch titled
Subject: mm: restart if multiple traversals raced
has been added to the -mm mm-unstable branch. Its filename is
mm-restart-if-multiple-traversals-raced.patch
This patch will shortly appear at
https://git.kernel.org/pub/scm/linux/kernel/git/akpm/25-new.git/tree/patches/mm-restart-if-multiple-traversals-raced.patch
This patch will later appear in the mm-unstable branch at
git://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm
Before you just go and hit "reply", please:
a) Consider who else should be cc'ed
b) Prefer to cc a suitable mailing list as well
c) Ideally: find the original patch on the mailing list and do a
reply-to-all to that, adding suitable additional cc's
*** Remember to use Documentation/process/submit-checklist.rst when testing your code ***
The -mm tree is included into linux-next via the mm-everything
branch at git://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm
and is updated there every 2-3 working days
------------------------------------------------------
From: Kinsey Ho <kinseyho@xxxxxxxxxx>
Subject: mm: restart if multiple traversals raced
Date: Tue, 27 Aug 2024 23:07:41 +0000
Currently, if multiple reclaimers raced on the same position, the
reclaimers which detect the race will still reclaim from the same memcg.
Instead, the reclaimers which detect the race should move on to the next
memcg in the hierarchy.
So, in the case where multiple traversals race, jump back to the start of
the mem_cgroup_iter() function to find the next memcg in the hierarchy to
reclaim from.
Link: https://lkml.kernel.org/r/20240827230753.2073580-5-kinseyho@xxxxxxxxxx
Signed-off-by: Kinsey Ho <kinseyho@xxxxxxxxxx>
Cc: Johannes Weiner <hannes@xxxxxxxxxxx>
Cc: Michal Hocko <mhocko@xxxxxxxxxx>
Cc: Michal Koutný <mkoutny@xxxxxxxx>
Cc: Muchun Song <muchun.song@xxxxxxxxx>
Cc: Roman Gushchin <roman.gushchin@xxxxxxxxx>
Cc: Shakeel Butt <shakeel.butt@xxxxxxxxx>
Cc: Tejun Heo <tj@xxxxxxxxxx>
Cc: Yosry Ahmed <yosryahmed@xxxxxxxxxx>
Cc: Zefan Li <lizefan.x@xxxxxxxxxxxxx>
Signed-off-by: Andrew Morton <akpm@xxxxxxxxxxxxxxxxxxxx>
---
include/linux/memcontrol.h | 4 ++--
mm/memcontrol.c | 22 ++++++++++++++--------
2 files changed, 16 insertions(+), 10 deletions(-)
--- a/include/linux/memcontrol.h~mm-restart-if-multiple-traversals-raced
+++ a/include/linux/memcontrol.h
@@ -57,7 +57,7 @@ enum memcg_memory_event {
struct mem_cgroup_reclaim_cookie {
pg_data_t *pgdat;
- unsigned int generation;
+ int generation;
};
#ifdef CONFIG_MEMCG
@@ -78,7 +78,7 @@ struct lruvec_stats;
struct mem_cgroup_reclaim_iter {
struct mem_cgroup *position;
/* scan generation, increased every round-trip */
- unsigned int generation;
+ atomic_t generation;
};
/*
--- a/mm/memcontrol.c~mm-restart-if-multiple-traversals-raced
+++ a/mm/memcontrol.c
@@ -986,7 +986,7 @@ struct mem_cgroup *mem_cgroup_iter(struc
struct mem_cgroup_reclaim_cookie *reclaim)
{
struct mem_cgroup_reclaim_iter *iter;
- struct cgroup_subsys_state *css = NULL;
+ struct cgroup_subsys_state *css;
struct mem_cgroup *memcg = NULL;
struct mem_cgroup *pos = NULL;
@@ -999,18 +999,20 @@ struct mem_cgroup *mem_cgroup_iter(struc
rcu_read_lock();
restart:
if (reclaim) {
+ int gen;
struct mem_cgroup_per_node *mz;
mz = root->nodeinfo[reclaim->pgdat->node_id];
iter = &mz->iter;
+ gen = atomic_read(&iter->generation);
/*
* On start, join the current reclaim iteration cycle.
* Exit when a concurrent walker completes it.
*/
if (!prev)
- reclaim->generation = iter->generation;
- else if (reclaim->generation != iter->generation)
+ reclaim->generation = gen;
+ else if (reclaim->generation != gen)
goto out_unlock;
pos = READ_ONCE(iter->position);
@@ -1018,8 +1020,7 @@ restart:
pos = prev;
}
- if (pos)
- css = &pos->css;
+ css = pos ? &pos->css : NULL;
for (;;) {
css = css_next_descendant_pre(css, &root->css);
@@ -1033,21 +1034,26 @@ restart:
* and kicking, and don't take an extra reference.
*/
if (css == &root->css || css_tryget(css)) {
- memcg = mem_cgroup_from_css(css);
break;
}
}
+ memcg = mem_cgroup_from_css(css);
+
if (reclaim) {
/*
* The position could have already been updated by a competing
* thread, so check that the value hasn't changed since we read
* it to avoid reclaiming from the same cgroup twice.
*/
- (void)cmpxchg(&iter->position, pos, memcg);
+ if (cmpxchg(&iter->position, pos, memcg) != pos) {
+ if (css && css != &root->css)
+ css_put(css);
+ goto restart;
+ }
if (!memcg) {
- iter->generation++;
+ atomic_inc(&iter->generation);
/*
* Reclaimers share the hierarchy walk, and a
_
Patches currently in -mm which might be from kinseyho@xxxxxxxxxx are
cgroup-clarify-css-sibling-linkage-is-protected-by-cgroup_mutex-or-rcu.patch
mm-dont-hold-css-refcnt-during-traversal.patch
mm-increment-gen-before-restarting-traversal.patch
mm-restart-if-multiple-traversals-raced.patch
mm-clean-up-mem_cgroup_iter.patch
