The patch titled
Subject: mm: memcg/slab: reparent memcg kmem_caches on cgroup removal
has been added to the -mm tree. Its filename is
mm-reparent-memcg-kmem_caches-on-cgroup-removal.patch
This patch should soon appear at
http://ozlabs.org/~akpm/mmots/broken-out/mm-reparent-memcg-kmem_caches-on-cgroup-removal.patch
and later at
http://ozlabs.org/~akpm/mmotm/broken-out/mm-reparent-memcg-kmem_caches-on-cgroup-removal.patch
Before you just go and hit "reply", please:
a) Consider who else should be cc'ed
b) Prefer to cc a suitable mailing list as well
c) Ideally: find the original patch on the mailing list and do a
reply-to-all to that, adding suitable additional cc's
*** Remember to use Documentation/process/submit-checklist.rst when testing your code ***
The -mm tree is included into linux-next and is updated
there every 3-4 working days
------------------------------------------------------
From: Roman Gushchin <guro@xxxxxx>
Subject: mm: memcg/slab: reparent memcg kmem_caches on cgroup removal
Let's reparent non-root kmem_caches on memcg offlining. This allows us to
release the memory cgroup without waiting for the last outstanding kernel
object (e.g. dentry used by another application).
Since the parent cgroup is already charged, everything we need to do is to
splice the list of kmem_caches to the parent's kmem_caches list, swap the
memcg pointer, drop the css refcounter for each kmem_cache and adjust the
parent's css refcounter.
Please, note that kmem_cache->memcg_params.memcg isn't a stable pointer
anymore. It's safe to read it under rcu_read_lock(), cgroup_mutex held,
or any other way that protects the memory cgroup from being released.
We can race with the slab allocation and deallocation paths. It's not a
big problem: parent's charge and slab global stats are always correct, and
we don't care anymore about the child usage and global stats. The child
cgroup is already offline, so we don't use or show it anywhere.
Local slab stats (NR_SLAB_RECLAIMABLE and NR_SLAB_UNRECLAIMABLE) aren't
used anywhere except count_shadow_nodes(). But even there it won't break
anything: after reparenting "nodes" will be 0 on child level (because
we're already reparenting shrinker lists), and on parent level page stats
always were 0, and this patch won't change anything.
Link: http://lkml.kernel.org/r/20190611231813.3148843-11-guro@xxxxxx
Signed-off-by: Roman Gushchin <guro@xxxxxx>
Cc: Christoph Lameter <cl@xxxxxxxxx>
Cc: Johannes Weiner <hannes@xxxxxxxxxxx>
Cc: Michal Hocko <mhocko@xxxxxxxx>
Cc: Shakeel Butt <shakeelb@xxxxxxxxxx>
Cc: Vladimir Davydov <vdavydov.dev@xxxxxxxxx>
Cc: Waiman Long <longman@xxxxxxxxxx>
Cc: David Rientjes <rientjes@xxxxxxxxxx>
Cc: Joonsoo Kim <iamjoonsoo.kim@xxxxxxx>
Cc: Pekka Enberg <penberg@xxxxxxxxxx>
Signed-off-by: Andrew Morton <akpm@xxxxxxxxxxxxxxxxxxxx>
---
include/linux/slab.h | 2 +-
mm/memcontrol.c | 14 ++++++++------
mm/slab.h | 26 ++++++++++++++++++++------
mm/slab_common.c | 19 +++++++++++++++++--
4 files changed, 46 insertions(+), 15 deletions(-)
--- a/include/linux/slab.h~mm-reparent-memcg-kmem_caches-on-cgroup-removal
+++ a/include/linux/slab.h
@@ -152,7 +152,7 @@ void kmem_cache_destroy(struct kmem_cach
int kmem_cache_shrink(struct kmem_cache *);
void memcg_create_kmem_cache(struct mem_cgroup *, struct kmem_cache *);
-void memcg_deactivate_kmem_caches(struct mem_cgroup *);
+void memcg_deactivate_kmem_caches(struct mem_cgroup *, struct mem_cgroup *);
/*
* Please use this macro to create slab caches. Simply specify the
--- a/mm/memcontrol.c~mm-reparent-memcg-kmem_caches-on-cgroup-removal
+++ a/mm/memcontrol.c
@@ -3284,15 +3284,15 @@ static void memcg_offline_kmem(struct me
*/
memcg->kmem_state = KMEM_ALLOCATED;
- memcg_deactivate_kmem_caches(memcg);
-
- kmemcg_id = memcg->kmemcg_id;
- BUG_ON(kmemcg_id < 0);
-
parent = parent_mem_cgroup(memcg);
if (!parent)
parent = root_mem_cgroup;
+ memcg_deactivate_kmem_caches(memcg, parent);
+
+ kmemcg_id = memcg->kmemcg_id;
+ BUG_ON(kmemcg_id < 0);
+
/*
* Change kmemcg_id of this cgroup and all its descendants to the
* parent's id, and then move all entries from this cgroup's list_lrus
@@ -3325,7 +3325,6 @@ static void memcg_free_kmem(struct mem_c
if (memcg->kmem_state == KMEM_ALLOCATED) {
WARN_ON(!list_empty(&memcg->kmem_caches));
static_branch_dec(&memcg_kmem_enabled_key);
- WARN_ON(page_counter_read(&memcg->kmem));
}
}
#else
@@ -4772,6 +4771,9 @@ mem_cgroup_css_alloc(struct cgroup_subsy
/* The following stuff does not apply to the root */
if (!parent) {
+#ifdef CONFIG_MEMCG_KMEM
+ INIT_LIST_HEAD(&memcg->kmem_caches);
+#endif
root_mem_cgroup = memcg;
return &memcg->css;
}
--- a/mm/slab_common.c~mm-reparent-memcg-kmem_caches-on-cgroup-removal
+++ a/mm/slab_common.c
@@ -252,7 +252,8 @@ static void memcg_unlink_cache(struct km
} else {
list_del(&s->memcg_params.children_node);
list_del(&s->memcg_params.kmem_caches_node);
- css_put(&s->memcg_params.memcg->css);
+ mem_cgroup_put(s->memcg_params.memcg);
+ WRITE_ONCE(s->memcg_params.memcg, NULL);
}
}
#else
@@ -790,11 +791,13 @@ unlock:
spin_unlock_irq(&memcg_kmem_wq_lock);
}
-void memcg_deactivate_kmem_caches(struct mem_cgroup *memcg)
+void memcg_deactivate_kmem_caches(struct mem_cgroup *memcg,
+ struct mem_cgroup *parent)
{
int idx;
struct memcg_cache_array *arr;
struct kmem_cache *s, *c;
+ unsigned int nr_reparented;
idx = memcg_cache_id(memcg);
@@ -812,6 +815,18 @@ void memcg_deactivate_kmem_caches(struct
kmemcg_cache_deactivate(c);
arr->entries[idx] = NULL;
}
+ nr_reparented = 0;
+ list_for_each_entry(s, &memcg->kmem_caches,
+ memcg_params.kmem_caches_node) {
+ WRITE_ONCE(s->memcg_params.memcg, parent);
+ css_put(&memcg->css);
+ nr_reparented++;
+ }
+ if (nr_reparented) {
+ list_splice_init(&memcg->kmem_caches,
+ &parent->kmem_caches);
+ css_get_many(&parent->css, nr_reparented);
+ }
mutex_unlock(&slab_mutex);
put_online_mems();
--- a/mm/slab.h~mm-reparent-memcg-kmem_caches-on-cgroup-removal
+++ a/mm/slab.h
@@ -261,6 +261,9 @@ static inline struct kmem_cache *memcg_r
* which do not have slab_cache pointer set.
* So this function assumes that the page can pass PageHead() and PageSlab()
* checks.
+ *
+ * The kmem_cache can be reparented asynchronously. The caller must ensure
+ * the memcg lifetime, e.g. by taking rcu_read_lock() or cgroup_mutex.
*/
static inline struct mem_cgroup *memcg_from_slab_page(struct page *page)
{
@@ -268,7 +271,7 @@ static inline struct mem_cgroup *memcg_f
s = READ_ONCE(page->slab_cache);
if (s && !is_root_cache(s))
- return s->memcg_params.memcg;
+ return READ_ONCE(s->memcg_params.memcg);
return NULL;
}
@@ -285,10 +288,18 @@ static __always_inline int memcg_charge_
struct lruvec *lruvec;
int ret;
- memcg = s->memcg_params.memcg;
+ rcu_read_lock();
+ memcg = READ_ONCE(s->memcg_params.memcg);
+ while (memcg && !css_tryget_online(&memcg->css))
+ memcg = parent_mem_cgroup(memcg);
+ rcu_read_unlock();
+
+ if (unlikely(!memcg))
+ return true;
+
ret = memcg_kmem_charge_memcg(page, gfp, order, memcg);
if (ret)
- return ret;
+ goto out;
lruvec = mem_cgroup_lruvec(page_pgdat(page), memcg);
mod_lruvec_state(lruvec, cache_vmstat_idx(s), 1 << order);
@@ -296,8 +307,9 @@ static __always_inline int memcg_charge_
/* transer try_charge() page references to kmem_cache */
percpu_ref_get_many(&s->memcg_params.refcnt, 1 << order);
css_put_many(&memcg->css, 1 << order);
-
- return 0;
+out:
+ css_put(&memcg->css);
+ return ret;
}
/*
@@ -310,10 +322,12 @@ static __always_inline void memcg_unchar
struct mem_cgroup *memcg;
struct lruvec *lruvec;
- memcg = s->memcg_params.memcg;
+ rcu_read_lock();
+ memcg = READ_ONCE(s->memcg_params.memcg);
lruvec = mem_cgroup_lruvec(page_pgdat(page), memcg);
mod_lruvec_state(lruvec, cache_vmstat_idx(s), -(1 << order));
memcg_kmem_uncharge_memcg(page, order, memcg);
+ rcu_read_unlock();
percpu_ref_put_many(&s->memcg_params.refcnt, 1 << order);
}
_
Patches currently in -mm which might be from guro@xxxxxx are
mm-postpone-kmem_cache-memcg-pointer-initialization-to-memcg_link_cache.patch
mm-rename-slab-delayed-deactivation-functions-and-fields.patch
mm-generalize-postponed-non-root-kmem_cache-deactivation.patch
mm-introduce-__memcg_kmem_uncharge_memcg.patch
mm-unify-slab-and-slub-page-accounting.patch
mm-dont-check-the-dying-flag-on-kmem_cache-creation.patch
mm-synchronize-access-to-kmem_cache-dying-flag-using-a-spinlock.patch
mm-rework-non-root-kmem_cache-lifecycle-management.patch
mm-stop-setting-page-mem_cgroup-pointer-for-slab-pages.patch
mm-reparent-memcg-kmem_caches-on-cgroup-removal.patch
