The patch titled
Subject: mm: memcg/slab: unify SLAB and SLUB page accounting
has been added to the -mm tree. Its filename is
mm-unify-slab-and-slub-page-accounting.patch
This patch should soon appear at
http://ozlabs.org/~akpm/mmots/broken-out/mm-unify-slab-and-slub-page-accounting.patch
and later at
http://ozlabs.org/~akpm/mmotm/broken-out/mm-unify-slab-and-slub-page-accounting.patch
Before you just go and hit "reply", please:
a) Consider who else should be cc'ed
b) Prefer to cc a suitable mailing list as well
c) Ideally: find the original patch on the mailing list and do a
reply-to-all to that, adding suitable additional cc's
*** Remember to use Documentation/process/submit-checklist.rst when testing your code ***
The -mm tree is included into linux-next and is updated
there every 3-4 working days
------------------------------------------------------
From: Roman Gushchin <guro@xxxxxx>
Subject: mm: memcg/slab: unify SLAB and SLUB page accounting
Currently the page accounting code is duplicated in SLAB and SLUB
internals. Let's move it into new (un)charge_slab_page helpers in the
slab_common.c file. These helpers will be responsible for statistics
(global and memcg-aware) and memcg charging. So they are replacing direct
memcg_(un)charge_slab() calls.
Link: http://lkml.kernel.org/r/20190611231813.3148843-6-guro@xxxxxx
Signed-off-by: Roman Gushchin <guro@xxxxxx>
Reviewed-by: Shakeel Butt <shakeelb@xxxxxxxxxx>
Acked-by: Christoph Lameter <cl@xxxxxxxxx>
Acked-by: Vladimir Davydov <vdavydov.dev@xxxxxxxxx>
Acked-by: Johannes Weiner <hannes@xxxxxxxxxxx>
Cc: Michal Hocko <mhocko@xxxxxxxx>
Cc: Waiman Long <longman@xxxxxxxxxx>
Cc: David Rientjes <rientjes@xxxxxxxxxx>
Cc: Joonsoo Kim <iamjoonsoo.kim@xxxxxxx>
Cc: Pekka Enberg <penberg@xxxxxxxxxx>
Signed-off-by: Andrew Morton <akpm@xxxxxxxxxxxxxxxxxxxx>
---
mm/slab.c | 19 +++----------------
mm/slab.h | 25 +++++++++++++++++++++++++
mm/slub.c | 14 ++------------
3 files changed, 30 insertions(+), 28 deletions(-)
--- a/mm/slab.c~mm-unify-slab-and-slub-page-accounting
+++ a/mm/slab.c
@@ -1360,7 +1360,6 @@ static struct page *kmem_getpages(struct
int nodeid)
{
struct page *page;
- int nr_pages;
flags |= cachep->allocflags;
@@ -1370,17 +1369,11 @@ static struct page *kmem_getpages(struct
return NULL;
}
- if (memcg_charge_slab(page, flags, cachep->gfporder, cachep)) {
+ if (charge_slab_page(page, flags, cachep->gfporder, cachep)) {
__free_pages(page, cachep->gfporder);
return NULL;
}
- nr_pages = (1 << cachep->gfporder);
- if (cachep->flags & SLAB_RECLAIM_ACCOUNT)
- mod_lruvec_page_state(page, NR_SLAB_RECLAIMABLE, nr_pages);
- else
- mod_lruvec_page_state(page, NR_SLAB_UNRECLAIMABLE, nr_pages);
-
__SetPageSlab(page);
/* Record if ALLOC_NO_WATERMARKS was set when allocating the slab */
if (sk_memalloc_socks() && page_is_pfmemalloc(page))
@@ -1395,12 +1388,6 @@ static struct page *kmem_getpages(struct
static void kmem_freepages(struct kmem_cache *cachep, struct page *page)
{
int order = cachep->gfporder;
- unsigned long nr_freed = (1 << order);
-
- if (cachep->flags & SLAB_RECLAIM_ACCOUNT)
- mod_lruvec_page_state(page, NR_SLAB_RECLAIMABLE, -nr_freed);
- else
- mod_lruvec_page_state(page, NR_SLAB_UNRECLAIMABLE, -nr_freed);
BUG_ON(!PageSlab(page));
__ClearPageSlabPfmemalloc(page);
@@ -1409,8 +1396,8 @@ static void kmem_freepages(struct kmem_c
page->mapping = NULL;
if (current->reclaim_state)
- current->reclaim_state->reclaimed_slab += nr_freed;
- memcg_uncharge_slab(page, order, cachep);
+ current->reclaim_state->reclaimed_slab += 1 << order;
+ uncharge_slab_page(page, order, cachep);
__free_pages(page, order);
}
--- a/mm/slab.h~mm-unify-slab-and-slub-page-accounting
+++ a/mm/slab.h
@@ -205,6 +205,12 @@ ssize_t slabinfo_write(struct file *file
void __kmem_cache_free_bulk(struct kmem_cache *, size_t, void **);
int __kmem_cache_alloc_bulk(struct kmem_cache *, gfp_t, size_t, void **);
+static inline int cache_vmstat_idx(struct kmem_cache *s)
+{
+ return (s->flags & SLAB_RECLAIM_ACCOUNT) ?
+ NR_SLAB_RECLAIMABLE : NR_SLAB_UNRECLAIMABLE;
+}
+
#ifdef CONFIG_MEMCG_KMEM
/* List of all root caches. */
@@ -361,6 +367,25 @@ static inline struct kmem_cache *virt_to
return page->slab_cache;
}
+static __always_inline int charge_slab_page(struct page *page,
+ gfp_t gfp, int order,
+ struct kmem_cache *s)
+{
+ int ret = memcg_charge_slab(page, gfp, order, s);
+
+ if (!ret)
+ mod_lruvec_page_state(page, cache_vmstat_idx(s), 1 << order);
+
+ return ret;
+}
+
+static __always_inline void uncharge_slab_page(struct page *page, int order,
+ struct kmem_cache *s)
+{
+ mod_lruvec_page_state(page, cache_vmstat_idx(s), -(1 << order));
+ memcg_uncharge_slab(page, order, s);
+}
+
static inline struct kmem_cache *cache_from_obj(struct kmem_cache *s, void *x)
{
struct kmem_cache *cachep;
--- a/mm/slub.c~mm-unify-slab-and-slub-page-accounting
+++ a/mm/slub.c
@@ -1488,7 +1488,7 @@ static inline struct page *alloc_slab_pa
else
page = __alloc_pages_node(node, flags, order);
- if (page && memcg_charge_slab(page, flags, order, s)) {
+ if (page && charge_slab_page(page, flags, order, s)) {
__free_pages(page, order);
page = NULL;
}
@@ -1681,11 +1681,6 @@ out:
if (!page)
return NULL;
- mod_lruvec_page_state(page,
- (s->flags & SLAB_RECLAIM_ACCOUNT) ?
- NR_SLAB_RECLAIMABLE : NR_SLAB_UNRECLAIMABLE,
- 1 << oo_order(oo));
-
inc_slabs_node(s, page_to_nid(page), page->objects);
return page;
@@ -1719,18 +1714,13 @@ static void __free_slab(struct kmem_cach
check_object(s, page, p, SLUB_RED_INACTIVE);
}
- mod_lruvec_page_state(page,
- (s->flags & SLAB_RECLAIM_ACCOUNT) ?
- NR_SLAB_RECLAIMABLE : NR_SLAB_UNRECLAIMABLE,
- -pages);
-
__ClearPageSlabPfmemalloc(page);
__ClearPageSlab(page);
page->mapping = NULL;
if (current->reclaim_state)
current->reclaim_state->reclaimed_slab += pages;
- memcg_uncharge_slab(page, order, s);
+ uncharge_slab_page(page, order, s);
__free_pages(page, order);
}
_
Patches currently in -mm which might be from guro@xxxxxx are
mm-postpone-kmem_cache-memcg-pointer-initialization-to-memcg_link_cache.patch
mm-rename-slab-delayed-deactivation-functions-and-fields.patch
mm-generalize-postponed-non-root-kmem_cache-deactivation.patch
mm-introduce-__memcg_kmem_uncharge_memcg.patch
mm-unify-slab-and-slub-page-accounting.patch
mm-dont-check-the-dying-flag-on-kmem_cache-creation.patch
mm-synchronize-access-to-kmem_cache-dying-flag-using-a-spinlock.patch
mm-rework-non-root-kmem_cache-lifecycle-management.patch
mm-stop-setting-page-mem_cgroup-pointer-for-slab-pages.patch
mm-reparent-memcg-kmem_caches-on-cgroup-removal.patch
