The patch titled
Subject: mm: memcg: remote memcg charging for kmem allocations
has been removed from the -mm tree. Its filename was
mm-memcg-remote-memcg-charging-for-kmem-allocations.patch
This patch was dropped because an updated version will be merged
------------------------------------------------------
From: Shakeel Butt <shakeelb@xxxxxxxxxx>
Subject: mm: memcg: remote memcg charging for kmem allocations
Patch series "Directed kmem charging", v5.
This patchset introduces memcg variant memory allocation functions. The
caller can explicitly pass the memcg to charge for kmem allocations.
Currently the kernel, for __GFP_ACCOUNT memory allocation requests,
extract the memcg of the current task to charge for the kmem allocation.
This patch series introduces kmem allocation functions where the caller
can pass the pointer to the remote memcg. The remote memcg will be
charged for the allocation instead of the memcg of the caller. However
the caller must have a reference to the remote memcg.
This patch (of 2):
Introduce the memcg variant for kmalloc[_node] and
kmem_cache_alloc[_node]. For kmem_cache_alloc, the kernel switches the
root kmem cache with the memcg specific kmem cache for __GFP_ACCOUNT
allocations to charge those allocations to the memcg. However, the memcg
to charge is extracted from the current task_struct. This patch
introduces the variant of kmem cache allocation functions where the memcg
can be provided explicitly by the caller instead of deducing the memcg
from the current task.
The kmalloc allocations are underlying served using the kmem caches unless
the size of the allocation request is larger than KMALLOC_MAX_CACHE_SIZE,
in which case, the kmem caches are bypassed and the request is routed
directly to page allocator. So, for __GFP_ACCOUNT kmalloc allocations,
the memcg of current task is charged. This patch introduces memcg variant
of kmalloc functions to allow callers to provide memcg for charging.
These functions are useful for use-cases where the allocations should be
charged to the memcg different from the memcg of the caller. One such
concrete use-case is the allocations for fsnotify event objects where the
objects should be charged to the listener instead of the producer.
One requirement to call these functions is that the caller must have the
reference to the memcg. Using kmalloc_memcg and kmem_cache_alloc_memcg
implicitly assumes that the caller is requesting a __GFP_ACCOUNT
allocation.
Link: http://lkml.kernel.org/r/20180416205150.113915-2-shakeelb@xxxxxxxxxx
Signed-off-by: Shakeel Butt <shakeelb@xxxxxxxxxx>
Cc: Michal Hocko <mhocko@xxxxxxxxxx>
Cc: Jan Kara <jack@xxxxxxx>
Cc: Amir Goldstein <amir73il@xxxxxxxxx>
Cc: Christoph Lameter <cl@xxxxxxxxx>
Cc: Pekka Enberg <penberg@xxxxxxxxxx>
Cc: David Rientjes <rientjes@xxxxxxxxxx>
Cc: Joonsoo Kim <iamjoonsoo.kim@xxxxxxx>
Cc: Greg Thelen <gthelen@xxxxxxxxxx>
Cc: Johannes Weiner <hannes@xxxxxxxxxxx>
Cc: Vladimir Davydov <vdavydov.dev@xxxxxxxxx>
Cc: Mel Gorman <mgorman@xxxxxxx>
Cc: Vlastimil Babka <vbabka@xxxxxxx>
Signed-off-by: Andrew Morton <akpm@xxxxxxxxxxxxxxxxxxxx>
---
include/linux/sched.h | 3 +
include/linux/sched/mm.h | 24 +++++++++++++++
include/linux/slab.h | 59 +++++++++++++++++++++++++++++++++++++
kernel/fork.c | 3 +
mm/memcontrol.c | 18 ++++++++++-
5 files changed, 105 insertions(+), 2 deletions(-)
diff -puN include/linux/sched.h~mm-memcg-remote-memcg-charging-for-kmem-allocations include/linux/sched.h
--- a/include/linux/sched.h~mm-memcg-remote-memcg-charging-for-kmem-allocations
+++ a/include/linux/sched.h
@@ -1137,6 +1137,9 @@ struct task_struct {
/* Number of pages to reclaim on returning to userland: */
unsigned int memcg_nr_pages_over_high;
+
+ /* Used by memcontrol for targeted memcg charge: */
+ struct mem_cgroup *target_memcg;
#endif
#ifdef CONFIG_UPROBES
diff -puN include/linux/sched/mm.h~mm-memcg-remote-memcg-charging-for-kmem-allocations include/linux/sched/mm.h
--- a/include/linux/sched/mm.h~mm-memcg-remote-memcg-charging-for-kmem-allocations
+++ a/include/linux/sched/mm.h
@@ -206,6 +206,30 @@ static inline void memalloc_noreclaim_re
current->flags = (current->flags & ~PF_MEMALLOC) | flags;
}
+#ifdef CONFIG_MEMCG
+static inline struct mem_cgroup *memalloc_memcg_save(struct mem_cgroup *memcg)
+{
+ struct mem_cgroup *old_memcg = current->target_memcg;
+
+ current->target_memcg = memcg;
+ return old_memcg;
+}
+
+static inline void memalloc_memcg_restore(struct mem_cgroup *memcg)
+{
+ current->target_memcg = memcg;
+}
+#else
+static inline struct mem_cgroup *memalloc_memcg_save(struct mem_cgroup *memcg)
+{
+ return NULL;
+}
+
+static inline void memalloc_memcg_restore(struct mem_cgroup *memcg)
+{
+}
+#endif /* CONFIG_MEMCG */
+
#ifdef CONFIG_MEMBARRIER
enum {
MEMBARRIER_STATE_PRIVATE_EXPEDITED_READY = (1U << 0),
diff -puN include/linux/slab.h~mm-memcg-remote-memcg-charging-for-kmem-allocations include/linux/slab.h
--- a/include/linux/slab.h~mm-memcg-remote-memcg-charging-for-kmem-allocations
+++ a/include/linux/slab.h
@@ -15,6 +15,7 @@
#include <linux/gfp.h>
#include <linux/types.h>
#include <linux/workqueue.h>
+#include <linux/sched/mm.h>
/*
@@ -374,6 +375,21 @@ static __always_inline void kfree_bulk(s
kmem_cache_free_bulk(NULL, size, p);
}
+/*
+ * Calling kmem_cache_alloc_memcg implicitly assumes that the caller
+ * wants a __GFP_ACCOUNT allocation.
+ */
+static __always_inline void *kmem_cache_alloc_memcg(struct kmem_cache *cachep,
+ gfp_t flags,
+ struct mem_cgroup *memcg)
+{
+ struct mem_cgroup *old_memcg = memalloc_memcg_save(memcg);
+ void *ptr = kmem_cache_alloc(cachep, flags | __GFP_ACCOUNT);
+
+ memalloc_memcg_restore(old_memcg);
+ return ptr;
+}
+
#ifdef CONFIG_NUMA
void *__kmalloc_node(size_t size, gfp_t flags, int node) __assume_kmalloc_alignment __malloc;
void *kmem_cache_alloc_node(struct kmem_cache *, gfp_t flags, int node) __assume_slab_alignment __malloc;
@@ -389,6 +405,21 @@ static __always_inline void *kmem_cache_
}
#endif
+/*
+ * Calling kmem_cache_alloc_node_memcg implicitly assumes that the caller
+ * wants a __GFP_ACCOUNT allocation.
+ */
+static __always_inline void *
+kmem_cache_alloc_node_memcg(struct kmem_cache *cachep, gfp_t flags, int node,
+ struct mem_cgroup *memcg)
+{
+ struct mem_cgroup *old_memcg = memalloc_memcg_save(memcg);
+ void *ptr = kmem_cache_alloc_node(cachep, flags | __GFP_ACCOUNT, node);
+
+ memalloc_memcg_restore(old_memcg);
+ return ptr;
+}
+
#ifdef CONFIG_TRACING
extern void *kmem_cache_alloc_trace(struct kmem_cache *, gfp_t, size_t) __assume_slab_alignment __malloc;
@@ -518,6 +549,20 @@ static __always_inline void *kmalloc(siz
}
/*
+ * Calling kmalloc_memcg implicitly assumes that the caller wants a
+ * __GFP_ACCOUNT allocation.
+ */
+static __always_inline void *kmalloc_memcg(size_t size, gfp_t flags,
+ struct mem_cgroup *memcg)
+{
+ struct mem_cgroup *old_memcg = memalloc_memcg_save(memcg);
+ void *ptr = kmalloc(size, flags | __GFP_ACCOUNT);
+
+ memalloc_memcg_restore(old_memcg);
+ return ptr;
+}
+
+/*
* Determine size used for the nth kmalloc cache.
* return size or 0 if a kmalloc cache for that
* size does not exist
@@ -554,6 +599,20 @@ static __always_inline void *kmalloc_nod
return __kmalloc_node(size, flags, node);
}
+/*
+ * Calling kmalloc_node_memcg implicitly assumes that the caller wants a
+ * __GFP_ACCOUNT allocation.
+ */
+static __always_inline void *
+kmalloc_node_memcg(size_t size, gfp_t flags, int node, struct mem_cgroup *memcg)
+{
+ struct mem_cgroup *old_memcg = memalloc_memcg_save(memcg);
+ void *ptr = kmalloc_node(size, flags | __GFP_ACCOUNT, node);
+
+ memalloc_memcg_restore(old_memcg);
+ return ptr;
+}
+
struct memcg_cache_array {
struct rcu_head rcu;
struct kmem_cache *entries[0];
diff -puN kernel/fork.c~mm-memcg-remote-memcg-charging-for-kmem-allocations kernel/fork.c
--- a/kernel/fork.c~mm-memcg-remote-memcg-charging-for-kmem-allocations
+++ a/kernel/fork.c
@@ -842,6 +842,9 @@ static struct task_struct *dup_task_stru
tsk->fail_nth = 0;
#endif
+#ifdef CONFIG_MEMCG
+ tsk->target_memcg = NULL;
+#endif
return tsk;
free_stack:
diff -puN mm/memcontrol.c~mm-memcg-remote-memcg-charging-for-kmem-allocations mm/memcontrol.c
--- a/mm/memcontrol.c~mm-memcg-remote-memcg-charging-for-kmem-allocations
+++ a/mm/memcontrol.c
@@ -701,6 +701,20 @@ static struct mem_cgroup *get_mem_cgroup
return memcg;
}
+static __always_inline struct mem_cgroup *get_mem_cgroup(
+ struct mem_cgroup *memcg, struct mm_struct *mm)
+{
+ if (unlikely(memcg)) {
+ rcu_read_lock();
+ if (css_tryget_online(&memcg->css)) {
+ rcu_read_unlock();
+ return memcg;
+ }
+ rcu_read_unlock();
+ }
+ return get_mem_cgroup_from_mm(mm);
+}
+
/**
* mem_cgroup_iter - iterate over memory cgroup hierarchy
* @root: hierarchy root
@@ -2261,7 +2275,7 @@ struct kmem_cache *memcg_kmem_get_cache(
if (current->memcg_kmem_skip_account)
return cachep;
- memcg = get_mem_cgroup_from_mm(current->mm);
+ memcg = get_mem_cgroup(current->target_memcg, current->mm);
kmemcg_id = READ_ONCE(memcg->kmemcg_id);
if (kmemcg_id < 0)
goto out;
@@ -2345,7 +2359,7 @@ int memcg_kmem_charge(struct page *page,
if (memcg_kmem_bypass())
return 0;
- memcg = get_mem_cgroup_from_mm(current->mm);
+ memcg = get_mem_cgroup(current->target_memcg, current->mm);
if (!mem_cgroup_is_root(memcg)) {
ret = memcg_kmem_charge_memcg(page, gfp, order, memcg);
if (!ret)
_
Patches currently in -mm which might be from shakeelb@xxxxxxxxxx are
mm-fix-race-between-kmem_cache-destroy-create-and-deactivate.patch
mm-fix-race-between-kmem_cache-destroy-create-and-deactivate-v2.patch
mm-fix-race-between-kmem_cache-destroy-create-and-deactivate-v3.patch
fs-fsnotify-account-fsnotify-metadata-to-kmemcg.patch
mm-memcontrol-drain-stocks-on-resize-limit.patch
mm-save-two-stranding-bit-in-gfp_mask.patch
--
To unsubscribe from this list: send the line "unsubscribe mm-commits" in
the body of a message to majordomo@xxxxxxxxxxxxxxx
More majordomo info at http://vger.kernel.org/majordomo-info.html
