On Tue, Jan 04, 2022 at 01:10:37AM +0100, Vlastimil Babka wrote:
> page->memcg_data is used with MEMCG_DATA_OBJCGS flag only for slab pages
> so convert all the related infrastructure to struct slab. Also use
> struct folio instead of struct page when resolving object pointers.
>
> This is not just mechanistic changing of types and names. Now in
> mem_cgroup_from_obj() we use folio_test_slab() to decide if we interpret
> the folio as a real slab instead of a large kmalloc, instead of relying
> on MEMCG_DATA_OBJCGS bit that used to be checked in page_objcgs_check().
> Similarly in memcg_slab_free_hook() where we can encounter
> kmalloc_large() pages (here the folio slab flag check is implied by
> virt_to_slab()). As a result, page_objcgs_check() can be dropped instead
> of converted.
>
> To avoid include cycles, move the inline definition of slab_objcgs()
> from memcontrol.h to mm/slab.h.
>
> Signed-off-by: Vlastimil Babka <vbabka@xxxxxxx>
> Cc: Johannes Weiner <hannes@xxxxxxxxxxx>
> Cc: Michal Hocko <mhocko@xxxxxxxxxx>
> Cc: Vladimir Davydov <vdavydov.dev@xxxxxxxxx>
> Cc: <cgroups@xxxxxxxxxxxxxxx>
> ---
> include/linux/memcontrol.h | 48 ------------------------
> mm/memcontrol.c | 47 ++++++++++++-----------
> mm/slab.h | 76 ++++++++++++++++++++++++++------------
> 3 files changed, 79 insertions(+), 92 deletions(-)
>
> diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h
> index 0c5c403f4be6..e34112f6a369 100644
> --- a/include/linux/memcontrol.h
> +++ b/include/linux/memcontrol.h
> @@ -536,45 +536,6 @@ static inline bool folio_memcg_kmem(struct folio *folio)
> return folio->memcg_data & MEMCG_DATA_KMEM;
> }
>
> -/*
> - * page_objcgs - get the object cgroups vector associated with a page
> - * @page: a pointer to the page struct
> - *
> - * Returns a pointer to the object cgroups vector associated with the page,
> - * or NULL. This function assumes that the page is known to have an
> - * associated object cgroups vector. It's not safe to call this function
> - * against pages, which might have an associated memory cgroup: e.g.
> - * kernel stack pages.
> - */
> -static inline struct obj_cgroup **page_objcgs(struct page *page)
> -{
> - unsigned long memcg_data = READ_ONCE(page->memcg_data);
> -
> - VM_BUG_ON_PAGE(memcg_data && !(memcg_data & MEMCG_DATA_OBJCGS), page);
> - VM_BUG_ON_PAGE(memcg_data & MEMCG_DATA_KMEM, page);
> -
> - return (struct obj_cgroup **)(memcg_data & ~MEMCG_DATA_FLAGS_MASK);
> -}
> -
> -/*
> - * page_objcgs_check - get the object cgroups vector associated with a page
> - * @page: a pointer to the page struct
> - *
> - * Returns a pointer to the object cgroups vector associated with the page,
> - * or NULL. This function is safe to use if the page can be directly associated
> - * with a memory cgroup.
> - */
> -static inline struct obj_cgroup **page_objcgs_check(struct page *page)
> -{
> - unsigned long memcg_data = READ_ONCE(page->memcg_data);
> -
> - if (!memcg_data || !(memcg_data & MEMCG_DATA_OBJCGS))
> - return NULL;
> -
> - VM_BUG_ON_PAGE(memcg_data & MEMCG_DATA_KMEM, page);
> -
> - return (struct obj_cgroup **)(memcg_data & ~MEMCG_DATA_FLAGS_MASK);
> -}
>
> #else
> static inline bool folio_memcg_kmem(struct folio *folio)
> @@ -582,15 +543,6 @@ static inline bool folio_memcg_kmem(struct folio *folio)
> return false;
> }
>
> -static inline struct obj_cgroup **page_objcgs(struct page *page)
> -{
> - return NULL;
> -}
> -
> -static inline struct obj_cgroup **page_objcgs_check(struct page *page)
> -{
> - return NULL;
> -}
> #endif
>
> static inline bool PageMemcgKmem(struct page *page)
> diff --git a/mm/memcontrol.c b/mm/memcontrol.c
> index f7b789e692a0..f4fdd5675991 100644
> --- a/mm/memcontrol.c
> +++ b/mm/memcontrol.c
> @@ -2816,31 +2816,31 @@ static inline void mod_objcg_mlstate(struct obj_cgroup *objcg,
> rcu_read_unlock();
> }
>
> -int memcg_alloc_page_obj_cgroups(struct page *page, struct kmem_cache *s,
> - gfp_t gfp, bool new_page)
> +int memcg_alloc_slab_cgroups(struct slab *slab, struct kmem_cache *s,
> + gfp_t gfp, bool new_slab)
> {
> - unsigned int objects = objs_per_slab(s, page_slab(page));
> + unsigned int objects = objs_per_slab(s, slab);
> unsigned long memcg_data;
> void *vec;
>
> gfp &= ~OBJCGS_CLEAR_MASK;
> vec = kcalloc_node(objects, sizeof(struct obj_cgroup *), gfp,
> - page_to_nid(page));
> + slab_nid(slab));
> if (!vec)
> return -ENOMEM;
>
> memcg_data = (unsigned long) vec | MEMCG_DATA_OBJCGS;
> - if (new_page) {
> + if (new_slab) {
> /*
> - * If the slab page is brand new and nobody can yet access
> - * it's memcg_data, no synchronization is required and
> - * memcg_data can be simply assigned.
> + * If the slab is brand new and nobody can yet access its
> + * memcg_data, no synchronization is required and memcg_data can
> + * be simply assigned.
> */
> - page->memcg_data = memcg_data;
> - } else if (cmpxchg(&page->memcg_data, 0, memcg_data)) {
> + slab->memcg_data = memcg_data;
> + } else if (cmpxchg(&slab->memcg_data, 0, memcg_data)) {
> /*
> - * If the slab page is already in use, somebody can allocate
> - * and assign obj_cgroups in parallel. In this case the existing
> + * If the slab is already in use, somebody can allocate and
> + * assign obj_cgroups in parallel. In this case the existing
> * objcg vector should be reused.
> */
> kfree(vec);
> @@ -2865,26 +2865,31 @@ int memcg_alloc_page_obj_cgroups(struct page *page, struct kmem_cache *s,
> */
> struct mem_cgroup *mem_cgroup_from_obj(void *p)
> {
> - struct page *page;
> + struct folio *folio;
>
> if (mem_cgroup_disabled())
> return NULL;
>
> - page = virt_to_head_page(p);
> + folio = virt_to_folio(p);
>
> /*
> * Slab objects are accounted individually, not per-page.
> * Memcg membership data for each individual object is saved in
> * the page->obj_cgroups.
^^^^^^^^^^^^^^^^^
slab->memcg_data
> */
> - if (page_objcgs_check(page)) {
> - struct obj_cgroup *objcg;
> + if (folio_test_slab(folio)) {
> + struct obj_cgroup **objcgs;
> + struct slab *slab;
> unsigned int off;
>
> - off = obj_to_index(page->slab_cache, page_slab(page), p);
> - objcg = page_objcgs(page)[off];
> - if (objcg)
> - return obj_cgroup_memcg(objcg);
> + slab = folio_slab(folio);
> + objcgs = slab_objcgs(slab);
> + if (!objcgs)
> + return NULL;
> +
> + off = obj_to_index(slab->slab_cache, slab, p);
> + if (objcgs[off])
> + return obj_cgroup_memcg(objcgs[off]);
>
> return NULL;
> }
There is a comment below, which needs some changes:
/*
* page_memcg_check() is used here, because page_has_obj_cgroups()
* check above could fail because the object cgroups vector wasn't set
* at that moment, but it can be set concurrently.
* page_memcg_check(page) will guarantee that a proper memory
* cgroup pointer or NULL will be returned.
*/
In reality the folio's slab flag can be cleared before releasing the objcgs \
vector. It seems that there is no such possibility at setting the flag,
it's always set before allocating and assigning the objcg vector.
> @@ -2896,7 +2901,7 @@ struct mem_cgroup *mem_cgroup_from_obj(void *p)
> * page_memcg_check(page) will guarantee that a proper memory
> * cgroup pointer or NULL will be returned.
> */
> - return page_memcg_check(page);
> + return page_memcg_check(folio_page(folio, 0));
> }
>
> __always_inline struct obj_cgroup *get_obj_cgroup_from_current(void)
> diff --git a/mm/slab.h b/mm/slab.h
> index bca9181e96d7..36e0022d8267 100644
> --- a/mm/slab.h
> +++ b/mm/slab.h
> @@ -412,15 +412,36 @@ static inline bool kmem_cache_debug_flags(struct kmem_cache *s, slab_flags_t fla
> }
>
> #ifdef CONFIG_MEMCG_KMEM
> -int memcg_alloc_page_obj_cgroups(struct page *page, struct kmem_cache *s,
> - gfp_t gfp, bool new_page);
> +/*
> + * slab_objcgs - get the object cgroups vector associated with a slab
> + * @slab: a pointer to the slab struct
> + *
> + * Returns a pointer to the object cgroups vector associated with the slab,
> + * or NULL. This function assumes that the slab is known to have an
> + * associated object cgroups vector. It's not safe to call this function
> + * against slabs with underlying pages, which might have an associated memory
> + * cgroup: e.g. kernel stack pages.
Hm, is it still true? I don't think so. It must be safe to call it for any
slab now.
The rest looks good to me, please feel free to add
Reviewed-by: Roman Gushchin <guro@xxxxxx>
after fixing these comments.
Thanks!
