Sorry for being late.
On Fri 02-10-20 10:25:56, Roman Gushchin wrote:
> Currently there are many open-coded reads of the page->mem_cgroup
> pointer, as well as a couple of read helpers, which are barely used.
>
> It creates an obstacle on a way to reuse some bits of the pointer
> for storing additional bits of information. In fact, we already do
> this for slab pages, where the last bit indicates that a pointer has
> an attached vector of objcg pointers instead of a regular memcg
> pointer.
>
> This commits uses 2 existing helpers and introduces a new helper to
> converts all read sides to calls of these helpers:
> struct mem_cgroup *page_memcg(struct page *page);
> struct mem_cgroup *page_memcg_rcu(struct page *page);
> struct mem_cgroup *page_memcg_check(struct page *page);
>
> page_memcg_check() is intended to be used in cases when the page
> can be a slab page and have a memcg pointer pointing at objcg vector.
> It does check the lowest bit, and if set, returns NULL.
> page_memcg() contains a VM_BUG_ON_PAGE() check for the page not
> being a slab page.
>
> To make sure nobody uses a direct access, struct page's
> mem_cgroup/obj_cgroups is converted to unsigned long memcg_data.
>
> Signed-off-by: Roman Gushchin <guro@xxxxxx>
> Acked-by: Johannes Weiner <hannes@xxxxxxxxxxx>
Acked-by: Michal Hocko <mhocko@xxxxxxxx>
> ---
> fs/buffer.c | 2 +-
> fs/iomap/buffered-io.c | 2 +-
> include/linux/memcontrol.h | 114 ++++++++++++++++++++++++++---
> include/linux/mm.h | 22 ------
> include/linux/mm_types.h | 5 +-
> include/trace/events/writeback.h | 2 +-
> kernel/fork.c | 7 +-
> mm/debug.c | 4 +-
> mm/huge_memory.c | 4 +-
> mm/memcontrol.c | 121 ++++++++++++++-----------------
> mm/page_alloc.c | 4 +-
> mm/page_io.c | 6 +-
> mm/slab.h | 9 +--
> mm/workingset.c | 2 +-
> 14 files changed, 184 insertions(+), 120 deletions(-)
>
> diff --git a/fs/buffer.c b/fs/buffer.c
> index bf4d8037f91b..64564ac7dcc5 100644
> --- a/fs/buffer.c
> +++ b/fs/buffer.c
> @@ -657,7 +657,7 @@ int __set_page_dirty_buffers(struct page *page)
> } while (bh != head);
> }
> /*
> - * Lock out page->mem_cgroup migration to keep PageDirty
> + * Lock out page's memcg migration to keep PageDirty
> * synchronized with per-memcg dirty page counters.
> */
> lock_page_memcg(page);
> diff --git a/fs/iomap/buffered-io.c b/fs/iomap/buffered-io.c
> index 8180061b9e16..f96094a32851 100644
> --- a/fs/iomap/buffered-io.c
> +++ b/fs/iomap/buffered-io.c
> @@ -650,7 +650,7 @@ iomap_set_page_dirty(struct page *page)
> return !TestSetPageDirty(page);
>
> /*
> - * Lock out page->mem_cgroup migration to keep PageDirty
> + * Lock out page's memcg migration to keep PageDirty
> * synchronized with per-memcg dirty page counters.
> */
> lock_page_memcg(page);
> diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h
> index e391e3c56de5..f95c1433461c 100644
> --- a/include/linux/memcontrol.h
> +++ b/include/linux/memcontrol.h
> @@ -343,6 +343,79 @@ struct mem_cgroup {
>
> extern struct mem_cgroup *root_mem_cgroup;
>
> +/*
> + * page_memcg - get the memory cgroup associated with a page
> + * @page: a pointer to the page struct
> + *
> + * Returns a pointer to the memory cgroup associated with the page,
> + * or NULL. This function assumes that the page is known to have a
> + * proper memory cgroup pointer. It's not safe to call this function
> + * against some type of pages, e.g. slab pages or ex-slab pages.
> + *
> + * Any of the following ensures page and memcg binding stability:
> + * - the page lock
> + * - LRU isolation
> + * - lock_page_memcg()
> + * - exclusive reference
> + */
> +static inline struct mem_cgroup *page_memcg(struct page *page)
> +{
> + VM_BUG_ON_PAGE(PageSlab(page), page);
> + return (struct mem_cgroup *)page->memcg_data;
> +}
> +
> +/*
> + * page_memcg_rcu - locklessly get the memory cgroup associated with a page
> + * @page: a pointer to the page struct
> + *
> + * Returns a pointer to the memory cgroup associated with the page,
> + * or NULL. This function assumes that the page is known to have a
> + * proper memory cgroup pointer. It's not safe to call this function
> + * against some type of pages, e.g. slab pages or ex-slab pages.
> + */
> +static inline struct mem_cgroup *page_memcg_rcu(struct page *page)
> +{
> + VM_BUG_ON_PAGE(PageSlab(page), page);
> + WARN_ON_ONCE(!rcu_read_lock_held());
> +
> + return (struct mem_cgroup *)READ_ONCE(page->memcg_data);
> +}
> +
> +/*
> + * page_memcg_check - get the memory cgroup associated with a page
> + * @page: a pointer to the page struct
> + *
> + * Returns a pointer to the memory cgroup associated with the page,
> + * or NULL. This function unlike page_memcg() can take any page
> + * as an argument. It has to be used in cases when it's not known if a page
> + * has an associated memory cgroup pointer or an object cgroups vector.
> + *
> + * Any of the following ensures page and memcg binding stability:
> + * - the page lock
> + * - LRU isolation
> + * - lock_page_memcg()
> + * - exclusive reference
> + */
> +static inline struct mem_cgroup *page_memcg_check(struct page *page)
> +{
> + /*
> + * Because page->memcg_data might be changed asynchronously
> + * for slab pages, READ_ONCE() should be used here.
> + */
> + unsigned long memcg_data = READ_ONCE(page->memcg_data);
> +
> + /*
> + * The lowest bit set means that memcg isn't a valid
> + * memcg pointer, but a obj_cgroups pointer.
> + * In this case the page is shared and doesn't belong
> + * to any specific memory cgroup.
> + */
> + if (memcg_data & 0x1UL)
> + return NULL;
> +
> + return (struct mem_cgroup *)memcg_data;
> +}
> +
> static __always_inline bool memcg_stat_item_in_bytes(int idx)
> {
> if (idx == MEMCG_PERCPU_B)
> @@ -743,15 +816,19 @@ static inline void mod_memcg_state(struct mem_cgroup *memcg,
> static inline void __mod_memcg_page_state(struct page *page,
> int idx, int val)
> {
> - if (page->mem_cgroup)
> - __mod_memcg_state(page->mem_cgroup, idx, val);
> + struct mem_cgroup *memcg = page_memcg(page);
> +
> + if (memcg)
> + __mod_memcg_state(memcg, idx, val);
> }
>
> static inline void mod_memcg_page_state(struct page *page,
> int idx, int val)
> {
> - if (page->mem_cgroup)
> - mod_memcg_state(page->mem_cgroup, idx, val);
> + struct mem_cgroup *memcg = page_memcg(page);
> +
> + if (memcg)
> + mod_memcg_state(memcg, idx, val);
> }
>
> static inline unsigned long lruvec_page_state(struct lruvec *lruvec,
> @@ -834,16 +911,17 @@ static inline void __mod_lruvec_page_state(struct page *page,
> enum node_stat_item idx, int val)
> {
> struct page *head = compound_head(page); /* rmap on tail pages */
> + struct mem_cgroup *memcg = page_memcg(head);
> pg_data_t *pgdat = page_pgdat(page);
> struct lruvec *lruvec;
>
> /* Untracked pages have no memcg, no lruvec. Update only the node */
> - if (!head->mem_cgroup) {
> + if (!memcg) {
> __mod_node_page_state(pgdat, idx, val);
> return;
> }
>
> - lruvec = mem_cgroup_lruvec(head->mem_cgroup, pgdat);
> + lruvec = mem_cgroup_lruvec(memcg, pgdat);
> __mod_lruvec_state(lruvec, idx, val);
> }
>
> @@ -878,8 +956,10 @@ static inline void count_memcg_events(struct mem_cgroup *memcg,
> static inline void count_memcg_page_event(struct page *page,
> enum vm_event_item idx)
> {
> - if (page->mem_cgroup)
> - count_memcg_events(page->mem_cgroup, idx, 1);
> + struct mem_cgroup *memcg = page_memcg(page);
> +
> + if (memcg)
> + count_memcg_events(memcg, idx, 1);
> }
>
> static inline void count_memcg_event_mm(struct mm_struct *mm,
> @@ -941,6 +1021,22 @@ void mem_cgroup_split_huge_fixup(struct page *head);
>
> struct mem_cgroup;
>
> +static inline struct mem_cgroup *page_memcg(struct page *page)
> +{
> + return NULL;
> +}
> +
> +static inline struct mem_cgroup *page_memcg_rcu(struct page *page)
> +{
> + WARN_ON_ONCE(!rcu_read_lock_held());
> + return NULL;
> +}
> +
> +static inline struct mem_cgroup *page_memcg_check(struct page *page)
> +{
> + return NULL;
> +}
> +
> static inline bool mem_cgroup_is_root(struct mem_cgroup *memcg)
> {
> return true;
> @@ -1430,7 +1526,7 @@ static inline void mem_cgroup_track_foreign_dirty(struct page *page,
> if (mem_cgroup_disabled())
> return;
>
> - if (unlikely(&page->mem_cgroup->css != wb->memcg_css))
> + if (unlikely(&page_memcg(page)->css != wb->memcg_css))
> mem_cgroup_track_foreign_dirty_slowpath(page, wb);
> }
>
> diff --git a/include/linux/mm.h b/include/linux/mm.h
> index 27c64d0d7520..6707801b014d 100644
> --- a/include/linux/mm.h
> +++ b/include/linux/mm.h
> @@ -1484,28 +1484,6 @@ static inline void set_page_links(struct page *page, enum zone_type zone,
> #endif
> }
>
> -#ifdef CONFIG_MEMCG
> -static inline struct mem_cgroup *page_memcg(struct page *page)
> -{
> - return page->mem_cgroup;
> -}
> -static inline struct mem_cgroup *page_memcg_rcu(struct page *page)
> -{
> - WARN_ON_ONCE(!rcu_read_lock_held());
> - return READ_ONCE(page->mem_cgroup);
> -}
> -#else
> -static inline struct mem_cgroup *page_memcg(struct page *page)
> -{
> - return NULL;
> -}
> -static inline struct mem_cgroup *page_memcg_rcu(struct page *page)
> -{
> - WARN_ON_ONCE(!rcu_read_lock_held());
> - return NULL;
> -}
> -#endif
> -
> /*
> * Some inline functions in vmstat.h depend on page_zone()
> */
> diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h
> index 5a9238f6caad..80f5d755c037 100644
> --- a/include/linux/mm_types.h
> +++ b/include/linux/mm_types.h
> @@ -199,10 +199,7 @@ struct page {
> atomic_t _refcount;
>
> #ifdef CONFIG_MEMCG
> - union {
> - struct mem_cgroup *mem_cgroup;
> - struct obj_cgroup **obj_cgroups;
> - };
> + unsigned long memcg_data;
> #endif
>
> /*
> diff --git a/include/trace/events/writeback.h b/include/trace/events/writeback.h
> index e7cbccc7c14c..39a40dfb578a 100644
> --- a/include/trace/events/writeback.h
> +++ b/include/trace/events/writeback.h
> @@ -257,7 +257,7 @@ TRACE_EVENT(track_foreign_dirty,
> __entry->ino = inode ? inode->i_ino : 0;
> __entry->memcg_id = wb->memcg_css->id;
> __entry->cgroup_ino = __trace_wb_assign_cgroup(wb);
> - __entry->page_cgroup_ino = cgroup_ino(page->mem_cgroup->css.cgroup);
> + __entry->page_cgroup_ino = cgroup_ino(page_memcg(page)->css.cgroup);
> ),
>
> TP_printk("bdi %s[%llu]: ino=%lu memcg_id=%u cgroup_ino=%lu page_cgroup_ino=%lu",
> diff --git a/kernel/fork.c b/kernel/fork.c
> index 9be04e54de7f..74e5776f2cea 100644
> --- a/kernel/fork.c
> +++ b/kernel/fork.c
> @@ -403,9 +403,10 @@ static int memcg_charge_kernel_stack(struct task_struct *tsk)
>
> for (i = 0; i < THREAD_SIZE / PAGE_SIZE; i++) {
> /*
> - * If memcg_kmem_charge_page() fails, page->mem_cgroup
> - * pointer is NULL, and memcg_kmem_uncharge_page() in
> - * free_thread_stack() will ignore this page.
> + * If memcg_kmem_charge_page() fails, page's
> + * memory cgroup pointer is NULL, and
> + * memcg_kmem_uncharge_page() in free_thread_stack()
> + * will ignore this page.
> */
> ret = memcg_kmem_charge_page(vm->pages[i], GFP_KERNEL,
> 0);
> diff --git a/mm/debug.c b/mm/debug.c
> index ccca576b2899..8a40b3fefbeb 100644
> --- a/mm/debug.c
> +++ b/mm/debug.c
> @@ -182,8 +182,8 @@ void __dump_page(struct page *page, const char *reason)
> pr_warn("page dumped because: %s\n", reason);
>
> #ifdef CONFIG_MEMCG
> - if (!page_poisoned && page->mem_cgroup)
> - pr_warn("page->mem_cgroup:%px\n", page->mem_cgroup);
> + if (!page_poisoned && page->memcg_data)
> + pr_warn("pages's memcg:%lx\n", page->memcg_data);
> #endif
> }
>
> diff --git a/mm/huge_memory.c b/mm/huge_memory.c
> index cba3812a5c3e..999f8a4dcb56 100644
> --- a/mm/huge_memory.c
> +++ b/mm/huge_memory.c
> @@ -470,7 +470,7 @@ pmd_t maybe_pmd_mkwrite(pmd_t pmd, struct vm_area_struct *vma)
> #ifdef CONFIG_MEMCG
> static inline struct deferred_split *get_deferred_split_queue(struct page *page)
> {
> - struct mem_cgroup *memcg = compound_head(page)->mem_cgroup;
> + struct mem_cgroup *memcg = page_memcg(compound_head(page));
> struct pglist_data *pgdat = NODE_DATA(page_to_nid(page));
>
> if (memcg)
> @@ -2766,7 +2766,7 @@ void deferred_split_huge_page(struct page *page)
> {
> struct deferred_split *ds_queue = get_deferred_split_queue(page);
> #ifdef CONFIG_MEMCG
> - struct mem_cgroup *memcg = compound_head(page)->mem_cgroup;
> + struct mem_cgroup *memcg = page_memcg(compound_head(page));
> #endif
> unsigned long flags;
>
> diff --git a/mm/memcontrol.c b/mm/memcontrol.c
> index 888cbc48311f..c2e6c75ed988 100644
> --- a/mm/memcontrol.c
> +++ b/mm/memcontrol.c
> @@ -533,7 +533,7 @@ struct cgroup_subsys_state *mem_cgroup_css_from_page(struct page *page)
> {
> struct mem_cgroup *memcg;
>
> - memcg = page->mem_cgroup;
> + memcg = page_memcg(page);
>
> if (!memcg || !cgroup_subsys_on_dfl(memory_cgrp_subsys))
> memcg = root_mem_cgroup;
> @@ -560,16 +560,7 @@ ino_t page_cgroup_ino(struct page *page)
> unsigned long ino = 0;
>
> rcu_read_lock();
> - memcg = page->mem_cgroup;
> -
> - /*
> - * The lowest bit set means that memcg isn't a valid
> - * memcg pointer, but a obj_cgroups pointer.
> - * In this case the page is shared and doesn't belong
> - * to any specific memory cgroup.
> - */
> - if ((unsigned long) memcg & 0x1UL)
> - memcg = NULL;
> + memcg = page_memcg_check(page);
>
> while (memcg && !(memcg->css.flags & CSS_ONLINE))
> memcg = parent_mem_cgroup(memcg);
> @@ -1050,7 +1041,7 @@ EXPORT_SYMBOL(get_mem_cgroup_from_mm);
> */
> struct mem_cgroup *get_mem_cgroup_from_page(struct page *page)
> {
> - struct mem_cgroup *memcg = page->mem_cgroup;
> + struct mem_cgroup *memcg = page_memcg(page);
>
> if (mem_cgroup_disabled())
> return NULL;
> @@ -1349,7 +1340,7 @@ struct lruvec *mem_cgroup_page_lruvec(struct page *page, struct pglist_data *pgd
> goto out;
> }
>
> - memcg = page->mem_cgroup;
> + memcg = page_memcg(page);
> /*
> * Swapcache readahead pages are added to the LRU - and
> * possibly migrated - before they are charged.
> @@ -2109,7 +2100,7 @@ void mem_cgroup_print_oom_group(struct mem_cgroup *memcg)
> }
>
> /**
> - * lock_page_memcg - lock a page->mem_cgroup binding
> + * lock_page_memcg - lock a page and memcg binding
> * @page: the page
> *
> * This function protects unlocked LRU pages from being moved to
> @@ -2141,7 +2132,7 @@ struct mem_cgroup *lock_page_memcg(struct page *page)
> if (mem_cgroup_disabled())
> return NULL;
> again:
> - memcg = head->mem_cgroup;
> + memcg = page_memcg(head);
> if (unlikely(!memcg))
> return NULL;
>
> @@ -2149,7 +2140,7 @@ struct mem_cgroup *lock_page_memcg(struct page *page)
> return memcg;
>
> spin_lock_irqsave(&memcg->move_lock, flags);
> - if (memcg != head->mem_cgroup) {
> + if (memcg != page_memcg(head)) {
> spin_unlock_irqrestore(&memcg->move_lock, flags);
> goto again;
> }
> @@ -2187,14 +2178,14 @@ void __unlock_page_memcg(struct mem_cgroup *memcg)
> }
>
> /**
> - * unlock_page_memcg - unlock a page->mem_cgroup binding
> + * unlock_page_memcg - unlock a page and memcg binding
> * @page: the page
> */
> void unlock_page_memcg(struct page *page)
> {
> struct page *head = compound_head(page);
>
> - __unlock_page_memcg(head->mem_cgroup);
> + __unlock_page_memcg(page_memcg(head));
> }
> EXPORT_SYMBOL(unlock_page_memcg);
>
> @@ -2884,7 +2875,7 @@ static void cancel_charge(struct mem_cgroup *memcg, unsigned int nr_pages)
>
> static void commit_charge(struct page *page, struct mem_cgroup *memcg)
> {
> - VM_BUG_ON_PAGE(page->mem_cgroup, page);
> + VM_BUG_ON_PAGE(page_memcg(page), page);
> /*
> * Any of the following ensures page->mem_cgroup stability:
> *
> @@ -2893,7 +2884,7 @@ static void commit_charge(struct page *page, struct mem_cgroup *memcg)
> * - lock_page_memcg()
> * - exclusive reference
> */
> - page->mem_cgroup = memcg;
> + page->memcg_data = (unsigned long)memcg;
> }
>
> #ifdef CONFIG_MEMCG_KMEM
> @@ -2908,8 +2899,7 @@ int memcg_alloc_page_obj_cgroups(struct page *page, struct kmem_cache *s,
> if (!vec)
> return -ENOMEM;
>
> - if (cmpxchg(&page->obj_cgroups, NULL,
> - (struct obj_cgroup **) ((unsigned long)vec | 0x1UL)))
> + if (cmpxchg(&page->memcg_data, 0, (unsigned long)vec | 0x1UL))
> kfree(vec);
> else
> kmemleak_not_leak(vec);
> @@ -2920,6 +2910,12 @@ int memcg_alloc_page_obj_cgroups(struct page *page, struct kmem_cache *s,
> /*
> * Returns a pointer to the memory cgroup to which the kernel object is charged.
> *
> + * A passed kernel object can be a slab object or a generic kernel page, so
> + * different mechanisms for getting the memory cgroup pointer should be used.
> + * In certain cases (e.g. kernel stacks or large kmallocs with SLUB) the caller
> + * can not know for sure how the kernel object is implemented.
> + * mem_cgroup_from_obj() can be safely used in such cases.
> + *
> * The caller must ensure the memcg lifetime, e.g. by taking rcu_read_lock(),
> * cgroup_mutex, etc.
> */
> @@ -2932,17 +2928,6 @@ struct mem_cgroup *mem_cgroup_from_obj(void *p)
>
> page = virt_to_head_page(p);
>
> - /*
> - * If page->mem_cgroup is set, it's either a simple mem_cgroup pointer
> - * or a pointer to obj_cgroup vector. In the latter case the lowest
> - * bit of the pointer is set.
> - * The page->mem_cgroup pointer can be asynchronously changed
> - * from NULL to (obj_cgroup_vec | 0x1UL), but can't be changed
> - * from a valid memcg pointer to objcg vector or back.
> - */
> - if (!page->mem_cgroup)
> - return NULL;
> -
> /*
> * Slab objects are accounted individually, not per-page.
> * Memcg membership data for each individual object is saved in
> @@ -2960,8 +2945,14 @@ struct mem_cgroup *mem_cgroup_from_obj(void *p)
> return NULL;
> }
>
> - /* All other pages use page->mem_cgroup */
> - return page->mem_cgroup;
> + /*
> + * page_memcg_check() is used here, because page_has_obj_cgroups()
> + * check above could fail because the object cgroups vector wasn't set
> + * at that moment, but it can be set concurrently.
> + * page_memcg_check(page) will guarantee that a proper memory
> + * cgroup pointer or NULL will be returned.
> + */
> + return page_memcg_check(page);
> }
>
> __always_inline struct obj_cgroup *get_obj_cgroup_from_current(void)
> @@ -3099,7 +3090,7 @@ int __memcg_kmem_charge_page(struct page *page, gfp_t gfp, int order)
> if (memcg && !mem_cgroup_is_root(memcg)) {
> ret = __memcg_kmem_charge(memcg, gfp, 1 << order);
> if (!ret) {
> - page->mem_cgroup = memcg;
> + page->memcg_data = (unsigned long)memcg;
> __SetPageKmemcg(page);
> return 0;
> }
> @@ -3115,7 +3106,7 @@ int __memcg_kmem_charge_page(struct page *page, gfp_t gfp, int order)
> */
> void __memcg_kmem_uncharge_page(struct page *page, int order)
> {
> - struct mem_cgroup *memcg = page->mem_cgroup;
> + struct mem_cgroup *memcg = page_memcg(page);
> unsigned int nr_pages = 1 << order;
>
> if (!memcg)
> @@ -3123,7 +3114,7 @@ void __memcg_kmem_uncharge_page(struct page *page, int order)
>
> VM_BUG_ON_PAGE(mem_cgroup_is_root(memcg), page);
> __memcg_kmem_uncharge(memcg, nr_pages);
> - page->mem_cgroup = NULL;
> + page->memcg_data = 0;
> css_put(&memcg->css);
>
> /* slab pages do not have PageKmemcg flag set */
> @@ -3274,7 +3265,7 @@ void obj_cgroup_uncharge(struct obj_cgroup *objcg, size_t size)
> */
> void mem_cgroup_split_huge_fixup(struct page *head)
> {
> - struct mem_cgroup *memcg = head->mem_cgroup;
> + struct mem_cgroup *memcg = page_memcg(head);
> int i;
>
> if (mem_cgroup_disabled())
> @@ -3282,7 +3273,7 @@ void mem_cgroup_split_huge_fixup(struct page *head)
>
> for (i = 1; i < HPAGE_PMD_NR; i++) {
> css_get(&memcg->css);
> - head[i].mem_cgroup = memcg;
> + head[i].memcg_data = (unsigned long)memcg;
> }
> }
> #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
> @@ -4658,7 +4649,7 @@ void mem_cgroup_wb_stats(struct bdi_writeback *wb, unsigned long *pfilepages,
> void mem_cgroup_track_foreign_dirty_slowpath(struct page *page,
> struct bdi_writeback *wb)
> {
> - struct mem_cgroup *memcg = page->mem_cgroup;
> + struct mem_cgroup *memcg = page_memcg(page);
> struct memcg_cgwb_frn *frn;
> u64 now = get_jiffies_64();
> u64 oldest_at = now;
> @@ -5627,14 +5618,14 @@ static int mem_cgroup_move_account(struct page *page,
>
> /*
> * Prevent mem_cgroup_migrate() from looking at
> - * page->mem_cgroup of its source page while we change it.
> + * page's memory cgroup of its source page while we change it.
> */
> ret = -EBUSY;
> if (!trylock_page(page))
> goto out;
>
> ret = -EINVAL;
> - if (page->mem_cgroup != from)
> + if (page_memcg(page) != from)
> goto out_unlock;
>
> pgdat = page_pgdat(page);
> @@ -5689,13 +5680,13 @@ static int mem_cgroup_move_account(struct page *page,
> /*
> * All state has been migrated, let's switch to the new memcg.
> *
> - * It is safe to change page->mem_cgroup here because the page
> + * It is safe to change page's memcg here because the page
> * is referenced, charged, isolated, and locked: we can't race
> * with (un)charging, migration, LRU putback, or anything else
> - * that would rely on a stable page->mem_cgroup.
> + * that would rely on a stable page's memory cgroup.
> *
> * Note that lock_page_memcg is a memcg lock, not a page lock,
> - * to save space. As soon as we switch page->mem_cgroup to a
> + * to save space. As soon as we switch page's memory cgroup to a
> * new memcg that isn't locked, the above state can change
> * concurrently again. Make sure we're truly done with it.
> */
> @@ -5704,7 +5695,7 @@ static int mem_cgroup_move_account(struct page *page,
> css_get(&to->css);
> css_put(&from->css);
>
> - page->mem_cgroup = to;
> + page->memcg_data = (unsigned long)to;
>
> __unlock_page_memcg(from);
>
> @@ -5770,7 +5761,7 @@ static enum mc_target_type get_mctgt_type(struct vm_area_struct *vma,
> * mem_cgroup_move_account() checks the page is valid or
> * not under LRU exclusion.
> */
> - if (page->mem_cgroup == mc.from) {
> + if (page_memcg(page) == mc.from) {
> ret = MC_TARGET_PAGE;
> if (is_device_private_page(page))
> ret = MC_TARGET_DEVICE;
> @@ -5814,7 +5805,7 @@ static enum mc_target_type get_mctgt_type_thp(struct vm_area_struct *vma,
> VM_BUG_ON_PAGE(!page || !PageHead(page), page);
> if (!(mc.flags & MOVE_ANON))
> return ret;
> - if (page->mem_cgroup == mc.from) {
> + if (page_memcg(page) == mc.from) {
> ret = MC_TARGET_PAGE;
> if (target) {
> get_page(page);
> @@ -6760,12 +6751,12 @@ int mem_cgroup_charge(struct page *page, struct mm_struct *mm, gfp_t gfp_mask)
> /*
> * Every swap fault against a single page tries to charge the
> * page, bail as early as possible. shmem_unuse() encounters
> - * already charged pages, too. page->mem_cgroup is protected
> - * by the page lock, which serializes swap cache removal, which
> - * in turn serializes uncharging.
> + * already charged pages, too. page and memcg binding is
> + * protected by the page lock, which serializes swap cache
> + * removal, which in turn serializes uncharging.
> */
> VM_BUG_ON_PAGE(!PageLocked(page), page);
> - if (compound_head(page)->mem_cgroup)
> + if (page_memcg(compound_head(page)))
> goto out;
>
> id = lookup_swap_cgroup_id(ent);
> @@ -6849,21 +6840,21 @@ static void uncharge_page(struct page *page, struct uncharge_gather *ug)
>
> VM_BUG_ON_PAGE(PageLRU(page), page);
>
> - if (!page->mem_cgroup)
> + if (!page_memcg(page))
> return;
>
> /*
> * Nobody should be changing or seriously looking at
> - * page->mem_cgroup at this point, we have fully
> + * page_memcg(page) at this point, we have fully
> * exclusive access to the page.
> */
>
> - if (ug->memcg != page->mem_cgroup) {
> + if (ug->memcg != page_memcg(page)) {
> if (ug->memcg) {
> uncharge_batch(ug);
> uncharge_gather_clear(ug);
> }
> - ug->memcg = page->mem_cgroup;
> + ug->memcg = page_memcg(page);
>
> /* pairs with css_put in uncharge_batch */
> css_get(&ug->memcg->css);
> @@ -6880,7 +6871,7 @@ static void uncharge_page(struct page *page, struct uncharge_gather *ug)
> }
>
> ug->dummy_page = page;
> - page->mem_cgroup = NULL;
> + page->memcg_data = 0;
> css_put(&ug->memcg->css);
> }
>
> @@ -6923,7 +6914,7 @@ void mem_cgroup_uncharge(struct page *page)
> return;
>
> /* Don't touch page->lru of any random page, pre-check: */
> - if (!page->mem_cgroup)
> + if (!page_memcg(page))
> return;
>
> uncharge_gather_clear(&ug);
> @@ -6973,11 +6964,11 @@ void mem_cgroup_migrate(struct page *oldpage, struct page *newpage)
> return;
>
> /* Page cache replacement: new page already charged? */
> - if (newpage->mem_cgroup)
> + if (page_memcg(newpage))
> return;
>
> /* Swapcache readahead pages can get replaced before being charged */
> - memcg = oldpage->mem_cgroup;
> + memcg = page_memcg(oldpage);
> if (!memcg)
> return;
>
> @@ -7172,7 +7163,7 @@ void mem_cgroup_swapout(struct page *page, swp_entry_t entry)
> if (cgroup_subsys_on_dfl(memory_cgrp_subsys))
> return;
>
> - memcg = page->mem_cgroup;
> + memcg = page_memcg(page);
>
> /* Readahead page, never charged */
> if (!memcg)
> @@ -7193,7 +7184,7 @@ void mem_cgroup_swapout(struct page *page, swp_entry_t entry)
> VM_BUG_ON_PAGE(oldid, page);
> mod_memcg_state(swap_memcg, MEMCG_SWAP, nr_entries);
>
> - page->mem_cgroup = NULL;
> + page->memcg_data = 0;
>
> if (!mem_cgroup_is_root(memcg))
> page_counter_uncharge(&memcg->memory, nr_entries);
> @@ -7236,7 +7227,7 @@ int mem_cgroup_try_charge_swap(struct page *page, swp_entry_t entry)
> if (!cgroup_subsys_on_dfl(memory_cgrp_subsys))
> return 0;
>
> - memcg = page->mem_cgroup;
> + memcg = page_memcg(page);
>
> /* Readahead page, never charged */
> if (!memcg)
> @@ -7317,7 +7308,7 @@ bool mem_cgroup_swap_full(struct page *page)
> if (cgroup_memory_noswap || !cgroup_subsys_on_dfl(memory_cgrp_subsys))
> return false;
>
> - memcg = page->mem_cgroup;
> + memcg = page_memcg(page);
> if (!memcg)
> return false;
>
> diff --git a/mm/page_alloc.c b/mm/page_alloc.c
> index eb0962976a0e..e9f0fe4a143e 100644
> --- a/mm/page_alloc.c
> +++ b/mm/page_alloc.c
> @@ -1057,7 +1057,7 @@ static inline bool page_expected_state(struct page *page,
> if (unlikely((unsigned long)page->mapping |
> page_ref_count(page) |
> #ifdef CONFIG_MEMCG
> - (unsigned long)page->mem_cgroup |
> + (unsigned long)page_memcg(page) |
> #endif
> (page->flags & check_flags)))
> return false;
> @@ -1082,7 +1082,7 @@ static const char *page_bad_reason(struct page *page, unsigned long flags)
> bad_reason = "PAGE_FLAGS_CHECK_AT_FREE flag(s) set";
> }
> #ifdef CONFIG_MEMCG
> - if (unlikely(page->mem_cgroup))
> + if (unlikely(page_memcg(page)))
> bad_reason = "page still charged to cgroup";
> #endif
> return bad_reason;
> diff --git a/mm/page_io.c b/mm/page_io.c
> index 433df1263349..9bca17ecc4df 100644
> --- a/mm/page_io.c
> +++ b/mm/page_io.c
> @@ -291,12 +291,14 @@ static inline void count_swpout_vm_event(struct page *page)
> static void bio_associate_blkg_from_page(struct bio *bio, struct page *page)
> {
> struct cgroup_subsys_state *css;
> + struct mem_cgroup *memcg;
>
> - if (!page->mem_cgroup)
> + memcg = page_memcg(page);
> + if (!memcg)
> return;
>
> rcu_read_lock();
> - css = cgroup_e_css(page->mem_cgroup->css.cgroup, &io_cgrp_subsys);
> + css = cgroup_e_css(memcg->css.cgroup, &io_cgrp_subsys);
> bio_associate_blkg_from_css(bio, css);
> rcu_read_unlock();
> }
> diff --git a/mm/slab.h b/mm/slab.h
> index 4a24e1702923..5ac89260f329 100644
> --- a/mm/slab.h
> +++ b/mm/slab.h
> @@ -242,18 +242,17 @@ static inline bool kmem_cache_debug_flags(struct kmem_cache *s, slab_flags_t fla
> static inline struct obj_cgroup **page_obj_cgroups(struct page *page)
> {
> /*
> - * page->mem_cgroup and page->obj_cgroups are sharing the same
> + * Page's memory cgroup and obj_cgroups vector are sharing the same
> * space. To distinguish between them in case we don't know for sure
> * that the page is a slab page (e.g. page_cgroup_ino()), let's
> * always set the lowest bit of obj_cgroups.
> */
> - return (struct obj_cgroup **)
> - ((unsigned long)page->obj_cgroups & ~0x1UL);
> + return (struct obj_cgroup **)(page->memcg_data & ~0x1UL);
> }
>
> static inline bool page_has_obj_cgroups(struct page *page)
> {
> - return ((unsigned long)page->obj_cgroups & 0x1UL);
> + return page->memcg_data & 0x1UL;
> }
>
> int memcg_alloc_page_obj_cgroups(struct page *page, struct kmem_cache *s,
> @@ -262,7 +261,7 @@ int memcg_alloc_page_obj_cgroups(struct page *page, struct kmem_cache *s,
> static inline void memcg_free_page_obj_cgroups(struct page *page)
> {
> kfree(page_obj_cgroups(page));
> - page->obj_cgroups = NULL;
> + page->memcg_data = 0;
> }
>
> static inline size_t obj_full_size(struct kmem_cache *s)
> diff --git a/mm/workingset.c b/mm/workingset.c
> index 8ed8e6296d8c..ef563f94a0e3 100644
> --- a/mm/workingset.c
> +++ b/mm/workingset.c
> @@ -257,7 +257,7 @@ void *workingset_eviction(struct page *page, struct mem_cgroup *target_memcg)
> struct lruvec *lruvec;
> int memcgid;
>
> - /* Page is fully exclusive and pins page->mem_cgroup */
> + /* Page is fully exclusive and pins page's memory cgroup pointer */
> VM_BUG_ON_PAGE(PageLRU(page), page);
> VM_BUG_ON_PAGE(page_count(page), page);
> VM_BUG_ON_PAGE(!PageLocked(page), page);
> --
> 2.26.2
--
Michal Hocko
SUSE Labs
