struct acctmem is used for MEMCG_DATA_KMEM allocations. We're still a
bit loose with our casting to folios instead of acctmem, but that's a
problem to solve later. The build asserts ensure that this carelessness
doesn't cause any new bugs today.
Signed-off-by: Matthew Wilcox (Oracle) <willy@xxxxxxxxxxxxx>
---
include/linux/memcontrol.h | 24 ++++++++++++++++++++++++
include/linux/mm_types.h | 6 +++---
mm/memcontrol.c | 7 ++++---
mm/page_alloc.c | 4 ++--
mm/page_owner.c | 2 +-
mm/slab.h | 2 +-
6 files changed, 35 insertions(+), 10 deletions(-)
diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h
index a787080f814f..19ee98abea0f 100644
--- a/include/linux/memcontrol.h
+++ b/include/linux/memcontrol.h
@@ -30,6 +30,30 @@ struct page;
struct mm_struct;
struct kmem_cache;
+/*
+ * For now, this data structure overlays struct page. Eventually it
+ * will be separately allocated and become a memdesc type of its own
+ * like slab and ptdesc. memcg_data is only valid on the first page
+ * of an allocation, but that allocation might not be compound!
+ */
+struct acctmem {
+ unsigned long __page_flags;
+ unsigned long __padding[5];
+ unsigned int ___padding[2];
+ unsigned long memcg_data;
+};
+#ifdef CONFIG_MEMCG
+static_assert(offsetof(struct page, __acct_memcg_data) ==
+ offsetof(struct acctmem, memcg_data));
+static_assert(offsetof(struct folio, memcg_data) ==
+ offsetof(struct acctmem, memcg_data));
+static_assert(sizeof(struct acctmem) <= sizeof(struct page));
+#endif
+
+#define page_acctmem(_page) (_Generic((_page), \
+ const struct page *: (const struct acctmem *)(_page), \
+ struct page *: (struct acctmem *)(_page)))
+
/* Cgroup-specific page state, on top of universal node page state */
enum memcg_stat_item {
MEMCG_SWAP = NR_VM_NODE_STAT_ITEMS,
diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h
index 2b694f9a4518..274b125df0df 100644
--- a/include/linux/mm_types.h
+++ b/include/linux/mm_types.h
@@ -181,7 +181,7 @@ struct page {
atomic_t _refcount;
#ifdef CONFIG_MEMCG
- unsigned long memcg_data;
+ unsigned long __acct_memcg_data;
#elif defined(CONFIG_SLAB_OBJ_EXT)
unsigned long _unused_slab_obj_exts;
#endif
@@ -410,7 +410,7 @@ FOLIO_MATCH(private, private);
FOLIO_MATCH(_mapcount, _mapcount);
FOLIO_MATCH(_refcount, _refcount);
#ifdef CONFIG_MEMCG
-FOLIO_MATCH(memcg_data, memcg_data);
+FOLIO_MATCH(__acct_memcg_data, memcg_data);
#endif
#if defined(WANT_PAGE_VIRTUAL)
FOLIO_MATCH(virtual, virtual);
@@ -499,7 +499,7 @@ TABLE_MATCH(rcu_head, pt_rcu_head);
TABLE_MATCH(page_type, __page_type);
TABLE_MATCH(_refcount, __page_refcount);
#ifdef CONFIG_MEMCG
-TABLE_MATCH(memcg_data, pt_memcg_data);
+TABLE_MATCH(__acct_memcg_data, pt_memcg_data);
#endif
#undef TABLE_MATCH
static_assert(sizeof(struct ptdesc) <= sizeof(struct page));
diff --git a/mm/memcontrol.c b/mm/memcontrol.c
index 506439a5dcfe..89c9d206c209 100644
--- a/mm/memcontrol.c
+++ b/mm/memcontrol.c
@@ -2661,6 +2661,7 @@ static int obj_cgroup_charge_pages(struct obj_cgroup *objcg, gfp_t gfp,
*/
int __memcg_kmem_charge_page(struct page *page, gfp_t gfp, int order)
{
+ struct acctmem *acctmem = page_acctmem(page);
struct obj_cgroup *objcg;
int ret = 0;
@@ -2669,7 +2670,7 @@ int __memcg_kmem_charge_page(struct page *page, gfp_t gfp, int order)
ret = obj_cgroup_charge_pages(objcg, gfp, 1 << order);
if (!ret) {
obj_cgroup_get(objcg);
- page->memcg_data = (unsigned long)objcg |
+ acctmem->memcg_data = (unsigned long)objcg |
MEMCG_DATA_KMEM;
return 0;
}
@@ -3039,7 +3040,7 @@ void __memcg_slab_free_hook(struct kmem_cache *s, struct slab *slab,
*/
void split_page_memcg(struct page *first, int order)
{
- unsigned long memcg_data = first->memcg_data;
+ unsigned long memcg_data = page_acctmem(first)->memcg_data;
struct obj_cgroup *objcg;
int i;
unsigned int nr = 1 << order;
@@ -3052,7 +3053,7 @@ void split_page_memcg(struct page *first, int order)
objcg = (void *)(memcg_data & ~OBJEXTS_FLAGS_MASK);
for (i = 1; i < nr; i++)
- first[i].memcg_data = memcg_data;
+ page_acctmem(first + i)->memcg_data = memcg_data;
obj_cgroup_get_many(objcg, nr - 1);
}
diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index 5523654c9759..07d9302882b2 100644
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -870,7 +870,7 @@ static inline bool page_expected_state(struct page *page,
if (unlikely((unsigned long)page->mapping |
page_ref_count(page) |
#ifdef CONFIG_MEMCG
- page->memcg_data |
+ page_acctmem(page)->memcg_data |
#endif
#ifdef CONFIG_PAGE_POOL
((page->pp_magic & ~0x3UL) == PP_SIGNATURE) |
@@ -898,7 +898,7 @@ static const char *page_bad_reason(struct page *page, unsigned long flags)
bad_reason = "PAGE_FLAGS_CHECK_AT_FREE flag(s) set";
}
#ifdef CONFIG_MEMCG
- if (unlikely(page->memcg_data))
+ if (unlikely(page_acctmem(page)->memcg_data))
bad_reason = "page still charged to cgroup";
#endif
#ifdef CONFIG_PAGE_POOL
diff --git a/mm/page_owner.c b/mm/page_owner.c
index 2d6360eaccbb..71e183f8988b 100644
--- a/mm/page_owner.c
+++ b/mm/page_owner.c
@@ -506,7 +506,7 @@ static inline int print_page_owner_memcg(char *kbuf, size_t count, int ret,
char name[80];
rcu_read_lock();
- memcg_data = READ_ONCE(page->memcg_data);
+ memcg_data = READ_ONCE(page_acctmem(page)->memcg_data);
if (!memcg_data)
goto out_unlock;
diff --git a/mm/slab.h b/mm/slab.h
index 632fedd71fea..ee9ab84f7c4d 100644
--- a/mm/slab.h
+++ b/mm/slab.h
@@ -103,7 +103,7 @@ SLAB_MATCH(flags, __page_flags);
SLAB_MATCH(compound_head, slab_cache); /* Ensure bit 0 is clear */
SLAB_MATCH(_refcount, __page_refcount);
#ifdef CONFIG_MEMCG
-SLAB_MATCH(memcg_data, obj_exts);
+SLAB_MATCH(__acct_memcg_data, obj_exts);
#elif defined(CONFIG_SLAB_OBJ_EXT)
SLAB_MATCH(_unused_slab_obj_exts, obj_exts);
#endif
--
2.43.0
