This can be used when there is concurrency on the struct page. If cpupid
is not in page flags, it can use a store.
Use this to replace the open-coded reset in wp_page_reuse.
Signed-off-by: Nicholas Piggin <npiggin@xxxxxxxxx>
---
include/linux/mm.h | 15 +++++++++++++++
mm/memory.c | 2 +-
2 files changed, 16 insertions(+), 1 deletion(-)
diff --git a/include/linux/mm.h b/include/linux/mm.h
index 086ab710383f..2524553ea898 100644
--- a/include/linux/mm.h
+++ b/include/linux/mm.h
@@ -1400,6 +1400,11 @@ static inline void __page_cpupid_reset_last(struct page *page)
{
page->_last_cpupid = -1 & LAST_CPUPID_MASK;
}
+
+static inline void page_cpupid_reset_last(struct page *page)
+{
+ WRITE_ONCE(page->_last_cpupid, -1 & LAST_CPUPID_MASK);
+}
#else
static inline int page_cpupid_last(struct page *page)
{
@@ -1412,6 +1417,12 @@ static inline void __page_cpupid_reset_last(struct page *page)
{
page->flags |= LAST_CPUPID_MASK << LAST_CPUPID_PGSHIFT;
}
+
+static inline void page_cpupid_reset_last(struct page *page)
+{
+ page_cpupid_xchg_last(page, (1 << LAST_CPUPID_SHIFT) - 1);
+}
+
#endif /* LAST_CPUPID_NOT_IN_PAGE_FLAGS */
#else /* !CONFIG_NUMA_BALANCING */
static inline int page_cpupid_xchg_last(struct page *page, int cpupid)
@@ -1453,6 +1464,10 @@ static inline void __page_cpupid_reset_last(struct page *page)
{
}
+static inline void page_cpupid_reset_last(struct page *page)
+{
+}
+
static inline bool cpupid_match_pid(struct task_struct *task, int cpupid)
{
return false;
diff --git a/mm/memory.c b/mm/memory.c
index 5efa07fb6cdc..bb37dc0f4dac 100644
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -2804,7 +2804,7 @@ static inline void wp_page_reuse(struct vm_fault *vmf)
* unrelated process.
*/
if (page)
- page_cpupid_xchg_last(page, (1 << LAST_CPUPID_SHIFT) - 1);
+ page_cpupid_reset_last(page);
flush_cache_page(vma, vmf->address, pte_pfn(vmf->orig_pte));
entry = pte_mkyoung(vmf->orig_pte);
--
2.23.0
