[PATCH] mpage clear

[Huang Ying <ying.huang@xxxxxxxxx>]

---
 mm/memory.c | 70 ++++++++++++++++++++++++++++++++++++++++++++++++++---
 1 file changed, 67 insertions(+), 3 deletions(-)

diff --git a/mm/memory.c b/mm/memory.c
index 3ccee51adfbb..1fdc548c4275 100644
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -6769,6 +6769,68 @@ static inline int process_huge_page(
 	return 0;
 }
 
+#define MPAGE_NRPAGES	(1<<4)
+#define MPAGE_SIZE	(PAGE_SIZE * MPAGE_NRPAGES)
+static inline int clear_huge_page(
+	unsigned long addr_hint, unsigned int nr_pages,
+	int (*process_subpage)(unsigned long addr, int idx, void *arg),
+	void *arg)
+{
+	int i, n, base, l, ret;
+	unsigned long addr = addr_hint &
+		~(((unsigned long)nr_pages << PAGE_SHIFT) - 1);
+	unsigned long nr_mpages = ((unsigned long)nr_pages << PAGE_SHIFT) / MPAGE_SIZE;
+
+	/* Process target subpage last to keep its cache lines hot */
+	might_sleep();
+	n = (addr_hint - addr) / MPAGE_SIZE;
+	if (2 * n <= nr_mpages) {
+		/* If target subpage in first half of huge page */
+		base = 0;
+		l = n;
+		/* Process subpages at the end of huge page */
+		for (i = nr_mpages - 1; i >= 2 * n; i--) {
+			cond_resched();
+			ret = process_subpage(addr + i * MPAGE_SIZE,
+					      i * MPAGE_NRPAGES, arg);
+			if (ret)
+				return ret;
+		}
+	} else {
+		/* If target subpage in second half of huge page */
+		base = nr_mpages - 2 * (nr_mpages - n);
+		l = nr_mpages - n;
+		/* Process subpages at the begin of huge page */
+		for (i = 0; i < base; i++) {
+			cond_resched();
+			ret = process_subpage(addr + i * MPAGE_SIZE,
+					      i * MPAGE_NRPAGES, arg);
+			if (ret)
+				return ret;
+		}
+	}
+	/*
+	 * Process remaining subpages in left-right-left-right pattern
+	 * towards the target subpage
+	 */
+	for (i = 0; i < l; i++) {
+		int left_idx = base + i;
+		int right_idx = base + 2 * l - 1 - i;
+
+		cond_resched();
+		ret = process_subpage(addr + left_idx * MPAGE_SIZE,
+				      left_idx * MPAGE_NRPAGES, arg);
+		if (ret)
+			return ret;
+		cond_resched();
+		ret = process_subpage(addr + right_idx * MPAGE_SIZE,
+				      right_idx * MPAGE_NRPAGES, arg);
+		if (ret)
+			return ret;
+	}
+	return 0;
+}
+
 static void clear_gigantic_page(struct folio *folio, unsigned long addr,
 				unsigned int nr_pages)
 {
@@ -6784,8 +6846,10 @@ static void clear_gigantic_page(struct folio *folio, unsigned long addr,
 static int clear_subpage(unsigned long addr, int idx, void *arg)
 {
 	struct folio *folio = arg;
+	int i;
 
-	clear_user_highpage(folio_page(folio, idx), addr);
+	for (i = 0; i < MPAGE_NRPAGES; i++)
+		clear_user_highpage(folio_page(folio, idx + i), addr + i * PAGE_SIZE);
 	return 0;
 }
 
@@ -6798,10 +6862,10 @@ void folio_zero_user(struct folio *folio, unsigned long addr_hint)
 {
 	unsigned int nr_pages = folio_nr_pages(folio);
 
-	if (unlikely(nr_pages > MAX_ORDER_NR_PAGES))
+	if (unlikely(nr_pages != HPAGE_PMD_NR))
 		clear_gigantic_page(folio, addr_hint, nr_pages);
 	else
-		process_huge_page(addr_hint, nr_pages, clear_subpage, folio);
+		clear_huge_page(addr_hint, nr_pages, clear_subpage, folio);
 }
 
 static int copy_user_gigantic_page(struct folio *dst, struct folio *src,
-- 
2.39.2