process_huge_pages() (and the subsequent call to process_subpage()) translate between base-addr[index], struct page *, first when calling clear/copy_subpage() which then translates to the parameters needed in clear/copy_user_highpage(). There's no runtime cost in doing this, but it's unnecessary complexity for something that only has two users. Accordingly, fold process_huge_page() back in its callers. Link: https://lore.kernel.org/lkml/20220606202109.1306034-1-ankur.a.arora@xxxxxxxxxx/ Suggested-by: Linus Torvalds <torvalds@xxxxxxxxxxxxxxxxxxxx> Signed-off-by: Ankur Arora <ankur.a.arora@xxxxxxxxxx> --- mm/memory.c | 126 ++++++++++++++++++++++++++++++---------------------- 1 file changed, 73 insertions(+), 53 deletions(-) diff --git a/mm/memory.c b/mm/memory.c index f456f3b5049c..d54bc27a35ca 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -5629,63 +5629,13 @@ EXPORT_SYMBOL(__might_fault); #endif #if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_HUGETLBFS) -/* - * Process all subpages of the specified huge page with the specified - * operation. The target subpage will be processed last to keep its - * cache lines hot. - */ -static inline void process_huge_page( - unsigned long addr_hint, unsigned int pages_per_huge_page, - void (*process_subpage)(unsigned long addr, int idx, void *arg), - void *arg) -{ - int i, n, base, l; - unsigned long addr = addr_hint & - ~(((unsigned long)pages_per_huge_page << PAGE_SHIFT) - 1); - - /* Process target subpage last to keep its cache lines hot */ - might_sleep(); - n = (addr_hint - addr) / PAGE_SIZE; - if (2 * n <= pages_per_huge_page) { - /* If target subpage in first half of huge page */ - base = 0; - l = n; - /* Process subpages at the end of huge page */ - for (i = pages_per_huge_page - 1; i >= 2 * n; i--) { - cond_resched(); - process_subpage(addr + i * PAGE_SIZE, i, arg); - } - } else { - /* If target subpage in second half of huge page */ - base = pages_per_huge_page - 2 * (pages_per_huge_page - n); - l = pages_per_huge_page - n; - /* Process subpages at the begin of huge page */ - for (i = 0; i < base; i++) { - cond_resched(); - process_subpage(addr + i * PAGE_SIZE, i, arg); - } - } - /* - * Process remaining subpages in left-right-left-right pattern - * towards the target subpage - */ - for (i = 0; i < l; i++) { - int left_idx = base + i; - int right_idx = base + 2 * l - 1 - i; - - cond_resched(); - process_subpage(addr + left_idx * PAGE_SIZE, left_idx, arg); - cond_resched(); - process_subpage(addr + right_idx * PAGE_SIZE, right_idx, arg); - } -} static void clear_gigantic_page(struct page *page, unsigned long addr, unsigned int pages_per_huge_page) { int i; - struct page *p; + struct page *p = page; might_sleep(); for (i = 0; i < pages_per_huge_page; i++) { @@ -5707,13 +5657,48 @@ void clear_huge_page(struct page *page, { unsigned long addr = addr_hint & ~(((unsigned long)pages_per_huge_page << PAGE_SHIFT) - 1); + int i, n, base, l; if (unlikely(pages_per_huge_page > MAX_ORDER_NR_PAGES)) { clear_gigantic_page(page, addr, pages_per_huge_page); return; } - process_huge_page(addr_hint, pages_per_huge_page, clear_subpage, page); + /* Process target subpage last to keep its cache lines hot */ + might_sleep(); + n = (addr_hint - addr) / PAGE_SIZE; + if (2 * n <= pages_per_huge_page) { + /* If target subpage in first half of huge page */ + base = 0; + l = n; + /* Process subpages at the end of huge page */ + for (i = pages_per_huge_page - 1; i >= 2 * n; i--) { + cond_resched(); + clear_subpage(addr + i * PAGE_SIZE, i, (void *)page); + } + } else { + /* If target subpage in second half of huge page */ + base = pages_per_huge_page - 2 * (pages_per_huge_page - n); + l = pages_per_huge_page - n; + /* Process subpages at the begin of huge page */ + for (i = 0; i < base; i++) { + cond_resched(); + clear_subpage(addr + i * PAGE_SIZE, i, (void *)page); + } + } + /* + * Process remaining subpages in left-right-left-right pattern + * towards the target subpage + */ + for (i = 0; i < l; i++) { + int left_idx = base + i; + int right_idx = base + 2 * l - 1 - i; + + cond_resched(); + clear_subpage(addr + left_idx * PAGE_SIZE, left_idx, (void *)page); + cond_resched(); + clear_subpage(addr + right_idx * PAGE_SIZE, right_idx, (void *)page); + } } static void copy_user_gigantic_page(struct page *dst, struct page *src, @@ -5759,6 +5744,7 @@ void copy_user_huge_page(struct page *dst, struct page *src, .src = src, .vma = vma, }; + int i, n, base, l; if (unlikely(pages_per_huge_page > MAX_ORDER_NR_PAGES)) { copy_user_gigantic_page(dst, src, addr, vma, @@ -5766,7 +5752,41 @@ void copy_user_huge_page(struct page *dst, struct page *src, return; } - process_huge_page(addr_hint, pages_per_huge_page, copy_subpage, &arg); + /* Process target subpage last to keep its cache lines hot */ + might_sleep(); + n = (addr_hint - addr) / PAGE_SIZE; + if (2 * n <= pages_per_huge_page) { + /* If target subpage in first half of huge page */ + base = 0; + l = n; + /* Process subpages at the end of huge page */ + for (i = pages_per_huge_page - 1; i >= 2 * n; i--) { + cond_resched(); + copy_subpage(addr + i * PAGE_SIZE, i, &arg); + } + } else { + /* If target subpage in second half of huge page */ + base = pages_per_huge_page - 2 * (pages_per_huge_page - n); + l = pages_per_huge_page - n; + /* Process subpages at the begin of huge page */ + for (i = 0; i < base; i++) { + cond_resched(); + copy_subpage(addr + i * PAGE_SIZE, i, &arg); + } + } + /* + * Process remaining subpages in left-right-left-right pattern + * towards the target subpage + */ + for (i = 0; i < l; i++) { + int left_idx = base + i; + int right_idx = base + 2 * l - 1 - i; + + cond_resched(); + copy_subpage(addr + left_idx * PAGE_SIZE, left_idx, &arg); + cond_resched(); + copy_subpage(addr + right_idx * PAGE_SIZE, right_idx, &arg); + } } long copy_huge_page_from_user(struct page *dst_page, -- 2.31.1
