We will soon support CoWing large folios, so will need support for
copying a contiguous range of pages in the case where there is a source
folio. Therefore, split __wp_page_copy_user() into 2 variants:
__wp_page_copy_user_pfn() copies a single pfn to a destination page.
This is used when CoWing from a source without a folio, and is always
only a single page copy.
__wp_page_copy_user_range() copies a range of pages from source to
destination and is used when the source has an underlying folio. For now
it is only used to copy a single page, but this will change in a future
commit.
In both cases, kmsan_copy_page_meta() is moved into these helper
functions so that the caller does not need to be concerned with calling
it multiple times for the range case.
No functional changes intended.
Signed-off-by: Ryan Roberts <ryan.roberts@xxxxxxx>
---
mm/memory.c | 41 +++++++++++++++++++++++++++++------------
1 file changed, 29 insertions(+), 12 deletions(-)
diff --git a/mm/memory.c b/mm/memory.c
index 7e2af54fe2e0..f2b7cfb2efc0 100644
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -2786,14 +2786,34 @@ static inline int pte_unmap_same(struct vm_fault *vmf)
return same;
}
+/*
+ * Return:
+ * 0: copied succeeded
+ * -EHWPOISON: copy failed due to hwpoison in source page
+ */
+static inline int __wp_page_copy_user_range(struct page *dst, struct page *src,
+ int nr, unsigned long addr,
+ struct vm_area_struct *vma)
+{
+ for (; nr != 0; nr--, dst++, src++, addr += PAGE_SIZE) {
+ if (copy_mc_user_highpage(dst, src, addr, vma)) {
+ memory_failure_queue(page_to_pfn(src), 0);
+ return -EHWPOISON;
+ }
+ kmsan_copy_page_meta(dst, src);
+ }
+
+ return 0;
+}
+
/*
* Return:
* 0: copied succeeded
* -EHWPOISON: copy failed due to hwpoison in source page
* -EAGAIN: copied failed (some other reason)
*/
-static inline int __wp_page_copy_user(struct page *dst, struct page *src,
- struct vm_fault *vmf)
+static inline int __wp_page_copy_user_pfn(struct page *dst,
+ struct vm_fault *vmf)
{
int ret;
void *kaddr;
@@ -2803,14 +2823,6 @@ static inline int __wp_page_copy_user(struct page *dst, struct page *src,
struct mm_struct *mm = vma->vm_mm;
unsigned long addr = vmf->address;
- if (likely(src)) {
- if (copy_mc_user_highpage(dst, src, addr, vma)) {
- memory_failure_queue(page_to_pfn(src), 0);
- return -EHWPOISON;
- }
- return 0;
- }
-
/*
* If the source page was a PFN mapping, we don't have
* a "struct page" for it. We do a best-effort copy by
@@ -2879,6 +2891,7 @@ static inline int __wp_page_copy_user(struct page *dst, struct page *src,
}
}
+ kmsan_copy_page_meta(dst, NULL);
ret = 0;
pte_unlock:
@@ -3372,7 +3385,12 @@ static vm_fault_t wp_page_copy(struct vm_fault *vmf)
if (!new_folio)
goto oom;
- ret = __wp_page_copy_user(&new_folio->page, vmf->page, vmf);
+ if (likely(old_folio))
+ ret = __wp_page_copy_user_range(&new_folio->page,
+ vmf->page,
+ 1, vmf->address, vma);
+ else
+ ret = __wp_page_copy_user_pfn(&new_folio->page, vmf);
if (ret) {
/*
* COW failed, if the fault was solved by other,
@@ -3388,7 +3406,6 @@ static vm_fault_t wp_page_copy(struct vm_fault *vmf)
delayacct_wpcopy_end();
return ret == -EHWPOISON ? VM_FAULT_HWPOISON : 0;
}
- kmsan_copy_page_meta(&new_folio->page, vmf->page);
}
if (mem_cgroup_charge(new_folio, mm, GFP_KERNEL))
--
2.25.1
