We will soon add support for CoWing large anonymous folios, so create a
ranged version of the ptep_clear_flush_notify() macro in preparation for
that. It is able call mmu_notifier_invalidate_range() once for the
entire range, but still calls ptep_clear_flush() per page since there is
no arch support for a batched version of this API yet.
No functional change intended.
Signed-off-by: Ryan Roberts <ryan.roberts@xxxxxxx>
---
include/linux/mmu_notifier.h | 31 +++++++++++++++++++++++++++++++
1 file changed, 31 insertions(+)
diff --git a/include/linux/mmu_notifier.h b/include/linux/mmu_notifier.h
index 64a3e051c3c4..527aa89959b4 100644
--- a/include/linux/mmu_notifier.h
+++ b/include/linux/mmu_notifier.h
@@ -595,6 +595,24 @@ static inline void mmu_notifier_range_init_owner(
___pte; \
})
+#define ptep_clear_flush_range_notify(__vma, __address, __ptep, __nr) \
+({ \
+ struct vm_area_struct *___vma = (__vma); \
+ unsigned long ___addr = (__address) & PAGE_MASK; \
+ pte_t *___ptep = (__ptep); \
+ int ___nr = (__nr); \
+ struct mm_struct *___mm = ___vma->vm_mm; \
+ int ___i; \
+ \
+ for (___i = 0; ___i < ___nr; ___i++) \
+ ptep_clear_flush(___vma, \
+ ___addr + (___i << PAGE_SHIFT), \
+ ___ptep + ___i); \
+ \
+ mmu_notifier_invalidate_range(___mm, ___addr, \
+ ___addr + (___nr << PAGE_SHIFT)); \
+})
+
#define pmdp_huge_clear_flush_notify(__vma, __haddr, __pmd) \
({ \
unsigned long ___haddr = __haddr & HPAGE_PMD_MASK; \
@@ -736,6 +754,19 @@ static inline void mmu_notifier_subscriptions_destroy(struct mm_struct *mm)
#define ptep_clear_young_notify ptep_test_and_clear_young
#define pmdp_clear_young_notify pmdp_test_and_clear_young
#define ptep_clear_flush_notify ptep_clear_flush
+#define ptep_clear_flush_range_notify(__vma, __address, __ptep, __nr) \
+({ \
+ struct vm_area_struct *___vma = (__vma); \
+ unsigned long ___addr = (__address) & PAGE_MASK; \
+ pte_t *___ptep = (__ptep); \
+ int ___nr = (__nr); \
+ int ___i; \
+ \
+ for (___i = 0; ___i < ___nr; ___i++) \
+ ptep_clear_flush(___vma, \
+ ___addr + (___i << PAGE_SHIFT), \
+ ___ptep + ___i); \
+})
#define pmdp_huge_clear_flush_notify pmdp_huge_clear_flush
#define pudp_huge_clear_flush_notify pudp_huge_clear_flush
#define set_pte_at_notify set_pte_at
--
2.25.1
