From: Jérôme Glisse <jglisse@xxxxxxxxxx>
CPU page table update can happens for many reasons, not only as a result
of a syscall (munmap(), mprotect(), mremap(), madvise(), ...) but also
as a result of kernel activities (memory compression, reclaim, migration,
...).
Users of mmu notifier API track changes to the CPU page table and take
specific action for them. While current API only provide range of virtual
address affected by the change, not why the changes is happening.
This patchset adds event information so that users of mmu notifier can
differentiate among broad category:
- UNMAP: munmap() or mremap()
- CLEAR: page table is cleared (migration, compaction, reclaim, ...)
- PROTECTION_VMA: change in access protections for the range
- PROTECTION_PAGE: change in access protections for page in the range
- SOFT_DIRTY: soft dirtyness tracking
Being able to identify munmap() and mremap() from other reasons why the
page table is cleared is important to allow user of mmu notifier to
update their own internal tracking structure accordingly (on munmap or
mremap it is not longer needed to track range of virtual address as it
becomes invalid).
Signed-off-by: Jérôme Glisse <jglisse@xxxxxxxxxx>
Cc: Andrew Morton <akpm@xxxxxxxxxxxxxxxxxxxx>
Cc: Matthew Wilcox <mawilcox@xxxxxxxxxxxxx>
Cc: Ross Zwisler <zwisler@xxxxxxxxxx>
Cc: Jan Kara <jack@xxxxxxx>
Cc: Dan Williams <dan.j.williams@xxxxxxxxx>
Cc: Paolo Bonzini <pbonzini@xxxxxxxxxx>
Cc: Radim Krčmář <rkrcmar@xxxxxxxxxx>
Cc: Michal Hocko <mhocko@xxxxxxxxxx>
Cc: Christian Koenig <christian.koenig@xxxxxxx>
Cc: Felix Kuehling <felix.kuehling@xxxxxxx>
Cc: Ralph Campbell <rcampbell@xxxxxxxxxx>
Cc: John Hubbard <jhubbard@xxxxxxxxxx>
Cc: kvm@xxxxxxxxxxxxxxx
Cc: linux-rdma@xxxxxxxxxxxxxxx
Cc: linux-fsdevel@xxxxxxxxxxxxxxx
Cc: dri-devel@xxxxxxxxxxxxxxxxxxxxx
---
fs/dax.c | 1 +
fs/proc/task_mmu.c | 1 +
include/linux/mmu_notifier.h | 33 +++++++++++++++++++++++++++++++++
kernel/events/uprobes.c | 1 +
mm/huge_memory.c | 4 ++++
mm/hugetlb.c | 4 ++++
mm/khugepaged.c | 1 +
mm/ksm.c | 2 ++
mm/madvise.c | 1 +
mm/memory.c | 5 +++++
mm/migrate.c | 2 ++
mm/mprotect.c | 1 +
mm/mremap.c | 1 +
mm/oom_kill.c | 1 +
mm/rmap.c | 2 ++
15 files changed, 60 insertions(+)
diff --git a/fs/dax.c b/fs/dax.c
index e22508ee19ec..83092c5ac5f0 100644
--- a/fs/dax.c
+++ b/fs/dax.c
@@ -761,6 +761,7 @@ static void dax_entry_mkclean(struct address_space *mapping, pgoff_t index,
struct mmu_notifier_range range;
unsigned long address;
+ range.event = MMU_NOTIFY_PROTECTION_PAGE;
range.mm = vma->vm_mm;
cond_resched();
diff --git a/fs/proc/task_mmu.c b/fs/proc/task_mmu.c
index 53d625925669..4abb1668eeb3 100644
--- a/fs/proc/task_mmu.c
+++ b/fs/proc/task_mmu.c
@@ -1144,6 +1144,7 @@ static ssize_t clear_refs_write(struct file *file, const char __user *buf,
range.start = 0;
range.end = -1UL;
range.mm = mm;
+ range.event = MMU_NOTIFY_SOFT_DIRTY;
mmu_notifier_invalidate_range_start(&range);
}
walk_page_range(0, mm->highest_vm_end, &clear_refs_walk);
diff --git a/include/linux/mmu_notifier.h b/include/linux/mmu_notifier.h
index cbeece8e47d4..3077d487be8b 100644
--- a/include/linux/mmu_notifier.h
+++ b/include/linux/mmu_notifier.h
@@ -25,10 +25,43 @@ struct mmu_notifier_mm {
spinlock_t lock;
};
+/*
+ * What event is triggering the invalidation:
+ *
+ * MMU_NOTIFY_UNMAP
+ * either munmap() that unmap the range or a mremap() that move the range
+ *
+ * MMU_NOTIFY_CLEAR
+ * clear page table entry (many reasons for this like madvise() or replacing
+ * a page by another one, ...).
+ *
+ * MMU_NOTIFY_PROTECTION_VMA
+ * update is due to protection change for the range ie using the vma access
+ * permission (vm_page_prot) to update the whole range is enough no need to
+ * inspect changes to the CPU page table (mprotect() syscall)
+ *
+ * MMU_NOTIFY_PROTECTION_PAGE
+ * update is due to change in read/write flag for pages in the range so to
+ * mirror those changes the user must inspect the CPU page table (from the
+ * end callback).
+ *
+ *
+ * MMU_NOTIFY_SOFT_DIRTY
+ * soft dirty accounting (still same page and same access flags)
+ */
+enum mmu_notifier_event {
+ MMU_NOTIFY_UNMAP = 0,
+ MMU_NOTIFY_CLEAR,
+ MMU_NOTIFY_PROTECTION_VMA,
+ MMU_NOTIFY_PROTECTION_PAGE,
+ MMU_NOTIFY_SOFT_DIRTY,
+};
+
struct mmu_notifier_range {
struct mm_struct *mm;
unsigned long start;
unsigned long end;
+ enum mmu_notifier_event event;
bool blockable;
};
diff --git a/kernel/events/uprobes.c b/kernel/events/uprobes.c
index aa7996ca361e..b6ef3be1c24e 100644
--- a/kernel/events/uprobes.c
+++ b/kernel/events/uprobes.c
@@ -174,6 +174,7 @@ static int __replace_page(struct vm_area_struct *vma, unsigned long addr,
struct mmu_notifier_range range;
struct mem_cgroup *memcg;
+ range.event = MMU_NOTIFY_CLEAR;
range.start = addr;
range.end = addr + PAGE_SIZE;
range.mm = mm;
diff --git a/mm/huge_memory.c b/mm/huge_memory.c
index 1a7a059dbf7d..4919be71ffd0 100644
--- a/mm/huge_memory.c
+++ b/mm/huge_memory.c
@@ -1182,6 +1182,7 @@ static vm_fault_t do_huge_pmd_wp_page_fallback(struct vm_fault *vmf,
cond_resched();
}
+ range.event = MMU_NOTIFY_CLEAR;
range.start = haddr;
range.end = range.start + HPAGE_PMD_SIZE;
range.mm = vma->vm_mm;
@@ -1347,6 +1348,7 @@ vm_fault_t do_huge_pmd_wp_page(struct vm_fault *vmf, pmd_t orig_pmd)
vma, HPAGE_PMD_NR);
__SetPageUptodate(new_page);
+ range.event = MMU_NOTIFY_CLEAR;
range.start = haddr;
range.end = range.start + HPAGE_PMD_SIZE;
range.mm = vma->vm_mm;
@@ -2029,6 +2031,7 @@ void __split_huge_pud(struct vm_area_struct *vma, pud_t *pud,
struct mm_struct *mm = vma->vm_mm;
struct mmu_notifier_range range;
+ range.event = MMU_NOTIFY_CLEAR;
range.start = address & HPAGE_PUD_MASK;
range.end = range.start + HPAGE_PUD_SIZE;
range.mm = mm;
@@ -2248,6 +2251,7 @@ void __split_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd,
struct mm_struct *mm = vma->vm_mm;
struct mmu_notifier_range range;
+ range.event = MMU_NOTIFY_CLEAR;
range.start = address & HPAGE_PMD_MASK;
range.end = range.start + HPAGE_PMD_SIZE;
range.mm = mm;
diff --git a/mm/hugetlb.c b/mm/hugetlb.c
index 4bfbdab44d51..9ffe34173834 100644
--- a/mm/hugetlb.c
+++ b/mm/hugetlb.c
@@ -3244,6 +3244,7 @@ int copy_hugetlb_page_range(struct mm_struct *dst, struct mm_struct *src,
cow = (vma->vm_flags & (VM_SHARED | VM_MAYWRITE)) == VM_MAYWRITE;
+ range.event = MMU_NOTIFY_CLEAR;
range.start = vma->vm_start;
range.end = vma->vm_end;
range.mm = src;
@@ -3344,6 +3345,7 @@ void __unmap_hugepage_range(struct mmu_gather *tlb, struct vm_area_struct *vma,
unsigned long sz = huge_page_size(h);
struct mmu_notifier_range range;
+ range.event = MMU_NOTIFY_CLEAR;
range.start = start;
range.end = end;
range.mm = mm;
@@ -3629,6 +3631,7 @@ static vm_fault_t hugetlb_cow(struct mm_struct *mm, struct vm_area_struct *vma,
__SetPageUptodate(new_page);
set_page_huge_active(new_page);
+ range.event = MMU_NOTIFY_CLEAR;
range.start = haddr;
range.end = range.start + huge_page_size(h);
range.mm = mm;
@@ -4346,6 +4349,7 @@ unsigned long hugetlb_change_protection(struct vm_area_struct *vma,
bool shared_pmd = false;
struct mmu_notifier_range range;
+ range.event = MMU_NOTIFY_PROTECTION_VMA;
range.start = start;
range.end = end;
range.mm = mm;
diff --git a/mm/khugepaged.c b/mm/khugepaged.c
index e9fe0c9a9f56..c5c78ba30b38 100644
--- a/mm/khugepaged.c
+++ b/mm/khugepaged.c
@@ -1016,6 +1016,7 @@ static void collapse_huge_page(struct mm_struct *mm,
pte = pte_offset_map(pmd, address);
pte_ptl = pte_lockptr(mm, pmd);
+ range.event = MMU_NOTIFY_CLEAR;
range.start = address;
range.end = range.start + HPAGE_PMD_SIZE;
range.mm = mm;
diff --git a/mm/ksm.c b/mm/ksm.c
index 262694d0cd4c..f8fbb92ca1bd 100644
--- a/mm/ksm.c
+++ b/mm/ksm.c
@@ -1050,6 +1050,7 @@ static int write_protect_page(struct vm_area_struct *vma, struct page *page,
BUG_ON(PageTransCompound(page));
+ range.event = MMU_NOTIFY_CLEAR;
range.start = pvmw.address;
range.end = range.start + PAGE_SIZE;
range.mm = mm;
@@ -1139,6 +1140,7 @@ static int replace_page(struct vm_area_struct *vma, struct page *page,
if (!pmd)
goto out;
+ range.event = MMU_NOTIFY_CLEAR;
range.start = addr;
range.end = addr + PAGE_SIZE;
range.mm = mm;
diff --git a/mm/madvise.c b/mm/madvise.c
index f20dd80ca21b..c415985d6a04 100644
--- a/mm/madvise.c
+++ b/mm/madvise.c
@@ -466,6 +466,7 @@ static int madvise_free_single_vma(struct vm_area_struct *vma,
if (!vma_is_anonymous(vma))
return -EINVAL;
+ range.event = MMU_NOTIFY_CLEAR;
range.start = max(vma->vm_start, start_addr);
if (range.start >= vma->vm_end)
return -EINVAL;
diff --git a/mm/memory.c b/mm/memory.c
index 36e0b83949fc..4ad63002d770 100644
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -1007,6 +1007,7 @@ int copy_page_range(struct mm_struct *dst_mm, struct mm_struct *src_mm,
* is_cow_mapping() returns true.
*/
is_cow = is_cow_mapping(vma->vm_flags);
+ range.event = MMU_NOTIFY_PROTECTION_PAGE;
range.start = addr;
range.end = end;
range.mm = src_mm;
@@ -1334,6 +1335,7 @@ void unmap_vmas(struct mmu_gather *tlb,
{
struct mmu_notifier_range range;
+ range.event = MMU_NOTIFY_UNMAP;
range.start = start_addr;
range.end = end_addr;
range.mm = vma->vm_mm;
@@ -1358,6 +1360,7 @@ void zap_page_range(struct vm_area_struct *vma, unsigned long start,
struct mmu_notifier_range range;
struct mmu_gather tlb;
+ range.event = MMU_NOTIFY_CLEAR;
range.start = start;
range.end = range.start + size;
range.mm = vma->vm_mm;
@@ -1387,6 +1390,7 @@ static void zap_page_range_single(struct vm_area_struct *vma, unsigned long addr
struct mmu_notifier_range range;
struct mmu_gather tlb;
+ range.event = MMU_NOTIFY_CLEAR;
range.start = address;
range.end = range.start + size;
range.mm = vma->vm_mm;
@@ -2260,6 +2264,7 @@ static vm_fault_t wp_page_copy(struct vm_fault *vmf)
struct mem_cgroup *memcg;
struct mmu_notifier_range range;
+ range.event = MMU_NOTIFY_CLEAR;
range.start = vmf->address & PAGE_MASK;
range.end = range.start + PAGE_SIZE;
range.mm = mm;
diff --git a/mm/migrate.c b/mm/migrate.c
index 4896dd9d8b28..a2caaabfc5a1 100644
--- a/mm/migrate.c
+++ b/mm/migrate.c
@@ -2306,6 +2306,7 @@ static void migrate_vma_collect(struct migrate_vma *migrate)
struct mmu_notifier_range range;
struct mm_walk mm_walk;
+ range.event = MMU_NOTIFY_CLEAR;
range.start = migrate->start;
range.end = migrate->end;
range.mm = mm_walk.mm;
@@ -2726,6 +2727,7 @@ static void migrate_vma_pages(struct migrate_vma *migrate)
if (!notified) {
notified = true;
+ range.event = MMU_NOTIFY_CLEAR;
range.start = addr;
range.end = migrate->end;
range.mm = mm;
diff --git a/mm/mprotect.c b/mm/mprotect.c
index f466adf31e12..6d41321b2f3e 100644
--- a/mm/mprotect.c
+++ b/mm/mprotect.c
@@ -186,6 +186,7 @@ static inline unsigned long change_pmd_range(struct vm_area_struct *vma,
/* invoke the mmu notifier if the pmd is populated */
if (!range.start) {
+ range.event = MMU_NOTIFY_PROTECTION_VMA;
range.start = addr;
range.end = end;
range.mm = mm;
diff --git a/mm/mremap.c b/mm/mremap.c
index db060acb4a8c..856a5e6bb226 100644
--- a/mm/mremap.c
+++ b/mm/mremap.c
@@ -203,6 +203,7 @@ unsigned long move_page_tables(struct vm_area_struct *vma,
old_end = old_addr + len;
flush_cache_range(vma, old_addr, old_end);
+ range.event = MMU_NOTIFY_UNMAP;
range.start = old_addr;
range.end = old_end;
range.mm = vma->vm_mm;
diff --git a/mm/oom_kill.c b/mm/oom_kill.c
index b29ab2624e95..f4bde1c34714 100644
--- a/mm/oom_kill.c
+++ b/mm/oom_kill.c
@@ -519,6 +519,7 @@ bool __oom_reap_task_mm(struct mm_struct *mm)
struct mmu_notifier_range range;
struct mmu_gather tlb;
+ range.event = MMU_NOTIFY_CLEAR;
range.start = vma->vm_start;
range.end = vma->vm_end;
range.mm = mm;
diff --git a/mm/rmap.c b/mm/rmap.c
index 09c5d9e5c766..b1afbbcc236a 100644
--- a/mm/rmap.c
+++ b/mm/rmap.c
@@ -896,6 +896,7 @@ static bool page_mkclean_one(struct page *page, struct vm_area_struct *vma,
* We have to assume the worse case ie pmd for invalidation. Note that
* the page can not be free from this function.
*/
+ range.event = MMU_NOTIFY_PROTECTION_PAGE;
range.mm = vma->vm_mm;
range.start = address;
range.end = min(vma->vm_end, range.start +
@@ -1372,6 +1373,7 @@ static bool try_to_unmap_one(struct page *page, struct vm_area_struct *vma,
* Note that the page can not be free in this function as call of
* try_to_unmap() must hold a reference on the page.
*/
+ range.event = MMU_NOTIFY_CLEAR;
range.mm = vma->vm_mm;
range.start = vma->vm_start;
range.end = min(vma->vm_end, range.start +
--
2.17.2
