Adding fast code paths to handle specifically only get and/or clear
operation of PAGE_IS_WRITTEN, increases its performance by 0-35%.
The results of some test cases are given below:
Test-case-1
t1 = (Get + WP) time
t2 = WP time
t1 t2
Without this patch: 140-170mcs 90-115mcs
With this patch: 110mcs 80mcs
Worst case diff: 35% faster 30% faster
Test-case-2
t3 = atomic Get and WP
t3
Without this patch: 120-140mcs
With this patch: 100-110mcs
Worst case diff: 21% faster
Signed-off-by: Muhammad Usama Anjum <usama.anjum@xxxxxxxxxxxxx>
---
The test to measure the performance can be found: https://is.gd/FtSKcD
8 8192 3 1 0 and 8 8192 3 1 1 arguments have been used to produce the
above mentioned results.
---
fs/proc/task_mmu.c | 38 ++++++++++++++++++++++++++++++++++++++
1 file changed, 38 insertions(+)
diff --git a/fs/proc/task_mmu.c b/fs/proc/task_mmu.c
index 6131dc04cceb3..2c3349b76b067 100644
--- a/fs/proc/task_mmu.c
+++ b/fs/proc/task_mmu.c
@@ -2107,6 +2107,43 @@ static int pagemap_scan_pmd_entry(pmd_t *pmd, unsigned long start,
return 0;
}
+ if (!p->vec_out) {
+ /* Fast path for performing exclusive WP */
+ for (addr = start; addr != end; pte++, addr += PAGE_SIZE) {
+ if (pte_uffd_wp(ptep_get(pte)))
+ continue;
+ make_uffd_wp_pte(vma, addr, pte);
+ if (!flush) {
+ start = addr;
+ flush = true;
+ }
+ }
+ goto flush_and_return;
+ }
+
+ if (!p->arg.category_anyof_mask && !p->arg.category_inverted &&
+ p->arg.category_mask == PAGE_IS_WRITTEN &&
+ p->arg.return_mask == PAGE_IS_WRITTEN) {
+ for (addr = start; addr < end; pte++, addr += PAGE_SIZE) {
+ unsigned long next = addr + PAGE_SIZE;
+
+ if (pte_uffd_wp(ptep_get(pte)))
+ continue;
+ ret = pagemap_scan_output(p->cur_vma_category | PAGE_IS_WRITTEN,
+ p, addr, &next);
+ if (next == addr)
+ break;
+ if (~p->arg.flags & PM_SCAN_WP_MATCHING)
+ continue;
+ make_uffd_wp_pte(vma, addr, pte);
+ if (!flush) {
+ start = addr;
+ flush = true;
+ }
+ }
+ goto flush_and_return;
+ }
+
for (addr = start; addr != end; pte++, addr += PAGE_SIZE) {
unsigned long categories = p->cur_vma_category |
pagemap_page_category(p, vma, addr, ptep_get(pte));
@@ -2131,6 +2168,7 @@ static int pagemap_scan_pmd_entry(pmd_t *pmd, unsigned long start,
}
}
+flush_and_return:
if (flush)
flush_tlb_range(vma, start, addr);
--
2.39.2
