MADV_DONTNEED[_LOCKED] and MADV_FREE internal logics can be invoked with
batched tlb flushes. madvise() is called for single address range, but
if there are multiple vmas for the range, tlb flush will happen multiple
times. Update do_madvise(), which is called for madvise(), to use that
in the efficient way, by passing an mmu_gather object that it
initializes before starting the work, and flushing the tlb entries at
once after all the internal works are done.
Signed-off-by: SeongJae Park <sj@xxxxxxxxxx>
---
mm/madvise.c | 68 ++++++++++++++++++++++++++++------------------------
1 file changed, 37 insertions(+), 31 deletions(-)
diff --git a/mm/madvise.c b/mm/madvise.c
index f1beadb6176a..0d292b8e1a0e 100644
--- a/mm/madvise.c
+++ b/mm/madvise.c
@@ -1635,6 +1635,32 @@ static void madvise_unlock(struct mm_struct *mm, int behavior)
mmap_read_unlock(mm);
}
+static bool madvise_batch_tlb_flush(int behavior)
+{
+ switch (behavior) {
+ case MADV_DONTNEED:
+ case MADV_DONTNEED_LOCKED:
+ return true;
+ default:
+ return false;
+ }
+}
+
+static void madvise_init_tlb(struct madvise_behavior *madv_behavior,
+ struct mm_struct *mm)
+{
+ if (!madvise_batch_tlb_flush(madv_behavior->behavior))
+ return;
+ tlb_gather_mmu(madv_behavior->tlb, mm);
+}
+
+static void madvise_finish_tlb(struct madvise_behavior *madv_behavior)
+{
+ if (!madvise_batch_tlb_flush(madv_behavior->behavior))
+ return;
+ tlb_finish_mmu(madv_behavior->tlb);
+}
+
static bool is_valid_madvise(unsigned long start, size_t len_in, int behavior)
{
size_t len;
@@ -1787,14 +1813,20 @@ static int madvise_do_behavior(struct mm_struct *mm,
int do_madvise(struct mm_struct *mm, unsigned long start, size_t len_in, int behavior)
{
int error;
- struct madvise_behavior madv_behavior = {.behavior = behavior};
+ struct mmu_gather tlb;
+ struct madvise_behavior madv_behavior = {
+ .behavior = behavior,
+ .tlb = &tlb,
+ };
if (madvise_should_skip(start, len_in, behavior, &error))
return error;
error = madvise_lock(mm, behavior);
if (error)
return error;
+ madvise_init_tlb(&madv_behavior, mm);
error = madvise_do_behavior(mm, start, len_in, &madv_behavior);
+ madvise_finish_tlb(&madv_behavior);
madvise_unlock(mm, behavior);
return error;
@@ -1805,32 +1837,6 @@ SYSCALL_DEFINE3(madvise, unsigned long, start, size_t, len_in, int, behavior)
return do_madvise(current->mm, start, len_in, behavior);
}
-static bool vector_madvise_batch_tlb_flush(int behavior)
-{
- switch (behavior) {
- case MADV_DONTNEED:
- case MADV_DONTNEED_LOCKED:
- return true;
- default:
- return false;
- }
-}
-
-static void vector_madvise_init_tlb(struct madvise_behavior *madv_behavior,
- struct mm_struct *mm)
-{
- if (!vector_madvise_batch_tlb_flush(madv_behavior->behavior))
- return;
- tlb_gather_mmu(madv_behavior->tlb, mm);
-}
-
-static void vector_madvise_finish_tlb(struct madvise_behavior *madv_behavior)
-{
- if (!vector_madvise_batch_tlb_flush(madv_behavior->behavior))
- return;
- tlb_finish_mmu(madv_behavior->tlb);
-}
-
/* Perform an madvise operation over a vector of addresses and lengths. */
static ssize_t vector_madvise(struct mm_struct *mm, struct iov_iter *iter,
int behavior)
@@ -1848,7 +1854,7 @@ static ssize_t vector_madvise(struct mm_struct *mm, struct iov_iter *iter,
ret = madvise_lock(mm, behavior);
if (ret)
return ret;
- vector_madvise_init_tlb(&madv_behavior, mm);
+ madvise_init_tlb(&madv_behavior, mm);
while (iov_iter_count(iter)) {
unsigned long start = (unsigned long)iter_iov_addr(iter);
@@ -1877,17 +1883,17 @@ static ssize_t vector_madvise(struct mm_struct *mm, struct iov_iter *iter,
}
/* Drop and reacquire lock to unwind race. */
- vector_madvise_finish_tlb(&madv_behavior);
+ madvise_finish_tlb(&madv_behavior);
madvise_unlock(mm, behavior);
madvise_lock(mm, behavior);
- vector_madvise_init_tlb(&madv_behavior, mm);
+ madvise_init_tlb(&madv_behavior, mm);
continue;
}
if (ret < 0)
break;
iov_iter_advance(iter, iter_iov_len(iter));
}
- vector_madvise_finish_tlb(&madv_behavior);
+ madvise_finish_tlb(&madv_behavior);
madvise_unlock(mm, behavior);
ret = (total_len - iov_iter_count(iter)) ? : ret;
--
2.39.5
