has_fast_aging should be set by subscribers that non-trivially implement fast-only versions of both test_young() and clear_young(). Fast aging must be opt-in. For a subscriber that has not been enlightened with "fast aging", the test/clear_young() will behave identically whether or not fast_only is given. Have KVM always opt out for now; specific architectures can opt-in later. Given that KVM is the only test/clear_young() implementer, we could instead add an equivalent check in KVM, but doing so would incur an indirect function call every time, even if the notifier ends up being a no-op. Add mm_has_fast_young_notifiers() in case a caller wants to know if it should skip many calls to the mmu notifiers that may not be necessary (like MGLRU look-around). Signed-off-by: James Houghton <jthoughton@xxxxxxxxxx> --- include/linux/mmu_notifier.h | 14 ++++++++++++++ mm/mmu_notifier.c | 20 ++++++++++++++++++++ virt/kvm/kvm_main.c | 1 + 3 files changed, 35 insertions(+) diff --git a/include/linux/mmu_notifier.h b/include/linux/mmu_notifier.h index e2dd57ca368b..37643fa43687 100644 --- a/include/linux/mmu_notifier.h +++ b/include/linux/mmu_notifier.h @@ -231,6 +231,7 @@ struct mmu_notifier { struct mm_struct *mm; struct rcu_head rcu; unsigned int users; + bool has_fast_aging; }; /** @@ -383,6 +384,7 @@ extern int __mmu_notifier_clear_young(struct mm_struct *mm, unsigned long end); extern int __mmu_notifier_test_young(struct mm_struct *mm, unsigned long address); +extern bool __mm_has_fast_young_notifiers(struct mm_struct *mm); extern int __mmu_notifier_invalidate_range_start(struct mmu_notifier_range *r); extern void __mmu_notifier_invalidate_range_end(struct mmu_notifier_range *r); extern void __mmu_notifier_arch_invalidate_secondary_tlbs(struct mm_struct *mm, @@ -428,6 +430,13 @@ static inline int mmu_notifier_test_young(struct mm_struct *mm, return 0; } +static inline bool mm_has_fast_young_notifiers(struct mm_struct *mm) +{ + if (mm_has_notifiers(mm)) + return __mm_has_fast_young_notifiers(mm); + return 0; +} + static inline void mmu_notifier_invalidate_range_start(struct mmu_notifier_range *range) { @@ -619,6 +628,11 @@ static inline int mmu_notifier_test_young(struct mm_struct *mm, return 0; } +static inline bool mm_has_fast_young_notifiers(struct mm_struct *mm) +{ + return 0; +} + static inline void mmu_notifier_invalidate_range_start(struct mmu_notifier_range *range) { diff --git a/mm/mmu_notifier.c b/mm/mmu_notifier.c index 8982e6139d07..c405e5b072cf 100644 --- a/mm/mmu_notifier.c +++ b/mm/mmu_notifier.c @@ -382,6 +382,26 @@ int __mmu_notifier_clear_flush_young(struct mm_struct *mm, return young; } +bool __mm_has_fast_young_notifiers(struct mm_struct *mm) +{ + struct mmu_notifier *subscription; + bool has_fast_aging = false; + int id; + + id = srcu_read_lock(&srcu); + hlist_for_each_entry_rcu(subscription, + &mm->notifier_subscriptions->list, hlist, + srcu_read_lock_held(&srcu)) { + if (subscription->has_fast_aging) { + has_fast_aging = true; + break; + } + } + srcu_read_unlock(&srcu, id); + + return has_fast_aging; +} + int __mmu_notifier_clear_young(struct mm_struct *mm, unsigned long start, unsigned long end) diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index 7d5b35cfc1ed..f6c369eccd2a 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -930,6 +930,7 @@ static const struct mmu_notifier_ops kvm_mmu_notifier_ops = { static int kvm_init_mmu_notifier(struct kvm *kvm) { kvm->mmu_notifier.ops = &kvm_mmu_notifier_ops; + kvm->mmu_notifier.has_fast_aging = false; return mmu_notifier_register(&kvm->mmu_notifier, current->mm); } -- 2.46.0.792.g87dc391469-goog