On Wed, Aug 7, 2024 at 11:23 AM Suren Baghdasaryan <surenb@xxxxxxxxxx> wrote: > > Add helper functions to speculatively perform operations without > read-locking mmap_lock, expecting that mmap_lock will not be > write-locked and mm is not modified from under us. > > Signed-off-by: Suren Baghdasaryan <surenb@xxxxxxxxxx> > Suggested-by: Peter Zijlstra <peterz@xxxxxxxxxxxxx> > Cc: Andrii Nakryiko <andrii.nakryiko@xxxxxxxxx> > --- This change makes sense and makes mm's seq a bit more useful and meaningful. I've also tested it locally with uprobe stress-test, and it seems to work great, I haven't run into any problems with a multi-hour stress test run so far. Thanks! Acked-by: Andrii Nakryiko <andrii@xxxxxxxxxx> > Discussion [1] follow-up. If proves to be useful can be included in that > patchset. Based on mm-unstable. > > [1] https://lore.kernel.org/all/20240730134605.GO33588@xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx/ > > include/linux/mm_types.h | 3 +++ > include/linux/mmap_lock.h | 53 +++++++++++++++++++++++++++++++-------- > kernel/fork.c | 3 --- > 3 files changed, 46 insertions(+), 13 deletions(-) > > diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h > index 003619fab20e..a426e6ced604 100644 > --- a/include/linux/mm_types.h > +++ b/include/linux/mm_types.h > @@ -887,6 +887,9 @@ struct mm_struct { > * Roughly speaking, incrementing the sequence number is > * equivalent to releasing locks on VMAs; reading the sequence > * number can be part of taking a read lock on a VMA. > + * Incremented every time mmap_lock is write-locked/unlocked. > + * Initialized to 0, therefore odd values indicate mmap_lock > + * is write-locked and even values that it's released. > * > * Can be modified under write mmap_lock using RELEASE > * semantics. > diff --git a/include/linux/mmap_lock.h b/include/linux/mmap_lock.h > index de9dc20b01ba..5410ce741d75 100644 > --- a/include/linux/mmap_lock.h > +++ b/include/linux/mmap_lock.h > @@ -71,15 +71,12 @@ static inline void mmap_assert_write_locked(const struct mm_struct *mm) > } > > #ifdef CONFIG_PER_VMA_LOCK > -/* > - * Drop all currently-held per-VMA locks. > - * This is called from the mmap_lock implementation directly before releasing > - * a write-locked mmap_lock (or downgrading it to read-locked). > - * This should normally NOT be called manually from other places. > - * If you want to call this manually anyway, keep in mind that this will release > - * *all* VMA write locks, including ones from further up the stack. > - */ > -static inline void vma_end_write_all(struct mm_struct *mm) > +static inline void init_mm_lock_seq(struct mm_struct *mm) > +{ > + mm->mm_lock_seq = 0; > +} > + > +static inline void inc_mm_lock_seq(struct mm_struct *mm) > { > mmap_assert_write_locked(mm); > /* > @@ -91,19 +88,52 @@ static inline void vma_end_write_all(struct mm_struct *mm) > */ > smp_store_release(&mm->mm_lock_seq, mm->mm_lock_seq + 1); > } > + > +static inline bool mmap_lock_speculation_start(struct mm_struct *mm, int *seq) > +{ > + /* Pairs with RELEASE semantics in inc_mm_lock_seq(). */ > + *seq = smp_load_acquire(&mm->mm_lock_seq); > + /* Allow speculation if mmap_lock is not write-locked */ > + return (*seq & 1) == 0; > +} > + > +static inline bool mmap_lock_speculation_end(struct mm_struct *mm, int seq) > +{ > + /* Pairs with RELEASE semantics in inc_mm_lock_seq(). */ > + return seq == smp_load_acquire(&mm->mm_lock_seq); > +} > + > #else > -static inline void vma_end_write_all(struct mm_struct *mm) {} > +static inline void init_mm_lock_seq(struct mm_struct *mm) {} > +static inline void inc_mm_lock_seq(struct mm_struct *mm) {} > +static inline bool mmap_lock_speculation_start(struct mm_struct *mm, int *seq) { return false; } > +static inline bool mmap_lock_speculation_end(struct mm_struct *mm, int seq) { return false; } > #endif > > +/* > + * Drop all currently-held per-VMA locks. > + * This is called from the mmap_lock implementation directly before releasing > + * a write-locked mmap_lock (or downgrading it to read-locked). > + * This should normally NOT be called manually from other places. > + * If you want to call this manually anyway, keep in mind that this will release > + * *all* VMA write locks, including ones from further up the stack. > + */ > +static inline void vma_end_write_all(struct mm_struct *mm) > +{ > + inc_mm_lock_seq(mm); > +} > + > static inline void mmap_init_lock(struct mm_struct *mm) > { > init_rwsem(&mm->mmap_lock); > + init_mm_lock_seq(mm); > } > > static inline void mmap_write_lock(struct mm_struct *mm) > { > __mmap_lock_trace_start_locking(mm, true); > down_write(&mm->mmap_lock); > + inc_mm_lock_seq(mm); > __mmap_lock_trace_acquire_returned(mm, true, true); > } > > @@ -111,6 +141,7 @@ static inline void mmap_write_lock_nested(struct mm_struct *mm, int subclass) > { > __mmap_lock_trace_start_locking(mm, true); > down_write_nested(&mm->mmap_lock, subclass); > + inc_mm_lock_seq(mm); > __mmap_lock_trace_acquire_returned(mm, true, true); > } > > @@ -120,6 +151,8 @@ static inline int mmap_write_lock_killable(struct mm_struct *mm) > > __mmap_lock_trace_start_locking(mm, true); > ret = down_write_killable(&mm->mmap_lock); > + if (!ret) > + inc_mm_lock_seq(mm); > __mmap_lock_trace_acquire_returned(mm, true, ret == 0); > return ret; > } > diff --git a/kernel/fork.c b/kernel/fork.c > index 3d590e51ce84..73e37af8a24d 100644 > --- a/kernel/fork.c > +++ b/kernel/fork.c > @@ -1259,9 +1259,6 @@ static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p, > seqcount_init(&mm->write_protect_seq); > mmap_init_lock(mm); > INIT_LIST_HEAD(&mm->mmlist); > -#ifdef CONFIG_PER_VMA_LOCK > - mm->mm_lock_seq = 0; > -#endif > mm_pgtables_bytes_init(mm); > mm->map_count = 0; > mm->locked_vm = 0; > > base-commit: 98808d08fc0f78ee638e0c0a88020fbbaf581ec6 > -- > 2.46.0.rc2.264.g509ed76dc8-goog >
