On Fri, May 03, 2019 at 04:50:59PM +0200, Peter Zijlstra wrote: > So how about something like so then? > --- a/kernel/locking/percpu-rwsem.c > +++ b/kernel/locking/percpu-rwsem.c > @@ -63,7 +66,7 @@ int __percpu_down_read(struct percpu_rw_ > * If !readers_block the critical section starts here, matched by the > * release in percpu_up_write(). > */ > - if (likely(!smp_load_acquire(&sem->readers_block))) > + if (likely(!atomic_read_acquire(&sem->block))) > return 1; > > /* > @@ -80,14 +83,8 @@ int __percpu_down_read(struct percpu_rw_ > * and reschedule on the preempt_enable() in percpu_down_read(). > */ > preempt_enable_no_resched(); > - > - /* > - * Avoid lockdep for the down/up_read() we already have them. > - */ > - __down_read(&sem->rw_sem); > + wait_event(sem->waiters, !atomic_read(&sem->block)); That should be: wait_event(sem->waiters, !atomic_read_acquire(&sem->block)); I suppose. > this_cpu_inc(*sem->read_count); > - __up_read(&sem->rw_sem); > - > preempt_disable(); > return 1; > } > @@ -104,7 +101,7 @@ void __percpu_up_read(struct percpu_rw_s > __this_cpu_dec(*sem->read_count); > > /* Prod writer to recheck readers_active */ > - rcuwait_wake_up(&sem->writer); > + wake_up(&sem->waiters); > } > EXPORT_SYMBOL_GPL(__percpu_up_read); > > @@ -139,18 +136,22 @@ static bool readers_active_check(struct > return true; > } > > +static inline bool acquire_block(struct percpu_rw_semaphore *sem) > +{ > + if (atomic_read(&sem->block)) > + return false; > + > + return atomic_xchg(&sem->block, 1) == 0; > +} > + > void percpu_down_write(struct percpu_rw_semaphore *sem) > { > + rwsem_acquire(&sem->dep_map, 0, 0, _RET_IP_); > + > /* Notify readers to take the slow path. */ > rcu_sync_enter(&sem->rss); > > - down_write(&sem->rw_sem); > - > - /* > - * Notify new readers to block; up until now, and thus throughout the > - * longish rcu_sync_enter() above, new readers could still come in. > - */ > - WRITE_ONCE(sem->readers_block, 1); > + wait_event_exclusive(sem->waiters, acquire_block(sem)); > > smp_mb(); /* D matches A */ And we can remove that smp_mb() and rely on the atomic_xchg() from acquire_block(). > > @@ -161,7 +162,7 @@ void percpu_down_write(struct percpu_rw_ > */ > > /* Wait for all now active readers to complete. */ > - rcuwait_wait_event(&sem->writer, readers_active_check(sem)); > + wait_event(sem->waiters, readers_active_check(sem)); > } > EXPORT_SYMBOL_GPL(percpu_down_write); > > @@ -177,12 +178,8 @@ void percpu_up_write(struct percpu_rw_se > * Therefore we force it through the slow path which guarantees an > * acquire and thereby guarantees the critical section's consistency. > */ > - smp_store_release(&sem->readers_block, 0); > - > - /* > - * Release the write lock, this will allow readers back in the game. > - */ > - up_write(&sem->rw_sem); > + atomic_set_release(&sem->block, 0); > + wake_up(&sem->waiters); > > /* > * Once this completes (at least one RCU-sched grace period hence) the > @@ -190,5 +187,21 @@ void percpu_up_write(struct percpu_rw_se > * exclusive write lock because its counting. > */ > rcu_sync_exit(&sem->rss); > + > + rwsem_release(&sem->dep_map, 1, _RET_IP_); > } > EXPORT_SYMBOL_GPL(percpu_up_write);