When a task is optimistically spinning on the owner, it may do it for a
long time if there is no other running task available in the run queue.
That can be long past the given timeout value.
To prevent that from happening, the rwsem_optimistic_spin() is now
modified to check for the timeout value, if specified, to see if it
should abort early.
Signed-off-by: Waiman Long <longman@xxxxxxxxxx>
---
kernel/locking/rwsem.c | 67 ++++++++++++++++++++++++++++--------------
1 file changed, 45 insertions(+), 22 deletions(-)
diff --git a/kernel/locking/rwsem.c b/kernel/locking/rwsem.c
index c0285749c338..49f052d68404 100644
--- a/kernel/locking/rwsem.c
+++ b/kernel/locking/rwsem.c
@@ -716,11 +716,13 @@ rwsem_owner_state(struct task_struct *owner, unsigned long flags, unsigned long
}
static noinline enum owner_state
-rwsem_spin_on_owner(struct rw_semaphore *sem, unsigned long nonspinnable)
+rwsem_spin_on_owner(struct rw_semaphore *sem, unsigned long nonspinnable,
+ ktime_t timeout)
{
struct task_struct *new, *owner;
unsigned long flags, new_flags;
enum owner_state state;
+ int loopcnt = 0;
owner = rwsem_owner_flags(sem, &flags);
state = rwsem_owner_state(owner, flags, nonspinnable);
@@ -749,16 +751,22 @@ rwsem_spin_on_owner(struct rw_semaphore *sem, unsigned long nonspinnable)
*/
barrier();
- if (need_resched() || !owner_on_cpu(owner)) {
- state = OWNER_NONSPINNABLE;
- break;
- }
+ if (need_resched() || !owner_on_cpu(owner))
+ goto stop_optspin;
+
+ if (timeout && !(++loopcnt & 0xf) &&
+ (sched_clock() >= ktime_to_ns(timeout)))
+ goto stop_optspin;
cpu_relax();
}
rcu_read_unlock();
return state;
+
+stop_optspin:
+ rcu_read_unlock();
+ return OWNER_NONSPINNABLE;
}
/*
@@ -786,12 +794,13 @@ static inline u64 rwsem_rspin_threshold(struct rw_semaphore *sem)
return sched_clock() + delta;
}
-static bool rwsem_optimistic_spin(struct rw_semaphore *sem, bool wlock)
+static bool rwsem_optimistic_spin(struct rw_semaphore *sem, bool wlock,
+ ktime_t timeout)
{
bool taken = false;
int prev_owner_state = OWNER_NULL;
int loop = 0;
- u64 rspin_threshold = 0;
+ u64 rspin_threshold = 0, curtime;
unsigned long nonspinnable = wlock ? RWSEM_WR_NONSPINNABLE
: RWSEM_RD_NONSPINNABLE;
@@ -801,6 +810,8 @@ static bool rwsem_optimistic_spin(struct rw_semaphore *sem, bool wlock)
if (!osq_lock(&sem->osq))
goto done;
+ curtime = timeout ? sched_clock() : 0;
+
/*
* Optimistically spin on the owner field and attempt to acquire the
* lock whenever the owner changes. Spinning will be stopped when:
@@ -810,7 +821,7 @@ static bool rwsem_optimistic_spin(struct rw_semaphore *sem, bool wlock)
for (;;) {
enum owner_state owner_state;
- owner_state = rwsem_spin_on_owner(sem, nonspinnable);
+ owner_state = rwsem_spin_on_owner(sem, nonspinnable, timeout);
if (!(owner_state & OWNER_SPINNABLE))
break;
@@ -823,6 +834,21 @@ static bool rwsem_optimistic_spin(struct rw_semaphore *sem, bool wlock)
if (taken)
break;
+ /*
+ * Check current time once every 16 iterations when
+ * 1) spinning on reader-owned rwsem; or
+ * 2) a timeout value is specified.
+ *
+ * This is to avoid calling sched_clock() too frequently
+ * so as to reduce the average latency between the times
+ * when the lock becomes free and when the spinner is
+ * ready to do a trylock.
+ */
+ if ((wlock && (owner_state == OWNER_READER)) || timeout) {
+ if (!(++loop & 0xf))
+ curtime = sched_clock();
+ }
+
/*
* Time-based reader-owned rwsem optimistic spinning
*/
@@ -838,23 +864,18 @@ static bool rwsem_optimistic_spin(struct rw_semaphore *sem, bool wlock)
if (rwsem_test_oflags(sem, nonspinnable))
break;
rspin_threshold = rwsem_rspin_threshold(sem);
- loop = 0;
}
- /*
- * Check time threshold once every 16 iterations to
- * avoid calling sched_clock() too frequently so
- * as to reduce the average latency between the times
- * when the lock becomes free and when the spinner
- * is ready to do a trylock.
- */
- else if (!(++loop & 0xf) && (sched_clock() > rspin_threshold)) {
+ else if (curtime > rspin_threshold) {
rwsem_set_nonspinnable(sem);
lockevent_inc(rwsem_opt_nospin);
break;
}
}
+ if (timeout && (ns_to_ktime(curtime) >= timeout))
+ break;
+
/*
* An RT task cannot do optimistic spinning if it cannot
* be sure the lock holder is running or live-lock may
@@ -968,7 +989,8 @@ static inline bool rwsem_can_spin_on_owner(struct rw_semaphore *sem,
return false;
}
-static inline bool rwsem_optimistic_spin(struct rw_semaphore *sem, bool wlock)
+static inline bool rwsem_optimistic_spin(struct rw_semaphore *sem, bool wlock,
+ ktime_t timeout)
{
return false;
}
@@ -982,7 +1004,8 @@ static inline bool rwsem_reader_phase_trylock(struct rw_semaphore *sem,
}
static inline int
-rwsem_spin_on_owner(struct rw_semaphore *sem, unsigned long nonspinnable)
+rwsem_spin_on_owner(struct rw_semaphore *sem, unsigned long nonspinnable,
+ ktime_t timeout)
{
return 0;
}
@@ -1036,7 +1059,7 @@ rwsem_down_read_slowpath(struct rw_semaphore *sem, int state)
*/
atomic_long_add(-RWSEM_READER_BIAS, &sem->count);
adjustment = 0;
- if (rwsem_optimistic_spin(sem, false)) {
+ if (rwsem_optimistic_spin(sem, false, 0)) {
/* rwsem_optimistic_spin() implies ACQUIRE on success */
/*
* Wake up other readers in the wait list if the front
@@ -1175,7 +1198,7 @@ rwsem_down_write_slowpath(struct rw_semaphore *sem, int state, ktime_t timeout)
/* do optimistic spinning and steal lock if possible */
if (rwsem_can_spin_on_owner(sem, RWSEM_WR_NONSPINNABLE) &&
- rwsem_optimistic_spin(sem, true)) {
+ rwsem_optimistic_spin(sem, true, timeout)) {
/* rwsem_optimistic_spin() implies ACQUIRE on success */
return sem;
}
@@ -1255,7 +1278,7 @@ rwsem_down_write_slowpath(struct rw_semaphore *sem, int state, ktime_t timeout)
* without sleeping.
*/
if ((wstate == WRITER_HANDOFF) &&
- (rwsem_spin_on_owner(sem, 0) == OWNER_NULL))
+ (rwsem_spin_on_owner(sem, 0, 0) == OWNER_NULL))
goto trylock_again;
/* Block until there are no active lockers. */
--
2.18.1
