There are use cases where a rwsem can be acquired by one task, but
released by another task. In thess cases, it may not be appropriate
for the lock waiters to spin on the task that acquires the lock.
One example will be the filesystem freeze/thaw code.
To handle such use cases, a new RWSEM_WRITER_OWNED_NOSPIN
flag can now be set in the owner field of the rwsem by the new
rwsem_set_writer_owned_nospin() function to indicate that the rwsem is
writer owned, but optimistic spinning on the rwsem should be disabled.
Later on, the new rwsem_set_writer_owned() function can be called to
set the new owner, if it is known. This function should not be called
without a prior rwsem_set_writer_owned_nospin() call.
Signed-off-by: Waiman Long <longman@xxxxxxxxxx>
---
include/linux/rwsem.h | 10 ++++++++++
kernel/locking/rwsem-xadd.c | 17 ++++++++---------
kernel/locking/rwsem.c | 16 +++++++++++++++-
kernel/locking/rwsem.h | 37 ++++++++++++++++++++++++++++++-------
4 files changed, 63 insertions(+), 17 deletions(-)
diff --git a/include/linux/rwsem.h b/include/linux/rwsem.h
index 56707d5..1ddf24b 100644
--- a/include/linux/rwsem.h
+++ b/include/linux/rwsem.h
@@ -145,6 +145,16 @@ static inline int rwsem_is_contended(struct rw_semaphore *sem)
*/
extern void downgrade_write(struct rw_semaphore *sem);
+#ifdef CONFIG_RWSEM_SPIN_ON_OWNER
+extern void rwsem_set_writer_owned_nospin(struct rw_semaphore *sem);
+extern void rwsem_set_writer_owned(struct rw_semaphore *sem,
+ struct task_struct *task);
+#else
+static inline void rwsem_set_writer_owned_nospin(struct rw_semaphore *sem) { }
+extern inline void rwsem_set_writer_owned(struct rw_semaphore *sem,
+ struct task_struct *task) { }
+#endif
+
#ifdef CONFIG_DEBUG_LOCK_ALLOC
/*
* nested locking. NOTE: rwsems are not allowed to recurse
diff --git a/kernel/locking/rwsem-xadd.c b/kernel/locking/rwsem-xadd.c
index e795908..a27dbb4 100644
--- a/kernel/locking/rwsem-xadd.c
+++ b/kernel/locking/rwsem-xadd.c
@@ -357,11 +357,8 @@ static inline bool rwsem_can_spin_on_owner(struct rw_semaphore *sem)
rcu_read_lock();
owner = READ_ONCE(sem->owner);
- if (!rwsem_owner_is_writer(owner)) {
- /*
- * Don't spin if the rwsem is readers owned.
- */
- ret = !rwsem_owner_is_reader(owner);
+ if (!owner || !is_rwsem_owner_spinnable(owner)) {
+ ret = !owner; /* !owner is spinnable */
goto done;
}
@@ -382,8 +379,10 @@ static noinline bool rwsem_spin_on_owner(struct rw_semaphore *sem)
{
struct task_struct *owner = READ_ONCE(sem->owner);
- if (!rwsem_owner_is_writer(owner))
- goto out;
+ if (!owner)
+ return true;
+ else if (!is_rwsem_owner_spinnable(owner))
+ return false;
rcu_read_lock();
while (sem->owner == owner) {
@@ -408,12 +407,12 @@ static noinline bool rwsem_spin_on_owner(struct rw_semaphore *sem)
cpu_relax();
}
rcu_read_unlock();
-out:
+
/*
* If there is a new owner or the owner is not set, we continue
* spinning.
*/
- return !rwsem_owner_is_reader(READ_ONCE(sem->owner));
+ return is_rwsem_owner_spinnable(READ_ONCE(sem->owner));
}
static bool rwsem_optimistic_spin(struct rw_semaphore *sem)
diff --git a/kernel/locking/rwsem.c b/kernel/locking/rwsem.c
index 30465a2..90e89ee 100644
--- a/kernel/locking/rwsem.c
+++ b/kernel/locking/rwsem.c
@@ -130,7 +130,8 @@ void up_read(struct rw_semaphore *sem)
void up_write(struct rw_semaphore *sem)
{
rwsem_release(&sem->dep_map, 1, _RET_IP_);
- DEBUG_RWSEMS_WARN_ON(sem->owner != current);
+ DEBUG_RWSEMS_WARN_ON((sem->owner != current) &&
+ (sem->owner != RWSEM_WRITER_OWNED_NOSPIN));
rwsem_clear_owner(sem);
__up_write(sem);
@@ -222,4 +223,17 @@ void up_read_non_owner(struct rw_semaphore *sem)
#endif
+#ifdef CONFIG_RWSEM_SPIN_ON_OWNER
+void rwsem_set_writer_owned_nospin(struct rw_semaphore *sem)
+{
+ __rwsem_set_writer_owned_nospin(sem);
+}
+EXPORT_SYMBOL(rwsem_set_writer_owned_nospin);
+void rwsem_set_writer_owned(struct rw_semaphore *sem, struct task_struct *task)
+{
+ DEBUG_RWSEMS_WARN_ON(sem->owner != RWSEM_WRITER_OWNED_NOSPIN);
+ __rwsem_set_writer_owned(sem, task);
+}
+EXPORT_SYMBOL(rwsem_set_writer_owned);
+#endif
diff --git a/kernel/locking/rwsem.h b/kernel/locking/rwsem.h
index a17cba8..bbbd5a3 100644
--- a/kernel/locking/rwsem.h
+++ b/kernel/locking/rwsem.h
@@ -11,10 +11,15 @@
* 2) RWSEM_READER_OWNED
* - lock is currently or previously owned by readers (lock is free
* or not set by owner yet)
- * 3) Other non-zero value
- * - a writer owns the lock
+ * 3) RWSEM_WRITER_OWNED_NOSPIN
+ * - lock is owned by a writer whose lock ownership may be transfered to
+ * another task and so spinning on the lock owner should be disabled.
+ * 4) Other non-zero value
+ * - a writer owns the lock and other writers can spin on the lock owner.
*/
-#define RWSEM_READER_OWNED ((struct task_struct *)1UL)
+#define RWSEM_READER_OWNED ((struct task_struct *)1UL)
+#define RWSEM_WRITER_OWNED_NOSPIN ((struct task_struct *)2UL)
+#define RWSEM_NOSPIN_MASK 3UL
#ifdef CONFIG_DEBUG_RWSEMS
# define DEBUG_RWSEMS_WARN_ON(c) DEBUG_LOCKS_WARN_ON(c)
@@ -51,14 +56,32 @@ static inline void rwsem_set_reader_owned(struct rw_semaphore *sem)
WRITE_ONCE(sem->owner, RWSEM_READER_OWNED);
}
-static inline bool rwsem_owner_is_writer(struct task_struct *owner)
+/*
+ * Mark the rwsem as writer owned, but optimistic spinning should be
+ * disabled.
+ *
+ * The caller must make sure that the rwsem is really writer owned
+ * and the lock won't be freed concurrently with this call.
+ */
+static inline void __rwsem_set_writer_owned_nospin(struct rw_semaphore *sem)
+{
+ WRITE_ONCE(sem->owner, RWSEM_WRITER_OWNED_NOSPIN);
+}
+
+static inline void __rwsem_set_writer_owned(struct rw_semaphore *sem,
+ struct task_struct *task)
{
- return owner && owner != RWSEM_READER_OWNED;
+ WRITE_ONCE(sem->owner, task);
}
-static inline bool rwsem_owner_is_reader(struct task_struct *owner)
+/*
+ * Return true if the a rwsem waiter can spin on the rwsem's owner
+ * and steal the lock.
+ * N.B. !owner is considered spinnable.
+ */
+static inline bool is_rwsem_owner_spinnable(struct task_struct *owner)
{
- return owner == RWSEM_READER_OWNED;
+ return !((unsigned long)owner & RWSEM_NOSPIN_MASK);
}
#else
static inline void rwsem_set_owner(struct rw_semaphore *sem)
--
1.8.3.1
