On Wed, Apr 17, 2024 at 03:03:05PM -0500, Elizabeth Figura wrote:
> Ach. I wrote this with the idea that the race isn't meaningful, but
> looking at it again you're right—there is a harmful race here.
>
> I think it should be fixable by moving the atomic_read inside the lock,
> though.
Right, I've ended up with the (as yet untested) below. I'll see if I can
find time later to actually test things.
---
--- a/drivers/misc/ntsync.c
+++ b/drivers/misc/ntsync.c
@@ -18,6 +18,7 @@
#include <linux/sched/signal.h>
#include <linux/slab.h>
#include <linux/spinlock.h>
+#include <linux/mutex.h>
#include <uapi/linux/ntsync.h>
#define NTSYNC_NAME "ntsync"
@@ -43,6 +44,7 @@ enum ntsync_type {
struct ntsync_obj {
spinlock_t lock;
+ int dev_locked;
enum ntsync_type type;
@@ -132,7 +134,7 @@ struct ntsync_device {
* wait_all_lock is taken first whenever multiple objects must be locked
* at the same time.
*/
- spinlock_t wait_all_lock;
+ struct mutex wait_all_lock;
struct file *file;
};
@@ -157,6 +159,56 @@ static bool is_signaled(struct ntsync_ob
}
/*
+ * XXX write coherent comment on the locking
+ */
+
+static void dev_lock_obj(struct ntsync_device *dev, struct ntsync_obj *obj)
+{
+ lockdep_assert_held(&dev->wait_all_lock);
+ WARN_ON_ONCE(obj->dev != dev);
+ spin_lock(&obj->lock);
+ obj->dev_locked = 1;
+ spin_unlock(&obj->lock);
+}
+
+static void dev_unlock_obj(struct ntsync_device *dev, struct ntsync_obj *obj)
+{
+ lockdep_assert_held(&dev->wait_all_lock);
+ WARN_ON_ONCE(obj->dev != dev);
+ spin_lock(&obj->lock);
+ obj->dev_locked = 0;
+ spin_unlock(&obj->lock);
+}
+
+static void obj_lock(struct ntsync_obj *obj)
+{
+ struct ntsync_device *dev = obj->dev;
+
+ for (;;) {
+ spin_lock(&obj->lock);
+ if (likely(!obj->dev_locked))
+ break;
+
+ spin_unlock(&obj->lock);
+ mutex_lock(&dev->wait_all_lock);
+ spin_lock(&obj->lock);
+ /*
+ * obj->dev_locked should be set and released under the same
+ * wait_all_lock section, since we now own this lock, it should
+ * be clear.
+ */
+ WARN_ON_ONCE(obj->dev_locked);
+ spin_unlock(&obj->lock);
+ mutex_unlock(&dev->wait_all_lock);
+ }
+}
+
+static void obj_unlock(struct ntsync_obj *obj)
+{
+ spin_unlock(&obj->lock);
+}
+
+/*
* "locked_obj" is an optional pointer to an object which is already locked and
* should not be locked again. This is necessary so that changing an object's
* state and waking it can be a single atomic operation.
@@ -175,7 +227,7 @@ static void try_wake_all(struct ntsync_d
for (i = 0; i < count; i++) {
if (q->entries[i].obj != locked_obj)
- spin_lock_nest_lock(&q->entries[i].obj->lock, &dev->wait_all_lock);
+ dev_lock_obj(dev, q->entries[i].obj);
}
for (i = 0; i < count; i++) {
@@ -211,7 +263,7 @@ static void try_wake_all(struct ntsync_d
for (i = 0; i < count; i++) {
if (q->entries[i].obj != locked_obj)
- spin_unlock(&q->entries[i].obj->lock);
+ dev_unlock_obj(dev, q->entries[i].obj);
}
}
@@ -231,6 +283,7 @@ static void try_wake_any_sem(struct ntsy
struct ntsync_q_entry *entry;
lockdep_assert_held(&sem->lock);
+ WARN_ON_ONCE(sem->type != NTSYNC_TYPE_SEM);
list_for_each_entry(entry, &sem->any_waiters, node) {
struct ntsync_q *q = entry->q;
@@ -251,6 +304,7 @@ static void try_wake_any_mutex(struct nt
struct ntsync_q_entry *entry;
lockdep_assert_held(&mutex->lock);
+ WARN_ON_ONCE(mutex->type != NTSYNC_TYPE_MUTEX);
list_for_each_entry(entry, &mutex->any_waiters, node) {
struct ntsync_q *q = entry->q;
@@ -302,6 +356,7 @@ static int post_sem_state(struct ntsync_
__u32 sum;
lockdep_assert_held(&sem->lock);
+ WARN_ON_ONCE(sem->type != NTSYNC_TYPE_SEM);
if (check_add_overflow(sem->u.sem.count, count, &sum) ||
sum > sem->u.sem.max)
@@ -316,6 +371,7 @@ static int ntsync_sem_post(struct ntsync
struct ntsync_device *dev = sem->dev;
__u32 __user *user_args = argp;
__u32 prev_count;
+ bool all = false;
__u32 args;
int ret;
@@ -325,28 +381,27 @@ static int ntsync_sem_post(struct ntsync
if (sem->type != NTSYNC_TYPE_SEM)
return -EINVAL;
- if (atomic_read(&sem->all_hint) > 0) {
- spin_lock(&dev->wait_all_lock);
- spin_lock_nest_lock(&sem->lock, &dev->wait_all_lock);
-
- prev_count = sem->u.sem.count;
- ret = post_sem_state(sem, args);
- if (!ret) {
+ obj_lock(sem);
+ all = atomic_read(&sem->all_hint);
+ if (unlikely(all)) {
+ obj_unlock(sem);
+ mutex_lock(&dev->wait_all_lock);
+ dev_lock_obj(dev, sem);
+ }
+
+ prev_count = sem->u.sem.count;
+ ret = post_sem_state(sem, args);
+ if (!ret) {
+ if (all)
try_wake_all_obj(dev, sem);
- try_wake_any_sem(sem);
- }
+ try_wake_any_sem(sem);
+ }
- spin_unlock(&sem->lock);
- spin_unlock(&dev->wait_all_lock);
+ if (all) {
+ dev_unlock_obj(dev, sem);
+ mutex_unlock(&dev->wait_all_lock);
} else {
- spin_lock(&sem->lock);
-
- prev_count = sem->u.sem.count;
- ret = post_sem_state(sem, args);
- if (!ret)
- try_wake_any_sem(sem);
-
- spin_unlock(&sem->lock);
+ obj_unlock(sem);
}
if (!ret && put_user(prev_count, user_args))
@@ -376,6 +431,7 @@ static int ntsync_mutex_unlock(struct nt
struct ntsync_mutex_args __user *user_args = argp;
struct ntsync_device *dev = mutex->dev;
struct ntsync_mutex_args args;
+ bool all = false;
__u32 prev_count;
int ret;
@@ -387,28 +443,27 @@ static int ntsync_mutex_unlock(struct nt
if (mutex->type != NTSYNC_TYPE_MUTEX)
return -EINVAL;
- if (atomic_read(&mutex->all_hint) > 0) {
- spin_lock(&dev->wait_all_lock);
- spin_lock_nest_lock(&mutex->lock, &dev->wait_all_lock);
-
- prev_count = mutex->u.mutex.count;
- ret = unlock_mutex_state(mutex, &args);
- if (!ret) {
+ obj_lock(mutex);
+ all = atomic_read(&mutex->all_hint);
+ if (unlikely(all)) {
+ obj_unlock(mutex);
+ mutex_lock(&dev->wait_all_lock);
+ dev_lock_obj(dev, mutex);
+ }
+
+ prev_count = mutex->u.mutex.count;
+ ret = unlock_mutex_state(mutex, &args);
+ if (!ret) {
+ if (all)
try_wake_all_obj(dev, mutex);
- try_wake_any_mutex(mutex);
- }
+ try_wake_any_mutex(mutex);
+ }
- spin_unlock(&mutex->lock);
- spin_unlock(&dev->wait_all_lock);
+ if (all) {
+ dev_unlock_obj(dev, mutex);
+ mutex_unlock(&dev->wait_all_lock);
} else {
- spin_lock(&mutex->lock);
-
- prev_count = mutex->u.mutex.count;
- ret = unlock_mutex_state(mutex, &args);
- if (!ret)
- try_wake_any_mutex(mutex);
-
- spin_unlock(&mutex->lock);
+ obj_unlock(mutex);
}
if (!ret && put_user(prev_count, &user_args->count))
@@ -437,6 +492,7 @@ static int kill_mutex_state(struct ntsyn
static int ntsync_mutex_kill(struct ntsync_obj *mutex, void __user *argp)
{
struct ntsync_device *dev = mutex->dev;
+ bool all = false;
__u32 owner;
int ret;
@@ -448,26 +504,26 @@ static int ntsync_mutex_kill(struct ntsy
if (mutex->type != NTSYNC_TYPE_MUTEX)
return -EINVAL;
- if (atomic_read(&mutex->all_hint) > 0) {
- spin_lock(&dev->wait_all_lock);
- spin_lock_nest_lock(&mutex->lock, &dev->wait_all_lock);
+ obj_lock(mutex);
+ all = atomic_read(&mutex->all_hint);
+ if (unlikely(all)) {
+ obj_unlock(mutex);
+ mutex_lock(&dev->wait_all_lock);
+ dev_lock_obj(dev, mutex);
+ }
- ret = kill_mutex_state(mutex, owner);
- if (!ret) {
+ ret = kill_mutex_state(mutex, owner);
+ if (!ret) {
+ if (all)
try_wake_all_obj(dev, mutex);
- try_wake_any_mutex(mutex);
- }
+ try_wake_any_mutex(mutex);
+ }
- spin_unlock(&mutex->lock);
- spin_unlock(&dev->wait_all_lock);
+ if (all) {
+ dev_unlock_obj(dev, mutex);
+ mutex_unlock(&dev->wait_all_lock);
} else {
- spin_lock(&mutex->lock);
-
- ret = kill_mutex_state(mutex, owner);
- if (!ret)
- try_wake_any_mutex(mutex);
-
- spin_unlock(&mutex->lock);
+ obj_unlock(mutex);
}
return ret;
@@ -477,35 +533,35 @@ static int ntsync_event_set(struct ntsyn
{
struct ntsync_device *dev = event->dev;
__u32 prev_state;
+ bool all = false;
if (event->type != NTSYNC_TYPE_EVENT)
return -EINVAL;
- if (atomic_read(&event->all_hint) > 0) {
- spin_lock(&dev->wait_all_lock);
- spin_lock_nest_lock(&event->lock, &dev->wait_all_lock);
+ obj_lock(event);
+ all = atomic_read(&event->all_hint);
+ if (unlikely(all)) {
+ obj_unlock(event);
+ mutex_lock(&dev->wait_all_lock);
+ dev_lock_obj(dev, event);
+ }
- prev_state = event->u.event.signaled;
- event->u.event.signaled = true;
+ prev_state = event->u.event.signaled;
+ event->u.event.signaled = true;
+ if (all)
try_wake_all_obj(dev, event);
- try_wake_any_event(event);
- if (pulse)
- event->u.event.signaled = false;
-
- spin_unlock(&event->lock);
- spin_unlock(&dev->wait_all_lock);
+ try_wake_any_event(event);
+ if (pulse)
+ event->u.event.signaled = false;
+
+ if (all) {
+ dev_unlock_obj(dev, event);
+ mutex_unlock(&dev->wait_all_lock);
} else {
- spin_lock(&event->lock);
-
- prev_state = event->u.event.signaled;
- event->u.event.signaled = true;
- try_wake_any_event(event);
- if (pulse)
- event->u.event.signaled = false;
-
- spin_unlock(&event->lock);
+ obj_unlock(event);
}
+
if (put_user(prev_state, (__u32 __user *)argp))
return -EFAULT;
@@ -984,7 +1040,7 @@ static int ntsync_wait_all(struct ntsync
/* queue ourselves */
- spin_lock(&dev->wait_all_lock);
+ mutex_lock(&dev->wait_all_lock);
for (i = 0; i < args.count; i++) {
struct ntsync_q_entry *entry = &q->entries[i];
@@ -1004,7 +1060,7 @@ static int ntsync_wait_all(struct ntsync
try_wake_all(dev, q, NULL);
- spin_unlock(&dev->wait_all_lock);
+ mutex_unlock(&dev->wait_all_lock);
/* sleep */
@@ -1012,7 +1068,7 @@ static int ntsync_wait_all(struct ntsync
/* and finally, unqueue */
- spin_lock(&dev->wait_all_lock);
+ mutex_lock(&dev->wait_all_lock);
for (i = 0; i < args.count; i++) {
struct ntsync_q_entry *entry = &q->entries[i];
@@ -1029,7 +1085,7 @@ static int ntsync_wait_all(struct ntsync
put_obj(obj);
}
- spin_unlock(&dev->wait_all_lock);
+ mutex_unlock(&dev->wait_all_lock);
signaled = atomic_read(&q->signaled);
if (signaled != -1) {
@@ -1056,7 +1112,7 @@ static int ntsync_char_open(struct inode
if (!dev)
return -ENOMEM;
- spin_lock_init(&dev->wait_all_lock);
+ mutex_init(&dev->wait_all_lock);
file->private_data = dev;
dev->file = file;
