When futexes are no longer u32 aligned, the lower offset bits are no
longer available to put type info in. However, since offset is the
offset within a page, there are plenty bits available on the top end.
After that, pass flags into futex_get_value_locked() for WAIT and
disallow FUTEX2_64 instead of mandating FUTEX2_32.
Signed-off-by: Peter Zijlstra (Intel) <peterz@xxxxxxxxxxxxx>
---
include/linux/futex.h | 11 ++++++-----
kernel/futex/core.c | 9 +++++++++
kernel/futex/futex.h | 4 ++--
kernel/futex/waitwake.c | 5 +++--
4 files changed, 20 insertions(+), 9 deletions(-)
--- a/include/linux/futex.h
+++ b/include/linux/futex.h
@@ -16,18 +16,19 @@ struct task_struct;
* The key type depends on whether it's a shared or private mapping.
* Don't rearrange members without looking at hash_futex().
*
- * offset is aligned to a multiple of sizeof(u32) (== 4) by definition.
- * We use the two low order bits of offset to tell what is the kind of key :
+ * offset is the position within a page and is in the range [0, PAGE_SIZE).
+ * The high bits of the offset indicate what kind of key this is:
* 00 : Private process futex (PTHREAD_PROCESS_PRIVATE)
* (no reference on an inode or mm)
* 01 : Shared futex (PTHREAD_PROCESS_SHARED)
* mapped on a file (reference on the underlying inode)
* 10 : Shared futex (PTHREAD_PROCESS_SHARED)
* (but private mapping on an mm, and reference taken on it)
-*/
+ */
-#define FUT_OFF_INODE 1 /* We set bit 0 if key has a reference on inode */
-#define FUT_OFF_MMSHARED 2 /* We set bit 1 if key has a reference on mm */
+#define FUT_OFF_INODE (PAGE_SIZE << 0)
+#define FUT_OFF_MMSHARED (PAGE_SIZE << 1)
+#define FUT_OFF_SIZE (PAGE_SIZE << 2)
union futex_key {
struct {
--- a/kernel/futex/core.c
+++ b/kernel/futex/core.c
@@ -308,6 +308,15 @@ int get_futex_key(void __user *uaddr, un
}
/*
+ * Encode the futex size in the offset. This makes cross-size
+ * wake-wait fail -- see futex_match().
+ *
+ * NOTE that cross-size wake-wait is fundamentally broken wrt
+ * FLAGS_NUMA but could possibly work for !NUMA.
+ */
+ key->both.offset |= FUT_OFF_SIZE * (flags & FLAGS_SIZE_MASK);
+
+ /*
* PROCESS_PRIVATE futexes are fast.
* As the mm cannot disappear under us and the 'key' only needs
* virtual address, we dont even have to find the underlying vma.
--- a/kernel/futex/futex.h
+++ b/kernel/futex/futex.h
@@ -79,8 +79,8 @@ static inline bool futex_flags_valid(uns
return false;
}
- /* Only 32bit futexes are implemented -- for now */
- if ((flags & FLAGS_SIZE_MASK) != FLAGS_SIZE_32)
+ /* 64bit futexes aren't implemented -- yet */
+ if ((flags & FLAGS_SIZE_MASK) == FLAGS_SIZE_64)
return false;
/*
--- a/kernel/futex/waitwake.c
+++ b/kernel/futex/waitwake.c
@@ -434,11 +434,12 @@ static int futex_wait_multiple_setup(str
for (i = 0; i < count; i++) {
u32 __user *uaddr = (u32 __user *)(unsigned long)vs[i].w.uaddr;
+ unsigned int flags = vs[i].w.flags;
struct futex_q *q = &vs[i].q;
u32 val = vs[i].w.val;
hb = futex_q_lock(q);
- ret = futex_get_value_locked(&uval, uaddr, FLAGS_SIZE_32);
+ ret = futex_get_value_locked(&uval, uaddr, flags);
if (!ret && uval == val) {
/*
@@ -606,7 +607,7 @@ int futex_wait_setup(u32 __user *uaddr,
retry_private:
*hb = futex_q_lock(q);
- ret = futex_get_value_locked(&uval, uaddr, FLAGS_SIZE_32);
+ ret = futex_get_value_locked(&uval, uaddr, flags);
if (ret) {
futex_q_unlock(*hb);
