This is a note to let you know that I've just added the patch titled
arm64: futex: Bound number of LDXR/STXR loops in FUTEX_WAKE_OP
to the 4.9-stable tree which can be found at:
http://www.kernel.org/git/?p=linux/kernel/git/stable/stable-queue.git;a=summary
The filename of the patch is:
arm64-futex-bound-number-of-ldxr-stxr-loops-in-futex_wake_op.patch
and it can be found in the queue-4.9 subdirectory.
If you, or anyone else, feels it should not be added to the stable tree,
please let <stable@xxxxxxxxxxxxxxx> know about it.
>From foo@baz Mon Mar 29 07:48:09 AM CEST 2021
From: Ben Hutchings <ben@xxxxxxxxxxxxxxx>
Date: Sun, 28 Mar 2021 22:42:51 +0200
Subject: arm64: futex: Bound number of LDXR/STXR loops in FUTEX_WAKE_OP
To: stable@xxxxxxxxxxxxxxx
Cc: Lee Jones <lee.jones@xxxxxxxxxx>, "Luis Claudio R. Goncalves" <lgoncalv@xxxxxxxxxx>, Florian Fainelli <f.fainelli@xxxxxxxxx>
Message-ID: <YGDqS7bKz9n1Jto3@xxxxxxxxxxxxxxx>
Content-Disposition: inline
From: Will Deacon <will.deacon@xxxxxxx>
commit 03110a5cb2161690ae5ac04994d47ed0cd6cef75 upstream.
Our futex implementation makes use of LDXR/STXR loops to perform atomic
updates to user memory from atomic context. This can lead to latency
problems if we end up spinning around the LL/SC sequence at the expense
of doing something useful.
Rework our futex atomic operations so that we return -EAGAIN if we fail
to update the futex word after 128 attempts. The core futex code will
reschedule if necessary and we'll try again later.
Fixes: 6170a97460db ("arm64: Atomic operations")
Signed-off-by: Will Deacon <will.deacon@xxxxxxx>
[bwh: Backported to 4.9: adjust context]
Signed-off-by: Ben Hutchings <ben@xxxxxxxxxxxxxxx>
Signed-off-by: Greg Kroah-Hartman <gregkh@xxxxxxxxxxxxxxxxxxx>
---
arch/arm64/include/asm/futex.h | 59 +++++++++++++++++++++++++----------------
1 file changed, 37 insertions(+), 22 deletions(-)
--- a/arch/arm64/include/asm/futex.h
+++ b/arch/arm64/include/asm/futex.h
@@ -26,7 +26,12 @@
#include <asm/errno.h>
#include <asm/sysreg.h>
+#define FUTEX_MAX_LOOPS 128 /* What's the largest number you can think of? */
+
#define __futex_atomic_op(insn, ret, oldval, uaddr, tmp, oparg) \
+do { \
+ unsigned int loops = FUTEX_MAX_LOOPS; \
+ \
asm volatile( \
ALTERNATIVE("nop", SET_PSTATE_PAN(0), ARM64_HAS_PAN, \
CONFIG_ARM64_PAN) \
@@ -34,21 +39,26 @@
"1: ldxr %w1, %2\n" \
insn "\n" \
"2: stlxr %w0, %w3, %2\n" \
-" cbnz %w0, 1b\n" \
-" dmb ish\n" \
+" cbz %w0, 3f\n" \
+" sub %w4, %w4, %w0\n" \
+" cbnz %w4, 1b\n" \
+" mov %w0, %w7\n" \
"3:\n" \
+" dmb ish\n" \
" .pushsection .fixup,\"ax\"\n" \
" .align 2\n" \
-"4: mov %w0, %w5\n" \
+"4: mov %w0, %w6\n" \
" b 3b\n" \
" .popsection\n" \
_ASM_EXTABLE(1b, 4b) \
_ASM_EXTABLE(2b, 4b) \
ALTERNATIVE("nop", SET_PSTATE_PAN(1), ARM64_HAS_PAN, \
CONFIG_ARM64_PAN) \
- : "=&r" (ret), "=&r" (oldval), "+Q" (*uaddr), "=&r" (tmp) \
- : "r" (oparg), "Ir" (-EFAULT) \
- : "memory")
+ : "=&r" (ret), "=&r" (oldval), "+Q" (*uaddr), "=&r" (tmp), \
+ "+r" (loops) \
+ : "r" (oparg), "Ir" (-EFAULT), "Ir" (-EAGAIN) \
+ : "memory"); \
+} while (0)
static inline int
arch_futex_atomic_op_inuser(int op, int oparg, int *oval, u32 __user *uaddr)
@@ -59,23 +69,23 @@ arch_futex_atomic_op_inuser(int op, int
switch (op) {
case FUTEX_OP_SET:
- __futex_atomic_op("mov %w3, %w4",
+ __futex_atomic_op("mov %w3, %w5",
ret, oldval, uaddr, tmp, oparg);
break;
case FUTEX_OP_ADD:
- __futex_atomic_op("add %w3, %w1, %w4",
+ __futex_atomic_op("add %w3, %w1, %w5",
ret, oldval, uaddr, tmp, oparg);
break;
case FUTEX_OP_OR:
- __futex_atomic_op("orr %w3, %w1, %w4",
+ __futex_atomic_op("orr %w3, %w1, %w5",
ret, oldval, uaddr, tmp, oparg);
break;
case FUTEX_OP_ANDN:
- __futex_atomic_op("and %w3, %w1, %w4",
+ __futex_atomic_op("and %w3, %w1, %w5",
ret, oldval, uaddr, tmp, ~oparg);
break;
case FUTEX_OP_XOR:
- __futex_atomic_op("eor %w3, %w1, %w4",
+ __futex_atomic_op("eor %w3, %w1, %w5",
ret, oldval, uaddr, tmp, oparg);
break;
default:
@@ -95,6 +105,7 @@ futex_atomic_cmpxchg_inatomic(u32 *uval,
u32 oldval, u32 newval)
{
int ret = 0;
+ unsigned int loops = FUTEX_MAX_LOOPS;
u32 val, tmp;
u32 __user *uaddr;
@@ -106,21 +117,25 @@ futex_atomic_cmpxchg_inatomic(u32 *uval,
ALTERNATIVE("nop", SET_PSTATE_PAN(0), ARM64_HAS_PAN, CONFIG_ARM64_PAN)
" prfm pstl1strm, %2\n"
"1: ldxr %w1, %2\n"
-" sub %w3, %w1, %w4\n"
-" cbnz %w3, 3f\n"
-"2: stlxr %w3, %w5, %2\n"
-" cbnz %w3, 1b\n"
-" dmb ish\n"
+" sub %w3, %w1, %w5\n"
+" cbnz %w3, 4f\n"
+"2: stlxr %w3, %w6, %2\n"
+" cbz %w3, 3f\n"
+" sub %w4, %w4, %w3\n"
+" cbnz %w4, 1b\n"
+" mov %w0, %w8\n"
"3:\n"
+" dmb ish\n"
+"4:\n"
" .pushsection .fixup,\"ax\"\n"
-"4: mov %w0, %w6\n"
-" b 3b\n"
+"5: mov %w0, %w7\n"
+" b 4b\n"
" .popsection\n"
- _ASM_EXTABLE(1b, 4b)
- _ASM_EXTABLE(2b, 4b)
+ _ASM_EXTABLE(1b, 5b)
+ _ASM_EXTABLE(2b, 5b)
ALTERNATIVE("nop", SET_PSTATE_PAN(1), ARM64_HAS_PAN, CONFIG_ARM64_PAN)
- : "+r" (ret), "=&r" (val), "+Q" (*uaddr), "=&r" (tmp)
- : "r" (oldval), "r" (newval), "Ir" (-EFAULT)
+ : "+r" (ret), "=&r" (val), "+Q" (*uaddr), "=&r" (tmp), "+r" (loops)
+ : "r" (oldval), "r" (newval), "Ir" (-EFAULT), "Ir" (-EAGAIN)
: "memory");
*uval = val;
Patches currently in stable-queue which might be from ben@xxxxxxxxxxxxxxx are
queue-4.9/futex-drop-hb-lock-before-enqueueing-on-the-rtmutex.patch
queue-4.9/futex-rework-futex_lock_pi-to-use-rt_mutex_-_proxy_lock.patch
queue-4.9/futex-avoid-freeing-an-active-timer.patch
queue-4.9/futex-rt_mutex-introduce-rt_mutex_init_waiter.patch
queue-4.9/futex-fix-incorrect-should_fail_futex-handling.patch
queue-4.9/futex-rt_mutex-fix-rt_mutex_cleanup_proxy_lock.patch
queue-4.9/futex-use-smp_store_release-in-mark_wake_futex.patch
queue-4.9/futex-fix-possible-missed-wakeup.patch
queue-4.9/locking-futex-allow-low-level-atomic-operations-to-return-eagain.patch
queue-4.9/futex-prevent-robust-futex-exit-race.patch
queue-4.9/futex-handle-transient-ownerless-rtmutex-state-correctly.patch
queue-4.9/futex-handle-early-deadlock-return-correctly.patch
queue-4.9/arm64-futex-bound-number-of-ldxr-stxr-loops-in-futex_wake_op.patch
