This is backport of the upstream patch 8238b4579866b7c1bb99883cfe102a43db5506ff
for the stable branch 5.15
wait_on_bit: add an acquire memory barrier
There are several places in the kernel where wait_on_bit is not followed
by a memory barrier (for example, in drivers/md/dm-bufio.c:new_read).
On architectures with weak memory ordering, it may happen that memory
accesses that follow wait_on_bit are reordered before wait_on_bit and
they may return invalid data.
Fix this class of bugs by introducing a new function "test_bit_acquire"
that works like test_bit, but has acquire memory ordering semantics.
Signed-off-by: Mikulas Patocka <mpatocka@xxxxxxxxxx>
Acked-by: Will Deacon <will@xxxxxxxxxx>
Cc: stable@xxxxxxxxxxxxxxx
Signed-off-by: Linus Torvalds <torvalds@xxxxxxxxxxxxxxxxxxxx>
---
arch/alpha/include/asm/bitops.h | 7 ++++++
arch/h8300/include/asm/bitops.h | 3 +-
arch/hexagon/include/asm/bitops.h | 15 +++++++++++++
arch/ia64/include/asm/bitops.h | 7 ++++++
arch/m68k/include/asm/bitops.h | 6 +++++
arch/s390/include/asm/bitops.h | 7 ++++++
arch/sh/include/asm/bitops-op32.h | 7 ++++++
arch/x86/include/asm/bitops.h | 21 +++++++++++++++++++
include/asm-generic/bitops/instrumented-non-atomic.h | 12 ++++++++++
include/asm-generic/bitops/non-atomic.h | 14 ++++++++++++
include/linux/buffer_head.h | 2 -
include/linux/wait_bit.h | 8 +++----
kernel/sched/wait_bit.c | 2 -
13 files changed, 104 insertions(+), 7 deletions(-)
Index: linux-stable/arch/x86/include/asm/bitops.h
===================================================================
--- linux-stable.orig/arch/x86/include/asm/bitops.h 2022-10-18 12:42:09.000000000 +0200
+++ linux-stable/arch/x86/include/asm/bitops.h 2022-10-18 12:42:09.000000000 +0200
@@ -207,6 +207,20 @@ static __always_inline bool constant_tes
(addr[nr >> _BITOPS_LONG_SHIFT])) != 0;
}
+static __always_inline bool constant_test_bit_acquire(long nr, const volatile unsigned long *addr)
+{
+ bool oldbit;
+
+ asm volatile("testb %2,%1"
+ CC_SET(nz)
+ : CC_OUT(nz) (oldbit)
+ : "m" (((unsigned char *)addr)[nr >> 3]),
+ "i" (1 << (nr & 7))
+ :"memory");
+
+ return oldbit;
+}
+
static __always_inline bool variable_test_bit(long nr, volatile const unsigned long *addr)
{
bool oldbit;
@@ -224,6 +238,13 @@ static __always_inline bool variable_tes
? constant_test_bit((nr), (addr)) \
: variable_test_bit((nr), (addr)))
+static __always_inline bool
+arch_test_bit_acquire(unsigned long nr, const volatile unsigned long *addr)
+{
+ return __builtin_constant_p(nr) ? constant_test_bit_acquire(nr, addr) :
+ variable_test_bit(nr, addr);
+}
+
/**
* __ffs - find first set bit in word
* @word: The word to search
Index: linux-stable/include/asm-generic/bitops/instrumented-non-atomic.h
===================================================================
--- linux-stable.orig/include/asm-generic/bitops/instrumented-non-atomic.h 2022-10-18 12:42:09.000000000 +0200
+++ linux-stable/include/asm-generic/bitops/instrumented-non-atomic.h 2022-10-18 12:42:09.000000000 +0200
@@ -135,4 +135,16 @@ static inline bool test_bit(long nr, con
return arch_test_bit(nr, addr);
}
+/**
+ * _test_bit_acquire - Determine, with acquire semantics, whether a bit is set
+ * @nr: bit number to test
+ * @addr: Address to start counting from
+ */
+static __always_inline bool
+test_bit_acquire(unsigned long nr, const volatile unsigned long *addr)
+{
+ instrument_atomic_read(addr + BIT_WORD(nr), sizeof(long));
+ return arch_test_bit_acquire(nr, addr);
+}
+
#endif /* _ASM_GENERIC_BITOPS_INSTRUMENTED_NON_ATOMIC_H */
Index: linux-stable/include/asm-generic/bitops/non-atomic.h
===================================================================
--- linux-stable.orig/include/asm-generic/bitops/non-atomic.h 2022-10-18 12:42:09.000000000 +0200
+++ linux-stable/include/asm-generic/bitops/non-atomic.h 2022-10-18 12:42:09.000000000 +0200
@@ -3,6 +3,7 @@
#define _ASM_GENERIC_BITOPS_NON_ATOMIC_H_
#include <asm/types.h>
+#include <asm/barrier.h>
/**
* arch___set_bit - Set a bit in memory
@@ -119,4 +120,17 @@ arch_test_bit(unsigned int nr, const vol
}
#define test_bit arch_test_bit
+/**
+ * arch_test_bit_acquire - Determine, with acquire semantics, whether a bit is set
+ * @nr: bit number to test
+ * @addr: Address to start counting from
+ */
+static __always_inline bool
+arch_test_bit_acquire(unsigned long nr, const volatile unsigned long *addr)
+{
+ unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr);
+ return 1UL & (smp_load_acquire(p) >> (nr & (BITS_PER_LONG-1)));
+}
+#define test_bit_acquire arch_test_bit_acquire
+
#endif /* _ASM_GENERIC_BITOPS_NON_ATOMIC_H_ */
Index: linux-stable/include/linux/buffer_head.h
===================================================================
--- linux-stable.orig/include/linux/buffer_head.h 2022-10-18 12:42:09.000000000 +0200
+++ linux-stable/include/linux/buffer_head.h 2022-10-18 12:42:09.000000000 +0200
@@ -166,7 +166,7 @@ static __always_inline int buffer_uptoda
* make it consistent with folio_test_uptodate
* pairs with smp_mb__before_atomic in set_buffer_uptodate
*/
- return (smp_load_acquire(&bh->b_state) & (1UL << BH_Uptodate)) != 0;
+ return test_bit_acquire(BH_Uptodate, &bh->b_state);
}
#define bh_offset(bh) ((unsigned long)(bh)->b_data & ~PAGE_MASK)
Index: linux-stable/include/linux/wait_bit.h
===================================================================
--- linux-stable.orig/include/linux/wait_bit.h 2022-10-18 12:42:09.000000000 +0200
+++ linux-stable/include/linux/wait_bit.h 2022-10-18 12:42:09.000000000 +0200
@@ -71,7 +71,7 @@ static inline int
wait_on_bit(unsigned long *word, int bit, unsigned mode)
{
might_sleep();
- if (!test_bit(bit, word))
+ if (!test_bit_acquire(bit, word))
return 0;
return out_of_line_wait_on_bit(word, bit,
bit_wait,
@@ -96,7 +96,7 @@ static inline int
wait_on_bit_io(unsigned long *word, int bit, unsigned mode)
{
might_sleep();
- if (!test_bit(bit, word))
+ if (!test_bit_acquire(bit, word))
return 0;
return out_of_line_wait_on_bit(word, bit,
bit_wait_io,
@@ -123,7 +123,7 @@ wait_on_bit_timeout(unsigned long *word,
unsigned long timeout)
{
might_sleep();
- if (!test_bit(bit, word))
+ if (!test_bit_acquire(bit, word))
return 0;
return out_of_line_wait_on_bit_timeout(word, bit,
bit_wait_timeout,
@@ -151,7 +151,7 @@ wait_on_bit_action(unsigned long *word,
unsigned mode)
{
might_sleep();
- if (!test_bit(bit, word))
+ if (!test_bit_acquire(bit, word))
return 0;
return out_of_line_wait_on_bit(word, bit, action, mode);
}
Index: linux-stable/kernel/sched/wait_bit.c
===================================================================
--- linux-stable.orig/kernel/sched/wait_bit.c 2022-10-18 12:42:09.000000000 +0200
+++ linux-stable/kernel/sched/wait_bit.c 2022-10-18 12:42:09.000000000 +0200
@@ -47,7 +47,7 @@ __wait_on_bit(struct wait_queue_head *wq
prepare_to_wait(wq_head, &wbq_entry->wq_entry, mode);
if (test_bit(wbq_entry->key.bit_nr, wbq_entry->key.flags))
ret = (*action)(&wbq_entry->key, mode);
- } while (test_bit(wbq_entry->key.bit_nr, wbq_entry->key.flags) && !ret);
+ } while (test_bit_acquire(wbq_entry->key.bit_nr, wbq_entry->key.flags) && !ret);
finish_wait(wq_head, &wbq_entry->wq_entry);
Index: linux-stable/arch/alpha/include/asm/bitops.h
===================================================================
--- linux-stable.orig/arch/alpha/include/asm/bitops.h 2022-10-18 12:42:09.000000000 +0200
+++ linux-stable/arch/alpha/include/asm/bitops.h 2022-10-18 12:42:09.000000000 +0200
@@ -289,6 +289,13 @@ test_bit(int nr, const volatile void * a
return (1UL & (((const int *) addr)[nr >> 5] >> (nr & 31))) != 0UL;
}
+static __always_inline bool
+test_bit_acquire(unsigned long nr, const volatile unsigned long *addr)
+{
+ unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr);
+ return 1UL & (smp_load_acquire(p) >> (nr & (BITS_PER_LONG-1)));
+}
+
/*
* ffz = Find First Zero in word. Undefined if no zero exists,
* so code should check against ~0UL first..
Index: linux-stable/arch/hexagon/include/asm/bitops.h
===================================================================
--- linux-stable.orig/arch/hexagon/include/asm/bitops.h 2022-10-18 12:42:09.000000000 +0200
+++ linux-stable/arch/hexagon/include/asm/bitops.h 2022-10-18 12:42:09.000000000 +0200
@@ -172,7 +172,22 @@ static inline int __test_bit(int nr, con
return retval;
}
+static inline int __test_bit_acquire(int nr, const volatile unsigned long *addr)
+{
+ int retval;
+
+ asm volatile(
+ "{P0 = tstbit(%1,%2); if (P0.new) %0 = #1; if (!P0.new) %0 = #0;}\n"
+ : "=&r" (retval)
+ : "r" (addr[BIT_WORD(nr)]), "r" (nr % BITS_PER_LONG)
+ : "p0", "memory"
+ );
+
+ return retval;
+}
+
#define test_bit(nr, addr) __test_bit(nr, addr)
+#define test_bit_acquire(nr, addr) __test_bit_acquire(nr, addr)
/*
* ffz - find first zero in word.
Index: linux-stable/arch/ia64/include/asm/bitops.h
===================================================================
--- linux-stable.orig/arch/ia64/include/asm/bitops.h 2022-10-18 12:42:09.000000000 +0200
+++ linux-stable/arch/ia64/include/asm/bitops.h 2022-10-18 12:42:09.000000000 +0200
@@ -337,6 +337,13 @@ test_bit (int nr, const volatile void *a
return 1 & (((const volatile __u32 *) addr)[nr >> 5] >> (nr & 31));
}
+static __always_inline bool
+test_bit_acquire(unsigned long nr, const volatile unsigned long *addr)
+{
+ unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr);
+ return 1UL & (smp_load_acquire(p) >> (nr & (BITS_PER_LONG-1)));
+}
+
/**
* ffz - find the first zero bit in a long word
* @x: The long word to find the bit in
Index: linux-stable/arch/m68k/include/asm/bitops.h
===================================================================
--- linux-stable.orig/arch/m68k/include/asm/bitops.h 2022-10-18 12:42:09.000000000 +0200
+++ linux-stable/arch/m68k/include/asm/bitops.h 2022-10-18 12:42:09.000000000 +0200
@@ -153,6 +153,12 @@ static inline int test_bit(int nr, const
return (vaddr[nr >> 5] & (1UL << (nr & 31))) != 0;
}
+static __always_inline bool
+test_bit_acquire(unsigned long nr, const volatile unsigned long *addr)
+{
+ unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr);
+ return 1UL & (smp_load_acquire(p) >> (nr & (BITS_PER_LONG-1)));
+}
static inline int bset_reg_test_and_set_bit(int nr,
volatile unsigned long *vaddr)
Index: linux-stable/arch/s390/include/asm/bitops.h
===================================================================
--- linux-stable.orig/arch/s390/include/asm/bitops.h 2022-10-18 12:42:09.000000000 +0200
+++ linux-stable/arch/s390/include/asm/bitops.h 2022-10-18 12:42:09.000000000 +0200
@@ -184,6 +184,13 @@ static inline bool arch_test_bit(unsigne
return *addr & mask;
}
+static __always_inline bool
+arch_test_bit_acquire(unsigned long nr, const volatile unsigned long *addr)
+{
+ unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr);
+ return 1UL & (smp_load_acquire(p) >> (nr & (BITS_PER_LONG-1)));
+}
+
static inline bool arch_test_and_set_bit_lock(unsigned long nr,
volatile unsigned long *ptr)
{
Index: linux-stable/arch/sh/include/asm/bitops-op32.h
===================================================================
--- linux-stable.orig/arch/sh/include/asm/bitops-op32.h 2022-10-18 12:42:09.000000000 +0200
+++ linux-stable/arch/sh/include/asm/bitops-op32.h 2022-10-18 12:42:09.000000000 +0200
@@ -138,4 +138,11 @@ static inline int test_bit(int nr, const
return 1UL & (addr[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG-1)));
}
+static __always_inline bool
+test_bit_acquire(unsigned long nr, const volatile unsigned long *addr)
+{
+ unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr);
+ return 1UL & (smp_load_acquire(p) >> (nr & (BITS_PER_LONG-1)));
+}
+
#endif /* __ASM_SH_BITOPS_OP32_H */
Index: linux-stable/arch/h8300/include/asm/bitops.h
===================================================================
--- linux-stable.orig/arch/h8300/include/asm/bitops.h 2022-10-18 12:42:09.000000000 +0200
+++ linux-stable/arch/h8300/include/asm/bitops.h 2022-10-18 12:42:09.000000000 +0200
@@ -87,7 +87,8 @@ static inline int test_bit(int nr, const
return ret;
}
-#define __test_bit(nr, addr) test_bit(nr, addr)
+#define __test_bit(nr, addr) test_bit(nr, addr)
+#define test_bit_acquire(nr, addr) test_bit(nr, addr)
#define H8300_GEN_TEST_BITOP(FNNAME, OP) \
static inline int FNNAME(int nr, void *addr) \
