Make the ISO bitops use 32-bit values internally so that on x86 we emit the
smaller BTRL/BTSL/BTCL instructions rather than BTRQ/BTSQ/BTCQ (which
require a prefix).
However, if we're going to do this, we really need to change the bit
numbers for test_bit(), set_bit(), test_and_set_bit(), etc. to be int
rather than long because BTR/BTS/BTC take a bit number that's the same size
as the memory variable size.
This means that BTSQ, for example, has a bit number in the range
-2^63..2^63-1 whereas BTSL only has a range of -2^31..2^31-1. So,
technically, the current inline-asm set_bit() and co. for non-constant bit
number are implemented incorrectly as they can't handle the full range of
the long bit number. However, in practice, it's probably not a problem.
Signed-off-by: David Howells <dhowells@xxxxxxxxxx>
---
include/asm-generic/iso-bitops.h | 57 +++++++++++++++++++++-----------------
1 file changed, 32 insertions(+), 25 deletions(-)
diff --git a/include/asm-generic/iso-bitops.h b/include/asm-generic/iso-bitops.h
index 64d5067e3a67..e87b91965e67 100644
--- a/include/asm-generic/iso-bitops.h
+++ b/include/asm-generic/iso-bitops.h
@@ -18,11 +18,12 @@
static __always_inline
bool test_bit(long bit, const volatile unsigned long *addr)
{
- unsigned long mask = 1UL << (bit & (BITS_PER_LONG - 1));
- unsigned long old;
+ const volatile unsigned int *addr32 = (const volatile unsigned int *)addr;
+ unsigned int mask = 1U << (bit & (32 - 1));
+ unsigned int old;
- addr += bit >> _BITOPS_LONG_SHIFT;
- old = __atomic_load_n(addr, __ATOMIC_RELAXED);
+ addr32 += bit >> 5;
+ old = __atomic_load_n(addr32, __ATOMIC_RELAXED);
return old & mask;
}
@@ -44,10 +45,11 @@ bool test_bit(long bit, const volatile unsigned long *addr)
static __always_inline
void iso_set_bit(long bit, volatile unsigned long *addr, int memorder)
{
- unsigned long mask = 1UL << (bit & (BITS_PER_LONG - 1));
+ volatile unsigned int *addr32 = (volatile unsigned int *)addr;
+ unsigned int mask = 1U << (bit & (32 - 1));
- addr += bit >> _BITOPS_LONG_SHIFT;
- __atomic_fetch_or(addr, mask, memorder);
+ addr32 += bit >> 5;
+ __atomic_fetch_or(addr32, mask, memorder);
}
#define set_bit(b, a) iso_set_bit((b), (a), __ATOMIC_ACQ_REL)
@@ -75,10 +77,11 @@ void iso_set_bit(long bit, volatile unsigned long *addr, int memorder)
static __always_inline
void iso_clear_bit(long bit, volatile unsigned long *addr, int memorder)
{
- unsigned long mask = 1UL << (bit & (BITS_PER_LONG - 1));
+ volatile unsigned int *addr32 = (volatile unsigned int *)addr;
+ unsigned int mask = 1U << (bit & (32 - 1));
- addr += bit >> _BITOPS_LONG_SHIFT;
- __atomic_fetch_and(addr, ~mask, memorder);
+ addr32 += bit >> 5;
+ __atomic_fetch_and(addr32, ~mask, memorder);
}
#define clear_bit(b, a) iso_clear_bit((b), (a), __ATOMIC_ACQ_REL)
@@ -105,10 +108,11 @@ void iso_clear_bit(long bit, volatile unsigned long *addr, int memorder)
static __always_inline
void iso_change_bit(long bit, volatile unsigned long *addr, int memorder)
{
- unsigned long mask = 1UL << (bit & (BITS_PER_LONG - 1));
+ volatile unsigned int *addr32 = (volatile unsigned int *)addr;
+ unsigned int mask = 1U << (bit & (32 - 1));
- addr += bit >> _BITOPS_LONG_SHIFT;
- __atomic_fetch_xor(addr, mask, memorder);
+ addr32 += bit >> 5;
+ __atomic_fetch_xor(addr32, mask, memorder);
}
#define change_bit(b, a) iso_change_bit((b), (a), __ATOMIC_ACQ_REL)
@@ -124,11 +128,12 @@ void iso_change_bit(long bit, volatile unsigned long *addr, int memorder)
static __always_inline
bool iso_test_and_set_bit(long bit, volatile unsigned long *addr, int memorder)
{
- unsigned long mask = 1UL << (bit & (BITS_PER_LONG - 1));
- unsigned long old;
+ volatile unsigned int *addr32 = (volatile unsigned int *)addr;
+ unsigned int mask = 1U << (bit & (32 - 1));
+ unsigned int old;
- addr += bit >> _BITOPS_LONG_SHIFT;
- old = __atomic_fetch_or(addr, mask, memorder);
+ addr32 += bit >> 5;
+ old = __atomic_fetch_or(addr32, mask, memorder);
return old & mask;
}
@@ -146,11 +151,12 @@ bool iso_test_and_set_bit(long bit, volatile unsigned long *addr, int memorder)
static __always_inline
bool iso_test_and_clear_bit(long bit, volatile unsigned long *addr, int memorder)
{
- unsigned long mask = 1UL << (bit & (BITS_PER_LONG - 1));
- unsigned long old;
+ volatile unsigned int *addr32 = (volatile unsigned int *)addr;
+ unsigned int mask = 1U << (bit & (32 - 1));
+ unsigned int old;
- addr += bit >> _BITOPS_LONG_SHIFT;
- old = __atomic_fetch_and(addr, ~mask, memorder);
+ addr32 += bit >> 5;
+ old = __atomic_fetch_and(addr32, ~mask, memorder);
return old & mask;
}
@@ -168,11 +174,12 @@ bool iso_test_and_clear_bit(long bit, volatile unsigned long *addr, int memorder
static __always_inline
bool iso_test_and_change_bit(long bit, volatile unsigned long *addr, int memorder)
{
- unsigned long mask = 1UL << (bit & (BITS_PER_LONG - 1));
- unsigned long old;
+ volatile unsigned int *addr32 = (volatile unsigned int *)addr;
+ unsigned int mask = 1U << (bit & (32 - 1));
+ unsigned int old;
- addr += bit >> _BITOPS_LONG_SHIFT;
- old = __atomic_fetch_xor(addr, mask, memorder);
+ addr32 += bit >> 5;
+ old = __atomic_fetch_xor(addr32, mask, memorder);
return old & mask;
}
--
To unsubscribe from this list: send the line "unsubscribe linux-arch" in
the body of a message to majordomo@xxxxxxxxxxxxxxx
More majordomo info at http://vger.kernel.org/majordomo-info.html
