On Wed, Jul 28, 2021 at 07:48:22PM +0800, Rui Wang wrote:
> From: wangrui <wangrui@xxxxxxxxxxx>
>
> This patch introduce a new atomic primitive 'and_or', It may be have three
> types of implemeations:
>
> * The generic implementation is based on arch_cmpxchg.
> * The hardware supports atomic 'and_or' of single instruction.
> * The hardware supports LL/SC style atomic operations:
>
> 1: ll v1, mem
> and t1, v1, arg1
> or t1, t1, arg2
> sc t1, mem
> beq t1, 0, 1b
>
> Now that all the architectures have implemented it.
>
> Signed-by-off: Rui Wang <wangrui@xxxxxxxxxxx>
> Signed-by-off: hev <r@xxxxxx>
> ---
> include/asm-generic/atomic-instrumented.h | 28 ++++++++++++
> include/asm-generic/atomic.h | 29 +++++++++++++
> include/linux/atomic-arch-fallback.h | 42 ++++++++++++++++++
> 22 files changed, 588 insertions(+)
> diff --git a/include/asm-generic/atomic-instrumented.h b/include/asm-generic/atomic-instrumented.h
> index bc45af52c93b..231a8386ac80 100644
> --- a/include/asm-generic/atomic-instrumented.h
> +++ b/include/asm-generic/atomic-instrumented.h
> @@ -441,6 +441,34 @@ atomic_fetch_xor_relaxed(int i, atomic_t *v)
> return arch_atomic_fetch_xor_relaxed(i, v);
> }
>
> +static __always_inline int
> +atomic_fetch_and_or(int i, int j, atomic_t *v)
> +{
> + instrument_atomic_read_write(v, sizeof(*v));
> + return arch_atomic_fetch_and_or(i, j, v);
> +}
> +
> +static __always_inline int
> +atomic_fetch_and_or_acquire(int i, int j, atomic_t *v)
> +{
> + instrument_atomic_read_write(v, sizeof(*v));
> + return arch_atomic_fetch_and_or_acquire(i, j, v);
> +}
> +
> +static __always_inline int
> +atomic_fetch_and_or_release(int i, int j, atomic_t *v)
> +{
> + instrument_atomic_read_write(v, sizeof(*v));
> + return arch_atomic_fetch_and_or_release(i, j, v);
> +}
> +
> +static __always_inline int
> +atomic_fetch_and_or_relaxed(int i, int j, atomic_t *v)
> +{
> + instrument_atomic_read_write(v, sizeof(*v));
> + return arch_atomic_fetch_and_or_relaxed(i, j, v);
> +}
> +
> static __always_inline int
> atomic_xchg(atomic_t *v, int i)
> {
> diff --git a/include/asm-generic/atomic.h b/include/asm-generic/atomic.h
> index 04b8be9f1a77..474e8cd8e58d 100644
> --- a/include/asm-generic/atomic.h
> +++ b/include/asm-generic/atomic.h
> @@ -50,6 +50,18 @@ static inline int generic_atomic_fetch_##op(int i, atomic_t *v) \
> return c; \
> }
>
> +#define ATOMIC_FETCH_OP2(op, c_op1, c_op2) \
> +static inline int generic_atomic_fetch_##op(int i, int j, atomic_t *v) \
> +{ \
> + int c, old; \
> + \
> + c = v->counter; \
> + while ((old = arch_cmpxchg(&v->counter, c, (c c_op1 i) c_op2 j)) != c) \
> + c = old; \
> + \
> + return c; \
> +}
> +
> #else
>
> #include <linux/irqflags.h>
> @@ -91,6 +103,20 @@ static inline int generic_atomic_fetch_##op(int i, atomic_t *v) \
> return ret; \
> }
>
> +#define ATOMIC_FETCH_OP2(op, c_op1, c_op2) \
> +static inline int generic_atomic_fetch_##op(int i, int j, atomic_t *v) \
> +{ \
> + unsigned long flags; \
> + int ret; \
> + \
> + raw_local_irq_save(flags); \
> + ret = v->counter; \
> + v->counter = (v->counter c_op1 i) c_op2 j; \
> + raw_local_irq_restore(flags); \
> + \
> + return ret; \
> +}
> +
> #endif /* CONFIG_SMP */
>
> ATOMIC_OP_RETURN(add, +)
> @@ -101,6 +127,7 @@ ATOMIC_FETCH_OP(sub, -)
> ATOMIC_FETCH_OP(and, &)
> ATOMIC_FETCH_OP(or, |)
> ATOMIC_FETCH_OP(xor, ^)
> +ATOMIC_FETCH_OP2(and_or, &, |)
>
> ATOMIC_OP(add, +)
> ATOMIC_OP(sub, -)
> @@ -109,6 +136,7 @@ ATOMIC_OP(or, |)
> ATOMIC_OP(xor, ^)
>
> #undef ATOMIC_FETCH_OP
> +#undef ATOMIC_FETCH_OP2
> #undef ATOMIC_OP_RETURN
> #undef ATOMIC_OP
>
> @@ -120,6 +148,7 @@ ATOMIC_OP(xor, ^)
> #define arch_atomic_fetch_and generic_atomic_fetch_and
> #define arch_atomic_fetch_or generic_atomic_fetch_or
> #define arch_atomic_fetch_xor generic_atomic_fetch_xor
> +#define arch_atomic_fetch_and_or generic_atomic_fetch_and_or
>
> #define arch_atomic_add generic_atomic_add
> #define arch_atomic_sub generic_atomic_sub
> diff --git a/include/linux/atomic-arch-fallback.h b/include/linux/atomic-arch-fallback.h
> index a3dba31df01e..92043a8d5b79 100644
> --- a/include/linux/atomic-arch-fallback.h
> +++ b/include/linux/atomic-arch-fallback.h
> @@ -891,6 +891,48 @@ arch_atomic_fetch_xor(int i, atomic_t *v)
>
> #endif /* arch_atomic_fetch_xor_relaxed */
>
> +#ifndef arch_atomic_fetch_and_or_relaxed
> +#define arch_atomic_fetch_and_or_acquire arch_atomic_fetch_and_or
> +#define arch_atomic_fetch_and_or_release arch_atomic_fetch_and_or
> +#define arch_atomic_fetch_and_or_relaxed arch_atomic_fetch_and_or
> +#else /* arch_atomic_fetch_and_or_relaxed */
> +
> +#ifndef arch_atomic_fetch_and_or_acquire
> +static __always_inline int
> +arch_atomic_fetch_and_or_acquire(int i, int j, atomic_t *v)
> +{
> + int ret = arch_atomic_fetch_and_or_relaxed(i, j, v);
> + __atomic_acquire_fence();
> + return ret;
> +}
> +#define arch_atomic_fetch_and_or_acquire arch_atomic_fetch_and_or_acquire
> +#endif
> +
> +#ifndef arch_atomic_fetch_and_or_release
> +static __always_inline int
> +arch_atomic_fetch_and_or_release(int i, int j, atomic_t *v)
> +{
> + __atomic_release_fence();
> + return arch_atomic_fetch_and_or_relaxed(i, j, v);
> +}
> +#define arch_atomic_fetch_and_or_release arch_atomic_fetch_and_or_release
> +#endif
> +
> +#ifndef arch_atomic_fetch_and_or
> +static __always_inline int
> +arch_atomic_fetch_and_or(int i, int j, atomic_t *v)
> +{
> + int ret;
> + __atomic_pre_full_fence();
> + ret = arch_atomic_fetch_and_or_relaxed(i, j, v);
> + __atomic_post_full_fence();
> + return ret;
> +}
> +#define arch_atomic_fetch_and_or arch_atomic_fetch_and_or
> +#endif
> +
> +#endif /* arch_atomic_fetch_and_or_relaxed */
> +
> #ifndef arch_atomic_xchg_relaxed
> #define arch_atomic_xchg_acquire arch_atomic_xchg
> #define arch_atomic_xchg_release arch_atomic_xchg
Urgh.. please start from something like the below and then run:
scripts/atomic/gen-atomics.sh
The below isn't quite right, because it'll use try_cmpxchg() for
atomic_andnot_or(), which by being a void atomic should be _relaxed. I'm
not entirely sure how to make that happen in a hurry.
---
diff --git a/scripts/atomic/atomics.tbl b/scripts/atomic/atomics.tbl
index fbee2f6190d9..3aaa0caa6b2d 100755
--- a/scripts/atomic/atomics.tbl
+++ b/scripts/atomic/atomics.tbl
@@ -39,3 +39,4 @@ inc_not_zero b v
inc_unless_negative b v
dec_unless_positive b v
dec_if_positive i v
+andnot_or vF v i:m i:o
diff --git a/scripts/atomic/fallbacks/andnot_or b/scripts/atomic/fallbacks/andnot_or
new file mode 100644
index 000000000000..f50e78d6c53a
--- /dev/null
+++ b/scripts/atomic/fallbacks/andnot_or
@@ -0,0 +1,15 @@
+cat <<EOF
+static __always_inline ${ret}
+arch_${atomic}_${pfx}andnot_or${sfx}${order}(${atomic}_t *v, ${int} m, ${int} o)
+{
+ ${retstmt}({
+ ${int} N, O = atomic_read(v);
+ do {
+ N = O;
+ N &= ~m;
+ N |= o;
+ } while (!arch_${atomic}_try_cmpxchg${order}(v, &O, N));
+ O;
+ });
+}
+EOF
diff --git a/scripts/atomic/fallbacks/fetch_andnot_or b/scripts/atomic/fallbacks/fetch_andnot_or
deleted file mode 100644
index e69de29bb2d1..000000000000
