From: wangrui <wangrui@xxxxxxxxxxx>
This patch introduce a new atomic primitive 'and_or', It may be have three
types of implemeations:
* The generic implementation is based on arch_cmpxchg.
* The hardware supports atomic 'and_or' of single instruction.
* The hardware supports LL/SC style atomic operations:
1: ll v1, mem
and t1, v1, arg1
or t1, t1, arg2
sc t1, mem
beq t1, 0, 1b
Now that all the architectures have implemented it.
Signed-by-off: Rui Wang <wangrui@xxxxxxxxxxx>
Signed-by-off: hev <r@xxxxxx>
---
arch/alpha/include/asm/atomic.h | 27 ++++++++++++
arch/arc/include/asm/atomic.h | 52 +++++++++++++++++++++++
arch/arm/include/asm/atomic.h | 44 +++++++++++++++++++
arch/arm64/include/asm/atomic.h | 16 +++++++
arch/arm64/include/asm/atomic_ll_sc.h | 33 ++++++++++++++
arch/hexagon/include/asm/atomic.h | 24 +++++++++++
arch/ia64/include/asm/atomic.h | 18 ++++++++
arch/m68k/include/asm/atomic.h | 36 ++++++++++++++++
arch/mips/include/asm/atomic.h | 41 ++++++++++++++++++
arch/openrisc/include/asm/atomic.h | 22 ++++++++++
arch/parisc/include/asm/atomic.h | 20 +++++++++
arch/powerpc/include/asm/atomic.h | 26 ++++++++++++
arch/riscv/include/asm/atomic.h | 25 +++++++++++
arch/s390/include/asm/atomic.h | 2 +
arch/s390/include/asm/atomic_ops.h | 25 +++++++++++
arch/sparc/include/asm/atomic_32.h | 2 +
arch/sparc/lib/atomic32.c | 17 ++++++++
arch/x86/include/asm/atomic.h | 10 +++++
arch/xtensa/include/asm/atomic.h | 49 +++++++++++++++++++++
include/asm-generic/atomic-instrumented.h | 28 ++++++++++++
include/asm-generic/atomic.h | 29 +++++++++++++
include/linux/atomic-arch-fallback.h | 42 ++++++++++++++++++
22 files changed, 588 insertions(+)
diff --git a/arch/alpha/include/asm/atomic.h b/arch/alpha/include/asm/atomic.h
index f2861a43a61e..deb05ac292b8 100644
--- a/arch/alpha/include/asm/atomic.h
+++ b/arch/alpha/include/asm/atomic.h
@@ -91,6 +91,25 @@ static inline int arch_atomic_fetch_##op##_relaxed(int i, atomic_t *v) \
return result; \
}
+#define ATOMIC_FETCH_OP2(op, asm_op1, asm_op2) \
+static inline int arch_atomic_fetch_##op##_relaxed(int i, int j, atomic_t *v) \
+{ \
+ long temp, result; \
+ __asm__ __volatile__( \
+ "1: ldl_l %2,%1\n" \
+ " " #asm_op1 " %2,%3,%0\n" \
+ " " #asm_op2 " %0,%4,%0\n" \
+ " stl_c %0,%1\n" \
+ " beq %0,2f\n" \
+ ".subsection 2\n" \
+ "2: br 1b\n" \
+ ".previous" \
+ :"=&r" (temp), "=m" (v->counter), "=&r" (result) \
+ :"Ir" (i), "Ir" (j), "m" (v->counter) : "memory"); \
+ smp_mb(); \
+ return result; \
+}
+
#define ATOMIC64_OP(op, asm_op) \
static __inline__ void arch_atomic64_##op(s64 i, atomic64_t * v) \
{ \
@@ -182,10 +201,17 @@ ATOMIC_OPS(andnot, bic)
ATOMIC_OPS(or, bis)
ATOMIC_OPS(xor, xor)
+#undef ATOMIC_OPS
+#define ATOMIC_OPS(op, asm_op1, asm_op2) \
+ ATOMIC_FETCH_OP2(op, asm_op1, asm_op2) \
+
+ATOMIC_OPS(and_or, and, bis)
+
#define arch_atomic_fetch_and_relaxed arch_atomic_fetch_and_relaxed
#define arch_atomic_fetch_andnot_relaxed arch_atomic_fetch_andnot_relaxed
#define arch_atomic_fetch_or_relaxed arch_atomic_fetch_or_relaxed
#define arch_atomic_fetch_xor_relaxed arch_atomic_fetch_xor_relaxed
+#define arch_atomic_fetch_and_or_relaxed arch_atomic_fetch_and_or_relaxed
#define arch_atomic64_fetch_and_relaxed arch_atomic64_fetch_and_relaxed
#define arch_atomic64_fetch_andnot_relaxed arch_atomic64_fetch_andnot_relaxed
@@ -197,6 +223,7 @@ ATOMIC_OPS(xor, xor)
#undef ATOMIC64_OP_RETURN
#undef ATOMIC64_OP
#undef ATOMIC_FETCH_OP
+#undef ATOMIC_FETCH_OP2
#undef ATOMIC_OP_RETURN
#undef ATOMIC_OP
diff --git a/arch/arc/include/asm/atomic.h b/arch/arc/include/asm/atomic.h
index 7a36d79b5b2f..1aa9e0f396d7 100644
--- a/arch/arc/include/asm/atomic.h
+++ b/arch/arc/include/asm/atomic.h
@@ -89,6 +89,35 @@ static inline int arch_atomic_fetch_##op(int i, atomic_t *v) \
return orig; \
}
+#define ATOMIC_FETCH_OP2(op, c_op1, c_op2, asm_op1, asm_op2) \
+static inline int arch_atomic_fetch_##op(int i, int j, atomic_t *v) \
+{ \
+ unsigned int val, orig; \
+ \
+ /* \
+ * Explicit full memory barrier needed before/after as \
+ * LLOCK/SCOND themselves don't provide any such semantics \
+ */ \
+ smp_mb(); \
+ \
+ __asm__ __volatile__( \
+ "1: llock %[orig], [%[ctr]] \n" \
+ " " #asm_op1 " %[val], %[orig], %[i] \n" \
+ " " #asm_op2 " %[val], %[val], %[j] \n" \
+ " scond %[val], [%[ctr]] \n" \
+ " bnz 1b \n" \
+ : [val] "=&r" (val), \
+ [orig] "=&r" (orig) \
+ : [ctr] "r" (&v->counter), \
+ [i] "ir" (i), \
+ [j] "ir" (j), \
+ : "cc"); \
+ \
+ smp_mb(); \
+ \
+ return orig; \
+}
+
#else /* !CONFIG_ARC_HAS_LLSC */
#ifndef CONFIG_SMP
@@ -170,6 +199,23 @@ static inline int arch_atomic_fetch_##op(int i, atomic_t *v) \
return orig; \
}
+#define ATOMIC_FETCH_OP2(op, c_op1, c_op2, asm_op1, asm_op2) \
+static inline int arch_atomic_fetch_##op(int i, int j, atomic_t *v) \
+{ \
+ unsigned long flags; \
+ unsigned long orig; \
+ \
+ /* \
+ * spin lock/unlock provides the needed smp_mb() before/after \
+ */ \
+ atomic_ops_lock(flags); \
+ orig = v->counter; \
+ v->counter = (orig c_op1 i) c_op2 j; \
+ atomic_ops_unlock(flags); \
+ \
+ return orig; \
+}
+
#endif /* !CONFIG_ARC_HAS_LLSC */
#define ATOMIC_OPS(op, c_op, asm_op) \
@@ -190,6 +236,12 @@ ATOMIC_OPS(andnot, &= ~, bic)
ATOMIC_OPS(or, |=, or)
ATOMIC_OPS(xor, ^=, xor)
+#undef ATOMIC_OPS
+#define ATOMIC_OPS(op, c_op1, c_op2, asm_op1, asm_op2) \
+ ATOMIC_FETCH_OP2(op, c_op1, c_op2, asm_op1, asm_op2)
+
+ATOMIC_OPS(and_or, &, |, and, or)
+
#define arch_atomic_andnot arch_atomic_andnot
#define arch_atomic_fetch_andnot arch_atomic_fetch_andnot
diff --git a/arch/arm/include/asm/atomic.h b/arch/arm/include/asm/atomic.h
index db8512d9a918..faddbc183ced 100644
--- a/arch/arm/include/asm/atomic.h
+++ b/arch/arm/include/asm/atomic.h
@@ -93,6 +93,28 @@ static inline int arch_atomic_fetch_##op##_relaxed(int i, atomic_t *v) \
return result; \
}
+#define ATOMIC_FETCH_OP2(op, c_op1, c_op2, asm_op1, asm_op2) \
+static inline int arch_atomic_fetch_##op##_relaxed(int i, int j, atomic_t *v) \
+{ \
+ unsigned long tmp; \
+ int result, val; \
+ \
+ prefetchw(&v->counter); \
+ \
+ __asm__ __volatile__("@ atomic_fetch_" #op "\n" \
+"1: ldrex %0, [%4]\n" \
+" " #asm_op1 " %1, %0, %5\n" \
+" " #asm_op2 " %1, %1, %6\n" \
+" strex %2, %1, [%4]\n" \
+" teq %2, #0\n" \
+" bne 1b" \
+ : "=&r" (result), "=&r" (val), "=&r" (tmp), "+Qo" (v->counter) \
+ : "r" (&v->counter), "Ir" (i), "Ir" (j) \
+ : "cc"); \
+ \
+ return result; \
+}
+
#define arch_atomic_add_return_relaxed arch_atomic_add_return_relaxed
#define arch_atomic_sub_return_relaxed arch_atomic_sub_return_relaxed
#define arch_atomic_fetch_add_relaxed arch_atomic_fetch_add_relaxed
@@ -102,6 +124,7 @@ static inline int arch_atomic_fetch_##op##_relaxed(int i, atomic_t *v) \
#define arch_atomic_fetch_andnot_relaxed arch_atomic_fetch_andnot_relaxed
#define arch_atomic_fetch_or_relaxed arch_atomic_fetch_or_relaxed
#define arch_atomic_fetch_xor_relaxed arch_atomic_fetch_xor_relaxed
+#define arch_atomic_fetch_and_or_relaxed arch_atomic_fetch_and_or_relaxed
static inline int arch_atomic_cmpxchg_relaxed(atomic_t *ptr, int old, int new)
{
@@ -197,6 +220,20 @@ static inline int arch_atomic_fetch_##op(int i, atomic_t *v) \
return val; \
}
+#define ATOMIC_FETCH_OP2(op, c_op1, c_op2, asm_op1, asm_op2) \
+static inline int arch_atomic_fetch_##op(int i, int j, atomic_t *v) \
+{ \
+ unsigned long flags; \
+ int val; \
+ \
+ raw_local_irq_save(flags); \
+ val = v->counter; \
+ v->counter = (val c_op1 i) c_op2 j; \
+ raw_local_irq_restore(flags); \
+ \
+ return val; \
+}
+
static inline int arch_atomic_cmpxchg(atomic_t *v, int old, int new)
{
int ret;
@@ -235,8 +272,15 @@ ATOMIC_OPS(andnot, &= ~, bic)
ATOMIC_OPS(or, |=, orr)
ATOMIC_OPS(xor, ^=, eor)
+#undef ATOMIC_OPS
+#define ATOMIC_OPS(op, c_op1, c_op2, asm_op1, asm_op2) \
+ ATOMIC_FETCH_OP2(op, c_op1, c_op2, asm_op1, asm_op2)
+
+ATOMIC_OPS(and_or, &, |, and, orr)
+
#undef ATOMIC_OPS
#undef ATOMIC_FETCH_OP
+#undef ATOMIC_FETCH_OP2
#undef ATOMIC_OP_RETURN
#undef ATOMIC_OP
diff --git a/arch/arm64/include/asm/atomic.h b/arch/arm64/include/asm/atomic.h
index c9979273d389..3f1cdd3e2ef9 100644
--- a/arch/arm64/include/asm/atomic.h
+++ b/arch/arm64/include/asm/atomic.h
@@ -43,6 +43,12 @@ static __always_inline int arch_##op##name(int i, atomic_t *v) \
ATOMIC_FETCH_OP(_release, op) \
ATOMIC_FETCH_OP( , op)
+#define ATOMIC_FETCH_OP2(name, op) \
+static __always_inline int arch_##op##name(int i, int j, atomic_t *v) \
+{ \
+ return __ll_sc_##op##name(i, j, v); \
+}
+
ATOMIC_FETCH_OPS(atomic_fetch_andnot)
ATOMIC_FETCH_OPS(atomic_fetch_or)
ATOMIC_FETCH_OPS(atomic_fetch_xor)
@@ -52,7 +58,17 @@ ATOMIC_FETCH_OPS(atomic_fetch_sub)
ATOMIC_FETCH_OPS(atomic_add_return)
ATOMIC_FETCH_OPS(atomic_sub_return)
+#undef ATOMIC_FETCH_OPS
+#define ATOMIC_FETCH_OPS(op) \
+ ATOMIC_FETCH_OP2(_relaxed, op) \
+ ATOMIC_FETCH_OP2(_acquire, op) \
+ ATOMIC_FETCH_OP2(_release, op) \
+ ATOMIC_FETCH_OP2( , op)
+
+ATOMIC_FETCH_OPS(atomic_fetch_and_or)
+
#undef ATOMIC_FETCH_OP
+#undef ATOMIC_FETCH_OP2
#undef ATOMIC_FETCH_OPS
#define ATOMIC64_OP(op) \
diff --git a/arch/arm64/include/asm/atomic_ll_sc.h b/arch/arm64/include/asm/atomic_ll_sc.h
index 13869b76b58c..90289c536ed6 100644
--- a/arch/arm64/include/asm/atomic_ll_sc.h
+++ b/arch/arm64/include/asm/atomic_ll_sc.h
@@ -97,6 +97,29 @@ __ll_sc_atomic_fetch_##op##name(int i, atomic_t *v) \
return result; \
}
+#define ATOMIC_FETCH_OP2(name, mb, acq, rel, cl, op, asm_op1, asm_op2, cstr) \
+static inline int \
+__ll_sc_atomic_fetch_##op##name(int i, int j, atomic_t *v) \
+{ \
+ unsigned long tmp; \
+ int val, result; \
+ \
+ asm volatile("// atomic_fetch_" #op #name "\n" \
+ __LL_SC_FALLBACK( \
+" prfm pstl1strm, %3\n" \
+"1: ld" #acq "xr %w0, %3\n" \
+" " #asm_op1 " %w1, %w0, %w4\n" \
+" " #asm_op2 " %w1, %w1, %w5\n" \
+" st" #rel "xr %w2, %w1, %3\n" \
+" cbnz %w2, 1b\n" \
+" " #mb ) \
+ : "=&r" (result), "=&r" (val), "=&r" (tmp), "+Q" (v->counter) \
+ : __stringify(cstr) "r" (i), __stringify(cstr) "r" (j) \
+ : cl); \
+ \
+ return result; \
+}
+
#define ATOMIC_OPS(...) \
ATOMIC_OP(__VA_ARGS__) \
ATOMIC_OP_RETURN( , dmb ish, , l, "memory", __VA_ARGS__)\
@@ -129,8 +152,18 @@ ATOMIC_OPS(xor, eor, K)
*/
ATOMIC_OPS(andnot, bic, )
+#undef ATOMIC_OPS
+#define ATOMIC_OPS(...) \
+ ATOMIC_FETCH_OP2 ( , dmb ish, , l, "memory", __VA_ARGS__)\
+ ATOMIC_FETCH_OP2 (_relaxed, , , , , __VA_ARGS__)\
+ ATOMIC_FETCH_OP2 (_acquire, , a, , "memory", __VA_ARGS__)\
+ ATOMIC_FETCH_OP2 (_release, , , l, "memory", __VA_ARGS__)
+
+ATOMIC_OPS(and_or, and, orr, K)
+
#undef ATOMIC_OPS
#undef ATOMIC_FETCH_OP
+#undef ATOMIC_FETCH_OP2
#undef ATOMIC_OP_RETURN
#undef ATOMIC_OP
diff --git a/arch/hexagon/include/asm/atomic.h b/arch/hexagon/include/asm/atomic.h
index 6e94f8d04146..d944e210085a 100644
--- a/arch/hexagon/include/asm/atomic.h
+++ b/arch/hexagon/include/asm/atomic.h
@@ -130,6 +130,24 @@ static inline int arch_atomic_fetch_##op(int i, atomic_t *v) \
return output; \
}
+#define ATOMIC_FETCH_OP2(op1, op2) \
+static inline int arch_atomic_fetch_##op1##_##op2(int i, int j, atomic_t *v)\
+{ \
+ int output, val; \
+ \
+ __asm__ __volatile__ ( \
+ "1: %0 = memw_locked(%2);\n" \
+ " %1 = "#op1 "(%0,%3);\n" \
+ " %1 = "#op2 "(%1,%4);\n" \
+ " memw_locked(%2,P3)=%1;\n" \
+ " if (!P3) jump 1b;\n" \
+ : "=&r" (output), "=&r" (val) \
+ : "r" (&v->counter), "r" (i), "r" (j) \
+ : "memory", "p3" \
+ ); \
+ return output; \
+}
+
#define ATOMIC_OPS(op) ATOMIC_OP(op) ATOMIC_OP_RETURN(op) ATOMIC_FETCH_OP(op)
ATOMIC_OPS(add)
@@ -142,8 +160,14 @@ ATOMIC_OPS(and)
ATOMIC_OPS(or)
ATOMIC_OPS(xor)
+#undef ATOMIC_OPS
+#define ATOMIC_OPS(op1, op2) ATOMIC_FETCH_OP2(op1, op2)
+
+ATOMIC_OPS(and, or)
+
#undef ATOMIC_OPS
#undef ATOMIC_FETCH_OP
+#undef ATOMIC_FETCH_OP2
#undef ATOMIC_OP_RETURN
#undef ATOMIC_OP
diff --git a/arch/ia64/include/asm/atomic.h b/arch/ia64/include/asm/atomic.h
index 266c429b9137..6190108dcd53 100644
--- a/arch/ia64/include/asm/atomic.h
+++ b/arch/ia64/include/asm/atomic.h
@@ -57,6 +57,21 @@ ia64_atomic_fetch_##op (int i, atomic_t *v) \
return old; \
}
+#define ATOMIC_FETCH_OP2(op, c_op1, c_op2) \
+static __inline__ int \
+ia64_atomic_fetch_##op (int i, int j, atomic_t *v) \
+{ \
+ __s32 old, new; \
+ CMPXCHG_BUGCHECK_DECL \
+ \
+ do { \
+ CMPXCHG_BUGCHECK(v); \
+ old = arch_atomic_read(v); \
+ new = (old c_op1 i) c_op2 j; \
+ } while (ia64_cmpxchg(acq, v, old, new, sizeof(atomic_t)) != old); \
+ return old; \
+}
+
#define ATOMIC_OPS(op, c_op) \
ATOMIC_OP(op, c_op) \
ATOMIC_FETCH_OP(op, c_op)
@@ -109,6 +124,7 @@ ATOMIC_OPS(sub, -)
ATOMIC_FETCH_OP(and, &)
ATOMIC_FETCH_OP(or, |)
ATOMIC_FETCH_OP(xor, ^)
+ATOMIC_FETCH_OP2(and_or, &, |)
#define arch_atomic_and(i,v) (void)ia64_atomic_fetch_and(i,v)
#define arch_atomic_or(i,v) (void)ia64_atomic_fetch_or(i,v)
@@ -117,9 +133,11 @@ ATOMIC_FETCH_OP(xor, ^)
#define arch_atomic_fetch_and(i,v) ia64_atomic_fetch_and(i,v)
#define arch_atomic_fetch_or(i,v) ia64_atomic_fetch_or(i,v)
#define arch_atomic_fetch_xor(i,v) ia64_atomic_fetch_xor(i,v)
+#define arch_atomic_fetch_and_or(i,j,v) ia64_atomic_fetch_and_or(i,j,v)
#undef ATOMIC_OPS
#undef ATOMIC_FETCH_OP
+#undef ATOMIC_FETCH_OP2
#undef ATOMIC_OP
#define ATOMIC64_OP(op, c_op) \
diff --git a/arch/m68k/include/asm/atomic.h b/arch/m68k/include/asm/atomic.h
index 8637bf8a2f65..480ecb6534a3 100644
--- a/arch/m68k/include/asm/atomic.h
+++ b/arch/m68k/include/asm/atomic.h
@@ -67,6 +67,22 @@ static inline int arch_atomic_fetch_##op(int i, atomic_t *v) \
return tmp; \
}
+#define ATOMIC_FETCH_OP2(op, c_op1, c_op2, asm_op1, asm_op2) \
+static inline int arch_atomic_fetch_##op(int i, int j, atomic_t *v) \
+{ \
+ int t, tmp; \
+ \
+ __asm__ __volatile__( \
+ "1: movel %2,%1\n" \
+ " " #asm_op1 "l %3,%1\n" \
+ " " #asm_op2 "l %4,%1\n" \
+ " casl %2,%1,%0\n" \
+ " jne 1b" \
+ : "+m" (*v), "=&d" (t), "=&d" (tmp) \
+ : "g" (i), "g" (j), "2" (arch_atomic_read(v))); \
+ return tmp; \
+}
+
#else
#define ATOMIC_OP_RETURN(op, c_op, asm_op) \
@@ -96,6 +112,20 @@ static inline int arch_atomic_fetch_##op(int i, atomic_t * v) \
return t; \
}
+#define ATOMIC_FETCH_OP2(op, c_op1, c_op2, asm_op1, asm_op2) \
+static inline int arch_atomic_fetch_##op(int i, int j, atomic_t * v) \
+{ \
+ unsigned long flags; \
+ int t; \
+ \
+ local_irq_save(flags); \
+ t = v->counter; \
+ v->counter = (t c_op1 i) c_op2 j; \
+ local_irq_restore(flags); \
+ \
+ return t; \
+}
+
#endif /* CONFIG_RMW_INSNS */
#define ATOMIC_OPS(op, c_op, asm_op) \
@@ -115,6 +145,12 @@ ATOMIC_OPS(and, &=, and)
ATOMIC_OPS(or, |=, or)
ATOMIC_OPS(xor, ^=, eor)
+#undef ATOMIC_OPS
+#define ATOMIC_OPS(op, c_op1, c_op2, asm_op1, asm_op2) \
+ ATOMIC_FETCH_OP2(op, c_op1, c_op2, asm_op1, asm_op2)
+
+ATOMIC_OPS(and_or, &, |, and, or)
+
#undef ATOMIC_OPS
#undef ATOMIC_FETCH_OP
#undef ATOMIC_OP_RETURN
diff --git a/arch/mips/include/asm/atomic.h b/arch/mips/include/asm/atomic.h
index 95e1f7f3597f..84319b1ab9b6 100644
--- a/arch/mips/include/asm/atomic.h
+++ b/arch/mips/include/asm/atomic.h
@@ -147,6 +147,39 @@ arch_##pfx##_fetch_##op##_relaxed(type i, pfx##_t * v) \
return result; \
}
+#define ATOMIC_FETCH_OP2(pfx, op, type, c_op1, c_op2, asm_op1, asm_op2, ll, sc)\
+static __inline__ type \
+arch_##pfx##_fetch_##op##_relaxed(type i, type j, pfx##_t * v) \
+{ \
+ int temp, result; \
+ \
+ if (!kernel_uses_llsc) { \
+ unsigned long flags; \
+ \
+ raw_local_irq_save(flags); \
+ result = v->counter; \
+ v->counter = (result c_op1 i) c_op2 j; \
+ raw_local_irq_restore(flags); \
+ return result; \
+ } \
+ \
+ __asm__ __volatile__( \
+ " .set push \n" \
+ " .set " MIPS_ISA_LEVEL " \n" \
+ " " __SYNC(full, loongson3_war) " \n" \
+ "1: " #ll " %0, %2 # " #pfx "_fetch_" #op "\n" \
+ " " #asm_op1 " %1, %0, %3 \n" \
+ " " #asm_op2 " %1, %1, %4 \n" \
+ " " #sc " %1, %2 \n" \
+ "\t" __SC_BEQZ "%1, 1b \n" \
+ " .set pop \n" \
+ : "=&r" (result), "=&r" (temp), \
+ "+" GCC_OFF_SMALL_ASM() (v->counter) \
+ : "Ir" (i), "Ir" (j) : __LLSC_CLOBBER); \
+ \
+ return result; \
+}
+
#undef ATOMIC_OPS
#define ATOMIC_OPS(pfx, op, type, c_op, asm_op, ll, sc) \
ATOMIC_OP(pfx, op, type, c_op, asm_op, ll, sc) \
@@ -179,9 +212,16 @@ ATOMIC_OPS(atomic, and, int, &=, and, ll, sc)
ATOMIC_OPS(atomic, or, int, |=, or, ll, sc)
ATOMIC_OPS(atomic, xor, int, ^=, xor, ll, sc)
+#undef ATOMIC_OPS
+#define ATOMIC_OPS(pfx, op, type, c_op1, c_op2, asm_op1, asm_op2, ll, sc) \
+ ATOMIC_FETCH_OP2(pfx, op, type, c_op1, c_op2, asm_op1, asm_op2, ll, sc)
+
+ATOMIC_OPS(atomic, and_or, int, &, |, and, or, ll, sc)
+
#define arch_atomic_fetch_and_relaxed arch_atomic_fetch_and_relaxed
#define arch_atomic_fetch_or_relaxed arch_atomic_fetch_or_relaxed
#define arch_atomic_fetch_xor_relaxed arch_atomic_fetch_xor_relaxed
+#define arch_atomic_fetch_and_or_relaxed arch_atomic_fetch_and_or_relaxed
#ifdef CONFIG_64BIT
ATOMIC_OPS(atomic64, and, s64, &=, and, lld, scd)
@@ -194,6 +234,7 @@ ATOMIC_OPS(atomic64, xor, s64, ^=, xor, lld, scd)
#undef ATOMIC_OPS
#undef ATOMIC_FETCH_OP
+#undef ATOMIC_FETCH_OP2
#undef ATOMIC_OP_RETURN
#undef ATOMIC_OP
diff --git a/arch/openrisc/include/asm/atomic.h b/arch/openrisc/include/asm/atomic.h
index 326167e4783a..04598ef16977 100644
--- a/arch/openrisc/include/asm/atomic.h
+++ b/arch/openrisc/include/asm/atomic.h
@@ -66,6 +66,25 @@ static inline int arch_atomic_fetch_##op(int i, atomic_t *v) \
return old; \
}
+#define ATOMIC_FETCH_OP2(op1, op2) \
+static inline int arch_atomic_fetch_##op1##_##op2(int i, int j, atomic_t *v) \
+{ \
+ int tmp, old; \
+ \
+ __asm__ __volatile__( \
+ "1: l.lwa %0,0(%2) \n" \
+ " l." #op1 " %1,%0,%3 \n" \
+ " l." #op2 " %1,%1,%4 \n" \
+ " l.swa 0(%2),%1 \n" \
+ " l.bnf 1b \n" \
+ " l.nop \n" \
+ : "=&r"(old), "=&r"(tmp) \
+ : "r"(&v->counter), "r"(i), "r"(j) \
+ : "cc", "memory"); \
+ \
+ return old; \
+}
+
ATOMIC_OP_RETURN(add)
ATOMIC_OP_RETURN(sub)
@@ -74,6 +93,7 @@ ATOMIC_FETCH_OP(sub)
ATOMIC_FETCH_OP(and)
ATOMIC_FETCH_OP(or)
ATOMIC_FETCH_OP(xor)
+ATOMIC_FETCH_OP2(and, or)
ATOMIC_OP(add)
ATOMIC_OP(sub)
@@ -82,6 +102,7 @@ ATOMIC_OP(or)
ATOMIC_OP(xor)
#undef ATOMIC_FETCH_OP
+#undef ATOMIC_FETCH_OP2
#undef ATOMIC_OP_RETURN
#undef ATOMIC_OP
@@ -92,6 +113,7 @@ ATOMIC_OP(xor)
#define arch_atomic_fetch_and arch_atomic_fetch_and
#define arch_atomic_fetch_or arch_atomic_fetch_or
#define arch_atomic_fetch_xor arch_atomic_fetch_xor
+#define arch_atomic_fetch_and_or arch_atomic_fetch_and_or
#define arch_atomic_add arch_atomic_add
#define arch_atomic_sub arch_atomic_sub
#define arch_atomic_and arch_atomic_and
diff --git a/arch/parisc/include/asm/atomic.h b/arch/parisc/include/asm/atomic.h
index dd5a299ada69..59b9685ed2b1 100644
--- a/arch/parisc/include/asm/atomic.h
+++ b/arch/parisc/include/asm/atomic.h
@@ -114,6 +114,20 @@ static __inline__ int arch_atomic_fetch_##op(int i, atomic_t *v) \
return ret; \
}
+#define ATOMIC_FETCH_OP2(op, c_op1, c_op2) \
+static __inline__ int arch_atomic_fetch_##op(int i, int j, atomic_t *v)\
+{ \
+ unsigned long flags; \
+ int ret; \
+ \
+ _atomic_spin_lock_irqsave(v, flags); \
+ ret = v->counter; \
+ v->counter = (ret c_op1 i) c_op2 j; \
+ _atomic_spin_unlock_irqrestore(v, flags); \
+ \
+ return ret; \
+}
+
#define ATOMIC_OPS(op, c_op) \
ATOMIC_OP(op, c_op) \
ATOMIC_OP_RETURN(op, c_op) \
@@ -131,6 +145,12 @@ ATOMIC_OPS(and, &=)
ATOMIC_OPS(or, |=)
ATOMIC_OPS(xor, ^=)
+#undef ATOMIC_OPS
+#define ATOMIC_OPS(op, c_op1, c_op2) \
+ ATOMIC_FETCH_OP2(op, c_op1, c_op2)
+
+ATOMIC_OPS(and_or, &, |)
+
#undef ATOMIC_OPS
#undef ATOMIC_FETCH_OP
#undef ATOMIC_OP_RETURN
diff --git a/arch/powerpc/include/asm/atomic.h b/arch/powerpc/include/asm/atomic.h
index a1732a79e92a..c2e966ab4b81 100644
--- a/arch/powerpc/include/asm/atomic.h
+++ b/arch/powerpc/include/asm/atomic.h
@@ -86,6 +86,24 @@ static inline int arch_atomic_fetch_##op##_relaxed(int a, atomic_t *v) \
return res; \
}
+#define ATOMIC_FETCH_OP2_RELAXED(op, asm_op1, asm_op2) \
+static inline int arch_atomic_fetch_##op##_relaxed(int a, int b, atomic_t *v)\
+{ \
+ int res, t; \
+ \
+ __asm__ __volatile__( \
+"1: lwarx %0,0,%5 # atomic_fetch_" #op "_relaxed\n" \
+ #asm_op1 " %1,%3,%0\n" \
+ #asm_op2 " %1,%4,%1\n" \
+" stwcx. %1,0,%5\n" \
+" bne- 1b\n" \
+ : "=&r" (res), "=&r" (t), "+m" (v->counter) \
+ : "r" (a), "r" (b), "r" (&v->counter) \
+ : "cc"); \
+ \
+ return res; \
+}
+
#define ATOMIC_OPS(op, asm_op) \
ATOMIC_OP(op, asm_op) \
ATOMIC_OP_RETURN_RELAXED(op, asm_op) \
@@ -109,12 +127,20 @@ ATOMIC_OPS(and, and)
ATOMIC_OPS(or, or)
ATOMIC_OPS(xor, xor)
+#undef ATOMIC_OPS
+#define ATOMIC_OPS(op, asm_op1, asm_op2) \
+ ATOMIC_FETCH_OP2_RELAXED(op, asm_op1, asm_op2)
+
+ATOMIC_OPS(and_or, and, or)
+
#define arch_atomic_fetch_and_relaxed arch_atomic_fetch_and_relaxed
#define arch_atomic_fetch_or_relaxed arch_atomic_fetch_or_relaxed
#define arch_atomic_fetch_xor_relaxed arch_atomic_fetch_xor_relaxed
+#define arch_atomic_fetch_and_or_relaxed arch_atomic_fetch_and_or_relaxed
#undef ATOMIC_OPS
#undef ATOMIC_FETCH_OP_RELAXED
+#undef ATOMIC_FETCH_OP2_RELAXED
#undef ATOMIC_OP_RETURN_RELAXED
#undef ATOMIC_OP
diff --git a/arch/riscv/include/asm/atomic.h b/arch/riscv/include/asm/atomic.h
index ac9bdf4fc404..572ca0ae2e76 100644
--- a/arch/riscv/include/asm/atomic.h
+++ b/arch/riscv/include/asm/atomic.h
@@ -110,6 +110,24 @@ c_type arch_atomic##prefix##_fetch_##op(c_type i, atomic##prefix##_t *v) \
return ret; \
}
+#define ATOMIC_FETCH_OP2(op, asm_op1, asm_op2, asm_type, c_type, prefix) \
+static __always_inline \
+c_type arch_atomic##prefix##_fetch_##op##_relaxed(c_type i, c_type j, \
+ atomic##prefix##_t *v) \
+{ \
+ register c_type ret, tmp; \
+ __asm__ __volatile__ ( \
+ "0: lr." #asm_type " %0, %2\n" \
+ " " #asm_op1 "%1, %0, %3\n" \
+ " " #asm_op2 "%1, %1, %4\n" \
+ " sc." #asm_type " %1, %1, %2\n" \
+ " bnez %1, 0b\n" \
+ : "=r" (ret), "=&r" (tmp), "+A" (v->counter) \
+ : "r" (i), "r" (j) \
+ : "memory"); \
+ return ret; \
+}
+
#define ATOMIC_OP_RETURN(op, asm_op, c_op, I, asm_type, c_type, prefix) \
static __always_inline \
c_type arch_atomic##prefix##_##op##_return_relaxed(c_type i, \
@@ -175,9 +193,15 @@ ATOMIC_OPS(and, and, i)
ATOMIC_OPS( or, or, i)
ATOMIC_OPS(xor, xor, i)
+#define ATOMIC_OPS(op, asm_op1, asm_op2, I) \
+ ATOMIC_FETCH_OP2(op, asm_op1, asm_op2, I, w, int,)
+
+ATOMIC_OPS(and_or, and, or, w)
+
#define arch_atomic_fetch_and_relaxed arch_atomic_fetch_and_relaxed
#define arch_atomic_fetch_or_relaxed arch_atomic_fetch_or_relaxed
#define arch_atomic_fetch_xor_relaxed arch_atomic_fetch_xor_relaxed
+#define arch_atomic_fetch_and_or_relaxed arch_atomic_fetch_and_or_relaxed
#define arch_atomic_fetch_and arch_atomic_fetch_and
#define arch_atomic_fetch_or arch_atomic_fetch_or
#define arch_atomic_fetch_xor arch_atomic_fetch_xor
@@ -194,6 +218,7 @@ ATOMIC_OPS(xor, xor, i)
#undef ATOMIC_OPS
#undef ATOMIC_FETCH_OP
+#undef ATOMIC_FETCH_OP2
#undef ATOMIC_OP_RETURN
/* This is required to provide a full barrier on success. */
diff --git a/arch/s390/include/asm/atomic.h b/arch/s390/include/asm/atomic.h
index 7138d189cc42..abebd658c1fa 100644
--- a/arch/s390/include/asm/atomic.h
+++ b/arch/s390/include/asm/atomic.h
@@ -62,6 +62,7 @@ static inline int arch_atomic_fetch_##op(int i, atomic_t *v) \
ATOMIC_OPS(and)
ATOMIC_OPS(or)
ATOMIC_OPS(xor)
+ATOMIC_OPS(and_or)
#undef ATOMIC_OPS
@@ -71,6 +72,7 @@ ATOMIC_OPS(xor)
#define arch_atomic_fetch_and arch_atomic_fetch_and
#define arch_atomic_fetch_or arch_atomic_fetch_or
#define arch_atomic_fetch_xor arch_atomic_fetch_xor
+#define arch_atomic_fetch_and_or arch_atomic_fetch_and_or
#define arch_atomic_xchg(v, new) (arch_xchg(&((v)->counter), new))
diff --git a/arch/s390/include/asm/atomic_ops.h b/arch/s390/include/asm/atomic_ops.h
index 50510e08b893..d396f2e2eb9a 100644
--- a/arch/s390/include/asm/atomic_ops.h
+++ b/arch/s390/include/asm/atomic_ops.h
@@ -154,6 +154,31 @@ __ATOMIC64_OPS(__atomic64_xor, "xgr")
#endif /* CONFIG_HAVE_MARCH_Z196_FEATURES */
+#define __ATOMIC_OP2(op_name, op1, op2) \
+static inline int op_name(int i, int j, int *ptr) \
+{ \
+ int old, new; \
+ \
+ asm volatile( \
+ "0: lr %[new],%[old]\n" \
+ op1 " %[new],%[i]\n" \
+ op2 " %[new],%[j]\n" \
+ " cs %[old],%[new],%[ptr]\n" \
+ " jl 0b" \
+ : [old] "=d" (old), [new] "=&d" (new), [ptr] "+Q" (*ptr)\
+ : [i] "d" (i), [j] "d" (j), "0" (*ptr) : "cc", "memory");\
+ return old; \
+}
+
+#define __ATOMIC_OPS(op_name, op1_string, op2_string) \
+ __ATOMIC_OP2(op_name, op1_string, op2_string) \
+ __ATOMIC_OP2(op_name##_barrier, op1_string, op2_string)
+
+__ATOMIC_OPS(__atomic_and_or, "ngr", "ogr")
+
+#undef __ATOMIC_OPS
+#undef __ATOMIC_OP2
+
static inline int __atomic_cmpxchg(int *ptr, int old, int new)
{
asm volatile(
diff --git a/arch/sparc/include/asm/atomic_32.h b/arch/sparc/include/asm/atomic_32.h
index d775daa83d12..d062b20eb64c 100644
--- a/arch/sparc/include/asm/atomic_32.h
+++ b/arch/sparc/include/asm/atomic_32.h
@@ -23,6 +23,7 @@ int arch_atomic_fetch_add(int, atomic_t *);
int arch_atomic_fetch_and(int, atomic_t *);
int arch_atomic_fetch_or(int, atomic_t *);
int arch_atomic_fetch_xor(int, atomic_t *);
+int arch_atomic_fetch_and_or(int, int, atomic_t *);
int arch_atomic_cmpxchg(atomic_t *, int, int);
int arch_atomic_xchg(atomic_t *, int);
int arch_atomic_fetch_add_unless(atomic_t *, int, int);
@@ -40,6 +41,7 @@ void arch_atomic_set(atomic_t *, int);
#define arch_atomic_and(i, v) ((void)arch_atomic_fetch_and((i), (v)))
#define arch_atomic_or(i, v) ((void)arch_atomic_fetch_or((i), (v)))
#define arch_atomic_xor(i, v) ((void)arch_atomic_fetch_xor((i), (v)))
+#define arch_atomic_and_or(i, j, v) ((void)arch_atomic_fetch_and_or((i), (j), (v)))
#define arch_atomic_sub_return(i, v) (arch_atomic_add_return(-(int)(i), (v)))
#define arch_atomic_fetch_sub(i, v) (arch_atomic_fetch_add (-(int)(i), (v)))
diff --git a/arch/sparc/lib/atomic32.c b/arch/sparc/lib/atomic32.c
index 8b81d0f00c97..aefb6d91985e 100644
--- a/arch/sparc/lib/atomic32.c
+++ b/arch/sparc/lib/atomic32.c
@@ -43,6 +43,21 @@ int arch_atomic_fetch_##op(int i, atomic_t *v) \
} \
EXPORT_SYMBOL(arch_atomic_fetch_##op);
+#define ATOMIC_FETCH_OP2(op, c_op1, c_op2) \
+int arch_atomic_fetch_##op(int i, int j, atomic_t *v) \
+{ \
+ int ret; \
+ unsigned long flags; \
+ spin_lock_irqsave(ATOMIC_HASH(v), flags); \
+ \
+ ret = v->counter; \
+ v->counter = (ret c_op1 i) c_op2 j; \
+ \
+ spin_unlock_irqrestore(ATOMIC_HASH(v), flags); \
+ return ret; \
+} \
+EXPORT_SYMBOL(arch_atomic_fetch_##op);
+
#define ATOMIC_OP_RETURN(op, c_op) \
int arch_atomic_##op##_return(int i, atomic_t *v) \
{ \
@@ -63,8 +78,10 @@ ATOMIC_FETCH_OP(add, +=)
ATOMIC_FETCH_OP(and, &=)
ATOMIC_FETCH_OP(or, |=)
ATOMIC_FETCH_OP(xor, ^=)
+ATOMIC_FETCH_OP2(and_or, &, |)
#undef ATOMIC_FETCH_OP
+#undef ATOMIC_FETCH_OP2
#undef ATOMIC_OP_RETURN
int arch_atomic_xchg(atomic_t *v, int new)
diff --git a/arch/x86/include/asm/atomic.h b/arch/x86/include/asm/atomic.h
index 5e754e895767..145dce45d02a 100644
--- a/arch/x86/include/asm/atomic.h
+++ b/arch/x86/include/asm/atomic.h
@@ -263,6 +263,16 @@ static __always_inline int arch_atomic_fetch_xor(int i, atomic_t *v)
}
#define arch_atomic_fetch_xor arch_atomic_fetch_xor
+static __always_inline int arch_atomic_fetch_and_or(int i, int j, atomic_t *v)
+{
+ int val = arch_atomic_read(v);
+
+ do { } while (!arch_atomic_try_cmpxchg(v, &val, (val & i) | j));
+
+ return val;
+}
+#define arch_atomic_fetch_and_or arch_atomic_fetch_and_or
+
#ifdef CONFIG_X86_32
# include <asm/atomic64_32.h>
#else
diff --git a/arch/xtensa/include/asm/atomic.h b/arch/xtensa/include/asm/atomic.h
index 4361fe4247e3..6b043cf74df2 100644
--- a/arch/xtensa/include/asm/atomic.h
+++ b/arch/xtensa/include/asm/atomic.h
@@ -177,6 +177,28 @@ static inline int arch_atomic_fetch_##op(int i, atomic_t * v) \
return result; \
}
+#define ATOMIC_FETCH_OP2(op1, op2) \
+static inline int arch_atomic_fetch_##op1##_##op2(int i, int j, atomic_t * v)\
+{ \
+ unsigned long tmp; \
+ int result; \
+ \
+ __asm__ __volatile__( \
+ "1: l32i %[tmp], %[mem]\n" \
+ " wsr %[tmp], scompare1\n" \
+ " " #op1 " %[result], %[tmp], %[i]\n" \
+ " " #op2 " %[result], %[result], %[j]\n" \
+ " s32c1i %[result], %[mem]\n" \
+ " bne %[result], %[tmp], 1b\n" \
+ : [result] "=&a" (result), [tmp] "=&a" (tmp), \
+ [mem] "+m" (*v) \
+ : [i] "a" (i), [j] "a" (j) \
+ : "memory" \
+ ); \
+ \
+ return result; \
+}
+
#else /* XCHAL_HAVE_S32C1I */
#define ATOMIC_OP(op) \
@@ -238,6 +260,28 @@ static inline int arch_atomic_fetch_##op(int i, atomic_t * v) \
return vval; \
}
+#define ATOMIC_FETCH_OP2(op1, op2) \
+static inline int arch_atomic_fetch_##op1##_##op2(int i, int j, atomic_t * v)\
+{ \
+ unsigned int tmp, vval; \
+ \
+ __asm__ __volatile__( \
+ " rsil a15,"__stringify(TOPLEVEL)"\n" \
+ " l32i %[result], %[mem]\n" \
+ " " #op1 " %[tmp], %[result], %[i]\n" \
+ " " #op2 " %[tmp], %[tmp], %[j]\n" \
+ " s32i %[tmp], %[mem]\n" \
+ " wsr a15, ps\n" \
+ " rsync\n" \
+ : [result] "=&a" (vval), [tmp] "=&a" (tmp), \
+ [mem] "+m" (*v) \
+ : [i] "a" (i), [j] "a" (j) \
+ : "a15", "memory" \
+ ); \
+ \
+ return vval; \
+}
+
#endif /* XCHAL_HAVE_S32C1I */
#define ATOMIC_OPS(op) ATOMIC_OP(op) ATOMIC_FETCH_OP(op) ATOMIC_OP_RETURN(op)
@@ -252,6 +296,11 @@ ATOMIC_OPS(and)
ATOMIC_OPS(or)
ATOMIC_OPS(xor)
+#undef ATOMIC_OPS
+#define ATOMIC_OPS(op1, op2) ATOMIC_FETCH_OP2(op1, op2)
+
+ATOMIC_OPS(and, or)
+
#undef ATOMIC_OPS
#undef ATOMIC_FETCH_OP
#undef ATOMIC_OP_RETURN
diff --git a/include/asm-generic/atomic-instrumented.h b/include/asm-generic/atomic-instrumented.h
index bc45af52c93b..231a8386ac80 100644
--- a/include/asm-generic/atomic-instrumented.h
+++ b/include/asm-generic/atomic-instrumented.h
@@ -441,6 +441,34 @@ atomic_fetch_xor_relaxed(int i, atomic_t *v)
return arch_atomic_fetch_xor_relaxed(i, v);
}
+static __always_inline int
+atomic_fetch_and_or(int i, int j, atomic_t *v)
+{
+ instrument_atomic_read_write(v, sizeof(*v));
+ return arch_atomic_fetch_and_or(i, j, v);
+}
+
+static __always_inline int
+atomic_fetch_and_or_acquire(int i, int j, atomic_t *v)
+{
+ instrument_atomic_read_write(v, sizeof(*v));
+ return arch_atomic_fetch_and_or_acquire(i, j, v);
+}
+
+static __always_inline int
+atomic_fetch_and_or_release(int i, int j, atomic_t *v)
+{
+ instrument_atomic_read_write(v, sizeof(*v));
+ return arch_atomic_fetch_and_or_release(i, j, v);
+}
+
+static __always_inline int
+atomic_fetch_and_or_relaxed(int i, int j, atomic_t *v)
+{
+ instrument_atomic_read_write(v, sizeof(*v));
+ return arch_atomic_fetch_and_or_relaxed(i, j, v);
+}
+
static __always_inline int
atomic_xchg(atomic_t *v, int i)
{
diff --git a/include/asm-generic/atomic.h b/include/asm-generic/atomic.h
index 04b8be9f1a77..474e8cd8e58d 100644
--- a/include/asm-generic/atomic.h
+++ b/include/asm-generic/atomic.h
@@ -50,6 +50,18 @@ static inline int generic_atomic_fetch_##op(int i, atomic_t *v) \
return c; \
}
+#define ATOMIC_FETCH_OP2(op, c_op1, c_op2) \
+static inline int generic_atomic_fetch_##op(int i, int j, atomic_t *v) \
+{ \
+ int c, old; \
+ \
+ c = v->counter; \
+ while ((old = arch_cmpxchg(&v->counter, c, (c c_op1 i) c_op2 j)) != c) \
+ c = old; \
+ \
+ return c; \
+}
+
#else
#include <linux/irqflags.h>
@@ -91,6 +103,20 @@ static inline int generic_atomic_fetch_##op(int i, atomic_t *v) \
return ret; \
}
+#define ATOMIC_FETCH_OP2(op, c_op1, c_op2) \
+static inline int generic_atomic_fetch_##op(int i, int j, atomic_t *v) \
+{ \
+ unsigned long flags; \
+ int ret; \
+ \
+ raw_local_irq_save(flags); \
+ ret = v->counter; \
+ v->counter = (v->counter c_op1 i) c_op2 j; \
+ raw_local_irq_restore(flags); \
+ \
+ return ret; \
+}
+
#endif /* CONFIG_SMP */
ATOMIC_OP_RETURN(add, +)
@@ -101,6 +127,7 @@ ATOMIC_FETCH_OP(sub, -)
ATOMIC_FETCH_OP(and, &)
ATOMIC_FETCH_OP(or, |)
ATOMIC_FETCH_OP(xor, ^)
+ATOMIC_FETCH_OP2(and_or, &, |)
ATOMIC_OP(add, +)
ATOMIC_OP(sub, -)
@@ -109,6 +136,7 @@ ATOMIC_OP(or, |)
ATOMIC_OP(xor, ^)
#undef ATOMIC_FETCH_OP
+#undef ATOMIC_FETCH_OP2
#undef ATOMIC_OP_RETURN
#undef ATOMIC_OP
@@ -120,6 +148,7 @@ ATOMIC_OP(xor, ^)
#define arch_atomic_fetch_and generic_atomic_fetch_and
#define arch_atomic_fetch_or generic_atomic_fetch_or
#define arch_atomic_fetch_xor generic_atomic_fetch_xor
+#define arch_atomic_fetch_and_or generic_atomic_fetch_and_or
#define arch_atomic_add generic_atomic_add
#define arch_atomic_sub generic_atomic_sub
diff --git a/include/linux/atomic-arch-fallback.h b/include/linux/atomic-arch-fallback.h
index a3dba31df01e..92043a8d5b79 100644
--- a/include/linux/atomic-arch-fallback.h
+++ b/include/linux/atomic-arch-fallback.h
@@ -891,6 +891,48 @@ arch_atomic_fetch_xor(int i, atomic_t *v)
#endif /* arch_atomic_fetch_xor_relaxed */
+#ifndef arch_atomic_fetch_and_or_relaxed
+#define arch_atomic_fetch_and_or_acquire arch_atomic_fetch_and_or
+#define arch_atomic_fetch_and_or_release arch_atomic_fetch_and_or
+#define arch_atomic_fetch_and_or_relaxed arch_atomic_fetch_and_or
+#else /* arch_atomic_fetch_and_or_relaxed */
+
+#ifndef arch_atomic_fetch_and_or_acquire
+static __always_inline int
+arch_atomic_fetch_and_or_acquire(int i, int j, atomic_t *v)
+{
+ int ret = arch_atomic_fetch_and_or_relaxed(i, j, v);
+ __atomic_acquire_fence();
+ return ret;
+}
+#define arch_atomic_fetch_and_or_acquire arch_atomic_fetch_and_or_acquire
+#endif
+
+#ifndef arch_atomic_fetch_and_or_release
+static __always_inline int
+arch_atomic_fetch_and_or_release(int i, int j, atomic_t *v)
+{
+ __atomic_release_fence();
+ return arch_atomic_fetch_and_or_relaxed(i, j, v);
+}
+#define arch_atomic_fetch_and_or_release arch_atomic_fetch_and_or_release
+#endif
+
+#ifndef arch_atomic_fetch_and_or
+static __always_inline int
+arch_atomic_fetch_and_or(int i, int j, atomic_t *v)
+{
+ int ret;
+ __atomic_pre_full_fence();
+ ret = arch_atomic_fetch_and_or_relaxed(i, j, v);
+ __atomic_post_full_fence();
+ return ret;
+}
+#define arch_atomic_fetch_and_or arch_atomic_fetch_and_or
+#endif
+
+#endif /* arch_atomic_fetch_and_or_relaxed */
+
#ifndef arch_atomic_xchg_relaxed
#define arch_atomic_xchg_acquire arch_atomic_xchg
#define arch_atomic_xchg_release arch_atomic_xchg
--
2.32.0
