From: Nadav Amit <namit@xxxxxxxxxx>
Introduce cpumask_atomic_or() and bitmask_atomic_or() to allow to
perform atomic or operations atomically on cpumasks. This will be used
by the next patch.
To be more efficient, skip atomic operations when no changes are needed.
Signed-off-by: Nadav Amit <namit@xxxxxxxxxx>
Cc: Mel Gorman <mgorman@xxxxxxxxxxxxxxxxxxx>
Cc: Andrea Arcangeli <aarcange@xxxxxxxxxx>
Cc: Andrew Morton <akpm@xxxxxxxxxxxxxxxxxxxx>
Cc: Andy Lutomirski <luto@xxxxxxxxxx>
Cc: Dave Hansen <dave.hansen@xxxxxxxxxxxxxxx>
Cc: Peter Zijlstra <peterz@xxxxxxxxxxxxx>
Cc: Thomas Gleixner <tglx@xxxxxxxxxxxxx>
Cc: Will Deacon <will@xxxxxxxxxx>
Cc: Yu Zhao <yuzhao@xxxxxxxxxx>
Cc: x86@xxxxxxxxxx
---
include/linux/bitmap.h | 5 +++++
include/linux/cpumask.h | 12 ++++++++++++
lib/bitmap.c | 25 +++++++++++++++++++++++++
3 files changed, 42 insertions(+)
diff --git a/include/linux/bitmap.h b/include/linux/bitmap.h
index 769b7a98e12f..c9a9b784b244 100644
--- a/include/linux/bitmap.h
+++ b/include/linux/bitmap.h
@@ -76,6 +76,7 @@
* bitmap_to_arr32(buf, src, nbits) Copy nbits from buf to u32[] dst
* bitmap_get_value8(map, start) Get 8bit value from map at start
* bitmap_set_value8(map, value, start) Set 8bit value to map at start
+ * bitmap_atomic_or(dst, src, nbits) *dst |= *src (atomically)
*
* Note, bitmap_zero() and bitmap_fill() operate over the region of
* unsigned longs, that is, bits behind bitmap till the unsigned long
@@ -577,6 +578,10 @@ static inline void bitmap_set_value8(unsigned long *map, unsigned long value,
map[index] |= value << offset;
}
+extern void bitmap_atomic_or(volatile unsigned long *dst,
+ const volatile unsigned long *bitmap, unsigned int bits);
+
+
#endif /* __ASSEMBLY__ */
#endif /* __LINUX_BITMAP_H */
diff --git a/include/linux/cpumask.h b/include/linux/cpumask.h
index 3d7e418aa113..0567d73a0192 100644
--- a/include/linux/cpumask.h
+++ b/include/linux/cpumask.h
@@ -699,6 +699,18 @@ static inline unsigned int cpumask_size(void)
return BITS_TO_LONGS(nr_cpumask_bits) * sizeof(long);
}
+/**
+ * cpumask_atomic_or - *dstp |= *srcp (*dstp is set atomically)
+ * @dstp: the cpumask result (and source which is or'd)
+ * @srcp: the source input
+ */
+static inline void cpumask_atomic_or(volatile struct cpumask *dstp,
+ const volatile struct cpumask *srcp)
+{
+ bitmap_atomic_or(cpumask_bits(dstp), cpumask_bits(srcp),
+ nr_cpumask_bits);
+}
+
/*
* cpumask_var_t: struct cpumask for stack usage.
*
diff --git a/lib/bitmap.c b/lib/bitmap.c
index 6df7b13727d3..50f1842ff891 100644
--- a/lib/bitmap.c
+++ b/lib/bitmap.c
@@ -1310,3 +1310,28 @@ void bitmap_to_arr32(u32 *buf, const unsigned long *bitmap, unsigned int nbits)
EXPORT_SYMBOL(bitmap_to_arr32);
#endif
+
+void bitmap_atomic_or(volatile unsigned long *dst,
+ const volatile unsigned long *bitmap, unsigned int bits)
+{
+ unsigned int k;
+ unsigned int nr = BITS_TO_LONGS(bits);
+
+ for (k = 0; k < nr; k++) {
+ unsigned long src = bitmap[k];
+
+ /*
+ * Skip atomic operations when no bits are changed. Do not use
+ * bitmap[k] directly to avoid redundant loads as bitmap
+ * variable is volatile.
+ */
+ if (!(src & ~dst[k]))
+ continue;
+
+ if (BITS_PER_LONG == 64)
+ atomic64_or(src, (atomic64_t*)&dst[k]);
+ else
+ atomic_or(src, (atomic_t*)&dst[k]);
+ }
+}
+EXPORT_SYMBOL(bitmap_atomic_or);
--
2.25.1
