From: ZhangPeng <zhangpeng362@xxxxxxxxxx>
Depending on whether counters is NULL, we can support two modes:
atomic mode and perpcu mode. We implement both modes by grouping
the s64 count and atomic64_t count_atomic in a union. At the same time,
we create the interface for adding and reading in atomic mode and for
switching atomic mode to percpu mode.
Suggested-by: Jan Kara <jack@xxxxxxx>
Signed-off-by: ZhangPeng <zhangpeng362@xxxxxxxxxx>
Signed-off-by: Kefeng Wang <wangkefeng.wang@xxxxxxxxxx>
---
include/linux/percpu_counter.h | 43 +++++++++++++++++++++++++++++++---
lib/percpu_counter.c | 31 ++++++++++++++++++++++--
2 files changed, 69 insertions(+), 5 deletions(-)
diff --git a/include/linux/percpu_counter.h b/include/linux/percpu_counter.h
index 3a44dd1e33d2..160f9734c0bb 100644
--- a/include/linux/percpu_counter.h
+++ b/include/linux/percpu_counter.h
@@ -21,7 +21,13 @@
struct percpu_counter {
raw_spinlock_t lock;
- s64 count;
+ /* Depending on whether counters is NULL, we can support two modes,
+ * atomic mode using count_atomic and perpcu mode using count.
+ */
+ union {
+ s64 count;
+ atomic64_t count_atomic;
+ };
#ifdef CONFIG_HOTPLUG_CPU
struct list_head list; /* All percpu_counters are on a list */
#endif
@@ -32,14 +38,14 @@ extern int percpu_counter_batch;
int __percpu_counter_init_many(struct percpu_counter *fbc, s64 amount,
gfp_t gfp, u32 nr_counters,
- struct lock_class_key *key);
+ struct lock_class_key *key, bool switch_mode);
#define percpu_counter_init_many(fbc, value, gfp, nr_counters) \
({ \
static struct lock_class_key __key; \
\
__percpu_counter_init_many(fbc, value, gfp, nr_counters,\
- &__key); \
+ &__key, false); \
})
@@ -130,6 +136,20 @@ static inline bool percpu_counter_initialized(struct percpu_counter *fbc)
return (fbc->counters != NULL);
}
+static inline s64 percpu_counter_atomic_read(struct percpu_counter *fbc)
+{
+ return atomic64_read(&fbc->count_atomic);
+}
+
+static inline void percpu_counter_atomic_add(struct percpu_counter *fbc,
+ s64 amount)
+{
+ atomic64_add(amount, &fbc->count_atomic);
+}
+
+int percpu_counter_switch_to_pcpu_many(struct percpu_counter *fbc,
+ u32 nr_counters);
+
#else /* !CONFIG_SMP */
struct percpu_counter {
@@ -260,6 +280,23 @@ static inline bool percpu_counter_initialized(struct percpu_counter *fbc)
static inline void percpu_counter_sync(struct percpu_counter *fbc)
{
}
+
+static inline s64 percpu_counter_atomic_read(struct percpu_counter *fbc)
+{
+ return fbc->count;
+}
+
+static inline void percpu_counter_atomic_add(struct percpu_counter *fbc,
+ s64 amount)
+{
+ percpu_counter_add(fbc, amount);
+}
+
+static inline int percpu_counter_switch_to_pcpu_many(struct percpu_counter *fbc,
+ u32 nr_counters)
+{
+ return 0;
+}
#endif /* CONFIG_SMP */
static inline void percpu_counter_inc(struct percpu_counter *fbc)
diff --git a/lib/percpu_counter.c b/lib/percpu_counter.c
index 44dd133594d4..95c4e038051a 100644
--- a/lib/percpu_counter.c
+++ b/lib/percpu_counter.c
@@ -153,7 +153,7 @@ EXPORT_SYMBOL(__percpu_counter_sum);
int __percpu_counter_init_many(struct percpu_counter *fbc, s64 amount,
gfp_t gfp, u32 nr_counters,
- struct lock_class_key *key)
+ struct lock_class_key *key, bool switch_mode)
{
unsigned long flags __maybe_unused;
size_t counter_size;
@@ -174,7 +174,8 @@ int __percpu_counter_init_many(struct percpu_counter *fbc, s64 amount,
#ifdef CONFIG_HOTPLUG_CPU
INIT_LIST_HEAD(&fbc[i].list);
#endif
- fbc[i].count = amount;
+ if (likely(!switch_mode))
+ fbc[i].count = amount;
fbc[i].counters = (void *)counters + (i * counter_size);
debug_percpu_counter_activate(&fbc[i]);
@@ -357,6 +358,32 @@ bool __percpu_counter_limited_add(struct percpu_counter *fbc,
return good;
}
+/*
+ * percpu_counter_switch_to_pcpu_many: Converts struct percpu_counters from
+ * atomic mode to percpu mode.
+ */
+int percpu_counter_switch_to_pcpu_many(struct percpu_counter *fbc,
+ u32 nr_counters)
+{
+ static struct lock_class_key __key;
+ unsigned long flags;
+ bool ret = 0;
+
+ if (percpu_counter_initialized(fbc))
+ return 0;
+
+ preempt_disable();
+ local_irq_save(flags);
+ if (likely(!percpu_counter_initialized(fbc)))
+ ret = __percpu_counter_init_many(fbc, 0,
+ GFP_ATOMIC|__GFP_NOWARN|__GFP_ZERO,
+ nr_counters, &__key, true);
+ local_irq_restore(flags);
+ preempt_enable();
+
+ return ret;
+}
+
static int __init percpu_counter_startup(void)
{
int ret;
--
2.25.1
