From: ZhangPeng <zhangpeng362@xxxxxxxxxx>
Add the struct percpu_counter fbc to struct lazy_percpu_counter.
Convert the u64 __percpu parameter of the lazy percpu counter function
to the struct percpu_counter parameter to prepare for converting
mm's rss stats into lazy_percpu_counter.
Signed-off-by: ZhangPeng <zhangpeng362@xxxxxxxxxx>
Signed-off-by: Kefeng Wang <wangkefeng.wang@xxxxxxxxxx>
---
include/linux/lazy-percpu-counter.h | 16 ++++--
lib/lazy-percpu-counter.c | 83 +++++++++++++++++++++++------
2 files changed, 77 insertions(+), 22 deletions(-)
diff --git a/include/linux/lazy-percpu-counter.h b/include/linux/lazy-percpu-counter.h
index 281b8dd88cb2..03ff24f0128d 100644
--- a/include/linux/lazy-percpu-counter.h
+++ b/include/linux/lazy-percpu-counter.h
@@ -20,15 +20,21 @@
#define _LINUX_LAZY_PERCPU_COUNTER_H
#include <linux/atomic.h>
+#include <linux/percpu_counter.h>
#include <asm/percpu.h>
struct lazy_percpu_counter {
atomic64_t v;
unsigned long last_wrap;
+ struct percpu_counter fbc;
};
-void lazy_percpu_counter_exit(struct lazy_percpu_counter *c);
+void lazy_percpu_counter_destroy_many(struct lazy_percpu_counter *c,
+ u32 nr_counters);
void lazy_percpu_counter_add_slowpath(struct lazy_percpu_counter *c, s64 i);
+s64 lazy_percpu_counter_read_positive(struct lazy_percpu_counter *c);
+s64 lazy_percpu_counter_sum(struct lazy_percpu_counter *c);
+s64 lazy_percpu_counter_sum_positive(struct lazy_percpu_counter *c);
/*
* We use the high bits of the atomic counter for a secondary counter, which is
@@ -48,13 +54,13 @@ void lazy_percpu_counter_add_slowpath(struct lazy_percpu_counter *c, s64 i);
*/
#define COUNTER_IS_PCPU_BIT 1
-static inline u64 __percpu *lazy_percpu_counter_is_pcpu(u64 v)
+static inline struct percpu_counter *lazy_percpu_counter_is_pcpu(u64 v)
{
if (!(v & COUNTER_IS_PCPU_BIT))
return NULL;
v ^= COUNTER_IS_PCPU_BIT;
- return (u64 __percpu *)(unsigned long)v;
+ return (struct percpu_counter *)(unsigned long)v;
}
/**
@@ -66,10 +72,10 @@ static inline u64 __percpu *lazy_percpu_counter_is_pcpu(u64 v)
static inline void lazy_percpu_counter_add(struct lazy_percpu_counter *c, s64 i)
{
u64 v = atomic64_read(&c->v);
- u64 __percpu *pcpu_v = lazy_percpu_counter_is_pcpu(v);
+ struct percpu_counter *pcpu_v = lazy_percpu_counter_is_pcpu(v);
if (likely(pcpu_v))
- this_cpu_add(*pcpu_v, i);
+ percpu_counter_add(pcpu_v, i);
else
lazy_percpu_counter_add_slowpath(c, i);
}
diff --git a/lib/lazy-percpu-counter.c b/lib/lazy-percpu-counter.c
index e1914207214d..c360903cc02a 100644
--- a/lib/lazy-percpu-counter.c
+++ b/lib/lazy-percpu-counter.c
@@ -15,45 +15,94 @@ static inline s64 lazy_percpu_counter_atomic_val(s64 v)
static void lazy_percpu_counter_switch_to_pcpu(struct lazy_percpu_counter *c)
{
- u64 __percpu *pcpu_v = alloc_percpu_gfp(u64, GFP_ATOMIC|__GFP_NOWARN);
u64 old, new, v;
+ unsigned long flags;
+ bool allocated = false;
- if (!pcpu_v)
- return;
-
+ local_irq_save(flags);
preempt_disable();
v = atomic64_read(&c->v);
do {
- if (lazy_percpu_counter_is_pcpu(v)) {
- free_percpu(pcpu_v);
- return;
+ if (lazy_percpu_counter_is_pcpu(v))
+ break;
+
+ if (!allocated) {
+ if (percpu_counter_init(&c->fbc, 0, GFP_ATOMIC|__GFP_NOWARN))
+ break;
+ allocated = true;
}
old = v;
- new = (unsigned long)pcpu_v | 1;
+ new = (unsigned long)&c->fbc | 1;
- *this_cpu_ptr(pcpu_v) = lazy_percpu_counter_atomic_val(v);
+ percpu_counter_set(&c->fbc, lazy_percpu_counter_atomic_val(v));
} while ((v = atomic64_cmpxchg(&c->v, old, new)) != old);
preempt_enable();
+ local_irq_restore(flags);
}
/**
- * lazy_percpu_counter_exit: Free resources associated with a
- * lazy_percpu_counter
+ * lazy_percpu_counter_destroy_many: Free resources associated with
+ * lazy_percpu_counters
*
- * @c: counter to exit
+ * @c: counters to exit
+ * @nr_counters: number of counters
*/
-void lazy_percpu_counter_exit(struct lazy_percpu_counter *c)
+void lazy_percpu_counter_destroy_many(struct lazy_percpu_counter *c,
+ u32 nr_counters)
+{
+ struct percpu_counter *pcpu_v;
+ u32 i;
+
+ for (i = 0; i < nr_counters; i++) {
+ pcpu_v = lazy_percpu_counter_is_pcpu(atomic64_read(&c[i].v));
+ if (pcpu_v)
+ percpu_counter_destroy(pcpu_v);
+ }
+}
+EXPORT_SYMBOL_GPL(lazy_percpu_counter_destroy_many);
+
+s64 lazy_percpu_counter_read_positive(struct lazy_percpu_counter *c)
+{
+ s64 v = atomic64_read(&c->v);
+ struct percpu_counter *pcpu_v = lazy_percpu_counter_is_pcpu(v);
+
+ if (pcpu_v)
+ return percpu_counter_read_positive(pcpu_v);
+
+ return lazy_percpu_counter_atomic_val(v);
+}
+EXPORT_SYMBOL_GPL(lazy_percpu_counter_read_positive);
+
+s64 lazy_percpu_counter_sum(struct lazy_percpu_counter *c)
+{
+ s64 v = atomic64_read(&c->v);
+ struct percpu_counter *pcpu_v = lazy_percpu_counter_is_pcpu(v);
+
+ if (pcpu_v)
+ return percpu_counter_sum(pcpu_v);
+
+ return lazy_percpu_counter_atomic_val(v);
+}
+EXPORT_SYMBOL_GPL(lazy_percpu_counter_sum);
+
+s64 lazy_percpu_counter_sum_positive(struct lazy_percpu_counter *c)
{
- free_percpu(lazy_percpu_counter_is_pcpu(atomic64_read(&c->v)));
+ s64 v = atomic64_read(&c->v);
+ struct percpu_counter *pcpu_v = lazy_percpu_counter_is_pcpu(v);
+
+ if (pcpu_v)
+ return percpu_counter_sum_positive(pcpu_v);
+
+ return lazy_percpu_counter_atomic_val(v);
}
-EXPORT_SYMBOL_GPL(lazy_percpu_counter_exit);
+EXPORT_SYMBOL_GPL(lazy_percpu_counter_sum_positive);
void lazy_percpu_counter_add_slowpath(struct lazy_percpu_counter *c, s64 i)
{
u64 atomic_i;
u64 old, v = atomic64_read(&c->v);
- u64 __percpu *pcpu_v;
+ struct percpu_counter *pcpu_v;
atomic_i = i << COUNTER_IS_PCPU_BIT;
atomic_i &= ~COUNTER_MOD_MASK;
@@ -62,7 +111,7 @@ void lazy_percpu_counter_add_slowpath(struct lazy_percpu_counter *c, s64 i)
do {
pcpu_v = lazy_percpu_counter_is_pcpu(v);
if (pcpu_v) {
- this_cpu_add(*pcpu_v, i);
+ percpu_counter_add(pcpu_v, i);
return;
}
--
2.25.1
