[PATCH v2 2/4] percpu_stats: Enable 64-bit counts in 32-bit architectures

[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]

 



The unsigned long type in 32-bit architectures is only 32-bit. This may
not be enough from some statistics counts that may well go over 2^32
over time. This patch optionally enables the use of 64-bit counts in
32-bit architecture, though it does add a bit of performance overhead
if enabled.

This patch adds a flags argument to the percpu_stats_init() function:

  int percpu_stats_init(struct percpu_stats *pcs, int num, int flags)

Currently, the following 2 flags are supported:

 1) PCPU_STAT_64BIT   - enable 64-bit counts
 2) PCPU_STAT_INTSAFE - make the 64-bit count update interrupt safe

The second flag isn't active if the first flag is not set.

Signed-off-by: Waiman Long <Waiman.Long@xxxxxxx>
---
 include/linux/percpu_stats.h |   28 +++++++++++-
 lib/percpu_stats.c           |   96 +++++++++++++++++++++++++++++++++++-------
 2 files changed, 105 insertions(+), 19 deletions(-)

diff --git a/include/linux/percpu_stats.h b/include/linux/percpu_stats.h
index ed6e8ac..641f211 100644
--- a/include/linux/percpu_stats.h
+++ b/include/linux/percpu_stats.h
@@ -6,15 +6,34 @@
  */
 #include <linux/percpu.h>
 #include <linux/types.h>
+#include <linux/u64_stats_sync.h>
+
+/*
+ * Supported flags for percpu_stats_init()
+ */
+#define PCPU_STAT_64BIT		1	/* Use 64-bit statistics count	  */
+#define PCPU_STAT_INTSAFE	2	/* Make percpu_add interrupt safe */
 
 struct percpu_stats {
-	unsigned long __percpu *stats;
+	union {
+		unsigned long __percpu	*stats;
+		uint64_t __percpu	*stats64;
+	};
+	struct u64_stats_sync sync;
 	int nstats;	/* Number of statistics counts in stats array */
+	int flags;
 };
 
 extern void percpu_stats_destroy(struct percpu_stats *pcs);
-extern int  percpu_stats_init(struct percpu_stats *pcs, int num);
+extern int  percpu_stats_init(struct percpu_stats *pcs, int num, int flags);
 extern uint64_t percpu_stats_sum(struct percpu_stats *pcs, int stat);
+extern void __percpu_stats_add(struct percpu_stats *pcs, int stat, int cnt);
+
+#ifdef CONFIG_64BIT
+#define PERCPU_STATS_FLAGS(pcs)	false
+#else
+#define PERCPU_STATS_FLAGS(pcs)	((pcs)->flags)
+#endif
 
 /**
  * percpu_stats_add - Add the given value to a statistics count
@@ -26,7 +45,10 @@ static inline void
 percpu_stats_add(struct percpu_stats *pcs, int stat, int cnt)
 {
 	BUG_ON((unsigned int)stat >= pcs->nstats);
-	this_cpu_add(pcs->stats[stat], cnt);
+	if (unlikely(PERCPU_STATS_FLAGS(pcs)))
+		__percpu_stats_add(pcs, stat, cnt);
+	else
+		this_cpu_add(pcs->stats[stat], cnt);
 }
 
 static inline void percpu_stats_inc(struct percpu_stats *pcs, int stat)
diff --git a/lib/percpu_stats.c b/lib/percpu_stats.c
index bc9f26d..2ec739e 100644
--- a/lib/percpu_stats.c
+++ b/lib/percpu_stats.c
@@ -5,29 +5,47 @@
 #include <linux/percpu_stats.h>
 #include <linux/bug.h>
 
+#ifdef CONFIG_64BIT
+/*
+ * Ignore PCPU_STAT_64BIT & PCPU_STAT_INTSAFE flags for 64-bit architectures
+ * as 64-bit count is the default.
+ */
+#define IS_STATS64(pcs)	false
+#define GET_FLAGS(f)	((f) & ~(PCPU_STAT_64BIT | PCPU_STAT_INTSAFE))
+#else
+#define IS_STATS64(pcs)	((pcs)->flags & PCPU_STAT_64BIT)
+#define GET_FLAGS(f)	(f)
+#endif
+
 /**
  * percpu_stats_init - allocate memory for the percpu statistics counts
- * @pcs: Pointer to percpu_stats structure
- * @num: Number of statistics counts to be used
+ * @pcs  : Pointer to percpu_stats structure
+ * @num  : Number of statistics counts to be used
+ * @flags: Optional feature bits
  * Return: 0 if successful, -ENOMEM if memory allocation fails.
  */
-int percpu_stats_init(struct percpu_stats *pcs, int num)
+int percpu_stats_init(struct percpu_stats *pcs, int num, int flags)
 {
-	int cpu;
+	int cpu, size;
 
+	pcs->flags  = GET_FLAGS(flags);
 	pcs->nstats = num;
-	pcs->stats  = __alloc_percpu(sizeof(unsigned long) * num,
-				     __alignof__(unsigned long));
-	if (!pcs->stats)
-		return -ENOMEM;
+	if (IS_STATS64(pcs)) {
+		size = sizeof(uint64_t) * num;
+		pcs->stats64 = __alloc_percpu(size, __alignof__(uint64_t));
+		if (!pcs->stats64)
+			return -ENOMEM;
+		u64_stats_init(&pcs->sync);
+	} else {
+		size = sizeof(unsigned long) * num;
+		pcs->stats = __alloc_percpu(size, __alignof__(unsigned long));
+		if (!pcs->stats)
+			return -ENOMEM;
+	}
 
-	for_each_possible_cpu(cpu) {
-		unsigned long *pstats =  per_cpu_ptr(pcs->stats, cpu);
-		int stat;
+	for_each_possible_cpu(cpu)
+		memset(per_cpu_ptr(pcs->stats, cpu), 0, size);
 
-		for (stat = 0; stat < pcs->nstats; stat++, pstats++)
-			*pstats = 0;
-	}
 	return 0;
 }
 EXPORT_SYMBOL(percpu_stats_init);
@@ -57,8 +75,54 @@ uint64_t percpu_stats_sum(struct percpu_stats *pcs, int stat)
 
 	BUG_ON((unsigned int)stat >= pcs->nstats);
 
-	for_each_possible_cpu(cpu)
-		sum += per_cpu(pcs->stats[stat], cpu);
+	if (IS_STATS64(pcs)) {
+		for_each_possible_cpu(cpu) {
+			uint64_t val;
+			unsigned int seq;
+
+			do {
+				seq = u64_stats_fetch_begin(&pcs->sync);
+				val = per_cpu(pcs->stats64[stat], cpu);
+			} while (u64_stats_fetch_retry(&pcs->sync, seq));
+			sum += val;
+		}
+	} else {
+		for_each_possible_cpu(cpu)
+			sum += per_cpu(pcs->stats[stat], cpu);
+	}
 	return sum;
 }
 EXPORT_SYMBOL(percpu_stats_sum);
+
+/**
+ * __percpu_stats_add - add given count to percpu value
+ * @pcs : Pointer to percpu_stats structure
+ * @stat: The statistics count that needs to be updated
+ * @cnt:  The value to be added to the statistics count
+ */
+void __percpu_stats_add(struct percpu_stats *pcs, int stat, int cnt)
+{
+	/*
+	 * u64_stats_update_begin/u64_stats_update_end alone are not safe
+	 * against recursive add on the same CPU caused by interrupt.
+	 * So we need to set the PCPU_STAT_INTSAFE flag if this is required.
+	 */
+	if (IS_STATS64(pcs)) {
+		uint64_t *pstats64;
+		unsigned long flags;
+
+		pstats64 = get_cpu_ptr(pcs->stats64);
+		if (pcs->flags & PCPU_STAT_INTSAFE)
+			local_irq_save(flags);
+
+		u64_stats_update_begin(&pcs->sync);
+		pstats64[stat] += cnt;
+		u64_stats_update_end(&pcs->sync);
+
+		if (pcs->flags & PCPU_STAT_INTSAFE)
+			local_irq_restore(flags);
+
+		put_cpu_ptr(pcs->stats64);
+	}
+}
+EXPORT_SYMBOL(__percpu_stats_add);
-- 
1.7.1

--
To unsubscribe from this list: send the line "unsubscribe linux-ext4" in
the body of a message to majordomo@xxxxxxxxxxxxxxx
More majordomo info at  http://vger.kernel.org/majordomo-info.html



[Index of Archives]     [Reiser Filesystem Development]     [Ceph FS]     [Kernel Newbies]     [Security]     [Netfilter]     [Bugtraq]     [Linux FS]     [Yosemite National Park]     [MIPS Linux]     [ARM Linux]     [Linux Security]     [Linux RAID]     [Samba]     [Device Mapper]     [Linux Media]

  Powered by Linux