Add torture test to verify percpu managed mode operations,
verifying that a percpu ref does not have non-zero count when
all users have dropped their reference and that there is no early
release of the ref while users hold references to it.
Signed-off-by: Neeraj Upadhyay <Neeraj.Upadhyay@xxxxxxx>
---
.../admin-guide/kernel-parameters.txt | 57 +++
lib/Kconfig.debug | 9 +
lib/Makefile | 1 +
lib/percpu-refcount-torture.c | 367 ++++++++++++++++++
lib/percpu-refcount.c | 49 ++-
lib/percpu-refcount.h | 6 +
6 files changed, 483 insertions(+), 6 deletions(-)
create mode 100644 lib/percpu-refcount-torture.c
create mode 100644 lib/percpu-refcount.h
diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt
index 0f02a1b04fe9..225f2dac294d 100644
--- a/Documentation/admin-guide/kernel-parameters.txt
+++ b/Documentation/admin-guide/kernel-parameters.txt
@@ -4677,6 +4677,63 @@
Default: 5000 ms
+ percpu_refcount_torture.busted_early_ref_release= [KNL]
+ Enable testing buggy release of percpu ref while
+ there are active users. Used for testing failure
+ scenarios in the test.
+
+ Default: 0 (disabled)
+
+ percpu_refcount_torture.busted_late_ref_release= [KNL]
+ Enable testing buggy non-zero reference count after
+ all ref users have dropped their reference. Used for
+ testing failure scenarios in the test.
+
+ Default: 0 (disabled)
+
+ percpu_refcount_torture.delay_us= [KNL]
+ Delay (in us) between reference increment and decrement
+ operations of ref users.
+
+ Default: 10
+
+ percpu_refcount_torture.niterations= [KNL]
+ Number of iterations of ref increment and decrement by
+ ref users.
+
+ Default: 100
+
+ percpu_refcount_torture.nrefs= [KNL]
+ Number of percpu ref instances.
+
+ Default: 2
+
+ percpu_refcount_torture.nusers= [KNL]
+ Number of percpu ref user threads which increment and
+ decrement a percpu ref.
+
+ Default: 2
+
+ percpu_refcount_torture.onoff_holdoff= [KNL]
+ Set time (s) after boot for CPU-hotplug testing.
+
+ Default: 0
+
+ percpu_refcount_torture.onoff_interval= [KNL]
+ Set time (jiffies) between CPU-hotplug operations,
+ or zero to disable CPU-hotplug testing.
+
+ percpu_refcount_torture.stutter= [KNL]
+ Set wait time (jiffies) between two iterations of
+ percpu ref operations.
+
+ Default: 0
+
+ percpu_refcount_torture.verbose= [KNL]
+ Enable additional printk() statements.
+
+ Default: 0 (Disabled)
+
pirq= [SMP,APIC] Manual mp-table setup
See Documentation/arch/x86/i386/IO-APIC.rst.
diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug
index a30c03a66172..7e0117e01f05 100644
--- a/lib/Kconfig.debug
+++ b/lib/Kconfig.debug
@@ -1611,6 +1611,15 @@ config SCF_TORTURE_TEST
module may be built after the fact on the running kernel to
be tested, if desired.
+config PERCPU_REFCOUNT_TORTURE_TEST
+ tristate "torture tests for percpu refcount"
+ select TORTURE_TEST
+ help
+ This options provides a kernel module that runs percpu
+ refcount torture tests for managed percpu refs. The kernel
+ module may be built after the fact on the running kernel
+ to be tested, if desired.
+
config CSD_LOCK_WAIT_DEBUG
bool "Debugging for csd_lock_wait(), called from smp_call_function*()"
depends on DEBUG_KERNEL
diff --git a/lib/Makefile b/lib/Makefile
index 322bb127b4dc..d0286f7dfb37 100644
--- a/lib/Makefile
+++ b/lib/Makefile
@@ -50,6 +50,7 @@ obj-y += bcd.o sort.o parser.o debug_locks.o random32.o \
once.o refcount.o rcuref.o usercopy.o errseq.o bucket_locks.o \
generic-radix-tree.o bitmap-str.o
obj-$(CONFIG_STRING_KUNIT_TEST) += string_kunit.o
+obj-$(CONFIG_PERCPU_REFCOUNT_TORTURE_TEST) += percpu-refcount-torture.o
obj-y += string_helpers.o
obj-$(CONFIG_STRING_HELPERS_KUNIT_TEST) += string_helpers_kunit.o
obj-y += hexdump.o
diff --git a/lib/percpu-refcount-torture.c b/lib/percpu-refcount-torture.c
new file mode 100644
index 000000000000..686f5a228b40
--- /dev/null
+++ b/lib/percpu-refcount-torture.c
@@ -0,0 +1,367 @@
+// SPDX-License-Identifier: GPL-2.0-only
+#include <linux/delay.h>
+#include <linux/jiffies.h>
+#include <linux/module.h>
+#include <linux/moduleparam.h>
+#include <linux/percpu-refcount.h>
+#include <linux/torture.h>
+
+#include "percpu-refcount.h"
+
+static int busted_early_ref_release;
+module_param(busted_early_ref_release, int, 0444);
+MODULE_PARM_DESC(busted_early_ref_release,
+ "Enable busted premature release of ref (default = 0), 0 = disable");
+
+static int busted_late_ref_release;
+module_param(busted_late_ref_release, int, 0444);
+MODULE_PARM_DESC(busted_late_ref_release,
+ "Enable busted late release of ref (default = 0), 0 = disable");
+
+static long delay_us = 10;
+module_param(delay_us, long, 0444);
+MODULE_PARM_DESC(delay_us,
+ "delay between reader refcount operations in microseconds (default = 10)");
+
+static long nrefs = 2;
+module_param(nrefs, long, 0444);
+MODULE_PARM_DESC(nrefs, "Number of percpu refs (default = 2)");
+
+static long niterations = 100;
+module_param(niterations, long, 0444);
+MODULE_PARM_DESC(niterations,
+ "Number of iterations of ref increment and decrement (default = 100)");
+
+static long nusers = 2;
+module_param(nusers, long, 0444);
+MODULE_PARM_DESC(nusers, "Number of refcount users (default = 2)");
+
+static int onoff_holdoff;
+module_param(onoff_holdoff, int, 0444);
+MODULE_PARM_DESC(onoff_holdoff, "Time after boot before CPU hotplugs (seconds)");
+
+static int onoff_interval;
+module_param(onoff_interval, int, 0444);
+MODULE_PARM_DESC(onoff_interval, "Time between CPU hotplugs (jiffies), 0=disable");
+
+static int stutter;
+module_param(stutter, int, 0444);
+MODULE_PARM_DESC(stutter, "Stutter period in jiffies (default = 0), 0 = disable");
+
+static int verbose = 1;
+module_param(verbose, int, 0444);
+MODULE_PARM_DESC(verbose, "Enable verbose debugging printk()s");
+
+static struct task_struct **ref_user_tasks;
+static struct task_struct *ref_manager_task;
+static struct task_struct **busted_early_release_tasks;
+static struct task_struct **busted_late_release_tasks;
+
+static struct percpu_ref *refs;
+static long *num_per_ref_users;
+
+static atomic_t running;
+static atomic_t *ref_running;
+
+static char *torture_type = "percpu-refcount";
+
+static int percpu_ref_manager_thread(void *data)
+{
+ int i;
+
+ while (atomic_read(&running) != 0) {
+ percpu_ref_test_flush_release_work();
+ stutter_wait("percpu_ref_manager_thread");
+ }
+ /* Ensure ordering with ref users */
+ smp_mb();
+
+ percpu_ref_test_flush_release_work();
+
+ for (i = 0; i < nrefs; i++) {
+ WARN(percpu_ref_test_is_percpu(&refs[i]),
+ "!!! released ref %d should be in atomic mode", i);
+ WARN(!percpu_ref_is_zero(&refs[i]),
+ "!!! released ref %d should have 0 refcount", i);
+ }
+
+ do {
+ stutter_wait("percpu_ref_manager_thread");
+ } while (!torture_must_stop());
+
+ torture_kthread_stopping("percpu_ref_manager_thread");
+
+ return 0;
+}
+
+static int percpu_ref_test_thread(void *data)
+{
+ struct percpu_ref *ref = (struct percpu_ref *)data;
+ int i = 0;
+
+ percpu_ref_get(ref);
+
+ do {
+ percpu_ref_get(ref);
+ udelay(delay_us);
+ percpu_ref_put(ref);
+ stutter_wait("percpu_ref_test_thread");
+ i++;
+ } while (i < niterations);
+
+ atomic_dec(&ref_running[ref - refs]);
+ /* Order ref release with ref_running[ref_idx] == 0 */
+ smp_mb();
+ percpu_ref_put(ref);
+ /* Order ref decrement with running == 0 */
+ smp_mb();
+ atomic_dec(&running);
+
+ do {
+ stutter_wait("percpu_ref_test_thread");
+ } while (!torture_must_stop());
+
+ torture_kthread_stopping("percpu_ref_test_thread");
+
+ return 0;
+}
+
+static int percpu_ref_busted_early_thread(void *data)
+{
+ struct percpu_ref *ref = (struct percpu_ref *)data;
+ int ref_idx = ref - refs;
+ int i = 0, j;
+
+ do {
+ /* Extra ref put momemtarily */
+ for (j = 0; j < num_per_ref_users[ref_idx]; j++)
+ percpu_ref_put(ref);
+ stutter_wait("percpu_ref_busted_early_thread");
+ for (j = 0; j < num_per_ref_users[ref_idx]; j++)
+ percpu_ref_get(ref);
+ i++;
+ stutter_wait("percpu_ref_busted_early_thread");
+ } while (i < niterations * 10);
+
+ do {
+ stutter_wait("percpu_ref_busted_early_thread");
+ } while (!torture_must_stop());
+
+ torture_kthread_stopping("percpu_ref_busted_early_thread");
+
+ return 0;
+}
+
+static int percpu_ref_busted_late_thread(void *data)
+{
+ struct percpu_ref *ref = (struct percpu_ref *)data;
+ int i = 0;
+
+ do {
+ /* Extra ref get momemtarily */
+ percpu_ref_get(ref);
+ stutter_wait("percpu_ref_busted_late_thread");
+ percpu_ref_put(ref);
+ i++;
+ } while (i < niterations);
+
+ do {
+ stutter_wait("percpu_ref_busted_late_thread");
+ } while (!torture_must_stop());
+
+ torture_kthread_stopping("percpu_ref_busted_late_thread");
+
+ return 0;
+}
+
+static void percpu_ref_test_cleanup(void)
+{
+ int i;
+
+ if (torture_cleanup_begin())
+ return;
+
+ if (busted_late_release_tasks) {
+ for (i = 0; i < nrefs; i++)
+ torture_stop_kthread(busted_late_task, busted_late_release_tasks[i]);
+ kfree(busted_late_release_tasks);
+ busted_late_release_tasks = NULL;
+ }
+
+ if (busted_early_release_tasks) {
+ for (i = 0; i < nrefs; i++)
+ torture_stop_kthread(busted_early_task, busted_early_release_tasks[i]);
+ kfree(busted_early_release_tasks);
+ busted_early_release_tasks = NULL;
+ }
+
+ if (ref_manager_task) {
+ torture_stop_kthread(ref_manager, ref_manager_task);
+ ref_manager_task = NULL;
+ }
+
+ if (ref_user_tasks) {
+ for (i = 0; i < nusers; i++)
+ torture_stop_kthread(ref_user, ref_user_tasks[i]);
+ kfree(ref_user_tasks);
+ ref_user_tasks = NULL;
+ }
+
+ kfree(ref_running);
+ ref_running = NULL;
+
+ kfree(num_per_ref_users);
+ num_per_ref_users = NULL;
+
+ if (refs) {
+ for (i = 0; i < nrefs; i++)
+ percpu_ref_exit(&refs[i]);
+ kfree(refs);
+ refs = NULL;
+ }
+
+ torture_cleanup_end();
+}
+
+static void percpu_ref_test_release(struct percpu_ref *ref)
+{
+ WARN(!!atomic_add_return(0, &ref_running[ref-refs]), "!!! Premature ref release");
+}
+
+static int __init percpu_ref_torture_init(void)
+{
+ DEFINE_TORTURE_RANDOM(rand);
+ struct torture_random_state *trsp = &rand;
+ int flags;
+ int err;
+ int ref_idx;
+ int i;
+
+ if (!torture_init_begin("percpu-refcount", verbose))
+ return -EBUSY;
+
+ atomic_set(&running, nusers);
+ /* Order @running with later increment and decrement operations */
+ smp_mb();
+
+ refs = kcalloc(nrefs, sizeof(refs[0]), GFP_KERNEL);
+ if (!refs) {
+ TOROUT_ERRSTRING("out of memory");
+ err = -ENOMEM;
+ goto init_err;
+ }
+ for (i = 0; i < nrefs; i++) {
+ flags = torture_random(trsp) & 1 ? PERCPU_REF_INIT_ATOMIC : PERCPU_REF_REL_MANAGED;
+ err = percpu_ref_init(&refs[i], percpu_ref_test_release,
+ flags, GFP_KERNEL);
+ if (err)
+ goto init_err;
+ if (!(flags & PERCPU_REF_REL_MANAGED))
+ percpu_ref_switch_to_managed(&refs[i]);
+ }
+
+ num_per_ref_users = kcalloc(nrefs, sizeof(num_per_ref_users[0]), GFP_KERNEL);
+ if (!num_per_ref_users) {
+ TOROUT_ERRSTRING("out of memory");
+ err = -ENOMEM;
+ goto init_err;
+ }
+ for (i = 0; i < nrefs; i++)
+ num_per_ref_users[i] = 0;
+
+ ref_user_tasks = kcalloc(nusers, sizeof(ref_user_tasks[0]), GFP_KERNEL);
+ if (!ref_user_tasks) {
+ TOROUT_ERRSTRING("out of memory");
+ err = -ENOMEM;
+ goto init_err;
+ }
+
+ ref_running = kcalloc(nrefs, sizeof(ref_running[0]), GFP_KERNEL);
+ if (!ref_running) {
+ TOROUT_ERRSTRING("out of memory");
+ err = -ENOMEM;
+ goto init_err;
+ }
+
+ for (i = 0; i < nusers; i++) {
+ ref_idx = torture_random(trsp) % nrefs;
+ atomic_inc(&ref_running[ref_idx]);
+ num_per_ref_users[ref_idx]++;
+ /* Order increments with subquent reads */
+ smp_mb();
+ err = torture_create_kthread(percpu_ref_test_thread,
+ &refs[ref_idx], ref_user_tasks[i]);
+ if (torture_init_error(err))
+ goto init_err;
+ }
+
+ err = torture_create_kthread(percpu_ref_manager_thread, NULL, ref_manager_task);
+ if (torture_init_error(err))
+ goto init_err;
+
+ /* Drop initial reference, after test threads have started running */
+ udelay(1);
+ for (i = 0; i < nrefs; i++)
+ percpu_ref_put(&refs[i]);
+
+
+ if (busted_early_ref_release) {
+ busted_early_release_tasks = kcalloc(nrefs,
+ sizeof(busted_early_release_tasks[0]),
+ GFP_KERNEL);
+ if (!busted_early_release_tasks) {
+ TOROUT_ERRSTRING("out of memory");
+ err = -ENOMEM;
+ goto init_err;
+ }
+ for (i = 0; i < nrefs; i++) {
+ err = torture_create_kthread(percpu_ref_busted_early_thread,
+ &refs[i], busted_early_release_tasks[i]);
+ if (torture_init_error(err))
+ goto init_err;
+ }
+ }
+
+ if (busted_late_ref_release) {
+ busted_late_release_tasks = kcalloc(nrefs, sizeof(busted_late_release_tasks[0]),
+ GFP_KERNEL);
+ if (!busted_late_release_tasks) {
+ TOROUT_ERRSTRING("out of memory");
+ err = -ENOMEM;
+ goto init_err;
+ }
+ for (i = 0; i < nrefs; i++) {
+ err = torture_create_kthread(percpu_ref_busted_late_thread,
+ &refs[i], busted_late_release_tasks[i]);
+ if (torture_init_error(err))
+ goto init_err;
+ }
+ }
+ if (stutter) {
+ err = torture_stutter_init(stutter, stutter);
+ if (torture_init_error(err))
+ goto init_err;
+ }
+
+ err = torture_onoff_init(onoff_holdoff * HZ, onoff_interval, NULL);
+ if (torture_init_error(err))
+ goto init_err;
+
+ torture_init_end();
+ return 0;
+init_err:
+ torture_init_end();
+ percpu_ref_test_cleanup();
+ return err;
+}
+
+static void __exit percpu_ref_torture_exit(void)
+{
+ percpu_ref_test_cleanup();
+}
+
+module_init(percpu_ref_torture_init);
+module_exit(percpu_ref_torture_exit);
+
+MODULE_LICENSE("GPL");
+MODULE_DESCRIPTION("percpu refcount torture test");
diff --git a/lib/percpu-refcount.c b/lib/percpu-refcount.c
index 7b97f9728c5b..7d0c85c7ce57 100644
--- a/lib/percpu-refcount.c
+++ b/lib/percpu-refcount.c
@@ -11,6 +11,8 @@
#include <linux/mm.h>
#include <linux/percpu-refcount.h>
+#include "percpu-refcount.h"
+
/*
* Initially, a percpu refcount is just a set of percpu counters. Initially, we
* don't try to detect the ref hitting 0 - which means that get/put can just
@@ -677,6 +679,7 @@ static void percpu_ref_release_work_fn(struct work_struct *work)
struct percpu_ref *ref;
int count = 0;
bool held;
+ struct llist_node *last_node = READ_ONCE(last_percpu_ref_node);
first = READ_ONCE(percpu_ref_manage_head.first);
if (!first)
@@ -711,7 +714,7 @@ static void percpu_ref_release_work_fn(struct work_struct *work)
* +----------+ +------+ +------+ +------+ +------+
*
*/
- if (last_percpu_ref_node == NULL || last_percpu_ref_node->next == NULL) {
+ if (last_node == NULL || last_node->next == NULL) {
retry_sentinel_get:
sen_node = percpu_ref_get_sen_node();
/*
@@ -741,11 +744,10 @@ static void percpu_ref_release_work_fn(struct work_struct *work)
head = prev->next;
}
} else {
- prev = last_percpu_ref_node;
+ prev = last_node;
head = prev->next;
}
- last_percpu_ref_node = NULL;
llist_for_each_safe(pos, next, head) {
/* Free sentinel node which is present in the list */
if (percpu_ref_is_sen_node(pos)) {
@@ -773,18 +775,53 @@ static void percpu_ref_release_work_fn(struct work_struct *work)
continue;
__percpu_ref_switch_to_percpu_checked(ref, false);
count++;
- if (count == max_scan_count) {
- last_percpu_ref_node = pos;
- break;
+ if (count == READ_ONCE(max_scan_count)) {
+ WRITE_ONCE(last_percpu_ref_node, pos);
+ goto queue_release_work;
}
prev = pos;
}
+ WRITE_ONCE(last_percpu_ref_node, NULL);
queue_release_work:
queue_delayed_work(percpu_ref_release_wq, &percpu_ref_release_work,
scan_interval);
}
+bool percpu_ref_test_is_percpu(struct percpu_ref *ref)
+{
+ unsigned long __percpu *percpu_count;
+
+ return __ref_is_percpu(ref, &percpu_count);
+}
+EXPORT_SYMBOL_GPL(percpu_ref_test_is_percpu);
+
+void percpu_ref_test_flush_release_work(void)
+{
+ int max_flush = READ_ONCE(max_scan_count);
+ int max_count = 1000;
+
+ /* Complete any executing release work */
+ flush_delayed_work(&percpu_ref_release_work);
+ /* Scan till the end of the llist */
+ WRITE_ONCE(max_scan_count, INT_MAX);
+ /* max scan count update visible to release work */
+ smp_mb();
+ flush_delayed_work(&percpu_ref_release_work);
+ /* max scan count update visible to release work */
+ smp_mb();
+ WRITE_ONCE(max_scan_count, 1);
+ /* max scan count update visible to work */
+ smp_mb();
+ flush_delayed_work(&percpu_ref_release_work);
+ while (READ_ONCE(last_percpu_ref_node) != NULL && max_count--)
+ flush_delayed_work(&percpu_ref_release_work);
+ /* max scan count update visible to work */
+ smp_mb();
+ WRITE_ONCE(max_scan_count, max_flush);
+}
+EXPORT_SYMBOL_GPL(percpu_ref_test_flush_release_work);
+
static __init int percpu_ref_setup(void)
{
percpu_ref_release_wq = alloc_workqueue("percpu_ref_release_wq",
diff --git a/lib/percpu-refcount.h b/lib/percpu-refcount.h
new file mode 100644
index 000000000000..be2ac0411194
--- /dev/null
+++ b/lib/percpu-refcount.h
@@ -0,0 +1,6 @@
+/* SPDX-License-Identifier: GPL-2.0+ */
+#ifndef __LIB_REFCOUNT_H
+#define __LIB_REFCOUNT_H
+bool percpu_ref_test_is_percpu(struct percpu_ref *ref);
+void percpu_ref_test_flush_release_work(void);
+#endif
--
2.34.1
