Some fault injection sites are placed in hotpaths and incur overhead
even if not enabled, due to one or more function calls leading up to
should_fail_ex() that returns false due to attr->probability == 0.
This overhead can be eliminated if the outermost call into the checks is
guarded with a static key, so add support for that. The framework should
be told that such static key exist for a fault_attr, by initializing
fault_attr->active with the static key address. When it's not NULL,
enable the static key from setup_fault_attr() when the fault probability
is non-zero.
Also wire up writing into debugfs "probability" file to enable or
disable the static key when transitioning between zero and non-zero
probability.
For now, do not add configfs interface support as the immediate plan is
to leverage this for should_failslab() and should_fail_alloc_page()
after other necessary preparatory changes, and none of the configfs
based fault injection users.
Signed-off-by: Vlastimil Babka <vbabka@xxxxxxx>
---
include/linux/fault-inject.h | 7 ++++++-
lib/fault-inject.c | 43 ++++++++++++++++++++++++++++++++++++++++++-
2 files changed, 48 insertions(+), 2 deletions(-)
diff --git a/include/linux/fault-inject.h b/include/linux/fault-inject.h
index 6d5edef09d45..cfe75cc1bac4 100644
--- a/include/linux/fault-inject.h
+++ b/include/linux/fault-inject.h
@@ -9,6 +9,7 @@
#include <linux/configfs.h>
#include <linux/ratelimit.h>
#include <linux/atomic.h>
+#include <linux/jump_label.h>
/*
* For explanation of the elements of this struct, see
@@ -30,13 +31,14 @@ struct fault_attr {
unsigned long count;
struct ratelimit_state ratelimit_state;
struct dentry *dname;
+ struct static_key *active;
};
enum fault_flags {
FAULT_NOWARN = 1 << 0,
};
-#define FAULT_ATTR_INITIALIZER { \
+#define FAULT_ATTR_INITIALIZER_KEY(_key) { \
.interval = 1, \
.times = ATOMIC_INIT(1), \
.require_end = ULONG_MAX, \
@@ -44,8 +46,11 @@ enum fault_flags {
.ratelimit_state = RATELIMIT_STATE_INIT_DISABLED, \
.verbose = 2, \
.dname = NULL, \
+ .active = (_key), \
}
+#define FAULT_ATTR_INITIALIZER FAULT_ATTR_INITIALIZER_KEY(NULL)
+
#define DECLARE_FAULT_ATTR(name) struct fault_attr name = FAULT_ATTR_INITIALIZER
int setup_fault_attr(struct fault_attr *attr, char *str);
bool should_fail_ex(struct fault_attr *attr, ssize_t size, int flags);
diff --git a/lib/fault-inject.c b/lib/fault-inject.c
index d608f9b48c10..93c46d2ec106 100644
--- a/lib/fault-inject.c
+++ b/lib/fault-inject.c
@@ -35,6 +35,9 @@ int setup_fault_attr(struct fault_attr *attr, char *str)
atomic_set(&attr->times, times);
atomic_set(&attr->space, space);
+ if (probability != 0 && attr->active)
+ static_key_slow_inc(attr->active);
+
return 1;
}
EXPORT_SYMBOL_GPL(setup_fault_attr);
@@ -166,6 +169,12 @@ EXPORT_SYMBOL_GPL(should_fail);
#ifdef CONFIG_FAULT_INJECTION_DEBUG_FS
+/*
+ * Protect updating probability from debugfs as that may trigger static key
+ * changes when changing between zero and non-zero.
+ */
+static DEFINE_MUTEX(probability_mutex);
+
static int debugfs_ul_set(void *data, u64 val)
{
*(unsigned long *)data = val;
@@ -186,6 +195,38 @@ static void debugfs_create_ul(const char *name, umode_t mode,
debugfs_create_file(name, mode, parent, value, &fops_ul);
}
+static int debugfs_prob_set(void *data, u64 val)
+{
+ struct fault_attr *attr = data;
+
+ mutex_lock(&probability_mutex);
+
+ if (attr->active) {
+ if (attr->probability != 0 && val == 0) {
+ static_key_slow_dec(attr->active);
+ } else if (attr->probability == 0 && val != 0) {
+ static_key_slow_inc(attr->active);
+ }
+ }
+
+ attr->probability = val;
+
+ mutex_unlock(&probability_mutex);
+
+ return 0;
+}
+
+static int debugfs_prob_get(void *data, u64 *val)
+{
+ struct fault_attr *attr = data;
+
+ *val = attr->probability;
+
+ return 0;
+}
+
+DEFINE_SIMPLE_ATTRIBUTE(fops_prob, debugfs_prob_get, debugfs_prob_set, "%llu\n");
+
#ifdef CONFIG_FAULT_INJECTION_STACKTRACE_FILTER
static int debugfs_stacktrace_depth_set(void *data, u64 val)
@@ -218,7 +259,7 @@ struct dentry *fault_create_debugfs_attr(const char *name,
if (IS_ERR(dir))
return dir;
- debugfs_create_ul("probability", mode, dir, &attr->probability);
+ debugfs_create_file("probability", mode, dir, &attr, &fops_prob);
debugfs_create_ul("interval", mode, dir, &attr->interval);
debugfs_create_atomic_t("times", mode, dir, &attr->times);
debugfs_create_atomic_t("space", mode, dir, &attr->space);
--
2.45.1
