Error injectable functions cannot be inlined and since some are called
from hot paths, this incurs overhead even if no error injection is
enabled for them.
To avoid this overhead when disabled, allow the callsites of error
injectable functions to put the calls behind a static key, which the
framework can control when error injection is enabled or disabled for
the function.
Introduce a new ALLOW_ERROR_INJECTION_KEY() macro that adds a parameter
with the static key's address, and store it in struct
error_injection_entry. This new field has caused a mismatch when
populating the injection list from the _error_injection_whitelist
section using the current STRUCT_ALIGN(), so change the alignment to 8.
During the population, copy the key's address also to struct ei_entry,
and make it possible to retrieve it by get_injection_key().
Finally, make the processing of writes to the debugfs inject file enable
the static key when the function is added to the injection list, and
disable when removed.
Signed-off-by: Vlastimil Babka <vbabka@xxxxxxx>
---
include/asm-generic/error-injection.h | 13 ++++++++++++-
include/asm-generic/vmlinux.lds.h | 2 +-
include/linux/error-injection.h | 12 ++++++++++--
kernel/fail_function.c | 10 ++++++++++
lib/error-inject.c | 19 +++++++++++++++++++
5 files changed, 52 insertions(+), 4 deletions(-)
diff --git a/include/asm-generic/error-injection.h b/include/asm-generic/error-injection.h
index b05253f68eaa..eed2731f3820 100644
--- a/include/asm-generic/error-injection.h
+++ b/include/asm-generic/error-injection.h
@@ -12,6 +12,7 @@ enum {
struct error_injection_entry {
unsigned long addr;
+ unsigned long static_key_addr;
int etype;
};
@@ -25,16 +26,26 @@ struct pt_regs;
* 'Error Injectable Functions' section.
*/
#define ALLOW_ERROR_INJECTION(fname, _etype) \
-static struct error_injection_entry __used \
+static struct error_injection_entry __used __aligned(8) \
__section("_error_injection_whitelist") \
_eil_addr_##fname = { \
.addr = (unsigned long)fname, \
.etype = EI_ETYPE_##_etype, \
}
+#define ALLOW_ERROR_INJECTION_KEY(fname, _etype, key) \
+static struct error_injection_entry __used __aligned(8) \
+ __section("_error_injection_whitelist") \
+ _eil_addr_##fname = { \
+ .addr = (unsigned long)fname, \
+ .static_key_addr = (unsigned long)key, \
+ .etype = EI_ETYPE_##_etype, \
+ }
+
void override_function_with_return(struct pt_regs *regs);
#else
#define ALLOW_ERROR_INJECTION(fname, _etype)
+#define ALLOW_ERROR_INJECTION_KEY(fname, _etype, key)
static inline void override_function_with_return(struct pt_regs *regs) { }
#endif
diff --git a/include/asm-generic/vmlinux.lds.h b/include/asm-generic/vmlinux.lds.h
index 5703526d6ebf..1b15a0af2a00 100644
--- a/include/asm-generic/vmlinux.lds.h
+++ b/include/asm-generic/vmlinux.lds.h
@@ -248,7 +248,7 @@
#ifdef CONFIG_FUNCTION_ERROR_INJECTION
#define ERROR_INJECT_WHITELIST() \
- STRUCT_ALIGN(); \
+ . = ALIGN(8); \
BOUNDED_SECTION(_error_injection_whitelist)
#else
#define ERROR_INJECT_WHITELIST()
diff --git a/include/linux/error-injection.h b/include/linux/error-injection.h
index 20e738f4eae8..48da027c0302 100644
--- a/include/linux/error-injection.h
+++ b/include/linux/error-injection.h
@@ -6,10 +6,13 @@
#include <linux/errno.h>
#include <asm-generic/error-injection.h>
+struct static_key;
+
#ifdef CONFIG_FUNCTION_ERROR_INJECTION
-extern bool within_error_injection_list(unsigned long addr);
-extern int get_injectable_error_type(unsigned long addr);
+bool within_error_injection_list(unsigned long addr);
+int get_injectable_error_type(unsigned long addr);
+struct static_key *get_injection_key(unsigned long addr);
#else /* !CONFIG_FUNCTION_ERROR_INJECTION */
@@ -23,6 +26,11 @@ static inline int get_injectable_error_type(unsigned long addr)
return -EOPNOTSUPP;
}
+static inline struct static_key *get_injection_key(unsigned long addr)
+{
+ return NULL;
+}
+
#endif
#endif /* _LINUX_ERROR_INJECTION_H */
diff --git a/kernel/fail_function.c b/kernel/fail_function.c
index d971a0189319..d39a9606a448 100644
--- a/kernel/fail_function.c
+++ b/kernel/fail_function.c
@@ -27,6 +27,7 @@ struct fei_attr {
struct list_head list;
struct kprobe kp;
unsigned long retval;
+ struct static_key *key;
};
static DEFINE_MUTEX(fei_lock);
static LIST_HEAD(fei_attr_list);
@@ -67,6 +68,11 @@ static struct fei_attr *fei_attr_new(const char *sym, unsigned long addr)
attr->kp.pre_handler = fei_kprobe_handler;
attr->kp.post_handler = fei_post_handler;
attr->retval = adjust_error_retval(addr, 0);
+
+ attr->key = get_injection_key(addr);
+ if (IS_ERR(attr->key))
+ attr->key = NULL;
+
INIT_LIST_HEAD(&attr->list);
}
return attr;
@@ -218,6 +224,8 @@ static int fei_open(struct inode *inode, struct file *file)
static void fei_attr_remove(struct fei_attr *attr)
{
+ if (attr->key)
+ static_key_slow_dec(attr->key);
fei_debugfs_remove_attr(attr);
unregister_kprobe(&attr->kp);
list_del(&attr->list);
@@ -295,6 +303,8 @@ static ssize_t fei_write(struct file *file, const char __user *buffer,
fei_attr_free(attr);
goto out;
}
+ if (attr->key)
+ static_key_slow_inc(attr->key);
fei_debugfs_add_attr(attr);
list_add_tail(&attr->list, &fei_attr_list);
ret = count;
diff --git a/lib/error-inject.c b/lib/error-inject.c
index 887acd9a6ea6..982fbedd9ad5 100644
--- a/lib/error-inject.c
+++ b/lib/error-inject.c
@@ -17,6 +17,7 @@ struct ei_entry {
struct list_head list;
unsigned long start_addr;
unsigned long end_addr;
+ struct static_key *key;
int etype;
void *priv;
};
@@ -54,6 +55,23 @@ int get_injectable_error_type(unsigned long addr)
return ei_type;
}
+struct static_key *get_injection_key(unsigned long addr)
+{
+ struct ei_entry *ent;
+ struct static_key *ei_key = ERR_PTR(-EINVAL);
+
+ mutex_lock(&ei_mutex);
+ list_for_each_entry(ent, &error_injection_list, list) {
+ if (addr >= ent->start_addr && addr < ent->end_addr) {
+ ei_key = ent->key;
+ break;
+ }
+ }
+ mutex_unlock(&ei_mutex);
+
+ return ei_key;
+}
+
/*
* Lookup and populate the error_injection_list.
*
@@ -86,6 +104,7 @@ static void populate_error_injection_list(struct error_injection_entry *start,
ent->start_addr = entry;
ent->end_addr = entry + size;
ent->etype = iter->etype;
+ ent->key = (struct static_key *) iter->static_key_addr;
ent->priv = priv;
INIT_LIST_HEAD(&ent->list);
list_add_tail(&ent->list, &error_injection_list);
--
2.45.2
