Adding support to register kprobe on multiple addresses within single kprobe object instance. It uses the CONFIG_KPROBES_ON_FTRACE feature (so it's only available for function entry addresses) and separated ftrace_ops object placed directly in the kprobe object. There's new CONFIG_HAVE_KPROBES_MULTI_ON_FTRACE config option, enabled for archs that support multi kprobes. It registers all the provided addresses in the ftrace_ops object filter and adds separate ftrace_ops callback. To register multi kprobe user provides array of addresses or symbols with their count, like: struct kprobe kp = {}; kp.multi.symbols = (const char **) symbols; kp.multi.cnt = cnt; ... err = register_kprobe(&kp); Signed-off-by: Jiri Olsa <jolsa@xxxxxxxxxx> --- arch/Kconfig | 3 + arch/x86/Kconfig | 1 + arch/x86/kernel/kprobes/ftrace.c | 48 ++++++-- include/linux/kprobes.h | 25 ++++ kernel/kprobes.c | 204 +++++++++++++++++++++++++------ 5 files changed, 235 insertions(+), 46 deletions(-) diff --git a/arch/Kconfig b/arch/Kconfig index d3c4ab249e9c..0131636e1ef8 100644 --- a/arch/Kconfig +++ b/arch/Kconfig @@ -191,6 +191,9 @@ config HAVE_OPTPROBES config HAVE_KPROBES_ON_FTRACE bool +config HAVE_KPROBES_MULTI_ON_FTRACE + bool + config ARCH_CORRECT_STACKTRACE_ON_KRETPROBE bool help diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index 5c2ccb85f2ef..0c870238016a 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -217,6 +217,7 @@ config X86 select HAVE_KERNEL_ZSTD select HAVE_KPROBES select HAVE_KPROBES_ON_FTRACE + select HAVE_KPROBES_MULTI_ON_FTRACE select HAVE_FUNCTION_ERROR_INJECTION select HAVE_KRETPROBES select HAVE_KVM diff --git a/arch/x86/kernel/kprobes/ftrace.c b/arch/x86/kernel/kprobes/ftrace.c index dd2ec14adb77..ac4d256b89c6 100644 --- a/arch/x86/kernel/kprobes/ftrace.c +++ b/arch/x86/kernel/kprobes/ftrace.c @@ -12,22 +12,14 @@ #include "common.h" -/* Ftrace callback handler for kprobes -- called under preempt disabled */ -void kprobe_ftrace_handler(unsigned long ip, unsigned long parent_ip, - struct ftrace_ops *ops, struct ftrace_regs *fregs) +static void ftrace_handler(struct kprobe *p, unsigned long ip, + struct ftrace_regs *fregs) { struct pt_regs *regs = ftrace_get_regs(fregs); - struct kprobe *p; struct kprobe_ctlblk *kcb; - int bit; - bit = ftrace_test_recursion_trylock(ip, parent_ip); - if (bit < 0) - return; - - p = get_kprobe((kprobe_opcode_t *)ip); if (unlikely(!p) || kprobe_disabled(p)) - goto out; + return; kcb = get_kprobe_ctlblk(); if (kprobe_running()) { @@ -57,11 +49,43 @@ void kprobe_ftrace_handler(unsigned long ip, unsigned long parent_ip, */ __this_cpu_write(current_kprobe, NULL); } -out: +} + +/* Ftrace callback handler for kprobes -- called under preempt disabled */ +void kprobe_ftrace_handler(unsigned long ip, unsigned long parent_ip, + struct ftrace_ops *ops, struct ftrace_regs *fregs) +{ + struct kprobe *p; + int bit; + + bit = ftrace_test_recursion_trylock(ip, parent_ip); + if (bit < 0) + return; + + p = get_kprobe((kprobe_opcode_t *)ip); + ftrace_handler(p, ip, fregs); + ftrace_test_recursion_unlock(bit); } NOKPROBE_SYMBOL(kprobe_ftrace_handler); +void kprobe_ftrace_multi_handler(unsigned long ip, unsigned long parent_ip, + struct ftrace_ops *ops, struct ftrace_regs *fregs) +{ + struct kprobe *p; + int bit; + + bit = ftrace_test_recursion_trylock(ip, parent_ip); + if (bit < 0) + return; + + p = container_of(ops, struct kprobe, multi.ops); + ftrace_handler(p, ip, fregs); + + ftrace_test_recursion_unlock(bit); +} +NOKPROBE_SYMBOL(kprobe_ftrace_multi_handler); + int arch_prepare_kprobe_ftrace(struct kprobe *p) { p->ainsn.insn = NULL; diff --git a/include/linux/kprobes.h b/include/linux/kprobes.h index a204df4fef96..03fd86ef69cb 100644 --- a/include/linux/kprobes.h +++ b/include/linux/kprobes.h @@ -68,6 +68,16 @@ struct kprobe { /* location of the probe point */ kprobe_opcode_t *addr; +#ifdef CONFIG_HAVE_KPROBES_MULTI_ON_FTRACE + /* location of the multi probe points */ + struct { + const char **symbols; + kprobe_opcode_t **addrs; + unsigned int cnt; + struct ftrace_ops ops; + } multi; +#endif + /* Allow user to indicate symbol name of the probe point */ const char *symbol_name; @@ -105,6 +115,7 @@ struct kprobe { * this flag is only for optimized_kprobe. */ #define KPROBE_FLAG_FTRACE 8 /* probe is using ftrace */ +#define KPROBE_FLAG_MULTI 16 /* probe multi addresses */ /* Has this kprobe gone ? */ static inline bool kprobe_gone(struct kprobe *p) @@ -130,6 +141,18 @@ static inline bool kprobe_ftrace(struct kprobe *p) return p->flags & KPROBE_FLAG_FTRACE; } +/* Is this ftrace multi kprobe ? */ +static inline bool kprobe_ftrace_multi(struct kprobe *p) +{ + return kprobe_ftrace(p) && (p->flags & KPROBE_FLAG_MULTI); +} + +/* Is this single kprobe ? */ +static inline bool kprobe_single(struct kprobe *p) +{ + return !(p->flags & KPROBE_FLAG_MULTI); +} + /* * Function-return probe - * Note: @@ -365,6 +388,8 @@ static inline void wait_for_kprobe_optimizer(void) { } #ifdef CONFIG_KPROBES_ON_FTRACE extern void kprobe_ftrace_handler(unsigned long ip, unsigned long parent_ip, struct ftrace_ops *ops, struct ftrace_regs *fregs); +extern void kprobe_ftrace_multi_handler(unsigned long ip, unsigned long parent_ip, + struct ftrace_ops *ops, struct ftrace_regs *fregs); extern int arch_prepare_kprobe_ftrace(struct kprobe *p); #else static inline int arch_prepare_kprobe_ftrace(struct kprobe *p) diff --git a/kernel/kprobes.c b/kernel/kprobes.c index c4060a8da050..e7729e20d85c 100644 --- a/kernel/kprobes.c +++ b/kernel/kprobes.c @@ -44,6 +44,7 @@ #include <asm/cacheflush.h> #include <asm/errno.h> #include <linux/uaccess.h> +#include <linux/ftrace.h> #define KPROBE_HASH_BITS 6 #define KPROBE_TABLE_SIZE (1 << KPROBE_HASH_BITS) @@ -1022,6 +1023,35 @@ static struct kprobe *alloc_aggr_kprobe(struct kprobe *p) } #endif /* CONFIG_OPTPROBES */ +static int check_kprobe_address(unsigned long addr) +{ + /* Ensure it is not in reserved area nor out of text */ + return !kernel_text_address(addr) || + within_kprobe_blacklist(addr) || + jump_label_text_reserved((void *) addr, (void *) addr) || + static_call_text_reserved((void *) addr, (void *) addr) || + find_bug(addr); +} + +static int check_ftrace_location(unsigned long addr, struct kprobe *p) +{ + unsigned long ftrace_addr; + + ftrace_addr = ftrace_location(addr); + if (ftrace_addr) { +#ifdef CONFIG_KPROBES_ON_FTRACE + /* Given address is not on the instruction boundary */ + if (addr != ftrace_addr) + return -EILSEQ; + if (p) + p->flags |= KPROBE_FLAG_FTRACE; +#else /* !CONFIG_KPROBES_ON_FTRACE */ + return -EINVAL; +#endif + } + return 0; +} + #ifdef CONFIG_KPROBES_ON_FTRACE static struct ftrace_ops kprobe_ftrace_ops __read_mostly = { .func = kprobe_ftrace_handler, @@ -1043,6 +1073,13 @@ static int __arm_kprobe_ftrace(struct kprobe *p, struct ftrace_ops *ops, lockdep_assert_held(&kprobe_mutex); +#ifdef CONFIG_HAVE_KPROBES_MULTI_ON_FTRACE + if (kprobe_ftrace_multi(p)) { + ret = register_ftrace_function(&p->multi.ops); + WARN(ret < 0, "Failed to register kprobe-multi-ftrace (error %d)\n", ret); + return ret; + } +#endif ret = ftrace_set_filter_ip(ops, (unsigned long)p->addr, 0, 0); if (WARN_ONCE(ret < 0, "Failed to arm kprobe-ftrace at %pS (error %d)\n", p->addr, ret)) return ret; @@ -1081,6 +1118,13 @@ static int __disarm_kprobe_ftrace(struct kprobe *p, struct ftrace_ops *ops, lockdep_assert_held(&kprobe_mutex); +#ifdef CONFIG_HAVE_KPROBES_MULTI_ON_FTRACE + if (kprobe_ftrace_multi(p)) { + ret = unregister_ftrace_function(&p->multi.ops); + WARN(ret < 0, "Failed to unregister kprobe-ftrace (error %d)\n", ret); + return ret; + } +#endif if (*cnt == 1) { ret = unregister_ftrace_function(ops); if (WARN(ret < 0, "Failed to unregister kprobe-ftrace (error %d)\n", ret)) @@ -1103,6 +1147,94 @@ static int disarm_kprobe_ftrace(struct kprobe *p) ipmodify ? &kprobe_ipmodify_ops : &kprobe_ftrace_ops, ipmodify ? &kprobe_ipmodify_enabled : &kprobe_ftrace_enabled); } + +#ifdef CONFIG_HAVE_KPROBES_MULTI_ON_FTRACE +/* + * In addition to standard kprobe address check for multi + * ftrace kprobes we also allow only: + * - ftrace managed function entry address + * - kernel core only address + */ +static unsigned long check_ftrace_addr(unsigned long addr) +{ + int err; + + if (!addr) + return -EINVAL; + err = check_ftrace_location(addr, NULL); + if (err) + return err; + if (check_kprobe_address(addr)) + return -EINVAL; + if (__module_text_address(addr)) + return -EINVAL; + return 0; +} + +static int check_ftrace_multi(struct kprobe *p) +{ + kprobe_opcode_t **addrs = p->multi.addrs; + const char **symbols = p->multi.symbols; + unsigned int i, cnt = p->multi.cnt; + unsigned long addr, *ips; + int err; + + if ((symbols && addrs) || (!symbols && !addrs)) + return -EINVAL; + + /* do we want sysctl for this? */ + if (cnt >= 20000) + return -E2BIG; + + ips = kmalloc(sizeof(*ips) * cnt, GFP_KERNEL); + if (!ips) + return -ENOMEM; + + for (i = 0; i < cnt; i++) { + if (symbols) + addr = (unsigned long) kprobe_lookup_name(symbols[i], 0); + else + addr = (unsigned long) addrs[i]; + ips[i] = addr; + } + + jump_label_lock(); + preempt_disable(); + + for (i = 0; i < cnt; i++) { + err = check_ftrace_addr(ips[i]); + if (err) + break; + } + + preempt_enable(); + jump_label_unlock(); + + if (err) + goto out; + + err = ftrace_set_filter_ips(&p->multi.ops, ips, cnt, 0, 0); + if (err) + goto out; + + p->multi.ops.func = kprobe_ftrace_multi_handler; + p->multi.ops.flags = FTRACE_OPS_FL_SAVE_REGS|FTRACE_OPS_FL_DYNAMIC; + + p->flags |= KPROBE_FLAG_MULTI|KPROBE_FLAG_FTRACE; + if (p->post_handler) + p->multi.ops.flags |= FTRACE_OPS_FL_IPMODIFY; + +out: + kfree(ips); + return err; +} + +static void free_ftrace_multi(struct kprobe *p) +{ + ftrace_free_filter(&p->multi.ops); +} +#endif + #else /* !CONFIG_KPROBES_ON_FTRACE */ static inline int arm_kprobe_ftrace(struct kprobe *p) { @@ -1489,6 +1621,9 @@ static struct kprobe *__get_valid_kprobe(struct kprobe *p) lockdep_assert_held(&kprobe_mutex); + if (kprobe_ftrace_multi(p)) + return p; + ap = get_kprobe(p->addr); if (unlikely(!ap)) return NULL; @@ -1520,41 +1655,18 @@ static inline int warn_kprobe_rereg(struct kprobe *p) return ret; } -static int check_ftrace_location(struct kprobe *p) -{ - unsigned long ftrace_addr; - - ftrace_addr = ftrace_location((unsigned long)p->addr); - if (ftrace_addr) { -#ifdef CONFIG_KPROBES_ON_FTRACE - /* Given address is not on the instruction boundary */ - if ((unsigned long)p->addr != ftrace_addr) - return -EILSEQ; - p->flags |= KPROBE_FLAG_FTRACE; -#else /* !CONFIG_KPROBES_ON_FTRACE */ - return -EINVAL; -#endif - } - return 0; -} - static int check_kprobe_address_safe(struct kprobe *p, struct module **probed_mod) { int ret; - ret = check_ftrace_location(p); + ret = check_ftrace_location((unsigned long) p->addr, p); if (ret) return ret; jump_label_lock(); preempt_disable(); - /* Ensure it is not in reserved area nor out of text */ - if (!kernel_text_address((unsigned long) p->addr) || - within_kprobe_blacklist((unsigned long) p->addr) || - jump_label_text_reserved(p->addr, p->addr) || - static_call_text_reserved(p->addr, p->addr) || - find_bug((unsigned long)p->addr)) { + if (check_kprobe_address((unsigned long) p->addr)) { ret = -EINVAL; goto out; } @@ -1599,13 +1711,16 @@ static unsigned long resolve_func_addr(kprobe_opcode_t *addr) return 0; } -int register_kprobe(struct kprobe *p) +static int check_addr(struct kprobe *p, struct module **probed_mod) { int ret; - struct kprobe *old_p; - struct module *probed_mod; kprobe_opcode_t *addr; +#ifdef CONFIG_HAVE_KPROBES_MULTI_ON_FTRACE + if (p->multi.cnt) + return check_ftrace_multi(p); +#endif + /* Adjust probe address from symbol */ addr = kprobe_addr(p); if (IS_ERR(addr)) @@ -1616,13 +1731,21 @@ int register_kprobe(struct kprobe *p) ret = warn_kprobe_rereg(p); if (ret) return ret; + return check_kprobe_address_safe(p, probed_mod); +} + +int register_kprobe(struct kprobe *p) +{ + struct module *probed_mod = NULL; + struct kprobe *old_p; + int ret; /* User can pass only KPROBE_FLAG_DISABLED to register_kprobe */ p->flags &= KPROBE_FLAG_DISABLED; p->nmissed = 0; INIT_LIST_HEAD(&p->list); - ret = check_kprobe_address_safe(p, &probed_mod); + ret = check_addr(p, &probed_mod); if (ret) return ret; @@ -1644,14 +1767,21 @@ int register_kprobe(struct kprobe *p) if (ret) goto out; - INIT_HLIST_NODE(&p->hlist); - hlist_add_head_rcu(&p->hlist, - &kprobe_table[hash_ptr(p->addr, KPROBE_HASH_BITS)]); + /* + * Multi ftrace kprobes do not have single address, + * so they are not stored in the kprobe_table hash. + */ + if (kprobe_single(p)) { + INIT_HLIST_NODE(&p->hlist); + hlist_add_head_rcu(&p->hlist, + &kprobe_table[hash_ptr(p->addr, KPROBE_HASH_BITS)]); + } if (!kprobes_all_disarmed && !kprobe_disabled(p)) { ret = arm_kprobe(p); if (ret) { - hlist_del_rcu(&p->hlist); + if (kprobe_single(p)) + hlist_del_rcu(&p->hlist); synchronize_rcu(); goto out; } @@ -1778,7 +1908,13 @@ static int __unregister_kprobe_top(struct kprobe *p) return 0; disarmed: - hlist_del_rcu(&ap->hlist); + if (kprobe_single(ap)) + hlist_del_rcu(&ap->hlist); + +#ifdef CONFIG_HAVE_KPROBES_MULTI_ON_FTRACE + if (kprobe_ftrace_multi(ap)) + free_ftrace_multi(ap); +#endif return 0; } -- 2.33.1