The new helper should be used to convert deltas of values
received by bpf_get_cpu_time_counter() into nanoseconds. It is not
designed to do full conversion of time counter values to
CLOCK_MONOTONIC_RAW nanoseconds and cannot guarantee monotonicity of 2
independent values, but rather to convert the difference of 2 close
enough values of CPU timestamp counter into nanoseconds.
This function is also JITted into just several instructions and adds as
low overhead as possible and perfectly suits benchmark use-cases.
Reviewed-by: Eduard Zingerman <eddyz87@xxxxxxxxx>
Acked-by: Andrii Nakryiko <andrii@xxxxxxxxxx>
Signed-off-by: Vadim Fedorenko <vadfed@xxxxxxxx>
---
arch/x86/net/bpf_jit_comp.c | 27 +++++++++++++++++++++++++++
arch/x86/net/bpf_jit_comp32.c | 27 +++++++++++++++++++++++++++
include/linux/bpf.h | 1 +
kernel/bpf/helpers.c | 14 +++++++++++++-
4 files changed, 68 insertions(+), 1 deletion(-)
diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c
index 92431ab1a21e..d21e0ab55c94 100644
--- a/arch/x86/net/bpf_jit_comp.c
+++ b/arch/x86/net/bpf_jit_comp.c
@@ -11,10 +11,12 @@
#include <linux/bpf.h>
#include <linux/memory.h>
#include <linux/sort.h>
+#include <linux/clocksource.h>
#include <asm/extable.h>
#include <asm/ftrace.h>
#include <asm/set_memory.h>
#include <asm/nospec-branch.h>
+#include <asm/timer.h>
#include <asm/text-patching.h>
#include <asm/unwind.h>
#include <asm/cfi.h>
@@ -2216,6 +2218,28 @@ st: if (is_imm8(insn->off))
break;
}
+ if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL &&
+ imm32 == BPF_CALL_IMM(bpf_cpu_time_counter_to_ns) &&
+ using_native_sched_clock() && sched_clock_stable()) {
+ struct cyc2ns_data data;
+ u32 mult, shift;
+
+ cyc2ns_read_begin(&data);
+ mult = data.cyc2ns_mul;
+ shift = data.cyc2ns_shift;
+ cyc2ns_read_end();
+ /* imul RAX, RDI, mult */
+ maybe_emit_mod(&prog, BPF_REG_1, BPF_REG_0, true);
+ EMIT2_off32(0x69, add_2reg(0xC0, BPF_REG_1, BPF_REG_0),
+ mult);
+
+ /* shr RAX, shift (which is less than 64) */
+ maybe_emit_1mod(&prog, BPF_REG_0, true);
+ EMIT3(0xC1, add_1reg(0xE8, BPF_REG_0), shift);
+
+ break;
+ }
+
func = (u8 *) __bpf_call_base + imm32;
if (src_reg == BPF_PSEUDO_CALL && tail_call_reachable) {
LOAD_TAIL_CALL_CNT_PTR(stack_depth);
@@ -3828,5 +3852,8 @@ bool bpf_jit_inlines_kfunc_call(s32 imm)
{
if (imm == BPF_CALL_IMM(bpf_get_cpu_time_counter))
return true;
+ if (imm == BPF_CALL_IMM(bpf_cpu_time_counter_to_ns) &&
+ using_native_sched_clock() && sched_clock_stable())
+ return true;
return false;
}
diff --git a/arch/x86/net/bpf_jit_comp32.c b/arch/x86/net/bpf_jit_comp32.c
index a549aea25f5f..a2069a3ee4a3 100644
--- a/arch/x86/net/bpf_jit_comp32.c
+++ b/arch/x86/net/bpf_jit_comp32.c
@@ -12,10 +12,12 @@
#include <linux/netdevice.h>
#include <linux/filter.h>
#include <linux/if_vlan.h>
+#include <linux/clocksource.h>
#include <asm/cacheflush.h>
#include <asm/set_memory.h>
#include <asm/nospec-branch.h>
#include <asm/asm-prototypes.h>
+#include <asm/timer.h>
#include <linux/bpf.h>
/*
@@ -2100,6 +2102,27 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image,
EMIT2(0x0F, 0x31);
break;
}
+ if (imm32 == BPF_CALL_IMM(bpf_cpu_time_counter_to_ns) &&
+ cpu_feature_enabled(X86_FEATURE_CONSTANT_TSC)) {
+ struct cyc2ns_data data;
+ u32 mult, shift;
+
+ cyc2ns_read_begin(&data);
+ mult = data.cyc2ns_mul;
+ shift = data.cyc2ns_shift;
+ cyc2ns_read_end();
+
+ /* move parameter to BPF_REG_0 */
+ emit_ia32_mov_r64(true, bpf2ia32[BPF_REG_0],
+ bpf2ia32[BPF_REG_1], true, true,
+ &prog, bpf_prog->aux);
+ /* multiply parameter by mut */
+ emit_ia32_mul_i64(bpf2ia32[BPF_REG_0],
+ mult, true, &prog);
+ /* shift parameter by shift which is less than 64 */
+ emit_ia32_rsh_i64(bpf2ia32[BPF_REG_0],
+ shift, true, &prog);
+ }
err = emit_kfunc_call(bpf_prog,
image + addrs[i],
@@ -2633,5 +2656,9 @@ bool bpf_jit_inlines_kfunc_call(s32 imm)
{
if (imm == BPF_CALL_IMM(bpf_get_cpu_time_counter))
return true;
+ if (imm == BPF_CALL_IMM(bpf_cpu_time_counter_to_ns) &&
+ using_native_sched_clock() && sched_clock_stable())
+ return true;
+
return false;
}
diff --git a/include/linux/bpf.h b/include/linux/bpf.h
index 6d540253cfb4..dd3c4ddfd60e 100644
--- a/include/linux/bpf.h
+++ b/include/linux/bpf.h
@@ -3336,6 +3336,7 @@ u64 bpf_get_raw_cpu_id(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5);
/* Inlined kfuncs */
#if IS_ENABLED(CONFIG_GENERIC_GETTIMEOFDAY)
u64 bpf_get_cpu_time_counter(void);
+u64 bpf_cpu_time_counter_to_ns(u64 cycles);
#endif
#if defined(CONFIG_NET)
diff --git a/kernel/bpf/helpers.c b/kernel/bpf/helpers.c
index 23f1a1606f8b..e4d461f2e98f 100644
--- a/kernel/bpf/helpers.c
+++ b/kernel/bpf/helpers.c
@@ -3079,8 +3079,19 @@ __bpf_kfunc u64 bpf_get_cpu_time_counter(void)
*/
return __arch_get_hw_counter(1, vd);
}
-#endif
+__bpf_kfunc u64 bpf_cpu_time_counter_to_ns(u64 cycles)
+{
+ const struct vdso_data *vd = __arch_get_k_vdso_data();
+
+ vd = &vd[CS_RAW];
+ /* kfunc implementation does less manipulations than vDSO
+ * implementation. BPF use-case assumes two measurements are close
+ * in time and can simplify the logic.
+ */
+ return mul_u64_u32_shr(cycles, vd->mult, vd->shift);
+}
+#endif
__bpf_kfunc_end_defs();
BTF_KFUNCS_START(generic_btf_ids)
@@ -3175,6 +3186,7 @@ BTF_ID_FLAGS(func, bpf_iter_kmem_cache_next, KF_ITER_NEXT | KF_RET_NULL | KF_SLE
BTF_ID_FLAGS(func, bpf_iter_kmem_cache_destroy, KF_ITER_DESTROY | KF_SLEEPABLE)
#if IS_ENABLED(CONFIG_GENERIC_GETTIMEOFDAY)
BTF_ID_FLAGS(func, bpf_get_cpu_time_counter, KF_FASTCALL)
+BTF_ID_FLAGS(func, bpf_cpu_time_counter_to_ns, KF_FASTCALL)
#endif
BTF_KFUNCS_END(common_btf_ids)
--
2.43.5
