On Wed, Oct 23, 2024 at 2:05 PM Vadim Fedorenko <vadfed@xxxxxxxx> wrote:
>
> New kfunc to return ARCH-specific timecounter. For x86 BPF JIT converts
> it into rdtsc ordered call. Other architectures will get JIT
> implementation too if supported. The fallback is to
> __arch_get_hw_counter().
arch_get_hw_counter is a great idea.
> Signed-off-by: Vadim Fedorenko <vadfed@xxxxxxxx>
> ---
> arch/x86/net/bpf_jit_comp.c | 23 +++++++++++++++++++++++
> arch/x86/net/bpf_jit_comp32.c | 11 +++++++++++
> kernel/bpf/helpers.c | 7 +++++++
> kernel/bpf/verifier.c | 11 +++++++++++
> 4 files changed, 52 insertions(+)
>
> diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c
> index 06b080b61aa5..55595a0fa55b 100644
> --- a/arch/x86/net/bpf_jit_comp.c
> +++ b/arch/x86/net/bpf_jit_comp.c
> @@ -2126,6 +2126,29 @@ st: if (is_imm8(insn->off))
> case BPF_JMP | BPF_CALL: {
> u8 *ip = image + addrs[i - 1];
>
> + if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL && !imm32) {
> + if (insn->dst_reg == 1) {
> + struct cpuinfo_x86 *c = &cpu_data(get_boot_cpu_id());
> +
> + /* Save RDX because RDTSC will use EDX:EAX to return u64 */
> + emit_mov_reg(&prog, true, AUX_REG, BPF_REG_3);
> + if (cpu_has(c, X86_FEATURE_LFENCE_RDTSC))
> + EMIT_LFENCE();
> + EMIT2(0x0F, 0x31);
> +
> + /* shl RDX, 32 */
> + maybe_emit_1mod(&prog, BPF_REG_3, true);
> + EMIT3(0xC1, add_1reg(0xE0, BPF_REG_3), 32);
> + /* or RAX, RDX */
> + maybe_emit_mod(&prog, BPF_REG_0, BPF_REG_3, true);
> + EMIT2(0x09, add_2reg(0xC0, BPF_REG_0, BPF_REG_3));
> + /* restore RDX from R11 */
> + emit_mov_reg(&prog, true, BPF_REG_3, AUX_REG);
This doesn't match
static inline u64 __arch_get_hw_counter(s32 clock_mode,
const struct vdso_data *vd)
{
if (likely(clock_mode == VDSO_CLOCKMODE_TSC))
return (u64)rdtsc_ordered() & S64_MAX;
- & is missing
- rdtsc vs rdtscp
but the later one is much slower (I was told).
So maybe instead of arch_get_hw_counter() it should be modelled
as JIT of sched_clock() ?
> +
> + break;
> + }
> + }
> +
> func = (u8 *) __bpf_call_base + imm32;
> if (tail_call_reachable) {
> LOAD_TAIL_CALL_CNT_PTR(bpf_prog->aux->stack_depth);
> diff --git a/arch/x86/net/bpf_jit_comp32.c b/arch/x86/net/bpf_jit_comp32.c
> index de0f9e5f9f73..c36ff18a044b 100644
> --- a/arch/x86/net/bpf_jit_comp32.c
> +++ b/arch/x86/net/bpf_jit_comp32.c
> @@ -2091,6 +2091,17 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image,
> if (insn->src_reg == BPF_PSEUDO_CALL)
> goto notyet;
>
> + if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL && !imm32) {
> + if (insn->dst_reg == 1) {
> + struct cpuinfo_x86 *c = &cpu_data(get_boot_cpu_id());
> +
> + if (cpu_has(c, X86_FEATURE_LFENCE_RDTSC))
> + EMIT3(0x0F, 0xAE, 0xE8);
> + EMIT2(0x0F, 0x31);
> + break;
> + }
> + }
> +
> if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL) {
> int err;
>
> diff --git a/kernel/bpf/helpers.c b/kernel/bpf/helpers.c
> index 5c3fdb29c1b1..6624b2465484 100644
> --- a/kernel/bpf/helpers.c
> +++ b/kernel/bpf/helpers.c
> @@ -23,6 +23,7 @@
> #include <linux/btf_ids.h>
> #include <linux/bpf_mem_alloc.h>
> #include <linux/kasan.h>
> +#include <asm/vdso/gettimeofday.h>
>
> #include "../../lib/kstrtox.h"
>
> @@ -3023,6 +3024,11 @@ __bpf_kfunc int bpf_copy_from_user_str(void *dst, u32 dst__sz, const void __user
> return ret + 1;
> }
>
> +__bpf_kfunc int bpf_get_hw_counter(void)
> +{
> + return __arch_get_hw_counter(1, NULL);
> +}
> +
> __bpf_kfunc_end_defs();
>
> BTF_KFUNCS_START(generic_btf_ids)
> @@ -3112,6 +3118,7 @@ BTF_ID_FLAGS(func, bpf_iter_bits_next, KF_ITER_NEXT | KF_RET_NULL)
> BTF_ID_FLAGS(func, bpf_iter_bits_destroy, KF_ITER_DESTROY)
> BTF_ID_FLAGS(func, bpf_copy_from_user_str, KF_SLEEPABLE)
> BTF_ID_FLAGS(func, bpf_get_kmem_cache)
> +BTF_ID_FLAGS(func, bpf_get_hw_counter, KF_FASTCALL)
> BTF_KFUNCS_END(common_btf_ids)
>
> static const struct btf_kfunc_id_set common_kfunc_set = {
> diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
> index f514247ba8ba..5f0e4f91ce48 100644
> --- a/kernel/bpf/verifier.c
> +++ b/kernel/bpf/verifier.c
> @@ -11260,6 +11260,7 @@ enum special_kfunc_type {
> KF_bpf_iter_css_task_new,
> KF_bpf_session_cookie,
> KF_bpf_get_kmem_cache,
> + KF_bpf_get_hw_counter,
> };
>
> BTF_SET_START(special_kfunc_set)
> @@ -11326,6 +11327,7 @@ BTF_ID(func, bpf_session_cookie)
> BTF_ID_UNUSED
> #endif
> BTF_ID(func, bpf_get_kmem_cache)
> +BTF_ID(func, bpf_get_hw_counter)
>
> static bool is_kfunc_ret_null(struct bpf_kfunc_call_arg_meta *meta)
> {
> @@ -20396,6 +20398,15 @@ static int fixup_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
> desc->func_id == special_kfunc_list[KF_bpf_rdonly_cast]) {
> insn_buf[0] = BPF_MOV64_REG(BPF_REG_0, BPF_REG_1);
> *cnt = 1;
> + } else if (IS_ENABLED(CONFIG_X86) &&
It's better to introduce bpf_jit_inlines_kfunc_call()
similar to bpf_jit_inlines_helper_call().
> + desc->func_id == special_kfunc_list[KF_bpf_get_hw_counter]) {
> + insn->imm = 0;
> + insn->code = BPF_JMP | BPF_CALL;
> + insn->src_reg = BPF_PSEUDO_KFUNC_CALL;
> + insn->dst_reg = 1; /* Implement enum for inlined fast calls */
Yes. Pls do it cleanly from the start.
Why rewrite though?
Can JIT match the addr of bpf_get_hw_counter ?
And no need to rewrite call insn ?
