Today doing a BPF tail call after a BPF to BPF call, that is from a
subprogram, is allowed only by the x86-64 BPF JIT. Mixing these features
requires support from JIT. Tail call count has to be tracked through BPF to
BPF calls, as well as through BPF tail calls to prevent unbounded chains of
tail calls.
arm64 BPF JIT stores the tail call count (TCC) in a dedicated
register (X26). This makes it easier to support bpf2bpf calls mixed with
tail calls than on x86 platform.
In order to keep the tail call count in tact throughout bpf2bpf calls, all
we need to do is tweak the program prologue generator. When emitting
prologue for a subprogram, we skip the block that initializes the tail call
count and emits a jump pad for the tail call.
With this change, a sample execution flow where a bpf2bpf call is followed
by a tail call would look like so:
int entry(struct __sk_buff *skb):
0xffffffc0090151d4: paciasp
0xffffffc0090151d8: stp x29, x30, [sp, #-16]!
0xffffffc0090151dc: mov x29, sp
0xffffffc0090151e0: stp x19, x20, [sp, #-16]!
0xffffffc0090151e4: stp x21, x22, [sp, #-16]!
0xffffffc0090151e8: stp x25, x26, [sp, #-16]!
0xffffffc0090151ec: stp x27, x28, [sp, #-16]!
0xffffffc0090151f0: mov x25, sp
0xffffffc0090151f4: mov x26, #0x0 // <- init TCC only
0xffffffc0090151f8: bti j // in main prog
0xffffffc0090151fc: sub x27, x25, #0x0
0xffffffc009015200: sub sp, sp, #0x10
0xffffffc009015204: mov w1, #0x0
0xffffffc009015208: mov x10, #0xffffffffffffffff
0xffffffc00901520c: strb w1, [x25, x10]
0xffffffc009015210: mov x10, #0xffffffffffffd25c
0xffffffc009015214: movk x10, #0x902, lsl #16
0xffffffc009015218: movk x10, #0xffc0, lsl #32
0xffffffc00901521c: blr x10 -------------------. // bpf2bpf call
0xffffffc009015220: add x7, x0, #0x0 <-------------.
0xffffffc009015224: add sp, sp, #0x10 | |
0xffffffc009015228: ldp x27, x28, [sp], #16 | |
0xffffffc00901522c: ldp x25, x26, [sp], #16 | |
0xffffffc009015230: ldp x21, x22, [sp], #16 | |
0xffffffc009015234: ldp x19, x20, [sp], #16 | |
0xffffffc009015238: ldp x29, x30, [sp], #16 | |
0xffffffc00901523c: add x0, x7, #0x0 | |
0xffffffc009015240: autiasp | |
0xffffffc009015244: ret | |
| |
int subprog_tail(struct __sk_buff *skb): | |
0xffffffc00902d25c: paciasp <----------------------' |
0xffffffc00902d260: stp x29, x30, [sp, #-16]! |
0xffffffc00902d264: mov x29, sp |
0xffffffc00902d268: stp x19, x20, [sp, #-16]! |
0xffffffc00902d26c: stp x21, x22, [sp, #-16]! |
0xffffffc00902d270: stp x25, x26, [sp, #-16]! |
0xffffffc00902d274: stp x27, x28, [sp, #-16]! |
0xffffffc00902d278: mov x25, sp |
0xffffffc00902d27c: sub x27, x25, #0x0 |
0xffffffc00902d280: sub sp, sp, #0x10 | // <- end of prologue, notice:
0xffffffc00902d284: add x19, x0, #0x0 | // 1) TCC not touched, and
0xffffffc00902d288: mov w0, #0x1 | // 2) no tail call jump pad
0xffffffc00902d28c: mov x10, #0xfffffffffffffffc |
0xffffffc00902d290: str w0, [x25, x10] |
0xffffffc00902d294: mov x20, #0xffffff80ffffffff |
0xffffffc00902d298: movk x20, #0xc033, lsl #16 |
0xffffffc00902d29c: movk x20, #0x4e00 |
0xffffffc00902d2a0: add x0, x19, #0x0 |
0xffffffc00902d2a4: add x1, x20, #0x0 |
0xffffffc00902d2a8: mov x2, #0x0 |
0xffffffc00902d2ac: mov w10, #0x24 |
0xffffffc00902d2b0: ldr w10, [x1, x10] |
0xffffffc00902d2b4: add w2, w2, #0x0 |
0xffffffc00902d2b8: cmp w2, w10 |
0xffffffc00902d2bc: b.cs 0xffffffc00902d2f8 |
0xffffffc00902d2c0: mov w10, #0x21 |
0xffffffc00902d2c4: cmp x26, x10 | // TCC >= MAX_TAIL_CALL_CNT?
0xffffffc00902d2c8: b.cs 0xffffffc00902d2f8 |
0xffffffc00902d2cc: add x26, x26, #0x1 | // TCC++
0xffffffc00902d2d0: mov w10, #0x110 |
0xffffffc00902d2d4: add x10, x1, x10 |
0xffffffc00902d2d8: lsl x11, x2, #3 |
0xffffffc00902d2dc: ldr x11, [x10, x11] |
0xffffffc00902d2e0: cbz x11, 0xffffffc00902d2f8 |
0xffffffc00902d2e4: mov w10, #0x30 |
0xffffffc00902d2e8: ldr x10, [x11, x10] |
0xffffffc00902d2ec: add x10, x10, #0x24 |
0xffffffc00902d2f0: add sp, sp, #0x10 | // <- destroy just current
0xffffffc00902d2f4: br x10 ---------------------. | // BPF stack frame
0xffffffc00902d2f8: mov x10, #0xfffffffffffffffc | | // before the tail call
0xffffffc00902d2fc: ldr w7, [x25, x10] | |
0xffffffc00902d300: add sp, sp, #0x10 | |
0xffffffc00902d304: ldp x27, x28, [sp], #16 | |
0xffffffc00902d308: ldp x25, x26, [sp], #16 | |
0xffffffc00902d30c: ldp x21, x22, [sp], #16 | |
0xffffffc00902d310: ldp x19, x20, [sp], #16 | |
0xffffffc00902d314: ldp x29, x30, [sp], #16 | |
0xffffffc00902d318: add x0, x7, #0x0 | |
0xffffffc00902d31c: autiasp | |
0xffffffc00902d320: ret | |
| |
int classifier_0(struct __sk_buff *skb): | |
0xffffffc008ff5874: paciasp | |
0xffffffc008ff5878: stp x29, x30, [sp, #-16]! | |
0xffffffc008ff587c: mov x29, sp | |
0xffffffc008ff5880: stp x19, x20, [sp, #-16]! | |
0xffffffc008ff5884: stp x21, x22, [sp, #-16]! | |
0xffffffc008ff5888: stp x25, x26, [sp, #-16]! | |
0xffffffc008ff588c: stp x27, x28, [sp, #-16]! | |
0xffffffc008ff5890: mov x25, sp | |
0xffffffc008ff5894: mov x26, #0x0 | |
0xffffffc008ff5898: bti j <----------------------' |
0xffffffc008ff589c: sub x27, x25, #0x0 |
0xffffffc008ff58a0: sub sp, sp, #0x0 |
0xffffffc008ff58a4: mov x0, #0xffffffc0ffffffff |
0xffffffc008ff58a8: movk x0, #0x8fc, lsl #16 |
0xffffffc008ff58ac: movk x0, #0x6000 |
0xffffffc008ff58b0: mov w1, #0x1 |
0xffffffc008ff58b4: str w1, [x0] |
0xffffffc008ff58b8: mov w7, #0x0 |
0xffffffc008ff58bc: mov sp, sp |
0xffffffc008ff58c0: ldp x27, x28, [sp], #16 |
0xffffffc008ff58c4: ldp x25, x26, [sp], #16 |
0xffffffc008ff58c8: ldp x21, x22, [sp], #16 |
0xffffffc008ff58cc: ldp x19, x20, [sp], #16 |
0xffffffc008ff58d0: ldp x29, x30, [sp], #16 |
0xffffffc008ff58d4: add x0, x7, #0x0 |
0xffffffc008ff58d8: autiasp |
0xffffffc008ff58dc: ret -------------------------------'
Signed-off-by: Jakub Sitnicki <jakub@xxxxxxxxxxxxxx>
---
arch/arm64/net/bpf_jit_comp.c | 9 ++++++++-
1 file changed, 8 insertions(+), 1 deletion(-)
diff --git a/arch/arm64/net/bpf_jit_comp.c b/arch/arm64/net/bpf_jit_comp.c
index 8ab4035dea27..71c3f4f7224e 100644
--- a/arch/arm64/net/bpf_jit_comp.c
+++ b/arch/arm64/net/bpf_jit_comp.c
@@ -246,6 +246,7 @@ static bool is_lsi_offset(int offset, int scale)
static int build_prologue(struct jit_ctx *ctx, bool ebpf_from_cbpf)
{
const struct bpf_prog *prog = ctx->prog;
+ const bool is_main_prog = prog->aux->func_idx == 0;
const u8 r6 = bpf2a64[BPF_REG_6];
const u8 r7 = bpf2a64[BPF_REG_7];
const u8 r8 = bpf2a64[BPF_REG_8];
@@ -299,7 +300,7 @@ static int build_prologue(struct jit_ctx *ctx, bool ebpf_from_cbpf)
/* Set up BPF prog stack base register */
emit(A64_MOV(1, fp, A64_SP), ctx);
- if (!ebpf_from_cbpf) {
+ if (!ebpf_from_cbpf && is_main_prog) {
/* Initialize tail_call_cnt */
emit(A64_MOVZ(1, tcc, 0, 0), ctx);
@@ -1529,3 +1530,9 @@ void bpf_jit_free_exec(void *addr)
{
return vfree(addr);
}
+
+/* Indicate the JIT backend supports mixing bpf2bpf and tailcalls. */
+bool bpf_jit_supports_subprog_tailcalls(void)
+{
+ return true;
+}
--
2.35.3
