Hi BPF Experts, I had a problem that verifier report "R1 invalid mem access 'inv'" when I attempted to rewrite packet destination ethernet MAC address in Cilium tunnel mode, I opened an issue with detail here https://github.com/cilium/cilium/issues/16571: I have couple of questions in general to try to understand the compiler, BPF byte code, and the verifier. 1 Why the BPF byte code changes so much with my simple C code change a: BPF byte code before C code change: 0000000000006068 <LBB12_410>: 3085: bf a2 00 00 00 00 00 00 r2 = r10 ; tunnel = map_lookup_elem(&TUNNEL_MAP, key); 3086: 07 02 00 00 78 ff ff ff r2 += -136 3087: 18 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 r1 = 0 ll 3089: 85 00 00 00 01 00 00 00 call 1 ; if (!tunnel) 3090: 15 00 06 01 00 00 00 00 if r0 == 0 goto +262 <LBB12_441> ; key.tunnel_id = seclabel; 3091: 18 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 r2 = 0 ll 3093: 67 02 00 00 20 00 00 00 r2 <<= 32 3094: 77 02 00 00 20 00 00 00 r2 >>= 32 3095: b7 01 00 00 06 00 00 00 r1 = 6 3096: 15 02 02 00 01 00 00 00 if r2 == 1 goto +2 <LBB12_413> 3097: 18 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 r1 = 0 ll 00000000000060d8 <LBB12_413>: ; return __encap_and_redirect_with_nodeid(ctx, tunnel->ip4, seclabel, monitor); b: BPF byte code after C code change: the C code diff change: diff --git a/bpf/lib/encap.h b/bpf/lib/encap.h index dfd87bd82..19199429d 100644 --- a/bpf/lib/encap.h +++ b/bpf/lib/encap.h @@ -187,6 +187,8 @@ encap_and_redirect_lxc(struct __ctx_buff *ctx, __u32 tunnel_endpoint, struct endpoint_key *key, __u32 seclabel, __u32 monitor) { struct endpoint_key *tunnel; +#define VTEP_MAC { .addr = { 0xce, 0x72, 0xa7, 0x03, 0x88, 0x58 } } + union macaddr vtep_mac = VTEP_MAC; if (tunnel_endpoint) { #ifdef ENABLE_IPSEC @@ -221,6 +223,8 @@ encap_and_redirect_lxc(struct __ctx_buff *ctx, __u32 tunnel_endpoint, seclabel); } #endif + if (eth_store_daddr(ctx, (__u8 *) &vtep_mac.addr, 0) < 0) + return DROP_WRITE_ERROR; return __encap_and_redirect_with_nodeid(ctx, tunnel->ip4, seclabel, monitor); } the result BPF byte code 0000000000004468 <LBB3_274>: 2189: bf a2 00 00 00 00 00 00 r2 = r10 ; tunnel = map_lookup_elem(&TUNNEL_MAP, key); 2190: 07 02 00 00 50 ff ff ff r2 += -176 2191: 18 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 r1 = 0 ll 2193: 85 00 00 00 01 00 00 00 call 1 2194: bf 07 00 00 00 00 00 00 r7 = r0 2195: 79 a6 48 ff 00 00 00 00 r6 = *(u64 *)(r10 - 184) ; if (!tunnel) 2196: 55 07 94 00 00 00 00 00 if r7 != 0 goto +148 <LBB3_289> 00000000000044a8 <LBB3_275>: ; __u8 new_ttl, ttl = ip4->ttl; 2197: 79 a1 38 ff 00 00 00 00 r1 = *(u64 *)(r10 - 200) 2198: 71 13 16 00 00 00 00 00 r3 = *(u8 *)(r1 + 22) ; if (ttl <= 1) 2199: 25 03 01 00 01 00 00 00 if r3 > 1 goto +1 <LBB3_277> 2200: 05 00 20 ff 00 00 00 00 goto -224 <LBB3_253> You can see that: before change: <LBB12_410> after change <LBB3_274> is different that <LBB12_410> has instructions 3091, 3092... but <LBB3_274> end with instruction 2196 before change: <LBB12_413> follows <LBB12_410> after change: <LBB3_275> follows <LBB3_274> <LBB12_413> and <LBB3_275> is very much different and <LBB3_275> instruction 2198 is the one with "R1 invalid mem access 'inv'" Why <LBB3_275> follows <LBB3_274> ? from C code, <LBB3_275> is not close to <LBB3_274>. 2, Can I assume the verifier is to simulate the order of BPF byte code execution in run time, like if without any jump or goto in <LBB3_274>, <LBB3_275> will be executed after <LBB3_274>? Enterprise Network Engineer F5 Networks Inc https://www.youtube.com/c/VincentLi
