Re: Kernel panic on bpf_skb_pull_data

[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]

 



On 03/24, Farbod Shahinfar wrote:
Hello everyone,

I am performing some test with BPF SK_SKB and I have encountered a
scenario resulting in kernel panic. I use a BPF_SK_SKB_STREAM_PARSER
program to parse a request which might be spanning multiple TCP
segments. If the end of request is detected in the parser program it
returns skb->len, passing the request to the
BPF_SK_SKB_STREAM_VERDICT program, and otherwise it returns 0, waiting
for more data to be received. You can find the BPF program attached
(bpf_test.c). Is there an assumption that the program violates?

To reproduce the crashing scenario, I use the python script attached
(client.py) which sends data in chunks toward the bpf program. Usually,
the kernel crashes on the 3rd segment.
�
To provide more information, I have attached some crash logs. I have
tested this on kernel version 6.1.0 (slightly modified) and version
6.2.8 (unmodified, obtained from kernel.org). It seems that the panic
happens when invoking the bpf_skb_pull_data.

Is this a known issue or is there any information that I can provide to help resolve it?

John, is it something already covered by your recent [0]?

0: https://lore.kernel.org/bpf/20230321215212.525630-1-john.fastabend@xxxxxxxxx/

Sincerely,
Farbod Shahinfar

PhD student at Politecnico di Milano
https://fshahinfar1.github.io/

kernel BUG at net/core/skbuff.c:2446!
[ 1549.847914] invalid opcode: 0000 [#1] PREEMPT SMP NOPTI
[ 1549.853139] CPU: 10 PID: 2008 Comm: python3 Tainted: G O 6.2.8 #4 [ 1549.860615] Hardware name: Dell Inc. PowerEdge C6525/04DK47, BIOS 2.0.3 01/15/2021
[ 1549.868181] RIP: 0010:__pskb_pull_tail+0x45c/0x4b0
[ 1549.872973] Code: 24 08 be 02 00 00 00 e8 42 cb ff ff 48 83 c4 10 31 c0 5b 41 5c 41 5d 41 5e 41 5f 5d e9 41 a0 26 00 48 8d 78 ff e9 42 fd ff ff <0f> 0b 48 c7 44 24 08 00 00 00 00 e9 9f fe ff ff 48 c7 44 24 08 00
[ 1549.891711] RSP: 0018:ffffad1fc05f8998 EFLAGS: 00010282
[ 1549.896939] RAX: 00000000fffffff2 RBX: 0000000000000001 RCX: 00000000000002c0 [ 1549.904070] RDX: ffff992f00ffe15b RSI: ffff992f00ffe000 RDI: ffff992f168adf00 [ 1549.911196] RBP: ffffad1fc05f89d0 R08: 00000000ffffff1c R09: ffff992f168ad300 [ 1549.918329] R10: 0000000000000001 R11: 0000000000000001 R12: 0000000000000001 [ 1549.925460] R13: ffff992f168adf00 R14: ffff992f00fff048 R15: ffff992f168adf00 [ 1549.932585] FS: 00007fd4a5e55740(0000) GS:ffff994dfee80000(0000) knlGS:0000000000000000
[ 1549.940670] CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[ 1549.946409] CR2: 0000000000453531 CR3: 000000010a688000 CR4: 0000000000350ee0
[ 1549.953542] Call Trace:
[ 1549.955997]  <IRQ>
[ 1549.958017]  skb_ensure_writable+0x8f/0xa0
[ 1549.962114]  sk_skb_pull_data+0x24/0x30
[ 1549.965953]  bpf_prog_5c4f327a1e142419_parser+0x1f/0xd8
[ 1549.971179]  sk_psock_strp_parse+0x50/0xc0
[ 1549.975279]  __strp_recv+0x1fb/0x630
[ 1549.978859]  strp_recv+0x27/0x30
[ 1549.982092]  tcp_read_sock+0x91/0x1c0
[ 1549.985757]  ? __pfx_strp_recv+0x10/0x10
[ 1549.989683]  strp_read_sock+0x68/0xb0
[ 1549.993348]  strp_data_ready+0x61/0xa0
[ 1549.997101]  sk_psock_strp_data_ready+0x86/0xa0
[ 1550.001635]  tcp_data_ready+0x33/0xe0
[ 1550.005301]  tcp_data_queue+0x469/0xd40
[ 1550.009142]  tcp_rcv_established+0x264/0x730
[ 1550.013411]  ? sk_filter_trim_cap+0x114/0x250
[ 1550.017772]  tcp_v4_do_rcv+0x15d/0x2b0
[ 1550.021524]  tcp_v4_rcv+0x11f1/0x1260
[ 1550.025192]  ip_protocol_deliver_rcu+0x3b/0x330
[ 1550.029724]  ip_local_deliver_finish+0x8a/0xb0
[ 1550.034171]  ip_local_deliver+0x73/0x120
[ 1550.038095]  ? __pfx_ip_local_deliver_finish+0x10/0x10
[ 1550.043234]  ip_rcv_finish+0xc2/0xd0
[ 1550.046816]  ip_rcv+0x57/0xf0
[ 1550.049787]  ? probe_sched_wakeup+0x39/0x40
[ 1550.053975]  __netif_receive_skb_one_core+0x8c/0xa0
[ 1550.058854]  __netif_receive_skb+0x15/0x60
[ 1550.062953]  process_backlog+0xa8/0x140
[ 1550.066793]  __napi_poll+0x31/0x1d0
[ 1550.070286]  net_rx_action+0x290/0x2e0
[ 1550.074037]  __do_softirq+0xf5/0x2d7
[ 1550.077618]  do_softirq+0x9a/0xc0
[ 1550.080935]  </IRQ>
[ 1550.083032]  <TASK>
[ 1550.085129]  __local_bh_enable_ip+0x7d/0x80
[ 1550.089318]  ip_finish_output2+0x19c/0x570
[ 1550.093417]  __ip_finish_output+0x1fe/0x2f0
[ 1550.097601]  ? add_wait_queue+0x6f/0x80
[ 1550.101443]  ip_finish_output+0x2e/0xd0
[ 1550.105282]  ip_output+0x7e/0x110
[ 1550.108601]  ? do_renameat2+0x244/0x5b0
[ 1550.112441]  ? __pfx_ip_finish_output+0x10/0x10
[ 1550.116974]  ip_local_out+0x62/0x70
[ 1550.120466]  __ip_queue_xmit+0x192/0x450
[ 1550.124392]  ip_queue_xmit+0x19/0x20
[ 1550.127971]  __tcp_transmit_skb+0xa56/0xb90
[ 1550.132157]  tcp_write_xmit+0x54d/0x12a0
[ 1550.136082]  ? _copy_from_iter+0x12a/0x5b0
[ 1550.140184]  __tcp_push_pending_frames+0x3b/0x110
[ 1550.144890]  tcp_push+0x10c/0x120
[ 1550.148207]  tcp_sendmsg_locked+0x491/0xc30
[ 1550.152397]  tcp_sendmsg+0x31/0x50
[ 1550.155800]  inet_sendmsg+0x47/0x80
[ 1550.159294]  sock_sendmsg+0x66/0x70
[ 1550.162786]  __sys_sendto+0x122/0x1b0
[ 1550.166454]  ? debug_smp_processor_id+0x1b/0x30
[ 1550.170984]  ? fpregs_assert_state_consistent+0x2b/0x60
[ 1550.176209]  ? exit_to_user_mode_prepare+0x49/0x1b0
[ 1550.181090]  __x64_sys_sendto+0x2d/0x40
[ 1550.184928]  do_syscall_64+0x3f/0x90
[ 1550.188509]  entry_SYSCALL_64_after_hwframe+0x72/0xdc
[ 1550.193561] RIP: 0033:0x7fd4a6141530
[ 1550.197140] Code: ff eb bc 0f 1f 80 00 00 00 00 f3 0f 1e fa 41 89 ca 64 8b 04 25 18 00 00 00 85 c0 75 1d 45 31 c9 45 31 c0 b8 2c 00 00 00 0f 05 <48> 3d 00 f0 ff ff 77 68 c3 0f 1f 80 00 00 00 00 55 48 83 ec 20 48 [ 1550.215879] RSP: 002b:00007ffe74631ee8 EFLAGS: 00000246 ORIG_RAX: 000000000000002c [ 1550.223443] RAX: ffffffffffffffda RBX: 00007ffe74631f80 RCX: 00007fd4a6141530 [ 1550.230570] RDX: 0000000000000001 RSI: 00007fd4a5671ad0 RDI: 0000000000000003 [ 1550.237702] RBP: 000000000113cfe0 R08: 0000000000000000 R09: 0000000000000000 [ 1550.244835] R10: 0000000000000000 R11: 0000000000000246 R12: 0000000000000001 [ 1550.251967] R13: 0000000000000000 R14: 00007ffe74631f80 R15: 0000000000624240
[ 1550.259102]  </TASK>
[ 1550.261293] Modules linked in: nfsv3 nfs_acl nfs lockd grace fscache netfs ipod(O) ipmi_ssif intel_rapl_msr intel_rapl_common amd64_edac edac_mce_amd kvm_amd binfmt_misc kvm crct10dif_pclmul ghash_clmulni_intel sha512_ssse3 aesni_intel crypto_simd cryptd mgag200 acpi_ipmi drm_shmem_helper rapl wmi_bmof drm_kms_helper i2c_algo_bit syscopyarea ipmi_si ipmi_devintf sysfillrect sysimgblt ccp k10temp ipmi_msghandler acpi_power_meter mac_hid sch_fq_codel drm sunrpc ip_tables x_tables autofs4 mlx5_ib ib_uverbs ib_core mlx5_core pci_hyperv_intf ahci mlxfw crc32_pclmul libahci psample i2c_piix4 tls wmi
[ 1550.313799] ---[ end trace 0000000000000000 ]---
[ 1550.318423] RIP: 0010:__pskb_pull_tail+0x45c/0x4b0
[ 1550.323223] Code: 24 08 be 02 00 00 00 e8 42 cb ff ff 48 83 c4 10 31 c0 5b 41 5c 41 5d 41 5e 41 5f 5d e9 41 a0 26 00 48 8d 78 ff e9 42 fd ff ff <0f> 0b 48 c7 44 24 08 00 00 00 00 e9 9f fe ff ff 48 c7 44 24 08 00
[ 1550.341972] RSP: 0018:ffffad1fc05f8998 EFLAGS: 00010282
[ 1550.347205] RAX: 00000000fffffff2 RBX: 0000000000000001 RCX: 00000000000002c0 [ 1550.354339] RDX: ffff992f00ffe15b RSI: ffff992f00ffe000 RDI: ffff992f168adf00 [ 1550.361478] RBP: ffffad1fc05f89d0 R08: 00000000ffffff1c R09: ffff992f168ad300 [ 1550.368611] R10: 0000000000000001 R11: 0000000000000001 R12: 0000000000000001 [ 1550.375746] R13: ffff992f168adf00 R14: ffff992f00fff048 R15: ffff992f168adf00 [ 1550.382886] FS: 00007fd4a5e55740(0000) GS:ffff994dfee80000(0000) knlGS:0000000000000000
[ 1550.390972] CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[ 1550.396719] CR2: 0000000000453531 CR3: 000000010a688000 CR4: 0000000000350ee0
[ 1550.403861] Kernel panic - not syncing: Fatal exception in interrupt
[ 1550.410763] Kernel Offset: 0x0 from 0xffffffff81000000 (relocation range: 0xffffffff80000000-0xffffffffbfffffff) [ 1550.420940] ---[ end Kernel panic - not syncing: Fatal exception in interrupt ]---


[  404.840245] kernel BUG at net/core/skbuff.c:2380!
[  404.845070] invalid opcode: 0000 [#1] PREEMPT SMP PTI
[ 404.846838] CPU: 2 PID: 1166 Comm: nc Not tainted 6.1.0-rc7-g83de108e3723-dirty #73 [ 404.848923] Hardware name: QEMU Standard PC (Q35 + ICH9, 2009), BIOS 1.15.0-1 04/01/2014
[  404.850796] RIP: 0010:__pskb_pull_tail+0x3fc/0x450
[ 404.852171] Code: 24 08 be 01 00 00 00 e8 e2 d0 ff ff 48 83 c4 10 31 c0 5b 41 5c 41 5d 41 5e 41 5f 5d c3 cc cc cc cc 48 8d 78 ff e9 96 fd ff ff <0f> 0b 48 c7 44 24 08 00 00 00 00 e9 9f fe ff ff 48 c7 44 24 08 00
[  404.856380] RSP: 0018:ffffa7aa800fca50 EFLAGS: 00010282
[ 404.857563] RAX: 00000000fffffff2 RBX: 0000000000000008 RCX: 0000000000000008 [ 404.859208] RDX: ffffa09f464b815d RSI: 00000000000002c0 RDI: 0000000000000000 [ 404.860790] RBP: ffffa7aa800fca88 R08: 00000000ffffff25 R09: 7fffffffffffffff [ 404.862379] R10: 0000000000000008 R11: ffffa09f44d9fc40 R12: 0000000000000008 [ 404.863963] R13: ffffa09f44d9fc00 R14: ffffa09f44d9fc00 R15: ffffa09f44d9f700 [ 404.865534] FS: 00007f98873c3740(0000) GS:ffffa0a0b7d00000(0000) knlGS:0000000000000000
[  404.867367] CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[ 404.868771] CR2: 00007ffe1f2db090 CR3: 00000001033c8002 CR4: 0000000000370ee0
[  404.870448] Call Trace:
[  404.871063]  <IRQ>
[  404.871663]  skb_ensure_writable+0x84/0xa0
[  404.872678]  sk_skb_pull_data+0x17/0x20
[  404.873650]  bpf_prog_fddaddf43f6741a5_parser+0x24/0x1e9
[  404.874942]  ? selinux_netlbl_sock_rcv_skb+0x2e/0x1b0
[  404.876160]  ? kmem_cache_alloc+0x33/0x1b0
[  404.877175]  sk_psock_strp_parse+0x3f/0xc0
[  404.878222]  __strp_recv+0x1f2/0x620
[  404.879143]  ? strp_process+0x40/0x40
[  404.880081]  tcp_read_sock+0x7f/0x1b0
[  404.881019]  strp_read_sock+0x5e/0xa0
[  404.882719]  strp_data_ready+0x54/0x80
[  404.883675]  sk_psock_strp_data_ready+0x68/0x80
[  404.884890]  tcp_data_queue+0x43a/0xce0
[  404.885950]  tcp_rcv_established+0x248/0x6e0
[  404.887026]  tcp_v4_do_rcv+0x147/0x290
[  404.887969]  tcp_v4_rcv+0xe59/0xf00
[  404.888865]  ip_protocol_deliver_rcu+0x2d/0x1f0
[  404.889985]  ip_local_deliver_finish+0x6e/0x90
[  404.891125]  ip_local_deliver+0x66/0x110
[  404.892116]  ip_rcv+0x4a/0xf0
[  404.893091]  __netif_receive_skb_one_core+0x86/0xa0
[  404.894300]  process_backlog+0xa3/0x150
[  404.895287]  __napi_poll+0x24/0x160
[  404.896249]  net_rx_action+0x291/0x350
[  404.897198]  __do_softirq+0xb3/0x28c
[  404.898183]  do_softirq+0x52/0x70
[  404.899064]  </IRQ>
[  404.899678]  <TASK>
[  404.900310]  __local_bh_enable_ip+0x5f/0x70
[  404.901457]  ip_finish_output2+0x179/0x500
[  404.902572]  ip_output+0x71/0x110
[  404.903481]  ? __ip_finish_output+0x2a0/0x2a0
[  404.904643]  __ip_queue_xmit+0x174/0x3d0
[  404.905693]  __tcp_transmit_skb+0xa38/0xb50
[  404.906764]  ? __alloc_skb+0x89/0x1b0
[  404.907704]  tcp_write_xmit+0x4dc/0x1160
[  404.908689]  __tcp_push_pending_frames+0x2d/0xc0
[  404.909830]  tcp_sendmsg_locked+0x291/0xbf0
[  404.910872]  tcp_sendmsg+0x23/0x40
[  404.911739]  sock_sendmsg+0x56/0x60
[  404.913378]  sock_write_iter+0x92/0xf0
[  404.914352]  vfs_write+0x356/0x3c0
[  404.915263]  ksys_write+0xa6/0xe0
[  404.916298]  do_syscall_64+0x38/0x90
[  404.917245]  entry_SYSCALL_64_after_hwframe+0x63/0xcd
[  404.918560] RIP: 0033:0x7f98874d4077
[ 404.919524] Code: 64 89 02 48 c7 c0 ff ff ff ff eb bb 0f 1f 80 00 00 00 00 f3 0f 1e fa 64 8b 04 25 18 00 00 00 85 c0 75 10 b8 01 00 00 00 0f 05 <48> 3d 00 f0 ff ff 77 51 c3 48 83 ec 28 48 89 54 24 18 48 89 74 24 [ 404.923807] RSP: 002b:00007ffe1f2db018 EFLAGS: 00000246 ORIG_RAX: 0000000000000001 [ 404.925817] RAX: ffffffffffffffda RBX: 0000000000000000 RCX: 00007f98874d4077 [ 404.927706] RDX: 0000000000000008 RSI: 00007ffe1f2df080 RDI: 0000000000000003 [ 404.929369] RBP: 00007ffe1f2db058 R08: 0000000000000004 R09: 0000000000000001 [ 404.931039] R10: 0000000000000000 R11: 0000000000000246 R12: 0000000000000008 [ 404.932717] R13: 00007ffe1f2df080 R14: 0000000000000003 R15: 0000000000000005
[  404.934448]  </TASK>
[  404.935141] Modules linked in:
[  404.936005] ---[ end trace 0000000000000000 ]---
[  404.937206] RIP: 0010:__pskb_pull_tail+0x3fc/0x450
[ 404.938475] Code: 24 08 be 01 00 00 00 e8 e2 d0 ff ff 48 83 c4 10 31 c0 5b 41 5c 41 5d 41 5e 41 5f 5d c3 cc cc cc cc 48 8d 78 ff e9 96 fd ff ff <0f> 0b 48 c7 44 24 08 00 00 00 00 e9 9f fe ff ff 48 c7 44 24 08 00
[  404.943339] RSP: 0018:ffffa7aa800fca50 EFLAGS: 00010282
[ 404.944681] RAX: 00000000fffffff2 RBX: 0000000000000008 RCX: 0000000000000008 [ 404.946518] RDX: ffffa09f464b815d RSI: 00000000000002c0 RDI: 0000000000000000 [ 404.948302] RBP: ffffa7aa800fca88 R08: 00000000ffffff25 R09: 7fffffffffffffff [ 404.950269] R10: 0000000000000008 R11: ffffa09f44d9fc40 R12: 0000000000000008 [ 404.951691] R13: ffffa09f44d9fc00 R14: ffffa09f44d9fc00 R15: ffffa09f44d9f700 [ 404.953292] FS: 00007f98873c3740(0000) GS:ffffa0a0b7d00000(0000) knlGS:0000000000000000
[  404.955084] CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[ 404.956425] CR2: 00007ffe1f2db090 CR3: 00000001033c8002 CR4: 0000000000370ee0
[  404.958010] Kernel panic - not syncing: Fatal exception in interrupt
[ 404.959636] Kernel Offset: 0x29200000 from 0xffffffff81000000 (relocation range: 0xffffffff80000000-0xffffffffbfffffff) [ 404.961972] ---[ end Kernel panic - not syncing: Fatal exception in interrupt ]---

import socket
import time
import argparse

def _send(sock, data):
     print(f'[Sending "{data}"]')
     sock.send(data.encode())

def _recv(sock):
     try:
         resp = sock.recv(2048)
         return resp.decode()
     except:
         print('[No data received]')
         return None


def parse_args():
     parser = argparse.ArgumentParser()
parser.add_argument('-k', help='Kernel crash scenario', action='store_true')
     parser.add_argument('--ip', default='localhost')
     parser.add_argument('--port', default=8080, type=int)
     args = parser.parse_args()
     return args


def main():
     args = parse_args()

     s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
     s.connect((args.ip, args.port))
     s.settimeout(1)

     # What request to send?
     req = ['hello world 1', 'hello world 2 END']
     if args.k:
         req = ['hello world 1', 'hello world 2', 'hello world 3',
                 'hello world 4', 'hello world 5 END']

     for r in req:
         _send(s, r)
         resp = _recv(s)
         if resp:
             print(resp)


     s.close()


if __name__ == '__main__':
     main()


#include <sys/types.h>
#include <sys/socket.h>
#include <linux/tcp.h>
#include <linux/bpf.h>
#include <bpf/bpf_helpers.h>
#include <bpf/bpf_endian.h>

/* Put state of each socket in this struct (This will be used in sockops.h as
  * part of per socket metadata) */
struct connection_state { };

#include "my_bpf/sockops.h"

#define OFFSET_MASK 0x0fff

/* struct arg { */
/* 	int test; */
/* }; */

/* struct { */
/* 	__uint(type,  BPF_MAP_TYPE_ARRAY); */
/* 	/1* __uint(map_flags, BPF_F_MMAPABLE); *1/ */
/* 	__type(key,   __u32); */
/* 	__type(value, struct arg); */
/* 	__uint(max_entries, 1); */
/* } arg_map SEC(".maps"); */

SEC("sk_skb/stream_parser")
int parser(struct __sk_buff *skb)
{

	void *data;
	void *data_end;
	__u16 len;

	/* Pull message data so that we can access it */
	if (bpf_skb_pull_data(skb, skb->len) != 0) {
		bpf_printk("Parser: Failed to load message data\n");
		return 0;
	}

	data = (void *)(long)skb->data;
	data_end = (void *)(long)skb->data_end;
	len = skb->len;

	char *ptr = data + ((len - 3) & 0x7fff);
	if ((void *)ptr < data || ((void *)ptr + 3 > data_end)) {
		bpf_printk("Parser: Not enough data!");
		return 0;
	}

	if (ptr[0] == 'E' && ptr[1] == 'N' && ptr[2] == 'D') {
		/* Found the end of request */
		return skb->len;
	}
	bpf_printk("@%d\n%s", (long)ptr - (long)data, ptr);

	return 0;
}

SEC("sk_skb/stream_verdict")
int verdict(struct __sk_buff *skb)
{
	return SK_PASS;
}

char _license[] SEC("license") = "GPL";





[Index of Archives]     [Linux Samsung SoC]     [Linux Rockchip SoC]     [Linux Actions SoC]     [Linux for Synopsys ARC Processors]     [Linux NFS]     [Linux NILFS]     [Linux USB Devel]     [Video for Linux]     [Linux Audio Users]     [Yosemite News]     [Linux Kernel]     [Linux SCSI]


  Powered by Linux