> > [ 75.793150] RIP: 0010:rb_insert_color+0x189/0x1480 > > What's in that line? Try, > > $ ./scripts/faddr2line vmlinux rb_insert_color+0x189/0x1480 rb_insert_color+0x189/0x1480: __rb_insert at /home/files/git/linux/lib/rbtree.c:131 (inlined by) rb_insert_color at /home/files/git/linux/lib/rbtree.c:452 > > What's steps to reproduce this? The steps is the kernel config provided (proc.config) and I double checked the attached C code from the qemu image (attached here). If the kernel does not immediately crash, a ^C will cause the fault to be noticed. The report from earlier is the report from the same code, my assumption was that the possible pool/redzone corruption is making it a bit tricky to pin down. If you would like alternative kernel settings please let me know, I can do that, also, my current test-bench has about 256 core's on x64, 64 of them are bare metal and 32 are arm64. Any possible preferred configuration tweaks I'm all ears, I'll be including some of these steps you suggested to me in any/additional upcoming threads (Thank you for that so far and future suggestions). Also, there is some occasionally varying stacks depending on the corruption, so this stack just now (another execution of test3.c); ./scripts/faddr2line vmlinux rcu_process_callbacks+0xd45/0x1650 rcu_process_callbacks+0xd45/0x1650: rcu_lock_release at include/linux/rcupdate.h:228 (inlined by) __rcu_reclaim at kernel/rcu/rcu.h:234 (inlined by) rcu_do_batch at kernel/rcu/tree.c:2452 (inlined by) invoke_rcu_callbacks at kernel/rcu/tree.c:2773 (inlined by) rcu_process_callbacks at kernel/rcu/tree.c:2754 (stack from just now) [12580.358392] ================================================================== [12580.360076] BUG: KASAN: double-free or invalid-free in rcu_process_callbacks+0xd45/0x1650 [12580.361738] [12580.362068] CPU: 1 PID: 16 Comm: ksoftirqd/1 Not tainted 5.0.0-rc1+ #5 [12580.363383] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS 1.11.1-1ubuntu1 04/01/2014 [12580.365223] Call Trace: [12580.365772] dump_stack+0x1d3/0x2c2 [12580.366518] ? dump_stack_print_info.cold.1+0x20/0x20 [12580.367608] ? printk+0xad/0xd3 [12580.368278] ? kmsg_dump_rewind_nolock+0xf0/0xf0 [12580.369261] print_address_description.cold.5+0x9/0x208 [12580.370393] ? rcu_process_callbacks+0xd45/0x1650 [12580.371376] kasan_report_invalid_free+0x64/0xa0 [12580.372356] ? rcu_process_callbacks+0xd45/0x1650 [12580.373358] __kasan_slab_free+0x138/0x150 [12580.374196] ? rcu_process_callbacks+0xd45/0x1650 [12580.375142] kasan_slab_free+0xe/0x10 [12580.375905] kfree+0xcf/0x220 [12580.376537] rcu_process_callbacks+0xd45/0x1650 [12580.377464] ? rcu_process_callbacks+0xcf8/0x1650 [12580.378431] ? rcu_fwd_progress_check+0xf0/0xf0 [12580.379371] ? compat_start_thread+0x80/0x80 [12580.380292] ? kasan_check_write+0x14/0x20 [12580.381145] ? finish_task_switch+0x2cb/0x880 [12580.382028] ? finish_task_switch+0x189/0x880 [12580.382920] ? preempt_notifier_register+0x210/0x210 [12580.383944] ? lock_repin_lock+0x450/0x450 [12580.384808] ? __do_softirq+0x27d/0xb6a [12580.385618] ? kasan_check_read+0x11/0x20 [12580.386461] ? rcu_is_watching+0x9d/0x160 [12580.387341] ? trace_hardirqs_on+0xce/0x310 [12580.388217] ? rcu_pm_notify+0xd0/0xd0 [12580.389008] __do_softirq+0x2eb/0xb6a [12580.389816] ? __irqentry_text_end+0x1f9d5b/0x1f9d5b [12580.390838] ? trace_hardirqs_off+0xc6/0x310 [12580.391729] ? smpboot_thread_fn+0x419/0x900 [12580.392611] ? trace_hardirqs_on+0x310/0x310 [12580.393503] ? check_same_owner+0x350/0x350 [12580.394368] ? takeover_tasklets+0xaa0/0xaa0 [12580.395268] ? takeover_tasklets+0xaa0/0xaa0 [12580.396153] run_ksoftirqd+0x8b/0x110 [12580.396922] smpboot_thread_fn+0x419/0x900 [12580.397785] ? sort_range+0x40/0x40 [12580.398513] ? __sanitizer_cov_trace_const_cmp8+0x18/0x20 [12580.399697] ? __kthread_parkme+0x106/0x1c0 [12580.400563] ? sort_range+0x40/0x40 [12580.401270] kthread+0x358/0x460 [12580.401956] ? kthread_bind+0x40/0x40 [12580.402741] ret_from_fork+0x24/0x30 [12580.403504] [12580.403844] Allocated by task 0: [12580.404524] (stack is not available) [12580.405276] [12580.405620] Freed by task 0: [12580.406223] (stack is not available) [12580.406955] [12580.407273] The buggy address belongs to the object at ffff88805b13e4f8 [12580.407273] which belongs to the cache kmemleak_object of size 360 [12580.409867] The buggy address is located 120 bytes inside of [12580.409867] 360-byte region [ffff88805b13e4f8, ffff88805b13e660) [12580.412182] The buggy address belongs to the page: [12580.413163] page:ffffea00016c4f80 count:1 mapcount:0 mapping:ffff88800fc13e40 index:0x0 [12580.414798] flags: 0x1fffc0000000200(slab) [12580.415653] raw: 01fffc0000000200 ffffea00016c7fc8 ffffea0001aba308 ffff88800fc13e40 [12580.417245] raw: 0000000000000000 ffff88805b13e000 0000000100000009 0000000000000000 [12580.418800] page dumped because: kasan: bad access detected [12580.419969] [12580.420300] Memory state around the buggy address: [12580.421303] ffff88805b13e400: fb fb fb fb fb fb fb fb fb fb fb fb fb fb fb fb [12580.422788] ffff88805b13e480: fb fb fb fb fb fb fb fc fc fc fc fc fc fc fc 00 [12580.424235] >ffff88805b13e500: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 [12580.425665] ^ [12580.427037] ffff88805b13e580: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 [12580.428479] ffff88805b13e600: 00 00 00 00 00 00 00 00 00 00 00 00 fc fc fc fc [12580.429970] ==================================================================
// autogenerated by syzkaller (https://github.com/google/syzkaller) #define _GNU_SOURCE #include <dirent.h> #include <endian.h> #include <errno.h> #include <fcntl.h> #include <setjmp.h> #include <signal.h> #include <stdint.h> #include <stdio.h> #include <stdlib.h> #include <string.h> #include <sys/ioctl.h> #include <sys/prctl.h> #include <sys/socket.h> #include <sys/stat.h> #include <sys/syscall.h> #include <sys/types.h> #include <sys/wait.h> #include <time.h> #include <unistd.h> #include <linux/genetlink.h> #include <linux/netlink.h> unsigned long long procid; static __thread int skip_segv; static __thread jmp_buf segv_env; static void segv_handler(int sig, siginfo_t* info, void* ctx) { uintptr_t addr = (uintptr_t)info->si_addr; const uintptr_t prog_start = 1 << 20; const uintptr_t prog_end = 100 << 20; if (__atomic_load_n(&skip_segv, __ATOMIC_RELAXED) && (addr < prog_start || addr > prog_end)) { _longjmp(segv_env, 1); } exit(sig); } static void install_segv_handler(void) { struct sigaction sa; memset(&sa, 0, sizeof(sa)); sa.sa_handler = SIG_IGN; syscall(SYS_rt_sigaction, 0x20, &sa, NULL, 8); syscall(SYS_rt_sigaction, 0x21, &sa, NULL, 8); memset(&sa, 0, sizeof(sa)); sa.sa_sigaction = segv_handler; sa.sa_flags = SA_NODEFER | SA_SIGINFO; sigaction(SIGSEGV, &sa, NULL); sigaction(SIGBUS, &sa, NULL); } #define NONFAILING(...) { __atomic_fetch_add(&skip_segv, 1, __ATOMIC_SEQ_CST); if (_setjmp(segv_env) == 0) { __VA_ARGS__; } __atomic_fetch_sub(&skip_segv, 1, __ATOMIC_SEQ_CST); } static void sleep_ms(uint64_t ms) { usleep(ms * 1000); } static uint64_t current_time_ms(void) { struct timespec ts; if (clock_gettime(CLOCK_MONOTONIC, &ts)) exit(1); return (uint64_t)ts.tv_sec * 1000 + (uint64_t)ts.tv_nsec / 1000000; } static long syz_open_dev(long a0, long a1, long a2) { if (a0 == 0xc || a0 == 0xb) { char buf[128]; sprintf(buf, "/dev/%s/%d:%d", a0 == 0xc ? "char" : "block", (uint8_t)a1, (uint8_t)a2); return open(buf, O_RDWR, 0); } else { char buf[1024]; char* hash; NONFAILING(strncpy(buf, (char*)a0, sizeof(buf) - 1)); buf[sizeof(buf) - 1] = 0; while ((hash = strchr(buf, '#'))) { *hash = '0' + (char)(a1 % 10); a1 /= 10; } return open(buf, a2, 0); } } static long syz_open_pts(long a0, long a1) { int ptyno = 0; if (ioctl(a0, TIOCGPTN, &ptyno)) return -1; char buf[128]; sprintf(buf, "/dev/pts/%d", ptyno); return open(buf, a1, 0); } static long syz_genetlink_get_family_id(long name) { char buf[512] = {0}; struct nlmsghdr* hdr = (struct nlmsghdr*)buf; struct genlmsghdr* genlhdr = (struct genlmsghdr*)NLMSG_DATA(hdr); struct nlattr* attr = (struct nlattr*)(genlhdr + 1); hdr->nlmsg_len = sizeof(*hdr) + sizeof(*genlhdr) + sizeof(*attr) + GENL_NAMSIZ; hdr->nlmsg_type = GENL_ID_CTRL; hdr->nlmsg_flags = NLM_F_REQUEST | NLM_F_ACK; genlhdr->cmd = CTRL_CMD_GETFAMILY; attr->nla_type = CTRL_ATTR_FAMILY_NAME; attr->nla_len = sizeof(*attr) + GENL_NAMSIZ; NONFAILING(strncpy((char*)(attr + 1), (char*)name, GENL_NAMSIZ)); struct iovec iov = {hdr, hdr->nlmsg_len}; struct sockaddr_nl addr = {0}; addr.nl_family = AF_NETLINK; int fd = socket(AF_NETLINK, SOCK_RAW, NETLINK_GENERIC); if (fd == -1) { return -1; } struct msghdr msg = {&addr, sizeof(addr), &iov, 1, NULL, 0, 0}; if (sendmsg(fd, &msg, 0) == -1) { close(fd); return -1; } ssize_t n = recv(fd, buf, sizeof(buf), 0); close(fd); if (n <= 0) { return -1; } if (hdr->nlmsg_type != GENL_ID_CTRL) { return -1; } for (; (char*)attr < buf + n; attr = (struct nlattr*)((char*)attr + NLMSG_ALIGN(attr->nla_len))) { if (attr->nla_type == CTRL_ATTR_FAMILY_ID) return *(uint16_t*)(attr + 1); } return -1; } static void kill_and_wait(int pid, int* status) { kill(-pid, SIGKILL); kill(pid, SIGKILL); int i; for (i = 0; i < 100; i++) { if (waitpid(-1, status, WNOHANG | __WALL) == pid) return; usleep(1000); } DIR* dir = opendir("/sys/fs/fuse/connections"); if (dir) { for (;;) { struct dirent* ent = readdir(dir); if (!ent) break; if (strcmp(ent->d_name, ".") == 0 || strcmp(ent->d_name, "..") == 0) continue; char abort[300]; snprintf(abort, sizeof(abort), "/sys/fs/fuse/connections/%s/abort", ent->d_name); int fd = open(abort, O_WRONLY); if (fd == -1) { continue; } if (write(fd, abort, 1) < 0) { } close(fd); } closedir(dir); } else { } while (waitpid(-1, status, __WALL) != pid) { } } #define SYZ_HAVE_SETUP_TEST 1 static void setup_test() { prctl(PR_SET_PDEATHSIG, SIGKILL, 0, 0, 0); setpgrp(); } #define SYZ_HAVE_RESET_TEST 1 static void reset_test() { int fd; for (fd = 3; fd < 30; fd++) close(fd); } static void execute_one(void); #define WAIT_FLAGS __WALL static void loop(void) { int iter; for (iter = 0;; iter++) { int pid = fork(); if (pid < 0) exit(1); if (pid == 0) { setup_test(); execute_one(); reset_test(); exit(0); } int status = 0; uint64_t start = current_time_ms(); for (;;) { if (waitpid(-1, &status, WNOHANG | WAIT_FLAGS) == pid) break; sleep_ms(1); if (current_time_ms() - start < 5 * 1000) continue; kill_and_wait(pid, &status); break; } } } uint64_t r[2] = {0xffffffffffffffff, 0xffffffffffffffff}; void execute_one(void) { long res = 0; NONFAILING(memcpy((void*)0x20000080, "/dev/sg#\x00", 9)); res = syz_open_dev(0x20000080, 0, 0); if (res != -1) r[0] = res; syscall(__NR_socket, 0xc, 0x800, 0x88); syscall(__NR_ioctl, 0xffffff9c, 0x8933, 0); NONFAILING(*(uint32_t*)0x200000c0 = 1); NONFAILING(*(uint32_t*)0x200000c4 = 0); NONFAILING(*(uint32_t*)0x200000c8 = 8); NONFAILING(memcpy((void*)0x200000cc, "\x86", 1)); syscall(__NR_ioctl, r[0], 1, 0x200000c0); syscall(__NR_openat, -1, 0, 0x88400, 0x100); syscall(__NR_socket, 0x11, 2, 0x300); syscall(__NR_write, -1, 0, 0); syscall(__NR_getsockopt, -1, 0x29, 0x39, 0, 0); syscall(__NR_read, -1, 0, 0); syscall(__NR_ioctl, -1, 0x5437, 0); syz_genetlink_get_family_id(0); syscall(__NR_getpid); NONFAILING(memcpy((void*)0x20000000, "/dev/loop#\x00", 11)); syz_open_dev(0x20000000, 0, 0); syz_open_dev(0, 2, 0x100); syscall(__NR_socket, 0x11, 3, 0x300); /* NONFAILING(*(uint64_t*)0x200003c0 = 0x200002c0); NONFAILING(*(uint16_t*)0x200002c0 = 0x10); NONFAILING(*(uint16_t*)0x200002c2 = 0); NONFAILING(*(uint32_t*)0x200002c4 = 0); NONFAILING(*(uint32_t*)0x200002c8 = 2); NONFAILING(*(uint32_t*)0x200003c8 = 0xc); NONFAILING(*(uint64_t*)0x200003d0 = 0x20000380); NONFAILING(*(uint64_t*)0x20000380 = 0); NONFAILING(*(uint64_t*)0x20000388 = 0); NONFAILING(*(uint64_t*)0x200003d8 = 1); NONFAILING(*(uint64_t*)0x200003e0 = 0); NONFAILING(*(uint64_t*)0x200003e8 = 0); NONFAILING(*(uint32_t*)0x200003f0 = 1); */ syscall(__NR_sendmsg, -1, 0x200003c0, 0x40040); syscall(__NR_openat, 0xffffffffffffff9c, 0, 0, 0); res = syscall(__NR_dup, -1); if (res != -1) r[1] = res; syscall(__NR_ioctl, r[1], 0x8903, 0); syscall(__NR_fdatasync, -1); syscall(__NR_shmctl, 0, 3, 0); syscall(__NR_openat, 0xffffffffffffff9c, 0, 0x20000, 0); syscall(__NR_creat, 0, 0); syscall(__NR_fcntl, -1, 4, 0x40003); syscall(__NR_write, -1, 0, 0); syscall(__NR_setitimer, 1, 0, 0); syscall(__NR_inotify_add_watch, -1, 0, 0x24000004); syscall(__NR_unlink, 0); NONFAILING(*(uint32_t*)0x20000180 = 0); NONFAILING(*(uint32_t*)0x20000184 = 0); NONFAILING(*(uint32_t*)0x20000188 = 0); NONFAILING(*(uint32_t*)0x2000018c = 0); NONFAILING(*(uint32_t*)0x20000190 = 0); NONFAILING(*(uint32_t*)0x20000194 = 0); syscall(__NR_capset, 0, 0x20000180); NONFAILING(*(uint64_t*)0x20000100 = 0); NONFAILING(*(uint64_t*)0x20000108 = 0); syscall(__NR_vmsplice, -1, 0x20000100, 1, 0); syscall(__NR_ioctl, -1, 0x89e0, 0); syscall(__NR_socket, 0xa, 2, 0); syscall(__NR_pipe2, 0, 0x84800); syscall(__NR_pipe, 0); syz_open_pts(0xffffff9c, 0x80000); syscall(__NR_socket, 0xa, 3, 0x3a); syscall(__NR_getsockopt, 0xffffff9c, 0x29, 0x22, 0, 0); syscall(__NR_readv, -1, 0, 0); syscall(__NR_creat, 0, 0); syscall(__NR_connect, -1, 0, 0); syscall(__NR_setsockopt, -1, 1, 0x19, 0, 0); syscall(__NR_sendmsg, -1, 0, 0x10); syscall(__NR_socket, 0x10, 3, 0x10); syscall(__NR_socket, 0x10, 3, 6); syscall(__NR_openat, 0xffffffffffffff9c, 0, 0x440040, 0); syscall(__NR_openat, 0xffffffffffffff9c, 0, 0x8000, 0); syscall(__NR_mmap, 0x20ff7000, 0x9000, 0, 0x5811, -1, 0); syscall(__NR_fstat, -1, 0); syscall(__NR_ioctl, -1, 0x40247007, 0); syscall(__NR_openat, 0xffffffffffffff9c, 0, 0x801, 0); syscall(__NR_ioctl, -1, 0x891b, 0); syscall(__NR_ioctl, -1, 0x541c, 0); syscall(__NR_openat, 0xffffffffffffff9c, 0, 0x200002, 0); syscall(__NR_socket, 2, 2, 0x88); syscall(__NR_readv, -1, 0, 0); syscall(__NR_pipe2, 0x20000000, 0); } int main(void) { syscall(__NR_mmap, 0x20000000, 0x1000000, 3, 0x32, -1, 0); install_segv_handler(); for (procid = 0; procid < 8; procid++) { if (fork() == 0) { loop(); } } sleep(1000000); return 0; }
