Hi,
I got a panic when running CentOS-6.5:
crash> bt
PID: 106074 TASK: ffff8839c1e32ae0 CPU: 4 COMMAND: "flushd4[cbd-sd-"
#0 [ffff8839c2a91900] machine_kexec at ffffffff81038fa9
#1 [ffff8839c2a91960] crash_kexec at ffffffff810c5992
#2 [ffff8839c2a91a30] oops_end at ffffffff81515c90
#3 [ffff8839c2a91a60] no_context at ffffffff81049f1b
#4 [ffff8839c2a91ab0] __bad_area_nosemaphore at ffffffff8104a1a5
#5 [ffff8839c2a91b00] bad_area_nosemaphore at ffffffff8104a273
#6 [ffff8839c2a91b10] __do_page_fault at ffffffff8104a9bf
#7 [ffff8839c2a91c30] do_page_fault at ffffffff81517bae
#8 [ffff8839c2a91c60] page_fault at ffffffff81514f95
[exception RIP: rb_next+1]
RIP: ffffffff81286e21 RSP: ffff8839c2a91d10 RFLAGS: 00010046
RAX: 0000000000000000 RBX: ffff88204b501c00 RCX: 0000000000000000
RDX: ffff88013bc56840 RSI: ffff88013bc568d8 RDI: 0000000000000010
RBP: ffff8839c2a91d60 R8: 0000000000000001 R9: 0000000000000001
R10: 0000000000000000 R11: 0000000000000000 R12: 0000000000000000
R13: 0000000000000000 R14: 0000000000000000 R15: 0000000000000000
ORIG_RAX: ffffffffffffffff CS: 0010 SS: 0018
#9 [ffff8839c2a91d18] pick_next_task_fair at ffffffff81068121
#10 [ffff8839c2a91d68] schedule at ffffffff81511e08
#11 [ffff8839c2a91e28] flushd_run at ffffffffa07a2cbd [cbd]
#12 [ffff8839c2a91ee8] kthread at ffffffff8109acd6
#13 [ffff8839c2a91f48] kernel_thread at ffffffff8100c20a
The [cbd] is a module developed by us, I think this bug has nothing to
do with it.
And the contents of rq in pick_next_task(struct rq *rq) is (see
attachement for full contents of struct rq):
struct rq {
lock = {
raw_lock = {
slock = 67109881
}
},
nr_running = 2,
cpu_load = {0, 5923, 14993, 13888, 9115},
last_load_update_tick = 4365159236,
nohz_balance_kick = 0 '\000',
skip_clock_update = 0,
load = {
weight = 2,
inv_weight = 0
},
nr_load_updates = 21530842,
nr_switches = 148355748,
cfs = {
load = {
weight = 2,
inv_weight = 0
},
nr_running = 1,
h_nr_running = 2,
exec_clock = 3309310258875,
min_vruntime = 1181294560093,
tasks_timeline = {
rb_node = 0x0
},
rb_leftmost = 0x0,
tasks = {
next = 0xffff88013bc568e8,
prev = 0xffff88013bc568e8
},
balance_iterator = 0xffff88013bc568e8,
curr = 0xffff88204b501e00,
next = 0x0,
last = 0x0,
skip = 0x0,
nr_spread_over = 5,
....
We can see that the value if rq->cfs.nr_running is not zero, but
rb_leftmost is null. With skip is null, this causes null deference
panic in pick_next_entity() of pick_next_task_fair().
Does anyone have encountered same problem or advice?
Thanks
struct rq {
lock = {
raw_lock = {
slock = 67109881
}
},
nr_running = 2,
cpu_load = {0, 5923, 14993, 13888, 9115},
last_load_update_tick = 4365159236,
nohz_balance_kick = 0 '\000',
skip_clock_update = 0,
load = {
weight = 2,
inv_weight = 0
},
nr_load_updates = 21530842,
nr_switches = 148355748,
cfs = {
load = {
weight = 2,
inv_weight = 0
},
nr_running = 1,
h_nr_running = 2,
exec_clock = 3309310258875,
min_vruntime = 1181294560093,
tasks_timeline = {
rb_node = 0x0
},
rb_leftmost = 0x0,
tasks = {
next = 0xffff88013bc568e8,
prev = 0xffff88013bc568e8
},
balance_iterator = 0xffff88013bc568e8,
curr = 0xffff88204b501e00,
next = 0x0,
last = 0x0,
skip = 0x0,
nr_spread_over = 5,
rq = 0xffff88013bc56840,
on_list = 1,
leaf_cfs_rq_list = {
next = 0xffff881b03781690,
prev = 0xffff882050d9ee90
},
tg = 0xffffffff81e25cc0 <init_task_group>,
task_weight = 0,
h_load = 1026,
load_avg = 0,
load_period = 0,
load_stamp = 1,
load_last = 0,
load_unacc_exec_time = 3309310258875,
load_contribution = 0,
runtime_enabled = 0,
runtime_expires = 0,
runtime_remaining = 0,
throttled_timestamp = 0,
throttled = 0,
throttle_count = 0,
throttled_list = {
next = 0xffff88013bc569b8,
prev = 0xffff88013bc569b8
}
},
rt = {
active = {
bitmap = {0, 68719476736},
queue = {{
next = 0xffff88013bc569d8,
prev = 0xffff88013bc569d8
}, {
next = 0xffff88013bc569e8,
prev = 0xffff88013bc569e8
}, {
next = 0xffff88013bc569f8,
prev = 0xffff88013bc569f8
}, {
next = 0xffff88013bc56a08,
prev = 0xffff88013bc56a08
}, {
next = 0xffff88013bc56a18,
prev = 0xffff88013bc56a18
}, {
next = 0xffff88013bc56a28,
prev = 0xffff88013bc56a28
}, {
next = 0xffff88013bc56a38,
prev = 0xffff88013bc56a38
}, {
next = 0xffff88013bc56a48,
prev = 0xffff88013bc56a48
}, {
next = 0xffff88013bc56a58,
prev = 0xffff88013bc56a58
}, {
next = 0xffff88013bc56a68,
prev = 0xffff88013bc56a68
}, {
next = 0xffff88013bc56a78,
prev = 0xffff88013bc56a78
}, {
next = 0xffff88013bc56a88,
prev = 0xffff88013bc56a88
}, {
next = 0xffff88013bc56a98,
prev = 0xffff88013bc56a98
}, {
next = 0xffff88013bc56aa8,
prev = 0xffff88013bc56aa8
}, {
next = 0xffff88013bc56ab8,
prev = 0xffff88013bc56ab8
}, {
next = 0xffff88013bc56ac8,
prev = 0xffff88013bc56ac8
}, {
next = 0xffff88013bc56ad8,
prev = 0xffff88013bc56ad8
}, {
next = 0xffff88013bc56ae8,
prev = 0xffff88013bc56ae8
}, {
next = 0xffff88013bc56af8,
prev = 0xffff88013bc56af8
}, {
next = 0xffff88013bc56b08,
prev = 0xffff88013bc56b08
}, {
next = 0xffff88013bc56b18,
prev = 0xffff88013bc56b18
}, {
next = 0xffff88013bc56b28,
prev = 0xffff88013bc56b28
}, {
next = 0xffff88013bc56b38,
prev = 0xffff88013bc56b38
}, {
next = 0xffff88013bc56b48,
prev = 0xffff88013bc56b48
}, {
next = 0xffff88013bc56b58,
prev = 0xffff88013bc56b58
}, {
next = 0xffff88013bc56b68,
prev = 0xffff88013bc56b68
}, {
next = 0xffff88013bc56b78,
prev = 0xffff88013bc56b78
}, {
next = 0xffff88013bc56b88,
prev = 0xffff88013bc56b88
}, {
next = 0xffff88013bc56b98,
prev = 0xffff88013bc56b98
}, {
next = 0xffff88013bc56ba8,
prev = 0xffff88013bc56ba8
}, {
next = 0xffff88013bc56bb8,
prev = 0xffff88013bc56bb8
}, {
next = 0xffff88013bc56bc8,
prev = 0xffff88013bc56bc8
}, {
next = 0xffff88013bc56bd8,
prev = 0xffff88013bc56bd8
}, {
next = 0xffff88013bc56be8,
prev = 0xffff88013bc56be8
}, {
next = 0xffff88013bc56bf8,
prev = 0xffff88013bc56bf8
}, {
next = 0xffff88013bc56c08,
prev = 0xffff88013bc56c08
}, {
next = 0xffff88013bc56c18,
prev = 0xffff88013bc56c18
}, {
next = 0xffff88013bc56c28,
prev = 0xffff88013bc56c28
}, {
next = 0xffff88013bc56c38,
prev = 0xffff88013bc56c38
}, {
next = 0xffff88013bc56c48,
prev = 0xffff88013bc56c48
}, {
next = 0xffff88013bc56c58,
prev = 0xffff88013bc56c58
}, {
next = 0xffff88013bc56c68,
prev = 0xffff88013bc56c68
}, {
next = 0xffff88013bc56c78,
prev = 0xffff88013bc56c78
}, {
next = 0xffff88013bc56c88,
prev = 0xffff88013bc56c88
}, {
next = 0xffff88013bc56c98,
prev = 0xffff88013bc56c98
}, {
next = 0xffff88013bc56ca8,
prev = 0xffff88013bc56ca8
}, {
next = 0xffff88013bc56cb8,
prev = 0xffff88013bc56cb8
}, {
next = 0xffff88013bc56cc8,
prev = 0xffff88013bc56cc8
}, {
next = 0xffff88013bc56cd8,
prev = 0xffff88013bc56cd8
}, {
next = 0xffff88013bc56ce8,
prev = 0xffff88013bc56ce8
}, {
next = 0xffff88013bc56cf8,
prev = 0xffff88013bc56cf8
}, {
next = 0xffff88013bc56d08,
prev = 0xffff88013bc56d08
}, {
next = 0xffff88013bc56d18,
prev = 0xffff88013bc56d18
}, {
next = 0xffff88013bc56d28,
prev = 0xffff88013bc56d28
}, {
next = 0xffff88013bc56d38,
prev = 0xffff88013bc56d38
}, {
next = 0xffff88013bc56d48,
prev = 0xffff88013bc56d48
}, {
next = 0xffff88013bc56d58,
prev = 0xffff88013bc56d58
}, {
next = 0xffff88013bc56d68,
prev = 0xffff88013bc56d68
}, {
next = 0xffff88013bc56d78,
prev = 0xffff88013bc56d78
}, {
next = 0xffff88013bc56d88,
prev = 0xffff88013bc56d88
}, {
next = 0xffff88013bc56d98,
prev = 0xffff88013bc56d98
}, {
next = 0xffff88013bc56da8,
prev = 0xffff88013bc56da8
}, {
next = 0xffff88013bc56db8,
prev = 0xffff88013bc56db8
}, {
next = 0xffff88013bc56dc8,
prev = 0xffff88013bc56dc8
}, {
next = 0xffff88013bc56dd8,
prev = 0xffff88013bc56dd8
}, {
next = 0xffff88013bc56de8,
prev = 0xffff88013bc56de8
}, {
next = 0xffff88013bc56df8,
prev = 0xffff88013bc56df8
}, {
next = 0xffff88013bc56e08,
prev = 0xffff88013bc56e08
}, {
next = 0xffff88013bc56e18,
prev = 0xffff88013bc56e18
}, {
next = 0xffff88013bc56e28,
prev = 0xffff88013bc56e28
}, {
next = 0xffff88013bc56e38,
prev = 0xffff88013bc56e38
}, {
next = 0xffff88013bc56e48,
prev = 0xffff88013bc56e48
}, {
next = 0xffff88013bc56e58,
prev = 0xffff88013bc56e58
}, {
next = 0xffff88013bc56e68,
prev = 0xffff88013bc56e68
}, {
next = 0xffff88013bc56e78,
prev = 0xffff88013bc56e78
}, {
next = 0xffff88013bc56e88,
prev = 0xffff88013bc56e88
}, {
next = 0xffff88013bc56e98,
prev = 0xffff88013bc56e98
}, {
next = 0xffff88013bc56ea8,
prev = 0xffff88013bc56ea8
}, {
next = 0xffff88013bc56eb8,
prev = 0xffff88013bc56eb8
}, {
next = 0xffff88013bc56ec8,
prev = 0xffff88013bc56ec8
}, {
next = 0xffff88013bc56ed8,
prev = 0xffff88013bc56ed8
}, {
next = 0xffff88013bc56ee8,
prev = 0xffff88013bc56ee8
}, {
next = 0xffff88013bc56ef8,
prev = 0xffff88013bc56ef8
}, {
next = 0xffff88013bc56f08,
prev = 0xffff88013bc56f08
}, {
next = 0xffff88013bc56f18,
prev = 0xffff88013bc56f18
}, {
next = 0xffff88013bc56f28,
prev = 0xffff88013bc56f28
}, {
next = 0xffff88013bc56f38,
prev = 0xffff88013bc56f38
}, {
next = 0xffff88013bc56f48,
prev = 0xffff88013bc56f48
}, {
next = 0xffff88013bc56f58,
prev = 0xffff88013bc56f58
}, {
next = 0xffff88013bc56f68,
prev = 0xffff88013bc56f68
}, {
next = 0xffff88013bc56f78,
prev = 0xffff88013bc56f78
}, {
next = 0xffff88013bc56f88,
prev = 0xffff88013bc56f88
}, {
next = 0xffff88013bc56f98,
prev = 0xffff88013bc56f98
}, {
next = 0xffff88013bc56fa8,
prev = 0xffff88013bc56fa8
}, {
next = 0xffff88013bc56fb8,
prev = 0xffff88013bc56fb8
}, {
next = 0xffff88013bc56fc8,
prev = 0xffff88013bc56fc8
}, {
next = 0xffff88013bc56fd8,
prev = 0xffff88013bc56fd8
}, {
next = 0xffff88013bc56fe8,
prev = 0xffff88013bc56fe8
}, {
next = 0xffff88013bc56ff8,
prev = 0xffff88013bc56ff8
}, {
next = 0xffff88013bc57008,
prev = 0xffff88013bc57008
}}
},
rt_nr_running = 0,
highest_prio = {
curr = 100,
next = 100
},
rt_nr_migratory = 0,
rt_nr_total = 0,
overloaded = 0,
pushable_tasks = {
prio_list = {
next = 0xffff88013bc57040,
prev = 0xffff88013bc57040
},
node_list = {
next = 0xffff88013bc57050,
prev = 0xffff88013bc57050
}
},
rt_throttled = 0,
rt_time = 0,
rt_runtime = 950000000,
rt_runtime_lock = {
raw_lock = {
slock = 3430075506
}
},
rt_nr_boosted = 0,
rq = 0xffff88013bc56840,
leaf_rt_rq_list = {
next = 0xffff88013bc570c0,
prev = 0xdead000000200200
},
tg = 0xffffffff81e25cc0 <init_task_group>,
rt_se = 0x0
},
leaf_cfs_rq_list = {
next = 0xffff88204b501c90,
prev = 0xffff8812ca1d8490
},
leaf_rt_rq_list = {
next = 0xffff88013bc570c0,
prev = 0xffff88013bc570c0
},
nr_uninterruptible = 17,
curr = 0xffff8839c1e32ae0,
idle = 0xffff884052c9aaa0,
next_balance = 4365159298,
prev_mm = 0x0,
clock = 70548181712501,
nr_iowait = {
counter = 0
},
rd = 0xffff882052f50000,
sd = 0xffff88013bc4fbe0,
idle_at_tick = 1 '\001',
post_schedule = 0,
active_balance = 0,
push_cpu = 20,
cpu = 4,
online = 1,
avg_load_per_task = 342,
migration_thread = 0xffff882052c4c040,
migration_queue = {
next = 0xffff88013bc57140,
prev = 0xffff88013bc57140
},
rt_avg = 11804,
age_stamp = 70548000000000,
idle_stamp = 0,
avg_idle = 1000000,
prev_steal_time = 0,
calc_load_update = 4365161392,
calc_load_active = 19,
hrtick_csd_pending = 0,
hrtick_csd = {
list = {
next = 0x0,
prev = 0x0
},
func = 0xffffffff8105aff0 <__hrtick_start>,
info = 0xffff88013bc56840,
flags = 0,
priv = 0
},
hrtick_timer = {
node = {
rb_parent_color = 0,
rb_right = 0x0,
rb_left = 0x0
},
_expires = {
tv64 = 0
},
_softexpires = {
tv64 = 0
},
function = 0xffffffff81059120 <hrtick>,
base = 0xffff88013bc11628,
state = 0,
start_pid = -1,
start_site = 0x0,
start_comm = "\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"
},
rq_sched_info = {
pcount = 76248357,
run_delay = 181146010801,
last_arrival = 0,
last_queued = 0,
bkl_count = 0
},
rq_cpu_time = 3471141075664,
yld_count = 673972,
sched_switch = 0,
sched_count = 149947296,
sched_goidle = 72140963,
ttwu_count = 76021337,
ttwu_local = 18157697,
bkl_count = 38109,
stop = 0xffff882052c55540
}
_______________________________________________
CentOS mailing list
CentOS@xxxxxxxxxx
https://lists.centos.org/mailman/listinfo/centos
