Null deference panic in CentOS-6.5

[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]




Hi,

I got a panic when running CentOS-6.5:

crash> bt
PID: 106074  TASK: ffff8839c1e32ae0  CPU: 4   COMMAND: "flushd4[cbd-sd-"
  #0 [ffff8839c2a91900] machine_kexec at ffffffff81038fa9
  #1 [ffff8839c2a91960] crash_kexec at ffffffff810c5992
  #2 [ffff8839c2a91a30] oops_end at ffffffff81515c90
  #3 [ffff8839c2a91a60] no_context at ffffffff81049f1b
  #4 [ffff8839c2a91ab0] __bad_area_nosemaphore at ffffffff8104a1a5
  #5 [ffff8839c2a91b00] bad_area_nosemaphore at ffffffff8104a273
  #6 [ffff8839c2a91b10] __do_page_fault at ffffffff8104a9bf
  #7 [ffff8839c2a91c30] do_page_fault at ffffffff81517bae
  #8 [ffff8839c2a91c60] page_fault at ffffffff81514f95
     [exception RIP: rb_next+1]
     RIP: ffffffff81286e21  RSP: ffff8839c2a91d10  RFLAGS: 00010046
     RAX: 0000000000000000  RBX: ffff88204b501c00  RCX: 0000000000000000
     RDX: ffff88013bc56840  RSI: ffff88013bc568d8  RDI: 0000000000000010
     RBP: ffff8839c2a91d60   R8: 0000000000000001   R9: 0000000000000001
     R10: 0000000000000000  R11: 0000000000000000  R12: 0000000000000000
     R13: 0000000000000000  R14: 0000000000000000  R15: 0000000000000000
     ORIG_RAX: ffffffffffffffff  CS: 0010  SS: 0018
  #9 [ffff8839c2a91d18] pick_next_task_fair at ffffffff81068121
#10 [ffff8839c2a91d68] schedule at ffffffff81511e08
#11 [ffff8839c2a91e28] flushd_run at ffffffffa07a2cbd [cbd]
#12 [ffff8839c2a91ee8] kthread at ffffffff8109acd6
#13 [ffff8839c2a91f48] kernel_thread at ffffffff8100c20a

The [cbd] is a module developed by us, I think this bug has nothing to
do with it.

And the contents of rq in pick_next_task(struct rq *rq) is (see
attachement for full contents of struct rq):

struct rq {
   lock = {
     raw_lock = {
       slock = 67109881
     }
   },
   nr_running = 2,
   cpu_load = {0, 5923, 14993, 13888, 9115},
   last_load_update_tick = 4365159236,
   nohz_balance_kick = 0 '\000',
   skip_clock_update = 0,
   load = {
     weight = 2,
     inv_weight = 0
   },
   nr_load_updates = 21530842,
   nr_switches = 148355748,
   cfs = {
     load = {
       weight = 2,
       inv_weight = 0
     },
     nr_running = 1,
     h_nr_running = 2,
     exec_clock = 3309310258875,
     min_vruntime = 1181294560093,
     tasks_timeline = {
       rb_node = 0x0
     },
     rb_leftmost = 0x0,
     tasks = {
       next = 0xffff88013bc568e8,
       prev = 0xffff88013bc568e8
     },
     balance_iterator = 0xffff88013bc568e8,
     curr = 0xffff88204b501e00,
     next = 0x0,
     last = 0x0,
     skip = 0x0,
     nr_spread_over = 5,
     ....

We can see that the value if rq->cfs.nr_running is not zero, but
rb_leftmost is null. With skip is null, this causes null deference
panic in pick_next_entity() of pick_next_task_fair().

Does anyone have encountered same problem or advice?

Thanks
struct rq {
  lock = {
    raw_lock = {
      slock = 67109881
    }
  }, 
  nr_running = 2, 
  cpu_load = {0, 5923, 14993, 13888, 9115}, 
  last_load_update_tick = 4365159236, 
  nohz_balance_kick = 0 '\000', 
  skip_clock_update = 0, 
  load = {
    weight = 2, 
    inv_weight = 0
  }, 
  nr_load_updates = 21530842, 
  nr_switches = 148355748, 
  cfs = {
    load = {
      weight = 2, 
      inv_weight = 0
    }, 
    nr_running = 1, 
    h_nr_running = 2, 
    exec_clock = 3309310258875, 
    min_vruntime = 1181294560093, 
    tasks_timeline = {
      rb_node = 0x0
    }, 
    rb_leftmost = 0x0, 
    tasks = {
      next = 0xffff88013bc568e8, 
      prev = 0xffff88013bc568e8
    }, 
    balance_iterator = 0xffff88013bc568e8, 
    curr = 0xffff88204b501e00, 
    next = 0x0, 
    last = 0x0, 
    skip = 0x0, 
    nr_spread_over = 5, 
    rq = 0xffff88013bc56840, 
    on_list = 1, 
    leaf_cfs_rq_list = {
      next = 0xffff881b03781690, 
      prev = 0xffff882050d9ee90
    }, 
    tg = 0xffffffff81e25cc0 <init_task_group>, 
    task_weight = 0, 
    h_load = 1026, 
    load_avg = 0, 
    load_period = 0, 
    load_stamp = 1, 
    load_last = 0, 
    load_unacc_exec_time = 3309310258875, 
    load_contribution = 0, 
    runtime_enabled = 0, 
    runtime_expires = 0, 
    runtime_remaining = 0, 
    throttled_timestamp = 0, 
    throttled = 0, 
    throttle_count = 0, 
    throttled_list = {
      next = 0xffff88013bc569b8, 
      prev = 0xffff88013bc569b8
    }
  }, 
  rt = {
    active = {
      bitmap = {0, 68719476736}, 
      queue = {{
          next = 0xffff88013bc569d8, 
          prev = 0xffff88013bc569d8
        }, {
          next = 0xffff88013bc569e8, 
          prev = 0xffff88013bc569e8
        }, {
          next = 0xffff88013bc569f8, 
          prev = 0xffff88013bc569f8
        }, {
          next = 0xffff88013bc56a08, 
          prev = 0xffff88013bc56a08
        }, {
          next = 0xffff88013bc56a18, 
          prev = 0xffff88013bc56a18
        }, {
          next = 0xffff88013bc56a28, 
          prev = 0xffff88013bc56a28
        }, {
          next = 0xffff88013bc56a38, 
          prev = 0xffff88013bc56a38
        }, {
          next = 0xffff88013bc56a48, 
          prev = 0xffff88013bc56a48
        }, {
          next = 0xffff88013bc56a58, 
          prev = 0xffff88013bc56a58
        }, {
          next = 0xffff88013bc56a68, 
          prev = 0xffff88013bc56a68
        }, {
          next = 0xffff88013bc56a78, 
          prev = 0xffff88013bc56a78
        }, {
          next = 0xffff88013bc56a88, 
          prev = 0xffff88013bc56a88
        }, {
          next = 0xffff88013bc56a98, 
          prev = 0xffff88013bc56a98
        }, {
          next = 0xffff88013bc56aa8, 
          prev = 0xffff88013bc56aa8
        }, {
          next = 0xffff88013bc56ab8, 
          prev = 0xffff88013bc56ab8
        }, {
          next = 0xffff88013bc56ac8, 
          prev = 0xffff88013bc56ac8
        }, {
          next = 0xffff88013bc56ad8, 
          prev = 0xffff88013bc56ad8
        }, {
          next = 0xffff88013bc56ae8, 
          prev = 0xffff88013bc56ae8
        }, {
          next = 0xffff88013bc56af8, 
          prev = 0xffff88013bc56af8
        }, {
          next = 0xffff88013bc56b08, 
          prev = 0xffff88013bc56b08
        }, {
          next = 0xffff88013bc56b18, 
          prev = 0xffff88013bc56b18
        }, {
          next = 0xffff88013bc56b28, 
          prev = 0xffff88013bc56b28
        }, {
          next = 0xffff88013bc56b38, 
          prev = 0xffff88013bc56b38
        }, {
          next = 0xffff88013bc56b48, 
          prev = 0xffff88013bc56b48
        }, {
          next = 0xffff88013bc56b58, 
          prev = 0xffff88013bc56b58
        }, {
          next = 0xffff88013bc56b68, 
          prev = 0xffff88013bc56b68
        }, {
          next = 0xffff88013bc56b78, 
          prev = 0xffff88013bc56b78
        }, {
          next = 0xffff88013bc56b88, 
          prev = 0xffff88013bc56b88
        }, {
          next = 0xffff88013bc56b98, 
          prev = 0xffff88013bc56b98
        }, {
          next = 0xffff88013bc56ba8, 
          prev = 0xffff88013bc56ba8
        }, {
          next = 0xffff88013bc56bb8, 
          prev = 0xffff88013bc56bb8
        }, {
          next = 0xffff88013bc56bc8, 
          prev = 0xffff88013bc56bc8
        }, {
          next = 0xffff88013bc56bd8, 
          prev = 0xffff88013bc56bd8
        }, {
          next = 0xffff88013bc56be8, 
          prev = 0xffff88013bc56be8
        }, {
          next = 0xffff88013bc56bf8, 
          prev = 0xffff88013bc56bf8
        }, {
          next = 0xffff88013bc56c08, 
          prev = 0xffff88013bc56c08
        }, {
          next = 0xffff88013bc56c18, 
          prev = 0xffff88013bc56c18
        }, {
          next = 0xffff88013bc56c28, 
          prev = 0xffff88013bc56c28
        }, {
          next = 0xffff88013bc56c38, 
          prev = 0xffff88013bc56c38
        }, {
          next = 0xffff88013bc56c48, 
          prev = 0xffff88013bc56c48
        }, {
          next = 0xffff88013bc56c58, 
          prev = 0xffff88013bc56c58
        }, {
          next = 0xffff88013bc56c68, 
          prev = 0xffff88013bc56c68
        }, {
          next = 0xffff88013bc56c78, 
          prev = 0xffff88013bc56c78
        }, {
          next = 0xffff88013bc56c88, 
          prev = 0xffff88013bc56c88
        }, {
          next = 0xffff88013bc56c98, 
          prev = 0xffff88013bc56c98
        }, {
          next = 0xffff88013bc56ca8, 
          prev = 0xffff88013bc56ca8
        }, {
          next = 0xffff88013bc56cb8, 
          prev = 0xffff88013bc56cb8
        }, {
          next = 0xffff88013bc56cc8, 
          prev = 0xffff88013bc56cc8
        }, {
          next = 0xffff88013bc56cd8, 
          prev = 0xffff88013bc56cd8
        }, {
          next = 0xffff88013bc56ce8, 
          prev = 0xffff88013bc56ce8
        }, {
          next = 0xffff88013bc56cf8, 
          prev = 0xffff88013bc56cf8
        }, {
          next = 0xffff88013bc56d08, 
          prev = 0xffff88013bc56d08
        }, {
          next = 0xffff88013bc56d18, 
          prev = 0xffff88013bc56d18
        }, {
          next = 0xffff88013bc56d28, 
          prev = 0xffff88013bc56d28
        }, {
          next = 0xffff88013bc56d38, 
          prev = 0xffff88013bc56d38
        }, {
          next = 0xffff88013bc56d48, 
          prev = 0xffff88013bc56d48
        }, {
          next = 0xffff88013bc56d58, 
          prev = 0xffff88013bc56d58
        }, {
          next = 0xffff88013bc56d68, 
          prev = 0xffff88013bc56d68
        }, {
          next = 0xffff88013bc56d78, 
          prev = 0xffff88013bc56d78
        }, {
          next = 0xffff88013bc56d88, 
          prev = 0xffff88013bc56d88
        }, {
          next = 0xffff88013bc56d98, 
          prev = 0xffff88013bc56d98
        }, {
          next = 0xffff88013bc56da8, 
          prev = 0xffff88013bc56da8
        }, {
          next = 0xffff88013bc56db8, 
          prev = 0xffff88013bc56db8
        }, {
          next = 0xffff88013bc56dc8, 
          prev = 0xffff88013bc56dc8
        }, {
          next = 0xffff88013bc56dd8, 
          prev = 0xffff88013bc56dd8
        }, {
          next = 0xffff88013bc56de8, 
          prev = 0xffff88013bc56de8
        }, {
          next = 0xffff88013bc56df8, 
          prev = 0xffff88013bc56df8
        }, {
          next = 0xffff88013bc56e08, 
          prev = 0xffff88013bc56e08
        }, {
          next = 0xffff88013bc56e18, 
          prev = 0xffff88013bc56e18
        }, {
          next = 0xffff88013bc56e28, 
          prev = 0xffff88013bc56e28
        }, {
          next = 0xffff88013bc56e38, 
          prev = 0xffff88013bc56e38
        }, {
          next = 0xffff88013bc56e48, 
          prev = 0xffff88013bc56e48
        }, {
          next = 0xffff88013bc56e58, 
          prev = 0xffff88013bc56e58
        }, {
          next = 0xffff88013bc56e68, 
          prev = 0xffff88013bc56e68
        }, {
          next = 0xffff88013bc56e78, 
          prev = 0xffff88013bc56e78
        }, {
          next = 0xffff88013bc56e88, 
          prev = 0xffff88013bc56e88
        }, {
          next = 0xffff88013bc56e98, 
          prev = 0xffff88013bc56e98
        }, {
          next = 0xffff88013bc56ea8, 
          prev = 0xffff88013bc56ea8
        }, {
          next = 0xffff88013bc56eb8, 
          prev = 0xffff88013bc56eb8
        }, {
          next = 0xffff88013bc56ec8, 
          prev = 0xffff88013bc56ec8
        }, {
          next = 0xffff88013bc56ed8, 
          prev = 0xffff88013bc56ed8
        }, {
          next = 0xffff88013bc56ee8, 
          prev = 0xffff88013bc56ee8
        }, {
          next = 0xffff88013bc56ef8, 
          prev = 0xffff88013bc56ef8
        }, {
          next = 0xffff88013bc56f08, 
          prev = 0xffff88013bc56f08
        }, {
          next = 0xffff88013bc56f18, 
          prev = 0xffff88013bc56f18
        }, {
          next = 0xffff88013bc56f28, 
          prev = 0xffff88013bc56f28
        }, {
          next = 0xffff88013bc56f38, 
          prev = 0xffff88013bc56f38
        }, {
          next = 0xffff88013bc56f48, 
          prev = 0xffff88013bc56f48
        }, {
          next = 0xffff88013bc56f58, 
          prev = 0xffff88013bc56f58
        }, {
          next = 0xffff88013bc56f68, 
          prev = 0xffff88013bc56f68
        }, {
          next = 0xffff88013bc56f78, 
          prev = 0xffff88013bc56f78
        }, {
          next = 0xffff88013bc56f88, 
          prev = 0xffff88013bc56f88
        }, {
          next = 0xffff88013bc56f98, 
          prev = 0xffff88013bc56f98
        }, {
          next = 0xffff88013bc56fa8, 
          prev = 0xffff88013bc56fa8
        }, {
          next = 0xffff88013bc56fb8, 
          prev = 0xffff88013bc56fb8
        }, {
          next = 0xffff88013bc56fc8, 
          prev = 0xffff88013bc56fc8
        }, {
          next = 0xffff88013bc56fd8, 
          prev = 0xffff88013bc56fd8
        }, {
          next = 0xffff88013bc56fe8, 
          prev = 0xffff88013bc56fe8
        }, {
          next = 0xffff88013bc56ff8, 
          prev = 0xffff88013bc56ff8
        }, {
          next = 0xffff88013bc57008, 
          prev = 0xffff88013bc57008
        }}
    }, 
    rt_nr_running = 0, 
    highest_prio = {
      curr = 100, 
      next = 100
    }, 
    rt_nr_migratory = 0, 
    rt_nr_total = 0, 
    overloaded = 0, 
    pushable_tasks = {
      prio_list = {
        next = 0xffff88013bc57040, 
        prev = 0xffff88013bc57040
      }, 
      node_list = {
        next = 0xffff88013bc57050, 
        prev = 0xffff88013bc57050
      }
    }, 
    rt_throttled = 0, 
    rt_time = 0, 
    rt_runtime = 950000000, 
    rt_runtime_lock = {
      raw_lock = {
        slock = 3430075506
      }
    }, 
    rt_nr_boosted = 0, 
    rq = 0xffff88013bc56840, 
    leaf_rt_rq_list = {
      next = 0xffff88013bc570c0, 
      prev = 0xdead000000200200
    }, 
    tg = 0xffffffff81e25cc0 <init_task_group>, 
    rt_se = 0x0
  }, 
  leaf_cfs_rq_list = {
    next = 0xffff88204b501c90, 
    prev = 0xffff8812ca1d8490
  }, 
  leaf_rt_rq_list = {
    next = 0xffff88013bc570c0, 
    prev = 0xffff88013bc570c0
  }, 
  nr_uninterruptible = 17, 
  curr = 0xffff8839c1e32ae0, 
  idle = 0xffff884052c9aaa0, 
  next_balance = 4365159298, 
  prev_mm = 0x0, 
  clock = 70548181712501, 
  nr_iowait = {
    counter = 0
  }, 
  rd = 0xffff882052f50000, 
  sd = 0xffff88013bc4fbe0, 
  idle_at_tick = 1 '\001', 
  post_schedule = 0, 
  active_balance = 0, 
  push_cpu = 20, 
  cpu = 4, 
  online = 1, 
  avg_load_per_task = 342, 
  migration_thread = 0xffff882052c4c040, 
  migration_queue = {
    next = 0xffff88013bc57140, 
    prev = 0xffff88013bc57140
  }, 
  rt_avg = 11804, 
  age_stamp = 70548000000000, 
  idle_stamp = 0, 
  avg_idle = 1000000, 
  prev_steal_time = 0, 
  calc_load_update = 4365161392, 
  calc_load_active = 19, 
  hrtick_csd_pending = 0, 
  hrtick_csd = {
    list = {
      next = 0x0, 
      prev = 0x0
    }, 
    func = 0xffffffff8105aff0 <__hrtick_start>, 
    info = 0xffff88013bc56840, 
    flags = 0, 
    priv = 0
  }, 
  hrtick_timer = {
    node = {
      rb_parent_color = 0, 
      rb_right = 0x0, 
      rb_left = 0x0
    }, 
    _expires = {
      tv64 = 0
    }, 
    _softexpires = {
      tv64 = 0
    }, 
    function = 0xffffffff81059120 <hrtick>, 
    base = 0xffff88013bc11628, 
    state = 0, 
    start_pid = -1, 
    start_site = 0x0, 
    start_comm = "\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"
  }, 
  rq_sched_info = {
    pcount = 76248357, 
    run_delay = 181146010801, 
    last_arrival = 0, 
    last_queued = 0, 
    bkl_count = 0
  }, 
  rq_cpu_time = 3471141075664, 
  yld_count = 673972, 
  sched_switch = 0, 
  sched_count = 149947296, 
  sched_goidle = 72140963, 
  ttwu_count = 76021337, 
  ttwu_local = 18157697, 
  bkl_count = 38109, 
  stop = 0xffff882052c55540
}
_______________________________________________
CentOS mailing list
CentOS@xxxxxxxxxx
https://lists.centos.org/mailman/listinfo/centos

[Index of Archives]     [CentOS]     [CentOS Announce]     [CentOS Development]     [CentOS ARM Devel]     [CentOS Docs]     [CentOS Virtualization]     [Carrier Grade Linux]     [Linux Media]     [Asterisk]     [DCCP]     [Netdev]     [Xorg]     [Linux USB]


  Powered by Linux