Hi all, I'm trying to trace the latency in my real-time application. For this purpose, i use the following approach prctl(0, 1); Critical Code prctl(0, 0); Later, when i have a look at /proc/latency_trace i see that there are function calls related with page fault. Later, i start and stop the tracer to see the messages coming from tracer. prctl(0, 1); prctl(0, 0); I see that most of them are tracing related. Is there a way to clear tracing related latency from the output or is this special to my configuration. you can find the output of trace attached to this email. My kernel 2.6.22.1-rt9 on an smp box. regards, sinan.
cmd pid ||||| time | caller \ / ||||| \ | / <...>-5554 1D... 0us : user_trace_start+0x15f/0x1ae (sys_prctl+0x21/0x1b2) <...>-5554 1.... 0us : rt_up+0xb/0x47 (user_trace_start+0x19b/0x1ae) <...>-5554 1...1 0us : rt_mutex_unlock+0xb/0x30 (rt_up+0x2f/0x47) <...>-5554 1.... 1us : do_syscall_trace+0xe/0x1d6 (syscall_exit_work+0x16/0x1b) <...>-5554 1.... 1us : audit_syscall_exit+0xe/0x361 (do_syscall_trace+0x83/0x1d6) <...>-5554 1.... 1us+: kfree+0xc/0xa0 (audit_syscall_exit+0x346/0x361) <...>-5554 1D... 4us : do_page_fault+0x25/0x576 (b79efd60 7 b7194d64) <...>-5554 1D... 4us : atomic_notifier_call_chain+0xb/0x36 (do_page_fault+0x129/0x576) <...>-5554 1D... 4us : __atomic_notifier_call_chain+0xc/0x4a (atomic_notifier_call_chain+0x34/0x36) <...>-5554 1D... 4us : __rcu_read_lock+0xb/0x9d (__atomic_notifier_call_chain+0x11/0x4a) <...>-5554 1D... 5us : __rcu_read_unlock+0x8/0x6e (__atomic_notifier_call_chain+0x42/0x4a) <...>-5554 1.... 5us : rt_down_read_trylock+0xc/0x4e (do_page_fault+0x186/0x576) <...>-5554 1.... 5us : __spin_lock_irqsave+0x8/0x3b (rt_down_read_trylock+0x13/0x4e) <...>-5554 1D..1 5us : __spin_unlock_irqrestore+0x8/0x2c (rt_down_read_trylock+0x42/0x4e) <...>-5554 1.... 6us : rt_mutex_trylock+0xb/0x33 (rt_down_read_trylock+0x4a/0x4e) <...>-5554 1.... 6us : find_vma+0xa/0x5e (do_page_fault+0x1bd/0x576) <...>-5554 1.... 7us : __handle_mm_fault+0xe/0xa66 (do_page_fault+0x24e/0x576) <...>-5554 1.... 7us : kmap+0xb/0x41 (__handle_mm_fault+0xd3/0xa66) <...>-5554 1.... 7us : page_address+0xe/0x7d (kmap+0x36/0x41) <...>-5554 1.... 7us : rt_spin_lock+0x8/0xf (__handle_mm_fault+0x982/0xa66) <...>-5554 1.... 8us : __rt_spin_lock+0x8/0x25 (rt_spin_lock+0xd/0xf) <...>-5554 1.... 8us : do_wp_page+0xe/0x4ac (__handle_mm_fault+0x9cb/0xa66) <...>-5554 1.... 8us : vm_normal_page+0xe/0x6a (do_wp_page+0x2d/0x4ac) <...>-5554 1.... 8us : can_share_swap_page+0xe/0x66 (do_wp_page+0x58/0x4ac) <...>-5554 1.... 9us : unlock_page+0x9/0x53 (do_wp_page+0x62/0x4ac) <...>-5554 1.... 9us : __wake_up_bit+0xc/0x39 (unlock_page+0x50/0x53) <...>-5554 1.... 9us : flush_tlb_page+0xc/0xac (do_wp_page+0x1ad/0x4ac) <...>-5554 1.... 10us : rt_spin_unlock+0x8/0x2b (do_wp_page+0x453/0x4ac) <...>-5554 1.... 10us : kunmap_virt+0xb/0x44 (do_wp_page+0x467/0x4ac) <...>-5554 1.... 10us : rt_up_read+0xc/0x4f (do_page_fault+0x2b2/0x576) <...>-5554 1.... 10us : __spin_lock_irqsave+0x8/0x3b (rt_up_read+0x13/0x4f) <...>-5554 1D..1 11us : __spin_unlock_irqrestore+0x8/0x2c (rt_up_read+0x43/0x4f) <...>-5554 1.... 11us+: rt_mutex_unlock+0xb/0x30 (rt_up_read+0x4b/0x4f) <...>-5554 1D... 13us : do_page_fault+0x25/0x576 (b79efe4b 7 b7ad20d0) <...>-5554 1D... 13us : atomic_notifier_call_chain+0xb/0x36 (do_page_fault+0x129/0x576) <...>-5554 1D... 14us : __atomic_notifier_call_chain+0xc/0x4a (atomic_notifier_call_chain+0x34/0x36) <...>-5554 1D... 14us : __rcu_read_lock+0xb/0x9d (__atomic_notifier_call_chain+0x11/0x4a) <...>-5554 1D... 14us : __rcu_read_unlock+0x8/0x6e (__atomic_notifier_call_chain+0x42/0x4a) <...>-5554 1.... 14us : rt_down_read_trylock+0xc/0x4e (do_page_fault+0x186/0x576) <...>-5554 1.... 15us : __spin_lock_irqsave+0x8/0x3b (rt_down_read_trylock+0x13/0x4e) <...>-5554 1D..1 15us : __spin_unlock_irqrestore+0x8/0x2c (rt_down_read_trylock+0x42/0x4e) <...>-5554 1.... 15us : rt_mutex_trylock+0xb/0x33 (rt_down_read_trylock+0x4a/0x4e) <...>-5554 1.... 15us : find_vma+0xa/0x5e (do_page_fault+0x1bd/0x576) <...>-5554 1.... 16us : __handle_mm_fault+0xe/0xa66 (do_page_fault+0x24e/0x576) <...>-5554 1.... 16us : kmap+0xb/0x41 (__handle_mm_fault+0xd3/0xa66) <...>-5554 1.... 16us : kmap_high+0x11/0x41c (kmap+0x3f/0x41) <...>-5554 1.... 16us : page_address+0xe/0x7d (kmap_high+0x108/0x41c) <...>-5554 1.... 17us : page_slot+0x8/0x1c (page_address+0x53/0x7d) <...>-5554 1.... 17us : rt_spin_lock+0x8/0xf (page_address+0x5f/0x7d) <...>-5554 1.... 17us : __rt_spin_lock+0x8/0x25 (rt_spin_lock+0xd/0xf) <...>-5554 1.... 17us : __page_address+0xa/0x3b (page_address+0x6b/0x7d) <...>-5554 1.... 18us : rt_spin_unlock+0x8/0x2b (page_address+0x74/0x7d) <...>-5554 1.... 18us : page_address+0xe/0x7d (kmap_high+0x15c/0x41c) <...>-5554 1.... 18us : page_slot+0x8/0x1c (page_address+0x53/0x7d) <...>-5554 1.... 19us : rt_spin_lock+0x8/0xf (page_address+0x5f/0x7d) <...>-5554 1.... 19us : __rt_spin_lock+0x8/0x25 (rt_spin_lock+0xd/0xf) <...>-5554 1.... 19us : __page_address+0xa/0x3b (page_address+0x6b/0x7d) <...>-5554 1.... 19us : rt_spin_unlock+0x8/0x2b (page_address+0x74/0x7d) <...>-5554 1.... 20us : rt_spin_lock+0x8/0xf (__handle_mm_fault+0x982/0xa66) <...>-5554 1.... 20us : __rt_spin_lock+0x8/0x25 (rt_spin_lock+0xd/0xf) <...>-5554 1.... 20us : do_wp_page+0xe/0x4ac (__handle_mm_fault+0x9cb/0xa66) <...>-5554 1.... 20us : vm_normal_page+0xe/0x6a (do_wp_page+0x2d/0x4ac) <...>-5554 1.... 21us : can_share_swap_page+0xe/0x66 (do_wp_page+0x58/0x4ac) <...>-5554 1.... 21us : unlock_page+0x9/0x53 (do_wp_page+0x62/0x4ac) <...>-5554 1.... 21us : __wake_up_bit+0xc/0x39 (unlock_page+0x50/0x53) <...>-5554 1.... 21us : flush_tlb_page+0xc/0xac (do_wp_page+0x1ad/0x4ac) <...>-5554 1.... 22us : rt_spin_unlock+0x8/0x2b (do_wp_page+0x453/0x4ac) <...>-5554 1.... 22us : kunmap_virt+0xb/0x44 (do_wp_page+0x467/0x4ac) <...>-5554 1.... 22us : kunmap+0x8/0x48 (kunmap_virt+0x42/0x44) <...>-5554 1.... 23us : kunmap_high+0xb/0xa9 (kunmap+0x46/0x48) <...>-5554 1.... 23us : page_address+0xe/0x7d (kunmap_high+0x13/0xa9) <...>-5554 1.... 23us : page_slot+0x8/0x1c (page_address+0x53/0x7d) <...>-5554 1.... 23us : rt_spin_lock+0x8/0xf (page_address+0x5f/0x7d) <...>-5554 1.... 23us : __rt_spin_lock+0x8/0x25 (rt_spin_lock+0xd/0xf) <...>-5554 1.... 24us : __page_address+0xa/0x3b (page_address+0x6b/0x7d) <...>-5554 1.... 24us : rt_spin_unlock+0x8/0x2b (page_address+0x74/0x7d) <...>-5554 1.... 24us : __wake_up+0xe/0x5b (kunmap_high+0x6b/0xa9) <...>-5554 1.... 24us : rt_spin_lock+0x8/0xf (__wake_up+0x19/0x5b) <...>-5554 1.... 25us : __rt_spin_lock+0x8/0x25 (rt_spin_lock+0xd/0xf) <...>-5554 1.... 25us : __wake_up_common+0xe/0x5e (__wake_up+0x38/0x5b) <...>-5554 1.... 25us : rt_spin_unlock+0x8/0x2b (__wake_up+0x3f/0x5b) <...>-5554 1.... 25us : __wake_up+0xe/0x5b (kunmap_high+0xa7/0xa9) <...>-5554 1.... 26us : rt_spin_lock+0x8/0xf (__wake_up+0x19/0x5b) <...>-5554 1.... 26us : __rt_spin_lock+0x8/0x25 (rt_spin_lock+0xd/0xf) <...>-5554 1.... 26us : __wake_up_common+0xe/0x5e (__wake_up+0x38/0x5b) <...>-5554 1.... 26us : rt_spin_unlock+0x8/0x2b (__wake_up+0x3f/0x5b) <...>-5554 1.... 27us : rt_up_read+0xc/0x4f (do_page_fault+0x2b2/0x576) <...>-5554 1.... 27us : __spin_lock_irqsave+0x8/0x3b (rt_up_read+0x13/0x4f) <...>-5554 1D..1 27us : __spin_unlock_irqrestore+0x8/0x2c (rt_up_read+0x43/0x4f) <...>-5554 1.... 27us : rt_mutex_unlock+0xb/0x30 (rt_up_read+0x4b/0x4f) <...>-5554 1D... 29us : do_page_fault+0x25/0x576 (b7a2012c 7 b7f51000) <...>-5554 1D... 29us : atomic_notifier_call_chain+0xb/0x36 (do_page_fault+0x129/0x576) <...>-5554 1D... 29us : __atomic_notifier_call_chain+0xc/0x4a (atomic_notifier_call_chain+0x34/0x36) <...>-5554 1D... 29us : __rcu_read_lock+0xb/0x9d (__atomic_notifier_call_chain+0x11/0x4a) <...>-5554 1D... 30us : __rcu_read_unlock+0x8/0x6e (__atomic_notifier_call_chain+0x42/0x4a) <...>-5554 1.... 30us : rt_down_read_trylock+0xc/0x4e (do_page_fault+0x186/0x576) <...>-5554 1.... 30us : __spin_lock_irqsave+0x8/0x3b (rt_down_read_trylock+0x13/0x4e) <...>-5554 1D..1 31us : __spin_unlock_irqrestore+0x8/0x2c (rt_down_read_trylock+0x42/0x4e) <...>-5554 1.... 31us : rt_mutex_trylock+0xb/0x33 (rt_down_read_trylock+0x4a/0x4e) <...>-5554 1.... 31us : find_vma+0xa/0x5e (do_page_fault+0x1bd/0x576) <...>-5554 1.... 31us : __handle_mm_fault+0xe/0xa66 (do_page_fault+0x24e/0x576) <...>-5554 1.... 32us : kmap+0xb/0x41 (__handle_mm_fault+0xd3/0xa66) <...>-5554 1.... 32us : kmap_high+0x11/0x41c (kmap+0x3f/0x41) <...>-5554 1.... 32us : page_address+0xe/0x7d (kmap_high+0x108/0x41c) <...>-5554 1.... 32us : page_slot+0x8/0x1c (page_address+0x53/0x7d) <...>-5554 1.... 32us : rt_spin_lock+0x8/0xf (page_address+0x5f/0x7d) <...>-5554 1.... 33us : __rt_spin_lock+0x8/0x25 (rt_spin_lock+0xd/0xf) <...>-5554 1.... 33us : __page_address+0xa/0x3b (page_address+0x6b/0x7d) <...>-5554 1.... 33us : rt_spin_unlock+0x8/0x2b (page_address+0x74/0x7d) <...>-5554 1.... 33us : page_address+0xe/0x7d (kmap_high+0x15c/0x41c) <...>-5554 1.... 34us : page_slot+0x8/0x1c (page_address+0x53/0x7d) <...>-5554 1.... 34us : rt_spin_lock+0x8/0xf (page_address+0x5f/0x7d) <...>-5554 1.... 34us : __rt_spin_lock+0x8/0x25 (rt_spin_lock+0xd/0xf) <...>-5554 1.... 34us : __page_address+0xa/0x3b (page_address+0x6b/0x7d) <...>-5554 1.... 35us : rt_spin_unlock+0x8/0x2b (page_address+0x74/0x7d) <...>-5554 1.... 35us : rt_spin_lock+0x8/0xf (__handle_mm_fault+0x982/0xa66) <...>-5554 1.... 35us : __rt_spin_lock+0x8/0x25 (rt_spin_lock+0xd/0xf) <...>-5554 1.... 35us : do_wp_page+0xe/0x4ac (__handle_mm_fault+0x9cb/0xa66) <...>-5554 1.... 36us : vm_normal_page+0xe/0x6a (do_wp_page+0x2d/0x4ac) <...>-5554 1.... 36us : can_share_swap_page+0xe/0x66 (do_wp_page+0x58/0x4ac) <...>-5554 1.... 36us : unlock_page+0x9/0x53 (do_wp_page+0x62/0x4ac) <...>-5554 1.... 36us : __wake_up_bit+0xc/0x39 (unlock_page+0x50/0x53) <...>-5554 1.... 37us : flush_tlb_page+0xc/0xac (do_wp_page+0x1ad/0x4ac) <...>-5554 1.... 37us : rt_spin_unlock+0x8/0x2b (do_wp_page+0x453/0x4ac) <...>-5554 1.... 37us : kunmap_virt+0xb/0x44 (do_wp_page+0x467/0x4ac) <...>-5554 1.... 38us : kunmap+0x8/0x48 (kunmap_virt+0x42/0x44) <...>-5554 1.... 38us : kunmap_high+0xb/0xa9 (kunmap+0x46/0x48) <...>-5554 1.... 38us : page_address+0xe/0x7d (kunmap_high+0x13/0xa9) <...>-5554 1.... 38us : page_slot+0x8/0x1c (page_address+0x53/0x7d) <...>-5554 1.... 38us : rt_spin_lock+0x8/0xf (page_address+0x5f/0x7d) <...>-5554 1.... 39us : __rt_spin_lock+0x8/0x25 (rt_spin_lock+0xd/0xf) <...>-5554 1.... 39us : __page_address+0xa/0x3b (page_address+0x6b/0x7d) <...>-5554 1.... 39us : rt_spin_unlock+0x8/0x2b (page_address+0x74/0x7d) <...>-5554 1.... 39us : __wake_up+0xe/0x5b (kunmap_high+0x6b/0xa9) <...>-5554 1.... 40us : rt_spin_lock+0x8/0xf (__wake_up+0x19/0x5b) <...>-5554 1.... 40us : __rt_spin_lock+0x8/0x25 (rt_spin_lock+0xd/0xf) <...>-5554 1.... 40us : __wake_up_common+0xe/0x5e (__wake_up+0x38/0x5b) <...>-5554 1.... 40us : rt_spin_unlock+0x8/0x2b (__wake_up+0x3f/0x5b) <...>-5554 1.... 41us : __wake_up+0xe/0x5b (kunmap_high+0xa7/0xa9) <...>-5554 1.... 41us : rt_spin_lock+0x8/0xf (__wake_up+0x19/0x5b) <...>-5554 1.... 41us : __rt_spin_lock+0x8/0x25 (rt_spin_lock+0xd/0xf) <...>-5554 1.... 41us : __wake_up_common+0xe/0x5e (__wake_up+0x38/0x5b) <...>-5554 1.... 41us : rt_spin_unlock+0x8/0x2b (__wake_up+0x3f/0x5b) <...>-5554 1.... 42us : rt_up_read+0xc/0x4f (do_page_fault+0x2b2/0x576) <...>-5554 1.... 42us : __spin_lock_irqsave+0x8/0x3b (rt_up_read+0x13/0x4f) <...>-5554 1D..1 42us : __spin_unlock_irqrestore+0x8/0x2c (rt_up_read+0x43/0x4f) <...>-5554 1.... 42us+: rt_mutex_unlock+0xb/0x30 (rt_up_read+0x4b/0x4f) <...>-5554 1.... 45us > sys_write+0x0/0x70 (00000001 b7f51000 000000d8) <...>-5554 1.... 45us : do_syscall_trace+0xe/0x1d6 (syscall_trace_entry+0x11/0x30) <...>-5554 1.... 46us : audit_syscall_entry+0xe/0x147 (do_syscall_trace+0x182/0x1d6) <...>-5554 1.... 46us : current_kernel_time+0xb/0x3e (audit_syscall_entry+0x11d/0x147) <...>-5554 1.... 46us : sys_write+0xd/0x70 (syscall_call+0x7/0xb) <...>-5554 1.... 46us : fget_light+0xb/0x82 (sys_write+0x1d/0x70) <...>-5554 1.... 47us : __rcu_read_lock+0xb/0x9d (fget_light+0x3c/0x82) <...>-5554 1.... 47us : __rcu_read_unlock+0x8/0x6e (fget_light+0x7b/0x82) <...>-5554 1.... 47us : vfs_write+0xe/0x177 (sys_write+0x4c/0x70) <...>-5554 1.... 48us : rw_verify_area+0xe/0xa4 (vfs_write+0x80/0x177) <...>-5554 1.... 48us : dummy_file_permission+0x8/0xc (vfs_write+0xa1/0x177) <...>-5554 1.... 48us : tty_write+0xe/0x1ee (vfs_write+0xce/0x177) <...>-5554 1.... 48us : tty_paranoia_check+0xc/0x77 (tty_write+0x31/0x1ee) <...>-5554 1.... 49us : tty_ldisc_ref_wait+0xd/0xa7 (tty_write+0x6c/0x1ee) <...>-5554 1.... 49us : tty_ldisc_try+0xa/0x3b (tty_ldisc_ref_wait+0x18/0xa7) <...>-5554 1.... 49us : rt_spin_lock+0x8/0xf (tty_ldisc_try+0x19/0x3b) <...>-5554 1.... 49us : __rt_spin_lock+0x8/0x25 (rt_spin_lock+0xd/0xf) <...>-5554 1.... 50us : rt_spin_unlock+0x8/0x2b (tty_ldisc_try+0x35/0x3b) <...>-5554 1.... 50us : _mutex_lock_interruptible+0xb/0x1d (tty_write+0x91/0x1ee) <...>-5554 1.... 50us : rt_mutex_lock_interruptible+0xd/0x52 (_mutex_lock_interruptible+0x1b/0x1d) <...>-5554 1.... 50us : copy_from_user+0xe/0x62 (tty_write+0x147/0x1ee) <...>-5554 1.... 51us : __copy_from_user_ll+0xa/0xdc (copy_from_user+0x3e/0x62) <...>-5554 1.... 51us : lock_kernel+0xa/0x28 (tty_write+0x150/0x1ee) <...>-5554 1.... 51us : rt_down+0xd/0x33 (lock_kernel+0x21/0x28) <...>-5554 1.... 51us : rt_mutex_lock+0xb/0x42 (rt_down+0x1a/0x33) <...>-5554 1.... 52us : write_chan+0xe/0x2ce (tty_write+0x16b/0x1ee) <...>-5554 1.... 52us : memcpy+0xa/0x2a (write_chan+0x39/0x2ce) <...>-5554 1.... 52us : add_wait_queue+0xa/0x36 (write_chan+0x81/0x2ce) <...>-5554 1.... 52us : rt_spin_lock+0x8/0xf (add_wait_queue+0x16/0x36) <...>-5554 1.... 52us : __rt_spin_lock+0x8/0x25 (rt_spin_lock+0xd/0xf) <...>-5554 1.... 53us : rt_spin_unlock+0x8/0x2b (add_wait_queue+0x32/0x36) <...>-5554 1.... 53us : tty_hung_up_p+0x8/0x1a (write_chan+0xb4/0x2ce) <...>-5554 1.... 53us : pty_write_room+0x8/0x2a (write_chan+0x102/0x2ce) <...>-5554 1.... 54us : pty_write+0xc/0x57 (write_chan+0x1f1/0x2ce) <...>-5554 1.... 54us : n_tty_receive_buf+0x11/0xfc2 (pty_write+0x4b/0x57) <...>-5554 1.... 55us : rt_spin_lock+0x8/0xf (n_tty_receive_buf+0x61/0xfc2) <...>-5554 1.... 55us : __rt_spin_lock+0x8/0x25 (rt_spin_lock+0xd/0xf) <...>-5554 1.... 55us : rt_spin_unlock+0x8/0x2b (n_tty_receive_buf+0x13c/0xfc2) <...>-5554 1.... 55us : n_tty_set_room+0x8/0x39 (n_tty_receive_buf+0xf06/0xfc2) <...>-5554 1.... 56us : kill_fasync+0xc/0x46 (n_tty_receive_buf+0xf40/0xfc2) <...>-5554 1.... 56us : __wake_up+0xe/0x5b (n_tty_receive_buf+0xf70/0xfc2) <...>-5554 1.... 56us : rt_spin_lock+0x8/0xf (__wake_up+0x19/0x5b) <...>-5554 1.... 56us : __rt_spin_lock+0x8/0x25 (rt_spin_lock+0xd/0xf) <...>-5554 1.... 57us : __wake_up_common+0xe/0x5e (__wake_up+0x38/0x5b) <...>-5554 1.... 57us : default_wake_function+0xb/0x34 (__wake_up_common+0x3c/0x5e) <...>-5554 1.... 57us : try_to_wake_up+0xe/0x3c8 (default_wake_function+0x32/0x34) <...>-5554 1.... 57us : try_to_wake_up+0x13/0x3c8 (c011e937 0 0) <...>-5554 1.... 58us : task_rq_lock+0xb/0x57 (try_to_wake_up+0x38/0x3c8) <...>-5554 1D... 58us : __spin_lock+0x8/0x23 (task_rq_lock+0x31/0x57) <...>-5554 1D..1 59us : idle_cpu+0x8/0x2d (try_to_wake_up+0x1cb/0x3c8) <...>-5554 1D..1 59us : __first_cpu+0x8/0x26 (try_to_wake_up+0x209/0x3c8) <...>-5554 1D..1 59us : idle_cpu+0x8/0x2d (try_to_wake_up+0x213/0x3c8) <...>-5554 1D..1 59us : __next_cpu+0xb/0x35 (try_to_wake_up+0x226/0x3c8) <...>-5554 1D..1 60us : find_next_bit+0xb/0x73 (__next_cpu+0x26/0x35) <...>-5554 1D..1 60us : idle_cpu+0x8/0x2d (try_to_wake_up+0x213/0x3c8) <...>-5554 1D..1 60us : __next_cpu+0xb/0x35 (try_to_wake_up+0x226/0x3c8) <...>-5554 1D..1 60us : find_next_bit+0xb/0x73 (__next_cpu+0x26/0x35) <...>-5554 1D..1 61us : set_task_cpu+0xe/0x106 (try_to_wake_up+0x24a/0x3c8) <...>-5554 1D..1 61us : __spin_unlock_irqrestore+0x8/0x2c (try_to_wake_up+0x255/0x3c8) <...>-5554 1.... 61us : task_rq_lock+0xb/0x57 (try_to_wake_up+0x267/0x3c8) <...>-5554 1D... 61us : __spin_lock+0x8/0x23 (task_rq_lock+0x31/0x57) <...>-5554 1D..1 62us : activate_task+0xe/0xd1 (try_to_wake_up+0x2eb/0x3c8) <...>-5554 1D..1 62us : __rq_clock+0xe/0xb1 (activate_task+0x27/0xd1) <...>-5554 1D..1 62us : sched_clock+0xa/0x5e (__rq_clock+0x22/0xb1) <...>-5554 1D..1 62us : activate_task+0x86/0xd1 <<...>-3658> (0 1) <...>-5554 1D..1 63us : enqueue_task+0xc/0x6c (activate_task+0xa4/0xd1) <...>-5554 1D..1 63us : sched_clock+0xa/0x5e (enqueue_task+0x2b/0x6c) <...>-5554 1D..1 63us : enqueue_task_fair+0xc/0x56 (enqueue_task+0x5f/0x6c) <...>-5554 1D..1 63us : enqueue_entity+0xe/0x229 (enqueue_task_fair+0x50/0x56) <...>-5554 1D..1 64us : update_curr+0xe/0x24a (enqueue_entity+0x27/0x229) <...>-5554 1D..1 64us : update_stats_enqueue+0xe/0xb4 (enqueue_entity+0x18b/0x229) <...>-5554 1D..1 64us : rb_insert_color+0xe/0xd3 (enqueue_entity+0x205/0x229) <...>-5554 1D..1 65us : update_curr_load+0xe/0xc9 (activate_task+0xb7/0xd1) <...>-5554 1D..1 65us : __spin_unlock_irqrestore+0x8/0x2c (try_to_wake_up+0x3be/0x3c8) <...>-5554 1.... 65us : rt_spin_unlock+0x8/0x2b (__wake_up+0x3f/0x5b) <...>-5554 1.... 66us : opost+0xe/0x1c3 (write_chan+0x21a/0x2ce) <...>-5554 1.... 66us : pty_write_room+0x8/0x2a (opost+0x20/0x1c3) <...>-5554 1.... 66us : tty_default_put_char+0xb/0x31 (opost+0x9b/0x1c3) <...>-5554 1.... 66us : pty_write+0xc/0x57 (tty_default_put_char+0x2f/0x31) <...>-5554 1.... 66us : n_tty_receive_buf+0x11/0xfc2 (pty_write+0x4b/0x57) <...>-5554 1.... 67us : rt_spin_lock+0x8/0xf (n_tty_receive_buf+0x61/0xfc2) <...>-5554 1.... 67us : __rt_spin_lock+0x8/0x25 (rt_spin_lock+0xd/0xf) <...>-5554 1.... 67us : rt_spin_unlock+0x8/0x2b (n_tty_receive_buf+0x13c/0xfc2) <...>-5554 1.... 67us : n_tty_set_room+0x8/0x39 (n_tty_receive_buf+0xf06/0xfc2) <...>-5554 1.... 68us : kill_fasync+0xc/0x46 (n_tty_receive_buf+0xf40/0xfc2) <...>-5554 1.... 68us : __wake_up+0xe/0x5b (n_tty_receive_buf+0xf70/0xfc2) <...>-5554 1.... 68us : rt_spin_lock+0x8/0xf (__wake_up+0x19/0x5b) <...>-5554 1.... 68us : __rt_spin_lock+0x8/0x25 (rt_spin_lock+0xd/0xf) <...>-5554 1.... 69us : __wake_up_common+0xe/0x5e (__wake_up+0x38/0x5b) <...>-5554 1.... 69us : default_wake_function+0xb/0x34 (__wake_up_common+0x3c/0x5e) <...>-5554 1.... 69us : try_to_wake_up+0xe/0x3c8 (default_wake_function+0x32/0x34) <...>-5554 1.... 69us : try_to_wake_up+0x13/0x3c8 (c011e937 0 0) <...>-5554 1.... 69us : task_rq_lock+0xb/0x57 (try_to_wake_up+0x38/0x3c8) <...>-5554 1D... 70us : __spin_lock+0x8/0x23 (task_rq_lock+0x31/0x57) <...>-5554 1D..1 70us : __spin_unlock_irqrestore+0x8/0x2c (try_to_wake_up+0x3be/0x3c8) <...>-5554 1.... 70us : rt_spin_unlock+0x8/0x2b (__wake_up+0x3f/0x5b) <...>-5554 1.... 71us : tty_default_put_char+0xb/0x31 (opost+0x1aa/0x1c3) <...>-5554 1.... 71us : pty_write+0xc/0x57 (tty_default_put_char+0x2f/0x31) <...>-5554 1.... 71us : n_tty_receive_buf+0x11/0xfc2 (pty_write+0x4b/0x57) <...>-5554 1.... 71us : rt_spin_lock+0x8/0xf (n_tty_receive_buf+0x61/0xfc2) <...>-5554 1.... 72us : __rt_spin_lock+0x8/0x25 (rt_spin_lock+0xd/0xf) <...>-5554 1.... 72us : rt_spin_unlock+0x8/0x2b (n_tty_receive_buf+0x13c/0xfc2) <...>-5554 1.... 72us : n_tty_set_room+0x8/0x39 (n_tty_receive_buf+0xf06/0xfc2) <...>-5554 1.... 72us : kill_fasync+0xc/0x46 (n_tty_receive_buf+0xf40/0xfc2) <...>-5554 1.... 73us : __wake_up+0xe/0x5b (n_tty_receive_buf+0xf70/0xfc2) <...>-5554 1.... 73us : rt_spin_lock+0x8/0xf (__wake_up+0x19/0x5b) <...>-5554 1.... 73us : __rt_spin_lock+0x8/0x25 (rt_spin_lock+0xd/0xf) <...>-5554 1.... 73us : __wake_up_common+0xe/0x5e (__wake_up+0x38/0x5b) <...>-5554 1.... 73us : default_wake_function+0xb/0x34 (__wake_up_common+0x3c/0x5e) <...>-5554 1.... 74us : try_to_wake_up+0xe/0x3c8 (default_wake_function+0x32/0x34) <...>-5554 1.... 74us : try_to_wake_up+0x13/0x3c8 (c011e937 0 0) <...>-5554 1.... 74us : task_rq_lock+0xb/0x57 (try_to_wake_up+0x38/0x3c8) <...>-5554 1D... 74us : __spin_lock+0x8/0x23 (task_rq_lock+0x31/0x57) <...>-5554 1D..1 75us : __spin_unlock_irqrestore+0x8/0x2c (try_to_wake_up+0x3be/0x3c8) <...>-5554 1.... 75us : rt_spin_unlock+0x8/0x2b (__wake_up+0x3f/0x5b) <...>-5554 1.... 75us : remove_wait_queue+0xa/0x3a (write_chan+0x2b5/0x2ce) <...>-5554 1.... 75us : rt_spin_lock+0x8/0xf (remove_wait_queue+0x13/0x3a) <...>-5554 1.... 76us : __rt_spin_lock+0x8/0x25 (rt_spin_lock+0xd/0xf) <...>-5554 1.... 76us : rt_spin_unlock+0x8/0x2b (remove_wait_queue+0x36/0x3a) <...>-5554 1.... 76us : unlock_kernel+0x8/0x2d (tty_write+0x172/0x1ee) <...>-5554 1.... 76us : rt_up+0xb/0x47 (unlock_kernel+0x2b/0x2d) <...>-5554 1...1 77us : rt_mutex_unlock+0xb/0x30 (rt_up+0x2f/0x47) <...>-5554 1.... 77us : current_fs_time+0xe/0x59 (tty_write+0x1be/0x1ee) <...>-5554 1.... 77us : timespec_trunc+0xe/0x45 (current_fs_time+0x51/0x59) <...>-5554 1.... 77us : _mutex_unlock+0xb/0x15 (tty_write+0x1d2/0x1ee) <...>-5554 1.... 78us : rt_mutex_unlock+0xb/0x30 (_mutex_unlock+0x13/0x15) <...>-5554 1.... 78us : tty_ldisc_deref+0xc/0x69 (tty_write+0x1dd/0x1ee) <...>-5554 1.... 78us : rt_spin_lock+0x8/0xf (tty_ldisc_deref+0x21/0x69) <...>-5554 1.... 78us : __rt_spin_lock+0x8/0x25 (rt_spin_lock+0xd/0xf) <...>-5554 1.... 79us : __wake_up+0xe/0x5b (tty_ldisc_deref+0x5b/0x69) <...>-5554 1.... 79us : rt_spin_lock+0x8/0xf (__wake_up+0x19/0x5b) <...>-5554 1.... 79us : __rt_spin_lock+0x8/0x25 (rt_spin_lock+0xd/0xf) <...>-5554 1.... 79us : __wake_up_common+0xe/0x5e (__wake_up+0x38/0x5b) <...>-5554 1.... 79us : rt_spin_unlock+0x8/0x2b (__wake_up+0x3f/0x5b) <...>-5554 1.... 80us : rt_spin_unlock+0x8/0x2b (tty_ldisc_deref+0x65/0x69) <...>-5554 1.... 80us : dnotify_parent+0xe/0x6f (vfs_write+0x124/0x177) <...>-5554 1.... 80us : rt_spin_lock+0x8/0xf (dnotify_parent+0x27/0x6f) <...>-5554 1.... 80us : __rt_spin_lock+0x8/0x25 (rt_spin_lock+0xd/0xf) <...>-5554 1.... 81us : rt_spin_unlock+0x8/0x2b (dnotify_parent+0x68/0x6f) <...>-5554 1.... 81us : inotify_dentry_parent_queue_event+0xe/0x8d (vfs_write+0x13f/0x177) <...>-5554 1.... 81us : inotify_inode_queue_event+0xe/0xc9 (vfs_write+0x163/0x177) <...>-5554 1.... 82us : fput+0x8/0x1e (sys_write+0x67/0x70) <...>-5554 1.... 82us : do_syscall_trace+0xe/0x1d6 (syscall_exit_work+0x16/0x1b) <...>-5554 1.... 82us : audit_syscall_exit+0xe/0x361 (do_syscall_trace+0x83/0x1d6) <...>-5554 1.... 82us : kfree+0xc/0xa0 (audit_syscall_exit+0x346/0x361) <...>-5554 1D... 84us : do_page_fault+0x25/0x576 (b7bee046 7 8087304) <...>-5554 1D... 84us : atomic_notifier_call_chain+0xb/0x36 (do_page_fault+0x129/0x576) <...>-5554 1D... 84us : __atomic_notifier_call_chain+0xc/0x4a (atomic_notifier_call_chain+0x34/0x36) <...>-5554 1D... 84us : __rcu_read_lock+0xb/0x9d (__atomic_notifier_call_chain+0x11/0x4a) <...>-5554 1D... 85us : __rcu_read_unlock+0x8/0x6e (__atomic_notifier_call_chain+0x42/0x4a) <...>-5554 1.... 85us : rt_down_read_trylock+0xc/0x4e (do_page_fault+0x186/0x576) <...>-5554 1.... 85us : __spin_lock_irqsave+0x8/0x3b (rt_down_read_trylock+0x13/0x4e) <...>-5554 1D..1 86us : __spin_unlock_irqrestore+0x8/0x2c (rt_down_read_trylock+0x42/0x4e) <...>-5554 1.... 86us : rt_mutex_trylock+0xb/0x33 (rt_down_read_trylock+0x4a/0x4e) <...>-5554 1.... 86us : find_vma+0xa/0x5e (do_page_fault+0x1bd/0x576) <...>-5554 1.... 87us : __handle_mm_fault+0xe/0xa66 (do_page_fault+0x24e/0x576) <...>-5554 1.... 87us : kmap+0xb/0x41 (__handle_mm_fault+0xd3/0xa66) <...>-5554 1.... 87us : kmap_high+0x11/0x41c (kmap+0x3f/0x41) <...>-5554 1.... 87us : page_address+0xe/0x7d (kmap_high+0x108/0x41c) <...>-5554 1.... 88us : page_slot+0x8/0x1c (page_address+0x53/0x7d) <...>-5554 1.... 88us : rt_spin_lock+0x8/0xf (page_address+0x5f/0x7d) <...>-5554 1.... 88us : __rt_spin_lock+0x8/0x25 (rt_spin_lock+0xd/0xf) <...>-5554 1.... 88us : __page_address+0xa/0x3b (page_address+0x6b/0x7d) <...>-5554 1.... 89us : rt_spin_unlock+0x8/0x2b (page_address+0x74/0x7d) <...>-5554 1.... 89us : page_address+0xe/0x7d (kmap_high+0x15c/0x41c) <...>-5554 1.... 89us : page_slot+0x8/0x1c (page_address+0x53/0x7d) <...>-5554 1.... 90us : rt_spin_lock+0x8/0xf (page_address+0x5f/0x7d) <...>-5554 1.... 90us : __rt_spin_lock+0x8/0x25 (rt_spin_lock+0xd/0xf) <...>-5554 1.... 90us : __page_address+0xa/0x3b (page_address+0x6b/0x7d) <...>-5554 1.... 90us : rt_spin_unlock+0x8/0x2b (page_address+0x74/0x7d) <...>-5554 1.... 91us : rt_spin_lock+0x8/0xf (__handle_mm_fault+0x982/0xa66) <...>-5554 1.... 91us : __rt_spin_lock+0x8/0x25 (rt_spin_lock+0xd/0xf) <...>-5554 1.... 91us : do_wp_page+0xe/0x4ac (__handle_mm_fault+0x9cb/0xa66) <...>-5554 1.... 91us : vm_normal_page+0xe/0x6a (do_wp_page+0x2d/0x4ac) <...>-5554 1.... 92us : can_share_swap_page+0xe/0x66 (do_wp_page+0x58/0x4ac) <...>-5554 1.... 92us : unlock_page+0x9/0x53 (do_wp_page+0x62/0x4ac) <...>-5554 1.... 92us : __wake_up_bit+0xc/0x39 (unlock_page+0x50/0x53) <...>-5554 1.... 92us : flush_tlb_page+0xc/0xac (do_wp_page+0x1ad/0x4ac) <...>-5554 1.... 93us : rt_spin_unlock+0x8/0x2b (do_wp_page+0x453/0x4ac) <...>-5554 1.... 93us : kunmap_virt+0xb/0x44 (do_wp_page+0x467/0x4ac) <...>-5554 1.... 93us : kunmap+0x8/0x48 (kunmap_virt+0x42/0x44) <...>-5554 1.... 94us : kunmap_high+0xb/0xa9 (kunmap+0x46/0x48) <...>-5554 1.... 94us : page_address+0xe/0x7d (kunmap_high+0x13/0xa9) <...>-5554 1.... 94us : page_slot+0x8/0x1c (page_address+0x53/0x7d) <...>-5554 1.... 94us : rt_spin_lock+0x8/0xf (page_address+0x5f/0x7d) <...>-5554 1.... 94us : __rt_spin_lock+0x8/0x25 (rt_spin_lock+0xd/0xf) <...>-5554 1.... 95us : __page_address+0xa/0x3b (page_address+0x6b/0x7d) <...>-5554 1.... 95us : rt_spin_unlock+0x8/0x2b (page_address+0x74/0x7d) <...>-5554 1.... 95us : __wake_up+0xe/0x5b (kunmap_high+0x6b/0xa9) <...>-5554 1.... 95us : rt_spin_lock+0x8/0xf (__wake_up+0x19/0x5b) <...>-5554 1.... 96us : __rt_spin_lock+0x8/0x25 (rt_spin_lock+0xd/0xf) <...>-5554 1.... 96us : __wake_up_common+0xe/0x5e (__wake_up+0x38/0x5b) <...>-5554 1.... 96us : rt_spin_unlock+0x8/0x2b (__wake_up+0x3f/0x5b) <...>-5554 1.... 96us : __wake_up+0xe/0x5b (kunmap_high+0xa7/0xa9) <...>-5554 1.... 97us : rt_spin_lock+0x8/0xf (__wake_up+0x19/0x5b) <...>-5554 1.... 97us : __rt_spin_lock+0x8/0x25 (rt_spin_lock+0xd/0xf) <...>-5554 1.... 97us : __wake_up_common+0xe/0x5e (__wake_up+0x38/0x5b) <...>-5554 1.... 97us : rt_spin_unlock+0x8/0x2b (__wake_up+0x3f/0x5b) <...>-5554 1.... 97us : rt_up_read+0xc/0x4f (do_page_fault+0x2b2/0x576) <...>-5554 1.... 98us : __spin_lock_irqsave+0x8/0x3b (rt_up_read+0x13/0x4f) <...>-5554 1D..1 98us : __spin_unlock_irqrestore+0x8/0x2c (rt_up_read+0x43/0x4f) <...>-5554 1.... 98us : rt_mutex_unlock+0xb/0x30 (rt_up_read+0x4b/0x4f) <...>-5554 1D... 99us : do_page_fault+0x25/0x576 (b7bf246e 7 b7f26628) <...>-5554 1D... 100us : atomic_notifier_call_chain+0xb/0x36 (do_page_fault+0x129/0x576) <...>-5554 1D... 100us : __atomic_notifier_call_chain+0xc/0x4a (atomic_notifier_call_chain+0x34/0x36) <...>-5554 1D... 100us : __rcu_read_lock+0xb/0x9d (__atomic_notifier_call_chain+0x11/0x4a) <...>-5554 1D... 100us : __rcu_read_unlock+0x8/0x6e (__atomic_notifier_call_chain+0x42/0x4a) <...>-5554 1.... 101us : rt_down_read_trylock+0xc/0x4e (do_page_fault+0x186/0x576) <...>-5554 1.... 101us : __spin_lock_irqsave+0x8/0x3b (rt_down_read_trylock+0x13/0x4e) <...>-5554 1D..1 101us : __spin_unlock_irqrestore+0x8/0x2c (rt_down_read_trylock+0x42/0x4e) <...>-5554 1.... 101us : rt_mutex_trylock+0xb/0x33 (rt_down_read_trylock+0x4a/0x4e) <...>-5554 1.... 102us : find_vma+0xa/0x5e (do_page_fault+0x1bd/0x576) <...>-5554 1.... 102us : __handle_mm_fault+0xe/0xa66 (do_page_fault+0x24e/0x576) <...>-5554 1.... 102us : kmap+0xb/0x41 (__handle_mm_fault+0xd3/0xa66) <...>-5554 1.... 102us : kmap_high+0x11/0x41c (kmap+0x3f/0x41) <...>-5554 1.... 103us : page_address+0xe/0x7d (kmap_high+0x108/0x41c) <...>-5554 1.... 103us : page_slot+0x8/0x1c (page_address+0x53/0x7d) <...>-5554 1.... 103us : rt_spin_lock+0x8/0xf (page_address+0x5f/0x7d) <...>-5554 1.... 103us : __rt_spin_lock+0x8/0x25 (rt_spin_lock+0xd/0xf) <...>-5554 1.... 104us : __page_address+0xa/0x3b (page_address+0x6b/0x7d) <...>-5554 1.... 104us : rt_spin_unlock+0x8/0x2b (page_address+0x74/0x7d) <...>-5554 1.... 104us : page_address+0xe/0x7d (kmap_high+0x15c/0x41c) <...>-5554 1.... 104us : page_slot+0x8/0x1c (page_address+0x53/0x7d) <...>-5554 1.... 105us : rt_spin_lock+0x8/0xf (page_address+0x5f/0x7d) <...>-5554 1.... 105us : __rt_spin_lock+0x8/0x25 (rt_spin_lock+0xd/0xf) <...>-5554 1.... 105us : __page_address+0xa/0x3b (page_address+0x6b/0x7d) <...>-5554 1.... 105us : rt_spin_unlock+0x8/0x2b (page_address+0x74/0x7d) <...>-5554 1.... 106us : rt_spin_lock+0x8/0xf (__handle_mm_fault+0x982/0xa66) <...>-5554 1.... 106us : __rt_spin_lock+0x8/0x25 (rt_spin_lock+0xd/0xf) <...>-5554 1.... 106us : do_wp_page+0xe/0x4ac (__handle_mm_fault+0x9cb/0xa66) <...>-5554 1.... 106us : vm_normal_page+0xe/0x6a (do_wp_page+0x2d/0x4ac) <...>-5554 1.... 107us : can_share_swap_page+0xe/0x66 (do_wp_page+0x58/0x4ac) <...>-5554 1.... 107us : unlock_page+0x9/0x53 (do_wp_page+0x62/0x4ac) <...>-5554 1.... 107us : __wake_up_bit+0xc/0x39 (unlock_page+0x50/0x53) <...>-5554 1.... 108us : flush_tlb_page+0xc/0xac (do_wp_page+0x1ad/0x4ac) <...>-5554 1.... 108us : rt_spin_unlock+0x8/0x2b (do_wp_page+0x453/0x4ac) <...>-5554 1.... 108us : kunmap_virt+0xb/0x44 (do_wp_page+0x467/0x4ac) <...>-5554 1.... 108us : kunmap+0x8/0x48 (kunmap_virt+0x42/0x44) <...>-5554 1.... 109us : kunmap_high+0xb/0xa9 (kunmap+0x46/0x48) <...>-5554 1.... 109us : page_address+0xe/0x7d (kunmap_high+0x13/0xa9) <...>-5554 1.... 109us : page_slot+0x8/0x1c (page_address+0x53/0x7d) <...>-5554 1.... 109us : rt_spin_lock+0x8/0xf (page_address+0x5f/0x7d) <...>-5554 1.... 109us : __rt_spin_lock+0x8/0x25 (rt_spin_lock+0xd/0xf) <...>-5554 1.... 110us : __page_address+0xa/0x3b (page_address+0x6b/0x7d) <...>-5554 1.... 110us : rt_spin_unlock+0x8/0x2b (page_address+0x74/0x7d) <...>-5554 1.... 110us : __wake_up+0xe/0x5b (kunmap_high+0x6b/0xa9) <...>-5554 1.... 110us : rt_spin_lock+0x8/0xf (__wake_up+0x19/0x5b) <...>-5554 1.... 111us : __rt_spin_lock+0x8/0x25 (rt_spin_lock+0xd/0xf) <...>-5554 1.... 111us : __wake_up_common+0xe/0x5e (__wake_up+0x38/0x5b) <...>-5554 1.... 111us : rt_spin_unlock+0x8/0x2b (__wake_up+0x3f/0x5b) <...>-5554 1.... 111us : __wake_up+0xe/0x5b (kunmap_high+0xa7/0xa9) <...>-5554 1.... 112us : rt_spin_lock+0x8/0xf (__wake_up+0x19/0x5b) <...>-5554 1.... 112us : __rt_spin_lock+0x8/0x25 (rt_spin_lock+0xd/0xf) <...>-5554 1.... 112us : __wake_up_common+0xe/0x5e (__wake_up+0x38/0x5b) <...>-5554 1.... 112us : rt_spin_unlock+0x8/0x2b (__wake_up+0x3f/0x5b) <...>-5554 1.... 113us : rt_up_read+0xc/0x4f (do_page_fault+0x2b2/0x576) <...>-5554 1.... 113us : __spin_lock_irqsave+0x8/0x3b (rt_up_read+0x13/0x4f) <...>-5554 1D..1 113us : __spin_unlock_irqrestore+0x8/0x2c (rt_up_read+0x43/0x4f) <...>-5554 1.... 113us+: rt_mutex_unlock+0xb/0x30 (rt_up_read+0x4b/0x4f) <...>-5554 1.... 117us > sys_prctl+0x0/0x1b2 (00000000 00000000 000000d8) <...>-5554 1.... 118us : do_syscall_trace+0xe/0x1d6 (syscall_trace_entry+0x11/0x30) <...>-5554 1.... 118us : audit_syscall_entry+0xe/0x147 (do_syscall_trace+0x182/0x1d6) <...>-5554 1.... 118us : current_kernel_time+0xb/0x3e (audit_syscall_entry+0x11d/0x147) <...>-5554 1.... 118us : sys_prctl+0xe/0x1b2 (syscall_call+0x7/0xb) <...>-5554 1.... 119us : user_trace_stop+0xe/0x3c4 (sys_prctl+0x2b/0x1b2) <...>-5554 1D... 119us : user_trace_stop+0x3f/0x3c4 (sys_prctl+0x2b/0x1b2)