We're still losing machines. I was able to get symbols loaded and a stack trace this time. It looks like a self-heal causes some kind of issue. Stack trace follows:
Stacktrace:
#0 0x00007f4812d945cc in ?? ()
No symbol table info available.
#1 0x00007f481216dd91 in glfs_io_async_cbk (ret=<optimized out>, frame=<optimized out>, data=0x7f481400f500) at glfs-fops.c:570
gio = 0x7f481400f500
#2 0x00007f4811f237fa in synctask_wrap (old_task=<optimized out>) at syncop.c:295
task = 0x7f48140f8a40
#3 0x00007f480ca1a7a0 in ?? () from /lib/x86_64-linux-gnu/libc.so.6
No symbol table info available.
#4 0x0000000000000000 in ?? ()
No symbol table info available.
StacktraceAddressSignature: /usr/bin/qemu-system-x86_64:11:x86_64:/usr/bin/qemu-system-x86_64+38c5cc:/usr/lib/x86_64-linux-gnu/libgfapi.so.0.0.0+8d91:/usr/lib/x86_64-linux-gnu/libglusterfs.so.0.0.0+477fa:/lib/x86_64-linux-gnu/libc-2.19.so+497a0
StacktraceSource:
#0 0x00007f4812d945cc in ?? ()
#1 0x00007f481216dd91 in glfs_io_async_cbk (ret=<optimized out>, frame=<optimized out>, data=0x7f481400f500) at glfs-fops.c:570
[Error: glfs-fops.c was not found in source tree]
#2 0x00007f4811f237fa in synctask_wrap (old_task=<optimized out>) at syncop.c:295
[Error: syncop.c was not found in source tree]
#3 0x00007f480ca1a7a0 in ?? () from /lib/x86_64-linux-gnu/libc.so.6
#4 0x0000000000000000 in ?? ()
StacktraceTop:
?? ()
glfs_io_async_cbk (ret=<optimized out>, frame=<optimized out>, data=0x7f481400f500) at glfs-fops.c:570
synctask_wrap (old_task=<optimized out>) at syncop.c:295
?? () from /lib/x86_64-linux-gnu/libc.so.6
?? ()
ThreadStacktrace:
.
Thread 28 (Thread 0x7f48129cc980 (LWP 51477)):
#0 0x00007f480cabec6f in __GI_ppoll (fds=0x7f48140fb800, nfds=6, timeout=<optimized out>, sigmask=0x0) at ../sysdeps/unix/sysv/linux/ppoll.c:56
resultvar = 18446744073709551102
oldtype = 0
tval = {tv_sec = 0, tv_nsec = 21949885}
result = <optimized out>
#1 0x00007f4812c39a69 in ?? ()
No symbol table info available.
#2 0x00007f4812bff754 in ?? ()
No symbol table info available.
#3 0x00007f4812aa6f36 in ?? ()
No symbol table info available.
#4 0x00007f480c9f2ec5 in __libc_start_main (main=0x7f4812aa59a0, argc=47, argv=0x7fff9babe848, init=<optimized out>, fini=<optimized out>, rtld_fini=<optimized out>, stack_end=0x7fff9babe838) at libc-start.c:287
result = <optimized out>
unwind_buf = {cancel_jmp_buf = {{jmp_buf = {0, 3324576305369744029, 139947527550051, 140735805122624, 0, 0, -3324655024350526819, -3408277451040324963}, mask_was_saved = 0}}, priv = {pad = {0x0, 0x0, 0x7fff9babe9c8, 0x7f4812a071c8}, data = {prev = 0x0, cleanup = 0x0, canceltype = -1683232312}}}
not_first_call = <optimized out>
#5 0x00007f4812aab48c in ?? ()
No symbol table info available.
.
Thread 27 (Thread 0x7f45b77fe700 (LWP 65826)):
#0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:238
No locals.
#1 0x00007f4811f251ef in syncenv_task (proc=proc@entry=0x7f4813fd1bb0) at syncop.c:493
env = 0x7f4813fcf270
task = 0x0
sleep_till = {tv_sec = 1425423008, tv_nsec = 0}
ret = <optimized out>
#2 0x00007f4811f25d80 in syncenv_processor (thdata=0x7f4813fd1bb0) at syncop.c:571
env = 0x7f4813fcf270
proc = 0x7f4813fd1bb0
task = <optimized out>
#3 0x00007f480cd9f182 in start_thread (arg=0x7f45b77fe700) at pthread_create.c:312
__res = <optimized out>
pd = 0x7f45b77fe700
now = <optimized out>
unwind_buf = {cancel_jmp_buf = {{jmp_buf = {139937408083712, 3324576305369744029, 0, 0, 139937408084416, 139937408083712, -3411430028351733091, -3408277985329642851}, mask_was_saved = 0}}, priv = {pad = {0x0, 0x0, 0x0, 0x0}, data = {prev = 0x0, cleanup = 0x0, canceltype = 0}}}
not_first_call = <optimized out>
pagesize_m1 = <optimized out>
sp = <optimized out>
freesize = <optimized out>
__PRETTY_FUNCTION__ = "start_thread"
#4 0x00007f480cacbefd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111
No locals.
.
Thread 26 (Thread 0x7f45d1ffb700 (LWP 58664)):
#0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:238
No locals.
#1 0x00007f4811f251ef in syncenv_task (proc=proc@entry=0x7f4813fd0cb0) at syncop.c:493
env = 0x7f4813fcf270
task = 0x0
sleep_till = {tv_sec = 1425423008, tv_nsec = 0}
ret = <optimized out>
#2 0x00007f4811f25d80 in syncenv_processor (thdata=0x7f4813fd0cb0) at syncop.c:571
env = 0x7f4813fcf270
proc = 0x7f4813fd0cb0
task = <optimized out>
#3 0x00007f480cd9f182 in start_thread (arg=0x7f45d1ffb700) at pthread_create.c:312
__res = <optimized out>
pd = 0x7f45d1ffb700
now = <optimized out>
unwind_buf = {cancel_jmp_buf = {{jmp_buf = {139937852667648, 3324576305369744029, 0, 0, 139937852668352, 139937852667648, -3411222222264696163, -3408277985329642851}, mask_was_saved = 0}}, priv = {pad = {0x0, 0x0, 0x0, 0x0}, data = {prev = 0x0, cleanup = 0x0, canceltype = 0}}}
not_first_call = <optimized out>
pagesize_m1 = <optimized out>
sp = <optimized out>
freesize = <optimized out>
__PRETTY_FUNCTION__ = "start_thread"
#4 0x00007f480cacbefd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111
No locals.
.
Thread 25 (Thread 0x7f45d2ffd700 (LWP 55067)):
#0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:238
No locals.
#1 0x00007f4811f251ef in syncenv_task (proc=proc@entry=0x7f4813fd0530) at syncop.c:493
env = 0x7f4813fcf270
task = 0x0
sleep_till = {tv_sec = 1425423008, tv_nsec = 0}
ret = <optimized out>
#2 0x00007f4811f25d80 in syncenv_processor (thdata=0x7f4813fd0530) at syncop.c:571
env = 0x7f4813fcf270
proc = 0x7f4813fd0530
task = <optimized out>
#3 0x00007f480cd9f182 in start_thread (arg=0x7f45d2ffd700) at pthread_create.c:312
__res = <optimized out>
pd = 0x7f45d2ffd700
now = <optimized out>
unwind_buf = {cancel_jmp_buf = {{jmp_buf = {139937869453056, 3324576305369744029, 0, 0, 139937869453760, 139937869453056, -3411228818260720995, -3408277985329642851}, mask_was_saved = 0}}, priv = {pad = {0x0, 0x0, 0x0, 0x0}, data = {prev = 0x0, cleanup = 0x0, canceltype = 0}}}
not_first_call = <optimized out>
pagesize_m1 = <optimized out>
sp = <optimized out>
freesize = <optimized out>
__PRETTY_FUNCTION__ = "start_thread"
#4 0x00007f480cacbefd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111
No locals.
.
Thread 24 (Thread 0x7f45b57fa700 (LWP 93253)):
#0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:238
No locals.
#1 0x00007f4811f251ef in syncenv_task (proc=proc@entry=0x7f4813fd2ab0) at syncop.c:493
env = 0x7f4813fcf270
task = 0x0
sleep_till = {tv_sec = 1425423008, tv_nsec = 0}
ret = <optimized out>
#2 0x00007f4811f25d80 in syncenv_processor (thdata=0x7f4813fd2ab0) at syncop.c:571
env = 0x7f4813fcf270
proc = 0x7f4813fd2ab0
task = <optimized out>
#3 0x00007f480cd9f182 in start_thread (arg=0x7f45b57fa700) at pthread_create.c:312
__res = <optimized out>
pd = 0x7f45b57fa700
now = <optimized out>
unwind_buf = {cancel_jmp_buf = {{jmp_buf = {139937374512896, 3324576305369744029, 0, 0, 139937374513600, 139937374512896, -3411425632452705635, -3408277985329642851}, mask_was_saved = 0}}, priv = {pad = {0x0, 0x0, 0x0, 0x0}, data = {prev = 0x0, cleanup = 0x0, canceltype = 0}}}
not_first_call = <optimized out>
pagesize_m1 = <optimized out>
sp = <optimized out>
freesize = <optimized out>
__PRETTY_FUNCTION__ = "start_thread"
#4 0x00007f480cacbefd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111
No locals.
.
Thread 23 (Thread 0x7f45b7fff700 (LWP 62731)):
#0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:238
No locals.
#1 0x00007f4811f251ef in syncenv_task (proc=proc@entry=0x7f4813fd17f0) at syncop.c:493
env = 0x7f4813fcf270
task = 0x0
sleep_till = {tv_sec = 1425423008, tv_nsec = 0}
ret = <optimized out>
#2 0x00007f4811f25d80 in syncenv_processor (thdata=0x7f4813fd17f0) at syncop.c:571
env = 0x7f4813fcf270
proc = 0x7f4813fd17f0
task = <optimized out>
#3 0x00007f480cd9f182 in start_thread (arg=0x7f45b7fff700) at pthread_create.c:312
__res = <optimized out>
pd = 0x7f45b7fff700
now = <optimized out>
unwind_buf = {cancel_jmp_buf = {{jmp_buf = {139937416476416, 3324576305369744029, 0, 0, 139937416477120, 139937416476416, -3411428928303234403, -3408277985329642851}, mask_was_saved = 0}}, priv = {pad = {0x0, 0x0, 0x0, 0x0}, data = {prev = 0x0, cleanup = 0x0, canceltype = 0}}}
not_first_call = <optimized out>
pagesize_m1 = <optimized out>
sp = <optimized out>
freesize = <optimized out>
__PRETTY_FUNCTION__ = "start_thread"
#4 0x00007f480cacbefd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111
No locals.
.
Thread 22 (Thread 0x7f45b5ffb700 (LWP 93252)):
#0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:238
No locals.
#1 0x00007f4811f251ef in syncenv_task (proc=proc@entry=0x7f4813fd26f0) at syncop.c:493
env = 0x7f4813fcf270
task = 0x0
sleep_till = {tv_sec = 1425423008, tv_nsec = 0}
ret = <optimized out>
#2 0x00007f4811f25d80 in syncenv_processor (thdata=0x7f4813fd26f0) at syncop.c:571
env = 0x7f4813fcf270
proc = 0x7f4813fd26f0
task = <optimized out>
#3 0x00007f480cd9f182 in start_thread (arg=0x7f45b5ffb700) at pthread_create.c:312
__res = <optimized out>
pd = 0x7f45b5ffb700
now = <optimized out>
unwind_buf = {cancel_jmp_buf = {{jmp_buf = {139937382905600, 3324576305369744029, 0, 0, 139937382906304, 139937382905600, -3411424532404206947, -3408277985329642851}, mask_was_saved = 0}}, priv = {pad = {0x0, 0x0, 0x0, 0x0}, data = {prev = 0x0, cleanup = 0x0, canceltype = 0}}}
not_first_call = <optimized out>
pagesize_m1 = <optimized out>
sp = <optimized out>
freesize = <optimized out>
__PRETTY_FUNCTION__ = "start_thread"
#4 0x00007f480cacbefd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111
No locals.
.
Thread 21 (Thread 0x7f45d0ff9700 (LWP 62730)):
#0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:238
No locals.
#1 0x00007f4811f251ef in syncenv_task (proc=proc@entry=0x7f4813fd1430) at syncop.c:493
env = 0x7f4813fcf270
task = 0x0
sleep_till = {tv_sec = 1425423008, tv_nsec = 0}
ret = <optimized out>
#2 0x00007f4811f25d80 in syncenv_processor (thdata=0x7f4813fd1430) at syncop.c:571
env = 0x7f4813fcf270
proc = 0x7f4813fd1430
task = <optimized out>
#3 0x00007f480cd9f182 in start_thread (arg=0x7f45d0ff9700) at pthread_create.c:312
__res = <optimized out>
pd = 0x7f45d0ff9700
now = <optimized out>
unwind_buf = {cancel_jmp_buf = {{jmp_buf = {139937835882240, 3324576305369744029, 0, 0, 139937835882944, 139937835882240, -3411224422361693539, -3408277985329642851}, mask_was_saved = 0}}, priv = {pad = {0x0, 0x0, 0x0, 0x0}, data = {prev = 0x0, cleanup = 0x0, canceltype = 0}}}
not_first_call = <optimized out>
pagesize_m1 = <optimized out>
sp = <optimized out>
freesize = <optimized out>
__PRETTY_FUNCTION__ = "start_thread"
#4 0x00007f480cacbefd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111
No locals.
.
Thread 20 (Thread 0x7f45d17fa700 (LWP 58665)):
#0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:238
No locals.
#1 0x00007f4811f251ef in syncenv_task (proc=proc@entry=0x7f4813fd1070) at syncop.c:493
env = 0x7f4813fcf270
task = 0x0
sleep_till = {tv_sec = 1425423008, tv_nsec = 0}
ret = <optimized out>
#2 0x00007f4811f25d80 in syncenv_processor (thdata=0x7f4813fd1070) at syncop.c:571
env = 0x7f4813fcf270
proc = 0x7f4813fd1070
task = <optimized out>
#3 0x00007f480cd9f182 in start_thread (arg=0x7f45d17fa700) at pthread_create.c:312
__res = <optimized out>
pd = 0x7f45d17fa700
now = <optimized out>
unwind_buf = {cancel_jmp_buf = {{jmp_buf = {139937844274944, 3324576305369744029, 0, 0, 139937844275648, 139937844274944, -3411223322313194851, -3408277985329642851}, mask_was_saved = 0}}, priv = {pad = {0x0, 0x0, 0x0, 0x0}, data = {prev = 0x0, cleanup = 0x0, canceltype = 0}}}
not_first_call = <optimized out>
pagesize_m1 = <optimized out>
sp = <optimized out>
freesize = <optimized out>
__PRETTY_FUNCTION__ = "start_thread"
#4 0x00007f480cacbefd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111
No locals.
.
Thread 19 (Thread 0x7f47d8e2e700 (LWP 51499)):
#0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:238
No locals.
#1 0x00007f4811f251ef in syncenv_task (proc=proc@entry=0x7f4813fcf9f0) at syncop.c:493
env = 0x7f4813fcf270
task = 0x0
sleep_till = {tv_sec = 1425423008, tv_nsec = 0}
ret = <optimized out>
#2 0x00007f4811f25d80 in syncenv_processor (thdata=0x7f4813fcf9f0) at syncop.c:571
env = 0x7f4813fcf270
proc = 0x7f4813fcf9f0
task = <optimized out>
#3 0x00007f480cd9f182 in start_thread (arg=0x7f47d8e2e700) at pthread_create.c:312
__res = <optimized out>
pd = 0x7f47d8e2e700
now = <optimized out>
unwind_buf = {cancel_jmp_buf = {{jmp_buf = {139946558154496, 3324576305369744029, 0, 0, 139946558155200, 139946558154496, -3410081107973078371, -3408277985329642851}, mask_was_saved = 0}}, priv = {pad = {0x0, 0x0, 0x0, 0x0}, data = {prev = 0x0, cleanup = 0x0, canceltype = 0}}}
not_first_call = <optimized out>
pagesize_m1 = <optimized out>
sp = <optimized out>
freesize = <optimized out>
__PRETTY_FUNCTION__ = "start_thread"
#4 0x00007f480cacbefd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111
No locals.
.
Thread 18 (Thread 0x7f4801c70700 (LWP 51479)):
#0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:238
No locals.
#1 0x00007f4811f251ef in syncenv_task (proc=proc@entry=0x7f4813fcf270) at syncop.c:493
env = 0x7f4813fcf270
task = 0x0
sleep_till = {tv_sec = 1425423008, tv_nsec = 0}
ret = <optimized out>
#2 0x00007f4811f25d80 in syncenv_processor (thdata=0x7f4813fcf270) at syncop.c:571
env = 0x7f4813fcf270
proc = 0x7f4813fcf270
task = <optimized out>
#3 0x00007f480cd9f182 in start_thread (arg=0x7f4801c70700) at pthread_create.c:312
__res = <optimized out>
pd = 0x7f4801c70700
now = <optimized out>
unwind_buf = {cancel_jmp_buf = {{jmp_buf = {139947244193536, 3324576305369744029, 0, 0, 139947244194240, 139947244193536, -3408302406323240291, -3408277985329642851}, mask_was_saved = 0}}, priv = {pad = {0x0, 0x0, 0x0, 0x0}, data = {prev = 0x0, cleanup = 0x0, canceltype = 0}}}
not_first_call = <optimized out>
pagesize_m1 = <optimized out>
sp = <optimized out>
freesize = <optimized out>
__PRETTY_FUNCTION__ = "start_thread"
#4 0x00007f480cacbefd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111
No locals.
.
Thread 17 (Thread 0x7f45d3fff700 (LWP 55065)):
#0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:238
No locals.
#1 0x00007f4811f251ef in syncenv_task (proc=proc@entry=0x7f4813fcfdb0) at syncop.c:493
env = 0x7f4813fcf270
task = 0x0
sleep_till = {tv_sec = 1425423008, tv_nsec = 0}
ret = <optimized out>
#2 0x00007f4811f25d80 in syncenv_processor (thdata=0x7f4813fcfdb0) at syncop.c:571
env = 0x7f4813fcf270
proc = 0x7f4813fcfdb0
task = <optimized out>
#3 0x00007f480cd9f182 in start_thread (arg=0x7f45d3fff700) at pthread_create.c:312
__res = <optimized out>
pd = 0x7f45d3fff700
now = <optimized out>
unwind_buf = {cancel_jmp_buf = {{jmp_buf = {139937886238464, 3324576305369744029, 0, 0, 139937886239168, 139937886238464, -3411226618163723619, -3408277985329642851}, mask_was_saved = 0}}, priv = {pad = {0x0, 0x0, 0x0, 0x0}, data = {prev = 0x0, cleanup = 0x0, canceltype = 0}}}
not_first_call = <optimized out>
pagesize_m1 = <optimized out>
sp = <optimized out>
freesize = <optimized out>
__PRETTY_FUNCTION__ = "start_thread"
#4 0x00007f480cacbefd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111
No locals.
.
Thread 16 (Thread 0x7f47ff7c2700 (LWP 51482)):
#0 0x00007f480cda6b9d in nanosleep () at ../sysdeps/unix/syscall-template.S:81
No locals.
#1 0x00007f4811f05f74 in gf_timer_proc (ctx=0x7f4813fb34e0) at timer.c:170
now = 11422000809582053
now_ts = {tv_sec = 11422000, tv_nsec = 809582053}
event = 0x7f47f80960c0
reg = 0x7f4813fd8750
sleepts = {tv_sec = 1, tv_nsec = 0}
__FUNCTION__ = "gf_timer_proc"
#2 0x00007f480cd9f182 in start_thread (arg=0x7f47ff7c2700) at pthread_create.c:312
__res = <optimized out>
pd = 0x7f47ff7c2700
now = <optimized out>
unwind_buf = {cancel_jmp_buf = {{jmp_buf = {139947205732096, 3324576305369744029, 0, 0, 139947205732800, 139947205732096, -3410145822392810851, -3408277985329642851}, mask_was_saved = 0}}, priv = {pad = {0x0, 0x0, 0x0, 0x0}, data = {prev = 0x0, cleanup = 0x0, canceltype = 0}}}
not_first_call = <optimized out>
pagesize_m1 = <optimized out>
sp = <optimized out>
freesize = <optimized out>
__PRETTY_FUNCTION__ = "start_thread"
#3 0x00007f480cacbefd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111
No locals.
.
Thread 15 (Thread 0x7f45b67fc700 (LWP 14268)):
#0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:238
No locals.
#1 0x00007f4811f251ef in syncenv_task (proc=proc@entry=0x7f4813fd2330) at syncop.c:493
env = 0x7f4813fcf270
task = 0x0
sleep_till = {tv_sec = 1425423008, tv_nsec = 0}
ret = <optimized out>
#2 0x00007f4811f25d80 in syncenv_processor (thdata=0x7f4813fd2330) at syncop.c:571
env = 0x7f4813fcf270
proc = 0x7f4813fd2330
task = <optimized out>
#3 0x00007f480cd9f182 in start_thread (arg=0x7f45b67fc700) at pthread_create.c:312
__res = <optimized out>
pd = 0x7f45b67fc700
now = <optimized out>
unwind_buf = {cancel_jmp_buf = {{jmp_buf = {139937391298304, 3324576305369744029, 0, 0, 139937391299008, 139937391298304, -3411432228448730467, -3408277985329642851}, mask_was_saved = 0}}, priv = {pad = {0x0, 0x0, 0x0, 0x0}, data = {prev = 0x0, cleanup = 0x0, canceltype = 0}}}
not_first_call = <optimized out>
pagesize_m1 = <optimized out>
sp = <optimized out>
freesize = <optimized out>
__PRETTY_FUNCTION__ = "start_thread"
#4 0x00007f480cacbefd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111
No locals.
.
Thread 14 (Thread 0x7f45d27fc700 (LWP 55068)):
#0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:238
No locals.
#1 0x00007f4811f251ef in syncenv_task (proc=proc@entry=0x7f4813fd08f0) at syncop.c:493
env = 0x7f4813fcf270
task = 0x0
sleep_till = {tv_sec = 1425423008, tv_nsec = 0}
ret = <optimized out>
#2 0x00007f4811f25d80 in syncenv_processor (thdata=0x7f4813fd08f0) at syncop.c:571
env = 0x7f4813fcf270
proc = 0x7f4813fd08f0
task = <optimized out>
#3 0x00007f480cd9f182 in start_thread (arg=0x7f45d27fc700) at pthread_create.c:312
__res = <optimized out>
pd = 0x7f45d27fc700
now = <optimized out>
unwind_buf = {cancel_jmp_buf = {{jmp_buf = {139937861060352, 3324576305369744029, 0, 0, 139937861061056, 139937861060352, -3411229918309219683, -3408277985329642851}, mask_was_saved = 0}}, priv = {pad = {0x0, 0x0, 0x0, 0x0}, data = {prev = 0x0, cleanup = 0x0, canceltype = 0}}}
not_first_call = <optimized out>
pagesize_m1 = <optimized out>
sp = <optimized out>
freesize = <optimized out>
__PRETTY_FUNCTION__ = "start_thread"
#4 0x00007f480cacbefd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111
No locals.
.
Thread 13 (Thread 0x7f45d37fe700 (LWP 55066)):
#0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:238
No locals.
#1 0x00007f4811f251ef in syncenv_task (proc=proc@entry=0x7f4813fd0170) at syncop.c:493
env = 0x7f4813fcf270
task = 0x0
sleep_till = {tv_sec = 1425423008, tv_nsec = 0}
ret = <optimized out>
#2 0x00007f4811f25d80 in syncenv_processor (thdata=0x7f4813fd0170) at syncop.c:571
env = 0x7f4813fcf270
proc = 0x7f4813fd0170
task = <optimized out>
#3 0x00007f480cd9f182 in start_thread (arg=0x7f45d37fe700) at pthread_create.c:312
__res = <optimized out>
pd = 0x7f45d37fe700
now = <optimized out>
unwind_buf = {cancel_jmp_buf = {{jmp_buf = {139937877845760, 3324576305369744029, 0, 0, 139937877846464, 139937877845760, -3411227718212222307, -3408277985329642851}, mask_was_saved = 0}}, priv = {pad = {0x0, 0x0, 0x0, 0x0}, data = {prev = 0x0, cleanup = 0x0, canceltype = 0}}}
not_first_call = <optimized out>
pagesize_m1 = <optimized out>
sp = <optimized out>
freesize = <optimized out>
__PRETTY_FUNCTION__ = "start_thread"
#4 0x00007f480cacbefd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111
No locals.
.
Thread 12 (Thread 0x7f47da5ff700 (LWP 51492)):
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
No locals.
#1 0x00007f4812d93e19 in ?? ()
No symbol table info available.
#2 0x00007f4812c73bb3 in ?? ()
No symbol table info available.
#3 0x00007f4812c73fb0 in ?? ()
No symbol table info available.
#4 0x00007f480cd9f182 in start_thread (arg=0x7f47da5ff700) at pthread_create.c:312
__res = <optimized out>
pd = 0x7f47da5ff700
now = <optimized out>
unwind_buf = {cancel_jmp_buf = {{jmp_buf = {139946583127808, 3324576305369744029, 0, 0, 139946583128512, 139946583127808, -3410086699483626851, -3408277985329642851}, mask_was_saved = 0}}, priv = {pad = {0x0, 0x0, 0x0, 0x0}, data = {prev = 0x0, cleanup = 0x0, canceltype = 0}}}
not_first_call = <optimized out>
pagesize_m1 = <optimized out>
sp = <optimized out>
freesize = <optimized out>
__PRETTY_FUNCTION__ = "start_thread"
#5 0x00007f480cacbefd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111
No locals.
.
Thread 11 (Thread 0x7f480146f700 (LWP 51480)):
#0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:238
No locals.
#1 0x00007f4811f251ef in syncenv_task (proc=proc@entry=0x7f4813fcf630) at syncop.c:493
env = 0x7f4813fcf270
task = 0x0
sleep_till = {tv_sec = 1425423008, tv_nsec = 0}
ret = <optimized out>
#2 0x00007f4811f25d80 in syncenv_processor (thdata=0x7f4813fcf630) at syncop.c:571
env = 0x7f4813fcf270
proc = 0x7f4813fcf630
task = <optimized out>
#3 0x00007f480cd9f182 in start_thread (arg=0x7f480146f700) at pthread_create.c:312
__res = <optimized out>
pd = 0x7f480146f700
now = <optimized out>
unwind_buf = {cancel_jmp_buf = {{jmp_buf = {139947235800832, 3324576305369744029, 0, 0, 139947235801536, 139947235800832, -3408303506371738979, -3408277985329642851}, mask_was_saved = 0}}, priv = {pad = {0x0, 0x0, 0x0, 0x0}, data = {prev = 0x0, cleanup = 0x0, canceltype = 0}}}
not_first_call = <optimized out>
pagesize_m1 = <optimized out>
sp = <optimized out>
freesize = <optimized out>
__PRETTY_FUNCTION__ = "start_thread"
#4 0x00007f480cacbefd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111
No locals.
.
Thread 10 (Thread 0x7f47f5d7b700 (LWP 51486)):
#0 0x00007f480cac2db7 in ioctl () at ../sysdeps/unix/syscall-template.S:81
No locals.
#1 0x00007f4812ce55e4 in ?? ()
No symbol table info available.
#2 0x00007f4812ce56c4 in ?? ()
No symbol table info available.
#3 0x00007f4812c85df2 in ?? ()
No symbol table info available.
#4 0x00007f480cd9f182 in start_thread (arg=0x7f47f5d7b700) at pthread_create.c:312
__res = <optimized out>
pd = 0x7f47f5d7b700
now = <optimized out>
unwind_buf = {cancel_jmp_buf = {{jmp_buf = {139947043960576, 3324576305369744029, 0, 0, 139947043961280, 139947043960576, -3410158238606392675, -3408277985329642851}, mask_was_saved = 0}}, priv = {pad = {0x0, 0x0, 0x0, 0x0}, data = {prev = 0x0, cleanup = 0x0, canceltype = 0}}}
not_first_call = <optimized out>
pagesize_m1 = <optimized out>
sp = <optimized out>
freesize = <optimized out>
__PRETTY_FUNCTION__ = "start_thread"
#5 0x00007f480cacbefd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111
No locals.
.
Thread 9 (Thread 0x7f47f6d7d700 (LWP 51484)):
#0 0x00007f480cac2db7 in ioctl () at ../sysdeps/unix/syscall-template.S:81
No locals.
#1 0x00007f4812ce55e4 in ?? ()
No symbol table info available.
#2 0x00007f4812ce56c4 in ?? ()
No symbol table info available.
#3 0x00007f4812c85df2 in ?? ()
No symbol table info available.
#4 0x00007f480cd9f182 in start_thread (arg=0x7f47f6d7d700) at pthread_create.c:312
__res = <optimized out>
pd = 0x7f47f6d7d700
now = <optimized out>
unwind_buf = {cancel_jmp_buf = {{jmp_buf = {139947060745984, 3324576305369744029, 0, 0, 139947060746688, 139947060745984, -3410164834602417507, -3408277985329642851}, mask_was_saved = 0}}, priv = {pad = {0x0, 0x0, 0x0, 0x0}, data = {prev = 0x0, cleanup = 0x0, canceltype = 0}}}
not_first_call = <optimized out>
pagesize_m1 = <optimized out>
sp = <optimized out>
freesize = <optimized out>
__PRETTY_FUNCTION__ = "start_thread"
#5 0x00007f480cacbefd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111
No locals.
.
Thread 8 (Thread 0x7f47db7fe700 (LWP 51490)):
#0 0x00007f480cac2db7 in ioctl () at ../sysdeps/unix/syscall-template.S:81
No locals.
#1 0x00007f4812ce55e4 in ?? ()
No symbol table info available.
#2 0x00007f4812ce56c4 in ?? ()
No symbol table info available.
#3 0x00007f4812c85df2 in ?? ()
No symbol table info available.
#4 0x00007f480cd9f182 in start_thread (arg=0x7f47db7fe700) at pthread_create.c:312
__res = <optimized out>
pd = 0x7f47db7fe700
now = <optimized out>
unwind_buf = {cancel_jmp_buf = {{jmp_buf = {139946601998080, 3324576305369744029, 0, 0, 139946601998784, 139946601998080, -3410084226119335267, -3408277985329642851}, mask_was_saved = 0}}, priv = {pad = {0x0, 0x0, 0x0, 0x0}, data = {prev = 0x0, cleanup = 0x0, canceltype = 0}}}
not_first_call = <optimized out>
pagesize_m1 = <optimized out>
sp = <optimized out>
freesize = <optimized out>
__PRETTY_FUNCTION__ = "start_thread"
#5 0x00007f480cacbefd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111
No locals.
.
Thread 7 (Thread 0x7f47dbfff700 (LWP 51489)):
#0 0x00007f480cac2db7 in ioctl () at ../sysdeps/unix/syscall-template.S:81
No locals.
#1 0x00007f4812ce55e4 in ?? ()
No symbol table info available.
#2 0x00007f4812ce56c4 in ?? ()
No symbol table info available.
#3 0x00007f4812c85df2 in ?? ()
No symbol table info available.
#4 0x00007f480cd9f182 in start_thread (arg=0x7f47dbfff700) at pthread_create.c:312
__res = <optimized out>
pd = 0x7f47dbfff700
now = <optimized out>
unwind_buf = {cancel_jmp_buf = {{jmp_buf = {139946610390784, 3324576305369744029, 0, 0, 139946610391488, 139946610390784, -3410083126070836579, -3408277985329642851}, mask_was_saved = 0}}, priv = {pad = {0x0, 0x0, 0x0, 0x0}, data = {prev = 0x0, cleanup = 0x0, canceltype = 0}}}
not_first_call = <optimized out>
pagesize_m1 = <optimized out>
sp = <optimized out>
freesize = <optimized out>
__PRETTY_FUNCTION__ = "start_thread"
#5 0x00007f480cacbefd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111
No locals.
.
Thread 6 (Thread 0x7f47f4d79700 (LWP 51488)):
#0 0x00007f480cac2db7 in ioctl () at ../sysdeps/unix/syscall-template.S:81
No locals.
#1 0x00007f4812ce55e4 in ?? ()
No symbol table info available.
#2 0x00007f4812ce56c4 in ?? ()
No symbol table info available.
#3 0x00007f4812c85df2 in ?? ()
No symbol table info available.
#4 0x00007f480cd9f182 in start_thread (arg=0x7f47f4d79700) at pthread_create.c:312
__res = <optimized out>
pd = 0x7f47f4d79700
now = <optimized out>
unwind_buf = {cancel_jmp_buf = {{jmp_buf = {139947027175168, 3324576305369744029, 0, 0, 139947027175872, 139947027175168, -3410160438703390051, -3408277985329642851}, mask_was_saved = 0}}, priv = {pad = {0x0, 0x0, 0x0, 0x0}, data = {prev = 0x0, cleanup = 0x0, canceltype = 0}}}
not_first_call = <optimized out>
pagesize_m1 = <optimized out>
sp = <optimized out>
freesize = <optimized out>
__PRETTY_FUNCTION__ = "start_thread"
#5 0x00007f480cacbefd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111
No locals.
.
Thread 5 (Thread 0x7f47f657c700 (LWP 51485)):
#0 0x00007f480cac2db7 in ioctl () at ../sysdeps/unix/syscall-template.S:81
No locals.
#1 0x00007f4812ce55e4 in ?? ()
No symbol table info available.
#2 0x00007f4812ce56c4 in ?? ()
No symbol table info available.
#3 0x00007f4812c85df2 in ?? ()
No symbol table info available.
#4 0x00007f480cd9f182 in start_thread (arg=0x7f47f657c700) at pthread_create.c:312
__res = <optimized out>
pd = 0x7f47f657c700
now = <optimized out>
unwind_buf = {cancel_jmp_buf = {{jmp_buf = {139947052353280, 3324576305369744029, 0, 0, 139947052353984, 139947052353280, -3410165934650916195, -3408277985329642851}, mask_was_saved = 0}}, priv = {pad = {0x0, 0x0, 0x0, 0x0}, data = {prev = 0x0, cleanup = 0x0, canceltype = 0}}}
not_first_call = <optimized out>
pagesize_m1 = <optimized out>
sp = <optimized out>
freesize = <optimized out>
__PRETTY_FUNCTION__ = "start_thread"
#5 0x00007f480cacbefd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111
No locals.
.
Thread 4 (Thread 0x7f47f557a700 (LWP 51487)):
#0 0x00007f480cac2db7 in ioctl () at ../sysdeps/unix/syscall-template.S:81
No locals.
#1 0x00007f4812ce55e4 in ?? ()
No symbol table info available.
#2 0x00007f4812ce56c4 in ?? ()
No symbol table info available.
#3 0x00007f4812c85df2 in ?? ()
No symbol table info available.
#4 0x00007f480cd9f182 in start_thread (arg=0x7f47f557a700) at pthread_create.c:312
__res = <optimized out>
pd = 0x7f47f557a700
now = <optimized out>
unwind_buf = {cancel_jmp_buf = {{jmp_buf = {139947035567872, 3324576305369744029, 0, 0, 139947035568576, 139947035567872, -3410159338654891363, -3408277985329642851}, mask_was_saved = 0}}, priv = {pad = {0x0, 0x0, 0x0, 0x0}, data = {prev = 0x0, cleanup = 0x0, canceltype = 0}}}
not_first_call = <optimized out>
pagesize_m1 = <optimized out>
sp = <optimized out>
freesize = <optimized out>
__PRETTY_FUNCTION__ = "start_thread"
#5 0x00007f480cacbefd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111
No locals.
.
Thread 3 (Thread 0x7f47f757e700 (LWP 51483)):
#0 0x00007f480cac2db7 in ioctl () at ../sysdeps/unix/syscall-template.S:81
No locals.
#1 0x00007f4812ce55e4 in ?? ()
No symbol table info available.
#2 0x00007f4812ce56c4 in ?? ()
No symbol table info available.
#3 0x00007f4812c85df2 in ?? ()
No symbol table info available.
#4 0x00007f480cd9f182 in start_thread (arg=0x7f47f757e700) at pthread_create.c:312
__res = <optimized out>
pd = 0x7f47f757e700
now = <optimized out>
unwind_buf = {cancel_jmp_buf = {{jmp_buf = {139947069138688, 3324576305369744029, 0, 0, 139947069139392, 139947069138688, -3410163734553918819, -3408277985329642851}, mask_was_saved = 0}}, priv = {pad = {0x0, 0x0, 0x0, 0x0}, data = {prev = 0x0, cleanup = 0x0, canceltype = 0}}}
not_first_call = <optimized out>
pagesize_m1 = <optimized out>
sp = <optimized out>
freesize = <optimized out>
__PRETTY_FUNCTION__ = "start_thread"
#5 0x00007f480cacbefd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111
No locals.
.
Thread 2 (Thread 0x7f48001d7700 (LWP 51481)):
#0 pthread_spin_lock () at ../nptl/sysdeps/x86_64/pthread_spin_lock.S:26
No locals.
#1 0x00007f4811f184f8 in iobuf_unref (iobuf=0x7f4813fbf770) at iobuf.c:735
ref = 0
#2 0x00007f47ff7ca949 in __socket_proto_state_machine (pollin=<synthetic pointer>, this=0x7f47f8034ed0) at socket.c:2046
count = <optimized out>
ret = <optimized out>
iobuf = <optimized out>
frag = 0x0
priv = 0x0
iobref = <optimized out>
vector = {{iov_base = 0x7f48129c3c00, iov_len = 36}, {iov_base = 0x0, iov_len = 0}}
in = 0x0
#3 socket_proto_state_machine (pollin=<synthetic pointer>, this=0x7f47f8034ed0) at socket.c:2099
priv = 0x7f47f8035a20
ret = 0
#4 socket_event_poll_in (this=this@entry=0x7f47f8034ed0) at socket.c:2115
ret = -1
pollin = 0x7f47f8001fe0
priv = 0x7f47f8035a20
#5 0x00007f47ff7cd1f4 in socket_event_handler (fd=<optimized out>, idx=2, data=data@entry=0x7f47f8034ed0, poll_in=1, poll_out=0, poll_err=0) at socket.c:2232
this = 0x7f47f8034ed0
priv = 0x7f47f8035a20
ret = 0
__FUNCTION__ = "socket_event_handler"
#6 0x00007f4811f3f26a in event_dispatch_epoll_handler (i=<optimized out>, events=0x7f47f80008e0, event_pool=0x7f4813fcf1d0) at event-epoll.c:384
data = 0x7f47f8034ed0
idx = <optimized out>
ret = -1
event_data = 0x7f47f80008f0
handler = 0x7f47ff7cd090 <socket_event_handler>
#7 event_dispatch_epoll (event_pool=0x7f4813fcf1d0) at event-epoll.c:445
events = 0x7f47f80008e0
i = <optimized out>
ret = <optimized out>
__FUNCTION__ = "event_dispatch_epoll"
#8 0x00007f481216b444 in glfs_poller (data=<optimized out>) at glfs.c:505
fs = <optimized out>
#9 0x00007f480cd9f182 in start_thread (arg=0x7f48001d7700) at pthread_create.c:312
__res = <optimized out>
pd = 0x7f48001d7700
now = <optimized out>
unwind_buf = {cancel_jmp_buf = {{jmp_buf = {139947216303872, 3324576305369744029, 0, 0, 139947216304576, 139947216303872, -3408304953775717731, -3408277985329642851}, mask_was_saved = 0}}, priv = {pad = {0x0, 0x0, 0x0, 0x0}, data = {prev = 0x0, cleanup = 0x0, canceltype = 0}}}
not_first_call = <optimized out>
pagesize_m1 = <optimized out>
sp = <optimized out>
freesize = <optimized out>
__PRETTY_FUNCTION__ = "start_thread"
#10 0x00007f480cacbefd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111
No locals.
.
Thread 1 (Thread 0x7f45b6ffd700 (LWP 14267)):
#0 0x00007f4812d945cc in ?? ()
No symbol table info available.
#1 0x00007f481216dd91 in glfs_io_async_cbk (ret=<optimized out>, frame=<optimized out>, data=0x7f481400f500) at glfs-fops.c:570
gio = 0x7f481400f500
#2 0x00007f4811f237fa in synctask_wrap (old_task=<optimized out>) at syncop.c:295
task = 0x7f48140f8a40
#3 0x00007f480ca1a7a0 in ?? () from /lib/x86_64-linux-gnu/libc.so.6
No symbol table info available.
#4 0x0000000000000000 in ?? ()
No symbol table info available.
Uname: Linux 3.13.0-37-generic x86_64
