### Libvirt version: ###
libvirt tagged v1.0.0 from git , with
f0e72b2f5c675f927d04545dc5095f9e5998f171 applied
### Problem: ###
Libvirtd segfaults
### Steps to reproduce: ###
This is difficult to reproduce; but appears to happen only when doing
'virsh destroys', and when they are ran in concurrent intervals, over
extended periods of time.
Here are the steps I have taken to reproduce this problem
(debian_6.0_amd64_kvm.img.gz is just a zipped up squeeze stock ISO
install):
First, copy this script to ease the steps (assumed to be at
/root/libvirt_crash.pl later):
#!/usr/bin/env perl
use strict;
use warnings;
use Getopt::Long qw(:config no_ignore_case);
use String::Random;
use UUID::Random;
my ($lvms,$sleep,$restarts_per_lvm);
GetOptions(
'lvms=i' => \$lvms,
'restarts_per_lvm=i' => \$restarts_per_lvm,
'sleep=i' => \$sleep,
);
die "USAGE:\n$0 [--lvms int | --restarts_per_lvm int | --sleep int
(optional) ]\n" unless ( $lvms && $restarts_per_lvm );
sub _create_lvm {
my $opts = shift;
print "\tlvcreate -n $opts->{lvm} -L 5G /dev/LVM\n";
system("lvcreate -n $opts->{lvm} -L 5G /dev/LVM");
}
sub _kpartx_lvm {
my $opts = shift;
die "_kpartx_lvm needs modifier!\n" unless ( $opts->{modifier} );
print "\tkpartx -p p -$opts->{modifier} -v
/dev/LVM/$opts->{lvm}\n";
system("kpartx -p p -$opts->{modifier} -v
/dev/LVM/$opts->{lvm}");
}
sub _remove_lvm {
my $opts = shift;
print "\tlvremove -f /dev/LVM/$opts->{lvm}\n";
system("lvremove -f /dev/LVM/$opts->{lvm}");
unlink("/xen/configs/$opts->{lvm}.cfg");
}
sub _display_lvm {
my $opts = shift;
print "\tparted -s /dev/LVM/$opts->{lvm} unit gb p\n";
system("parted -s /dev/LVM/$opts->{lvm} unit gb p");
}
sub _dd_template {
my $opts = shift;
print "\tdd if=/root/ssullivan/debian_6.0_amd64_kvm.img.gz
bs=1M | gunzip -c | dd of=/dev/LVM/$opts->{lvm} bs=1M\n";
system("dd if=/root/ssullivan/debian_6.0_amd64_kvm.img.gz
bs=1M | gunzip -c | dd of=/dev/LVM/$opts->{lvm} bs=1M");
}
sub _e2fsck_lvm {
my $opts = shift;
print "\te2fsck -y -f /dev/mapper/LVM-$opts->{lvm}p1\n";
system("e2fsck -y -f /dev/mapper/LVM-$opts->{lvm}p1");
print "\te2fsck -y -f /dev/mapper/LVM-$opts->{lvm}p3\n";
system("e2fsck -y -f /dev/mapper/LVM-$opts->{lvm}p3")
}
sub _place_virtconf {
my $opts = shift;
my $uuid = UUID::Random::generate;
my $virt_conf = <<END;
<domain type="kvm">
<name>$opts->{lvm}</name>
<uuid>$uuid</uuid>
<memory>450560</memory>
<currentMemory>450560</currentMemory>
<vcpu>2</vcpu>
<cpu>
<topology sockets="2" cores="4" threads="1"/>
</cpu>
<os>
<type arch="x86_64" machine="pc-1.1">hvm</type>
<boot dev="hd"/>
</os>
<features>
<acpi/>
<apic/>
<pae/>
</features>
<clock offset="localtime"/>
<on_poweroff>destroy</on_poweroff>
<on_reboot>restart</on_reboot>
<on_crash>restart</on_crash>
<devices>
<emulator>/opt/libexec/qemu-kvm-wrapper</emulator>
<disk type="block" device="disk">
<driver name="qemu" cache="none"/>
<source dev="/dev/LVM/$opts->{lvm}"/>
<target dev="vda" bus="virtio"/>
</disk>
<serial type="pty">
<target port="1"/>
</serial>
<console type="pty">
<target port="1"/>
</console>
<input type="tablet" bus="usb"/>
<input type="mouse" bus="ps2"/>
<graphics type="vnc" port="-1" autoport="yes" keymap="en-us"/>
</devices>
</domain>
END
open(my $fh, '>', "/xen/configs/$opts->{lvm}.cfg") || die
"Error opening [/xen/configs/$opts->{lvm}.cfg] [$!]";
print $fh "$virt_conf\n";
close($fh) || die "Error closing
[/xen/configs/$opts->{lvm}.cfg] [$!]";
}
sub _shutdown_lvm {
my $opts = shift;
print "\tvirsh destroy $opts->{lvm}\n";
system("virsh destroy $opts->{lvm}");
_virsh_list();
}
sub _virsh_list {
print "\tvirsh list\n";
system('virsh list');
}
sub _start_lvm {
my $opts = shift;
print "\tvirsh create /xen/configs/$opts->{lvm}.cfg\n";
system("virsh create /xen/configs/$opts->{lvm}.cfg");
_virsh_list();
}
my $rnd = new String::Random;
my $count = 0;
while ( $count < $lvms ) {
my $id = $rnd->randpattern("cccccccc");
print "####\nLVM PASS: $count\nLVM: $id\n####\n";
print "Creating LVM...\n";
_create_lvm({ lvm => $id });
print "DD'ing template...\n";
_dd_template({ lvm => $id });
print "kpartxA LVM...\n";
_kpartx_lvm({ lvm => $id, modifier => 'a' });
print "e2fsck LVM..\n";
_e2fsck_lvm({ lvm => $id });
print "kpartxD LVM...\n";
_kpartx_lvm({ lvm => $id, modifier => 'd' });
print "Placing libvirt conf...\n";
_place_virtconf({ lvm => $id });
print "Partition setup:\n";
_display_lvm({ lvm => $id });
print "Performing $restarts_per_lvm restarts on $id...\n";
my $restart_cnt = 0;
while ( $restart_cnt < $restarts_per_lvm ) {
print "####\nRESTART PASS: $restart_cnt\nLVM:
$id\n####\n";
print "Shutting down LVM...\n";
_shutdown_lvm({ lvm => $id });
sleep $sleep if ( $sleep );
print "Starting LVM...\n";
_start_lvm({ lvm => $id });
$restart_cnt++;
}
print "Removing LVM...\n";
_shutdown_lvm({ lvm => $id });
_remove_lvm({ lvm => $id });
$count++;
}
I then started 21 of these scripts, with the below commands
(preferably in a screen to reduce spamming):
perl /root/libvirt_crash.pl --lvms 1000 --restarts_per_lvm 2000 &
perl /root/libvirt_crash.pl --lvms 1000 --restarts_per_lvm 2000 &
perl /root/libvirt_crash.pl --lvms 1000 --restarts_per_lvm 2000 &
perl /root/libvirt_crash.pl --lvms 1000 --restarts_per_lvm 2000 &
perl /root/libvirt_crash.pl --lvms 1000 --restarts_per_lvm 2000
--sleep 10 &
perl /root/libvirt_crash.pl --lvms 1000 --restarts_per_lvm 2000
--sleep 10 &
perl /root/libvirt_crash.pl --lvms 1000 --restarts_per_lvm 2000 &
perl /root/libvirt_crash.pl --lvms 1000 --restarts_per_lvm 2000 &
perl /root/libvirt_crash.pl --lvms 1000 --restarts_per_lvm 2000 &
perl /root/libvirt_crash.pl --lvms 1000 --restarts_per_lvm 2000 &
perl /root/libvirt_crash.pl --lvms 6000 --restarts_per_lvm 30 &
perl /root/libvirt_crash.pl --lvms 6000 --restarts_per_lvm 30 &
perl /root/libvirt_crash.pl --lvms 6000 --restarts_per_lvm 30 &
perl /root/libvirt_crash.pl --lvms 6000 --restarts_per_lvm 30 &
perl /root/libvirt_crash.pl --lvms 6000 --restarts_per_lvm 30 &
perl /root/libvirt_crash.pl --lvms 6000 --restarts_per_lvm 30 &
perl /root/libvirt_crash.pl --lvms 6000 --restarts_per_lvm 30 &
perl /root/libvirt_crash.pl --lvms 6000 --restarts_per_lvm 30 &
perl /root/libvirt_crash.pl --lvms 6000 --restarts_per_lvm 30 &
perl /root/libvirt_crash.pl --lvms 6000 --restarts_per_lvm 30 &
perl /root/libvirt_crash.pl --lvms 6000 --restarts_per_lvm 30 &
I already had libvirtd started under GDB at this point. I waited about
3-6 hours, and I found this in my GDB session:
2012-12-27 17:23:35.922+0000: 7445: error : qemuMonitorIO:614 :
internal error End of file from monitor
2012-12-27 17:23:36.558+0000: 7445: error : qemuMonitorIO:614 :
internal error End of file from monitor
Program received signal SIGSEGV, Segmentation fault.
[Switching to Thread 0x7fff8e1fc700 (LWP 31142)]
qemuDomainObjBeginJobInternal (driver=0x7fffe401d740,
driver_locked=true, obj=0x7fff80001b00, job=QEMU_JOB_DESTROY,
asyncJob=QEMU_ASYNC_JOB_NONE) at qemu/qemu_domain.c:768
768 priv->jobs_queued++;
Missing separate debuginfos, use: debuginfo-install
audit-libs-2.1.3-3.el6.x86_64 augeas-libs-0.9.0-1.el6.x86_64
avahi-libs-0.6.25-11.el6.x86_64 cyrus-sasl-gssapi-2.1.23-13.el6.x86_64
cyrus-sasl-lib-2.1.23-13.el6.x86_64
cyrus-sasl-md5-2.1.23-13.el6.x86_64
cyrus-sasl-plain-2.1.23-13.el6.x86_64 db4-4.7.25-16.el6.x86_64
dbus-libs-1.2.24-5.el6_1.x86_64
device-mapper-libs-1.02.66-6.el6.x86_64 glibc-2.12-1.47.el6.x86_64
gnutls-2.8.5-4.el6.x86_64 keyutils-libs-1.4-3.el6.x86_64
krb5-libs-1.9-22.el6_2.1.x86_64 libblkid-2.17.2-12.4.el6.x86_64
libcap-ng-0.6.4-3.el6_0.1.x86_64 libcom_err-1.41.12-11.el6.x86_64
libcurl-7.19.7-26.el6_1.2.x86_64 libgcrypt-1.4.5-9.el6.x86_64
libgpg-error-1.7-4.el6.x86_64 libidn-1.18-2.el6.x86_64
libnl-1.1-14.el6.x86_64 libpcap-1.0.0-6.20091201git117cb5.el6.x86_64
libpciaccess-0.12.1-1.el6.x86_64 libselinux-2.0.94-5.2.el6.x86_64
libsepol-2.0.41-4.el6.x86_64 libtasn1-2.3-3.el6.x86_64
libudev-147-2.40.el6.x86_64 libuuid-2.17.2-12.4.el6.x86_64
libxml2-2.7.6-4.el6.x86_64 libxslt-1.1.26-2.el6.x86_64
netcf-libs-0.1.9-2.el6.x86_64 nspr-4.8.8-3.el6.x86_64
nss-3.12.10-17.el6_2.x86_64 nss-softokn-freebl-3.12.9-11.el6.x86_64
nss-util-3.12.10-2.el6.x86_64 numactl-2.0.3-9.el6.x86_64
openldap-2.4.23-20.el6.x86_64 openssl-1.0.0-20.el6.x86_64
yajl-1.0.7-3.el6.x86_64 zlib-1.2.3-27.el6.x86_64
(gdb)
Here is the output of 'thread apply all bt' from this session:
Thread 30 (Thread 0x7fff8e1fc700 (LWP 31142)):
#0 qemuDomainObjBeginJobInternal (driver=0x7fffe401d740,
driver_locked=true, obj=0x7fff80001b00, job=QEMU_JOB_DESTROY,
asyncJob=QEMU_ASYNC_JOB_NONE) at qemu/qemu_domain.c:768
#1 0x00007fffeac2b223 in qemuDomainDestroyFlags (dom=<value optimized
out>, flags=<value optimized out>) at qemu/qemu_driver.c:2052
#2 0x00000039f10f97df in virDomainDestroy (domain=0x7fff741b8540) at
libvirt.c:2201
#3 0x0000000000428e22 in remoteDispatchDomainDestroy (server=<value
optimized out>, client=<value optimized out>, msg=<value optimized
out>, rerr=0x7fff8e1fbbe0, args=<value optimized out>, ret=<value
optimized out>) at remote_dispatch.h:1277
#4 remoteDispatchDomainDestroyHelper (server=<value optimized out>,
client=<value optimized out>, msg=<value optimized out>,
rerr=0x7fff8e1fbbe0, args=<value optimized out>, ret=<value optimized
out>) at remote_dispatch.h:1255
#5 0x00000039f1146152 in virNetServerProgramDispatchCall
(prog=0x6884a0, server=0x67fe60, client=0x68eb50, msg=0x6930b0) at
rpc/virnetserverprogram.c:431
#6 virNetServerProgramDispatch (prog=0x6884a0, server=0x67fe60,
client=0x68eb50, msg=0x6930b0) at rpc/virnetserverprogram.c:304
#7 0x00000039f1143fee in virNetServerProcessMsg (srv=<value optimized
out>, client=0x68eb50, prog=<value optimized out>, msg=0x6930b0) at
rpc/virnetserver.c:171
#8 0x00000039f1144a8b in virNetServerHandleJob (jobOpaque=<value
optimized out>, opaque=<value optimized out>) at rpc/virnetserver.c:192
#9 0x00000039f10643ec in virThreadPoolWorker (opaque=<value optimized
out>) at util/threadpool.c:144
#10 0x00000039f1063cd9 in virThreadHelper (data=<value optimized out>)
at util/threads-pthread.c:161
#11 0x000000300a2077f1 in start_thread () from /lib64/libpthread.so.0
#12 0x0000003009ae570d in clone () from /lib64/libc.so.6
Thread 29 (Thread 0x7fff8ebfd700 (LWP 20241)):
#0 0x000000300a20b3dc in pthread_cond_wait@@GLIBC_2.3.2 () from
/lib64/libpthread.so.0
#1 0x00000039f1063eb6 in virCondWait (c=<value optimized out>,
m=<value optimized out>) at util/threads-pthread.c:117
#2 0x00000039f1064483 in virThreadPoolWorker (opaque=<value optimized
out>) at util/threadpool.c:103
#3 0x00000039f1063cd9 in virThreadHelper (data=<value optimized out>)
at util/threads-pthread.c:161
#4 0x000000300a2077f1 in start_thread () from /lib64/libpthread.so.0
#5 0x0000003009ae570d in clone () from /lib64/libc.so.6
Thread 28 (Thread 0x7fff8f5fe700 (LWP 20235)):
#0 0x000000300a20b3dc in pthread_cond_wait@@GLIBC_2.3.2 () from
/lib64/libpthread.so.0
#1 0x00000039f1063eb6 in virCondWait (c=<value optimized out>,
m=<value optimized out>) at util/threads-pthread.c:117
#2 0x00000039f1064483 in virThreadPoolWorker (opaque=<value optimized
out>) at util/threadpool.c:103
#3 0x00000039f1063cd9 in virThreadHelper (data=<value optimized out>)
at util/threads-pthread.c:161
#4 0x000000300a2077f1 in start_thread () from /lib64/libpthread.so.0
#5 0x0000003009ae570d in clone () from /lib64/libc.so.6
Thread 27 (Thread 0x7fff8ffff700 (LWP 29349)):
#0 0x000000300a20b3dc in pthread_cond_wait@@GLIBC_2.3.2 () from
/lib64/libpthread.so.0
#1 0x00000039f1063eb6 in virCondWait (c=<value optimized out>,
m=<value optimized out>) at util/threads-pthread.c:117
#2 0x00000039f1064483 in virThreadPoolWorker (opaque=<value optimized
out>) at util/threadpool.c:103
#3 0x00000039f1063cd9 in virThreadHelper (data=<value optimized out>)
at util/threads-pthread.c:161
#4 0x000000300a2077f1 in start_thread () from /lib64/libpthread.so.0
#5 0x0000003009ae570d in clone () from /lib64/libc.so.6
Thread 26 (Thread 0x7fffb8dfa700 (LWP 29348)):
#0 0x000000300a20b3dc in pthread_cond_wait@@GLIBC_2.3.2 () from
/lib64/libpthread.so.0
#1 0x00000039f1063eb6 in virCondWait (c=<value optimized out>,
m=<value optimized out>) at util/threads-pthread.c:117
#2 0x00000039f1064483 in virThreadPoolWorker (opaque=<value optimized
out>) at util/threadpool.c:103
#3 0x00000039f1063cd9 in virThreadHelper (data=<value optimized out>)
at util/threads-pthread.c:161
#4 0x000000300a2077f1 in start_thread () from /lib64/libpthread.so.0
#5 0x0000003009ae570d in clone () from /lib64/libc.so.6
Thread 25 (Thread 0x7fffb97fb700 (LWP 28577)):
#0 0x000000300a20b3dc in pthread_cond_wait@@GLIBC_2.3.2 () from
/lib64/libpthread.so.0
#1 0x00000039f1063eb6 in virCondWait (c=<value optimized out>,
m=<value optimized out>) at util/threads-pthread.c:117
#2 0x00000039f1064483 in virThreadPoolWorker (opaque=<value optimized
out>) at util/threadpool.c:103
#3 0x00000039f1063cd9 in virThreadHelper (data=<value optimized out>)
at util/threads-pthread.c:161
#4 0x000000300a2077f1 in start_thread () from /lib64/libpthread.so.0
#5 0x0000003009ae570d in clone () from /lib64/libc.so.6
Thread 24 (Thread 0x7fffba1fc700 (LWP 28576)):
#0 0x000000300a20b3dc in pthread_cond_wait@@GLIBC_2.3.2 () from
/lib64/libpthread.so.0
#1 0x00000039f1063eb6 in virCondWait (c=<value optimized out>,
m=<value optimized out>) at util/threads-pthread.c:117
#2 0x00000039f1064483 in virThreadPoolWorker (opaque=<value optimized
out>) at util/threadpool.c:103
#3 0x00000039f1063cd9 in virThreadHelper (data=<value optimized out>)
at util/threads-pthread.c:161
#4 0x000000300a2077f1 in start_thread () from /lib64/libpthread.so.0
#5 0x0000003009ae570d in clone () from /lib64/libc.so.6
Thread 23 (Thread 0x7fffbabfd700 (LWP 28180)):
#0 0x000000300a20b3dc in pthread_cond_wait@@GLIBC_2.3.2 () from
/lib64/libpthread.so.0
#1 0x00000039f1063eb6 in virCondWait (c=<value optimized out>,
m=<value optimized out>) at util/threads-pthread.c:117
#2 0x00000039f1064483 in virThreadPoolWorker (opaque=<value optimized
out>) at util/threadpool.c:103
#3 0x00000039f1063cd9 in virThreadHelper (data=<value optimized out>)
at util/threads-pthread.c:161
#4 0x000000300a2077f1 in start_thread () from /lib64/libpthread.so.0
#5 0x0000003009ae570d in clone () from /lib64/libc.so.6
Thread 22 (Thread 0x7fffbb5fe700 (LWP 28178)):
#0 0x000000300a20b3dc in pthread_cond_wait@@GLIBC_2.3.2 () from
/lib64/libpthread.so.0
#1 0x00000039f1063eb6 in virCondWait (c=<value optimized out>,
m=<value optimized out>) at util/threads-pthread.c:117
#2 0x00000039f1064483 in virThreadPoolWorker (opaque=<value optimized
out>) at util/threadpool.c:103
#3 0x00000039f1063cd9 in virThreadHelper (data=<value optimized out>)
at util/threads-pthread.c:161
#4 0x000000300a2077f1 in start_thread () from /lib64/libpthread.so.0
#5 0x0000003009ae570d in clone () from /lib64/libc.so.6
Thread 21 (Thread 0x7fffe0dfa700 (LWP 12076)):
#0 0x000000300a20b3dc in pthread_cond_wait@@GLIBC_2.3.2 () from
/lib64/libpthread.so.0
#1 0x00000039f1063eb6 in virCondWait (c=<value optimized out>,
m=<value optimized out>) at util/threads-pthread.c:117
#2 0x00000039f1064483 in virThreadPoolWorker (opaque=<value optimized
out>) at util/threadpool.c:103
#3 0x00000039f1063cd9 in virThreadHelper (data=<value optimized out>)
at util/threads-pthread.c:161
#4 0x000000300a2077f1 in start_thread () from /lib64/libpthread.so.0
#5 0x0000003009ae570d in clone () from /lib64/libc.so.6
Thread 20 (Thread 0x7fffe17fb700 (LWP 5934)):
#0 0x000000300a20b3dc in pthread_cond_wait@@GLIBC_2.3.2 () from
/lib64/libpthread.so.0
#1 0x00000039f1063eb6 in virCondWait (c=<value optimized out>,
m=<value optimized out>) at util/threads-pthread.c:117
#2 0x00000039f1064483 in virThreadPoolWorker (opaque=<value optimized
out>) at util/threadpool.c:103
#3 0x00000039f1063cd9 in virThreadHelper (data=<value optimized out>)
at util/threads-pthread.c:161
#4 0x000000300a2077f1 in start_thread () from /lib64/libpthread.so.0
#5 0x0000003009ae570d in clone () from /lib64/libc.so.6
Thread 19 (Thread 0x7fffe21fc700 (LWP 5904)):
#0 0x000000300a20b3dc in pthread_cond_wait@@GLIBC_2.3.2 () from
/lib64/libpthread.so.0
#1 0x00000039f1063eb6 in virCondWait (c=<value optimized out>,
m=<value optimized out>) at util/threads-pthread.c:117
#2 0x00000039f1064483 in virThreadPoolWorker (opaque=<value optimized
out>) at util/threadpool.c:103
#3 0x00000039f1063cd9 in virThreadHelper (data=<value optimized out>)
at util/threads-pthread.c:161
#4 0x000000300a2077f1 in start_thread () from /lib64/libpthread.so.0
#5 0x0000003009ae570d in clone () from /lib64/libc.so.6
Thread 18 (Thread 0x7fffe2bfd700 (LWP 5860)):
#0 0x000000300a20b3dc in pthread_cond_wait@@GLIBC_2.3.2 () from
/lib64/libpthread.so.0
#1 0x00000039f1063eb6 in virCondWait (c=<value optimized out>,
m=<value optimized out>) at util/threads-pthread.c:117
#2 0x00000039f1064483 in virThreadPoolWorker (opaque=<value optimized
out>) at util/threadpool.c:103
#3 0x00000039f1063cd9 in virThreadHelper (data=<value optimized out>)
at util/threads-pthread.c:161
#4 0x000000300a2077f1 in start_thread () from /lib64/libpthread.so.0
#5 0x0000003009ae570d in clone () from /lib64/libc.so.6
Thread 17 (Thread 0x7fffbbfff700 (LWP 855)):
#0 0x000000300a20b3dc in pthread_cond_wait@@GLIBC_2.3.2 () from
/lib64/libpthread.so.0
#1 0x00000039f1063eb6 in virCondWait (c=<value optimized out>,
m=<value optimized out>) at util/threads-pthread.c:117
#2 0x00000039f1064483 in virThreadPoolWorker (opaque=<value optimized
out>) at util/threadpool.c:103
#3 0x00000039f1063cd9 in virThreadHelper (data=<value optimized out>)
at util/threads-pthread.c:161
#4 0x000000300a2077f1 in start_thread () from /lib64/libpthread.so.0
#5 0x0000003009ae570d in clone () from /lib64/libc.so.6
Thread 16 (Thread 0x7fffe97c4700 (LWP 835)):
#0 0x000000300a20b3dc in pthread_cond_wait@@GLIBC_2.3.2 () from
/lib64/libpthread.so.0
#1 0x00000039f1063eb6 in virCondWait (c=<value optimized out>,
m=<value optimized out>) at util/threads-pthread.c:117
#2 0x00000039f1064483 in virThreadPoolWorker (opaque=<value optimized
out>) at util/threadpool.c:103
#3 0x00000039f1063cd9 in virThreadHelper (data=<value optimized out>)
at util/threads-pthread.c:161
#4 0x000000300a2077f1 in start_thread () from /lib64/libpthread.so.0
#5 0x0000003009ae570d in clone () from /lib64/libc.so.6
Thread 11 (Thread 0x7fffec73f700 (LWP 7457)):
#0 0x000000300a20b3dc in pthread_cond_wait@@GLIBC_2.3.2 () from
/lib64/libpthread.so.0
#1 0x00000039f1063eb6 in virCondWait (c=<value optimized out>,
m=<value optimized out>) at util/threads-pthread.c:117
#2 0x00000039f1064483 in virThreadPoolWorker (opaque=<value optimized
out>) at util/threadpool.c:103
#3 0x00000039f1063cd9 in virThreadHelper (data=<value optimized out>)
at util/threads-pthread.c:161
#4 0x000000300a2077f1 in start_thread () from /lib64/libpthread.so.0
#5 0x0000003009ae570d in clone () from /lib64/libc.so.6
Thread 10 (Thread 0x7fffed140700 (LWP 7456)):
#0 0x000000300a20b3dc in pthread_cond_wait@@GLIBC_2.3.2 () from
/lib64/libpthread.so.0
#1 0x00000039f1063eb6 in virCondWait (c=<value optimized out>,
m=<value optimized out>) at util/threads-pthread.c:117
#2 0x00000039f1064483 in virThreadPoolWorker (opaque=<value optimized
out>) at util/threadpool.c:103
#3 0x00000039f1063cd9 in virThreadHelper (data=<value optimized out>)
at util/threads-pthread.c:161
#4 0x000000300a2077f1 in start_thread () from /lib64/libpthread.so.0
#5 0x0000003009ae570d in clone () from /lib64/libc.so.6
Thread 9 (Thread 0x7fffedb41700 (LWP 7455)):
#0 0x000000300a20b3dc in pthread_cond_wait@@GLIBC_2.3.2 () from
/lib64/libpthread.so.0
#1 0x00000039f1063eb6 in virCondWait (c=<value optimized out>,
m=<value optimized out>) at util/threads-pthread.c:117
#2 0x00000039f1064483 in virThreadPoolWorker (opaque=<value optimized
out>) at util/threadpool.c:103
#3 0x00000039f1063cd9 in virThreadHelper (data=<value optimized out>)
at util/threads-pthread.c:161
#4 0x000000300a2077f1 in start_thread () from /lib64/libpthread.so.0
#5 0x0000003009ae570d in clone () from /lib64/libc.so.6
Thread 8 (Thread 0x7fffee542700 (LWP 7454)):
#0 0x000000300a20b3dc in pthread_cond_wait@@GLIBC_2.3.2 () from
/lib64/libpthread.so.0
#1 0x00000039f1063eb6 in virCondWait (c=<value optimized out>,
m=<value optimized out>) at util/threads-pthread.c:117
#2 0x00000039f1064483 in virThreadPoolWorker (opaque=<value optimized
out>) at util/threadpool.c:103
#3 0x00000039f1063cd9 in virThreadHelper (data=<value optimized out>)
at util/threads-pthread.c:161
#4 0x000000300a2077f1 in start_thread () from /lib64/libpthread.so.0
#5 0x0000003009ae570d in clone () from /lib64/libc.so.6
Thread 7 (Thread 0x7fffeef43700 (LWP 7453)):
#0 0x000000300a20b3dc in pthread_cond_wait@@GLIBC_2.3.2 () from
/lib64/libpthread.so.0
#1 0x00000039f1063eb6 in virCondWait (c=<value optimized out>,
m=<value optimized out>) at util/threads-pthread.c:117
#2 0x00000039f1064483 in virThreadPoolWorker (opaque=<value optimized
out>) at util/threadpool.c:103
#3 0x00000039f1063cd9 in virThreadHelper (data=<value optimized out>)
at util/threads-pthread.c:161
#4 0x000000300a2077f1 in start_thread () from /lib64/libpthread.so.0
#5 0x0000003009ae570d in clone () from /lib64/libc.so.6
Thread 6 (Thread 0x7fffef944700 (LWP 7452)):
#0 0x000000300a20b3dc in pthread_cond_wait@@GLIBC_2.3.2 () from
/lib64/libpthread.so.0
#1 0x00000039f1063eb6 in virCondWait (c=<value optimized out>,
m=<value optimized out>) at util/threads-pthread.c:117
#2 0x00000039f1064483 in virThreadPoolWorker (opaque=<value optimized
out>) at util/threadpool.c:103
#3 0x00000039f1063cd9 in virThreadHelper (data=<value optimized out>)
at util/threads-pthread.c:161
#4 0x000000300a2077f1 in start_thread () from /lib64/libpthread.so.0
#5 0x0000003009ae570d in clone () from /lib64/libc.so.6
Thread 5 (Thread 0x7ffff0345700 (LWP 7451)):
#0 0x000000300a20b3dc in pthread_cond_wait@@GLIBC_2.3.2 () from
/lib64/libpthread.so.0
#1 0x00000039f1063eb6 in virCondWait (c=<value optimized out>,
m=<value optimized out>) at util/threads-pthread.c:117
#2 0x00000039f1064483 in virThreadPoolWorker (opaque=<value optimized
out>) at util/threadpool.c:103
#3 0x00000039f1063cd9 in virThreadHelper (data=<value optimized out>)
at util/threads-pthread.c:161
#4 0x000000300a2077f1 in start_thread () from /lib64/libpthread.so.0
#5 0x0000003009ae570d in clone () from /lib64/libc.so.6
Thread 4 (Thread 0x7ffff0d46700 (LWP 7450)):
#0 0x000000300a20b3dc in pthread_cond_wait@@GLIBC_2.3.2 () from
/lib64/libpthread.so.0
#1 0x00000039f1063eb6 in virCondWait (c=<value optimized out>,
m=<value optimized out>) at util/threads-pthread.c:117
#2 0x00000039f1064483 in virThreadPoolWorker (opaque=<value optimized
out>) at util/threadpool.c:103
#3 0x00000039f1063cd9 in virThreadHelper (data=<value optimized out>)
at util/threads-pthread.c:161
#4 0x000000300a2077f1 in start_thread () from /lib64/libpthread.so.0
#5 0x0000003009ae570d in clone () from /lib64/libc.so.6
Thread 3 (Thread 0x7ffff1747700 (LWP 7449)):
#0 0x000000300a20b3dc in pthread_cond_wait@@GLIBC_2.3.2 () from
/lib64/libpthread.so.0
#1 0x00000039f1063eb6 in virCondWait (c=<value optimized out>,
m=<value optimized out>) at util/threads-pthread.c:117
#2 0x00000039f1064483 in virThreadPoolWorker (opaque=<value optimized
out>) at util/threadpool.c:103
#3 0x00000039f1063cd9 in virThreadHelper (data=<value optimized out>)
at util/threads-pthread.c:161
#4 0x000000300a2077f1 in start_thread () from /lib64/libpthread.so.0
#5 0x0000003009ae570d in clone () from /lib64/libc.so.6
Thread 2 (Thread 0x7ffff2148700 (LWP 7448)):
#0 0x000000300a20b3dc in pthread_cond_wait@@GLIBC_2.3.2 () from
/lib64/libpthread.so.0
#1 0x00000039f1063eb6 in virCondWait (c=<value optimized out>,
m=<value optimized out>) at util/threads-pthread.c:117
#2 0x00000039f1064483 in virThreadPoolWorker (opaque=<value optimized
out>) at util/threadpool.c:103
#3 0x00000039f1063cd9 in virThreadHelper (data=<value optimized out>)
at util/threads-pthread.c:161
#4 0x000000300a2077f1 in start_thread () from /lib64/libpthread.so.0
#5 0x0000003009ae570d in clone () from /lib64/libc.so.6
Thread 1 (Thread 0x7ffff7fda800 (LWP 7445)):
#0 0x0000003009a46f60 in vfprintf () from /lib64/libc.so.6
#1 0x0000003009afc970 in __vsnprintf_chk () from /lib64/libc.so.6
#2 0x0000003009afc8aa in __snprintf_chk () from /lib64/libc.so.6
#3 0x00000039f108144a in snprintf (when=<value optimized out>,
buf=0x7fffffffd470 "2012-12-27 17:23:36.561+0000") at
/usr/include/bits/stdio2.h:65
#4 virTimeStringThenRaw (when=<value optimized out>,
buf=0x7fffffffd470 "2012-12-27 17:23:36.561+0000") at util/virtime.c:219
#5 0x00000039f108167a in virTimeStringNowRaw (buf=0x7fffffffd470
"2012-12-27 17:23:36.561+0000") at util/virtime.c:195
#6 0x00000039f105834b in virLogVMessage (source=VIR_LOG_FROM_FILE,
priority=<value optimized out>, filename=0x39f11ad70b
"util/event_poll.c", linenr=378, funcname=0x39f11adcc0
"virEventPollMakePollFDs", fmt=<value optimized out>,
vargs=0x7fffffffd4f0) at util/logging.c:822
#7 0x00000039f10586ec in virLogMessage (source=<value optimized out>,
priority=<value optimized out>, filename=<value optimized out>,
linenr=<value optimized out>, funcname=<value optimized out>,
fmt=<value optimized out>) at util/logging.c:753
#8 0x00000039f1052241 in virEventPollMakePollFDs () at
util/event_poll.c:374
#9 virEventPollRunOnce () at util/event_poll.c:605
#10 0x00000039f1051666 in virEventRunDefaultImpl () at util/event.c:247
#11 0x00000039f11437dd in virNetServerRun (srv=0x67fe60) at
rpc/virnetserver.c:1004
#12 0x000000000040c5a3 in main (argc=<value optimized out>,
argv=<value optimized out>) at libvirtd.c:1354
(gdb)
Does anyone have any insights as to the cause of this, or any
potential solutions? If there's anything I can add that would help let
me know.