Hi Peter, I have attached a trace via coredump. Best Regards Am 30.07.20 um 17:03 schrieb Peter Krempa: > On Thu, Jul 30, 2020 at 16:13:09 +0200, Alex Walender wrote: >> Dear libvirt community, >> >> >> Using recent Ubuntu Stein Cloud Packages, we are observing random >> libvirtd live-migration crashes on the target host. >> Libvirt is having a SEGFAULT with the qemu driver. Transferring block >> devices usually works without issues. >> However, the following memory transfer is causing the target libvirtd >> randomly to close down its socket, resulting in a roll-backed migration >> process.I can reproduce this with large VMs, which have a large memory pool. > Could you please also attach the backtrace of the libvirtd process once > it crashes? That would be helpful. > -- M.Sc Alex Walender de.NBI Cloud Bielefeld Administrator Center for Biotechnology (CeBiTec) University of Bielefeld 33594 Bielefeld Germany room: M3-118 phone: +49 (521) 106 2907
PID: 2386 (libvirtd)
UID: 0 (root)
GID: 0 (root)
Signal: 11 (SEGV)
Timestamp: Fri 2020-07-31 12:50:32 CEST (44min ago)
Command Line: /usr/sbin/libvirtd -l
Executable: /usr/sbin/libvirtd
Control Group: /system.slice/libvirtd.service
Unit: libvirtd.service
Slice: system.slice
Boot ID: de7898777fd9439b9aaa276a91ce1e39
Machine ID: 2ee16e1076e24d98a71de0d47772986d
Hostname: hornbach
Storage: /var/lib/systemd/coredump/core.libvirtd.0.de7898777fd9439b9aaa276a91ce1e39.2386.1596192632000000.lz4
Message: Process 2386 (libvirtd) of user 0 dumped core.
Stack trace of thread 2433:
#0 0x00007febd50af20b qemuMonitorSend (libvirt_driver_qemu.so)
#1 0x00007febd50c0a69 n/a (libvirt_driver_qemu.so)
#2 0x00007febd50cdd6c qemuMonitorJSONGetIOThreads (libvirt_driver_qemu.so)
#3 0x00007febd50d43ee n/a (libvirt_driver_qemu.so)
#4 0x00007febd50d44c8 n/a (libvirt_driver_qemu.so)
#5 0x00007febd50d528a n/a (libvirt_driver_qemu.so)
#6 0x00007fec1db43e49 virConnectGetAllDomainStats (libvirt.so.0)
#7 0x000055a8e44dc30b n/a (libvirtd)
#8 0x00007fec1da5a47c virNetServerProgramDispatch (libvirt.so.0)
#9 0x00007fec1da60888 n/a (libvirt.so.0)
#10 0x00007fec1d991161 n/a (libvirt.so.0)
#11 0x00007fec1d9904d8 n/a (libvirt.so.0)
#12 0x00007fec1d4176db start_thread (libpthread.so.0)
#13 0x00007fec1d140a3f __clone (libc.so.6)
Stack trace of thread 2438:
#0 0x00007fec1d41d9f3 pthread_cond_wait@@GLIBC_2.3.2 (libpthread.so.0)
#1 0x00007fec1d990736 virCondWait (libvirt.so.0)
#2 0x00007fec1d9911a8 n/a (libvirt.so.0)
#3 0x00007fec1d9904d8 n/a (libvirt.so.0)
#4 0x00007fec1d4176db start_thread (libpthread.so.0)
#5 0x00007fec1d140a3f __clone (libc.so.6)
Stack trace of thread 2431:
#0 0x00007fec1d42111d __lll_lock_wait (libpthread.so.0)
#1 0x00007fec1d41bf02 __pthread_mutex_cond_lock (libpthread.so.0)
#2 0x00007fec1d41d8bf pthread_cond_wait@@GLIBC_2.3.2 (libpthread.so.0)
#3 0x00007fec1d990736 virCondWait (libvirt.so.0)
#4 0x00007febd50af1fc qemuMonitorSend (libvirt_driver_qemu.so)
#5 0x00007febd50c0a69 n/a (libvirt_driver_qemu.so)
#6 0x00007febd50cdd6c qemuMonitorJSONGetIOThreads (libvirt_driver_qemu.so)
#7 0x00007febd50d43ee n/a (libvirt_driver_qemu.so)
#8 0x00007febd50d44c8 n/a (libvirt_driver_qemu.so)
#9 0x00007febd50d528a n/a (libvirt_driver_qemu.so)
#10 0x00007fec1db43e49 virConnectGetAllDomainStats (libvirt.so.0)
#11 0x000055a8e44dc30b n/a (libvirtd)
#12 0x00007fec1da5a47c virNetServerProgramDispatch (libvirt.so.0)
#13 0x00007fec1da60888 n/a (libvirt.so.0)
#14 0x00007fec1d991161 n/a (libvirt.so.0)
#15 0x00007fec1d9904d8 n/a (libvirt.so.0)
#16 0x00007fec1d4176db start_thread (libpthread.so.0)
#17 0x00007fec1d140a3f __clone (libc.so.6)
Stack trace of thread 2434:
#0 0x00007fec1d42111d __lll_lock_wait (libpthread.so.0)
#1 0x00007fec1d41bf02 __pthread_mutex_cond_lock (libpthread.so.0)
#2 0x00007fec1d41d8bf pthread_cond_wait@@GLIBC_2.3.2 (libpthread.so.0)
#3 0x00007fec1d990736 virCondWait (libvirt.so.0)
#4 0x00007febd50af1fc qemuMonitorSend (libvirt_driver_qemu.so)
#5 0x00007febd50c0a69 n/a (libvirt_driver_qemu.so)
#6 0x00007febd50cdd6c qemuMonitorJSONGetIOThreads (libvirt_driver_qemu.so)
#7 0x00007febd50d43ee n/a (libvirt_driver_qemu.so)
#8 0x00007febd50d44c8 n/a (libvirt_driver_qemu.so)
#9 0x00007febd50d528a n/a (libvirt_driver_qemu.so)
#10 0x00007fec1db43e49 virConnectGetAllDomainStats (libvirt.so.0)
#11 0x000055a8e44dc30b n/a (libvirtd)
#12 0x00007fec1da5a47c virNetServerProgramDispatch (libvirt.so.0)
#13 0x00007fec1da60888 n/a (libvirt.so.0)
#14 0x00007fec1d991161 n/a (libvirt.so.0)
#15 0x00007fec1d9904d8 n/a (libvirt.so.0)
#16 0x00007fec1d4176db start_thread (libpthread.so.0)
#17 0x00007fec1d140a3f __clone (libc.so.6)
Stack trace of thread 2435:
#0 0x00007fec1d42111d __lll_lock_wait (libpthread.so.0)
#1 0x00007fec1d41bf02 __pthread_mutex_cond_lock (libpthread.so.0)
#2 0x00007fec1d41d8bf pthread_cond_wait@@GLIBC_2.3.2 (libpthread.so.0)
#3 0x00007fec1d990736 virCondWait (libvirt.so.0)
#4 0x00007febd50af1fc qemuMonitorSend (libvirt_driver_qemu.so)
#5 0x00007febd50c0a69 n/a (libvirt_driver_qemu.so)
#6 0x00007febd50cdd6c qemuMonitorJSONGetIOThreads (libvirt_driver_qemu.so)
#7 0x00007febd50d43ee n/a (libvirt_driver_qemu.so)
#8 0x00007febd50d44c8 n/a (libvirt_driver_qemu.so)
#9 0x00007febd50d528a n/a (libvirt_driver_qemu.so)
#10 0x00007fec1db43e49 virConnectGetAllDomainStats (libvirt.so.0)
#11 0x000055a8e44dc30b n/a (libvirtd)
#12 0x00007fec1da5a47c virNetServerProgramDispatch (libvirt.so.0)
#13 0x00007fec1da60888 n/a (libvirt.so.0)
#14 0x00007fec1d991161 n/a (libvirt.so.0)
#15 0x00007fec1d9904d8 n/a (libvirt.so.0)
#16 0x00007fec1d4176db start_thread (libpthread.so.0)
#17 0x00007fec1d140a3f __clone (libc.so.6)
Stack trace of thread 2437:
#0 0x00007fec1d41d9f3 pthread_cond_wait@@GLIBC_2.3.2 (libpthread.so.0)
#1 0x00007fec1d990736 virCondWait (libvirt.so.0)
#2 0x00007fec1d9911a8 n/a (libvirt.so.0)
#3 0x00007fec1d9904d8 n/a (libvirt.so.0)
#4 0x00007fec1d4176db start_thread (libpthread.so.0)
#5 0x00007fec1d140a3f __clone (libc.so.6)
Stack trace of thread 2432:
#0 0x00007fec1d0af9ac n/a (libc.so.6)
#1 0x00007fec1d0b70ab cfree (libc.so.6)
#2 0x00007fec1d90712b virFree (libvirt.so.0)
#3 0x00007fec1d994a08 virTypedParamsFree (libvirt.so.0)
#4 0x00007febd50d5446 n/a (libvirt_driver_qemu.so)
#5 0x00007fec1db43e49 virConnectGetAllDomainStats (libvirt.so.0)
#6 0x000055a8e44dc30b n/a (libvirtd)
#7 0x00007fec1da5a47c virNetServerProgramDispatch (libvirt.so.0)
#8 0x00007fec1da60888 n/a (libvirt.so.0)
#9 0x00007fec1d991161 n/a (libvirt.so.0)
#10 0x00007fec1d9904d8 n/a (libvirt.so.0)
#11 0x00007fec1d4176db start_thread (libpthread.so.0)
#12 0x00007fec1d140a3f __clone (libc.so.6)
Stack trace of thread 2386:
#0 0x00007fec1d133cf9 __poll (libc.so.6)
#1 0x00007fec1d92ea25 virEventPollRunOnce (libvirt.so.0)
#2 0x00007fec1d92d561 virEventRunDefaultImpl (libvirt.so.0)
#3 0x00007fec1da600d5 virNetDaemonRun (libvirt.so.0)
#4 0x000055a8e44cfabe n/a (libvirtd)
#5 0x00007fec1d040b97 __libc_start_main (libc.so.6)
#6 0x000055a8e44d10fa n/a (libvirtd)
Stack trace of thread 2440:
#0 0x00007fec1d41d9f3 pthread_cond_wait@@GLIBC_2.3.2 (libpthread.so.0)
#1 0x00007fec1d990736 virCondWait (libvirt.so.0)
#2 0x00007fec1d9911a8 n/a (libvirt.so.0)
#3 0x00007fec1d9904d8 n/a (libvirt.so.0)
#4 0x00007fec1d4176db start_thread (libpthread.so.0)
#5 0x00007fec1d140a3f __clone (libc.so.6)
Stack trace of thread 2436:
#0 0x00007fec1d41d9f3 pthread_cond_wait@@GLIBC_2.3.2 (libpthread.so.0)
#1 0x00007fec1d990736 virCondWait (libvirt.so.0)
#2 0x00007fec1d9911a8 n/a (libvirt.so.0)
#3 0x00007fec1d9904d8 n/a (libvirt.so.0)
#4 0x00007fec1d4176db start_thread (libpthread.so.0)
#5 0x00007fec1d140a3f __clone (libc.so.6)
Stack trace of thread 2439:
#0 0x00007fec1d41d9f3 pthread_cond_wait@@GLIBC_2.3.2 (libpthread.so.0)
#1 0x00007fec1d990736 virCondWait (libvirt.so.0)
#2 0x00007fec1d9911a8 n/a (libvirt.so.0)
#3 0x00007fec1d9904d8 n/a (libvirt.so.0)
#4 0x00007fec1d4176db start_thread (libpthread.so.0)
#5 0x00007fec1d140a3f __clone (libc.so.6)
Stack trace of thread 20436:
#0 0x00007fec1d41d9f3 pthread_cond_wait@@GLIBC_2.3.2 (libpthread.so.0)
#1 0x00007fec1d990736 virCondWait (libvirt.so.0)
#2 0x00007fec1d991273 n/a (libvirt.so.0)
#3 0x00007fec1d9904d8 n/a (libvirt.so.0)
#4 0x00007fec1d4176db start_thread (libpthread.so.0)
#5 0x00007fec1d140a3f __clone (libc.so.6)
Stack trace of thread 2463:
#0 0x00007fec1d41d9f3 pthread_cond_wait@@GLIBC_2.3.2 (libpthread.so.0)
#1 0x00007fec1d990736 virCondWait (libvirt.so.0)
#2 0x00007fec1d991273 n/a (libvirt.so.0)
#3 0x00007fec1d9904d8 n/a (libvirt.so.0)
#4 0x00007fec1d4176db start_thread (libpthread.so.0)
#5 0x00007fec1d140a3f __clone (libc.so.6)
Stack trace of thread 2461:
#0 0x00007fec1d41d9f3 pthread_cond_wait@@GLIBC_2.3.2 (libpthread.so.0)
#1 0x00007fec1d990736 virCondWait (libvirt.so.0)
#2 0x00007fec1d991273 n/a (libvirt.so.0)
#3 0x00007fec1d9904d8 n/a (libvirt.so.0)
#4 0x00007fec1d4176db start_thread (libpthread.so.0)
#5 0x00007fec1d140a3f __clone (libc.so.6)
Stack trace of thread 2460:
#0 0x00007fec1d41d9f3 pthread_cond_wait@@GLIBC_2.3.2 (libpthread.so.0)
#1 0x00007fec1d990736 virCondWait (libvirt.so.0)
#2 0x00007fec1d991273 n/a (libvirt.so.0)
#3 0x00007fec1d9904d8 n/a (libvirt.so.0)
#4 0x00007fec1d4176db start_thread (libpthread.so.0)
#5 0x00007fec1d140a3f __clone (libc.so.6)
Stack trace of thread 2462:
#0 0x00007fec1d41d9f3 pthread_cond_wait@@GLIBC_2.3.2 (libpthread.so.0)
#1 0x00007fec1d990736 virCondWait (libvirt.so.0)
#2 0x00007fec1d991273 n/a (libvirt.so.0)
#3 0x00007fec1d9904d8 n/a (libvirt.so.0)
#4 0x00007fec1d4176db start_thread (libpthread.so.0)
#5 0x00007fec1d140a3f __clone (libc.so.6)
Stack trace of thread 2464:
#0 0x00007fec1d41d9f3 pthread_cond_wait@@GLIBC_2.3.2 (libpthread.so.0)
#1 0x00007fec1d990736 virCondWait (libvirt.so.0)
#2 0x00007fec1d991273 n/a (libvirt.so.0)
#3 0x00007fec1d9904d8 n/a (libvirt.so.0)
#4 0x00007fec1d4176db start_thread (libpthread.so.0)
#5 0x00007fec1d140a3f __clone (libc.so.6)
Stack trace of thread 2480:
#0 0x00007fec1d41d9f3 pthread_cond_wait@@GLIBC_2.3.2 (libpthread.so.0)
#1 0x00007fec1d990736 virCondWait (libvirt.so.0)
#2 0x00007febfc4cd804 n/a (libvirt_driver_nodedev.so)
#3 0x00007fec1d990502 n/a (libvirt.so.0)
#4 0x00007fec1d4176db start_thread (libpthread.so.0)
#5 0x00007fec1d140a3f __clone (libc.so.6)
Attachment:
signature.asc
Description: OpenPGP digital signature
