I'm running a 14-node pacemaker/corosync cluster. I recently ditched the
Ubuntu 12.04 corosync and pacemaker pages opting to compile from source
instead. I successfully built Pacemaker 1.1.9 and Corosync 2.3.0,
however, Corosync often does not start on boot. Even worse, it tends to
die randomly once it is started, leaving no error messages in the log. I
can downgrade to Corosync 1.4.5 and all is fine. I have no explanation
as to why 2.3.0 fails so frequently. I've included some debugging
information. Any help would be much appreciated.
Thanks.
- Rob P.
strace output
sendmsg(11, {msg_name(16)={sa_family=AF_INET, sin_port=htons(9000),
sin_addr=inet_addr("226.94.1.3")},
msg_iov(1)=[{"\376\376\0\0\2\0\"\377w\4\1\n\2w\4\1\n\2\0\n\1\4w\0\0\0\0\0\0\0\0\0"...,
87}], msg_controllen=0, msg_flags=0}, MSG_NOSIGNAL) = 87
sendmsg(10, {msg_name(0)=NULL,
msg_iov(1)=[{"\376\376\0\0\2\0\"\377w\4\1\n\2w\4\1\n\2\0\n\1\4w\0\0\0\0\0\0\0\0\0"...,
87}], msg_controllen=0, msg_flags=0}, MSG_NOSIGNAL) = 87
clock_gettime(CLOCK_MONOTONIC, {57355, 710432961}) = 0
clock_gettime(CLOCK_MONOTONIC, {57355, 710674580}) = 0
clock_gettime(CLOCK_MONOTONIC, {57355, 711018759}) = 0
epoll_wait(4, {{EPOLLIN, {u32=2, u64=5813885752995479554}}}, 12, 0) = 1
clock_gettime(CLOCK_MONOTONIC, {57355, 711595469}) = 0
clock_gettime(CLOCK_MONOTONIC, {57355, 711888254}) = 0
epoll_wait(4, {{EPOLLIN, {u32=2, u64=5813885752995479554}}}, 12, 257) = 1
clock_gettime(CLOCK_MONOTONIC, {57355, 712421751}) = 0
clock_gettime(CLOCK_MONOTONIC, {57355, 712715186}) = 0
recvmsg(9, {msg_name(0)={sa_family=0x7c50 /* AF_??? */,
sa_data="\234\217A\177\0\0\20s\234\217A\177\0\0"},
msg_iov(1)=[{"\376\376\0\0\2\0\"\377w\4\1\n\2w\4\1\n\2\0\n\1\4w\0\0\0\0\0\0\0\0\0"...,
10000}], msg_controllen=0, msg_flags=0}, MSG_DONTWAIT|MSG_NOSIGNAL) = 87
clock_gettime(CLOCK_MONOTONIC, {57355, 713490334}) = 0
epoll_wait(4, {}, 12, 0) = 0
clock_gettime(CLOCK_MONOTONIC, {57355, 714051544}) = 0
clock_gettime(CLOCK_MONOTONIC, {57355, 714348259}) = 0
clock_gettime(CLOCK_MONOTONIC, {57355, 714558026}) = 0
epoll_wait(4, {}, 12, 0) = 0
clock_gettime(CLOCK_MONOTONIC, {57355, 715203138}) = 0
clock_gettime(CLOCK_MONOTONIC, {57355, 715411760}) = 0
epoll_wait(4, {{EPOLLIN, {u32=1, u64=7284734596711710721}}}, 12, 254) = 1
clock_gettime(CLOCK_MONOTONIC, {57355, 930356426}) = 0
clock_gettime(CLOCK_MONOTONIC, {57355, 930649677}) = 0
recvmsg(8, {msg_name(16)={sa_family=AF_INET, sin_port=htons(8999),
sin_addr=inet_addr("10.1.4.124")},
msg_iov(1)=[{"\376\376\0\0\5\0\"\377|\4\1\nw\4\1\n\2\0\n\1\4w\0\0\0\0\0\0\0\0\0\0"...,
10000}], msg_controllen=0, msg_flags=0}, MSG_DONTWAIT|MSG_NOSIGNAL) = 42
sendmsg(11, {msg_name(16)={sa_family=AF_INET, sin_port=htons(9000),
sin_addr=inet_addr("10.1.4.121")},
msg_iov(1)=[{"\376\376\0\0\0\0\"\377w\4\1\n\256\0\0\0\10=\0\0\256\0\0\0\0\0\0\0w\4\1\n"...,
74}], msg_controllen=0, msg_flags=0}, MSG_NOSIGNAL) = 74
clock_gettime(CLOCK_MONOTONIC, {57355, 931677894}) = 0
clock_gettime(CLOCK_MONOTONIC, {57355, 931933853}) = 0
epoll_wait(4, {}, 12, 0) = 0
clock_gettime(CLOCK_MONOTONIC, {57355, 932546611}) = 0
clock_gettime(CLOCK_MONOTONIC, {57355, 932960908}) = 0
clock_gettime(CLOCK_MONOTONIC, {57355, 933318531}) = 0
epoll_wait(4, {}, 12, 0) = 0
clock_gettime(CLOCK_MONOTONIC, {57355, 933818440}) = 0
clock_gettime(CLOCK_MONOTONIC, {57355, 934122715}) = 0
epoll_wait(4, {{EPOLLIN, {u32=1, u64=7284734596711710721}}}, 12, 35) = 1
clock_gettime(CLOCK_MONOTONIC, {57355, 939059907}) = 0
clock_gettime(CLOCK_MONOTONIC, {57355, 939326101}) = 0
recvmsg(8, {msg_name(16)={sa_family=AF_INET, sin_port=htons(8999),
sin_addr=inet_addr("10.1.4.124")},
msg_iov(1)=[{"\376\376\0\0\1\2\"\377|\4\1\n\2|\4\1\n\2\0\n\1\4|\0\0\0\0\0\0\0\0\0"...,
10000}], msg_controllen=0, msg_flags=0}, MSG_DONTWAIT|MSG_NOSIGNAL) = 1472
clock_gettime(CLOCK_MONOTONIC, {57355, 939958334}) = 0
epoll_wait(4, {{EPOLLIN, {u32=1, u64=7284734596711710721}}, {EPOLLIN,
{u32=3, u64=8736411904714473475}}}, 12, 0) = 2
clock_gettime(CLOCK_MONOTONIC, {57355, 940609120}) = 0
clock_gettime(CLOCK_MONOTONIC, {57355, 940881911}) = 0
recvmsg(8, {msg_name(16)={sa_family=AF_INET, sin_port=htons(8999),
sin_addr=inet_addr("10.1.4.124")},
msg_iov(1)=[{"\376\376\0\0\1\2\"\377|\4\1\n\2|\4\1\n\2\0\n\1\4|\0\0\0\0\0\0\0\0\0"...,
10000}], msg_controllen=0, msg_flags=0}, MSG_DONTWAIT|MSG_NOSIGNAL) = 617
sendto(23, "\0", 1, MSG_NOSIGNAL, NULL, 0) = 1
recvmsg(12, {msg_name(16)={sa_family=AF_INET, sin_port=htons(8999),
sin_addr=inet_addr("10.1.4.133")},
msg_iov(1)=[{"\376\376\0\0\0\0\"\377\205\4\1\n\262\0\0\0\20=\0\0\262\0\0\0\0\0\0\0w\4\1\n"...,
10000}], msg_controllen=0, msg_flags=0}, MSG_DONTWAIT|MSG_NOSIGNAL) = 74
poll([{fd=8, events=POLLIN}], 1, 0) = 1 ([{fd=8, revents=POLLIN}])
recvmsg(8, {msg_name(16)={sa_family=AF_INET, sin_port=htons(8999),
sin_addr=inet_addr("10.1.4.133")},
msg_iov(1)=[{"\376\376\0\0\5\0\"\377\205\4\1\nw\4\1\n\2\0\n\1\4w\0\0\0\0\0\0\0\0\0\0"...,
10000}], msg_controllen=0, msg_flags=0}, MSG_DONTWAIT|MSG_NOSIGNAL) = 42
sendmsg(11, {msg_name(16)={sa_family=AF_INET, sin_port=htons(9000),
sin_addr=inet_addr("10.1.4.121")},
msg_iov(1)=[{"\376\376\0\0\0\0\"\377w\4\1\n\256\0\0\0\10=\0\0\256\0\0\0\0\0\0\0w\4\1\n"...,
74}], msg_controllen=0, msg_flags=0}, MSG_NOSIGNAL) = 74
clock_gettime(CLOCK_MONOTONIC, {57355, 943922220}) = 0
poll([{fd=8, events=POLLIN}], 1, 0) = 1 ([{fd=8, revents=POLLIN}])
recvmsg(8, {msg_name(16)={sa_family=AF_INET, sin_port=htons(8999),
sin_addr=inet_addr("10.1.4.133")},
msg_iov(1)=[{"\376\376\0\0\1\2\"\377\205\4\1\n\2\205\4\1\n\2\0\n\1\4\205\0\0\0\0\0\0\0\0\0"...,
10000}], msg_controllen=0, msg_flags=0}, MSG_DONTWAIT|MSG_NOSIGNAL) = 1472
poll([{fd=8, events=POLLIN}], 1, 0) = 1 ([{fd=8, revents=POLLIN}])
recvmsg(8, {msg_name(16)={sa_family=AF_INET, sin_port=htons(8999),
sin_addr=inet_addr("10.1.4.133")},
msg_iov(1)=[{"\376\376\0\0\1\2\"\377\205\4\1\n\2\205\4\1\n\2\0\n\1\4\205\0\0\0\0\0\0\0\0\0"...,
10000}], msg_controllen=0, msg_flags=0}, MSG_DONTWAIT|MSG_NOSIGNAL) = 584
sendto(23, "\0", 1, MSG_NOSIGNAL, NULL, 0) = 1
poll([{fd=8, events=POLLIN}], 1, 0) = 0 (Timeout)
poll([{fd=9, events=POLLIN}], 1, 0) = 0 (Timeout)
clock_gettime(CLOCK_MONOTONIC, {57355, 946686072}) = 0
sendmsg(11, {msg_name(16)={sa_family=AF_INET, sin_port=htons(9000),
sin_addr=inet_addr("10.1.4.121")},
msg_iov(1)=[{"\376\376\0\0\0\0\"\377w\4\1\n\262\0\0\0\21=\0\0\262\0\0\0\0\0\0\0w\4\1\n"...,
74}], msg_controllen=0, msg_flags=0}, MSG_NOSIGNAL) = 74
clock_gettime(CLOCK_MONOTONIC, {57355, 952769552}) = 0
clock_gettime(CLOCK_MONOTONIC, {57355, 952985732}) = 0
clock_gettime(CLOCK_MONOTONIC, {57355, 953217383}) = 0
clock_gettime(CLOCK_MONOTONIC, {57355, 953489345}) = 0
epoll_wait(4, {}, 12, 0) = 0
clock_gettime(CLOCK_MONOTONIC, {57355, 954019395}) = 0
clock_gettime(CLOCK_MONOTONIC, {57355, 954314404}) = 0
epoll_wait(4, {{EPOLLIN, {u32=3, u64=8736411904714473475}}}, 12, 586) = 1
clock_gettime(CLOCK_MONOTONIC, {57355, 954907616}) = 0
clock_gettime(CLOCK_MONOTONIC, {57355, 955138096}) = 0
recvmsg(12, {msg_name(16)={sa_family=AF_INET, sin_port=htons(8999),
sin_addr=inet_addr("10.1.4.133")},
msg_iov(1)=[{"\376\376\0\0\0\0\"\377\205\4\1\n\262\0\0\0\31=\0\0\262\0\0\0\0\0\0\0w\4\1\n"...,
10000}], msg_controllen=0, msg_flags=0}, MSG_DONTWAIT|MSG_NOSIGNAL) = 74
clock_gettime(CLOCK_MONOTONIC, {57355, 955849968}) = 0
poll([{fd=8, events=POLLIN}], 1, 0) = 0 (Timeout)
poll([{fd=9, events=POLLIN}], 1, 0) = 0 (Timeout)
clock_gettime(CLOCK_MONOTONIC, {57355, 956746383}) = 0
--- SIGBUS (Bus error) @ 0 (0) ---
Process 30095 detached
corosync-blackbox output
root@krusty:~# corosync-blackbox
Failed to initialize the cmap API. Error CS_ERR_LIBRARY
Failed to initialize the cmap API. Error CS_ERR_LIBRARY
Dumping the contents of /var/lib/corosync/fdata
[debug] shm size:8388608; real_size:8388608; rb->word_size:2097152
[debug] read total of: 8388620
Ringbuffer:
->NORMAL
->write_pt [673]
->read_pt [0]
->size [2097152 words]
=>free [8385912 bytes]
=>used [2692 bytes]
debug May 07 15:26:52 handle_new_connection(476):2147483648: IPC
credentials authenticated (13438-14116-18)
debug May 07 15:26:52 qb_ipcs_shm_connect(294):9: connecting to client
[14116]
debug May 07 15:26:52 qb_rb_open_2(226):9: shm size:zd; real_size:zd;
rb->word_size:1048576
debug May 07 15:26:52 qb_rb_open_2(226):9: shm size:zd; real_size:zd;
rb->word_size:1048576
debug May 07 15:26:52 qb_rb_open_2(226):9: shm size:zd; real_size:zd;
rb->word_size:1048576
debug May 07 15:26:52 cs_ipcs_connection_created(269):8: connection
created
debug May 07 15:26:52 cmap_lib_init_fn(306):9: lib_init_fn:
conn=0x7feb08b0ea00
debug May 07 15:26:52 qb_ipcs_dispatch_connection_request(723):9: HUP
conn (13438-14116-18)
debug May 07 15:26:52 qb_ipcs_disconnect(565):9:
qb_ipcs_disconnect(13438-14116-18) state:2
debug May 07 15:26:52 _del(117):9: epoll_ctl(del): Bad file descriptor (9)
debug May 07 15:26:52 cs_ipcs_connection_closed(414):8:
cs_ipcs_connection_closed()
debug May 07 15:26:52 cmap_lib_exit_fn(325):9: exit_fn for
conn=0x7feb08b0ea00
debug May 07 15:26:52 cs_ipcs_connection_destroyed(387):8:
cs_ipcs_connection_destroyed()
trace May 07 15:26:52 qb_rb_close(279):9: ENTERING qb_rb_close()
trace May 07 15:26:52 my_posix_sem_destroy(91):9: ENTERING
my_posix_sem_destroy()
debug May 07 15:26:52 qb_rb_close(290):9: Free'ing ringbuffer:
/dev/shm/qb-cmap-response-13438-14116-18-header
trace May 07 15:26:52 qb_rb_close(279):9: ENTERING qb_rb_close()
trace May 07 15:26:52 my_posix_sem_destroy(91):9: ENTERING
my_posix_sem_destroy()
debug May 07 15:26:52 qb_rb_close(290):9: Free'ing ringbuffer:
/dev/shm/qb-cmap-event-13438-14116-18-header
trace May 07 15:26:52 qb_rb_close(279):9: ENTERING qb_rb_close()
trace May 07 15:26:52 my_posix_sem_destroy(91):9: ENTERING
my_posix_sem_destroy()
debug May 07 15:26:52 qb_rb_close(290):9: Free'ing ringbuffer:
/dev/shm/qb-cmap-request-13438-14116-18-header
debug May 07 15:26:52 handle_new_connection(476):2147483648: IPC
credentials authenticated (13438-14118-18)
debug May 07 15:26:52 qb_ipcs_shm_connect(294):9: connecting to client
[14118]
debug May 07 15:26:52 qb_rb_open_2(226):9: shm size:zd; real_size:zd;
rb->word_size:1048576
debug May 07 15:26:52 qb_rb_open_2(226):9: shm size:zd; real_size:zd;
rb->word_size:1048576
debug May 07 15:26:52 qb_rb_open_2(226):9: shm size:zd; real_size:zd;
rb->word_size:1048576
debug May 07 15:26:52 cs_ipcs_connection_created(269):8: connection
created
debug May 07 15:26:52 cmap_lib_init_fn(306):9: lib_init_fn:
conn=0x7feb08b0ea00
ERROR: qb_rb_chunk_read failed: Connection timed out
[trace] ENTERING qb_rb_close()
[debug] Free'ing ringbuffer: /dev/shm/qb-create_from_file-header
core dump
Reading symbols from /usr/sbin/corosync...done.
[New LWP 30095]
[New LWP 30096]
warning: Can't read pathname for load map: Input/output error.
[Thread debugging using libthread_db enabled]
Using host libthread_db library "/lib/x86_64-linux-gnu/libthread_db.so.1".
Core was generated by `corosync'.
Program terminated with signal 7, Bus error.
#0 0x00007f418e4ea823 in qb_rb_chunk_alloc (rb=0x7f418f9c0d20,
len=<optimized out>) at ringbuffer.c:456
456 rb->shared_data[write_pt] = 0;
(gdb) thread apply all bt
Thread 2 (Thread 0x7f418bf6f700 (LWP 30096)):
#0 0x00007f418e2cdfd0 in sem_wait () from
/lib/x86_64-linux-gnu/libpthread.so.0
#1 0x00007f418e4f45f3 in qb_logt_worker_thread (data=<optimized out>)
at log_thread.c:71
#2 0x00007f418e2c7e9a in start_thread () from
/lib/x86_64-linux-gnu/libpthread.so.0
#3 0x00007f418dff3cbd in clone () from /lib/x86_64-linux-gnu/libc.so.6
#4 0x0000000000000000 in ?? ()
Thread 1 (Thread 0x7f418ed96700 (LWP 30095)):
#0 0x00007f418e4ea823 in qb_rb_chunk_alloc (rb=0x7f418f9c0d20,
len=<optimized out>) at ringbuffer.c:456
#1 0x00007f418e4f4bb9 in _blackbox_vlogger (target=<optimized out>,
cs=0x7f418fa3e4d8, timestamp=1368027286,
ap=0x7fff3408ebf8) at log_blackbox.c:74
#2 0x00007f418e4f2dea in qb_log_real_va_ (cs=0x7f418fa3e4d8,
ap=0x7fff3408ee88) at log.c:184
#3 0x00007f418edbdf00 in _logsys_log_printf (level=<optimized out>,
subsys=<optimized out>,
function_name=<optimized out>, file_name=<optimized out>,
file_line=<optimized out>, format=<optimized out>)
at main.c:898
#4 0x00007f418e95f7bd in messages_free (token_aru=<optimized out>,
instance=0x7f418b732010) at totemsrp.c:2462
#5 message_handler_orf_token (instance=0x7f418b732010, msg=<optimized
out>, endian_conversion_needed=<optimized out>,
msg_len=<optimized out>) at totemsrp.c:3611
#6 0x00007f418e9617f7 in message_handler_orf_token (instance=<optimized
out>, msg=<optimized out>,
msg_len=<optimized out>, endian_conversion_needed=<optimized out>)
at totemsrp.c:3532
#7 0x00007f418e95bcc1 in rrp_deliver_fn (context=0x7f418f9ce850,
msg=0x7f418fa0f868, msg_len=70) at totemrrp.c:1783
#8 0x00007f418e956ad2 in net_deliver_fn (fd=<optimized out>,
revents=<optimized out>, data=0x7f418fa0f800)
at totemudp.c:521
#9 0x00007f418e4ec05f in _poll_dispatch_and_take_back_ (item=<optimized
out>, p=<optimized out>) at loop_poll.c:98
#10 0x00007f418e4ebbd7 in qb_loop_run_level (level=0x7f418f9c7268) at
loop.c:43
#11 qb_loop_run (lp=<optimized out>) at loop.c:204
#12 0x00007f418edacbe5 in main (argc=<optimized out>, argv=<optimized
out>, envp=<optimized out>) at main.c:1309
syslog
May 8 10:26:23 krusty corosync[30095]: [QUORUM] got nodeinfo message
from cluster node 167838851
May 8 10:26:23 krusty corosync[30095]: [QUORUM] nodeinfo
message[167838851]: votes: 1, expected: 17 flags: 1
May 8 10:26:23 krusty corosync[30095]: [QUORUM] flags: quorate: Yes
Leaving: No WFA Status: No First: No Qdevice: No QdeviceAlive: No
QdeviceCastVote: No QdeviceMasterWins: No
May 8 10:26:23 krusty corosync[30095]: [QUORUM] total_votes=8,
expected_votes=17
May 8 10:26:23 krusty corosync[30095]: [QUORUM] node 167838839
state=1, votes=1, expected=17
May 8 10:26:23 krusty corosync[30095]: [QUORUM] node 167838841
state=2, votes=1, expected=17
May 8 10:26:23 krusty corosync[30095]: [QUORUM] node 167838842
state=1, votes=1, expected=17
May 8 10:26:23 krusty corosync[30095]: [QUORUM] node 167838844
state=1, votes=1, expected=17
May 8 10:26:23 krusty corosync[30095]: [QUORUM] node 167838846
state=1, votes=1, expected=17
May 8 10:26:23 krusty corosync[30095]: [QUORUM] node 167838849
state=1, votes=1, expected=17
May 8 10:26:23 krusty corosync[30095]: [QUORUM] node 167838850
state=1, votes=1, expected=17
May 8 10:26:23 krusty corosync[30095]: [QUORUM] node 167838851
state=1, votes=1, expected=17
May 8 10:26:23 krusty corosync[30095]: [QUORUM] node 167838853
state=1, votes=1, expected=17
May 8 10:26:23 krusty corosync[30095]: [QUORUM] quorum lost, blocking
activity
May 8 10:26:23 krusty corosync[30095]: [QUORUM] got nodeinfo message
from cluster node 167838851
May 8 10:26:23 krusty corosync[30095]: [QUORUM] nodeinfo message[0]:
votes: 0, expected: 0 flags: 0
May 8 10:26:23 krusty corosync[30095]: [QUORUM] got nodeinfo message
from cluster node 167838853
May 8 10:26:23 krusty corosync[30095]: [QUORUM] nodeinfo
message[167838853]: votes: 1, expected: 17 flags: 1
May 8 10:26:23 krusty corosync[30095]: [QUORUM] flags: quorate: Yes
Leaving: No WFA Status: No First: No Qdevice: No QdeviceAlive: No
QdeviceCastVote: No QdeviceMasterWins: No
May 8 10:26:23 krusty corosync[30095]: [QUORUM] total_votes=8,
expected_votes=17
May 8 10:26:23 krusty corosync[30095]: [QUORUM] node 167838839
state=1, votes=1, expected=17
May 8 10:26:23 krusty corosync[30095]: [QUORUM] node 167838841
state=2, votes=1, expected=17
May 8 10:26:23 krusty corosync[30095]: [QUORUM] node 167838842
state=1, votes=1, expected=17
May 8 10:26:23 krusty corosync[30095]: [QUORUM] node 167838844
state=1, votes=1, expected=17
May 8 10:26:23 krusty corosync[30095]: [QUORUM] node 167838846
state=1, votes=1, expected=17
May 8 10:26:23 krusty corosync[30095]: [QUORUM] node 167838849
state=1, votes=1, expected=17
May 8 10:26:23 krusty corosync[30095]: [QUORUM] node 167838850
state=1, votes=1, expected=17
May 8 10:26:23 krusty corosync[30095]: [QUORUM] node 167838851
state=1, votes=1, expected=17
May 8 10:26:23 krusty corosync[30095]: [QUORUM] node 167838853
state=1, votes=1, expected=17
May 8 10:26:23 krusty corosync[30095]: [QUORUM] got nodeinfo message
from cluster node 167838853
May 8 10:26:23 krusty corosync[30095]: [QUORUM] nodeinfo message[0]:
votes: 0, expected: 0 flags: 0
May 8 10:26:23 krusty corosync[30095]: [QUORUM] got nodeinfo message
from cluster node 167838839
May 8 10:26:23 krusty corosync[30095]: [QUORUM] nodeinfo
message[167838839]: votes: 1, expected: 17 flags: 1
May 8 10:26:23 krusty corosync[30095]: [QUORUM] flags: quorate: Yes
Leaving: No WFA Status: No First: No Qdevice: No QdeviceAlive: No
QdeviceCastVote: No QdeviceMasterWins: No
May 8 10:26:23 krusty corosync[30095]: [QUORUM] total_votes=8,
expected_votes=17
May 8 10:26:23 krusty corosync[30095]: [QUORUM] node 167838839
state=1, votes=1, expected=17
May 8 10:26:23 krusty corosync[30095]: [QUORUM] node 167838841
state=2, votes=1, expected=17
May 8 10:26:23 krusty corosync[30095]: [QUORUM] node 167838842
state=1, votes=1, expected=17
May 8 10:26:23 krusty corosync[30095]: [QUORUM] node 167838844
state=1, votes=1, expected=17
May 8 10:26:23 krusty corosync[30095]: [QUORUM] node 167838846
state=1, votes=1, expected=17
May 8 10:26:23 krusty corosync[30095]: [QUORUM] node 167838849
state=1, votes=1, expected=17
May 8 10:26:23 krusty corosync[30095]: [QUORUM] node 167838850
state=1, votes=1, expected=17
May 8 10:26:23 krusty corosync[30095]: [QUORUM] node 167838851
state=1, votes=1, expected=17
May 8 10:26:23 krusty corosync[30095]: [QUORUM] node 167838853
state=1, votes=1, expected=17
May 8 10:26:23 krusty corosync[30095]: [QUORUM] got nodeinfo message
from cluster node 167838839
May 8 10:26:23 krusty corosync[30095]: [QUORUM] nodeinfo message[0]:
votes: 0, expected: 0 flags: 0
May 8 10:26:23 krusty corosync[30095]: [QUORUM] got nodeinfo message
from cluster node 167838842
May 8 10:26:23 krusty corosync[30095]: [QUORUM] nodeinfo
message[167838842]: votes: 1, expected: 17 flags: 1
May 8 10:26:23 krusty corosync[30095]: [QUORUM] flags: quorate: Yes
Leaving: No WFA Status: No First: No Qdevice: No QdeviceAlive: No
QdeviceCastVote: No QdeviceMasterWins: No
May 8 10:26:23 krusty corosync[30095]: [QUORUM] total_votes=8,
expected_votes=17
May 8 10:26:23 krusty corosync[30095]: [QUORUM] node 167838839
state=1, votes=1, expected=17
May 8 10:26:23 krusty corosync[30095]: [QUORUM] node 167838841
state=2, votes=1, expected=17
May 8 10:26:23 krusty corosync[30095]: [QUORUM] node 167838842
state=1, votes=1, expected=17
May 8 10:26:23 krusty corosync[30095]: [QUORUM] node 167838844
state=1, votes=1, expected=17
May 8 10:26:23 krusty corosync[30095]: [QUORUM] node 167838846
state=1, votes=1, expected=17
May 8 10:26:23 krusty corosync[30095]: [QUORUM] node 167838849
state=1, votes=1, expected=17
May 8 10:26:23 krusty corosync[30095]: [QUORUM] node 167838850
state=1, votes=1, expected=17
May 8 10:26:23 krusty corosync[30095]: [QUORUM] node 167838851
state=1, votes=1, expected=17
May 8 10:26:23 krusty corosync[30095]: [QUORUM] node 167838853
state=1, votes=1, expected=17
May 8 10:26:23 krusty corosync[30095]: [QUORUM] got nodeinfo message
from cluster node 167838842
May 8 10:26:23 krusty corosync[30095]: [QUORUM] nodeinfo message[0]:
votes: 0, expected: 0 flags: 0
May 8 10:26:23 krusty corosync[30095]: [QUORUM] got nodeinfo message
from cluster node 167838844
May 8 10:26:23 krusty corosync[30095]: [QUORUM] nodeinfo
message[167838844]: votes: 1, expected: 17 flags: 1
May 8 10:26:23 krusty corosync[30095]: [QUORUM] flags: quorate: Yes
Leaving: No WFA Status: No First: No Qdevice: No QdeviceAlive: No
QdeviceCastVote: No QdeviceMasterWins: No
May 8 10:26:23 krusty corosync[30095]: [QUORUM] total_votes=8,
expected_votes=17
May 8 10:26:23 krusty corosync[30095]: [QUORUM] node 167838839
state=1, votes=1, expected=17
May 8 10:26:23 krusty corosync[30095]: [QUORUM] node 167838841
state=2, votes=1, expected=17
May 8 10:26:23 krusty corosync[30095]: [QUORUM] node 167838842
state=1, votes=1, expected=17
May 8 10:26:23 krusty corosync[30095]: [QUORUM] node 167838844
state=1, votes=1, expected=17
May 8 10:26:23 krusty corosync[30095]: [QUORUM] node 167838846
state=1, votes=1, expected=17
May 8 10:26:23 krusty corosync[30095]: [QUORUM] node 167838849
state=1, votes=1, expected=17
May 8 10:26:23 krusty corosync[30095]: [QUORUM] node 167838850
state=1, votes=1, expected=17
May 8 10:26:23 krusty corosync[30095]: [QUORUM] node 167838851
state=1, votes=1, expected=17
May 8 10:26:23 krusty corosync[30095]: [QUORUM] node 167838853
state=1, votes=1, expected=17
May 8 10:26:23 krusty corosync[30095]: [QUORUM] got nodeinfo message
from cluster node 167838844
May 8 10:26:23 krusty corosync[30095]: [QUORUM] nodeinfo message[0]:
votes: 0, expected: 0 flags: 0
May 8 10:26:23 krusty corosync[30095]: [QUORUM] got nodeinfo message
from cluster node 167838846
May 8 10:26:23 krusty corosync[30095]: [QUORUM] nodeinfo
message[167838846]: votes: 1, expected: 17 flags: 1
May 8 10:26:23 krusty corosync[30095]: [QUORUM] flags: quorate: Yes
Leaving: No WFA Status: No First: No Qdevice: No QdeviceAlive: No
QdeviceCastVote: No QdeviceMasterWins: No
May 8 10:26:23 krusty corosync[30095]: [QUORUM] total_votes=8,
expected_votes=17
May 8 10:26:23 krusty corosync[30095]: [QUORUM] node 167838839
state=1, votes=1, expected=17
May 8 10:26:23 krusty corosync[30095]: [QUORUM] node 167838841
state=2, votes=1, expected=17
May 8 10:26:23 krusty corosync[30095]: [QUORUM] node 167838842
state=1, votes=1, expected=17
May 8 10:26:23 krusty corosync[30095]: [QUORUM] node 167838844
state=1, votes=1, expected=17
May 8 10:26:23 krusty corosync[30095]: [QUORUM] node 167838846
state=1, votes=1, expected=17
May 8 10:26:23 krusty corosync[30095]: [QUORUM] node 167838849
state=1, votes=1, expected=17
May 8 10:26:23 krusty rsyslogd-2177: imuxsock begins to drop messages
from pid 30095 due to rate-limiting
May 8 10:26:23 krusty crmd[30689]: notice: pcmk_quorum_notification:
Membership 47416: quorum lost (8)
May 8 10:26:24 krusty crmd[30689]: notice: pcmk_quorum_notification:
Membership 47420: quorum acquired (9)
May 8 10:26:25 krusty crmd[30689]: notice: corosync_node_name: Unable
to get node name for nodeid 167838841
May 8 10:27:37 krusty stonith-ng[30687]: notice: unpack_config: On
loss of CCM Quorum: Ignore
May 8 10:27:37 krusty stonith-ng[30687]: warning: create_node:
Detected multiple node entries with uname=quimby - this is rarely intended
May 8 10:27:37 krusty stonith-ng[30687]: warning: create_node:
Detected multiple node entries with uname=todd - this is rarely intended
May 8 10:27:37 krusty stonith-ng[30687]: warning: create_node:
Detected multiple node entries with uname=rod - this is rarely intended
May 8 10:27:37 krusty stonith-ng[30687]: warning: create_node:
Detected multiple node entries with uname=maude - this is rarely intended
May 8 10:27:37 krusty stonith-ng[30687]: warning: create_node:
Detected multiple node entries with uname=krusty - this is rarely intended
May 8 10:27:37 krusty stonith-ng[30687]: warning: create_node:
Detected multiple node entries with uname=itchy - this is rarely intended
May 8 10:27:37 krusty stonith-ng[30687]: warning: create_node:
Detected multiple node entries with uname=sideshowbob - this is rarely
intended
May 8 10:27:37 krusty stonith-ng[30687]: warning: create_node:
Detected multiple node entries with uname=willie - this is rarely intended
May 8 10:27:37 krusty stonith-ng[30687]: warning: create_node:
Detected multiple node entries with uname=ned - this is rarely intended
May 8 10:27:37 krusty stonith-ng[30687]: warning: create_node:
Detected multiple node entries with uname=moe - this is rarely intended
May 8 10:27:37 krusty stonith-ng[30687]: warning: create_node:
Detected multiple node entries with uname=nelson - this is rarely intended
May 8 10:27:37 krusty stonith-ng[30687]: warning: create_node:
Detected multiple node entries with uname=scratchy - this is rarely intended
May 8 10:27:37 krusty stonith-ng[30687]: warning: create_node:
Detected multiple node entries with uname=fattony - this is rarely intended
May 8 10:27:37 krusty stonith-ng[30687]: warning: create_node:
Detected multiple node entries with uname=barney - this is rarely intended
May 8 10:32:47 krusty stonith-ng[30687]: notice: unpack_config: On
loss of CCM Quorum: Ignore
May 8 10:32:47 krusty stonith-ng[30687]: warning: create_node:
Detected multiple node entries with uname=quimby - this is rarely intended
May 8 10:32:47 krusty stonith-ng[30687]: warning: create_node:
Detected multiple node entries with uname=todd - this is rarely intended
May 8 10:32:47 krusty stonith-ng[30687]: warning: create_node:
Detected multiple node entries with uname=rod - this is rarely intended
May 8 10:32:47 krusty stonith-ng[30687]: warning: create_node:
Detected multiple node entries with uname=maude - this is rarely intended
May 8 10:32:47 krusty stonith-ng[30687]: warning: create_node:
Detected multiple node entries with uname=krusty - this is rarely intended
May 8 10:32:47 krusty stonith-ng[30687]: warning: create_node:
Detected multiple node entries with uname=itchy - this is rarely intended
May 8 10:32:47 krusty stonith-ng[30687]: warning: create_node:
Detected multiple node entries with uname=sideshowbob - this is rarely
intended
May 8 10:32:47 krusty stonith-ng[30687]: warning: create_node:
Detected multiple node entries with uname=willie - this is rarely intended
May 8 10:32:47 krusty stonith-ng[30687]: warning: create_node:
Detected multiple node entries with uname=ned - this is rarely intended
May 8 10:32:47 krusty stonith-ng[30687]: warning: create_node:
Detected multiple node entries with uname=moe - this is rarely intended
May 8 10:32:47 krusty stonith-ng[30687]: warning: create_node:
Detected multiple node entries with uname=nelson - this is rarely intended
May 8 10:32:47 krusty stonith-ng[30687]: warning: create_node:
Detected multiple node entries with uname=scratchy - this is rarely intended
May 8 10:32:47 krusty stonith-ng[30687]: warning: create_node:
Detected multiple node entries with uname=fattony - this is rarely intended
May 8 10:32:47 krusty stonith-ng[30687]: warning: create_node:
Detected multiple node entries with uname=barney - this is rarely intended
May 8 10:34:23 krusty stonith-ng[30687]: notice: unpack_config: On
loss of CCM Quorum: Ignore
May 8 10:34:23 krusty stonith-ng[30687]: warning: create_node:
Detected multiple node entries with uname=quimby - this is rarely intended
May 8 10:34:23 krusty stonith-ng[30687]: warning: create_node:
Detected multiple node entries with uname=todd - this is rarely intended
May 8 10:34:23 krusty stonith-ng[30687]: warning: create_node:
Detected multiple node entries with uname=rod - this is rarely intended
May 8 10:34:23 krusty stonith-ng[30687]: warning: create_node:
Detected multiple node entries with uname=maude - this is rarely intended
May 8 10:34:23 krusty stonith-ng[30687]: warning: create_node:
Detected multiple node entries with uname=krusty - this is rarely intended
May 8 10:34:23 krusty stonith-ng[30687]: warning: create_node:
Detected multiple node entries with uname=itchy - this is rarely intended
May 8 10:34:23 krusty stonith-ng[30687]: warning: create_node:
Detected multiple node entries with uname=sideshowbob - this is rarely
intended
May 8 10:34:23 krusty stonith-ng[30687]: warning: create_node:
Detected multiple node entries with uname=willie - this is rarely intended
May 8 10:34:23 krusty stonith-ng[30687]: warning: create_node:
Detected multiple node entries with uname=ned - this is rarely intended
May 8 10:34:23 krusty stonith-ng[30687]: warning: create_node:
Detected multiple node entries with uname=moe - this is rarely intended
May 8 10:34:23 krusty stonith-ng[30687]: warning: create_node:
Detected multiple node entries with uname=nelson - this is rarely intended
May 8 10:34:23 krusty stonith-ng[30687]: warning: create_node:
Detected multiple node entries with uname=scratchy - this is rarely intended
May 8 10:34:23 krusty stonith-ng[30687]: warning: create_node:
Detected multiple node entries with uname=fattony - this is rarely intended
May 8 10:34:23 krusty stonith-ng[30687]: warning: create_node:
Detected multiple node entries with uname=barney - this is rarely intended
May 8 10:34:23 krusty stonith-ng[30687]: warning: unpack_rsc_location:
No resource (con=cli-standby-res_IPaddr2_NXTProduction,
rsc=res_IPaddr2_NXTProduction)
May 8 10:34:23 krusty stonith-ng[30687]: warning: unpack_rsc_location:
No resource (con=cli-prefer-res_IPaddr2_NXTProduction,
rsc=res_IPaddr2_NXTProduction)
May 8 10:34:46 krusty pacemakerd[30683]: error:
cfg_connection_destroy: Connection destroyed
May 8 10:34:46 krusty pacemakerd[30683]: notice:
pcmk_shutdown_worker: Shuting down Pacemaker
May 8 10:34:46 krusty pacemakerd[30683]: notice: stop_child: Stopping
crmd: Sent -15 to process 30689
May 8 10:34:46 krusty crmd[30689]: notice: crm_shutdown: Requesting
shutdown, upper limit is 1200000ms
May 8 10:34:46 krusty pacemakerd[30683]: error:
cpg_connection_destroy: Connection destroyed
May 8 10:34:46 krusty stonith-ng[30687]: error: pcmk_cpg_dispatch:
Connection to the CPG API failed: 2
May 8 10:34:46 krusty stonith-ng[30687]: error:
stonith_peer_ais_destroy: AIS connection terminated
May 8 10:34:46 krusty crmd[30689]: error: crm_ipc_read: Connection
to stonith-ng failed
May 8 10:34:46 krusty crmd[30689]: error: mainloop_gio_callback:
Connection to stonith-ng[0x1cce4e0] closed (I/O condition=17)
May 8 10:34:46 krusty crmd[30689]: crit:
tengine_stonith_connection_destroy: Fencing daemon connection failed
May 8 10:34:46 krusty attrd[30688]: error: pcmk_cpg_dispatch:
Connection to the CPG API failed: 2
May 8 10:34:46 krusty attrd[30688]: crit: attrd_ais_destroy: Lost
connection to Corosync service!
May 8 10:34:46 krusty attrd[30688]: notice: main: Exiting...
May 8 10:34:46 krusty attrd[30688]: notice: main: Disconnecting
client 0x22e9eb0, pid=30689...
May 8 10:34:46 krusty attrd[30688]: error:
attrd_cib_connection_destroy: Connection to the CIB terminated...
May 8 10:34:46 krusty cib[30686]: error: pcmk_cpg_dispatch:
Connection to the CPG API failed: 2
May 8 10:34:46 krusty cib[30686]: error: cib_ais_destroy: Corosync
connection lost! Exiting.
May 8 10:34:47 krusty crmd[30689]: error: te_connect_stonith:
Sign-in failed: triggered a retry
May 8 10:34:47 krusty crmd[30689]: error: crm_ipc_read: Connection
to cib_shm failed
May 8 10:34:47 krusty crmd[30689]: error: mainloop_gio_callback:
Connection to cib_shm[0x1baf020] closed (I/O condition=17)
May 8 10:34:47 krusty crmd[30689]: error:
crmd_cib_connection_destroy: Connection to the CIB terminated...
May 8 10:34:47 krusty crmd[30689]: error: crmd_quorum_destroy:
connection terminated
_______________________________________________
discuss mailing list
discuss@xxxxxxxxxxxx
http://lists.corosync.org/mailman/listinfo/discuss
