Hi CephFolks,
I have a cluster 14.2.21-22/Ubuntu 18.04 with 3 mon’s. After going down/restart of 1 mon(amon3) it stucks on probing and its also out of quorum, we have changed nothing and it was working, regardless we have checked tcp ports / mtu's of mons are open and reachable.
appreciate any help you can provide.
ceph health detail
HEALTH_WARN 1/3 mons down, quorum amon1,amon2
MON_DOWN 1/3 mons down, quorum amon1,amon2
mon.amon3 (rank 0) addr [v2:172.16.12.83:3300/0,v1:172.16.12.83:6789/0] is down (out of quorum)
all mon daemons are up and running, monmaptool printout is also equal on all mon nodes and on other healthy mon nodes log there are entries from amon3:
Log from healthy amon1 log:
2021-09-10 10:49:16.657 7fc8285f3700 0 log_channel(audit) log [DBG] : from='client.? 172.16.12.83:0/245515484' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch
2021-09-10 11:04:16.586 7fc8285f3700 0 log_channel(audit) log [DBG] : from='client.? 172.16.12.83:0/2472330429' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch
2021-09-10 11:06:06.245 7fc8285f3700 0 log_channel(audit) log [DBG] : from='client.? 172.16.12.83:0/1444461156' entity='client.admin' cmd=[{"prefix": "mon getmap"}]: dispatch
2021-09-10 11:19:16.581 7fc8285f3700 0 log_channel(audit) log [DBG] : from='client.? 172.16.12.83:0/964654151' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch
#########
monmaptool --print /tmp/monmap
monmaptool: monmap file /tmp/monmap
epoch 14
fsid 10a40588-d89e-4fac-be88-afcedc1a7372
last_changed 2021-06-22 13:35:56.211202
created 2018-07-10 12:37:03.124657
min_mon_release 14 (nautilus)
0: [v2:172.16.12.83:3300/0,v1:172.16.12.83:6789/0] mon.amon3
1: [v2:172.16.12.81:3300/0,v1:172.16.12.81:6789/0] mon.amon1
2: [v2:172.16.12.82:3300/0,v1:172.16.12.82:6789/0] mon.amon2
##########
ceph config get mon.*
WHO MASK LEVEL OPTION VALUE RO
mon advanced mon_crush_min_required_version firefly *
mon advanced mon_warn_on_insecure_global_id_reclaim_allowed false
##########
ceph --admin-daemon /var/run/ceph/ceph-mon.amon3.asok mon_status
{
"name": "amon3",
"rank": 0,
"state": "probing",
"election_epoch": 0,
"quorum": [],
"features": {
"required_con": "2449958747315912708",
"required_mon": [
"kraken",
"luminous",
"mimic",
"osdmap-prune",
"nautilus"
],
"quorum_con": "0",
"quorum_mon": []
},
"outside_quorum": [
"amon3"
],
"extra_probe_peers": [],
"sync_provider": [],
"monmap": {
"epoch": 14,
"fsid": "10a40588-d89e-4fac-be88-afcedc1a7372",
"modified": "2021-06-22 13:35:56.211202",
"created": "2018-07-10 12:37:03.124657",
"min_mon_release": 14,
"min_mon_release_name": "nautilus",
"features": {
"persistent": [
"kraken",
"luminous",
"mimic",
"osdmap-prune",
"nautilus"
],
"optional": []
},
"mons": [
{
"rank": 0,
"name": "amon3",
"public_addrs": {
"addrvec": [
{
"type": "v2",
"addr": "172.16.12.83:3300",
"nonce": 0
},
{
"type": "v1",
"addr": "172.16.12.83:6789",
"nonce": 0
}
]
},
"addr": "172.16.12.83:6789/0",
"public_addr": "172.16.12.83:6789/0"
},
{
"rank": 1,
"name": "amon1",
"public_addrs": {
"addrvec": [
{
"type": "v2",
"addr": "172.16.12.81:3300",
"nonce": 0
},
{
"type": "v1",
"addr": "172.16.12.81:6789",
"nonce": 0
}
]
},
"addr": "172.16.12.81:6789/0",
"public_addr": "172.16.12.81:6789/0"
},
{
"rank": 2,
"name": "amon2",
"public_addrs": {
"addrvec": [
{
"type": "v2",
"addr": "172.16.12.82:3300",
"nonce": 0
},
{
"type": "v1",
"addr": "172.16.12.82:6789",
"nonce": 0
}
]
},
"addr": "172.16.12.82:6789/0",
"public_addr": "172.16.12.82:6789/0"
}
]
},
"feature_map": {
"mon": [
{
"features": "0x3ffddff8ffecffff",
"release": "luminous",
"num": 1
}
]
}
}
#########
ceph --admin-daemon /var/run/ceph/ceph-mon.amon2.asok mon_status
{
"name": "amon2",
"rank": 2,
"state": "peon",
"election_epoch": 4612,
"quorum": [
1,
2
],
"quorum_age": 65650,
"features": {
"required_con": "2449958747315912708",
"required_mon": [
"kraken",
"luminous",
"mimic",
"osdmap-prune",
"nautilus"
],
"quorum_con": "4611087854035861503",
"quorum_mon": [
"kraken",
"luminous",
"mimic",
"osdmap-prune",
"nautilus"
]
},
"outside_quorum": [],
"extra_probe_peers": [],
"sync_provider": [],
"monmap": {
"epoch": 14,
"fsid": "10a40588-d89e-4fac-be88-afcedc1a7372",
"modified": "2021-06-22 11:35:56.211202",
"created": "2018-07-10 10:37:03.124657",
"min_mon_release": 14,
"min_mon_release_name": "nautilus",
"features": {
"persistent": [
"kraken",
"luminous",
"mimic",
"osdmap-prune",
"nautilus"
],
"optional": []
},
"mons": [
{
"rank": 0,
"name": "amon3",
"public_addrs": {
"addrvec": [
{
"type": "v2",
"addr": "172.16.12.83:3300",
"nonce": 0
},
{
"type": "v1",
"addr": "172.16.12.83:6789",
"nonce": 0
}
]
},
"addr": "172.16.12.83:6789/0",
"public_addr": "172.16.12.83:6789/0"
},
{
"rank": 1,
"name": "amon1",
"public_addrs": {
"addrvec": [
{
"type": "v2",
"addr": "172.16.12.81:3300",
"nonce": 0
},
{
"type": "v1",
"addr": "172.16.12.81:6789",
"nonce": 0
}
]
},
"addr": "172.16.12.81:6789/0",
"public_addr": "172.16.12.81:6789/0"
},
{
"rank": 2,
"name": "amon2",
"public_addrs": {
"addrvec": [
{
"type": "v2",
"addr": "172.16.12.82:3300",
"nonce": 0
},
{
"type": "v1",
"addr": "172.16.12.82:6789",
"nonce": 0
}
]
},
"addr": "172.16.12.82:6789/0",
"public_addr": "172.16.12.82:6789/0"
}
]
},
"feature_map": {
"mon": [
{
"features": "0x3ffddff8ffecffff",
"release": "luminous",
"num": 1
}
],
"osd": [
{
"features": "0x3ffddff8ffacffff",
"release": "luminous",
"num": 18
},
{
"features": "0x3ffddff8ffecffff",
"release": "luminous",
"num": 11
}
],
"client": [
{
"features": "0x3ffddff8ffacffff",
"release": "luminous",
"num": 30
},
{
"features": "0x3ffddff8ffecffff",
"release": "luminous",
"num": 7
}
],
"mgr": [
{
"features": "0x3ffddff8ffecffff",
"release": "luminous",
"num": 2
}
]
}
}
###########
ceph --admin-daemon /var/run/ceph/ceph-mon.amon1.asok mon_status
{
"name": "amon1",
"rank": 1,
"state": "leader",
"election_epoch": 4612,
"quorum": [
1,
2
],
"quorum_age": 65744,
"features": {
"required_con": "2449958747315912708",
"required_mon": [
"kraken",
"luminous",
"mimic",
"osdmap-prune",
"nautilus"
],
"quorum_con": "4611087854035861503",
"quorum_mon": [
"kraken",
"luminous",
"mimic",
"osdmap-prune",
"nautilus"
]
},
"outside_quorum": [],
"extra_probe_peers": [
{
"addrvec": [
{
"type": "v2",
"addr": "172.16.12.82:3300",
"nonce": 0
},
{
"type": "v1",
"addr": "172.16.12.82:6789",
"nonce": 0
}
]
}
],
"sync_provider": [],
"monmap": {
"epoch": 14,
"fsid": "10a40588-d89e-4fac-be88-afcedc1a7372",
"modified": "2021-06-22 13:35:56.211202",
"created": "2018-07-10 12:37:03.124657",
"min_mon_release": 14,
"min_mon_release_name": "nautilus",
"features": {
"persistent": [
"kraken",
"luminous",
"mimic",
"osdmap-prune",
"nautilus"
],
"optional": []
},
"mons": [
{
"rank": 0,
"name": "amon3",
"public_addrs": {
"addrvec": [
{
"type": "v2",
"addr": "172.16.12.83:3300",
"nonce": 0
},
{
"type": "v1",
"addr": "172.16.12.83:6789",
"nonce": 0
}
]
},
"addr": "172.16.12.83:6789/0",
"public_addr": "172.16.12.83:6789/0"
},
{
"rank": 1,
"name": "amon1",
"public_addrs": {
"addrvec": [
{
"type": "v2",
"addr": "172.16.12.81:3300",
"nonce": 0
},
{
"type": "v1",
"addr": "172.16.12.81:6789",
"nonce": 0
}
]
},
"addr": "172.16.12.81:6789/0",
"public_addr": "172.16.12.81:6789/0"
},
{
"rank": 2,
"name": "amon2",
"public_addrs": {
"addrvec": [
{
"type": "v2",
"addr": "172.16.12.82:3300",
"nonce": 0
},
{
"type": "v1",
"addr": "172.16.12.82:6789",
"nonce": 0
}
]
},
"addr": "172.16.12.82:6789/0",
"public_addr": "172.16.12.82:6789/0"
}
]
},
"feature_map": {
"mon": [
{
"features": "0x3ffddff8ffecffff",
"release": "luminous",
"num": 1
}
],
"osd": [
{
"features": "0x3ffddff8ffacffff",
"release": "luminous",
"num": 39
},
{
"features": "0x3ffddff8ffecffff",
"release": "luminous",
"num": 8
}
],
"client": [
{
"features": "0x3ffddff8ffacfffb",
"release": "luminous",
"num": 1
},
{
"features": "0x3ffddff8ffacffff",
"release": "luminous",
"num": 32
},
{
"features": "0x3ffddff8ffecffff",
"release": "luminous",
"num": 12
}
],
"mgr": [
{
"features": "0x3ffddff8ffecffff",
"release": "luminous",
"num": 1
}
]
}
}
########
Amon3 log
2021-09-10 12:06:16.409 7f1ead2033c0 0 starting mon.amon3 rank 0 at public addrs [v2:172.16.12.83:3300/0,v1:172.16.12.83:6789/0] at bind addrs [v2:172.16.12.83:3300/0,v1:172.16.12.83:6789/0] mon_data /var/lib/ceph/mon/ceph-amon3 fsid 10a40588-d89e-4fac-be88-afcedc1a7372
2021-09-10 12:06:16.413 7f1ead2033c0 1 mon.amon3@-1(???) e14 preinit fsid 10a40588-d89e-4fac-be88-afcedc1a7372
2021-09-10 12:06:16.413 7f1ead2033c0 10 mon.amon3@-1(???) e14 check_fsid cluster_uuid contains '10a40588-d89e-4fac-be88-afcedc1a7372'
2021-09-10 12:06:16.413 7f1ead2033c0 10 mon.amon3@-1(???) e14 features compat={},rocompat={},incompat={1=initial feature set (~v.18),3=single paxos with k/v store (v0.?),4=support erasure code pools,5=new-style osdmap encoding,6=support isa/lrc erasure code,7=support shec erasure code,8=support monmap features,9=luminous ondisk layout,10=mimic ondisk layout,11=nautilus ondisk layout}
2021-09-10 12:06:16.413 7f1ead2033c0 10 mon.amon3@-1(???) e14 calc_quorum_requirements required_features 2449958747315912708
2021-09-10 12:06:16.413 7f1ead2033c0 10 mon.amon3@-1(???) e14 required_features 2449958747315912708
2021-09-10 12:06:16.413 7f1ead2033c0 10 mon.amon3@-1(???) e14 has_ever_joined = 1
2021-09-10 12:06:16.413 7f1ead2033c0 1 mon.amon3@-1(???) e14 preinit clean up potentially inconsistent store state
2021-09-10 12:06:16.413 7f1ead2033c0 10 mon.amon3@-1(???) e14 sync_last_committed_floor 353360965
2021-09-10 12:06:16.413 7f1ead2033c0 10 mon.amon3@-1(???) e14 init_paxos
2021-09-10 12:06:16.413 7f1ead2033c0 5 mon.amon3@-1(???).mds e0 Unable to load 'last_metadata'
2021-09-10 12:06:16.413 7f1ead2033c0 10 mon.amon3@-1(???).health init
2021-09-10 12:06:16.413 7f1ead2033c0 10 mon.amon3@-1(???).config init
2021-09-10 12:06:16.413 7f1ead2033c0 10 mon.amon3@-1(???) e14 refresh_from_paxos
2021-09-10 12:06:16.413 7f1ead2033c0 10 mon.amon3@-1(???).log v0 update_from_paxos
2021-09-10 12:06:16.413 7f1ead2033c0 10 mon.amon3@-1(???).log v0 update_from_paxos version 0 summary v 0
2021-09-10 12:06:16.413 7f1ead2033c0 10 mon.amon3@-1(???).auth v0 update_from_paxos
2021-09-10 12:06:16.413 7f1ead2033c0 10 mon.amon3@-1(???).config load_config got 0 keys
2021-09-10 12:06:16.413 7f1ead2033c0 10 mon.amon3@-1(???).mgrstat 0
2021-09-10 12:06:16.413 7f1ead2033c0 10 mon.amon3@-1(???).mgrstat check_subs
2021-09-10 12:06:16.413 7f1ead2033c0 10 mon.amon3@-1(???).health update_from_paxos
2021-09-10 12:06:16.413 7f1ead2033c0 1 -- [v2:172.16.12.83:3300/0,v1:172.16.12.83:6789/0] learned_addr learned my addr [v2:172.16.12.83:3300/0,v1:172.16.12.83:6789/0] (peer_addr_for_me v2:172.16.12.83:3300/0)
2021-09-10 12:06:16.413 7f1ead2033c0 1 -- [v2:172.16.12.83:3300/0,v1:172.16.12.83:6789/0] _finish_bind bind my_addrs is [v2:172.16.12.83:3300/0,v1:172.16.12.83:6789/0]
2021-09-10 12:06:16.413 7f1ead2033c0 1 -- [v2:172.16.12.83:3300/0,v1:172.16.12.83:6789/0] start start
2021-09-10 12:06:16.413 7f1ead2033c0 1 -- start start
2021-09-10 12:06:16.413 7f1ead2033c0 2 mon.amon3@-1(???) e14 init
2021-09-10 12:06:16.417 7f1ead2033c0 1 Processor -- start
2021-09-10 12:06:16.417 7f1ead2033c0 1 Processor -- start
2021-09-10 12:06:16.417 7f1ead2033c0 10 mon.amon3@-1(???).mgr e0 prime_mgr_client
2021-09-10 12:06:16.417 7f1ead2033c0 10 mon.amon3@-1(probing) e14 bootstrap
2021-09-10 12:06:16.417 7f1ead2033c0 10 mon.amon3@-1(probing) e14 sync_reset_requester
2021-09-10 12:06:16.417 7f1ead2033c0 10 mon.amon3@-1(probing) e14 unregister_cluster_logger - not registered
2021-09-10 12:06:16.417 7f1ead2033c0 10 mon.amon3@-1(probing) e14 cancel_probe_timeout (none scheduled)
2021-09-10 12:06:16.417 7f1ead2033c0 10 mon.amon3@-1(probing) e14 monmap e14: 3 mons at {amon1=[v2:172.16.12.81:3300/0,v1:172.16.12.81:6789/0],amon2=[v2:172.16.12.82:3300/0,v1:172.16.12.82:6789/0],amon3=[v2:172.16.12.83:3300/0,v1:172.16.12.83:6789/0]}
2021-09-10 12:06:16.417 7f1ead2033c0 0 mon.amon3@-1(probing) e14 my rank is now 0 (was -1)
2021-09-10 12:06:16.417 7f1ead2033c0 1 -- [v2:172.16.12.83:3300/0,v1:172.16.12.83:6789/0] shutdown_connections
2021-09-10 12:06:16.417 7f1ead2033c0 10 mon.amon3@0(probing) e14 _reset
2021-09-10 12:06:16.417 7f1ead2033c0 10 mon.amon3@0(probing).auth v0 _set_mon_num_rank num 0 rank 0
2021-09-10 12:06:16.417 7f1ead2033c0 10 mon.amon3@0(probing) e14 cancel_probe_timeout (none scheduled)
2021-09-10 12:06:16.417 7f1ead2033c0 10 mon.amon3@0(probing) e14 timecheck_finish
2021-09-10 12:06:16.417 7f1ead2033c0 10 mon.amon3@0(probing) e14 scrub_event_cancel
2021-09-10 12:06:16.417 7f1ead2033c0 10 mon.amon3@0(probing) e14 scrub_reset
2021-09-10 12:06:16.417 7f1ead2033c0 10 mon.amon3@0(probing) e14 cancel_probe_timeout (none scheduled)
2021-09-10 12:06:16.417 7f1ead2033c0 10 mon.amon3@0(probing) e14 reset_probe_timeout 0x55ffa1907860 after 2 seconds
2021-09-10 12:06:16.417 7f1ead2033c0 10 mon.amon3@0(probing) e14 probing other monitors
2021-09-10 12:06:16.417 7f1ead2033c0 1 -- [v2:172.16.12.83:3300/0,v1:172.16.12.83:6789/0] send_to--> mon [v2:172.16.12.81:3300/0,v1:172.16.12.81:6789/0] -- mon_probe(probe 10a40588-d89e-4fac-be88-afcedc1a7372 name amon3 mon_release 14) v7 -- ?+0 0x55ffa0956a00
2021-09-10 12:06:16.417 7f1ead2033c0 1 --2- [v2:172.16.12.83:3300/0,v1:172.16.12.83:6789/0] >> [v2:172.16.12.81:3300/0,v1:172.16.12.81:6789/0] conn(0x55ffa096f600 0x55ffa15d1c00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 rx=0 tx=0).connect
2021-09-10 12:06:16.417 7f1ead2033c0 1 -- [v2:172.16.12.83:3300/0,v1:172.16.12.83:6789/0] --> [v2:172.16.12.81:3300/0,v1:172.16.12.81:6789/0] -- mon_probe(probe 10a40588-d89e-4fac-be88-afcedc1a7372 name amon3 mon_release 14) v7 -- 0x55ffa0956a00 con 0x55ffa096f600
2021-09-10 12:06:16.417 7f1ead2033c0 1 -- [v2:172.16.12.83:3300/0,v1:172.16.12.83:6789/0] send_to--> mon [v2:172.16.12.82:3300/0,v1:172.16.12.82:6789/0] -- mon_probe(probe 10a40588-d89e-4fac-be88-afcedc1a7372 name amon3 mon_release 14) v7 -- ?+0 0x55ffa0956780
2021-09-10 12:06:16.417 7f1ead2033c0 1 --2- [v2:172.16.12.83:3300/0,v1:172.16.12.83:6789/0] >> [v2:172.16.12.82:3300/0,v1:172.16.12.82:6789/0] conn(0x55ffa096fa80 0x55ffa15d2300 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 rx=0 tx=0).connect
2021-09-10 12:06:16.417 7f1ead2033c0 1 -- [v2:172.16.12.83:3300/0,v1:172.16.12.83:6789/0] --> [v2:172.16.12.82:3300/0,v1:172.16.12.82:6789/0] -- mon_probe(probe 10a40588-d89e-4fac-be88-afcedc1a7372 name amon3 mon_release 14) v7 -- 0x55ffa0956780 con 0x55ffa096fa80
2021-09-10 12:06:16.417 7f1e99e04700 1 --2- [v2:172.16.12.83:3300/0,v1:172.16.12.83:6789/0] >> [v2:172.16.12.81:3300/0,v1:172.16.12.81:6789/0] conn(0x55ffa096f600 0x55ffa15d1c00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0
2021-09-10 12:06:16.417 7f1e99e04700 10 mon.amon3@0(probing) e14 ms_get_authorizer for mon
2021-09-10 12:06:16.417 7f1e99e04700 1 -- [v2:172.16.12.83:3300/0,v1:172.16.12.83:6789/0] >> [v2:172.16.12.81:3300/0,v1:172.16.12.81:6789/0] conn(0x55ffa096f600 msgr2=0x55ffa15d1c00 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_bulk peer close file descriptor 34
2021-09-10 12:06:16.417 7f1e99e04700 1 -- [v2:172.16.12.83:3300/0,v1:172.16.12.83:6789/0] >> [v2:172.16.12.81:3300/0,v1:172.16.12.81:6789/0] conn(0x55ffa096f600 msgr2=0x55ffa15d1c00 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_until read failed
2021-09-10 12:06:16.417 7f1e99e04700 1 --2- [v2:172.16.12.83:3300/0,v1:172.16.12.83:6789/0] >> [v2:172.16.12.81:3300/0,v1:172.16.12.81:6789/0] conn(0x55ffa096f600 0x55ffa15d1c00 crc :-1 s=SESSION_CONNECTING pgs=0 cs=0 l=0 rev1=1 rx=0 tx=0).handle_read_frame_preamble_main read frame preamble failed r=-1 ((1) Operation not permitted)
2021-09-10 12:06:16.417 7f1e99e04700 1 --2- [v2:172.16.12.83:3300/0,v1:172.16.12.83:6789/0] >> [v2:172.16.12.81:3300/0,v1:172.16.12.81:6789/0] conn(0x55ffa096f600 0x55ffa15d1c00 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=1 rx=0 tx=0)._fault waiting 0.200000
2021-09-10 12:06:16.421 7f1e99603700 1 --2- [v2:172.16.12.83:3300/0,v1:172.16.12.83:6789/0] >> [v2:172.16.12.82:3300/0,v1:172.16.12.82:6789/0] conn(0x55ffa096fa80 0x55ffa15d2300 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0
2021-09-10 12:06:16.421 7f1e99603700 10 mon.amon3@0(probing) e14 ms_get_authorizer for mon
2021-09-10 12:06:16.425 7f1e99603700 1 -- [v2:172.16.12.83:3300/0,v1:172.16.12.83:6789/0] >> [v2:172.16.12.82:3300/0,v1:172.16.12.82:6789/0] conn(0x55ffa096fa80 msgr2=0x55ffa15d2300 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_bulk peer close file descriptor 37
2021-09-10 12:06:16.425 7f1e99603700 1 -- [v2:172.16.12.83:3300/0,v1:172.16.12.83:6789/0] >> [v2:172.16.12.82:3300/0,v1:172.16.12.82:6789/0] conn(0x55ffa096fa80 msgr2=0x55ffa15d2300 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_until read failed
2021-09-10 12:06:16.425 7f1e99603700 1 --2- [v2:172.16.12.83:3300/0,v1:172.16.12.83:6789/0] >> [v2:172.16.12.82:3300/0,v1:172.16.12.82:6789/0] conn(0x55ffa096fa80 0x55ffa15d2300 crc :-1 s=SESSION_CONNECTING pgs=0 cs=0 l=0 rev1=1 rx=0 tx=0).handle_read_frame_preamble_main read frame preamble failed r=-1 ((1) Operation not permitted)
2021-09-10 12:06:16.425 7f1e99603700 1 --2- [v2:172.16.12.83:3300/0,v1:172.16.12.83:6789/0] >> [v2:172.16.12.82:3300/0,v1:172.16.12.82:6789/0] conn(0x55ffa096fa80 0x55ffa15d2300 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=1 rx=0 tx=0)._fault waiting 0.200000
2021-09-10 12:06:16.617 7f1e99e04700 1 --2- [v2:172.16.12.83:3300/0,v1:172.16.12.83:6789/0] >> [v2:172.16.12.81:3300/0,v1:172.16.12.81:6789/0] conn(0x55ffa096f600 0x55ffa15d1c00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=1 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0
2021-09-10 12:06:16.617 7f1e99e04700 10 mon.amon3@0(probing) e14 ms_get_authorizer for mon
2021-09-10 12:06:16.621 7f1e99e04700 1 -- [v2:172.16.12.83:3300/0,v1:172.16.12.83:6789/0] >> [v2:172.16.12.81:3300/0,v1:172.16.12.81:6789/0] conn(0x55ffa096f600 msgr2=0x55ffa15d1c00 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_bulk peer close file descriptor 34
2021-09-10 12:06:16.621 7f1e99e04700 1 -- [v2:172.16.12.83:3300/0,v1:172.16.12.83:6789/0] >> [v2:172.16.12.81:3300/0,v1:172.16.12.81:6789/0] conn(0x55ffa096f600 msgr2=0x55ffa15d1c00 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_until read failed
2021-09-10 12:06:16.621 7f1e99e04700 1 --2- [v2:172.16.12.83:3300/0,v1:172.16.12.83:6789/0] >> [v2:172.16.12.81:3300/0,v1:172.16.12.81:6789/0] conn(0x55ffa096f600 0x55ffa15d1c00 crc :-1 s=SESSION_CONNECTING pgs=0 cs=0 l=0 rev1=1 rx=0 tx=0).handle_read_frame_preamble_main read frame preamble failed r=-1 ((1) Operation not permitted)
2021-09-10 12:06:16.621 7f1e99e04700 1 --2- [v2:172.16.12.83:3300/0,v1:172.16.12.83:6789/0] >> [v2:172.16.12.81:3300/0,v1:172.16.12.81:6789/0] conn(0x55ffa096f600 0x55ffa15d1c00 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=1 rx=0 tx=0)._fault waiting 0.400000
2021-09-10 12:06:16.625 7f1e99603700 1 --2- [v2:172.16.12.83:3300/0,v1:172.16.12.83:6789/0] >> [v2:172.16.12.82:3300/0,v1:172.16.12.82:6789/0] conn(0x55ffa096fa80 0x55ffa15d2300 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=1 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0
2021-09-10 12:06:16.625 7f1e99603700 10 mon.amon3@0(probing) e14 ms_get_authorizer for mon
2021-09-10 12:06:16.625 7f1e99603700 1 -- [v2:172.16.12.83:3300/0,v1:172.16.12.83:6789/0] >> [v2:172.16.12.82:3300/0,v1:172.16.12.82:6789/0] conn(0x55ffa096fa80 msgr2=0x55ffa15d2300 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_bulk peer close file descriptor 34
2021-09-10 12:06:16.625 7f1e99603700 1 -- [v2:172.16.12.83:3300/0,v1:172.16.12.83:6789/0] >> [v2:172.16.12.82:3300/0,v1:172.16.12.82:6789/0] conn(0x55ffa096fa80 msgr2=0x55ffa15d2300 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_until read failed
2021-09-10 12:06:16.625 7f1e99603700 1 --2- [v2:172.16.12.83:3300/0,v1:172.16.12.83:6789/0] >> [v2:172.16.12.82:3300/0,v1:172.16.12.82:6789/0] conn(0x55ffa096fa80 0x55ffa15d2300 crc :-1 s=SESSION_CONNECTING pgs=0 cs=0 l=0 rev1=1 rx=0 tx=0).handle_read_frame_preamble_main read frame preamble failed r=-1 ((1) Operation not permitted)
2021-09-10 12:06:16.625 7f1e99603700 1 --2- [v2:172.16.12.83:3300/0,v1:172.16.12.83:6789/0] >> [v2:172.16.12.82:3300/0,v1:172.16.12.82:6789/0] conn(0x55ffa096fa80 0x55ffa15d2300 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=1 rx=0 tx=0)._fault waiting 0.400000
2021-09-10 12:06:17.021 7f1e99e04700 1 --2- [v2:172.16.12.83:3300/0,v1:172.16.12.83:6789/0] >> [v2:172.16.12.81:3300/0,v1:172.16.12.81:6789/0] conn(0x55ffa096f600 0x55ffa15d1c00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=1 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0
2021-09-10 12:06:17.021 7f1e99e04700 10 mon.amon3@0(probing) e14 ms_get_authorizer for mon
2021-09-10 12:06:17.021 7f1e99e04700 1 -- [v2:172.16.12.83:3300/0,v1:172.16.12.83:6789/0] >> [v2:172.16.12.81:3300/0,v1:172.16.12.81:6789/0] conn(0x55ffa096f600 msgr2=0x55ffa15d1c00 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_bulk peer close file descriptor 34
2021-09-10 12:06:17.021 7f1e99e04700 1 -- [v2:172.16.12.83:3300/0,v1:172.16.12.83:6789/0] >> [v2:172.16.12.81:3300/0,v1:172.16.12.81:6789/0] conn(0x55ffa096f600 msgr2=0x55ffa15d1c00 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_until read failed
2021-09-10 12:06:17.021 7f1e99e04700 1 --2- [v2:172.16.12.83:3300/0,v1:172.16.12.83:6789/0] >> [v2:172.16.12.81:3300/0,v1:172.16.12.81:6789/0] conn(0x55ffa096f600 0x55ffa15d1c00 crc :-1 s=SESSION_CONNECTING pgs=0 cs=0 l=0 rev1=1 rx=0 tx=0).handle_read_frame_preamble_main read frame preamble failed r=-1 ((1) Operation not permitted)
2021-09-10 12:06:17.021 7f1e99e04700 1 --2- [v2:172.16.12.83:3300/0,v1:172.16.12.83:6789/0] >> [v2:172.16.12.81:3300/0,v1:172.16.12.81:6789/0] conn(0x55ffa096f600 0x55ffa15d1c00 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=1 rx=0 tx=0)._fault waiting 0.800000
2021-09-10 12:06:17.029 7f1e99603700 1 --2- [v2:172.16.12.83:3300/0,v1:172.16.12.83:6789/0] >> [v2:172.16.12.82:3300/0,v1:172.16.12.82:6789/0] conn(0x55ffa096fa80 0x55ffa15d2300 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=1 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0
2021-09-10 12:06:17.029 7f1e99603700 10 mon.amon3@0(probing) e14 ms_get_authorizer for mon
2021-09-10 12:06:17.029 7f1e99603700 1 -- [v2:172.16.12.83:3300/0,v1:172.16.12.83:6789/0] >> [v2:172.16.12.82:3300/0,v1:172.16.12.82:6789/0] conn(0x55ffa096fa80 msgr2=0x55ffa15d2300 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_bulk peer close file descriptor 34
2021-09-10 12:06:17.029 7f1e99603700 1 -- [v2:172.16.12.83:3300/0,v1:172.16.12.83:6789/0] >> [v2:172.16.12.82:3300/0,v1:172.16.12.82:6789/0] conn(0x55ffa096fa80 msgr2=0x55ffa15d2300 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_until read failed
2021-09-10 12:06:17.029 7f1e99603700 1 --2- [v2:172.16.12.83:3300/0,v1:172.16.12.83:6789/0] >> [v2:172.16.12.82:3300/0,v1:172.16.12.82:6789/0] conn(0x55ffa096fa80 0x55ffa15d2300 crc :-1 s=SESSION_CONNECTING pgs=0 cs=0 l=0 rev1=1 rx=0 tx=0).handle_read_frame_preamble_main read frame preamble failed r=-1 ((1) Operation not permitted)
2021-09-10 12:06:17.029 7f1e99603700 1 --2- [v2:172.16.12.83:3300/0,v1:172.16.12.83:6789/0] >> [v2:172.16.12.82:3300/0,v1:172.16.12.82:6789/0] conn(0x55ffa096fa80 0x55ffa15d2300 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=1 rx=0 tx=0)._fault waiting 0.800000
2021-09-10 12:06:17.613 7f1e94dfa700 1 --1- [v2:172.16.12.83:3300/0,v1:172.16.12.83:6789/0] >> conn(0x55ffa193c000 0x55ffa1944000 :6789 s=ACCEPTING pgs=0 cs=0 l=0).send_server_banner sd=34 legacy v1:172.16.12.83:6789/0 socket_addr v1:172.16.12.83:6789/0 target_addr v1:172.16.12.19:41104/0
2021-09-10 12:06:17.613 7f1e955fb700 10 mon.amon3@0(probing) e14 ms_handle_accept con 0x55ffa193c000 no session
2021-09-10 12:06:17.613 7f1e955fb700 1 -- [v2:172.16.12.83:3300/0,v1:172.16.12.83:6789/0] <== client.? v1:172.16.12.19:0/10663437 1 ==== auth(proto 0 30 bytes epoch 0) v1 ==== 60+0+0 (unknown 900162395 0 0) 0x55ffa194c000 con 0x55ffa193c000
2021-09-10 12:06:17.613 7f1e955fb700 10 mon.amon3@0(probing) e14 _ms_dispatch new session 0x55ffa194c240 MonSession(client.? v1:172.16.12.19:0/10663437 is open , features 0x3ffddff8ffecffff (luminous)) features 0x3ffddff8ffecffff
2021-09-10 12:06:17.613 7f1e955fb700 5 mon.amon3@0(probing) e14 waitlisting message auth(proto 0 30 bytes epoch 0) v1
2021-09-10 12:06:17.613 7f1e94dfa700 1 -- [v2:172.16.12.83:3300/0,v1:172.16.12.83:6789/0] >> v1:172.16.12.19:0/10663437 conn(0x55ffa193c000 legacy=0x55ffa1944000 unknown :6789 s=STATE_CONNECTION_ESTABLISHED l=1).read_bulk peer close file descriptor 34
2021-09-10 12:06:17.613 7f1e94dfa700 1 -- [v2:172.16.12.83:3300/0,v1:172.16.12.83:6789/0] >> v1:172.16.12.19:0/10663437 conn(0x55ffa193c000 legacy=0x55ffa1944000 unknown :6789 s=STATE_CONNECTION_ESTABLISHED l=1).read_until read failed
2021-09-10 12:06:17.613 7f1e94dfa700 1 --1- [v2:172.16.12.83:3300/0,v1:172.16.12.83:6789/0] >> v1:172.16.12.19:0/10663437 conn(0x55ffa193c000 0x55ffa1944000 :6789 s=OPENED pgs=2 cs=1 l=1).handle_message read tag failed
2021-09-10 12:06:17.613 7f1e94dfa700 1 --1- [v2:172.16.12.83:3300/0,v1:172.16.12.83:6789/0] >> v1:172.16.12.19:0/10663437 conn(0x55ffa193c000 0x55ffa1944000 :6789 s=OPENED pgs=2 cs=1 l=1).fault on lossy channel, failing
2021-09-10 12:06:17.613 7f1e955fb700 10 mon.amon3@0(probing) e14 ms_handle_reset 0x55ffa193c000 v1:172.16.12.19:0/10663437
2021-09-10 12:06:17.613 7f1e955fb700 10 mon.amon3@0(probing) e14 reset/close on session client.? v1:172.16.12.19:0/10663437
2021-09-10 12:06:17.613 7f1e955fb700 10 mon.amon3@0(probing) e14 remove_session 0x55ffa194c240 client.? v1:172.16.12.19:0/10663437 features 0x3ffddff8ffecffff
2021-09-10 12:06:17.613 7f1e94dfa700 1 --2- [v2:172.16.12.83:3300/0,v1:172.16.12.83:6789/0] >> conn(0x55ffa193c480 0x55ffa15d2a00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 rx=0 tx=0).accept
2021-09-10 12:06:17.613 7f1e94dfa700 1 --2- [v2:172.16.12.83:3300/0,v1:172.16.12.83:6789/0] >> conn(0x55ffa193c480 0x55ffa15d2a00 unknown :-1 s=BANNER_ACCEPTING pgs=0 cs=0 l=0 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0
2021-09-10 12:06:17.613 7f1e94dfa700 10 mon.amon3@0(probing) e14 handle_auth_request con 0x55ffa193c480 (start) method 1 payload 22
2021-09-10 12:06:17.613 7f1e94dfa700 10 mon.amon3@0(probing) e14 handle_auth_request haven't formed initial quorum, EBUSY
2021-09-10 12:06:17.613 7f1e94dfa700 1 --2- [v2:172.16.12.83:3300/0,v1:172.16.12.83:6789/0] >> conn(0x55ffa193c480 0x55ffa15d2a00 crc :-1 s=AUTH_ACCEPTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).stop
2021-09-10 12:06:17.613 7f1e955fb700 10 mon.amon3@0(probing) e14 ms_handle_reset 0x55ffa193c480 -
2021-09-10 12:06:17.621 7f1e94dfa700 1 --2- [v2:172.16.12.83:3300/0,v1:172.16.12.83:6789/0] >> conn(0x55ffa193c900 0x55ffa15d3100 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 rx=0 tx=0).accept
2021-09-10 12:06:17.621 7f1e94dfa700 1 --2- [v2:172.16.12.83:3300/0,v1:172.16.12.83:6789/0] >> conn(0x55ffa193c900 0x55ffa15d3100 unknown :-1 s=BANNER_ACCEPTING pgs=0 cs=0 l=0 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0
2021-09-10 12:06:17.621 7f1e94dfa700 10 mon.amon3@0(probing) e14 handle_auth_request con 0x55ffa193c900 (start) method 1 payload 22
2021-09-10 12:06:17.621 7f1e94dfa700 10 mon.amon3@0(probing) e14 handle_auth_request haven't formed initial quorum, EBUSY
2021-09-10 12:06:17.621 7f1e94dfa700 1 --2- [v2:172.16.12.83:3300/0,v1:172.16.12.83:6789/0] >> conn(0x55ffa193c900 0x55ffa15d3100 crc :-1 s=AUTH_ACCEPTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).stop
2021-09-10 12:06:17.621 7f1e955fb700 10 mon.amon3@0(probing) e14 ms_handle_reset 0x55ffa193c900 -
2021-09-10 12:06:17.825 7f1e99e04700 1 --2- [v2:172.16.12.83:3300/0,v1:172.16.12.83:6789/0] >> [v2:172.16.12.81:3300/0,v1:172.16.12.81:6789/0] conn(0x55ffa096f600 0x55ffa15d1c00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=1 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0
2021-09-10 12:06:17.825 7f1e99e04700 10 mon.amon3@0(probing) e14 ms_get_authorizer for mon
2021-09-10 12:06:17.825 7f1e99e04700 1 -- [v2:172.16.12.83:3300/0,v1:172.16.12.83:6789/0] >> [v2:172.16.12.81:3300/0,v1:172.16.12.81:6789/0] conn(0x55ffa096f600 msgr2=0x55ffa15d1c00 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_bulk peer close file descriptor 34
2021-09-10 12:06:17.825 7f1e99e04700 1 -- [v2:172.16.12.83:3300/0,v1:172.16.12.83:6789/0] >> [v2:172.16.12.81:3300/0,v1:172.16.12.81:6789/0] conn(0x55ffa096f600 msgr2=0x55ffa15d1c00 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_until read failed
2021-09-10 12:06:17.825 7f1e99e04700 1 --2- [v2:172.16.12.83:3300/0,v1:172.16.12.83:6789/0] >> [v2:172.16.12.81:3300/0,v1:172.16.12.81:6789/0] conn(0x55ffa096f600 0x55ffa15d1c00 crc :-1 s=SESSION_CONNECTING pgs=0 cs=0 l=0 rev1=1 rx=0 tx=0).handle_read_frame_preamble_main read frame preamble failed r=-1 ((1) Operation not permitted)
2021-09-10 12:06:17.825 7f1e99e04700 1 --2- [v2:172.16.12.83:3300/0,v1:172.16.12.83:6789/0] >> [v2:172.16.12.81:3300/0,v1:172.16.12.81:6789/0] conn(0x55ffa096f600 0x55ffa15d1c00 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=1 rx=0 tx=0)._fault waiting 1.600000
2021-09-10 12:06:17.829 7f1e99603700 1 --2- [v2:172.16.12.83:3300/0,v1:172.16.12.83:6789/0] >> [v2:172.16.12.82:3300/0,v1:172.16.12.82:6789/0] conn(0x55ffa096fa80 0x55ffa15d2300 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=1 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0
2021-09-10 12:06:17.829 7f1e99603700 10 mon.amon3@0(probing) e14 ms_get_authorizer for mon
2021-09-10 12:06:17.829 7f1e99603700 1 -- [v2:172.16.12.83:3300/0,v1:172.16.12.83:6789/0] >> [v2:172.16.12.82:3300/0,v1:172.16.12.82:6789/0] conn(0x55ffa096fa80 msgr2=0x55ffa15d2300 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_bulk peer close file descriptor 34
2021-09-10 12:06:17.829 7f1e99603700 1 -- [v2:172.16.12.83:3300/0,v1:172.16.12.83:6789/0] >> [v2:172.16.12.82:3300/0,v1:172.16.12.82:6789/0] conn(0x55ffa096fa80 msgr2=0x55ffa15d2300 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_until read failed
2021-09-10 12:06:17.829 7f1e99603700 1 --2- [v2:172.16.12.83:3300/0,v1:172.16.12.83:6789/0] >> [v2:172.16.12.82:3300/0,v1:172.16.12.82:6789/0] conn(0x55ffa096fa80 0x55ffa15d2300 crc :-1 s=SESSION_CONNECTING pgs=0 cs=0 l=0 rev1=1 rx=0 tx=0).handle_read_frame_preamble_main read frame preamble failed r=-1 ((1) Operation not permitted)
2021-09-10 12:06:17.829 7f1e99603700 1 --2- [v2:172.16.12.83:3300/0,v1:172.16.12.83:6789/0] >> [v2:172.16.12.82:3300/0,v1:172.16.12.82:6789/0] conn(0x55ffa096fa80 0x55ffa15d2300 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=1 rx=0 tx=0)._fault waiting 1.600000
2021-09-10 12:06:18.417 7f1e97e00700 4 mon.amon3@0(probing) e14 probe_timeout 0x55ffa1907860
2021-09-10 12:06:18.417 7f1e97e00700 10 mon.amon3@0(probing) e14 bootstrap
2021-09-10 12:06:18.417 7f1e97e00700 10 mon.amon3@0(probing) e14 sync_reset_requester
2021-09-10 12:06:18.417 7f1e97e00700 10 mon.amon3@0(probing) e14 unregister_cluster_logger - not registered
2021-09-10 12:06:18.417 7f1e97e00700 10 mon.amon3@0(probing) e14 cancel_probe_timeout (none scheduled)
2021-09-10 12:06:18.417 7f1e97e00700 10 mon.amon3@0(probing) e14 monmap e14: 3 mons at {amon1=[v2:172.16.12.81:3300/0,v1:172.16.12.81:6789/0],amon2=[v2:172.16.12.82:3300/0,v1:172.16.12.82:6789/0],amon3=[v2:172.16.12.83:3300/0,v1:172.16.12.83:6789/0]}
2021-09-10 12:06:18.417 7f1e97e00700 10 mon.amon3@0(probing) e14 _reset
2021-09-10 12:06:18.417 7f1e97e00700 10 mon.amon3@0(probing).auth v0 _set_mon_num_rank num 0 rank 0
2021-09-10 12:06:18.417 7f1e97e00700 10 mon.amon3@0(probing) e14 cancel_probe_timeout (none scheduled)
2021-09-10 12:06:18.417 7f1e97e00700 10 mon.amon3@0(probing) e14 timecheck_finish
2021-09-10 12:06:18.417 7f1e97e00700 10 mon.amon3@0(probing) e14 scrub_event_cancel
2021-09-10 12:06:18.417 7f1e97e00700 10 mon.amon3@0(probing) e14 scrub_reset
2021-09-10 12:06:18.417 7f1e97e00700 10 mon.amon3@0(probing) e14 cancel_probe_timeout (none scheduled)
2021-09-10 12:06:18.417 7f1e97e00700 10 mon.amon3@0(probing) e14 reset_probe_timeout 0x55ffa1907890 after 2 seconds
2021-09-10 12:06:18.417 7f1e97e00700 10 mon.amon3@0(probing) e14 probing other monitors
BR
Max
_______________________________________________
ceph-users mailing list -- ceph-users@xxxxxxx
To unsubscribe send an email to ceph-users-leave@xxxxxxx
