Hi all,
we had deployed a cluster ceph with three nodes pacific with ubuntu 20.04, after we had tryed to restart one node, but when it’s comes up we see:
root@tst2-ceph01:~# ceph status
cluster:
id: be115adc-edf0-11eb-8509-c5c80111fd98
health: HEALTH_WARN
6 failed cephadm daemon(s)
1 osds down
1 host (6 osds) down
Degraded data redundancy: 1 pg undersized
services:
mon: 3 daemons, quorum tst2-ceph01.tstsddc.csi.it,tst2-ceph02,tst2-ceph03 (age 44s)
mgr: tst2-ceph03.fmrcvf(active, since 65m), standbys: tst2-ceph01.tstsddc.csi.it.ydtoyd
osd: 18 osds: 12 up (since 65m), 13 in (since 18m)
data:
pools: 1 pools, 1 pgs
objects: 0 objects, 0 B
usage: 757 GiB used, 13 TiB / 14 TiB avail
pgs: 1 active+undersized
root@tst2-ceph01:~# ceph osd tree
ID CLASS WEIGHT TYPE NAME STATUS REWEIGHT PRI-AFF
-1 20.73944 root default
-7 6.91315 host tst2-ceph01
2 hdd 1.15219 osd.2 down 0 1.00000
5 hdd 1.15219 osd.5 down 0 1.00000
8 hdd 1.15219 osd.8 down 0 1.00000
11 hdd 1.15219 osd.11 down 0 1.00000
14 hdd 1.15219 osd.14 down 0 1.00000
17 hdd 1.15219 osd.17 down 1.00000 1.00000
-3 6.91315 host tst2-ceph02
0 hdd 1.15219 osd.0 up 1.00000 1.00000
3 hdd 1.15219 osd.3 up 1.00000 1.00000
6 hdd 1.15219 osd.6 up 1.00000 1.00000
9 hdd 1.15219 osd.9 up 1.00000 1.00000
12 hdd 1.15219 osd.12 up 1.00000 1.00000
15 hdd 1.15219 osd.15 up 1.00000 1.00000
-5 6.91315 host tst2-ceph03
1 hdd 1.15219 osd.1 up 1.00000 1.00000
4 hdd 1.15219 osd.4 up 1.00000 1.00000
7 hdd 1.15219 osd.7 up 1.00000 1.00000
10 hdd 1.15219 osd.10 up 1.00000 1.00000
13 hdd 1.15219 osd.13 up 1.00000 1.00000
16 hdd 1.15219 osd.16 up 1.00000 1.00000
The services on node:
● ceph-0c7a175e-ebbc-11eb-8509-c5c80111fd98@alertmanager.tst2-ceph01.service<mailto:ceph-0c7a175e-ebbc-11eb-8509-c5c80111fd98@alertmanager.tst2-ceph01.service> loaded failed failed >
● ceph-0c7a175e-ebbc-11eb-8509-c5c80111fd98@crash.tst2-ceph01.service<mailto:ceph-0c7a175e-ebbc-11eb-8509-c5c80111fd98@crash.tst2-ceph01.service> loaded failed failed >
● ceph-0c7a175e-ebbc-11eb-8509-c5c80111fd98@grafana.tst2-ceph01.service<mailto:ceph-0c7a175e-ebbc-11eb-8509-c5c80111fd98@grafana.tst2-ceph01.service> loaded failed failed >
● ceph-0c7a175e-ebbc-11eb-8509-c5c80111fd98@mgr.tst2-ceph01.fthmip.service<mailto:ceph-0c7a175e-ebbc-11eb-8509-c5c80111fd98@mgr.tst2-ceph01.fthmip.service> loaded failed failed >
● ceph-0c7a175e-ebbc-11eb-8509-c5c80111fd98@mon.tst2-ceph01.service<mailto:ceph-0c7a175e-ebbc-11eb-8509-c5c80111fd98@mon.tst2-ceph01.service> loaded failed failed >
● ceph-0c7a175e-ebbc-11eb-8509-c5c80111fd98@node-exporter.tst2-ceph01.service<mailto:ceph-0c7a175e-ebbc-11eb-8509-c5c80111fd98@node-exporter.tst2-ceph01.service> loaded failed failed >
● ceph-0c7a175e-ebbc-11eb-8509-c5c80111fd98@osd.0.service<mailto:ceph-0c7a175e-ebbc-11eb-8509-c5c80111fd98@osd.0.service> loaded failed failed >
● ceph-0c7a175e-ebbc-11eb-8509-c5c80111fd98@osd.1.service<mailto:ceph-0c7a175e-ebbc-11eb-8509-c5c80111fd98@osd.1.service> loaded failed failed >
● ceph-0c7a175e-ebbc-11eb-8509-c5c80111fd98@osd.2.service<mailto:ceph-0c7a175e-ebbc-11eb-8509-c5c80111fd98@osd.2.service> loaded failed failed >
● ceph-0c7a175e-ebbc-11eb-8509-c5c80111fd98@osd.3.service<mailto:ceph-0c7a175e-ebbc-11eb-8509-c5c80111fd98@osd.3.service> loaded failed failed >
● ceph-0c7a175e-ebbc-11eb-8509-c5c80111fd98@osd.4.service<mailto:ceph-0c7a175e-ebbc-11eb-8509-c5c80111fd98@osd.4.service> loaded failed failed >
● ceph-0c7a175e-ebbc-11eb-8509-c5c80111fd98@osd.5.service<mailto:ceph-0c7a175e-ebbc-11eb-8509-c5c80111fd98@osd.5.service> loaded failed failed >
● ceph-0c7a175e-ebbc-11eb-8509-c5c80111fd98@osd.6.service<mailto:ceph-0c7a175e-ebbc-11eb-8509-c5c80111fd98@osd.6.service> loaded failed failed >
● ceph-0c7a175e-ebbc-11eb-8509-c5c80111fd98@prometheus.tst2-ceph01.service<mailto:ceph-0c7a175e-ebbc-11eb-8509-c5c80111fd98@prometheus.tst2-ceph01.service> loaded failed failed >
● ceph-33b6f6c8-edee-11eb-8509-c5c80111fd98@alertmanager.tst2-ceph01.service<mailto:ceph-33b6f6c8-edee-11eb-8509-c5c80111fd98@alertmanager.tst2-ceph01.service> loaded failed failed >
● ceph-33b6f6c8-edee-11eb-8509-c5c80111fd98@crash.tst2-ceph01.service<mailto:ceph-33b6f6c8-edee-11eb-8509-c5c80111fd98@crash.tst2-ceph01.service> loaded failed failed >
● ceph-33b6f6c8-edee-11eb-8509-c5c80111fd98@grafana.tst2-ceph01.service<mailto:ceph-33b6f6c8-edee-11eb-8509-c5c80111fd98@grafana.tst2-ceph01.service> loaded failed failed >
● ceph-33b6f6c8-edee-11eb-8509-c5c80111fd98@xxxxxxxxxxxxxxxxxxxxxxxxxxxxxx.vudokp.service<mailto:ceph-33b6f6c8-edee-11eb-8509-c5c80111fd98@xxxxxxxxxxxxxxxxxxxxxxxxxxxxxx.vudokp.service> loaded failed failed >
● ceph-33b6f6c8-edee-11eb-8509-c5c80111fd98@xxxxxxxxxxxxxxxxxxxxxxxxxxxxxx.service<mailto:ceph-33b6f6c8-edee-11eb-8509-c5c80111fd98@xxxxxxxxxxxxxxxxxxxxxxxxxxxxxx.service> loaded failed failed >
● ceph-33b6f6c8-edee-11eb-8509-c5c80111fd98@node-exporter.tst2-ceph01.service<mailto:ceph-33b6f6c8-edee-11eb-8509-c5c80111fd98@node-exporter.tst2-ceph01.service> loaded failed failed >
● ceph-33b6f6c8-edee-11eb-8509-c5c80111fd98@prometheus.tst2-ceph01.service<mailto:ceph-33b6f6c8-edee-11eb-8509-c5c80111fd98@prometheus.tst2-ceph01.service> loaded failed failed >
● ceph-4ae1bc3c-ebde-11eb-8509-c5c80111fd98@alertmanager.tst2-ceph01.service<mailto:ceph-4ae1bc3c-ebde-11eb-8509-c5c80111fd98@alertmanager.tst2-ceph01.service> loaded failed failed >
● ceph-4ae1bc3c-ebde-11eb-8509-c5c80111fd98@crash.tst2-ceph01.service<mailto:ceph-4ae1bc3c-ebde-11eb-8509-c5c80111fd98@crash.tst2-ceph01.service> loaded failed failed >
● ceph-4ae1bc3c-ebde-11eb-8509-c5c80111fd98@grafana.tst2-ceph01.service<mailto:ceph-4ae1bc3c-ebde-11eb-8509-c5c80111fd98@grafana.tst2-ceph01.service> loaded failed failed >
● ceph-4ae1bc3c-ebde-11eb-8509-c5c80111fd98@mgr.tst2-ceph01.yfpcnr.service<mailto:ceph-4ae1bc3c-ebde-11eb-8509-c5c80111fd98@mgr.tst2-ceph01.yfpcnr.service> loaded failed failed >
● ceph-4ae1bc3c-ebde-11eb-8509-c5c80111fd98@mon.tst2-ceph01.service<mailto:ceph-4ae1bc3c-ebde-11eb-8509-c5c80111fd98@mon.tst2-ceph01.service> loaded failed failed >
● ceph-4ae1bc3c-ebde-11eb-8509-c5c80111fd98@node-exporter.tst2-ceph01.service<mailto:ceph-4ae1bc3c-ebde-11eb-8509-c5c80111fd98@node-exporter.tst2-ceph01.service> loaded failed failed >
● ceph-4ae1bc3c-ebde-11eb-8509-c5c80111fd98@osd.11.service<mailto:ceph-4ae1bc3c-ebde-11eb-8509-c5c80111fd98@osd.11.service> loaded failed failed >
● ceph-4ae1bc3c-ebde-11eb-8509-c5c80111fd98@osd.14.service<mailto:ceph-4ae1bc3c-ebde-11eb-8509-c5c80111fd98@osd.14.service> loaded failed failed >
● ceph-4ae1bc3c-ebde-11eb-8509-c5c80111fd98@osd.16.service<mailto:ceph-4ae1bc3c-ebde-11eb-8509-c5c80111fd98@osd.16.service> loaded failed failed >
● ceph-4ae1bc3c-ebde-11eb-8509-c5c80111fd98@osd.2.service<mailto:ceph-4ae1bc3c-ebde-11eb-8509-c5c80111fd98@osd.2.service> loaded failed failed >
● ceph-4ae1bc3c-ebde-11eb-8509-c5c80111fd98@osd.5.service<mailto:ceph-4ae1bc3c-ebde-11eb-8509-c5c80111fd98@osd.5.service> loaded failed failed >
● ceph-4ae1bc3c-ebde-11eb-8509-c5c80111fd98@osd.8.service<mailto:ceph-4ae1bc3c-ebde-11eb-8509-c5c80111fd98@osd.8.service> loaded failed failed >
● ceph-4ae1bc3c-ebde-11eb-8509-c5c80111fd98@prometheus.tst2-ceph01.service<mailto:ceph-4ae1bc3c-ebde-11eb-8509-c5c80111fd98@prometheus.tst2-ceph01.service> loaded failed failed >
● ceph-7b1db4e2-ede0-11eb-8509-c5c80111fd98@alertmanager.tst2-ceph01.service<mailto:ceph-7b1db4e2-ede0-11eb-8509-c5c80111fd98@alertmanager.tst2-ceph01.service> loaded failed failed >
● ceph-7b1db4e2-ede0-11eb-8509-c5c80111fd98@crash.tst2-ceph01.service<mailto:ceph-7b1db4e2-ede0-11eb-8509-c5c80111fd98@crash.tst2-ceph01.service> loaded failed failed >
● ceph-7b1db4e2-ede0-11eb-8509-c5c80111fd98@grafana.tst2-ceph01.service<mailto:ceph-7b1db4e2-ede0-11eb-8509-c5c80111fd98@grafana.tst2-ceph01.service> loaded failed failed >
● ceph-7b1db4e2-ede0-11eb-8509-c5c80111fd98@mgr.tst2-ceph01.nmjrrz.service<mailto:ceph-7b1db4e2-ede0-11eb-8509-c5c80111fd98@mgr.tst2-ceph01.nmjrrz.service> loaded failed failed >
● ceph-7b1db4e2-ede0-11eb-8509-c5c80111fd98@mon.tst2-ceph01.service<mailto:ceph-7b1db4e2-ede0-11eb-8509-c5c80111fd98@mon.tst2-ceph01.service> loaded failed failed >
● ceph-7b1db4e2-ede0-11eb-8509-c5c80111fd98@osd.11.service<mailto:ceph-7b1db4e2-ede0-11eb-8509-c5c80111fd98@osd.11.service> loaded failed failed >
● ceph-7b1db4e2-ede0-11eb-8509-c5c80111fd98@osd.14.service<mailto:ceph-7b1db4e2-ede0-11eb-8509-c5c80111fd98@osd.14.service> loaded failed failed >
● ceph-7b1db4e2-ede0-11eb-8509-c5c80111fd98@osd.17.service<mailto:ceph-7b1db4e2-ede0-11eb-8509-c5c80111fd98@osd.17.service> loaded failed failed >
● ceph-7b1db4e2-ede0-11eb-8509-c5c80111fd98@osd.2.service<mailto:ceph-7b1db4e2-ede0-11eb-8509-c5c80111fd98@osd.2.service> loaded failed failed >
● ceph-7b1db4e2-ede0-11eb-8509-c5c80111fd98@osd.5.service<mailto:ceph-7b1db4e2-ede0-11eb-8509-c5c80111fd98@osd.5.service> loaded failed failed >
● ceph-7b1db4e2-ede0-11eb-8509-c5c80111fd98@osd.8.service<mailto:ceph-7b1db4e2-ede0-11eb-8509-c5c80111fd98@osd.8.service> loaded failed failed >
● ceph-7b1db4e2-ede0-11eb-8509-c5c80111fd98@prometheus.tst2-ceph01.service<mailto:ceph-7b1db4e2-ede0-11eb-8509-c5c80111fd98@prometheus.tst2-ceph01.service> loaded failed failed >
● ceph-8b937a98-eb86-11eb-8509-c5c80111fd98@alertmanager.tst2-ceph01.service<mailto:ceph-8b937a98-eb86-11eb-8509-c5c80111fd98@alertmanager.tst2-ceph01.service> loaded failed failed >
● ceph-8b937a98-eb86-11eb-8509-c5c80111fd98@crash.tst2-ceph01.service<mailto:ceph-8b937a98-eb86-11eb-8509-c5c80111fd98@crash.tst2-ceph01.service> loaded failed failed >
● ceph-8b937a98-eb86-11eb-8509-c5c80111fd98@grafana.tst2-ceph01.service<mailto:ceph-8b937a98-eb86-11eb-8509-c5c80111fd98@grafana.tst2-ceph01.service> loaded failed failed >
● ceph-8b937a98-eb86-11eb-8509-c5c80111fd98@mgr.tst2-ceph01.kwyejx.service<mailto:ceph-8b937a98-eb86-11eb-8509-c5c80111fd98@mgr.tst2-ceph01.kwyejx.service> loaded failed failed >
● ceph-8b937a98-eb86-11eb-8509-c5c80111fd98@mon.tst2-ceph01.service<mailto:ceph-8b937a98-eb86-11eb-8509-c5c80111fd98@mon.tst2-ceph01.service> loaded failed failed >
● ceph-8b937a98-eb86-11eb-8509-c5c80111fd98@node-exporter.tst2-ceph01.service<mailto:ceph-8b937a98-eb86-11eb-8509-c5c80111fd98@node-exporter.tst2-ceph01.service> loaded failed failed >
● ceph-8b937a98-eb86-11eb-8509-c5c80111fd98@osd.0.service<mailto:ceph-8b937a98-eb86-11eb-8509-c5c80111fd98@osd.0.service> loaded failed failed >
● ceph-8b937a98-eb86-11eb-8509-c5c80111fd98@prometheus.tst2-ceph01.service<mailto:ceph-8b937a98-eb86-11eb-8509-c5c80111fd98@prometheus.tst2-ceph01.service> loaded failed failed >
ceph-be115adc-edf0-11eb-8509-c5c80111fd98@alertmanager.tst2-ceph01.service<mailto:ceph-be115adc-edf0-11eb-8509-c5c80111fd98@alertmanager.tst2-ceph01.service> loaded active running >
ceph-be115adc-edf0-11eb-8509-c5c80111fd98@crash.tst2-ceph01.service<mailto:ceph-be115adc-edf0-11eb-8509-c5c80111fd98@crash.tst2-ceph01.service> loaded active running >
ceph-be115adc-edf0-11eb-8509-c5c80111fd98@grafana.tst2-ceph01.service<mailto:ceph-be115adc-edf0-11eb-8509-c5c80111fd98@grafana.tst2-ceph01.service> loaded active running >
ceph-be115adc-edf0-11eb-8509-c5c80111fd98@xxxxxxxxxxxxxxxxxxxxxxxxxxxxxx.ydtoyd.service<mailto:ceph-be115adc-edf0-11eb-8509-c5c80111fd98@xxxxxxxxxxxxxxxxxxxxxxxxxxxxxx.ydtoyd.service> loaded active running >
ceph-be115adc-edf0-11eb-8509-c5c80111fd98@xxxxxxxxxxxxxxxxxxxxxxxxxxxxxx.service<mailto:ceph-be115adc-edf0-11eb-8509-c5c80111fd98@xxxxxxxxxxxxxxxxxxxxxxxxxxxxxx.service> loaded active running >
ceph-be115adc-edf0-11eb-8509-c5c80111fd98@node-exporter.tst2-ceph01.service<mailto:ceph-be115adc-edf0-11eb-8509-c5c80111fd98@node-exporter.tst2-ceph01.service> loaded active running >
● ceph-be115adc-edf0-11eb-8509-c5c80111fd98@osd.11.service<mailto:ceph-be115adc-edf0-11eb-8509-c5c80111fd98@osd.11.service> loaded failed failed >
● ceph-be115adc-edf0-11eb-8509-c5c80111fd98@osd.14.service<mailto:ceph-be115adc-edf0-11eb-8509-c5c80111fd98@osd.14.service> loaded failed failed >
● ceph-be115adc-edf0-11eb-8509-c5c80111fd98@osd.17.service<mailto:ceph-be115adc-edf0-11eb-8509-c5c80111fd98@osd.17.service> loaded failed failed >
● ceph-be115adc-edf0-11eb-8509-c5c80111fd98@osd.2.service<mailto:ceph-be115adc-edf0-11eb-8509-c5c80111fd98@osd.2.service> loaded failed failed >
● ceph-be115adc-edf0-11eb-8509-c5c80111fd98@osd.5.service<mailto:ceph-be115adc-edf0-11eb-8509-c5c80111fd98@osd.5.service> loaded failed failed >
● ceph-be115adc-edf0-11eb-8509-c5c80111fd98@osd.8.service<mailto:ceph-be115adc-edf0-11eb-8509-c5c80111fd98@osd.8.service> loaded failed failed >
And recovery doesn’t start.
Any suggest?
Thank you.
Andrea
_______________________________________________
ceph-users mailing list -- ceph-users@xxxxxxx
To unsubscribe send an email to ceph-users-leave@xxxxxxx
