Hello all,
I have an issue that developed after restarting my cluster. I shut down a
healthy working cluster. Before shutting down the nodes, I issued the
following:
sudo ceph osd set noout
sudo ceph osd set nobackfill
sudo ceph osd set norecover
When I restarted all the nodes, several of the OSDs will not restart and
connect to the cluster.
I am running 16.2.6 using podman for the containers.
I have 5 nodes, holding 3 OSDs on each. Each OSD is encrypted. There are also
3 monitor nodes.
ceph orch commands all hang at present.
I would be grateful for any suggestions how to restart these OSDs. I have
tried restarting the nodes. Both by a reboot and using the systemctl commands.
Thanks in advance,
Stephen.
ceph osd tree:
ID CLASS WEIGHT TYPE NAME STATUS REWEIGHT PRI-AFF
-1 63.67683 root default
-9 12.73537 host cephnode01
3 hdd 7.27739 osd.3 up 1.00000 1.00000
7 hdd 2.72899 osd.7 up 1.00000 1.00000
14 hdd 2.72899 osd.14 up 1.00000 1.00000
-7 12.73537 host cephnode02
0 hdd 7.27739 osd.0 down 1.00000 1.00000
4 hdd 2.72899 osd.4 down 1.00000 1.00000
13 hdd 2.72899 osd.13 up 1.00000 1.00000
-11 12.73537 host cephnode03
8 hdd 7.27739 osd.8 down 1.00000 1.00000
9 hdd 2.72899 osd.9 down 1.00000 1.00000
10 hdd 2.72899 osd.10 down 1.00000 1.00000
-3 12.73537 host cephnode04
1 hdd 7.27739 osd.1 down 1.00000 1.00000
6 hdd 2.72899 osd.6 down 1.00000 1.00000
12 hdd 2.72899 osd.12 down 1.00000 1.00000
-5 12.73537 host cephnode06
2 hdd 7.27739 osd.2 down 1.00000 1.00000
5 hdd 2.72899 osd.5 down 1.00000 1.00000
11 hdd 2.72899 osd.11 down 1.00000 1.00000
On each node I can run cephadm ceph-volume lvm list
CEPHNODE01
Inferring fsid d82ced8c-1f6e-11ec-a2e4-00fd45fcaf9c
Using recent ceph image quay.io/ceph/
ceph@sha256:a39254df9fb8b100968e8ccc85c4e4b130a3d6fa5655294d0c3d3321eb48c6be
====== osd.14 ======
[block] /dev/ceph-3f95b986-2657-458b-90d6-b80936b2401c/osd-block-
c596c94f-9c86-47e5-877f-6595456951cc
block device /dev/ceph-3f95b986-2657-458b-90d6-
b80936b2401c/osd-block-c596c94f-9c86-47e5-877f-6595456951cc
block uuid G9XLUI-6PJz-Lzrn-t757-yHHu-Zt4y-zrNSve
cephx lockbox secret AQBmVI5hAtj4MxAAP2lC3tVuiBktKQ9iwuWqWg==
cluster fsid d82ced8c-1f6e-11ec-a2e4-00fd45fcaf9c
cluster name ceph
crush device class None
encrypted 1
osd fsid c596c94f-9c86-47e5-877f-6595456951cc
osd id 14
osdspec affinity default_drive_group
type block
vdo 0
devices /dev/sdb
====== osd.3 =======
[block] /dev/ceph-db09a349-07e6-4b56-9e59-2044c36c3cbd/osd-block-
de08e91e-b00e-450a-8246-eb28f8f7a6ad
block device /dev/ceph-
db09a349-07e6-4b56-9e59-2044c36c3cbd/osd-block-de08e91e-b00e-450a-8246-
eb28f8f7a6ad
block uuid iYSNKF-SYSh-j78d-LEN3-x7aw-Xh2k-f2QHzq
cephx lockbox secret AQDQzFFhAjRRKBAAGUz3luwO4g1ezBLueTHT3Q==
cluster fsid d82ced8c-1f6e-11ec-a2e4-00fd45fcaf9c
cluster name ceph
crush device class None
encrypted 1
osd fsid de08e91e-b00e-450a-8246-eb28f8f7a6ad
osd id 3
osdspec affinity default_drive_group
type block
vdo 0
devices /dev/sda
====== osd.7 =======
[block] /dev/ceph-abeba17c-fe9a-4a8f-ae77-bb890f192d49/osd-block-
b6bd047b-5702-4567-89a0-991161b8b684
block device /dev/ceph-abeba17c-fe9a-4a8f-ae77-
bb890f192d49/osd-block-b6bd047b-5702-4567-89a0-991161b8b684
block uuid o7BJQE-gosc-XAw9-RHai-yf3y-UFyt-gDbtSp
cephx lockbox secret AQBo+FFh0U7KCBAAdQm8eSU+FcWhNFNz3V7ZNA==
cluster fsid d82ced8c-1f6e-11ec-a2e4-00fd45fcaf9c
cluster name ceph
crush device class None
encrypted 1
osd fsid b6bd047b-5702-4567-89a0-991161b8b684
osd id 7
osdspec affinity default_drive_group
type block
vdo 0
devices /dev/sdc
CEPHNODE02
Inferring fsid d82ced8c-1f6e-11ec-a2e4-00fd45fcaf9c
Using recent ceph image quay.io/ceph/
ceph@sha256:285d2abb6e74bdc6e15e1af585aa19f132045651b7a80eb77b9cec8a785ff330
====== osd.0 =======
[block] /dev/ceph-779b00ee-dbb5-4542-859c-b6338b01efe0/osd-
block-2f5d17d3-3308-4f42-867c-7ec8639bde18
block device /dev/ceph-779b00ee-dbb5-4542-859c-
b6338b01efe0/osd-block-2f5d17d3-3308-4f42-867c-7ec8639bde18
block uuid 6oP2t6-fuMx-25XU-KiIt-A28Q-fzng-K7ouf6
cephx lockbox secret AQDNzFFh11ooIRAAru8xYXAXL1/n/75AOJm2KA==
cluster fsid d82ced8c-1f6e-11ec-a2e4-00fd45fcaf9c
cluster name ceph
crush device class None
encrypted 1
osd fsid 2f5d17d3-3308-4f42-867c-7ec8639bde18
osd id 0
osdspec affinity default_drive_group
type block
vdo 0
devices /dev/sda
====== osd.13 ======
[block] /dev/ceph-41cbaf58-d703-4bae-9095-f1b590b8337b/osd-
block-99e1e1b2-3cfe-48e0-8627-acfe5008c1c5
block device /dev/ceph-41cbaf58-d703-4bae-9095-
f1b590b8337b/osd-block-99e1e1b2-3cfe-48e0-8627-acfe5008c1c5
block uuid fwR6Nz-3DbF-Ac2K-jhVp-DtsV-A1mt-VMUefw
cephx lockbox secret AQAyQ45hPpzqDxAAx2vyba/tTeQmZl1Sllh8Mg==
cluster fsid d82ced8c-1f6e-11ec-a2e4-00fd45fcaf9c
cluster name ceph
crush device class None
encrypted 1
osd fsid 99e1e1b2-3cfe-48e0-8627-acfe5008c1c5
osd id 13
osdspec affinity default_drive_group
type block
vdo 0
devices /dev/sdb
====== osd.4 =======
[block] /dev/ceph-09c84540-06e3-496f-bf90-45f59748768a/osd-
block-60ad9fb3-8144-4d06-83e6-b90e9d52dcaa
block device /dev/ceph-09c84540-06e3-496f-
bf90-45f59748768a/osd-block-60ad9fb3-8144-4d06-83e6-b90e9d52dcaa
block uuid h352xv-O3aB-qRAQ-rWUa-alhz-7b9H-o30rfs
cephx lockbox secret AQB+/FFhtwkAIxAAWVWk87T5duTfEIDtQGGqjg==
cluster fsid d82ced8c-1f6e-11ec-a2e4-00fd45fcaf9c
cluster name ceph
crush device class None
encrypted 1
osd fsid 60ad9fb3-8144-4d06-83e6-b90e9d52dcaa
osd id 4
osdspec affinity default_drive_group
type block
vdo 0
devices /dev/sdc
CEPHNODE03
Inferring fsid d82ced8c-1f6e-11ec-a2e4-00fd45fcaf9c
Using recent ceph image quay.io/ceph/
ceph@sha256:285d2abb6e74bdc6e15e1af585aa19f132045651b7a80eb77b9cec8a785ff330
====== osd.10 ======
[block] /dev/ceph-98254eee-65b4-4f21-bf90-475b6f0f4a21/osd-
block-8302c697-b8d2-4278-9295-f66f95524f91
block device /dev/ceph-98254eee-65b4-4f21-
bf90-475b6f0f4a21/osd-block-8302c697-b8d2-4278-9295-f66f95524f91
block uuid b1kxhl-Fzwm-nyFr-uKci-8Ono-BddF-x2TgZ9
cephx lockbox secret AQAmxIJh7mXnExAAYhZ9c3tHzupkkuzBiapBMg==
cluster fsid d82ced8c-1f6e-11ec-a2e4-00fd45fcaf9c
cluster name ceph
crush device class None
encrypted 1
osd fsid 8302c697-b8d2-4278-9295-f66f95524f91
osd id 10
osdspec affinity default_drive_group
type block
vdo 0
devices /dev/sdb
====== osd.8 =======
[block] /dev/ceph-f62bce7f-08f0-436a-bd03-550ac0d0a895/osd-
block-672668b6-a3e9-4819-ab4c-f892e0fa79d0
block device /dev/ceph-f62bce7f-08f0-436a-
bd03-550ac0d0a895/osd-block-672668b6-a3e9-4819-ab4c-f892e0fa79d0
block uuid 2gfdhU-1mEg-bIh6-yKZ1-jRrC-oc3a-I1RzLb
cephx lockbox secret AQA9TIJhZqg+ORAAHliN1xXtv+k88Rzc4KjRdQ==
cluster fsid d82ced8c-1f6e-11ec-a2e4-00fd45fcaf9c
cluster name ceph
crush device class None
encrypted 1
osd fsid 672668b6-a3e9-4819-ab4c-f892e0fa79d0
osd id 8
osdspec affinity default_drive_group
type block
vdo 0
devices /dev/sda
====== osd.9 =======
[block] /dev/ceph-fb56ad2f-ea67-499b-807c-6b1ecebdb6a3/osd-block-
cdb04867-5463-44a7-9592-c268a3de9540
block device /dev/ceph-fb56ad2f-
ea67-499b-807c-6b1ecebdb6a3/osd-block-cdb04867-5463-44a7-9592-c268a3de9540
block uuid sRurSt-Lww2-DGel-8oY7-RjNu-2sOT-1diFgX
cephx lockbox secret AQBZTIJh+l1ULBAAEyZcAi4opoVxQQjM3YkuTw==
cluster fsid d82ced8c-1f6e-11ec-a2e4-00fd45fcaf9c
cluster name ceph
crush device class None
encrypted 1
osd fsid cdb04867-5463-44a7-9592-c268a3de9540
osd id 9
osdspec affinity default_drive_group
type block
vdo 0
devices /dev/sdc
CEPHNODE04
Inferring fsid d82ced8c-1f6e-11ec-a2e4-00fd45fcaf9c
Using recent ceph image quay.io/ceph/
ceph@sha256:285d2abb6e74bdc6e15e1af585aa19f132045651b7a80eb77b9cec8a785ff330
====== osd.1 =======
[block] /dev/ceph-b62d8c20-614e-484c-9ec3-c175ed248f11/osd-
block-20eb9d49-5b58-4d1f-a3b3-a68a5d41cc2c
block device /dev/ceph-b62d8c20-614e-484c-9ec3-
c175ed248f11/osd-block-20eb9d49-5b58-4d1f-a3b3-a68a5d41cc2c
block uuid 26UC4l-6sjQ-neN0-ORAf-8sKI-RVuz-HBHStq
cephx lockbox secret AQDNzFFh5SLzIBAApb/ID6Vg7wgYK8+MKqHLKw==
cluster fsid d82ced8c-1f6e-11ec-a2e4-00fd45fcaf9c
cluster name ceph
crush device class None
encrypted 1
osd fsid 20eb9d49-5b58-4d1f-a3b3-a68a5d41cc2c
osd id 1
osdspec affinity default_drive_group
type block
vdo 0
devices /dev/sda
====== osd.12 ======
[block] /dev/ceph-aece66f3-89cd-4b87-a811-e4cca52a3dbd/osd-
block-40a3f337-32d4-4539-9c67-6ba4e95e2124
block device /dev/ceph-aece66f3-89cd-4b87-a811-
e4cca52a3dbd/osd-block-40a3f337-32d4-4539-9c67-6ba4e95e2124
block uuid ZW6SAr-zmNK-3EYp-ityS-pIld-U1VW-0ACnpJ
cephx lockbox secret AQApxIphr515EBAA4TqEVVeauRE5W1GyNuIUDw==
cluster fsid d82ced8c-1f6e-11ec-a2e4-00fd45fcaf9c
cluster name ceph
crush device class None
encrypted 1
osd fsid 40a3f337-32d4-4539-9c67-6ba4e95e2124
osd id 12
osdspec affinity default_drive_group
type block
vdo 0
devices /dev/sdb
====== osd.6 =======
[block] /dev/ceph-78000997-726a-4b41-b02e-7c7bc01afa06/osd-block-
cfe5e317-015a-42f7-97e1-5e3f559801de
block device /dev/ceph-78000997-726a-4b41-
b02e-7c7bc01afa06/osd-block-cfe5e317-015a-42f7-97e1-5e3f559801de
block uuid apPTYN-JSVy-RggI-7qVi-9Bh3-VvyD-eEQgMm
cephx lockbox secret AQBj+FFhpuB+AxAAJ6z3owugu/QXADfeME8lyg==
cluster fsid d82ced8c-1f6e-11ec-a2e4-00fd45fcaf9c
cluster name ceph
crush device class None
encrypted 1
osd fsid cfe5e317-015a-42f7-97e1-5e3f559801de
osd id 6
osdspec affinity default_drive_group
type block
vdo 0
devices /dev/sdc
CEPHNODE06
Inferring fsid d82ced8c-1f6e-11ec-a2e4-00fd45fcaf9c
Using recent ceph image quay.io/ceph/
ceph@sha256:285d2abb6e74bdc6e15e1af585aa19f132045651b7a80eb77b9cec8a785ff330
====== osd.11 ======
[block] /dev/ceph-a1fbc61a-9278-4fa0-88da-c154cebec32e/osd-
block-05604866-7fb4-4983-97d3-a94e5ab34434
block device /dev/ceph-a1fbc61a-9278-4fa0-88da-
c154cebec32e/osd-block-05604866-7fb4-4983-97d3-a94e5ab34434
block uuid RYIpUA-Yzdp-yrBF-T2wj-xDa3-27I7-SVAY3L
cephx lockbox secret AQDMwodhgQTnAhAA4ZDrc3givfBjxa+pB0rwZA==
cluster fsid d82ced8c-1f6e-11ec-a2e4-00fd45fcaf9c
cluster name ceph
crush device class None
encrypted 1
osd fsid 05604866-7fb4-4983-97d3-a94e5ab34434
osd id 11
osdspec affinity default_drive_group
type block
vdo 0
devices /dev/sdb
====== osd.2 =======
[block] /dev/ceph-f0ae899f-42d8-4cf8-bfdb-1588c262ce11/osd-block-
d09bcdb8-65b4-469a-8d83-e38879fcc765
block device /dev/ceph-f0ae899f-42d8-4cf8-
bfdb-1588c262ce11/osd-block-d09bcdb8-65b4-469a-8d83-e38879fcc765
block uuid g1afxW-p52c-d553-dxcJ-G51x-LYPt-m41VCx
cephx lockbox secret AQDNzFFhPA96JRAAJwvk0mMFyje7tiH7IwhW/Q==
cluster fsid d82ced8c-1f6e-11ec-a2e4-00fd45fcaf9c
cluster name ceph
crush device class None
encrypted 1
osd fsid d09bcdb8-65b4-469a-8d83-e38879fcc765
osd id 2
osdspec affinity default_drive_group
type block
vdo 0
devices /dev/sda
====== osd.5 =======
[block] /dev/ceph-d63ba95e-9116-44ee-a67e-8b082299a8eb/osd-
block-1d5da947-22cc-407b-b6d1-fd198770c857
block device /dev/ceph-d63ba95e-9116-44ee-
a67e-8b082299a8eb/osd-block-1d5da947-22cc-407b-b6d1-fd198770c857
block uuid xuLljc-PWlb-kpb9-UcMG-XB1N-tIo3-NSSOlu
cephx lockbox secret AQBj+FFhE5IoEhAAgY6RWjMGHNHHNMkQraaupA==
cluster fsid d82ced8c-1f6e-11ec-a2e4-00fd45fcaf9c
cluster name ceph
crush device class None
encrypted 1
osd fsid 1d5da947-22cc-407b-b6d1-fd198770c857
osd id 5
osdspec affinity default_drive_group
type block
vdo 0
devices /dev/sdc
_______________________________________________
ceph-users mailing list -- ceph-users@xxxxxxx
To unsubscribe send an email to ceph-users-leave@xxxxxxx
