These repeat for every host, only after upgrading from prev release
Quincy to 17.2.7. As a result, the cluster is always warned, never
indicates healthy.
root@noc1:~# ceph health detail
HEALTH_WARN failed to probe daemons or devices
[WRN] CEPHADM_REFRESH_FAILED: failed to probe daemons or devices
host sysmon1 `cephadm ceph-volume` failed: cephadm exited with an
error code: 1, stderr: Inferring config
/var/lib/ceph/4067126d-01cb-40af-824a-881c130140f8/mon.sysmon1/config
Non-zero exit code 1 from /usr/bin/docker run --rm --ipc=host
--stop-signal=SIGTERM --ulimit nofile=1048576 --net=host --entrypoint
/usr/sbin/ceph-volume --privileged --group-add=disk --init -e
CONTAINER_IMAGE=quay.io/ceph/ceph@sha2
56:92e8fa7d8ca17a7a5bbfde6e596fdfecc8e165fcb94d86493f4e6c7b1f326e4e -e
NODE_NAME=sysmon1 -e CEPH_USE_RANDOM_NONCE=1 -e
CEPH_VOLUME_SKIP_RESTORECON=yes -e CEPH_VOLUME_DEBUG=1 -v
/var/run/ceph/4067126d-01cb-40af-824a-881c130140f8:/var
/run/ceph:z -v
/var/log/ceph/4067126d-01cb-40af-824a-881c130140f8:/var/log/ceph:z -v
/var/lib/ceph/4067126d-01cb-40af-824a-881c130140f8/crash:/var/lib/ceph/crash:z
-v /dev:/dev -v /run/udev:/run/udev -v /sys:/sys -v /run/lvm:/run/lv
m -v /run/lock/lvm:/run/lock/lvm -v /:/rootfs -v
/tmp/ceph-tmpl1e27bun:/etc/ceph/ceph.conf:z
quay.io/ceph/ceph@sha256:92e8fa7d8ca17a7a5bbfde6e596fdfecc8e165fcb94d86493f4e6c7b1f326e4e
inventory --format=json-pretty --filter-for-batch
/usr/bin/docker: stderr stderr: Unknown device, --name=, --path=, or
absolute path in /dev/ or /sys expected.
/usr/bin/docker: stderr stderr: Unknown device, --name=, --path=, or
absolute path in /dev/ or /sys expected.
/usr/bin/docker: stderr stderr: Unknown device, --name=, --path=, or
absolute path in /dev/ or /sys expected.
/usr/bin/docker: stderr stderr: Unknown device, --name=, --path=, or
absolute path in /dev/ or /sys expected.
/usr/bin/docker: stderr stderr: Unknown device, --name=, --path=, or
absolute path in /dev/ or /sys expected.
/usr/bin/docker: stderr stderr: Unknown device, --name=, --path=, or
absolute path in /dev/ or /sys expected.
/usr/bin/docker: stderr Traceback (most recent call last):
/usr/bin/docker: stderr File "/usr/sbin/ceph-volume", line 11, in
<module>
/usr/bin/docker: stderr load_entry_point('ceph-volume==1.0.0',
'console_scripts', 'ceph-volume')()
/usr/bin/docker: stderr File
"/usr/lib/python3.6/site-packages/ceph_volume/main.py", line 41, in
__init__
/usr/bin/docker: stderr self.main(self.argv)
/usr/bin/docker: stderr File
"/usr/lib/python3.6/site-packages/ceph_volume/decorators.py", line 59,
in newfunc
/usr/bin/docker: stderr return f(*a, **kw)
/usr/bin/docker: stderr File
"/usr/lib/python3.6/site-packages/ceph_volume/main.py", line 153, in main
/usr/bin/docker: stderr terminal.dispatch(self.mapper, subcommand_args)
/usr/bin/docker: stderr File
"/usr/lib/python3.6/site-packages/ceph_volume/terminal.py", line 194, in
dispatch
/usr/bin/docker: stderr instance.main()
/usr/bin/docker: stderr File
"/usr/lib/python3.6/site-packages/ceph_volume/inventory/main.py", line
60, in main
/usr/bin/docker: stderr list_all=self.args.list_all))
/usr/bin/docker: stderr File
"/usr/lib/python3.6/site-packages/ceph_volume/util/device.py", line 50,
in __init__
/usr/bin/docker: stderr sys_info.devices.keys()]
/usr/bin/docker: stderr File
"/usr/lib/python3.6/site-packages/ceph_volume/util/device.py", line 49,
in <listcomp>
/usr/bin/docker: stderr all_devices_vgs=all_devices_vgs) for k in
/usr/bin/docker: stderr File
"/usr/lib/python3.6/site-packages/ceph_volume/util/device.py", line 147,
in __init__
/usr/bin/docker: stderr self.available_lvm,
self.rejected_reasons_lvm = self._check_lvm_reject_reasons()
/usr/bin/docker: stderr File
"/usr/lib/python3.6/site-packages/ceph_volume/util/device.py", line 646,
in _check_lvm_reject_reasons
/usr/bin/docker: stderr
rejected.extend(self._check_generic_reject_reasons())
/usr/bin/docker: stderr File
"/usr/lib/python3.6/site-packages/ceph_volume/util/device.py", line 601,
in _check_generic_reject_reasons
/usr/bin/docker: stderr if self.is_acceptable_device:
/usr/bin/docker: stderr File
"/usr/lib/python3.6/site-packages/ceph_volume/util/device.py", line 502,
in is_acceptable_device
/usr/bin/docker: stderr return self.is_device or self.is_partition
or self.is_lv
/usr/bin/docker: stderr File
"/usr/lib/python3.6/site-packages/ceph_volume/util/device.py", line 482,
in is_partition
/usr/bin/docker: stderr return self.blkid_api['TYPE'] == 'part'
/usr/bin/docker: stderr KeyError: 'TYPE'
Traceback (most recent call last):
File
"/var/lib/ceph/4067126d-01cb-40af-824a-881c130140f8/cephadm.8b92cafd937eb89681ee011f9e70f85937fd09c4bd61ed4a59981d275a1f255b",
line 9679, in <module>
main()
File
"/var/lib/ceph/4067126d-01cb-40af-824a-881c130140f8/cephadm.8b92cafd937eb89681ee011f9e70f85937fd09c4bd61ed4a59981d275a1f255b",
line 9667, in main
r = ctx.func(ctx)
File
"/var/lib/ceph/4067126d-01cb-40af-824a-881c130140f8/cephadm.8b92cafd937eb89681ee011f9e70f85937fd09c4bd61ed4a59981d275a1f255b",
line 2116, in _infer_config
return func(ctx)
File
"/var/lib/ceph/4067126d-01cb-40af-824a-881c130140f8/cephadm.8b92cafd937eb89681ee011f9e70f85937fd09c4bd61ed4a59981d275a1f255b",
line 2032, in _infer_fsid
return func(ctx)
File
"/var/lib/ceph/4067126d-01cb-40af-824a-881c130140f8/cephadm.8b92cafd937eb89681ee011f9e70f85937fd09c4bd61ed4a59981d275a1f255b",
line 2144, in _infer_image
return func(ctx)
File
"/var/lib/ceph/4067126d-01cb-40af-824a-881c130140f8/cephadm.8b92cafd937eb89681ee011f9e70f85937fd09c4bd61ed4a59981d275a1f255b",
line 2019, in _validate_fsid
return func(ctx)
File
"/var/lib/ceph/4067126d-01cb-40af-824a-881c130140f8/cephadm.8b92cafd937eb89681ee011f9e70f85937fd09c4bd61ed4a59981d275a1f255b",
line 6272, in command_ceph_volume
out, err, code = call_throws(ctx, c.run_cmd(),
verbosity=CallVerbosity.QUIET_UNLESS_ERROR)
File
"/var/lib/ceph/4067126d-01cb-40af-824a-881c130140f8/cephadm.8b92cafd937eb89681ee011f9e70f85937fd09c4bd61ed4a59981d275a1f255b",
line 1807, in call_throws
raise RuntimeError('Failed command: %s' % ' '.join(command))
RuntimeError: Failed command: /usr/bin/docker run --rm --ipc=host
--stop-signal=SIGTERM --ulimit nofile=1048576 --net=host --entrypoint
/usr/sbin/ceph-volume --privileged --group-add=disk --init -e
CONTAINER_IMAGE=quay.io/ceph/ceph@
sha256:92e8fa7d8ca17a7a5bbfde6e596fdfecc8e165fcb94d86493f4e6c7b1f326e4e
-e NODE_NAME=sysmon1 -e CEPH_USE_RANDOM_NONCE=1 -e
CEPH_VOLUME_SKIP_RESTORECON=yes -e CEPH_VOLUME_DEBUG=1 -v
/var/run/ceph/4067126d-01cb-40af-824a-881c130140f8:
/var/run/ceph:z -v
/var/log/ceph/4067126d-01cb-40af-824a-881c130140f8:/var/log/ceph:z -v
/var/lib/ceph/4067126d-01cb-40af-824a-881c130140f8/crash:/var/lib/ceph/crash:z
-v /dev:/dev -v /run/udev:/run/udev -v /sys:/sys -v /run/lvm:/ru
n/lvm -v /run/lock/lvm:/run/lock/lvm -v /:/rootfs -v
/tmp/ceph-tmpl1e27bun:/etc/ceph/ceph.conf:z
quay.io/ceph/ceph@sha256:92e8fa7d8ca17a7a5bbfde6e596fdfecc8e165fcb94d86493f4e6c7b1f326e4e
inventory --format=json-pretty --filter-for-b
atch
host noc3 `cephadm ceph-volume` failed: cephadm exited with an error
code: 1, stderr: Inferring config
/var/lib/ceph/4067126d-01cb-40af-824a-881c130140f8/mon.noc3/config
Non-zero exit code 1 from /usr/bin/docker run --rm --ipc=host
--stop-signal=SIGTERM --ulimit nofile=1048576 --net=host --entrypoint
/usr/sbin/ceph-volume --privileged --group-add=disk --init -e
CONTAINER_IMAGE=quay.io/ceph/ceph@sha2
56:92e8fa7d8ca17a7a5bbfde6e596fdfecc8e165fcb94d86493f4e6c7b1f326e4e -e
NODE_NAME=noc3 -e CEPH_USE_RANDOM_NONCE=1 -e
CEPH_VOLUME_SKIP_RESTORECON=yes -e CEPH_VOLUME_DEBUG=1 -v
/var/run/ceph/4067126d-01cb-40af-824a-881c130140f8:/var/ru
n/ceph:z -v
/var/log/ceph/4067126d-01cb-40af-824a-881c130140f8:/var/log/ceph:z -v
/var/lib/ceph/4067126d-01cb-40af-824a-881c130140f8/crash:/var/lib/ceph/crash:z
-v /dev:/dev -v /run/udev:/run/udev -v /sys:/sys -v /run/lvm:/run/lvm -
v /run/lock/lvm:/run/lock/lvm -v /:/rootfs -v
/tmp/ceph-tmpjjmsgag7:/etc/ceph/ceph.conf:z
quay.io/ceph/ceph@sha256:92e8fa7d8ca17a7a5bbfde6e596fdfecc8e165fcb94d86493f4e6c7b1f326e4e
inventory --format=json-pretty --filter-for-batch
/usr/bin/docker: stderr Traceback (most recent call last):
/usr/bin/docker: stderr File "/usr/sbin/ceph-volume", line 11, in
<module>
/usr/bin/docker: stderr load_entry_point('ceph-volume==1.0.0',
'console_scripts', 'ceph-volume')()
/usr/bin/docker: stderr File
"/usr/lib/python3.6/site-packages/ceph_volume/main.py", line 41, in
__init__
/usr/bin/docker: stderr self.main(self.argv)
/usr/bin/docker: stderr File
"/usr/lib/python3.6/site-packages/ceph_volume/decorators.py", line 59,
in newfunc
/usr/bin/docker: stderr return f(*a, **kw)
/usr/bin/docker: stderr File
"/usr/lib/python3.6/site-packages/ceph_volume/main.py", line 153, in main
/usr/bin/docker: stderr terminal.dispatch(self.mapper, subcommand_args)
/usr/bin/docker: stderr File
"/usr/lib/python3.6/site-packages/ceph_volume/terminal.py", line 194, in
dispatch
/usr/bin/docker: stderr instance.main()
/usr/bin/docker: stderr File
"/usr/lib/python3.6/site-packages/ceph_volume/inventory/main.py", line
60, in main
/usr/bin/docker: stderr list_all=self.args.list_all))
/usr/bin/docker: stderr File
"/usr/lib/python3.6/site-packages/ceph_volume/util/device.py", line 50,
in __init__
/usr/bin/docker: stderr sys_info.devices.keys()]
/usr/bin/docker: stderr File
"/usr/lib/python3.6/site-packages/ceph_volume/util/device.py", line 49,
in <listcomp>
/usr/bin/docker: stderr all_devices_vgs=all_devices_vgs) for k in
/usr/bin/docker: stderr File
"/usr/lib/python3.6/site-packages/ceph_volume/util/device.py", line 147,
in __init__
/usr/bin/docker: stderr self.available_lvm,
self.rejected_reasons_lvm = self._check_lvm_reject_reasons()
/usr/bin/docker: stderr File
"/usr/lib/python3.6/site-packages/ceph_volume/util/device.py", line 646,
in _check_lvm_reject_reasons
/usr/bin/docker: stderr
rejected.extend(self._check_generic_reject_reasons())
/usr/bin/docker: stderr File
"/usr/lib/python3.6/site-packages/ceph_volume/util/device.py", line 601,
in _check_generic_reject_reasons
/usr/bin/docker: stderr if self.is_acceptable_device:
/usr/bin/docker: stderr File
"/usr/lib/python3.6/site-packages/ceph_volume/util/device.py", line 502,
in is_acceptable_device
/usr/bin/docker: stderr return self.is_device or self.is_partition
or self.is_lv
/usr/bin/docker: stderr File
"/usr/lib/python3.6/site-packages/ceph_volume/util/device.py", line 482,
in is_partition
/usr/bin/docker: stderr return self.blkid_api['TYPE'] == 'part'
/usr/bin/docker: stderr KeyError: 'TYPE'
Traceback (most recent call last):
File
"/var/lib/ceph/4067126d-01cb-40af-824a-881c130140f8/cephadm.8b92cafd937eb89681ee011f9e70f85937fd09c4bd61ed4a59981d275a1f255b",
line 9679, in <module>
main()
File
"/var/lib/ceph/4067126d-01cb-40af-824a-881c130140f8/cephadm.8b92cafd937eb89681ee011f9e70f85937fd09c4bd61ed4a59981d275a1f255b",
line 9667, in main
r = ctx.func(ctx)
File
"/var/lib/ceph/4067126d-01cb-40af-824a-881c130140f8/cephadm.8b92cafd937eb89681ee011f9e70f85937fd09c4bd61ed4a59981d275a1f255b",
line 2116, in _infer_config
return func(ctx)
File
"/var/lib/ceph/4067126d-01cb-40af-824a-881c130140f8/cephadm.8b92cafd937eb89681ee011f9e70f85937fd09c4bd61ed4a59981d275a1f255b",
line 2032, in _infer_fsid
return func(ctx)
File
"/var/lib/ceph/4067126d-01cb-40af-824a-881c130140f8/cephadm.8b92cafd937eb89681ee011f9e70f85937fd09c4bd61ed4a59981d275a1f255b",
line 2144, in _infer_image
return func(ctx)
File
"/var/lib/ceph/4067126d-01cb-40af-824a-881c130140f8/cephadm.8b92cafd937eb89681ee011f9e70f85937fd09c4bd61ed4a59981d275a1f255b",
line 2019, in _validate_fsid
return func(ctx)
File
"/var/lib/ceph/4067126d-01cb-40af-824a-881c130140f8/cephadm.8b92cafd937eb89681ee011f9e70f85937fd09c4bd61ed4a59981d275a1f255b",
line 6272, in command_ceph_volume
out, err, code = call_throws(ctx, c.run_cmd(),
verbosity=CallVerbosity.QUIET_UNLESS_ERROR)
File
"/var/lib/ceph/4067126d-01cb-40af-824a-881c130140f8/cephadm.8b92cafd937eb89681ee011f9e70f85937fd09c4bd61ed4a59981d275a1f255b",
line 1807, in call_throws
raise RuntimeError('Failed command: %s' % ' '.join(command))
RuntimeError: Failed command: /usr/bin/docker run --rm --ipc=host
--stop-signal=SIGTERM --ulimit nofile=1048576 --net=host --entrypoint
/usr/sbin/ceph-volume --privileged --group-add=disk --init -e
CONTAINER_IMAGE=quay.io/ceph/ceph@
sha256:92e8fa7d8ca17a7a5bbfde6e596fdfecc8e165fcb94d86493f4e6c7b1f326e4e
-e NODE_NAME=noc3 -e CEPH_USE_RANDOM_NONCE=1 -e
CEPH_VOLUME_SKIP_RESTORECON=yes -e CEPH_VOLUME_DEBUG=1 -v
/var/run/ceph/4067126d-01cb-40af-824a-881c130140f8:/va
r/run/ceph:z -v
/var/log/ceph/4067126d-01cb-40af-824a-881c130140f8:/var/log/ceph:z -v
/var/lib/ceph/4067126d-01cb-40af-824a-881c130140f8/crash:/var/lib/ceph/crash:z
-v /dev:/dev -v /run/udev:/run/udev -v /sys:/sys -v /run/lvm:/run/l
vm -v /run/lock/lvm:/run/lock/lvm -v /:/rootfs -v
/tmp/ceph-tmpjjmsgag7:/etc/ceph/ceph.conf:z
quay.io/ceph/ceph@sha256:92e8fa7d8ca17a7a5bbfde6e596fdfecc8e165fcb94d86493f4e6c7b1f326e4e
inventory --format=json-pretty --filter-for-batc
h
....
_______________________________________________
ceph-users mailing list -- ceph-users@xxxxxxx
To unsubscribe send an email to ceph-users-leave@xxxxxxx
