Hi,
have you tried restarting all osds?
Am 08.07.2020 um 15:43 schrieb Ml Ml:
> Hello,
>
> ceph is stuck since 4 days with 0.064% misplaced and i dunno why. Can
> anyone help me to get it fixed?
> I did restart some OSDs and reweight them again to get some data
> moving but that did not help.
>
> root@node01:~ # ceph -s
> cluster:
> id: 251c937e-0b55-48c1-8f34-96e84e4023d4
> health: HEALTH_WARN
> 1803/2799972 objects misplaced (0.064%)
> mon node02 is low on available space
>
> services:
> mon: 3 daemons, quorum node01,node02,node03
> mgr: node03(active), standbys: node01, node02
> osd: 16 osds: 16 up, 16 in; 1 remapped pgs
>
> data:
> pools: 1 pools, 512 pgs
> objects: 933.32k objects, 2.68TiB
> usage: 9.54TiB used, 5.34TiB / 14.9TiB avail
> pgs: 1803/2799972 objects misplaced (0.064%)
> 511 active+clean
> 1 active+clean+remapped
>
> io:
> client: 131KiB/s rd, 8.57MiB/s wr, 28op/s rd, 847op/s wr
>
> root@node01:~ # ceph health detail
> HEALTH_WARN 1803/2800179 objects misplaced (0.064%); mon node02 is low
> on available space
> OBJECT_MISPLACED 1803/2800179 objects misplaced (0.064%)
> MON_DISK_LOW mon node02 is low on available space
> mon.node02 has 28% avail
> root@node01:~ # ceph versions
> {
> "mon": {
> "ceph version 12.2.13 (98af9a6b9a46b2d562a0de4b09263d70aeb1c9dd)
> luminous (stable)": 3
> },
> "mgr": {
> "ceph version 12.2.13 (98af9a6b9a46b2d562a0de4b09263d70aeb1c9dd)
> luminous (stable)": 3
> },
> "osd": {
> "ceph version 12.2.13 (98af9a6b9a46b2d562a0de4b09263d70aeb1c9dd)
> luminous (stable)": 16
> },
> "mds": {},
> "overall": {
> "ceph version 12.2.13 (98af9a6b9a46b2d562a0de4b09263d70aeb1c9dd)
> luminous (stable)": 22
> }
> }
>
> root@node02:~ # df -h
> Filesystem Size Used Avail Use% Mounted on
> udev 63G 0 63G 0% /dev
> tmpfs 13G 1.3G 12G 11% /run
> /dev/sda3 46G 31G 14G 70% /
> tmpfs 63G 57M 63G 1% /dev/shm
> tmpfs 5.0M 0 5.0M 0% /run/lock
> tmpfs 63G 0 63G 0% /sys/fs/cgroup
> /dev/sda1 922M 206M 653M 24% /boot
> /dev/fuse 30M 144K 30M 1% /etc/pve
> /dev/sde1 93M 5.4M 88M 6% /var/lib/ceph/osd/ceph-11
> /dev/sdf1 93M 5.4M 88M 6% /var/lib/ceph/osd/ceph-14
> /dev/sdc1 889G 676G 214G 77% /var/lib/ceph/osd/ceph-3
> /dev/sdb1 889G 667G 222G 76% /var/lib/ceph/osd/ceph-2
> /dev/sdd1 93M 5.4M 88M 6% /var/lib/ceph/osd/ceph-7
> tmpfs 13G 0 13G 0% /run/user/0
>
> root@node02:~ # ceph osd tree
> ID CLASS WEIGHT TYPE NAME STATUS REWEIGHT PRI-AFF
> -1 14.34781 root default
> -2 4.25287 host node01
> 0 hdd 0.85999 osd.0 up 0.80005 1.00000
> 1 hdd 0.86749 osd.1 up 0.85004 1.00000
> 6 hdd 0.87270 osd.6 up 0.90002 1.00000
> 12 hdd 0.78000 osd.12 up 0.95001 1.00000
> 13 hdd 0.87270 osd.13 up 0.95001 1.00000
> -3 3.91808 host node02
> 2 hdd 0.70000 osd.2 up 0.80005 1.00000
> 3 hdd 0.59999 osd.3 up 0.85004 1.00000
> 7 hdd 0.87270 osd.7 up 0.85004 1.00000
> 11 hdd 0.87270 osd.11 up 0.75006 1.00000
> 14 hdd 0.87270 osd.14 up 0.85004 1.00000
> -4 6.17686 host node03
> 4 hdd 0.87000 osd.4 up 1.00000 1.00000
> 5 hdd 0.87000 osd.5 up 1.00000 1.00000
> 8 hdd 0.87270 osd.8 up 1.00000 1.00000
> 10 hdd 0.87270 osd.10 up 1.00000 1.00000
> 15 hdd 0.87270 osd.15 up 1.00000 1.00000
> 16 hdd 1.81879 osd.16 up 1.00000 1.00000
>
> root@node01:~ # ceph osd df tree
> ID CLASS WEIGHT REWEIGHT SIZE USE DATA OMAP META
> AVAIL %USE VAR PGS TYPE NAME
> -1 14.55780 - 14.9TiB 9.45TiB 7.46TiB 1.47GiB 23.2GiB
> 5.43TiB 63.52 1.00 - root default
> -2 4.27286 - 4.35TiB 3.15TiB 2.41TiB 486MiB 7.62GiB
> 1.21TiB 72.32 1.14 - host node01
> 0 hdd 0.85999 0.80005 888GiB 619GiB 269GiB 92.3MiB 0B
> 269GiB 69.72 1.10 89 osd.0
> 1 hdd 0.86749 0.85004 888GiB 641GiB 248GiB 109MiB 0B
> 248GiB 72.12 1.14 92 osd.1
> 6 hdd 0.87270 0.90002 894GiB 634GiB 632GiB 98.9MiB 2.65GiB
> 259GiB 70.99 1.12 107 osd.6
> 12 hdd 0.79999 0.95001 894GiB 664GiB 661GiB 94.4MiB 2.52GiB
> 230GiB 74.31 1.17 112 osd.12
> 13 hdd 0.87270 0.95001 894GiB 665GiB 663GiB 91.7MiB 2.46GiB
> 229GiB 74.43 1.17 112 osd.13
> -3 4.10808 - 4.35TiB 3.17TiB 2.18TiB 479MiB 6.99GiB
> 1.18TiB 72.86 1.15 - host node02
> 2 hdd 0.78999 0.75006 888GiB 654GiB 235GiB 95.6MiB 0B
> 235GiB 73.57 1.16 94 osd.2
> 3 hdd 0.70000 0.80005 888GiB 737GiB 151GiB 114MiB 0B
> 151GiB 82.98 1.31 105 osd.3
> 7 hdd 0.87270 0.85004 894GiB 612GiB 610GiB 88.9MiB 2.43GiB
> 281GiB 68.50 1.08 103 osd.7
> 11 hdd 0.87270 0.75006 894GiB 576GiB 574GiB 81.8MiB 2.19GiB
> 317GiB 64.47 1.01 97 osd.11
> 14 hdd 0.87270 0.85004 894GiB 669GiB 666GiB 98.8MiB 2.37GiB
> 225GiB 74.85 1.18 112 osd.14
> -4 6.17686 - 6.17TiB 3.13TiB 2.86TiB 541MiB 8.58GiB
> 3.04TiB 50.73 0.80 - host node03
> 4 hdd 0.87000 1.00000 888GiB 504GiB 384GiB 124MiB 0B
> 384GiB 56.72 0.89 72 osd.4
> 5 hdd 0.87000 1.00000 888GiB 520GiB 368GiB 96.2MiB 0B
> 368GiB 58.57 0.92 75 osd.5
> 8 hdd 0.87270 1.00000 894GiB 508GiB 505GiB 80.2MiB 2.07GiB
> 386GiB 56.80 0.89 85 osd.8
> 10 hdd 0.87270 1.00000 894GiB 374GiB 373GiB 51.9MiB 1.73GiB
> 519GiB 41.88 0.66 63 osd.10
> 15 hdd 0.87270 1.00000 894GiB 504GiB 502GiB 60.1MiB 1.99GiB
> 390GiB 56.37 0.89 84 osd.15
> 16 hdd 1.81879 1.00000 1.82TiB 797GiB 794GiB 129MiB 2.79GiB
> 1.04TiB 42.77 0.67 134 osd.16
> TOTAL 14.9TiB 9.45TiB 7.46TiB 1.47GiB 23.2GiB 5.43TiB 63.52
> MIN/MAX VAR: 0.66/1.31 STDDEV: 11.59
>
> root@node02:~ # df -h
> Filesystem Size Used Avail Use% Mounted on
> udev 63G 0 63G 0% /dev
> tmpfs 13G 1.4G 12G 11% /run
> /dev/sda3 46G 31G 14G 70% /
> tmpfs 63G 63M 63G 1% /dev/shm
> tmpfs 5.0M 0 5.0M 0% /run/lock
> tmpfs 63G 0 63G 0% /sys/fs/cgroup
> /dev/sda1 922M 206M 653M 24% /boot
> /dev/fuse 30M 140K 30M 1% /etc/pve
> /dev/sde1 93M 5.4M 88M 6% /var/lib/ceph/osd/ceph-11
> /dev/sdf1 93M 5.4M 88M 6% /var/lib/ceph/osd/ceph-14
> /dev/sdc1 889G 738G 152G 83% /var/lib/ceph/osd/ceph-3
> /dev/sdb1 889G 654G 235G 74% /var/lib/ceph/osd/ceph-2
> /dev/sdd1 93M 5.4M 88M 6% /var/lib/ceph/osd/ceph-7
> tmpfs 13G 0 13G 0% /run/user/0
>
>
>
> Any idea?
>
> Thanks,
> Michael
> _______________________________________________
> ceph-users mailing list -- ceph-users@xxxxxxx
> To unsubscribe send an email to ceph-users-leave@xxxxxxx
>
--
Mit freundlichen Grüßen,
Florian Wiessner
Smart Weblications GmbH
Martinsberger Str. 1
D-95119 Naila
fon.: +49 9282 9638 200
fax.: +49 9282 9638 205
24/7: +49 900 144 000 00 - 0,99 EUR/Min*
http://www.smart-weblications.de
--
Sitz der Gesellschaft: Naila
Geschäftsführer: Florian Wiessner
HRB-Nr.: HRB 3840 Amtsgericht Hof
*aus dem dt. Festnetz, ggf. abweichende Preise aus dem Mobilfunknetz
_______________________________________________
ceph-users mailing list -- ceph-users@xxxxxxx
To unsubscribe send an email to ceph-users-leave@xxxxxxx
