Hello,
ceph is stuck since 4 days with 0.064% misplaced and i dunno why. Can
anyone help me to get it fixed?
I did restart some OSDs and reweight them again to get some data
moving but that did not help.
root@node01:~ # ceph -s
cluster:
id: 251c937e-0b55-48c1-8f34-96e84e4023d4
health: HEALTH_WARN
1803/2799972 objects misplaced (0.064%)
mon node02 is low on available space
services:
mon: 3 daemons, quorum node01,node02,node03
mgr: node03(active), standbys: node01, node02
osd: 16 osds: 16 up, 16 in; 1 remapped pgs
data:
pools: 1 pools, 512 pgs
objects: 933.32k objects, 2.68TiB
usage: 9.54TiB used, 5.34TiB / 14.9TiB avail
pgs: 1803/2799972 objects misplaced (0.064%)
511 active+clean
1 active+clean+remapped
io:
client: 131KiB/s rd, 8.57MiB/s wr, 28op/s rd, 847op/s wr
root@node01:~ # ceph health detail
HEALTH_WARN 1803/2800179 objects misplaced (0.064%); mon node02 is low
on available space
OBJECT_MISPLACED 1803/2800179 objects misplaced (0.064%)
MON_DISK_LOW mon node02 is low on available space
mon.node02 has 28% avail
root@node01:~ # ceph versions
{
"mon": {
"ceph version 12.2.13 (98af9a6b9a46b2d562a0de4b09263d70aeb1c9dd)
luminous (stable)": 3
},
"mgr": {
"ceph version 12.2.13 (98af9a6b9a46b2d562a0de4b09263d70aeb1c9dd)
luminous (stable)": 3
},
"osd": {
"ceph version 12.2.13 (98af9a6b9a46b2d562a0de4b09263d70aeb1c9dd)
luminous (stable)": 16
},
"mds": {},
"overall": {
"ceph version 12.2.13 (98af9a6b9a46b2d562a0de4b09263d70aeb1c9dd)
luminous (stable)": 22
}
}
root@node02:~ # df -h
Filesystem Size Used Avail Use% Mounted on
udev 63G 0 63G 0% /dev
tmpfs 13G 1.3G 12G 11% /run
/dev/sda3 46G 31G 14G 70% /
tmpfs 63G 57M 63G 1% /dev/shm
tmpfs 5.0M 0 5.0M 0% /run/lock
tmpfs 63G 0 63G 0% /sys/fs/cgroup
/dev/sda1 922M 206M 653M 24% /boot
/dev/fuse 30M 144K 30M 1% /etc/pve
/dev/sde1 93M 5.4M 88M 6% /var/lib/ceph/osd/ceph-11
/dev/sdf1 93M 5.4M 88M 6% /var/lib/ceph/osd/ceph-14
/dev/sdc1 889G 676G 214G 77% /var/lib/ceph/osd/ceph-3
/dev/sdb1 889G 667G 222G 76% /var/lib/ceph/osd/ceph-2
/dev/sdd1 93M 5.4M 88M 6% /var/lib/ceph/osd/ceph-7
tmpfs 13G 0 13G 0% /run/user/0
root@node02:~ # ceph osd tree
ID CLASS WEIGHT TYPE NAME STATUS REWEIGHT PRI-AFF
-1 14.34781 root default
-2 4.25287 host node01
0 hdd 0.85999 osd.0 up 0.80005 1.00000
1 hdd 0.86749 osd.1 up 0.85004 1.00000
6 hdd 0.87270 osd.6 up 0.90002 1.00000
12 hdd 0.78000 osd.12 up 0.95001 1.00000
13 hdd 0.87270 osd.13 up 0.95001 1.00000
-3 3.91808 host node02
2 hdd 0.70000 osd.2 up 0.80005 1.00000
3 hdd 0.59999 osd.3 up 0.85004 1.00000
7 hdd 0.87270 osd.7 up 0.85004 1.00000
11 hdd 0.87270 osd.11 up 0.75006 1.00000
14 hdd 0.87270 osd.14 up 0.85004 1.00000
-4 6.17686 host node03
4 hdd 0.87000 osd.4 up 1.00000 1.00000
5 hdd 0.87000 osd.5 up 1.00000 1.00000
8 hdd 0.87270 osd.8 up 1.00000 1.00000
10 hdd 0.87270 osd.10 up 1.00000 1.00000
15 hdd 0.87270 osd.15 up 1.00000 1.00000
16 hdd 1.81879 osd.16 up 1.00000 1.00000
root@node01:~ # ceph osd df tree
ID CLASS WEIGHT REWEIGHT SIZE USE DATA OMAP META
AVAIL %USE VAR PGS TYPE NAME
-1 14.55780 - 14.9TiB 9.45TiB 7.46TiB 1.47GiB 23.2GiB
5.43TiB 63.52 1.00 - root default
-2 4.27286 - 4.35TiB 3.15TiB 2.41TiB 486MiB 7.62GiB
1.21TiB 72.32 1.14 - host node01
0 hdd 0.85999 0.80005 888GiB 619GiB 269GiB 92.3MiB 0B
269GiB 69.72 1.10 89 osd.0
1 hdd 0.86749 0.85004 888GiB 641GiB 248GiB 109MiB 0B
248GiB 72.12 1.14 92 osd.1
6 hdd 0.87270 0.90002 894GiB 634GiB 632GiB 98.9MiB 2.65GiB
259GiB 70.99 1.12 107 osd.6
12 hdd 0.79999 0.95001 894GiB 664GiB 661GiB 94.4MiB 2.52GiB
230GiB 74.31 1.17 112 osd.12
13 hdd 0.87270 0.95001 894GiB 665GiB 663GiB 91.7MiB 2.46GiB
229GiB 74.43 1.17 112 osd.13
-3 4.10808 - 4.35TiB 3.17TiB 2.18TiB 479MiB 6.99GiB
1.18TiB 72.86 1.15 - host node02
2 hdd 0.78999 0.75006 888GiB 654GiB 235GiB 95.6MiB 0B
235GiB 73.57 1.16 94 osd.2
3 hdd 0.70000 0.80005 888GiB 737GiB 151GiB 114MiB 0B
151GiB 82.98 1.31 105 osd.3
7 hdd 0.87270 0.85004 894GiB 612GiB 610GiB 88.9MiB 2.43GiB
281GiB 68.50 1.08 103 osd.7
11 hdd 0.87270 0.75006 894GiB 576GiB 574GiB 81.8MiB 2.19GiB
317GiB 64.47 1.01 97 osd.11
14 hdd 0.87270 0.85004 894GiB 669GiB 666GiB 98.8MiB 2.37GiB
225GiB 74.85 1.18 112 osd.14
-4 6.17686 - 6.17TiB 3.13TiB 2.86TiB 541MiB 8.58GiB
3.04TiB 50.73 0.80 - host node03
4 hdd 0.87000 1.00000 888GiB 504GiB 384GiB 124MiB 0B
384GiB 56.72 0.89 72 osd.4
5 hdd 0.87000 1.00000 888GiB 520GiB 368GiB 96.2MiB 0B
368GiB 58.57 0.92 75 osd.5
8 hdd 0.87270 1.00000 894GiB 508GiB 505GiB 80.2MiB 2.07GiB
386GiB 56.80 0.89 85 osd.8
10 hdd 0.87270 1.00000 894GiB 374GiB 373GiB 51.9MiB 1.73GiB
519GiB 41.88 0.66 63 osd.10
15 hdd 0.87270 1.00000 894GiB 504GiB 502GiB 60.1MiB 1.99GiB
390GiB 56.37 0.89 84 osd.15
16 hdd 1.81879 1.00000 1.82TiB 797GiB 794GiB 129MiB 2.79GiB
1.04TiB 42.77 0.67 134 osd.16
TOTAL 14.9TiB 9.45TiB 7.46TiB 1.47GiB 23.2GiB 5.43TiB 63.52
MIN/MAX VAR: 0.66/1.31 STDDEV: 11.59
root@node02:~ # df -h
Filesystem Size Used Avail Use% Mounted on
udev 63G 0 63G 0% /dev
tmpfs 13G 1.4G 12G 11% /run
/dev/sda3 46G 31G 14G 70% /
tmpfs 63G 63M 63G 1% /dev/shm
tmpfs 5.0M 0 5.0M 0% /run/lock
tmpfs 63G 0 63G 0% /sys/fs/cgroup
/dev/sda1 922M 206M 653M 24% /boot
/dev/fuse 30M 140K 30M 1% /etc/pve
/dev/sde1 93M 5.4M 88M 6% /var/lib/ceph/osd/ceph-11
/dev/sdf1 93M 5.4M 88M 6% /var/lib/ceph/osd/ceph-14
/dev/sdc1 889G 738G 152G 83% /var/lib/ceph/osd/ceph-3
/dev/sdb1 889G 654G 235G 74% /var/lib/ceph/osd/ceph-2
/dev/sdd1 93M 5.4M 88M 6% /var/lib/ceph/osd/ceph-7
tmpfs 13G 0 13G 0% /run/user/0
Any idea?
Thanks,
Michael
_______________________________________________
ceph-users mailing list -- ceph-users@xxxxxxx
To unsubscribe send an email to ceph-users-leave@xxxxxxx
