Hello List,
any ideas why my OSDs are that unbalanced ?
root@ceph01:~# ceph -s
cluster:
id: 5436dd5d-83d4-4dc8-a93b-60ab5db145df
health: HEALTH_WARN
1 nearfull osd(s)
4 pool(s) nearfull
services:
mon: 3 daemons, quorum ceph03,ceph01,ceph02 (age 2w)
mgr: ceph03(active, since 4M), standbys: ceph02.jwvivm
mds: backup:1 {0=backup.ceph06.hdjehi=up:active} 3 up:standby
osd: 56 osds: 56 up (since 29h), 56 in (since 3d)
task status:
scrub status:
mds.backup.ceph06.hdjehi: idle
data:
pools: 4 pools, 1185 pgs
objects: 24.29M objects, 44 TiB
usage: 151 TiB used, 55 TiB / 206 TiB avail
pgs: 675 active+clean
476 active+clean+snaptrim_wait
30 active+clean+snaptrim
4 active+clean+scrubbing+deep
root@ceph01:~# ceph osd df tree
ID CLASS WEIGHT REWEIGHT SIZE RAW USE DATA OMAP
META AVAIL %USE VAR PGS STATUS TYPE NAME
-1 206.79979 - 206 TiB 151 TiB 151 TiB 36 GiB
503 GiB 55 TiB 73.23 1.00 - root default
-2 28.89995 - 29 TiB 20 TiB 20 TiB 5.5 GiB
74 GiB 8.9 TiB 69.19 0.94 - host ceph01
0 hdd 2.70000 1.00000 2.7 TiB 1.8 TiB 1.8 TiB 590 MiB
6.9 GiB 908 GiB 66.81 0.91 44 up osd.0
1 hdd 2.70000 1.00000 2.7 TiB 1.6 TiB 1.6 TiB 411 MiB
6.5 GiB 1.1 TiB 60.43 0.83 39 up osd.1
4 hdd 2.70000 1.00000 2.7 TiB 1.8 TiB 1.8 TiB 501 MiB
6.8 GiB 898 GiB 67.15 0.92 43 up osd.4
8 hdd 2.70000 1.00000 2.7 TiB 2.0 TiB 2.0 TiB 453 MiB
7.0 GiB 700 GiB 74.39 1.02 47 up osd.8
11 hdd 1.70000 1.00000 1.7 TiB 1.3 TiB 1.3 TiB 356 MiB
5.6 GiB 433 GiB 75.39 1.03 31 up osd.11
12 hdd 2.70000 1.00000 2.7 TiB 2.1 TiB 2.1 TiB 471 MiB
7.0 GiB 591 GiB 78.40 1.07 48 up osd.12
14 hdd 2.70000 1.00000 2.7 TiB 1.6 TiB 1.6 TiB 448 MiB
6.0 GiB 1.1 TiB 59.68 0.82 38 up osd.14
18 hdd 2.70000 1.00000 2.7 TiB 1.7 TiB 1.7 TiB 515 MiB
6.2 GiB 980 GiB 64.15 0.88 41 up osd.18
22 hdd 1.70000 1.00000 1.7 TiB 1.2 TiB 1.2 TiB 360 MiB
4.2 GiB 491 GiB 72.06 0.98 29 up osd.22
30 hdd 1.70000 1.00000 1.7 TiB 1.2 TiB 1.2 TiB 366 MiB
4.7 GiB 558 GiB 68.26 0.93 28 up osd.30
33 hdd 1.59999 1.00000 1.6 TiB 1.2 TiB 1.2 TiB 406 MiB
4.9 GiB 427 GiB 74.28 1.01 29 up osd.33
64 hdd 3.29999 1.00000 3.3 TiB 2.4 TiB 2.4 TiB 736 MiB
8.6 GiB 915 GiB 73.22 1.00 60 up osd.64
-3 29.69995 - 30 TiB 22 TiB 22 TiB 5.4 GiB
81 GiB 7.9 TiB 73.20 1.00 - host ceph02
2 hdd 1.70000 1.00000 1.7 TiB 1.3 TiB 1.2 TiB 402 MiB
5.2 GiB 476 GiB 72.93 1.00 30 up osd.2
3 hdd 2.70000 1.00000 2.7 TiB 2.0 TiB 2.0 TiB 653 MiB
7.8 GiB 652 GiB 76.15 1.04 49 up osd.3
7 hdd 2.70000 1.00000 2.7 TiB 2.5 TiB 2.5 TiB 456 MiB
7.7 GiB 209 GiB 92.36 1.26 56 up osd.7
9 hdd 2.70000 1.00000 2.7 TiB 1.9 TiB 1.9 TiB 434 MiB
7.2 GiB 781 GiB 71.46 0.98 46 up osd.9
13 hdd 2.39999 1.00000 2.4 TiB 1.6 TiB 1.6 TiB 451 MiB
6.1 GiB 823 GiB 66.28 0.91 38 up osd.13
16 hdd 2.70000 1.00000 2.7 TiB 1.6 TiB 1.6 TiB 375 MiB
6.4 GiB 1.1 TiB 59.84 0.82 39 up osd.16
19 hdd 1.70000 1.00000 1.7 TiB 1.1 TiB 1.1 TiB 323 MiB
4.7 GiB 601 GiB 65.80 0.90 27 up osd.19
23 hdd 2.70000 1.00000 2.7 TiB 2.2 TiB 2.2 TiB 471 MiB
7.7 GiB 520 GiB 80.99 1.11 50 up osd.23
24 hdd 1.70000 1.00000 1.7 TiB 1.4 TiB 1.4 TiB 371 MiB
5.5 GiB 273 GiB 84.44 1.15 32 up osd.24
28 hdd 2.70000 1.00000 2.7 TiB 1.9 TiB 1.9 TiB 428 MiB
7.4 GiB 818 GiB 70.07 0.96 44 up osd.28
31 hdd 2.70000 1.00000 2.7 TiB 2.0 TiB 2.0 TiB 516 MiB
7.4 GiB 660 GiB 75.85 1.04 48 up osd.31
32 hdd 3.29999 1.00000 3.3 TiB 2.2 TiB 2.2 TiB 661 MiB
7.9 GiB 1.2 TiB 64.86 0.89 52 up osd.32
-4 26.29996 - 26 TiB 18 TiB 18 TiB 4.3 GiB
73 GiB 8.0 TiB 69.58 0.95 - host ceph03
5 hdd 1.70000 1.00000 1.7 TiB 1.2 TiB 1.2 TiB 298 MiB
5.2 GiB 541 GiB 69.21 0.95 29 up osd.5
6 hdd 1.70000 1.00000 1.7 TiB 1.0 TiB 1.0 TiB 321 MiB
4.4 GiB 697 GiB 60.34 0.82 25 up osd.6
10 hdd 2.70000 1.00000 2.7 TiB 1.9 TiB 1.9 TiB 431 MiB
7.5 GiB 796 GiB 70.89 0.97 46 up osd.10
15 hdd 2.70000 1.00000 2.7 TiB 1.9 TiB 1.9 TiB 500 MiB
6.6 GiB 805 GiB 70.55 0.96 44 up osd.15
17 hdd 1.59999 1.00000 1.6 TiB 1.1 TiB 1.1 TiB 377 MiB
4.9 GiB 530 GiB 68.05 0.93 27 up osd.17
20 hdd 1.70000 1.00000 1.7 TiB 1.0 TiB 1.0 TiB 223 MiB
4.7 GiB 685 GiB 61.03 0.83 25 up osd.20
21 hdd 2.70000 1.00000 2.7 TiB 1.7 TiB 1.7 TiB 392 MiB
6.7 GiB 951 GiB 65.23 0.89 42 up osd.21
25 hdd 1.70000 1.00000 1.7 TiB 1.1 TiB 1.1 TiB 157 MiB
5.1 GiB 601 GiB 65.83 0.90 27 up osd.25
26 hdd 2.70000 1.00000 2.7 TiB 2.1 TiB 2.1 TiB 512 MiB
7.6 GiB 573 GiB 79.06 1.08 50 up osd.26
27 hdd 2.70000 1.00000 2.7 TiB 1.9 TiB 1.9 TiB 473 MiB
7.6 GiB 805 GiB 70.55 0.96 46 up osd.27
29 hdd 2.70000 1.00000 2.7 TiB 2.1 TiB 2.1 TiB 478 MiB
7.3 GiB 539 GiB 80.29 1.10 50 up osd.29
63 hdd 1.70000 1.00000 1.7 TiB 1.1 TiB 1.1 TiB 195 MiB
5.1 GiB 646 GiB 63.23 0.86 26 up osd.63
-11 24.79999 - 25 TiB 18 TiB 18 TiB 4.1 GiB
59 GiB 6.3 TiB 74.51 1.02 - host ceph04
34 hdd 5.20000 1.00000 5.2 TiB 3.9 TiB 3.8 TiB 954 MiB
13 GiB 1.4 TiB 73.48 1.00 94 up osd.34
42 hdd 5.20000 1.00000 5.2 TiB 3.9 TiB 3.8 TiB 841 MiB
13 GiB 1.4 TiB 73.43 1.00 94 up osd.42
44 hdd 7.20000 1.00000 7.2 TiB 5.5 TiB 5.5 TiB 1.2 GiB
17 GiB 1.6 TiB 77.54 1.06 133 up osd.44
45 hdd 7.20000 1.00000 7.2 TiB 5.2 TiB 5.2 TiB 1.2 GiB
16 GiB 1.9 TiB 73.03 1.00 125 up osd.45
-13 30.09998 - 30 TiB 22 TiB 22 TiB 5.1 GiB
72 GiB 8.0 TiB 73.48 1.00 - host ceph05
39 hdd 7.20000 1.00000 7.2 TiB 5.6 TiB 5.6 TiB 1.4 GiB
17 GiB 1.6 TiB 77.89 1.06 126 up osd.39
40 hdd 7.20000 1.00000 7.2 TiB 5.3 TiB 5.3 TiB 1.2 GiB
17 GiB 1.9 TiB 73.87 1.01 124 up osd.40
41 hdd 7.20000 1.00000 7.2 TiB 5.4 TiB 5.3 TiB 1.1 GiB
17 GiB 1.8 TiB 74.92 1.02 128 up osd.41
43 hdd 5.20000 1.00000 5.2 TiB 3.7 TiB 3.7 TiB 853 MiB
13 GiB 1.5 TiB 71.28 0.97 91 up osd.43
60 hdd 3.29999 1.00000 3.3 TiB 2.1 TiB 2.1 TiB 573 MiB
7.9 GiB 1.2 TiB 63.53 0.87 52 up osd.60
-9 17.59999 - 18 TiB 12 TiB 12 TiB 3.0 GiB
40 GiB 5.3 TiB 70.10 0.96 - host ceph06
35 hdd 7.20000 1.00000 7.2 TiB 5.2 TiB 5.2 TiB 1.3 GiB
16 GiB 1.9 TiB 72.80 0.99 125 up osd.35
36 hdd 5.20000 1.00000 5.2 TiB 3.5 TiB 3.5 TiB 804 MiB
12 GiB 1.8 TiB 66.50 0.91 85 up osd.36
38 hdd 5.20000 1.00000 5.2 TiB 3.7 TiB 3.7 TiB 978 MiB
12 GiB 1.6 TiB 70.02 0.96 88 up osd.38
-15 24.89998 - 25 TiB 18 TiB 18 TiB 4.3 GiB
58 GiB 6.5 TiB 73.75 1.01 - host ceph07
66 hdd 7.20000 1.00000 7.2 TiB 5.3 TiB 5.3 TiB 1.1 GiB
17 GiB 1.8 TiB 74.74 1.02 126 up osd.66
67 hdd 7.20000 1.00000 7.2 TiB 5.3 TiB 5.3 TiB 1.2 GiB
17 GiB 1.8 TiB 74.51 1.02 121 up osd.67
68 hdd 3.29999 1.00000 3.3 TiB 2.3 TiB 2.3 TiB 720 MiB
7.9 GiB 1.0 TiB 68.63 0.94 55 up osd.68
69 hdd 7.20000 1.00000 7.2 TiB 5.3 TiB 5.3 TiB 1.2 GiB
17 GiB 1.8 TiB 74.40 1.02 129 up osd.69
-17 24.50000 - 24 TiB 20 TiB 20 TiB 4.1 GiB
47 GiB 4.4 TiB 82.08 1.12 - host ceph08
37 hdd 9.50000 1.00000 9.5 TiB 7.8 TiB 7.7 TiB 1.5 GiB
18 GiB 1.8 TiB 81.39 1.11 166 up osd.37
46 hdd 5.00000 1.00000 5.0 TiB 4.0 TiB 3.9 TiB 889 MiB
9.3 GiB 1.0 TiB 79.67 1.09 87 up osd.46
47 hdd 5.00000 1.00000 5.0 TiB 4.2 TiB 4.2 TiB 863 MiB
9.7 GiB 817 GiB 83.90 1.15 90 up osd.47
48 hdd 5.00000 1.00000 5.0 TiB 4.2 TiB 4.2 TiB 969 MiB
10 GiB 813 GiB 83.99 1.15 91 up osd.48
TOTAL 206 TiB 151 TiB 151 TiB 36 GiB
503 GiB 55 TiB 73.23
MIN/MAX VAR: 0.82/1.26 STDDEV: 7.00
root@ceph01:~# ceph balancer status
{
"active": true,
"last_optimize_duration": "0:00:00.016174",
"last_optimize_started": "Fri Apr 16 12:54:47 2021",
"mode": "upmap",
"optimize_result": "Unable to find further optimization, or
pool(s) pg_num is decreasing, or distribution is already perfect",
"plans": []
}
root@ceph01:~# ceph versions
{
"mon": {
"ceph version 15.2.5
(2c93eff00150f0cc5f106a559557a58d3d7b6f1f) octopus (stable)": 3
},
"mgr": {
"ceph version 15.2.5
(2c93eff00150f0cc5f106a559557a58d3d7b6f1f) octopus (stable)": 2
},
"osd": {
"ceph version 15.2.5
(2c93eff00150f0cc5f106a559557a58d3d7b6f1f) octopus (stable)": 56
},
"mds": {
"ceph version 15.2.5
(2c93eff00150f0cc5f106a559557a58d3d7b6f1f) octopus (stable)": 4
},
"overall": {
"ceph version 15.2.5
(2c93eff00150f0cc5f106a559557a58d3d7b6f1f) octopus (stable)": 65
}
}
root@ceph01:~# ceph osd crush rule ls
replicated_ruleset
root@ceph01:~# ceph osd crush rule dump replicated_ruleset
{
"rule_id": 0,
"rule_name": "replicated_ruleset",
"ruleset": 0,
"type": 1,
"min_size": 1,
"max_size": 10,
"steps": [
{
"op": "take",
"item": -1,
"item_name": "default"
},
{
"op": "chooseleaf_firstn",
"num": 0,
"type": "host"
},
{
"op": "emit"
}
]
}
Cheers,
Michael
_______________________________________________
ceph-users mailing list -- ceph-users@xxxxxxx
To unsubscribe send an email to ceph-users-leave@xxxxxxx
