On 1/5/19 4:17 PM, Kevin Olbrich wrote:
root@adminnode:~# ceph osd tree
ID CLASS WEIGHT TYPE NAME STATUS REWEIGHT PRI-AFF
-1 30.82903 root default
-16 30.82903 datacenter dc01
-19 30.82903 pod dc01-agg01
-10 17.43365 rack dc01-rack02
-4 7.20665 host node1001
0 hdd 0.90999 osd.0 up 1.00000 1.00000
1 hdd 0.90999 osd.1 up 1.00000 1.00000
5 hdd 0.90999 osd.5 up 1.00000 1.00000
29 hdd 0.90970 osd.29 up 1.00000 1.00000
32 hdd 0.90970 osd.32 down 0 1.00000
33 hdd 0.90970 osd.33 up 1.00000 1.00000
2 ssd 0.43700 osd.2 up 1.00000 1.00000
3 ssd 0.43700 osd.3 up 1.00000 1.00000
4 ssd 0.43700 osd.4 up 1.00000 1.00000
30 ssd 0.43660 osd.30 up 1.00000 1.00000
-7 6.29724 host node1002
9 hdd 0.90999 osd.9 up 1.00000 1.00000
10 hdd 0.90999 osd.10 up 1.00000 1.00000
11 hdd 0.90999 osd.11 up 1.00000 1.00000
12 hdd 0.90999 osd.12 up 1.00000 1.00000
35 hdd 0.90970 osd.35 up 1.00000 1.00000
6 ssd 0.43700 osd.6 up 1.00000 1.00000
7 ssd 0.43700 osd.7 up 1.00000 1.00000
8 ssd 0.43700 osd.8 up 1.00000 1.00000
31 ssd 0.43660 osd.31 up 1.00000 1.00000
-28 2.18318 host node1005
34 ssd 0.43660 osd.34 up 1.00000 1.00000
36 ssd 0.87329 osd.36 up 1.00000 1.00000
37 ssd 0.87329 osd.37 up 1.00000 1.00000
-29 1.74658 host node1006
42 ssd 0.87329 osd.42 up 1.00000 1.00000
43 ssd 0.87329 osd.43 up 1.00000 1.00000
-11 13.39537 rack dc01-rack03
-22 5.38794 host node1003
17 hdd 0.90999 osd.17 up 1.00000 1.00000
18 hdd 0.90999 osd.18 up 1.00000 1.00000
24 hdd 0.90999 osd.24 up 1.00000 1.00000
26 hdd 0.90999 osd.26 up 1.00000 1.00000
13 ssd 0.43700 osd.13 up 1.00000 1.00000
14 ssd 0.43700 osd.14 up 1.00000 1.00000
15 ssd 0.43700 osd.15 up 1.00000 1.00000
16 ssd 0.43700 osd.16 up 1.00000 1.00000
-25 5.38765 host node1004
23 hdd 0.90999 osd.23 up 1.00000 1.00000
25 hdd 0.90999 osd.25 up 1.00000 1.00000
27 hdd 0.90999 osd.27 up 1.00000 1.00000
28 hdd 0.90970 osd.28 up 1.00000 1.00000
19 ssd 0.43700 osd.19 up 1.00000 1.00000
20 ssd 0.43700 osd.20 up 1.00000 1.00000
21 ssd 0.43700 osd.21 up 1.00000 1.00000
22 ssd 0.43700 osd.22 up 1.00000 1.00000
-30 2.61978 host node1007
38 ssd 0.43660 osd.38 up 1.00000 1.00000
39 ssd 0.43660 osd.39 up 1.00000 1.00000
40 ssd 0.87329 osd.40 up 1.00000 1.00000
41 ssd 0.87329 osd.41 up 1.00000 1.00000
========================================================
root@adminnode:~# ceph osd df tree
ID CLASS WEIGHT REWEIGHT SIZE USE AVAIL %USE VAR PGS
TYPE NAME
-1 30.82903 - 29.9TiB 14.0TiB 16.0TiB 46.65 1.00 -
root default
-16 30.82903 - 29.9TiB 14.0TiB 16.0TiB 46.65 1.00 -
datacenter dc01
-19 30.82903 - 29.9TiB 14.0TiB 16.0TiB 46.65 1.00 -
pod dc01-agg01
-10 17.43365 - 16.5TiB 7.31TiB 9.21TiB 44.26 0.95 -
rack dc01-rack02
-4 7.20665 - 6.29TiB 2.76TiB 3.54TiB 43.83 0.94 -
host node1001
0 hdd 0.90999 1.00000 932GiB 356GiB 575GiB 38.22 0.82 95
osd.0
1 hdd 0.90999 1.00000 932GiB 397GiB 534GiB 42.66 0.91 106
osd.1
5 hdd 0.90999 1.00000 932GiB 284GiB 647GiB 30.50 0.65 76
osd.5
29 hdd 0.90970 1.00000 932GiB 366GiB 566GiB 39.29 0.84 98
osd.29
32 hdd 0.90970 0 0B 0B 0B 0 0 0
osd.32
33 hdd 0.90970 1.00000 932GiB 369GiB 563GiB 39.57 0.85 99
osd.33
2 ssd 0.43700 1.00000 447GiB 271GiB 176GiB 60.67 1.30 50
osd.2
3 ssd 0.43700 1.00000 447GiB 249GiB 198GiB 55.62 1.19 58
osd.3
4 ssd 0.43700 1.00000 447GiB 297GiB 150GiB 66.39 1.42 56
osd.4
30 ssd 0.43660 1.00000 447GiB 236GiB 211GiB 52.85 1.13 48
osd.30
-7 6.29724 - 6.29TiB 2.74TiB 3.55TiB 43.53 0.93 -
host node1002
9 hdd 0.90999 1.00000 932GiB 354GiB 578GiB 37.96 0.81 95
osd.9
10 hdd 0.90999 1.00000 932GiB 357GiB 575GiB 38.28 0.82 96
osd.10
11 hdd 0.90999 1.00000 932GiB 318GiB 613GiB 34.18 0.73 86
osd.11
12 hdd 0.90999 1.00000 932GiB 373GiB 558GiB 40.09 0.86 100
osd.12
35 hdd 0.90970 1.00000 932GiB 343GiB 588GiB 36.83 0.79 92
osd.35
6 ssd 0.43700 1.00000 447GiB 269GiB 178GiB 60.20 1.29 60
osd.6
7 ssd 0.43700 1.00000 447GiB 249GiB 198GiB 55.69 1.19 56
osd.7
8 ssd 0.43700 1.00000 447GiB 286GiB 161GiB 63.95 1.37 56
osd.8
31 ssd 0.43660 1.00000 447GiB 257GiB 190GiB 57.47 1.23 55
osd.31
-28 2.18318 - 2.18TiB 968GiB 1.24TiB 43.29 0.93 -
host node1005
34 ssd 0.43660 1.00000 447GiB 202GiB 245GiB 45.14 0.97 47
osd.34
36 ssd 0.87329 1.00000 894GiB 405GiB 489GiB 45.28 0.97 91
osd.36
37 ssd 0.87329 1.00000 894GiB 361GiB 533GiB 40.38 0.87 79
osd.37
-29 1.74658 - 1.75TiB 888GiB 900GiB 49.65 1.06 -
host node1006
42 ssd 0.87329 1.00000 894GiB 417GiB 477GiB 46.68 1.00 92
osd.42
43 ssd 0.87329 1.00000 894GiB 471GiB 424GiB 52.63 1.13 90
osd.43
-11 13.39537 - 13.4TiB 6.64TiB 6.75TiB 49.60 1.06 -
rack dc01-rack03
-22 5.38794 - 5.39TiB 2.70TiB 2.69TiB 50.14 1.07 -
host node1003
17 hdd 0.90999 1.00000 932GiB 371GiB 560GiB 39.83 0.85 100
osd.17
18 hdd 0.90999 1.00000 932GiB 390GiB 542GiB 41.82 0.90 105
osd.18
24 hdd 0.90999 1.00000 932GiB 352GiB 580GiB 37.77 0.81 94
osd.24
26 hdd 0.90999 1.00000 932GiB 387GiB 545GiB 41.54 0.89 104
osd.26
13 ssd 0.43700 1.00000 447GiB 319GiB 128GiB 71.32 1.53 77
osd.13
14 ssd 0.43700 1.00000 447GiB 303GiB 144GiB 67.76 1.45 70
osd.14
15 ssd 0.43700 1.00000 447GiB 361GiB 86.4GiB 80.67 1.73 77
osd.15
16 ssd 0.43700 1.00000 447GiB 283GiB 164GiB 63.29 1.36 71
osd.16
-25 5.38765 - 5.39TiB 2.83TiB 2.56TiB 52.55 1.13 -
host node1004
23 hdd 0.90999 1.00000 932GiB 382GiB 549GiB 41.05 0.88 102
osd.23
25 hdd 0.90999 1.00000 932GiB 412GiB 520GiB 44.20 0.95 111
osd.25
27 hdd 0.90999 1.00000 932GiB 385GiB 546GiB 41.36 0.89 103
osd.27
28 hdd 0.90970 1.00000 932GiB 462GiB 469GiB 49.64 1.06 124
osd.28
19 ssd 0.43700 1.00000 447GiB 314GiB 133GiB 70.22 1.51 75
osd.19
20 ssd 0.43700 1.00000 447GiB 327GiB 120GiB 73.06 1.57 76
osd.20
21 ssd 0.43700 1.00000 447GiB 324GiB 123GiB 72.45 1.55 77
osd.21
22 ssd 0.43700 1.00000 447GiB 292GiB 156GiB 65.21 1.40 68
osd.22
-30 2.61978 - 2.62TiB 1.11TiB 1.51TiB 42.43 0.91 -
host node1007
38 ssd 0.43660 1.00000 447GiB 165GiB 283GiB 36.82 0.79 46
osd.38
39 ssd 0.43660 1.00000 447GiB 156GiB 292GiB 34.79 0.75 42
osd.39
40 ssd 0.87329 1.00000 894GiB 429GiB 466GiB 47.94 1.03 98
osd.40
41 ssd 0.87329 1.00000 894GiB 389GiB 505GiB 43.55 0.93 103
osd.41
TOTAL 29.9TiB 14.0TiB 16.0TiB 46.65
MIN/MAX VAR: 0.65/1.73 STDDEV: 13.30
=============================================================
root@adminnode:~# ceph df && ceph -v
GLOBAL:
SIZE AVAIL RAW USED %RAW USED
29.9TiB 16.0TiB 14.0TiB 46.65
POOLS:
NAME ID USED %USED MAX AVAIL OBJECTS
rbd_vms_ssd 2 986GiB 49.83 993GiB 262606
rbd_vms_hdd 3 3.76TiB 48.94 3.92TiB 992255
rbd_vms_ssd_01 4 372KiB 0 662GiB 148
rbd_vms_ssd_01_ec 6 2.85TiB 68.81 1.29TiB 770506
ceph version 12.2.8 (ae699615bac534ea496ee965ac6192cb7e0e07c0) luminous (stable)
Looks good. You should always delete your down osd's from crush map
before replace it. After delete this osd try balancer again.
k
_______________________________________________
ceph-users mailing list
ceph-users@xxxxxxxxxxxxxx
http://lists.ceph.com/listinfo.cgi/ceph-users-ceph.com
