On Sat, Oct 27, 2018, 9:14 PM Alex Litvak <alexander.v.litvak@xxxxxxxxx> wrote:
I have a cluster using 2 roots. I attempted to reweigh osds under the
"default" root used by pool rbd, cephfs-data, cephfs-meta using Cern
script: crush-reweight-by-utilization.py. I ran it first and it showed
4 candidates (per script default ), it shows final weight and single
step movements.
./crush-reweight-by-utilization.py --pool=rbd
osd.36 (1.273109 >= 0.675607) [1.000000 -> 0.990000]
osd.0 (1.243042 >= 0.675607) [1.000000 -> 0.990000]
osd.2 (1.231539 >= 0.675607) [1.000000 -> 0.990000]
osd.19 (1.228613 >= 0.675607) [1.000000 -> 0.990000]
Script advises on all osds in the pool (36 of them if mentioned, see
below). Is it safe to take osd.36 as only one osd and reweigh it first?
I attempted to do it and each step caused some more pgs stuck in
active+unmapped mode. I didn't proceed to the end at the moment, but if
I do continue with osd.36 should pgs distribute correctly or my
assumption is wrong? Should I use some other approach, i.e. reweighing
all osds in the pool or recalculating the weights completely?
This is my first attempt to re-balance cluster properly so any clues are
appreciated.
Below are various diagnostics in anticipation of questions.
Thank you in advance
./crush-reweight-by-utilization.py --pool=rbd --num-osds=36
osd.36 (1.273079 >= 0.675594) [1.000000 -> 0.990000]
osd.0 (1.243019 >= 0.675594) [1.000000 -> 0.990000]
osd.2 (1.231513 >= 0.675594) [1.000000 -> 0.990000]
osd.19 (1.228569 >= 0.675594) [1.000000 -> 0.990000]
osd.16 (1.228071 >= 0.675594) [1.000000 -> 0.990000]
osd.46 (1.220588 >= 0.675594) [1.000000 -> 0.990000]
osd.23 (1.215887 >= 0.675594) [1.000000 -> 0.990000]
osd.7 (1.204189 >= 0.675594) [1.000000 -> 0.990000]
osd.10 (1.202385 >= 0.675594) [1.000000 -> 0.990000]
osd.40 (1.186002 >= 0.675594) [1.000000 -> 0.990000]
osd.43 (1.180218 >= 0.675594) [1.000000 -> 0.990000]
osd.21 (1.180050 >= 0.675594) [1.000000 -> 0.990000]
osd.15 (1.162953 >= 0.675594) [1.000000 -> 0.990000]
osd.1 (1.155985 >= 0.675594) [1.000000 -> 0.990000]
osd.44 (1.151496 >= 0.675594) [1.000000 -> 0.990000]
osd.39 (1.149947 >= 0.675594) [1.000000 -> 0.990000]
osd.22 (1.148013 >= 0.675594) [1.000000 -> 0.990000]
osd.8 (1.143455 >= 0.675594) [1.000000 -> 0.990000]
osd.37 (1.130054 >= 0.675594) [1.000000 -> 0.990000]
osd.18 (1.126777 >= 0.675594) [1.000000 -> 0.990000]
osd.17 (1.125752 >= 0.675594) [1.000000 -> 0.990000]
osd.9 (1.124679 >= 0.675594) [1.000000 -> 0.990000]
osd.42 (1.110069 >= 0.675594) [1.000000 -> 0.990000]
osd.4 (1.108986 >= 0.675594) [1.000000 -> 0.990000]
osd.45 (1.102144 >= 0.675594) [1.000000 -> 0.990000]
osd.12 (1.085402 >= 0.675594) [1.000000 -> 0.990000]
osd.38 (1.083698 >= 0.675594) [1.000000 -> 0.990000]
osd.5 (1.076138 >= 0.675594) [1.000000 -> 0.990000]
osd.11 (1.075955 >= 0.675594) [1.000000 -> 0.990000]
osd.13 (1.070176 >= 0.675594) [1.000000 -> 0.990000]
osd.20 (1.063759 >= 0.675594) [1.000000 -> 0.990000]
osd.14 (1.052357 >= 0.675594) [1.000000 -> 0.990000]
osd.41 (1.035255 >= 0.675594) [1.000000 -> 0.990000]
osd.3 (1.013664 >= 0.675594) [1.000000 -> 0.990000]
osd.47 (1.011428 >= 0.675594) [1.000000 -> 0.990000]
osd.6 (1.000170 >= 0.675594) [1.000000 -> 0.990000]
# ceph osd df tree
ID WEIGHT REWEIGHT SIZE USE AVAIL %USE VAR TYPE NAME
-10 18.00000 - 20100G 7127G 12973G 35.46 0.63 root 12g
-9 18.00000 - 20100G 7127G 12973G 35.46 0.63 datacenter
la-12g
-5 6.00000 - 6700G 2375G 4324G 35.45 0.63 host
oss4-la-12g
24 1.00000 1.00000 1116G 409G 706G 36.71 0.65
osd.24
26 1.00000 1.00000 1116G 373G 743G 33.43 0.59
osd.26
28 1.00000 1.00000 1116G 414G 702G 37.10 0.66
osd.28
30 1.00000 1.00000 1116G 453G 663G 40.60 0.72
osd.30
32 1.00000 1.00000 1116G 342G 774G 30.65 0.54
osd.32
34 1.00000 1.00000 1116G 382G 734G 34.23 0.61
osd.34
-6 6.00000 - 6700G 2375G 4324G 35.45 0.63 host
oss5-la-12g
25 1.00000 1.00000 1116G 383G 733G 34.32 0.61
osd.25
27 1.00000 1.00000 1116G 388G 728G 34.75 0.62
osd.27
29 1.00000 1.00000 1116G 381G 734G 34.19 0.61
osd.29
31 1.00000 1.00000 1116G 424G 692G 38.00 0.67
osd.31
33 1.00000 1.00000 1116G 418G 698G 37.46 0.67
osd.33
35 1.00000 1.00000 1116G 379G 736G 34.02 0.60
osd.35
-7 6.00000 - 6700G 2376G 4323G 35.47 0.63 host
oss6-la-12g
48 1.00000 1.00000 1116G 410G 705G 36.79 0.65
osd.48
49 1.00000 1.00000 1116G 415G 701G 37.22 0.66
osd.49
50 1.00000 1.00000 1116G 354G 761G 31.78 0.56
osd.50
51 1.00000 1.00000 1116G 414G 701G 37.16 0.66
osd.51
52 1.00000 1.00000 1116G 416G 700G 37.27 0.66
osd.52
53 1.00000 1.00000 1116G 363G 752G 32.57 0.58
osd.53
-1 36.00000 - 66952G 41883G 25069G 62.56 1.11 root default
-4 36.00000 - 66952G 41883G 25069G 62.56 1.11 datacenter la
-2 12.00000 - 22334G 13961G 8373G 62.51 1.11 host
oss1-la
0 1.00000 1.00000 1861G 1272G 588G 68.37 1.21 osd.0
2 1.00000 1.00000 1861G 1261G 600G 67.74 1.20 osd.2
4 1.00000 1.00000 1861G 1135G 726G 61.00 1.08 osd.4
6 1.00000 1.00000 1861G 1024G 837G 55.02 0.98 osd.6
8 1.00000 1.00000 1861G 1170G 690G 62.90 1.12 osd.8
10 1.00000 1.00000 1861G 1231G 630G 66.14 1.17
osd.10
14 1.00000 1.00000 1861G 1077G 784G 57.89 1.03
osd.14
16 1.00000 1.00000 1861G 1257G 604G 67.55 1.20
osd.16
18 1.00000 1.00000 1861G 1153G 707G 61.98 1.10
osd.18
20 1.00000 1.00000 1861G 1089G 772G 58.51 1.04
osd.20
22 1.00000 1.00000 1861G 1175G 686G 63.15 1.12
osd.22
12 1.00000 1.00000 1856G 1111G 745G 59.86 1.06
osd.12
-3 12.00000 - 22339G 13961G 8378G 62.50 1.11 host
oss2-la
1 1.00000 1.00000 1861G 1183G 677G 63.59 1.13 osd.1
3 1.00000 1.00000 1861G 1037G 823G 55.76 0.99 osd.3
5 1.00000 1.00000 1861G 1101G 759G 59.19 1.05 osd.5
7 1.00000 1.00000 1861G 1233G 628G 66.24 1.18 osd.7
9 1.00000 1.00000 1861G 1151G 709G 61.86 1.10 osd.9
11 1.00000 1.00000 1861G 1101G 759G 59.18 1.05
osd.11
13 1.00000 1.00000 1861G 1095G 765G 58.87 1.05
osd.13
15 1.00000 1.00000 1861G 1190G 670G 63.97 1.14
osd.15
17 1.00000 1.00000 1861G 1152G 708G 61.92 1.10
osd.17
19 1.00000 1.00000 1861G 1258G 603G 67.58 1.20
osd.19
21 1.00000 1.00000 1861G 1208G 653G 64.91 1.15
osd.21
23 1.00000 1.00000 1861G 1245G 616G 66.88 1.19
osd.23
-13 12.00000 - 22279G 13961G 8318G 62.66 1.11 host
oss3-la
36 1.00000 1.00000 1856G 1303G 552G 70.22 1.25
osd.36
37 1.00000 1.00000 1856G 1157G 699G 62.33 1.11
osd.37
38 1.00000 1.00000 1856G 1109G 746G 59.77 1.06
osd.38
39 1.00000 1.00000 1856G 1177G 679G 63.42 1.13
osd.39
40 1.00000 1.00000 1856G 1214G 642G 65.41 1.16
osd.40
41 1.00000 1.00000 1856G 1060G 796G 57.10 1.01
osd.41
42 1.00000 1.00000 1856G 1136G 719G 61.22 1.09
osd.42
43 1.00000 1.00000 1856G 1208G 648G 65.09 1.16
osd.43
44 1.00000 1.00000 1856G 1179G 677G 63.51 1.13
osd.44
45 1.00000 1.00000 1856G 1128G 727G 60.79 1.08
osd.45
46 1.00000 1.00000 1856G 1249G 606G 67.32 1.20
osd.46
47 1.00000 1.00000 1856G 1035G 820G 55.78 0.99 osd.47
ceph -s
cluster 96e9619a-4828-4700-989a-fcf152286758
health HEALTH_OK
monmap e2: 3 mons at
{mds1mgs1-la=10.0.40.43:6789/0,mds2mgs2-la=10.0.40.44:6789/0,mds3mgs3-la=10.0.40.45:6789/0}
election epoch 714, quorum 0,1,2
mds1mgs1-la,mds2mgs2-la,mds3mgs3-la
mdsmap e234: 1/1/1 up {0=mds1mgs1-la=up:active}, 2 up:standby
osdmap e20335: 54 osds: 54 up, 54 in
pgmap v105042048: 5120 pgs, 4 pools, 16381 GB data, 7249 kobjects
49010 GB used, 38042 GB / 87053 GB avail
5119 active+clean
1 active+clean+scrubbing+deep
client io 125 kB/s rd, 1411 kB/s wr, 119 op/s
# ceph osd crush dump
{
"devices": [
{
"id": 0,
"name": "osd.0"
},
{
"id": 1,
"name": "osd.1"
},
{
"id": 2,
"name": "osd.2"
},
{
"id": 3,
"name": "osd.3"
},
{
"id": 4,
"name": "osd.4"
},
{
"id": 5,
"name": "osd.5"
},
{
"id": 6,
"name": "osd.6"
},
{
"id": 7,
"name": "osd.7"
},
{
"id": 8,
"name": "osd.8"
},
{
"id": 9,
"name": "osd.9"
},
{
"id": 10,
"name": "osd.10"
},
{
"id": 11,
"name": "osd.11"
},
{
"id": 12,
"name": "osd.12"
},
{
"id": 13,
"name": "osd.13"
},
{
"id": 14,
"name": "osd.14"
},
{
"id": 15,
"name": "osd.15"
},
{
"id": 16,
"name": "osd.16"
},
{
"id": 17,
"name": "osd.17"
},
{
"id": 18,
"name": "osd.18"
},
{
"id": 19,
"name": "osd.19"
},
{
"id": 20,
"name": "osd.20"
},
{
"id": 21,
"name": "osd.21"
},
{
"id": 22,
"name": "osd.22"
},
{
"id": 23,
"name": "osd.23"
},
{
"id": 24,
"name": "osd.24"
},
{
"id": 25,
"name": "osd.25"
},
{
"id": 26,
"name": "osd.26"
},
{
"id": 27,
"name": "osd.27"
},
{
"id": 28,
"name": "osd.28"
},
{
"id": 29,
"name": "osd.29"
},
{
"id": 30,
"name": "osd.30"
},
{
"id": 31,
"name": "osd.31"
},
{
"id": 32,
"name": "osd.32"
},
{
"id": 33,
"name": "osd.33"
},
{
"id": 34,
"name": "osd.34"
},
{
"id": 35,
"name": "osd.35"
},
{
"id": 36,
"name": "osd.36"
},
{
"id": 37,
"name": "osd.37"
},
{
"id": 38,
"name": "osd.38"
},
{
"id": 39,
"name": "osd.39"
},
{
"id": 40,
"name": "osd.40"
},
{
"id": 41,
"name": "osd.41"
},
{
"id": 42,
"name": "osd.42"
},
{
"id": 43,
"name": "osd.43"
},
{
"id": 44,
"name": "osd.44"
},
{
"id": 45,
"name": "osd.45"
},
{
"id": 46,
"name": "osd.46"
},
{
"id": 47,
"name": "osd.47"
},
{
"id": 48,
"name": "osd.48"
},
{
"id": 49,
"name": "osd.49"
},
{
"id": 50,
"name": "osd.50"
},
{
"id": 51,
"name": "osd.51"
},
{
"id": 52,
"name": "osd.52"
},
{
"id": 53,
"name": "osd.53"
}
],
"types": [
{
"type_id": 0,
"name": "osd"
},
{
"type_id": 1,
"name": "host"
},
{
"type_id": 2,
"name": "chassis"
},
{
"type_id": 3,
"name": "rack"
},
{
"type_id": 4,
"name": "row"
},
{
"type_id": 5,
"name": "pdu"
},
{
"type_id": 6,
"name": "pod"
},
{
"type_id": 7,
"name": "room"
},
{
"type_id": 8,
"name": "datacenter"
},
{
"type_id": 9,
"name": "region"
},
{
"type_id": 10,
"name": "root"
}
],
"buckets": [
{
"id": -1,
"name": "default",
"type_id": 10,
"type_name": "root",
"weight": 2359296,
"alg": "straw",
"hash": "rjenkins1",
"items": [
{
"id": -4,
"weight": 2359296,
"pos": 0
}
]
},
{
"id": -2,
"name": "oss1-la",
"type_id": 1,
"type_name": "host",
"weight": 786432,
"alg": "straw",
"hash": "rjenkins1",
"items": [
{
"id": 0,
"weight": 65536,
"pos": 0
},
{
"id": 2,
"weight": 65536,
"pos": 1
},
{
"id": 4,
"weight": 65536,
"pos": 2
},
{
"id": 6,
"weight": 65536,
"pos": 3
},
{
"id": 8,
"weight": 65536,
"pos": 4
},
{
"id": 10,
"weight": 65536,
"pos": 5
},
{
"id": 14,
"weight": 65536,
"pos": 6
},
{
"id": 16,
"weight": 65536,
"pos": 7
},
{
"id": 18,
"weight": 65536,
"pos": 8
},
{
"id": 20,
"weight": 65536,
"pos": 9
},
{
"id": 22,
"weight": 65536,
"pos": 10
},
{
"id": 12,
"weight": 65536,
"pos": 11
}
]
},
{
"id": -3,
"name": "oss2-la",
"type_id": 1,
"type_name": "host",
"weight": 786432,
"alg": "straw",
"hash": "rjenkins1",
"items": [
{
"id": 1,
"weight": 65536,
"pos": 0
},
{
"id": 3,
"weight": 65536,
"pos": 1
},
{
"id": 5,
"weight": 65536,
"pos": 2
},
{
"id": 7,
"weight": 65536,
"pos": 3
},
{
"id": 9,
"weight": 65536,
"pos": 4
},
{
"id": 11,
"weight": 65536,
"pos": 5
},
{
"id": 13,
"weight": 65536,
"pos": 6
},
{
"id": 15,
"weight": 65536,
"pos": 7
},
{
"id": 17,
"weight": 65536,
"pos": 8
},
{
"id": 19,
"weight": 65536,
"pos": 9
},
{
"id": 21,
"weight": 65536,
"pos": 10
},
{
"id": 23,
"weight": 65536,
"pos": 11
}
]
},
{
"id": -4,
"name": "la",
"type_id": 8,
"type_name": "datacenter",
"weight": 2359296,
"alg": "straw",
"hash": "rjenkins1",
"items": [
{
"id": -2,
"weight": 786432,
"pos": 0
},
{
"id": -3,
"weight": 786432,
"pos": 1
},
{
"id": -13,
"weight": 786432,
"pos": 2
}
]
},
{
"id": -5,
"name": "oss4-la-12g",
"type_id": 1,
"type_name": "host",
"weight": 393216,
"alg": "straw2",
"hash": "rjenkins1",
"items": [
{
"id": 24,
"weight": 65536,
"pos": 0
},
{
"id": 26,
"weight": 65536,
"pos": 1
},
{
"id": 28,
"weight": 65536,
"pos": 2
},
{
"id": 30,
"weight": 65536,
"pos": 3
},
{
"id": 32,
"weight": 65536,
"pos": 4
},
{
"id": 34,
"weight": 65536,
"pos": 5
}
]
},
{
"id": -6,
"name": "oss5-la-12g",
"type_id": 1,
"type_name": "host",
"weight": 393216,
"alg": "straw2",
"hash": "rjenkins1",
"items": [
{
"id": 25,
"weight": 65536,
"pos": 0
},
{
"id": 27,
"weight": 65536,
"pos": 1
},
{
"id": 29,
"weight": 65536,
"pos": 2
},
{
"id": 31,
"weight": 65536,
"pos": 3
},
{
"id": 33,
"weight": 65536,
"pos": 4
},
{
"id": 35,
"weight": 65536,
"pos": 5
}
]
},
{
"id": -7,
"name": "oss6-la-12g",
"type_id": 1,
"type_name": "host",
"weight": 393216,
"alg": "straw2",
"hash": "rjenkins1",
"items": [
{
"id": 48,
"weight": 65536,
"pos": 0
},
{
"id": 49,
"weight": 65536,
"pos": 1
},
{
"id": 50,
"weight": 65536,
"pos": 2
},
{
"id": 51,
"weight": 65536,
"pos": 3
},
{
"id": 52,
"weight": 65536,
"pos": 4
},
{
"id": 53,
"weight": 65536,
"pos": 5
}
]
},
{
"id": -9,
"name": "la-12g",
"type_id": 8,
"type_name": "datacenter",
"weight": 1179648,
"alg": "straw2",
"hash": "rjenkins1",
"items": [
{
"id": -5,
"weight": 393216,
"pos": 0
},
{
"id": -6,
"weight": 393216,
"pos": 1
},
{
"id": -7,
"weight": 393216,
"pos": 2
}
]
},
{
"id": -10,
"name": "12g",
"type_id": 10,
"type_name": "root",
"weight": 1179648,
"alg": "straw2",
"hash": "rjenkins1",
"items": [
{
"id": -9,
"weight": 1179648,
"pos": 0
}
]
},
{
"id": -13,
"name": "oss3-la",
"type_id": 1,
"type_name": "host",
"weight": 786432,
"alg": "straw",
"hash": "rjenkins1",
"items": [
{
"id": 36,
"weight": 65536,
"pos": 0
},
{
"id": 37,
"weight": 65536,
"pos": 1
},
{
"id": 38,
"weight": 65536,
"pos": 2
},
{
"id": 39,
"weight": 65536,
"pos": 3
},
{
"id": 40,
"weight": 65536,
"pos": 4
},
{
"id": 41,
"weight": 65536,
"pos": 5
},
{
"id": 42,
"weight": 65536,
"pos": 6
},
{
"id": 43,
"weight": 65536,
"pos": 7
},
{
"id": 44,
"weight": 65536,
"pos": 8
},
{
"id": 45,
"weight": 65536,
"pos": 9
},
{
"id": 46,
"weight": 65536,
"pos": 10
},
{
"id": 47,
"weight": 65536,
"pos": 11
}
]
}
],
"rules": [
{
"rule_id": 0,
"rule_name": "replicated_ruleset",
"ruleset": 0,
"type": 1,
"min_size": 1,
"max_size": 10,
"steps": [
{
"op": "take",
"item": -1,
"item_name": "default"
},
{
"op": "chooseleaf_firstn",
"num": 0,
"type": "host"
},
{
"op": "emit"
}
]
},
{
"rule_id": 1,
"rule_name": "2nodes_3reps",
"ruleset": 1,
"type": 1,
"min_size": 3,
"max_size": 4,
"steps": [
{
"op": "take",
"item": -1,
"item_name": "default"
},
{
"op": "choose_firstn",
"num": 0,
"type": "host"
},
{
"op": "choose_firstn",
"num": 2,
"type": "osd"
},
{
"op": "emit"
}
]
},
{
"rule_id": 2,
"rule_name": "replicated_12g",
"ruleset": 2,
"type": 1,
"min_size": 1,
"max_size": 10,
"steps": [
{
"op": "take",
"item": -10,
"item_name": "12g"
},
{
"op": "chooseleaf_firstn",
"num": 0,
"type": "host"
},
{
"op": "emit"
}
]
}
],
"tunables": {
"choose_local_tries": 0,
"choose_local_fallback_tries": 0,
"choose_total_tries": 50,
"chooseleaf_descend_once": 1,
"chooseleaf_vary_r": 0,
"straw_calc_version": 0,
"allowed_bucket_algs": 22,
"profile": "bobtail",
"optimal_tunables": 0,
"legacy_tunables": 0,
"require_feature_tunables": 1,
"require_feature_tunables2": 1,
"require_feature_tunables3": 0,
"has_v2_rules": 0,
"has_v3_rules": 0,
"has_v4_buckets": 1
}
}
_______________________________________________
ceph-users mailing list
ceph-users@xxxxxxxxxxxxxx
http://lists.ceph.com/listinfo.cgi/ceph-users-ceph.com
_______________________________________________ ceph-users mailing list ceph-users@xxxxxxxxxxxxxx http://lists.ceph.com/listinfo.cgi/ceph-users-ceph.com
