Hi,
There is a cluster of ceph. To accelerate dokinuli ssd into it, the card: I ssd gathered in a separate pool and a prescribed rule. But I'm getting the following error msg:
[ceph@ceph-adm test]$ ceph -s
cluster a73f8be5-0fa6-4a82-aa9b-22121246398e
health HEALTH_WARN
64 pgs stuck inactive
64 pgs stuck unclean
monmap e1: 3 mons at {srv-osd01=10.30.15.20:6789/0,srv-osd02=10.30.15.21:6789/0,srv-osd03=10.30.15.22:6789/0 }
election epoch 76, quorum 0,1,2 srv-osd01,srv-osd02,srv-osd03
osdmap e9500: 135 osds: 135 up, 135 in
flags sortbitwise
pgmap v16447007: 4168 pgs, 3 pools, 5829 GB data, 1488 kobjects
17469 GB used, 270 TB / 287 TB avail
4104 active+clean
64 creating
client io 179 kB/s rd, 8781 kB/s wr, 177 op/s
If you change the value of ruleset 0 to ruleset 2 all will work.
Possibly due to the removal of osd.52?
ceph.conf
[global]
fsid = a73f8be5-0fa6-4a82-aa9b-22121246398e
mon_initial_members = srv-osd01, srv-osd02, srv-osd03
mon_host = 10.30.15.20,10.30.15.21,10.30.15.22
auth_cluster_required = cephx
auth_service_required = cephx
auth_client_required = cephx
filestore_xattr_use_omap = true
public network = 10.30.15.0/24
cluster network = 10.40.10.0/24
osd_crush_update_on_start = false
My crushrule map:
[ceph@ceph-adm test]$ cat map.decompile
# begin crush map
tunable choose_local_tries 0
tunable choose_local_fallback_tries 0
tunable choose_total_tries 50
tunable chooseleaf_descend_once 1
tunable straw_calc_version 1
# devices
device 0 osd.0
device 1 osd.1
device 2 osd.2
device 3 osd.3
device 4 osd.4
device 5 osd.5
device 6 osd.6
device 7 osd.7
device 8 osd.8
device 9 osd.9
device 10 osd.10
device 11 osd.11
device 12 osd.12
device 13 osd.13
device 14 osd.14
device 15 osd.15
device 16 osd.16
device 17 osd.17
device 18 osd.18
device 19 osd.19
device 20 osd.20
device 21 osd.21
device 22 osd.22
device 23 osd.23
device 24 osd.24
device 25 osd.25
device 26 osd.26
device 27 osd.27
device 28 osd.28
device 29 osd.29
device 30 osd.30
device 31 osd.31
device 32 osd.32
device 33 osd.33
device 34 osd.34
device 35 osd.35
device 36 osd.36
device 37 osd.37
device 38 osd.38
device 39 osd.39
device 40 osd.40
device 41 osd.41
device 42 osd.42
device 43 osd.43
device 44 osd.44
device 45 osd.45
device 46 osd.46
device 47 osd.47
device 48 osd.48
device 49 osd.49
device 50 osd.50
device 51 osd.51
device 52 device52
device 53 osd.53
device 54 osd.54
device 55 osd.55
device 56 osd.56
device 57 osd.57
device 58 osd.58
device 59 osd.59
device 60 osd.60
device 61 osd.61
device 62 osd.62
device 63 osd.63
device 64 osd.64
device 65 osd.65
device 66 osd.66
device 67 osd.67
device 68 osd.68
device 69 osd.69
device 70 osd.70
device 71 osd.71
device 72 osd.72
device 73 osd.73
device 74 osd.74
device 75 osd.75
device 76 osd.76
device 77 osd.77
device 78 osd.78
device 79 osd.79
device 80 osd.80
device 81 osd.81
device 82 osd.82
device 83 osd.83
device 84 osd.84
device 85 osd.85
device 86 osd.86
device 87 osd.87
device 88 osd.88
device 89 osd.89
device 90 osd.90
device 91 osd.91
device 92 osd.92
device 93 osd.93
device 94 osd.94
device 95 osd.95
device 96 osd.96
device 97 osd.97
device 98 osd.98
device 99 osd.99
device 100 osd.100
device 101 osd.101
device 102 osd.102
device 103 osd.103
device 104 osd.104
device 105 osd.105
device 106 osd.106
device 107 osd.107
device 108 osd.108
device 109 osd.109
device 110 osd.110
device 111 osd.111
device 112 osd.112
device 113 osd.113
device 114 osd.114
device 115 osd.115
device 116 osd.116
device 117 osd.117
device 118 osd.118
device 119 osd.119
device 120 osd.120
device 121 osd.121
device 122 osd.122
device 123 osd.123
device 124 osd.124
device 125 osd.125
device 126 osd.126
device 127 osd.127
device 128 osd.128
device 129 osd.129
device 130 osd.130
device 131 osd.131
device 132 osd.132
device 133 osd.133
device 134 osd.134
device 135 osd.135
# types
type 0 osd
type 1 host
type 2 chassis
type 3 rack
type 4 row
type 5 pdu
type 6 pod
type 7 room
type 8 datacenter
type 9 region
type 10 root
# buckets
host srv-osd01-ssd {
id -2 # do not change unnecessarily
# weight 0.854
alg straw
hash 0 # rjenkins1
item osd.0 weight 0.427
item osd.1 weight 0.427
}
host srv-osd02-ssd {
id -3 # do not change unnecessarily
# weight 0.854
alg straw
hash 0 # rjenkins1
item osd.2 weight 0.427
item osd.3 weight 0.427
}
host srv-osd03-ssd {
id -4 # do not change unnecessarily
# weight 0.854
alg straw
hash 0 # rjenkins1
item osd.4 weight 0.427
item osd.5 weight 0.427
}
host srv-osd04-ssd {
id -5 # do not change unnecessarily
# weight 0.854
alg straw
hash 0 # rjenkins1
item osd.6 weight 0.427
item osd.7 weight 0.427
}
host srv-osd05-ssd {
id -6 # do not change unnecessarily
# weight 0.344
alg straw
hash 0 # rjenkins1
item osd.135 weight 0.172
item osd.124 weight 0.172
}
host srv-osd06-ssd {
id -7 # do not change unnecessarily
# weight 0.344
alg straw
hash 0 # rjenkins1
item osd.48 weight 0.172
item osd.49 weight 0.172
}
host srv-osd07-ssd {
id -8 # do not change unnecessarily
# weight 0.344
alg straw
hash 0 # rjenkins1
item osd.50 weight 0.172
item osd.51 weight 0.172
}
host srv-osd08-ssd {
id -9 # do not change unnecessarily
# weight 0.172
alg straw
hash 0 # rjenkins1
item osd.53 weight 0.172
}
root ssd {
id -1 # do not change unnecessarily
# weight 4.530
alg straw
hash 0 # rjenkins1
item srv-osd01-ssd weight 0.854
item srv-osd02-ssd weight 0.854
item srv-osd03-ssd weight 0.854
item srv-osd04-ssd weight 0.854
item srv-osd05-ssd weight 0.314
item srv-osd06-ssd weight 0.314
item srv-osd07-ssd weight 0.314
item srv-osd08-ssd weight 0.172
}
host srv-osd01-hdd {
id -102 # do not change unnecessarily
# weight 54.470
alg straw
hash 0 # rjenkins1
item osd.8 weight 5.447
item osd.9 weight 5.447
item osd.10 weight 5.447
item osd.11 weight 5.447
item osd.12 weight 5.447
item osd.13 weight 5.447
item osd.14 weight 5.447
item osd.15 weight 5.447
item osd.16 weight 5.447
item osd.17 weight 5.447
}
host srv-osd02-hdd {
id -103 # do not change unnecessarily
# weight 54.470
alg straw
hash 0 # rjenkins1
item osd.18 weight 5.447
item osd.19 weight 5.447
item osd.20 weight 5.447
item osd.21 weight 5.447
item osd.22 weight 5.447
item osd.23 weight 5.447
item osd.24 weight 5.447
item osd.25 weight 5.447
item osd.26 weight 5.447
item osd.46 weight 5.447
}
host srv-osd03-hdd {
id -104 # do not change unnecessarily
# weight 54.470
alg straw
hash 0 # rjenkins1
item osd.27 weight 5.447
item osd.28 weight 5.447
item osd.29 weight 5.447
item osd.30 weight 5.447
item osd.31 weight 5.447
item osd.32 weight 5.447
item osd.33 weight 5.447
item osd.34 weight 5.447
item osd.35 weight 5.447
item osd.36 weight 5.447
}
host srv-osd04-hdd {
id -105 # do not change unnecessarily
# weight 54.470
alg straw
hash 0 # rjenkins1
item osd.37 weight 5.447
item osd.38 weight 5.447
item osd.39 weight 5.447
item osd.40 weight 5.447
item osd.41 weight 5.447
item osd.42 weight 5.447
item osd.43 weight 5.447
item osd.44 weight 5.447
item osd.45 weight 5.447
item osd.47 weight 5.447
}
host srv-osd05-hdd {
id -106 # do not change unnecessarily
# weight 16.160
alg straw
hash 0 # rjenkins1
item osd.114 weight 0.808
item osd.115 weight 0.808
item osd.116 weight 0.808
item osd.117 weight 0.808
item osd.118 weight 0.808
item osd.119 weight 0.808
item osd.120 weight 0.808
item osd.121 weight 0.808
item osd.122 weight 0.808
item osd.123 weight 0.808
item osd.125 weight 0.808
item osd.126 weight 0.808
item osd.127 weight 0.808
item osd.128 weight 0.808
item osd.129 weight 0.808
item osd.130 weight 0.808
item osd.131 weight 0.808
item osd.132 weight 0.808
item osd.133 weight 0.808
item osd.134 weight 0.808
}
host srv-osd06-hdd {
id -107 # do not change unnecessarily
# weight 16.160
alg straw
hash 0 # rjenkins1
item osd.55 weight 0.808
item osd.56 weight 0.808
item osd.57 weight 0.808
item osd.58 weight 0.808
item osd.59 weight 0.808
item osd.60 weight 0.808
item osd.61 weight 0.808
item osd.62 weight 0.808
item osd.63 weight 0.808
item osd.64 weight 0.808
item osd.65 weight 0.808
item osd.66 weight 0.808
item osd.67 weight 0.808
item osd.68 weight 0.808
item osd.69 weight 0.808
item osd.70 weight 0.808
item osd.71 weight 0.808
item osd.72 weight 0.808
item osd.73 weight 0.808
item osd.54 weight 0.808
}
host srv-osd07-hdd {
id -108 # do not change unnecessarily
# weight 16.160
alg straw
hash 0 # rjenkins1
item osd.74 weight 0.808
item osd.75 weight 0.808
item osd.76 weight 0.808
item osd.77 weight 0.808
item osd.78 weight 0.808
item osd.79 weight 0.808
item osd.80 weight 0.808
item osd.81 weight 0.808
item osd.82 weight 0.808
item osd.83 weight 0.808
item osd.84 weight 0.808
item osd.85 weight 0.808
item osd.86 weight 0.808
item osd.87 weight 0.808
item osd.88 weight 0.808
item osd.89 weight 0.808
item osd.90 weight 0.808
item osd.91 weight 0.808
item osd.92 weight 0.808
item osd.93 weight 0.808
}
host srv-osd08-hdd {
id -109 # do not change unnecessarily
# weight 16.160
alg straw
hash 0 # rjenkins1
item osd.94 weight 0.808
item osd.95 weight 0.808
item osd.96 weight 0.808
item osd.97 weight 0.808
item osd.98 weight 0.808
item osd.99 weight 0.808
item osd.100 weight 0.808
item osd.101 weight 0.808
item osd.102 weight 0.808
item osd.103 weight 0.808
item osd.104 weight 0.808
item osd.105 weight 0.808
item osd.106 weight 0.808
item osd.107 weight 0.808
item osd.108 weight 0.808
item osd.109 weight 0.808
item osd.110 weight 0.808
item osd.111 weight 0.808
item osd.112 weight 0.808
item osd.113 weight 0.808
}
root hdd {
id -100 # do not change unnecessarily
# weight 282.632
alg straw
hash 0 # rjenkins1
item srv-osd01-hdd weight 54.466
item srv-osd02-hdd weight 54.466
item srv-osd03-hdd weight 54.466
item srv-osd04-hdd weight 54.466
item srv-osd05-hdd weight 16.192
item srv-osd06-hdd weight 16.192
item srv-osd07-hdd weight 16.192
item srv-osd08-hdd weight 16.192
}
# rules
rule ssd {
ruleset 0
type replicated
min_size 1
max_size 10
step take ssd
step chooseleaf firstn 0 type host
step emit
}
rule hdd {
ruleset 1
type replicated
min_size 1
max_size 10
step take hdd
step chooseleaf firstn 0 type host
step emit
}
# end crush map
[ceph@ceph-adm test]$ ceph osd tree
ID WEIGHT TYPE NAME UP/DOWN REWEIGHT PRIMARY-AFFINITY
-100 282.63190 root hdd
-102 54.46599 host srv-osd01-hdd
8 5.44699 osd.8 up 1.00000 1.00000
9 5.44699 osd.9 up 1.00000 1.00000
10 5.44699 osd.10 up 1.00000 1.00000
11 5.44699 osd.11 up 1.00000 1.00000
12 5.44699 osd.12 up 1.00000 1.00000
13 5.44699 osd.13 up 1.00000 1.00000
14 5.44699 osd.14 up 1.00000 1.00000
15 5.44699 osd.15 up 1.00000 1.00000
16 5.44699 osd.16 up 1.00000 1.00000
17 5.44699 osd.17 up 1.00000 1.00000
-103 54.46599 host srv-osd02-hdd
18 5.44699 osd.18 up 1.00000 1.00000
19 5.44699 osd.19 up 1.00000 1.00000
20 5.44699 osd.20 up 1.00000 1.00000
21 5.44699 osd.21 up 1.00000 1.00000
22 5.44699 osd.22 up 1.00000 1.00000
23 5.44699 osd.23 up 1.00000 1.00000
24 5.44699 osd.24 up 1.00000 1.00000
25 5.44699 osd.25 up 1.00000 1.00000
26 5.44699 osd.26 up 1.00000 1.00000
46 5.44699 osd.46 up 1.00000 1.00000
-104 54.46599 host srv-osd03-hdd
27 5.44699 osd.27 up 1.00000 1.00000
28 5.44699 osd.28 up 1.00000 1.00000
29 5.44699 osd.29 up 1.00000 1.00000
30 5.44699 osd.30 up 1.00000 1.00000
31 5.44699 osd.31 up 1.00000 1.00000
32 5.44699 osd.32 up 1.00000 1.00000
33 5.44699 osd.33 up 1.00000 1.00000
34 5.44699 osd.34 up 1.00000 1.00000
35 5.44699 osd.35 up 1.00000 1.00000
36 5.44699 osd.36 up 1.00000 1.00000
-105 54.46599 host srv-osd04-hdd
37 5.44699 osd.37 up 1.00000 1.00000
38 5.44699 osd.38 up 1.00000 1.00000
39 5.44699 osd.39 up 1.00000 1.00000
40 5.44699 osd.40 up 1.00000 1.00000
41 5.44699 osd.41 up 1.00000 1.00000
42 5.44699 osd.42 up 1.00000 1.00000
43 5.44699 osd.43 up 1.00000 1.00000
44 5.44699 osd.44 up 1.00000 1.00000
45 5.44699 osd.45 up 1.00000 1.00000
47 5.44699 osd.47 up 1.00000 1.00000
-106 16.19199 host srv-osd05-hdd
114 0.80800 osd.114 up 1.00000 1.00000
115 0.80800 osd.115 up 0.81648 1.00000
116 0.80800 osd.116 up 1.00000 1.00000
117 0.80800 osd.117 up 1.00000 1.00000
118 0.80800 osd.118 up 0.70801 1.00000
119 0.80800 osd.119 up 1.00000 1.00000
120 0.80800 osd.120 up 1.00000 1.00000
121 0.80800 osd.121 up 1.00000 1.00000
122 0.80800 osd.122 up 1.00000 1.00000
123 0.80800 osd.123 up 1.00000 1.00000
125 0.80800 osd.125 up 1.00000 1.00000
126 0.80800 osd.126 up 1.00000 1.00000
127 0.80800 osd.127 up 1.00000 1.00000
128 0.80800 osd.128 up 1.00000 1.00000
129 0.80800 osd.129 up 1.00000 1.00000
130 0.80800 osd.130 up 0.79892 1.00000
131 0.80800 osd.131 up 1.00000 1.00000
132 0.80800 osd.132 up 1.00000 1.00000
133 0.80800 osd.133 up 1.00000 1.00000
134 0.80800 osd.134 up 1.00000 1.00000
-107 16.19199 host srv-osd06-hdd
55 0.80800 osd.55 up 1.00000 1.00000
56 0.80800 osd.56 up 0.82719 1.00000
57 0.80800 osd.57 up 1.00000 1.00000
58 0.80800 osd.58 up 1.00000 1.00000
59 0.80800 osd.59 up 1.00000 1.00000
60 0.80800 osd.60 up 1.00000 1.00000
61 0.80800 osd.61 up 1.00000 1.00000
62 0.80800 osd.62 up 1.00000 1.00000
63 0.80800 osd.63 up 0.75986 1.00000
64 0.80800 osd.64 up 1.00000 1.00000
65 0.80800 osd.65 up 1.00000 1.00000
66 0.80800 osd.66 up 1.00000 1.00000
67 0.80800 osd.67 up 1.00000 1.00000
68 0.80800 osd.68 up 1.00000 1.00000
69 0.80800 osd.69 up 1.00000 1.00000
70 0.80800 osd.70 up 0.81743 1.00000
71 0.80800 osd.71 up 1.00000 1.00000
72 0.80800 osd.72 up 1.00000 1.00000
73 0.80800 osd.73 up 1.00000 1.00000
54 0.80800 osd.54 up 1.00000 1.00000
-108 16.19199 host srv-osd07-hdd
74 0.80800 osd.74 up 0.73679 1.00000
75 0.80800 osd.75 up 1.00000 1.00000
76 0.80800 osd.76 up 1.00000 1.00000
77 0.80800 osd.77 up 1.00000 1.00000
78 0.80800 osd.78 up 1.00000 1.00000
79 0.80800 osd.79 up 0.86931 1.00000
80 0.80800 osd.80 up 1.00000 1.00000
81 0.80800 osd.81 up 1.00000 1.00000
82 0.80800 osd.82 up 1.00000 1.00000
83 0.80800 osd.83 up 0.82845 1.00000
84 0.80800 osd.84 up 1.00000 1.00000
85 0.80800 osd.85 up 1.00000 1.00000
86 0.80800 osd.86 up 1.00000 1.00000
87 0.80800 osd.87 up 0.85548 1.00000
88 0.80800 osd.88 up 1.00000 1.00000
89 0.80800 osd.89 up 1.00000 1.00000
90 0.80800 osd.90 up 1.00000 1.00000
91 0.80800 osd.91 up 0.83315 1.00000
92 0.80800 osd.92 up 1.00000 1.00000
93 0.80800 osd.93 up 1.00000 1.00000
-109 16.19199 host srv-osd08-hdd
94 0.80800 osd.94 up 0.83136 1.00000
95 0.80800 osd.95 up 1.00000 1.00000
96 0.80800 osd.96 up 1.00000 1.00000
97 0.80800 osd.97 up 1.00000 1.00000
98 0.80800 osd.98 up 1.00000 1.00000
99 0.80800 osd.99 up 1.00000 1.00000
100 0.80800 osd.100 up 1.00000 1.00000
101 0.80800 osd.101 up 1.00000 1.00000
102 0.80800 osd.102 up 1.00000 1.00000
103 0.80800 osd.103 up 1.00000 1.00000
104 0.80800 osd.104 up 1.00000 1.00000
105 0.80800 osd.105 up 1.00000 1.00000
106 0.80800 osd.106 up 1.00000 1.00000
107 0.80800 osd.107 up 1.00000 1.00000
108 0.80800 osd.108 up 1.00000 1.00000
109 0.80800 osd.109 up 0.85403 1.00000
110 0.80800 osd.110 up 0.68864 1.00000
111 0.80800 osd.111 up 0.82031 1.00000
112 0.80800 osd.112 up 1.00000 1.00000
113 0.80800 osd.113 up 1.00000 1.00000
-1 4.52994 root ssd
-2 0.85399 host srv-osd01-ssd
0 0.42699 osd.0 up 0.00015 1.00000
1 0.42699 osd.1 up 0.00012 1.00000
-3 0.85399 host srv-osd02-ssd
2 0.42699 osd.2 up 0.00014 1.00000
3 0.42699 osd.3 up 0.00018 1.00000
-4 0.85399 host srv-osd03-ssd
4 0.42699 osd.4 up 0.00012 1.00000
5 0.42699 osd.5 up 0.00015 1.00000
-5 0.85399 host srv-osd04-ssd
6 0.42699 osd.6 up 0.00018 1.00000
7 0.42699 osd.7 up 0.00026 1.00000
-6 0.31400 host srv-osd05-ssd
135 0.17200 osd.135 up 0.00006 1.00000
124 0.17200 osd.124 up 0.00021 1.00000
-7 0.31400 host srv-osd06-ssd
48 0.17200 osd.48 up 0.00006 1.00000
49 0.17200 osd.49 up 0.00014 1.00000
-8 0.31400 host srv-osd07-ssd
50 0.17200 osd.50 up 0.00014 1.00000
51 0.17200 osd.51 up 0.00018 1.00000
-9 0.17200 host srv-osd08-ssd
53 0.17200 osd.53 up 0.00005 1.00000
_______________________________________________ ceph-users mailing list ceph-users@xxxxxxxxxxxxxx http://lists.ceph.com/listinfo.cgi/ceph-users-ceph.com
