Yes, I changed original crushmap, need to take nodes gstore1, gstore2,
cstore5 for new cluster. I have only crushmap from the failed cluster
downloaded immediately after cluster was crushed. It on attachment.
2013/7/17 Gregory Farnum <greg@xxxxxxxxxxx>:
> Have you changed either of these maps since you originally switched to
> use rule 3?
>
> Can you compare them to what you have on your test cluster? In
> particular I see that you have 0 weight for all the buckets in the
> crush pool, which I expect to misbehave but not to cause the OSD to
> crash everywhere.
> Software Engineer #42 @ http://inktank.com | http://ceph.com
>
>
> On Tue, Jul 16, 2013 at 4:00 PM, Vladislav Gorbunov <vadikgo@xxxxxxxxx> wrote:
>> output is in the attached files
>>
>> 2013/7/17 Gregory Farnum <greg@xxxxxxxxxxx>:
>>> The maps in the OSDs only would have gotten there from the monitors.
>>> If a bad map somehow got distributed to the OSDs then cleaning it up
>>> is unfortunately going to take a lot of work without any well-defined
>>> processes.
>>> So if you could just do "ceph osd crush dump" and "ceph osd dump" and
>>> provide the output from those commands, we can look at what the map
>>> actually has and go from there.
>>> -Greg
>>> Software Engineer #42 @ http://inktank.com | http://ceph.com
>>>
>>>
>>> On Tue, Jul 16, 2013 at 3:22 PM, Vladislav Gorbunov <vadikgo@xxxxxxxxx> wrote:
>>>> Gregory, thank for you help!
>>>> After all osd servers downed, i'am back rule set for the iscsi pool
>>>> back to default rule 0:
>>>> ceph osd pool set iscsi crush_ruleset 0
>>>> it does not help, all osd not started, except without data, with weight 0.
>>>> next i remove ruleset iscsi from crush map. It does not help too. And
>>>> after that i post crushmap to this mail list.
>>>> Is any method to extract crush map from downed osd server and inject
>>>> it to the mon server? from /var/lib/ceph/osd/ceph-2/current/omap
>>>> folder?
>>>>
>>>> 2013/7/17 Gregory Farnum <greg@xxxxxxxxxxx>:
>>>>> I notice that your first dump of the crush map didn't include rule #3.
>>>>> Are you sure you've injected it into the cluster? Try extracting it
>>>>> from the monitors and looking at that map directly, instead of a
>>>>> locally cached version.
>>>>> You mentioned some problem with OSDs being positioned wrong too, so
>>>>> you might look at "ceph osd tree" and look at the shape of the map.
>>>>> But it sounds to me like maybe there's a disconnect between what
>>>>> you've put into the cluster, and what you're looking at.
>>>>> -Greg
>>>>> Software Engineer #42 @ http://inktank.com | http://ceph.com
# begin crush map
# devices
device 0 osd.0
device 1 osd.1
device 2 osd.2
device 3 osd.3
device 4 osd.4
device 5 osd.5
device 6 osd.6
device 7 osd.7
device 8 osd.8
device 9 osd.9
device 10 osd.10
device 11 osd.11
device 12 osd.12
device 13 osd.13
device 14 osd.14
device 15 osd.15
device 16 osd.16
device 17 osd.17
device 18 osd.18
device 19 osd.19
device 20 osd.20
device 21 osd.21
device 22 osd.22
device 23 osd.23
device 24 osd.24
device 25 osd.25
device 26 osd.26
device 27 osd.27
device 28 osd.28
device 29 osd.29
device 30 osd.30
device 31 osd.31
device 32 osd.32
device 33 osd.33
device 34 osd.34
device 35 osd.35
device 36 osd.36
device 37 osd.37
device 38 osd.38
device 39 osd.39
device 40 osd.40
device 41 osd.41
device 42 osd.42
device 43 osd.43
device 44 osd.44
device 45 osd.45
device 46 osd.46
device 47 osd.47
device 48 osd.48
device 49 osd.49
device 50 osd.50
device 51 osd.51
device 52 osd.52
device 53 osd.53
device 54 osd.54
device 55 osd.55
device 56 osd.56
device 57 osd.57
device 58 osd.58
device 59 osd.59
device 60 osd.60
device 61 osd.61
device 62 osd.62
device 63 osd.63
device 64 osd.64
device 65 osd.65
device 66 osd.66
device 67 osd.67
device 68 osd.68
device 69 osd.69
device 70 osd.70
device 71 osd.71
device 72 osd.72
device 73 osd.73
device 74 osd.74
device 75 osd.75
device 76 osd.76
device 77 osd.77
device 78 osd.78
device 79 osd.79
device 80 osd.80
device 81 osd.81
device 82 osd.82
device 83 osd.83
device 84 osd.84
device 85 osd.85
device 86 osd.86
device 87 osd.87
device 88 osd.88
device 89 osd.89
device 90 osd.90
device 91 osd.91
device 92 osd.92
device 93 osd.93
device 94 osd.94
device 95 osd.95
device 96 osd.96
device 97 osd.97
device 98 osd.98
device 99 osd.99
device 100 osd.100
device 101 osd.101
device 102 osd.102
device 103 osd.103
device 104 osd.104
device 105 osd.105
# types
type 0 osd
type 1 host
type 2 rack
type 3 row
type 4 room
type 5 datacenter
type 6 pool
# buckets
host gstore5 {
id -9 # do not change unnecessarily
# weight 5.000
alg straw
hash 0 # rjenkins1
item osd.2 weight 5.000
}
host cstore1 {
id -10 # do not change unnecessarily
# weight 16.000
alg straw
hash 0 # rjenkins1
item osd.5 weight 1.000
item osd.6 weight 1.000
item osd.7 weight 1.000
item osd.8 weight 1.000
item osd.9 weight 1.000
item osd.10 weight 1.000
item osd.11 weight 1.000
item osd.12 weight 1.000
item osd.13 weight 1.000
item osd.14 weight 1.000
item osd.4 weight 1.000
item osd.47 weight 1.000
item osd.48 weight 1.000
item osd.49 weight 1.000
item osd.50 weight 1.000
item osd.51 weight 1.000
}
host cstore2 {
id -11 # do not change unnecessarily
# weight 20.000
alg straw
hash 0 # rjenkins1
item osd.15 weight 1.000
item osd.16 weight 1.000
item osd.17 weight 1.000
item osd.18 weight 1.000
item osd.19 weight 1.000
item osd.20 weight 1.000
item osd.21 weight 1.000
item osd.22 weight 1.000
item osd.23 weight 1.000
item osd.24 weight 1.000
item osd.41 weight 1.000
item osd.42 weight 1.000
item osd.43 weight 1.000
item osd.44 weight 1.000
item osd.45 weight 1.000
item osd.46 weight 1.000
item osd.52 weight 1.000
item osd.53 weight 1.000
item osd.54 weight 1.000
item osd.55 weight 1.000
item osd.56 weight 0.000
item osd.57 weight 0.000
}
host cstore3 {
id -12 # do not change unnecessarily
# weight 16.000
alg straw
hash 0 # rjenkins1
item osd.25 weight 1.000
item osd.26 weight 1.000
item osd.27 weight 1.000
item osd.28 weight 1.000
item osd.29 weight 1.000
item osd.30 weight 1.000
item osd.31 weight 1.000
item osd.32 weight 1.000
item osd.33 weight 1.000
item osd.34 weight 1.000
item osd.35 weight 1.000
item osd.36 weight 1.000
item osd.37 weight 1.000
item osd.38 weight 1.000
item osd.39 weight 1.000
item osd.40 weight 1.000
}
host cstore4 {
id -13 # do not change unnecessarily
# weight 7.640
alg straw
hash 0 # rjenkins1
item osd.62 weight 0.550
item osd.63 weight 0.550
item osd.64 weight 0.550
item osd.65 weight 0.550
item osd.66 weight 0.550
item osd.67 weight 0.550
item osd.68 weight 0.550
item osd.69 weight 0.550
item osd.70 weight 0.270
item osd.71 weight 0.270
item osd.72 weight 0.270
item osd.73 weight 0.270
item osd.74 weight 0.270
item osd.75 weight 0.270
item osd.76 weight 0.270
item osd.77 weight 0.270
item osd.78 weight 0.270
item osd.79 weight 0.270
item osd.80 weight 0.270
item osd.81 weight 0.270
}
host cstore5 {
id -14 # do not change unnecessarily
# weight 3.320
alg straw
hash 0 # rjenkins1
item osd.0 weight 0.400
item osd.1 weight 0.500
item osd.3 weight 0.400
item osd.58 weight 0.400
item osd.59 weight 0.540
item osd.60 weight 0.540
item osd.61 weight 0.540
}
rack unknownrack {
id -3 # do not change unnecessarily
# weight 67.960
alg straw
hash 0 # rjenkins1
item gstore5 weight 5.000
item cstore1 weight 16.000
item cstore2 weight 20.000
item cstore3 weight 16.000
item cstore4 weight 7.640
item cstore5 weight 3.320
}
pool default {
id -1 # do not change unnecessarily
# weight 67.960
alg straw
hash 0 # rjenkins1
item unknownrack weight 67.960
}
host gstore1 {
id -6 # do not change unnecessarily
# weight 32.760
alg straw
hash 0 # rjenkins1
item osd.82 weight 2.730
item osd.83 weight 2.730
item osd.84 weight 2.730
item osd.85 weight 2.730
item osd.86 weight 2.730
item osd.87 weight 2.730
item osd.88 weight 2.730
item osd.89 weight 2.730
item osd.90 weight 2.730
item osd.91 weight 2.730
item osd.92 weight 2.730
item osd.93 weight 2.730
}
datacenter datacenter-cod {
id -4 # do not change unnecessarily
# weight 32.760
alg straw
hash 0 # rjenkins1
item gstore1 weight 32.760
}
host gstore2 {
id -7 # do not change unnecessarily
# weight 32.760
alg straw
hash 0 # rjenkins1
item osd.94 weight 2.730
item osd.95 weight 2.730
item osd.96 weight 2.730
item osd.97 weight 2.730
item osd.98 weight 2.730
item osd.99 weight 2.730
item osd.100 weight 2.730
item osd.101 weight 2.730
item osd.102 weight 2.730
item osd.103 weight 2.730
item osd.104 weight 2.730
item osd.105 weight 2.730
}
datacenter datacenter-rcod {
id -5 # do not change unnecessarily
# weight 32.760
alg straw
hash 0 # rjenkins1
item gstore2 weight 32.760
}
pool iscsi {
id -2 # do not change unnecessarily
# weight 65.520
alg straw
hash 0 # rjenkins1
item datacenter-cod weight 32.760
item datacenter-rcod weight 32.760
}
# rules
rule data {
ruleset 0
type replicated
min_size 1
max_size 10
step take default
step chooseleaf firstn 0 type host
step emit
}
rule metadata {
ruleset 1
type replicated
min_size 1
max_size 10
step take default
step chooseleaf firstn 0 type host
step emit
}
rule rbd {
ruleset 2
type replicated
min_size 1
max_size 10
step take default
step chooseleaf firstn 0 type host
step emit
}
rule iscsi {
ruleset 3
type replicated
min_size 1
max_size 10
step take iscsi
step chooseleaf firstn 0 type datacenter
step chooseleaf firstn 0 type host
step emit
}
# end crush map
_______________________________________________
ceph-users mailing list
ceph-users@xxxxxxxxxxxxxx
http://lists.ceph.com/listinfo.cgi/ceph-users-ceph.com
