Dear ml,
we are currently trying to wrap our heads around a HEALTH_ERR problem on
our Luminous 12.2.12 cluster (upgraded from Jewel a couple of weeks
ago). Before attempting a 'ceph pg repair' we would like to have a
better understanding of what has happened.
ceph -s reports:
cluster:
id: 7705608d-cbef-477a-865d-f5ae4c03370a
health: HEALTH_ERR
14 scrub errors
Possible data damage: 5 pgs inconsistent
services:
mon: 5 daemons, quorum mon-k3-38,mon-k5-41,mon-l2-40,mon-l7-40,mon-k4-32
mgr: mon-l2-40(active), standbys: mon-l7-40, mon-k3-38, mon-k5-41,
mon-k4-32
osd: 1332 osds: 1332 up, 1332 in
data:
pools: 4 pools, 49160 pgs
objects: 342.85M objects, 1.23PiB
usage: 3.73PiB used, 1.39PiB / 5.11PiB avail
pgs: 49112 active+clean
34 active+clean+scrubbing+deep
9 active+clean+scrubbing
5 active+clean+inconsistent
io:
client: 450MiB/s rd, 411MiB/s wr, 5.10kop/s rd, 2.33kop/s wr
It all seem to have originated from the deletion of an old RBD image
snapshot (cinder volume snapshot) that caused:
- an unusual load spike on most OSD nodes.
- a number of osds being concurrently marked down by mon "despite being
running"
We are now observing an increasing number of scrub errors as
deep-scrubbing progresses, with all inconsistent PGs belonging to the
same pool and all the missing objects to the same RBD image whose
snapshot has been deleted.
Attempting to troubleshoot the situation we realized we don't have a
clear understanding of how an image with snapshots evolves over time:
a RBD image has a preset number of objects but once a snapshot has been
taken and the image starts to diverge from the snapshot more objects
will eventually be needed, correct?
I am attaching the output of 'rados list-inconsistent-obj' ran against
the inconsistent PGs.
We can see two kinds of errors:
- "union_shard_errors": "missing"
- "errors:" "object_info_inconsistency",
"snapset_inconsistency"
The pool has size=3 and min_size=2, the image size is 5TB with 4MB objects.
Has anyone experienced a similar issue? I could not find anything
relevant in the issue tracker but I'll be happy to open a case if this
turns out to be a bug.
Thanks in advance for any hints,
Kind regards,
Mattia Belluco
{
"epoch": 381704,
"inconsistents": [
{
"object": {
"name": "rbd_data.20c737083e5fdc6e.00000000000dc733",
"nspace": "",
"locator": "",
"snap": 16970,
"version": 4532671
},
"errors": [],
"union_shard_errors": [
"missing"
],
"selected_object_info": {
"oid": {
"oid": "rbd_data.20c737083e5fdc6e.00000000000dc733",
"key": "",
"snapid": 16970,
"hash": 3945368553,
"max": 0,
"pool": 36,
"namespace": ""
},
"version": "326149'4861369",
"prior_version": "262931'4535183",
"last_reqid": "osd.987.0:56883287",
"user_version": 4532671,
"size": 4194304,
"mtime": "2019-03-06 16:04:40.619294",
"local_mtime": "2019-03-06 16:04:40.630542",
"lost": 0,
"flags": [
"dirty",
"data_digest",
"omap_digest"
],
"legacy_snaps": [
16970
],
"truncate_seq": 0,
"truncate_size": 0,
"data_digest": "0x1b5e4a89",
"omap_digest": "0xffffffff",
"expected_object_size": 0,
"expected_write_size": 0,
"alloc_hint_flags": 0,
"manifest": {
"type": 0,
"redirect_target": {
"oid": "",
"key": "",
"snapid": 0,
"hash": 0,
"max": 0,
"pool": -9223372036854775808,
"namespace": ""
}
},
"watchers": {}
},
"shards": [
{
"osd": 190,
"primary": false,
"errors": [],
"size": 4194304
},
{
"osd": 1254,
"primary": true,
"errors": [
"missing"
]
},
{
"osd": 1317,
"primary": false,
"errors": [],
"size": 4194304
}
]
},
{
"object": {
"name": "rbd_data.20c737083e5fdc6e.00000000000dc733",
"nspace": "",
"locator": "",
"snap": "head",
"version": 5283556
},
"errors": [
"object_info_inconsistency",
"snapset_inconsistency"
],
"union_shard_errors": [],
"selected_object_info": {
"oid": {
"oid": "rbd_data.20c737083e5fdc6e.00000000000dc733",
"key": "",
"snapid": -2,
"hash": 3945368553,
"max": 0,
"pool": 36,
"namespace": ""
},
"version": "381700'5459156",
"prior_version": "369764'5299802",
"last_reqid": "osd.1254.0:491362",
"user_version": 5283556,
"size": 4194304,
"mtime": "2019-08-30 11:01:06.681079",
"local_mtime": "2019-08-30 11:01:06.683473",
"lost": 0,
"flags": [
"dirty",
"data_digest",
"omap_digest"
],
"legacy_snaps": [],
"truncate_seq": 0,
"truncate_size": 0,
"data_digest": "0x54ec13a2",
"omap_digest": "0xffffffff",
"expected_object_size": 4194304,
"expected_write_size": 4194304,
"alloc_hint_flags": 0,
"manifest": {
"type": 0,
"redirect_target": {
"oid": "",
"key": "",
"snapid": 0,
"hash": 0,
"max": 0,
"pool": -9223372036854775808,
"namespace": ""
}
},
"watchers": {}
},
"shards": [
{
"osd": 190,
"primary": false,
"errors": [],
"size": 4194304,
"object_info": {
"oid": {
"oid": "rbd_data.20c737083e5fdc6e.00000000000dc733",
"key": "",
"snapid": -2,
"hash": 3945368553,
"max": 0,
"pool": 36,
"namespace": ""
},
"version": "369764'5299802",
"prior_version": "369764'5299801",
"last_reqid": "osd.1254.0:491362",
"user_version": 5283556,
"size": 4194304,
"mtime": "2019-08-30 11:01:06.681079",
"local_mtime": "2019-08-30 11:01:06.683473",
"lost": 0,
"flags": [
"dirty",
"data_digest",
"omap_digest"
],
"legacy_snaps": [],
"truncate_seq": 0,
"truncate_size": 0,
"data_digest": "0x54ec13a2",
"omap_digest": "0xffffffff",
"expected_object_size": 4194304,
"expected_write_size": 4194304,
"alloc_hint_flags": 0,
"manifest": {
"type": 0,
"redirect_target": {
"oid": "",
"key": "",
"snapid": 0,
"hash": 0,
"max": 0,
"pool": -9223372036854775808,
"namespace": ""
}
},
"watchers": {}
},
"snapset": {
"snap_context": {
"seq": 16970,
"snaps": [
16970
]
},
"head_exists": 1,
"clones": [
{
"snap": 16970,
"size": 4194304,
"overlap": "[1048576~327680]",
"snaps": [
16970
]
}
]
}
},
{
"osd": 1254,
"primary": true,
"errors": [],
"size": 4194304,
"object_info": {
"oid": {
"oid": "rbd_data.20c737083e5fdc6e.00000000000dc733",
"key": "",
"snapid": -2,
"hash": 3945368553,
"max": 0,
"pool": 36,
"namespace": ""
},
"version": "381700'5459156",
"prior_version": "369764'5299802",
"last_reqid": "osd.1254.0:491362",
"user_version": 5283556,
"size": 4194304,
"mtime": "2019-08-30 11:01:06.681079",
"local_mtime": "2019-08-30 11:01:06.683473",
"lost": 0,
"flags": [
"dirty",
"data_digest",
"omap_digest"
],
"legacy_snaps": [],
"truncate_seq": 0,
"truncate_size": 0,
"data_digest": "0x54ec13a2",
"omap_digest": "0xffffffff",
"expected_object_size": 4194304,
"expected_write_size": 4194304,
"alloc_hint_flags": 0,
"manifest": {
"type": 0,
"redirect_target": {
"oid": "",
"key": "",
"snapid": 0,
"hash": 0,
"max": 0,
"pool": -9223372036854775808,
"namespace": ""
}
},
"watchers": {}
},
"snapset": {
"snap_context": {
"seq": 16970,
"snaps": []
},
"head_exists": 1,
"clones": []
}
},
{
"osd": 1317,
"primary": false,
"errors": [],
"size": 4194304,
"object_info": {
"oid": {
"oid": "rbd_data.20c737083e5fdc6e.00000000000dc733",
"key": "",
"snapid": -2,
"hash": 3945368553,
"max": 0,
"pool": 36,
"namespace": ""
},
"version": "369764'5299802",
"prior_version": "369764'5299801",
"last_reqid": "osd.1254.0:491362",
"user_version": 5283556,
"size": 4194304,
"mtime": "2019-08-30 11:01:06.681079",
"local_mtime": "2019-08-30 11:01:06.683473",
"lost": 0,
"flags": [
"dirty",
"data_digest",
"omap_digest"
],
"legacy_snaps": [],
"truncate_seq": 0,
"truncate_size": 0,
"data_digest": "0x54ec13a2",
"omap_digest": "0xffffffff",
"expected_object_size": 4194304,
"expected_write_size": 4194304,
"alloc_hint_flags": 0,
"manifest": {
"type": 0,
"redirect_target": {
"oid": "",
"key": "",
"snapid": 0,
"hash": 0,
"max": 0,
"pool": -9223372036854775808,
"namespace": ""
}
},
"watchers": {}
},
"snapset": {
"snap_context": {
"seq": 16970,
"snaps": [
16970
]
},
"head_exists": 1,
"clones": [
{
"snap": 16970,
"size": 4194304,
"overlap": "[1048576~327680]",
"snaps": [
16970
]
}
]
}
}
]
}
]
}
{
"epoch": 381675,
"inconsistents": [
{
"object": {
"name": "rbd_data.20c737083e5fdc6e.0000000000014376",
"nspace": "",
"locator": "",
"snap": 16970,
"version": 4967730
},
"errors": [],
"union_shard_errors": [
"missing"
],
"selected_object_info": {
"oid": {
"oid": "rbd_data.20c737083e5fdc6e.0000000000014376",
"key": "",
"snapid": 16970,
"hash": 633050641,
"max": 0,
"pool": 36,
"namespace": ""
},
"version": "374722'6800514",
"prior_version": "189139'4972773",
"last_reqid": "osd.566.0:17126097",
"user_version": 4967730,
"size": 4194304,
"mtime": "2018-07-16 19:04:04.504694",
"local_mtime": "2018-07-16 19:04:04.507996",
"lost": 0,
"flags": [
"dirty",
"data_digest",
"omap_digest"
],
"legacy_snaps": [],
"truncate_seq": 0,
"truncate_size": 0,
"data_digest": "0xba655f8a",
"omap_digest": "0xffffffff",
"expected_object_size": 0,
"expected_write_size": 0,
"alloc_hint_flags": 0,
"manifest": {
"type": 0,
"redirect_target": {
"oid": "",
"key": "",
"snapid": 0,
"hash": 0,
"max": 0,
"pool": -9223372036854775808,
"namespace": ""
}
},
"watchers": {}
},
"shards": [
{
"osd": 566,
"primary": true,
"errors": [
"missing"
]
},
{
"osd": 575,
"primary": false,
"errors": [],
"size": 4194304
},
{
"osd": 1285,
"primary": false,
"errors": [],
"size": 4194304
}
]
},
{
"object": {
"name": "rbd_data.20c737083e5fdc6e.0000000000014376",
"nspace": "",
"locator": "",
"snap": "head",
"version": 6800517
},
"errors": [
"object_info_inconsistency",
"snapset_inconsistency"
],
"union_shard_errors": [],
"selected_object_info": {
"oid": {
"oid": "rbd_data.20c737083e5fdc6e.0000000000014376",
"key": "",
"snapid": -2,
"hash": 633050641,
"max": 0,
"pool": 36,
"namespace": ""
},
"version": "381672'6869082",
"prior_version": "375620'6805877",
"last_reqid": "osd.566.0:417488",
"user_version": 6800517,
"size": 4194304,
"mtime": "2019-09-04 20:11:33.677490",
"local_mtime": "2019-09-04 20:11:33.682616",
"lost": 0,
"flags": [
"dirty",
"data_digest",
"omap_digest"
],
"legacy_snaps": [],
"truncate_seq": 0,
"truncate_size": 0,
"data_digest": "0x42796279",
"omap_digest": "0xffffffff",
"expected_object_size": 4194304,
"expected_write_size": 4194304,
"alloc_hint_flags": 0,
"manifest": {
"type": 0,
"redirect_target": {
"oid": "",
"key": "",
"snapid": 0,
"hash": 0,
"max": 0,
"pool": -9223372036854775808,
"namespace": ""
}
},
"watchers": {}
},
"shards": [
{
"osd": 566,
"primary": true,
"errors": [],
"size": 4194304,
"object_info": {
"oid": {
"oid": "rbd_data.20c737083e5fdc6e.0000000000014376",
"key": "",
"snapid": -2,
"hash": 633050641,
"max": 0,
"pool": 36,
"namespace": ""
},
"version": "381672'6869082",
"prior_version": "375620'6805877",
"last_reqid": "osd.566.0:417488",
"user_version": 6800517,
"size": 4194304,
"mtime": "2019-09-04 20:11:33.677490",
"local_mtime": "2019-09-04 20:11:33.682616",
"lost": 0,
"flags": [
"dirty",
"data_digest",
"omap_digest"
],
"legacy_snaps": [],
"truncate_seq": 0,
"truncate_size": 0,
"data_digest": "0x42796279",
"omap_digest": "0xffffffff",
"expected_object_size": 4194304,
"expected_write_size": 4194304,
"alloc_hint_flags": 0,
"manifest": {
"type": 0,
"redirect_target": {
"oid": "",
"key": "",
"snapid": 0,
"hash": 0,
"max": 0,
"pool": -9223372036854775808,
"namespace": ""
}
},
"watchers": {}
},
"snapset": {
"snap_context": {
"seq": 16970,
"snaps": []
},
"head_exists": 1,
"clones": []
}
},
{
"osd": 575,
"primary": false,
"errors": [],
"size": 4194304,
"object_info": {
"oid": {
"oid": "rbd_data.20c737083e5fdc6e.0000000000014376",
"key": "",
"snapid": -2,
"hash": 633050641,
"max": 0,
"pool": 36,
"namespace": ""
},
"version": "375620'6805877",
"prior_version": "374722'6800517",
"last_reqid": "osd.566.0:417488",
"user_version": 6800517,
"size": 4194304,
"mtime": "2019-09-04 20:11:33.677490",
"local_mtime": "2019-09-04 20:11:33.682616",
"lost": 0,
"flags": [
"dirty",
"data_digest",
"omap_digest"
],
"legacy_snaps": [],
"truncate_seq": 0,
"truncate_size": 0,
"data_digest": "0x42796279",
"omap_digest": "0xffffffff",
"expected_object_size": 4194304,
"expected_write_size": 4194304,
"alloc_hint_flags": 0,
"manifest": {
"type": 0,
"redirect_target": {
"oid": "",
"key": "",
"snapid": 0,
"hash": 0,
"max": 0,
"pool": -9223372036854775808,
"namespace": ""
}
},
"watchers": {}
},
"snapset": {
"snap_context": {
"seq": 16970,
"snaps": [
16970
]
},
"head_exists": 1,
"clones": [
{
"snap": 16970,
"size": 4194304,
"overlap": "[1048576~3145728]",
"snaps": [
16970
]
}
]
}
},
{
"osd": 1285,
"primary": false,
"errors": [],
"size": 4194304,
"object_info": {
"oid": {
"oid": "rbd_data.20c737083e5fdc6e.0000000000014376",
"key": "",
"snapid": -2,
"hash": 633050641,
"max": 0,
"pool": 36,
"namespace": ""
},
"version": "375620'6805877",
"prior_version": "374722'6800517",
"last_reqid": "osd.566.0:417488",
"user_version": 6800517,
"size": 4194304,
"mtime": "2019-09-04 20:11:33.677490",
"local_mtime": "2019-09-04 20:11:33.682616",
"lost": 0,
"flags": [
"dirty",
"data_digest",
"omap_digest"
],
"legacy_snaps": [],
"truncate_seq": 0,
"truncate_size": 0,
"data_digest": "0x42796279",
"omap_digest": "0xffffffff",
"expected_object_size": 4194304,
"expected_write_size": 4194304,
"alloc_hint_flags": 0,
"manifest": {
"type": 0,
"redirect_target": {
"oid": "",
"key": "",
"snapid": 0,
"hash": 0,
"max": 0,
"pool": -9223372036854775808,
"namespace": ""
}
},
"watchers": {}
},
"snapset": {
"snap_context": {
"seq": 16970,
"snaps": [
16970
]
},
"head_exists": 1,
"clones": [
{
"snap": 16970,
"size": 4194304,
"overlap": "[1048576~3145728]",
"snaps": [
16970
]
}
]
}
}
]
}
]
}
{
"epoch": 381666,
"inconsistents": [
{
"object": {
"name": "rbd_data.20c737083e5fdc6e.00000000000d1cf4",
"nspace": "",
"locator": "",
"snap": 16970,
"version": 5534061
},
"errors": [],
"union_shard_errors": [
"missing"
],
"selected_object_info": {
"oid": {
"oid": "rbd_data.20c737083e5fdc6e.00000000000d1cf4",
"key": "",
"snapid": 16970,
"hash": 20194156,
"max": 0,
"pool": 36,
"namespace": ""
},
"version": "359776'6893919",
"prior_version": "256671'5539617",
"last_reqid": "osd.912.0:39602580",
"user_version": 5534061,
"size": 4194304,
"mtime": "2019-02-18 09:31:12.202842",
"local_mtime": "2019-02-18 09:31:12.210501",
"lost": 0,
"flags": [
"dirty",
"data_digest",
"omap_digest"
],
"legacy_snaps": [
16970
],
"truncate_seq": 0,
"truncate_size": 0,
"data_digest": "0x8bb2310d",
"omap_digest": "0xffffffff",
"expected_object_size": 0,
"expected_write_size": 0,
"alloc_hint_flags": 0,
"manifest": {
"type": 0,
"redirect_target": {
"oid": "",
"key": "",
"snapid": 0,
"hash": 0,
"max": 0,
"pool": -9223372036854775808,
"namespace": ""
}
},
"watchers": {}
},
"shards": [
{
"osd": 384,
"primary": false,
"errors": [],
"size": 4194304,
"omap_digest": "0xffffffff",
"data_digest": "0x8bb2310d"
},
{
"osd": 539,
"primary": false,
"errors": [],
"size": 4194304,
"omap_digest": "0xffffffff",
"data_digest": "0x8bb2310d"
},
{
"osd": 912,
"primary": true,
"errors": [
"missing"
]
}
]
},
{
"object": {
"name": "rbd_data.20c737083e5fdc6e.00000000000d1cf4",
"nspace": "",
"locator": "",
"snap": "head",
"version": 6908382
},
"errors": [
"object_info_inconsistency",
"snapset_inconsistency"
],
"union_shard_errors": [],
"selected_object_info": {
"oid": {
"oid": "rbd_data.20c737083e5fdc6e.00000000000d1cf4",
"key": "",
"snapid": -2,
"hash": 20194156,
"max": 0,
"pool": 36,
"namespace": ""
},
"version": "381662'7131617",
"prior_version": "373678'7023961",
"last_reqid": "osd.912.0:322487",
"user_version": 6908382,
"size": 4194304,
"mtime": "2019-08-14 19:34:18.929576",
"local_mtime": "2019-08-14 19:34:18.938024",
"lost": 0,
"flags": [
"dirty",
"data_digest",
"omap_digest"
],
"legacy_snaps": [],
"truncate_seq": 0,
"truncate_size": 0,
"data_digest": "0x04110e3b",
"omap_digest": "0xffffffff",
"expected_object_size": 4194304,
"expected_write_size": 4194304,
"alloc_hint_flags": 0,
"manifest": {
"type": 0,
"redirect_target": {
"oid": "",
"key": "",
"snapid": 0,
"hash": 0,
"max": 0,
"pool": -9223372036854775808,
"namespace": ""
}
},
"watchers": {}
},
"shards": [
{
"osd": 384,
"primary": false,
"errors": [],
"size": 4194304,
"omap_digest": "0xffffffff",
"data_digest": "0x04110e3b",
"object_info": {
"oid": {
"oid": "rbd_data.20c737083e5fdc6e.00000000000d1cf4",
"key": "",
"snapid": -2,
"hash": 20194156,
"max": 0,
"pool": 36,
"namespace": ""
},
"version": "373678'7023961",
"prior_version": "361773'6942508",
"last_reqid": "osd.912.0:322487",
"user_version": 6908382,
"size": 4194304,
"mtime": "2019-08-14 19:34:18.929576",
"local_mtime": "2019-08-14 19:34:18.938024",
"lost": 0,
"flags": [
"dirty",
"data_digest",
"omap_digest"
],
"legacy_snaps": [],
"truncate_seq": 0,
"truncate_size": 0,
"data_digest": "0x04110e3b",
"omap_digest": "0xffffffff",
"expected_object_size": 4194304,
"expected_write_size": 4194304,
"alloc_hint_flags": 0,
"manifest": {
"type": 0,
"redirect_target": {
"oid": "",
"key": "",
"snapid": 0,
"hash": 0,
"max": 0,
"pool": -9223372036854775808,
"namespace": ""
}
},
"watchers": {}
},
"snapset": {
"snap_context": {
"seq": 16970,
"snaps": [
16970
]
},
"head_exists": 1,
"clones": [
{
"snap": 16970,
"size": 4194304,
"overlap": "[]",
"snaps": [
16970
]
}
]
}
},
{
"osd": 539,
"primary": false,
"errors": [],
"size": 4194304,
"omap_digest": "0xffffffff",
"data_digest": "0x04110e3b",
"object_info": {
"oid": {
"oid": "rbd_data.20c737083e5fdc6e.00000000000d1cf4",
"key": "",
"snapid": -2,
"hash": 20194156,
"max": 0,
"pool": 36,
"namespace": ""
},
"version": "373678'7023961",
"prior_version": "361773'6942508",
"last_reqid": "osd.912.0:322487",
"user_version": 6908382,
"size": 4194304,
"mtime": "2019-08-14 19:34:18.929576",
"local_mtime": "2019-08-14 19:34:18.938024",
"lost": 0,
"flags": [
"dirty",
"data_digest",
"omap_digest"
],
"legacy_snaps": [],
"truncate_seq": 0,
"truncate_size": 0,
"data_digest": "0x04110e3b",
"omap_digest": "0xffffffff",
"expected_object_size": 4194304,
"expected_write_size": 4194304,
"alloc_hint_flags": 0,
"manifest": {
"type": 0,
"redirect_target": {
"oid": "",
"key": "",
"snapid": 0,
"hash": 0,
"max": 0,
"pool": -9223372036854775808,
"namespace": ""
}
},
"watchers": {}
},
"snapset": {
"snap_context": {
"seq": 16970,
"snaps": [
16970
]
},
"head_exists": 1,
"clones": [
{
"snap": 16970,
"size": 4194304,
"overlap": "[]",
"snaps": [
16970
]
}
]
}
},
{
"osd": 912,
"primary": true,
"errors": [],
"size": 4194304,
"omap_digest": "0xffffffff",
"data_digest": "0x04110e3b",
"object_info": {
"oid": {
"oid": "rbd_data.20c737083e5fdc6e.00000000000d1cf4",
"key": "",
"snapid": -2,
"hash": 20194156,
"max": 0,
"pool": 36,
"namespace": ""
},
"version": "381662'7131617",
"prior_version": "373678'7023961",
"last_reqid": "osd.912.0:322487",
"user_version": 6908382,
"size": 4194304,
"mtime": "2019-08-14 19:34:18.929576",
"local_mtime": "2019-08-14 19:34:18.938024",
"lost": 0,
"flags": [
"dirty",
"data_digest",
"omap_digest"
],
"legacy_snaps": [],
"truncate_seq": 0,
"truncate_size": 0,
"data_digest": "0x04110e3b",
"omap_digest": "0xffffffff",
"expected_object_size": 4194304,
"expected_write_size": 4194304,
"alloc_hint_flags": 0,
"manifest": {
"type": 0,
"redirect_target": {
"oid": "",
"key": "",
"snapid": 0,
"hash": 0,
"max": 0,
"pool": -9223372036854775808,
"namespace": ""
}
},
"watchers": {}
},
"snapset": {
"snap_context": {
"seq": 16970,
"snaps": []
},
"head_exists": 1,
"clones": []
}
}
]
}
]
}
{
"epoch": 381674,
"inconsistents": [
{
"object": {
"name": "rbd_data.20c737083e5fdc6e.000000000003dee4",
"nspace": "",
"locator": "",
"snap": 16970,
"version": 2670409
},
"errors": [],
"union_shard_errors": [
"missing"
],
"selected_object_info": {
"oid": {
"oid": "rbd_data.20c737083e5fdc6e.000000000003dee4",
"key": "",
"snapid": 16970,
"hash": 935143073,
"max": 0,
"pool": 36,
"namespace": ""
},
"version": "368645'5004394",
"prior_version": "189111'2672847",
"last_reqid": "osd.793.0:10861702",
"user_version": 2670409,
"size": 4194304,
"mtime": "2018-07-17 06:30:03.725532",
"local_mtime": "2018-07-17 06:30:03.729200",
"lost": 0,
"flags": [
"dirty",
"data_digest",
"omap_digest"
],
"legacy_snaps": [
16970
],
"truncate_seq": 0,
"truncate_size": 0,
"data_digest": "0x42ece340",
"omap_digest": "0xffffffff",
"expected_object_size": 0,
"expected_write_size": 0,
"alloc_hint_flags": 0,
"manifest": {
"type": 0,
"redirect_target": {
"oid": "",
"key": "",
"snapid": 0,
"hash": 0,
"max": 0,
"pool": -9223372036854775808,
"namespace": ""
}
},
"watchers": {}
},
"shards": [
{
"osd": 793,
"primary": true,
"errors": [
"missing"
]
},
{
"osd": 977,
"primary": false,
"errors": [
"missing"
]
},
{
"osd": 1109,
"primary": false,
"errors": [],
"size": 4194304
}
]
},
{
"object": {
"name": "rbd_data.20c737083e5fdc6e.000000000003dee4",
"nspace": "",
"locator": "",
"snap": "head",
"version": 5055818
},
"errors": [
"object_info_inconsistency",
"snapset_inconsistency"
],
"union_shard_errors": [],
"selected_object_info": {
"oid": {
"oid": "rbd_data.20c737083e5fdc6e.000000000003dee4",
"key": "",
"snapid": -2,
"hash": 935143073,
"max": 0,
"pool": 36,
"namespace": ""
},
"version": "381672'5154353",
"prior_version": "375648'5080222",
"last_reqid": "osd.793.0:690956",
"user_version": 5055818,
"size": 4194304,
"mtime": "2019-09-02 09:17:10.290331",
"local_mtime": "2019-09-02 09:17:10.292028",
"lost": 0,
"flags": [
"dirty",
"data_digest",
"omap_digest"
],
"legacy_snaps": [],
"truncate_seq": 0,
"truncate_size": 0,
"data_digest": "0xe6e32110",
"omap_digest": "0xffffffff",
"expected_object_size": 4194304,
"expected_write_size": 4194304,
"alloc_hint_flags": 0,
"manifest": {
"type": 0,
"redirect_target": {
"oid": "",
"key": "",
"snapid": 0,
"hash": 0,
"max": 0,
"pool": -9223372036854775808,
"namespace": ""
}
},
"watchers": {}
},
"shards": [
{
"osd": 793,
"primary": true,
"errors": [],
"size": 4194304,
"object_info": {
"oid": {
"oid": "rbd_data.20c737083e5fdc6e.000000000003dee4",
"key": "",
"snapid": -2,
"hash": 935143073,
"max": 0,
"pool": 36,
"namespace": ""
},
"version": "381672'5154353",
"prior_version": "375648'5080222",
"last_reqid": "osd.793.0:690956",
"user_version": 5055818,
"size": 4194304,
"mtime": "2019-09-02 09:17:10.290331",
"local_mtime": "2019-09-02 09:17:10.292028",
"lost": 0,
"flags": [
"dirty",
"data_digest",
"omap_digest"
],
"legacy_snaps": [],
"truncate_seq": 0,
"truncate_size": 0,
"data_digest": "0xe6e32110",
"omap_digest": "0xffffffff",
"expected_object_size": 4194304,
"expected_write_size": 4194304,
"alloc_hint_flags": 0,
"manifest": {
"type": 0,
"redirect_target": {
"oid": "",
"key": "",
"snapid": 0,
"hash": 0,
"max": 0,
"pool": -9223372036854775808,
"namespace": ""
}
},
"watchers": {}
},
"snapset": {
"snap_context": {
"seq": 16970,
"snaps": []
},
"head_exists": 1,
"clones": []
}
},
{
"osd": 977,
"primary": false,
"errors": [],
"size": 4194304,
"object_info": {
"oid": {
"oid": "rbd_data.20c737083e5fdc6e.000000000003dee4",
"key": "",
"snapid": -2,
"hash": 935143073,
"max": 0,
"pool": 36,
"namespace": ""
},
"version": "381672'5154353",
"prior_version": "375648'5080222",
"last_reqid": "osd.793.0:690956",
"user_version": 5055818,
"size": 4194304,
"mtime": "2019-09-02 09:17:10.290331",
"local_mtime": "2019-09-02 09:17:10.292028",
"lost": 0,
"flags": [
"dirty",
"data_digest",
"omap_digest"
],
"legacy_snaps": [],
"truncate_seq": 0,
"truncate_size": 0,
"data_digest": "0xe6e32110",
"omap_digest": "0xffffffff",
"expected_object_size": 4194304,
"expected_write_size": 4194304,
"alloc_hint_flags": 0,
"manifest": {
"type": 0,
"redirect_target": {
"oid": "",
"key": "",
"snapid": 0,
"hash": 0,
"max": 0,
"pool": -9223372036854775808,
"namespace": ""
}
},
"watchers": {}
},
"snapset": {
"snap_context": {
"seq": 16970,
"snaps": []
},
"head_exists": 1,
"clones": []
}
},
{
"osd": 1109,
"primary": false,
"errors": [],
"size": 4194304,
"object_info": {
"oid": {
"oid": "rbd_data.20c737083e5fdc6e.000000000003dee4",
"key": "",
"snapid": -2,
"hash": 935143073,
"max": 0,
"pool": 36,
"namespace": ""
},
"version": "375648'5080222",
"prior_version": "373124'5060007",
"last_reqid": "osd.793.0:690956",
"user_version": 5055818,
"size": 4194304,
"mtime": "2019-09-02 09:17:10.290331",
"local_mtime": "2019-09-02 09:17:10.292028",
"lost": 0,
"flags": [
"dirty",
"data_digest",
"omap_digest"
],
"legacy_snaps": [],
"truncate_seq": 0,
"truncate_size": 0,
"data_digest": "0xe6e32110",
"omap_digest": "0xffffffff",
"expected_object_size": 4194304,
"expected_write_size": 4194304,
"alloc_hint_flags": 0,
"manifest": {
"type": 0,
"redirect_target": {
"oid": "",
"key": "",
"snapid": 0,
"hash": 0,
"max": 0,
"pool": -9223372036854775808,
"namespace": ""
}
},
"watchers": {}
},
"snapset": {
"snap_context": {
"seq": 16970,
"snaps": [
16970
]
},
"head_exists": 1,
"clones": [
{
"snap": 16970,
"size": 4194304,
"overlap": "[]",
"snaps": [
16970
]
}
]
}
}
]
}
]
}
{
"epoch": 381710,
"inconsistents": [
{
"object": {
"name": "rbd_data.ef6bab128c15569.00000000001818e9",
"nspace": "",
"locator": "",
"snap": "head",
"version": 6997940
},
"errors": [],
"union_shard_errors": [
"read_error"
],
"selected_object_info": {
"oid": {
"oid": "rbd_data.ef6bab128c15569.00000000001818e9",
"key": "",
"snapid": -2,
"hash": 4006504529,
"max": 0,
"pool": 36,
"namespace": ""
},
"version": "381726'6997940",
"prior_version": "380185'6898931",
"last_reqid": "client.787178776.0:734147066",
"user_version": 6997940,
"size": 4194304,
"mtime": "2019-09-18 10:18:22.606477",
"local_mtime": "2019-09-18 10:18:22.613620",
"lost": 0,
"flags": [
"dirty",
"data_digest",
"omap_digest"
],
"legacy_snaps": [],
"truncate_seq": 0,
"truncate_size": 0,
"data_digest": "0x8323446c",
"omap_digest": "0xffffffff",
"expected_object_size": 0,
"expected_write_size": 0,
"alloc_hint_flags": 0,
"manifest": {
"type": 0,
"redirect_target": {
"oid": "",
"key": "",
"snapid": 0,
"hash": 0,
"max": 0,
"pool": -9223372036854775808,
"namespace": ""
}
},
"watchers": {}
},
"shards": [
{
"osd": 1182,
"primary": false,
"errors": [],
"size": 4194304,
"omap_digest": "0xffffffff",
"data_digest": "0x8323446c"
},
{
"osd": 1207,
"primary": true,
"errors": [
"read_error"
],
"size": 4194304
},
{
"osd": 1296,
"primary": false,
"errors": [],
"size": 4194304,
"omap_digest": "0xffffffff",
"data_digest": "0x8323446c"
}
]
}
]
}
_______________________________________________ ceph-users mailing list ceph-users@xxxxxxxxxxxxxx http://lists.ceph.com/listinfo.cgi/ceph-users-ceph.com
