Hi
I have recently discovered that one of my pg is in inconsistent state. I have checked filesystem on osd.3, and re-run deep-scrub few times. Osd uses xfs. Any suggestions how to fix it?
# ceph -v
ceph version 0.61.8 (a6fdcca3bddbc9f177e4e2bf0d9cdd85006b028b)
# ceph pg deep-scrub 350.5e
2013-09-03 16:15:51.961722 7f9189d5c700 0 log [ERR] : 350.5e osd.3: soid 21b651de/rbd_data.163bb74b0dc51.0000000000000007/head//350 digest 1420341435 != known digest 3593795552
2013-09-03 16:15:53.114372 7f9189d5c700 0 log [ERR] : 350.5e deep-scrub 0 missing, 1 inconsistent objects
2013-09-03 16:15:53.114379 7f9189d5c700 0 log [ERR] : 350.5e deep-scrub 1 errors
{ "state": "active+clean+inconsistent",
"epoch": 6429,
"up": [
3,
5],
"acting": [
3,
5],
"info": { "pgid": "350.5e",
"last_update": "6429'972570",
"last_complete": "6429'972570",
"log_tail": "6398'969569",
"last_backfill": "MAX",
"purged_snaps": "[]",
"history": { "epoch_created": 2132,
"last_epoch_started": 6429,
"last_epoch_clean": 6429,
"last_epoch_split": 0,
"same_up_since": 6428,
"same_interval_since": 6428,
"same_primary_since": 6428,
"last_scrub": "6429'972567",
"last_scrub_stamp": "2013-09-03 16:15:53.114382",
"last_deep_scrub": "6429'972567",
"last_deep_scrub_stamp": "2013-09-03 16:15:53.114382",
"last_clean_scrub_stamp": "2013-09-03 16:04:57.077937"},
"stats": { "version": "6429'972570",
"reported": "6429'1178349",
"state": "active+clean+inconsistent",
"last_fresh": "2013-09-03 16:16:23.630653",
"last_change": "2013-09-03 16:15:53.114402",
"last_active": "2013-09-03 16:16:23.630653",
"last_clean": "2013-09-03 16:16:23.630653",
"last_became_active": "0.000000",
"last_unstale": "2013-09-03 16:16:23.630653",
"mapping_epoch": 6426,
"log_start": "6398'969569",
"ondisk_log_start": "6398'969569",
"created": 2132,
"last_epoch_clean": 6429,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "6429'972567",
"last_scrub_stamp": "2013-09-03 16:15:53.114382",
"last_deep_scrub": "6429'972567",
"last_deep_scrub_stamp": "2013-09-03 16:15:53.114382",
"last_clean_scrub_stamp": "2013-09-03 16:04:57.077937",
"log_size": 0,
"ondisk_log_size": 0,
"stats_invalid": "0",
"stat_sum": { "num_bytes": 571404305,
"num_objects": 78,
"num_object_clones": 0,
"num_object_copies": 0,
"num_objects_missing_on_primary": 0,
"num_objects_degraded": 0,
"num_objects_unfound": 0,
"num_read": 1037151,
"num_read_kb": 1982418,
"num_write": 972739,
"num_write_kb": 12430480,
"num_scrub_errors": 1,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 1,
"num_objects_recovered": 1320,
"num_bytes_recovered": 10111942741,
"num_keys_recovered": 124},
"stat_cat_sum": {},
"up": [
3,
5],
"acting": [
3,
5]},
"empty": 0,
"dne": 0,
"incomplete": 0,
"last_epoch_started": 6429},
"recovery_state": [
{ "name": "Started\/Primary\/Active",
"enter_time": "2013-09-03 16:14:59.621528",
"might_have_unfound": [
{ "osd": 5,
"status": "already probed"}],
"recovery_progress": { "backfill_target": -1,
"waiting_on_backfill": 0,
"backfill_pos": "0\/\/0\/\/-1",
"backfill_info": { "begin": "0\/\/0\/\/-1",
"end": "0\/\/0\/\/-1",
"objects": []},
"peer_backfill_info": { "begin": "0\/\/0\/\/-1",
"end": "0\/\/0\/\/-1",
"objects": []},
"backfills_in_flight": [],
"pull_from_peer": [],
"pushing": []},
"scrub": { "scrubber.epoch_start": "6428",
"scrubber.active": 0,
"scrubber.block_writes": 0,
"scrubber.finalizing": 0,
"scrubber.waiting_on": 0,
"scrubber.waiting_on_whom": []}},
{ "name": "Started",
"enter_time": "2013-09-03 16:14:58.192177"}]}
regards
-- Maciej Gałkiewicz
Shelly Cloud Sp. z o. o., Sysadmin
http://shellycloud.com/, macias@xxxxxxxxxxxxxxx
KRS: 0000440358 REGON: 101504426
_______________________________________________ ceph-users mailing list ceph-users@xxxxxxxxxxxxxx http://lists.ceph.com/listinfo.cgi/ceph-users-ceph.com
