On Tue, Sep 3, 2013 at 7:20 AM, Maciej Gałkiewicz <macias@xxxxxxxxxxxxxxx> wrote: > Hi > > I have recently discovered that one of my pg is in inconsistent state. I > have checked filesystem on osd.3, and re-run deep-scrub few times. Osd uses > xfs. Any suggestions how to fix it? You can use "repair" instead of "scrub" or "deep-scrub" — but right now the "fix" just copies the primary's copy to anybody who disagrees. You need to evaluate data integrity yourself (if you have three copies, probably just look for the two that match) and if the primary is the bad copy then delete it on the node before running repair. At least, that was how it went last time I saw this come up; Sam might have some newer recommendations. -Greg Software Engineer #42 @ http://inktank.com | http://ceph.com > > # ceph -v > ceph version 0.61.8 (a6fdcca3bddbc9f177e4e2bf0d9cdd85006b028b) > # ceph pg deep-scrub 350.5e > > 2013-09-03 16:15:51.961722 7f9189d5c700 0 log [ERR] : 350.5e osd.3: soid > 21b651de/rbd_data.163bb74b0dc51.0000000000000007/head//350 digest 1420341435 > != known digest 3593795552 > 2013-09-03 16:15:53.114372 7f9189d5c700 0 log [ERR] : 350.5e deep-scrub 0 > missing, 1 inconsistent objects > 2013-09-03 16:15:53.114379 7f9189d5c700 0 log [ERR] : 350.5e deep-scrub 1 > errors > > > { "state": "active+clean+inconsistent", > "epoch": 6429, > "up": [ > 3, > 5], > "acting": [ > 3, > 5], > "info": { "pgid": "350.5e", > "last_update": "6429'972570", > "last_complete": "6429'972570", > "log_tail": "6398'969569", > "last_backfill": "MAX", > "purged_snaps": "[]", > "history": { "epoch_created": 2132, > "last_epoch_started": 6429, > "last_epoch_clean": 6429, > "last_epoch_split": 0, > "same_up_since": 6428, > "same_interval_since": 6428, > "same_primary_since": 6428, > "last_scrub": "6429'972567", > "last_scrub_stamp": "2013-09-03 16:15:53.114382", > "last_deep_scrub": "6429'972567", > "last_deep_scrub_stamp": "2013-09-03 16:15:53.114382", > "last_clean_scrub_stamp": "2013-09-03 16:04:57.077937"}, > "stats": { "version": "6429'972570", > "reported": "6429'1178349", > "state": "active+clean+inconsistent", > "last_fresh": "2013-09-03 16:16:23.630653", > "last_change": "2013-09-03 16:15:53.114402", > "last_active": "2013-09-03 16:16:23.630653", > "last_clean": "2013-09-03 16:16:23.630653", > "last_became_active": "0.000000", > "last_unstale": "2013-09-03 16:16:23.630653", > "mapping_epoch": 6426, > "log_start": "6398'969569", > "ondisk_log_start": "6398'969569", > "created": 2132, > "last_epoch_clean": 6429, > "parent": "0.0", > "parent_split_bits": 0, > "last_scrub": "6429'972567", > "last_scrub_stamp": "2013-09-03 16:15:53.114382", > "last_deep_scrub": "6429'972567", > "last_deep_scrub_stamp": "2013-09-03 16:15:53.114382", > "last_clean_scrub_stamp": "2013-09-03 16:04:57.077937", > "log_size": 0, > "ondisk_log_size": 0, > "stats_invalid": "0", > "stat_sum": { "num_bytes": 571404305, > "num_objects": 78, > "num_object_clones": 0, > "num_object_copies": 0, > "num_objects_missing_on_primary": 0, > "num_objects_degraded": 0, > "num_objects_unfound": 0, > "num_read": 1037151, > "num_read_kb": 1982418, > "num_write": 972739, > "num_write_kb": 12430480, > "num_scrub_errors": 1, > "num_shallow_scrub_errors": 0, > "num_deep_scrub_errors": 1, > "num_objects_recovered": 1320, > "num_bytes_recovered": 10111942741, > "num_keys_recovered": 124}, > "stat_cat_sum": {}, > "up": [ > 3, > 5], > "acting": [ > 3, > 5]}, > "empty": 0, > "dne": 0, > "incomplete": 0, > "last_epoch_started": 6429}, > "recovery_state": [ > { "name": "Started\/Primary\/Active", > "enter_time": "2013-09-03 16:14:59.621528", > "might_have_unfound": [ > { "osd": 5, > "status": "already probed"}], > "recovery_progress": { "backfill_target": -1, > "waiting_on_backfill": 0, > "backfill_pos": "0\/\/0\/\/-1", > "backfill_info": { "begin": "0\/\/0\/\/-1", > "end": "0\/\/0\/\/-1", > "objects": []}, > "peer_backfill_info": { "begin": "0\/\/0\/\/-1", > "end": "0\/\/0\/\/-1", > "objects": []}, > "backfills_in_flight": [], > "pull_from_peer": [], > "pushing": []}, > "scrub": { "scrubber.epoch_start": "6428", > "scrubber.active": 0, > "scrubber.block_writes": 0, > "scrubber.finalizing": 0, > "scrubber.waiting_on": 0, > "scrubber.waiting_on_whom": []}}, > { "name": "Started", > "enter_time": "2013-09-03 16:14:58.192177"}]} > > regards > -- > Maciej Gałkiewicz > Shelly Cloud Sp. z o. o., Sysadmin > http://shellycloud.com/, macias@xxxxxxxxxxxxxxx > KRS: 0000440358 REGON: 101504426 > > _______________________________________________ > ceph-users mailing list > ceph-users@xxxxxxxxxxxxxx > http://lists.ceph.com/listinfo.cgi/ceph-users-ceph.com > _______________________________________________ ceph-users mailing list ceph-users@xxxxxxxxxxxxxx http://lists.ceph.com/listinfo.cgi/ceph-users-ceph.com
 |