Hi, After some hardware errors one of pg in our backup server is ‘incomplete’. I do export pg without problems like here: https://ceph.com/community/incomplete-pgs-oh-my/ After remove pg from all osd’s and import pg to one of osd pg is still ‘incomplete’. I want to recover only some pice of data from this rbd so if I lost something then nothing happened. How can I tell ceph to accept this pg as complete and clean? >> ceph health detail HEALTH_WARN 1 pgs incomplete; 1 pgs stuck inactive; 1 pgs stuck unclean pg 0.109 is stuck inactive since forever, current state incomplete, last acting [9,13] pg 0.109 is stuck unclean since forever, current state incomplete, last acting [9,13] pg 0.109 is incomplete, acting [9,13] >> Ceph pg 0.109 query In attachment. Regards, Mateusz |
{ "state": "incomplete",
"snap_trimq": "[]",
"epoch": 6310,
"up": [
9,
13],
"acting": [
9,
13],
"info": { "pgid": "0.109",
"last_update": "0'0",
"last_complete": "0'0",
"log_tail": "0'0",
"last_user_version": 0,
"last_backfill": "MAX",
"purged_snaps": "[]",
"history": { "epoch_created": 1,
"last_epoch_started": 4101,
"last_epoch_clean": 4089,
"last_epoch_split": 0,
"same_up_since": 6306,
"same_interval_since": 6306,
"same_primary_since": 6304,
"last_scrub": "3249'1096189",
"last_scrub_stamp": "2015-06-05 14:50:50.378387",
"last_deep_scrub": "3084'1088300",
"last_deep_scrub_stamp": "2015-05-31 13:56:29.394517",
"last_clean_scrub_stamp": "2015-06-05 14:50:50.378387"},
"stats": { "version": "0'0",
"reported_seq": "7",
"reported_epoch": "6310",
"state": "incomplete",
"last_fresh": "2015-06-18 12:54:14.562011",
"last_change": "2015-06-18 12:53:05.172499",
"last_active": "0.000000",
"last_clean": "0.000000",
"last_became_active": "0.000000",
"last_unstale": "2015-06-18 12:54:14.562011",
"last_undegraded": "2015-06-18 12:54:14.562011",
"last_fullsized": "2015-06-18 12:54:14.562011",
"mapping_epoch": 6306,
"log_start": "0'0",
"ondisk_log_start": "0'0",
"created": 1,
"last_epoch_clean": 4089,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "3249'1096189",
"last_scrub_stamp": "2015-06-05 14:50:50.378387",
"last_deep_scrub": "3084'1088300",
"last_deep_scrub_stamp": "2015-05-31 13:56:29.394517",
"last_clean_scrub_stamp": "2015-06-05 14:50:50.378387",
"log_size": 0,
"ondisk_log_size": 0,
"stats_invalid": "0",
"stat_sum": { "num_bytes": 0,
"num_objects": 0,
"num_object_clones": 0,
"num_object_copies": 0,
"num_objects_missing_on_primary": 0,
"num_objects_degraded": 0,
"num_objects_misplaced": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 0,
"num_whiteouts": 0,
"num_read": 0,
"num_read_kb": 0,
"num_write": 0,
"num_write_kb": 0,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 0,
"num_bytes_recovered": 0,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0},
"stat_cat_sum": {},
"up": [
9,
13],
"acting": [
9,
13],
"blocked_by": [],
"up_primary": 9,
"acting_primary": 9},
"empty": 1,
"dne": 0,
"incomplete": 0,
"last_epoch_started": 0,
"hit_set_history": { "current_last_update": "0'0",
"current_last_stamp": "0.000000",
"current_info": { "begin": "0.000000",
"end": "0.000000",
"version": "0'0"},
"history": []}},
"peer_info": [
{ "peer": "2",
"pgid": "0.109",
"last_update": "0'0",
"last_complete": "0'0",
"log_tail": "0'0",
"last_user_version": 0,
"last_backfill": "MAX",
"purged_snaps": "[]",
"history": { "epoch_created": 0,
"last_epoch_started": 0,
"last_epoch_clean": 0,
"last_epoch_split": 0,
"same_up_since": 0,
"same_interval_since": 0,
"same_primary_since": 0,
"last_scrub": "0'0",
"last_scrub_stamp": "0.000000",
"last_deep_scrub": "0'0",
"last_deep_scrub_stamp": "0.000000",
"last_clean_scrub_stamp": "0.000000"},
"stats": { "version": "0'0",
"reported_seq": "0",
"reported_epoch": "0",
"state": "inactive",
"last_fresh": "0.000000",
"last_change": "0.000000",
"last_active": "0.000000",
"last_clean": "0.000000",
"last_became_active": "0.000000",
"last_unstale": "0.000000",
"last_undegraded": "0.000000",
"last_fullsized": "0.000000",
"mapping_epoch": 0,
"log_start": "0'0",
"ondisk_log_start": "0'0",
"created": 0,
"last_epoch_clean": 0,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "0'0",
"last_scrub_stamp": "0.000000",
"last_deep_scrub": "0'0",
"last_deep_scrub_stamp": "0.000000",
"last_clean_scrub_stamp": "0.000000",
"log_size": 0,
"ondisk_log_size": 0,
"stats_invalid": "0",
"stat_sum": { "num_bytes": 0,
"num_objects": 0,
"num_object_clones": 0,
"num_object_copies": 0,
"num_objects_missing_on_primary": 0,
"num_objects_degraded": 0,
"num_objects_misplaced": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 0,
"num_whiteouts": 0,
"num_read": 0,
"num_read_kb": 0,
"num_write": 0,
"num_write_kb": 0,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 0,
"num_bytes_recovered": 0,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0},
"stat_cat_sum": {},
"up": [],
"acting": [],
"blocked_by": [],
"up_primary": -1,
"acting_primary": -1},
"empty": 1,
"dne": 1,
"incomplete": 0,
"last_epoch_started": 0,
"hit_set_history": { "current_last_update": "0'0",
"current_last_stamp": "0.000000",
"current_info": { "begin": "0.000000",
"end": "0.000000",
"version": "0'0"},
"history": []}},
{ "peer": "4",
"pgid": "0.109",
"last_update": "0'0",
"last_complete": "0'0",
"log_tail": "0'0",
"last_user_version": 0,
"last_backfill": "MAX",
"purged_snaps": "[]",
"history": { "epoch_created": 0,
"last_epoch_started": 0,
"last_epoch_clean": 0,
"last_epoch_split": 0,
"same_up_since": 0,
"same_interval_since": 0,
"same_primary_since": 0,
"last_scrub": "0'0",
"last_scrub_stamp": "0.000000",
"last_deep_scrub": "0'0",
"last_deep_scrub_stamp": "0.000000",
"last_clean_scrub_stamp": "0.000000"},
"stats": { "version": "0'0",
"reported_seq": "0",
"reported_epoch": "0",
"state": "inactive",
"last_fresh": "0.000000",
"last_change": "0.000000",
"last_active": "0.000000",
"last_clean": "0.000000",
"last_became_active": "0.000000",
"last_unstale": "0.000000",
"last_undegraded": "0.000000",
"last_fullsized": "0.000000",
"mapping_epoch": 0,
"log_start": "0'0",
"ondisk_log_start": "0'0",
"created": 0,
"last_epoch_clean": 0,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "0'0",
"last_scrub_stamp": "0.000000",
"last_deep_scrub": "0'0",
"last_deep_scrub_stamp": "0.000000",
"last_clean_scrub_stamp": "0.000000",
"log_size": 0,
"ondisk_log_size": 0,
"stats_invalid": "0",
"stat_sum": { "num_bytes": 0,
"num_objects": 0,
"num_object_clones": 0,
"num_object_copies": 0,
"num_objects_missing_on_primary": 0,
"num_objects_degraded": 0,
"num_objects_misplaced": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 0,
"num_whiteouts": 0,
"num_read": 0,
"num_read_kb": 0,
"num_write": 0,
"num_write_kb": 0,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 0,
"num_bytes_recovered": 0,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0},
"stat_cat_sum": {},
"up": [],
"acting": [],
"blocked_by": [],
"up_primary": -1,
"acting_primary": -1},
"empty": 1,
"dne": 1,
"incomplete": 0,
"last_epoch_started": 0,
"hit_set_history": { "current_last_update": "0'0",
"current_last_stamp": "0.000000",
"current_info": { "begin": "0.000000",
"end": "0.000000",
"version": "0'0"},
"history": []}},
{ "peer": "7",
"pgid": "0.109",
"last_update": "0'0",
"last_complete": "0'0",
"log_tail": "0'0",
"last_user_version": 0,
"last_backfill": "MAX",
"purged_snaps": "[]",
"history": { "epoch_created": 0,
"last_epoch_started": 0,
"last_epoch_clean": 0,
"last_epoch_split": 0,
"same_up_since": 0,
"same_interval_since": 0,
"same_primary_since": 0,
"last_scrub": "0'0",
"last_scrub_stamp": "0.000000",
"last_deep_scrub": "0'0",
"last_deep_scrub_stamp": "0.000000",
"last_clean_scrub_stamp": "0.000000"},
"stats": { "version": "0'0",
"reported_seq": "0",
"reported_epoch": "0",
"state": "inactive",
"last_fresh": "0.000000",
"last_change": "0.000000",
"last_active": "0.000000",
"last_clean": "0.000000",
"last_became_active": "0.000000",
"last_unstale": "0.000000",
"last_undegraded": "0.000000",
"last_fullsized": "0.000000",
"mapping_epoch": 0,
"log_start": "0'0",
"ondisk_log_start": "0'0",
"created": 0,
"last_epoch_clean": 0,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "0'0",
"last_scrub_stamp": "0.000000",
"last_deep_scrub": "0'0",
"last_deep_scrub_stamp": "0.000000",
"last_clean_scrub_stamp": "0.000000",
"log_size": 0,
"ondisk_log_size": 0,
"stats_invalid": "0",
"stat_sum": { "num_bytes": 0,
"num_objects": 0,
"num_object_clones": 0,
"num_object_copies": 0,
"num_objects_missing_on_primary": 0,
"num_objects_degraded": 0,
"num_objects_misplaced": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 0,
"num_whiteouts": 0,
"num_read": 0,
"num_read_kb": 0,
"num_write": 0,
"num_write_kb": 0,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 0,
"num_bytes_recovered": 0,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0},
"stat_cat_sum": {},
"up": [],
"acting": [],
"blocked_by": [],
"up_primary": -1,
"acting_primary": -1},
"empty": 1,
"dne": 1,
"incomplete": 0,
"last_epoch_started": 0,
"hit_set_history": { "current_last_update": "0'0",
"current_last_stamp": "0.000000",
"current_info": { "begin": "0.000000",
"end": "0.000000",
"version": "0'0"},
"history": []}},
{ "peer": "13",
"pgid": "0.109",
"last_update": "3933'1096450",
"last_complete": "3933'1096450",
"log_tail": "3099'1093197",
"last_user_version": 1096450,
"last_backfill": "MAX",
"purged_snaps": "[]",
"history": { "epoch_created": 1,
"last_epoch_started": 4101,
"last_epoch_clean": 4089,
"last_epoch_split": 0,
"same_up_since": 6306,
"same_interval_since": 6306,
"same_primary_since": 6304,
"last_scrub": "3249'1096189",
"last_scrub_stamp": "2015-06-05 14:50:50.378387",
"last_deep_scrub": "3084'1088300",
"last_deep_scrub_stamp": "2015-05-31 13:56:29.394517",
"last_clean_scrub_stamp": "2015-06-05 14:50:50.378387"},
"stats": { "version": "3933'1096450",
"reported_seq": "3898664",
"reported_epoch": "6297",
"state": "down+peering",
"last_fresh": "2015-06-18 12:53:04.990905",
"last_change": "2015-06-18 12:53:04.990905",
"last_active": "2015-06-07 20:02:45.980123",
"last_clean": "2015-06-07 20:02:45.980123",
"last_became_active": "0.000000",
"last_unstale": "2015-06-18 12:53:04.990905",
"last_undegraded": "2015-06-18 12:53:04.990905",
"last_fullsized": "2015-06-18 12:53:04.990905",
"mapping_epoch": 6304,
"log_start": "3099'1093197",
"ondisk_log_start": "3099'1093197",
"created": 1,
"last_epoch_clean": 4089,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "3249'1096189",
"last_scrub_stamp": "2015-06-05 14:50:50.378387",
"last_deep_scrub": "3084'1088300",
"last_deep_scrub_stamp": "2015-05-31 13:56:29.394517",
"last_clean_scrub_stamp": "2015-06-05 14:50:50.378387",
"log_size": 3253,
"ondisk_log_size": 3253,
"stats_invalid": "0",
"stat_sum": { "num_bytes": 19001393152,
"num_objects": 4547,
"num_object_clones": 0,
"num_object_copies": 9094,
"num_objects_missing_on_primary": 0,
"num_objects_degraded": 0,
"num_objects_misplaced": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 4547,
"num_whiteouts": 0,
"num_read": 1836023,
"num_read_kb": 27544496,
"num_write": 1865919,
"num_write_kb": 36504180,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 9804,
"num_bytes_recovered": 40980742144,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0},
"stat_cat_sum": {},
"up": [
9,
13],
"acting": [
9,
13],
"blocked_by": [
2,
4,
7],
"up_primary": 9,
"acting_primary": 9},
"empty": 0,
"dne": 0,
"incomplete": 0,
"last_epoch_started": 4065,
"hit_set_history": { "current_last_update": "0'0",
"current_last_stamp": "0.000000",
"current_info": { "begin": "0.000000",
"end": "0.000000",
"version": "0'0"},
"history": []}}],
"recovery_state": [
{ "name": "Started\/Primary\/Peering",
"enter_time": "2015-06-18 12:53:05.169236",
"past_intervals": [
{ "first": 4064,
"last": 4087,
"maybe_went_rw": 1,
"up": [
13,
7],
"acting": [
13,
7],
"primary": 13,
"up_primary": 13},
{ "first": 4088,
"last": 4094,
"maybe_went_rw": 1,
"up": [
7],
"acting": [
7],
"primary": 7,
"up_primary": 7},
{ "first": 4095,
"last": 4099,
"maybe_went_rw": 1,
"up": [
7,
4],
"acting": [
7,
4],
"primary": 7,
"up_primary": 7},
{ "first": 4100,
"last": 4194,
"maybe_went_rw": 1,
"up": [
7,
4],
"acting": [
7],
"primary": 7,
"up_primary": 7},
{ "first": 4195,
"last": 4201,
"maybe_went_rw": 1,
"up": [
4],
"acting": [
4],
"primary": 4,
"up_primary": 4},
{ "first": 4202,
"last": 4269,
"maybe_went_rw": 1,
"up": [
4,
2],
"acting": [
4,
2],
"primary": 4,
"up_primary": 4},
{ "first": 4270,
"last": 4287,
"maybe_went_rw": 1,
"up": [
7,
4],
"acting": [
7,
4],
"primary": 7,
"up_primary": 7},
{ "first": 4288,
"last": 4330,
"maybe_went_rw": 1,
"up": [
9,
7],
"acting": [
9,
7],
"primary": 9,
"up_primary": 9},
{ "first": 4331,
"last": 5203,
"maybe_went_rw": 1,
"up": [
9,
13],
"acting": [
9,
13],
"primary": 9,
"up_primary": 9},
{ "first": 5204,
"last": 5233,
"maybe_went_rw": 1,
"up": [
13],
"acting": [
13],
"primary": 13,
"up_primary": 13},
{ "first": 5234,
"last": 5464,
"maybe_went_rw": 1,
"up": [
13,
7],
"acting": [
13,
7],
"primary": 13,
"up_primary": 13},
{ "first": 5465,
"last": 5757,
"maybe_went_rw": 1,
"up": [
9,
13],
"acting": [
9,
13],
"primary": 9,
"up_primary": 9},
{ "first": 5758,
"last": 5760,
"maybe_went_rw": 1,
"up": [
9],
"acting": [
9],
"primary": 9,
"up_primary": 9},
{ "first": 5761,
"last": 5762,
"maybe_went_rw": 0,
"up": [],
"acting": [],
"primary": -1,
"up_primary": -1},
{ "first": 5763,
"last": 5764,
"maybe_went_rw": 1,
"up": [
9],
"acting": [
9],
"primary": 9,
"up_primary": 9},
{ "first": 5765,
"last": 5767,
"maybe_went_rw": 1,
"up": [
9,
13],
"acting": [
9,
13],
"primary": 9,
"up_primary": 9},
{ "first": 5768,
"last": 5769,
"maybe_went_rw": 1,
"up": [
9],
"acting": [
9],
"primary": 9,
"up_primary": 9},
{ "first": 5770,
"last": 5771,
"maybe_went_rw": 1,
"up": [
9,
13],
"acting": [
9,
13],
"primary": 9,
"up_primary": 9},
{ "first": 5772,
"last": 5774,
"maybe_went_rw": 1,
"up": [
13],
"acting": [
13],
"primary": 13,
"up_primary": 13},
{ "first": 5775,
"last": 5792,
"maybe_went_rw": 1,
"up": [
9,
13],
"acting": [
9,
13],
"primary": 9,
"up_primary": 9},
{ "first": 5793,
"last": 5794,
"maybe_went_rw": 1,
"up": [
13],
"acting": [
13],
"primary": 13,
"up_primary": 13},
{ "first": 5795,
"last": 5796,
"maybe_went_rw": 0,
"up": [],
"acting": [],
"primary": -1,
"up_primary": -1},
{ "first": 5797,
"last": 5798,
"maybe_went_rw": 1,
"up": [
9],
"acting": [
9],
"primary": 9,
"up_primary": 9},
{ "first": 5799,
"last": 5803,
"maybe_went_rw": 1,
"up": [
9,
13],
"acting": [
9,
13],
"primary": 9,
"up_primary": 9},
{ "first": 5804,
"last": 5805,
"maybe_went_rw": 1,
"up": [
9],
"acting": [
9],
"primary": 9,
"up_primary": 9},
{ "first": 5806,
"last": 5807,
"maybe_went_rw": 1,
"up": [
9,
13],
"acting": [
9,
13],
"primary": 9,
"up_primary": 9},
{ "first": 5808,
"last": 5809,
"maybe_went_rw": 1,
"up": [
9],
"acting": [
9],
"primary": 9,
"up_primary": 9},
{ "first": 5810,
"last": 5812,
"maybe_went_rw": 0,
"up": [],
"acting": [],
"primary": -1,
"up_primary": -1},
{ "first": 5813,
"last": 5815,
"maybe_went_rw": 1,
"up": [
9],
"acting": [
9],
"primary": 9,
"up_primary": 9},
{ "first": 5816,
"last": 5929,
"maybe_went_rw": 1,
"up": [
9,
7],
"acting": [
9,
7],
"primary": 9,
"up_primary": 9},
{ "first": 5930,
"last": 5934,
"maybe_went_rw": 1,
"up": [
7],
"acting": [
7],
"primary": 7,
"up_primary": 7},
{ "first": 5935,
"last": 6060,
"maybe_went_rw": 1,
"up": [
9,
7],
"acting": [
9,
7],
"primary": 9,
"up_primary": 9},
{ "first": 6061,
"last": 6062,
"maybe_went_rw": 1,
"up": [
9],
"acting": [
9],
"primary": 9,
"up_primary": 9},
{ "first": 6063,
"last": 6073,
"maybe_went_rw": 1,
"up": [
9,
7],
"acting": [
9,
7],
"primary": 9,
"up_primary": 9},
{ "first": 6074,
"last": 6077,
"maybe_went_rw": 1,
"up": [
9,
13],
"acting": [
9,
13],
"primary": 9,
"up_primary": 9},
{ "first": 6078,
"last": 6126,
"maybe_went_rw": 1,
"up": [
9,
7],
"acting": [
9,
7],
"primary": 9,
"up_primary": 9},
{ "first": 6127,
"last": 6295,
"maybe_went_rw": 1,
"up": [
9,
13],
"acting": [
9,
13],
"primary": 9,
"up_primary": 9},
{ "first": 6296,
"last": 6297,
"maybe_went_rw": 1,
"up": [
13],
"acting": [
13],
"primary": 13,
"up_primary": 13},
{ "first": 6298,
"last": 6303,
"maybe_went_rw": 0,
"up": [],
"acting": [],
"primary": -1,
"up_primary": -1},
{ "first": 6304,
"last": 6305,
"maybe_went_rw": 1,
"up": [
9],
"acting": [
9],
"primary": 9,
"up_primary": 9}],
"probing_osds": [
"2",
"4",
"7",
"9",
"13"],
"down_osds_we_would_probe": [],
"peering_blocked_by": []},
{ "name": "Started",
"enter_time": "2015-06-18 12:53:05.169150"}],
"agent_state": {}}_______________________________________________ ceph-users mailing list ceph-users@xxxxxxxxxxxxxx http://lists.ceph.com/listinfo.cgi/ceph-users-ceph.com
