Hi,guys
my cluster face a network problem so it occur some error.after solve
network problem.
latency of some osds in one node is high,using ceph osd perf,which come to
3000+
so I delete this osd from cluster,keep osd data device.
after recover and backfill,then I face the problem describe in title.ceph
health detail is :
pg 4.438 is active+recovering+degraded+remapped, acting [7,11], 1 unfound
pg 4.438 is stuck unclean for 135368.626141, current state
active+recovering+degraded+remapped, last acting [7,11]
recovery 1062/4842087 objects degraded (0.022%); 1/2028378 unfound (0.000%)
root@node-67:~# ceph pg map 4.438
osdmap e42522 pg 4.438 (4.438) -> up [34,20,30] acting [7,11]
I can see the pg data in deleted osd.6,which have some different with the
existing osd.7 and osd.11
can I copy the pg data to the new up set? ignore acting set?
some info is below,the output of pg query is the attachment.
root@node-67:~# ceph pg 4.438 list_missing
{ "offset": { "oid": "",
"key": "",
"snapid": 0,
"hash": 0,
"max": 0,
"pool": -1,
"namespace": ""},
"num_missing": 1,
"num_unfound": 1,
"objects": [
{ "oid": { "oid": "rbd_data.188b9163c78e9.00000000000015f2",
"key": "",
"snapid": -2,
"hash": 2427198520,
"max": 0,
"pool": 4,
"namespace": ""},
"need": "39188'2314230",
"have": "39174'2314229",
"locations": []}],
rootceph pg 4.438 mark_unfound_lost revert
Error EINVAL: pg has 1 unfound objects but we haven't probed all sources,
not marking lost
root@node-67:~# ceph pg 4.438 mark_unfound_lost delete
Error EINVAL: pg has 1 unfound objects but we haven't probed all sources,
not marking lost
-------------------------
hnuzhoulin@xxxxxxxxx
{ "state": "active+recovering+degraded+remapped",
"epoch": 42479,
"up": [
34,
20,
30],
"acting": [
7,
11],
"backfill_targets": [
"20",
"30",
"34"],
"actingbackfill": [
"7",
"11",
"20",
"30",
"34"],
"info": { "pgid": "4.438",
"last_update": "42466'2314233",
"last_complete": "39174'2314229",
"log_tail": "36831'2311191",
"last_user_version": 2314233,
"last_backfill": "MAX",
"purged_snaps": "[1~6,8~3,c~2]",
"history": { "epoch_created": 3,
"last_epoch_started": 41203,
"last_epoch_clean": 39193,
"last_epoch_split": 0,
"same_up_since": 41202,
"same_interval_since": 41202,
"same_primary_since": 41080,
"last_scrub": "36831'2232695",
"last_scrub_stamp": "2016-03-12 00:04:39.729981",
"last_deep_scrub": "36831'2232695",
"last_deep_scrub_stamp": "2016-03-12 00:04:39.729981",
"last_clean_scrub_stamp": "2016-03-12 00:04:39.729981"},
"stats": { "version": "42466'2314233",
"reported_seq": "2867271",
"reported_epoch": "42479",
"state": "active+recovering+degraded+remapped",
"last_fresh": "2016-03-19 14:55:31.313305",
"last_change": "2016-03-19 10:45:38.030496",
"last_active": "2016-03-19 14:55:31.313305",
"last_clean": "2016-03-18 07:29:45.298075",
"last_became_active": "0.000000",
"last_unstale": "2016-03-19 14:55:31.313305",
"mapping_epoch": 41195,
"log_start": "36831'2311191",
"ondisk_log_start": "36831'2311191",
"created": 3,
"last_epoch_clean": 39193,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "36831'2232695",
"last_scrub_stamp": "2016-03-12 00:04:39.729981",
"last_deep_scrub": "36831'2232695",
"last_deep_scrub_stamp": "2016-03-12 00:04:39.729981",
"last_clean_scrub_stamp": "2016-03-12 00:04:39.729981",
"log_size": 3042,
"ondisk_log_size": 3042,
"stats_invalid": "0",
"stat_sum": { "num_bytes": 1923686400,
"num_objects": 353,
"num_object_clones": 22,
"num_object_copies": 1765,
"num_objects_missing_on_primary": 1,
"num_objects_degraded": 1061,
"num_objects_unfound": 1,
"num_objects_dirty": 353,
"num_whiteouts": 0,
"num_read": 671585,
"num_read_kb": 6314988,
"num_write": 4628325,
"num_write_kb": 58300392,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 2362,
"num_bytes_recovered": 10920329216,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0},
"stat_cat_sum": {},
"up": [
34,
20,
30],
"acting": [
7,
11],
"up_primary": 34,
"acting_primary": 7},
"empty": 0,
"dne": 0,
"incomplete": 0,
"last_epoch_started": 41203,
"hit_set_history": { "current_last_update": "0'0",
"current_last_stamp": "0.000000",
"current_info": { "begin": "0.000000",
"end": "0.000000",
"version": "0'0"},
"history": []}},
"peer_info": [
{ "peer": "11",
"pgid": "4.438",
"last_update": "42466'2314233",
"last_complete": "0'0",
"log_tail": "36831'2311191",
"last_user_version": 2314232,
"last_backfill": "MAX",
"purged_snaps": "[1~6,8~3,c~2]",
"history": { "epoch_created": 3,
"last_epoch_started": 41203,
"last_epoch_clean": 39193,
"last_epoch_split": 0,
"same_up_since": 41202,
"same_interval_since": 41202,
"same_primary_since": 41080,
"last_scrub": "36831'2232695",
"last_scrub_stamp": "2016-03-12 00:04:39.729981",
"last_deep_scrub": "36831'2232695",
"last_deep_scrub_stamp": "2016-03-12 00:04:39.729981",
"last_clean_scrub_stamp": "2016-03-12 00:04:39.729981"},
"stats": { "version": "40764'2314231",
"reported_seq": "2865717",
"reported_epoch": "40764",
"state": "active+recovery_wait+remapped",
"last_fresh": "2016-03-19 09:31:03.008054",
"last_change": "2016-03-19 09:25:28.584436",
"last_active": "2016-03-19 09:31:03.008054",
"last_clean": "2016-03-18 07:29:45.298075",
"last_became_active": "0.000000",
"last_unstale": "2016-03-19 09:31:03.008054",
"mapping_epoch": 41195,
"log_start": "36831'2311191",
"ondisk_log_start": "36831'2311191",
"created": 3,
"last_epoch_clean": 39193,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "36831'2232695",
"last_scrub_stamp": "2016-03-12 00:04:39.729981",
"last_deep_scrub": "36831'2232695",
"last_deep_scrub_stamp": "2016-03-12 00:04:39.729981",
"last_clean_scrub_stamp": "2016-03-12 00:04:39.729981",
"log_size": 3040,
"ondisk_log_size": 3040,
"stats_invalid": "0",
"stat_sum": { "num_bytes": 1923686400,
"num_objects": 353,
"num_object_clones": 22,
"num_object_copies": 2118,
"num_objects_missing_on_primary": 4,
"num_objects_degraded": 1065,
"num_objects_unfound": 0,
"num_objects_dirty": 353,
"num_whiteouts": 0,
"num_read": 671584,
"num_read_kb": 6314988,
"num_write": 4628323,
"num_write_kb": 58300388,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 2362,
"num_bytes_recovered": 10920329216,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0},
"stat_cat_sum": {},
"up": [
34,
20,
30],
"acting": [
7,
11],
"up_primary": 34,
"acting_primary": 7},
"empty": 0,
"dne": 0,
"incomplete": 0,
"last_epoch_started": 41203,
"hit_set_history": { "current_last_update": "0'0",
"current_last_stamp": "0.000000",
"current_info": { "begin": "0.000000",
"end": "0.000000",
"version": "0'0"},
"history": []}},
{ "peer": "20",
"pgid": "4.438",
"last_update": "42466'2314233",
"last_complete": "42466'2314233",
"log_tail": "36831'2311230",
"last_user_version": 2314232,
"last_backfill": "0\/\/0\/\/-1",
"purged_snaps": "[]",
"history": { "epoch_created": 3,
"last_epoch_started": 41203,
"last_epoch_clean": 39193,
"last_epoch_split": 0,
"same_up_since": 41202,
"same_interval_since": 41202,
"same_primary_since": 41080,
"last_scrub": "36831'2232695",
"last_scrub_stamp": "2016-03-12 00:04:39.729981",
"last_deep_scrub": "36831'2232695",
"last_deep_scrub_stamp": "2016-03-12 00:04:39.729981",
"last_clean_scrub_stamp": "2016-03-12 00:04:39.729981"},
"stats": { "version": "0'0",
"reported_seq": "0",
"reported_epoch": "0",
"state": "inactive",
"last_fresh": "0.000000",
"last_change": "0.000000",
"last_active": "0.000000",
"last_clean": "0.000000",
"last_became_active": "0.000000",
"last_unstale": "0.000000",
"mapping_epoch": 41195,
"log_start": "0'0",
"ondisk_log_start": "0'0",
"created": 0,
"last_epoch_clean": 0,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "0'0",
"last_scrub_stamp": "0.000000",
"last_deep_scrub": "0'0",
"last_deep_scrub_stamp": "0.000000",
"last_clean_scrub_stamp": "0.000000",
"log_size": 0,
"ondisk_log_size": 0,
"stats_invalid": "0",
"stat_sum": { "num_bytes": 0,
"num_objects": 0,
"num_object_clones": 0,
"num_object_copies": 0,
"num_objects_missing_on_primary": 0,
"num_objects_degraded": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 0,
"num_whiteouts": 0,
"num_read": 0,
"num_read_kb": 0,
"num_write": 0,
"num_write_kb": 0,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 0,
"num_bytes_recovered": 0,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0},
"stat_cat_sum": {},
"up": [
34,
20,
30],
"acting": [
7,
11],
"up_primary": 34,
"acting_primary": 7},
"empty": 0,
"dne": 0,
"incomplete": 1,
"last_epoch_started": 41203,
"hit_set_history": { "current_last_update": "0'0",
"current_last_stamp": "0.000000",
"current_info": { "begin": "0.000000",
"end": "0.000000",
"version": "0'0"},
"history": []}},
{ "peer": "30",
"pgid": "4.438",
"last_update": "42466'2314233",
"last_complete": "42466'2314233",
"log_tail": "36831'2311230",
"last_user_version": 2314232,
"last_backfill": "0\/\/0\/\/-1",
"purged_snaps": "[]",
"history": { "epoch_created": 3,
"last_epoch_started": 41203,
"last_epoch_clean": 39193,
"last_epoch_split": 0,
"same_up_since": 41202,
"same_interval_since": 41202,
"same_primary_since": 41080,
"last_scrub": "36831'2232695",
"last_scrub_stamp": "2016-03-12 00:04:39.729981",
"last_deep_scrub": "36831'2232695",
"last_deep_scrub_stamp": "2016-03-12 00:04:39.729981",
"last_clean_scrub_stamp": "2016-03-12 00:04:39.729981"},
"stats": { "version": "0'0",
"reported_seq": "0",
"reported_epoch": "0",
"state": "inactive",
"last_fresh": "0.000000",
"last_change": "0.000000",
"last_active": "0.000000",
"last_clean": "0.000000",
"last_became_active": "0.000000",
"last_unstale": "0.000000",
"mapping_epoch": 41195,
"log_start": "0'0",
"ondisk_log_start": "0'0",
"created": 0,
"last_epoch_clean": 0,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "0'0",
"last_scrub_stamp": "0.000000",
"last_deep_scrub": "0'0",
"last_deep_scrub_stamp": "0.000000",
"last_clean_scrub_stamp": "0.000000",
"log_size": 0,
"ondisk_log_size": 0,
"stats_invalid": "0",
"stat_sum": { "num_bytes": 0,
"num_objects": 0,
"num_object_clones": 0,
"num_object_copies": 0,
"num_objects_missing_on_primary": 0,
"num_objects_degraded": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 0,
"num_whiteouts": 0,
"num_read": 0,
"num_read_kb": 0,
"num_write": 0,
"num_write_kb": 0,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 0,
"num_bytes_recovered": 0,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0},
"stat_cat_sum": {},
"up": [
34,
20,
30],
"acting": [
7,
11],
"up_primary": 34,
"acting_primary": 7},
"empty": 0,
"dne": 0,
"incomplete": 1,
"last_epoch_started": 41203,
"hit_set_history": { "current_last_update": "0'0",
"current_last_stamp": "0.000000",
"current_info": { "begin": "0.000000",
"end": "0.000000",
"version": "0'0"},
"history": []}},
{ "peer": "31",
"pgid": "4.438",
"last_update": "39188'2314230",
"last_complete": "39188'2314230",
"log_tail": "36831'2311230",
"last_user_version": 0,
"last_backfill": "0\/\/0\/\/-1",
"purged_snaps": "[]",
"history": { "epoch_created": 3,
"last_epoch_started": 40707,
"last_epoch_clean": 39193,
"last_epoch_split": 0,
"same_up_since": 41202,
"same_interval_since": 41202,
"same_primary_since": 41080,
"last_scrub": "36831'2232695",
"last_scrub_stamp": "2016-03-12 00:04:39.729981",
"last_deep_scrub": "36831'2232695",
"last_deep_scrub_stamp": "2016-03-12 00:04:39.729981",
"last_clean_scrub_stamp": "2016-03-12 00:04:39.729981"},
"stats": { "version": "0'0",
"reported_seq": "0",
"reported_epoch": "0",
"state": "inactive",
"last_fresh": "0.000000",
"last_change": "0.000000",
"last_active": "0.000000",
"last_clean": "0.000000",
"last_became_active": "0.000000",
"last_unstale": "0.000000",
"mapping_epoch": 41195,
"log_start": "0'0",
"ondisk_log_start": "0'0",
"created": 0,
"last_epoch_clean": 0,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "0'0",
"last_scrub_stamp": "0.000000",
"last_deep_scrub": "0'0",
"last_deep_scrub_stamp": "0.000000",
"last_clean_scrub_stamp": "0.000000",
"log_size": 0,
"ondisk_log_size": 0,
"stats_invalid": "0",
"stat_sum": { "num_bytes": 0,
"num_objects": 0,
"num_object_clones": 0,
"num_object_copies": 0,
"num_objects_missing_on_primary": 0,
"num_objects_degraded": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 0,
"num_whiteouts": 0,
"num_read": 0,
"num_read_kb": 0,
"num_write": 0,
"num_write_kb": 0,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 0,
"num_bytes_recovered": 0,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0},
"stat_cat_sum": {},
"up": [
34,
20,
30],
"acting": [
7,
11],
"up_primary": 34,
"acting_primary": 7},
"empty": 0,
"dne": 0,
"incomplete": 1,
"last_epoch_started": 0,
"hit_set_history": { "current_last_update": "0'0",
"current_last_stamp": "0.000000",
"current_info": { "begin": "0.000000",
"end": "0.000000",
"version": "0'0"},
"history": []}},
{ "peer": "34",
"pgid": "4.438",
"last_update": "42466'2314233",
"last_complete": "42466'2314233",
"log_tail": "36831'2311230",
"last_user_version": 2314232,
"last_backfill": "0\/\/0\/\/-1",
"purged_snaps": "[]",
"history": { "epoch_created": 3,
"last_epoch_started": 41203,
"last_epoch_clean": 39193,
"last_epoch_split": 0,
"same_up_since": 41202,
"same_interval_since": 41202,
"same_primary_since": 41080,
"last_scrub": "36831'2232695",
"last_scrub_stamp": "2016-03-12 00:04:39.729981",
"last_deep_scrub": "36831'2232695",
"last_deep_scrub_stamp": "2016-03-12 00:04:39.729981",
"last_clean_scrub_stamp": "2016-03-12 00:04:39.729981"},
"stats": { "version": "0'0",
"reported_seq": "0",
"reported_epoch": "0",
"state": "inactive",
"last_fresh": "0.000000",
"last_change": "0.000000",
"last_active": "0.000000",
"last_clean": "0.000000",
"last_became_active": "0.000000",
"last_unstale": "0.000000",
"mapping_epoch": 41195,
"log_start": "0'0",
"ondisk_log_start": "0'0",
"created": 0,
"last_epoch_clean": 0,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "0'0",
"last_scrub_stamp": "0.000000",
"last_deep_scrub": "0'0",
"last_deep_scrub_stamp": "0.000000",
"last_clean_scrub_stamp": "0.000000",
"log_size": 0,
"ondisk_log_size": 0,
"stats_invalid": "0",
"stat_sum": { "num_bytes": 0,
"num_objects": 0,
"num_object_clones": 0,
"num_object_copies": 0,
"num_objects_missing_on_primary": 0,
"num_objects_degraded": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 0,
"num_whiteouts": 0,
"num_read": 0,
"num_read_kb": 0,
"num_write": 0,
"num_write_kb": 0,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 0,
"num_bytes_recovered": 0,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0},
"stat_cat_sum": {},
"up": [
34,
20,
30],
"acting": [
7,
11],
"up_primary": 34,
"acting_primary": 7},
"empty": 0,
"dne": 0,
"incomplete": 1,
"last_epoch_started": 41203,
"hit_set_history": { "current_last_update": "0'0",
"current_last_stamp": "0.000000",
"current_info": { "begin": "0.000000",
"end": "0.000000",
"version": "0'0"},
"history": []}}],
"recovery_state": [
{ "name": "Started\/Primary\/Active",
"enter_time": "2016-03-19 10:24:50.186833",
"might_have_unfound": [
{ "osd": "6",
"status": "osd is down"},
{ "osd": "11",
"status": "already probed"},
{ "osd": "20",
"status": "already probed"},
{ "osd": "30",
"status": "already probed"},
{ "osd": "31",
"status": "already probed"},
{ "osd": "34",
"status": "already probed"}],
"recovery_progress": { "backfill_targets": [
"20",
"30",
"34"],
"waiting_on_backfill": [],
"last_backfill_started": "0\/\/0\/\/-1",
"backfill_info": { "begin": "0\/\/0\/\/-1",
"end": "0\/\/0\/\/-1",
"objects": []},
"peer_backfill_info": [],
"backfills_in_flight": [],
"recovering": [],
"pg_backend": { "pull_from_peer": [],
"pushing": []}},
"scrub": { "scrubber.epoch_start": "0",
"scrubber.active": 0,
"scrubber.block_writes": 0,
"scrubber.finalizing": 0,
"scrubber.waiting_on": 0,
"scrubber.waiting_on_whom": []}},
{ "name": "Started",
"enter_time": "2016-03-19 10:24:49.182580"}],
"agent_state": {}}_______________________________________________
ceph-users mailing list
ceph-users@xxxxxxxxxxxxxx
http://lists.ceph.com/listinfo.cgi/ceph-users-ceph.com
