On 06. okt. 2016 13:41, Ronny Aasen wrote:
hello
I have a few osd's in my cluster that are regularly crashing.
[snip]
ofcourse having 3 osd's dying regularly is not good for my health. so i
have set noout, to avoid heavy recoveries.
googeling this error messages gives exactly 1 hit:
https://github.com/ceph/ceph/pull/6946
where it saies: "the shard must be removed so it can be reconstructed"
but with my 3 osd's failing, i am not certain witch of them contain the
broken shard. (or perhaps all 3 of them?)
a bit reluctant to delete on all 3. I have 4+2 erasure coding.
( erasure size 6 min_size 4 ) so finding out witch one is bad would be
nice.
hope someone have an idea how to progress.
kind regards
Ronny Aasen
i again have this problem with crashing osd's. a more detailed log is on
the tail of this mail.
Does anyone have any suggestions on how i can identify what shard that
needs to be removed to allow the EC to recover. ?
and more importantly how i can stop the osd's from crashing?
kind regards
Ronny Aasen
-- query of pg in question --
# ceph pg 5.26 query
{
"state": "active+undersized+degraded+remapped+wait_backfill",
"snap_trimq": "[]",
"epoch": 138744,
"up": [
27,
109,
2147483647,
2147483647,
62,
75
],
"acting": [
2147483647,
2147483647,
32,
107,
62,
38
],
"backfill_targets": [
"27(0)",
"75(5)",
"109(1)"
],
"actingbackfill": [
"27(0)",
"32(2)",
"38(5)",
"62(4)",
"75(5)",
"107(3)",
"109(1)"
],
"info": {
"pgid": "5.26s2",
"last_update": "84093'35622",
"last_complete": "84093'35622",
"log_tail": "82361'32622",
"last_user_version": 0,
"last_backfill": "MAX",
"purged_snaps": "[1~7]",
"history": {
"epoch_created": 61149,
"last_epoch_started": 138692,
"last_epoch_clean": 136567,
"last_epoch_split": 0,
"same_up_since": 138691,
"same_interval_since": 138691,
"same_primary_since": 138691,
"last_scrub": "84093'35622",
"last_scrub_stamp": "2016-10-18 06:18:28.253508",
"last_deep_scrub": "84093'35622",
"last_deep_scrub_stamp": "2016-10-14 05:33:56.701167",
"last_clean_scrub_stamp": "2016-10-14 05:33:56.701167"
},
"stats": {
"version": "84093'35622",
"reported_seq": "210475",
"reported_epoch": "138730",
"state": "active+undersized+degraded+remapped+wait_backfill",
"last_fresh": "2016-10-19 12:40:32.982617",
"last_change": "2016-10-19 12:03:29.377914",
"last_active": "2016-10-19 12:40:32.982617",
"last_peered": "2016-10-19 12:40:32.982617",
"last_clean": "2016-07-19 12:03:54.814292",
"last_became_active": "0.000000",
"last_became_peered": "0.000000",
"last_unstale": "2016-10-19 12:40:32.982617",
"last_undegraded": "2016-10-19 12:02:03.030755",
"last_fullsized": "2016-10-19 12:02:03.030755",
"mapping_epoch": 138627,
"log_start": "82361'32622",
"ondisk_log_start": "82361'32622",
"created": 61149,
"last_epoch_clean": 136567,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "84093'35622",
"last_scrub_stamp": "2016-10-18 06:18:28.253508",
"last_deep_scrub": "84093'35622",
"last_deep_scrub_stamp": "2016-10-14 05:33:56.701167",
"last_clean_scrub_stamp": "2016-10-14 05:33:56.701167",
"log_size": 3000,
"ondisk_log_size": 3000,
"stats_invalid": "0",
"stat_sum": {
"num_bytes": 99736657920,
"num_objects": 12026,
"num_object_clones": 0,
"num_object_copies": 84182,
"num_objects_missing_on_primary": 0,
"num_objects_degraded": 24052,
"num_objects_misplaced": 90583,
"num_objects_unfound": 0,
"num_objects_dirty": 12026,
"num_whiteouts": 0,
"num_read": 86122,
"num_read_kb": 9446184,
"num_write": 35622,
"num_write_kb": 182277312,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 0,
"num_bytes_recovered": 0,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0
},
"up": [
27,
109,
2147483647,
2147483647,
62,
75
],
"acting": [
2147483647,
2147483647,
32,
107,
62,
38
],
"blocked_by": [],
"up_primary": 27,
"acting_primary": 32
},
"empty": 0,
"dne": 0,
"incomplete": 0,
"last_epoch_started": 138692,
"hit_set_history": {
"current_last_update": "0'0",
"current_last_stamp": "0.000000",
"current_info": {
"begin": "0.000000",
"end": "0.000000",
"version": "0'0",
"using_gmt": "1"
},
"history": []
}
},
"peer_info": [
{
"peer": "27(0)",
"pgid": "5.26s0",
"last_update": "84093'35622",
"last_complete": "84093'35622",
"log_tail": "82361'32622",
"last_user_version": 0,
"last_backfill":
"5\/7b5e6226\/rbd_data.3a3938238e1f29.00000000001e94e9\/head",
"purged_snaps": "[1~7]",
"history": {
"epoch_created": 61149,
"last_epoch_started": 138692,
"last_epoch_clean": 136567,
"last_epoch_split": 0,
"same_up_since": 138691,
"same_interval_since": 138691,
"same_primary_since": 138691,
"last_scrub": "84093'35622",
"last_scrub_stamp": "2016-10-18 06:18:28.253508",
"last_deep_scrub": "84093'35622",
"last_deep_scrub_stamp": "2016-10-14 05:33:56.701167",
"last_clean_scrub_stamp": "2016-10-14 05:33:56.701167"
},
"stats": {
"version": "0'0",
"reported_seq": "0",
"reported_epoch": "0",
"state": "inactive",
"last_fresh": "0.000000",
"last_change": "0.000000",
"last_active": "0.000000",
"last_peered": "0.000000",
"last_clean": "0.000000",
"last_became_active": "0.000000",
"last_became_peered": "0.000000",
"last_unstale": "0.000000",
"last_undegraded": "0.000000",
"last_fullsized": "0.000000",
"mapping_epoch": 138627,
"log_start": "0'0",
"ondisk_log_start": "0'0",
"created": 0,
"last_epoch_clean": 0,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "0'0",
"last_scrub_stamp": "0.000000",
"last_deep_scrub": "0'0",
"last_deep_scrub_stamp": "0.000000",
"last_clean_scrub_stamp": "0.000000",
"log_size": 0,
"ondisk_log_size": 0,
"stats_invalid": "0",
"stat_sum": {
"num_bytes": 15566770176,
"num_objects": 1875,
"num_object_clones": 0,
"num_object_copies": 0,
"num_objects_missing_on_primary": 0,
"num_objects_degraded": 0,
"num_objects_misplaced": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 1875,
"num_whiteouts": 0,
"num_read": 0,
"num_read_kb": 0,
"num_write": 0,
"num_write_kb": 0,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 0,
"num_bytes_recovered": 0,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0
},
"up": [
27,
109,
2147483647,
2147483647,
62,
75
],
"acting": [
2147483647,
2147483647,
32,
107,
62,
38
],
"blocked_by": [],
"up_primary": 27,
"acting_primary": 32
},
"empty": 0,
"dne": 0,
"incomplete": 1,
"last_epoch_started": 138692,
"hit_set_history": {
"current_last_update": "0'0",
"current_last_stamp": "0.000000",
"current_info": {
"begin": "0.000000",
"end": "0.000000",
"version": "0'0",
"using_gmt": "1"
},
"history": []
}
},
{
"peer": "38(3)",
"pgid": "5.26s3",
"last_update": "84093'35622",
"last_complete": "84093'35622",
"log_tail": "82361'32622",
"last_user_version": 0,
"last_backfill": "-1\/0\/\/0",
"purged_snaps": "[1~7]",
"history": {
"epoch_created": 61149,
"last_epoch_started": 136617,
"last_epoch_clean": 136567,
"last_epoch_split": 0,
"same_up_since": 138691,
"same_interval_since": 138691,
"same_primary_since": 138691,
"last_scrub": "84093'35622",
"last_scrub_stamp": "2016-10-18 06:18:28.253508",
"last_deep_scrub": "84093'35622",
"last_deep_scrub_stamp": "2016-10-14 05:33:56.701167",
"last_clean_scrub_stamp": "2016-10-14 05:33:56.701167"
},
"stats": {
"version": "0'0",
"reported_seq": "0",
"reported_epoch": "0",
"state": "inactive",
"last_fresh": "0.000000",
"last_change": "0.000000",
"last_active": "0.000000",
"last_peered": "0.000000",
"last_clean": "0.000000",
"last_became_active": "0.000000",
"last_became_peered": "0.000000",
"last_unstale": "0.000000",
"last_undegraded": "0.000000",
"last_fullsized": "0.000000",
"mapping_epoch": 138627,
"log_start": "0'0",
"ondisk_log_start": "0'0",
"created": 0,
"last_epoch_clean": 0,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "0'0",
"last_scrub_stamp": "0.000000",
"last_deep_scrub": "0'0",
"last_deep_scrub_stamp": "0.000000",
"last_clean_scrub_stamp": "0.000000",
"log_size": 0,
"ondisk_log_size": 0,
"stats_invalid": "0",
"stat_sum": {
"num_bytes": 0,
"num_objects": 0,
"num_object_clones": 0,
"num_object_copies": 0,
"num_objects_missing_on_primary": 0,
"num_objects_degraded": 0,
"num_objects_misplaced": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 0,
"num_whiteouts": 0,
"num_read": 0,
"num_read_kb": 0,
"num_write": 0,
"num_write_kb": 0,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 0,
"num_bytes_recovered": 0,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0
},
"up": [
27,
109,
2147483647,
2147483647,
62,
75
],
"acting": [
2147483647,
2147483647,
32,
107,
62,
38
],
"blocked_by": [],
"up_primary": 27,
"acting_primary": 32
},
"empty": 0,
"dne": 0,
"incomplete": 1,
"last_epoch_started": 136617,
"hit_set_history": {
"current_last_update": "0'0",
"current_last_stamp": "0.000000",
"current_info": {
"begin": "0.000000",
"end": "0.000000",
"version": "0'0",
"using_gmt": "1"
},
"history": []
}
},
{
"peer": "38(5)",
"pgid": "5.26s5",
"last_update": "84093'35622",
"last_complete": "84093'35622",
"log_tail": "82361'32622",
"last_user_version": 0,
"last_backfill": "MAX",
"purged_snaps": "[1~7]",
"history": {
"epoch_created": 61149,
"last_epoch_started": 138692,
"last_epoch_clean": 136567,
"last_epoch_split": 0,
"same_up_since": 138691,
"same_interval_since": 138691,
"same_primary_since": 138691,
"last_scrub": "84093'35622",
"last_scrub_stamp": "2016-10-18 06:18:28.253508",
"last_deep_scrub": "84093'35622",
"last_deep_scrub_stamp": "2016-10-14 05:33:56.701167",
"last_clean_scrub_stamp": "2016-10-14 05:33:56.701167"
},
"stats": {
"version": "84093'35622",
"reported_seq": "210384",
"reported_epoch": "136129",
"state": "active+remapped+backfilling",
"last_fresh": "2016-10-16 22:40:02.720542",
"last_change": "2016-10-16 20:11:41.616308",
"last_active": "2016-10-16 22:40:02.720542",
"last_peered": "2016-10-16 22:40:02.720542",
"last_clean": "2016-07-19 12:03:54.814292",
"last_became_active": "0.000000",
"last_became_peered": "0.000000",
"last_unstale": "2016-10-16 22:40:02.720542",
"last_undegraded": "2016-10-16 22:40:02.720542",
"last_fullsized": "2016-10-16 22:40:02.720542",
"mapping_epoch": 138627,
"log_start": "82361'32622",
"ondisk_log_start": "82361'32622",
"created": 61149,
"last_epoch_clean": 135426,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "84093'35622",
"last_scrub_stamp": "2016-10-14 05:33:56.701167",
"last_deep_scrub": "84093'35622",
"last_deep_scrub_stamp": "2016-10-14 05:33:56.701167",
"last_clean_scrub_stamp": "2016-10-14 05:33:56.701167",
"log_size": 3000,
"ondisk_log_size": 3000,
"stats_invalid": "0",
"stat_sum": {
"num_bytes": 99736657920,
"num_objects": 12026,
"num_object_clones": 0,
"num_object_copies": 120260,
"num_objects_missing_on_primary": 0,
"num_objects_degraded": 0,
"num_objects_misplaced": 48120,
"num_objects_unfound": 0,
"num_objects_dirty": 12026,
"num_whiteouts": 0,
"num_read": 86122,
"num_read_kb": 9446184,
"num_write": 35622,
"num_write_kb": 182277312,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 0,
"num_bytes_recovered": 0,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0
},
"up": [
27,
109,
2147483647,
2147483647,
62,
75
],
"acting": [
2147483647,
2147483647,
32,
107,
62,
38
],
"blocked_by": [],
"up_primary": 27,
"acting_primary": 32
},
"empty": 0,
"dne": 0,
"incomplete": 0,
"last_epoch_started": 138692,
"hit_set_history": {
"current_last_update": "0'0",
"current_last_stamp": "0.000000",
"current_info": {
"begin": "0.000000",
"end": "0.000000",
"version": "0'0",
"using_gmt": "1"
},
"history": []
}
},
{
"peer": "62(4)",
"pgid": "5.26s4",
"last_update": "84093'35622",
"last_complete": "84093'35622",
"log_tail": "82361'32582",
"last_user_version": 384692,
"last_backfill": "MAX",
"purged_snaps": "[1~7]",
"history": {
"epoch_created": 61149,
"last_epoch_started": 138692,
"last_epoch_clean": 136567,
"last_epoch_split": 0,
"same_up_since": 138691,
"same_interval_since": 138691,
"same_primary_since": 138691,
"last_scrub": "84093'35622",
"last_scrub_stamp": "2016-10-18 06:18:28.253508",
"last_deep_scrub": "84093'35622",
"last_deep_scrub_stamp": "2016-10-14 05:33:56.701167",
"last_clean_scrub_stamp": "2016-10-14 05:33:56.701167"
},
"stats": {
"version": "84093'35620",
"reported_seq": "210463",
"reported_epoch": "138692",
"state": "active+undersized+degraded",
"last_fresh": "2016-07-21 08:54:40.103916",
"last_change": "2016-07-21 08:54:39.880548",
"last_active": "2016-07-21 08:54:40.103916",
"last_peered": "2016-07-21 08:54:40.103916",
"last_clean": "2016-07-19 12:03:54.814292",
"last_became_active": "0.000000",
"last_became_peered": "0.000000",
"last_unstale": "2016-07-21 08:54:40.103916",
"last_undegraded": "2016-07-21 08:54:37.733453",
"last_fullsized": "2016-07-21 08:54:37.733453",
"mapping_epoch": 138627,
"log_start": "82361'32582",
"ondisk_log_start": "82361'32582",
"created": 61149,
"last_epoch_clean": 84093,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "67261'27554",
"last_scrub_stamp": "2016-06-21 05:49:33.802327",
"last_deep_scrub": "67261'27554",
"last_deep_scrub_stamp": "2016-06-21 05:49:33.802327",
"last_clean_scrub_stamp": "2016-06-21 05:49:33.802327",
"log_size": 3038,
"ondisk_log_size": 3038,
"stats_invalid": "0",
"stat_sum": {
"num_bytes": 99736657920,
"num_objects": 12026,
"num_object_clones": 0,
"num_object_copies": 72150,
"num_objects_missing_on_primary": 0,
"num_objects_degraded": 12025,
"num_objects_misplaced": 12025,
"num_objects_unfound": 0,
"num_objects_dirty": 12026,
"num_whiteouts": 0,
"num_read": 86122,
"num_read_kb": 9446184,
"num_write": 35622,
"num_write_kb": 182277312,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 0,
"num_bytes_recovered": 0,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0
},
"up": [
27,
109,
2147483647,
2147483647,
62,
75
],
"acting": [
2147483647,
2147483647,
32,
107,
62,
38
],
"blocked_by": [],
"up_primary": 27,
"acting_primary": 32
},
"empty": 0,
"dne": 0,
"incomplete": 0,
"last_epoch_started": 138692,
"hit_set_history": {
"current_last_update": "0'0",
"current_last_stamp": "0.000000",
"current_info": {
"begin": "0.000000",
"end": "0.000000",
"version": "0'0",
"using_gmt": "1"
},
"history": []
}
},
{
"peer": "75(5)",
"pgid": "5.26s5",
"last_update": "84093'35622",
"last_complete": "84093'35622",
"log_tail": "82361'32622",
"last_user_version": 0,
"last_backfill":
"5\/7b5e6226\/rbd_data.3a3938238e1f29.00000000001e94e9\/head",
"purged_snaps": "[1~7]",
"history": {
"epoch_created": 61149,
"last_epoch_started": 138692,
"last_epoch_clean": 136567,
"last_epoch_split": 0,
"same_up_since": 138691,
"same_interval_since": 138691,
"same_primary_since": 138691,
"last_scrub": "84093'35622",
"last_scrub_stamp": "2016-10-18 06:18:28.253508",
"last_deep_scrub": "84093'35622",
"last_deep_scrub_stamp": "2016-10-14 05:33:56.701167",
"last_clean_scrub_stamp": "2016-10-14 05:33:56.701167"
},
"stats": {
"version": "0'0",
"reported_seq": "0",
"reported_epoch": "0",
"state": "inactive",
"last_fresh": "0.000000",
"last_change": "0.000000",
"last_active": "0.000000",
"last_peered": "0.000000",
"last_clean": "0.000000",
"last_became_active": "0.000000",
"last_became_peered": "0.000000",
"last_unstale": "0.000000",
"last_undegraded": "0.000000",
"last_fullsized": "0.000000",
"mapping_epoch": 138627,
"log_start": "0'0",
"ondisk_log_start": "0'0",
"created": 0,
"last_epoch_clean": 0,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "0'0",
"last_scrub_stamp": "0.000000",
"last_deep_scrub": "0'0",
"last_deep_scrub_stamp": "0.000000",
"last_clean_scrub_stamp": "0.000000",
"log_size": 0,
"ondisk_log_size": 0,
"stats_invalid": "0",
"stat_sum": {
"num_bytes": 15566770176,
"num_objects": 1875,
"num_object_clones": 0,
"num_object_copies": 0,
"num_objects_missing_on_primary": 0,
"num_objects_degraded": 0,
"num_objects_misplaced": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 1875,
"num_whiteouts": 0,
"num_read": 0,
"num_read_kb": 0,
"num_write": 0,
"num_write_kb": 0,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 0,
"num_bytes_recovered": 0,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0
},
"up": [
27,
109,
2147483647,
2147483647,
62,
75
],
"acting": [
2147483647,
2147483647,
32,
107,
62,
38
],
"blocked_by": [],
"up_primary": 27,
"acting_primary": 32
},
"empty": 0,
"dne": 0,
"incomplete": 1,
"last_epoch_started": 138692,
"hit_set_history": {
"current_last_update": "0'0",
"current_last_stamp": "0.000000",
"current_info": {
"begin": "0.000000",
"end": "0.000000",
"version": "0'0",
"using_gmt": "1"
},
"history": []
}
},
{
"peer": "107(3)",
"pgid": "5.26s3",
"last_update": "84093'35622",
"last_complete": "84093'35622",
"log_tail": "82361'32622",
"last_user_version": 0,
"last_backfill": "MAX",
"purged_snaps": "[1~7]",
"history": {
"epoch_created": 61149,
"last_epoch_started": 138692,
"last_epoch_clean": 136567,
"last_epoch_split": 0,
"same_up_since": 138691,
"same_interval_since": 138691,
"same_primary_since": 138691,
"last_scrub": "84093'35622",
"last_scrub_stamp": "2016-10-18 06:18:28.253508",
"last_deep_scrub": "84093'35622",
"last_deep_scrub_stamp": "2016-10-14 05:33:56.701167",
"last_clean_scrub_stamp": "2016-10-14 05:33:56.701167"
},
"stats": {
"version": "84093'35622",
"reported_seq": "210394",
"reported_epoch": "138136",
"state": "down+remapped+peering",
"last_fresh": "2016-10-19 07:37:44.345580",
"last_change": "2016-10-19 06:44:35.544182",
"last_active": "2016-10-16 22:40:02.720542",
"last_peered": "2016-10-16 22:40:02.720542",
"last_clean": "2016-07-19 12:03:54.814292",
"last_became_active": "0.000000",
"last_became_peered": "0.000000",
"last_unstale": "2016-10-19 07:37:44.345580",
"last_undegraded": "2016-10-19 07:37:44.345580",
"last_fullsized": "2016-10-19 07:37:44.345580",
"mapping_epoch": 138627,
"log_start": "82361'32622",
"ondisk_log_start": "82361'32622",
"created": 61149,
"last_epoch_clean": 136567,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "84093'35622",
"last_scrub_stamp": "2016-10-18 06:18:28.253508",
"last_deep_scrub": "84093'35622",
"last_deep_scrub_stamp": "2016-10-14 05:33:56.701167",
"last_clean_scrub_stamp": "2016-10-14 05:33:56.701167",
"log_size": 3000,
"ondisk_log_size": 3000,
"stats_invalid": "0",
"stat_sum": {
"num_bytes": 99736657920,
"num_objects": 12026,
"num_object_clones": 0,
"num_object_copies": 72156,
"num_objects_missing_on_primary": 0,
"num_objects_degraded": 0,
"num_objects_misplaced": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 12026,
"num_whiteouts": 0,
"num_read": 86122,
"num_read_kb": 9446184,
"num_write": 35622,
"num_write_kb": 182277312,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 0,
"num_bytes_recovered": 0,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0
},
"up": [
27,
109,
2147483647,
2147483647,
62,
75
],
"acting": [
2147483647,
2147483647,
32,
107,
62,
38
],
"blocked_by": [],
"up_primary": 27,
"acting_primary": 32
},
"empty": 0,
"dne": 0,
"incomplete": 0,
"last_epoch_started": 138692,
"hit_set_history": {
"current_last_update": "0'0",
"current_last_stamp": "0.000000",
"current_info": {
"begin": "0.000000",
"end": "0.000000",
"version": "0'0",
"using_gmt": "1"
},
"history": []
}
},
{
"peer": "109(1)",
"pgid": "5.26s1",
"last_update": "84093'35622",
"last_complete": "84093'35622",
"log_tail": "82361'32622",
"last_user_version": 0,
"last_backfill":
"5\/7b5e6226\/rbd_data.3a3938238e1f29.00000000001e94e9\/head",
"purged_snaps": "[1~7]",
"history": {
"epoch_created": 61149,
"last_epoch_started": 138692,
"last_epoch_clean": 136567,
"last_epoch_split": 0,
"same_up_since": 138691,
"same_interval_since": 138691,
"same_primary_since": 138691,
"last_scrub": "84093'35622",
"last_scrub_stamp": "2016-10-18 06:18:28.253508",
"last_deep_scrub": "84093'35622",
"last_deep_scrub_stamp": "2016-10-14 05:33:56.701167",
"last_clean_scrub_stamp": "2016-10-14 05:33:56.701167"
},
"stats": {
"version": "0'0",
"reported_seq": "0",
"reported_epoch": "0",
"state": "inactive",
"last_fresh": "0.000000",
"last_change": "0.000000",
"last_active": "0.000000",
"last_peered": "0.000000",
"last_clean": "0.000000",
"last_became_active": "0.000000",
"last_became_peered": "0.000000",
"last_unstale": "0.000000",
"last_undegraded": "0.000000",
"last_fullsized": "0.000000",
"mapping_epoch": 138627,
"log_start": "0'0",
"ondisk_log_start": "0'0",
"created": 0,
"last_epoch_clean": 0,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "0'0",
"last_scrub_stamp": "0.000000",
"last_deep_scrub": "0'0",
"last_deep_scrub_stamp": "0.000000",
"last_clean_scrub_stamp": "0.000000",
"log_size": 0,
"ondisk_log_size": 0,
"stats_invalid": "0",
"stat_sum": {
"num_bytes": 15566770176,
"num_objects": 1875,
"num_object_clones": 0,
"num_object_copies": 0,
"num_objects_missing_on_primary": 0,
"num_objects_degraded": 0,
"num_objects_misplaced": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 1875,
"num_whiteouts": 0,
"num_read": 0,
"num_read_kb": 0,
"num_write": 0,
"num_write_kb": 0,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 0,
"num_bytes_recovered": 0,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0
},
"up": [
27,
109,
2147483647,
2147483647,
62,
75
],
"acting": [
2147483647,
2147483647,
32,
107,
62,
38
],
"blocked_by": [],
"up_primary": 27,
"acting_primary": 32
},
"empty": 0,
"dne": 0,
"incomplete": 1,
"last_epoch_started": 138692,
"hit_set_history": {
"current_last_update": "0'0",
"current_last_stamp": "0.000000",
"current_info": {
"begin": "0.000000",
"end": "0.000000",
"version": "0'0",
"using_gmt": "1"
},
"history": []
}
}
],
"recovery_state": [
{
"name": "Started\/Primary\/Active",
"enter_time": "2016-10-19 12:02:03.031149",
"might_have_unfound": [],
"recovery_progress": {
"backfill_targets": [
"27(0)",
"75(5)",
"109(1)"
],
"waiting_on_backfill": [],
"last_backfill_started":
"5\/7b5e6226\/rbd_data.3a3938238e1f29.00000000001e94e9\/head",
"backfill_info": {
"begin": "-1\/0\/\/0",
"end": "-1\/0\/\/0",
"objects": []
},
"peer_backfill_info": [],
"backfills_in_flight": [],
"recovering": [],
"pg_backend": {
"recovery_ops": [],
"read_ops": []
}
},
"scrub": {
"scrubber.epoch_start": "0",
"scrubber.active": 0,
"scrubber.waiting_on": 0,
"scrubber.waiting_on_whom": []
}
},
{
"name": "Started",
"enter_time": "2016-10-19 12:02:01.684000"
}
],
"agent_state": {}
}
--log of one osd--
-150> 2016-10-19 11:20:22.245891 7f019182c700 10 osd.30 pg_epoch:
138618 pg[5.26s0( v 84093'35622 (82361'32622,84093'35622]
local-les=138613 n=12026 ec=61149 les/c 138613/136567
138612/138612/138448) [27,109,22,30,62,75]/[30,22,32,107,62,2147483647]
r=0 lpr=138612 pi=136565-138611/23 rops=5
bft=22(2),27(0),30(3),75(5),109(1) crt=0'0 lcod 0'0 mlcod 0'0
active+undersized+degraded+remapped+backfilling] handle_sub_read_reply:
reply ECSubReadReply(tid=1, attrs_read=0)
-149> 2016-10-19 11:20:22.245929 7f019182c700 10 osd.30 pg_epoch:
138618 pg[5.26s0( v 84093'35622 (82361'32622,84093'35622]
local-les=138613 n=12026 ec=61149 les/c 138613/136567
138612/138612/138448) [27,109,22,30,62,75]/[30,22,32,107,62,2147483647]
r=0 lpr=138612 pi=136565-138611/23 rops=5
bft=22(2),27(0),30(3),75(5),109(1) crt=0'0 lcod 0'0 mlcod 0'0
active+undersized+degraded+remapped+backfilling] handle_sub_read_reply
readop not complete: ReadOp(tid=1,
to_read={5/9a2f6226/rbd_data.3a3938238e1f29.00000000001e81d8/head=read_request_t(to_read=[0,8388608,0],
need=22(1),30(0),32(2),107(3),
want_attrs=1),5/169f6226/rbd_data.3a3938238e1f29.00000000001b2a67/head=read_request_t(to_read=[0,8388608,0],
need=22(1),30(0),32(2),107(3),
want_attrs=1),5/25107226/rbd_data.3a3938238e1f29.00000000000b92bd/head=read_request_t(to_read=[0,8388608,0],
need=22(1),30(0),32(2),107(3),
want_attrs=1),5/46207226/rbd_data.3a3938238e1f29.00000000002d80ca/head=read_request_t(to_read=[0,8388608,0],
need=22(1),30(0),32(2),107(3),
want_attrs=1),5/1b907226/rbd_data.3a3938238e1f29.00000000002963de/head=read_request_t(to_read=[0,8388608,0],
need=22(1),30(0),32(2),107(3), want_attrs=1)},
complete={5/9a2f6226/rbd_data.3a3938238e1f29.00000000001e81d8/head=read_result_t(r=0,
errors={}, attrs={_=buffer::list(len=275,
buffer::ptr(10486388~275 0x3b064274 in raw 0x3a664000 len
10488660 nref 20)
),hinfo_key=buffer::list(len=42,
buffer::ptr(10486680~42 0x3b064398 in raw 0x3a664000 len
10488660 nref 20)
),snapset=buffer::list(len=31,
buffer::ptr(10486737~31 0x3b0643d1 in raw 0x3a664000 len
10488660 nref 20)
)}, returned=(0, 8388608, [22(1),2097152, 30(0),2097152,
107(3),2097152]),5/169f6226/rbd_data.3a3938238e1f29.00000000001b2a67/head=read_result_t(r=0,
errors={}, attrs={_=buffer::list(len=275,
buffer::ptr(10486860~275 0x3b06444c in raw 0x3a664000 len
10488660 nref 20)
),hinfo_key=buffer::list(len=42,
buffer::ptr(10487152~42 0x3b064570 in raw 0x3a664000 len
10488660 nref 20)
),snapset=buffer::list(len=31,
buffer::ptr(10487209~31 0x3b0645a9 in raw 0x3a664000 len
10488660 nref 20)
)}, returned=(0, 8388608, [22(1),2097152, 30(0),2097152,
107(3),2097152]),5/25107226/rbd_data.3a3938238e1f29.00000000000b92bd/head=read_result_t(r=0,
errors={}, attrs={_=buffer::list(len=275,
buffer::ptr(10487332~275 0x3b064624 in raw 0x3a664000 len
10488660 nref 20)
),hinfo_key=buffer::list(len=42,
buffer::ptr(10487624~42 0x3b064748 in raw 0x3a664000 len
10488660 nref 20)
),snapset=buffer::list(len=31,
buffer::ptr(10487681~31 0x3b064781 in raw 0x3a664000 len
10488660 nref 20)
)}, returned=(0, 8388608, [22(1),2097152, 30(0),2097152,
107(3),2097152]),5/46207226/rbd_data.3a3938238e1f29.00000000002d80ca/head=read_result_t(r=0,
errors={}, attrs={_=buffer::list(len=275,
buffer::ptr(10487804~275 0x3b0647fc in raw 0x3a664000 len
10488660 nref 20)
),hinfo_key=buffer::list(len=42,
buffer::ptr(10488096~42 0x3b064920 in raw 0x3a664000 len
10488660 nref 20)
),snapset=buffer::list(len=31,
buffer::ptr(10488153~31 0x3b064959 in raw 0x3a664000 len
10488660 nref 20)
)}, returned=(0, 8388608, [22(1),2097152, 30(0),2097152,
107(3),2097152]),5/1b907226/rbd_data.3a3938238e1f29.00000000002963de/head=read_result_t(r=0,
errors={}, attrs={_=buffer::list(len=275,
buffer::ptr(10488276~275 0x3b0649d4 in raw 0x3a664000 len
10488660 nref 20)
),hinfo_key=buffer::list(len=42,
buffer::ptr(10488568~42 0x3b064af8 in raw 0x3a664000 len
10488660 nref 20)
),snapset=buffer::list(len=31,
buffer::ptr(10488625~31 0x3b064b31 in raw 0x3a664000 len
10488660 nref 20)
)}, returned=(0, 8388608, [22(1),2097152, 30(0),2097152,
107(3),2097152])}, priority=1,
obj_to_source={5/9a2f6226/rbd_data.3a3938238e1f29.00000000001e81d8/head=22(1),30(0),32(2),107(3),5/169f6226/rbd_data.3a3938238e1f29.00000000001b2a67/head=22(1),30(0),32(2),107(3),5/25107226/rbd_data.3a3938238e1f29.00000000000b92bd/head=22(1),30(0),32(2),107(3),5/46207226/rbd_data.3a3938238e1f29.00000000002d80ca/head=22(1),30(0),32(2),107(3),5/1b907226/rbd_data.3a3938238e1f29.00000000002963de/head=22(1),30(0),32(2),107(3)},
source_to_obj={22(1)=5/9a2f6226/rbd_data.3a3938238e1f29.00000000001e81d8/head,5/169f6226/rbd_data.3a3938238e1f29.00000000001b2a67/head,5/25107226/rbd_data.3a3938238e1f29.00000000000b92bd/head,5/46207226/rbd_data.3a3938238e1f29.00000000002d80ca/head,5/1b907226/rbd_data.3a3938238e1f29.00000000002963de/head,30(0)=5/9a2f6226/rbd_data.3a3938238e1f29.00000000001e81d8/head,5/169f6226/rbd_data.3a3938238e1f29.00000000001b2a67/head,5/25107226/rbd_data.3a3938238e1f29.00000000000b92bd/head,5/46207226/rbd_data.3a3938238e1f29.00000000002d80ca/head,5/1b907226/rbd_data.3a3938238e1f29.00000000002963de/head,32(2)=5/9a2f6226/rbd_data.3a3938238e1f29.00000000001e81d8/head,5/169f6226/rbd_data.3a3938238e1f29.00000000001b2a67/head,5/25107226/rbd_data.3a3938238e1f29.00000000000b92bd/head,5/46207226/rbd_data.3a3938238e1f29.00000000002d80ca/head,5/1b907226/rbd_data.3a3938238e1f29.00000000002963de/head,107(3)=5/9a2f6226/rbd_data.3a3938238e1f29.00000000001e81d8/head,5/169f6226/rbd_data.3a3938238e1f29.000
00000001b2a67/head,5/25107226/rbd_data.3a3938238e1f29.00000000000b92bd/head,5/46207226/rbd_data.3a3938238e1f29.00000000002d80ca/head,5/1b907226/rbd_data.3a3938238e1f29.00000000002963de/head},
in_progress=32(2))
-148> 2016-10-19 11:20:22.246044 7f019182c700 5
filestore(/var/lib/ceph/osd/ceph-30) queue_transactions existing
osr(5.26s0 0x1288fd00)/0x1288fd00
-147> 2016-10-19 11:20:22.246061 7f019182c700 5
filestore(/var/lib/ceph/osd/ceph-30) queue_transactions (writeahead)
13466211 0x16200c40
-146> 2016-10-19 11:20:22.246082 7f019182c700 10 osd.30 138618
dequeue_op 0x1d452c00 finish
-145> 2016-10-19 11:20:22.246088 7f019182c700 5 -- op tracker --
seq: 12580, time: 2016-10-19 11:20:22.246088, event: done, op:
MOSDECSubOpReadReply(5.26s0 138617 ECSubReadReply(tid=1, attrs_read=0))
-144> 2016-10-19 11:20:22.252966 7f01a495b700 5
filestore(/var/lib/ceph/osd/ceph-30) _journaled_ahead 0x1cb92490 seq
13466207 osr(5.26s0 0x1288fd00) 0x1cc2c700
-143> 2016-10-19 11:20:22.252989 7f01a495b700 5
filestore(/var/lib/ceph/osd/ceph-30) queue_op 0x1cb92490 seq 13466207
osr(5.26s0 0x1288fd00) 32 bytes (queue has 6 ops and 111729 bytes)
-142> 2016-10-19 11:20:22.253003 7f01a495b700 10
filestore(/var/lib/ceph/osd/ceph-30) queueing ondisk 0x24e9fca0
-141> 2016-10-19 11:20:22.253012 7f01a3959700 5
filestore(/var/lib/ceph/osd/ceph-30) _do_op 0x1cb92490 seq 13466207
osr(5.26s0 0x1288fd00)/0x1288fd00 start
-140> 2016-10-19 11:20:22.253021 7f01a3959700 10
filestore(/var/lib/ceph/osd/ceph-30) _do_transaction on 0x1cc2c700
-139> 2016-10-19 11:20:22.253026 7f01a3959700 10
filestore(/var/lib/ceph/osd/ceph-30) _do_op 0x1cb92490 seq 13466207 r =
0, finisher 0x24e9ce80 0
-138> 2016-10-19 11:20:22.253030 7f01a3959700 10
filestore(/var/lib/ceph/osd/ceph-30) _finish_op 0x1cb92490 seq 13466207
osr(5.26s0 0x1288fd00)/0x1288fd00
-137> 2016-10-19 11:20:22.253011 7f01a495b700 5
filestore(/var/lib/ceph/osd/ceph-30) _journaled_ahead 0x1ca23250 seq
13466208 osr(5.26s0 0x1288fd00) 0x2b019b80
-136> 2016-10-19 11:20:22.253048 7f01a495b700 5
filestore(/var/lib/ceph/osd/ceph-30) queue_op 0x1ca23250 seq 13466208
osr(5.26s0 0x1288fd00) 32 bytes (queue has 5 ops and 111697 bytes)
-135> 2016-10-19 11:20:22.253055 7f01a495b700 10
filestore(/var/lib/ceph/osd/ceph-30) queueing ondisk 0x2d414200
-134> 2016-10-19 11:20:22.253060 7f01a3959700 5
filestore(/var/lib/ceph/osd/ceph-30) _do_op 0x1ca23250 seq 13466208
osr(5.26s0 0x1288fd00)/0x1288fd00 start
-133> 2016-10-19 11:20:22.253067 7f01a3959700 10
filestore(/var/lib/ceph/osd/ceph-30) _do_transaction on 0x2b019b80
-132> 2016-10-19 11:20:22.253070 7f01a3959700 10
filestore(/var/lib/ceph/osd/ceph-30) _do_op 0x1ca23250 seq 13466208 r =
0, finisher 0x2d415b40 0
-131> 2016-10-19 11:20:22.253072 7f01a3959700 10
filestore(/var/lib/ceph/osd/ceph-30) _finish_op 0x1ca23250 seq 13466208
osr(5.26s0 0x1288fd00)/0x1288fd00
-130> 2016-10-19 11:20:22.260485 7f019903b700 1 --
10.24.11.25:6811/14492 <== osd.37 10.24.12.25:0/32309 1900 ====
osd_ping(ping e138618 stamp 2016-10-19 11:20:22.259983) v2 ==== 47+0+0
(1564552750 0 0) 0x23765000 con 0x1fc1b9c0
-129> 2016-10-19 11:20:22.260520 7f019903b700 1 --
10.24.11.25:6811/14492 --> 10.24.12.25:0/32309 -- osd_ping(ping_reply
e138618 stamp 2016-10-19 11:20:22.259983) v2 -- ?+0 0x2b4b8600 con
0x1fc1b9c0
-128> 2016-10-19 11:20:22.260497 7f0197838700 1 --
10.24.12.25:6801/14492 <== osd.37 10.24.12.25:0/32309 1900 ====
osd_ping(ping e138618 stamp 2016-10-19 11:20:22.259983) v2 ==== 47+0+0
(1564552750 0 0) 0x252a9e00 con 0x1dae9ce0
-127> 2016-10-19 11:20:22.260536 7f019903b700 10 osd.30 138618
note_peer_epoch osd.37 has 138618
-126> 2016-10-19 11:20:22.260596 7f0197838700 1 --
10.24.12.25:6801/14492 --> 10.24.12.25:0/32309 -- osd_ping(ping_reply
e138618 stamp 2016-10-19 11:20:22.259983) v2 -- ?+0 0x25082400 con
0x1dae9ce0
-125> 2016-10-19 11:20:22.289145 7f01a495b700 5
filestore(/var/lib/ceph/osd/ceph-30) _journaled_ahead 0x1ca23340 seq
13466209 osr(5.26s0 0x1288fd00) 0x2b019100
-124> 2016-10-19 11:20:22.289164 7f01a495b700 5
filestore(/var/lib/ceph/osd/ceph-30) queue_op 0x1ca23340 seq 13466209
osr(5.26s0 0x1288fd00) 32 bytes (queue has 4 ops and 111665 bytes)
-123> 2016-10-19 11:20:22.289174 7f01a495b700 10
filestore(/var/lib/ceph/osd/ceph-30) queueing ondisk 0x2d415ae0
-122> 2016-10-19 11:20:22.289183 7f01a495b700 5
filestore(/var/lib/ceph/osd/ceph-30) _journaled_ahead 0x1cbfe170 seq
13466210 osr(5.45s4 0x3d94220) 0x1c4d8840
-121> 2016-10-19 11:20:22.289192 7f01a495b700 5
filestore(/var/lib/ceph/osd/ceph-30) queue_op 0x1cbfe170 seq 13466210
osr(5.45s4 0x3d94220) 1021 bytes (queue has 4 ops and 111665 bytes)
-120> 2016-10-19 11:20:22.289221 7f01a3959700 5
filestore(/var/lib/ceph/osd/ceph-30) _do_op 0x1ca23340 seq 13466209
osr(5.26s0 0x1288fd00)/0x1288fd00 start
-119> 2016-10-19 11:20:22.289230 7f01a3959700 10
filestore(/var/lib/ceph/osd/ceph-30) _do_transaction on 0x2b019100
-118> 2016-10-19 11:20:22.289235 7f01a3959700 10
filestore(/var/lib/ceph/osd/ceph-30) _do_op 0x1ca23340 seq 13466209 r =
0, finisher 0x2d415c40 0
-117> 2016-10-19 11:20:22.289239 7f01a3959700 10
filestore(/var/lib/ceph/osd/ceph-30) _finish_op 0x1ca23340 seq 13466209
osr(5.26s0 0x1288fd00)/0x1288fd00
-116> 2016-10-19 11:20:22.289248 7f01a3959700 5
filestore(/var/lib/ceph/osd/ceph-30) _do_op 0x1cbfe170 seq 13466210
osr(5.45s4 0x3d94220)/0x3d94220 start
-115> 2016-10-19 11:20:22.289251 7f01a3959700 10
filestore(/var/lib/ceph/osd/ceph-30) _do_transaction on 0x1c4d8840
-114> 2016-10-19 11:20:22.289261 7f01a3959700 15
filestore(/var/lib/ceph/osd/ceph-30) _omap_setkeys
5.45s4_head/5/45//head/18446744073709551615/4
-113> 2016-10-19 11:20:22.289315 7f01a3959700 10 filestore oid:
5/45//head/18446744073709551615/4 not skipping op, *spos 13466210.0.0
-112> 2016-10-19 11:20:22.289320 7f01a3959700 10 filestore >
header.spos 0.0.0
-111> 2016-10-19 11:20:22.289364 7f01a3959700 10
filestore(/var/lib/ceph/osd/ceph-30) _do_op 0x1cbfe170 seq 13466210 r =
0, finisher 0x1c9ceb30 0
-110> 2016-10-19 11:20:22.289368 7f01a3959700 10
filestore(/var/lib/ceph/osd/ceph-30) _finish_op 0x1cbfe170 seq 13466210
osr(5.45s4 0x3d94220)/0x3d94220
-109> 2016-10-19 11:20:22.289519 7f01a415a700 10
filestore(/var/lib/ceph/osd/ceph-30) write
meta/-1/281b090b/inc_osdmap.138618/0 0~203 = 203
-108> 2016-10-19 11:20:22.289546 7f01a415a700 15
filestore(/var/lib/ceph/osd/ceph-30) write
meta/-1/df5656d4/osdmap.138618/0 0~109464
-107> 2016-10-19 11:20:22.297611 7f01a495b700 5
filestore(/var/lib/ceph/osd/ceph-30) _journaled_ahead 0x1ca23160 seq
13466211 osr(5.26s0 0x1288fd00) 0x16200c40
-106> 2016-10-19 11:20:22.297633 7f01a495b700 5
filestore(/var/lib/ceph/osd/ceph-30) queue_op 0x1ca23160 seq 13466211
osr(5.26s0 0x1288fd00) 32 bytes (queue has 2 ops and 110612 bytes)
-105> 2016-10-19 11:20:22.297651 7f01a495b700 10
filestore(/var/lib/ceph/osd/ceph-30) queueing ondisk 0x2337b220
-104> 2016-10-19 11:20:22.297740 7f01a3959700 5
filestore(/var/lib/ceph/osd/ceph-30) _do_op 0x1ca23160 seq 13466211
osr(5.26s0 0x1288fd00)/0x1288fd00 start
-103> 2016-10-19 11:20:22.297760 7f01a3959700 10
filestore(/var/lib/ceph/osd/ceph-30) _do_transaction on 0x16200c40
-102> 2016-10-19 11:20:22.297769 7f01a3959700 10
filestore(/var/lib/ceph/osd/ceph-30) _do_op 0x1ca23160 seq 13466211 r =
0, finisher 0x23379aa0 0
-101> 2016-10-19 11:20:22.297776 7f01a3959700 10
filestore(/var/lib/ceph/osd/ceph-30) _finish_op 0x1ca23160 seq 13466211
osr(5.26s0 0x1288fd00)/0x1288fd00
-100> 2016-10-19 11:20:22.315558 7f019b840700 1 --
10.24.12.25:6800/14492 <== osd.32 10.24.12.23:6806/15481 349 ====
osd_map(138618..138618 src has 89976..138618) v3 ==== 243+0+0
(3207388896 0 0) 0x21c2c6c0 con 0x1d76ac00
-99> 2016-10-19 11:20:22.315601 7f019b840700 10 osd.30 138618
do_waiters -- start
-98> 2016-10-19 11:20:22.315607 7f019b840700 10 osd.30 138618
do_waiters -- finish
-97> 2016-10-19 11:20:22.315621 7f019b840700 3 osd.30 138618
handle_osd_map epochs [138618,138618], i have 138618, src has [89976,138618]
-96> 2016-10-19 11:20:22.315626 7f019b840700 10 osd.30 138618 no
new maps here, dropping
-95> 2016-10-19 11:20:22.315629 7f019b840700 10 osd.30 138618
do_waiters -- start
-94> 2016-10-19 11:20:22.315630 7f019b840700 10 osd.30 138618
do_waiters -- finish
-93> 2016-10-19 11:20:22.328048 7f01867ec700 1 --
10.24.12.25:6800/14492 <== osd.32 10.24.12.23:6806/15481 350 ====
MOSDECSubOpReadReply(5.26s0 138617 ECSubReadReply(tid=1, attrs_read=0))
v1 ==== 8389148+0+0 (3765373542 0 0) 0x2165d680 con 0x1d76ac00
-92> 2016-10-19 11:20:22.328094 7f01867ec700 5 -- op tracker --
seq: 12581, time: 2016-10-19 11:20:22.317251, event: header_read, op:
MOSDECSubOpReadReply(5.26s0 138617 ECSubReadReply(tid=1, attrs_read=0))
-91> 2016-10-19 11:20:22.328118 7f01867ec700 5 -- op tracker --
seq: 12581, time: 2016-10-19 11:20:22.317253, event: throttled, op:
MOSDECSubOpReadReply(5.26s0 138617 ECSubReadReply(tid=1, attrs_read=0))
-90> 2016-10-19 11:20:22.328130 7f01867ec700 5 -- op tracker --
seq: 12581, time: 2016-10-19 11:20:22.328015, event: all_read, op:
MOSDECSubOpReadReply(5.26s0 138617 ECSubReadReply(tid=1, attrs_read=0))
-89> 2016-10-19 11:20:22.328145 7f01867ec700 5 -- op tracker --
seq: 12581, time: 0.000000, event: dispatched, op:
MOSDECSubOpReadReply(5.26s0 138617 ECSubReadReply(tid=1, attrs_read=0))
-88> 2016-10-19 11:20:22.328163 7f01867ec700 10 osd.30 138618
handle_replica_op MOSDECSubOpReadReply(5.26s0 138617
ECSubReadReply(tid=1, attrs_read=0)) v1 epoch 138617
-87> 2016-10-19 11:20:22.328181 7f01867ec700 10 osd.30 138618
osd.32 10.24.12.23:6806/15481 has old map 138617 < 138618
-86> 2016-10-19 11:20:22.328197 7f01867ec700 15 osd.30 138618
enqueue_op 0x2ee07100 prio 127 cost 0 latency 0.010945
MOSDECSubOpReadReply(5.26s0 138617 ECSubReadReply(tid=1, attrs_read=0)) v1
-85> 2016-10-19 11:20:22.328334 7f018f027700 10 osd.30 138618
dequeue_op 0x2ee07100 prio 127 cost 0 latency 0.011082
MOSDECSubOpReadReply(5.26s0 138617 ECSubReadReply(tid=1, attrs_read=0))
v1 pg pg[5.26s0( v 84093'35622 (82361'32622,84093'35622]
local-les=138613 n=12026 ec=61149 les/c 138613/136567
138612/138612/138448) [27,109,22,30,62,75]/[30,22,32,107,62,2147483647]
r=0 lpr=138612 pi=136565-138611/23 rops=5
bft=22(2),27(0),30(3),75(5),109(1) crt=0'0 lcod 0'0 mlcod 0'0
active+undersized+degraded+remapped+backfilling]
-84> 2016-10-19 11:20:22.328380 7f018f027700 10 osd.30 138618
osd.32 10.24.12.23:6806/15481 has old map 138617 < 138618
-83> 2016-10-19 11:20:22.328389 7f018f027700 10 osd.30 138618
osd.32 10.24.12.23:6806/15481 has old map 138617 < 138618
-82> 2016-10-19 11:20:22.328394 7f018f027700 10 osd.30 138618
note_peer_epoch osd.32 has 138618
-81> 2016-10-19 11:20:22.328396 7f018f027700 10 osd.30 138618
send_incremental_map 138617 -> 138618 to 0x1d76ac00 10.24.12.23:6806/15481
-80> 2016-10-19 11:20:22.328433 7f018f027700 1 --
10.24.12.25:6800/14492 --> 10.24.12.23:6806/15481 --
osd_map(138618..138618 src has 89976..138618) v3 -- ?+0 0x21a4c1c0 con
0x1d76ac00
-79> 2016-10-19 11:20:22.328458 7f018f027700 5 -- op tracker --
seq: 12581, time: 2016-10-19 11:20:22.328458, event: reached_pg, op:
MOSDECSubOpReadReply(5.26s0 138617 ECSubReadReply(tid=1, attrs_read=0))
-78> 2016-10-19 11:20:22.328470 7f018f027700 10 osd.30 pg_epoch:
138618 pg[5.26s0( v 84093'35622 (82361'32622,84093'35622]
local-les=138613 n=12026 ec=61149 les/c 138613/136567
138612/138612/138448) [27,109,22,30,62,75]/[30,22,32,107,62,2147483647]
r=0 lpr=138612 pi=136565-138611/23 rops=5
bft=22(2),27(0),30(3),75(5),109(1) crt=0'0 lcod 0'0 mlcod 0'0
active+undersized+degraded+remapped+backfilling] handle_message:
MOSDECSubOpReadReply(5.26s0 138617 ECSubReadReply(tid=1, attrs_read=0)) v1
-77> 2016-10-19 11:20:22.328496 7f018f027700 10 osd.30 pg_epoch:
138618 pg[5.26s0( v 84093'35622 (82361'32622,84093'35622]
local-les=138613 n=12026 ec=61149 les/c 138613/136567
138612/138612/138448) [27,109,22,30,62,75]/[30,22,32,107,62,2147483647]
r=0 lpr=138612 pi=136565-138611/23 rops=5
bft=22(2),27(0),30(3),75(5),109(1) crt=0'0 lcod 0'0 mlcod 0'0
active+undersized+degraded+remapped+backfilling] handle_sub_read_reply:
reply ECSubReadReply(tid=1, attrs_read=0)
-76> 2016-10-19 11:20:22.328542 7f018f027700 10 osd.30 pg_epoch:
138618 pg[5.26s0( v 84093'35622 (82361'32622,84093'35622]
local-les=138613 n=12026 ec=61149 les/c 138613/136567
138612/138612/138448) [27,109,22,30,62,75]/[30,22,32,107,62,2147483647]
r=0 lpr=138612 pi=136565-138611/23 rops=5
bft=22(2),27(0),30(3),75(5),109(1) crt=0'0 lcod 0'0 mlcod 0'0
active+undersized+degraded+remapped+backfilling] handle_sub_read_reply
readop complete: ReadOp(tid=1,
to_read={5/9a2f6226/rbd_data.3a3938238e1f29.00000000001e81d8/head=read_request_t(to_read=[0,8388608,0],
need=22(1),30(0),32(2),107(3),
want_attrs=1),5/169f6226/rbd_data.3a3938238e1f29.00000000001b2a67/head=read_request_t(to_read=[0,8388608,0],
need=22(1),30(0),32(2),107(3),
want_attrs=1),5/25107226/rbd_data.3a3938238e1f29.00000000000b92bd/head=read_request_t(to_read=[0,8388608,0],
need=22(1),30(0),32(2),107(3),
want_attrs=1),5/46207226/rbd_data.3a3938238e1f29.00000000002d80ca/head=read_request_t(to_read=[0,8388608,0],
need=22(1),30(0),32(2),107(3),
want_attrs=1),5/1b907226/rbd_data.3a3938238e1f29.00000000002963de/head=read_request_t(to_read=[0,8388608,0],
need=22(1),30(0),32(2),107(3), want_attrs=1)},
complete={5/9a2f6226/rbd_data.3a3938238e1f29.00000000001e81d8/head=read_result_t(r=0,
errors={}, attrs={_=buffer::list(len=275,
buffer::ptr(10486388~275 0x3b064274 in raw 0x3a664000 len
10488660 nref 20)
),hinfo_key=buffer::list(len=42,
buffer::ptr(10486680~42 0x3b064398 in raw 0x3a664000 len
10488660 nref 20)
),snapset=buffer::list(len=31,
buffer::ptr(10486737~31 0x3b0643d1 in raw 0x3a664000 len
10488660 nref 20)
)}, returned=(0, 8388608, [22(1),2097152, 30(0),2097152, 32(2),2097152,
107(3),2097152]),5/169f6226/rbd_data.3a3938238e1f29.00000000001b2a67/head=read_result_t(r=0,
errors={}, attrs={_=buffer::list(len=275,
buffer::ptr(10486860~275 0x3b06444c in raw 0x3a664000 len
10488660 nref 20)
),hinfo_key=buffer::list(len=42,
buffer::ptr(10487152~42 0x3b064570 in raw 0x3a664000 len
10488660 nref 20)
),snapset=buffer::list(len=31,
buffer::ptr(10487209~31 0x3b0645a9 in raw 0x3a664000 len
10488660 nref 20)
)}, returned=(0, 8388608, [22(1),2097152, 30(0),2097152, 32(2),2097152,
107(3),2097152]),5/25107226/rbd_data.3a3938238e1f29.00000000000b92bd/head=read_result_t(r=0,
errors={}, attrs={_=buffer::list(len=275,
buffer::ptr(10487332~275 0x3b064624 in raw 0x3a664000 len
10488660 nref 20)
),hinfo_key=buffer::list(len=42,
buffer::ptr(10487624~42 0x3b064748 in raw 0x3a664000 len
10488660 nref 20)
),snapset=buffer::list(len=31,
buffer::ptr(10487681~31 0x3b064781 in raw 0x3a664000 len
10488660 nref 20)
)}, returned=(0, 8388608, [22(1),2097152, 30(0),2097152, 32(2),2097152,
107(3),2097152]),5/46207226/rbd_data.3a3938238e1f29.00000000002d80ca/head=read_result_t(r=0,
errors={}, attrs={_=buffer::list(len=275,
buffer::ptr(10487804~275 0x3b0647fc in raw 0x3a664000 len
10488660 nref 20)
),hinfo_key=buffer::list(len=42,
buffer::ptr(10488096~42 0x3b064920 in raw 0x3a664000 len
10488660 nref 20)
),snapset=buffer::list(len=31,
buffer::ptr(10488153~31 0x3b064959 in raw 0x3a664000 len
10488660 nref 20)
)}, returned=(0, 8388608, [22(1),2097152, 30(0),2097152, 32(2),0,
107(3),2097152]),5/1b907226/rbd_data.3a3938238e1f29.00000000002963de/head=read_result_t(r=0,
errors={}, attrs={_=buffer::list(len=275,
buffer::ptr(10488276~275 0x3b0649d4 in raw 0x3a664000 len
10488660 nref 20)
),hinfo_key=buffer::list(len=42,
buffer::ptr(10488568~42 0x3b064af8 in raw 0x3a664000 len
10488660 nref 20)
),snapset=buffer::list(len=31,
buffer::ptr(10488625~31 0x3b064b31 in raw 0x3a664000 len
10488660 nref 20)
)}, returned=(0, 8388608, [22(1),2097152, 30(0),2097152, 32(2),2097152,
107(3),2097152])}, priority=1,
obj_to_source={5/9a2f6226/rbd_data.3a3938238e1f29.00000000001e81d8/head=22(1),30(0),32(2),107(3),5/169f6226/rbd_data.3a3938238e1f29.00000000001b2a67/head=22(1),30(0),32(2),107(3),5/25107226/rbd_data.3a3938238e1f29.00000000000b92bd/head=22(1),30(0),32(2),107(3),5/46207226/rbd_data.3a3938238e1f29.00000000002d80ca/head=22(1),30(0),32(2),107(3),5/1b907226/rbd_data.3a3938238e1f29.00000000002963de/head=22(1),30(0),32(2),107(3)},
source_to_obj={22(1)=5/9a2f6226/rbd_data.3a3938238e1f29.00000000001e81d8/head,5/169f6226/rbd_data.3a3938238e1f29.00000000001b2a67/head,5/25107226/rbd_data.3a3938238e1f29.00000000000b92bd/head,5/46207226/rbd_data.3a3938238e1f29.00000000002d80ca/head,5/1b907226/rbd_data.3a3938238e1f29.00000000002963de/head,30(0)=5/9a2f6226/rbd_data.3a3938238e1f29.00000000001e81d8/head,5/169f6226/rbd_data.3a3938238e1f29.00000000001b2a67/head,5/25107226/rbd_data.3a3938238e1f29.00000000000b92bd/head,5/46207226/rbd_data.3a3938238e1f29.00000000002d80ca/head,5/1b907226/rbd_data.3a3938238e1f29.00000000002963de/head,32(2)=5/9a2f6226/rbd_data.3a3938238e1f29.00000000001e81d8/head,5/169f6226/rbd_data.3a3938238e1f29.00000000001b2a67/head,5/25107226/rbd_data.3a3938238e1f29.00000000000b92bd/head,5/46207226/rbd_data.3a3938238e1f29.00000000002d80ca/head,5/1b907226/rbd_data.3a3938238e1f29.00000000002963de/head,107(3)=5/9a2f6226/rbd_data.3a3938238e1f29.00000000001e81d8/head,5/169f6226/rbd_data.3a3938238e1f29.000
00000001b2a67/head,5/25107226/rbd_data.3a3938238e1f29.00000000000b92bd/head,5/46207226/rbd_data.3a3938238e1f29.00000000002d80ca/head,5/1b907226/rbd_data.3a3938238e1f29.00000000002963de/head},
in_progress=)
-75> 2016-10-19 11:20:22.328670 7f018f027700 10 osd.30 pg_epoch:
138618 pg[5.26s0( v 84093'35622 (82361'32622,84093'35622]
local-les=138613 n=12026 ec=61149 les/c 138613/136567
138612/138612/138448) [27,109,22,30,62,75]/[30,22,32,107,62,2147483647]
r=0 lpr=138612 pi=136565-138611/23 rops=5
bft=22(2),27(0),30(3),75(5),109(1) crt=0'0 lcod 0'0 mlcod 0'0
active+undersized+degraded+remapped+backfilling]
handle_recovery_read_complete: returned
5/9a2f6226/rbd_data.3a3938238e1f29.00000000001e81d8/head (0, 8388608,
[22(1),2097152, 30(0),2097152, 32(2),2097152, 107(3),2097152])
-74> 2016-10-19 11:20:22.328716 7f018f027700 10 osd.30 pg_epoch:
138618 pg[5.26s0( v 84093'35622 (82361'32622,84093'35622]
local-les=138613 n=12026 ec=61149 les/c 138613/136567
138612/138612/138448) [27,109,22,30,62,75]/[30,22,32,107,62,2147483647]
r=0 lpr=138612 pi=136565-138611/23 rops=5
bft=22(2),27(0),30(3),75(5),109(1) crt=0'0 lcod 0'0 mlcod 0'0
active+undersized+degraded+remapped+backfilling]
handle_recovery_read_complete: [0,2097152, 1,2097152, 2,2097152, 3,2097152]
-73> 2016-10-19 11:20:22.362043 7f018f027700 10 osd.30 pg_epoch:
138618 pg[5.26s0( v 84093'35622 (82361'32622,84093'35622]
local-les=138613 n=12026 ec=61149 les/c 138613/136567
138612/138612/138448) [27,109,22,30,62,75]/[30,22,32,107,62,2147483647]
r=0 lpr=138612 pi=136565-138611/23 rops=5
bft=22(2),27(0),30(3),75(5),109(1) crt=0'0 lcod 0'0 mlcod 0'0
active+undersized+degraded+remapped+backfilling] continue_recovery_op:
continuing
RecoveryOp(hoid=5/9a2f6226/rbd_data.3a3938238e1f29.00000000001e81d8/head
v=67187'26733 missing_on=22(2),27(0),30(3),75(5),109(1)
missing_on_shards=0,1,2,3,5
recovery_info=ObjectRecoveryInfo(5/9a2f6226/rbd_data.3a3938238e1f29.00000000001e81d8/head@67187'26733,
copy_subset: [], clone_subset: {})
recovery_progress=ObjectRecoveryProgress(first, data_recovered_to:0,
data_complete:false, omap_recovered_to:, omap_complete:true)
pending_read=0 obc refcount=3 state=READING waiting_on_pushes=
extent_requested=0,8388608
-72> 2016-10-19 11:20:22.362567 7f018f027700 10 osd.30 pg_epoch:
138618 pg[5.26s0( v 84093'35622 (82361'32622,84093'35622]
local-les=138613 n=12026 ec=61149 les/c 138613/136567
138612/138612/138448) [27,109,22,30,62,75]/[30,22,32,107,62,2147483647]
r=0 lpr=138612 pi=136565-138611/23 rops=5
bft=22(2),27(0),30(3),75(5),109(1) crt=0'0 lcod 0'0 mlcod 0'0
active+undersized+degraded+remapped+backfilling] continue_recovery_op:
before_progress=ObjectRecoveryProgress(first, data_recovered_to:0,
data_complete:false, omap_recovered_to:, omap_complete:true),
after_progress=ObjectRecoveryProgress(!first, data_recovered_to:8388608,
data_complete:true, omap_recovered_to:, omap_complete:true),
pop.data.length()=2097152, size=8388608
-71> 2016-10-19 11:20:22.362994 7f018f027700 10 osd.30 pg_epoch:
138618 pg[5.26s0( v 84093'35622 (82361'32622,84093'35622]
local-les=138613 n=12026 ec=61149 les/c 138613/136567
138612/138612/138448) [27,109,22,30,62,75]/[30,22,32,107,62,2147483647]
r=0 lpr=138612 pi=136565-138611/23 rops=5
bft=22(2),27(0),30(3),75(5),109(1) crt=0'0 lcod 0'0 mlcod 0'0
active+undersized+degraded+remapped+backfilling] continue_recovery_op:
before_progress=ObjectRecoveryProgress(first, data_recovered_to:0,
data_complete:false, omap_recovered_to:, omap_complete:true),
after_progress=ObjectRecoveryProgress(!first, data_recovered_to:8388608,
data_complete:true, omap_recovered_to:, omap_complete:true),
pop.data.length()=2097152, size=8388608
-70> 2016-10-19 11:20:22.363546 7f018f027700 10 osd.30 pg_epoch:
138618 pg[5.26s0( v 84093'35622 (82361'32622,84093'35622]
local-les=138613 n=12026 ec=61149 les/c 138613/136567
138612/138612/138448) [27,109,22,30,62,75]/[30,22,32,107,62,2147483647]
r=0 lpr=138612 pi=136565-138611/23 rops=5
bft=22(2),27(0),30(3),75(5),109(1) crt=0'0 lcod 0'0 mlcod 0'0
active+undersized+degraded+remapped+backfilling] continue_recovery_op:
before_progress=ObjectRecoveryProgress(first, data_recovered_to:0,
data_complete:false, omap_recovered_to:, omap_complete:true),
after_progress=ObjectRecoveryProgress(!first, data_recovered_to:8388608,
data_complete:true, omap_recovered_to:, omap_complete:true),
pop.data.length()=2097152, size=8388608
-69> 2016-10-19 11:20:22.363631 7f0197838700 1 --
10.24.12.25:6801/14492 <== osd.62 10.24.12.21:0/13047 1815 ====
osd_ping(ping e138618 stamp 2016-10-19 11:20:22.361898) v2 ==== 47+0+0
(655118554 0 0) 0x2d6a5200 con 0x1fc01ce0
-68> 2016-10-19 11:20:22.363653 7f0197838700 1 --
10.24.12.25:6801/14492 --> 10.24.12.21:0/13047 -- osd_ping(ping_reply
e138618 stamp 2016-10-19 11:20:22.361898) v2 -- ?+0 0x252a9e00 con
0x1fc01ce0
-67> 2016-10-19 11:20:22.363695 7f019903b700 1 --
10.24.11.25:6811/14492 <== osd.62 10.24.12.21:0/13047 1815 ====
osd_ping(ping e138618 stamp 2016-10-19 11:20:22.361898) v2 ==== 47+0+0
(655118554 0 0) 0x2d7ef000 con 0x1fc1d080
-66> 2016-10-19 11:20:22.363711 7f019903b700 1 --
10.24.11.25:6811/14492 --> 10.24.12.21:0/13047 -- osd_ping(ping_reply
e138618 stamp 2016-10-19 11:20:22.361898) v2 -- ?+0 0x23765000 con
0x1fc1d080
-65> 2016-10-19 11:20:22.364030 7f018f027700 10 osd.30 pg_epoch:
138618 pg[5.26s0( v 84093'35622 (82361'32622,84093'35622]
local-les=138613 n=12026 ec=61149 les/c 138613/136567
138612/138612/138448) [27,109,22,30,62,75]/[30,22,32,107,62,2147483647]
r=0 lpr=138612 pi=136565-138611/23 rops=5
bft=22(2),27(0),30(3),75(5),109(1) crt=0'0 lcod 0'0 mlcod 0'0
active+undersized+degraded+remapped+backfilling] continue_recovery_op:
before_progress=ObjectRecoveryProgress(first, data_recovered_to:0,
data_complete:false, omap_recovered_to:, omap_complete:true),
after_progress=ObjectRecoveryProgress(!first, data_recovered_to:8388608,
data_complete:true, omap_recovered_to:, omap_complete:true),
pop.data.length()=2097152, size=8388608
-64> 2016-10-19 11:20:22.364497 7f018f027700 10 osd.30 pg_epoch:
138618 pg[5.26s0( v 84093'35622 (82361'32622,84093'35622]
local-les=138613 n=12026 ec=61149 les/c 138613/136567
138612/138612/138448) [27,109,22,30,62,75]/[30,22,32,107,62,2147483647]
r=0 lpr=138612 pi=136565-138611/23 rops=5
bft=22(2),27(0),30(3),75(5),109(1) crt=0'0 lcod 0'0 mlcod 0'0
active+undersized+degraded+remapped+backfilling] continue_recovery_op:
before_progress=ObjectRecoveryProgress(first, data_recovered_to:0,
data_complete:false, omap_recovered_to:, omap_complete:true),
after_progress=ObjectRecoveryProgress(!first, data_recovered_to:8388608,
data_complete:true, omap_recovered_to:, omap_complete:true),
pop.data.length()=2097152, size=8388608
-63> 2016-10-19 11:20:22.367061 7f018f027700 10 osd.30 pg_epoch:
138618 pg[5.26s0( v 84093'35622 (82361'32622,84093'35622]
local-les=138613 n=12026 ec=61149 les/c 138613/136567
138612/138612/138448) [27,109,22,30,62,75]/[30,22,32,107,62,2147483647]
r=0 lpr=138612 pi=136565-138611/23 rops=5
bft=22(2),27(0),30(3),75(5),109(1) crt=0'0 lcod 0'0 mlcod 0'0
active+undersized+degraded+remapped+backfilling] continue_recovery_op:
READING return
RecoveryOp(hoid=5/9a2f6226/rbd_data.3a3938238e1f29.00000000001e81d8/head
v=67187'26733 missing_on=22(2),27(0),30(3),75(5),109(1)
missing_on_shards=0,1,2,3,5
recovery_info=ObjectRecoveryInfo(5/9a2f6226/rbd_data.3a3938238e1f29.00000000001e81d8/head@67187'26733,
copy_subset: [], clone_subset: {})
recovery_progress=ObjectRecoveryProgress(!first,
data_recovered_to:8388608, data_complete:true, omap_recovered_to:,
omap_complete:true) pending_read=0 obc refcount=3 state=WRITING
waiting_on_pushes=22(2),27(0),30(3),75(5),109(1) extent_requested=0,8388608
-62> 2016-10-19 11:20:22.367111 7f018f027700 10 osd.30 pg_epoch:
138618 pg[5.26s0( v 84093'35622 (82361'32622,84093'35622]
local-les=138613 n=12026 ec=61149 les/c 138613/136567
138612/138612/138448) [27,109,22,30,62,75]/[30,22,32,107,62,2147483647]
r=0 lpr=138612 pi=136565-138611/23 rops=5
bft=22(2),27(0),30(3),75(5),109(1) crt=0'0 lcod 0'0 mlcod 0'0
active+undersized+degraded+remapped+backfilling]
handle_recovery_read_complete: returned
5/169f6226/rbd_data.3a3938238e1f29.00000000001b2a67/head (0, 8388608,
[22(1),2097152, 30(0),2097152, 32(2),2097152, 107(3),2097152])
-61> 2016-10-19 11:20:22.367144 7f018f027700 10 osd.30 pg_epoch:
138618 pg[5.26s0( v 84093'35622 (82361'32622,84093'35622]
local-les=138613 n=12026 ec=61149 les/c 138613/136567
138612/138612/138448) [27,109,22,30,62,75]/[30,22,32,107,62,2147483647]
r=0 lpr=138612 pi=136565-138611/23 rops=5
bft=22(2),27(0),30(3),75(5),109(1) crt=0'0 lcod 0'0 mlcod 0'0
active+undersized+degraded+remapped+backfilling]
handle_recovery_read_complete: [0,2097152, 1,2097152, 2,2097152, 3,2097152]
-60> 2016-10-19 11:20:22.386084 7f01a415a700 10
filestore(/var/lib/ceph/osd/ceph-30) write
meta/-1/df5656d4/osdmap.138618/0 0~109464 = 109464
-59> 2016-10-19 11:20:22.386108 7f01a415a700 15
filestore(/var/lib/ceph/osd/ceph-30) write
meta/-1/23c2fcde/osd_superblock/0 0~440
-58> 2016-10-19 11:20:22.386125 7f01a415a700 10
filestore(/var/lib/ceph/osd/ceph-30) write
meta/-1/23c2fcde/osd_superblock/0 0~440 = 440
-57> 2016-10-19 11:20:22.386130 7f01a415a700 10
filestore(/var/lib/ceph/osd/ceph-30) _do_op 0x1cbfe710 seq 13466205 r =
0, finisher 0x1c9d6d30 0x284f8be0
-56> 2016-10-19 11:20:22.386134 7f01a415a700 10
filestore(/var/lib/ceph/osd/ceph-30) _finish_op 0x1cbfe710 seq 13466205
osr(default 0x3e149f8)/0x3e149f8
-55> 2016-10-19 11:20:22.397115 7f018f027700 10 osd.30 pg_epoch:
138618 pg[5.26s0( v 84093'35622 (82361'32622,84093'35622]
local-les=138613 n=12026 ec=61149 les/c 138613/136567
138612/138612/138448) [27,109,22,30,62,75]/[30,22,32,107,62,2147483647]
r=0 lpr=138612 pi=136565-138611/23 rops=5
bft=22(2),27(0),30(3),75(5),109(1) crt=0'0 lcod 0'0 mlcod 0'0
active+undersized+degraded+remapped+backfilling] continue_recovery_op:
continuing
RecoveryOp(hoid=5/169f6226/rbd_data.3a3938238e1f29.00000000001b2a67/head
v=81277'29571 missing_on=22(2),27(0),30(3),75(5),109(1)
missing_on_shards=0,1,2,3,5
recovery_info=ObjectRecoveryInfo(5/169f6226/rbd_data.3a3938238e1f29.00000000001b2a67/head@81277'29571,
copy_subset: [], clone_subset: {})
recovery_progress=ObjectRecoveryProgress(first, data_recovered_to:0,
data_complete:false, omap_recovered_to:, omap_complete:true)
pending_read=0 obc refcount=3 state=READING waiting_on_pushes=
extent_requested=0,8388608
-54> 2016-10-19 11:20:22.397945 7f018f027700 10 osd.30 pg_epoch:
138618 pg[5.26s0( v 84093'35622 (82361'32622,84093'35622]
local-les=138613 n=12026 ec=61149 les/c 138613/136567
138612/138612/138448) [27,109,22,30,62,75]/[30,22,32,107,62,2147483647]
r=0 lpr=138612 pi=136565-138611/23 rops=5
bft=22(2),27(0),30(3),75(5),109(1) crt=0'0 lcod 0'0 mlcod 0'0
active+undersized+degraded+remapped+backfilling] continue_recovery_op:
before_progress=ObjectRecoveryProgress(first, data_recovered_to:0,
data_complete:false, omap_recovered_to:, omap_complete:true),
after_progress=ObjectRecoveryProgress(!first, data_recovered_to:8388608,
data_complete:true, omap_recovered_to:, omap_complete:true),
pop.data.length()=2097152, size=8388608
-53> 2016-10-19 11:20:22.398610 7f018f027700 10 osd.30 pg_epoch:
138618 pg[5.26s0( v 84093'35622 (82361'32622,84093'35622]
local-les=138613 n=12026 ec=61149 les/c 138613/136567
138612/138612/138448) [27,109,22,30,62,75]/[30,22,32,107,62,2147483647]
r=0 lpr=138612 pi=136565-138611/23 rops=5
bft=22(2),27(0),30(3),75(5),109(1) crt=0'0 lcod 0'0 mlcod 0'0
active+undersized+degraded+remapped+backfilling] continue_recovery_op:
before_progress=ObjectRecoveryProgress(first, data_recovered_to:0,
data_complete:false, omap_recovered_to:, omap_complete:true),
after_progress=ObjectRecoveryProgress(!first, data_recovered_to:8388608,
data_complete:true, omap_recovered_to:, omap_complete:true),
pop.data.length()=2097152, size=8388608
-52> 2016-10-19 11:20:22.399512 7f018f027700 10 osd.30 pg_epoch:
138618 pg[5.26s0( v 84093'35622 (82361'32622,84093'35622]
local-les=138613 n=12026 ec=61149 les/c 138613/136567
138612/138612/138448) [27,109,22,30,62,75]/[30,22,32,107,62,2147483647]
r=0 lpr=138612 pi=136565-138611/23 rops=5
bft=22(2),27(0),30(3),75(5),109(1) crt=0'0 lcod 0'0 mlcod 0'0
active+undersized+degraded+remapped+backfilling] continue_recovery_op:
before_progress=ObjectRecoveryProgress(first, data_recovered_to:0,
data_complete:false, omap_recovered_to:, omap_complete:true),
after_progress=ObjectRecoveryProgress(!first, data_recovered_to:8388608,
data_complete:true, omap_recovered_to:, omap_complete:true),
pop.data.length()=2097152, size=8388608
-51> 2016-10-19 11:20:22.400290 7f018f027700 10 osd.30 pg_epoch:
138618 pg[5.26s0( v 84093'35622 (82361'32622,84093'35622]
local-les=138613 n=12026 ec=61149 les/c 138613/136567
138612/138612/138448) [27,109,22,30,62,75]/[30,22,32,107,62,2147483647]
r=0 lpr=138612 pi=136565-138611/23 rops=5
bft=22(2),27(0),30(3),75(5),109(1) crt=0'0 lcod 0'0 mlcod 0'0
active+undersized+degraded+remapped+backfilling] continue_recovery_op:
before_progress=ObjectRecoveryProgress(first, data_recovered_to:0,
data_complete:false, omap_recovered_to:, omap_complete:true),
after_progress=ObjectRecoveryProgress(!first, data_recovered_to:8388608,
data_complete:true, omap_recovered_to:, omap_complete:true),
pop.data.length()=2097152, size=8388608
-50> 2016-10-19 11:20:22.401039 7f018f027700 10 osd.30 pg_epoch:
138618 pg[5.26s0( v 84093'35622 (82361'32622,84093'35622]
local-les=138613 n=12026 ec=61149 les/c 138613/136567
138612/138612/138448) [27,109,22,30,62,75]/[30,22,32,107,62,2147483647]
r=0 lpr=138612 pi=136565-138611/23 rops=5
bft=22(2),27(0),30(3),75(5),109(1) crt=0'0 lcod 0'0 mlcod 0'0
active+undersized+degraded+remapped+backfilling] continue_recovery_op:
before_progress=ObjectRecoveryProgress(first, data_recovered_to:0,
data_complete:false, omap_recovered_to:, omap_complete:true),
after_progress=ObjectRecoveryProgress(!first, data_recovered_to:8388608,
data_complete:true, omap_recovered_to:, omap_complete:true),
pop.data.length()=2097152, size=8388608
-49> 2016-10-19 11:20:22.402395 7f018f027700 10 osd.30 pg_epoch:
138618 pg[5.26s0( v 84093'35622 (82361'32622,84093'35622]
local-les=138613 n=12026 ec=61149 les/c 138613/136567
138612/138612/138448) [27,109,22,30,62,75]/[30,22,32,107,62,2147483647]
r=0 lpr=138612 pi=136565-138611/23 rops=5
bft=22(2),27(0),30(3),75(5),109(1) crt=0'0 lcod 0'0 mlcod 0'0
active+undersized+degraded+remapped+backfilling] continue_recovery_op:
READING return
RecoveryOp(hoid=5/169f6226/rbd_data.3a3938238e1f29.00000000001b2a67/head
v=81277'29571 missing_on=22(2),27(0),30(3),75(5),109(1)
missing_on_shards=0,1,2,3,5
recovery_info=ObjectRecoveryInfo(5/169f6226/rbd_data.3a3938238e1f29.00000000001b2a67/head@81277'29571,
copy_subset: [], clone_subset: {})
recovery_progress=ObjectRecoveryProgress(!first,
data_recovered_to:8388608, data_complete:true, omap_recovered_to:,
omap_complete:true) pending_read=0 obc refcount=3 state=WRITING
waiting_on_pushes=22(2),27(0),30(3),75(5),109(1) extent_requested=0,8388608
-48> 2016-10-19 11:20:22.402425 7f018f027700 10 osd.30 pg_epoch:
138618 pg[5.26s0( v 84093'35622 (82361'32622,84093'35622]
local-les=138613 n=12026 ec=61149 les/c 138613/136567
138612/138612/138448) [27,109,22,30,62,75]/[30,22,32,107,62,2147483647]
r=0 lpr=138612 pi=136565-138611/23 rops=5
bft=22(2),27(0),30(3),75(5),109(1) crt=0'0 lcod 0'0 mlcod 0'0
active+undersized+degraded+remapped+backfilling]
handle_recovery_read_complete: returned
5/25107226/rbd_data.3a3938238e1f29.00000000000b92bd/head (0, 8388608,
[22(1),2097152, 30(0),2097152, 32(2),2097152, 107(3),2097152])
-47> 2016-10-19 11:20:22.402448 7f018f027700 10 osd.30 pg_epoch:
138618 pg[5.26s0( v 84093'35622 (82361'32622,84093'35622]
local-les=138613 n=12026 ec=61149 les/c 138613/136567
138612/138612/138448) [27,109,22,30,62,75]/[30,22,32,107,62,2147483647]
r=0 lpr=138612 pi=136565-138611/23 rops=5
bft=22(2),27(0),30(3),75(5),109(1) crt=0'0 lcod 0'0 mlcod 0'0
active+undersized+degraded+remapped+backfilling]
handle_recovery_read_complete: [0,2097152, 1,2097152, 2,2097152, 3,2097152]
-46> 2016-10-19 11:20:22.415973 7f0197838700 1 --
10.24.12.25:6801/14492 <== osd.13 10.24.12.26:0/9540 1869 ====
osd_ping(ping e138618 stamp 2016-10-19 11:20:22.413088) v2 ==== 47+0+0
(3341986049 0 0) 0x216a2200 con 0x1bea62c0
-45> 2016-10-19 11:20:22.416000 7f0197838700 1 --
10.24.12.25:6801/14492 --> 10.24.12.26:0/9540 -- osd_ping(ping_reply
e138618 stamp 2016-10-19 11:20:22.413088) v2 -- ?+0 0x2d6a5200 con
0x1bea62c0
-44> 2016-10-19 11:20:22.416016 7f0197838700 10 osd.30 138618
note_peer_epoch osd.13 has 138618
-43> 2016-10-19 11:20:22.416084 7f019903b700 1 --
10.24.11.25:6811/14492 <== osd.13 10.24.12.26:0/9540 1869 ====
osd_ping(ping e138618 stamp 2016-10-19 11:20:22.413088) v2 ==== 47+0+0
(3341986049 0 0) 0x2843fa00 con 0x1dbb67e0
-42> 2016-10-19 11:20:22.416100 7f019903b700 1 --
10.24.11.25:6811/14492 --> 10.24.12.26:0/9540 -- osd_ping(ping_reply
e138618 stamp 2016-10-19 11:20:22.413088) v2 -- ?+0 0x2d7ef000 con
0x1dbb67e0
-41> 2016-10-19 11:20:22.426935 7f018f027700 10 osd.30 pg_epoch:
138618 pg[5.26s0( v 84093'35622 (82361'32622,84093'35622]
local-les=138613 n=12026 ec=61149 les/c 138613/136567
138612/138612/138448) [27,109,22,30,62,75]/[30,22,32,107,62,2147483647]
r=0 lpr=138612 pi=136565-138611/23 rops=5
bft=22(2),27(0),30(3),75(5),109(1) crt=0'0 lcod 0'0 mlcod 0'0
active+undersized+degraded+remapped+backfilling] continue_recovery_op:
continuing
RecoveryOp(hoid=5/25107226/rbd_data.3a3938238e1f29.00000000000b92bd/head
v=81277'29572 missing_on=22(2),27(0),30(3),75(5),109(1)
missing_on_shards=0,1,2,3,5
recovery_info=ObjectRecoveryInfo(5/25107226/rbd_data.3a3938238e1f29.00000000000b92bd/head@81277'29572,
copy_subset: [], clone_subset: {})
recovery_progress=ObjectRecoveryProgress(first, data_recovered_to:0,
data_complete:false, omap_recovered_to:, omap_complete:true)
pending_read=0 obc refcount=3 state=READING waiting_on_pushes=
extent_requested=0,8388608
-40> 2016-10-19 11:20:22.428431 7f018f027700 10 osd.30 pg_epoch:
138618 pg[5.26s0( v 84093'35622 (82361'32622,84093'35622]
local-les=138613 n=12026 ec=61149 les/c 138613/136567
138612/138612/138448) [27,109,22,30,62,75]/[30,22,32,107,62,2147483647]
r=0 lpr=138612 pi=136565-138611/23 rops=5
bft=22(2),27(0),30(3),75(5),109(1) crt=0'0 lcod 0'0 mlcod 0'0
active+undersized+degraded+remapped+backfilling] continue_recovery_op:
before_progress=ObjectRecoveryProgress(first, data_recovered_to:0,
data_complete:false, omap_recovered_to:, omap_complete:true),
after_progress=ObjectRecoveryProgress(!first, data_recovered_to:8388608,
data_complete:true, omap_recovered_to:, omap_complete:true),
pop.data.length()=2097152, size=8388608
-39> 2016-10-19 11:20:22.429710 7f018f027700 10 osd.30 pg_epoch:
138618 pg[5.26s0( v 84093'35622 (82361'32622,84093'35622]
local-les=138613 n=12026 ec=61149 les/c 138613/136567
138612/138612/138448) [27,109,22,30,62,75]/[30,22,32,107,62,2147483647]
r=0 lpr=138612 pi=136565-138611/23 rops=5
bft=22(2),27(0),30(3),75(5),109(1) crt=0'0 lcod 0'0 mlcod 0'0
active+undersized+degraded+remapped+backfilling] continue_recovery_op:
before_progress=ObjectRecoveryProgress(first, data_recovered_to:0,
data_complete:false, omap_recovered_to:, omap_complete:true),
after_progress=ObjectRecoveryProgress(!first, data_recovered_to:8388608,
data_complete:true, omap_recovered_to:, omap_complete:true),
pop.data.length()=2097152, size=8388608
-38> 2016-10-19 11:20:22.431818 7f018f027700 10 osd.30 pg_epoch:
138618 pg[5.26s0( v 84093'35622 (82361'32622,84093'35622]
local-les=138613 n=12026 ec=61149 les/c 138613/136567
138612/138612/138448) [27,109,22,30,62,75]/[30,22,32,107,62,2147483647]
r=0 lpr=138612 pi=136565-138611/23 rops=5
bft=22(2),27(0),30(3),75(5),109(1) crt=0'0 lcod 0'0 mlcod 0'0
active+undersized+degraded+remapped+backfilling] continue_recovery_op:
before_progress=ObjectRecoveryProgress(first, data_recovered_to:0,
data_complete:false, omap_recovered_to:, omap_complete:true),
after_progress=ObjectRecoveryProgress(!first, data_recovered_to:8388608,
data_complete:true, omap_recovered_to:, omap_complete:true),
pop.data.length()=2097152, size=8388608
-37> 2016-10-19 11:20:22.433601 7f018f027700 10 osd.30 pg_epoch:
138618 pg[5.26s0( v 84093'35622 (82361'32622,84093'35622]
local-les=138613 n=12026 ec=61149 les/c 138613/136567
138612/138612/138448) [27,109,22,30,62,75]/[30,22,32,107,62,2147483647]
r=0 lpr=138612 pi=136565-138611/23 rops=5
bft=22(2),27(0),30(3),75(5),109(1) crt=0'0 lcod 0'0 mlcod 0'0
active+undersized+degraded+remapped+backfilling] continue_recovery_op:
before_progress=ObjectRecoveryProgress(first, data_recovered_to:0,
data_complete:false, omap_recovered_to:, omap_complete:true),
after_progress=ObjectRecoveryProgress(!first, data_recovered_to:8388608,
data_complete:true, omap_recovered_to:, omap_complete:true),
pop.data.length()=2097152, size=8388608
-36> 2016-10-19 11:20:22.435297 7f018f027700 10 osd.30 pg_epoch:
138618 pg[5.26s0( v 84093'35622 (82361'32622,84093'35622]
local-les=138613 n=12026 ec=61149 les/c 138613/136567
138612/138612/138448) [27,109,22,30,62,75]/[30,22,32,107,62,2147483647]
r=0 lpr=138612 pi=136565-138611/23 rops=5
bft=22(2),27(0),30(3),75(5),109(1) crt=0'0 lcod 0'0 mlcod 0'0
active+undersized+degraded+remapped+backfilling] continue_recovery_op:
before_progress=ObjectRecoveryProgress(first, data_recovered_to:0,
data_complete:false, omap_recovered_to:, omap_complete:true),
after_progress=ObjectRecoveryProgress(!first, data_recovered_to:8388608,
data_complete:true, omap_recovered_to:, omap_complete:true),
pop.data.length()=2097152, size=8388608
-35> 2016-10-19 11:20:22.435614 7f0197838700 1 --
10.24.12.25:6801/14492 <== osd.21 10.24.12.21:0/28469 1879 ====
osd_ping(ping e138618 stamp 2016-10-19 11:20:22.433240) v2 ==== 47+0+0
(335991690 0 0) 0x124e6e00 con 0x1d6fb9c0
-34> 2016-10-19 11:20:22.435639 7f0197838700 1 --
10.24.12.25:6801/14492 --> 10.24.12.21:0/28469 -- osd_ping(ping_reply
e138618 stamp 2016-10-19 11:20:22.433240) v2 -- ?+0 0x216a2200 con
0x1d6fb9c0
-33> 2016-10-19 11:20:22.435653 7f0197838700 10 osd.30 138618
note_peer_epoch osd.21 has 138618
-32> 2016-10-19 11:20:22.436295 7f019903b700 1 --
10.24.11.25:6811/14492 <== osd.21 10.24.12.21:0/28469 1879 ====
osd_ping(ping e138618 stamp 2016-10-19 11:20:22.433240) v2 ==== 47+0+0
(335991690 0 0) 0x24903000 con 0x1dab9860
-31> 2016-10-19 11:20:22.436334 7f019903b700 1 --
10.24.11.25:6811/14492 --> 10.24.12.21:0/28469 -- osd_ping(ping_reply
e138618 stamp 2016-10-19 11:20:22.433240) v2 -- ?+0 0x2843fa00 con
0x1dab9860
-30> 2016-10-19 11:20:22.437245 7f018f027700 10 osd.30 pg_epoch:
138618 pg[5.26s0( v 84093'35622 (82361'32622,84093'35622]
local-les=138613 n=12026 ec=61149 les/c 138613/136567
138612/138612/138448) [27,109,22,30,62,75]/[30,22,32,107,62,2147483647]
r=0 lpr=138612 pi=136565-138611/23 rops=5
bft=22(2),27(0),30(3),75(5),109(1) crt=0'0 lcod 0'0 mlcod 0'0
active+undersized+degraded+remapped+backfilling] continue_recovery_op:
READING return
RecoveryOp(hoid=5/25107226/rbd_data.3a3938238e1f29.00000000000b92bd/head
v=81277'29572 missing_on=22(2),27(0),30(3),75(5),109(1)
missing_on_shards=0,1,2,3,5
recovery_info=ObjectRecoveryInfo(5/25107226/rbd_data.3a3938238e1f29.00000000000b92bd/head@81277'29572,
copy_subset: [], clone_subset: {})
recovery_progress=ObjectRecoveryProgress(!first,
data_recovered_to:8388608, data_complete:true, omap_recovered_to:,
omap_complete:true) pending_read=0 obc refcount=3 state=WRITING
waiting_on_pushes=22(2),27(0),30(3),75(5),109(1) extent_requested=0,8388608
-29> 2016-10-19 11:20:22.437305 7f018f027700 10 osd.30 pg_epoch:
138618 pg[5.26s0( v 84093'35622 (82361'32622,84093'35622]
local-les=138613 n=12026 ec=61149 les/c 138613/136567
138612/138612/138448) [27,109,22,30,62,75]/[30,22,32,107,62,2147483647]
r=0 lpr=138612 pi=136565-138611/23 rops=5
bft=22(2),27(0),30(3),75(5),109(1) crt=0'0 lcod 0'0 mlcod 0'0
active+undersized+degraded+remapped+backfilling]
handle_recovery_read_complete: returned
5/46207226/rbd_data.3a3938238e1f29.00000000002d80ca/head (0, 8388608,
[22(1),2097152, 30(0),2097152, 32(2),0, 107(3),2097152])
-28> 2016-10-19 11:20:22.437338 7f018f027700 10 osd.30 pg_epoch:
138618 pg[5.26s0( v 84093'35622 (82361'32622,84093'35622]
local-les=138613 n=12026 ec=61149 les/c 138613/136567
138612/138612/138448) [27,109,22,30,62,75]/[30,22,32,107,62,2147483647]
r=0 lpr=138612 pi=136565-138611/23 rops=5
bft=22(2),27(0),30(3),75(5),109(1) crt=0'0 lcod 0'0 mlcod 0'0
active+undersized+degraded+remapped+backfilling]
handle_recovery_read_complete: [0,2097152, 1,2097152, 2,0, 3,2097152]
-27> 2016-10-19 11:20:22.437352 7f018f027700 -1 osd.30 pg_epoch:
138618 pg[5.26s0( v 84093'35622 (82361'32622,84093'35622]
local-les=138613 n=12026 ec=61149 les/c 138613/136567
138612/138612/138448) [27,109,22,30,62,75]/[30,22,32,107,62,2147483647]
r=0 lpr=138612 pi=136565-138611/23 rops=5
bft=22(2),27(0),30(3),75(5),109(1) crt=0'0 lcod 0'0 mlcod 0'0
active+undersized+degraded+remapped+backfilling]
handle_recovery_read_complete: inconsistent shard sizes
5/46207226/rbd_data.3a3938238e1f29.00000000002d80ca/head the offending
shard must be manually removed after verifying there are enough shards
to recover (0, 8388608, [22(1),0, 30(0),0, 32(2),0, 107(3),0])
-26> 2016-10-19 11:20:22.490045 7f0197838700 1 --
10.24.12.25:6801/14492 <== osd.109 10.24.12.22:0/1551947 1866 ====
osd_ping(ping e138617 stamp 2016-10-19 11:20:22.488992) v2 ==== 47+0+0
(2142451185 0 0) 0x2ed1d800 con 0x1d866160
-25> 2016-10-19 11:20:22.490082 7f0197838700 1 --
10.24.12.25:6801/14492 --> 10.24.12.22:0/1551947 -- osd_ping(ping_reply
e138618 stamp 2016-10-19 11:20:22.488992) v2 -- ?+0 0x124e6e00 con
0x1d866160
-24> 2016-10-19 11:20:22.490106 7f0197838700 10 osd.30 138618
send_incremental_map 138617 -> 138618 to 0x1d84b9c0 10.24.12.22:6811/1551947
-23> 2016-10-19 11:20:22.490045 7f019903b700 1 --
10.24.11.25:6811/14492 <== osd.109 10.24.12.22:0/1551947 1866 ====
osd_ping(ping e138617 stamp 2016-10-19 11:20:22.488992) v2 ==== 47+0+0
(2142451185 0 0) 0x2b49f200 con 0x1fb33700
-22> 2016-10-19 11:20:22.490287 7f0197838700 1 --
10.24.12.25:6800/14492 --> 10.24.12.22:6811/1551947 --
osd_map(138618..138618 src has 89976..138618) v3 -- ?+0 0x2eecad00 con
0x1d84b9c0
-21> 2016-10-19 11:20:22.490308 7f0197838700 10 osd.30 138618
note_peer_epoch osd.109 has 138618
-20> 2016-10-19 11:20:22.490421 7f019903b700 1 --
10.24.11.25:6811/14492 --> 10.24.12.22:0/1551947 -- osd_ping(ping_reply
e138618 stamp 2016-10-19 11:20:22.488992) v2 -- ?+0 0x24903000 con
0x1fb33700
-19> 2016-10-19 11:20:22.585992 7f0197838700 1 --
10.24.12.25:6801/14492 <== osd.107 10.24.11.22:0/26894 1864 ====
osd_ping(ping e138618 stamp 2016-10-19 11:20:22.585470) v2 ==== 47+0+0
(3478576791 0 0) 0x23fb9a00 con 0x1fc01b80
-18> 2016-10-19 11:20:22.586054 7f0197838700 1 --
10.24.12.25:6801/14492 --> 10.24.11.22:0/26894 -- osd_ping(ping_reply
e138618 stamp 2016-10-19 11:20:22.585470) v2 -- ?+0 0x2ed1d800 con
0x1fc01b80
-17> 2016-10-19 11:20:22.586069 7f019903b700 1 --
10.24.11.25:6811/14492 <== osd.107 10.24.11.22:0/26894 1864 ====
osd_ping(ping e138618 stamp 2016-10-19 11:20:22.585470) v2 ==== 47+0+0
(3478576791 0 0) 0x3924d600 con 0x1fc1cf20
-16> 2016-10-19 11:20:22.586096 7f019903b700 1 --
10.24.11.25:6811/14492 --> 10.24.11.22:0/26894 -- osd_ping(ping_reply
e138618 stamp 2016-10-19 11:20:22.585470) v2 -- ?+0 0x2b49f200 con
0x1fc1cf20
-15> 2016-10-19 11:20:22.592451 7f019903b700 1 --
10.24.11.25:6811/14492 <== osd.12 10.24.12.23:0/11535 1816 ====
osd_ping(ping e138618 stamp 2016-10-19 11:20:22.591219) v2 ==== 47+0+0
(981476777 0 0) 0x2af28600 con 0x1dab7e40
-14> 2016-10-19 11:20:22.592451 7f0197838700 1 --
10.24.12.25:6801/14492 <== osd.12 10.24.12.23:0/11535 1816 ====
osd_ping(ping e138618 stamp 2016-10-19 11:20:22.591219) v2 ==== 47+0+0
(981476777 0 0) 0x21b71200 con 0x1fc08c60
-13> 2016-10-19 11:20:22.592484 7f0197838700 1 --
10.24.12.25:6801/14492 --> 10.24.12.23:0/11535 -- osd_ping(ping_reply
e138618 stamp 2016-10-19 11:20:22.591219) v2 -- ?+0 0x23fb9a00 con
0x1fc08c60
-12> 2016-10-19 11:20:22.592500 7f0197838700 10 osd.30 138618
note_peer_epoch osd.12 has 138618
-11> 2016-10-19 11:20:22.592516 7f019903b700 1 --
10.24.11.25:6811/14492 --> 10.24.12.23:0/11535 -- osd_ping(ping_reply
e138618 stamp 2016-10-19 11:20:22.591219) v2 -- ?+0 0x3924d600 con
0x1dab7e40
-10> 2016-10-19 11:20:22.627004 7f019903b700 1 --
10.24.11.25:6811/14492 <== osd.43 10.24.11.24:0/14808 1853 ====
osd_ping(ping e138618 stamp 2016-10-19 11:20:22.626779) v2 ==== 47+0+0
(1511592291 0 0) 0x231ed800 con 0x1d87b4a0
-9> 2016-10-19 11:20:22.627064 7f019903b700 1 --
10.24.11.25:6811/14492 --> 10.24.11.24:0/14808 -- osd_ping(ping_reply
e138618 stamp 2016-10-19 11:20:22.626779) v2 -- ?+0 0x2af28600 con
0x1d87b4a0
-8> 2016-10-19 11:20:22.627088 7f019903b700 10 osd.30 138618
note_peer_epoch osd.43 has 138618
-7> 2016-10-19 11:20:22.627318 7f0197838700 1 --
10.24.12.25:6801/14492 <== osd.43 10.24.11.24:0/14808 1853 ====
osd_ping(ping e138618 stamp 2016-10-19 11:20:22.626779) v2 ==== 47+0+0
(1511592291 0 0) 0x303fba00 con 0x1d631fa0
-6> 2016-10-19 11:20:22.627358 7f0197838700 1 --
10.24.12.25:6801/14492 --> 10.24.11.24:0/14808 -- osd_ping(ping_reply
e138618 stamp 2016-10-19 11:20:22.626779) v2 -- ?+0 0x21b71200 con
0x1d631fa0
-5> 2016-10-19 11:20:22.666142 7f0197838700 1 --
10.24.12.25:6801/14492 <== osd.42 10.24.12.24:0/16828 1867 ====
osd_ping(ping e138618 stamp 2016-10-19 11:20:22.666036) v2 ==== 47+0+0
(1984048023 0 0) 0x24f39e00 con 0x1d724160
-4> 2016-10-19 11:20:22.666192 7f0197838700 1 --
10.24.12.25:6801/14492 --> 10.24.12.24:0/16828 -- osd_ping(ping_reply
e138618 stamp 2016-10-19 11:20:22.666036) v2 -- ?+0 0x303fba00 con
0x1d724160
-3> 2016-10-19 11:20:22.666216 7f0197838700 10 osd.30 138618
note_peer_epoch osd.42 has 138618
-2> 2016-10-19 11:20:22.666671 7f019903b700 1 --
10.24.11.25:6811/14492 <== osd.42 10.24.12.24:0/16828 1867 ====
osd_ping(ping e138618 stamp 2016-10-19 11:20:22.666036) v2 ==== 47+0+0
(1984048023 0 0) 0x2d7ed400 con 0x1d51f440
-1> 2016-10-19 11:20:22.666692 7f019903b700 1 --
10.24.11.25:6811/14492 --> 10.24.12.24:0/16828 -- osd_ping(ping_reply
e138618 stamp 2016-10-19 11:20:22.666036) v2 -- ?+0 0x231ed800 con
0x1d51f440
0> 2016-10-19 11:20:22.696498 7f018f027700 -1 osd/ECBackend.cc:
In function 'void ECBackend::handle_recovery_read_complete(const
hobject_t&, boost::tuples::tuple<long unsigned int, long unsigned int,
std::map<pg_shard_t, ceph::buffer::list> >&,
boost::optional<std::map<std::basic_string<char>, ceph::buffer::list> >,
RecoveryMessages*)' thread 7f018f027700 time 2016-10-19 11:20:22.450562
osd/ECBackend.cc: 373: FAILED assert(0)
ceph version 0.94.9 (fe6d859066244b97b24f09d46552afc2071e6f90)
1: (ceph::__ceph_assert_fail(char const*, char const*, int, char
const*)+0x76) [0xc0f196]
2: (ECBackend::handle_recovery_read_complete(hobject_t const&,
boost::tuples::tuple<unsigned long, unsigned long, std::map<pg_shard_t,
ceph::buffer::list, std::less<pg_shard_t>,
std::allocator<std::pair<pg_shard_t const, ceph::buffer::list> > >,
boost::tuples::null_type, boost::tuples::null_type,
boost::tuples::null_type, boost::tuples::null_type,
boost::tuples::null_type, boost::tuples::null_type,
boost::tuples::null_type>&, boost::optional<std::map<std::string,
ceph::buffer::list, std::less<std::string>,
std::allocator<std::pair<std::string const, ceph::buffer::list> > > >,
RecoveryMessages*)+0xb29) [0xa5c2b9]
3: (OnRecoveryReadComplete::finish(std::pair<RecoveryMessages*,
ECBackend::read_result_t&>&)+0x83) [0xa6b383]
4: (ECBackend::complete_read_op(ECBackend::ReadOp&,
RecoveryMessages*)+0x4a) [0xa4e58a]
5: (ECBackend::handle_sub_read_reply(pg_shard_t, ECSubReadReply&,
RecoveryMessages*)+0x972) [0xa53082]
6: (ECBackend::handle_message(std::tr1::shared_ptr<OpRequest>)+0x17e)
[0xa5b38e]
7: (ReplicatedPG::do_request(std::tr1::shared_ptr<OpRequest>&,
ThreadPool::TPHandle&)+0x159) [0x831ba9]
8: (OSD::dequeue_op(boost::intrusive_ptr<PG>,
std::tr1::shared_ptr<OpRequest>, ThreadPool::TPHandle&)+0x3cf) [0x688cff]
9: (OSD::ShardedOpWQ::_process(unsigned int,
ceph::heartbeat_handle_d*)+0x338) [0x689238]
10: (ShardedThreadPool::shardedthreadpool_worker(unsigned int)+0x89e)
[0xbfe69e]
11: (ShardedThreadPool::WorkThreadSharded::entry()+0x10) [0xc007f0]
12: (()+0x80a4) [0x7f01b1a9c0a4]
13: (clone()+0x6d) [0x7f01afff787d]
NOTE: a copy of the executable, or `objdump -rdS <executable>` is
needed to interpret this.
--- logging levels ---
0/ 5 none
0/ 1 lockdep
0/ 1 context
1/ 1 crush
1/ 5 mds
1/ 5 mds_balancer
1/ 5 mds_locker
1/ 5 mds_log
1/ 5 mds_log_expire
1/ 5 mds_migrator
0/ 1 buffer
0/ 1 timer
0/ 1 filer
0/ 1 striper
0/ 1 objecter
0/ 5 rados
0/ 5 rbd
0/ 5 rbd_replay
0/ 5 journaler
0/ 5 objectcacher
0/ 5 client
15/15 osd
0/ 5 optracker
0/ 5 objclass
15/15 filestore
1/ 3 keyvaluestore
1/ 3 journal
0/ 5 ms
1/ 5 mon
0/10 monc
1/ 5 paxos
0/ 5 tp
1/ 5 auth
1/ 5 crypto
1/ 1 finisher
1/ 5 heartbeatmap
1/ 5 perfcounter
1/ 5 rgw
1/10 civetweb
1/ 5 javaclient
1/ 5 asok
1/ 1 throttle
0/ 0 refs
1/ 5 xio
-2/-2 (syslog threshold)
-1/-1 (stderr threshold)
max_recent 10000
max_new 1000
log_file /var/log/ceph/ceph-osd.30.log
--- end dump of recent events ---
2016-10-19 11:20:22.823891 7f018f027700 -1 *** Caught signal (Aborted) **
in thread 7f018f027700
ceph version 0.94.9 (fe6d859066244b97b24f09d46552afc2071e6f90)
1: /usr/bin/ceph-osd() [0xb0c4d3]
2: (()+0xf8d0) [0x7f01b1aa38d0]
3: (gsignal()+0x37) [0x7f01aff44067]
4: (abort()+0x148) [0x7f01aff45448]
5: (__gnu_cxx::__verbose_terminate_handler()+0x15d) [0x7f01b0831b3d]
6: (()+0x5ebb6) [0x7f01b082fbb6]
7: (()+0x5ec01) [0x7f01b082fc01]
8: (()+0x5ee19) [0x7f01b082fe19]
9: (ceph::__ceph_assert_fail(char const*, char const*, int, char
const*)+0x247) [0xc0f367]
10: (ECBackend::handle_recovery_read_complete(hobject_t const&,
boost::tuples::tuple<unsigned long, unsigned long, std::map<pg_shard_t,
ceph::buffer::list, std::less<pg_shard_t>,
std::allocator<std::pair<pg_shard_t const, ceph::buffer::list> > >,
boost::tuples::null_type, boost::tuples::null_type,
boost::tuples::null_type, boost::tuples::null_type,
boost::tuples::null_type, boost::tuples::null_type,
boost::tuples::null_type>&, boost::optional<std::map<std::string,
ceph::buffer::list, std::less<std::string>,
std::allocator<std::pair<std::string const, ceph::buffer::list> > > >,
RecoveryMessages*)+0xb29) [0xa5c2b9]
11: (OnRecoveryReadComplete::finish(std::pair<RecoveryMessages*,
ECBackend::read_result_t&>&)+0x83) [0xa6b383]
12: (ECBackend::complete_read_op(ECBackend::ReadOp&,
RecoveryMessages*)+0x4a) [0xa4e58a]
13: (ECBackend::handle_sub_read_reply(pg_shard_t, ECSubReadReply&,
RecoveryMessages*)+0x972) [0xa53082]
14:
(ECBackend::handle_message(std::tr1::shared_ptr<OpRequest>)+0x17e)
[0xa5b38e]
15: (ReplicatedPG::do_request(std::tr1::shared_ptr<OpRequest>&,
ThreadPool::TPHandle&)+0x159) [0x831ba9]
16: (OSD::dequeue_op(boost::intrusive_ptr<PG>,
std::tr1::shared_ptr<OpRequest>, ThreadPool::TPHandle&)+0x3cf) [0x688cff]
17: (OSD::ShardedOpWQ::_process(unsigned int,
ceph::heartbeat_handle_d*)+0x338) [0x689238]
18: (ShardedThreadPool::shardedthreadpool_worker(unsigned int)+0x89e)
[0xbfe69e]
19: (ShardedThreadPool::WorkThreadSharded::entry()+0x10) [0xc007f0]
20: (()+0x80a4) [0x7f01b1a9c0a4]
21: (clone()+0x6d) [0x7f01afff787d]
NOTE: a copy of the executable, or `objdump -rdS <executable>` is
needed to interpret this.
--- begin dump of recent events ---
-4> 2016-10-19 11:20:22.745796 7f019903b700 1 --
10.24.11.25:6811/14492 <== osd.102 10.24.11.21:0/23005 1869 ====
osd_ping(ping e138618 stamp 2016-10-19 11:20:22.743943) v2 ==== 47+0+0
(3370376478 0 0) 0x2b4bb200 con 0x3e07c80
-3> 2016-10-19 11:20:22.745822 7f019903b700 1 --
10.24.11.25:6811/14492 --> 10.24.11.21:0/23005 -- osd_ping(ping_reply
e138618 stamp 2016-10-19 11:20:22.743943) v2 -- ?+0 0x2d7ed400 con 0x3e07c80
-2> 2016-10-19 11:20:22.746703 7f0197838700 1 --
10.24.12.25:6801/14492 <== osd.102 10.24.11.21:0/23005 1869 ====
osd_ping(ping e138618 stamp 2016-10-19 11:20:22.743943) v2 ==== 47+0+0
(3370376478 0 0) 0x25084400 con 0x1d632940
-1> 2016-10-19 11:20:22.746719 7f0197838700 1 --
10.24.12.25:6801/14492 --> 10.24.11.21:0/23005 -- osd_ping(ping_reply
e138618 stamp 2016-10-19 11:20:22.743943) v2 -- ?+0 0x24f39e00 con
0x1d632940
0> 2016-10-19 11:20:22.823891 7f018f027700 -1 *** Caught signal
(Aborted) **
in thread 7f018f027700
ceph version 0.94.9 (fe6d859066244b97b24f09d46552afc2071e6f90)
1: /usr/bin/ceph-osd() [0xb0c4d3]
2: (()+0xf8d0) [0x7f01b1aa38d0]
3: (gsignal()+0x37) [0x7f01aff44067]
4: (abort()+0x148) [0x7f01aff45448]
5: (__gnu_cxx::__verbose_terminate_handler()+0x15d) [0x7f01b0831b3d]
6: (()+0x5ebb6) [0x7f01b082fbb6]
7: (()+0x5ec01) [0x7f01b082fc01]
8: (()+0x5ee19) [0x7f01b082fe19]
9: (ceph::__ceph_assert_fail(char const*, char const*, int, char
const*)+0x247) [0xc0f367]
10: (ECBackend::handle_recovery_read_complete(hobject_t const&,
boost::tuples::tuple<unsigned long, unsigned long, std::map<pg_shard_t,
ceph::buffer::list, std::less<pg_shard_t>,
std::allocator<std::pair<pg_shard_t const, ceph::buffer::list> > >,
boost::tuples::null_type, boost::tuples::null_type,
boost::tuples::null_type, boost::tuples::null_type,
boost::tuples::null_type, boost::tuples::null_type,
boost::tuples::null_type>&, boost::optional<std::map<std::string,
ceph::buffer::list, std::less<std::string>,
std::allocator<std::pair<std::string const, ceph::buffer::list> > > >,
RecoveryMessages*)+0xb29) [0xa5c2b9]
11: (OnRecoveryReadComplete::finish(std::pair<RecoveryMessages*,
ECBackend::read_result_t&>&)+0x83) [0xa6b383]
12: (ECBackend::complete_read_op(ECBackend::ReadOp&,
RecoveryMessages*)+0x4a) [0xa4e58a]
13: (ECBackend::handle_sub_read_reply(pg_shard_t, ECSubReadReply&,
RecoveryMessages*)+0x972) [0xa53082]
14:
(ECBackend::handle_message(std::tr1::shared_ptr<OpRequest>)+0x17e)
[0xa5b38e]
15: (ReplicatedPG::do_request(std::tr1::shared_ptr<OpRequest>&,
ThreadPool::TPHandle&)+0x159) [0x831ba9]
16: (OSD::dequeue_op(boost::intrusive_ptr<PG>,
std::tr1::shared_ptr<OpRequest>, ThreadPool::TPHandle&)+0x3cf) [0x688cff]
17: (OSD::ShardedOpWQ::_process(unsigned int,
ceph::heartbeat_handle_d*)+0x338) [0x689238]
18: (ShardedThreadPool::shardedthreadpool_worker(unsigned int)+0x89e)
[0xbfe69e]
19: (ShardedThreadPool::WorkThreadSharded::entry()+0x10) [0xc007f0]
20: (()+0x80a4) [0x7f01b1a9c0a4]
21: (clone()+0x6d) [0x7f01afff787d]
NOTE: a copy of the executable, or `objdump -rdS <executable>` is
needed to interpret this.
--- logging levels ---
0/ 5 none
0/ 1 lockdep
0/ 1 context
1/ 1 crush
1/ 5 mds
1/ 5 mds_balancer
1/ 5 mds_locker
1/ 5 mds_log
1/ 5 mds_log_expire
1/ 5 mds_migrator
0/ 1 buffer
0/ 1 timer
0/ 1 filer
0/ 1 striper
0/ 1 objecter
0/ 5 rados
0/ 5 rbd
0/ 5 rbd_replay
0/ 5 journaler
0/ 5 objectcacher
0/ 5 client
15/15 osd
0/ 5 optracker
0/ 5 objclass
15/15 filestore
1/ 3 keyvaluestore
1/ 3 journal
0/ 5 ms
1/ 5 mon
0/10 monc
1/ 5 paxos
0/ 5 tp
1/ 5 auth
1/ 5 crypto
1/ 1 finisher
1/ 5 heartbeatmap
1/ 5 perfcounter
1/ 5 rgw
1/10 civetweb
1/ 5 javaclient
1/ 5 asok
1/ 1 throttle
0/ 0 refs
1/ 5 xio
-2/-2 (syslog threshold)
-1/-1 (stderr threshold)
max_recent 10000
max_new 1000
log_file /var/log/ceph/ceph-osd.30.log
--- end dump of recent events ---
_______________________________________________
ceph-users mailing list
ceph-users@xxxxxxxxxxxxxx
http://lists.ceph.com/listinfo.cgi/ceph-users-ceph.com
