---------- Пересылаемое сообщение ---------
От: Voloshanenko Igor <igor.voloshanenko@xxxxxxxxx>
Дата: вторник, 18 августа 2015 г.
Тема: Repair inconsistent pgs..
Кому: Irek Fasikhov <malmyzh@xxxxxxxxx>
Some additional inforamtion (Tnx Irek for questions!)
Pool values:
root@test:~# ceph osd pool get cold-storage size
size: 3
root@test:~# ceph osd pool get cold-storage min_size
min_size: 2
Broken pgs dump
PG_1 #########
{
"state": "active+clean+inconsistent",
"snap_trimq": "[]",
"epoch": 17541,
"up": [
56,
10,
42
],
"acting": [
56,
10,
42
],
"actingbackfill": [
"10",
"42",
"56"
],
"info": {
"pgid": "2.c4",
"last_update": "17541'29153",
"last_complete": "17541'29153",
"log_tail": "16746'26095",
"last_user_version": 401173,
"last_backfill": "MAX",
"purged_snaps": "[1~1,6~1,8~3,11~2,17~2,1f~2,25~1,28~1,2c~5,32~4,37~1,39~7,41~5,47~16,5e~19,cb~1,ce~2,d4~7,dc~1,de~1,e6~4,102~1,105~6,10d~1,119~1,150~1,15d~2,160~3,16d~1,16f~5,178~1,184~2,194~1,1a2~1,1a5~1,1ac~2,1c7~1,1cb~2,1ce~1]",
"history": {
"epoch_created": 98,
"last_epoch_started": 17531,
"last_epoch_clean": 17541,
"last_epoch_split": 0,
"same_up_since": 17139,
"same_interval_since": 17530,
"same_primary_since": 17530,
"last_scrub": "17541'29114",
"last_scrub_stamp": "2015-08-18 07:37:04.567973",
"last_deep_scrub": "17541'29114",
"last_deep_scrub_stamp": "2015-08-18 07:37:04.567973",
"last_clean_scrub_stamp": "2015-08-05 17:23:45.251731"
},
"stats": {
"version": "17541'29153",
"reported_seq": "21552",
"reported_epoch": "17541",
"state": "active+clean+inconsistent",
"last_fresh": "2015-08-18 07:48:37.667036",
"last_change": "2015-08-18 07:37:04.568541",
"last_active": "2015-08-18 07:48:37.667036",
"last_peered": "2015-08-18 07:48:37.667036",
"last_clean": "2015-08-18 07:48:37.667036",
"last_became_active": "0.000000",
"last_became_peered": "0.000000",
"last_unstale": "2015-08-18 07:48:37.667036",
"last_undegraded": "2015-08-18 07:48:37.667036",
"last_fullsized": "2015-08-18 07:48:37.667036",
"mapping_epoch": 17140,
"log_start": "16746'26095",
"ondisk_log_start": "16746'26095",
"created": 98,
"last_epoch_clean": 17541,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "17541'29114",
"last_scrub_stamp": "2015-08-18 07:37:04.567973",
"last_deep_scrub": "17541'29114",
"last_deep_scrub_stamp": "2015-08-18 07:37:04.567973",
"last_clean_scrub_stamp": "2015-08-05 17:23:45.251731",
"log_size": 3058,
"ondisk_log_size": 3058,
"stats_invalid": "0",
"stat_sum": {
"num_bytes": 2236608990,
"num_objects": 307,
"num_object_clones": 7,
"num_object_copies": 921,
"num_objects_missing_on_primary": 0,
"num_objects_degraded": 0,
"num_objects_misplaced": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 307,
"num_whiteouts": 0,
"num_read": 15694,
"num_read_kb": 401354,
"num_write": 55720,
"num_write_kb": 2539827,
"num_scrub_errors": 1,
"num_shallow_scrub_errors": 1,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 1842,
"num_bytes_recovered": 13419653940,
"num_keys_recovered": 36,
"num_objects_omap": 1,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0
},
"up": [
56,
10,
42
],
"acting": [
56,
10,
42
],
"blocked_by": [],
"up_primary": 56,
"acting_primary": 56
},
"empty": 0,
"dne": 0,
"incomplete": 0,
"last_epoch_started": 17531,
"hit_set_history": {
"current_last_update": "0'0",
"current_last_stamp": "0.000000",
"current_info": {
"begin": "0.000000",
"end": "0.000000",
"version": "0'0"
},
"history": []
}
},
"peer_info": [
{
"peer": "10",
"pgid": "2.c4",
"last_update": "17541'29153",
"last_complete": "17541'29153",
"log_tail": "16746'25703",
"last_user_version": 400914,
"last_backfill": "MAX",
"purged_snaps": "[1~1,6~1,8~3,11~2,17~2,1f~2,25~1,28~1,2c~5,32~4,37~1,39~7,41~5,47~16,5e~19,cb~1,ce~2,d4~7,dc~1,de~1,e6~4,102~1,105~6,10d~1,119~1,150~1,15d~2,160~3,16d~1,16f~5,178~1,184~2,194~1,1a2~1,1a5~1,1ac~2,1c7~1,1cb~2,1ce~1]",
"history": {
"epoch_created": 98,
"last_epoch_started": 17531,
"last_epoch_clean": 17541,
"last_epoch_split": 0,
"same_up_since": 17139,
"same_interval_since": 17530,
"same_primary_since": 17530,
"last_scrub": "17541'29114",
"last_scrub_stamp": "2015-08-18 07:37:04.567973",
"last_deep_scrub": "17541'29114",
"last_deep_scrub_stamp": "2015-08-18 07:37:04.567973",
"last_clean_scrub_stamp": "2015-08-05 17:23:45.251731"
},
"stats": {
"version": "17527'28894",
"reported_seq": "21277",
"reported_epoch": "17529",
"state": "active+remapped+inconsistent+backfilling",
"last_fresh": "2015-08-18 05:07:02.140274",
"last_change": "2015-08-18 05:05:47.908928",
"last_active": "2015-08-18 05:07:02.140274",
"last_peered": "2015-08-18 05:07:02.140274",
"last_clean": "2015-08-18 02:52:55.811384",
"last_became_active": "0.000000",
"last_became_peered": "0.000000",
"last_unstale": "2015-08-18 05:07:02.140274",
"last_undegraded": "2015-08-18 05:07:02.140274",
"last_fullsized": "2015-08-18 05:07:02.140274",
"mapping_epoch": 17140,
"log_start": "16746'25642",
"ondisk_log_start": "16746'25642",
"created": 98,
"last_epoch_clean": 17117,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "17129'28702",
"last_scrub_stamp": "2015-08-18 02:52:55.810489",
"last_deep_scrub": "17129'28702",
"last_deep_scrub_stamp": "2015-08-18 02:52:55.810489",
"last_clean_scrub_stamp": "2015-08-05 17:23:45.251731",
"log_size": 3252,
"ondisk_log_size": 3252,
"stats_invalid": "0",
"stat_sum": {
"num_bytes": 2236608990,
"num_objects": 307,
"num_object_clones": 7,
"num_object_copies": 1842,
"num_objects_missing_on_primary": 0,
"num_objects_degraded": 0,
"num_objects_misplaced": 924,
"num_objects_unfound": 0,
"num_objects_dirty": 307,
"num_whiteouts": 0,
"num_read": 15475,
"num_read_kb": 401354,
"num_write": 55202,
"num_write_kb": 2538487,
"num_scrub_errors": 1,
"num_shallow_scrub_errors": 1,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 1842,
"num_bytes_recovered": 13419653940,
"num_keys_recovered": 36,
"num_objects_omap": 1,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0
},
"up": [
56,
10,
42
],
"acting": [
56,
10,
42
],
"blocked_by": [],
"up_primary": 56,
"acting_primary": 56
},
"empty": 0,
"dne": 0,
"incomplete": 0,
"last_epoch_started": 17531,
"hit_set_history": {
"current_last_update": "0'0",
"current_last_stamp": "0.000000",
"current_info": {
"begin": "0.000000",
"end": "0.000000",
"version": "0'0"
},
"history": []
}
},
{
"peer": "42",
"pgid": "2.c4",
"last_update": "17541'29153",
"last_complete": "17541'29153",
"log_tail": "16746'25703",
"last_user_version": 400914,
"last_backfill": "MAX",
"purged_snaps": "[1~1,6~1,8~3,11~2,17~2,1f~2,25~1,28~1,2c~5,32~4,37~1,39~7,41~5,47~16,5e~19,cb~1,ce~2,d4~7,dc~1,de~1,e6~4,102~1,105~6,10d~1,119~1,150~1,15d~2,160~3,16d~1,16f~5,178~1,184~2,194~1,1a2~1,1a5~1,1ac~2,1c7~1,1cb~2,1ce~1]",
"history": {
"epoch_created": 98,
"last_epoch_started": 17531,
"last_epoch_clean": 17541,
"last_epoch_split": 0,
"same_up_since": 17139,
"same_interval_since": 17530,
"same_primary_since": 17530,
"last_scrub": "17541'29114",
"last_scrub_stamp": "2015-08-18 07:37:04.567973",
"last_deep_scrub": "17541'29114",
"last_deep_scrub_stamp": "2015-08-18 07:37:04.567973",
"last_clean_scrub_stamp": "2015-08-05 17:23:45.251731"
},
"stats": {
"version": "17527'28894",
"reported_seq": "21277",
"reported_epoch": "17529",
"state": "active+remapped+inconsistent+backfilling",
"last_fresh": "2015-08-18 05:07:02.140274",
"last_change": "2015-08-18 05:05:47.908928",
"last_active": "2015-08-18 05:07:02.140274",
"last_peered": "2015-08-18 05:07:02.140274",
"last_clean": "2015-08-18 02:52:55.811384",
"last_became_active": "0.000000",
"last_became_peered": "0.000000",
"last_unstale": "2015-08-18 05:07:02.140274",
"last_undegraded": "2015-08-18 05:07:02.140274",
"last_fullsized": "2015-08-18 05:07:02.140274",
"mapping_epoch": 17140,
"log_start": "16746'25642",
"ondisk_log_start": "16746'25642",
"created": 98,
"last_epoch_clean": 17117,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "17129'28702",
"last_scrub_stamp": "2015-08-18 02:52:55.810489",
"last_deep_scrub": "17129'28702",
"last_deep_scrub_stamp": "2015-08-18 02:52:55.810489",
"last_clean_scrub_stamp": "2015-08-05 17:23:45.251731",
"log_size": 3252,
"ondisk_log_size": 3252,
"stats_invalid": "0",
"stat_sum": {
"num_bytes": 2236608990,
"num_objects": 307,
"num_object_clones": 7,
"num_object_copies": 1842,
"num_objects_missing_on_primary": 0,
"num_objects_degraded": 0,
"num_objects_misplaced": 924,
"num_objects_unfound": 0,
"num_objects_dirty": 307,
"num_whiteouts": 0,
"num_read": 15475,
"num_read_kb": 401354,
"num_write": 55202,
"num_write_kb": 2538487,
"num_scrub_errors": 1,
"num_shallow_scrub_errors": 1,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 1842,
"num_bytes_recovered": 13419653940,
"num_keys_recovered": 36,
"num_objects_omap": 1,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0
},
"up": [
56,
10,
42
],
"acting": [
56,
10,
42
],
"blocked_by": [],
"up_primary": 56,
"acting_primary": 56
},
"empty": 0,
"dne": 0,
"incomplete": 0,
"last_epoch_started": 17531,
"hit_set_history": {
"current_last_update": "0'0",
"current_last_stamp": "0.000000",
"current_info": {
"begin": "0.000000",
"end": "0.000000",
"version": "0'0"
},
"history": []
}
}
],
"recovery_state": [
{
"name": "Started\/Primary\/Active",
"enter_time": "2015-08-18 05:07:04.026226",
"might_have_unfound": [],
"recovery_progress": {
"backfill_targets": [],
"waiting_on_backfill": [],
"last_backfill_started": "0\/\/0\/\/-1",
"backfill_info": {
"begin": "0\/\/0\/\/-1",
"end": "0\/\/0\/\/-1",
"objects": []
},
"peer_backfill_info": [],
"backfills_in_flight": [],
"recovering": [],
"pg_backend": {
"pull_from_peer": [],
"pushing": []
}
},
"scrub": {
"scrubber.epoch_start": "17530",
"scrubber.active": 0,
"scrubber.waiting_on": 0,
"scrubber.waiting_on_whom": []
}
},
{
"name": "Started",
"enter_time": "2015-08-18 05:07:02.925534"
}
],
"agent_state": {}
}
PG_2 ############
{
"state": "active+clean+inconsistent",
"snap_trimq": "[]",
"epoch": 17541,
"up": [
56,
15,
29
],
"acting": [
56,
15,
29
],
"actingbackfill": [
"15",
"29",
"56"
],
"info": {
"pgid": "2.490",
"last_update": "17541'38040",
"last_complete": "17541'38040",
"log_tail": "16745'34960",
"last_user_version": 155503,
"last_backfill": "MAX",
"purged_snaps": "[1~1,6~1,8~3,11~2,17~2,1f~2,25~1,28~1,2c~5,32~4,37~1,39~7,41~5,47~16,5e~19,cb~1,ce~2,d4~7,dc~1,de~1,e6~4,102~1,105~6,10d~1,119~1,150~1,15d~2,160~3,16d~1,16f~5,178~1,184~2,194~1,1a2~1,1a5~1,1ac~2,1c7~1,1cb~2,1ce~1]",
"history": {
"epoch_created": 98,
"last_epoch_started": 17143,
"last_epoch_clean": 17541,
"last_epoch_split": 0,
"same_up_since": 17139,
"same_interval_since": 17142,
"same_primary_since": 17142,
"last_scrub": "17541'38035",
"last_scrub_stamp": "2015-08-18 07:37:13.983075",
"last_deep_scrub": "17541'38035",
"last_deep_scrub_stamp": "2015-08-18 07:37:13.983075",
"last_clean_scrub_stamp": "2015-08-06 06:24:06.552297"
},
"stats": {
"version": "17541'38040",
"reported_seq": "29406",
"reported_epoch": "17541",
"state": "active+clean+inconsistent",
"last_fresh": "2015-08-18 07:45:42.152254",
"last_change": "2015-08-18 07:37:13.983710",
"last_active": "2015-08-18 07:45:42.152254",
"last_peered": "2015-08-18 07:45:42.152254",
"last_clean": "2015-08-18 07:45:42.152254",
"last_became_active": "0.000000",
"last_became_peered": "0.000000",
"last_unstale": "2015-08-18 07:45:42.152254",
"last_undegraded": "2015-08-18 07:45:42.152254",
"last_fullsized": "2015-08-18 07:45:42.152254",
"mapping_epoch": 17139,
"log_start": "16745'34960",
"ondisk_log_start": "16745'34960",
"created": 98,
"last_epoch_clean": 17541,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "17541'38035",
"last_scrub_stamp": "2015-08-18 07:37:13.983075",
"last_deep_scrub": "17541'38035",
"last_deep_scrub_stamp": "2015-08-18 07:37:13.983075",
"last_clean_scrub_stamp": "2015-08-06 06:24:06.552297",
"log_size": 3080,
"ondisk_log_size": 3080,
"stats_invalid": "0",
"stat_sum": {
"num_bytes": 1308410368,
"num_objects": 318,
"num_object_clones": 4,
"num_object_copies": 954,
"num_objects_missing_on_primary": 0,
"num_objects_degraded": 0,
"num_objects_misplaced": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 318,
"num_whiteouts": 0,
"num_read": 19307,
"num_read_kb": 300579,
"num_write": 73378,
"num_write_kb": 2887251,
"num_scrub_errors": 17,
"num_shallow_scrub_errors": 17,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 2322,
"num_bytes_recovered": 9568430592,
"num_keys_recovered": 45,
"num_objects_omap": 1,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0
},
"up": [
56,
15,
29
],
"acting": [
56,
15,
29
],
"blocked_by": [],
"up_primary": 56,
"acting_primary": 56
},
"empty": 0,
"dne": 0,
"incomplete": 0,
"last_epoch_started": 17143,
"hit_set_history": {
"current_last_update": "0'0",
"current_last_stamp": "0.000000",
"current_info": {
"begin": "0.000000",
"end": "0.000000",
"version": "0'0"
},
"history": []
}
},
"peer_info": [
{
"peer": "15",
"pgid": "2.490",
"last_update": "17541'38040",
"last_complete": "17541'38040",
"log_tail": "16745'34855",
"last_user_version": 155325,
"last_backfill": "MAX",
"purged_snaps": "[1~1,6~1,8~3,11~2,17~2,1f~2,25~1,28~1,2c~5,32~4,37~1,39~7,41~5,47~16,5e~19,cb~1,ce~2,d4~7,dc~1,de~1,e6~4,102~1,105~6,10d~1,119~1,150~1,15d~2,160~3,16d~1,16f~5,178~1,184~2,194~1,1a2~1,1a5~1,1ac~2,1c7~1,1cb~2,1ce~1]",
"history": {
"epoch_created": 98,
"last_epoch_started": 17143,
"last_epoch_clean": 17541,
"last_epoch_split": 0,
"same_up_since": 17139,
"same_interval_since": 17142,
"same_primary_since": 17142,
"last_scrub": "17541'38035",
"last_scrub_stamp": "2015-08-18 07:37:13.983075",
"last_deep_scrub": "17541'38035",
"last_deep_scrub_stamp": "2015-08-18 07:37:13.983075",
"last_clean_scrub_stamp": "2015-08-06 06:24:06.552297"
},
"stats": {
"version": "17057'37860",
"reported_seq": "29201",
"reported_epoch": "17142",
"state": "active+remapped+inconsistent",
"last_fresh": "2015-08-18 02:54:29.982572",
"last_change": "2015-08-18 02:54:29.982572",
"last_active": "2015-08-18 02:54:29.982572",
"last_peered": "2015-08-18 02:54:29.982572",
"last_clean": "2015-08-18 02:45:44.150922",
"last_became_active": "0.000000",
"last_became_peered": "0.000000",
"last_unstale": "2015-08-18 02:54:29.982572",
"last_undegraded": "2015-08-18 02:54:29.982572",
"last_fullsized": "2015-08-18 02:54:29.982572",
"mapping_epoch": 17139,
"log_start": "16745'34855",
"ondisk_log_start": "16745'34855",
"created": 98,
"last_epoch_clean": 17141,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "17057'37860",
"last_scrub_stamp": "2015-08-18 02:47:56.695560",
"last_deep_scrub": "17057'37860",
"last_deep_scrub_stamp": "2015-08-18 02:47:56.695560",
"last_clean_scrub_stamp": "2015-08-06 06:24:06.552297",
"log_size": 3005,
"ondisk_log_size": 3005,
"stats_invalid": "0",
"stat_sum": {
"num_bytes": 1308410368,
"num_objects": 318,
"num_object_clones": 4,
"num_object_copies": 1590,
"num_objects_missing_on_primary": 0,
"num_objects_degraded": 0,
"num_objects_misplaced": 636,
"num_objects_unfound": 0,
"num_objects_dirty": 318,
"num_whiteouts": 0,
"num_read": 19124,
"num_read_kb": 299799,
"num_write": 73022,
"num_write_kb": 2883469,
"num_scrub_errors": 17,
"num_shallow_scrub_errors": 17,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 2322,
"num_bytes_recovered": 9568430592,
"num_keys_recovered": 45,
"num_objects_omap": 1,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0
},
"up": [
56,
15,
29
],
"acting": [
56,
15,
29
],
"blocked_by": [],
"up_primary": 56,
"acting_primary": 56
},
"empty": 0,
"dne": 0,
"incomplete": 0,
"last_epoch_started": 17143,
"hit_set_history": {
"current_last_update": "0'0",
"current_last_stamp": "0.000000",
"current_info": {
"begin": "0.000000",
"end": "0.000000",
"version": "0'0"
},
"history": []
}
},
{
"peer": "29",
"pgid": "2.490",
"last_update": "17541'38040",
"last_complete": "17541'38040",
"log_tail": "16745'34860",
"last_user_version": 0,
"last_backfill": "MAX",
"purged_snaps": "[1~1,6~1,8~3,11~2,17~2,1f~2,25~1,28~1,2c~5,32~4,37~1,39~7,41~5,47~16,5e~19,cb~1,ce~2,d4~7,dc~1,de~1,e6~4,102~1,105~6,10d~1,119~1,150~1,15d~2,160~3,16d~1,16f~5,178~1,184~2,194~1,1a2~1,1a5~1,1ac~2,1c7~1,1cb~2,1ce~1]",
"history": {
"epoch_created": 98,
"last_epoch_started": 17143,
"last_epoch_clean": 17541,
"last_epoch_split": 0,
"same_up_since": 17139,
"same_interval_since": 17142,
"same_primary_since": 17142,
"last_scrub": "17541'38035",
"last_scrub_stamp": "2015-08-18 07:37:13.983075",
"last_deep_scrub": "17541'38035",
"last_deep_scrub_stamp": "2015-08-18 07:37:13.983075",
"last_clean_scrub_stamp": "2015-08-06 06:24:06.552297"
},
"stats": {
"version": "17057'37860",
"reported_seq": "29198",
"reported_epoch": "17141",
"state": "active+remapped+inconsistent+backfilling",
"last_fresh": "2015-08-18 02:54:29.979923",
"last_change": "2015-08-18 02:53:45.383431",
"last_active": "2015-08-18 02:54:29.979923",
"last_peered": "2015-08-18 02:54:29.979923",
"last_clean": "2015-08-18 02:45:44.150922",
"last_became_active": "0.000000",
"last_became_peered": "0.000000",
"last_unstale": "2015-08-18 02:54:29.979923",
"last_undegraded": "2015-08-18 02:54:29.979923",
"last_fullsized": "2015-08-18 02:54:29.979923",
"mapping_epoch": 17139,
"log_start": "16745'34855",
"ondisk_log_start": "16745'34855",
"created": 98,
"last_epoch_clean": 17079,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "17057'37860",
"last_scrub_stamp": "2015-08-18 02:47:56.695560",
"last_deep_scrub": "17057'37860",
"last_deep_scrub_stamp": "2015-08-18 02:47:56.695560",
"last_clean_scrub_stamp": "2015-08-06 06:24:06.552297",
"log_size": 3005,
"ondisk_log_size": 3005,
"stats_invalid": "0",
"stat_sum": {
"num_bytes": 1308410368,
"num_objects": 318,
"num_object_clones": 4,
"num_object_copies": 1590,
"num_objects_missing_on_primary": 0,
"num_objects_degraded": 0,
"num_objects_misplaced": 638,
"num_objects_unfound": 0,
"num_objects_dirty": 318,
"num_whiteouts": 0,
"num_read": 19124,
"num_read_kb": 299799,
"num_write": 73022,
"num_write_kb": 2883469,
"num_scrub_errors": 17,
"num_shallow_scrub_errors": 17,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 2322,
"num_bytes_recovered": 9568430592,
"num_keys_recovered": 45,
"num_objects_omap": 1,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0
},
"up": [
56,
15,
29
],
"acting": [
56,
15,
29
],
"blocked_by": [],
"up_primary": 56,
"acting_primary": 56
},
"empty": 0,
"dne": 0,
"incomplete": 0,
"last_epoch_started": 17143,
"hit_set_history": {
"current_last_update": "0'0",
"current_last_stamp": "0.000000",
"current_info": {
"begin": "0.000000",
"end": "0.000000",
"version": "0'0"
},
"history": []
}
}
],
"recovery_state": [
{
"name": "Started\/Primary\/Active",
"enter_time": "2015-08-18 02:54:32.017624",
"might_have_unfound": [],
"recovery_progress": {
"backfill_targets": [],
"waiting_on_backfill": [],
"last_backfill_started": "0\/\/0\/\/-1",
"backfill_info": {
"begin": "0\/\/0\/\/-1",
"end": "0\/\/0\/\/-1",
"objects": []
},
"peer_backfill_info": [],
"backfills_in_flight": [],
"recovering": [],
"pg_backend": {
"pull_from_peer": [],
"pushing": []
}
},
"scrub": {
"scrubber.epoch_start": "17142",
"scrubber.active": 0,
"scrubber.waiting_on": 0,
"scrubber.waiting_on_whom": []
}
},
{
"name": "Started",
"enter_time": "2015-08-18 02:54:30.940793"
}
],
"agent_state": {}
}
2015-08-18 8:40 GMT+03:00 Irek Fasikhov <malmyzh@xxxxxxxxx>:
Да просто recovery и repair немного разные вещи ;).Ну да ладно. не поняли просто друг друга.У тебя чему равен параметр min_size на пуле? и так же покажи вывод ceph pg $PG queryС уважением, Фасихов Ирек НургаязовичМоб.: +792290457572015-08-18 8:36 GMT+03:00 Voloshanenko Igor <igor.voloshanenko@xxxxxxxxx>:Hi Irek, Please read careful )))You proposal was the first, i try to do... That's why i asked about help... (2015-08-18 8:34 GMT+03:00 Irek Fasikhov <malmyzh@xxxxxxxxx>:Hi, Igor.You need to repair the PG.for i in `ceph pg dump| grep inconsistent | grep -v 'inconsistent+repair' | awk {'print$1'}`;do ceph pg repair $i;doneС уважением, Фасихов Ирек НургаязовичМоб.: +792290457572015-08-18 8:27 GMT+03:00 Voloshanenko Igor <igor.voloshanenko@xxxxxxxxx>:_______________________________________________Hi all, at our production cluster, due high rebalancing ((( we have 2 pgs in inconsistent state...root@temp:~# ceph health detail | grep incHEALTH_ERR 2 pgs inconsistent; 18 scrub errorspg 2.490 is active+clean+inconsistent, acting [56,15,29]pg 2.c4 is active+clean+inconsistent, acting [56,10,42]From OSD logs, after recovery attempt:root@test:~# ceph pg dump | grep -i incons | cut -f 1 | while read i; do ceph pg repair ${i} ; donedumped all in format plaininstructing pg 2.490 on osd.56 to repairinstructing pg 2.c4 on osd.56 to repair/var/log/ceph/ceph-osd.56.log:51:2015-08-18 07:26:37.035910 7f94663b3700 -1 log_channel(cluster) log [ERR] : deep-scrub 2.490 f5759490/rbd_data.1631755377d7e.00000000000004da/head//2 expected clone 90c59490/rbd_data.eb486436f2beb.0000000000007a65/141//2/var/log/ceph/ceph-osd.56.log:52:2015-08-18 07:26:37.035960 7f94663b3700 -1 log_channel(cluster) log [ERR] : deep-scrub 2.490 fee49490/rbd_data.12483d3ba0794b.000000000000522f/head//2 expected clone f5759490/rbd_data.1631755377d7e.00000000000004da/141//2/var/log/ceph/ceph-osd.56.log:53:2015-08-18 07:26:37.036133 7f94663b3700 -1 log_channel(cluster) log [ERR] : deep-scrub 2.490 a9b39490/rbd_data.12483d3ba0794b.00000000000037b3/head//2 expected clone fee49490/rbd_data.12483d3ba0794b.000000000000522f/141//2/var/log/ceph/ceph-osd.56.log:54:2015-08-18 07:26:37.036243 7f94663b3700 -1 log_channel(cluster) log [ERR] : deep-scrub 2.490 bac19490/rbd_data.1238e82ae8944a.000000000000032e/head//2 expected clone a9b39490/rbd_data.12483d3ba0794b.00000000000037b3/141//2/var/log/ceph/ceph-osd.56.log:55:2015-08-18 07:26:37.036289 7f94663b3700 -1 log_channel(cluster) log [ERR] : deep-scrub 2.490 98519490/rbd_data.123e9c2ae8944a.0000000000000807/head//2 expected clone bac19490/rbd_data.1238e82ae8944a.000000000000032e/141//2/var/log/ceph/ceph-osd.56.log:56:2015-08-18 07:26:37.036314 7f94663b3700 -1 log_channel(cluster) log [ERR] : deep-scrub 2.490 c3c09490/rbd_data.1238e82ae8944a.0000000000000c2b/head//2 expected clone 98519490/rbd_data.123e9c2ae8944a.0000000000000807/141//2/var/log/ceph/ceph-osd.56.log:57:2015-08-18 07:26:37.036363 7f94663b3700 -1 log_channel(cluster) log [ERR] : deep-scrub 2.490 28809490/rbd_data.edea7460fe42b.00000000000001d9/head//2 expected clone c3c09490/rbd_data.1238e82ae8944a.0000000000000c2b/141//2/var/log/ceph/ceph-osd.56.log:58:2015-08-18 07:26:37.036432 7f94663b3700 -1 log_channel(cluster) log [ERR] : deep-scrub 2.490 e1509490/rbd_data.1423897545e146.00000000000009a6/head//2 expected clone 28809490/rbd_data.edea7460fe42b.00000000000001d9/141//2/var/log/ceph/ceph-osd.56.log:59:2015-08-18 07:26:38.548765 7f94663b3700 -1 log_channel(cluster) log [ERR] : 2.490 deep-scrub 17 errorsSo, how i can solve "expected clone" situation by hand?Thank in advance!
ceph-users mailing list
ceph-users@xxxxxxxxxxxxxx
http://lists.ceph.com/listinfo.cgi/ceph-users-ceph.com
_______________________________________________ ceph-users mailing list ceph-users@xxxxxxxxxxxxxx http://lists.ceph.com/listinfo.cgi/ceph-users-ceph.com
