Good morning,
the osd.61 actually just crashed and the disk is still intact. However,
after 8 hours of rebuilding, the unfound objects are still missing:
root@server1:~# ceph -s
cluster:
id: 26c0c5a8-d7ce-49ac-b5a7-bfd9d0ba81ab
health: HEALTH_WARN
noscrub,nodeep-scrub flag(s) set
111436/3017766 objects misplaced (3.693%)
9377/1005922 objects unfound (0.932%)
Reduced data availability: 84 pgs inactive
Degraded data redundancy: 277034/3017766 objects degraded (9.180%), 84 pgs unclean, 84 pgs degraded, 84 pgs undersized
mon server2 is low on available space
services:
mon: 3 daemons, quorum server5,server3,server2
mgr: server5(active), standbys: server2, 2, 0, server3
osd: 54 osds: 54 up, 54 in; 84 remapped pgs
flags noscrub,nodeep-scrub
data:
pools: 3 pools, 1344 pgs
objects: 982k objects, 3837 GB
usage: 10618 GB used, 39030 GB / 49648 GB avail
pgs: 6.250% pgs not active
277034/3017766 objects degraded (9.180%)
111436/3017766 objects misplaced (3.693%)
9377/1005922 objects unfound (0.932%)
1260 active+clean
84 recovery_wait+undersized+degraded+remapped+peered
io:
client: 68960 B/s rd, 20722 kB/s wr, 12 op/s rd, 77 op/s wr
We tried restarting osd.61, but ceph health detail does not change
anymore:
HEALTH_WARN noscrub,nodeep-scrub flag(s) set; 111436/3017886 objects misplaced (3.69
3%); 9377/1005962 objects unfound (0.932%); Reduced data availability: 84 pgs inacti
ve; Degraded data redundancy: 277034/3017886 objects degraded (9.180%), 84 pgs uncle
an, 84 pgs degraded, 84 pgs undersized; mon server2 is low on available space
OSDMAP_FLAGS noscrub,nodeep-scrub flag(s) set
OBJECT_MISPLACED 111436/3017886 objects misplaced (3.693%)
OBJECT_UNFOUND 9377/1005962 objects unfound (0.932%)
pg 4.fa has 117 unfound objects
pg 4.ff has 107 unfound objects
pg 4.fd has 113 unfound objects
...
pg 4.2a has 108 unfound objects
PG_AVAILABILITY Reduced data availability: 84 pgs inactive
pg 4.2a is stuck inactive for 64117.189552, current state recovery_wait+undersiz
ed+degraded+remapped+peered, last acting [61]
pg 4.31 is stuck inactive for 64117.147636, current state recovery_wait+undersiz
ed+degraded+remapped+peered, last acting [61]
pg 4.32 is stuck inactive for 64117.178461, current state recovery_wait+undersiz
ed+degraded+remapped+peered, last acting [61]
pg 4.34 is stuck inactive for 64117.150475, current state recovery_wait+undersiz
ed+degraded+remapped+peered, last acting [61]
...
PG_DEGRADED Degraded data redundancy: 277034/3017886 objects degraded (9.180%), 84 pgs unclean, 84 pgs degraded, 84 pgs undersized
pg 4.2a is stuck unclean for 131612.984555, current state recovery_wait+undersized+degraded+remapped+peered, last acting [61]
pg 4.31 is stuck undersized for 221.568468, current state recovery_wait+undersized+degraded+remapped+peered, last acting [61]
Is there any chance to recover those pgs or did we actually lose data
with a 2 disk failure?
And is there any way out of this besides going with
ceph pg {pg-id} mark_unfound_lost revert|delete
?
Best,
Nico
p.s.: the ceph 4.2a query:
{
"state": "recovery_wait+undersized+degraded+remapped+peered",
"snap_trimq": "[]",
"epoch": 17879,
"up": [
17,
13,
25
],
"acting": [
61
],
"backfill_targets": [
"13",
"17",
"25"
],
"actingbackfill": [
"13",
"17",
"25",
"61"
],
"info": {
"pgid": "4.2a",
"last_update": "17529'53875",
"last_complete": "17217'45447",
"log_tail": "17090'43812",
"last_user_version": 53875,
"last_backfill": "MAX",
"last_backfill_bitwise": 0,
"purged_snaps": [
{
"start": "1",
"length": "3"
},
{
"start": "6",
"length": "8"
},
{
"start": "10",
"length": "2"
}
],
"history": {
"epoch_created": 9134,
"epoch_pool_created": 9134,
"last_epoch_started": 17528,
"last_interval_started": 17527,
"last_epoch_clean": 17079,
"last_interval_clean": 17078,
"last_epoch_split": 0,
"last_epoch_marked_full": 0,
"same_up_since": 17143,
"same_interval_since": 17878,
"same_primary_since": 17878,
"last_scrub": "17090'44622",
"last_scrub_stamp": "2018-01-21 09:37:09.888508",
"last_deep_scrub": "17090'42219",
"last_deep_scrub_stamp": "2018-01-20 05:05:45.372052",
"last_clean_scrub_stamp": "2018-01-21 09:37:09.888508"
},
"stats": {
"version": "17529'53875",
"reported_seq": "2930796",
"reported_epoch": "17879",
"state": "recovery_wait+undersized+degraded+remapped+peered",
"last_fresh": "2018-01-23 08:51:45.177851",
"last_change": "2018-01-23 08:51:45.177851",
"last_active": "2018-01-22 15:00:44.639725",
"last_peered": "2018-01-23 08:51:45.177851",
"last_clean": "2018-01-21 20:15:48.844722",
"last_became_active": "2018-01-22 14:53:07.957356",
"last_became_peered": "2018-01-23 08:45:40.566385",
"last_unstale": "2018-01-23 08:51:45.177851",
"last_undegraded": "2018-01-23 08:45:40.268969",
"last_fullsized": "2018-01-23 08:45:40.268969",
"mapping_epoch": 17878,
"log_start": "17090'43812",
"ondisk_log_start": "17090'43812",
"created": 9134,
"last_epoch_clean": 17079,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "17090'44622",
"last_scrub_stamp": "2018-01-21 09:37:09.888508",
"last_deep_scrub": "17090'42219",
"last_deep_scrub_stamp": "2018-01-20 05:05:45.372052",
"last_clean_scrub_stamp": "2018-01-21 09:37:09.888508",
"log_size": 10063,
"ondisk_log_size": 10063,
"stats_invalid": false,
"dirty_stats_invalid": false,
"omap_stats_invalid": false,
"hitset_stats_invalid": false,
"hitset_bytes_stats_invalid": false,
"pin_stats_invalid": false,
"stat_sum": {
"num_bytes": 6580056064,
"num_objects": 1631,
"num_object_clones": 0,
"num_object_copies": 4893,
"num_objects_missing_on_primary": 108,
"num_objects_missing": 0,
"num_objects_degraded": 3370,
"num_objects_misplaced": 1523,
"num_objects_unfound": 108,
"num_objects_dirty": 1631,
"num_whiteouts": 0,
"num_read": 2876946,
"num_read_kb": 11999400,
"num_write": 56903,
"num_write_kb": 9930660,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 402,
"num_bytes_recovered": 1574608896,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0,
"num_flush": 0,
"num_flush_kb": 0,
"num_evict": 0,
"num_evict_kb": 0,
"num_promote": 0,
"num_flush_mode_high": 0,
"num_flush_mode_low": 0,
"num_evict_mode_some": 0,
"num_evict_mode_full": 0,
"num_objects_pinned": 0,
"num_legacy_snapsets": 0
},
"up": [
17,
13,
25
],
"acting": [
61
],
"blocked_by": [],
"up_primary": 17,
"acting_primary": 61
},
"empty": 0,
"dne": 0,
"incomplete": 0,
"last_epoch_started": 17528,
"hit_set_history": {
"current_last_update": "0'0",
"history": []
}
},
"peer_info": [
{
"peer": "8",
"pgid": "4.2a",
"last_update": "17101'44876",
"last_complete": "17101'44876",
"log_tail": "17090'43371",
"last_user_version": 44876,
"last_backfill": "MIN",
"last_backfill_bitwise": 1,
"purged_snaps": [
{
"start": "1",
"length": "3"
},
{
"start": "6",
"length": "8"
},
{
"start": "10",
"length": "2"
}
],
"history": {
"epoch_created": 9134,
"epoch_pool_created": 9134,
"last_epoch_started": 17528,
"last_interval_started": 17527,
"last_epoch_clean": 17079,
"last_interval_clean": 17078,
"last_epoch_split": 0,
"last_epoch_marked_full": 0,
"same_up_since": 17143,
"same_interval_since": 17878,
"same_primary_since": 17878,
"last_scrub": "17090'44622",
"last_scrub_stamp": "2018-01-21 09:37:09.888508",
"last_deep_scrub": "17090'42219",
"last_deep_scrub_stamp": "2018-01-20 05:05:45.372052",
"last_clean_scrub_stamp": "2018-01-21 09:37:09.888508"
},
"stats": {
"version": "0'0",
"reported_seq": "0",
"reported_epoch": "0",
"state": "unknown",
"last_fresh": "0.000000",
"last_change": "0.000000",
"last_active": "0.000000",
"last_peered": "0.000000",
"last_clean": "0.000000",
"last_became_active": "0.000000",
"last_became_peered": "0.000000",
"last_unstale": "0.000000",
"last_undegraded": "0.000000",
"last_fullsized": "0.000000",
"mapping_epoch": 17878,
"log_start": "0'0",
"ondisk_log_start": "0'0",
"created": 0,
"last_epoch_clean": 0,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "0'0",
"last_scrub_stamp": "0.000000",
"last_deep_scrub": "0'0",
"last_deep_scrub_stamp": "0.000000",
"last_clean_scrub_stamp": "0.000000",
"log_size": 0,
"ondisk_log_size": 0,
"stats_invalid": false,
"dirty_stats_invalid": false,
"omap_stats_invalid": false,
"hitset_stats_invalid": false,
"hitset_bytes_stats_invalid": false,
"pin_stats_invalid": false,
"stat_sum": {
"num_bytes": 0,
"num_objects": 0,
"num_object_clones": 0,
"num_object_copies": 0,
"num_objects_missing_on_primary": 0,
"num_objects_missing": 0,
"num_objects_degraded": 0,
"num_objects_misplaced": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 0,
"num_whiteouts": 0,
"num_read": 0,
"num_read_kb": 0,
"num_write": 0,
"num_write_kb": 0,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 0,
"num_bytes_recovered": 0,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0,
"num_flush": 0,
"num_flush_kb": 0,
"num_evict": 0,
"num_evict_kb": 0,
"num_promote": 0,
"num_flush_mode_high": 0,
"num_flush_mode_low": 0,
"num_evict_mode_some": 0,
"num_evict_mode_full": 0,
"num_objects_pinned": 0,
"num_legacy_snapsets": 0
},
"up": [
17,
13,
25
],
"acting": [
61
],
"blocked_by": [],
"up_primary": 17,
"acting_primary": 61
},
"empty": 0,
"dne": 0,
"incomplete": 1,
"last_epoch_started": 17099,
"hit_set_history": {
"current_last_update": "0'0",
"history": []
}
},
{
"peer": "13",
"pgid": "4.2a",
"last_update": "17529'53875",
"last_complete": "17529'53875",
"log_tail": "17090'43812",
"last_user_version": 53875,
"last_backfill": "MIN",
"last_backfill_bitwise": 1,
"purged_snaps": [
{
"start": "1",
"length": "3"
},
{
"start": "6",
"length": "8"
},
{
"start": "10",
"length": "2"
}
],
"history": {
"epoch_created": 9134,
"epoch_pool_created": 9134,
"last_epoch_started": 17528,
"last_interval_started": 17527,
"last_epoch_clean": 17079,
"last_interval_clean": 17078,
"last_epoch_split": 0,
"last_epoch_marked_full": 0,
"same_up_since": 17143,
"same_interval_since": 17878,
"same_primary_since": 17878,
"last_scrub": "17090'44622",
"last_scrub_stamp": "2018-01-21 09:37:09.888508",
"last_deep_scrub": "17090'42219",
"last_deep_scrub_stamp": "2018-01-20 05:05:45.372052",
"last_clean_scrub_stamp": "2018-01-21 09:37:09.888508"
},
"stats": {
"version": "0'0",
"reported_seq": "0",
"reported_epoch": "0",
"state": "unknown",
"last_fresh": "0.000000",
"last_change": "0.000000",
"last_active": "0.000000",
"last_peered": "0.000000",
"last_clean": "0.000000",
"last_became_active": "0.000000",
"last_became_peered": "0.000000",
"last_unstale": "0.000000",
"last_undegraded": "0.000000",
"last_fullsized": "0.000000",
"mapping_epoch": 17878,
"log_start": "0'0",
"ondisk_log_start": "0'0",
"created": 0,
"last_epoch_clean": 0,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "0'0",
"last_scrub_stamp": "0.000000",
"last_deep_scrub": "0'0",
"last_deep_scrub_stamp": "0.000000",
"last_clean_scrub_stamp": "0.000000",
"log_size": 0,
"ondisk_log_size": 0,
"stats_invalid": false,
"dirty_stats_invalid": false,
"omap_stats_invalid": false,
"hitset_stats_invalid": false,
"hitset_bytes_stats_invalid": false,
"pin_stats_invalid": false,
"stat_sum": {
"num_bytes": 0,
"num_objects": 0,
"num_object_clones": 0,
"num_object_copies": 0,
"num_objects_missing_on_primary": 0,
"num_objects_missing": 0,
"num_objects_degraded": 0,
"num_objects_misplaced": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 0,
"num_whiteouts": 0,
"num_read": 0,
"num_read_kb": 0,
"num_write": 0,
"num_write_kb": 0,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 0,
"num_bytes_recovered": 0,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0,
"num_flush": 0,
"num_flush_kb": 0,
"num_evict": 0,
"num_evict_kb": 0,
"num_promote": 0,
"num_flush_mode_high": 0,
"num_flush_mode_low": 0,
"num_evict_mode_some": 0,
"num_evict_mode_full": 0,
"num_objects_pinned": 0,
"num_legacy_snapsets": 0
},
"up": [
17,
13,
25
],
"acting": [
61
],
"blocked_by": [],
"up_primary": 17,
"acting_primary": 61
},
"empty": 0,
"dne": 0,
"incomplete": 1,
"last_epoch_started": 17528,
"hit_set_history": {
"current_last_update": "0'0",
"history": []
}
},
{
"peer": "17",
"pgid": "4.2a",
"last_update": "17529'53875",
"last_complete": "17529'53875",
"log_tail": "17090'43812",
"last_user_version": 53875,
"last_backfill": "MIN",
"last_backfill_bitwise": 1,
"purged_snaps": [
{
"start": "1",
"length": "3"
},
{
"start": "6",
"length": "8"
},
{
"start": "10",
"length": "2"
}
],
"history": {
"epoch_created": 9134,
"epoch_pool_created": 9134,
"last_epoch_started": 17528,
"last_interval_started": 17527,
"last_epoch_clean": 17079,
"last_interval_clean": 17078,
"last_epoch_split": 0,
"last_epoch_marked_full": 0,
"same_up_since": 17143,
"same_interval_since": 17878,
"same_primary_since": 17878,
"last_scrub": "17090'44622",
"last_scrub_stamp": "2018-01-21 09:37:09.888508",
"last_deep_scrub": "17090'42219",
"last_deep_scrub_stamp": "2018-01-20 05:05:45.372052",
"last_clean_scrub_stamp": "2018-01-21 09:37:09.888508"
},
"stats": {
"version": "17529'53875",
"reported_seq": "61",
"reported_epoch": "17877",
"state": "peering",
"last_fresh": "2018-01-23 08:45:38.255735",
"last_change": "2018-01-23 08:45:38.225305",
"last_active": "0.000000",
"last_peered": "0.000000",
"last_clean": "0.000000",
"last_became_active": "0.000000",
"last_became_peered": "0.000000",
"last_unstale": "2018-01-23 08:45:38.255735",
"last_undegraded": "2018-01-23 08:45:38.255735",
"last_fullsized": "2018-01-23 08:45:38.255735",
"mapping_epoch": 17878,
"log_start": "17090'43812",
"ondisk_log_start": "17090'43812",
"created": 9134,
"last_epoch_clean": 17079,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "17090'44622",
"last_scrub_stamp": "2018-01-21 09:37:09.888508",
"last_deep_scrub": "17090'42219",
"last_deep_scrub_stamp": "2018-01-20 05:05:45.372052",
"last_clean_scrub_stamp": "2018-01-21 09:37:09.888508",
"log_size": 10063,
"ondisk_log_size": 10063,
"stats_invalid": false,
"dirty_stats_invalid": false,
"omap_stats_invalid": false,
"hitset_stats_invalid": false,
"hitset_bytes_stats_invalid": false,
"pin_stats_invalid": false,
"stat_sum": {
"num_bytes": 0,
"num_objects": 0,
"num_object_clones": 0,
"num_object_copies": 0,
"num_objects_missing_on_primary": 0,
"num_objects_missing": 0,
"num_objects_degraded": 0,
"num_objects_misplaced": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 0,
"num_whiteouts": 0,
"num_read": 0,
"num_read_kb": 0,
"num_write": 0,
"num_write_kb": 0,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 0,
"num_bytes_recovered": 0,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0,
"num_flush": 0,
"num_flush_kb": 0,
"num_evict": 0,
"num_evict_kb": 0,
"num_promote": 0,
"num_flush_mode_high": 0,
"num_flush_mode_low": 0,
"num_evict_mode_some": 0,
"num_evict_mode_full": 0,
"num_objects_pinned": 0,
"num_legacy_snapsets": 0
},
"up": [
17,
13,
25
],
"acting": [
61
],
"blocked_by": [],
"up_primary": 17,
"acting_primary": 61
},
"empty": 0,
"dne": 0,
"incomplete": 1,
"last_epoch_started": 17528,
"hit_set_history": {
"current_last_update": "0'0",
"history": []
}
},
{
"peer": "21",
"pgid": "4.2a",
"last_update": "17116'44885",
"last_complete": "17116'44885",
"log_tail": "17090'43385",
"last_user_version": 0,
"last_backfill": "MIN",
"last_backfill_bitwise": 1,
"purged_snaps": [
{
"start": "1",
"length": "3"
},
{
"start": "6",
"length": "8"
},
{
"start": "10",
"length": "2"
}
],
"history": {
"epoch_created": 9134,
"epoch_pool_created": 9134,
"last_epoch_started": 17137,
"last_interval_started": 17136,
"last_epoch_clean": 17079,
"last_interval_clean": 17078,
"last_epoch_split": 0,
"last_epoch_marked_full": 0,
"same_up_since": 17143,
"same_interval_since": 17878,
"same_primary_since": 17878,
"last_scrub": "17090'44622",
"last_scrub_stamp": "2018-01-21 09:37:09.888508",
"last_deep_scrub": "17090'42219",
"last_deep_scrub_stamp": "2018-01-20 05:05:45.372052",
"last_clean_scrub_stamp": "2018-01-21 09:37:09.888508"
},
"stats": {
"version": "0'0",
"reported_seq": "0",
"reported_epoch": "0",
"state": "unknown",
"last_fresh": "0.000000",
"last_change": "0.000000",
"last_active": "0.000000",
"last_peered": "0.000000",
"last_clean": "0.000000",
"last_became_active": "0.000000",
"last_became_peered": "0.000000",
"last_unstale": "0.000000",
"last_undegraded": "0.000000",
"last_fullsized": "0.000000",
"mapping_epoch": 17878,
"log_start": "0'0",
"ondisk_log_start": "0'0",
"created": 0,
"last_epoch_clean": 0,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "0'0",
"last_scrub_stamp": "0.000000",
"last_deep_scrub": "0'0",
"last_deep_scrub_stamp": "0.000000",
"last_clean_scrub_stamp": "0.000000",
"log_size": 0,
"ondisk_log_size": 0,
"stats_invalid": false,
"dirty_stats_invalid": false,
"omap_stats_invalid": false,
"hitset_stats_invalid": false,
"hitset_bytes_stats_invalid": false,
"pin_stats_invalid": false,
"stat_sum": {
"num_bytes": 0,
"num_objects": 0,
"num_object_clones": 0,
"num_object_copies": 0,
"num_objects_missing_on_primary": 0,
"num_objects_missing": 0,
"num_objects_degraded": 0,
"num_objects_misplaced": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 0,
"num_whiteouts": 0,
"num_read": 0,
"num_read_kb": 0,
"num_write": 0,
"num_write_kb": 0,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 0,
"num_bytes_recovered": 0,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0,
"num_flush": 0,
"num_flush_kb": 0,
"num_evict": 0,
"num_evict_kb": 0,
"num_promote": 0,
"num_flush_mode_high": 0,
"num_flush_mode_low": 0,
"num_evict_mode_some": 0,
"num_evict_mode_full": 0,
"num_objects_pinned": 0,
"num_legacy_snapsets": 0
},
"up": [
17,
13,
25
],
"acting": [
61
],
"blocked_by": [],
"up_primary": 17,
"acting_primary": 61
},
"empty": 0,
"dne": 0,
"incomplete": 1,
"last_epoch_started": 17135,
"hit_set_history": {
"current_last_update": "0'0",
"history": []
}
},
{
"peer": "24",
"pgid": "4.2a",
"last_update": "17138'45024",
"last_complete": "17138'45024",
"log_tail": "17090'43385",
"last_user_version": 45024,
"last_backfill": "MIN",
"last_backfill_bitwise": 1,
"purged_snaps": [
{
"start": "1",
"length": "3"
},
{
"start": "6",
"length": "8"
},
{
"start": "10",
"length": "2"
}
],
"history": {
"epoch_created": 9134,
"epoch_pool_created": 9134,
"last_epoch_started": 17137,
"last_interval_started": 17136,
"last_epoch_clean": 17079,
"last_interval_clean": 17078,
"last_epoch_split": 0,
"last_epoch_marked_full": 0,
"same_up_since": 17143,
"same_interval_since": 17878,
"same_primary_since": 17878,
"last_scrub": "17090'44622",
"last_scrub_stamp": "2018-01-21 09:37:09.888508",
"last_deep_scrub": "17090'42219",
"last_deep_scrub_stamp": "2018-01-20 05:05:45.372052",
"last_clean_scrub_stamp": "2018-01-21 09:37:09.888508"
},
"stats": {
"version": "0'0",
"reported_seq": "0",
"reported_epoch": "0",
"state": "unknown",
"last_fresh": "0.000000",
"last_change": "0.000000",
"last_active": "0.000000",
"last_peered": "0.000000",
"last_clean": "0.000000",
"last_became_active": "0.000000",
"last_became_peered": "0.000000",
"last_unstale": "0.000000",
"last_undegraded": "0.000000",
"last_fullsized": "0.000000",
"mapping_epoch": 17878,
"log_start": "0'0",
"ondisk_log_start": "0'0",
"created": 0,
"last_epoch_clean": 0,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "0'0",
"last_scrub_stamp": "0.000000",
"last_deep_scrub": "0'0",
"last_deep_scrub_stamp": "0.000000",
"last_clean_scrub_stamp": "0.000000",
"log_size": 0,
"ondisk_log_size": 0,
"stats_invalid": false,
"dirty_stats_invalid": false,
"omap_stats_invalid": false,
"hitset_stats_invalid": false,
"hitset_bytes_stats_invalid": false,
"pin_stats_invalid": false,
"stat_sum": {
"num_bytes": 0,
"num_objects": 0,
"num_object_clones": 0,
"num_object_copies": 0,
"num_objects_missing_on_primary": 0,
"num_objects_missing": 0,
"num_objects_degraded": 0,
"num_objects_misplaced": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 0,
"num_whiteouts": 0,
"num_read": 0,
"num_read_kb": 0,
"num_write": 0,
"num_write_kb": 0,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 0,
"num_bytes_recovered": 0,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0,
"num_flush": 0,
"num_flush_kb": 0,
"num_evict": 0,
"num_evict_kb": 0,
"num_promote": 0,
"num_flush_mode_high": 0,
"num_flush_mode_low": 0,
"num_evict_mode_some": 0,
"num_evict_mode_full": 0,
"num_objects_pinned": 0,
"num_legacy_snapsets": 0
},
"up": [
17,
13,
25
],
"acting": [
61
],
"blocked_by": [],
"up_primary": 17,
"acting_primary": 61
},
"empty": 0,
"dne": 0,
"incomplete": 1,
"last_epoch_started": 17137,
"hit_set_history": {
"current_last_update": "0'0",
"history": []
}
},
{
"peer": "25",
"pgid": "4.2a",
"last_update": "17529'53875",
"last_complete": "17529'53875",
"log_tail": "17090'43812",
"last_user_version": 53875,
"last_backfill": "MIN",
"last_backfill_bitwise": 1,
"purged_snaps": [
{
"start": "1",
"length": "3"
},
{
"start": "6",
"length": "8"
},
{
"start": "10",
"length": "2"
}
],
"history": {
"epoch_created": 9134,
"epoch_pool_created": 9134,
"last_epoch_started": 17528,
"last_interval_started": 17527,
"last_epoch_clean": 17079,
"last_interval_clean": 17078,
"last_epoch_split": 0,
"last_epoch_marked_full": 0,
"same_up_since": 17143,
"same_interval_since": 17878,
"same_primary_since": 17878,
"last_scrub": "17090'44622",
"last_scrub_stamp": "2018-01-21 09:37:09.888508",
"last_deep_scrub": "17090'42219",
"last_deep_scrub_stamp": "2018-01-20 05:05:45.372052",
"last_clean_scrub_stamp": "2018-01-21 09:37:09.888508"
},
"stats": {
"version": "0'0",
"reported_seq": "0",
"reported_epoch": "0",
"state": "unknown",
"last_fresh": "0.000000",
"last_change": "0.000000",
"last_active": "0.000000",
"last_peered": "0.000000",
"last_clean": "0.000000",
"last_became_active": "0.000000",
"last_became_peered": "0.000000",
"last_unstale": "0.000000",
"last_undegraded": "0.000000",
"last_fullsized": "0.000000",
"mapping_epoch": 17878,
"log_start": "0'0",
"ondisk_log_start": "0'0",
"created": 0,
"last_epoch_clean": 0,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "0'0",
"last_scrub_stamp": "0.000000",
"last_deep_scrub": "0'0",
"last_deep_scrub_stamp": "0.000000",
"last_clean_scrub_stamp": "0.000000",
"log_size": 0,
"ondisk_log_size": 0,
"stats_invalid": false,
"dirty_stats_invalid": false,
"omap_stats_invalid": false,
"hitset_stats_invalid": false,
"hitset_bytes_stats_invalid": false,
"pin_stats_invalid": false,
"stat_sum": {
"num_bytes": 0,
"num_objects": 0,
"num_object_clones": 0,
"num_object_copies": 0,
"num_objects_missing_on_primary": 0,
"num_objects_missing": 0,
"num_objects_degraded": 0,
"num_objects_misplaced": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 0,
"num_whiteouts": 0,
"num_read": 0,
"num_read_kb": 0,
"num_write": 0,
"num_write_kb": 0,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 0,
"num_bytes_recovered": 0,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0,
"num_flush": 0,
"num_flush_kb": 0,
"num_evict": 0,
"num_evict_kb": 0,
"num_promote": 0,
"num_flush_mode_high": 0,
"num_flush_mode_low": 0,
"num_evict_mode_some": 0,
"num_evict_mode_full": 0,
"num_objects_pinned": 0,
"num_legacy_snapsets": 0
},
"up": [
17,
13,
25
],
"acting": [
61
],
"blocked_by": [],
"up_primary": 17,
"acting_primary": 61
},
"empty": 0,
"dne": 0,
"incomplete": 1,
"last_epoch_started": 17528,
"hit_set_history": {
"current_last_update": "0'0",
"history": []
}
}
],
"recovery_state": [
{
"name": "Started/Primary/Active",
"enter_time": "2018-01-23 08:45:40.269014",
"might_have_unfound": [
{
"osd": "8",
"status": "already probed"
},
{
"osd": "13",
"status": "already probed"
},
{
"osd": "17",
"status": "already probed"
},
{
"osd": "21",
"status": "already probed"
},
{
"osd": "24",
"status": "already probed"
},
{
"osd": "25",
"status": "already probed"
},
{
"osd": "63",
"status": "osd is down"
},
{
"osd": "64",
"status": "osd is down"
}
],
"recovery_progress": {
"backfill_targets": [
"13",
"17",
"25"
],
"waiting_on_backfill": [],
"last_backfill_started": "MIN",
"backfill_info": {
"begin": "MIN",
"end": "MIN",
"objects": []
},
"peer_backfill_info": [],
"backfills_in_flight": [],
"recovering": [],
"pg_backend": {
"pull_from_peer": [],
"pushing": []
}
},
"scrub": {
"scrubber.epoch_start": "0",
"scrubber.active": false,
"scrubber.state": "INACTIVE",
"scrubber.start": "MIN",
"scrubber.end": "MIN",
"scrubber.subset_last_update": "0'0",
"scrubber.deep": false,
"scrubber.seed": 0,
"scrubber.waiting_on": 0,
"scrubber.waiting_on_whom": []
}
},
{
"name": "Started",
"enter_time": "2018-01-23 08:45:40.237202"
}
],
"agent_state": {}
}
David Turner <drakonstein@xxxxxxxxx> writes:
> Weight the remaining disks you added to 0.0. They seem to be a bad batch.
> This will start moving their data off of them and back onto the rest of the
> cluster. I generally suggest not to add storage in more than what you can
> afford to lose, unless you trust your burn-in process. So if you have a
> host failure domain and size=3, I wouldn't add storage in more than 2 nodes
> at a time in case the disks die. That way you are much less likely to have
> scares.
>
> I assume this disk was in a third node leaving you with 3 failed disks
> across 3 hosts? It doesn't seem like these drives are going to work out
> and I would immediately weight all newly added disks to 0.0 and get back to
> a point where you are no longer backfilling/recovering PGs and see where
> things are at from there.
>
> On Mon, Jan 22, 2018 at 1:33 PM Nico Schottelius <
> nico.schottelius@xxxxxxxxxxx> wrote:
>
>>
>> While writing, yet another disk (osd.61 now) died and now we have
>> 172 pgs down:
>>
>> [19:32:35] server2:~# ceph -s
>> cluster:
>> id: 26c0c5a8-d7ce-49ac-b5a7-bfd9d0ba81ab
>> health: HEALTH_WARN
>> noscrub,nodeep-scrub flag(s) set
>> 21033/2263701 objects misplaced (0.929%)
>> Reduced data availability: 186 pgs inactive, 172 pgs down
>> Degraded data redundancy: 67370/2263701 objects degraded
>> (2.976%), 219 pgs unclean, 46 pgs degraded, 46 pgs undersized
>> mon server2 is low on available space
>>
>> services:
>> mon: 3 daemons, quorum server5,server3,server2
>> mgr: server5(active), standbys: server2, 2, 0, server3
>> osd: 54 osds: 53 up, 53 in; 47 remapped pgs
>> flags noscrub,nodeep-scrub
>>
>> data:
>> pools: 3 pools, 1344 pgs
>> objects: 736k objects, 2889 GB
>> usage: 8517 GB used, 36474 GB / 44991 GB avail
>> pgs: 13.839% pgs not active
>> 67370/2263701 objects degraded (2.976%)
>> 21033/2263701 objects misplaced (0.929%)
>> 1125 active+clean
>> 172 down
>> 26 active+undersized+degraded+remapped+backfilling
>> 14 undersized+degraded+remapped+backfilling+peered
>> 6 active+undersized+degraded+remapped+backfill_wait
>> 1 active+remapped+backfill_wait
>>
>> io:
>> client: 835 kB/s rd, 262 kB/s wr, 16 op/s rd, 25 op/s wr
>> recovery: 102 MB/s, 26 objects/s
>>
>> What is the most sensible way to get out of this situation?
>>
>>
>>
>>
>>
>> David Turner <drakonstein@xxxxxxxxx> writes:
>>
>> > I do remember seeing that exactly. As the number of recovery_wait pgs
>> > decreased, the number of unfound objects decreased until they were all
>> > found. Unfortunately it blocked some IO from happening during the
>> > recovery, but in the long run we ended up with full data integrity again.
>> >
>> > On Mon, Jan 22, 2018 at 1:03 PM Nico Schottelius <
>> > nico.schottelius@xxxxxxxxxxx> wrote:
>> >
>> >>
>> >> Hey David,
>> >>
>> >> thanks for the fast answer. All our pools are running with size=3,
>> >> min_size=2 and the two disks were in 2 different hosts.
>> >>
>> >> What I am a bit worried about is the output of "ceph pg 4.fa query" (see
>> >> below) that indicates that ceph already queried all other hosts and did
>> >> not find the data anywhere.
>> >>
>> >> Do you remember having seen something similar?
>> >>
>> >> Best,
>> >>
>> >> Nico
>> >>
>> >> David Turner <drakonstein@xxxxxxxxx> writes:
>> >>
>> >> > I have had the same problem before with unfound objects that happened
>> >> while
>> >> > backfilling after losing a drive. We didn't lose drives outside of the
>> >> > failure domains and ultimately didn't lose any data, but we did have
>> to
>> >> > wait until after all of the PGs in recovery_wait state were caught up.
>> >> So
>> >> > if the 2 disks you lost were in the same host and your CRUSH rules are
>> >> set
>> >> > so that you can lose a host without losing data, then the cluster will
>> >> > likely find all of the objects by the time it's done backfilling.
>> With
>> >> > only losing 2 disks, I wouldn't worry about the missing objects not
>> >> > becoming found unless you're pool size=2.
>> >> >
>> >> > On Mon, Jan 22, 2018 at 11:47 AM Nico Schottelius <
>> >> > nico.schottelius@xxxxxxxxxxx> wrote:
>> >> >
>> >> >>
>> >> >> Hello,
>> >> >>
>> >> >> we added about 7 new disks yesterday/today and our cluster became
>> very
>> >> >> slow. While the rebalancing took place, 2 of the 7 new added disks
>> >> >> died.
>> >> >>
>> >> >> Our cluster is still recovering, however we spotted that there are a
>> lot
>> >> >> of unfound objects.
>> >> >>
>> >> >> We lost osd.63 and osd.64, which seem not to be involved into the
>> sample
>> >> >> pg that has unfound objects.
>> >> >>
>> >> >> We were wondering why there are unfound objects, where they are
>> coming
>> >> >> from and if there is a way to recover them?
>> >> >>
>> >> >> Any help appreciated,
>> >> >>
>> >> >> Best,
>> >> >>
>> >> >> Nico
>> >> >>
>> >> >>
>> >> >> Our status is:
>> >> >>
>> >> >> cluster:
>> >> >> id: 26c0c5a8-d7ce-49ac-b5a7-bfd9d0ba81ab
>> >> >> health: HEALTH_WARN
>> >> >> 261953/3006663 objects misplaced (8.712%)
>> >> >> 9377/1002221 objects unfound (0.936%)
>> >> >> Reduced data availability: 176 pgs inactive
>> >> >> Degraded data redundancy: 609338/3006663 objects degraded
>> >> >> (20.266%), 243 pgs unclea
>> >> >> n, 222 pgs degraded, 213 pgs undersized
>> >> >> mon server2 is low on available space
>> >> >>
>> >> >> services:
>> >> >> mon: 3 daemons, quorum server5,server3,server2
>> >> >> mgr: server5(active), standbys: 2, server2, 0, server3
>> >> >> osd: 54 osds: 54 up, 54 in; 234 remapped pgs
>> >> >>
>> >> >> data:
>> >> >> pools: 3 pools, 1344 pgs
>> >> >> objects: 978k objects, 3823 GB
>> >> >> usage: 9350 GB used, 40298 GB / 49648 GB avail
>> >> >> pgs: 13.095% pgs not active
>> >> >> 609338/3006663 objects degraded (20.266%)
>> >> >> 261953/3006663 objects misplaced (8.712%)
>> >> >> 9377/1002221 objects unfound (0.936%)
>> >> >> 1101 active+clean
>> >> >> 84 recovery_wait+undersized+degraded+remapped+peered
>> >> >> 82 undersized+degraded+remapped+backfill_wait+peered
>> >> >> 23 active+undersized+degraded+remapped+backfill_wait
>> >> >> 18 active+remapped+backfill_wait
>> >> >> 14 active+undersized+degraded+remapped+backfilling
>> >> >> 10 undersized+degraded+remapped+backfilling+peered
>> >> >> 9 active+recovery_wait+degraded
>> >> >> 3 active+remapped+backfilling
>> >> >>
>> >> >> io:
>> >> >> client: 624 kB/s rd, 3255 kB/s wr, 22 op/s rd, 66 op/s wr
>> >> >> recovery: 90148 kB/s, 22 objects/s
>> >> >>
>> >> >> Looking at the unfound objects:
>> >> >>
>> >> >> [17:32:17] server1:~# ceph health detail
>> >> >> HEALTH_WARN 263745/3006663 objects misplaced (8.772%); 9377/1002221
>> >> >> objects unfound (0.936%); Reduced data availability: 176 pgs
>> inactive;
>> >> >> Degraded data redundancy: 612398/3006663 objects degraded (20.368%),
>> 244
>> >> >> pgs unclean, 223 pgs degraded, 214 pgs undersized; mon server2 is
>> low on
>> >> >> available space
>> >> >> OBJECT_MISPLACED 263745/3006663 objects misplaced (8.772%)
>> >> >> OBJECT_UNFOUND 9377/1002221 objects unfound (0.936%)
>> >> >> pg 4.fa has 117 unfound objects
>> >> >> pg 4.ff has 107 unfound objects
>> >> >> pg 4.fd has 113 unfound objects
>> >> >> pg 4.f0 has 120 unfound objects
>> >> >> ....
>> >> >>
>> >> >>
>> >> >> Output from ceph pg 4.fa query:
>> >> >>
>> >> >> {
>> >> >> "state": "recovery_wait+undersized+degraded+remapped+peered",
>> >> >> "snap_trimq": "[]",
>> >> >> "epoch": 17561,
>> >> >> "up": [
>> >> >> 8,
>> >> >> 17,
>> >> >> 25
>> >> >> ],
>> >> >> "acting": [
>> >> >> 61
>> >> >> ],
>> >> >> "backfill_targets": [
>> >> >> "8",
>> >> >> "17",
>> >> >> "25"
>> >> >> ],
>> >> >> "actingbackfill": [
>> >> >> "8",
>> >> >> "17",
>> >> >> "25",
>> >> >> "61"
>> >> >> ],
>> >> >> "info": {
>> >> >> "pgid": "4.fa",
>> >> >> "last_update": "17529'85051",
>> >> >> "last_complete": "17217'77468",
>> >> >> "log_tail": "17091'75034",
>> >> >> "last_user_version": 85051,
>> >> >> "last_backfill": "MAX",
>> >> >> "last_backfill_bitwise": 0,
>> >> >> "purged_snaps": [
>> >> >> {
>> >> >> "start": "1",
>> >> >> "length": "3"
>> >> >> },
>> >> >> {
>> >> >> "start": "6",
>> >> >> "length": "8"
>> >> >> },
>> >> >> {
>> >> >> "start": "10",
>> >> >> "length": "2"
>> >> >> }
>> >> >> ],
>> >> >> "history": {
>> >> >> "epoch_created": 9134,
>> >> >> "epoch_pool_created": 9134,
>> >> >> "last_epoch_started": 17528,
>> >> >> "last_interval_started": 17527,
>> >> >> "last_epoch_clean": 17079,
>> >> >> "last_interval_clean": 17078,
>> >> >> "last_epoch_split": 0,
>> >> >> "last_epoch_marked_full": 0,
>> >> >> "same_up_since": 17143,
>> >> >> "same_interval_since": 17530,
>> >> >> "same_primary_since": 17515,
>> >> >> "last_scrub": "17090'57357",
>> >> >> "last_scrub_stamp": "2018-01-20 20:45:32.616142",
>> >> >> "last_deep_scrub": "17082'54734",
>> >> >> "last_deep_scrub_stamp": "2018-01-15 21:09:34.121488",
>> >> >> "last_clean_scrub_stamp": "2018-01-20 20:45:32.616142"
>> >> >> },
>> >> >> "stats": {
>> >> >> "version": "17529'85051",
>> >> >> "reported_seq": "218453",
>> >> >> "reported_epoch": "17561",
>> >> >> "state":
>> >> "recovery_wait+undersized+degraded+remapped+peered",
>> >> >> "last_fresh": "2018-01-22 17:42:28.196701",
>> >> >> "last_change": "2018-01-22 15:00:46.507189",
>> >> >> "last_active": "2018-01-22 15:00:44.635399",
>> >> >> "last_peered": "2018-01-22 17:42:28.196701",
>> >> >> "last_clean": "2018-01-21 20:15:48.267209",
>> >> >> "last_became_active": "2018-01-22 14:53:07.918893",
>> >> >> "last_became_peered": "2018-01-22 15:00:46.437493",
>> >> >> "last_unstale": "2018-01-22 17:42:28.196701",
>> >> >> "last_undegraded": "2018-01-22 15:00:45.664468",
>> >> >> "last_fullsized": "2018-01-22 15:00:45.664468",
>> >> >> "mapping_epoch": 17530,
>> >> >> "log_start": "17091'75034",
>> >> >> "ondisk_log_start": "17091'75034",
>> >> >> "created": 9134,
>> >> >> "last_epoch_clean": 17079,
>> >> >> "parent": "0.0",
>> >> >> "parent_split_bits": 0,
>> >> >> "last_scrub": "17090'57357",
>> >> >> "last_scrub_stamp": "2018-01-20 20:45:32.616142",
>> >> >> "last_deep_scrub": "17082'54734",
>> >> >> "last_deep_scrub_stamp": "2018-01-15 21:09:34.121488",
>> >> >> "last_clean_scrub_stamp": "2018-01-20 20:45:32.616142",
>> >> >> "log_size": 10017,
>> >> >> "ondisk_log_size": 10017,
>> >> >> "stats_invalid": false,
>> >> >> "dirty_stats_invalid": false,
>> >> >> "omap_stats_invalid": false,
>> >> >> "hitset_stats_invalid": false,
>> >> >> "hitset_bytes_stats_invalid": false,
>> >> >> "pin_stats_invalid": false,
>> >> >> "stat_sum": {
>> >> >> "num_bytes": 6112366592,
>> >> >> "num_objects": 1503,
>> >> >> "num_object_clones": 0,
>> >> >> "num_object_copies": 4509,
>> >> >> "num_objects_missing_on_primary": 117,
>> >> >> "num_objects_missing": 0,
>> >> >> "num_objects_degraded": 3123,
>> >> >> "num_objects_misplaced": 1386,
>> >> >> "num_objects_unfound": 117,
>> >> >> "num_objects_dirty": 1503,
>> >> >> "num_whiteouts": 0,
>> >> >> "num_read": 165528,
>> >> >> "num_read_kb": 3252584,
>> >> >> "num_write": 87725,
>> >> >> "num_write_kb": 14125516,
>> >> >> "num_scrub_errors": 0,
>> >> >> "num_shallow_scrub_errors": 0,
>> >> >> "num_deep_scrub_errors": 0,
>> >> >> "num_objects_recovered": 244,
>> >> >> "num_bytes_recovered": 963108864,
>> >> >> "num_keys_recovered": 0,
>> >> >> "num_objects_omap": 0,
>> >> >> "num_objects_hit_set_archive": 0,
>> >> >> "num_bytes_hit_set_archive": 0,
>> >> >> "num_flush": 0,
>> >> >> "num_flush_kb": 0,
>> >> >> "num_evict": 0,
>> >> >> "num_evict_kb": 0,
>> >> >> "num_promote": 0,
>> >> >> "num_flush_mode_high": 0,
>> >> >> "num_flush_mode_low": 0,
>> >> >> "num_evict_mode_some": 0,
>> >> >> "num_evict_mode_full": 0,
>> >> >> "num_objects_pinned": 0,
>> >> >> "num_legacy_snapsets": 0
>> >> >> },
>> >> >> "up": [
>> >> >> 8,
>> >> >> 17,
>> >> >> 25
>> >> >> ],
>> >> >> "acting": [
>> >> >> 61
>> >> >> ],
>> >> >> "blocked_by": [],
>> >> >> "up_primary": 8,
>> >> >> "acting_primary": 61
>> >> >> },
>> >> >> "empty": 0,
>> >> >> "dne": 0,
>> >> >> "incomplete": 0,
>> >> >> "last_epoch_started": 17528,
>> >> >> "hit_set_history": {
>> >> >> "current_last_update": "0'0",
>> >> >> "history": []
>> >> >> }
>> >> >> },
>> >> >> "peer_info": [
>> >> >> {
>> >> >> "peer": "8",
>> >> >> "pgid": "4.fa",
>> >> >> "last_update": "17529'85051",
>> >> >> "last_complete": "17529'85051",
>> >> >> "log_tail": "17091'75034",
>> >> >> "last_user_version": 85051,
>> >> >> "last_backfill": "MIN",
>> >> >> "last_backfill_bitwise": 1,
>> >> >> "purged_snaps": [
>> >> >> {
>> >> >> "start": "1",
>> >> >> "length": "3"
>> >> >> },
>> >> >> {
>> >> >> "start": "6",
>> >> >> "length": "8"
>> >> >> },
>> >> >> {
>> >> >> "start": "10",
>> >> >> "length": "2"
>> >> >> }
>> >> >> ],
>> >> >> "history": {
>> >> >> "epoch_created": 9134,
>> >> >> "epoch_pool_created": 9134,
>> >> >> "last_epoch_started": 17528,
>> >> >> "last_interval_started": 17527,
>> >> >> "last_epoch_clean": 17079,
>> >> >> "last_interval_clean": 17078,
>> >> >> "last_epoch_split": 0,
>> >> >> "last_epoch_marked_full": 0,
>> >> >> "same_up_since": 17143,
>> >> >> "same_interval_since": 17530,
>> >> >> "same_primary_since": 17515,
>> >> >> "last_scrub": "17090'57357",
>> >> >> "last_scrub_stamp": "2018-01-20 20:45:32.616142",
>> >> >> "last_deep_scrub": "17082'54734",
>> >> >> "last_deep_scrub_stamp": "2018-01-15
>> 21:09:34.121488",
>> >> >> "last_clean_scrub_stamp": "2018-01-20
>> 20:45:32.616142"
>> >> >> },
>> >> >> "stats": {
>> >> >> "version": "0'0",
>> >> >> "reported_seq": "3",
>> >> >> "reported_epoch": "17097",
>> >> >> "state": "peering",
>> >> >> "last_fresh": "2018-01-21 20:15:49.798018",
>> >> >> "last_change": "2018-01-21 20:15:49.786423",
>> >> >> "last_active": "0.000000",
>> >> >> "last_peered": "0.000000",
>> >> >> "last_clean": "0.000000",
>> >> >> "last_became_active": "0.000000",
>> >> >> "last_became_peered": "0.000000",
>> >> >> "last_unstale": "2018-01-21 20:15:49.798018",
>> >> >> "last_undegraded": "2018-01-21 20:15:49.798018",
>> >> >> "last_fullsized": "2018-01-21 20:15:49.798018",
>> >> >> "mapping_epoch": 17530,
>> >> >> "log_start": "0'0",
>> >> >> "ondisk_log_start": "0'0",
>> >> >> "created": 9134,
>> >> >> "last_epoch_clean": 17079,
>> >> >> "parent": "0.0",
>> >> >> "parent_split_bits": 0,
>> >> >> "last_scrub": "17090'57357",
>> >> >> "last_scrub_stamp": "2018-01-20 20:45:32.616142",
>> >> >> "last_deep_scrub": "17082'54734",
>> >> >> "last_deep_scrub_stamp": "2018-01-15
>> 21:09:34.121488",
>> >> >> "last_clean_scrub_stamp": "2018-01-20
>> 20:45:32.616142",
>> >> >> "log_size": 0,
>> >> >> "ondisk_log_size": 0,
>> >> >> "stats_invalid": false,
>> >> >> "dirty_stats_invalid": false,
>> >> >> "omap_stats_invalid": false,
>> >> >> "hitset_stats_invalid": false,
>> >> >> "hitset_bytes_stats_invalid": false,
>> >> >> "pin_stats_invalid": false,
>> >> >> "stat_sum": {
>> >> >> "num_bytes": 0,
>> >> >> "num_objects": 0,
>> >> >> "num_object_clones": 0,
>> >> >> "num_object_copies": 0,
>> >> >> "num_objects_missing_on_primary": 0,
>> >> >> "num_objects_missing": 0,
>> >> >> "num_objects_degraded": 0,
>> >> >> "num_objects_misplaced": 0,
>> >> >> "num_objects_unfound": 0,
>> >> >> "num_objects_dirty": 0,
>> >> >> "num_whiteouts": 0,
>> >> >> "num_read": 0,
>> >> >> "num_read_kb": 0,
>> >> >> "num_write": 0,
>> >> >> "num_write_kb": 0,
>> >> >> "num_scrub_errors": 0,
>> >> >> "num_shallow_scrub_errors": 0,
>> >> >> "num_deep_scrub_errors": 0,
>> >> >> "num_objects_recovered": 0,
>> >> >> "num_bytes_recovered": 0,
>> >> >> "num_keys_recovered": 0,
>> >> >> "num_objects_omap": 0,
>> >> >> "num_objects_hit_set_archive": 0,
>> >> >> "num_bytes_hit_set_archive": 0,
>> >> >> "num_flush": 0,
>> >> >> "num_flush_kb": 0,
>> >> >> "num_evict": 0,
>> >> >> "num_evict_kb": 0,
>> >> >> "num_promote": 0,
>> >> >> "num_flush_mode_high": 0,
>> >> >> "num_flush_mode_low": 0,
>> >> >> "num_evict_mode_some": 0,
>> >> >> "num_evict_mode_full": 0,
>> >> >> "num_objects_pinned": 0,
>> >> >> "num_legacy_snapsets": 0
>> >> >> },
>> >> >> "up": [
>> >> >> 8,
>> >> >> 17,
>> >> >> 25
>> >> >> ],
>> >> >> "acting": [
>> >> >> 61
>> >> >> ],
>> >> >> "blocked_by": [],
>> >> >> "up_primary": 8,
>> >> >> "acting_primary": 61
>> >> >> },
>> >> >> "empty": 0,
>> >> >> "dne": 0,
>> >> >> "incomplete": 1,
>> >> >> "last_epoch_started": 17528,
>> >> >> "hit_set_history": {
>> >> >> "current_last_update": "0'0",
>> >> >> "history": []
>> >> >> }
>> >> >> },
>> >> >> {
>> >> >> "peer": "17",
>> >> >> "pgid": "4.fa",
>> >> >> "last_update": "17529'85051",
>> >> >> "last_complete": "17529'85051",
>> >> >> "log_tail": "17091'75034",
>> >> >> "last_user_version": 85051,
>> >> >> "last_backfill": "MIN",
>> >> >> "last_backfill_bitwise": 1,
>> >> >> "purged_snaps": [
>> >> >> {
>> >> >> "start": "1",
>> >> >> "length": "3"
>> >> >> },
>> >> >> {
>> >> >> "start": "6",
>> >> >> "length": "8"
>> >> >> },
>> >> >> {
>> >> >> "start": "10",
>> >> >> "length": "2"
>> >> >> }
>> >> >> ],
>> >> >> "history": {
>> >> >> "epoch_created": 9134,
>> >> >> "epoch_pool_created": 9134,
>> >> >> "last_epoch_started": 17528,
>> >> >> "last_interval_started": 17527,
>> >> >> "last_epoch_clean": 17079,
>> >> >> "last_interval_clean": 17078,
>> >> >> "last_epoch_split": 0,
>> >> >> "last_epoch_marked_full": 0,
>> >> >> "same_up_since": 17143,
>> >> >> "same_interval_since": 17530,
>> >> >> "same_primary_since": 17515,
>> >> >> "last_scrub": "17090'57357",
>> >> >> "last_scrub_stamp": "2018-01-20 20:45:32.616142",
>> >> >> "last_deep_scrub": "17082'54734",
>> >> >> "last_deep_scrub_stamp": "2018-01-15
>> 21:09:34.121488",
>> >> >> "last_clean_scrub_stamp": "2018-01-20
>> 20:45:32.616142"
>> >> >> },
>> >> >> "stats": {
>> >> >> "version": "0'0",
>> >> >> "reported_seq": "0",
>> >> >> "reported_epoch": "0",
>> >> >> "state": "unknown",
>> >> >> "last_fresh": "0.000000",
>> >> >> "last_change": "0.000000",
>> >> >> "last_active": "0.000000",
>> >> >> "last_peered": "0.000000",
>> >> >> "last_clean": "0.000000",
>> >> >> "last_became_active": "0.000000",
>> >> >> "last_became_peered": "0.000000",
>> >> >> "last_unstale": "0.000000",
>> >> >> "last_undegraded": "0.000000",
>> >> >> "last_fullsized": "0.000000",
>> >> >> "mapping_epoch": 17530,
>> >> >> "log_start": "0'0",
>> >> >> "ondisk_log_start": "0'0",
>> >> >> "created": 0,
>> >> >> "last_epoch_clean": 0,
>> >> >> "parent": "0.0",
>> >> >> "parent_split_bits": 0,
>> >> >> "last_scrub": "0'0",
>> >> >> "last_scrub_stamp": "0.000000",
>> >> >> "last_deep_scrub": "0'0",
>> >> >> "last_deep_scrub_stamp": "0.000000",
>> >> >> "last_clean_scrub_stamp": "0.000000",
>> >> >> "log_size": 0,
>> >> >> "ondisk_log_size": 0,
>> >> >> "stats_invalid": false,
>> >> >> "dirty_stats_invalid": false,
>> >> >> "omap_stats_invalid": false,
>> >> >> "hitset_stats_invalid": false,
>> >> >> "hitset_bytes_stats_invalid": false,
>> >> >> "pin_stats_invalid": false,
>> >> >> "stat_sum": {
>> >> >> "num_bytes": 0,
>> >> >> "num_objects": 0,
>> >> >> "num_object_clones": 0,
>> >> >> "num_object_copies": 0,
>> >> >> "num_objects_missing_on_primary": 0,
>> >> >> "num_objects_missing": 0,
>> >> >> "num_objects_degraded": 0,
>> >> >> "num_objects_misplaced": 0,
>> >> >> "num_objects_unfound": 0,
>> >> >> "num_objects_dirty": 0,
>> >> >> "num_whiteouts": 0,
>> >> >> "num_read": 0,
>> >> >> "num_read_kb": 0,
>> >> >> "num_write": 0,
>> >> >> "num_write_kb": 0,
>> >> >> "num_scrub_errors": 0,
>> >> >> "num_shallow_scrub_errors": 0,
>> >> >> "num_deep_scrub_errors": 0,
>> >> >> "num_objects_recovered": 0,
>> >> >> "num_bytes_recovered": 0,
>> >> >> "num_keys_recovered": 0,
>> >> >> "num_objects_omap": 0,
>> >> >> "num_objects_hit_set_archive": 0,
>> >> >> "num_bytes_hit_set_archive": 0,
>> >> >> "num_flush": 0,
>> >> >> "num_flush_kb": 0,
>> >> >> "num_evict": 0,
>> >> >> "num_evict_kb": 0,
>> >> >> "num_promote": 0,
>> >> >> "num_flush_mode_high": 0,
>> >> >> "num_flush_mode_low": 0,
>> >> >> "num_evict_mode_some": 0,
>> >> >> "num_evict_mode_full": 0,
>> >> >> "num_objects_pinned": 0,
>> >> >> "num_legacy_snapsets": 0
>> >> >> },
>> >> >> "up": [
>> >> >> 8,
>> >> >> 17,
>> >> >> 25
>> >> >> ],
>> >> >> "acting": [
>> >> >> 61
>> >> >> ],
>> >> >> "blocked_by": [],
>> >> >> "up_primary": 8,
>> >> >> "acting_primary": 61
>> >> >> },
>> >> >> "empty": 0,
>> >> >> "dne": 0,
>> >> >> "incomplete": 1,
>> >> >> "last_epoch_started": 17528,
>> >> >> "hit_set_history": {
>> >> >> "current_last_update": "0'0",
>> >> >> "history": []
>> >> >> }
>> >> >> },
>> >> >> {
>> >> >> "peer": "20",
>> >> >> "pgid": "4.fa",
>> >> >> "last_update": "17138'75260",
>> >> >> "last_complete": "17138'75260",
>> >> >> "log_tail": "17090'73735",
>> >> >> "last_user_version": 75260,
>> >> >> "last_backfill": "MIN",
>> >> >> "last_backfill_bitwise": 1,
>> >> >> "purged_snaps": [
>> >> >> {
>> >> >> "start": "1",
>> >> >> "length": "3"
>> >> >> },
>> >> >> {
>> >> >> "start": "6",
>> >> >> "length": "8"
>> >> >> },
>> >> >> {
>> >> >> "start": "10",
>> >> >> "length": "2"
>> >> >> }
>> >> >> ],
>> >> >> "history": {
>> >> >> "epoch_created": 9134,
>> >> >> "epoch_pool_created": 9134,
>> >> >> "last_epoch_started": 17121,
>> >> >> "last_interval_started": 17120,
>> >> >> "last_epoch_clean": 17079,
>> >> >> "last_interval_clean": 17078,
>> >> >> "last_epoch_split": 0,
>> >> >> "last_epoch_marked_full": 0,
>> >> >> "same_up_since": 17143,
>> >> >> "same_interval_since": 17530,
>> >> >> "same_primary_since": 17515,
>> >> >> "last_scrub": "17090'57357",
>> >> >> "last_scrub_stamp": "2018-01-20 20:45:32.616142",
>> >> >> "last_deep_scrub": "17082'54734",
>> >> >> "last_deep_scrub_stamp": "2018-01-15
>> 21:09:34.121488",
>> >> >> "last_clean_scrub_stamp": "2018-01-20
>> 20:45:32.616142"
>> >> >> },
>> >> >> "stats": {
>> >> >> "version": "0'0",
>> >> >> "reported_seq": "0",
>> >> >> "reported_epoch": "0",
>> >> >> "state": "unknown",
>> >> >> "last_fresh": "0.000000",
>> >> >> "last_change": "0.000000",
>> >> >> "last_active": "0.000000",
>> >> >> "last_peered": "0.000000",
>> >> >> "last_clean": "0.000000",
>> >> >> "last_became_active": "0.000000",
>> >> >> "last_became_peered": "0.000000",
>> >> >> "last_unstale": "0.000000",
>> >> >> "last_undegraded": "0.000000",
>> >> >> "last_fullsized": "0.000000",
>> >> >> "mapping_epoch": 17530,
>> >> >> "log_start": "0'0",
>> >> >> "ondisk_log_start": "0'0",
>> >> >> "created": 0,
>> >> >> "last_epoch_clean": 0,
>> >> >> "parent": "0.0",
>> >> >> "parent_split_bits": 0,
>> >> >> "last_scrub": "0'0",
>> >> >> "last_scrub_stamp": "0.000000",
>> >> >> "last_deep_scrub": "0'0",
>> >> >> "last_deep_scrub_stamp": "0.000000",
>> >> >> "last_clean_scrub_stamp": "0.000000",
>> >> >> "log_size": 0,
>> >> >> "ondisk_log_size": 0,
>> >> >> "stats_invalid": false,
>> >> >> "dirty_stats_invalid": false,
>> >> >> "omap_stats_invalid": false,
>> >> >> "hitset_stats_invalid": false,
>> >> >> "hitset_bytes_stats_invalid": false,
>> >> >> "pin_stats_invalid": false,
>> >> >> "stat_sum": {
>> >> >> "num_bytes": 0,
>> >> >> "num_objects": 0,
>> >> >> "num_object_clones": 0,
>> >> >> "num_object_copies": 0,
>> >> >> "num_objects_missing_on_primary": 0,
>> >> >> "num_objects_missing": 0,
>> >> >> "num_objects_degraded": 0,
>> >> >> "num_objects_misplaced": 0,
>> >> >> "num_objects_unfound": 0,
>> >> >> "num_objects_dirty": 0,
>> >> >> "num_whiteouts": 0,
>> >> >> "num_read": 0,
>> >> >> "num_read_kb": 0,
>> >> >> "num_write": 0,
>> >> >> "num_write_kb": 0,
>> >> >> "num_scrub_errors": 0,
>> >> >> "num_shallow_scrub_errors": 0,
>> >> >> "num_deep_scrub_errors": 0,
>> >> >> "num_objects_recovered": 0,
>> >> >> "num_bytes_recovered": 0,
>> >> >> "num_keys_recovered": 0,
>> >> >> "num_objects_omap": 0,
>> >> >> "num_objects_hit_set_archive": 0,
>> >> >> "num_bytes_hit_set_archive": 0,
>> >> >> "num_flush": 0,
>> >> >> "num_flush_kb": 0,
>> >> >> "num_evict": 0,
>> >> >> "num_evict_kb": 0,
>> >> >> "num_promote": 0,
>> >> >> "num_flush_mode_high": 0,
>> >> >> "num_flush_mode_low": 0,
>> >> >> "num_evict_mode_some": 0,
>> >> >> "num_evict_mode_full": 0,
>> >> >> "num_objects_pinned": 0,
>> >> >> "num_legacy_snapsets": 0
>> >> >> },
>> >> >> "up": [
>> >> >> 8,
>> >> >> 17,
>> >> >> 25
>> >> >> ],
>> >> >> "acting": [
>> >> >> 61
>> >> >> ],
>> >> >> "blocked_by": [],
>> >> >> "up_primary": 8,
>> >> >> "acting_primary": 61
>> >> >> },
>> >> >> "empty": 0,
>> >> >> "dne": 0,
>> >> >> "incomplete": 1,
>> >> >> "last_epoch_started": 17121,
>> >> >> "hit_set_history": {
>> >> >> "current_last_update": "0'0",
>> >> >> "history": []
>> >> >> }
>> >> >> },
>> >> >> {
>> >> >> "peer": "25",
>> >> >> "pgid": "4.fa",
>> >> >> "last_update": "17529'85051",
>> >> >> "last_complete": "17529'85051",
>> >> >> "log_tail": "17091'75034",
>> >> >> "last_user_version": 85051,
>> >> >> "last_backfill": "MIN",
>> >> >> "last_backfill_bitwise": 1,
>> >> >> "purged_snaps": [
>> >> >> {
>> >> >> "start": "1",
>> >> >> "length": "3"
>> >> >> },
>> >> >> {
>> >> >> "start": "6",
>> >> >> "length": "8"
>> >> >> },
>> >> >> {
>> >> >> "start": "10",
>> >> >> "length": "2"
>> >> >> }
>> >> >> ],
>> >> >> "history": {
>> >> >> "epoch_created": 9134,
>> >> >> "epoch_pool_created": 9134,
>> >> >> "last_epoch_started": 17528,
>> >> >> "last_interval_started": 17527,
>> >> >> "last_epoch_clean": 17079,
>> >> >> "last_interval_clean": 17078,
>> >> >> "last_epoch_split": 0,
>> >> >> "last_epoch_marked_full": 0,
>> >> >> "same_up_since": 17143,
>> >> >> "same_interval_since": 17530,
>> >> >> "same_primary_since": 17515,
>> >> >> "last_scrub": "17090'57357",
>> >> >> "last_scrub_stamp": "2018-01-20 20:45:32.616142",
>> >> >> "last_deep_scrub": "17082'54734",
>> >> >> "last_deep_scrub_stamp": "2018-01-15
>> 21:09:34.121488",
>> >> >> "last_clean_scrub_stamp": "2018-01-20
>> 20:45:32.616142"
>> >> >> },
>> >> >> "stats": {
>> >> >> "version": "0'0",
>> >> >> "reported_seq": "0",
>> >> >> "reported_epoch": "0",
>> >> >> "state": "unknown",
>> >> >> "last_fresh": "0.000000",
>> >> >> "last_change": "0.000000",
>> >> >> "last_active": "0.000000",
>> >> >> "last_peered": "0.000000",
>> >> >> "last_clean": "0.000000",
>> >> >> "last_became_active": "0.000000",
>> >> >> "last_became_peered": "0.000000",
>> >> >> "last_unstale": "0.000000",
>> >> >> "last_undegraded": "0.000000",
>> >> >> "last_fullsized": "0.000000",
>> >> >> "mapping_epoch": 17530,
>> >> >> "log_start": "0'0",
>> >> >> "ondisk_log_start": "0'0",
>> >> >> "created": 0,
>> >> >> "last_epoch_clean": 0,
>> >> >> "parent": "0.0",
>> >> >> "parent_split_bits": 0,
>> >> >> "last_scrub": "0'0",
>> >> >> "last_scrub_stamp": "0.000000",
>> >> >> "last_deep_scrub": "0'0",
>> >> >> "last_deep_scrub_stamp": "0.000000",
>> >> >> "last_clean_scrub_stamp": "0.000000",
>> >> >> "log_size": 0,
>> >> >> "ondisk_log_size": 0,
>> >> >> "stats_invalid": false,
>> >> >> "dirty_stats_invalid": false,
>> >> >> "omap_stats_invalid": false,
>> >> >> "hitset_stats_invalid": false,
>> >> >> "hitset_bytes_stats_invalid": false,
>> >> >> "pin_stats_invalid": false,
>> >> >> "stat_sum": {
>> >> >> "num_bytes": 0,
>> >> >> "num_objects": 0,
>> >> >> "num_object_clones": 0,
>> >> >> "num_object_copies": 0,
>> >> >> "num_objects_missing_on_primary": 0,
>> >> >> "num_objects_missing": 0,
>> >> >> "num_objects_degraded": 0,
>> >> >> "num_objects_misplaced": 0,
>> >> >> "num_objects_unfound": 0,
>> >> >> "num_objects_dirty": 0,
>> >> >> "num_whiteouts": 0,
>> >> >> "num_read": 0,
>> >> >> "num_read_kb": 0,
>> >> >> "num_write": 0,
>> >> >> "num_write_kb": 0,
>> >> >> "num_scrub_errors": 0,
>> >> >> "num_shallow_scrub_errors": 0,
>> >> >> "num_deep_scrub_errors": 0,
>> >> >> "num_objects_recovered": 0,
>> >> >> "num_bytes_recovered": 0,
>> >> >> "num_keys_recovered": 0,
>> >> >> "num_objects_omap": 0,
>> >> >> "num_objects_hit_set_archive": 0,
>> >> >> "num_bytes_hit_set_archive": 0,
>> >> >> "num_flush": 0,
>> >> >> "num_flush_kb": 0,
>> >> >> "num_evict": 0,
>> >> >> "num_evict_kb": 0,
>> >> >> "num_promote": 0,
>> >> >> "num_flush_mode_high": 0,
>> >> >> "num_flush_mode_low": 0,
>> >> >> "num_evict_mode_some": 0,
>> >> >> "num_evict_mode_full": 0,
>> >> >> "num_objects_pinned": 0,
>> >> >> "num_legacy_snapsets": 0
>> >> >> },
>> >> >> "up": [
>> >> >> 8,
>> >> >> 17,
>> >> >> 25
>> >> >> ],
>> >> >> "acting": [
>> >> >> 61
>> >> >> ],
>> >> >> "blocked_by": [],
>> >> >> "up_primary": 8,
>> >> >> "acting_primary": 61
>> >> >> },
>> >> >> "empty": 0,
>> >> >> "dne": 0,
>> >> >> "incomplete": 1,
>> >> >> "last_epoch_started": 17528,
>> >> >> "hit_set_history": {
>> >> >> "current_last_update": "0'0",
>> >> >> "history": []
>> >> >> }
>> >> >> }
>> >> >> ],
>> >> >> "recovery_state": [
>> >> >> {
>> >> >> "name": "Started/Primary/Active",
>> >> >> "enter_time": "2018-01-22 15:00:45.664496",
>> >> >> "might_have_unfound": [
>> >> >> {
>> >> >> "osd": "8",
>> >> >> "status": "already probed"
>> >> >> },
>> >> >> {
>> >> >> "osd": "17",
>> >> >> "status": "already probed"
>> >> >> },
>> >> >> {
>> >> >> "osd": "20",
>> >> >> "status": "already probed"
>> >> >> },
>> >> >> {
>> >> >> "osd": "25",
>> >> >> "status": "already probed"
>> >> >> },
>> >> >> {
>> >> >> "osd": "63",
>> >> >> "status": "osd is down"
>> >> >> },
>> >> >> {
>> >> >> "osd": "64",
>> >> >> "status": "osd is down"
>> >> >> }
>> >> >> ],
>> >> >> "recovery_progress": {
>> >> >> "backfill_targets": [
>> >> >> "8",
>> >> >> "17",
>> >> >> "25"
>> >> >> ],
>> >> >> "waiting_on_backfill": [],
>> >> >> "last_backfill_started": "MIN",
>> >> >> "backfill_info": {
>> >> >> "begin": "MIN",
>> >> >> "end": "MIN",
>> >> >> "objects": []
>> >> >> },
>> >> >> "peer_backfill_info": [],
>> >> >> "backfills_in_flight": [],
>> >> >> "recovering": [],
>> >> >> "pg_backend": {
>> >> >> "pull_from_peer": [],
>> >> >> "pushing": []
>> >> >> }
>> >> >> },
>> >> >> "scrub": {
>> >> >> "scrubber.epoch_start": "17078",
>> >> >> "scrubber.active": false,
>> >> >> "scrubber.state": "INACTIVE",
>> >> >> "scrubber.start": "MIN",
>> >> >> "scrubber.end": "MIN",
>> >> >> "scrubber.subset_last_update": "0'0",
>> >> >> "scrubber.deep": false,
>> >> >> "scrubber.seed": 0,
>> >> >> "scrubber.waiting_on": 0,
>> >> >> "scrubber.waiting_on_whom": []
>> >> >> }
>> >> >> },
>> >> >> {
>> >> >> "name": "Started",
>> >> >> "enter_time": "2018-01-22 15:00:45.613486"
>> >> >> }
>> >> >> ],
>> >> >> "agent_state": {}
>> >> >> }
>> >> >>
>> >> >>
>> >> >> ID CLASS WEIGHT TYPE NAME STATUS REWEIGHT PRI-AFF
>> >> >> -1 48.94531 root default
>> >> >> -6 11.88382 host server1
>> >> >> 25 hdd 4.59999 osd.25 up 1.00000 1.00000
>> >> >> 61 hdd 4.59999 osd.61 up 1.00000 1.00000
>> >> >> 48 ssd 0.22198 osd.48 up 1.00000 1.00000
>> >> >> 49 ssd 0.22198 osd.49 up 1.00000 1.00000
>> >> >> 50 ssd 0.22198 osd.50 up 1.00000 1.00000
>> >> >> 51 ssd 0.22699 osd.51 up 1.00000 1.00000
>> >> >> 52 ssd 0.22198 osd.52 up 1.00000 1.00000
>> >> >> 53 ssd 0.22198 osd.53 up 1.00000 1.00000
>> >> >> 54 ssd 0.22198 osd.54 up 1.00000 1.00000
>> >> >> 55 ssd 0.22699 osd.55 up 1.00000 1.00000
>> >> >> 56 ssd 0.22198 osd.56 up 1.00000 1.00000
>> >> >> 57 ssd 0.22198 osd.57 up 1.00000 1.00000
>> >> >> 58 ssd 0.22699 osd.58 up 1.00000 1.00000
>> >> >> 59 ssd 0.22699 osd.59 up 1.00000 1.00000
>> >> >> -2 11.95193 host server2
>> >> >> 21 hdd 4.59999 osd.21 up 1.00000 1.00000
>> >> >> 24 hdd 4.59999 osd.24 up 1.00000 1.00000
>> >> >> 0 ssd 0.68799 osd.0 up 1.00000 1.00000
>> >> >> 4 ssd 0.68799 osd.4 up 1.00000 1.00000
>>
--
Modern, affordable, Swiss Virtual Machines. Visit www.datacenterlight.ch
_______________________________________________
ceph-users mailing list
ceph-users@xxxxxxxxxxxxxx
http://lists.ceph.com/listinfo.cgi/ceph-users-ceph.com
