Dear Etienne
Is a way there you can enforce mon to rejoin a quorum ? I tried to restart
it but nothing changed. I guess it is the cause If I am not mistaken.
below is pg querry output
root@ceph-mon2:~# ceph pg 5.10 query
{
"snap_trimq": "[]",
"snap_trimq_len": 0,
"state": "active+recovery_unfound+undersized+degraded+remapped",
"epoch": 2880,
"up": [
7,
5,
4
],
"acting": [
7,
5
],
"backfill_targets": [
"4"
],
"acting_recovery_backfill": [
"4",
"5",
"7"
],
"info": {
"pgid": "5.10",
"last_update": "2880'3790033",
"last_complete": "0'0",
"log_tail": "2833'3782663",
"last_user_version": 3790033,
"last_backfill": "MAX",
"purged_snaps": [],
"history": {
"epoch_created": 37,
"epoch_pool_created": 37,
"last_epoch_started": 2875,
"last_interval_started": 2874,
"last_epoch_clean": 2629,
"last_interval_clean": 2628,
"last_epoch_split": 0,
"last_epoch_marked_full": 0,
"same_up_since": 2865,
"same_interval_since": 2874,
"same_primary_since": 2866,
"last_scrub": "2332'3609539",
"last_scrub_stamp": "2021-10-25T20:05:38.843924+0000",
"last_deep_scrub": "2332'3609539",
"last_deep_scrub_stamp": "2021-10-20T19:25:39.076801+0000",
"last_clean_scrub_stamp": "2021-10-25T20:05:38.843924+0000",
"prior_readable_until_ub": 0
},
"stats": {
"version": "2880'3790033",
"reported_seq": "4090762",
"reported_epoch": "2880",
"state": "active+recovery_unfound+undersized+degraded+remapped",
"last_fresh": "2021-10-29T06:58:04.048288+0000",
"last_change": "2021-10-29T06:54:53.204703+0000",
"last_active": "2021-10-29T06:58:04.048288+0000",
"last_peered": "2021-10-29T06:58:04.048288+0000",
"last_clean": "2021-10-26T20:59:44.290622+0000",
"last_became_active": "2021-10-29T06:54:52.257112+0000",
"last_became_peered": "2021-10-29T06:54:52.257112+0000",
"last_unstale": "2021-10-29T06:58:04.048288+0000",
"last_undegraded": "2021-10-29T06:54:52.541786+0000",
"last_fullsized": "2021-10-29T06:54:52.207129+0000",
"mapping_epoch": 2874,
"log_start": "2833'3782663",
"ondisk_log_start": "2833'3782663",
"created": 37,
"last_epoch_clean": 2629,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "2332'3609539",
"last_scrub_stamp": "2021-10-25T20:05:38.843924+0000",
"last_deep_scrub": "2332'3609539",
"last_deep_scrub_stamp": "2021-10-20T19:25:39.076801+0000",
"last_clean_scrub_stamp": "2021-10-25T20:05:38.843924+0000",
"log_size": 7370,
"ondisk_log_size": 7370,
"stats_invalid": false,
"dirty_stats_invalid": false,
"omap_stats_invalid": false,
"hitset_stats_invalid": false,
"hitset_bytes_stats_invalid": false,
"pin_stats_invalid": false,
"manifest_stats_invalid": false,
"snaptrimq_len": 0,
"stat_sum": {
"num_bytes": 4191127570,
"num_objects": 1021,
"num_object_clones": 0,
"num_object_copies": 3063,
"num_objects_missing_on_primary": 1,
"num_objects_missing": 1,
"num_objects_degraded": 1023,
"num_objects_misplaced": 0,
"num_objects_unfound": 1,
"num_objects_dirty": 1021,
"num_whiteouts": 0,
"num_read": 304669,
"num_read_kb": 7496973,
"num_write": 3773209,
"num_write_kb": 64044388,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 616,
"num_bytes_recovered": 2329521682,
"num_keys_recovered": 8,
"num_objects_omap": 1,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0,
"num_flush": 0,
"num_flush_kb": 0,
"num_evict": 0,
"num_evict_kb": 0,
"num_promote": 0,
"num_flush_mode_high": 0,
"num_flush_mode_low": 0,
"num_evict_mode_some": 0,
"num_evict_mode_full": 0,
"num_objects_pinned": 0,
"num_legacy_snapsets": 0,
"num_large_omap_objects": 0,
"num_objects_manifest": 0,
"num_omap_bytes": 0,
"num_omap_keys": 0,
"num_objects_repaired": 0
},
"up": [
7,
5,
4
],
"acting": [
7,
5
],
"avail_no_missing": [],
"object_location_counts": [
{
"shards": "5,7",
"objects": 1021
}
],
"blocked_by": [],
"up_primary": 7,
"acting_primary": 7,
"purged_snaps": []
},
"empty": 0,
"dne": 0,
"incomplete": 0,
"last_epoch_started": 2875,
"hit_set_history": {
"current_last_update": "0'0",
"history": []
}
},
"peer_info": [
{
"peer": "0",
"pgid": "5.10",
"last_update": "2873'3790006",
"last_complete": "0'0",
"log_tail": "2833'3782663",
"last_user_version": 3790006,
"last_backfill": "MAX",
"purged_snaps": [],
"history": {
"epoch_created": 37,
"epoch_pool_created": 37,
"last_epoch_started": 2875,
"last_interval_started": 2874,
"last_epoch_clean": 2629,
"last_interval_clean": 2628,
"last_epoch_split": 0,
"last_epoch_marked_full": 0,
"same_up_since": 2865,
"same_interval_since": 2874,
"same_primary_since": 2866,
"last_scrub": "2332'3609539",
"last_scrub_stamp": "2021-10-25T20:05:38.843924+0000",
"last_deep_scrub": "2332'3609539",
"last_deep_scrub_stamp": "2021-10-20T19:25:39.076801+0000",
"last_clean_scrub_stamp": "2021-10-25T20:05:38.843924+0000",
"prior_readable_until_ub": 0
},
"stats": {
"version": "2873'3790003",
"reported_seq": "4090680",
"reported_epoch": "2873",
"state": "active+recovery_unfound+degraded+remapped",
"last_fresh": "2021-10-29T06:54:36.801708+0000",
"last_change": "2021-10-29T06:54:05.714722+0000",
"last_active": "2021-10-29T06:54:36.801708+0000",
"last_peered": "2021-10-29T06:54:36.801708+0000",
"last_clean": "2021-10-26T20:59:44.290622+0000",
"last_became_active": "2021-10-29T06:54:01.731778+0000",
"last_became_peered": "2021-10-29T06:54:01.731778+0000",
"last_unstale": "2021-10-29T06:54:36.801708+0000",
"last_undegraded": "2021-10-29T06:54:05.128093+0000",
"last_fullsized": "2021-10-29T06:54:36.801708+0000",
"mapping_epoch": 2874,
"log_start": "2833'3782663",
"ondisk_log_start": "2833'3782663",
"created": 37,
"last_epoch_clean": 2629,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "2332'3609539",
"last_scrub_stamp": "2021-10-25T20:05:38.843924+0000",
"last_deep_scrub": "2332'3609539",
"last_deep_scrub_stamp": "2021-10-20T19:25:39.076801+0000",
"last_clean_scrub_stamp": "2021-10-25T20:05:38.843924+0000",
"log_size": 7340,
"ondisk_log_size": 7340,
"stats_invalid": false,
"dirty_stats_invalid": false,
"omap_stats_invalid": false,
"hitset_stats_invalid": false,
"hitset_bytes_stats_invalid": false,
"pin_stats_invalid": false,
"manifest_stats_invalid": false,
"snaptrimq_len": 0,
"stat_sum": {
"num_bytes": 4191127570,
"num_objects": 1021,
"num_object_clones": 0,
"num_object_copies": 3063,
"num_objects_missing_on_primary": 1,
"num_objects_missing": 1,
"num_objects_degraded": 3,
"num_objects_misplaced": 1020,
"num_objects_unfound": 1,
"num_objects_dirty": 1021,
"num_whiteouts": 0,
"num_read": 304635,
"num_read_kb": 7496561,
"num_write": 3773182,
"num_write_kb": 64044279,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 616,
"num_bytes_recovered": 2329521682,
"num_keys_recovered": 8,
"num_objects_omap": 1,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0,
"num_flush": 0,
"num_flush_kb": 0,
"num_evict": 0,
"num_evict_kb": 0,
"num_promote": 0,
"num_flush_mode_high": 0,
"num_flush_mode_low": 0,
"num_evict_mode_some": 0,
"num_evict_mode_full": 0,
"num_objects_pinned": 0,
"num_legacy_snapsets": 0,
"num_large_omap_objects": 0,
"num_objects_manifest": 0,
"num_omap_bytes": 0,
"num_omap_keys": 0,
"num_objects_repaired": 0
},
"up": [
7,
5,
4
],
"acting": [
7,
5
],
"avail_no_missing": [],
"object_location_counts": [
{
"shards": "0,5,7",
"objects": 1021
}
],
"blocked_by": [],
"up_primary": 7,
"acting_primary": 7,
"purged_snaps": []
},
"empty": 0,
"dne": 0,
"incomplete": 0,
"last_epoch_started": 2867,
"hit_set_history": {
"current_last_update": "0'0",
"history": []
}
},
{
"peer": "4",
"pgid": "5.10",
"last_update": "2880'3790033",
"last_complete": "2880'3790033",
"log_tail": "2833'3782663",
"last_user_version": 3790006,
"last_backfill": "MIN",
"purged_snaps": [],
"history": {
"epoch_created": 37,
"epoch_pool_created": 37,
"last_epoch_started": 2875,
"last_interval_started": 2874,
"last_epoch_clean": 2629,
"last_interval_clean": 2628,
"last_epoch_split": 0,
"last_epoch_marked_full": 0,
"same_up_since": 2865,
"same_interval_since": 2874,
"same_primary_since": 2866,
"last_scrub": "2332'3609539",
"last_scrub_stamp": "2021-10-25T20:05:38.843924+0000",
"last_deep_scrub": "2332'3609539",
"last_deep_scrub_stamp": "2021-10-20T19:25:39.076801+0000",
"last_clean_scrub_stamp": "2021-10-25T20:05:38.843924+0000",
"prior_readable_until_ub": 0
},
"stats": {
"version": "2332'3609527",
"reported_seq": "3877429",
"reported_epoch": "2332",
"state": "active+clean+scrubbing+deep",
"last_fresh": "2021-10-20T19:25:37.446983+0000",
"last_change": "2021-10-20T19:24:19.080037+0000",
"last_active": "2021-10-20T19:25:37.446983+0000",
"last_peered": "2021-10-20T19:25:37.446983+0000",
"last_clean": "2021-10-20T19:25:37.446983+0000",
"last_became_active": "2021-10-20T07:56:08.752581+0000",
"last_became_peered": "2021-10-20T07:56:08.752581+0000",
"last_unstale": "2021-10-20T19:25:37.446983+0000",
"last_undegraded": "2021-10-20T19:25:37.446983+0000",
"last_fullsized": "2021-10-20T19:25:37.446983+0000",
"mapping_epoch": 2874,
"log_start": "2274'3601800",
"ondisk_log_start": "2274'3601800",
"created": 37,
"last_epoch_clean": 2316,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "2281'3608971",
"last_scrub_stamp": "2021-10-19T11:20:35.197342+0000",
"last_deep_scrub": "2214'3267745",
"last_deep_scrub_stamp": "2021-10-13T08:46:48.003857+0000",
"last_clean_scrub_stamp": "2021-10-19T11:20:35.197342+0000",
"log_size": 7727,
"ondisk_log_size": 7727,
"stats_invalid": false,
"dirty_stats_invalid": false,
"omap_stats_invalid": false,
"hitset_stats_invalid": false,
"hitset_bytes_stats_invalid": false,
"pin_stats_invalid": false,
"manifest_stats_invalid": false,
"snaptrimq_len": 0,
"stat_sum": {
"num_bytes": 2414569490,
"num_objects": 0,
"num_object_clones": 0,
"num_object_copies": 0,
"num_objects_missing_on_primary": 0,
"num_objects_missing": 1021,
"num_objects_degraded": 0,
"num_objects_misplaced": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 0,
"num_whiteouts": 0,
"num_read": 0,
"num_read_kb": 0,
"num_write": 0,
"num_write_kb": 0,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 0,
"num_bytes_recovered": 0,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0,
"num_flush": 0,
"num_flush_kb": 0,
"num_evict": 0,
"num_evict_kb": 0,
"num_promote": 0,
"num_flush_mode_high": 0,
"num_flush_mode_low": 0,
"num_evict_mode_some": 0,
"num_evict_mode_full": 0,
"num_objects_pinned": 0,
"num_legacy_snapsets": 0,
"num_large_omap_objects": 0,
"num_objects_manifest": 0,
"num_omap_bytes": 0,
"num_omap_keys": 0,
"num_objects_repaired": 0
},
"up": [
7,
5,
4
],
"acting": [
7,
5
],
"avail_no_missing": [],
"object_location_counts": [],
"blocked_by": [],
"up_primary": 7,
"acting_primary": 7,
"purged_snaps": []
},
"empty": 0,
"dne": 0,
"incomplete": 1,
"last_epoch_started": 2875,
"hit_set_history": {
"current_last_update": "0'0",
"history": []
}
},
{
"peer": "5",
"pgid": "5.10",
"last_update": "2880'3790033",
"last_complete": "0'0",
"log_tail": "2833'3782663",
"last_user_version": 3790006,
"last_backfill": "MAX",
"purged_snaps": [],
"history": {
"epoch_created": 37,
"epoch_pool_created": 37,
"last_epoch_started": 2875,
"last_interval_started": 2874,
"last_epoch_clean": 2629,
"last_interval_clean": 2628,
"last_epoch_split": 0,
"last_epoch_marked_full": 0,
"same_up_since": 2865,
"same_interval_since": 2874,
"same_primary_since": 2866,
"last_scrub": "2332'3609539",
"last_scrub_stamp": "2021-10-25T20:05:38.843924+0000",
"last_deep_scrub": "2332'3609539",
"last_deep_scrub_stamp": "2021-10-20T19:25:39.076801+0000",
"last_clean_scrub_stamp": "2021-10-25T20:05:38.843924+0000",
"prior_readable_until_ub": 0
},
"stats": {
"version": "2873'3790003",
"reported_seq": "4090680",
"reported_epoch": "2873",
"state": "active+recovery_unfound+degraded+remapped",
"last_fresh": "2021-10-29T06:54:36.801708+0000",
"last_change": "2021-10-29T06:54:05.714722+0000",
"last_active": "2021-10-29T06:54:36.801708+0000",
"last_peered": "2021-10-29T06:54:36.801708+0000",
"last_clean": "2021-10-26T20:59:44.290622+0000",
"last_became_active": "2021-10-29T06:54:01.731778+0000",
"last_became_peered": "2021-10-29T06:54:01.731778+0000",
"last_unstale": "2021-10-29T06:54:36.801708+0000",
"last_undegraded": "2021-10-29T06:54:05.128093+0000",
"last_fullsized": "2021-10-29T06:54:36.801708+0000",
"mapping_epoch": 2874,
"log_start": "2833'3782663",
"ondisk_log_start": "2833'3782663",
"created": 37,
"last_epoch_clean": 2629,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "2332'3609539",
"last_scrub_stamp": "2021-10-25T20:05:38.843924+0000",
"last_deep_scrub": "2332'3609539",
"last_deep_scrub_stamp": "2021-10-20T19:25:39.076801+0000",
"last_clean_scrub_stamp": "2021-10-25T20:05:38.843924+0000",
"log_size": 7340,
"ondisk_log_size": 7340,
"stats_invalid": false,
"dirty_stats_invalid": false,
"omap_stats_invalid": false,
"hitset_stats_invalid": false,
"hitset_bytes_stats_invalid": false,
"pin_stats_invalid": false,
"manifest_stats_invalid": false,
"snaptrimq_len": 0,
"stat_sum": {
"num_bytes": 4191127570,
"num_objects": 1021,
"num_object_clones": 0,
"num_object_copies": 3063,
"num_objects_missing_on_primary": 1,
"num_objects_missing": 1,
"num_objects_degraded": 3,
"num_objects_misplaced": 1020,
"num_objects_unfound": 1,
"num_objects_dirty": 1021,
"num_whiteouts": 0,
"num_read": 304635,
"num_read_kb": 7496561,
"num_write": 3773182,
"num_write_kb": 64044279,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 616,
"num_bytes_recovered": 2329521682,
"num_keys_recovered": 8,
"num_objects_omap": 1,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0,
"num_flush": 0,
"num_flush_kb": 0,
"num_evict": 0,
"num_evict_kb": 0,
"num_promote": 0,
"num_flush_mode_high": 0,
"num_flush_mode_low": 0,
"num_evict_mode_some": 0,
"num_evict_mode_full": 0,
"num_objects_pinned": 0,
"num_legacy_snapsets": 0,
"num_large_omap_objects": 0,
"num_objects_manifest": 0,
"num_omap_bytes": 0,
"num_omap_keys": 0,
"num_objects_repaired": 0
},
"up": [
7,
5,
4
],
"acting": [
7,
5
],
"avail_no_missing": [],
"object_location_counts": [
{
"shards": "0,5,7",
"objects": 1021
}
],
"blocked_by": [],
"up_primary": 7,
"acting_primary": 7,
"purged_snaps": []
},
"empty": 0,
"dne": 0,
"incomplete": 0,
"last_epoch_started": 2875,
"hit_set_history": {
"current_last_update": "0'0",
"history": []
}
}
],
"recovery_state": [
{
"name": "Started/Primary/Active",
"enter_time": "2021-10-29T06:54:52.207211+0000",
"might_have_unfound": [
{
"osd": "0",
"status": "already probed"
},
{
"osd": "4",
"status": "already probed"
},
{
"osd": "5",
"status": "already probed"
}
],
"recovery_progress": {
"backfill_targets": [
"4"
],
"waiting_on_backfill": [],
"last_backfill_started": "MIN",
"backfill_info": {
"begin": "MIN",
"end": "MIN",
"objects": []
},
"peer_backfill_info": [],
"backfills_in_flight": [],
"recovering": [],
"pg_backend": {
"pull_from_peer": [],
"pushing": []
}
}
},
{
"name": "Started",
"enter_time": "2021-10-29T06:54:51.175040+0000"
},
{
"scrubber.epoch_start": "0",
"scrubber.active": false,
"scrubber.state": "INACTIVE",
"scrubber.start": "MIN",
"scrubber.end": "MIN",
"scrubber.max_end": "MIN",
"scrubber.subset_last_update": "0'0",
"scrubber.deep": false,
"scrubber.waiting_on_whom": []
}
],
"agent_state": {}
Regards
On Fri, Oct 29, 2021 at 10:56 AM Etienne Menguy <etienne.menguy@xxxxxxxx>
wrote:
> With “ceph pg x.y query” you can check why it’s complaining.
>
> x.y for pg id, like 5.77
>
> It would also be interesting to check why mon fails to rejoin quorum, it
> may give you hints at your OSD issues.
>
> -
> Etienne Menguy
> etienne.menguy@xxxxxxxx
>
>
>
>
> On 29 Oct 2021, at 10:34, Michel Niyoyita <micou12@xxxxxxxxx> wrote:
>
> Hello Etienne
>
> This is the ceph -s output
>
> root@ceph-mon1:~# ceph -s
> cluster:
> id: 43f5d6b4-74b0-4281-92ab-940829d3ee5e
> health: HEALTH_ERR
> 1/3 mons down, quorum ceph-mon1,ceph-mon3
> 14/47681 objects unfound (0.029%)
> 1 scrub errors
> Possible data damage: 13 pgs recovery_unfound, 1 pg
> inconsistent
> Degraded data redundancy: 42/143043 objects degraded (0.029%),
> 13 pgs degraded
> 2 slow ops, oldest one blocked for 2897 sec, daemons
> [osd.0,osd.7] have slow ops.
>
> services:
> mon: 3 daemons, quorum ceph-mon1,ceph-mon3 (age 2h), out of quorum:
> ceph-mon4
> mgr: ceph-mon1(active, since 25h), standbys: ceph-mon2
> osd: 12 osds: 12 up (since 97m), 12 in (since 25h); 10 remapped pgs
>
> data:
> pools: 5 pools, 225 pgs
> objects: 47.68k objects, 204 GiB
> usage: 603 GiB used, 4.1 TiB / 4.7 TiB avail
> pgs: 42/143043 objects degraded (0.029%)
> 2460/143043 objects misplaced (1.720%)
> 14/47681 objects unfound (0.029%)
> 211 active+clean
> 10 active+recovery_unfound+degraded+remapped
> 3 active+recovery_unfound+degraded
> 1 active+clean+inconsistent
>
> io:
> client: 2.0 KiB/s rd, 88 KiB/s wr, 2 op/s rd, 12 op/s wr
>
> On Fri, Oct 29, 2021 at 10:09 AM Etienne Menguy <etienne.menguy@xxxxxxxx>
> wrote:
>
>> Hi,
>>
>> Please share “ceph -s” output.
>>
>> -
>> Etienne Menguy
>> etienne.menguy@xxxxxxxx
>>
>>
>>
>>
>> On 29 Oct 2021, at 10:03, Michel Niyoyita <micou12@xxxxxxxxx> wrote:
>>
>> Hello team
>>
>> I am running a ceph cluster with 3 monitors and 4 OSDs nodes running 3osd
>> each , I deployed my ceph cluster using ansible and ubuntu 20.04 as OS ,
>> the ceph version is Octopus. yesterday , My server which hosts OSDs nodes
>> restarted because of power issue and to comeback on its status one of the
>> monitor is out of quorum and some Pg marks as damaged . please help me to
>> solve this issue. below are health detail status I am finding. and the 4
>> OSDs node are the same which are running monitors (3 of them).
>>
>> Best regards.
>>
>> Michel
>>
>>
>> root@ceph-mon1:~# ceph health detail
>> HEALTH_ERR 1/3 mons down, quorum ceph-mon1,ceph-mon3; 14/47195 objects
>> unfound (0.030%); Possible data damage: 13 pgs recovery_unfound; Degraded
>> data redundancy: 42/141585 objects degraded (0.030%), 13 pgs degraded; 2
>> slow ops, oldest one blocked for 322 sec, daemons [osd.0,osd.7] have slow
>> ops.
>> [WRN] MON_DOWN: 1/3 mons down, quorum ceph-mon1,ceph-mon3
>> mon.ceph-mon4 (rank 2) addr [v2:
>> 10.10.29.154:3300/0,v1:10.10.29.154:6789/0] is down (out of quorum)
>> [WRN] OBJECT_UNFOUND: 14/47195 objects unfound (0.030%)
>> pg 5.77 has 1 unfound objects
>> pg 5.6d has 2 unfound objects
>> pg 5.6a has 1 unfound objects
>> pg 5.65 has 1 unfound objects
>> pg 5.4a has 1 unfound objects
>> pg 5.30 has 1 unfound objects
>> pg 5.28 has 1 unfound objects
>> pg 5.25 has 1 unfound objects
>> pg 5.19 has 1 unfound objects
>> pg 5.1a has 1 unfound objects
>> pg 5.1 has 1 unfound objects
>> pg 5.b has 1 unfound objects
>> pg 5.8 has 1 unfound objects
>> [ERR] PG_DAMAGED: Possible data damage: 13 pgs recovery_unfound
>> pg 5.1 is active+recovery_unfound+degraded+remapped, acting [5,8,7], 1
>> unfound
>> pg 5.8 is active+recovery_unfound+degraded+remapped, acting [6,11,8], 1
>> unfound
>> pg 5.b is active+recovery_unfound+degraded+remapped, acting [7,0,5], 1
>> unfound
>> pg 5.19 is active+recovery_unfound+degraded+remapped, acting [0,5,7], 1
>> unfound
>> pg 5.1a is active+recovery_unfound+degraded, acting [10,11,8], 1
>> unfound
>> pg 5.25 is active+recovery_unfound+degraded+remapped, acting [0,10,11],
>> 1 unfound
>> pg 5.28 is active+recovery_unfound+degraded+remapped, acting [6,11,8],
>> 1 unfound
>> pg 5.30 is active+recovery_unfound+degraded+remapped, acting [7,5,0], 1
>> unfound
>> pg 5.4a is active+recovery_unfound+degraded, acting [0,11,7], 1 unfound
>> pg 5.65 is active+recovery_unfound+degraded+remapped, acting [0,10,11],
>> 1 unfound
>> pg 5.6a is active+recovery_unfound+degraded, acting [0,11,7], 1 unfound
>> pg 5.6d is active+recovery_unfound+degraded+remapped, acting [7,2,0], 2
>> unfound
>> pg 5.77 is active+recovery_unfound+degraded+remapped, acting [5,6,8], 1
>> unfound
>> [WRN] PG_DEGRADED: Degraded data redundancy: 42/141585 objects degraded
>> (0.030%), 13 pgs degraded
>> pg 5.1 is active+recovery_unfound+degraded+remapped, acting [5,8,7], 1
>> unfound
>> pg 5.8 is active+recovery_unfound+degraded+remapped, acting [6,11,8], 1
>> unfound
>> pg 5.b is active+recovery_unfound+degraded+remapped, acting [7,0,5], 1
>> unfound
>> pg 5.19 is active+recovery_unfound+degraded+remapped, acting [0,5,7], 1
>> unfound
>> pg 5.1a is active+recovery_unfound+degraded, acting [10,11,8], 1
>> unfound
>> pg 5.25 is active+recovery_unfound+degraded+remapped, acting [0,10,11],
>> 1 unfound
>> pg 5.28 is active+recovery_unfound+degraded+remapped, acting [6,11,8],
>> 1 unfound
>> pg 5.30 is active+recovery_unfound+degraded+remapped, acting [7,5,0], 1
>> unfound
>> pg 5.4a is active+recovery_unfound+degraded, acting [0,11,7], 1 unfound
>> pg 5.65 is active+recovery_unfound+degraded+remapped, acting [0,10,11],
>> 1 unfound
>> pg 5.6a is active+recovery_unfound+degraded, acting [0,11,7], 1 unfound
>> pg 5.6d is active+recovery_unfound+degraded+remapped, acting [7,2,0], 2
>> unfound
>> pg 5.77 is active+recovery_unfound+degraded+remapped, acting [5,6,8], 1
>> unfound
>> [WRN] SLOW_OPS: 2 slow ops, oldest one blocked for 322 sec, daemons
>> [osd.0,osd.7] have slow ops.
>> _______________________________________________
>> ceph-users mailing list -- ceph-users@xxxxxxx
>> To unsubscribe send an email to ceph-users-leave@xxxxxxx
>>
>>
>>
>
_______________________________________________
ceph-users mailing list -- ceph-users@xxxxxxx
To unsubscribe send an email to ceph-users-leave@xxxxxxx
