Hi all,
hope someone can help me. After restarting a node of
my 2-node-cluster suddenly I get this:
root@yak2 /var/www/projects # ceph -s
cluster:
id: 749b2473-9300-4535-97a6-ee6d55008a1b
health: HEALTH_WARN
Reduced data availability: 200 pgs
inactive
services:
mon: 3 daemons, quorum yak1,yak2,yak0
mgr:
yak0.planwerk6.de(active),
standbys:
yak1.planwerk6.de,
yak2.planwerk6.de
mds: cephfs-1/1/1 up {0=
yak1.planwerk6.de=up:active},
1 up:standby
osd: 2 osds: 2 up, 2 in
data:
pools: 2 pools, 200 pgs
objects: 0 objects, 0 B
usage: 0 B used, 0 B / 0 B avail
pgs: 100.000% pgs unknown
200 unknown
And this:
root@yak2 /var/www/projects # ceph health detail
HEALTH_WARN Reduced data availability: 200 pgs
inactive
PG_AVAILABILITY Reduced data availability: 200 pgs
inactive
pg 1.34 is stuck inactive for 3506.815664, current
state unknown, last acting []
pg 1.35 is stuck inactive for 3506.815664, current
state unknown, last acting []
pg 1.36 is stuck inactive for 3506.815664, current
state unknown, last acting []
pg 1.37 is stuck inactive for 3506.815664, current
state unknown, last acting []
pg 1.38 is stuck inactive for 3506.815664, current
state unknown, last acting []
pg 1.39 is stuck inactive for 3506.815664, current
state unknown, last acting []
pg 1.3a is stuck inactive for 3506.815664, current
state unknown, last acting []
pg 1.3b is stuck inactive for 3506.815664, current
state unknown, last acting []
pg 1.3c is stuck inactive for 3506.815664, current
state unknown, last acting []
pg 1.3d is stuck inactive for 3506.815664, current
state unknown, last acting []
pg 1.3e is stuck inactive for 3506.815664, current
state unknown, last acting []
pg 1.3f is stuck inactive for 3506.815664, current
state unknown, last acting []
pg 1.40 is stuck inactive for 3506.815664, current
state unknown, last acting []
pg 1.41 is stuck inactive for 3506.815664, current
state unknown, last acting []
pg 1.42 is stuck inactive for 3506.815664, current
state unknown, last acting []
pg 1.43 is stuck inactive for 3506.815664, current
state unknown, last acting []
pg 1.44 is stuck inactive for 3506.815664, current
state unknown, last acting []
pg 1.45 is stuck inactive for 3506.815664, current
state unknown, last acting []
pg 1.46 is stuck inactive for 3506.815664, current
state unknown, last acting []
pg 1.47 is stuck inactive for 3506.815664, current
state unknown, last acting []
pg 1.48 is stuck inactive for 3506.815664, current
state unknown, last acting []
pg 1.49 is stuck inactive for 3506.815664, current
state unknown, last acting []
pg 1.4a is stuck inactive for 3506.815664, current
state unknown, last acting []
pg 1.4b is stuck inactive for 3506.815664, current
state unknown, last acting []
pg 1.4c is stuck inactive for 3506.815664, current
state unknown, last acting []
pg 1.4d is stuck inactive for 3506.815664, current
state unknown, last acting []
pg 2.34 is stuck inactive for 3506.815664, current
state unknown, last acting []
pg 2.35 is stuck inactive for 3506.815664, current
state unknown, last acting []
pg 2.36 is stuck inactive for 3506.815664, current
state unknown, last acting []
pg 2.38 is stuck inactive for 3506.815664, current
state unknown, last acting []
pg 2.39 is stuck inactive for 3506.815664, current
state unknown, last acting []
pg 2.3a is stuck inactive for 3506.815664, current
state unknown, last acting []
pg 2.3b is stuck inactive for 3506.815664, current
state unknown, last acting []
pg 2.3c is stuck inactive for 3506.815664, current
state unknown, last acting []
pg 2.3d is stuck inactive for 3506.815664, current
state unknown, last acting []
pg 2.3e is stuck inactive for 3506.815664, current
state unknown, last acting []
pg 2.3f is stuck inactive for 3506.815664, current
state unknown, last acting []
pg 2.40 is stuck inactive for 3506.815664, current
state unknown, last acting []
pg 2.41 is stuck inactive for 3506.815664, current
state unknown, last acting []
pg 2.42 is stuck inactive for 3506.815664, current
state unknown, last acting []
pg 2.43 is stuck inactive for 3506.815664, current
state unknown, last acting []
pg 2.44 is stuck inactive for 3506.815664, current
state unknown, last acting []
pg 2.45 is stuck inactive for 3506.815664, current
state unknown, last acting []
pg 2.46 is stuck inactive for 3506.815664, current
state unknown, last acting []
pg 2.47 is stuck inactive for 3506.815664, current
state unknown, last acting []
pg 2.48 is stuck inactive for 3506.815664, current
state unknown, last acting []
pg 2.49 is stuck inactive for 3506.815664, current
state unknown, last acting []
pg 2.4a is stuck inactive for 3506.815664, current
state unknown, last acting []
pg 2.4b is stuck inactive for 3506.815664, current
state unknown, last acting []
pg 2.4e is stuck inactive for 3506.815664, current
state unknown, last acting []
pg 2.4f is stuck inactive for 3506.815664, current
state unknown, last acting []
But if I query an individual PG I get this:
root@yak1 /var/www/projects # ceph pg 1.49 query
{
"state": "active+clean",
"snap_trimq": "[]",
"snap_trimq_len": 0,
"epoch": 162,
"up": [
0,
1
],
"acting": [
0,
1
],
"acting_recovery_backfill": [
"0",
"1"
],
"info": {
"pgid": "1.49",
"last_update": "127'38077",
"last_complete": "127'38077",
"log_tail": "127'35000",
"last_user_version": 38077,
"last_backfill": "MAX",
"last_backfill_bitwise": 0,
"purged_snaps": [],
"history": {
"epoch_created": 10,
"epoch_pool_created": 10,
"last_epoch_started": 159,
"last_interval_started": 158,
"last_epoch_clean": 159,
"last_interval_clean": 158,
"last_epoch_split": 0,
"last_epoch_marked_full": 0,
"same_up_since": 158,
"same_interval_since": 158,
"same_primary_since": 135,
"last_scrub": "127'36909",
"last_scrub_stamp": "2019-02-20
15:02:45.204342",
"last_deep_scrub": "127'36714",
"last_deep_scrub_stamp": "2019-02-16
07:55:15.205861",
"last_clean_scrub_stamp": "2019-02-20
15:02:45.204342"
},
"stats": {
"version": "127'38077",
"reported_seq": "58934",
"reported_epoch": "162",
"state": "active+clean",
"last_fresh": "2019-02-20
19:56:56.740536",
"last_change": "2019-02-20
19:52:27.063812",
"last_active": "2019-02-20
19:56:56.740536",
"last_peered": "2019-02-20
19:56:56.740536",
"last_clean": "2019-02-20
19:56:56.740536",
"last_became_active": "2019-02-20
19:52:27.062689",
"last_became_peered": "2019-02-20
19:52:27.062689",
"last_unstale": "2019-02-20
19:56:56.740536",
"last_undegraded": "2019-02-20
19:56:56.740536",
"last_fullsized": "2019-02-20
19:56:56.740536",
"mapping_epoch": 158,
"log_start": "127'35000",
"ondisk_log_start": "127'35000",
"created": 10,
"last_epoch_clean": 159,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "127'36909",
"last_scrub_stamp": "2019-02-20
15:02:45.204342",
"last_deep_scrub": "127'36714",
"last_deep_scrub_stamp": "2019-02-16
07:55:15.205861",
"last_clean_scrub_stamp": "2019-02-20
15:02:45.204342",
"log_size": 3077,
"ondisk_log_size": 3077,
"stats_invalid": false,
"dirty_stats_invalid": false,
"omap_stats_invalid": false,
"hitset_stats_invalid": false,
"hitset_bytes_stats_invalid": false,
"pin_stats_invalid": false,
"manifest_stats_invalid": true,
"snaptrimq_len": 0,
"stat_sum": {
"num_bytes": 478347970,
"num_objects": 12052,
"num_object_clones": 0,
"num_object_copies": 24104,
"num_objects_missing_on_primary": 0,
"num_objects_missing": 0,
"num_objects_degraded": 0,
"num_objects_misplaced": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 12052,
"num_whiteouts": 0,
"num_read": 20186,
"num_read_kb": 1952018,
"num_write": 38927,
"num_write_kb": 484756,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 6,
"num_bytes_recovered": 4101,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0,
"num_flush": 0,
"num_flush_kb": 0,
"num_evict": 0,
"num_evict_kb": 0,
"num_promote": 0,
"num_flush_mode_high": 0,
"num_flush_mode_low": 0,
"num_evict_mode_some": 0,
"num_evict_mode_full": 0,
"num_objects_pinned": 0,
"num_legacy_snapsets": 0,
"num_large_omap_objects": 0,
"num_objects_manifest": 0
},
"up": [
0,
1
],
"acting": [
0,
1
],
"blocked_by": [],
"up_primary": 0,
"acting_primary": 0,
"purged_snaps": []
},
"empty": 0,
"dne": 0,
"incomplete": 0,
"last_epoch_started": 159,
"hit_set_history": {
"current_last_update": "0'0",
"history": []
}
},
"peer_info": [
{
"peer": "1",
"pgid": "1.49",
"last_update": "127'38077",
"last_complete": "127'38077",
"log_tail": "127'35000",
"last_user_version": 38077,
"last_backfill": "MAX",
"last_backfill_bitwise": 0,
"purged_snaps": [],
"history": {
"epoch_created": 10,
"epoch_pool_created": 10,
"last_epoch_started": 159,
"last_interval_started": 158,
"last_epoch_clean": 159,
"last_interval_clean": 158,
"last_epoch_split": 0,
"last_epoch_marked_full": 0,
"same_up_since": 158,
"same_interval_since": 158,
"same_primary_since": 135,
"last_scrub": "127'36909",
"last_scrub_stamp": "2019-02-20
15:02:45.204342",
"last_deep_scrub": "127'36714",
"last_deep_scrub_stamp": "2019-02-16
07:55:15.205861",
"last_clean_scrub_stamp": "2019-02-20
15:02:45.204342"
},
"stats": {
"version": "127'38077",
"reported_seq": "58745",
"reported_epoch": "134",
"state": "active+undersized+degraded",
"last_fresh": "2019-02-20
19:06:19.180016",
"last_change": "2019-02-20
19:04:39.483332",
"last_active": "2019-02-20
19:06:19.180016",
"last_peered": "2019-02-20
19:06:19.180016",
"last_clean": "2019-02-20
18:23:33.675145",
"last_became_active": "2019-02-20
19:04:39.483332",
"last_became_peered": "2019-02-20
19:04:39.483332",
"last_unstale": "2019-02-20
19:06:19.180016",
"last_undegraded": "2019-02-20
19:04:39.477829",
"last_fullsized": "2019-02-20
19:04:39.477717",
"mapping_epoch": 158,
"log_start": "127'35000",
"ondisk_log_start": "127'35000",
"created": 10,
"last_epoch_clean": 124,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "127'36909",
"last_scrub_stamp": "2019-02-20
15:02:45.204342",
"last_deep_scrub": "127'36714",
"last_deep_scrub_stamp": "2019-02-16
07:55:15.205861",
"last_clean_scrub_stamp": "2019-02-20
15:02:45.204342",
"log_size": 3077,
"ondisk_log_size": 3077,
"stats_invalid": false,
"dirty_stats_invalid": false,
"omap_stats_invalid": false,
"hitset_stats_invalid": false,
"hitset_bytes_stats_invalid": false,
"pin_stats_invalid": false,
"manifest_stats_invalid": true,
"snaptrimq_len": 0,
"stat_sum": {
"num_bytes": 478347970,
"num_objects": 12052,
"num_object_clones": 0,
"num_object_copies": 24104,
"num_objects_missing_on_primary":
0,
"num_objects_missing": 0,
"num_objects_degraded": 12052,
"num_objects_misplaced": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 12052,
"num_whiteouts": 0,
"num_read": 20186,
"num_read_kb": 1952018,
"num_write": 38927,
"num_write_kb": 484756,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 6,
"num_bytes_recovered": 4101,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0,
"num_flush": 0,
"num_flush_kb": 0,
"num_evict": 0,
"num_evict_kb": 0,
"num_promote": 0,
"num_flush_mode_high": 0,
"num_flush_mode_low": 0,
"num_evict_mode_some": 0,
"num_evict_mode_full": 0,
"num_objects_pinned": 0,
"num_legacy_snapsets": 0,
"num_large_omap_objects": 0,
"num_objects_manifest": 0
},
"up": [
0,
1
],
"acting": [
0,
1
],
"blocked_by": [],
"up_primary": 0,
"acting_primary": 0,
"purged_snaps": []
},
"empty": 0,
"dne": 0,
"incomplete": 0,
"last_epoch_started": 159,
"hit_set_history": {
"current_last_update": "0'0",
"history": []
}
}
],
"recovery_state": [
{
"name": "Started/Primary/Active",
"enter_time": "2019-02-20
19:52:27.027151",
"might_have_unfound": [],
"recovery_progress": {
"backfill_targets": [],
"waiting_on_backfill": [],
"last_backfill_started": "MIN",
"backfill_info": {
"begin": "MIN",
"end": "MIN",
"objects": []
},
"peer_backfill_info": [],
"backfills_in_flight": [],
"recovering": [],
"pg_backend": {
"pull_from_peer": [],
"pushing": []
}
},
"scrub": {
"scrubber.epoch_start": "0",
"scrubber.active": false,
"scrubber.state": "INACTIVE",
"scrubber.start": "MIN",
"scrubber.end": "MIN",
"scrubber.max_end": "MIN",
"scrubber.subset_last_update": "0'0",
"scrubber.deep": false,
"scrubber.waiting_on_whom": []
}
},
{
"name": "Started",
"enter_time": "2019-02-20 19:52:25.976144"
}
],
"agent_state": {}
}
I wonder what it all means and how to get out of this
situation. The cluster seems to work normally. But
it's quite disconcerting as you can probably imagine.
Could it be a firewall issue? I'm not aware of any
changes and I don't see any peering problems...
Thank you
Ranjan
