Hallo,
all of a sudden, 3 of my OSDs failed, showing similar messages in
the log:
.....
-5> 2022-03-28 14:19:02.451 7fc20fe99700 5 osd.145 pg_epoch:
616454 pg[70.2c6s1( empty local-lis/les=612106/612107 n=0
ec=148456/148456 lis/c
612106/612106 les/c/f 612107/612107/0 612106/612106/612101)
[168,145,102,96,112,124,128,134,56,34]p168(0) r=1 lpr=616429 crt=0'0
unknown mbc={}]
enter Started
-4> 2022-03-28 14:19:02.451 7fc20fe99700 5 osd.145 pg_epoch:
616454 pg[70.2c6s1( empty local-lis/les=612106/612107 n=0
ec=148456/148456 lis/c
612106/612106 les/c/f 612107/612107/0 612106/612106/612101)
[168,145,102,96,112,124,128,134,56,34]p168(0) r=1 lpr=616429 crt=0'0
unknown mbc={}]
enter Start
-3> 2022-03-28 14:19:02.451 7fc20fe99700 1 osd.145 pg_epoch:
616454 pg[70.2c6s1( empty local-lis/les=612106/612107 n=0
ec=148456/148456 lis/c
612106/612106 les/c/f 612107/612107/0 612106/612106/612101)
[168,145,102,96,112,124,128,134,56,34]p168(0) r=1 lpr=616429 crt=0'0
unknown mbc={}]
state<Start>: transitioning to Stray
-2> 2022-03-28 14:19:02.451 7fc20fe99700 5 osd.145 pg_epoch:
616454 pg[70.2c6s1( empty local-lis/les=612106/612107 n=0
ec=148456/148456 lis/c
612106/612106 les/c/f 612107/612107/0 612106/612106/612101)
[168,145,102,96,112,124,128,134,56,34]p168(0) r=1 lpr=616429 crt=0'0
unknown mbc={}]
exit Start 0.000008 0 0.000000
-1> 2022-03-28 14:19:02.451 7fc20fe99700 5 osd.145 pg_epoch:
616454 pg[70.2c6s1( empty local-lis/les=612106/612107 n=0
ec=148456/148456 lis/c
612106/612106 les/c/f 612107/612107/0 612106/612106/612101)
[168,145,102,96,112,124,128,134,56,34]p168(0) r=1 lpr=616429 crt=0'0
unknown mbc={}]
enter Started/Stray
0> 2022-03-28 14:19:02.451 7fc20f698700 -1 *** Caught signal
(Aborted) **
in thread 7fc20f698700 thread_name:tp_osd_tp
ceph version 14.2.22 (ca74598065096e6fcbd8433c8779a2be0c889351)
nautilus (stable)
1: (()+0x12ce0) [0x7fc2327dcce0]
2: (gsignal()+0x10f) [0x7fc231452a4f]
3: (abort()+0x127) [0x7fc231425db5]
4: (ceph::__ceph_abort(char const*, int, char const*,
std::__cxx11::basic_string<char, std::char_traits<char>,
std::allocator<char> > const&)+0x1b4) [0x55b8139cb671]
5: (PG::check_past_interval_bounds() const+0xc16) [0x55b813b586f6]
6: (PG::RecoveryState::Reset::react(PG::AdvMap const&)+0x3e8)
[0x55b813b963d8]
7: (boost::statechart::simple_state<PG::RecoveryState::Reset,
PG::RecoveryState::RecoveryMachine, boost::mpl::list<mpl_::na, mpl_::na,
mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na,
mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na,
mpl_::na, mpl_::na, mpl_::na, mpl_::na>,
(boost::statechart::history_mode)0>::react_impl(boost::statechart::event_base
const&, void const*)+0x7d) [0x55b813bdd32d]
8: (PG::handle_advance_map(std::shared_ptr<OSDMap const>,
std::shared_ptr<OSDMap const>, std::vector<int, std::allocator<int> >&,
int, std::vector<int, std::allocator<int> >&, int,
PG::RecoveryCtx*)+0x39d) [0x55b813b7b5fd]
9: (OSD::advance_pg(unsigned int, PG*, ThreadPool::TPHandle&,
PG::RecoveryCtx*)+0x2e9) [0x55b813ad14e9]
10: (OSD::dequeue_peering_evt(OSDShard*, PG*,
std::shared_ptr<PGPeeringEvent>, ThreadPool::TPHandle&)+0xaa)
[0x55b813ae345a]
11: (PGPeeringItem::run(OSD*, OSDShard*, boost::intrusive_ptr<PG>&,
ThreadPool::TPHandle&)+0x55) [0x55b813d66c15]
12: (OSD::ShardedOpWQ::_process(unsigned int,
ceph::heartbeat_handle_d*)+0x1366) [0x55b813adff46]
13: (ShardedThreadPool::shardedthreadpool_worker(unsigned int)+0x5c4)
[0x55b8140dc944]
14: (ShardedThreadPool::WorkThreadSharded::entry()+0x14) [0x55b8140df514]
15: (()+0x81cf) [0x7fc2327d21cf]
16: (clone()+0x43) [0x7fc23143dd83]
Trying to "activate --all", rebotting server, and such, did not help.
I am now stuck with one PG (85.25) down, find below the output from "query".
The PG belongs to a 3+2 erasure-coded pool.
As the devices corresponding to the 3 down OSDs are properly mounted, is
there a way to get PG.ID=85.25 from the devices and copy it elsewhere?
Actually, I tried to find 85.25 in the 3 down OSDs with command:
~]# ceph-objectstore-tool --data-path /var/lib/ceph/osd/cephpa1-158/
--no-mon-config --pgid 85.25 --op export --file /tmp/pg_85-25
PG '85.25' not found
which puzzled me... is there a way to search such PG.ID over the
whole cluster?
Thanks for your help!
Fulvio
====================
~]# ceph --cluster cephpa1 health detail | grep down
.....
PG_AVAILABILITY Reduced data availability: 1 pg inactive, 1 pg down
pg 85.25 is down+remapped, acting
[2147483647,2147483647,96,2147483647,2147483647]
~]# ceph --cluster cephpa1 pg 85.25 query
{
"state": "down+remapped",
"snap_trimq": "[]",
"snap_trimq_len": 0,
"epoch": 617667,
"up": [
2147483647,
2147483647,
2147483647,
2147483647,
2147483647
],
"acting": [
2147483647,
2147483647,
96,
2147483647,
2147483647
],
"info": {
"pgid": "85.25s2",
"last_update": "606021'521273",
"last_complete": "606021'521273",
"log_tail": "605873'518175",
"last_user_version": 521273,
"last_backfill": "MAX",
"last_backfill_bitwise": 1,
"purged_snaps": [],
"history": {
"epoch_created": 478130,
"epoch_pool_created": 478130,
"last_epoch_started": 611605,
"last_interval_started": 611604,
"last_epoch_clean": 595152,
"last_interval_clean": 595151,
"last_epoch_split": 0,
"last_epoch_marked_full": 0,
"same_up_since": 617661,
"same_interval_since": 617661,
"same_primary_since": 616519,
"last_scrub": "597862'517979",
"last_scrub_stamp": "2022-03-15 05:08:42.561720",
"last_deep_scrub": "597454'511650",
"last_deep_scrub_stamp": "2022-03-09 18:05:34.205121",
"last_clean_scrub_stamp": "2022-03-15 05:08:42.561720"
},
"stats": {
"version": "606021'521273",
"reported_seq": "1733225",
"reported_epoch": "617667",
"state": "down+remapped",
"last_fresh": "2022-03-28 14:17:48.920542",
"last_change": "2022-03-28 14:15:42.025193",
"last_active": "2022-03-25 15:24:42.557667",
"last_peered": "2022-03-25 15:24:25.912029",
"last_clean": "2022-03-15 01:50:02.979366",
"last_became_active": "2022-03-25 12:44:52.379711",
"last_became_peered": "2022-03-25 12:44:52.379711",
"last_unstale": "2022-03-28 14:17:48.920542",
"last_undegraded": "2022-03-28 14:17:48.920542",
"last_fullsized": "2022-03-28 14:17:48.920542",
"mapping_epoch": 617661,
"log_start": "605873'518175",
"ondisk_log_start": "605873'518175",
"created": 478130,
"last_epoch_clean": 595152,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "597862'517979",
"last_scrub_stamp": "2022-03-15 05:08:42.561720",
"last_deep_scrub": "597454'511650",
"last_deep_scrub_stamp": "2022-03-09 18:05:34.205121",
"last_clean_scrub_stamp": "2022-03-15 05:08:42.561720",
"log_size": 3098,
"ondisk_log_size": 3098,
"stats_invalid": false,
"dirty_stats_invalid": false,
"omap_stats_invalid": false,
"hitset_stats_invalid": false,
"hitset_bytes_stats_invalid": false,
"pin_stats_invalid": false,
"manifest_stats_invalid": false,
"snaptrimq_len": 0,
"stat_sum": {
"num_bytes": 165108592640,
"num_objects": 39402,
"num_object_clones": 0,
"num_object_copies": 197010,
"num_objects_missing_on_primary": 0,
"num_objects_missing": 0,
"num_objects_degraded": 0,
"num_objects_misplaced": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 39402,
"num_whiteouts": 0,
"num_read": 338705,
"num_read_kb": 124297874,
"num_write": 316971,
"num_write_kb": 311968402,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 196778,
"num_bytes_recovered": 824577241088,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0,
"num_flush": 0,
"num_flush_kb": 0,
"num_evict": 0,
"num_evict_kb": 0,
"num_promote": 0,
"num_flush_mode_high": 0,
"num_flush_mode_low": 0,
"num_evict_mode_some": 0,
"num_evict_mode_full": 0,
"num_objects_pinned": 0,
"num_legacy_snapsets": 0,
"num_large_omap_objects": 0,
"num_objects_manifest": 0,
"num_omap_bytes": 0,
"num_omap_keys": 0,
"num_objects_repaired": 0
},
"up": [
2147483647,
2147483647,
2147483647,
2147483647,
2147483647
],
"acting": [
2147483647,
2147483647,
96,
2147483647,
2147483647
],
"avail_no_missing": [],
"object_location_counts": [],
"blocked_by": [
121
],
"up_primary": -1,
"acting_primary": 96,
"purged_snaps": []
},
"empty": 0,
"dne": 0,
"incomplete": 0,
"last_epoch_started": 616455,
"hit_set_history": {
"current_last_update": "0'0",
"history": []
}
},
"peer_info": [],
"recovery_state": [
{
"name": "Started/Primary/Peering/Down",
"enter_time": "2022-03-28 14:15:42.025185",
"comment": "not enough up instances of this PG to go active"
},
{
"name": "Started/Primary/Peering",
"enter_time": "2022-03-28 14:15:42.025128",
"past_intervals": [
{
"first": "595151",
"last": "617660",
"all_participants": [
{
"osd": 77,
"shard": 0
},
{
"osd": 82,
"shard": 2
},
{
"osd": 90,
"shard": 0
},
{
"osd": 91,
"shard": 4
},
{
"osd": 96,
"shard": 2
},
{
"osd": 121,
"shard": 3
},
{
"osd": 140,
"shard": 1
},
{
"osd": 159,
"shard": 4
}
],
"intervals": [
{
"first": "610237",
"last": "611219",
"acting": "96(2),121(3),159(4)"
},
{
"first": "611604",
"last": "616452",
"acting": "121(3),140(1),159(4)"
},
{
"first": "616461",
"last": "616518",
"acting": "96(2),140(1),159(4)"
}
]
}
],
"probing_osds": [
"77(0)",
"82(2)",
"90(0)",
"91(4)",
"96(2)",
"140(1)",
"159(4)"
],
"blocked": "peering is blocked due to down osds",
"down_osds_we_would_probe": [
121
],
"peering_blocked_by": [
{
"osd": 121,
"current_lost_at": 0,
"comment": "starting or marking this osd lost may
let us proceed"
}
]
},
{
"name": "Started",
"enter_time": "2022-03-28 14:15:42.025068"
}
],
"agent_state": {}
}
[
--
Fulvio Galeazzi
GARR-CSD Department
skype: fgaleazzi70
tel.: +39-334-6533-250
_______________________________________________
ceph-users mailing list -- ceph-users@xxxxxxx
To unsubscribe send an email to ceph-users-leave@xxxxxxx
