Hello everyone,
We are seeing instability in 20.04.3 using HWE kernel and Ceph 16.2.6
w/Podman.
We have OSDs that fail after <24 hours and I'm not sure why.
Seeing this:
ceph crash info
2021-10-12T14:32:49.169552Z_d1ee94f7-1aaa-4221-abeb-68bd56d3c763
{
"backtrace": [
"/lib64/libpthread.so.0(+0x12b20) [0x7f4d31099b20]",
"pthread_cond_wait()",
"(std::condition_variable::wait(std::unique_lock<std::mutex>&)+0x10)
[0x7f4d306de8f0]",
"(Throttle::_wait(long, std::unique_lock<std::mutex>&)+0x10d)
[0x55c52a0f077d]",
"(Throttle::get(long, long)+0xb9) [0x55c52a0f1199]",
"(BlueStore::BlueStoreThrottle::try_start_transaction(KeyValueDB&,
BlueStore::TransContext&, std::chrono::time_point<ceph::mono_clock,
std::chrono::duration<unsigned long, std::ratio<1l, 1000000000l> > >)+0x29)
[0x55c529f362c9]",
"(BlueStore::queue_transactions(boost::intrusive_ptr<ObjectStore::CollectionImpl>&,
std::vector<ceph::os::Transaction, std::allocator<ceph::os::Transaction>
&, boost::intrusive_ptr<TrackedOp>, ThreadPool::TPHandle*)+0x854)
[0x55c529fb7664]",
"(non-virtual thunk to
PrimaryLogPG::queue_transactions(std::vector<ceph::os::Transaction,
std::allocator<ceph::os::Transaction> >&,
boost::intrusive_ptr<OpRequest>)+0x58) [0x55c529c0ee98]",
"(ReplicatedBackend::submit_transaction(hobject_t const&,
object_stat_sum_t const&, eversion_t const&, std::unique_ptr<PGTransaction,
std::default_delete<PGTransaction> >&&, eversion_t const&, eversion_t
const&, std::vector<pg_log_entry_t, std::allocator<pg_log_entry_t> >&&,
std::optional<pg_hit_set_history_t>&, Context*, unsigned long, osd_reqid_t,
boost::intrusive_ptr<OpRequest>)+0xcad) [0x55c529dfbedd]",
"(PrimaryLogPG::issue_repop(PrimaryLogPG::RepGather*,
PrimaryLogPG::OpContext*)+0xcf0) [0x55c529b7a630]",
"(PrimaryLogPG::execute_ctx(PrimaryLogPG::OpContext*)+0x115d)
[0x55c529bd65ed]",
"(PrimaryLogPG::do_op(boost::intrusive_ptr<OpRequest>&)+0x2de2)
[0x55c529bdf162]",
"(PrimaryLogPG::do_request(boost::intrusive_ptr<OpRequest>&,
ThreadPool::TPHandle&)+0xd1c) [0x55c529be64ac]",
"(OSD::dequeue_op(boost::intrusive_ptr<PG>,
boost::intrusive_ptr<OpRequest>, ThreadPool::TPHandle&)+0x309)
[0x55c529a6f1b9]",
"(ceph::osd::scheduler::PGOpItem::run(OSD*, OSDShard*,
boost::intrusive_ptr<PG>&, ThreadPool::TPHandle&)+0x68) [0x55c529ccc868]",
"(OSD::ShardedOpWQ::_process(unsigned int,
ceph::heartbeat_handle_d*)+0xa58) [0x55c529a8f1e8]",
"(ShardedThreadPool::shardedthreadpool_worker(unsigned int)+0x5c4)
[0x55c52a0fa6c4]",
"(ShardedThreadPool::WorkThreadSharded::entry()+0x14)
[0x55c52a0fd364]",
"/lib64/libpthread.so.0(+0x814a) [0x7f4d3108f14a]",
"clone()"
],
"ceph_version": "16.2.6",
"crash_id":
"2021-10-12T14:32:49.169552Z_d1ee94f7-1aaa-4221-abeb-68bd56d3c763",
"entity_name": "osd.14",
"os_id": "centos",
"os_name": "CentOS Linux",
"os_version": "8",
"os_version_id": "8",
"process_name": "ceph-osd",
"stack_sig":
"46b81ca079908da081327cbc114a9c1801dfdbb81303b85fff0d4107a1aeeabe",
"timestamp": "2021-10-12T14:32:49.169552Z",
"utsname_hostname": "<HOSTNAME_REMOVED>",
"utsname_machine": "x86_64",
"utsname_release": "5.11.0-37-generic",
"utsname_sysname": "Linux",
"utsname_version": "#41~20.04.2-Ubuntu SMP Fri Sep 24 09:06:38 UTC
2021"
dmesg on host shows:
[66258.080040] BUG: kernel NULL pointer dereference, address:
00000000000000c0
[66258.080067] #PF: supervisor read access in kernel mode
[66258.080081] #PF: error_code(0x0000) - not-present page
[66258.080093] PGD 0 P4D 0
[66258.080105] Oops: 0000 [#1] SMP NOPTI
[66258.080115] CPU: 35 PID: 4955 Comm: zabbix_agentd Not tainted
5.11.0-37-generic #41~20.04.2-Ubuntu
[66258.080137] Hardware name: Supermicro SSG-6049P-E1CR60L+/X11DSC+, BIOS
3.3 02/21/2020
[66258.080154] RIP: 0010:blk_mq_put_rq_ref+0xa/0x60
[66258.080171] Code: 15 0f b6 d3 4c 89 e7 be 01 00 00 00 e8 cf fe ff ff 5b
41 5c 5d c3 0f 0b 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 8b 47 10
<48> 8b 80 c0 00 00 00 48 89 e5 48 3b 78 40 74 1f 4c 8d 87 e8 00 00
[66258.080210] RSP: 0018:ffffa251249fbbc0 EFLAGS: 00010283
[66258.080224] RAX: 0000000000000000 RBX: ffffa251249fbc48 RCX:
0000000000000002
[66258.080240] RDX: 0000000000000001 RSI: 0000000000000202 RDI:
ffff9382f4d8e000
[66258.080256] RBP: ffffa251249fbbf8 R08: 0000000000000000 R09:
0000000000000035
[66258.080272] R10: abcc77118461cefd R11: ffff93b382257076 R12:
ffff9382f4d8e000
[66258.080288] R13: ffff9382f5670c00 R14: 0000000000000000 R15:
0000000000000001
[66258.080304] FS: 00007fedc0ef46c0(0000) GS:ffff93e13f4c0000(0000)
knlGS:0000000000000000
[66258.080322] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[66258.080335] CR2: 00000000000000c0 CR3: 000000011cc44005 CR4:
00000000007706e0
[66258.080351] DR0: 0000000000000000 DR1: 0000000000000000 DR2:
0000000000000000
[66258.080367] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7:
0000000000000400
[66258.080383] PKRU: 55555554
[66258.080391] Call Trace:
[66258.080401] ? bt_iter+0x54/0x90
[66258.080413] blk_mq_queue_tag_busy_iter+0x18b/0x2d0
[66258.080427] ? blk_mq_hctx_mark_pending+0x70/0x70
[66258.080440] ? blk_mq_hctx_mark_pending+0x70/0x70
[66258.080452] blk_mq_in_flight+0x38/0x60
[66258.080463] diskstats_show+0x75/0x2b0
[66258.080475] traverse+0x78/0x200
[66258.080485] seq_lseek+0x61/0xd0
[66258.080495] proc_reg_llseek+0x77/0xa0
[66258.080507] ksys_lseek+0x68/0xb0
[66258.080519] __x64_sys_lseek+0x1a/0x20
[66258.080530] do_syscall_64+0x38/0x90
[66258.080542] entry_SYSCALL_64_after_hwframe+0x44/0xa9
[66258.080557] RIP: 0033:0x7fedc237293b
[66258.081098] Code: c3 48 8b 15 8f 96 00 00 f7 d8 64 89 02 b8 ff ff ff ff
eb be 66 2e 0f 1f 84 00 00 00 00 00 90 f3 0f 1e fa b8 08 00 00 00 0f 05
<48> 3d 00 f0 ff ff 77 05 c3 0f 1f 40 00 48 8b 15 59 96 00 00 f7 d8
[66258.082122] RSP: 002b:00007fffb1cfd8e8 EFLAGS: 00000206 ORIG_RAX:
0000000000000008
[66258.082633] RAX: ffffffffffffffda RBX: 0000000000000004 RCX:
00007fedc237293b
[66258.083129] RDX: 0000000000000000 RSI: 00000000000022dc RDI:
0000000000000007
[66258.083612] RBP: 00007fffb1cfd950 R08: 0000000000000002 R09:
0000000061659bf0
[66258.084080] R10: 0000000000000000 R11: 0000000000000206 R12:
000055a02d1acd4d
[66258.084536] R13: 0000000000000003 R14: 0000000000000000 R15:
0000000000000000
[66258.084979] Modules linked in: binfmt_misc overlay bonding nls_iso8859_1
dm_multipath scsi_dh_rdac scsi_dh_emc scsi_dh_alua intel_rapl_msr
intel_rapl_common isst_if_common skx_edac nfit x86_pkg_temp_thermal
coretemp kvm_intel ipmi_ssif kvm rapl intel_cstate efi_pstore joydev
input_leds intel_pch_thermal mei_me mei ioatdma acpi_ipmi ipmi_si
ipmi_devintf ipmi_msghandler acpi_power_meter mac_hid sch_fq_codel msr
ip_tables x_tables autofs4 btrfs blake2b_generic raid10 raid456
async_raid6_recov async_memcpy async_pq async_xor async_tx xor raid6_pq
libcrc32c raid0 multipath linear mlx5_ib ib_uverbs ib_core hid_generic
usbhid hid ses enclosure scsi_transport_sas raid1 crct10dif_pclmul
crc32_pclmul ast drm_vram_helper i2c_algo_bit drm_ttm_helper
ghash_clmulni_intel ttm mlx5_core drm_kms_helper aesni_intel syscopyarea
crypto_simd sysfillrect sysimgblt cryptd glue_helper fb_sys_fops cec
pci_hyperv_intf rc_core mlxfw megaraid_sas tls drm ixgbe xfrm_algo dca mdio
ahci i2c_i801 vmd xhci_pci lpc_ich
[66258.085039] libahci i2c_smbus xhci_pci_renesas wmi
[66258.089332] CR2: 00000000000000c0
[66258.089825] ---[ end trace c1ae715ae7a3e043 ]---
[66258.144753] RIP: 0010:blk_mq_put_rq_ref+0xa/0x60
[66258.145285] Code: 15 0f b6 d3 4c 89 e7 be 01 00 00 00 e8 cf fe ff ff 5b
41 5c 5d c3 0f 0b 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 8b 47 10
<48> 8b 80 c0 00 00 00 48 89 e5 48 3b 78 40 74 1f 4c 8d 87 e8 00 00
[66258.146271] RSP: 0018:ffffa251249fbbc0 EFLAGS: 00010283
[66258.146768] RAX: 0000000000000000 RBX: ffffa251249fbc48 RCX:
0000000000000002
[66258.147268] RDX: 0000000000000001 RSI: 0000000000000202 RDI:
ffff9382f4d8e000
[66258.147779] RBP: ffffa251249fbbf8 R08: 0000000000000000 R09:
0000000000000035
[66258.148309] R10: abcc77118461cefd R11: ffff93b382257076 R12:
ffff9382f4d8e000
[66258.148802] R13: ffff9382f5670c00 R14: 0000000000000000 R15:
0000000000000001
[66258.149291] FS: 00007fedc0ef46c0(0000) GS:ffff93e13f4c0000(0000)
knlGS:0000000000000000
[66258.149785] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[66258.150281] CR2: 00000000000000c0 CR3: 000000011cc44005 CR4:
00000000007706e0
[66258.150791] DR0: 0000000000000000 DR1: 0000000000000000 DR2:
0000000000000000
[66258.151290] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7:
0000000000000400
[66258.151789] PKRU: 55555554
[69435.862190] perf: interrupt took too long (2513 > 2500), lowering
kernel.perf_event_max_sample_rate to 79500
Any guidance is much appreciated.
Thanks,
Marco
_______________________________________________
ceph-users mailing list -- ceph-users@xxxxxxx
To unsubscribe send an email to ceph-users-leave@xxxxxxx
