Hello again,
i've been able to extract the traces regarding this error and i'll provide more information:
Mount options (excluding helper provided): 'recover_session=clean,fsc=XXXX'
Kernel version: 6.10.2
Ceph versions:
{
"mon": {
"ceph version 18.2.1 (e3fce6809130d78ac0058fc87e537ecd926cd213) reef (stable)": 2,
"ceph version 18.2.2 (531c0d11a1c5d39fbfe6aa8a521f023abf3bf3e2) reef (stable)": 1
},
"mgr": {
"ceph version 18.2.1 (e3fce6809130d78ac0058fc87e537ecd926cd213) reef (stable)": 2,
"ceph version 18.2.2 (531c0d11a1c5d39fbfe6aa8a521f023abf3bf3e2) reef (stable)": 1
},
"osd": {
"ceph version 18.2.1 (e3fce6809130d78ac0058fc87e537ecd926cd213) reef (stable)": 12,
"ceph version 18.2.2 (531c0d11a1c5d39fbfe6aa8a521f023abf3bf3e2) reef (stable)": 6
},
"mds": {
"ceph version 18.2.1 (e3fce6809130d78ac0058fc87e537ecd926cd213) reef (stable)": 4,
"ceph version 18.2.2 (531c0d11a1c5d39fbfe6aa8a521f023abf3bf3e2) reef (stable)": 2
},
"overall": {
"ceph version 18.2.1 (e3fce6809130d78ac0058fc87e537ecd926cd213) reef (stable)": 20,
"ceph version 18.2.2 (531c0d11a1c5d39fbfe6aa8a521f023abf3bf3e2) reef (stable)": 10
}
}
Kernel log snippet:
[Aug 8 06:33] libceph: mon1 (2)REDACTED:3300 session established [ +0.142682] libceph: client363957071 fsid REDACTED-FSID [ +0.644346] libceph: mon2 (2)REDACTED:3300 session established [ +0.153025] libceph: client363957201 fsid REDACTED-FSID [ +0.894635] libceph: mon1 (2)REDACTED:3300 session established [ +0.139688] libceph: client363957074 fsid REDACTED-FSID [ +0.569369] libceph: mon2 (2)REDACTED:3300 session established [ +0.297118] libceph: client363957204 fsid REDACTED-FSID [ +0.547002] libceph: mon1 (2)REDACTED:3300 session established [ +0.147304] libceph: client363957077 fsid REDACTED-FSID [ +0.961183] libceph: mon1 (2)REDACTED:3300 session established [ +0.186801] libceph: client363957080 fsid REDACTED-FSID [ +0.624825] libceph: mon1 (2)REDACTED:3300 session established [ +0.390715] libceph: client363957083 fsid REDACTED-FSID [ +0.975917] libceph: mon0 (2)REDACTED:3300 session established [ +0.300246] libceph: client363957292 fsid REDACTED-FSID [ +0.820914] libceph: mon0 (2)REDACTED:3300 session established [ +0.219877] libceph: client363957295 fsid REDACTED-FSID [Aug 8 06:47] BUG: Bad page state in process kworker/0:2 pfn:11218b [ +0.000014] page: refcount:0 mapcount:0 mapping:0000000037efe36d index:0x0 pfn:0x11218b [ +0.000011] aops:exit_ceph [ceph] ino:10000a30a34 dentry name:"cookies.txt" [ +0.000030] flags: 0x200000000000008(uptodate|node=0|zone=2) [ +0.000010] raw: 0200000000000008 dead000000000100 dead000000000122 ffff9b084bfb2d38 [ +0.000006] raw: 0000000000000000 0000000000000000 00000000ffffffff 0000000000000000 [ +0.000003] page dumped because: non-NULL mapping [ +0.000004] Modules linked in: snd_usb_audio snd_usbmidi_lib snd_ump snd_rawmidi snd_seq_device hid_generic usbhid hid ceph libceph cmac ctr ccm bridge stp llc wireguard curve25519_x86_64 libchacha20poly1305 chacha_x86_64 poly1305_x86_64 ip6_udp_tunnel udp_tunnel libcurve25519_generic libchacha nft_reject_inet nf_reject_ipv4 nf_reject_ipv6 nft_reject nft_ct nft_chain_nat nf_nat nf_conntrack nf_defrag_ipv6 nf_defrag_ipv4 nf_tables nfnetlink cachefiles netfs joydev mousedev uvcvideo uvc btusb videobuf2_vmalloc btrtl videobuf2_memops btmtk videobuf2_v4l2 btbcm videobuf2_common btintel videodev mc bluetooth ecdh_generic ecc ext4 crc16 mbcache jbd2 intel_rapl_msr intel_rapl_common edac_mce_amd kvm_amd snd_soc_dmic snd_acp3x_rn snd_acp3x_pdm_dma kvm input_leds psmouse rapl serio_raw pcspkr efi_pstore acpi_cpufreq snd_sof_amd_acp63 snd_sof_amd_vangogh snd_sof_amd_rembrandt snd_sof_amd_renoir snd_hda_codec_realtek wmi_bmof snd_sof_amd_acp snd_hda_codec_generic snd_sof_pci snd_hda_scodec_component rtw89_8852ae snd_sof [ +0.000221] rtw89_8852a snd_sof_utils snd_sof_xtensa_dsp snd_hda_codec_hdmi rtw89_pci snd_soc_core snd_hda_intel rtw89_core snd_compress snd_ctl_led snd_pcm_dmaengine snd_hda_codec radeon snd_intel_dspcfg mac80211 snd_hda_core snd_intel_sdw_acpi snd_hwdep thinkpad_acpi snd_pci_acp6x snd_pcm nvram snd_pci_acp5x sparse_keymap af_packet snd_timer snd_rn_pci_acp3x platform_profile cfg80211 snd_acp_config snd snd_soc_acpi tpm_crb ipmi_devintf r8169 k10temp snd_pci_acp3x sp5100_tco ccp soundcore dm_multipath rfkill libarc4 realtek ipmi_msghandler thermal tpm_tis ac ucsi_acpi tpm_tis_core tun typec_ucsi tpm typec roles rng_core evdev mac_hid efivarfs dm_crypt encrypted_keys trusted dm_mod rtsx_pci_sdmmc mmc_core i2c_scmi crct10dif_pclmul crc32_pclmul ghash_clmulni_intel sha512_ssse3 sha256_ssse3 sha1_ssse3 aesni_intel crypto_simd cryptd ehci_pci ehci_hcd i2c_piix4 rtsx_pci amdgpu drm_exec amdxcp drm_buddy gpu_sched drm_suballoc_helper drm_ttm_helper ttm drm_display_helper drm_kms_helper drm agpgart i2c_algo_bit video [ +0.000262] xhci_pci xhci_hcd usbcore usb_common nvme nvme_core t10_pi crc64_rocksoft_generic crc64_rocksoft hwmon crc64 battery button wmi loop btrfs blake2b_generic libcrc32c crc32c_generic crc32c_intel xor raid6_pq [ +0.000058] CPU: 0 PID: 10405 Comm: kworker/0:2 Not tainted 6.10.2-2-full #3-Alpine [ +0.000009] Hardware name: LENOVO 20UH0036GE/20UH0036GE, BIOS R1CET77W(1.46 ) 10/26/2023 [ +0.000005] Workqueue: ceph-msgr ceph_con_workfn [libceph] [ +0.000026] Call Trace: [ +0.000004] <TASK> [ +0.000005] dump_stack_lvl+0x5c/0x80 [ +0.000012] dump_stack+0x10/0x16 [ +0.000008] bad_page.cold+0x62/0x90 [ +0.000008] free_page_is_bad_report+0x75/0x90 [ +0.000007] free_unref_folios+0x3a3/0x5f0 [ +0.000008] ? srso_return_thunk+0x5/0x5f [ +0.000009] folios_put_refs+0xdf/0x120 [ +0.000010] release_pages+0xc2/0x110 [ +0.000007] ? srso_return_thunk+0x5/0x5f [ +0.000006] ? lock_timer_base+0x76/0xa0 [ +0.000008] ? srso_return_thunk+0x5/0x5f [ +0.000006] ? __mod_timer+0x115/0x330 [ +0.000010] ? srso_return_thunk+0x5/0x5f [ +0.000005] ? __mod_lruvec_state+0x31/0x40 [ +0.000008] ? srso_return_thunk+0x5/0x5f [ +0.000006] ? __lruvec_stat_mod_folio+0x6b/0xb0 [ +0.000008] ? srso_return_thunk+0x5/0x5f [ +0.000005] ? srso_return_thunk+0x5/0x5f [ +0.000006] ? folio_memcg_unlock+0x32/0x70 [ +0.000007] ? srso_return_thunk+0x5/0x5f [ +0.000005] ? __folio_end_writeback+0xc2/0x310 [ +0.000011] writepages_finish+0x24d/0x3e0 [ceph] [ +0.000025] ? ceph_msg_revoke_incoming+0x3a/0xc0 [libceph] [ +0.000024] handle_reply+0x59e/0x710 [libceph] [ +0.000027] ? srso_return_thunk+0x5/0x5f [ +0.000010] ? srso_return_thunk+0x5/0x5f [ +0.000006] ? sg_init_table+0x15/0x40 [ +0.000007] ? srso_return_thunk+0x5/0x5f [ +0.000006] ? srso_return_thunk+0x5/0x5f [ +0.000005] ? scatterwalk_map_and_copy+0x4c/0x70 [ +0.000014] osd_dispatch+0x54/0xa50 [libceph] [ +0.000023] ? srso_return_thunk+0x5/0x5f [ +0.000006] ? free_unref_page+0x24f/0x3b0 [ +0.000007] ? generic_gcmaes_decrypt+0x4c/0x70 [aesni_intel] [ +0.000014] ceph_con_process_message+0x6a/0x80 [libceph] [ +0.000021] ceph_con_v2_try_read+0xba3/0x16f0 [libceph] [ +0.000023] ? srso_return_thunk+0x5/0x5f [ +0.000005] ? update_load_avg+0x7b/0x6d0 [ +0.000010] ? srso_return_thunk+0x5/0x5f [ +0.000005] ? set_next_entity+0x5e/0xc0 [ +0.000007] ? srso_return_thunk+0x5/0x5f [ +0.000007] ceph_con_workfn+0x154/0x520 [libceph] [ +0.000021] ? srso_return_thunk+0x5/0x5f [ +0.000007] process_one_work+0x16e/0x290 [ +0.000011] worker_thread+0x31a/0x450 [ +0.000007] ? __pfx_worker_thread+0x10/0x10 [ +0.000007] kthread+0xdf/0x110 [ +0.000007] ? __pfx_kthread+0x10/0x10 [ +0.000007] ret_from_fork+0x38/0x60 [ +0.000007] ? __pfx_kthread+0x10/0x10 [ +0.000004] ret_from_fork_asm+0x1a/0x30 [ +0.000008] </TASK> [ +0.000050] Disabling lock debugging due to kernel taint [ +0.000001] BUG: Bad page state in process kworker/0:2 pfn:11218c [ +0.000002] page: refcount:0 mapcount:0 mapping:0000000037efe36d index:0x1 pfn:0x11218c [ +0.000002] aops:exit_ceph [ceph] ino:10000a30a34 dentry name:"cookies.txt" [ +0.000007] flags: 0x200000000000008(uptodate|node=0|zone=2) [ +0.000002] raw: 0200000000000008 dead000000000100 dead000000000122 ffff9b084bfb2d38 [ +0.000002] raw: 0000000000000001 0000000000000000 00000000ffffffff 0000000000000000 [ +0.000002] page dumped because: non-NULL mapping [ +0.000001] Modules linked in: snd_usb_audio snd_usbmidi_lib snd_ump snd_rawmidi snd_seq_device hid_generic usbhid hid ceph libceph cmac ctr ccm bridge stp llc wireguard curve25519_x86_64 libchacha20poly1305 chacha_x86_64 poly1305_x86_64 ip6_udp_tunnel udp_tunnel libcurve25519_generic libchacha nft_reject_inet nf_reject_ipv4 nf_reject_ipv6 nft_reject nft_ct nft_chain_nat nf_nat nf_conntrack nf_defrag_ipv6 nf_defrag_ipv4 nf_tables nfnetlink cachefiles netfs joydev mousedev uvcvideo uvc btusb videobuf2_vmalloc btrtl videobuf2_memops btmtk videobuf2_v4l2 btbcm videobuf2_common btintel videodev mc bluetooth ecdh_generic ecc ext4 crc16 mbcache jbd2 intel_rapl_msr intel_rapl_common edac_mce_amd kvm_amd snd_soc_dmic snd_acp3x_rn snd_acp3x_pdm_dma kvm input_leds psmouse rapl serio_raw pcspkr efi_pstore acpi_cpufreq snd_sof_amd_acp63 snd_sof_amd_vangogh snd_sof_amd_rembrandt snd_sof_amd_renoir snd_hda_codec_realtek wmi_bmof snd_sof_amd_acp snd_hda_codec_generic snd_sof_pci snd_hda_scodec_component rtw89_8852ae snd_sof [ +0.000067] rtw89_8852a snd_sof_utils snd_sof_xtensa_dsp snd_hda_codec_hdmi rtw89_pci snd_soc_core snd_hda_intel rtw89_core snd_compress snd_ctl_led snd_pcm_dmaengine snd_hda_codec radeon snd_intel_dspcfg mac80211 snd_hda_core snd_intel_sdw_acpi snd_hwdep thinkpad_acpi snd_pci_acp6x snd_pcm nvram snd_pci_acp5x sparse_keymap af_packet snd_timer snd_rn_pci_acp3x platform_profile cfg80211 snd_acp_config snd snd_soc_acpi tpm_crb ipmi_devintf r8169 k10temp snd_pci_acp3x sp5100_tco ccp soundcore dm_multipath rfkill libarc4 realtek ipmi_msghandler thermal tpm_tis ac ucsi_acpi tpm_tis_core tun typec_ucsi tpm typec roles rng_core evdev mac_hid efivarfs dm_crypt encrypted_keys trusted dm_mod rtsx_pci_sdmmc mmc_core i2c_scmi crct10dif_pclmul crc32_pclmul ghash_clmulni_intel sha512_ssse3 sha256_ssse3 sha1_ssse3 aesni_intel crypto_simd cryptd ehci_pci ehci_hcd i2c_piix4 rtsx_pci amdgpu drm_exec amdxcp drm_buddy gpu_sched drm_suballoc_helper drm_ttm_helper ttm drm_display_helper drm_kms_helper drm agpgart i2c_algo_bit video [ +0.000076] xhci_pci xhci_hcd usbcore usb_common nvme nvme_core t10_pi crc64_rocksoft_generic crc64_rocksoft hwmon crc64 battery button wmi loop btrfs blake2b_generic libcrc32c crc32c_generic crc32c_intel xor raid6_pq [ +0.000018] CPU: 0 PID: 10405 Comm: kworker/0:2 Tainted: G B 6.10.2-2-full #3-Alpine [ +0.000003] Hardware name: LENOVO 20UH0036GE/20UH0036GE, BIOS R1CET77W(1.46 ) 10/26/2023 [ +0.000001] Workqueue: ceph-msgr ceph_con_workfn [libceph] [ +0.000008] Call Trace: [ +0.000001] <TASK> [ +0.000001] dump_stack_lvl+0x5c/0x80 [ +0.000003] dump_stack+0x10/0x16 [ +0.000003] bad_page.cold+0x62/0x90 [ +0.000002] free_page_is_bad_report+0x75/0x90 [ +0.000002] free_unref_folios+0x3a3/0x5f0 [ +0.000003] ? srso_return_thunk+0x5/0x5f [ +0.000003] folios_put_refs+0xdf/0x120 [ +0.000003] release_pages+0xc2/0x110 [ +0.000002] ? srso_return_thunk+0x5/0x5f [ +0.000002] ? lock_timer_base+0x76/0xa0 [ +0.000003] ? srso_return_thunk+0x5/0x5f [ +0.000001] ? __mod_timer+0x115/0x330 [ +0.000003] ? srso_return_thunk+0x5/0x5f [ +0.000002] ? __mod_lruvec_state+0x31/0x40 [ +0.000003] ? srso_return_thunk+0x5/0x5f [ +0.000002] ? __lruvec_stat_mod_folio+0x6b/0xb0 [ +0.000002] ? srso_return_thunk+0x5/0x5f [ +0.000002] ? srso_return_thunk+0x5/0x5f [ +0.000002] ? folio_memcg_unlock+0x32/0x70 [ +0.000002] ? srso_return_thunk+0x5/0x5f [ +0.000002] ? __folio_end_writeback+0xc2/0x310 [ +0.000003] writepages_finish+0x24d/0x3e0 [ceph] [ +0.000008] ? ceph_msg_revoke_incoming+0x3a/0xc0 [libceph] [ +0.000007] handle_reply+0x59e/0x710 [libceph] [ +0.000008] ? srso_return_thunk+0x5/0x5f [ +0.000003] ? srso_return_thunk+0x5/0x5f [ +0.000002] ? sg_init_table+0x15/0x40 [ +0.000002] ? srso_return_thunk+0x5/0x5f [ +0.000002] ? srso_return_thunk+0x5/0x5f [ +0.000001] ? scatterwalk_map_and_copy+0x4c/0x70 [ +0.000005] osd_dispatch+0x54/0xa50 [libceph] [ +0.000007] ? srso_return_thunk+0x5/0x5f [ +0.000002] ? free_unref_page+0x24f/0x3b0 [ +0.000002] ? generic_gcmaes_decrypt+0x4c/0x70 [aesni_intel] [ +0.000005] ceph_con_process_message+0x6a/0x80 [libceph] [ +0.000007] ceph_con_v2_try_read+0xba3/0x16f0 [libceph] [ +0.000007] ? srso_return_thunk+0x5/0x5f [ +0.000002] ? update_load_avg+0x7b/0x6d0 [ +0.000003] ? srso_return_thunk+0x5/0x5f [ +0.000002] ? set_next_entity+0x5e/0xc0 [ +0.000002] ? srso_return_thunk+0x5/0x5f [ +0.000002] ceph_con_workfn+0x154/0x520 [libceph] [ +0.000007] ? srso_return_thunk+0x5/0x5f [ +0.000002] process_one_work+0x16e/0x290 [ +0.000004] worker_thread+0x31a/0x450 [ +0.000002] ? __pfx_worker_thread+0x10/0x10 [ +0.000002] kthread+0xdf/0x110 [ +0.000002] ? __pfx_kthread+0x10/0x10 [ +0.000002] ret_from_fork+0x38/0x60 [ +0.000003] ? __pfx_kthread+0x10/0x10 [ +0.000002] ret_from_fork_asm+0x1a/0x30 [ +0.000004] </TASK> [ +0.000001] BUG: Bad page state in process kworker/0:2 pfn:11218d [ +0.000002] page: refcount:0 mapcount:0 mapping:0000000037efe36d index:0x2 pfn:0x11218d [ +0.000001] aops:exit_ceph [ceph] ino:10000a30a34 dentry name:"cookies.txt" [ +0.000007] flags: 0x200000000000008(uptodate|node=0|zone=2) [ +0.000002] raw: 0200000000000008 dead000000000100 dead000000000122 ffff9b084bfb2d38 [ +0.000002] raw: 0000000000000002 0000000000000000 00000000ffffffff 0000000000000000 [ +0.000001] page dumped because: non-NULL mapping [ +0.000001] Modules linked in: snd_usb_audio snd_usbmidi_lib snd_ump snd_rawmidi snd_seq_device hid_generic usbhid hid ceph libceph cmac ctr ccm bridge stp llc wireguard curve25519_x86_64 libchacha20poly1305 chacha_x86_64 poly1305_x86_64 ip6_udp_tunnel udp_tunnel libcurve25519_generic libchacha nft_reject_inet nf_reject_ipv4 nf_reject_ipv6 nft_reject nft_ct nft_chain_nat nf_nat nf_conntrack nf_defrag_ipv6 nf_defrag_ipv4 nf_tables nfnetlink cachefiles netfs joydev mousedev uvcvideo uvc btusb videobuf2_vmalloc btrtl videobuf2_memops btmtk videobuf2_v4l2 btbcm videobuf2_common btintel videodev mc bluetooth ecdh_generic ecc ext4 crc16 mbcache jbd2 intel_rapl_msr intel_rapl_common edac_mce_amd kvm_amd snd_soc_dmic snd_acp3x_rn snd_acp3x_pdm_dma kvm input_leds psmouse rapl serio_raw pcspkr efi_pstore acpi_cpufreq snd_sof_amd_acp63 snd_sof_amd_vangogh snd_sof_amd_rembrandt snd_sof_amd_renoir snd_hda_codec_realtek wmi_bmof snd_sof_amd_acp snd_hda_codec_generic snd_sof_pci snd_hda_scodec_component rtw89_8852ae snd_sof [ +0.000067] rtw89_8852a snd_sof_utils snd_sof_xtensa_dsp snd_hda_codec_hdmi rtw89_pci snd_soc_core snd_hda_intel rtw89_core snd_compress snd_ctl_led snd_pcm_dmaengine snd_hda_codec radeon snd_intel_dspcfg mac80211 snd_hda_core snd_intel_sdw_acpi snd_hwdep thinkpad_acpi snd_pci_acp6x snd_pcm nvram snd_pci_acp5x sparse_keymap af_packet snd_timer snd_rn_pci_acp3x platform_profile cfg80211 snd_acp_config snd snd_soc_acpi tpm_crb ipmi_devintf r8169 k10temp snd_pci_acp3x sp5100_tco ccp soundcore dm_multipath rfkill libarc4 realtek ipmi_msghandler thermal tpm_tis ac ucsi_acpi tpm_tis_core tun typec_ucsi tpm typec roles rng_core evdev mac_hid efivarfs dm_crypt encrypted_keys trusted dm_mod rtsx_pci_sdmmc mmc_core i2c_scmi crct10dif_pclmul crc32_pclmul ghash_clmulni_intel sha512_ssse3 sha256_ssse3 sha1_ssse3 aesni_intel crypto_simd cryptd ehci_pci ehci_hcd i2c_piix4 rtsx_pci amdgpu drm_exec amdxcp drm_buddy gpu_sched drm_suballoc_helper drm_ttm_helper ttm drm_display_helper drm_kms_helper drm agpgart i2c_algo_bit video [ +0.000071] xhci_pci xhci_hcd usbcore usb_common nvme nvme_core t10_pi crc64_rocksoft_generic crc64_rocksoft hwmon crc64 battery button wmi loop btrfs blake2b_generic libcrc32c crc32c_generic crc32c_intel xor raid6_pq [ +0.000018] CPU: 0 PID: 10405 Comm: kworker/0:2 Tainted: G B 6.10.2-2-full #3-Alpine [ +0.000002] Hardware name: LENOVO 20UH0036GE/20UH0036GE, BIOS R1CET77W(1.46 ) 10/26/2023 [ +0.000001] Workqueue: ceph-msgr ceph_con_workfn [libceph] [ +0.000007] Call Trace: [ +0.000001] <TASK> [ +0.000001] dump_stack_lvl+0x5c/0x80 [ +0.000003] dump_stack+0x10/0x16 [ +0.000003] bad_page.cold+0x62/0x90 [ +0.000002] free_page_is_bad_report+0x75/0x90 [ +0.000002] free_unref_folios+0x3a3/0x5f0 [ +0.000002] ? srso_return_thunk+0x5/0x5f [ +0.000003] folios_put_refs+0xdf/0x120 [ +0.000003] release_pages+0xc2/0x110 [ +0.000002] ? srso_return_thunk+0x5/0x5f [ +0.000002] ? lock_timer_base+0x76/0xa0 [ +0.000003] ? srso_return_thunk+0x5/0x5f [ +0.000002] ? __mod_timer+0x115/0x330 [ +0.000003] ? srso_return_thunk+0x5/0x5f [ +0.000002] ? __mod_lruvec_state+0x31/0x40 [ +0.000002] ? srso_return_thunk+0x5/0x5f [ +0.000002] ? __lruvec_stat_mod_folio+0x6b/0xb0 [ +0.000003] ? srso_return_thunk+0x5/0x5f [ +0.000002] ? srso_return_thunk+0x5/0x5f [ +0.000002] ? folio_memcg_unlock+0x32/0x70 [ +0.000003] ? srso_return_thunk+0x5/0x5f [ +0.000002] ? __folio_end_writeback+0xc2/0x310 [ +0.000003] writepages_finish+0x24d/0x3e0 [ceph] [ +0.000008] ? ceph_msg_revoke_incoming+0x3a/0xc0 [libceph] [ +0.000007] handle_reply+0x59e/0x710 [libceph] [ +0.000007] ? srso_return_thunk+0x5/0x5f [ +0.000003] ? srso_return_thunk+0x5/0x5f [ +0.000002] ? sg_init_table+0x15/0x40 [ +0.000002] ? srso_return_thunk+0x5/0x5f [ +0.000002] ? srso_return_thunk+0x5/0x5f [ +0.000001] ? scatterwalk_map_and_copy+0x4c/0x70 [ +0.000005] osd_dispatch+0x54/0xa50 [libceph] [ +0.000007] ? srso_return_thunk+0x5/0x5f [ +0.000001] ? free_unref_page+0x24f/0x3b0 [ +0.000003] ? generic_gcmaes_decrypt+0x4c/0x70 [aesni_intel] [ +0.000004] ceph_con_process_message+0x6a/0x80 [libceph] [ +0.000007] ceph_con_v2_try_read+0xba3/0x16f0 [libceph] [ +0.000007] ? srso_return_thunk+0x5/0x5f [ +0.000001] ? update_load_avg+0x7b/0x6d0 [ +0.000003] ? srso_return_thunk+0x5/0x5f [ +0.000002] ? set_next_entity+0x5e/0xc0 [ +0.000002] ? srso_return_thunk+0x5/0x5f [ +0.000003] ceph_con_workfn+0x154/0x520 [libceph] [ +0.000006] ? srso_return_thunk+0x5/0x5f [ +0.000002] process_one_work+0x16e/0x290 [ +0.000004] worker_thread+0x31a/0x450 [ +0.000002] ? __pfx_worker_thread+0x10/0x10 [ +0.000002] kthread+0xdf/0x110 [ +0.000002] ? __pfx_kthread+0x10/0x10 [ +0.000003] ret_from_fork+0x38/0x60 [ +0.000003] ? __pfx_kthread+0x10/0x10 [ +0.000002] ret_from_fork_asm+0x1a/0x30 [ +0.000004] </TASK> [Aug 8 07:10] BUG: Bad page state in process vim pfn:2452e9 [ +0.000013] page: refcount:0 mapcount:0 mapping:00000000a56f0863 index:0x0 pfn:0x2452e9 [ +0.000009] aops:exit_ceph [ceph] ino:10000bde243 dentry name:"table" [ +0.000026] flags: 0x200000000000008(uptodate|node=0|zone=2) [ +0.000010] raw: 0200000000000008 dead000000000100 dead000000000122 ffff9b08ddd2cc78 [ +0.000007] raw: 0000000000000000 0000000000000000 00000000ffffffff 0000000000000000 [ +0.000003] page dumped because: non-NULL mapping [ +0.000004] Modules linked in: snd_usb_audio snd_usbmidi_lib snd_ump snd_rawmidi snd_seq_device hid_generic usbhid hid ceph libceph cmac ctr ccm bridge stp llc wireguard curve25519_x86_64 libchacha20poly1305 chacha_x86_64 poly1305_x86_64 ip6_udp_tunnel udp_tunnel libcurve25519_generic libchacha nft_reject_inet nf_reject_ipv4 nf_reject_ipv6 nft_reject nft_ct nft_chain_nat nf_nat nf_conntrack nf_defrag_ipv6 nf_defrag_ipv4 nf_tables nfnetlink cachefiles netfs joydev mousedev uvcvideo uvc btusb videobuf2_vmalloc btrtl videobuf2_memops btmtk videobuf2_v4l2 btbcm videobuf2_common btintel videodev mc bluetooth ecdh_generic ecc ext4 crc16 mbcache jbd2 intel_rapl_msr intel_rapl_common edac_mce_amd kvm_amd snd_soc_dmic snd_acp3x_rn snd_acp3x_pdm_dma kvm input_leds psmouse rapl serio_raw pcspkr efi_pstore acpi_cpufreq snd_sof_amd_acp63 snd_sof_amd_vangogh snd_sof_amd_rembrandt snd_sof_amd_renoir snd_hda_codec_realtek wmi_bmof snd_sof_amd_acp snd_hda_codec_generic snd_sof_pci snd_hda_scodec_component rtw89_8852ae snd_sof [ +0.000235] rtw89_8852a snd_sof_utils snd_sof_xtensa_dsp snd_hda_codec_hdmi rtw89_pci snd_soc_core snd_hda_intel rtw89_core snd_compress snd_ctl_led snd_pcm_dmaengine snd_hda_codec radeon snd_intel_dspcfg mac80211 snd_hda_core snd_intel_sdw_acpi snd_hwdep thinkpad_acpi snd_pci_acp6x snd_pcm nvram snd_pci_acp5x sparse_keymap af_packet snd_timer snd_rn_pci_acp3x platform_profile cfg80211 snd_acp_config snd snd_soc_acpi tpm_crb ipmi_devintf r8169 k10temp snd_pci_acp3x sp5100_tco ccp soundcore dm_multipath rfkill libarc4 realtek ipmi_msghandler thermal tpm_tis ac ucsi_acpi tpm_tis_core tun typec_ucsi tpm typec roles rng_core evdev mac_hid efivarfs dm_crypt encrypted_keys trusted dm_mod rtsx_pci_sdmmc mmc_core i2c_scmi crct10dif_pclmul crc32_pclmul ghash_clmulni_intel sha512_ssse3 sha256_ssse3 sha1_ssse3 aesni_intel crypto_simd cryptd ehci_pci ehci_hcd i2c_piix4 rtsx_pci amdgpu drm_exec amdxcp drm_buddy gpu_sched drm_suballoc_helper drm_ttm_helper ttm drm_display_helper drm_kms_helper drm agpgart i2c_algo_bit video [ +0.000226] xhci_pci xhci_hcd usbcore usb_common nvme nvme_core t10_pi crc64_rocksoft_generic crc64_rocksoft hwmon crc64 battery button wmi loop btrfs blake2b_generic libcrc32c crc32c_generic crc32c_intel xor raid6_pq [ +0.000058] CPU: 13 PID: 15521 Comm: vim Tainted: G B 6.10.2-2-full #3-Alpine [ +0.000008] Hardware name: LENOVO 20UH0036GE/20UH0036GE, BIOS R1CET77W(1.46 ) 10/26/2023 [ +0.000005] Call Trace: [ +0.000005] <TASK> [ +0.000004] dump_stack_lvl+0x5c/0x80 [ +0.000013] dump_stack+0x10/0x16 [ +0.000007] bad_page.cold+0x62/0x90 [ +0.000008] free_page_is_bad_report+0x75/0x90 [ +0.000007] free_unref_folios+0x3a3/0x5f0 [ +0.000007] ? srso_return_thunk+0x5/0x5f [ +0.000010] folios_put_refs+0xdf/0x120 [ +0.000010] __folio_batch_release+0x51/0x70 [ +0.000007] __filemap_fdatawait_range+0xe2/0xf0 [ +0.000020] file_write_and_wait_range+0x7e/0xa0 [ +0.000008] ceph_fsync+0x33/0x670 [ceph] [ +0.000026] ? srso_return_thunk+0x5/0x5f [ +0.000005] ? vfs_write+0x24c/0x440 [ +0.000013] vfs_fsync_range+0x46/0x90 [ +0.000010] __x64_sys_fsync+0x37/0x70 [ +0.000009] x64_sys_call+0x178a/0x1d00 [ +0.000007] do_syscall_64+0x4b/0x110 [ +0.000008] entry_SYSCALL_64_after_hwframe+0x76/0x7e [ +0.000008] RIP: 0033:0x7f6e621e5992 [ +0.000006] Code: c3 8b 07 85 c0 75 24 49 89 fb 48 89 f0 48 89 d7 48 89 ce 4c 89 c2 4d 89 ca 4c 8b 44 24 08 4c 8b 4c 24 10 4c 89 5c 24 08 0f 05 <c3> e9 01 cc ff ff 41 54 b8 02 00 00 00 55 48 89 f5 be 00 88 08 00 [ +0.000006] RSP: 002b:00007ffe1ad84678 EFLAGS: 00000246 ORIG_RAX: 000000000000004a [ +0.000008] RAX: ffffffffffffffda RBX: 000000000000004a RCX: 00007f6e621e5992 [ +0.000006] RDX: 0000000000000000 RSI: 0000000000000000 RDI: 0000000000000003 [ +0.000004] RBP: 00007f6e60977800 R08: 0000000000000000 R09: 0000000000000000 [ +0.000004] R10: 0000000000000000 R11: 0000000000000246 R12: 0000000000000783 [ +0.000005] R13: 0000000000000000 R14: 0000000000000000 R15: 0000000000000053 [ +0.000010] </TASK> [Aug 8 07:20] BUG: Bad page state in process vim pfn:2e69d8 [ +0.000008] page: refcount:0 mapcount:0 mapping:00000000ed6ddda8 index:0x0 pfn:0x2e69d8 [ +0.000007] aops:exit_ceph [ceph] ino:10000bde439 dentry name:"table" [ +0.000019] flags: 0x200000000000008(uptodate|node=0|zone=2) [ +0.000007] raw: 0200000000000008 dead000000000100 dead000000000122 ffff9b08b3ad4c78 [ +0.000005] raw: 0000000000000000 0000000000000000 00000000ffffffff 0000000000000000 [ +0.000003] page dumped because: non-NULL mapping [ +0.000002] Modules linked in: snd_usb_audio snd_usbmidi_lib snd_ump snd_rawmidi snd_seq_device hid_generic usbhid hid ceph libceph cmac ctr ccm bridge stp llc wireguard curve25519_x86_64 libchacha20poly1305 chacha_x86_64 poly1305_x86_64 ip6_udp_tunnel udp_tunnel libcurve25519_generic libchacha nft_reject_inet nf_reject_ipv4 nf_reject_ipv6 nft_reject nft_ct nft_chain_nat nf_nat nf_conntrack nf_defrag_ipv6 nf_defrag_ipv4 nf_tables nfnetlink cachefiles netfs joydev mousedev uvcvideo uvc btusb videobuf2_vmalloc btrtl videobuf2_memops btmtk videobuf2_v4l2 btbcm videobuf2_common btintel videodev mc bluetooth ecdh_generic ecc ext4 crc16 mbcache jbd2 intel_rapl_msr intel_rapl_common edac_mce_amd kvm_amd snd_soc_dmic snd_acp3x_rn snd_acp3x_pdm_dma kvm input_leds psmouse rapl serio_raw pcspkr efi_pstore acpi_cpufreq snd_sof_amd_acp63 snd_sof_amd_vangogh snd_sof_amd_rembrandt snd_sof_amd_renoir snd_hda_codec_realtek wmi_bmof snd_sof_amd_acp snd_hda_codec_generic snd_sof_pci snd_hda_scodec_component rtw89_8852ae snd_sof [ +0.000181] rtw89_8852a snd_sof_utils snd_sof_xtensa_dsp snd_hda_codec_hdmi rtw89_pci snd_soc_core snd_hda_intel rtw89_core snd_compress snd_ctl_led snd_pcm_dmaengine snd_hda_codec radeon snd_intel_dspcfg mac80211 snd_hda_core snd_intel_sdw_acpi snd_hwdep thinkpad_acpi snd_pci_acp6x snd_pcm nvram snd_pci_acp5x sparse_keymap af_packet snd_timer snd_rn_pci_acp3x platform_profile cfg80211 snd_acp_config snd snd_soc_acpi tpm_crb ipmi_devintf r8169 k10temp snd_pci_acp3x sp5100_tco ccp soundcore dm_multipath rfkill libarc4 realtek ipmi_msghandler thermal tpm_tis ac ucsi_acpi tpm_tis_core tun typec_ucsi tpm typec roles rng_core evdev mac_hid efivarfs dm_crypt encrypted_keys trusted dm_mod rtsx_pci_sdmmc mmc_core i2c_scmi crct10dif_pclmul crc32_pclmul ghash_clmulni_intel sha512_ssse3 sha256_ssse3 sha1_ssse3 aesni_intel crypto_simd cryptd ehci_pci ehci_hcd i2c_piix4 rtsx_pci amdgpu drm_exec amdxcp drm_buddy gpu_sched drm_suballoc_helper drm_ttm_helper ttm drm_display_helper drm_kms_helper drm agpgart i2c_algo_bit video [ +0.000177] xhci_pci xhci_hcd usbcore usb_common nvme nvme_core t10_pi crc64_rocksoft_generic crc64_rocksoft hwmon crc64 battery button wmi loop btrfs blake2b_generic libcrc32c crc32c_generic crc32c_intel xor raid6_pq [ +0.000044] CPU: 10 PID: 15521 Comm: vim Tainted: G B 6.10.2-2-full #3-Alpine [ +0.000007] Hardware name: LENOVO 20UH0036GE/20UH0036GE, BIOS R1CET77W(1.46 ) 10/26/2023 [ +0.000004] Call Trace: [ +0.000004] <TASK> [ +0.000003] dump_stack_lvl+0x5c/0x80 [ +0.000010] dump_stack+0x10/0x16 [ +0.000006] bad_page.cold+0x62/0x90 [ +0.000005] free_page_is_bad_report+0x75/0x90 [ +0.000006] free_unref_folios+0x3a3/0x5f0 [ +0.000005] ? srso_return_thunk+0x5/0x5f [ +0.000008] folios_put_refs+0xdf/0x120 [ +0.000008] __folio_batch_release+0x51/0x70 [ +0.000006] __filemap_fdatawait_range+0xe2/0xf0 [ +0.000014] file_write_and_wait_range+0x7e/0xa0 [ +0.000007] ceph_fsync+0x33/0x670 [ceph] [ +0.000019] ? srso_return_thunk+0x5/0x5f [ +0.000004] ? vfs_write+0x24c/0x440 [ +0.000010] vfs_fsync_range+0x46/0x90 [ +0.000007] __x64_sys_fsync+0x37/0x70 [ +0.000007] x64_sys_call+0x178a/0x1d00 [ +0.000005] do_syscall_64+0x4b/0x110 [ +0.000007] entry_SYSCALL_64_after_hwframe+0x76/0x7e [ +0.000006] RIP: 0033:0x7f6e621e5992 [ +0.000005] Code: c3 8b 07 85 c0 75 24 49 89 fb 48 89 f0 48 89 d7 48 89 ce 4c 89 c2 4d 89 ca 4c 8b 44 24 08 4c 8b 4c 24 10 4c 89 5c 24 08 0f 05 <c3> e9 01 cc ff ff 41 54 b8 02 00 00 00 55 48 89 f5 be 00 88 08 00 [ +0.000004] RSP: 002b:00007ffe1ad84678 EFLAGS: 00000246 ORIG_RAX: 000000000000004a [ +0.000007] RAX: ffffffffffffffda RBX: 000000000000004a RCX: 00007f6e621e5992 [ +0.000003] RDX: 0000000000000000 RSI: 0000000000000000 RDI: 0000000000000003 [ +0.000003] RBP: 00007f6e60977800 R08: 0000000000000000 R09: 0000000000000000 [ +0.000004] R10: 0000000000000000 R11: 0000000000000246 R12: 000000000000080e [ +0.000003] R13: 0000000000000000 R14: 0000000000000000 R15: 0000000000000056 [ +0.000008] </TASK> [Aug 8 07:21] ceph: [REDACTED-FSID 363957080]: mds0 caps stale [ +4.863866] rcu: INFO: rcu_preempt detected stalls on CPUs/tasks: [ +0.000008] rcu: Tasks blocked on level-0 rcu_node (CPUs 0-15): P13110 [ +0.000014] rcu: (detected by 10, t=60002 jiffies, g=1157737, q=247405 ncpus=16) [ +0.000008] task:kworker/12:1 state:R running task stack:0 pid:13110 tgid:13110 ppid:2 flags:0x00004000 [ +0.000015] Workqueue: ceph-msgr ceph_con_workfn [libceph] [ +0.000031] Call Trace: [ +0.000005] <TASK> [ +0.000005] ? srso_return_thunk+0x5/0x5f [ +0.000010] __schedule+0x2d1/0x820 [ +0.000010] ? srso_return_thunk+0x5/0x5f [ +0.000006] ? irqentry_exit+0x20/0x50 [ +0.000006] ? srso_return_thunk+0x5/0x5f [ +0.000006] ? sysvec_apic_timer_interrupt+0x8f/0xa0 [ +0.000009] ? srso_return_thunk+0x5/0x5f [ +0.000005] ? irqentry_exit+0x20/0x50 [ +0.000006] ? srso_return_thunk+0x5/0x5f [ +0.000006] ? sysvec_apic_timer_interrupt+0x8f/0xa0 [ +0.000007] ? srso_return_thunk+0x5/0x5f [ +0.000006] ? irqentry_exit+0x20/0x50 [ +0.000006] ? srso_return_thunk+0x5/0x5f [ +0.000006] ? sysvec_apic_timer_interrupt+0x8f/0xa0 [ +0.000008] ? srso_return_thunk+0x5/0x5f [ +0.000006] ? asm_sysvec_apic_timer_interrupt+0x1b/0x20 [ +0.000010] ? srso_return_thunk+0x5/0x5f [ +0.000006] ? xas_load+0xd/0xd0 [ +0.000008] ? srso_return_thunk+0x5/0x5f [ +0.000005] ? xas_find+0x157/0x1a0 [ +0.000007] ? srso_return_thunk+0x5/0x5f [ +0.000004] ? find_lock_entries+0xa6/0x270 [ +0.000009] ? truncate_inode_pages_range+0xd4/0x450 [ +0.000010] ? srso_return_thunk+0x5/0x5f [ +0.000005] ? skcipher_walk_first+0x3d/0x100 [ +0.000007] ? srso_return_thunk+0x5/0x5f [ +0.000004] ? skcipher_walk_done+0xbc/0x250 [ +0.000006] ? srso_return_thunk+0x5/0x5f [ +0.000006] ? gcmaes_crypt_by_sg+0x2e5/0x390 [aesni_intel] [ +0.000009] ? srso_return_thunk+0x5/0x5f [ +0.000012] ? srso_return_thunk+0x5/0x5f [ +0.000007] ? srso_return_thunk+0x5/0x5f [ +0.000005] ? __inode_wait_for_writeback+0x72/0xf0 [ +0.000010] ? truncate_inode_pages_final+0x3b/0x50 [ +0.000009] ? ceph_evict_inode+0x41/0x230 [ceph] [ +0.000017] ? evict+0xc3/0x1c0 [ +0.000007] ? iput+0xf1/0x1f0 [ +0.000006] ? dentry_unlink_inode+0xc4/0x130 [ +0.000008] ? __dentry_kill+0x6e/0x190 [ +0.000005] ? dput+0xe2/0x1a0 [ +0.000005] ? ceph_mdsc_release_request+0x143/0x250 [ceph] [ +0.000021] ? mds_dispatch+0x108e/0x1b80 [ceph] [ +0.000018] ? gcmaes_decrypt+0xa3/0xd0 [aesni_intel] [ +0.000009] ? generic_gcmaes_decrypt+0x4c/0x70 [aesni_intel] [ +0.000010] ? ceph_con_process_message+0x6a/0x80 [libceph] [ +0.000018] ? ceph_con_v2_try_read+0xba3/0x16f0 [libceph] [ +0.000019] ? srso_return_thunk+0x5/0x5f [ +0.000004] ? update_load_avg+0x7b/0x6d0 [ +0.000008] ? srso_return_thunk+0x5/0x5f [ +0.000004] ? set_next_entity+0x5e/0xc0 [ +0.000006] ? srso_return_thunk+0x5/0x5f [ +0.000006] ? ceph_con_workfn+0x154/0x520 [libceph] [ +0.000016] ? srso_return_thunk+0x5/0x5f [ +0.000005] ? process_one_work+0x16e/0x290 [ +0.000009] ? worker_thread+0x31a/0x450 [ +0.000006] ? __pfx_worker_thread+0x10/0x10 [ +0.000005] ? kthread+0xdf/0x110 [ +0.000005] ? __pfx_kthread+0x10/0x10 [ +0.000006] ? ret_from_fork+0x38/0x60 [ +0.000007] ? __pfx_kthread+0x10/0x10 [ +0.000005] ? ret_from_fork_asm+0x1a/0x30 [ +0.000009] </TASK> [Aug 8 07:24] rcu: INFO: rcu_preempt detected stalls on CPUs/tasks: [ +0.000009] rcu: Tasks blocked on level-0 rcu_node (CPUs 0-15): P13110 [ +0.000011] rcu: (detected by 10, t=240007 jiffies, g=1157737, q=532450 ncpus=16) [ +0.000007] task:kworker/12:1 state:R running task stack:0 pid:13110 tgid:13110 ppid:2 flags:0x00004000 [ +0.000013] Workqueue: ceph-msgr ceph_con_workfn [libceph] [ +0.000024] Call Trace: [ +0.000004] <TASK> [ +0.000004] ? srso_return_thunk+0x5/0x5f [ +0.000008] __schedule+0x2d1/0x820 [ +0.000008] ? srso_return_thunk+0x5/0x5f [ +0.000005] ? irqentry_exit+0x20/0x50 [ +0.000005] ? srso_return_thunk+0x5/0x5f [ +0.000004] ? irqentry_exit+0x20/0x50 [ +0.000004] ? srso_return_thunk+0x5/0x5f [ +0.000004] ? sysvec_apic_timer_interrupt+0x8f/0xa0 [ +0.000008] ? srso_return_thunk+0x5/0x5f [ +0.000004] ? asm_sysvec_apic_timer_interrupt+0x1b/0x20 [ +0.000006] ? srso_return_thunk+0x5/0x5f [ +0.000004] ? irqentry_exit+0x20/0x50 [ +0.000005] ? srso_return_thunk+0x5/0x5f [ +0.000004] ? sysvec_apic_timer_interrupt+0x8f/0xa0 [ +0.000007] ? srso_return_thunk+0x5/0x5f [ +0.000004] ? asm_sysvec_apic_timer_interrupt+0x1b/0x20 [ +0.000008] ? srso_return_thunk+0x5/0x5f [ +0.000004] ? xas_load+0xd/0xd0 [ +0.000007] ? xas_find+0x157/0x1a0 [ +0.000005] ? srso_return_thunk+0x5/0x5f [ +0.000005] ? find_lock_entries+0x152/0x270 [ +0.000009] ? truncate_inode_pages_range+0xd4/0x450 [ +0.000010] ? srso_return_thunk+0x5/0x5f [ +0.000005] ? skcipher_walk_first+0x3d/0x100 [ +0.000008] ? srso_return_thunk+0x5/0x5f [ +0.000004] ? skcipher_walk_done+0xbc/0x250 [ +0.000006] ? srso_return_thunk+0x5/0x5f [ +0.000006] ? gcmaes_crypt_by_sg+0x2e5/0x390 [aesni_intel] [ +0.000008] ? srso_return_thunk+0x5/0x5f [ +0.000010] ? srso_return_thunk+0x5/0x5f [ +0.000004] ? srso_return_thunk+0x5/0x5f [ +0.000005] ? __inode_wait_for_writeback+0x72/0xf0 [ +0.000007] ? truncate_inode_pages_final+0x3b/0x50 [ +0.000007] ? ceph_evict_inode+0x41/0x230 [ceph] [ +0.000018] ? evict+0xc3/0x1c0 [ +0.000007] ? iput+0xf1/0x1f0 [ +0.000006] ? dentry_unlink_inode+0xc4/0x130 [ +0.000007] ? __dentry_kill+0x6e/0x190 [ +0.000005] ? dput+0xe2/0x1a0 [ +0.000006] ? ceph_mdsc_release_request+0x143/0x250 [ceph] [ +0.000021] ? mds_dispatch+0x108e/0x1b80 [ceph] [ +0.000018] ? gcmaes_decrypt+0xa3/0xd0 [aesni_intel] [ +0.000009] ? generic_gcmaes_decrypt+0x4c/0x70 [aesni_intel] [ +0.000010] ? ceph_con_process_message+0x6a/0x80 [libceph] [ +0.000019] ? ceph_con_v2_try_read+0xba3/0x16f0 [libceph] [ +0.000018] ? srso_return_thunk+0x5/0x5f [ +0.000005] ? update_load_avg+0x7b/0x6d0 [ +0.000007] ? srso_return_thunk+0x5/0x5f [ +0.000005] ? set_next_entity+0x5e/0xc0 [ +0.000005] ? srso_return_thunk+0x5/0x5f [ +0.000006] ? ceph_con_workfn+0x154/0x520 [libceph] [ +0.000017] ? srso_return_thunk+0x5/0x5f [ +0.000005] ? process_one_work+0x16e/0x290 [ +0.000009] ? worker_thread+0x31a/0x450 [ +0.000006] ? __pfx_worker_thread+0x10/0x10 [ +0.000005] ? kthread+0xdf/0x110 [ +0.000005] ? __pfx_kthread+0x10/0x10 [ +0.000006] ? ret_from_fork+0x38/0x60 [ +0.000007] ? __pfx_kthread+0x10/0x10 [ +0.000005] ? ret_from_fork_asm+0x1a/0x30 [ +0.000009] </TASK> [Aug 8 07:25] ceph: [REDACTED-FSID 363957080]: mds0 hung
-- Alex D. RedXen System & Infrastructure Administration https://redxen.eu/
Attachment:
signature.asc
Description: PGP signature
_______________________________________________ ceph-users mailing list -- ceph-users@xxxxxxx To unsubscribe send an email to ceph-users-leave@xxxxxxx
