Hello!
On a two nodes ceph cluster, use mainly as s3 provider, i have manyRadosGW crashs which probably explain why i have fail of large (multipart) files uploads.Indeed, i havent no issue with small file (<10 Gb).
If someone here can help me dig to this issue, i'll be great!
Thanks a lot for your work!
Best regards,Kwame
cat /etc/redhat-release
Red Hat Enterprise Linux release 8.2 (Ootpa)
ceph -v
ceph version 15.2.4 (7447c15c6ff58d7fce91843b705a268a1917325c) octopus (stable)
# same issue with 15.2.8 and 15.2.9
ceph -s
cluster:
id: 33ccf2a3-2efc-4def-a857-ade988228e80
health: HEALTH_WARN
965 daemons have recently crashed
services:
mon: 2 daemons, quorum domaster3,domaster1 (age 10h)
mgr: domaster1(active, since 10h), standbys: domaster3
mds: 2 up:standby
osd: 4 osds: 4 up (since 10h), 4 in (since 8d)
rgw: 2 daemons active (domaster1, domaster3)
task status:
data:
pools: 8 pools, 273 pgs
objects: 34.98k objects, 93 GiB
usage: 285 GiB used, 315 GiB / 600 GiB avail
pgs: 273 active+clean
io:
client: 20 MiB/s rd, 13 op/s rd, 0 op/s wr
ceph crash ls-new<snip>
2021-03-24T07:05:03.257297Z_76400e5d-7334-4628-af67-5f5069a27f4e client.rgw.domaster1 *
2021-03-24T07:24:50.300691Z_a015c60f-c5e6-4be1-841e-e6c8720e2fb7 client.rgw.domaster3 *
2021-03-24T07:25:19.027295Z_0a1998fe-87d1-4a38-99e6-618ed36bcc24 client.rgw.domaster1 *
ceph crash info 2021-03-24T07:25:19.027295Z_0a1998fe-87d1-4a38-99e6-618ed36bcc24
{
"backtrace": [
"(()+0x12dd0) [0x7f8de7e6ddd0]",
"(gsignal()+0x10f) [0x7f8de64b270f]",
"(abort()+0x127) [0x7f8de649cb25]",
"(()+0x364928) [0x7f8df2a05928]",
"(RGWRESTStreamRWRequest::do_send_prepare(RGWAccessKey*, std::map<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::less<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > >&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, ceph::buffer::v15_2_0::list*)+0x74d) [0x7f8df2e7f60d]",
"(RGWRESTStreamRWRequest::send_prepare(RGWAccessKey*, std::map<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::less<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > >&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, ceph::buffer::v15_2_0::list*)+0x66) [0x7f8df2e7f7a6]",
"(RGWRESTStreamRWRequest::send_request(RGWAccessKey*, std::map<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::less<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > >&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, RGWHTTPManager*, ceph::buffer::v15_2_0::list*)+0x18) [0x7f8df2e7f808]",
"(RGWRESTReadResource::aio_read()+0x50) [0x7f8df2e87980]",
"(RGWReadRawRESTResourceCR::send_request()+0x75) [0x7f8df2cabfc5]",
"(RGWSimpleCoroutine::state_send_request()+0x13) [0x7f8df2d8f103]",
"(RGWSimpleCoroutine::operate()+0xac) [0x7f8df2d94cdc]",
"(RGWCoroutinesStack::operate(RGWCoroutinesEnv*)+0x67) [0x7f8df2d925c7]",
"(RGWCoroutinesManager::run(std::__cxx11::list<RGWCoroutinesStack*, std::allocator<RGWCoroutinesStack*> >&)+0x271) [0x7f8df2d933f1]",
"(RGWSyncLogTrimThread::process()+0x200) [0x7f8df2e5d2b0]",
"(RGWRadosThread::Worker::entry()+0x176) [0x7f8df2e25936]",
"(()+0x82de) [0x7f8de7e632de]",
"(clone()+0x43) [0x7f8de6576e83]"
],
"ceph_version": "15.2.4",
"crash_id": "2021-03-24T07:25:19.027295Z_0a1998fe-87d1-4a38-99e6-618ed36bcc24",
"entity_name": "client.rgw.domaster1",
"os_id": "rhel",
"os_name": "Red Hat Enterprise Linux",
"os_version": "8.2 (Ootpa)",
"os_version_id": "8.2",
"process_name": "radosgw",
"stack_sig": "39a3d8cc71898dd0f486ea1cace46babf5c4cdbf18d64adaf1505e7d447ee44d",
"timestamp": "2021-03-24T07:25:19.027295Z",
"utsname_hostname": "domaster1",
"utsname_machine": "x86_64",
"utsname_release": "4.18.0-193.6.3.el8_2.x86_64",
"utsname_sysname": "Linux",
"utsname_version": "#1 SMP Mon Jun 1 20:24:55 UTC 2020"
}
_______________________________________________
ceph-users mailing list -- ceph-users@xxxxxxx
To unsubscribe send an email to ceph-users-leave@xxxxxxx
