Unfortunately, I can't think of a workaround that doesn't involve a code change. I've created a tracker (https://tracker.ceph.com/issues/52745) and am working towards a fix for this, but I'm not sure how to deal with it using the current 16.2.6 image. Maybe others will have some ideas. On Mon, Sep 27, 2021 at 10:10 AM Marco Pizzolo <marcopizzolo@xxxxxxxxx> wrote: > Good morning Adam, Ceph users, > > Is there something we can do to extend the acceptable response size? > Trying to understand if there is some viable workaround that we can > implement. > > Thanks, > Marco > > On Fri, Sep 24, 2021 at 2:59 PM Marco Pizzolo <marcopizzolo@xxxxxxxxx> > wrote: > >> Hi Adam, >> >> I really appreciate your time reviewing. >> >> You're absolutely correct. We have 60 OSDs in 2 of the nodes. We do see >> the errors as you predicated. >> >> All of our nodes will have 60 OSDs, so I'm hoping there's a fix for this. >> >> Thanks for the quick response. >> >> Using recent ceph image >> quay.io/ceph/ceph@sha256:43b9d5bb30fe76c831a97b47abd92290516fcc4e1455aba455cbcfc85f166b36 >> Traceback (most recent call last): >> File "/usr/lib/python3.8/asyncio/streams.py", line 540, in readline >> line = await self.readuntil(sep) >> File "/usr/lib/python3.8/asyncio/streams.py", line 635, in readuntil >> raise exceptions.LimitOverrunError( >> asyncio.exceptions.LimitOverrunError: Separator is found, but chunk is >> longer than limit >> >> During handling of the above exception, another exception occurred: >> >> Traceback (most recent call last): >> File "/usr/sbin/cephadm", line 8479, in <module> >> main() >> File "/usr/sbin/cephadm", line 8467, in main >> r = ctx.func(ctx) >> File "/usr/sbin/cephadm", line 1732, in _infer_config >> return func(ctx) >> File "/usr/sbin/cephadm", line 1702, in _infer_fsid >> return func(ctx) >> File "/usr/sbin/cephadm", line 1760, in _infer_image >> return func(ctx) >> File "/usr/sbin/cephadm", line 1660, in _validate_fsid >> return func(ctx) >> File "/usr/sbin/cephadm", line 4753, in command_ceph_volume >> out, err, code = call_throws(ctx, c.run_cmd()) >> File "/usr/sbin/cephadm", line 1460, in call_throws >> out, err, ret = call(ctx, command, desc, verbosity, timeout, **kwargs) >> File "/usr/sbin/cephadm", line 1442, in call >> stdout, stderr, returncode = async_run(run_with_timeout()) >> File "/usr/lib/python3.8/asyncio/runners.py", line 44, in run >> return loop.run_until_complete(main) >> File "/usr/lib/python3.8/asyncio/base_events.py", line 616, in >> run_until_complete >> return future.result() >> File "/usr/sbin/cephadm", line 1433, in run_with_timeout >> stdout, stderr = await asyncio.gather(tee(process.stdout), >> File "/usr/sbin/cephadm", line 1415, in tee >> async for line in reader: >> File "/usr/lib/python3.8/asyncio/streams.py", line 738, in __anext__ >> val = await self.readline() >> File "/usr/lib/python3.8/asyncio/streams.py", line 549, in readline >> raise ValueError(e.args[0]) >> ValueError: Separator is found, but chunk is longer than limit >> >> On Fri, Sep 24, 2021 at 2:42 PM Adam King <adking@xxxxxxxxxx> wrote: >> >>> It looks like the output from a ceph-volume command was too long to >>> handle. If you run "cephadm ceph-volume -- inventory --format=json" (add >>> "--with-lsm" if you've turned on device_enhanced_scan) manually on each >>> host do any of them fail in a similar fashion? >>> >>> On Fri, Sep 24, 2021 at 1:37 PM Marco Pizzolo <marcopizzolo@xxxxxxxxx> >>> wrote: >>> >>>> Hello Everyone, >>>> >>>> If you have any suggestions on the cause, or what we can do I'd >>>> certainly >>>> appreciate it. >>>> >>>> I'm seeing the following on a newly stood up cluster using Podman on >>>> Ubuntu >>>> 20.04.3 HWE: >>>> >>>> Thank you very much >>>> >>>> Marco >>>> >>>> >>>> >>>> Sep 24, 2021, 1:24:30 PM [ERR] cephadm exited with an error code: 1, >>>> stderr:Inferring config >>>> >>>> /var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/mon.prdrepoceph01/config >>>> Traceback (most recent call last): >>>> File "/usr/lib/python3.8/asyncio/streams.py", line 540, in readline >>>> line = await self.readuntil(sep) >>>> File "/usr/lib/python3.8/asyncio/streams.py", line 635, in readuntil >>>> raise exceptions.LimitOverrunError( >>>> asyncio.exceptions.LimitOverrunError: Separator is found, but chunk is >>>> longer than limit >>>> >>>> During handling of the above exception, another exception occurred: >>>> >>>> Traceback (most recent call last): >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 8479, in <module> >>>> main() >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 8467, in main >>>> r = ctx.func(ctx) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1732, in _infer_config >>>> return func(ctx) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1673, in _infer_fsid >>>> return func(ctx) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1760, in _infer_image >>>> return func(ctx) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1660, in _validate_fsid >>>> return func(ctx) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 4753, in command_ceph_volume >>>> out, err, code = call_throws(ctx, c.run_cmd()) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1460, in call_throws >>>> out, err, ret = call(ctx, command, desc, verbosity, timeout, >>>> **kwargs) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1442, in call >>>> stdout, stderr, returncode = async_run(run_with_timeout()) >>>> File "/usr/lib/python3.8/asyncio/runners.py", line 44, in run >>>> return loop.run_until_complete(main) >>>> File "/usr/lib/python3.8/asyncio/base_events.py", line 616, in >>>> run_until_complete >>>> return future.result() >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1433, in run_with_timeout >>>> stdout, stderr = await asyncio.gather(tee(process.stdout), >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1415, in tee >>>> async for line in reader: >>>> File "/usr/lib/python3.8/asyncio/streams.py", line 738, in __anext__ >>>> val = await self.readline() >>>> File "/usr/lib/python3.8/asyncio/streams.py", line 549, in readline >>>> raise ValueError(e.args[0]) >>>> ValueError: Separator is found, but chunk is longer than limit >>>> Traceback (most recent call last): >>>> File "/usr/share/ceph/mgr/cephadm/serve.py", line 1366, in >>>> _remote_connection >>>> yield (conn, connr) >>>> File "/usr/share/ceph/mgr/cephadm/serve.py", line 1263, in >>>> _run_cephadm >>>> code, '\n'.join(err))) >>>> orchestrator._interface.OrchestratorError: cephadm exited with an error >>>> code: 1, stderr:Inferring config >>>> >>>> /var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/mon.prdrepoceph01/config >>>> Traceback (most recent call last): >>>> File "/usr/lib/python3.8/asyncio/streams.py", line 540, in readline >>>> line = await self.readuntil(sep) >>>> File "/usr/lib/python3.8/asyncio/streams.py", line 635, in readuntil >>>> raise exceptions.LimitOverrunError( >>>> asyncio.exceptions.LimitOverrunError: Separator is found, but chunk is >>>> longer than limit >>>> >>>> During handling of the above exception, another exception occurred: >>>> >>>> Traceback (most recent call last): >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 8479, in <module> >>>> main() >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 8467, in main >>>> r = ctx.func(ctx) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1732, in _infer_config >>>> return func(ctx) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1673, in _infer_fsid >>>> return func(ctx) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1760, in _infer_image >>>> return func(ctx) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1660, in _validate_fsid >>>> return func(ctx) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 4753, in command_ceph_volume >>>> out, err, code = call_throws(ctx, c.run_cmd()) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1460, in call_throws >>>> out, err, ret = call(ctx, command, desc, verbosity, timeout, >>>> **kwargs) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1442, in call >>>> stdout, stderr, returncode = async_run(run_with_timeout()) >>>> File "/usr/lib/python3.8/asyncio/runners.py", line 44, in run >>>> return loop.run_until_complete(main) >>>> File "/usr/lib/python3.8/asyncio/base_events.py", line 616, in >>>> run_until_complete >>>> return future.result() >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1433, in run_with_timeout >>>> stdout, stderr = await asyncio.gather(tee(process.stdout), >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1415, in tee >>>> async for line in reader: >>>> File "/usr/lib/python3.8/asyncio/streams.py", line 738, in __anext__ >>>> val = await self.readline() >>>> File "/usr/lib/python3.8/asyncio/streams.py", line 549, in readline >>>> raise ValueError(e.args[0]) >>>> ValueError: Separator is found, but chunk is longer than limit >>>> Sep 24, 2021, 1:24:28 PM [ERR] cephadm exited with an error code: 1, >>>> stderr:Inferring config >>>> >>>> /var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/mon.prdrepoceph02/config >>>> Traceback (most recent call last): >>>> File "/usr/lib/python3.8/asyncio/streams.py", line 540, in readline >>>> line = await self.readuntil(sep) >>>> File "/usr/lib/python3.8/asyncio/streams.py", line 635, in readuntil >>>> raise exceptions.LimitOverrunError( >>>> asyncio.exceptions.LimitOverrunError: Separator is found, but chunk is >>>> longer than limit >>>> >>>> During handling of the above exception, another exception occurred: >>>> >>>> Traceback (most recent call last): >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 8479, in <module> >>>> main() >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 8467, in main >>>> r = ctx.func(ctx) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1732, in _infer_config >>>> return func(ctx) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1673, in _infer_fsid >>>> return func(ctx) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1760, in _infer_image >>>> return func(ctx) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1660, in _validate_fsid >>>> return func(ctx) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 4753, in command_ceph_volume >>>> out, err, code = call_throws(ctx, c.run_cmd()) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1460, in call_throws >>>> out, err, ret = call(ctx, command, desc, verbosity, timeout, >>>> **kwargs) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1442, in call >>>> stdout, stderr, returncode = async_run(run_with_timeout()) >>>> File "/usr/lib/python3.8/asyncio/runners.py", line 44, in run >>>> return loop.run_until_complete(main) >>>> File "/usr/lib/python3.8/asyncio/base_events.py", line 616, in >>>> run_until_complete >>>> return future.result() >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1433, in run_with_timeout >>>> stdout, stderr = await asyncio.gather(tee(process.stdout), >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1415, in tee >>>> async for line in reader: >>>> File "/usr/lib/python3.8/asyncio/streams.py", line 738, in __anext__ >>>> val = await self.readline() >>>> File "/usr/lib/python3.8/asyncio/streams.py", line 549, in readline >>>> raise ValueError(e.args[0]) >>>> ValueError: Separator is found, but chunk is longer than limit >>>> Traceback (most recent call last): >>>> File "/usr/share/ceph/mgr/cephadm/serve.py", line 1366, in >>>> _remote_connection >>>> yield (conn, connr) >>>> File "/usr/share/ceph/mgr/cephadm/serve.py", line 1263, in >>>> _run_cephadm >>>> code, '\n'.join(err))) >>>> orchestrator._interface.OrchestratorError: cephadm exited with an error >>>> code: 1, stderr:Inferring config >>>> >>>> /var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/mon.prdrepoceph02/config >>>> Traceback (most recent call last): >>>> File "/usr/lib/python3.8/asyncio/streams.py", line 540, in readline >>>> line = await self.readuntil(sep) >>>> File "/usr/lib/python3.8/asyncio/streams.py", line 635, in readuntil >>>> raise exceptions.LimitOverrunError( >>>> asyncio.exceptions.LimitOverrunError: Separator is found, but chunk is >>>> longer than limit >>>> >>>> During handling of the above exception, another exception occurred: >>>> >>>> Traceback (most recent call last): >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 8479, in <module> >>>> main() >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 8467, in main >>>> r = ctx.func(ctx) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1732, in _infer_config >>>> return func(ctx) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1673, in _infer_fsid >>>> return func(ctx) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1760, in _infer_image >>>> return func(ctx) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1660, in _validate_fsid >>>> return func(ctx) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 4753, in command_ceph_volume >>>> out, err, code = call_throws(ctx, c.run_cmd()) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1460, in call_throws >>>> out, err, ret = call(ctx, command, desc, verbosity, timeout, >>>> **kwargs) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1442, in call >>>> stdout, stderr, returncode = async_run(run_with_timeout()) >>>> File "/usr/lib/python3.8/asyncio/runners.py", line 44, in run >>>> return loop.run_until_complete(main) >>>> File "/usr/lib/python3.8/asyncio/base_events.py", line 616, in >>>> run_until_complete >>>> return future.result() >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1433, in run_with_timeout >>>> stdout, stderr = await asyncio.gather(tee(process.stdout), >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1415, in tee >>>> async for line in reader: >>>> File "/usr/lib/python3.8/asyncio/streams.py", line 738, in __anext__ >>>> val = await self.readline() >>>> File "/usr/lib/python3.8/asyncio/streams.py", line 549, in readline >>>> raise ValueError(e.args[0]) >>>> ValueError: Separator is found, but chunk is longer than limit >>>> Sep 24, 2021, 1:22:45 PM [ERR] cephadm exited with an error code: 1, >>>> stderr:Inferring config >>>> >>>> /var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/mon.prdrepoceph01/config >>>> Traceback (most recent call last): >>>> File "/usr/lib/python3.8/asyncio/streams.py", line 540, in readline >>>> line = await self.readuntil(sep) >>>> File "/usr/lib/python3.8/asyncio/streams.py", line 635, in readuntil >>>> raise exceptions.LimitOverrunError( >>>> asyncio.exceptions.LimitOverrunError: Separator is found, but chunk is >>>> longer than limit >>>> >>>> During handling of the above exception, another exception occurred: >>>> >>>> Traceback (most recent call last): >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 8479, in <module> >>>> main() >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 8467, in main >>>> r = ctx.func(ctx) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1732, in _infer_config >>>> return func(ctx) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1673, in _infer_fsid >>>> return func(ctx) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1760, in _infer_image >>>> return func(ctx) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1660, in _validate_fsid >>>> return func(ctx) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 4753, in command_ceph_volume >>>> out, err, code = call_throws(ctx, c.run_cmd()) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1460, in call_throws >>>> out, err, ret = call(ctx, command, desc, verbosity, timeout, >>>> **kwargs) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1442, in call >>>> stdout, stderr, returncode = async_run(run_with_timeout()) >>>> File "/usr/lib/python3.8/asyncio/runners.py", line 44, in run >>>> return loop.run_until_complete(main) >>>> File "/usr/lib/python3.8/asyncio/base_events.py", line 616, in >>>> run_until_complete >>>> return future.result() >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1433, in run_with_timeout >>>> stdout, stderr = await asyncio.gather(tee(process.stdout), >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1415, in tee >>>> async for line in reader: >>>> File "/usr/lib/python3.8/asyncio/streams.py", line 738, in __anext__ >>>> val = await self.readline() >>>> File "/usr/lib/python3.8/asyncio/streams.py", line 549, in readline >>>> raise ValueError(e.args[0]) >>>> ValueError: Separator is found, but chunk is longer than limit >>>> Traceback (most recent call last): >>>> File "/usr/share/ceph/mgr/cephadm/serve.py", line 1366, in >>>> _remote_connection >>>> yield (conn, connr) >>>> File "/usr/share/ceph/mgr/cephadm/serve.py", line 1263, in >>>> _run_cephadm >>>> code, '\n'.join(err))) >>>> orchestrator._interface.OrchestratorError: cephadm exited with an error >>>> code: 1, stderr:Inferring config >>>> >>>> /var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/mon.prdrepoceph01/config >>>> Traceback (most recent call last): >>>> File "/usr/lib/python3.8/asyncio/streams.py", line 540, in readline >>>> line = await self.readuntil(sep) >>>> File "/usr/lib/python3.8/asyncio/streams.py", line 635, in readuntil >>>> raise exceptions.LimitOverrunError( >>>> asyncio.exceptions.LimitOverrunError: Separator is found, but chunk is >>>> longer than limit >>>> >>>> During handling of the above exception, another exception occurred: >>>> >>>> Traceback (most recent call last): >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 8479, in <module> >>>> main() >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 8467, in main >>>> r = ctx.func(ctx) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1732, in _infer_config >>>> return func(ctx) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1673, in _infer_fsid >>>> return func(ctx) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1760, in _infer_image >>>> return func(ctx) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1660, in _validate_fsid >>>> return func(ctx) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 4753, in command_ceph_volume >>>> out, err, code = call_throws(ctx, c.run_cmd()) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1460, in call_throws >>>> out, err, ret = call(ctx, command, desc, verbosity, timeout, >>>> **kwargs) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1442, in call >>>> stdout, stderr, returncode = async_run(run_with_timeout()) >>>> File "/usr/lib/python3.8/asyncio/runners.py", line 44, in run >>>> return loop.run_until_complete(main) >>>> File "/usr/lib/python3.8/asyncio/base_events.py", line 616, in >>>> run_until_complete >>>> return future.result() >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1433, in run_with_timeout >>>> stdout, stderr = await asyncio.gather(tee(process.stdout), >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1415, in tee >>>> async for line in reader: >>>> File "/usr/lib/python3.8/asyncio/streams.py", line 738, in __anext__ >>>> val = await self.readline() >>>> File "/usr/lib/python3.8/asyncio/streams.py", line 549, in readline >>>> raise ValueError(e.args[0]) >>>> ValueError: Separator is found, but chunk is longer than limit >>>> Sep 24, 2021, 1:22:45 PM [ERR] cephadm exited with an error code: 1, >>>> stderr:Inferring config >>>> >>>> /var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/mon.prdrepoceph02/config >>>> Traceback (most recent call last): >>>> File "/usr/lib/python3.8/asyncio/streams.py", line 540, in readline >>>> line = await self.readuntil(sep) >>>> File "/usr/lib/python3.8/asyncio/streams.py", line 635, in readuntil >>>> raise exceptions.LimitOverrunError( >>>> asyncio.exceptions.LimitOverrunError: Separator is found, but chunk is >>>> longer than limit >>>> >>>> During handling of the above exception, another exception occurred: >>>> >>>> Traceback (most recent call last): >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 8479, in <module> >>>> main() >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 8467, in main >>>> r = ctx.func(ctx) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1732, in _infer_config >>>> return func(ctx) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1673, in _infer_fsid >>>> return func(ctx) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1760, in _infer_image >>>> return func(ctx) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1660, in _validate_fsid >>>> return func(ctx) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 4753, in command_ceph_volume >>>> out, err, code = call_throws(ctx, c.run_cmd()) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1460, in call_throws >>>> out, err, ret = call(ctx, command, desc, verbosity, timeout, >>>> **kwargs) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1442, in call >>>> stdout, stderr, returncode = async_run(run_with_timeout()) >>>> File "/usr/lib/python3.8/asyncio/runners.py", line 44, in run >>>> return loop.run_until_complete(main) >>>> File "/usr/lib/python3.8/asyncio/base_events.py", line 616, in >>>> run_until_complete >>>> return future.result() >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1433, in run_with_timeout >>>> stdout, stderr = await asyncio.gather(tee(process.stdout), >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1415, in tee >>>> async for line in reader: >>>> File "/usr/lib/python3.8/asyncio/streams.py", line 738, in __anext__ >>>> val = await self.readline() >>>> File "/usr/lib/python3.8/asyncio/streams.py", line 549, in readline >>>> raise ValueError(e.args[0]) >>>> ValueError: Separator is found, but chunk is longer than limit >>>> Traceback (most recent call last): >>>> File "/usr/share/ceph/mgr/cephadm/serve.py", line 1366, in >>>> _remote_connection >>>> yield (conn, connr) >>>> File "/usr/share/ceph/mgr/cephadm/serve.py", line 1263, in >>>> _run_cephadm >>>> code, '\n'.join(err))) >>>> orchestrator._interface.OrchestratorError: cephadm exited with an error >>>> code: 1, stderr:Inferring config >>>> >>>> /var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/mon.prdrepoceph02/config >>>> Traceback (most recent call last): >>>> File "/usr/lib/python3.8/asyncio/streams.py", line 540, in readline >>>> line = await self.readuntil(sep) >>>> File "/usr/lib/python3.8/asyncio/streams.py", line 635, in readuntil >>>> raise exceptions.LimitOverrunError( >>>> asyncio.exceptions.LimitOverrunError: Separator is found, but chunk is >>>> longer than limit >>>> >>>> During handling of the above exception, another exception occurred: >>>> >>>> Traceback (most recent call last): >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 8479, in <module> >>>> main() >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 8467, in main >>>> r = ctx.func(ctx) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1732, in _infer_config >>>> return func(ctx) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1673, in _infer_fsid >>>> return func(ctx) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1760, in _infer_image >>>> return func(ctx) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1660, in _validate_fsid >>>> return func(ctx) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 4753, in command_ceph_volume >>>> out, err, code = call_throws(ctx, c.run_cmd()) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1460, in call_throws >>>> out, err, ret = call(ctx, command, desc, verbosity, timeout, >>>> **kwargs) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1442, in call >>>> stdout, stderr, returncode = async_run(run_with_timeout()) >>>> File "/usr/lib/python3.8/asyncio/runners.py", line 44, in run >>>> return loop.run_until_complete(main) >>>> File "/usr/lib/python3.8/asyncio/base_events.py", line 616, in >>>> run_until_complete >>>> return future.result() >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1433, in run_with_timeout >>>> stdout, stderr = await asyncio.gather(tee(process.stdout), >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1415, in tee >>>> async for line in reader: >>>> File "/usr/lib/python3.8/asyncio/streams.py", line 738, in __anext__ >>>> val = await self.readline() >>>> File "/usr/lib/python3.8/asyncio/streams.py", line 549, in readline >>>> raise ValueError(e.args[0]) >>>> ValueError: Separator is found, but chunk is longer than limit >>>> Sep 24, 2021, 1:21:13 PM [ERR] cephadm exited with an error code: 1, >>>> stderr:Inferring config >>>> >>>> /var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/mon.prdrepoceph01/config >>>> Traceback (most recent call last): >>>> File "/usr/lib/python3.8/asyncio/streams.py", line 540, in readline >>>> line = await self.readuntil(sep) >>>> File "/usr/lib/python3.8/asyncio/streams.py", line 635, in readuntil >>>> raise exceptions.LimitOverrunError( >>>> asyncio.exceptions.LimitOverrunError: Separator is found, but chunk is >>>> longer than limit >>>> >>>> During handling of the above exception, another exception occurred: >>>> >>>> Traceback (most recent call last): >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 8479, in <module> >>>> main() >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 8467, in main >>>> r = ctx.func(ctx) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1732, in _infer_config >>>> return func(ctx) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1673, in _infer_fsid >>>> return func(ctx) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1760, in _infer_image >>>> return func(ctx) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1660, in _validate_fsid >>>> return func(ctx) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 4753, in command_ceph_volume >>>> out, err, code = call_throws(ctx, c.run_cmd()) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1460, in call_throws >>>> out, err, ret = call(ctx, command, desc, verbosity, timeout, >>>> **kwargs) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1442, in call >>>> stdout, stderr, returncode = async_run(run_with_timeout()) >>>> File "/usr/lib/python3.8/asyncio/runners.py", line 44, in run >>>> return loop.run_until_complete(main) >>>> File "/usr/lib/python3.8/asyncio/base_events.py", line 616, in >>>> run_until_complete >>>> return future.result() >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1433, in run_with_timeout >>>> stdout, stderr = await asyncio.gather(tee(process.stdout), >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1415, in tee >>>> async for line in reader: >>>> File "/usr/lib/python3.8/asyncio/streams.py", line 738, in __anext__ >>>> val = await self.readline() >>>> File "/usr/lib/python3.8/asyncio/streams.py", line 549, in readline >>>> raise ValueError(e.args[0]) >>>> ValueError: Separator is found, but chunk is longer than limit >>>> Traceback (most recent call last): >>>> File "/usr/share/ceph/mgr/cephadm/serve.py", line 1366, in >>>> _remote_connection >>>> yield (conn, connr) >>>> File "/usr/share/ceph/mgr/cephadm/serve.py", line 1263, in >>>> _run_cephadm >>>> code, '\n'.join(err))) >>>> orchestrator._interface.OrchestratorError: cephadm exited with an error >>>> code: 1, stderr:Inferring config >>>> >>>> /var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/mon.prdrepoceph01/config >>>> Traceback (most recent call last): >>>> File "/usr/lib/python3.8/asyncio/streams.py", line 540, in readline >>>> line = await self.readuntil(sep) >>>> File "/usr/lib/python3.8/asyncio/streams.py", line 635, in readuntil >>>> raise exceptions.LimitOverrunError( >>>> asyncio.exceptions.LimitOverrunError: Separator is found, but chunk is >>>> longer than limit >>>> >>>> During handling of the above exception, another exception occurred: >>>> >>>> Traceback (most recent call last): >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 8479, in <module> >>>> main() >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 8467, in main >>>> r = ctx.func(ctx) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1732, in _infer_config >>>> return func(ctx) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1673, in _infer_fsid >>>> return func(ctx) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1760, in _infer_image >>>> return func(ctx) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1660, in _validate_fsid >>>> return func(ctx) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 4753, in command_ceph_volume >>>> out, err, code = call_throws(ctx, c.run_cmd()) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1460, in call_throws >>>> out, err, ret = call(ctx, command, desc, verbosity, timeout, >>>> **kwargs) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1442, in call >>>> stdout, stderr, returncode = async_run(run_with_timeout()) >>>> File "/usr/lib/python3.8/asyncio/runners.py", line 44, in run >>>> return loop.run_until_complete(main) >>>> File "/usr/lib/python3.8/asyncio/base_events.py", line 616, in >>>> run_until_complete >>>> return future.result() >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1433, in run_with_timeout >>>> stdout, stderr = await asyncio.gather(tee(process.stdout), >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1415, in tee >>>> async for line in reader: >>>> File "/usr/lib/python3.8/asyncio/streams.py", line 738, in __anext__ >>>> val = await self.readline() >>>> File "/usr/lib/python3.8/asyncio/streams.py", line 549, in readline >>>> raise ValueError(e.args[0]) >>>> ValueError: Separator is found, but chunk is longer than limit >>>> Sep 24, 2021, 1:21:13 PM [ERR] cephadm exited with an error code: 1, >>>> stderr:Inferring config >>>> >>>> /var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/mon.prdrepoceph02/config >>>> Traceback (most recent call last): >>>> File "/usr/lib/python3.8/asyncio/streams.py", line 540, in readline >>>> line = await self.readuntil(sep) >>>> File "/usr/lib/python3.8/asyncio/streams.py", line 635, in readuntil >>>> raise exceptions.LimitOverrunError( >>>> asyncio.exceptions.LimitOverrunError: Separator is found, but chunk is >>>> longer than limit >>>> >>>> During handling of the above exception, another exception occurred: >>>> >>>> Traceback (most recent call last): >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 8479, in <module> >>>> main() >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 8467, in main >>>> r = ctx.func(ctx) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1732, in _infer_config >>>> return func(ctx) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1673, in _infer_fsid >>>> return func(ctx) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1760, in _infer_image >>>> return func(ctx) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1660, in _validate_fsid >>>> return func(ctx) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 4753, in command_ceph_volume >>>> out, err, code = call_throws(ctx, c.run_cmd()) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1460, in call_throws >>>> out, err, ret = call(ctx, command, desc, verbosity, timeout, >>>> **kwargs) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1442, in call >>>> stdout, stderr, returncode = async_run(run_with_timeout()) >>>> File "/usr/lib/python3.8/asyncio/runners.py", line 44, in run >>>> return loop.run_until_complete(main) >>>> File "/usr/lib/python3.8/asyncio/base_events.py", line 616, in >>>> run_until_complete >>>> return future.result() >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1433, in run_with_timeout >>>> stdout, stderr = await asyncio.gather(tee(process.stdout), >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1415, in tee >>>> async for line in reader: >>>> File "/usr/lib/python3.8/asyncio/streams.py", line 738, in __anext__ >>>> val = await self.readline() >>>> File "/usr/lib/python3.8/asyncio/streams.py", line 549, in readline >>>> raise ValueError(e.args[0]) >>>> ValueError: Separator is found, but chunk is longer than limit >>>> Traceback (most recent call last): >>>> File "/usr/share/ceph/mgr/cephadm/serve.py", line 1366, in >>>> _remote_connection >>>> yield (conn, connr) >>>> File "/usr/share/ceph/mgr/cephadm/serve.py", line 1263, in >>>> _run_cephadm >>>> code, '\n'.join(err))) >>>> orchestrator._interface.OrchestratorError: cephadm exited with an error >>>> code: 1, stderr:Inferring config >>>> >>>> /var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/mon.prdrepoceph02/config >>>> Traceback (most recent call last): >>>> File "/usr/lib/python3.8/asyncio/streams.py", line 540, in readline >>>> line = await self.readuntil(sep) >>>> File "/usr/lib/python3.8/asyncio/streams.py", line 635, in readuntil >>>> raise exceptions.LimitOverrunError( >>>> asyncio.exceptions.LimitOverrunError: Separator is found, but chunk is >>>> longer than limit >>>> >>>> During handling of the above exception, another exception occurred: >>>> >>>> Traceback (most recent call last): >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 8479, in <module> >>>> main() >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 8467, in main >>>> r = ctx.func(ctx) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1732, in _infer_config >>>> return func(ctx) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1673, in _infer_fsid >>>> return func(ctx) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1760, in _infer_image >>>> return func(ctx) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1660, in _validate_fsid >>>> return func(ctx) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 4753, in command_ceph_volume >>>> out, err, code = call_throws(ctx, c.run_cmd()) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1460, in call_throws >>>> out, err, ret = call(ctx, command, desc, verbosity, timeout, >>>> **kwargs) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1442, in call >>>> stdout, stderr, returncode = async_run(run_with_timeout()) >>>> File "/usr/lib/python3.8/asyncio/runners.py", line 44, in run >>>> return loop.run_until_complete(main) >>>> File "/usr/lib/python3.8/asyncio/base_events.py", line 616, in >>>> run_until_complete >>>> return future.result() >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1433, in run_with_timeout >>>> stdout, stderr = await asyncio.gather(tee(process.stdout), >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1415, in tee >>>> async for line in reader: >>>> File "/usr/lib/python3.8/asyncio/streams.py", line 738, in __anext__ >>>> val = await self.readline() >>>> File "/usr/lib/python3.8/asyncio/streams.py", line 549, in readline >>>> raise ValueError(e.args[0]) >>>> ValueError: Separator is found, but chunk is longer than limit >>>> Sep 24, 2021, 1:20:00 PM [WRN] overall HEALTH_WARN failed to probe >>>> daemons >>>> or devices >>>> Sep 24, 2021, 1:19:41 PM [ERR] cephadm exited with an error code: 1, >>>> stderr:Inferring config >>>> >>>> /var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/mon.prdrepoceph01/config >>>> Traceback (most recent call last): >>>> File "/usr/lib/python3.8/asyncio/streams.py", line 540, in readline >>>> line = await self.readuntil(sep) >>>> File "/usr/lib/python3.8/asyncio/streams.py", line 635, in readuntil >>>> raise exceptions.LimitOverrunError( >>>> asyncio.exceptions.LimitOverrunError: Separator is found, but chunk is >>>> longer than limit >>>> >>>> During handling of the above exception, another exception occurred: >>>> >>>> Traceback (most recent call last): >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 8479, in <module> >>>> main() >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 8467, in main >>>> r = ctx.func(ctx) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1732, in _infer_config >>>> return func(ctx) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1673, in _infer_fsid >>>> return func(ctx) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1760, in _infer_image >>>> return func(ctx) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1660, in _validate_fsid >>>> return func(ctx) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 4753, in command_ceph_volume >>>> out, err, code = call_throws(ctx, c.run_cmd()) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1460, in call_throws >>>> out, err, ret = call(ctx, command, desc, verbosity, timeout, >>>> **kwargs) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1442, in call >>>> stdout, stderr, returncode = async_run(run_with_timeout()) >>>> File "/usr/lib/python3.8/asyncio/runners.py", line 44, in run >>>> return loop.run_until_complete(main) >>>> File "/usr/lib/python3.8/asyncio/base_events.py", line 616, in >>>> run_until_complete >>>> return future.result() >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1433, in run_with_timeout >>>> stdout, stderr = await asyncio.gather(tee(process.stdout), >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1415, in tee >>>> async for line in reader: >>>> File "/usr/lib/python3.8/asyncio/streams.py", line 738, in __anext__ >>>> val = await self.readline() >>>> File "/usr/lib/python3.8/asyncio/streams.py", line 549, in readline >>>> raise ValueError(e.args[0]) >>>> ValueError: Separator is found, but chunk is longer than limit >>>> Traceback (most recent call last): >>>> File "/usr/share/ceph/mgr/cephadm/serve.py", line 1366, in >>>> _remote_connection >>>> yield (conn, connr) >>>> File "/usr/share/ceph/mgr/cephadm/serve.py", line 1263, in >>>> _run_cephadm >>>> code, '\n'.join(err))) >>>> orchestrator._interface.OrchestratorError: cephadm exited with an error >>>> code: 1, stderr:Inferring config >>>> >>>> /var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/mon.prdrepoceph01/config >>>> Traceback (most recent call last): >>>> File "/usr/lib/python3.8/asyncio/streams.py", line 540, in readline >>>> line = await self.readuntil(sep) >>>> File "/usr/lib/python3.8/asyncio/streams.py", line 635, in readuntil >>>> raise exceptions.LimitOverrunError( >>>> asyncio.exceptions.LimitOverrunError: Separator is found, but chunk is >>>> longer than limit >>>> >>>> During handling of the above exception, another exception occurred: >>>> >>>> Traceback (most recent call last): >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 8479, in <module> >>>> main() >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 8467, in main >>>> r = ctx.func(ctx) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1732, in _infer_config >>>> return func(ctx) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1673, in _infer_fsid >>>> return func(ctx) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1760, in _infer_image >>>> return func(ctx) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1660, in _validate_fsid >>>> return func(ctx) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 4753, in command_ceph_volume >>>> out, err, code = call_throws(ctx, c.run_cmd()) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1460, in call_throws >>>> out, err, ret = call(ctx, command, desc, verbosity, timeout, >>>> **kwargs) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1442, in call >>>> stdout, stderr, returncode = async_run(run_with_timeout()) >>>> File "/usr/lib/python3.8/asyncio/runners.py", line 44, in run >>>> return loop.run_until_complete(main) >>>> File "/usr/lib/python3.8/asyncio/base_events.py", line 616, in >>>> run_until_complete >>>> return future.result() >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1433, in run_with_timeout >>>> stdout, stderr = await asyncio.gather(tee(process.stdout), >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1415, in tee >>>> async for line in reader: >>>> File "/usr/lib/python3.8/asyncio/streams.py", line 738, in __anext__ >>>> val = await self.readline() >>>> File "/usr/lib/python3.8/asyncio/streams.py", line 549, in readline >>>> raise ValueError(e.args[0]) >>>> ValueError: Separator is found, but chunk is longer than limit >>>> Sep 24, 2021, 1:19:40 PM [ERR] cephadm exited with an error code: 1, >>>> stderr:Inferring config >>>> >>>> /var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/mon.prdrepoceph02/config >>>> Traceback (most recent call last): >>>> File "/usr/lib/python3.8/asyncio/streams.py", line 540, in readline >>>> line = await self.readuntil(sep) >>>> File "/usr/lib/python3.8/asyncio/streams.py", line 635, in readuntil >>>> raise exceptions.LimitOverrunError( >>>> asyncio.exceptions.LimitOverrunError: Separator is found, but chunk is >>>> longer than limit >>>> >>>> During handling of the above exception, another exception occurred: >>>> >>>> Traceback (most recent call last): >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 8479, in <module> >>>> main() >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 8467, in main >>>> r = ctx.func(ctx) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1732, in _infer_config >>>> return func(ctx) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1673, in _infer_fsid >>>> return func(ctx) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1760, in _infer_image >>>> return func(ctx) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1660, in _validate_fsid >>>> return func(ctx) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 4753, in command_ceph_volume >>>> out, err, code = call_throws(ctx, c.run_cmd()) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1460, in call_throws >>>> out, err, ret = call(ctx, command, desc, verbosity, timeout, >>>> **kwargs) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1442, in call >>>> stdout, stderr, returncode = async_run(run_with_timeout()) >>>> File "/usr/lib/python3.8/asyncio/runners.py", line 44, in run >>>> return loop.run_until_complete(main) >>>> File "/usr/lib/python3.8/asyncio/base_events.py", line 616, in >>>> run_until_complete >>>> return future.result() >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1433, in run_with_timeout >>>> stdout, stderr = await asyncio.gather(tee(process.stdout), >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1415, in tee >>>> async for line in reader: >>>> File "/usr/lib/python3.8/asyncio/streams.py", line 738, in __anext__ >>>> val = await self.readline() >>>> File "/usr/lib/python3.8/asyncio/streams.py", line 549, in readline >>>> raise ValueError(e.args[0]) >>>> ValueError: Separator is found, but chunk is longer than limit >>>> Traceback (most recent call last): >>>> File "/usr/share/ceph/mgr/cephadm/serve.py", line 1366, in >>>> _remote_connection >>>> yield (conn, connr) >>>> File "/usr/share/ceph/mgr/cephadm/serve.py", line 1263, in >>>> _run_cephadm >>>> code, '\n'.join(err))) >>>> orchestrator._interface.OrchestratorError: cephadm exited with an error >>>> code: 1, stderr:Inferring config >>>> >>>> /var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/mon.prdrepoceph02/config >>>> Traceback (most recent call last): >>>> File "/usr/lib/python3.8/asyncio/streams.py", line 540, in readline >>>> line = await self.readuntil(sep) >>>> File "/usr/lib/python3.8/asyncio/streams.py", line 635, in readuntil >>>> raise exceptions.LimitOverrunError( >>>> asyncio.exceptions.LimitOverrunError: Separator is found, but chunk is >>>> longer than limit >>>> >>>> During handling of the above exception, another exception occurred: >>>> >>>> Traceback (most recent call last): >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 8479, in <module> >>>> main() >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 8467, in main >>>> r = ctx.func(ctx) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1732, in _infer_config >>>> return func(ctx) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1673, in _infer_fsid >>>> return func(ctx) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1760, in _infer_image >>>> return func(ctx) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1660, in _validate_fsid >>>> return func(ctx) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 4753, in command_ceph_volume >>>> out, err, code = call_throws(ctx, c.run_cmd()) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1460, in call_throws >>>> out, err, ret = call(ctx, command, desc, verbosity, timeout, >>>> **kwargs) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1442, in call >>>> stdout, stderr, returncode = async_run(run_with_timeout()) >>>> File "/usr/lib/python3.8/asyncio/runners.py", line 44, in run >>>> return loop.run_until_complete(main) >>>> File "/usr/lib/python3.8/asyncio/base_events.py", line 616, in >>>> run_until_complete >>>> return future.result() >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1433, in run_with_timeout >>>> stdout, stderr = await asyncio.gather(tee(process.stdout), >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1415, in tee >>>> async for line in reader: >>>> File "/usr/lib/python3.8/asyncio/streams.py", line 738, in __anext__ >>>> val = await self.readline() >>>> File "/usr/lib/python3.8/asyncio/streams.py", line 549, in readline >>>> raise ValueError(e.args[0]) >>>> ValueError: Separator is found, but chunk is longer than limit >>>> Sep 24, 2021, 1:18:09 PM [ERR] cephadm exited with an error code: 1, >>>> stderr:Inferring config >>>> >>>> /var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/mon.prdrepoceph02/config >>>> Traceback (most recent call last): >>>> File "/usr/lib/python3.8/asyncio/streams.py", line 540, in readline >>>> line = await self.readuntil(sep) >>>> File "/usr/lib/python3.8/asyncio/streams.py", line 635, in readuntil >>>> raise exceptions.LimitOverrunError( >>>> asyncio.exceptions.LimitOverrunError: Separator is found, but chunk is >>>> longer than limit >>>> >>>> During handling of the above exception, another exception occurred: >>>> >>>> Traceback (most recent call last): >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 8479, in <module> >>>> main() >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 8467, in main >>>> r = ctx.func(ctx) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1732, in _infer_config >>>> return func(ctx) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1673, in _infer_fsid >>>> return func(ctx) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1760, in _infer_image >>>> return func(ctx) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1660, in _validate_fsid >>>> return func(ctx) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 4753, in command_ceph_volume >>>> out, err, code = call_throws(ctx, c.run_cmd()) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1460, in call_throws >>>> out, err, ret = call(ctx, command, desc, verbosity, timeout, >>>> **kwargs) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1442, in call >>>> stdout, stderr, returncode = async_run(run_with_timeout()) >>>> File "/usr/lib/python3.8/asyncio/runners.py", line 44, in run >>>> return loop.run_until_complete(main) >>>> File "/usr/lib/python3.8/asyncio/base_events.py", line 616, in >>>> run_until_complete >>>> return future.result() >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1433, in run_with_timeout >>>> stdout, stderr = await asyncio.gather(tee(process.stdout), >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1415, in tee >>>> async for line in reader: >>>> File "/usr/lib/python3.8/asyncio/streams.py", line 738, in __anext__ >>>> val = await self.readline() >>>> File "/usr/lib/python3.8/asyncio/streams.py", line 549, in readline >>>> raise ValueError(e.args[0]) >>>> ValueError: Separator is found, but chunk is longer than limit >>>> Traceback (most recent call last): >>>> File "/usr/share/ceph/mgr/cephadm/serve.py", line 1366, in >>>> _remote_connection >>>> yield (conn, connr) >>>> File "/usr/share/ceph/mgr/cephadm/serve.py", line 1263, in >>>> _run_cephadm >>>> code, '\n'.join(err))) >>>> orchestrator._interface.OrchestratorError: cephadm exited with an error >>>> code: 1, stderr:Inferring config >>>> >>>> /var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/mon.prdrepoceph02/config >>>> Traceback (most recent call last): >>>> File "/usr/lib/python3.8/asyncio/streams.py", line 540, in readline >>>> line = await self.readuntil(sep) >>>> File "/usr/lib/python3.8/asyncio/streams.py", line 635, in readuntil >>>> raise exceptions.LimitOverrunError( >>>> asyncio.exceptions.LimitOverrunError: Separator is found, but chunk is >>>> longer than limit >>>> >>>> During handling of the above exception, another exception occurred: >>>> >>>> Traceback (most recent call last): >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 8479, in <module> >>>> main() >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 8467, in main >>>> r = ctx.func(ctx) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1732, in _infer_config >>>> return func(ctx) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1673, in _infer_fsid >>>> return func(ctx) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1760, in _infer_image >>>> return func(ctx) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1660, in _validate_fsid >>>> return func(ctx) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 4753, in command_ceph_volume >>>> out, err, code = call_throws(ctx, c.run_cmd()) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1460, in call_throws >>>> out, err, ret = call(ctx, command, desc, verbosity, timeout, >>>> **kwargs) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1442, in call >>>> stdout, stderr, returncode = async_run(run_with_timeout()) >>>> File "/usr/lib/python3.8/asyncio/runners.py", line 44, in run >>>> return loop.run_until_complete(main) >>>> File "/usr/lib/python3.8/asyncio/base_events.py", line 616, in >>>> run_until_complete >>>> return future.result() >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1433, in run_with_timeout >>>> stdout, stderr = await asyncio.gather(tee(process.stdout), >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1415, in tee >>>> async for line in reader: >>>> File "/usr/lib/python3.8/asyncio/streams.py", line 738, in __anext__ >>>> val = await self.readline() >>>> File "/usr/lib/python3.8/asyncio/streams.py", line 549, in readline >>>> raise ValueError(e.args[0]) >>>> ValueError: Separator is found, but chunk is longer than limit >>>> Sep 24, 2021, 1:18:09 PM [ERR] cephadm exited with an error code: 1, >>>> stderr:Inferring config >>>> >>>> /var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/mon.prdrepoceph01/config >>>> Traceback (most recent call last): >>>> File "/usr/lib/python3.8/asyncio/streams.py", line 540, in readline >>>> line = await self.readuntil(sep) >>>> File "/usr/lib/python3.8/asyncio/streams.py", line 635, in readuntil >>>> raise exceptions.LimitOverrunError( >>>> asyncio.exceptions.LimitOverrunError: Separator is found, but chunk is >>>> longer than limit >>>> >>>> During handling of the above exception, another exception occurred: >>>> >>>> Traceback (most recent call last): >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 8479, in <module> >>>> main() >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 8467, in main >>>> r = ctx.func(ctx) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1732, in _infer_config >>>> return func(ctx) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1673, in _infer_fsid >>>> return func(ctx) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1760, in _infer_image >>>> return func(ctx) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1660, in _validate_fsid >>>> return func(ctx) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 4753, in command_ceph_volume >>>> out, err, code = call_throws(ctx, c.run_cmd()) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1460, in call_throws >>>> out, err, ret = call(ctx, command, desc, verbosity, timeout, >>>> **kwargs) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1442, in call >>>> stdout, stderr, returncode = async_run(run_with_timeout()) >>>> File "/usr/lib/python3.8/asyncio/runners.py", line 44, in run >>>> return loop.run_until_complete(main) >>>> File "/usr/lib/python3.8/asyncio/base_events.py", line 616, in >>>> run_until_complete >>>> return future.result() >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1433, in run_with_timeout >>>> stdout, stderr = await asyncio.gather(tee(process.stdout), >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1415, in tee >>>> async for line in reader: >>>> File "/usr/lib/python3.8/asyncio/streams.py", line 738, in __anext__ >>>> val = await self.readline() >>>> File "/usr/lib/python3.8/asyncio/streams.py", line 549, in readline >>>> raise ValueError(e.args[0]) >>>> ValueError: Separator is found, but chunk is longer than limit >>>> Traceback (most recent call last): >>>> File "/usr/share/ceph/mgr/cephadm/serve.py", line 1366, in >>>> _remote_connection >>>> yield (conn, connr) >>>> File "/usr/share/ceph/mgr/cephadm/serve.py", line 1263, in >>>> _run_cephadm >>>> code, '\n'.join(err))) >>>> orchestrator._interface.OrchestratorError: cephadm exited with an error >>>> code: 1, stderr:Inferring config >>>> >>>> /var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/mon.prdrepoceph01/config >>>> Traceback (most recent call last): >>>> File "/usr/lib/python3.8/asyncio/streams.py", line 540, in readline >>>> line = await self.readuntil(sep) >>>> File "/usr/lib/python3.8/asyncio/streams.py", line 635, in readuntil >>>> raise exceptions.LimitOverrunError( >>>> asyncio.exceptions.LimitOverrunError: Separator is found, but chunk is >>>> longer than limit >>>> >>>> During handling of the above exception, another exception occurred: >>>> >>>> Traceback (most recent call last): >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 8479, in <module> >>>> main() >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 8467, in main >>>> r = ctx.func(ctx) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1732, in _infer_config >>>> return func(ctx) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1673, in _infer_fsid >>>> return func(ctx) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1760, in _infer_image >>>> return func(ctx) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1660, in _validate_fsid >>>> return func(ctx) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 4753, in command_ceph_volume >>>> out, err, code = call_throws(ctx, c.run_cmd()) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1460, in call_throws >>>> out, err, ret = call(ctx, command, desc, verbosity, timeout, >>>> **kwargs) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1442, in call >>>> stdout, stderr, returncode = async_run(run_with_timeout()) >>>> File "/usr/lib/python3.8/asyncio/runners.py", line 44, in run >>>> return loop.run_until_complete(main) >>>> File "/usr/lib/python3.8/asyncio/base_events.py", line 616, in >>>> run_until_complete >>>> return future.result() >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1433, in run_with_timeout >>>> stdout, stderr = await asyncio.gather(tee(process.stdout), >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1415, in tee >>>> async for line in reader: >>>> File "/usr/lib/python3.8/asyncio/streams.py", line 738, in __anext__ >>>> val = await self.readline() >>>> File "/usr/lib/python3.8/asyncio/streams.py", line 549, in readline >>>> raise ValueError(e.args[0]) >>>> ValueError: Separator is found, but chunk is longer than limit >>>> Sep 24, 2021, 1:16:37 PM [ERR] cephadm exited with an error code: 1, >>>> stderr:Inferring config >>>> >>>> /var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/mon.prdrepoceph01/config >>>> Traceback (most recent call last): >>>> File "/usr/lib/python3.8/asyncio/streams.py", line 540, in readline >>>> line = await self.readuntil(sep) >>>> File "/usr/lib/python3.8/asyncio/streams.py", line 635, in readuntil >>>> raise exceptions.LimitOverrunError( >>>> asyncio.exceptions.LimitOverrunError: Separator is found, but chunk is >>>> longer than limit >>>> >>>> During handling of the above exception, another exception occurred: >>>> >>>> Traceback (most recent call last): >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 8479, in <module> >>>> main() >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 8467, in main >>>> r = ctx.func(ctx) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1732, in _infer_config >>>> return func(ctx) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1673, in _infer_fsid >>>> return func(ctx) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1760, in _infer_image >>>> return func(ctx) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1660, in _validate_fsid >>>> return func(ctx) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 4753, in command_ceph_volume >>>> out, err, code = call_throws(ctx, c.run_cmd()) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1460, in call_throws >>>> out, err, ret = call(ctx, command, desc, verbosity, timeout, >>>> **kwargs) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1442, in call >>>> stdout, stderr, returncode = async_run(run_with_timeout()) >>>> File "/usr/lib/python3.8/asyncio/runners.py", line 44, in run >>>> return loop.run_until_complete(main) >>>> File "/usr/lib/python3.8/asyncio/base_events.py", line 616, in >>>> run_until_complete >>>> return future.result() >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1433, in run_with_timeout >>>> stdout, stderr = await asyncio.gather(tee(process.stdout), >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1415, in tee >>>> async for line in reader: >>>> File "/usr/lib/python3.8/asyncio/streams.py", line 738, in __anext__ >>>> val = await self.readline() >>>> File "/usr/lib/python3.8/asyncio/streams.py", line 549, in readline >>>> raise ValueError(e.args[0]) >>>> ValueError: Separator is found, but chunk is longer than limit >>>> Traceback (most recent call last): >>>> File "/usr/share/ceph/mgr/cephadm/serve.py", line 1366, in >>>> _remote_connection >>>> yield (conn, connr) >>>> File "/usr/share/ceph/mgr/cephadm/serve.py", line 1263, in >>>> _run_cephadm >>>> code, '\n'.join(err))) >>>> orchestrator._interface.OrchestratorError: cephadm exited with an error >>>> code: 1, stderr:Inferring config >>>> >>>> /var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/mon.prdrepoceph01/config >>>> Traceback (most recent call last): >>>> File "/usr/lib/python3.8/asyncio/streams.py", line 540, in readline >>>> line = await self.readuntil(sep) >>>> File "/usr/lib/python3.8/asyncio/streams.py", line 635, in readuntil >>>> raise exceptions.LimitOverrunError( >>>> asyncio.exceptions.LimitOverrunError: Separator is found, but chunk is >>>> longer than limit >>>> >>>> During handling of the above exception, another exception occurred: >>>> >>>> Traceback (most recent call last): >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 8479, in <module> >>>> main() >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 8467, in main >>>> r = ctx.func(ctx) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1732, in _infer_config >>>> return func(ctx) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1673, in _infer_fsid >>>> return func(ctx) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1760, in _infer_image >>>> return func(ctx) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1660, in _validate_fsid >>>> return func(ctx) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 4753, in command_ceph_volume >>>> out, err, code = call_throws(ctx, c.run_cmd()) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1460, in call_throws >>>> out, err, ret = call(ctx, command, desc, verbosity, timeout, >>>> **kwargs) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1442, in call >>>> stdout, stderr, returncode = async_run(run_with_timeout()) >>>> File "/usr/lib/python3.8/asyncio/runners.py", line 44, in run >>>> return loop.run_until_complete(main) >>>> File "/usr/lib/python3.8/asyncio/base_events.py", line 616, in >>>> run_until_complete >>>> return future.result() >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1433, in run_with_timeout >>>> stdout, stderr = await asyncio.gather(tee(process.stdout), >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1415, in tee >>>> async for line in reader: >>>> File "/usr/lib/python3.8/asyncio/streams.py", line 738, in __anext__ >>>> val = await self.readline() >>>> File "/usr/lib/python3.8/asyncio/streams.py", line 549, in readline >>>> raise ValueError(e.args[0]) >>>> ValueError: Separator is found, but chunk is longer than limit >>>> Sep 24, 2021, 1:16:37 PM [ERR] cephadm exited with an error code: 1, >>>> stderr:Inferring config >>>> >>>> /var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/mon.prdrepoceph02/config >>>> Traceback (most recent call last): >>>> File "/usr/lib/python3.8/asyncio/streams.py", line 540, in readline >>>> line = await self.readuntil(sep) >>>> File "/usr/lib/python3.8/asyncio/streams.py", line 635, in readuntil >>>> raise exceptions.LimitOverrunError( >>>> asyncio.exceptions.LimitOverrunError: Separator is found, but chunk is >>>> longer than limit >>>> >>>> During handling of the above exception, another exception occurred: >>>> >>>> Traceback (most recent call last): >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 8479, in <module> >>>> main() >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 8467, in main >>>> r = ctx.func(ctx) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1732, in _infer_config >>>> return func(ctx) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1673, in _infer_fsid >>>> return func(ctx) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1760, in _infer_image >>>> return func(ctx) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1660, in _validate_fsid >>>> return func(ctx) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 4753, in command_ceph_volume >>>> out, err, code = call_throws(ctx, c.run_cmd()) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1460, in call_throws >>>> out, err, ret = call(ctx, command, desc, verbosity, timeout, >>>> **kwargs) >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1442, in call >>>> stdout, stderr, returncode = async_run(run_with_timeout()) >>>> File "/usr/lib/python3.8/asyncio/runners.py", line 44, in run >>>> return loop.run_until_complete(main) >>>> File "/usr/lib/python3.8/asyncio/base_events.py", line 616, in >>>> run_until_complete >>>> return future.result() >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1433, in run_with_timeout >>>> stdout, stderr = await asyncio.gather(tee(process.stdout), >>>> File >>>> >>>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>>> line 1415, in tee >>>> async for line in reader: >>>> File "/usr/lib/python3.8/asyncio/streams.py", line 738, in __anext__ >>>> val = await self.readline() >>>> File "/usr/lib/python3.8/asyncio/streams.py", line 549, in readline >>>> raise ValueError(e.args[0]) >>>> ValueError: Separator is found, but chunk is longer than limit >>>> Traceback (most recent call last): >>>> File "/usr/share/ceph/mgr/cephadm/serve.py", line 1366, in >>>> _remote >>> >>> _______________________________________________ ceph-users mailing list -- ceph-users@xxxxxxx To unsubscribe send an email to ceph-users-leave@xxxxxxx
 |