Good morning Adam, Ceph users, Is there something we can do to extend the acceptable response size? Trying to understand if there is some viable workaround that we can implement. Thanks, Marco On Fri, Sep 24, 2021 at 2:59 PM Marco Pizzolo <marcopizzolo@xxxxxxxxx> wrote: > Hi Adam, > > I really appreciate your time reviewing. > > You're absolutely correct. We have 60 OSDs in 2 of the nodes. We do see > the errors as you predicated. > > All of our nodes will have 60 OSDs, so I'm hoping there's a fix for this. > > Thanks for the quick response. > > Using recent ceph image > quay.io/ceph/ceph@sha256:43b9d5bb30fe76c831a97b47abd92290516fcc4e1455aba455cbcfc85f166b36 > Traceback (most recent call last): > File "/usr/lib/python3.8/asyncio/streams.py", line 540, in readline > line = await self.readuntil(sep) > File "/usr/lib/python3.8/asyncio/streams.py", line 635, in readuntil > raise exceptions.LimitOverrunError( > asyncio.exceptions.LimitOverrunError: Separator is found, but chunk is > longer than limit > > During handling of the above exception, another exception occurred: > > Traceback (most recent call last): > File "/usr/sbin/cephadm", line 8479, in <module> > main() > File "/usr/sbin/cephadm", line 8467, in main > r = ctx.func(ctx) > File "/usr/sbin/cephadm", line 1732, in _infer_config > return func(ctx) > File "/usr/sbin/cephadm", line 1702, in _infer_fsid > return func(ctx) > File "/usr/sbin/cephadm", line 1760, in _infer_image > return func(ctx) > File "/usr/sbin/cephadm", line 1660, in _validate_fsid > return func(ctx) > File "/usr/sbin/cephadm", line 4753, in command_ceph_volume > out, err, code = call_throws(ctx, c.run_cmd()) > File "/usr/sbin/cephadm", line 1460, in call_throws > out, err, ret = call(ctx, command, desc, verbosity, timeout, **kwargs) > File "/usr/sbin/cephadm", line 1442, in call > stdout, stderr, returncode = async_run(run_with_timeout()) > File "/usr/lib/python3.8/asyncio/runners.py", line 44, in run > return loop.run_until_complete(main) > File "/usr/lib/python3.8/asyncio/base_events.py", line 616, in > run_until_complete > return future.result() > File "/usr/sbin/cephadm", line 1433, in run_with_timeout > stdout, stderr = await asyncio.gather(tee(process.stdout), > File "/usr/sbin/cephadm", line 1415, in tee > async for line in reader: > File "/usr/lib/python3.8/asyncio/streams.py", line 738, in __anext__ > val = await self.readline() > File "/usr/lib/python3.8/asyncio/streams.py", line 549, in readline > raise ValueError(e.args[0]) > ValueError: Separator is found, but chunk is longer than limit > > On Fri, Sep 24, 2021 at 2:42 PM Adam King <adking@xxxxxxxxxx> wrote: > >> It looks like the output from a ceph-volume command was too long to >> handle. If you run "cephadm ceph-volume -- inventory --format=json" (add >> "--with-lsm" if you've turned on device_enhanced_scan) manually on each >> host do any of them fail in a similar fashion? >> >> On Fri, Sep 24, 2021 at 1:37 PM Marco Pizzolo <marcopizzolo@xxxxxxxxx> >> wrote: >> >>> Hello Everyone, >>> >>> If you have any suggestions on the cause, or what we can do I'd certainly >>> appreciate it. >>> >>> I'm seeing the following on a newly stood up cluster using Podman on >>> Ubuntu >>> 20.04.3 HWE: >>> >>> Thank you very much >>> >>> Marco >>> >>> >>> >>> Sep 24, 2021, 1:24:30 PM [ERR] cephadm exited with an error code: 1, >>> stderr:Inferring config >>> >>> /var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/mon.prdrepoceph01/config >>> Traceback (most recent call last): >>> File "/usr/lib/python3.8/asyncio/streams.py", line 540, in readline >>> line = await self.readuntil(sep) >>> File "/usr/lib/python3.8/asyncio/streams.py", line 635, in readuntil >>> raise exceptions.LimitOverrunError( >>> asyncio.exceptions.LimitOverrunError: Separator is found, but chunk is >>> longer than limit >>> >>> During handling of the above exception, another exception occurred: >>> >>> Traceback (most recent call last): >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 8479, in <module> >>> main() >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 8467, in main >>> r = ctx.func(ctx) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1732, in _infer_config >>> return func(ctx) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1673, in _infer_fsid >>> return func(ctx) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1760, in _infer_image >>> return func(ctx) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1660, in _validate_fsid >>> return func(ctx) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 4753, in command_ceph_volume >>> out, err, code = call_throws(ctx, c.run_cmd()) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1460, in call_throws >>> out, err, ret = call(ctx, command, desc, verbosity, timeout, >>> **kwargs) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1442, in call >>> stdout, stderr, returncode = async_run(run_with_timeout()) >>> File "/usr/lib/python3.8/asyncio/runners.py", line 44, in run >>> return loop.run_until_complete(main) >>> File "/usr/lib/python3.8/asyncio/base_events.py", line 616, in >>> run_until_complete >>> return future.result() >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1433, in run_with_timeout >>> stdout, stderr = await asyncio.gather(tee(process.stdout), >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1415, in tee >>> async for line in reader: >>> File "/usr/lib/python3.8/asyncio/streams.py", line 738, in __anext__ >>> val = await self.readline() >>> File "/usr/lib/python3.8/asyncio/streams.py", line 549, in readline >>> raise ValueError(e.args[0]) >>> ValueError: Separator is found, but chunk is longer than limit >>> Traceback (most recent call last): >>> File "/usr/share/ceph/mgr/cephadm/serve.py", line 1366, in >>> _remote_connection >>> yield (conn, connr) >>> File "/usr/share/ceph/mgr/cephadm/serve.py", line 1263, in _run_cephadm >>> code, '\n'.join(err))) >>> orchestrator._interface.OrchestratorError: cephadm exited with an error >>> code: 1, stderr:Inferring config >>> >>> /var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/mon.prdrepoceph01/config >>> Traceback (most recent call last): >>> File "/usr/lib/python3.8/asyncio/streams.py", line 540, in readline >>> line = await self.readuntil(sep) >>> File "/usr/lib/python3.8/asyncio/streams.py", line 635, in readuntil >>> raise exceptions.LimitOverrunError( >>> asyncio.exceptions.LimitOverrunError: Separator is found, but chunk is >>> longer than limit >>> >>> During handling of the above exception, another exception occurred: >>> >>> Traceback (most recent call last): >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 8479, in <module> >>> main() >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 8467, in main >>> r = ctx.func(ctx) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1732, in _infer_config >>> return func(ctx) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1673, in _infer_fsid >>> return func(ctx) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1760, in _infer_image >>> return func(ctx) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1660, in _validate_fsid >>> return func(ctx) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 4753, in command_ceph_volume >>> out, err, code = call_throws(ctx, c.run_cmd()) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1460, in call_throws >>> out, err, ret = call(ctx, command, desc, verbosity, timeout, >>> **kwargs) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1442, in call >>> stdout, stderr, returncode = async_run(run_with_timeout()) >>> File "/usr/lib/python3.8/asyncio/runners.py", line 44, in run >>> return loop.run_until_complete(main) >>> File "/usr/lib/python3.8/asyncio/base_events.py", line 616, in >>> run_until_complete >>> return future.result() >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1433, in run_with_timeout >>> stdout, stderr = await asyncio.gather(tee(process.stdout), >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1415, in tee >>> async for line in reader: >>> File "/usr/lib/python3.8/asyncio/streams.py", line 738, in __anext__ >>> val = await self.readline() >>> File "/usr/lib/python3.8/asyncio/streams.py", line 549, in readline >>> raise ValueError(e.args[0]) >>> ValueError: Separator is found, but chunk is longer than limit >>> Sep 24, 2021, 1:24:28 PM [ERR] cephadm exited with an error code: 1, >>> stderr:Inferring config >>> >>> /var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/mon.prdrepoceph02/config >>> Traceback (most recent call last): >>> File "/usr/lib/python3.8/asyncio/streams.py", line 540, in readline >>> line = await self.readuntil(sep) >>> File "/usr/lib/python3.8/asyncio/streams.py", line 635, in readuntil >>> raise exceptions.LimitOverrunError( >>> asyncio.exceptions.LimitOverrunError: Separator is found, but chunk is >>> longer than limit >>> >>> During handling of the above exception, another exception occurred: >>> >>> Traceback (most recent call last): >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 8479, in <module> >>> main() >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 8467, in main >>> r = ctx.func(ctx) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1732, in _infer_config >>> return func(ctx) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1673, in _infer_fsid >>> return func(ctx) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1760, in _infer_image >>> return func(ctx) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1660, in _validate_fsid >>> return func(ctx) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 4753, in command_ceph_volume >>> out, err, code = call_throws(ctx, c.run_cmd()) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1460, in call_throws >>> out, err, ret = call(ctx, command, desc, verbosity, timeout, >>> **kwargs) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1442, in call >>> stdout, stderr, returncode = async_run(run_with_timeout()) >>> File "/usr/lib/python3.8/asyncio/runners.py", line 44, in run >>> return loop.run_until_complete(main) >>> File "/usr/lib/python3.8/asyncio/base_events.py", line 616, in >>> run_until_complete >>> return future.result() >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1433, in run_with_timeout >>> stdout, stderr = await asyncio.gather(tee(process.stdout), >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1415, in tee >>> async for line in reader: >>> File "/usr/lib/python3.8/asyncio/streams.py", line 738, in __anext__ >>> val = await self.readline() >>> File "/usr/lib/python3.8/asyncio/streams.py", line 549, in readline >>> raise ValueError(e.args[0]) >>> ValueError: Separator is found, but chunk is longer than limit >>> Traceback (most recent call last): >>> File "/usr/share/ceph/mgr/cephadm/serve.py", line 1366, in >>> _remote_connection >>> yield (conn, connr) >>> File "/usr/share/ceph/mgr/cephadm/serve.py", line 1263, in _run_cephadm >>> code, '\n'.join(err))) >>> orchestrator._interface.OrchestratorError: cephadm exited with an error >>> code: 1, stderr:Inferring config >>> >>> /var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/mon.prdrepoceph02/config >>> Traceback (most recent call last): >>> File "/usr/lib/python3.8/asyncio/streams.py", line 540, in readline >>> line = await self.readuntil(sep) >>> File "/usr/lib/python3.8/asyncio/streams.py", line 635, in readuntil >>> raise exceptions.LimitOverrunError( >>> asyncio.exceptions.LimitOverrunError: Separator is found, but chunk is >>> longer than limit >>> >>> During handling of the above exception, another exception occurred: >>> >>> Traceback (most recent call last): >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 8479, in <module> >>> main() >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 8467, in main >>> r = ctx.func(ctx) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1732, in _infer_config >>> return func(ctx) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1673, in _infer_fsid >>> return func(ctx) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1760, in _infer_image >>> return func(ctx) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1660, in _validate_fsid >>> return func(ctx) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 4753, in command_ceph_volume >>> out, err, code = call_throws(ctx, c.run_cmd()) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1460, in call_throws >>> out, err, ret = call(ctx, command, desc, verbosity, timeout, >>> **kwargs) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1442, in call >>> stdout, stderr, returncode = async_run(run_with_timeout()) >>> File "/usr/lib/python3.8/asyncio/runners.py", line 44, in run >>> return loop.run_until_complete(main) >>> File "/usr/lib/python3.8/asyncio/base_events.py", line 616, in >>> run_until_complete >>> return future.result() >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1433, in run_with_timeout >>> stdout, stderr = await asyncio.gather(tee(process.stdout), >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1415, in tee >>> async for line in reader: >>> File "/usr/lib/python3.8/asyncio/streams.py", line 738, in __anext__ >>> val = await self.readline() >>> File "/usr/lib/python3.8/asyncio/streams.py", line 549, in readline >>> raise ValueError(e.args[0]) >>> ValueError: Separator is found, but chunk is longer than limit >>> Sep 24, 2021, 1:22:45 PM [ERR] cephadm exited with an error code: 1, >>> stderr:Inferring config >>> >>> /var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/mon.prdrepoceph01/config >>> Traceback (most recent call last): >>> File "/usr/lib/python3.8/asyncio/streams.py", line 540, in readline >>> line = await self.readuntil(sep) >>> File "/usr/lib/python3.8/asyncio/streams.py", line 635, in readuntil >>> raise exceptions.LimitOverrunError( >>> asyncio.exceptions.LimitOverrunError: Separator is found, but chunk is >>> longer than limit >>> >>> During handling of the above exception, another exception occurred: >>> >>> Traceback (most recent call last): >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 8479, in <module> >>> main() >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 8467, in main >>> r = ctx.func(ctx) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1732, in _infer_config >>> return func(ctx) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1673, in _infer_fsid >>> return func(ctx) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1760, in _infer_image >>> return func(ctx) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1660, in _validate_fsid >>> return func(ctx) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 4753, in command_ceph_volume >>> out, err, code = call_throws(ctx, c.run_cmd()) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1460, in call_throws >>> out, err, ret = call(ctx, command, desc, verbosity, timeout, >>> **kwargs) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1442, in call >>> stdout, stderr, returncode = async_run(run_with_timeout()) >>> File "/usr/lib/python3.8/asyncio/runners.py", line 44, in run >>> return loop.run_until_complete(main) >>> File "/usr/lib/python3.8/asyncio/base_events.py", line 616, in >>> run_until_complete >>> return future.result() >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1433, in run_with_timeout >>> stdout, stderr = await asyncio.gather(tee(process.stdout), >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1415, in tee >>> async for line in reader: >>> File "/usr/lib/python3.8/asyncio/streams.py", line 738, in __anext__ >>> val = await self.readline() >>> File "/usr/lib/python3.8/asyncio/streams.py", line 549, in readline >>> raise ValueError(e.args[0]) >>> ValueError: Separator is found, but chunk is longer than limit >>> Traceback (most recent call last): >>> File "/usr/share/ceph/mgr/cephadm/serve.py", line 1366, in >>> _remote_connection >>> yield (conn, connr) >>> File "/usr/share/ceph/mgr/cephadm/serve.py", line 1263, in _run_cephadm >>> code, '\n'.join(err))) >>> orchestrator._interface.OrchestratorError: cephadm exited with an error >>> code: 1, stderr:Inferring config >>> >>> /var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/mon.prdrepoceph01/config >>> Traceback (most recent call last): >>> File "/usr/lib/python3.8/asyncio/streams.py", line 540, in readline >>> line = await self.readuntil(sep) >>> File "/usr/lib/python3.8/asyncio/streams.py", line 635, in readuntil >>> raise exceptions.LimitOverrunError( >>> asyncio.exceptions.LimitOverrunError: Separator is found, but chunk is >>> longer than limit >>> >>> During handling of the above exception, another exception occurred: >>> >>> Traceback (most recent call last): >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 8479, in <module> >>> main() >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 8467, in main >>> r = ctx.func(ctx) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1732, in _infer_config >>> return func(ctx) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1673, in _infer_fsid >>> return func(ctx) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1760, in _infer_image >>> return func(ctx) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1660, in _validate_fsid >>> return func(ctx) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 4753, in command_ceph_volume >>> out, err, code = call_throws(ctx, c.run_cmd()) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1460, in call_throws >>> out, err, ret = call(ctx, command, desc, verbosity, timeout, >>> **kwargs) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1442, in call >>> stdout, stderr, returncode = async_run(run_with_timeout()) >>> File "/usr/lib/python3.8/asyncio/runners.py", line 44, in run >>> return loop.run_until_complete(main) >>> File "/usr/lib/python3.8/asyncio/base_events.py", line 616, in >>> run_until_complete >>> return future.result() >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1433, in run_with_timeout >>> stdout, stderr = await asyncio.gather(tee(process.stdout), >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1415, in tee >>> async for line in reader: >>> File "/usr/lib/python3.8/asyncio/streams.py", line 738, in __anext__ >>> val = await self.readline() >>> File "/usr/lib/python3.8/asyncio/streams.py", line 549, in readline >>> raise ValueError(e.args[0]) >>> ValueError: Separator is found, but chunk is longer than limit >>> Sep 24, 2021, 1:22:45 PM [ERR] cephadm exited with an error code: 1, >>> stderr:Inferring config >>> >>> /var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/mon.prdrepoceph02/config >>> Traceback (most recent call last): >>> File "/usr/lib/python3.8/asyncio/streams.py", line 540, in readline >>> line = await self.readuntil(sep) >>> File "/usr/lib/python3.8/asyncio/streams.py", line 635, in readuntil >>> raise exceptions.LimitOverrunError( >>> asyncio.exceptions.LimitOverrunError: Separator is found, but chunk is >>> longer than limit >>> >>> During handling of the above exception, another exception occurred: >>> >>> Traceback (most recent call last): >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 8479, in <module> >>> main() >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 8467, in main >>> r = ctx.func(ctx) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1732, in _infer_config >>> return func(ctx) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1673, in _infer_fsid >>> return func(ctx) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1760, in _infer_image >>> return func(ctx) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1660, in _validate_fsid >>> return func(ctx) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 4753, in command_ceph_volume >>> out, err, code = call_throws(ctx, c.run_cmd()) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1460, in call_throws >>> out, err, ret = call(ctx, command, desc, verbosity, timeout, >>> **kwargs) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1442, in call >>> stdout, stderr, returncode = async_run(run_with_timeout()) >>> File "/usr/lib/python3.8/asyncio/runners.py", line 44, in run >>> return loop.run_until_complete(main) >>> File "/usr/lib/python3.8/asyncio/base_events.py", line 616, in >>> run_until_complete >>> return future.result() >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1433, in run_with_timeout >>> stdout, stderr = await asyncio.gather(tee(process.stdout), >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1415, in tee >>> async for line in reader: >>> File "/usr/lib/python3.8/asyncio/streams.py", line 738, in __anext__ >>> val = await self.readline() >>> File "/usr/lib/python3.8/asyncio/streams.py", line 549, in readline >>> raise ValueError(e.args[0]) >>> ValueError: Separator is found, but chunk is longer than limit >>> Traceback (most recent call last): >>> File "/usr/share/ceph/mgr/cephadm/serve.py", line 1366, in >>> _remote_connection >>> yield (conn, connr) >>> File "/usr/share/ceph/mgr/cephadm/serve.py", line 1263, in _run_cephadm >>> code, '\n'.join(err))) >>> orchestrator._interface.OrchestratorError: cephadm exited with an error >>> code: 1, stderr:Inferring config >>> >>> /var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/mon.prdrepoceph02/config >>> Traceback (most recent call last): >>> File "/usr/lib/python3.8/asyncio/streams.py", line 540, in readline >>> line = await self.readuntil(sep) >>> File "/usr/lib/python3.8/asyncio/streams.py", line 635, in readuntil >>> raise exceptions.LimitOverrunError( >>> asyncio.exceptions.LimitOverrunError: Separator is found, but chunk is >>> longer than limit >>> >>> During handling of the above exception, another exception occurred: >>> >>> Traceback (most recent call last): >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 8479, in <module> >>> main() >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 8467, in main >>> r = ctx.func(ctx) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1732, in _infer_config >>> return func(ctx) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1673, in _infer_fsid >>> return func(ctx) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1760, in _infer_image >>> return func(ctx) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1660, in _validate_fsid >>> return func(ctx) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 4753, in command_ceph_volume >>> out, err, code = call_throws(ctx, c.run_cmd()) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1460, in call_throws >>> out, err, ret = call(ctx, command, desc, verbosity, timeout, >>> **kwargs) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1442, in call >>> stdout, stderr, returncode = async_run(run_with_timeout()) >>> File "/usr/lib/python3.8/asyncio/runners.py", line 44, in run >>> return loop.run_until_complete(main) >>> File "/usr/lib/python3.8/asyncio/base_events.py", line 616, in >>> run_until_complete >>> return future.result() >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1433, in run_with_timeout >>> stdout, stderr = await asyncio.gather(tee(process.stdout), >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1415, in tee >>> async for line in reader: >>> File "/usr/lib/python3.8/asyncio/streams.py", line 738, in __anext__ >>> val = await self.readline() >>> File "/usr/lib/python3.8/asyncio/streams.py", line 549, in readline >>> raise ValueError(e.args[0]) >>> ValueError: Separator is found, but chunk is longer than limit >>> Sep 24, 2021, 1:21:13 PM [ERR] cephadm exited with an error code: 1, >>> stderr:Inferring config >>> >>> /var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/mon.prdrepoceph01/config >>> Traceback (most recent call last): >>> File "/usr/lib/python3.8/asyncio/streams.py", line 540, in readline >>> line = await self.readuntil(sep) >>> File "/usr/lib/python3.8/asyncio/streams.py", line 635, in readuntil >>> raise exceptions.LimitOverrunError( >>> asyncio.exceptions.LimitOverrunError: Separator is found, but chunk is >>> longer than limit >>> >>> During handling of the above exception, another exception occurred: >>> >>> Traceback (most recent call last): >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 8479, in <module> >>> main() >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 8467, in main >>> r = ctx.func(ctx) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1732, in _infer_config >>> return func(ctx) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1673, in _infer_fsid >>> return func(ctx) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1760, in _infer_image >>> return func(ctx) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1660, in _validate_fsid >>> return func(ctx) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 4753, in command_ceph_volume >>> out, err, code = call_throws(ctx, c.run_cmd()) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1460, in call_throws >>> out, err, ret = call(ctx, command, desc, verbosity, timeout, >>> **kwargs) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1442, in call >>> stdout, stderr, returncode = async_run(run_with_timeout()) >>> File "/usr/lib/python3.8/asyncio/runners.py", line 44, in run >>> return loop.run_until_complete(main) >>> File "/usr/lib/python3.8/asyncio/base_events.py", line 616, in >>> run_until_complete >>> return future.result() >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1433, in run_with_timeout >>> stdout, stderr = await asyncio.gather(tee(process.stdout), >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1415, in tee >>> async for line in reader: >>> File "/usr/lib/python3.8/asyncio/streams.py", line 738, in __anext__ >>> val = await self.readline() >>> File "/usr/lib/python3.8/asyncio/streams.py", line 549, in readline >>> raise ValueError(e.args[0]) >>> ValueError: Separator is found, but chunk is longer than limit >>> Traceback (most recent call last): >>> File "/usr/share/ceph/mgr/cephadm/serve.py", line 1366, in >>> _remote_connection >>> yield (conn, connr) >>> File "/usr/share/ceph/mgr/cephadm/serve.py", line 1263, in _run_cephadm >>> code, '\n'.join(err))) >>> orchestrator._interface.OrchestratorError: cephadm exited with an error >>> code: 1, stderr:Inferring config >>> >>> /var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/mon.prdrepoceph01/config >>> Traceback (most recent call last): >>> File "/usr/lib/python3.8/asyncio/streams.py", line 540, in readline >>> line = await self.readuntil(sep) >>> File "/usr/lib/python3.8/asyncio/streams.py", line 635, in readuntil >>> raise exceptions.LimitOverrunError( >>> asyncio.exceptions.LimitOverrunError: Separator is found, but chunk is >>> longer than limit >>> >>> During handling of the above exception, another exception occurred: >>> >>> Traceback (most recent call last): >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 8479, in <module> >>> main() >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 8467, in main >>> r = ctx.func(ctx) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1732, in _infer_config >>> return func(ctx) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1673, in _infer_fsid >>> return func(ctx) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1760, in _infer_image >>> return func(ctx) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1660, in _validate_fsid >>> return func(ctx) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 4753, in command_ceph_volume >>> out, err, code = call_throws(ctx, c.run_cmd()) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1460, in call_throws >>> out, err, ret = call(ctx, command, desc, verbosity, timeout, >>> **kwargs) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1442, in call >>> stdout, stderr, returncode = async_run(run_with_timeout()) >>> File "/usr/lib/python3.8/asyncio/runners.py", line 44, in run >>> return loop.run_until_complete(main) >>> File "/usr/lib/python3.8/asyncio/base_events.py", line 616, in >>> run_until_complete >>> return future.result() >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1433, in run_with_timeout >>> stdout, stderr = await asyncio.gather(tee(process.stdout), >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1415, in tee >>> async for line in reader: >>> File "/usr/lib/python3.8/asyncio/streams.py", line 738, in __anext__ >>> val = await self.readline() >>> File "/usr/lib/python3.8/asyncio/streams.py", line 549, in readline >>> raise ValueError(e.args[0]) >>> ValueError: Separator is found, but chunk is longer than limit >>> Sep 24, 2021, 1:21:13 PM [ERR] cephadm exited with an error code: 1, >>> stderr:Inferring config >>> >>> /var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/mon.prdrepoceph02/config >>> Traceback (most recent call last): >>> File "/usr/lib/python3.8/asyncio/streams.py", line 540, in readline >>> line = await self.readuntil(sep) >>> File "/usr/lib/python3.8/asyncio/streams.py", line 635, in readuntil >>> raise exceptions.LimitOverrunError( >>> asyncio.exceptions.LimitOverrunError: Separator is found, but chunk is >>> longer than limit >>> >>> During handling of the above exception, another exception occurred: >>> >>> Traceback (most recent call last): >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 8479, in <module> >>> main() >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 8467, in main >>> r = ctx.func(ctx) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1732, in _infer_config >>> return func(ctx) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1673, in _infer_fsid >>> return func(ctx) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1760, in _infer_image >>> return func(ctx) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1660, in _validate_fsid >>> return func(ctx) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 4753, in command_ceph_volume >>> out, err, code = call_throws(ctx, c.run_cmd()) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1460, in call_throws >>> out, err, ret = call(ctx, command, desc, verbosity, timeout, >>> **kwargs) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1442, in call >>> stdout, stderr, returncode = async_run(run_with_timeout()) >>> File "/usr/lib/python3.8/asyncio/runners.py", line 44, in run >>> return loop.run_until_complete(main) >>> File "/usr/lib/python3.8/asyncio/base_events.py", line 616, in >>> run_until_complete >>> return future.result() >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1433, in run_with_timeout >>> stdout, stderr = await asyncio.gather(tee(process.stdout), >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1415, in tee >>> async for line in reader: >>> File "/usr/lib/python3.8/asyncio/streams.py", line 738, in __anext__ >>> val = await self.readline() >>> File "/usr/lib/python3.8/asyncio/streams.py", line 549, in readline >>> raise ValueError(e.args[0]) >>> ValueError: Separator is found, but chunk is longer than limit >>> Traceback (most recent call last): >>> File "/usr/share/ceph/mgr/cephadm/serve.py", line 1366, in >>> _remote_connection >>> yield (conn, connr) >>> File "/usr/share/ceph/mgr/cephadm/serve.py", line 1263, in _run_cephadm >>> code, '\n'.join(err))) >>> orchestrator._interface.OrchestratorError: cephadm exited with an error >>> code: 1, stderr:Inferring config >>> >>> /var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/mon.prdrepoceph02/config >>> Traceback (most recent call last): >>> File "/usr/lib/python3.8/asyncio/streams.py", line 540, in readline >>> line = await self.readuntil(sep) >>> File "/usr/lib/python3.8/asyncio/streams.py", line 635, in readuntil >>> raise exceptions.LimitOverrunError( >>> asyncio.exceptions.LimitOverrunError: Separator is found, but chunk is >>> longer than limit >>> >>> During handling of the above exception, another exception occurred: >>> >>> Traceback (most recent call last): >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 8479, in <module> >>> main() >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 8467, in main >>> r = ctx.func(ctx) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1732, in _infer_config >>> return func(ctx) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1673, in _infer_fsid >>> return func(ctx) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1760, in _infer_image >>> return func(ctx) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1660, in _validate_fsid >>> return func(ctx) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 4753, in command_ceph_volume >>> out, err, code = call_throws(ctx, c.run_cmd()) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1460, in call_throws >>> out, err, ret = call(ctx, command, desc, verbosity, timeout, >>> **kwargs) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1442, in call >>> stdout, stderr, returncode = async_run(run_with_timeout()) >>> File "/usr/lib/python3.8/asyncio/runners.py", line 44, in run >>> return loop.run_until_complete(main) >>> File "/usr/lib/python3.8/asyncio/base_events.py", line 616, in >>> run_until_complete >>> return future.result() >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1433, in run_with_timeout >>> stdout, stderr = await asyncio.gather(tee(process.stdout), >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1415, in tee >>> async for line in reader: >>> File "/usr/lib/python3.8/asyncio/streams.py", line 738, in __anext__ >>> val = await self.readline() >>> File "/usr/lib/python3.8/asyncio/streams.py", line 549, in readline >>> raise ValueError(e.args[0]) >>> ValueError: Separator is found, but chunk is longer than limit >>> Sep 24, 2021, 1:20:00 PM [WRN] overall HEALTH_WARN failed to probe >>> daemons >>> or devices >>> Sep 24, 2021, 1:19:41 PM [ERR] cephadm exited with an error code: 1, >>> stderr:Inferring config >>> >>> /var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/mon.prdrepoceph01/config >>> Traceback (most recent call last): >>> File "/usr/lib/python3.8/asyncio/streams.py", line 540, in readline >>> line = await self.readuntil(sep) >>> File "/usr/lib/python3.8/asyncio/streams.py", line 635, in readuntil >>> raise exceptions.LimitOverrunError( >>> asyncio.exceptions.LimitOverrunError: Separator is found, but chunk is >>> longer than limit >>> >>> During handling of the above exception, another exception occurred: >>> >>> Traceback (most recent call last): >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 8479, in <module> >>> main() >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 8467, in main >>> r = ctx.func(ctx) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1732, in _infer_config >>> return func(ctx) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1673, in _infer_fsid >>> return func(ctx) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1760, in _infer_image >>> return func(ctx) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1660, in _validate_fsid >>> return func(ctx) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 4753, in command_ceph_volume >>> out, err, code = call_throws(ctx, c.run_cmd()) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1460, in call_throws >>> out, err, ret = call(ctx, command, desc, verbosity, timeout, >>> **kwargs) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1442, in call >>> stdout, stderr, returncode = async_run(run_with_timeout()) >>> File "/usr/lib/python3.8/asyncio/runners.py", line 44, in run >>> return loop.run_until_complete(main) >>> File "/usr/lib/python3.8/asyncio/base_events.py", line 616, in >>> run_until_complete >>> return future.result() >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1433, in run_with_timeout >>> stdout, stderr = await asyncio.gather(tee(process.stdout), >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1415, in tee >>> async for line in reader: >>> File "/usr/lib/python3.8/asyncio/streams.py", line 738, in __anext__ >>> val = await self.readline() >>> File "/usr/lib/python3.8/asyncio/streams.py", line 549, in readline >>> raise ValueError(e.args[0]) >>> ValueError: Separator is found, but chunk is longer than limit >>> Traceback (most recent call last): >>> File "/usr/share/ceph/mgr/cephadm/serve.py", line 1366, in >>> _remote_connection >>> yield (conn, connr) >>> File "/usr/share/ceph/mgr/cephadm/serve.py", line 1263, in _run_cephadm >>> code, '\n'.join(err))) >>> orchestrator._interface.OrchestratorError: cephadm exited with an error >>> code: 1, stderr:Inferring config >>> >>> /var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/mon.prdrepoceph01/config >>> Traceback (most recent call last): >>> File "/usr/lib/python3.8/asyncio/streams.py", line 540, in readline >>> line = await self.readuntil(sep) >>> File "/usr/lib/python3.8/asyncio/streams.py", line 635, in readuntil >>> raise exceptions.LimitOverrunError( >>> asyncio.exceptions.LimitOverrunError: Separator is found, but chunk is >>> longer than limit >>> >>> During handling of the above exception, another exception occurred: >>> >>> Traceback (most recent call last): >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 8479, in <module> >>> main() >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 8467, in main >>> r = ctx.func(ctx) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1732, in _infer_config >>> return func(ctx) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1673, in _infer_fsid >>> return func(ctx) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1760, in _infer_image >>> return func(ctx) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1660, in _validate_fsid >>> return func(ctx) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 4753, in command_ceph_volume >>> out, err, code = call_throws(ctx, c.run_cmd()) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1460, in call_throws >>> out, err, ret = call(ctx, command, desc, verbosity, timeout, >>> **kwargs) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1442, in call >>> stdout, stderr, returncode = async_run(run_with_timeout()) >>> File "/usr/lib/python3.8/asyncio/runners.py", line 44, in run >>> return loop.run_until_complete(main) >>> File "/usr/lib/python3.8/asyncio/base_events.py", line 616, in >>> run_until_complete >>> return future.result() >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1433, in run_with_timeout >>> stdout, stderr = await asyncio.gather(tee(process.stdout), >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1415, in tee >>> async for line in reader: >>> File "/usr/lib/python3.8/asyncio/streams.py", line 738, in __anext__ >>> val = await self.readline() >>> File "/usr/lib/python3.8/asyncio/streams.py", line 549, in readline >>> raise ValueError(e.args[0]) >>> ValueError: Separator is found, but chunk is longer than limit >>> Sep 24, 2021, 1:19:40 PM [ERR] cephadm exited with an error code: 1, >>> stderr:Inferring config >>> >>> /var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/mon.prdrepoceph02/config >>> Traceback (most recent call last): >>> File "/usr/lib/python3.8/asyncio/streams.py", line 540, in readline >>> line = await self.readuntil(sep) >>> File "/usr/lib/python3.8/asyncio/streams.py", line 635, in readuntil >>> raise exceptions.LimitOverrunError( >>> asyncio.exceptions.LimitOverrunError: Separator is found, but chunk is >>> longer than limit >>> >>> During handling of the above exception, another exception occurred: >>> >>> Traceback (most recent call last): >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 8479, in <module> >>> main() >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 8467, in main >>> r = ctx.func(ctx) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1732, in _infer_config >>> return func(ctx) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1673, in _infer_fsid >>> return func(ctx) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1760, in _infer_image >>> return func(ctx) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1660, in _validate_fsid >>> return func(ctx) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 4753, in command_ceph_volume >>> out, err, code = call_throws(ctx, c.run_cmd()) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1460, in call_throws >>> out, err, ret = call(ctx, command, desc, verbosity, timeout, >>> **kwargs) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1442, in call >>> stdout, stderr, returncode = async_run(run_with_timeout()) >>> File "/usr/lib/python3.8/asyncio/runners.py", line 44, in run >>> return loop.run_until_complete(main) >>> File "/usr/lib/python3.8/asyncio/base_events.py", line 616, in >>> run_until_complete >>> return future.result() >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1433, in run_with_timeout >>> stdout, stderr = await asyncio.gather(tee(process.stdout), >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1415, in tee >>> async for line in reader: >>> File "/usr/lib/python3.8/asyncio/streams.py", line 738, in __anext__ >>> val = await self.readline() >>> File "/usr/lib/python3.8/asyncio/streams.py", line 549, in readline >>> raise ValueError(e.args[0]) >>> ValueError: Separator is found, but chunk is longer than limit >>> Traceback (most recent call last): >>> File "/usr/share/ceph/mgr/cephadm/serve.py", line 1366, in >>> _remote_connection >>> yield (conn, connr) >>> File "/usr/share/ceph/mgr/cephadm/serve.py", line 1263, in _run_cephadm >>> code, '\n'.join(err))) >>> orchestrator._interface.OrchestratorError: cephadm exited with an error >>> code: 1, stderr:Inferring config >>> >>> /var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/mon.prdrepoceph02/config >>> Traceback (most recent call last): >>> File "/usr/lib/python3.8/asyncio/streams.py", line 540, in readline >>> line = await self.readuntil(sep) >>> File "/usr/lib/python3.8/asyncio/streams.py", line 635, in readuntil >>> raise exceptions.LimitOverrunError( >>> asyncio.exceptions.LimitOverrunError: Separator is found, but chunk is >>> longer than limit >>> >>> During handling of the above exception, another exception occurred: >>> >>> Traceback (most recent call last): >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 8479, in <module> >>> main() >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 8467, in main >>> r = ctx.func(ctx) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1732, in _infer_config >>> return func(ctx) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1673, in _infer_fsid >>> return func(ctx) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1760, in _infer_image >>> return func(ctx) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1660, in _validate_fsid >>> return func(ctx) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 4753, in command_ceph_volume >>> out, err, code = call_throws(ctx, c.run_cmd()) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1460, in call_throws >>> out, err, ret = call(ctx, command, desc, verbosity, timeout, >>> **kwargs) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1442, in call >>> stdout, stderr, returncode = async_run(run_with_timeout()) >>> File "/usr/lib/python3.8/asyncio/runners.py", line 44, in run >>> return loop.run_until_complete(main) >>> File "/usr/lib/python3.8/asyncio/base_events.py", line 616, in >>> run_until_complete >>> return future.result() >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1433, in run_with_timeout >>> stdout, stderr = await asyncio.gather(tee(process.stdout), >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1415, in tee >>> async for line in reader: >>> File "/usr/lib/python3.8/asyncio/streams.py", line 738, in __anext__ >>> val = await self.readline() >>> File "/usr/lib/python3.8/asyncio/streams.py", line 549, in readline >>> raise ValueError(e.args[0]) >>> ValueError: Separator is found, but chunk is longer than limit >>> Sep 24, 2021, 1:18:09 PM [ERR] cephadm exited with an error code: 1, >>> stderr:Inferring config >>> >>> /var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/mon.prdrepoceph02/config >>> Traceback (most recent call last): >>> File "/usr/lib/python3.8/asyncio/streams.py", line 540, in readline >>> line = await self.readuntil(sep) >>> File "/usr/lib/python3.8/asyncio/streams.py", line 635, in readuntil >>> raise exceptions.LimitOverrunError( >>> asyncio.exceptions.LimitOverrunError: Separator is found, but chunk is >>> longer than limit >>> >>> During handling of the above exception, another exception occurred: >>> >>> Traceback (most recent call last): >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 8479, in <module> >>> main() >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 8467, in main >>> r = ctx.func(ctx) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1732, in _infer_config >>> return func(ctx) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1673, in _infer_fsid >>> return func(ctx) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1760, in _infer_image >>> return func(ctx) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1660, in _validate_fsid >>> return func(ctx) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 4753, in command_ceph_volume >>> out, err, code = call_throws(ctx, c.run_cmd()) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1460, in call_throws >>> out, err, ret = call(ctx, command, desc, verbosity, timeout, >>> **kwargs) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1442, in call >>> stdout, stderr, returncode = async_run(run_with_timeout()) >>> File "/usr/lib/python3.8/asyncio/runners.py", line 44, in run >>> return loop.run_until_complete(main) >>> File "/usr/lib/python3.8/asyncio/base_events.py", line 616, in >>> run_until_complete >>> return future.result() >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1433, in run_with_timeout >>> stdout, stderr = await asyncio.gather(tee(process.stdout), >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1415, in tee >>> async for line in reader: >>> File "/usr/lib/python3.8/asyncio/streams.py", line 738, in __anext__ >>> val = await self.readline() >>> File "/usr/lib/python3.8/asyncio/streams.py", line 549, in readline >>> raise ValueError(e.args[0]) >>> ValueError: Separator is found, but chunk is longer than limit >>> Traceback (most recent call last): >>> File "/usr/share/ceph/mgr/cephadm/serve.py", line 1366, in >>> _remote_connection >>> yield (conn, connr) >>> File "/usr/share/ceph/mgr/cephadm/serve.py", line 1263, in _run_cephadm >>> code, '\n'.join(err))) >>> orchestrator._interface.OrchestratorError: cephadm exited with an error >>> code: 1, stderr:Inferring config >>> >>> /var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/mon.prdrepoceph02/config >>> Traceback (most recent call last): >>> File "/usr/lib/python3.8/asyncio/streams.py", line 540, in readline >>> line = await self.readuntil(sep) >>> File "/usr/lib/python3.8/asyncio/streams.py", line 635, in readuntil >>> raise exceptions.LimitOverrunError( >>> asyncio.exceptions.LimitOverrunError: Separator is found, but chunk is >>> longer than limit >>> >>> During handling of the above exception, another exception occurred: >>> >>> Traceback (most recent call last): >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 8479, in <module> >>> main() >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 8467, in main >>> r = ctx.func(ctx) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1732, in _infer_config >>> return func(ctx) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1673, in _infer_fsid >>> return func(ctx) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1760, in _infer_image >>> return func(ctx) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1660, in _validate_fsid >>> return func(ctx) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 4753, in command_ceph_volume >>> out, err, code = call_throws(ctx, c.run_cmd()) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1460, in call_throws >>> out, err, ret = call(ctx, command, desc, verbosity, timeout, >>> **kwargs) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1442, in call >>> stdout, stderr, returncode = async_run(run_with_timeout()) >>> File "/usr/lib/python3.8/asyncio/runners.py", line 44, in run >>> return loop.run_until_complete(main) >>> File "/usr/lib/python3.8/asyncio/base_events.py", line 616, in >>> run_until_complete >>> return future.result() >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1433, in run_with_timeout >>> stdout, stderr = await asyncio.gather(tee(process.stdout), >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1415, in tee >>> async for line in reader: >>> File "/usr/lib/python3.8/asyncio/streams.py", line 738, in __anext__ >>> val = await self.readline() >>> File "/usr/lib/python3.8/asyncio/streams.py", line 549, in readline >>> raise ValueError(e.args[0]) >>> ValueError: Separator is found, but chunk is longer than limit >>> Sep 24, 2021, 1:18:09 PM [ERR] cephadm exited with an error code: 1, >>> stderr:Inferring config >>> >>> /var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/mon.prdrepoceph01/config >>> Traceback (most recent call last): >>> File "/usr/lib/python3.8/asyncio/streams.py", line 540, in readline >>> line = await self.readuntil(sep) >>> File "/usr/lib/python3.8/asyncio/streams.py", line 635, in readuntil >>> raise exceptions.LimitOverrunError( >>> asyncio.exceptions.LimitOverrunError: Separator is found, but chunk is >>> longer than limit >>> >>> During handling of the above exception, another exception occurred: >>> >>> Traceback (most recent call last): >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 8479, in <module> >>> main() >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 8467, in main >>> r = ctx.func(ctx) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1732, in _infer_config >>> return func(ctx) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1673, in _infer_fsid >>> return func(ctx) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1760, in _infer_image >>> return func(ctx) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1660, in _validate_fsid >>> return func(ctx) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 4753, in command_ceph_volume >>> out, err, code = call_throws(ctx, c.run_cmd()) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1460, in call_throws >>> out, err, ret = call(ctx, command, desc, verbosity, timeout, >>> **kwargs) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1442, in call >>> stdout, stderr, returncode = async_run(run_with_timeout()) >>> File "/usr/lib/python3.8/asyncio/runners.py", line 44, in run >>> return loop.run_until_complete(main) >>> File "/usr/lib/python3.8/asyncio/base_events.py", line 616, in >>> run_until_complete >>> return future.result() >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1433, in run_with_timeout >>> stdout, stderr = await asyncio.gather(tee(process.stdout), >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1415, in tee >>> async for line in reader: >>> File "/usr/lib/python3.8/asyncio/streams.py", line 738, in __anext__ >>> val = await self.readline() >>> File "/usr/lib/python3.8/asyncio/streams.py", line 549, in readline >>> raise ValueError(e.args[0]) >>> ValueError: Separator is found, but chunk is longer than limit >>> Traceback (most recent call last): >>> File "/usr/share/ceph/mgr/cephadm/serve.py", line 1366, in >>> _remote_connection >>> yield (conn, connr) >>> File "/usr/share/ceph/mgr/cephadm/serve.py", line 1263, in _run_cephadm >>> code, '\n'.join(err))) >>> orchestrator._interface.OrchestratorError: cephadm exited with an error >>> code: 1, stderr:Inferring config >>> >>> /var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/mon.prdrepoceph01/config >>> Traceback (most recent call last): >>> File "/usr/lib/python3.8/asyncio/streams.py", line 540, in readline >>> line = await self.readuntil(sep) >>> File "/usr/lib/python3.8/asyncio/streams.py", line 635, in readuntil >>> raise exceptions.LimitOverrunError( >>> asyncio.exceptions.LimitOverrunError: Separator is found, but chunk is >>> longer than limit >>> >>> During handling of the above exception, another exception occurred: >>> >>> Traceback (most recent call last): >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 8479, in <module> >>> main() >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 8467, in main >>> r = ctx.func(ctx) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1732, in _infer_config >>> return func(ctx) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1673, in _infer_fsid >>> return func(ctx) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1760, in _infer_image >>> return func(ctx) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1660, in _validate_fsid >>> return func(ctx) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 4753, in command_ceph_volume >>> out, err, code = call_throws(ctx, c.run_cmd()) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1460, in call_throws >>> out, err, ret = call(ctx, command, desc, verbosity, timeout, >>> **kwargs) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1442, in call >>> stdout, stderr, returncode = async_run(run_with_timeout()) >>> File "/usr/lib/python3.8/asyncio/runners.py", line 44, in run >>> return loop.run_until_complete(main) >>> File "/usr/lib/python3.8/asyncio/base_events.py", line 616, in >>> run_until_complete >>> return future.result() >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1433, in run_with_timeout >>> stdout, stderr = await asyncio.gather(tee(process.stdout), >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1415, in tee >>> async for line in reader: >>> File "/usr/lib/python3.8/asyncio/streams.py", line 738, in __anext__ >>> val = await self.readline() >>> File "/usr/lib/python3.8/asyncio/streams.py", line 549, in readline >>> raise ValueError(e.args[0]) >>> ValueError: Separator is found, but chunk is longer than limit >>> Sep 24, 2021, 1:16:37 PM [ERR] cephadm exited with an error code: 1, >>> stderr:Inferring config >>> >>> /var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/mon.prdrepoceph01/config >>> Traceback (most recent call last): >>> File "/usr/lib/python3.8/asyncio/streams.py", line 540, in readline >>> line = await self.readuntil(sep) >>> File "/usr/lib/python3.8/asyncio/streams.py", line 635, in readuntil >>> raise exceptions.LimitOverrunError( >>> asyncio.exceptions.LimitOverrunError: Separator is found, but chunk is >>> longer than limit >>> >>> During handling of the above exception, another exception occurred: >>> >>> Traceback (most recent call last): >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 8479, in <module> >>> main() >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 8467, in main >>> r = ctx.func(ctx) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1732, in _infer_config >>> return func(ctx) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1673, in _infer_fsid >>> return func(ctx) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1760, in _infer_image >>> return func(ctx) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1660, in _validate_fsid >>> return func(ctx) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 4753, in command_ceph_volume >>> out, err, code = call_throws(ctx, c.run_cmd()) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1460, in call_throws >>> out, err, ret = call(ctx, command, desc, verbosity, timeout, >>> **kwargs) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1442, in call >>> stdout, stderr, returncode = async_run(run_with_timeout()) >>> File "/usr/lib/python3.8/asyncio/runners.py", line 44, in run >>> return loop.run_until_complete(main) >>> File "/usr/lib/python3.8/asyncio/base_events.py", line 616, in >>> run_until_complete >>> return future.result() >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1433, in run_with_timeout >>> stdout, stderr = await asyncio.gather(tee(process.stdout), >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1415, in tee >>> async for line in reader: >>> File "/usr/lib/python3.8/asyncio/streams.py", line 738, in __anext__ >>> val = await self.readline() >>> File "/usr/lib/python3.8/asyncio/streams.py", line 549, in readline >>> raise ValueError(e.args[0]) >>> ValueError: Separator is found, but chunk is longer than limit >>> Traceback (most recent call last): >>> File "/usr/share/ceph/mgr/cephadm/serve.py", line 1366, in >>> _remote_connection >>> yield (conn, connr) >>> File "/usr/share/ceph/mgr/cephadm/serve.py", line 1263, in _run_cephadm >>> code, '\n'.join(err))) >>> orchestrator._interface.OrchestratorError: cephadm exited with an error >>> code: 1, stderr:Inferring config >>> >>> /var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/mon.prdrepoceph01/config >>> Traceback (most recent call last): >>> File "/usr/lib/python3.8/asyncio/streams.py", line 540, in readline >>> line = await self.readuntil(sep) >>> File "/usr/lib/python3.8/asyncio/streams.py", line 635, in readuntil >>> raise exceptions.LimitOverrunError( >>> asyncio.exceptions.LimitOverrunError: Separator is found, but chunk is >>> longer than limit >>> >>> During handling of the above exception, another exception occurred: >>> >>> Traceback (most recent call last): >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 8479, in <module> >>> main() >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 8467, in main >>> r = ctx.func(ctx) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1732, in _infer_config >>> return func(ctx) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1673, in _infer_fsid >>> return func(ctx) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1760, in _infer_image >>> return func(ctx) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1660, in _validate_fsid >>> return func(ctx) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 4753, in command_ceph_volume >>> out, err, code = call_throws(ctx, c.run_cmd()) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1460, in call_throws >>> out, err, ret = call(ctx, command, desc, verbosity, timeout, >>> **kwargs) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1442, in call >>> stdout, stderr, returncode = async_run(run_with_timeout()) >>> File "/usr/lib/python3.8/asyncio/runners.py", line 44, in run >>> return loop.run_until_complete(main) >>> File "/usr/lib/python3.8/asyncio/base_events.py", line 616, in >>> run_until_complete >>> return future.result() >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1433, in run_with_timeout >>> stdout, stderr = await asyncio.gather(tee(process.stdout), >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1415, in tee >>> async for line in reader: >>> File "/usr/lib/python3.8/asyncio/streams.py", line 738, in __anext__ >>> val = await self.readline() >>> File "/usr/lib/python3.8/asyncio/streams.py", line 549, in readline >>> raise ValueError(e.args[0]) >>> ValueError: Separator is found, but chunk is longer than limit >>> Sep 24, 2021, 1:16:37 PM [ERR] cephadm exited with an error code: 1, >>> stderr:Inferring config >>> >>> /var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/mon.prdrepoceph02/config >>> Traceback (most recent call last): >>> File "/usr/lib/python3.8/asyncio/streams.py", line 540, in readline >>> line = await self.readuntil(sep) >>> File "/usr/lib/python3.8/asyncio/streams.py", line 635, in readuntil >>> raise exceptions.LimitOverrunError( >>> asyncio.exceptions.LimitOverrunError: Separator is found, but chunk is >>> longer than limit >>> >>> During handling of the above exception, another exception occurred: >>> >>> Traceback (most recent call last): >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 8479, in <module> >>> main() >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 8467, in main >>> r = ctx.func(ctx) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1732, in _infer_config >>> return func(ctx) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1673, in _infer_fsid >>> return func(ctx) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1760, in _infer_image >>> return func(ctx) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1660, in _validate_fsid >>> return func(ctx) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 4753, in command_ceph_volume >>> out, err, code = call_throws(ctx, c.run_cmd()) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1460, in call_throws >>> out, err, ret = call(ctx, command, desc, verbosity, timeout, >>> **kwargs) >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1442, in call >>> stdout, stderr, returncode = async_run(run_with_timeout()) >>> File "/usr/lib/python3.8/asyncio/runners.py", line 44, in run >>> return loop.run_until_complete(main) >>> File "/usr/lib/python3.8/asyncio/base_events.py", line 616, in >>> run_until_complete >>> return future.result() >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1433, in run_with_timeout >>> stdout, stderr = await asyncio.gather(tee(process.stdout), >>> File >>> >>> "/var/lib/ceph/91f80988-1cb1-11ec-867e-59d6b2eb9d9d/cephadm.f46dc95b01feeedb28941a48e2f1d0abb51139ca828de11150ea7122a8e3549c", >>> line 1415, in tee >>> async for line in reader: >>> File "/usr/lib/python3.8/asyncio/streams.py", line 738, in __anext__ >>> val = await self.readline() >>> File "/usr/lib/python3.8/asyncio/streams.py", line 549, in readline >>> raise ValueError(e.args[0]) >>> ValueError: Separator is found, but chunk is longer than limit >>> Traceback (most recent call last): >>> File "/usr/share/ceph/mgr/cephadm/serve.py", line 1366, in >>> _remote >> >> _______________________________________________ ceph-users mailing list -- ceph-users@xxxxxxx To unsubscribe send an email to ceph-users-leave@xxxxxxx
 |