Hi all,
in various Rook operated Ceph clusters I have seen OSDs going into a CrashLoop due to
debug 2020-12-16T13:19:25.500+0000 7fc4c3f13f40 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1608124765507105, "job": 1, "event": "recovery_started", "log_files": [1400, 1402]}
debug 2020-12-16T13:19:25.500+0000 7fc4c3f13f40 4 rocksdb: [db/db_impl_open.cc:583] Recovering log #1400 mode 0
debug 2020-12-16T13:19:27.724+0000 7fc4c3f13f40 1 bluefs _allocate failed to allocate 0x43ce43d on bdev 1, free 0x2e50000; fallback to bdev 2
debug 2020-12-16T13:19:27.724+0000 7fc4c3f13f40 1 bluefs _allocate unable to allocate 0x43ce43d on bdev 2, free 0xffffffffffffffff; fallback to slow device expander
debug 2020-12-16T13:19:27.724+0000 7fc4c3f13f40 -1 bluestore(/var/lib/ceph/osd/ceph-1) allocate_bluefs_freespace failed to allocate on 0x3d1b0000 min_size 0x43d0000 > allocated total 0x300000 bluefs_shared_alloc_size 0x10000 allocated 0x300000 available 0x b019c8000
debug 2020-12-16T13:19:27.724+0000 7fc4c3f13f40 -1 bluefs _allocate failed to expand slow device to fit +0x43ce43d
debug 2020-12-16T13:19:27.724+0000 7fc4c3f13f40 -1 bluefs _flush_range allocated: 0x0 offset: 0x0 length: 0x43ce43d
/home/jenkins-build/build/workspace/ceph-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos8/DIST/centos8/MACHINE_SIZE/gigantic/release/15.2.6/rpm/el8/BUILD/ceph-15.2.6/src/os/bluestore/BlueFS.cc: In function 'int BlueFS::_flush_range(BlueFS::FileWriter*, uint64_t, uint64_t)' thread 7fc4c3f13f40 time 2020-12-16T13:19:27.731533+0000
/home/jenkins-build/build/workspace/ceph-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos8/DIST/centos8/MACHINE_SIZE/gigantic/release/15.2.6/rpm/el8/BUILD/ceph-15.2.6/src/os/bluestore/BlueFS.cc: 2721: ceph_abort_msg("bluefs enospc")
The OSD is not really full:
# ceph-bluestore-tool bluefs-bdev-expand --path /var/lib/ceph/osd/ceph-1
inferring bluefs devices from bluestore path
1 : device size 0x18ffc00000 : own 0x[bffe10000~fffe0000] = 0xfffe0000 : using 0xfd190000(4.0 GiB) : bluestore has 0x82b360000(33 GiB) available
Expanding DB/WAL...
Expanding the underlying block device by just 1 gig followed by "ceph-bluestore-tool bluefs-bdev-expand" and "ceph-bluestore-tool repair" resolves the situation. In general, larger OSDs seem to reduce the likeliness for this issue.
Ceph version is v15.2.6.
Is this a known bug?
Ceph report and logs are attached.
Thanks for your help
Stephan
report 1067799142
{
"cluster_fingerprint": "f71b5f78-64c9-4e77-b523-d2de5e5f1a01",
"version": "15.2.6",
"commit": "cb8c61a60551b72614257d632a574d420064c17a",
"timestamp": "2020-12-16T14:46:26.701139+0000",
"tag": "",
"health": {
"status": "HEALTH_WARN",
"checks": {
"MDS_SLOW_METADATA_IO": {
"severity": "HEALTH_WARN",
"summary": {
"message": "2 MDSs report slow metadata IOs",
"count": 2
},
"detail": [
{
"message": "mdscephfs-a(mds.0): 100+ slow metadata IOs are blocked > 30 secs, oldest blocked for 9181 secs"
},
{
"message": "mdscephfs-b(mds.0): 1 slow metadata IOs are blocked > 30 secs, oldest blocked for 9051 secs"
}
],
"muted": false
},
"MDS_SLOW_REQUEST": {
"severity": "HEALTH_WARN",
"summary": {
"message": "1 MDSs report slow requests",
"count": 1
},
"detail": [
{
"message": "mdscephfs-a(mds.0): 129 slow requests are blocked > 30 secs"
}
],
"muted": false
},
"MON_DOWN": {
"severity": "HEALTH_WARN",
"summary": {
"message": "1/3 mons down, quorum f,k",
"count": 1
},
"detail": [
{
"message": "mon.m (rank 2) addr [v2:172.31.163.187:3300/0,v1:172.31.163.187:6789/0] is down (out of quorum)"
}
],
"muted": false
},
"PG_AVAILABILITY": {
"severity": "HEALTH_WARN",
"summary": {
"message": "Reduced data availability: 25 pgs inactive, 25 pgs down",
"count": 50
},
"detail": [
{
"message": "pg 1.9 is down, acting [3,0]"
},
{
"message": "pg 1.a is down, acting [3,2]"
},
{
"message": "pg 1.d is down, acting [0,3]"
},
{
"message": "pg 1.e is down, acting [3,2]"
},
{
"message": "pg 1.11 is down, acting [3,0]"
},
{
"message": "pg 1.17 is down, acting [3,0]"
},
{
"message": "pg 1.1c is down, acting [3,0]"
},
{
"message": "pg 2.2 is down, acting [3,0]"
},
{
"message": "pg 2.7 is down, acting [3,2]"
},
{
"message": "pg 2.9 is down, acting [3,2]"
},
{
"message": "pg 2.d is down, acting [3,0]"
},
{
"message": "pg 2.10 is down, acting [3,2]"
},
{
"message": "pg 2.12 is down, acting [3,0]"
},
{
"message": "pg 2.1b is down, acting [3,0]"
},
{
"message": "pg 2.1c is down, acting [3,2]"
},
{
"message": "pg 2.1e is down, acting [3,0]"
},
{
"message": "pg 3.0 is down, acting [0,3]"
},
{
"message": "pg 3.2 is down, acting [2,3]"
},
{
"message": "pg 3.3 is down, acting [3,0]"
},
{
"message": "pg 3.8 is down, acting [3,0]"
},
{
"message": "pg 3.a is down, acting [3,0]"
},
{
"message": "pg 3.e is down, acting [3,2]"
},
{
"message": "pg 3.12 is down, acting [2,3]"
},
{
"message": "pg 3.13 is down, acting [3,2]"
},
{
"message": "pg 3.1b is down, acting [3,0]"
}
],
"muted": false
},
"SLOW_OPS": {
"severity": "HEALTH_WARN",
"summary": {
"message": "602 slow ops, oldest one blocked for 8123 sec, daemons [osd.0,osd.2,mon.f] have slow ops.",
"count": 1
},
"detail": [],
"muted": false
}
},
"mutes": []
},
"monmap_first_committed": 1,
"monmap_last_committed": 24,
"monmap": {
"epoch": 24,
"fsid": "39941002-eb2d-498a-bab4-04961fb33a42",
"modified": "2020-12-16T09:55:13.010421Z",
"created": "2019-11-21T16:05:56.597779Z",
"min_mon_release": 15,
"min_mon_release_name": "octopus",
"features": {
"persistent": [
"kraken",
"luminous",
"mimic",
"osdmap-prune",
"nautilus",
"octopus"
],
"optional": []
},
"mons": [
{
"rank": 0,
"name": "f",
"public_addrs": {
"addrvec": [
{
"type": "v2",
"addr": "172.31.36.158:3300",
"nonce": 0
},
{
"type": "v1",
"addr": "172.31.36.158:6789",
"nonce": 0
}
]
},
"addr": "172.31.36.158:6789/0",
"public_addr": "172.31.36.158:6789/0",
"priority": 0,
"weight": 0
},
{
"rank": 1,
"name": "k",
"public_addrs": {
"addrvec": [
{
"type": "v2",
"addr": "172.31.225.8:3300",
"nonce": 0
},
{
"type": "v1",
"addr": "172.31.225.8:6789",
"nonce": 0
}
]
},
"addr": "172.31.225.8:6789/0",
"public_addr": "172.31.225.8:6789/0",
"priority": 0,
"weight": 0
},
{
"rank": 2,
"name": "m",
"public_addrs": {
"addrvec": [
{
"type": "v2",
"addr": "172.31.163.187:3300",
"nonce": 0
},
{
"type": "v1",
"addr": "172.31.163.187:6789",
"nonce": 0
}
]
},
"addr": "172.31.163.187:6789/0",
"public_addr": "172.31.163.187:6789/0",
"priority": 0,
"weight": 0
}
]
},
"quorum": [
0,
1
],
"osdmap": {
"epoch": 9019,
"fsid": "39941002-eb2d-498a-bab4-04961fb33a42",
"created": "2019-11-21T16:05:57.640347+0000",
"modified": "2020-12-16T14:42:15.846697+0000",
"last_up_change": "2020-12-16T14:10:00.415027+0000",
"last_in_change": "2020-12-16T14:10:00.415027+0000",
"flags": "sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit",
"flags_num": 5799936,
"flags_set": [
"pglog_hardlimit",
"purged_snapdirs",
"recovery_deletes",
"sortbitwise"
],
"crush_version": 14,
"full_ratio": 0.94999998807907104,
"backfillfull_ratio": 0.89999997615814209,
"nearfull_ratio": 0.85000002384185791,
"cluster_snapshot": "",
"pool_max": 4,
"max_osd": 4,
"require_min_compat_client": "luminous",
"min_compat_client": "jewel",
"require_osd_release": "octopus",
"pools": [
{
"pool": 1,
"pool_name": "rbdpool",
"create_time": "2019-11-21T16:07:32.986975+0000",
"flags": 8193,
"flags_names": "hashpspool,selfmanaged_snaps",
"type": 1,
"size": 2,
"min_size": 1,
"crush_rule": 1,
"object_hash": 2,
"pg_autoscale_mode": "on",
"pg_num": 32,
"pg_placement_num": 32,
"pg_placement_num_target": 32,
"pg_num_target": 32,
"pg_num_pending": 32,
"last_pg_merge_meta": {
"source_pgid": "0.0",
"ready_epoch": 0,
"last_epoch_started": 0,
"last_epoch_clean": 0,
"source_version": "0'0",
"target_version": "0'0"
},
"last_change": "8657",
"last_force_op_resend": "0",
"last_force_op_resend_prenautilus": "0",
"last_force_op_resend_preluminous": "1613",
"auid": 0,
"snap_mode": "selfmanaged",
"snap_seq": 3,
"snap_epoch": 19,
"pool_snaps": [],
"removed_snaps": "[]",
"quota_max_bytes": 0,
"quota_max_objects": 0,
"tiers": [],
"tier_of": -1,
"read_tier": -1,
"write_tier": -1,
"cache_mode": "none",
"target_max_bytes": 0,
"target_max_objects": 0,
"cache_target_dirty_ratio_micro": 400000,
"cache_target_dirty_high_ratio_micro": 600000,
"cache_target_full_ratio_micro": 800000,
"cache_min_flush_age": 0,
"cache_min_evict_age": 0,
"erasure_code_profile": "",
"hit_set_params": {
"type": "none"
},
"hit_set_period": 0,
"hit_set_count": 0,
"use_gmt_hitset": true,
"min_read_recency_for_promote": 0,
"min_write_recency_for_promote": 0,
"hit_set_grade_decay_rate": 0,
"hit_set_search_last_n": 0,
"grade_table": [],
"stripe_width": 0,
"expected_num_objects": 0,
"fast_read": false,
"options": {},
"application_metadata": {
"rbd": {}
}
},
{
"pool": 2,
"pool_name": "cephfs-metadata",
"create_time": "2019-12-02T09:55:42.819120+0000",
"flags": 1,
"flags_names": "hashpspool",
"type": 1,
"size": 2,
"min_size": 1,
"crush_rule": 2,
"object_hash": 2,
"pg_autoscale_mode": "on",
"pg_num": 32,
"pg_placement_num": 32,
"pg_placement_num_target": 32,
"pg_num_target": 32,
"pg_num_pending": 32,
"last_pg_merge_meta": {
"source_pgid": "0.0",
"ready_epoch": 0,
"last_epoch_started": 0,
"last_epoch_clean": 0,
"source_version": "0'0",
"target_version": "0'0"
},
"last_change": "8658",
"last_force_op_resend": "0",
"last_force_op_resend_prenautilus": "0",
"last_force_op_resend_preluminous": "3081",
"auid": 0,
"snap_mode": "selfmanaged",
"snap_seq": 0,
"snap_epoch": 0,
"pool_snaps": [],
"removed_snaps": "[]",
"quota_max_bytes": 0,
"quota_max_objects": 0,
"tiers": [],
"tier_of": -1,
"read_tier": -1,
"write_tier": -1,
"cache_mode": "none",
"target_max_bytes": 0,
"target_max_objects": 0,
"cache_target_dirty_ratio_micro": 400000,
"cache_target_dirty_high_ratio_micro": 600000,
"cache_target_full_ratio_micro": 800000,
"cache_min_flush_age": 0,
"cache_min_evict_age": 0,
"erasure_code_profile": "",
"hit_set_params": {
"type": "none"
},
"hit_set_period": 0,
"hit_set_count": 0,
"use_gmt_hitset": true,
"min_read_recency_for_promote": 0,
"min_write_recency_for_promote": 0,
"hit_set_grade_decay_rate": 0,
"hit_set_search_last_n": 0,
"grade_table": [],
"stripe_width": 0,
"expected_num_objects": 0,
"fast_read": false,
"options": {
"pg_autoscale_bias": 4,
"pg_num_min": 16,
"recovery_priority": 5
},
"application_metadata": {
"cephfs": {
"metadata": "cephfs"
}
}
},
{
"pool": 3,
"pool_name": "cephfs-data0",
"create_time": "2019-12-02T09:55:49.464990+0000",
"flags": 1,
"flags_names": "hashpspool",
"type": 1,
"size": 2,
"min_size": 1,
"crush_rule": 3,
"object_hash": 2,
"pg_autoscale_mode": "on",
"pg_num": 32,
"pg_placement_num": 32,
"pg_placement_num_target": 32,
"pg_num_target": 32,
"pg_num_pending": 32,
"last_pg_merge_meta": {
"source_pgid": "3.20",
"ready_epoch": 7759,
"last_epoch_started": 7758,
"last_epoch_clean": 7758,
"source_version": "7750'2574810",
"target_version": "7750'2576879"
},
"last_change": "8660",
"last_force_op_resend": "0",
"last_force_op_resend_prenautilus": "7759",
"last_force_op_resend_preluminous": "7757",
"auid": 0,
"snap_mode": "selfmanaged",
"snap_seq": 0,
"snap_epoch": 0,
"pool_snaps": [],
"removed_snaps": "[]",
"quota_max_bytes": 0,
"quota_max_objects": 0,
"tiers": [],
"tier_of": -1,
"read_tier": -1,
"write_tier": -1,
"cache_mode": "none",
"target_max_bytes": 0,
"target_max_objects": 0,
"cache_target_dirty_ratio_micro": 400000,
"cache_target_dirty_high_ratio_micro": 600000,
"cache_target_full_ratio_micro": 800000,
"cache_min_flush_age": 0,
"cache_min_evict_age": 0,
"erasure_code_profile": "",
"hit_set_params": {
"type": "none"
},
"hit_set_period": 0,
"hit_set_count": 0,
"use_gmt_hitset": true,
"min_read_recency_for_promote": 0,
"min_write_recency_for_promote": 0,
"hit_set_grade_decay_rate": 0,
"hit_set_search_last_n": 0,
"grade_table": [],
"stripe_width": 0,
"expected_num_objects": 0,
"fast_read": false,
"options": {},
"application_metadata": {
"cephfs": {}
}
},
{
"pool": 4,
"pool_name": "device_health_metrics",
"create_time": "2020-04-20T15:49:40.015449+0000",
"flags": 1,
"flags_names": "hashpspool",
"type": 1,
"size": 2,
"min_size": 1,
"crush_rule": 0,
"object_hash": 2,
"pg_autoscale_mode": "on",
"pg_num": 1,
"pg_placement_num": 1,
"pg_placement_num_target": 1,
"pg_num_target": 1,
"pg_num_pending": 1,
"last_pg_merge_meta": {
"source_pgid": "0.0",
"ready_epoch": 0,
"last_epoch_started": 0,
"last_epoch_clean": 0,
"source_version": "0'0",
"target_version": "0'0"
},
"last_change": "3328",
"last_force_op_resend": "0",
"last_force_op_resend_prenautilus": "0",
"last_force_op_resend_preluminous": "0",
"auid": 0,
"snap_mode": "selfmanaged",
"snap_seq": 0,
"snap_epoch": 0,
"pool_snaps": [],
"removed_snaps": "[]",
"quota_max_bytes": 0,
"quota_max_objects": 0,
"tiers": [],
"tier_of": -1,
"read_tier": -1,
"write_tier": -1,
"cache_mode": "none",
"target_max_bytes": 0,
"target_max_objects": 0,
"cache_target_dirty_ratio_micro": 400000,
"cache_target_dirty_high_ratio_micro": 600000,
"cache_target_full_ratio_micro": 800000,
"cache_min_flush_age": 0,
"cache_min_evict_age": 0,
"erasure_code_profile": "",
"hit_set_params": {
"type": "none"
},
"hit_set_period": 0,
"hit_set_count": 0,
"use_gmt_hitset": true,
"min_read_recency_for_promote": 0,
"min_write_recency_for_promote": 0,
"hit_set_grade_decay_rate": 0,
"hit_set_search_last_n": 0,
"grade_table": [],
"stripe_width": 0,
"expected_num_objects": 0,
"fast_read": false,
"options": {
"pg_num_min": 1
},
"application_metadata": {
"mgr_devicehealth": {}
}
}
],
"osds": [
{
"osd": 0,
"uuid": "03afe048-c50a-4417-a989-d64d83d49981",
"up": 1,
"in": 1,
"weight": 1,
"primary_affinity": 1,
"last_clean_begin": 8536,
"last_clean_end": 8564,
"up_from": 8567,
"up_thru": 9015,
"down_at": 8565,
"lost_at": 0,
"public_addrs": {
"addrvec": [
{
"type": "v2",
"addr": "172.30.33.165:6800",
"nonce": 1609012285
},
{
"type": "v1",
"addr": "172.30.33.165:6801",
"nonce": 1609012285
}
]
},
"cluster_addrs": {
"addrvec": [
{
"type": "v2",
"addr": "172.30.33.165:6802",
"nonce": 1609012285
},
{
"type": "v1",
"addr": "172.30.33.165:6803",
"nonce": 1609012285
}
]
},
"heartbeat_back_addrs": {
"addrvec": [
{
"type": "v2",
"addr": "172.30.33.165:6806",
"nonce": 1609012285
},
{
"type": "v1",
"addr": "172.30.33.165:6807",
"nonce": 1609012285
}
]
},
"heartbeat_front_addrs": {
"addrvec": [
{
"type": "v2",
"addr": "172.30.33.165:6804",
"nonce": 1609012285
},
{
"type": "v1",
"addr": "172.30.33.165:6805",
"nonce": 1609012285
}
]
},
"public_addr": "172.30.33.165:6801/1609012285",
"cluster_addr": "172.30.33.165:6803/1609012285",
"heartbeat_back_addr": "172.30.33.165:6807/1609012285",
"heartbeat_front_addr": "172.30.33.165:6805/1609012285",
"state": [
"exists",
"up"
]
},
{
"osd": 1,
"uuid": "6de39020-523e-4797-8d7f-b5f88a5af71b",
"up": 0,
"in": 0,
"weight": 0,
"primary_affinity": 1,
"last_clean_begin": 8766,
"last_clean_end": 8797,
"up_from": 8800,
"up_thru": 8848,
"down_at": 8850,
"lost_at": 0,
"public_addrs": {
"addrvec": [
{
"type": "v2",
"addr": "172.30.161.39:6800",
"nonce": 1103809042
},
{
"type": "v1",
"addr": "172.30.161.39:6801",
"nonce": 1103809042
}
]
},
"cluster_addrs": {
"addrvec": [
{
"type": "v2",
"addr": "172.30.161.39:6802",
"nonce": 1103809042
},
{
"type": "v1",
"addr": "172.30.161.39:6803",
"nonce": 1103809042
}
]
},
"heartbeat_back_addrs": {
"addrvec": [
{
"type": "v2",
"addr": "172.30.161.39:6806",
"nonce": 1103809042
},
{
"type": "v1",
"addr": "172.30.161.39:6807",
"nonce": 1103809042
}
]
},
"heartbeat_front_addrs": {
"addrvec": [
{
"type": "v2",
"addr": "172.30.161.39:6804",
"nonce": 1103809042
},
{
"type": "v1",
"addr": "172.30.161.39:6805",
"nonce": 1103809042
}
]
},
"public_addr": "172.30.161.39:6801/1103809042",
"cluster_addr": "172.30.161.39:6803/1103809042",
"heartbeat_back_addr": "172.30.161.39:6807/1103809042",
"heartbeat_front_addr": "172.30.161.39:6805/1103809042",
"state": [
"autoout",
"exists"
]
},
{
"osd": 2,
"uuid": "ea6894a5-2dd2-489a-85cf-37ff9eabffec",
"up": 1,
"in": 1,
"weight": 1,
"primary_affinity": 1,
"last_clean_begin": 8542,
"last_clean_end": 8570,
"up_from": 8578,
"up_thru": 9008,
"down_at": 8577,
"lost_at": 0,
"public_addrs": {
"addrvec": [
{
"type": "v2",
"addr": "172.30.201.2:6800",
"nonce": 3460969467
},
{
"type": "v1",
"addr": "172.30.201.2:6801",
"nonce": 3460969467
}
]
},
"cluster_addrs": {
"addrvec": [
{
"type": "v2",
"addr": "172.30.201.2:6802",
"nonce": 3460969467
},
{
"type": "v1",
"addr": "172.30.201.2:6803",
"nonce": 3460969467
}
]
},
"heartbeat_back_addrs": {
"addrvec": [
{
"type": "v2",
"addr": "172.30.201.2:6806",
"nonce": 3460969467
},
{
"type": "v1",
"addr": "172.30.201.2:6807",
"nonce": 3460969467
}
]
},
"heartbeat_front_addrs": {
"addrvec": [
{
"type": "v2",
"addr": "172.30.201.2:6804",
"nonce": 3460969467
},
{
"type": "v1",
"addr": "172.30.201.2:6805",
"nonce": 3460969467
}
]
},
"public_addr": "172.30.201.2:6801/3460969467",
"cluster_addr": "172.30.201.2:6803/3460969467",
"heartbeat_back_addr": "172.30.201.2:6807/3460969467",
"heartbeat_front_addr": "172.30.201.2:6805/3460969467",
"state": [
"exists",
"up"
]
},
{
"osd": 3,
"uuid": "bd68fabb-371b-4624-ada1-1e532dd9b6c2",
"up": 1,
"in": 1,
"weight": 1,
"primary_affinity": 1,
"last_clean_begin": 8928,
"last_clean_end": 8962,
"up_from": 9007,
"up_thru": 9008,
"down_at": 8963,
"lost_at": 0,
"public_addrs": {
"addrvec": [
{
"type": "v2",
"addr": "172.30.107.5:6800",
"nonce": 1472157578
},
{
"type": "v1",
"addr": "172.30.107.5:6801",
"nonce": 1472157578
}
]
},
"cluster_addrs": {
"addrvec": [
{
"type": "v2",
"addr": "172.30.107.5:6802",
"nonce": 1472157578
},
{
"type": "v1",
"addr": "172.30.107.5:6803",
"nonce": 1472157578
}
]
},
"heartbeat_back_addrs": {
"addrvec": [
{
"type": "v2",
"addr": "172.30.107.5:6806",
"nonce": 1472157578
},
{
"type": "v1",
"addr": "172.30.107.5:6807",
"nonce": 1472157578
}
]
},
"heartbeat_front_addrs": {
"addrvec": [
{
"type": "v2",
"addr": "172.30.107.5:6804",
"nonce": 1472157578
},
{
"type": "v1",
"addr": "172.30.107.5:6805",
"nonce": 1472157578
}
]
},
"public_addr": "172.30.107.5:6801/1472157578",
"cluster_addr": "172.30.107.5:6803/1472157578",
"heartbeat_back_addr": "172.30.107.5:6807/1472157578",
"heartbeat_front_addr": "172.30.107.5:6805/1472157578",
"state": [
"exists",
"up"
]
}
],
"osd_xinfo": [
{
"osd": 0,
"down_stamp": "2020-11-30T07:43:58.886656+0000",
"laggy_probability": 0,
"laggy_interval": 0,
"features": 4540138292836696063,
"old_weight": 0,
"last_purged_snaps_scrub": "2020-12-16T07:44:08.318462+0000",
"dead_epoch": 8565
},
{
"osd": 1,
"down_stamp": "2020-11-30T07:43:59.893241+0000",
"laggy_probability": 0,
"laggy_interval": 0,
"features": 4540138292836696063,
"old_weight": 65536,
"last_purged_snaps_scrub": "2020-12-16T07:44:07.193968+0000",
"dead_epoch": 8850
},
{
"osd": 2,
"down_stamp": "2020-12-02T20:50:38.561773+0000",
"laggy_probability": 0,
"laggy_interval": 0,
"features": 4540138292836696063,
"old_weight": 0,
"last_purged_snaps_scrub": "2020-12-16T07:44:08.708138+0000",
"dead_epoch": 8534
},
{
"osd": 3,
"down_stamp": "2020-12-16T13:41:17.899806+0000",
"laggy_probability": 0,
"laggy_interval": 0,
"features": 4540138292836696063,
"old_weight": 0,
"last_purged_snaps_scrub": "2020-12-16T07:44:15.719293+0000",
"dead_epoch": 8569
}
],
"pg_upmap": [],
"pg_upmap_items": [],
"pg_temp": [
{
"pgid": "1.b",
"osds": [
2,
0
]
},
{
"pgid": "3.4",
"osds": [
0,
2
]
},
{
"pgid": "3.9",
"osds": [
2,
0
]
},
{
"pgid": "3.14",
"osds": [
2,
0
]
}
],
"primary_temp": [],
"blacklist": {
"172.30.33.182:6800/3419146026": "2020-12-17T14:42:15.846634+0000",
"172.30.33.182:0/2229539348": "2020-12-17T14:42:15.846634+0000",
"172.30.33.182:6800/2559671592": "2020-12-17T14:34:11.404103+0000",
"172.30.33.182:6801/770553481": "2020-12-17T14:24:35.084957+0000",
"172.30.33.182:0/679977084": "2020-12-17T14:24:35.084957+0000",
"172.30.33.182:6800/341100026": "2020-12-17T14:13:29.464284+0000",
"172.30.33.182:0/650646842": "2020-12-17T14:13:29.464284+0000",
"172.30.33.182:6800/2767151552": "2020-12-17T13:58:47.701698+0000",
"172.30.33.182:0/2325382332": "2020-12-17T13:58:47.701698+0000",
"172.30.33.182:6801/3419146026": "2020-12-17T14:42:15.846634+0000",
"172.30.33.182:6801/706685806": "2020-12-17T13:51:42.212621+0000",
"172.30.33.182:6800/706685806": "2020-12-17T13:51:42.212621+0000",
"172.30.33.182:0/1830539806": "2020-12-17T13:51:42.212621+0000",
"172.30.33.182:0/1606846417": "2020-12-17T07:40:38.144070+0000",
"172.30.33.182:0/2727871142": "2020-12-17T13:07:09.779879+0000",
"172.30.33.182:6801/2767151552": "2020-12-17T13:58:47.701698+0000",
"172.30.33.182:6801/3066679254": "2020-12-17T13:27:16.110761+0000",
"172.30.33.182:0/3953563216": "2020-12-17T14:05:33.402279+0000",
"172.30.33.182:6801/2724987499": "2020-12-17T07:40:38.144070+0000",
"172.30.33.182:0/2983455564": "2020-12-17T07:40:38.144070+0000",
"172.30.33.182:6800/331145036": "2020-12-17T09:55:38.892627+0000",
"172.30.33.182:6800/2724987499": "2020-12-17T07:40:38.144070+0000",
"172.30.33.182:6801/331145036": "2020-12-17T09:55:38.892627+0000",
"172.30.33.182:0/1264159784": "2020-12-17T12:57:29.225073+0000",
"172.30.33.182:0/308452087": "2020-12-17T13:45:42.266017+0000",
"172.30.33.182:6800/2014607231": "2020-12-17T14:05:33.402279+0000",
"172.30.33.182:6801/2770464765": "2020-12-17T12:24:24.311640+0000",
"172.30.33.182:0/1008950751": "2020-12-17T07:40:38.144070+0000",
"172.30.33.182:0/2181242141": "2020-12-17T09:55:38.892627+0000",
"172.30.33.182:0/4292547686": "2020-12-17T12:24:24.311640+0000",
"172.30.33.182:6801/1968208880": "2020-12-17T13:07:09.779879+0000",
"172.30.33.182:0/552173175": "2020-12-17T12:24:24.311640+0000",
"172.30.33.182:6800/2770464765": "2020-12-17T12:24:24.311640+0000",
"172.30.33.182:6801/341100026": "2020-12-17T14:13:29.464284+0000",
"172.30.33.182:6800/2003928880": "2020-12-17T13:16:59.956451+0000",
"172.30.33.182:6801/2014607231": "2020-12-17T14:05:33.402279+0000",
"172.30.33.182:6801/2519773534": "2020-12-17T13:37:38.338856+0000",
"172.30.33.182:6801/1599274580": "2020-12-17T12:57:29.225073+0000",
"172.30.33.182:0/2532175852": "2020-12-17T12:26:13.870499+0000",
"172.30.33.182:6800/770553481": "2020-12-17T14:24:35.084957+0000",
"172.30.33.182:6800/90505859": "2020-12-17T12:26:13.870499+0000",
"172.30.33.182:6800/1599274580": "2020-12-17T12:57:29.225073+0000",
"172.30.33.182:0/1994564048": "2020-12-17T12:24:24.311640+0000",
"172.30.33.182:6800/1968208880": "2020-12-17T13:07:09.779879+0000",
"172.30.33.182:0/1850821225": "2020-12-17T14:34:11.404103+0000",
"172.30.33.182:6801/2003928880": "2020-12-17T13:16:59.956451+0000",
"172.30.33.182:0/3673069545": "2020-12-17T13:16:59.956451+0000",
"172.30.33.182:0/2941856325": "2020-12-17T13:27:16.110761+0000",
"172.30.33.182:0/1537744287": "2020-12-17T09:55:38.892627+0000",
"172.30.33.182:6801/90505859": "2020-12-17T12:26:13.870499+0000",
"172.30.33.182:6800/2519773534": "2020-12-17T13:37:38.338856+0000",
"172.30.33.182:6800/3066679254": "2020-12-17T13:27:16.110761+0000",
"172.30.33.182:6801/1037859658": "2020-12-17T13:45:42.266017+0000",
"172.30.33.182:6801/2559671592": "2020-12-17T14:34:11.404103+0000",
"172.30.33.182:0/2583385031": "2020-12-17T13:37:38.338856+0000",
"172.30.33.182:6800/1037859658": "2020-12-17T13:45:42.266017+0000"
},
"erasure_code_profiles": {
"default": {
"k": "2",
"m": "1",
"plugin": "jerasure",
"technique": "reed_sol_van"
}
},
"removed_snaps_queue": [],
"new_removed_snaps": [],
"new_purged_snaps": [],
"crush_node_flags": {},
"device_class_flags": {}
},
"osd_metadata": [
{
"id": 0,
"arch": "x86_64",
"back_addr": "[v2:172.30.33.165:6802/1609012285,v1:172.30.33.165:6803/1609012285]",
"back_iface": "eth0",
"bluefs": "1",
"bluefs_dedicated_db": "0",
"bluefs_dedicated_wal": "0",
"bluefs_single_shared_device": "1",
"bluestore_bdev_access_mode": "blk",
"bluestore_bdev_block_size": "4096",
"bluestore_bdev_dev_node": "/dev/dm-0",
"bluestore_bdev_devices": "sdb",
"bluestore_bdev_driver": "KernelDevice",
"bluestore_bdev_partition_path": "/dev/dm-0",
"bluestore_bdev_rotational": "0",
"bluestore_bdev_size": "106300440576",
"bluestore_bdev_support_discard": "0",
"bluestore_bdev_type": "ssd",
"ceph_release": "octopus",
"ceph_version": "ceph version 15.2.6 (cb8c61a60551b72614257d632a574d420064c17a) octopus (stable)",
"ceph_version_short": "15.2.6",
"container_hostname": "rook-ceph-osd-0-cbff5dcd5-pwkqn",
"container_image": "repo.pruef.local:50443/ceph/ceph:v15.2.6-20201119",
"cpu": "Intel(R) Xeon(R) CPU E5-2620 v4 @ 2.10GHz",
"default_device_class": "ssd",
"device_ids": "",
"device_paths": "sdb=/dev/disk/by-path/pci-0000:03:00.0-scsi-0:0:1:0",
"devices": "sdb",
"distro": "centos",
"distro_description": "CentOS Linux 8 (Core)",
"distro_version": "8",
"front_addr": "[v2:172.30.33.165:6800/1609012285,v1:172.30.33.165:6801/1609012285]",
"front_iface": "eth0",
"hb_back_addr": "[v2:172.30.33.165:6806/1609012285,v1:172.30.33.165:6807/1609012285]",
"hb_front_addr": "[v2:172.30.33.165:6804/1609012285,v1:172.30.33.165:6805/1609012285]",
"hostname": "pruef-bbs-worker2",
"journal_rotational": "0",
"kernel_description": "#46~18.04.1-Ubuntu SMP Fri Jul 10 07:21:24 UTC 2020",
"kernel_version": "5.4.0-42-generic",
"mem_cgroup_limit": "2147483648",
"mem_swap_kb": "0",
"mem_total_kb": "28758928",
"network_numa_unknown_ifaces": "eth0",
"objectstore_numa_unknown_devices": "sdb",
"os": "Linux",
"osd_data": "/var/lib/ceph/osd/ceph-0",
"osd_objectstore": "bluestore",
"osdspec_affinity": "",
"pod_name": "rook-ceph-osd-0-cbff5dcd5-pwkqn",
"pod_namespace": "rook-ceph",
"rotational": "0"
},
{
"id": 1,
"arch": "x86_64",
"back_addr": "[v2:172.30.161.39:6802/1103809042,v1:172.30.161.39:6803/1103809042]",
"back_iface": "eth0",
"bluefs": "1",
"bluefs_dedicated_db": "0",
"bluefs_dedicated_wal": "0",
"bluefs_single_shared_device": "1",
"bluestore_bdev_access_mode": "blk",
"bluestore_bdev_block_size": "4096",
"bluestore_bdev_dev_node": "/dev/dm-0",
"bluestore_bdev_devices": "sdb",
"bluestore_bdev_driver": "KernelDevice",
"bluestore_bdev_partition_path": "/dev/dm-0",
"bluestore_bdev_rotational": "0",
"bluestore_bdev_size": "107369988096",
"bluestore_bdev_support_discard": "0",
"bluestore_bdev_type": "ssd",
"ceph_release": "octopus",
"ceph_version": "ceph version 15.2.6 (cb8c61a60551b72614257d632a574d420064c17a) octopus (stable)",
"ceph_version_short": "15.2.6",
"container_hostname": "rook-ceph-osd-1-797f9d8b95-z46wj",
"container_image": "repo.pruef.local:50443/ceph/ceph:v15.2.6-20201119",
"cpu": "Intel(R) Xeon(R) CPU E5-2620 v4 @ 2.10GHz",
"default_device_class": "ssd",
"device_ids": "",
"device_paths": "sdb=/dev/disk/by-path/pci-0000:03:00.0-scsi-0:0:1:0",
"devices": "sdb",
"distro": "centos",
"distro_description": "CentOS Linux 8 (Core)",
"distro_version": "8",
"front_addr": "[v2:172.30.161.39:6800/1103809042,v1:172.30.161.39:6801/1103809042]",
"front_iface": "eth0",
"hb_back_addr": "[v2:172.30.161.39:6806/1103809042,v1:172.30.161.39:6807/1103809042]",
"hb_front_addr": "[v2:172.30.161.39:6804/1103809042,v1:172.30.161.39:6805/1103809042]",
"hostname": "pruef-bbs-worker4",
"journal_rotational": "0",
"kernel_description": "#46~18.04.1-Ubuntu SMP Fri Jul 10 07:21:24 UTC 2020",
"kernel_version": "5.4.0-42-generic",
"mem_cgroup_limit": "2147483648",
"mem_swap_kb": "0",
"mem_total_kb": "28758928",
"network_numa_unknown_ifaces": "eth0",
"objectstore_numa_unknown_devices": "sdb",
"os": "Linux",
"osd_data": "/var/lib/ceph/osd/ceph-1",
"osd_objectstore": "bluestore",
"osdspec_affinity": "",
"pod_name": "rook-ceph-osd-1-797f9d8b95-z46wj",
"pod_namespace": "rook-ceph",
"rotational": "0"
},
{
"id": 2,
"arch": "x86_64",
"back_addr": "[v2:172.30.201.2:6802/3460969467,v1:172.30.201.2:6803/3460969467]",
"back_iface": "eth0",
"bluefs": "1",
"bluefs_dedicated_db": "0",
"bluefs_dedicated_wal": "0",
"bluefs_single_shared_device": "1",
"bluestore_bdev_access_mode": "blk",
"bluestore_bdev_block_size": "4096",
"bluestore_bdev_dev_node": "/dev/dm-0",
"bluestore_bdev_devices": "sdb",
"bluestore_bdev_driver": "KernelDevice",
"bluestore_bdev_partition_path": "/dev/dm-0",
"bluestore_bdev_rotational": "0",
"bluestore_bdev_size": "106300440576",
"bluestore_bdev_support_discard": "0",
"bluestore_bdev_type": "ssd",
"ceph_release": "octopus",
"ceph_version": "ceph version 15.2.6 (cb8c61a60551b72614257d632a574d420064c17a) octopus (stable)",
"ceph_version_short": "15.2.6",
"container_hostname": "rook-ceph-osd-2-68746c555b-dqmnj",
"container_image": "repo.pruef.local:50443/ceph/ceph:v15.2.6-20201119",
"cpu": "Intel(R) Xeon(R) CPU E5-2620 v4 @ 2.10GHz",
"default_device_class": "ssd",
"device_ids": "",
"device_paths": "sdb=/dev/disk/by-path/pci-0000:03:00.0-scsi-0:0:1:0",
"devices": "sdb",
"distro": "centos",
"distro_description": "CentOS Linux 8 (Core)",
"distro_version": "8",
"front_addr": "[v2:172.30.201.2:6800/3460969467,v1:172.30.201.2:6801/3460969467]",
"front_iface": "eth0",
"hb_back_addr": "[v2:172.30.201.2:6806/3460969467,v1:172.30.201.2:6807/3460969467]",
"hb_front_addr": "[v2:172.30.201.2:6804/3460969467,v1:172.30.201.2:6805/3460969467]",
"hostname": "pruef-bbs-worker3",
"journal_rotational": "0",
"kernel_description": "#46~18.04.1-Ubuntu SMP Fri Jul 10 07:21:24 UTC 2020",
"kernel_version": "5.4.0-42-generic",
"mem_cgroup_limit": "2147483648",
"mem_swap_kb": "0",
"mem_total_kb": "28758928",
"network_numa_unknown_ifaces": "eth0",
"objectstore_numa_unknown_devices": "sdb",
"os": "Linux",
"osd_data": "/var/lib/ceph/osd/ceph-2",
"osd_objectstore": "bluestore",
"osdspec_affinity": "",
"pod_name": "rook-ceph-osd-2-68746c555b-dqmnj",
"pod_namespace": "rook-ceph",
"rotational": "0"
},
{
"id": 3,
"arch": "x86_64",
"back_addr": "[v2:172.30.107.5:6802/1472157578,v1:172.30.107.5:6803/1472157578]",
"back_iface": "eth0",
"bluefs": "1",
"bluefs_dedicated_db": "0",
"bluefs_dedicated_wal": "0",
"bluefs_single_shared_device": "1",
"bluestore_bdev_access_mode": "blk",
"bluestore_bdev_block_size": "4096",
"bluestore_bdev_dev_node": "/dev/dm-0",
"bluestore_bdev_devices": "sdb",
"bluestore_bdev_driver": "KernelDevice",
"bluestore_bdev_partition_path": "/dev/dm-0",
"bluestore_bdev_rotational": "0",
"bluestore_bdev_size": "106300440576",
"bluestore_bdev_support_discard": "0",
"bluestore_bdev_type": "ssd",
"ceph_release": "octopus",
"ceph_version": "ceph version 15.2.6 (cb8c61a60551b72614257d632a574d420064c17a) octopus (stable)",
"ceph_version_short": "15.2.6",
"container_hostname": "rook-ceph-osd-3-6d4c68c766-tjm8t",
"container_image": "repo.pruef.local:50443/ceph/ceph:v15.2.6-20201119",
"cpu": "AMD Opteron(tm) Processor 6378",
"default_device_class": "ssd",
"device_ids": "",
"device_paths": "sdb=/dev/disk/by-path/pci-0000:03:00.0-scsi-0:0:1:0",
"devices": "sdb",
"distro": "centos",
"distro_description": "CentOS Linux 8 (Core)",
"distro_version": "8",
"front_addr": "[v2:172.30.107.5:6800/1472157578,v1:172.30.107.5:6801/1472157578]",
"front_iface": "eth0",
"hb_back_addr": "[v2:172.30.107.5:6806/1472157578,v1:172.30.107.5:6807/1472157578]",
"hb_front_addr": "[v2:172.30.107.5:6804/1472157578,v1:172.30.107.5:6805/1472157578]",
"hostname": "pruef-bbs-worker1",
"journal_rotational": "0",
"kernel_description": "#46~18.04.1-Ubuntu SMP Fri Jul 10 07:21:24 UTC 2020",
"kernel_version": "5.4.0-42-generic",
"mem_cgroup_limit": "2147483648",
"mem_swap_kb": "0",
"mem_total_kb": "32887688",
"network_numa_unknown_ifaces": "eth0",
"objectstore_numa_unknown_devices": "sdb",
"os": "Linux",
"osd_data": "/var/lib/ceph/osd/ceph-3",
"osd_objectstore": "bluestore",
"osdspec_affinity": "",
"pod_name": "rook-ceph-osd-3-6d4c68c766-tjm8t",
"pod_namespace": "rook-ceph",
"rotational": "0"
}
],
"osdmap_clean_epochs": {
"min_last_epoch_clean": 8838,
"last_epoch_clean": {
"per_pool": [
{
"poolid": 1,
"floor": 8838
},
{
"poolid": 2,
"floor": 8838
},
{
"poolid": 3,
"floor": 8838
},
{
"poolid": 4,
"floor": 8844
}
]
},
"osd_epochs": [
{
"id": 0,
"epoch": 9019
},
{
"id": 1,
"epoch": 8844
},
{
"id": 2,
"epoch": 9019
},
{
"id": 3,
"epoch": 9018
}
]
},
"osdmap_first_committed": 8513,
"osdmap_last_committed": 9019,
"crushmap": {
"devices": [
{
"id": 0,
"name": "osd.0",
"class": "ssd"
},
{
"id": 1,
"name": "osd.1",
"class": "ssd"
},
{
"id": 2,
"name": "osd.2",
"class": "ssd"
},
{
"id": 3,
"name": "osd.3",
"class": "ssd"
}
],
"types": [
{
"type_id": 0,
"name": "osd"
},
{
"type_id": 1,
"name": "host"
},
{
"type_id": 2,
"name": "chassis"
},
{
"type_id": 3,
"name": "rack"
},
{
"type_id": 4,
"name": "row"
},
{
"type_id": 5,
"name": "pdu"
},
{
"type_id": 6,
"name": "pod"
},
{
"type_id": 7,
"name": "room"
},
{
"type_id": 8,
"name": "datacenter"
},
{
"type_id": 9,
"name": "zone"
},
{
"type_id": 10,
"name": "region"
},
{
"type_id": 11,
"name": "root"
}
],
"buckets": [
{
"id": -1,
"name": "default",
"type_id": 11,
"type_name": "root",
"weight": 15819,
"alg": "straw2",
"hash": "rjenkins1",
"items": [
{
"id": -4,
"weight": 6278,
"pos": 0
},
{
"id": -8,
"weight": 9541,
"pos": 1
}
]
},
{
"id": -2,
"name": "default~ssd",
"type_id": 11,
"type_name": "root",
"weight": 15819,
"alg": "straw2",
"hash": "rjenkins1",
"items": [
{
"id": -6,
"weight": 6278,
"pos": 0
},
{
"id": -10,
"weight": 9541,
"pos": 1
}
]
},
{
"id": -3,
"name": "pruef-bbs-worker2",
"type_id": 1,
"type_name": "host",
"weight": 3139,
"alg": "straw2",
"hash": "rjenkins1",
"items": [
{
"id": 0,
"weight": 3139,
"pos": 0
}
]
},
{
"id": -4,
"name": "dc1",
"type_id": 9,
"type_name": "zone",
"weight": 6278,
"alg": "straw2",
"hash": "rjenkins1",
"items": [
{
"id": -3,
"weight": 3139,
"pos": 0
},
{
"id": -13,
"weight": 3139,
"pos": 1
}
]
},
{
"id": -5,
"name": "pruef-bbs-worker2~ssd",
"type_id": 1,
"type_name": "host",
"weight": 3139,
"alg": "straw2",
"hash": "rjenkins1",
"items": [
{
"id": 0,
"weight": 3139,
"pos": 0
}
]
},
{
"id": -6,
"name": "dc1~ssd",
"type_id": 9,
"type_name": "zone",
"weight": 6278,
"alg": "straw2",
"hash": "rjenkins1",
"items": [
{
"id": -5,
"weight": 3139,
"pos": 0
},
{
"id": -14,
"weight": 3139,
"pos": 1
}
]
},
{
"id": -7,
"name": "pruef-bbs-worker4",
"type_id": 1,
"type_name": "host",
"weight": 6402,
"alg": "straw2",
"hash": "rjenkins1",
"items": [
{
"id": 1,
"weight": 6402,
"pos": 0
}
]
},
{
"id": -8,
"name": "dc2",
"type_id": 9,
"type_name": "zone",
"weight": 9541,
"alg": "straw2",
"hash": "rjenkins1",
"items": [
{
"id": -7,
"weight": 6402,
"pos": 0
},
{
"id": -11,
"weight": 3139,
"pos": 1
}
]
},
{
"id": -9,
"name": "pruef-bbs-worker4~ssd",
"type_id": 1,
"type_name": "host",
"weight": 6402,
"alg": "straw2",
"hash": "rjenkins1",
"items": [
{
"id": 1,
"weight": 6402,
"pos": 0
}
]
},
{
"id": -10,
"name": "dc2~ssd",
"type_id": 9,
"type_name": "zone",
"weight": 9541,
"alg": "straw2",
"hash": "rjenkins1",
"items": [
{
"id": -9,
"weight": 6402,
"pos": 0
},
{
"id": -12,
"weight": 3139,
"pos": 1
}
]
},
{
"id": -11,
"name": "pruef-bbs-worker3",
"type_id": 1,
"type_name": "host",
"weight": 3139,
"alg": "straw2",
"hash": "rjenkins1",
"items": [
{
"id": 2,
"weight": 3139,
"pos": 0
}
]
},
{
"id": -12,
"name": "pruef-bbs-worker3~ssd",
"type_id": 1,
"type_name": "host",
"weight": 3139,
"alg": "straw2",
"hash": "rjenkins1",
"items": [
{
"id": 2,
"weight": 3139,
"pos": 0
}
]
},
{
"id": -13,
"name": "pruef-bbs-worker1",
"type_id": 1,
"type_name": "host",
"weight": 3139,
"alg": "straw2",
"hash": "rjenkins1",
"items": [
{
"id": 3,
"weight": 3139,
"pos": 0
}
]
},
{
"id": -14,
"name": "pruef-bbs-worker1~ssd",
"type_id": 1,
"type_name": "host",
"weight": 3139,
"alg": "straw2",
"hash": "rjenkins1",
"items": [
{
"id": 3,
"weight": 3139,
"pos": 0
}
]
}
],
"rules": [
{
"rule_id": 0,
"rule_name": "replicated_rule",
"ruleset": 0,
"type": 1,
"min_size": 1,
"max_size": 10,
"steps": [
{
"op": "take",
"item": -1,
"item_name": "default"
},
{
"op": "chooseleaf_firstn",
"num": 0,
"type": "host"
},
{
"op": "emit"
}
]
},
{
"rule_id": 1,
"rule_name": "rbdpool",
"ruleset": 1,
"type": 1,
"min_size": 1,
"max_size": 10,
"steps": [
{
"op": "take",
"item": -1,
"item_name": "default"
},
{
"op": "chooseleaf_firstn",
"num": 0,
"type": "host"
},
{
"op": "emit"
}
]
},
{
"rule_id": 2,
"rule_name": "cephfs-metadata",
"ruleset": 2,
"type": 1,
"min_size": 1,
"max_size": 10,
"steps": [
{
"op": "take",
"item": -2,
"item_name": "default~ssd"
},
{
"op": "chooseleaf_firstn",
"num": 0,
"type": "host"
},
{
"op": "emit"
}
]
},
{
"rule_id": 3,
"rule_name": "cephfs-data0",
"ruleset": 3,
"type": 1,
"min_size": 1,
"max_size": 10,
"steps": [
{
"op": "take",
"item": -1,
"item_name": "default"
},
{
"op": "chooseleaf_firstn",
"num": 0,
"type": "host"
},
{
"op": "emit"
}
]
}
],
"tunables": {
"choose_local_tries": 0,
"choose_local_fallback_tries": 0,
"choose_total_tries": 50,
"chooseleaf_descend_once": 1,
"chooseleaf_vary_r": 1,
"chooseleaf_stable": 1,
"straw_calc_version": 1,
"allowed_bucket_algs": 54,
"profile": "jewel",
"optimal_tunables": 1,
"legacy_tunables": 0,
"minimum_required_version": "jewel",
"require_feature_tunables": 1,
"require_feature_tunables2": 1,
"has_v2_rules": 0,
"require_feature_tunables3": 1,
"has_v3_rules": 0,
"has_v4_buckets": 1,
"require_feature_tunables5": 1,
"has_v5_rules": 0
},
"choose_args": {}
},
"fsmap": {
"epoch": 143755,
"default_fscid": 1,
"compat": {
"compat": {},
"ro_compat": {},
"incompat": {
"feature_1": "base v0.20",
"feature_2": "client writeable ranges",
"feature_3": "default file layouts on dirs",
"feature_4": "dir inode in separate object",
"feature_5": "mds uses versioned encoding",
"feature_6": "dirfrag is stored in omap",
"feature_8": "no anchor table",
"feature_9": "file layout v2",
"feature_10": "snaprealm v2"
}
},
"feature_flags": {
"enable_multiple": false,
"ever_enabled_multiple": false
},
"standbys": [],
"filesystems": [
{
"mdsmap": {
"epoch": 143755,
"flags": 50,
"ever_allowed_features": 32,
"explicitly_allowed_features": 32,
"created": "2019-12-02T09:55:55.279019+0000",
"modified": "2020-12-16T14:46:25.570842+0000",
"tableserver": 0,
"root": 0,
"session_timeout": 60,
"session_autoclose": 300,
"min_compat_client": "0 (unknown)",
"max_file_size": 1099511627776,
"last_failure": 0,
"last_failure_osd_epoch": 0,
"compat": {
"compat": {},
"ro_compat": {},
"incompat": {
"feature_1": "base v0.20",
"feature_2": "client writeable ranges",
"feature_3": "default file layouts on dirs",
"feature_4": "dir inode in separate object",
"feature_5": "mds uses versioned encoding",
"feature_6": "dirfrag is stored in omap",
"feature_8": "no anchor table",
"feature_9": "file layout v2",
"feature_10": "snaprealm v2"
}
},
"max_mds": 1,
"in": [
0
],
"up": {
"mds_0": 14004638
},
"failed": [],
"damaged": [],
"stopped": [],
"info": {
"gid_14004638": {
"gid": 14004638,
"name": "cephfs-a",
"rank": 0,
"incarnation": 140943,
"state": "up:active",
"state_seq": 296914,
"addr": "172.30.201.31:6801/1916468523",
"addrs": {
"addrvec": [
{
"type": "v2",
"addr": "172.30.201.31:6800",
"nonce": 1916468523
},
{
"type": "v1",
"addr": "172.30.201.31:6801",
"nonce": 1916468523
}
]
},
"join_fscid": -1,
"export_targets": [],
"features": 4540138292836696063,
"flags": 0
},
"gid_14646337": {
"gid": 14646337,
"name": "cephfs-b",
"rank": 0,
"incarnation": 0,
"state": "up:standby-replay",
"state_seq": 25537,
"addr": "172.30.161.21:6801/175446176",
"addrs": {
"addrvec": [
{
"type": "v2",
"addr": "172.30.161.21:6800",
"nonce": 175446176
},
{
"type": "v1",
"addr": "172.30.161.21:6801",
"nonce": 175446176
}
]
},
"join_fscid": -1,
"export_targets": [],
"features": 4540138292836696063,
"flags": 0
}
},
"data_pools": [
3
],
"metadata_pool": 2,
"enabled": true,
"fs_name": "cephfs",
"balancer": "",
"standby_count_wanted": 1
},
"id": 1
}
]
},
"mdsmap_first_committed": 143050,
"mdsmap_last_committed": 143755,
"auth": {
"first_committed": 11251,
"last_committed": 11371,
"num_secrets": 19
},
"num_pg": 97,
"num_pg_active": 72,
"num_pg_unknown": 0,
"num_osd": 3,
"pool_sum": {
"stat_sum": {
"num_bytes": 57245739111,
"num_objects": 3430078,
"num_object_clones": 0,
"num_object_copies": 6860156,
"num_objects_missing_on_primary": 0,
"num_objects_missing": 0,
"num_objects_degraded": 0,
"num_objects_misplaced": 354664,
"num_objects_unfound": 0,
"num_objects_dirty": 3430078,
"num_whiteouts": 0,
"num_read": 24259364,
"num_read_kb": 269991629,
"num_write": 34345357,
"num_write_kb": 2155285838,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 10850234,
"num_bytes_recovered": 184571569207,
"num_keys_recovered": 5870408,
"num_objects_omap": 677,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0,
"num_flush": 0,
"num_flush_kb": 0,
"num_evict": 0,
"num_evict_kb": 0,
"num_promote": 0,
"num_flush_mode_high": 0,
"num_flush_mode_low": 0,
"num_evict_mode_some": 0,
"num_evict_mode_full": 0,
"num_objects_pinned": 0,
"num_legacy_snapsets": 0,
"num_large_omap_objects": 0,
"num_objects_manifest": 0,
"num_omap_bytes": 904835969,
"num_omap_keys": 1744156,
"num_objects_repaired": 0
},
"store_stats": {
"total": 0,
"available": 0,
"internally_reserved": 0,
"allocated": 0,
"data_stored": 0,
"data_compressed": 0,
"data_compressed_allocated": 0,
"data_compressed_original": 0,
"omap_allocated": 0,
"internal_metadata": 0
},
"log_size": 448055,
"ondisk_log_size": 448055,
"up": 194,
"acting": 194,
"num_store_stats": 0
},
"osd_sum": {
"up_from": 0,
"seq": 0,
"num_pgs": 177,
"num_osds": 3,
"num_per_pool_osds": 3,
"num_per_pool_omap_osds": 3,
"kb": 311427072,
"kb_used": 160997120,
"kb_used_data": 151041616,
"kb_used_omap": 2838297,
"kb_used_meta": 7116198,
"kb_avail": 150429952,
"statfs": {
"total": 318901321728,
"available": 154040270848,
"internally_reserved": 10193403904,
"allocated": 154666614784,
"data_stored": 101848516609,
"data_compressed": 0,
"data_compressed_allocated": 0,
"data_compressed_original": 0,
"omap_allocated": 2906416156,
"internal_metadata": 7286987748
},
"hb_peers": [],
"snap_trim_queue_len": 0,
"num_snap_trimming": 0,
"num_shards_repaired": 0,
"op_queue_age_hist": {
"histogram": [
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
1,
2,
3,
6,
13,
26,
51,
102,
205,
703,
639
],
"upper_bound": 16777216
},
"perf_stat": {
"commit_latency_ms": 0,
"apply_latency_ms": 0,
"commit_latency_ns": 0,
"apply_latency_ns": 0
},
"alerts": [],
"network_ping_times": []
},
"osd_sum_by_class": {
"ssd": {
"up_from": 0,
"seq": 0,
"num_pgs": 177,
"num_osds": 3,
"num_per_pool_osds": 3,
"num_per_pool_omap_osds": 3,
"kb": 311427072,
"kb_used": 160997120,
"kb_used_data": 151041616,
"kb_used_omap": 2838297,
"kb_used_meta": 7116198,
"kb_avail": 150429952,
"statfs": {
"total": 318901321728,
"available": 154040270848,
"internally_reserved": 10193403904,
"allocated": 154666614784,
"data_stored": 101848516609,
"data_compressed": 0,
"data_compressed_allocated": 0,
"data_compressed_original": 0,
"omap_allocated": 2906416156,
"internal_metadata": 7286987748
},
"hb_peers": [],
"snap_trim_queue_len": 0,
"num_snap_trimming": 0,
"num_shards_repaired": 0,
"op_queue_age_hist": {
"histogram": [
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
1,
2,
3,
6,
13,
26,
51,
102,
205,
703,
639
],
"upper_bound": 16777216
},
"perf_stat": {
"commit_latency_ms": 0,
"apply_latency_ms": 0,
"commit_latency_ns": 0,
"apply_latency_ns": 0
},
"alerts": [],
"network_ping_times": []
}
},
"pool_stats": [
{
"poolid": 4,
"num_pg": 1,
"stat_sum": {
"num_bytes": 0,
"num_objects": 0,
"num_object_clones": 0,
"num_object_copies": 0,
"num_objects_missing_on_primary": 0,
"num_objects_missing": 0,
"num_objects_degraded": 0,
"num_objects_misplaced": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 0,
"num_whiteouts": 0,
"num_read": 0,
"num_read_kb": 0,
"num_write": 0,
"num_write_kb": 0,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 0,
"num_bytes_recovered": 0,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0,
"num_flush": 0,
"num_flush_kb": 0,
"num_evict": 0,
"num_evict_kb": 0,
"num_promote": 0,
"num_flush_mode_high": 0,
"num_flush_mode_low": 0,
"num_evict_mode_some": 0,
"num_evict_mode_full": 0,
"num_objects_pinned": 0,
"num_legacy_snapsets": 0,
"num_large_omap_objects": 0,
"num_objects_manifest": 0,
"num_omap_bytes": 0,
"num_omap_keys": 0,
"num_objects_repaired": 0
},
"store_stats": {
"total": 0,
"available": 0,
"internally_reserved": 0,
"allocated": 0,
"data_stored": 0,
"data_compressed": 0,
"data_compressed_allocated": 0,
"data_compressed_original": 0,
"omap_allocated": 0,
"internal_metadata": 0
},
"log_size": 0,
"ondisk_log_size": 0,
"up": 2,
"acting": 2,
"num_store_stats": 2
},
{
"poolid": 3,
"num_pg": 32,
"stat_sum": {
"num_bytes": 43519665207,
"num_objects": 3425317,
"num_object_clones": 0,
"num_object_copies": 6850634,
"num_objects_missing_on_primary": 0,
"num_objects_missing": 0,
"num_objects_degraded": 0,
"num_objects_misplaced": 354529,
"num_objects_unfound": 0,
"num_objects_dirty": 3425317,
"num_whiteouts": 0,
"num_read": 1213449,
"num_read_kb": 8794181,
"num_write": 18819861,
"num_write_kb": 124507830,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 10827786,
"num_bytes_recovered": 132979727344,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0,
"num_flush": 0,
"num_flush_kb": 0,
"num_evict": 0,
"num_evict_kb": 0,
"num_promote": 0,
"num_flush_mode_high": 0,
"num_flush_mode_low": 0,
"num_evict_mode_some": 0,
"num_evict_mode_full": 0,
"num_objects_pinned": 0,
"num_legacy_snapsets": 0,
"num_large_omap_objects": 0,
"num_objects_manifest": 0,
"num_omap_bytes": 0,
"num_omap_keys": 0,
"num_objects_repaired": 0
},
"store_stats": {
"total": 0,
"available": 0,
"internally_reserved": 0,
"allocated": 134729744384,
"data_stored": 81958998585,
"data_compressed": 0,
"data_compressed_allocated": 0,
"data_compressed_original": 0,
"omap_allocated": 0,
"internal_metadata": 0
},
"log_size": 134736,
"ondisk_log_size": 134736,
"up": 64,
"acting": 64,
"num_store_stats": 3
},
{
"poolid": 1,
"num_pg": 32,
"stat_sum": {
"num_bytes": 13453389948,
"num_objects": 4025,
"num_object_clones": 0,
"num_object_copies": 8050,
"num_objects_missing_on_primary": 0,
"num_objects_missing": 0,
"num_objects_degraded": 0,
"num_objects_misplaced": 135,
"num_objects_unfound": 0,
"num_objects_dirty": 4025,
"num_whiteouts": 0,
"num_read": 977375,
"num_read_kb": 188709292,
"num_write": 4598965,
"num_write_kb": 1979021483,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 19340,
"num_bytes_recovered": 50404528112,
"num_keys_recovered": 230,
"num_objects_omap": 15,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0,
"num_flush": 0,
"num_flush_kb": 0,
"num_evict": 0,
"num_evict_kb": 0,
"num_promote": 0,
"num_flush_mode_high": 0,
"num_flush_mode_low": 0,
"num_evict_mode_some": 0,
"num_evict_mode_full": 0,
"num_objects_pinned": 0,
"num_legacy_snapsets": 0,
"num_large_omap_objects": 0,
"num_objects_manifest": 0,
"num_omap_bytes": 1232,
"num_omap_keys": 70,
"num_objects_repaired": 0
},
"store_stats": {
"total": 0,
"available": 0,
"internally_reserved": 0,
"allocated": 19415613440,
"data_stored": 19413727480,
"data_compressed": 0,
"data_compressed_allocated": 0,
"data_compressed_original": 0,
"omap_allocated": 559833,
"internal_metadata": 0
},
"log_size": 147191,
"ondisk_log_size": 147191,
"up": 64,
"acting": 64,
"num_store_stats": 3
},
{
"poolid": 2,
"num_pg": 32,
"stat_sum": {
"num_bytes": 272683956,
"num_objects": 736,
"num_object_clones": 0,
"num_object_copies": 1472,
"num_objects_missing_on_primary": 0,
"num_objects_missing": 0,
"num_objects_degraded": 0,
"num_objects_misplaced": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 736,
"num_whiteouts": 0,
"num_read": 22068540,
"num_read_kb": 72488156,
"num_write": 10926531,
"num_write_kb": 51756525,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 3108,
"num_bytes_recovered": 1187313751,
"num_keys_recovered": 5870178,
"num_objects_omap": 662,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0,
"num_flush": 0,
"num_flush_kb": 0,
"num_evict": 0,
"num_evict_kb": 0,
"num_promote": 0,
"num_flush_mode_high": 0,
"num_flush_mode_low": 0,
"num_evict_mode_some": 0,
"num_evict_mode_full": 0,
"num_objects_pinned": 0,
"num_legacy_snapsets": 0,
"num_large_omap_objects": 0,
"num_objects_manifest": 0,
"num_omap_bytes": 904834737,
"num_omap_keys": 1744086,
"num_objects_repaired": 0
},
"store_stats": {
"total": 0,
"available": 0,
"internally_reserved": 0,
"allocated": 464470016,
"data_stored": 464217286,
"data_compressed": 0,
"data_compressed_allocated": 0,
"data_compressed_original": 0,
"omap_allocated": 2905856323,
"internal_metadata": 0
},
"log_size": 166128,
"ondisk_log_size": 166128,
"up": 64,
"acting": 64,
"num_store_stats": 3
}
],
"osd_stats": [
{
"osd": 0,
"seq": 36794985060424
},
{
"osd": 1,
"seq": 0
},
{
"osd": 2,
"seq": 36842229700642
},
{
"osd": 3,
"seq": 38684770435438
}
],
"num_pg_by_state": [
{
"state": "active+clean+laggy",
"num": 1
},
{
"state": "active+remapped+backfill_wait",
"num": 4
},
{
"state": "down",
"num": 25
},
{
"state": "active+clean",
"num": 67
}
],
"num_pg_by_osd": [
{
"osd": 3,
"num_primary_pg": 35,
"num_acting_pg": 57,
"num_up_not_acting_pg": 4
},
{
"osd": 2,
"num_primary_pg": 32,
"num_acting_pg": 64,
"num_up_not_acting_pg": 0
},
{
"osd": 0,
"num_primary_pg": 30,
"num_acting_pg": 73,
"num_up_not_acting_pg": 0
}
],
"purged_snaps": [
{
"pool": 1,
"purged_snaps": {}
},
{
"pool": 2,
"purged_snaps": {}
},
{
"pool": 3,
"purged_snaps": {}
},
{
"pool": 4,
"purged_snaps": {}
}
],
"servicemap": {
"epoch": 75,
"modified": "2020-12-15T10:24:04.682164+0000",
"services": {
"mds": {
"daemons": {
"summary": "",
"cephfs-a": {
"start_epoch": 0,
"start_stamp": "0.000000",
"gid": 0,
"addr": "(unrecognized address family 0)/0",
"metadata": {},
"task_status": {
"scrub status": "idle"
}
},
"cephfs-b": {
"start_epoch": 0,
"start_stamp": "0.000000",
"gid": 0,
"addr": "(unrecognized address family 0)/0",
"metadata": {},
"task_status": {
"scrub status": "idle"
}
}
}
}
}
},
"paxos": {
"first_committed": 36786059,
"last_committed": 36786566,
"last_pn": 234100,
"accepted_pn": 234100
}
}debug 2020-12-16T13:19:21.772+0000 7fc4c3f13f40 0 set uid:gid to 167:167 (ceph:ceph)
debug 2020-12-16T13:19:21.772+0000 7fc4c3f13f40 0 ceph version 15.2.6 (cb8c61a60551b72614257d632a574d420064c17a) octopus (stable), process ceph-osd, pid 1
debug 2020-12-16T13:19:21.772+0000 7fc4c3f13f40 0 pidfile_write: ignore empty --pid-file
debug 2020-12-16T13:19:21.776+0000 7fc4c3f13f40 1 bdev create path /var/lib/ceph/osd/ceph-1/block type kernel
debug 2020-12-16T13:19:21.776+0000 7fc4c3f13f40 1 bdev(0x55a0b0fd0000 /var/lib/ceph/osd/ceph-1/block) open path /var/lib/ceph/osd/ceph-1/block
debug 2020-12-16T13:19:21.776+0000 7fc4c3f13f40 1 bdev(0x55a0b0fd0000 /var/lib/ceph/osd/ceph-1/block) open size 107369988096 (0x18ffc00000, 100 GiB) block_size 4096 (4 KiB) non-rotational discard not supported
debug 2020-12-16T13:19:21.776+0000 7fc4c3f13f40 1 bluestore(/var/lib/ceph/osd/ceph-1) _set_cache_sizes cache_size 3221225472 meta 0.4 kv 0.4 data 0.2
debug 2020-12-16T13:19:21.776+0000 7fc4c3f13f40 1 bdev create path /var/lib/ceph/osd/ceph-1/block type kernel
debug 2020-12-16T13:19:21.776+0000 7fc4c3f13f40 1 bdev(0x55a0b0fd0700 /var/lib/ceph/osd/ceph-1/block) open path /var/lib/ceph/osd/ceph-1/block
debug 2020-12-16T13:19:21.776+0000 7fc4c3f13f40 1 bdev(0x55a0b0fd0700 /var/lib/ceph/osd/ceph-1/block) open size 107369988096 (0x18ffc00000, 100 GiB) block_size 4096 (4 KiB) non-rotational discard not supported
debug 2020-12-16T13:19:21.776+0000 7fc4c3f13f40 1 bluefs add_block_device bdev 1 path /var/lib/ceph/osd/ceph-1/block size 100 GiB
debug 2020-12-16T13:19:21.776+0000 7fc4c3f13f40 1 bdev(0x55a0b0fd0700 /var/lib/ceph/osd/ceph-1/block) close
debug 2020-12-16T13:19:22.084+0000 7fc4c3f13f40 1 bdev(0x55a0b0fd0000 /var/lib/ceph/osd/ceph-1/block) close
debug 2020-12-16T13:19:22.316+0000 7fc4c3f13f40 0 starting osd.1 osd_data /var/lib/ceph/osd/ceph-1 /var/lib/ceph/osd/ceph-1/journal
debug 2020-12-16T13:19:22.316+0000 7fc4c3f13f40 -1 Falling back to public interface
debug 2020-12-16T13:19:22.328+0000 7fc4c3f13f40 0 load: jerasure load: lrc load: isa
debug 2020-12-16T13:19:22.328+0000 7fc4c3f13f40 1 bdev create path /var/lib/ceph/osd/ceph-1/block type kernel
debug 2020-12-16T13:19:22.328+0000 7fc4c3f13f40 1 bdev(0x55a0b0fd0000 /var/lib/ceph/osd/ceph-1/block) open path /var/lib/ceph/osd/ceph-1/block
debug 2020-12-16T13:19:22.328+0000 7fc4c3f13f40 1 bdev(0x55a0b0fd0000 /var/lib/ceph/osd/ceph-1/block) open size 107369988096 (0x18ffc00000, 100 GiB) block_size 4096 (4 KiB) non-rotational discard not supported
debug 2020-12-16T13:19:22.328+0000 7fc4c3f13f40 1 bluestore(/var/lib/ceph/osd/ceph-1) _set_cache_sizes cache_size 3221225472 meta 0.4 kv 0.4 data 0.2
debug 2020-12-16T13:19:22.328+0000 7fc4c3f13f40 1 bdev(0x55a0b0fd0000 /var/lib/ceph/osd/ceph-1/block) close
debug 2020-12-16T13:19:22.624+0000 7fc4c3f13f40 0 osd.1:0.OSDShard using op scheduler ClassedOpQueueScheduler(queue=WeightedPriorityQueue, cutoff=196)
debug 2020-12-16T13:19:22.624+0000 7fc4c3f13f40 0 osd.1:1.OSDShard using op scheduler ClassedOpQueueScheduler(queue=WeightedPriorityQueue, cutoff=196)
debug 2020-12-16T13:19:22.624+0000 7fc4c3f13f40 0 osd.1:2.OSDShard using op scheduler ClassedOpQueueScheduler(queue=WeightedPriorityQueue, cutoff=196)
debug 2020-12-16T13:19:22.624+0000 7fc4c3f13f40 0 osd.1:3.OSDShard using op scheduler ClassedOpQueueScheduler(queue=WeightedPriorityQueue, cutoff=196)
debug 2020-12-16T13:19:22.624+0000 7fc4c3f13f40 0 osd.1:4.OSDShard using op scheduler ClassedOpQueueScheduler(queue=WeightedPriorityQueue, cutoff=196)
debug 2020-12-16T13:19:22.624+0000 7fc4c3f13f40 0 osd.1:5.OSDShard using op scheduler ClassedOpQueueScheduler(queue=WeightedPriorityQueue, cutoff=196)
debug 2020-12-16T13:19:22.624+0000 7fc4c3f13f40 0 osd.1:6.OSDShard using op scheduler ClassedOpQueueScheduler(queue=WeightedPriorityQueue, cutoff=196)
debug 2020-12-16T13:19:22.624+0000 7fc4c3f13f40 0 osd.1:7.OSDShard using op scheduler ClassedOpQueueScheduler(queue=WeightedPriorityQueue, cutoff=196)
debug 2020-12-16T13:19:22.624+0000 7fc4c3f13f40 1 bluestore(/var/lib/ceph/osd/ceph-1) _mount path /var/lib/ceph/osd/ceph-1
debug 2020-12-16T13:19:22.624+0000 7fc4c3f13f40 1 bdev create path /var/lib/ceph/osd/ceph-1/block type kernel
debug 2020-12-16T13:19:22.624+0000 7fc4c3f13f40 1 bdev(0x55a0b0fd0000 /var/lib/ceph/osd/ceph-1/block) open path /var/lib/ceph/osd/ceph-1/block
debug 2020-12-16T13:19:22.624+0000 7fc4c3f13f40 1 bdev(0x55a0b0fd0000 /var/lib/ceph/osd/ceph-1/block) open size 107369988096 (0x18ffc00000, 100 GiB) block_size 4096 (4 KiB) non-rotational discard not supported
debug 2020-12-16T13:19:22.624+0000 7fc4c3f13f40 1 bluestore(/var/lib/ceph/osd/ceph-1) _set_cache_sizes cache_size 3221225472 meta 0.4 kv 0.4 data 0.2
debug 2020-12-16T13:19:22.624+0000 7fc4c3f13f40 1 bdev create path /var/lib/ceph/osd/ceph-1/block type kernel
debug 2020-12-16T13:19:22.624+0000 7fc4c3f13f40 1 bdev(0x55a0b0fd0700 /var/lib/ceph/osd/ceph-1/block) open path /var/lib/ceph/osd/ceph-1/block
debug 2020-12-16T13:19:22.624+0000 7fc4c3f13f40 1 bdev(0x55a0b0fd0700 /var/lib/ceph/osd/ceph-1/block) open size 107369988096 (0x18ffc00000, 100 GiB) block_size 4096 (4 KiB) non-rotational discard not supported
debug 2020-12-16T13:19:22.624+0000 7fc4c3f13f40 1 bluefs add_block_device bdev 1 path /var/lib/ceph/osd/ceph-1/block size 100 GiB
debug 2020-12-16T13:19:22.624+0000 7fc4c3f13f40 1 bluefs mount
debug 2020-12-16T13:19:22.624+0000 7fc4c3f13f40 1 bluefs _init_alloc id 1 alloc_size 0x10000 size 0x18ffc00000
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 0 set rocksdb option compaction_readahead_size = 2097152
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 0 set rocksdb option compression = kNoCompression
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 0 set rocksdb option max_background_compactions = 2
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 0 set rocksdb option max_write_buffer_number = 4
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 0 set rocksdb option min_write_buffer_number_to_merge = 1
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 0 set rocksdb option recycle_log_file_num = 4
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 0 set rocksdb option writable_file_max_buffer_size = 0
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 0 set rocksdb option write_buffer_size = 268435456
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 0 set rocksdb option compaction_readahead_size = 2097152
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 0 set rocksdb option compression = kNoCompression
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 0 set rocksdb option max_background_compactions = 2
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 0 set rocksdb option max_write_buffer_number = 4
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 0 set rocksdb option min_write_buffer_number_to_merge = 1
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 0 set rocksdb option recycle_log_file_num = 4
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 0 set rocksdb option writable_file_max_buffer_size = 0
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 0 set rocksdb option write_buffer_size = 268435456
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 1 rocksdb: do_open column families: [default]
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: RocksDB version: 6.1.2
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Git sha rocksdb_build_git_sha:@0@
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Compile date Nov 17 2020
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: DB SUMMARY
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: CURRENT file: CURRENT
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: IDENTITY file: IDENTITY
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: MANIFEST file: MANIFEST-001369 size: 15227 Bytes
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: SST files in db dir, Total Num: 59, files: 001195.sst 001226.sst 001227.sst 001228.sst 001229.sst 001230.sst 001250.sst 001251.sst 001262.sst
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Write Ahead Log file in db.wal: 001400.log size: 255396293 ; 001402.log size: 249498342 ;
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.error_if_exists: 0
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.create_if_missing: 0
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.paranoid_checks: 1
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.env: 0x55a0b0301f80
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.info_log: 0x55a0b030ad80
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.max_file_opening_threads: 16
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.statistics: (nil)
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.use_fsync: 0
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.max_log_file_size: 0
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.max_manifest_file_size: 1073741824
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.log_file_time_to_roll: 0
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.keep_log_file_num: 1000
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.recycle_log_file_num: 4
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.allow_fallocate: 1
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.allow_mmap_reads: 0
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.allow_mmap_writes: 0
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.use_direct_reads: 0
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.create_missing_column_families: 0
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.db_log_dir:
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.wal_dir: db.wal
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.table_cache_numshardbits: 6
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.max_subcompactions: 1
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.max_background_flushes: -1
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.WAL_ttl_seconds: 0
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.WAL_size_limit_MB: 0
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.manifest_preallocation_size: 4194304
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.is_fd_close_on_exec: 1
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.advise_random_on_open: 1
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.db_write_buffer_size: 0
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.write_buffer_manager: 0x55a0b1009110
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.access_hint_on_compaction_start: 1
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.new_table_reader_for_compaction_inputs: 1
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.random_access_max_buffer_size: 1048576
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.use_adaptive_mutex: 0
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.rate_limiter: (nil)
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.wal_recovery_mode: 0
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.enable_thread_tracking: 0
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.enable_pipelined_write: 0
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.allow_concurrent_memtable_write: 1
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.write_thread_max_yield_usec: 100
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.write_thread_slow_yield_usec: 3
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.row_cache: None
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.wal_filter: None
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.avoid_flush_during_recovery: 0
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.allow_ingest_behind: 0
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.preserve_deletes: 0
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.two_write_queues: 0
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.manual_wal_flush: 0
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.atomic_flush: 0
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.max_background_jobs: 2
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.max_background_compactions: 2
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.avoid_flush_during_shutdown: 0
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.writable_file_max_buffer_size: 0
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.delayed_write_rate : 16777216
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.max_total_wal_size: 0
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.stats_dump_period_sec: 600
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.stats_persist_period_sec: 600
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.stats_history_buffer_size: 1048576
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.max_open_files: -1
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.bytes_per_sync: 0
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.wal_bytes_per_sync: 0
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.compaction_readahead_size: 2097152
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Compression algorithms supported:
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: kZSTDNotFinalCompression supported: 0
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: kZSTD supported: 0
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: kXpressCompression supported: 0
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: kLZ4HCCompression supported: 1
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: kLZ4Compression supported: 1
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: kBZip2Compression supported: 0
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: kZlibCompression supported: 1
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: kSnappyCompression supported: 1
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Fast CRC32 supported: Supported on x86
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: [db/db_impl_readonly.cc:22] Opening the db in read only mode
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: [db/version_set.cc:3543] Recovering from manifest file: db/MANIFEST-001369
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: [db/column_family.cc:477] --------------- Options for column family [default]:
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.comparator: leveldb.BytewiseComparator
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.merge_operator: .T:int64_array.b:bitwise_xor
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.compaction_filter: None
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.compaction_filter_factory: None
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.memtable_factory: SkipListFactory
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.table_factory: BlockBasedTable
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55a0b0386438)
cache_index_and_filter_blocks: 1
cache_index_and_filter_blocks_with_high_priority: 1
pin_l0_filter_and_index_blocks_in_cache: 0
pin_top_level_index_and_filter: 1
index_type: 0
data_block_index_type: 0
data_block_hash_table_util_ratio: 0.750000
hash_index_allow_collision: 1
checksum: 1
no_block_cache: 0
block_cache: 0x55a0b02e0b10
block_cache_name: BinnedLRUCache
block_cache_options:
capacity : 1288490188
num_shard_bits : 4
strict_capacity_limit : 0
high_pri_pool_ratio: 0.000
block_cache_compressed: (nil)
persistent_cache: (nil)
block_size: 4096
block_size_deviation: 10
block_restart_interval: 16
index_block_restart_interval: 1
metadata_block_size: 4096
partition_filters: 0
use_delta_encoding: 1
filter_policy: rocksdb.BuiltinBloomFilter
whole_key_filtering: 1
verify_compression: 0
read_amp_bytes_per_bit: 0
format_version: 2
enable_index_compression: 1
block_align: 0
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.write_buffer_size: 268435456
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.max_write_buffer_number: 4
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.compression: NoCompression
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.bottommost_compression: Disabled
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.prefix_extractor: nullptr
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.num_levels: 7
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.min_write_buffer_number_to_merge: 1
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.bottommost_compression_opts.level: 32767
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.bottommost_compression_opts.strategy: 0
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.bottommost_compression_opts.enabled: false
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.compression_opts.window_bits: -14
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.compression_opts.level: 32767
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.compression_opts.strategy: 0
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.compression_opts.max_dict_bytes: 0
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.compression_opts.enabled: false
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.level0_file_num_compaction_trigger: 4
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.level0_slowdown_writes_trigger: 20
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.level0_stop_writes_trigger: 36
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.target_file_size_base: 67108864
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.target_file_size_multiplier: 1
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.max_bytes_for_level_base: 268435456
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 0
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.max_sequential_skip_in_iterations: 8
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.max_compaction_bytes: 1677721600
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.arena_block_size: 33554432
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.rate_limit_delay_max_milliseconds: 100
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.disable_auto_compactions: 0
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.compaction_style: kCompactionStyleLevel
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.compaction_options_universal.size_ratio: 1
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.table_properties_collectors:
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.inplace_update_support: 0
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.inplace_update_num_locks: 10000
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.memtable_whole_key_filtering: 0
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.memtable_huge_page_size: 0
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.bloom_locality: 0
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.max_successive_merges: 0
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.optimize_filters_for_hits: 0
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.paranoid_file_checks: 0
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.force_consistency_checks: 0
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.report_bg_io_stats: 0
debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.ttl: 0
debug 2020-12-16T13:19:22.684+0000 7fc4c3f13f40 4 rocksdb: [db/version_set.cc:3757] Recovered from manifest file:db/MANIFEST-001369 succeeded,manifest_file_number is 1369, next_file_number is 1403, last_sequence is 144598951, log_number is 1400,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0
debug 2020-12-16T13:19:22.684+0000 7fc4c3f13f40 4 rocksdb: [db/version_set.cc:3766] Column family [default] (ID 0), log number is 1400
debug 2020-12-16T13:19:22.684+0000 7fc4c3f13f40 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1608124762690294, "job": 1, "event": "recovery_started", "log_files": [1400, 1402]}
debug 2020-12-16T13:19:22.684+0000 7fc4c3f13f40 4 rocksdb: [db/db_impl_open.cc:583] Recovering log #1400 mode 0
debug 2020-12-16T13:19:24.644+0000 7fc4c3f13f40 4 rocksdb: [db/db_impl_open.cc:583] Recovering log #1402 mode 0
debug 2020-12-16T13:19:24.648+0000 7fc4c3f13f40 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1608124764655926, "job": 1, "event": "recovery_finished"}
debug 2020-12-16T13:19:24.648+0000 7fc4c3f13f40 1 bluestore(/var/lib/ceph/osd/ceph-1) _open_db opened rocksdb path db options compression=kNoCompression,max_write_buffer_number=4,min_write_buffer_number_to_merge=1,recycle_log_file_num=4,write_buffer_size=268435456,writable_file_max_buffer_size=0,compaction_readahead_size=2097152,max_background_compactions=2
debug 2020-12-16T13:19:24.676+0000 7fc4c3f13f40 1 freelist init
debug 2020-12-16T13:19:24.676+0000 7fc4c3f13f40 1 freelist _init_from_label
debug 2020-12-16T13:19:24.680+0000 7fc4c3f13f40 1 bluestore(/var/lib/ceph/osd/ceph-1) _open_alloc opening allocation metadata
debug 2020-12-16T13:19:25.312+0000 7fc4c3f13f40 1 bluestore(/var/lib/ceph/osd/ceph-1) _open_alloc loaded 48 GiB in 666948 extents available 44 GiB
debug 2020-12-16T13:19:25.312+0000 7fc4c3f13f40 4 rocksdb: [db/db_impl.cc:390] Shutdown: canceling all background work
debug 2020-12-16T13:19:25.316+0000 7fc4c3f13f40 4 rocksdb: [db/db_impl.cc:563] Shutdown complete
debug 2020-12-16T13:19:25.316+0000 7fc4c3f13f40 1 bluefs umount
debug 2020-12-16T13:19:25.316+0000 7fc4c3f13f40 1 bdev(0x55a0b0fd0700 /var/lib/ceph/osd/ceph-1/block) close
debug 2020-12-16T13:19:25.416+0000 7fc4c3f13f40 1 bdev create path /var/lib/ceph/osd/ceph-1/block type kernel
debug 2020-12-16T13:19:25.416+0000 7fc4c3f13f40 1 bdev(0x55a0b0fd0700 /var/lib/ceph/osd/ceph-1/block) open path /var/lib/ceph/osd/ceph-1/block
debug 2020-12-16T13:19:25.416+0000 7fc4c3f13f40 1 bdev(0x55a0b0fd0700 /var/lib/ceph/osd/ceph-1/block) open size 107369988096 (0x18ffc00000, 100 GiB) block_size 4096 (4 KiB) non-rotational discard not supported
debug 2020-12-16T13:19:25.416+0000 7fc4c3f13f40 1 bluefs add_block_device bdev 1 path /var/lib/ceph/osd/ceph-1/block size 100 GiB
debug 2020-12-16T13:19:25.416+0000 7fc4c3f13f40 1 bluefs mount
debug 2020-12-16T13:19:25.416+0000 7fc4c3f13f40 1 bluefs _init_alloc id 1 alloc_size 0x10000 size 0x18ffc00000
debug 2020-12-16T13:19:25.464+0000 7fc4c3f13f40 0 set rocksdb option compaction_readahead_size = 2097152
debug 2020-12-16T13:19:25.464+0000 7fc4c3f13f40 0 set rocksdb option compression = kNoCompression
debug 2020-12-16T13:19:25.464+0000 7fc4c3f13f40 0 set rocksdb option max_background_compactions = 2
debug 2020-12-16T13:19:25.464+0000 7fc4c3f13f40 0 set rocksdb option max_write_buffer_number = 4
debug 2020-12-16T13:19:25.464+0000 7fc4c3f13f40 0 set rocksdb option min_write_buffer_number_to_merge = 1
debug 2020-12-16T13:19:25.464+0000 7fc4c3f13f40 0 set rocksdb option recycle_log_file_num = 4
debug 2020-12-16T13:19:25.468+0000 7fc4c3f13f40 0 set rocksdb option writable_file_max_buffer_size = 0
debug 2020-12-16T13:19:25.468+0000 7fc4c3f13f40 0 set rocksdb option write_buffer_size = 268435456
debug 2020-12-16T13:19:25.468+0000 7fc4c3f13f40 0 set rocksdb option compaction_readahead_size = 2097152
debug 2020-12-16T13:19:25.468+0000 7fc4c3f13f40 0 set rocksdb option compression = kNoCompression
debug 2020-12-16T13:19:25.468+0000 7fc4c3f13f40 0 set rocksdb option max_background_compactions = 2
debug 2020-12-16T13:19:25.468+0000 7fc4c3f13f40 0 set rocksdb option max_write_buffer_number = 4
debug 2020-12-16T13:19:25.468+0000 7fc4c3f13f40 0 set rocksdb option min_write_buffer_number_to_merge = 1
debug 2020-12-16T13:19:25.468+0000 7fc4c3f13f40 0 set rocksdb option recycle_log_file_num = 4
debug 2020-12-16T13:19:25.468+0000 7fc4c3f13f40 0 set rocksdb option writable_file_max_buffer_size = 0
debug 2020-12-16T13:19:25.468+0000 7fc4c3f13f40 0 set rocksdb option write_buffer_size = 268435456
debug 2020-12-16T13:19:25.468+0000 7fc4c3f13f40 1 rocksdb: do_open column families: [default]
debug 2020-12-16T13:19:25.468+0000 7fc4c3f13f40 4 rocksdb: RocksDB version: 6.1.2
debug 2020-12-16T13:19:25.468+0000 7fc4c3f13f40 4 rocksdb: Git sha rocksdb_build_git_sha:@0@
debug 2020-12-16T13:19:25.468+0000 7fc4c3f13f40 4 rocksdb: Compile date Nov 17 2020
debug 2020-12-16T13:19:25.472+0000 7fc4c3f13f40 4 rocksdb: DB SUMMARY
debug 2020-12-16T13:19:25.472+0000 7fc4c3f13f40 4 rocksdb: CURRENT file: CURRENT
debug 2020-12-16T13:19:25.472+0000 7fc4c3f13f40 4 rocksdb: IDENTITY file: IDENTITY
debug 2020-12-16T13:19:25.472+0000 7fc4c3f13f40 4 rocksdb: MANIFEST file: MANIFEST-001369 size: 15227 Bytes
debug 2020-12-16T13:19:25.472+0000 7fc4c3f13f40 4 rocksdb: SST files in db dir, Total Num: 59, files: 001195.sst 001226.sst 001227.sst 001228.sst 001229.sst 001230.sst 001250.sst 001251.sst 001262.sst
debug 2020-12-16T13:19:25.472+0000 7fc4c3f13f40 4 rocksdb: Write Ahead Log file in db.wal: 001400.log size: 255396293 ; 001402.log size: 249498342 ;
debug 2020-12-16T13:19:25.472+0000 7fc4c3f13f40 4 rocksdb: Options.error_if_exists: 0
debug 2020-12-16T13:19:25.472+0000 7fc4c3f13f40 4 rocksdb: Options.create_if_missing: 0
debug 2020-12-16T13:19:25.472+0000 7fc4c3f13f40 4 rocksdb: Options.paranoid_checks: 1
debug 2020-12-16T13:19:25.472+0000 7fc4c3f13f40 4 rocksdb: Options.env: 0x55a0b12b1740
debug 2020-12-16T13:19:25.472+0000 7fc4c3f13f40 4 rocksdb: Options.info_log: 0x55a0b038f6a0
debug 2020-12-16T13:19:25.472+0000 7fc4c3f13f40 4 rocksdb: Options.max_file_opening_threads: 16
debug 2020-12-16T13:19:25.472+0000 7fc4c3f13f40 4 rocksdb: Options.statistics: (nil)
debug 2020-12-16T13:19:25.472+0000 7fc4c3f13f40 4 rocksdb: Options.use_fsync: 0
debug 2020-12-16T13:19:25.472+0000 7fc4c3f13f40 4 rocksdb: Options.max_log_file_size: 0
debug 2020-12-16T13:19:25.472+0000 7fc4c3f13f40 4 rocksdb: Options.max_manifest_file_size: 1073741824
debug 2020-12-16T13:19:25.472+0000 7fc4c3f13f40 4 rocksdb: Options.log_file_time_to_roll: 0
debug 2020-12-16T13:19:25.472+0000 7fc4c3f13f40 4 rocksdb: Options.keep_log_file_num: 1000
debug 2020-12-16T13:19:25.472+0000 7fc4c3f13f40 4 rocksdb: Options.recycle_log_file_num: 4
debug 2020-12-16T13:19:25.472+0000 7fc4c3f13f40 4 rocksdb: Options.allow_fallocate: 1
debug 2020-12-16T13:19:25.472+0000 7fc4c3f13f40 4 rocksdb: Options.allow_mmap_reads: 0
debug 2020-12-16T13:19:25.472+0000 7fc4c3f13f40 4 rocksdb: Options.allow_mmap_writes: 0
debug 2020-12-16T13:19:25.472+0000 7fc4c3f13f40 4 rocksdb: Options.use_direct_reads: 0
debug 2020-12-16T13:19:25.472+0000 7fc4c3f13f40 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0
debug 2020-12-16T13:19:25.476+0000 7fc4c3f13f40 4 rocksdb: Options.create_missing_column_families: 0
debug 2020-12-16T13:19:25.476+0000 7fc4c3f13f40 4 rocksdb: Options.db_log_dir:
debug 2020-12-16T13:19:25.476+0000 7fc4c3f13f40 4 rocksdb: Options.wal_dir: db.wal
debug 2020-12-16T13:19:25.476+0000 7fc4c3f13f40 4 rocksdb: Options.table_cache_numshardbits: 6
debug 2020-12-16T13:19:25.476+0000 7fc4c3f13f40 4 rocksdb: Options.max_subcompactions: 1
debug 2020-12-16T13:19:25.476+0000 7fc4c3f13f40 4 rocksdb: Options.max_background_flushes: -1
debug 2020-12-16T13:19:25.476+0000 7fc4c3f13f40 4 rocksdb: Options.WAL_ttl_seconds: 0
debug 2020-12-16T13:19:25.476+0000 7fc4c3f13f40 4 rocksdb: Options.WAL_size_limit_MB: 0
debug 2020-12-16T13:19:25.476+0000 7fc4c3f13f40 4 rocksdb: Options.manifest_preallocation_size: 4194304
debug 2020-12-16T13:19:25.476+0000 7fc4c3f13f40 4 rocksdb: Options.is_fd_close_on_exec: 1
debug 2020-12-16T13:19:25.476+0000 7fc4c3f13f40 4 rocksdb: Options.advise_random_on_open: 1
debug 2020-12-16T13:19:25.476+0000 7fc4c3f13f40 4 rocksdb: Options.db_write_buffer_size: 0
debug 2020-12-16T13:19:25.476+0000 7fc4c3f13f40 4 rocksdb: Options.write_buffer_manager: 0x55a0b157c8a0
debug 2020-12-16T13:19:25.476+0000 7fc4c3f13f40 4 rocksdb: Options.access_hint_on_compaction_start: 1
debug 2020-12-16T13:19:25.476+0000 7fc4c3f13f40 4 rocksdb: Options.new_table_reader_for_compaction_inputs: 1
debug 2020-12-16T13:19:25.476+0000 7fc4c3f13f40 4 rocksdb: Options.random_access_max_buffer_size: 1048576
debug 2020-12-16T13:19:25.476+0000 7fc4c3f13f40 4 rocksdb: Options.use_adaptive_mutex: 0
debug 2020-12-16T13:19:25.476+0000 7fc4c3f13f40 4 rocksdb: Options.rate_limiter: (nil)
debug 2020-12-16T13:19:25.476+0000 7fc4c3f13f40 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0
debug 2020-12-16T13:19:25.476+0000 7fc4c3f13f40 4 rocksdb: Options.wal_recovery_mode: 0
debug 2020-12-16T13:19:25.476+0000 7fc4c3f13f40 4 rocksdb: Options.enable_thread_tracking: 0
debug 2020-12-16T13:19:25.476+0000 7fc4c3f13f40 4 rocksdb: Options.enable_pipelined_write: 0
debug 2020-12-16T13:19:25.476+0000 7fc4c3f13f40 4 rocksdb: Options.allow_concurrent_memtable_write: 1
debug 2020-12-16T13:19:25.476+0000 7fc4c3f13f40 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1
debug 2020-12-16T13:19:25.480+0000 7fc4c3f13f40 4 rocksdb: Options.write_thread_max_yield_usec: 100
debug 2020-12-16T13:19:25.480+0000 7fc4c3f13f40 4 rocksdb: Options.write_thread_slow_yield_usec: 3
debug 2020-12-16T13:19:25.480+0000 7fc4c3f13f40 4 rocksdb: Options.row_cache: None
debug 2020-12-16T13:19:25.480+0000 7fc4c3f13f40 4 rocksdb: Options.wal_filter: None
debug 2020-12-16T13:19:25.480+0000 7fc4c3f13f40 4 rocksdb: Options.avoid_flush_during_recovery: 0
debug 2020-12-16T13:19:25.480+0000 7fc4c3f13f40 4 rocksdb: Options.allow_ingest_behind: 0
debug 2020-12-16T13:19:25.480+0000 7fc4c3f13f40 4 rocksdb: Options.preserve_deletes: 0
debug 2020-12-16T13:19:25.480+0000 7fc4c3f13f40 4 rocksdb: Options.two_write_queues: 0
debug 2020-12-16T13:19:25.480+0000 7fc4c3f13f40 4 rocksdb: Options.manual_wal_flush: 0
debug 2020-12-16T13:19:25.480+0000 7fc4c3f13f40 4 rocksdb: Options.atomic_flush: 0
debug 2020-12-16T13:19:25.480+0000 7fc4c3f13f40 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0
debug 2020-12-16T13:19:25.480+0000 7fc4c3f13f40 4 rocksdb: Options.max_background_jobs: 2
debug 2020-12-16T13:19:25.480+0000 7fc4c3f13f40 4 rocksdb: Options.max_background_compactions: 2
debug 2020-12-16T13:19:25.480+0000 7fc4c3f13f40 4 rocksdb: Options.avoid_flush_during_shutdown: 0
debug 2020-12-16T13:19:25.480+0000 7fc4c3f13f40 4 rocksdb: Options.writable_file_max_buffer_size: 0
debug 2020-12-16T13:19:25.480+0000 7fc4c3f13f40 4 rocksdb: Options.delayed_write_rate : 16777216
debug 2020-12-16T13:19:25.480+0000 7fc4c3f13f40 4 rocksdb: Options.max_total_wal_size: 0
debug 2020-12-16T13:19:25.480+0000 7fc4c3f13f40 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000
debug 2020-12-16T13:19:25.480+0000 7fc4c3f13f40 4 rocksdb: Options.stats_dump_period_sec: 600
debug 2020-12-16T13:19:25.480+0000 7fc4c3f13f40 4 rocksdb: Options.stats_persist_period_sec: 600
debug 2020-12-16T13:19:25.480+0000 7fc4c3f13f40 4 rocksdb: Options.stats_history_buffer_size: 1048576
debug 2020-12-16T13:19:25.480+0000 7fc4c3f13f40 4 rocksdb: Options.max_open_files: -1
debug 2020-12-16T13:19:25.480+0000 7fc4c3f13f40 4 rocksdb: Options.bytes_per_sync: 0
debug 2020-12-16T13:19:25.480+0000 7fc4c3f13f40 4 rocksdb: Options.wal_bytes_per_sync: 0
debug 2020-12-16T13:19:25.480+0000 7fc4c3f13f40 4 rocksdb: Options.compaction_readahead_size: 2097152
debug 2020-12-16T13:19:25.480+0000 7fc4c3f13f40 4 rocksdb: Compression algorithms supported:
debug 2020-12-16T13:19:25.480+0000 7fc4c3f13f40 4 rocksdb: kZSTDNotFinalCompression supported: 0
debug 2020-12-16T13:19:25.480+0000 7fc4c3f13f40 4 rocksdb: kZSTD supported: 0
debug 2020-12-16T13:19:25.480+0000 7fc4c3f13f40 4 rocksdb: kXpressCompression supported: 0
debug 2020-12-16T13:19:25.480+0000 7fc4c3f13f40 4 rocksdb: kLZ4HCCompression supported: 1
debug 2020-12-16T13:19:25.480+0000 7fc4c3f13f40 4 rocksdb: kLZ4Compression supported: 1
debug 2020-12-16T13:19:25.480+0000 7fc4c3f13f40 4 rocksdb: kBZip2Compression supported: 0
debug 2020-12-16T13:19:25.480+0000 7fc4c3f13f40 4 rocksdb: kZlibCompression supported: 1
debug 2020-12-16T13:19:25.480+0000 7fc4c3f13f40 4 rocksdb: kSnappyCompression supported: 1
debug 2020-12-16T13:19:25.480+0000 7fc4c3f13f40 4 rocksdb: Fast CRC32 supported: Supported on x86
debug 2020-12-16T13:19:25.480+0000 7fc4c3f13f40 4 rocksdb: [db/version_set.cc:3543] Recovering from manifest file: db/MANIFEST-001369
debug 2020-12-16T13:19:25.480+0000 7fc4c3f13f40 4 rocksdb: [db/column_family.cc:477] --------------- Options for column family [default]:
debug 2020-12-16T13:19:25.484+0000 7fc4c3f13f40 4 rocksdb: Options.comparator: leveldb.BytewiseComparator
debug 2020-12-16T13:19:25.484+0000 7fc4c3f13f40 4 rocksdb: Options.merge_operator: .T:int64_array.b:bitwise_xor
debug 2020-12-16T13:19:25.484+0000 7fc4c3f13f40 4 rocksdb: Options.compaction_filter: None
debug 2020-12-16T13:19:25.484+0000 7fc4c3f13f40 4 rocksdb: Options.compaction_filter_factory: None
debug 2020-12-16T13:19:25.484+0000 7fc4c3f13f40 4 rocksdb: Options.memtable_factory: SkipListFactory
debug 2020-12-16T13:19:25.484+0000 7fc4c3f13f40 4 rocksdb: Options.table_factory: BlockBasedTable
debug 2020-12-16T13:19:25.484+0000 7fc4c3f13f40 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55a0b0386438)
cache_index_and_filter_blocks: 1
cache_index_and_filter_blocks_with_high_priority: 1
pin_l0_filter_and_index_blocks_in_cache: 0
pin_top_level_index_and_filter: 1
index_type: 0
data_block_index_type: 0
data_block_hash_table_util_ratio: 0.750000
hash_index_allow_collision: 1
checksum: 1
no_block_cache: 0
block_cache: 0x55a0b02e1510
block_cache_name: BinnedLRUCache
block_cache_options:
capacity : 1288490188
num_shard_bits : 4
strict_capacity_limit : 0
high_pri_pool_ratio: 0.000
block_cache_compressed: (nil)
persistent_cache: (nil)
block_size: 4096
block_size_deviation: 10
block_restart_interval: 16
index_block_restart_interval: 1
metadata_block_size: 4096
partition_filters: 0
use_delta_encoding: 1
filter_policy: rocksdb.BuiltinBloomFilter
whole_key_filtering: 1
verify_compression: 0
read_amp_bytes_per_bit: 0
format_version: 2
enable_index_compression: 1
block_align: 0
debug 2020-12-16T13:19:25.484+0000 7fc4c3f13f40 4 rocksdb: Options.write_buffer_size: 268435456
debug 2020-12-16T13:19:25.484+0000 7fc4c3f13f40 4 rocksdb: Options.max_write_buffer_number: 4
debug 2020-12-16T13:19:25.484+0000 7fc4c3f13f40 4 rocksdb: Options.compression: NoCompression
debug 2020-12-16T13:19:25.484+0000 7fc4c3f13f40 4 rocksdb: Options.bottommost_compression: Disabled
debug 2020-12-16T13:19:25.484+0000 7fc4c3f13f40 4 rocksdb: Options.prefix_extractor: nullptr
debug 2020-12-16T13:19:25.484+0000 7fc4c3f13f40 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr
debug 2020-12-16T13:19:25.484+0000 7fc4c3f13f40 4 rocksdb: Options.num_levels: 7
debug 2020-12-16T13:19:25.484+0000 7fc4c3f13f40 4 rocksdb: Options.min_write_buffer_number_to_merge: 1
debug 2020-12-16T13:19:25.484+0000 7fc4c3f13f40 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0
debug 2020-12-16T13:19:25.484+0000 7fc4c3f13f40 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14
debug 2020-12-16T13:19:25.484+0000 7fc4c3f13f40 4 rocksdb: Options.bottommost_compression_opts.level: 32767
debug 2020-12-16T13:19:25.484+0000 7fc4c3f13f40 4 rocksdb: Options.bottommost_compression_opts.strategy: 0
debug 2020-12-16T13:19:25.484+0000 7fc4c3f13f40 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0
debug 2020-12-16T13:19:25.484+0000 7fc4c3f13f40 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0
debug 2020-12-16T13:19:25.484+0000 7fc4c3f13f40 4 rocksdb: Options.bottommost_compression_opts.enabled: false
debug 2020-12-16T13:19:25.484+0000 7fc4c3f13f40 4 rocksdb: Options.compression_opts.window_bits: -14
debug 2020-12-16T13:19:25.484+0000 7fc4c3f13f40 4 rocksdb: Options.compression_opts.level: 32767
debug 2020-12-16T13:19:25.484+0000 7fc4c3f13f40 4 rocksdb: Options.compression_opts.strategy: 0
debug 2020-12-16T13:19:25.484+0000 7fc4c3f13f40 4 rocksdb: Options.compression_opts.max_dict_bytes: 0
debug 2020-12-16T13:19:25.484+0000 7fc4c3f13f40 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0
debug 2020-12-16T13:19:25.484+0000 7fc4c3f13f40 4 rocksdb: Options.compression_opts.enabled: false
debug 2020-12-16T13:19:25.484+0000 7fc4c3f13f40 4 rocksdb: Options.level0_file_num_compaction_trigger: 4
debug 2020-12-16T13:19:25.484+0000 7fc4c3f13f40 4 rocksdb: Options.level0_slowdown_writes_trigger: 20
debug 2020-12-16T13:19:25.488+0000 7fc4c3f13f40 4 rocksdb: Options.level0_stop_writes_trigger: 36
debug 2020-12-16T13:19:25.488+0000 7fc4c3f13f40 4 rocksdb: Options.target_file_size_base: 67108864
debug 2020-12-16T13:19:25.488+0000 7fc4c3f13f40 4 rocksdb: Options.target_file_size_multiplier: 1
debug 2020-12-16T13:19:25.488+0000 7fc4c3f13f40 4 rocksdb: Options.max_bytes_for_level_base: 268435456
debug 2020-12-16T13:19:25.488+0000 7fc4c3f13f40 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 0
debug 2020-12-16T13:19:25.488+0000 7fc4c3f13f40 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000
debug 2020-12-16T13:19:25.488+0000 7fc4c3f13f40 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1
debug 2020-12-16T13:19:25.488+0000 7fc4c3f13f40 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1
debug 2020-12-16T13:19:25.488+0000 7fc4c3f13f40 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1
debug 2020-12-16T13:19:25.488+0000 7fc4c3f13f40 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1
debug 2020-12-16T13:19:25.488+0000 7fc4c3f13f40 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1
debug 2020-12-16T13:19:25.488+0000 7fc4c3f13f40 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1
debug 2020-12-16T13:19:25.488+0000 7fc4c3f13f40 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1
debug 2020-12-16T13:19:25.488+0000 7fc4c3f13f40 4 rocksdb: Options.max_sequential_skip_in_iterations: 8
debug 2020-12-16T13:19:25.488+0000 7fc4c3f13f40 4 rocksdb: Options.max_compaction_bytes: 1677721600
debug 2020-12-16T13:19:25.488+0000 7fc4c3f13f40 4 rocksdb: Options.arena_block_size: 33554432
debug 2020-12-16T13:19:25.488+0000 7fc4c3f13f40 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736
debug 2020-12-16T13:19:25.488+0000 7fc4c3f13f40 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944
debug 2020-12-16T13:19:25.488+0000 7fc4c3f13f40 4 rocksdb: Options.rate_limit_delay_max_milliseconds: 100
debug 2020-12-16T13:19:25.488+0000 7fc4c3f13f40 4 rocksdb: Options.disable_auto_compactions: 0
debug 2020-12-16T13:19:25.488+0000 7fc4c3f13f40 4 rocksdb: Options.compaction_style: kCompactionStyleLevel
debug 2020-12-16T13:19:25.488+0000 7fc4c3f13f40 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio
debug 2020-12-16T13:19:25.488+0000 7fc4c3f13f40 4 rocksdb: Options.compaction_options_universal.size_ratio: 1
debug 2020-12-16T13:19:25.488+0000 7fc4c3f13f40 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2
debug 2020-12-16T13:19:25.488+0000 7fc4c3f13f40 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295
debug 2020-12-16T13:19:25.488+0000 7fc4c3f13f40 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200
debug 2020-12-16T13:19:25.488+0000 7fc4c3f13f40 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1
debug 2020-12-16T13:19:25.488+0000 7fc4c3f13f40 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize
debug 2020-12-16T13:19:25.488+0000 7fc4c3f13f40 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824
debug 2020-12-16T13:19:25.488+0000 7fc4c3f13f40 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0
debug 2020-12-16T13:19:25.488+0000 7fc4c3f13f40 4 rocksdb: Options.table_properties_collectors:
debug 2020-12-16T13:19:25.488+0000 7fc4c3f13f40 4 rocksdb: Options.inplace_update_support: 0
debug 2020-12-16T13:19:25.488+0000 7fc4c3f13f40 4 rocksdb: Options.inplace_update_num_locks: 10000
debug 2020-12-16T13:19:25.488+0000 7fc4c3f13f40 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000
debug 2020-12-16T13:19:25.488+0000 7fc4c3f13f40 4 rocksdb: Options.memtable_whole_key_filtering: 0
debug 2020-12-16T13:19:25.488+0000 7fc4c3f13f40 4 rocksdb: Options.memtable_huge_page_size: 0
debug 2020-12-16T13:19:25.488+0000 7fc4c3f13f40 4 rocksdb: Options.bloom_locality: 0
debug 2020-12-16T13:19:25.488+0000 7fc4c3f13f40 4 rocksdb: Options.max_successive_merges: 0
debug 2020-12-16T13:19:25.492+0000 7fc4c3f13f40 4 rocksdb: Options.optimize_filters_for_hits: 0
debug 2020-12-16T13:19:25.492+0000 7fc4c3f13f40 4 rocksdb: Options.paranoid_file_checks: 0
debug 2020-12-16T13:19:25.492+0000 7fc4c3f13f40 4 rocksdb: Options.force_consistency_checks: 0
debug 2020-12-16T13:19:25.492+0000 7fc4c3f13f40 4 rocksdb: Options.report_bg_io_stats: 0
debug 2020-12-16T13:19:25.492+0000 7fc4c3f13f40 4 rocksdb: Options.ttl: 0
debug 2020-12-16T13:19:25.500+0000 7fc4c3f13f40 4 rocksdb: [db/version_set.cc:3757] Recovered from manifest file:db/MANIFEST-001369 succeeded,manifest_file_number is 1369, next_file_number is 1403, last_sequence is 144598951, log_number is 1400,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0
debug 2020-12-16T13:19:25.500+0000 7fc4c3f13f40 4 rocksdb: [db/version_set.cc:3766] Column family [default] (ID 0), log number is 1400
debug 2020-12-16T13:19:25.500+0000 7fc4c3f13f40 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1608124765507105, "job": 1, "event": "recovery_started", "log_files": [1400, 1402]}
debug 2020-12-16T13:19:25.500+0000 7fc4c3f13f40 4 rocksdb: [db/db_impl_open.cc:583] Recovering log #1400 mode 0
debug 2020-12-16T13:19:27.724+0000 7fc4c3f13f40 1 bluefs _allocate failed to allocate 0x43ce43d on bdev 1, free 0x2e50000; fallback to bdev 2
debug 2020-12-16T13:19:27.724+0000 7fc4c3f13f40 1 bluefs _allocate unable to allocate 0x43ce43d on bdev 2, free 0xffffffffffffffff; fallback to slow device expander
debug 2020-12-16T13:19:27.724+0000 7fc4c3f13f40 -1 bluestore(/var/lib/ceph/osd/ceph-1) allocate_bluefs_freespace failed to allocate on 0x3d1b0000 min_size 0x43d0000 > allocated total 0x300000 bluefs_shared_alloc_size 0x10000 allocated 0x300000 available 0x b019c8000
debug 2020-12-16T13:19:27.724+0000 7fc4c3f13f40 -1 bluefs _allocate failed to expand slow device to fit +0x43ce43d
debug 2020-12-16T13:19:27.724+0000 7fc4c3f13f40 -1 bluefs _flush_range allocated: 0x0 offset: 0x0 length: 0x43ce43d
/home/jenkins-build/build/workspace/ceph-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos8/DIST/centos8/MACHINE_SIZE/gigantic/release/15.2.6/rpm/el8/BUILD/ceph-15.2.6/src/os/bluestore/BlueFS.cc: In function 'int BlueFS::_flush_range(BlueFS::FileWriter*, uint64_t, uint64_t)' thread 7fc4c3f13f40 time 2020-12-16T13:19:27.731533+0000
/home/jenkins-build/build/workspace/ceph-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos8/DIST/centos8/MACHINE_SIZE/gigantic/release/15.2.6/rpm/el8/BUILD/ceph-15.2.6/src/os/bluestore/BlueFS.cc: 2721: ceph_abort_msg("bluefs enospc")
ceph version 15.2.6 (cb8c61a60551b72614257d632a574d420064c17a) octopus (stable)
1: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)+0xe5) [0x55a0a4e0a4fe]
2: (BlueFS::_flush_range(BlueFS::FileWriter*, unsigned long, unsigned long)+0x1a43) [0x55a0a5488073]
3: (BlueFS::_flush(BlueFS::FileWriter*, bool, bool*)+0x110) [0x55a0a5488760]
4: (BlueFS::_flush(BlueFS::FileWriter*, bool, std::unique_lock<std::mutex>&)+0x32) [0x55a0a54994f2]
5: (BlueRocksWritableFile::Flush()+0x72) [0x55a0a54b87a2]
6: (rocksdb::WritableFileWriter::Flush()+0x1fc) [0x55a0a5a7ed9c]
7: (rocksdb::WritableFileWriter::Sync(bool)+0x1f) [0x55a0a5a7ef1f]
8: (rocksdb::BuildTable(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, rocksdb::Env*, rocksdb::ImmutableCFOptions const&, rocksdb::MutableCFOptions const&, rocksdb::EnvOptions const&, rocksdb::TableCache*, rocksdb::InternalIteratorBase<rocksdb::Slice>*, std::vector<std::unique_ptr<rocksdb::FragmentedRangeTombstoneIterator, std::default_delete<rocksdb::FragmentedRangeTombstoneIterator> >, std::allocator<std::unique_ptr<rocksdb::FragmentedRangeTombstoneIterator, std::default_delete<rocksdb::FragmentedRangeTombstoneIterator> > > >, rocksdb::FileMetaData*, rocksdb::InternalKeyComparator const&, std::vector<std::unique_ptr<rocksdb::IntTblPropCollectorFactory, std::default_delete<rocksdb::IntTblPropCollectorFactory> >, std::allocator<std::unique_ptr<rocksdb::IntTblPropCollectorFactory, std::default_delete<rocksdb::IntTblPropCollectorFactory> > > > const*, unsigned int, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::vector<unsigned long, std::allocator<unsigned long> >, unsigned long, rocksdb::SnapshotChecker*, rocksdb::CompressionType, unsigned long, rocksdb::CompressionOptions const&, bool, rocksdb::InternalStats*, rocksdb::TableFileCreationReason, rocksdb::EventLogger*, int, rocksdb::Env::IOPriority, rocksdb::TableProperties*, int, unsigned long, unsigned long, rocksdb::Env::WriteLifeTimeHint)+0x2be5) [0x55a0a5aa6085]
9: (rocksdb::DBImpl::WriteLevel0TableForRecovery(int, rocksdb::ColumnFamilyData*, rocksdb::MemTable*, rocksdb::VersionEdit*)+0xc07) [0x55a0a5956c97]
10: (rocksdb::DBImpl::RecoverLogFiles(std::vector<unsigned long, std::allocator<unsigned long> > const&, unsigned long*, bool)+0x12dd) [0x55a0a595879d]
11: (rocksdb::DBImpl::Recover(std::vector<rocksdb::ColumnFamilyDescriptor, std::allocator<rocksdb::ColumnFamilyDescriptor> > const&, bool, bool, bool)+0xb09) [0x55a0a5959c59]
12: (rocksdb::DBImpl::Open(rocksdb::DBOptions const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::vector<rocksdb::ColumnFamilyDescriptor, std::allocator<rocksdb::ColumnFamilyDescriptor> > const&, std::vector<rocksdb::ColumnFamilyHandle*, std::allocator<rocksdb::ColumnFamilyHandle*> >*, rocksdb::DB**, bool, bool)+0x9f5) [0x55a0a59544f5]
13: (rocksdb::DB::Open(rocksdb::DBOptions const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::vector<rocksdb::ColumnFamilyDescriptor, std::allocator<rocksdb::ColumnFamilyDescriptor> > const&, std::vector<rocksdb::ColumnFamilyHandle*, std::allocator<rocksdb::ColumnFamilyHandle*> >*, rocksdb::DB**)+0x15) [0x55a0a5955845]
14: (RocksDBStore::do_open(std::ostream&, bool, bool, std::vector<KeyValueDB::ColumnFamily, std::allocator<KeyValueDB::ColumnFamily> > const*)+0x24e4) [0x55a0a58e6984]
15: (BlueStore::_open_db(bool, bool, bool)+0x1239) [0x55a0a5386079]
16: (BlueStore::_open_db_and_around(bool)+0x19b) [0x55a0a539d0fb]
17: (BlueStore::_mount(bool, bool)+0x847) [0x55a0a53ef787]
18: (OSD::init()+0x380) [0x55a0a4f18dd0]
19: (main()+0x47e4) [0x55a0a4e6c1b4]
20: (__libc_start_main()+0xf3) [0x7fc4c08db6a3]
21: (_start()+0x2e) [0x55a0a4e9a39e]
debug 2020-12-16T13:19:27.732+0000 7fc4c3f13f40 -1 /home/jenkins-build/build/workspace/ceph-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos8/DIST/centos8/MACHINE_SIZE/gigantic/release/15.2.6/rpm/el8/BUILD/ceph-15.2.6/src/os/bluestore/BlueFS.cc: In function 'int BlueFS::_flush_range(BlueFS::FileWriter*, uint64_t, uint64_t)' thread 7fc4c3f13f40 time 2020-12-16T13:19:27.731533+0000
/home/jenkins-build/build/workspace/ceph-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos8/DIST/centos8/MACHINE_SIZE/gigantic/release/15.2.6/rpm/el8/BUILD/ceph-15.2.6/src/os/bluestore/BlueFS.cc: 2721: ceph_abort_msg("bluefs enospc")
ceph version 15.2.6 (cb8c61a60551b72614257d632a574d420064c17a) octopus (stable)
1: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)+0xe5) [0x55a0a4e0a4fe]
2: (BlueFS::_flush_range(BlueFS::FileWriter*, unsigned long, unsigned long)+0x1a43) [0x55a0a5488073]
3: (BlueFS::_flush(BlueFS::FileWriter*, bool, bool*)+0x110) [0x55a0a5488760]
4: (BlueFS::_flush(BlueFS::FileWriter*, bool, std::unique_lock<std::mutex>&)+0x32) [0x55a0a54994f2]
5: (BlueRocksWritableFile::Flush()+0x72) [0x55a0a54b87a2]
6: (rocksdb::WritableFileWriter::Flush()+0x1fc) [0x55a0a5a7ed9c]
7: (rocksdb::WritableFileWriter::Sync(bool)+0x1f) [0x55a0a5a7ef1f]
8: (rocksdb::BuildTable(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, rocksdb::Env*, rocksdb::ImmutableCFOptions const&, rocksdb::MutableCFOptions const&, rocksdb::EnvOptions const&, rocksdb::TableCache*, rocksdb::InternalIteratorBase<rocksdb::Slice>*, std::vector<std::unique_ptr<rocksdb::FragmentedRangeTombstoneIterator, std::default_delete<rocksdb::FragmentedRangeTombstoneIterator> >, std::allocator<std::unique_ptr<rocksdb::FragmentedRangeTombstoneIterator, std::default_delete<rocksdb::FragmentedRangeTombstoneIterator> > > >, rocksdb::FileMetaData*, rocksdb::InternalKeyComparator const&, std::vector<std::unique_ptr<rocksdb::IntTblPropCollectorFactory, std::default_delete<rocksdb::IntTblPropCollectorFactory> >, std::allocator<std::unique_ptr<rocksdb::IntTblPropCollectorFactory, std::default_delete<rocksdb::IntTblPropCollectorFactory> > > > const*, unsigned int, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::vector<unsigned long, std::allocator<unsigned long> >, unsigned long, rocksdb::SnapshotChecker*, rocksdb::CompressionType, unsigned long, rocksdb::CompressionOptions const&, bool, rocksdb::InternalStats*, rocksdb::TableFileCreationReason, rocksdb::EventLogger*, int, rocksdb::Env::IOPriority, rocksdb::TableProperties*, int, unsigned long, unsigned long, rocksdb::Env::WriteLifeTimeHint)+0x2be5) [0x55a0a5aa6085]
9: (rocksdb::DBImpl::WriteLevel0TableForRecovery(int, rocksdb::ColumnFamilyData*, rocksdb::MemTable*, rocksdb::VersionEdit*)+0xc07) [0x55a0a5956c97]
10: (rocksdb::DBImpl::RecoverLogFiles(std::vector<unsigned long, std::allocator<unsigned long> > const&, unsigned long*, bool)+0x12dd) [0x55a0a595879d]
11: (rocksdb::DBImpl::Recover(std::vector<rocksdb::ColumnFamilyDescriptor, std::allocator<rocksdb::ColumnFamilyDescriptor> > const&, bool, bool, bool)+0xb09) [0x55a0a5959c59]
12: (rocksdb::DBImpl::Open(rocksdb::DBOptions const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::vector<rocksdb::ColumnFamilyDescriptor, std::allocator<rocksdb::ColumnFamilyDescriptor> > const&, std::vector<rocksdb::ColumnFamilyHandle*, std::allocator<rocksdb::ColumnFamilyHandle*> >*, rocksdb::DB**, bool, bool)+0x9f5) [0x55a0a59544f5]
13: (rocksdb::DB::Open(rocksdb::DBOptions const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::vector<rocksdb::ColumnFamilyDescriptor, std::allocator<rocksdb::ColumnFamilyDescriptor> > const&, std::vector<rocksdb::ColumnFamilyHandle*, std::allocator<rocksdb::ColumnFamilyHandle*> >*, rocksdb::DB**)+0x15) [0x55a0a5955845]
14: (RocksDBStore::do_open(std::ostream&, bool, bool, std::vector<KeyValueDB::ColumnFamily, std::allocator<KeyValueDB::ColumnFamily> > const*)+0x24e4) [0x55a0a58e6984]
15: (BlueStore::_open_db(bool, bool, bool)+0x1239) [0x55a0a5386079]
16: (BlueStore::_open_db_and_around(bool)+0x19b) [0x55a0a539d0fb]
17: (BlueStore::_mount(bool, bool)+0x847) [0x55a0a53ef787]
18: (OSD::init()+0x380) [0x55a0a4f18dd0]
19: (main()+0x47e4) [0x55a0a4e6c1b4]
20: (__libc_start_main()+0xf3) [0x7fc4c08db6a3]
21: (_start()+0x2e) [0x55a0a4e9a39e]
*** Caught signal (Aborted) **
in thread 7fc4c3f13f40 thread_name:ceph-osd
ceph version 15.2.6 (cb8c61a60551b72614257d632a574d420064c17a) octopus (stable)
1: (()+0x12dd0) [0x7fc4c1c86dd0]
2: (gsignal()+0x10f) [0x7fc4c08ef70f]
3: (abort()+0x127) [0x7fc4c08d9b25]
4: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)+0x1b6) [0x55a0a4e0a5cf]
5: (BlueFS::_flush_range(BlueFS::FileWriter*, unsigned long, unsigned long)+0x1a43) [0x55a0a5488073]
6: (BlueFS::_flush(BlueFS::FileWriter*, bool, bool*)+0x110) [0x55a0a5488760]
7: (BlueFS::_flush(BlueFS::FileWriter*, bool, std::unique_lock<std::mutex>&)+0x32) [0x55a0a54994f2]
8: (BlueRocksWritableFile::Flush()+0x72) [0x55a0a54b87a2]
9: (rocksdb::WritableFileWriter::Flush()+0x1fc) [0x55a0a5a7ed9c]
10: (rocksdb::WritableFileWriter::Sync(bool)+0x1f) [0x55a0a5a7ef1f]
11: (rocksdb::BuildTable(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, rocksdb::Env*, rocksdb::ImmutableCFOptions const&, rocksdb::MutableCFOptions const&, rocksdb::EnvOptions const&, rocksdb::TableCache*, rocksdb::InternalIteratorBase<rocksdb::Slice>*, std::vector<std::unique_ptr<rocksdb::FragmentedRangeTombstoneIterator, std::default_delete<rocksdb::FragmentedRangeTombstoneIterator> >, std::allocator<std::unique_ptr<rocksdb::FragmentedRangeTombstoneIterator, std::default_delete<rocksdb::FragmentedRangeTombstoneIterator> > > >, rocksdb::FileMetaData*, rocksdb::InternalKeyComparator const&, std::vector<std::unique_ptr<rocksdb::IntTblPropCollectorFactory, std::default_delete<rocksdb::IntTblPropCollectorFactory> >, std::allocator<std::unique_ptr<rocksdb::IntTblPropCollectorFactory, std::default_delete<rocksdb::IntTblPropCollectorFactory> > > > const*, unsigned int, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::vector<unsigned long, std::allocator<unsigned long> >, unsigned long, rocksdb::SnapshotChecker*, rocksdb::CompressionType, unsigned long, rocksdb::CompressionOptions const&, bool, rocksdb::InternalStats*, rocksdb::TableFileCreationReason, rocksdb::EventLogger*, int, rocksdb::Env::IOPriority, rocksdb::TableProperties*, int, unsigned long, unsigned long, rocksdb::Env::WriteLifeTimeHint)+0x2be5) [0x55a0a5aa6085]
12: (rocksdb::DBImpl::WriteLevel0TableForRecovery(int, rocksdb::ColumnFamilyData*, rocksdb::MemTable*, rocksdb::VersionEdit*)+0xc07) [0x55a0a5956c97]
13: (rocksdb::DBImpl::RecoverLogFiles(std::vector<unsigned long, std::allocator<unsigned long> > const&, unsigned long*, bool)+0x12dd) [0x55a0a595879d]
14: (rocksdb::DBImpl::Recover(std::vector<rocksdb::ColumnFamilyDescriptor, std::allocator<rocksdb::ColumnFamilyDescriptor> > const&, bool, bool, bool)+0xb09) [0x55a0a5959c59]
15: (rocksdb::DBImpl::Open(rocksdb::DBOptions const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::vector<rocksdb::ColumnFamilyDescriptor, std::allocator<rocksdb::ColumnFamilyDescriptor> > const&, std::vector<rocksdb::ColumnFamilyHandle*, std::allocator<rocksdb::ColumnFamilyHandle*> >*, rocksdb::DB**, bool, bool)+0x9f5) [0x55a0a59544f5]
16: (rocksdb::DB::Open(rocksdb::DBOptions const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::vector<rocksdb::ColumnFamilyDescriptor, std::allocator<rocksdb::ColumnFamilyDescriptor> > const&, std::vector<rocksdb::ColumnFamilyHandle*, std::allocator<rocksdb::ColumnFamilyHandle*> >*, rocksdb::DB**)+0x15) [0x55a0a5955845]
17: (RocksDBStore::do_open(std::ostream&, bool, bool, std::vector<KeyValueDB::ColumnFamily, std::allocator<KeyValueDB::ColumnFamily> > const*)+0x24e4) [0x55a0a58e6984]
18: (BlueStore::_open_db(bool, bool, bool)+0x1239) [0x55a0a5386079]
19: (BlueStore::_open_db_and_around(bool)+0x19b) [0x55a0a539d0fb]
20: (BlueStore::_mount(bool, bool)+0x847) [0x55a0a53ef787]
21: (OSD::init()+0x380) [0x55a0a4f18dd0]
22: (main()+0x47e4) [0x55a0a4e6c1b4]
23: (__libc_start_main()+0xf3) [0x7fc4c08db6a3]
24: (_start()+0x2e) [0x55a0a4e9a39e]
debug 2020-12-16T13:19:27.740+0000 7fc4c3f13f40 -1 *** Caught signal (Aborted) **
in thread 7fc4c3f13f40 thread_name:ceph-osd
ceph version 15.2.6 (cb8c61a60551b72614257d632a574d420064c17a) octopus (stable)
1: (()+0x12dd0) [0x7fc4c1c86dd0]
2: (gsignal()+0x10f) [0x7fc4c08ef70f]
3: (abort()+0x127) [0x7fc4c08d9b25]
4: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)+0x1b6) [0x55a0a4e0a5cf]
5: (BlueFS::_flush_range(BlueFS::FileWriter*, unsigned long, unsigned long)+0x1a43) [0x55a0a5488073]
6: (BlueFS::_flush(BlueFS::FileWriter*, bool, bool*)+0x110) [0x55a0a5488760]
7: (BlueFS::_flush(BlueFS::FileWriter*, bool, std::unique_lock<std::mutex>&)+0x32) [0x55a0a54994f2]
8: (BlueRocksWritableFile::Flush()+0x72) [0x55a0a54b87a2]
9: (rocksdb::WritableFileWriter::Flush()+0x1fc) [0x55a0a5a7ed9c]
10: (rocksdb::WritableFileWriter::Sync(bool)+0x1f) [0x55a0a5a7ef1f]
11: (rocksdb::BuildTable(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, rocksdb::Env*, rocksdb::ImmutableCFOptions const&, rocksdb::MutableCFOptions const&, rocksdb::EnvOptions const&, rocksdb::TableCache*, rocksdb::InternalIteratorBase<rocksdb::Slice>*, std::vector<std::unique_ptr<rocksdb::FragmentedRangeTombstoneIterator, std::default_delete<rocksdb::FragmentedRangeTombstoneIterator> >, std::allocator<std::unique_ptr<rocksdb::FragmentedRangeTombstoneIterator, std::default_delete<rocksdb::FragmentedRangeTombstoneIterator> > > >, rocksdb::FileMetaData*, rocksdb::InternalKeyComparator const&, std::vector<std::unique_ptr<rocksdb::IntTblPropCollectorFactory, std::default_delete<rocksdb::IntTblPropCollectorFactory> >, std::allocator<std::unique_ptr<rocksdb::IntTblPropCollectorFactory, std::default_delete<rocksdb::IntTblPropCollectorFactory> > > > const*, unsigned int, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::vector<unsigned long, std::allocator<unsigned long> >, unsigned long, rocksdb::SnapshotChecker*, rocksdb::CompressionType, unsigned long, rocksdb::CompressionOptions const&, bool, rocksdb::InternalStats*, rocksdb::TableFileCreationReason, rocksdb::EventLogger*, int, rocksdb::Env::IOPriority, rocksdb::TableProperties*, int, unsigned long, unsigned long, rocksdb::Env::WriteLifeTimeHint)+0x2be5) [0x55a0a5aa6085]
12: (rocksdb::DBImpl::WriteLevel0TableForRecovery(int, rocksdb::ColumnFamilyData*, rocksdb::MemTable*, rocksdb::VersionEdit*)+0xc07) [0x55a0a5956c97]
13: (rocksdb::DBImpl::RecoverLogFiles(std::vector<unsigned long, std::allocator<unsigned long> > const&, unsigned long*, bool)+0x12dd) [0x55a0a595879d]
14: (rocksdb::DBImpl::Recover(std::vector<rocksdb::ColumnFamilyDescriptor, std::allocator<rocksdb::ColumnFamilyDescriptor> > const&, bool, bool, bool)+0xb09) [0x55a0a5959c59]
15: (rocksdb::DBImpl::Open(rocksdb::DBOptions const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::vector<rocksdb::ColumnFamilyDescriptor, std::allocator<rocksdb::ColumnFamilyDescriptor> > const&, std::vector<rocksdb::ColumnFamilyHandle*, std::allocator<rocksdb::ColumnFamilyHandle*> >*, rocksdb::DB**, bool, bool)+0x9f5) [0x55a0a59544f5]
16: (rocksdb::DB::Open(rocksdb::DBOptions const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::vector<rocksdb::ColumnFamilyDescriptor, std::allocator<rocksdb::ColumnFamilyDescriptor> > const&, std::vector<rocksdb::ColumnFamilyHandle*, std::allocator<rocksdb::ColumnFamilyHandle*> >*, rocksdb::DB**)+0x15) [0x55a0a5955845]
17: (RocksDBStore::do_open(std::ostream&, bool, bool, std::vector<KeyValueDB::ColumnFamily, std::allocator<KeyValueDB::ColumnFamily> > const*)+0x24e4) [0x55a0a58e6984]
18: (BlueStore::_open_db(bool, bool, bool)+0x1239) [0x55a0a5386079]
19: (BlueStore::_open_db_and_around(bool)+0x19b) [0x55a0a539d0fb]
20: (BlueStore::_mount(bool, bool)+0x847) [0x55a0a53ef787]
21: (OSD::init()+0x380) [0x55a0a4f18dd0]
22: (main()+0x47e4) [0x55a0a4e6c1b4]
23: (__libc_start_main()+0xf3) [0x7fc4c08db6a3]
24: (_start()+0x2e) [0x55a0a4e9a39e]
NOTE: a copy of the executable, or `objdump -rdS <executable>` is needed to interpret this.
--- begin dump of recent events ---
debug -696> 2020-12-16T13:19:21.748+0000 7fc4c3f13f40 5 asok(0x55a0b0344000) register_command assert hook 0x55a0b02985b0
debug -695> 2020-12-16T13:19:21.748+0000 7fc4c3f13f40 5 asok(0x55a0b0344000) register_command abort hook 0x55a0b02985b0
debug -694> 2020-12-16T13:19:21.748+0000 7fc4c3f13f40 5 asok(0x55a0b0344000) register_command leak_some_memory hook 0x55a0b02985b0
debug -693> 2020-12-16T13:19:21.748+0000 7fc4c3f13f40 5 asok(0x55a0b0344000) register_command perfcounters_dump hook 0x55a0b02985b0
debug -692> 2020-12-16T13:19:21.748+0000 7fc4c3f13f40 5 asok(0x55a0b0344000) register_command 1 hook 0x55a0b02985b0
debug -691> 2020-12-16T13:19:21.748+0000 7fc4c3f13f40 5 asok(0x55a0b0344000) register_command perf dump hook 0x55a0b02985b0
debug -690> 2020-12-16T13:19:21.748+0000 7fc4c3f13f40 5 asok(0x55a0b0344000) register_command perfcounters_schema hook 0x55a0b02985b0
debug -689> 2020-12-16T13:19:21.748+0000 7fc4c3f13f40 5 asok(0x55a0b0344000) register_command perf histogram dump hook 0x55a0b02985b0
debug -688> 2020-12-16T13:19:21.748+0000 7fc4c3f13f40 5 asok(0x55a0b0344000) register_command 2 hook 0x55a0b02985b0
debug -687> 2020-12-16T13:19:21.748+0000 7fc4c3f13f40 5 asok(0x55a0b0344000) register_command perf schema hook 0x55a0b02985b0
debug -686> 2020-12-16T13:19:21.748+0000 7fc4c3f13f40 5 asok(0x55a0b0344000) register_command perf histogram schema hook 0x55a0b02985b0
debug -685> 2020-12-16T13:19:21.748+0000 7fc4c3f13f40 5 asok(0x55a0b0344000) register_command perf reset hook 0x55a0b02985b0
debug -684> 2020-12-16T13:19:21.748+0000 7fc4c3f13f40 5 asok(0x55a0b0344000) register_command config show hook 0x55a0b02985b0
debug -683> 2020-12-16T13:19:21.748+0000 7fc4c3f13f40 5 asok(0x55a0b0344000) register_command config help hook 0x55a0b02985b0
debug -682> 2020-12-16T13:19:21.748+0000 7fc4c3f13f40 5 asok(0x55a0b0344000) register_command config set hook 0x55a0b02985b0
debug -681> 2020-12-16T13:19:21.748+0000 7fc4c3f13f40 5 asok(0x55a0b0344000) register_command config unset hook 0x55a0b02985b0
debug -680> 2020-12-16T13:19:21.748+0000 7fc4c3f13f40 5 asok(0x55a0b0344000) register_command config get hook 0x55a0b02985b0
debug -679> 2020-12-16T13:19:21.748+0000 7fc4c3f13f40 5 asok(0x55a0b0344000) register_command config diff hook 0x55a0b02985b0
debug -678> 2020-12-16T13:19:21.748+0000 7fc4c3f13f40 5 asok(0x55a0b0344000) register_command config diff get hook 0x55a0b02985b0
debug -677> 2020-12-16T13:19:21.748+0000 7fc4c3f13f40 5 asok(0x55a0b0344000) register_command injectargs hook 0x55a0b02985b0
debug -676> 2020-12-16T13:19:21.748+0000 7fc4c3f13f40 5 asok(0x55a0b0344000) register_command log flush hook 0x55a0b02985b0
debug -675> 2020-12-16T13:19:21.748+0000 7fc4c3f13f40 5 asok(0x55a0b0344000) register_command log dump hook 0x55a0b02985b0
debug -674> 2020-12-16T13:19:21.748+0000 7fc4c3f13f40 5 asok(0x55a0b0344000) register_command log reopen hook 0x55a0b02985b0
debug -673> 2020-12-16T13:19:21.748+0000 7fc4c3f13f40 5 asok(0x55a0b0344000) register_command dump_mempools hook 0x55a0b0f18068
debug -672> 2020-12-16T13:19:21.764+0000 7fc4c3f13f40 10 monclient: get_monmap_and_config
debug -671> 2020-12-16T13:19:21.764+0000 7fc4c3f13f40 10 monclient: build_initial_monmap
debug -670> 2020-12-16T13:19:21.764+0000 7fc4c3f13f40 10 monclient: monmap:
epoch 0
fsid 39941002-eb2d-498a-bab4-04961fb33a42
last_changed 2020-12-16T13:19:21.772470+0000
created 2020-12-16T13:19:21.772470+0000
min_mon_release 0 (unknown)
0: [v2:172.31.36.158:3300/0,v1:172.31.36.158:6789/0] mon.noname-b
1: [v2:172.31.225.8:3300/0,v1:172.31.225.8:6789/0] mon.noname-a
debug -669> 2020-12-16T13:19:21.768+0000 7fc4c3f13f40 5 AuthRegistry(0x55a0b032e940) adding auth protocol: cephx
debug -668> 2020-12-16T13:19:21.768+0000 7fc4c3f13f40 5 AuthRegistry(0x55a0b032e940) adding auth protocol: cephx
debug -667> 2020-12-16T13:19:21.768+0000 7fc4c3f13f40 5 AuthRegistry(0x55a0b032e940) adding auth protocol: cephx
debug -666> 2020-12-16T13:19:21.768+0000 7fc4c3f13f40 5 AuthRegistry(0x55a0b032e940) adding auth protocol: none
debug -665> 2020-12-16T13:19:21.768+0000 7fc4c3f13f40 5 AuthRegistry(0x55a0b032e940) adding con mode: secure
debug -664> 2020-12-16T13:19:21.768+0000 7fc4c3f13f40 5 AuthRegistry(0x55a0b032e940) adding con mode: crc
debug -663> 2020-12-16T13:19:21.768+0000 7fc4c3f13f40 5 AuthRegistry(0x55a0b032e940) adding con mode: secure
debug -662> 2020-12-16T13:19:21.768+0000 7fc4c3f13f40 5 AuthRegistry(0x55a0b032e940) adding con mode: crc
debug -661> 2020-12-16T13:19:21.768+0000 7fc4c3f13f40 5 AuthRegistry(0x55a0b032e940) adding con mode: secure
debug -660> 2020-12-16T13:19:21.768+0000 7fc4c3f13f40 5 AuthRegistry(0x55a0b032e940) adding con mode: crc
debug -659> 2020-12-16T13:19:21.768+0000 7fc4c3f13f40 5 AuthRegistry(0x55a0b032e940) adding con mode: crc
debug -658> 2020-12-16T13:19:21.768+0000 7fc4c3f13f40 5 AuthRegistry(0x55a0b032e940) adding con mode: secure
debug -657> 2020-12-16T13:19:21.768+0000 7fc4c3f13f40 5 AuthRegistry(0x55a0b032e940) adding con mode: crc
debug -656> 2020-12-16T13:19:21.768+0000 7fc4c3f13f40 5 AuthRegistry(0x55a0b032e940) adding con mode: secure
debug -655> 2020-12-16T13:19:21.768+0000 7fc4c3f13f40 5 AuthRegistry(0x55a0b032e940) adding con mode: crc
debug -654> 2020-12-16T13:19:21.768+0000 7fc4c3f13f40 5 AuthRegistry(0x55a0b032e940) adding con mode: secure
debug -653> 2020-12-16T13:19:21.768+0000 7fc4c3f13f40 2 auth: KeyRing::load: loaded key file /var/lib/ceph/osd/ceph-1/keyring
debug -652> 2020-12-16T13:19:21.768+0000 7fc4c3f13f40 10 monclient: init
debug -651> 2020-12-16T13:19:21.768+0000 7fc4c3f13f40 5 AuthRegistry(0x7ffcedef1a30) adding auth protocol: cephx
debug -650> 2020-12-16T13:19:21.768+0000 7fc4c3f13f40 5 AuthRegistry(0x7ffcedef1a30) adding auth protocol: cephx
debug -649> 2020-12-16T13:19:21.768+0000 7fc4c3f13f40 5 AuthRegistry(0x7ffcedef1a30) adding auth protocol: cephx
debug -648> 2020-12-16T13:19:21.768+0000 7fc4c3f13f40 5 AuthRegistry(0x7ffcedef1a30) adding auth protocol: none
debug -647> 2020-12-16T13:19:21.768+0000 7fc4c3f13f40 5 AuthRegistry(0x7ffcedef1a30) adding con mode: secure
debug -646> 2020-12-16T13:19:21.768+0000 7fc4c3f13f40 5 AuthRegistry(0x7ffcedef1a30) adding con mode: crc
debug -645> 2020-12-16T13:19:21.768+0000 7fc4c3f13f40 5 AuthRegistry(0x7ffcedef1a30) adding con mode: secure
debug -644> 2020-12-16T13:19:21.768+0000 7fc4c3f13f40 5 AuthRegistry(0x7ffcedef1a30) adding con mode: crc
debug -643> 2020-12-16T13:19:21.768+0000 7fc4c3f13f40 5 AuthRegistry(0x7ffcedef1a30) adding con mode: secure
debug -642> 2020-12-16T13:19:21.768+0000 7fc4c3f13f40 5 AuthRegistry(0x7ffcedef1a30) adding con mode: crc
debug -641> 2020-12-16T13:19:21.768+0000 7fc4c3f13f40 5 AuthRegistry(0x7ffcedef1a30) adding con mode: crc
debug -640> 2020-12-16T13:19:21.768+0000 7fc4c3f13f40 5 AuthRegistry(0x7ffcedef1a30) adding con mode: secure
debug -639> 2020-12-16T13:19:21.768+0000 7fc4c3f13f40 5 AuthRegistry(0x7ffcedef1a30) adding con mode: crc
debug -638> 2020-12-16T13:19:21.768+0000 7fc4c3f13f40 5 AuthRegistry(0x7ffcedef1a30) adding con mode: secure
debug -637> 2020-12-16T13:19:21.768+0000 7fc4c3f13f40 5 AuthRegistry(0x7ffcedef1a30) adding con mode: crc
debug -636> 2020-12-16T13:19:21.768+0000 7fc4c3f13f40 5 AuthRegistry(0x7ffcedef1a30) adding con mode: secure
debug -635> 2020-12-16T13:19:21.768+0000 7fc4c3f13f40 2 auth: KeyRing::load: loaded key file /var/lib/ceph/osd/ceph-1/keyring
debug -634> 2020-12-16T13:19:21.768+0000 7fc4c3f13f40 2 auth: KeyRing::load: loaded key file /var/lib/ceph/osd/ceph-1/keyring
debug -633> 2020-12-16T13:19:21.768+0000 7fc4c3f13f40 10 monclient: _reopen_session rank -1
debug -632> 2020-12-16T13:19:21.768+0000 7fc4c3f13f40 10 monclient: _add_conns ranks=[1,0]
debug -631> 2020-12-16T13:19:21.768+0000 7fc4c3f13f40 10 monclient(hunting): picked mon.noname-a con 0x55a0b0342c00 addr [v2:172.31.225.8:3300/0,v1:172.31.225.8:6789/0]
debug -630> 2020-12-16T13:19:21.768+0000 7fc4c3f13f40 10 monclient(hunting): picked mon.noname-b con 0x55a0b0343000 addr [v2:172.31.36.158:3300/0,v1:172.31.36.158:6789/0]
debug -629> 2020-12-16T13:19:21.768+0000 7fc4c3f13f40 10 monclient(hunting): start opening mon connection
debug -628> 2020-12-16T13:19:21.768+0000 7fc4c3f13f40 10 monclient(hunting): start opening mon connection
debug -627> 2020-12-16T13:19:21.768+0000 7fc4c3f13f40 10 monclient(hunting): _renew_subs
debug -626> 2020-12-16T13:19:21.768+0000 7fc4c3f13f40 10 monclient(hunting): authenticate will time out at 2020-12-16T13:24:21.774889+0000
debug -625> 2020-12-16T13:19:21.768+0000 7fc4bd495700 10 monclient(hunting): get_auth_request con 0x55a0b0343000 auth_method 0
debug -624> 2020-12-16T13:19:21.768+0000 7fc4bd495700 10 monclient(hunting): get_auth_request method 2 preferred_modes [2,1]
debug -623> 2020-12-16T13:19:21.768+0000 7fc4bd495700 10 monclient(hunting): _init_auth method 2
debug -622> 2020-12-16T13:19:21.768+0000 7fc4bdc96700 10 monclient(hunting): get_auth_request con 0x55a0b0342c00 auth_method 0
debug -621> 2020-12-16T13:19:21.768+0000 7fc4bdc96700 10 monclient(hunting): get_auth_request method 2 preferred_modes [2,1]
debug -620> 2020-12-16T13:19:21.768+0000 7fc4bdc96700 10 monclient(hunting): _init_auth method 2
debug -619> 2020-12-16T13:19:21.768+0000 7fc4bd495700 10 monclient(hunting): handle_auth_reply_more payload 9
debug -618> 2020-12-16T13:19:21.768+0000 7fc4bd495700 10 monclient(hunting): handle_auth_reply_more payload_len 9
debug -617> 2020-12-16T13:19:21.768+0000 7fc4bd495700 10 monclient(hunting): handle_auth_reply_more responding with 36 bytes
debug -616> 2020-12-16T13:19:21.768+0000 7fc4bdc96700 10 monclient(hunting): handle_auth_reply_more payload 9
debug -615> 2020-12-16T13:19:21.768+0000 7fc4bdc96700 10 monclient(hunting): handle_auth_reply_more payload_len 9
debug -614> 2020-12-16T13:19:21.768+0000 7fc4bdc96700 10 monclient(hunting): handle_auth_reply_more responding with 36 bytes
debug -613> 2020-12-16T13:19:21.772+0000 7fc4bd495700 10 monclient(hunting): handle_auth_done global_id 14765571 payload 450
debug -612> 2020-12-16T13:19:21.772+0000 7fc4bd495700 10 monclient: _finish_hunting 0
debug -611> 2020-12-16T13:19:21.772+0000 7fc4bd495700 1 monclient: found mon.noname-b
debug -610> 2020-12-16T13:19:21.772+0000 7fc4bd495700 10 monclient: _send_mon_message to mon.noname-b at v2:172.31.36.158:3300/0
debug -609> 2020-12-16T13:19:21.772+0000 7fc4bcc94700 10 monclient: handle_monmap mon_map magic: 0 v1
debug -608> 2020-12-16T13:19:21.772+0000 7fc4bcc94700 10 monclient: got monmap 24 from mon.noname-b (according to old e24)
debug -607> 2020-12-16T13:19:21.772+0000 7fc4bcc94700 10 monclient: dump:
epoch 24
fsid 39941002-eb2d-498a-bab4-04961fb33a42
last_changed 2020-12-16T09:55:13.010421+0000
created 2019-11-21T16:05:56.597779+0000
min_mon_release 15 (octopus)
0: [v2:172.31.36.158:3300/0,v1:172.31.36.158:6789/0] mon.f
1: [v2:172.31.225.8:3300/0,v1:172.31.225.8:6789/0] mon.k
2: [v2:172.31.163.187:3300/0,v1:172.31.163.187:6789/0] mon.m
debug -606> 2020-12-16T13:19:21.772+0000 7fc4bcc94700 10 monclient: _finish_auth 0
debug -605> 2020-12-16T13:19:21.772+0000 7fc4bcc94700 10 monclient: _check_auth_rotating renewing rotating keys (they expired before 2020-12-16T13:18:51.777665+0000)
debug -604> 2020-12-16T13:19:21.772+0000 7fc4bcc94700 10 monclient: _send_mon_message to mon.f at v2:172.31.36.158:3300/0
debug -603> 2020-12-16T13:19:21.772+0000 7fc4bcc94700 10 monclient: handle_config config(7 keys) v1
debug -602> 2020-12-16T13:19:21.772+0000 7fc4bcc94700 10 monclient: handle_monmap mon_map magic: 0 v1
debug -601> 2020-12-16T13:19:21.772+0000 7fc4bcc94700 10 monclient: got monmap 24 from mon.f (according to old e24)
debug -600> 2020-12-16T13:19:21.772+0000 7fc4bcc94700 10 monclient: dump:
epoch 24
fsid 39941002-eb2d-498a-bab4-04961fb33a42
last_changed 2020-12-16T09:55:13.010421+0000
created 2019-11-21T16:05:56.597779+0000
min_mon_release 15 (octopus)
0: [v2:172.31.36.158:3300/0,v1:172.31.36.158:6789/0] mon.f
1: [v2:172.31.225.8:3300/0,v1:172.31.225.8:6789/0] mon.k
2: [v2:172.31.163.187:3300/0,v1:172.31.163.187:6789/0] mon.m
debug -599> 2020-12-16T13:19:21.772+0000 7fc4bb491700 4 set_mon_vals no callback set
debug -598> 2020-12-16T13:19:21.772+0000 7fc4bb491700 10 set_mon_vals bluestore_warn_on_legacy_statfs = false
debug -597> 2020-12-16T13:19:21.772+0000 7fc4c3f13f40 5 monclient: authenticate success, global_id 14765571
debug -596> 2020-12-16T13:19:21.772+0000 7fc4bb491700 10 set_mon_vals log_file =
debug -595> 2020-12-16T13:19:21.772+0000 7fc4bb491700 10 set_mon_vals mon_allow_pool_delete = true
debug -594> 2020-12-16T13:19:21.772+0000 7fc4c3f13f40 10 monclient: get_monmap_and_config success
debug -593> 2020-12-16T13:19:21.772+0000 7fc4bb491700 10 set_mon_vals mon_cluster_log_file =
debug -592> 2020-12-16T13:19:21.772+0000 7fc4c3f13f40 10 monclient: shutdown
debug -591> 2020-12-16T13:19:21.772+0000 7fc4bb491700 10 set_mon_vals osd_pool_default_pg_autoscale_mode = on
debug -590> 2020-12-16T13:19:21.772+0000 7fc4bb491700 10 set_mon_vals rbd_default_features = 3
debug -589> 2020-12-16T13:19:21.772+0000 7fc4c3f13f40 0 set uid:gid to 167:167 (ceph:ceph)
debug -588> 2020-12-16T13:19:21.772+0000 7fc4c3f13f40 0 ceph version 15.2.6 (cb8c61a60551b72614257d632a574d420064c17a) octopus (stable), process ceph-osd, pid 1
debug -587> 2020-12-16T13:19:21.772+0000 7fc4c3f13f40 0 pidfile_write: ignore empty --pid-file
debug -586> 2020-12-16T13:19:21.772+0000 7fc4c3f13f40 5 asok(0x55a0b0344000) init /var/run/ceph/ceph-osd.1.asok
debug -585> 2020-12-16T13:19:21.772+0000 7fc4c3f13f40 5 asok(0x55a0b0344000) bind_and_listen /var/run/ceph/ceph-osd.1.asok
debug -584> 2020-12-16T13:19:21.772+0000 7fc4c3f13f40 5 asok(0x55a0b0344000) register_command 0 hook 0x55a0b029c368
debug -583> 2020-12-16T13:19:21.772+0000 7fc4c3f13f40 5 asok(0x55a0b0344000) register_command version hook 0x55a0b029c368
debug -582> 2020-12-16T13:19:21.772+0000 7fc4c3f13f40 5 asok(0x55a0b0344000) register_command git_version hook 0x55a0b029c368
debug -581> 2020-12-16T13:19:21.772+0000 7fc4c3f13f40 5 asok(0x55a0b0344000) register_command help hook 0x55a0b02983c0
debug -580> 2020-12-16T13:19:21.772+0000 7fc4c3f13f40 5 asok(0x55a0b0344000) register_command get_command_descriptions hook 0x55a0b0298070
debug -579> 2020-12-16T13:19:21.772+0000 7fc4bc493700 5 asok(0x55a0b0344000) entry start
debug -578> 2020-12-16T13:19:21.772+0000 7fc4c3f13f40 5 object store type is bluestore
debug -577> 2020-12-16T13:19:21.776+0000 7fc4c3f13f40 1 bdev create path /var/lib/ceph/osd/ceph-1/block type kernel
debug -576> 2020-12-16T13:19:21.776+0000 7fc4c3f13f40 1 bdev(0x55a0b0fd0000 /var/lib/ceph/osd/ceph-1/block) open path /var/lib/ceph/osd/ceph-1/block
debug -575> 2020-12-16T13:19:21.776+0000 7fc4c3f13f40 1 bdev(0x55a0b0fd0000 /var/lib/ceph/osd/ceph-1/block) open size 107369988096 (0x18ffc00000, 100 GiB) block_size 4096 (4 KiB) non-rotational discard not supported
debug -574> 2020-12-16T13:19:21.776+0000 7fc4c3f13f40 1 bluestore(/var/lib/ceph/osd/ceph-1) _set_cache_sizes cache_size 3221225472 meta 0.4 kv 0.4 data 0.2
debug -573> 2020-12-16T13:19:21.776+0000 7fc4c3f13f40 5 asok(0x55a0b0344000) register_command bluestore bluefs available hook 0x55a0b02980b0
debug -572> 2020-12-16T13:19:21.776+0000 7fc4c3f13f40 5 asok(0x55a0b0344000) register_command bluefs stats hook 0x55a0b02980b0
debug -571> 2020-12-16T13:19:21.776+0000 7fc4c3f13f40 1 bdev create path /var/lib/ceph/osd/ceph-1/block type kernel
debug -570> 2020-12-16T13:19:21.776+0000 7fc4c3f13f40 1 bdev(0x55a0b0fd0700 /var/lib/ceph/osd/ceph-1/block) open path /var/lib/ceph/osd/ceph-1/block
debug -569> 2020-12-16T13:19:21.776+0000 7fc4c3f13f40 1 bdev(0x55a0b0fd0700 /var/lib/ceph/osd/ceph-1/block) open size 107369988096 (0x18ffc00000, 100 GiB) block_size 4096 (4 KiB) non-rotational discard not supported
debug -568> 2020-12-16T13:19:21.776+0000 7fc4c3f13f40 1 bluefs add_block_device bdev 1 path /var/lib/ceph/osd/ceph-1/block size 100 GiB
debug -567> 2020-12-16T13:19:21.776+0000 7fc4c3f13f40 5 asok(0x55a0b0344000) unregister_commands bluefs stats
debug -566> 2020-12-16T13:19:21.776+0000 7fc4c3f13f40 5 asok(0x55a0b0344000) unregister_commands bluestore bluefs available
debug -565> 2020-12-16T13:19:21.776+0000 7fc4c3f13f40 1 bdev(0x55a0b0fd0700 /var/lib/ceph/osd/ceph-1/block) close
debug -564> 2020-12-16T13:19:22.084+0000 7fc4c3f13f40 1 bdev(0x55a0b0fd0000 /var/lib/ceph/osd/ceph-1/block) close
debug -563> 2020-12-16T13:19:22.316+0000 7fc4c3f13f40 5 AuthRegistry(0x55a0b032e940) adding auth protocol: cephx
debug -562> 2020-12-16T13:19:22.316+0000 7fc4c3f13f40 5 AuthRegistry(0x55a0b032e940) adding auth protocol: cephx
debug -561> 2020-12-16T13:19:22.316+0000 7fc4c3f13f40 5 AuthRegistry(0x55a0b032e940) adding auth protocol: cephx
debug -560> 2020-12-16T13:19:22.316+0000 7fc4c3f13f40 5 AuthRegistry(0x55a0b032e940) adding auth protocol: none
debug -559> 2020-12-16T13:19:22.316+0000 7fc4c3f13f40 5 AuthRegistry(0x55a0b032e940) adding con mode: secure
debug -558> 2020-12-16T13:19:22.316+0000 7fc4c3f13f40 5 AuthRegistry(0x55a0b032e940) adding con mode: crc
debug -557> 2020-12-16T13:19:22.316+0000 7fc4c3f13f40 5 AuthRegistry(0x55a0b032e940) adding con mode: secure
debug -556> 2020-12-16T13:19:22.316+0000 7fc4c3f13f40 5 AuthRegistry(0x55a0b032e940) adding con mode: crc
debug -555> 2020-12-16T13:19:22.316+0000 7fc4c3f13f40 5 AuthRegistry(0x55a0b032e940) adding con mode: secure
debug -554> 2020-12-16T13:19:22.316+0000 7fc4c3f13f40 5 AuthRegistry(0x55a0b032e940) adding con mode: crc
debug -553> 2020-12-16T13:19:22.316+0000 7fc4c3f13f40 5 AuthRegistry(0x55a0b032e940) adding con mode: crc
debug -552> 2020-12-16T13:19:22.316+0000 7fc4c3f13f40 5 AuthRegistry(0x55a0b032e940) adding con mode: secure
debug -551> 2020-12-16T13:19:22.316+0000 7fc4c3f13f40 5 AuthRegistry(0x55a0b032e940) adding con mode: crc
debug -550> 2020-12-16T13:19:22.316+0000 7fc4c3f13f40 5 AuthRegistry(0x55a0b032e940) adding con mode: secure
debug -549> 2020-12-16T13:19:22.316+0000 7fc4c3f13f40 5 AuthRegistry(0x55a0b032e940) adding con mode: crc
debug -548> 2020-12-16T13:19:22.316+0000 7fc4c3f13f40 5 AuthRegistry(0x55a0b032e940) adding con mode: secure
debug -547> 2020-12-16T13:19:22.316+0000 7fc4c3f13f40 2 auth: KeyRing::load: loaded key file /var/lib/ceph/osd/ceph-1/keyring
debug -546> 2020-12-16T13:19:22.316+0000 7fc4c3f13f40 5 AuthRegistry(0x55a0b032f940) adding auth protocol: cephx
debug -545> 2020-12-16T13:19:22.316+0000 7fc4c3f13f40 5 AuthRegistry(0x55a0b032f940) adding auth protocol: cephx
debug -544> 2020-12-16T13:19:22.316+0000 7fc4c3f13f40 5 AuthRegistry(0x55a0b032f940) adding auth protocol: cephx
debug -543> 2020-12-16T13:19:22.316+0000 7fc4c3f13f40 5 AuthRegistry(0x55a0b032f940) adding auth protocol: none
debug -542> 2020-12-16T13:19:22.316+0000 7fc4c3f13f40 5 AuthRegistry(0x55a0b032f940) adding con mode: secure
debug -541> 2020-12-16T13:19:22.316+0000 7fc4c3f13f40 5 AuthRegistry(0x55a0b032f940) adding con mode: crc
debug -540> 2020-12-16T13:19:22.316+0000 7fc4c3f13f40 5 AuthRegistry(0x55a0b032f940) adding con mode: secure
debug -539> 2020-12-16T13:19:22.316+0000 7fc4c3f13f40 5 AuthRegistry(0x55a0b032f940) adding con mode: crc
debug -538> 2020-12-16T13:19:22.316+0000 7fc4c3f13f40 5 AuthRegistry(0x55a0b032f940) adding con mode: secure
debug -537> 2020-12-16T13:19:22.316+0000 7fc4c3f13f40 5 AuthRegistry(0x55a0b032f940) adding con mode: crc
debug -536> 2020-12-16T13:19:22.316+0000 7fc4c3f13f40 5 AuthRegistry(0x55a0b032f940) adding con mode: crc
debug -535> 2020-12-16T13:19:22.316+0000 7fc4c3f13f40 5 AuthRegistry(0x55a0b032f940) adding con mode: secure
debug -534> 2020-12-16T13:19:22.316+0000 7fc4c3f13f40 5 AuthRegistry(0x55a0b032f940) adding con mode: crc
debug -533> 2020-12-16T13:19:22.316+0000 7fc4c3f13f40 5 AuthRegistry(0x55a0b032f940) adding con mode: secure
debug -532> 2020-12-16T13:19:22.316+0000 7fc4c3f13f40 5 AuthRegistry(0x55a0b032f940) adding con mode: crc
debug -531> 2020-12-16T13:19:22.316+0000 7fc4c3f13f40 5 AuthRegistry(0x55a0b032f940) adding con mode: secure
debug -530> 2020-12-16T13:19:22.316+0000 7fc4c3f13f40 2 auth: KeyRing::load: loaded key file /var/lib/ceph/osd/ceph-1/keyring
debug -529> 2020-12-16T13:19:22.316+0000 7fc4c3f13f40 5 AuthRegistry(0x55a0b0330940) adding auth protocol: cephx
debug -528> 2020-12-16T13:19:22.316+0000 7fc4c3f13f40 5 AuthRegistry(0x55a0b0330940) adding auth protocol: cephx
debug -527> 2020-12-16T13:19:22.316+0000 7fc4c3f13f40 5 AuthRegistry(0x55a0b0330940) adding auth protocol: cephx
debug -526> 2020-12-16T13:19:22.316+0000 7fc4c3f13f40 5 AuthRegistry(0x55a0b0330940) adding auth protocol: none
debug -525> 2020-12-16T13:19:22.316+0000 7fc4c3f13f40 5 AuthRegistry(0x55a0b0330940) adding con mode: secure
debug -524> 2020-12-16T13:19:22.316+0000 7fc4c3f13f40 5 AuthRegistry(0x55a0b0330940) adding con mode: crc
debug -523> 2020-12-16T13:19:22.316+0000 7fc4c3f13f40 5 AuthRegistry(0x55a0b0330940) adding con mode: secure
debug -522> 2020-12-16T13:19:22.316+0000 7fc4c3f13f40 5 AuthRegistry(0x55a0b0330940) adding con mode: crc
debug -521> 2020-12-16T13:19:22.316+0000 7fc4c3f13f40 5 AuthRegistry(0x55a0b0330940) adding con mode: secure
debug -520> 2020-12-16T13:19:22.316+0000 7fc4c3f13f40 5 AuthRegistry(0x55a0b0330940) adding con mode: crc
debug -519> 2020-12-16T13:19:22.316+0000 7fc4c3f13f40 5 AuthRegistry(0x55a0b0330940) adding con mode: crc
debug -518> 2020-12-16T13:19:22.316+0000 7fc4c3f13f40 5 AuthRegistry(0x55a0b0330940) adding con mode: secure
[root@rook-ceph-osd-1-7648fb96d8-ndrdm /]# ceph-bluestore-tool bluefs-bdev-expand --path /var/lib/ceph/osd/ceph-1
inferring bluefs devices from bluestore path
1 : device size 0x18ffc00000 : own 0x[bffe10000~fffe0000] = 0xfffe0000 : using 0xfd190000(4.0 GiB) : bluestore has 0x82b360000(33 GiB) available
Expanding DB/WAL...
[root@rook-ceph-osd-1-7648fb96d8-ndrdm /]# ceph-bluestore-tool repair --path /var/lib/ceph/osd/ceph-1
2020-12-16T13:24:20.065+0000 7fc4240cc240 -1 bluestore(/var/lib/ceph/osd/ceph-1) allocate_bluefs_freespace failed to allocate on 0x3d1b0000 min_size 0x43d0000 > allocated total 0x300000 bluefs_shared_alloc_size 0x10000 allocated 0x300000 available 0x b019c8000
2020-12-16T13:24:20.065+0000 7fc4240cc240 -1 bluefs _allocate failed to expand slow device to fit +0x43ce0aa
2020-12-16T13:24:20.065+0000 7fc4240cc240 -1 bluefs _flush_range allocated: 0x0 offset: 0x0 length: 0x43ce0aa
/home/jenkins-build/build/workspace/ceph-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos8/DIST/centos8/MACHINE_SIZE/gigantic/release/15.2.6/rpm/el8/BUILD/ceph-15.2.6/src/os/bluestore/BlueFS.cc: In function 'int BlueFS::_flush_range(BlueFS::FileWriter*, uint64_t, uint64_t)' thread 7fc4240cc240 time 2020-12-16T13:24:20.069595+0000
/home/jenkins-build/build/workspace/ceph-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos8/DIST/centos8/MACHINE_SIZE/gigantic/release/15.2.6/rpm/el8/BUILD/ceph-15.2.6/src/os/bluestore/BlueFS.cc: 2721: ceph_abort_msg("bluefs enospc")
ceph version 15.2.6 (cb8c61a60551b72614257d632a574d420064c17a) octopus (stable)
1: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)+0xe5) [0x7fc41a1e82cc]
2: (BlueFS::_flush_range(BlueFS::FileWriter*, unsigned long, unsigned long)+0x1a43) [0x55a433337073]
3: (BlueFS::_flush(BlueFS::FileWriter*, bool, bool*)+0x110) [0x55a433337760]
4: (BlueFS::_flush(BlueFS::FileWriter*, bool, std::unique_lock<std::mutex>&)+0x32) [0x55a4333484f2]
5: (BlueRocksWritableFile::Flush()+0x72) [0x55a4335204a2]
6: (rocksdb::WritableFileWriter::Flush()+0x1fc) [0x55a43374c95c]
7: (rocksdb::WritableFileWriter::Sync(bool)+0x1f) [0x55a43374cadf]
8: (rocksdb::BuildTable(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, rocksdb::Env*, rocksdb::ImmutableCFOptions const&, rocksdb::MutableCFOptions const&, rocksdb::EnvOptions const&, rocksdb::TableCache*, rocksdb::InternalIteratorBase<rocksdb::Slice>*, std::vector<std::unique_ptr<rocksdb::FragmentedRangeTombstoneIterator, std::default_delete<rocksdb::FragmentedRangeTombstoneIterator> >, std::allocator<std::unique_ptr<rocksdb::FragmentedRangeTombstoneIterator, std::default_delete<rocksdb::FragmentedRangeTombstoneIterator> > > >, rocksdb::FileMetaData*, rocksdb::InternalKeyComparator const&, std::vector<std::unique_ptr<rocksdb::IntTblPropCollectorFactory, std::default_delete<rocksdb::IntTblPropCollectorFactory> >, std::allocator<std::unique_ptr<rocksdb::IntTblPropCollectorFactory, std::default_delete<rocksdb::IntTblPropCollectorFactory> > > > const*, unsigned int, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::vector<unsigned long, std::allocator<unsigned long> >, unsigned long, rocksdb::SnapshotChecker*, rocksdb::CompressionType, unsigned long, rocksdb::CompressionOptions const&, bool, rocksdb::InternalStats*, rocksdb::TableFileCreationReason, rocksdb::EventLogger*, int, rocksdb::Env::IOPriority, rocksdb::TableProperties*, int, unsigned long, unsigned long, rocksdb::Env::WriteLifeTimeHint)+0x2be5) [0x55a433774085]
9: (rocksdb::DBImpl::WriteLevel0TableForRecovery(int, rocksdb::ColumnFamilyData*, rocksdb::MemTable*, rocksdb::VersionEdit*)+0xc07) [0x55a4336217c7]
10: (rocksdb::DBImpl::RecoverLogFiles(std::vector<unsigned long, std::allocator<unsigned long> > const&, unsigned long*, bool)+0x12dd) [0x55a4336232cd]
11: (rocksdb::DBImpl::Recover(std::vector<rocksdb::ColumnFamilyDescriptor, std::allocator<rocksdb::ColumnFamilyDescriptor> > const&, bool, bool, bool)+0xb09) [0x55a433624789]
12: (rocksdb::DBImpl::Open(rocksdb::DBOptions const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::vector<rocksdb::ColumnFamilyDescriptor, std::allocator<rocksdb::ColumnFamilyDescriptor> > const&, std::vector<rocksdb::ColumnFamilyHandle*, std::allocator<rocksdb::ColumnFamilyHandle*> >*, rocksdb::DB**, bool, bool)+0x9f5) [0x55a43361f025]
13: (rocksdb::DB::Open(rocksdb::DBOptions const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::vector<rocksdb::ColumnFamilyDescriptor, std::allocator<rocksdb::ColumnFamilyDescriptor> > const&, std::vector<rocksdb::ColumnFamilyHandle*, std::allocator<rocksdb::ColumnFamilyHandle*> >*, rocksdb::DB**)+0x15) [0x55a433620375]
14: (RocksDBStore::do_open(std::ostream&, bool, bool, std::vector<KeyValueDB::ColumnFamily, std::allocator<KeyValueDB::ColumnFamily> > const*)+0x24e4) [0x55a4335af714]
15: (BlueStore::_open_db(bool, bool, bool)+0x1239) [0x55a433395989]
16: (BlueStore::_open_db_and_around(bool)+0x19b) [0x55a4333aca0b]
17: (BlueStore::_fsck(BlueStore::FSCKDepth, bool)+0x1e1) [0x55a4333fe0a1]
18: (main()+0x136e) [0x55a4332f916e]
19: (__libc_start_main()+0xf3) [0x7fc417a176a3]
20: (_start()+0x2e) [0x55a43331b65e]
*** Caught signal (Aborted) **
in thread 7fc4240cc240 thread_name:ceph-bluestore-
2020-12-16T13:24:20.069+0000 7fc4240cc240 -1 /home/jenkins-build/build/workspace/ceph-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos8/DIST/centos8/MACHINE_SIZE/gigantic/release/15.2.6/rpm/el8/BUILD/ceph-15.2.6/src/os/bluestore/BlueFS.cc: In function 'int BlueFS::_flush_range(BlueFS::FileWriter*, uint64_t, uint64_t)' thread 7fc4240cc240 time 2020-12-16T13:24:20.069595+0000
/home/jenkins-build/build/workspace/ceph-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos8/DIST/centos8/MACHINE_SIZE/gigantic/release/15.2.6/rpm/el8/BUILD/ceph-15.2.6/src/os/bluestore/BlueFS.cc: 2721: ceph_abort_msg("bluefs enospc")
ceph version 15.2.6 (cb8c61a60551b72614257d632a574d420064c17a) octopus (stable)
1: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)+0xe5) [0x7fc41a1e82cc]
2: (BlueFS::_flush_range(BlueFS::FileWriter*, unsigned long, unsigned long)+0x1a43) [0x55a433337073]
3: (BlueFS::_flush(BlueFS::FileWriter*, bool, bool*)+0x110) [0x55a433337760]
4: (BlueFS::_flush(BlueFS::FileWriter*, bool, std::unique_lock<std::mutex>&)+0x32) [0x55a4333484f2]
5: (BlueRocksWritableFile::Flush()+0x72) [0x55a4335204a2]
6: (rocksdb::WritableFileWriter::Flush()+0x1fc) [0x55a43374c95c]
7: (rocksdb::WritableFileWriter::Sync(bool)+0x1f) [0x55a43374cadf]
8: (rocksdb::BuildTable(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, rocksdb::Env*, rocksdb::ImmutableCFOptions const&, rocksdb::MutableCFOptions const&, rocksdb::EnvOptions const&, rocksdb::TableCache*, rocksdb::InternalIteratorBase<rocksdb::Slice>*, std::vector<std::unique_ptr<rocksdb::FragmentedRangeTombstoneIterator, std::default_delete<rocksdb::FragmentedRangeTombstoneIterator> >, std::allocator<std::unique_ptr<rocksdb::FragmentedRangeTombstoneIterator, std::default_delete<rocksdb::FragmentedRangeTombstoneIterator> > > >, rocksdb::FileMetaData*, rocksdb::InternalKeyComparator const&, std::vector<std::unique_ptr<rocksdb::IntTblPropCollectorFactory, std::default_delete<rocksdb::IntTblPropCollectorFactory> >, std::allocator<std::unique_ptr<rocksdb::IntTblPropCollectorFactory, std::default_delete<rocksdb::IntTblPropCollectorFactory> > > > const*, unsigned int, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::vector<unsigned long, std::allocator<unsigned long> >, unsigned long, rocksdb::SnapshotChecker*, rocksdb::CompressionType, unsigned long, rocksdb::CompressionOptions const&, bool, rocksdb::InternalStats*, rocksdb::TableFileCreationReason, rocksdb::EventLogger*, int, rocksdb::Env::IOPriority, rocksdb::TableProperties*, int, unsigned long, unsigned long, rocksdb::Env::WriteLifeTimeHint)+0x2be5) [0x55a433774085]
9: (rocksdb::DBImpl::WriteLevel0TableForRecovery(int, rocksdb::ColumnFamilyData*, rocksdb::MemTable*, rocksdb::VersionEdit*)+0xc07) [0x55a4336217c7]
10: (rocksdb::DBImpl::RecoverLogFiles(std::vector<unsigned long, std::allocator<unsigned long> > const&, unsigned long*, bool)+0x12dd) [0x55a4336232cd]
11: (rocksdb::DBImpl::Recover(std::vector<rocksdb::ColumnFamilyDescriptor, std::allocator<rocksdb::ColumnFamilyDescriptor> > const&, bool, bool, bool)+0xb09) [0x55a433624789]
12: (rocksdb::DBImpl::Open(rocksdb::DBOptions const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::vector<rocksdb::ColumnFamilyDescriptor, std::allocator<rocksdb::ColumnFamilyDescriptor> > const&, std::vector<rocksdb::ColumnFamilyHandle*, std::allocator<rocksdb::ColumnFamilyHandle*> >*, rocksdb::DB**, bool, bool)+0x9f5) [0x55a43361f025]
13: (rocksdb::DB::Open(rocksdb::DBOptions const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::vector<rocksdb::ColumnFamilyDescriptor, std::allocator<rocksdb::ColumnFamilyDescriptor> > const&, std::vector<rocksdb::ColumnFamilyHandle*, std::allocator<rocksdb::ColumnFamilyHandle*> >*, rocksdb::DB**)+0x15) [0x55a433620375]
14: (RocksDBStore::do_open(std::ostream&, bool, bool, std::vector<KeyValueDB::ColumnFamily, std::allocator<KeyValueDB::ColumnFamily> > const*)+0x24e4) [0x55a4335af714]
15: (BlueStore::_open_db(bool, bool, bool)+0x1239) [0x55a433395989]
16: (BlueStore::_open_db_and_around(bool)+0x19b) [0x55a4333aca0b]
17: (BlueStore::_fsck(BlueStore::FSCKDepth, bool)+0x1e1) [0x55a4333fe0a1]
18: (main()+0x136e) [0x55a4332f916e]
19: (__libc_start_main()+0xf3) [0x7fc417a176a3]
20: (_start()+0x2e) [0x55a43331b65e]
ceph version 15.2.6 (cb8c61a60551b72614257d632a574d420064c17a) octopus (stable)
1: (()+0x12dd0) [0x7fc4193e6dd0]
2: (gsignal()+0x10f) [0x7fc417a2b70f]
3: (abort()+0x127) [0x7fc417a15b25]
4: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)+0x1b6) [0x7fc41a1e839d]
5: (BlueFS::_flush_range(BlueFS::FileWriter*, unsigned long, unsigned long)+0x1a43) [0x55a433337073]
6: (BlueFS::_flush(BlueFS::FileWriter*, bool, bool*)+0x110) [0x55a433337760]
7: (BlueFS::_flush(BlueFS::FileWriter*, bool, std::unique_lock<std::mutex>&)+0x32) [0x55a4333484f2]
8: (BlueRocksWritableFile::Flush()+0x72) [0x55a4335204a2]
9: (rocksdb::WritableFileWriter::Flush()+0x1fc) [0x55a43374c95c]
10: (rocksdb::WritableFileWriter::Sync(bool)+0x1f) [0x55a43374cadf]
11: (rocksdb::BuildTable(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, rocksdb::Env*, rocksdb::ImmutableCFOptions const&, rocksdb::MutableCFOptions const&, rocksdb::EnvOptions const&, rocksdb::TableCache*, rocksdb::InternalIteratorBase<rocksdb::Slice>*, std::vector<std::unique_ptr<rocksdb::FragmentedRangeTombstoneIterator, std::default_delete<rocksdb::FragmentedRangeTombstoneIterator> >, std::allocator<std::unique_ptr<rocksdb::FragmentedRangeTombstoneIterator, std::default_delete<rocksdb::FragmentedRangeTombstoneIterator> > > >, rocksdb::FileMetaData*, rocksdb::InternalKeyComparator const&, std::vector<std::unique_ptr<rocksdb::IntTblPropCollectorFactory, std::default_delete<rocksdb::IntTblPropCollectorFactory> >, std::allocator<std::unique_ptr<rocksdb::IntTblPropCollectorFactory, std::default_delete<rocksdb::IntTblPropCollectorFactory> > > > const*, unsigned int, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::vector<unsigned long, std::allocator<unsigned long> >, unsigned long, rocksdb::SnapshotChecker*, rocksdb::CompressionType, unsigned long, rocksdb::CompressionOptions const&, bool, rocksdb::InternalStats*, rocksdb::TableFileCreationReason, rocksdb::EventLogger*, int, rocksdb::Env::IOPriority, rocksdb::TableProperties*, int, unsigned long, unsigned long, rocksdb::Env::WriteLifeTimeHint)+0x2be5) [0x55a433774085]
12: (rocksdb::DBImpl::WriteLevel0TableForRecovery(int, rocksdb::ColumnFamilyData*, rocksdb::MemTable*, rocksdb::VersionEdit*)+0xc07) [0x55a4336217c7]
13: (rocksdb::DBImpl::RecoverLogFiles(std::vector<unsigned long, std::allocator<unsigned long> > const&, unsigned long*, bool)+0x12dd) [0x55a4336232cd]
14: (rocksdb::DBImpl::Recover(std::vector<rocksdb::ColumnFamilyDescriptor, std::allocator<rocksdb::ColumnFamilyDescriptor> > const&, bool, bool, bool)+0xb09) [0x55a433624789]
15: (rocksdb::DBImpl::Open(rocksdb::DBOptions const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::vector<rocksdb::ColumnFamilyDescriptor, std::allocator<rocksdb::ColumnFamilyDescriptor> > const&, std::vector<rocksdb::ColumnFamilyHandle*, std::allocator<rocksdb::ColumnFamilyHandle*> >*, rocksdb::DB**, bool, bool)+0x9f5) [0x55a43361f025]
16: (rocksdb::DB::Open(rocksdb::DBOptions const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::vector<rocksdb::ColumnFamilyDescriptor, std::allocator<rocksdb::ColumnFamilyDescriptor> > const&, std::vector<rocksdb::ColumnFamilyHandle*, std::allocator<rocksdb::ColumnFamilyHandle*> >*, rocksdb::DB**)+0x15) [0x55a433620375]
17: (RocksDBStore::do_open(std::ostream&, bool, bool, std::vector<KeyValueDB::ColumnFamily, std::allocator<KeyValueDB::ColumnFamily> > const*)+0x24e4) [0x55a4335af714]
18: (BlueStore::_open_db(bool, bool, bool)+0x1239) [0x55a433395989]
19: (BlueStore::_open_db_and_around(bool)+0x19b) [0x55a4333aca0b]
20: (BlueStore::_fsck(BlueStore::FSCKDepth, bool)+0x1e1) [0x55a4333fe0a1]
21: (main()+0x136e) [0x55a4332f916e]
22: (__libc_start_main()+0xf3) [0x7fc417a176a3]
23: (_start()+0x2e) [0x55a43331b65e]
2020-12-16T13:24:20.069+0000 7fc4240cc240 -1 *** Caught signal (Aborted) **
in thread 7fc4240cc240 thread_name:ceph-bluestore-
ceph version 15.2.6 (cb8c61a60551b72614257d632a574d420064c17a) octopus (stable)
1: (()+0x12dd0) [0x7fc4193e6dd0]
2: (gsignal()+0x10f) [0x7fc417a2b70f]
3: (abort()+0x127) [0x7fc417a15b25]
4: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)+0x1b6) [0x7fc41a1e839d]
5: (BlueFS::_flush_range(BlueFS::FileWriter*, unsigned long, unsigned long)+0x1a43) [0x55a433337073]
6: (BlueFS::_flush(BlueFS::FileWriter*, bool, bool*)+0x110) [0x55a433337760]
7: (BlueFS::_flush(BlueFS::FileWriter*, bool, std::unique_lock<std::mutex>&)+0x32) [0x55a4333484f2]
8: (BlueRocksWritableFile::Flush()+0x72) [0x55a4335204a2]
9: (rocksdb::WritableFileWriter::Flush()+0x1fc) [0x55a43374c95c]
10: (rocksdb::WritableFileWriter::Sync(bool)+0x1f) [0x55a43374cadf]
11: (rocksdb::BuildTable(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, rocksdb::Env*, rocksdb::ImmutableCFOptions const&, rocksdb::MutableCFOptions const&, rocksdb::EnvOptions const&, rocksdb::TableCache*, rocksdb::InternalIteratorBase<rocksdb::Slice>*, std::vector<std::unique_ptr<rocksdb::FragmentedRangeTombstoneIterator, std::default_delete<rocksdb::FragmentedRangeTombstoneIterator> >, std::allocator<std::unique_ptr<rocksdb::FragmentedRangeTombstoneIterator, std::default_delete<rocksdb::FragmentedRangeTombstoneIterator> > > >, rocksdb::FileMetaData*, rocksdb::InternalKeyComparator const&, std::vector<std::unique_ptr<rocksdb::IntTblPropCollectorFactory, std::default_delete<rocksdb::IntTblPropCollectorFactory> >, std::allocator<std::unique_ptr<rocksdb::IntTblPropCollectorFactory, std::default_delete<rocksdb::IntTblPropCollectorFactory> > > > const*, unsigned int, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::vector<unsigned long, std::allocator<unsigned long> >, unsigned long, rocksdb::SnapshotChecker*, rocksdb::CompressionType, unsigned long, rocksdb::CompressionOptions const&, bool, rocksdb::InternalStats*, rocksdb::TableFileCreationReason, rocksdb::EventLogger*, int, rocksdb::Env::IOPriority, rocksdb::TableProperties*, int, unsigned long, unsigned long, rocksdb::Env::WriteLifeTimeHint)+0x2be5) [0x55a433774085]
12: (rocksdb::DBImpl::WriteLevel0TableForRecovery(int, rocksdb::ColumnFamilyData*, rocksdb::MemTable*, rocksdb::VersionEdit*)+0xc07) [0x55a4336217c7]
13: (rocksdb::DBImpl::RecoverLogFiles(std::vector<unsigned long, std::allocator<unsigned long> > const&, unsigned long*, bool)+0x12dd) [0x55a4336232cd]
14: (rocksdb::DBImpl::Recover(std::vector<rocksdb::ColumnFamilyDescriptor, std::allocator<rocksdb::ColumnFamilyDescriptor> > const&, bool, bool, bool)+0xb09) [0x55a433624789]
15: (rocksdb::DBImpl::Open(rocksdb::DBOptions const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::vector<rocksdb::ColumnFamilyDescriptor, std::allocator<rocksdb::ColumnFamilyDescriptor> > const&, std::vector<rocksdb::ColumnFamilyHandle*, std::allocator<rocksdb::ColumnFamilyHandle*> >*, rocksdb::DB**, bool, bool)+0x9f5) [0x55a43361f025]
16: (rocksdb::DB::Open(rocksdb::DBOptions const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::vector<rocksdb::ColumnFamilyDescriptor, std::allocator<rocksdb::ColumnFamilyDescriptor> > const&, std::vector<rocksdb::ColumnFamilyHandle*, std::allocator<rocksdb::ColumnFamilyHandle*> >*, rocksdb::DB**)+0x15) [0x55a433620375]
17: (RocksDBStore::do_open(std::ostream&, bool, bool, std::vector<KeyValueDB::ColumnFamily, std::allocator<KeyValueDB::ColumnFamily> > const*)+0x24e4) [0x55a4335af714]
18: (BlueStore::_open_db(bool, bool, bool)+0x1239) [0x55a433395989]
19: (BlueStore::_open_db_and_around(bool)+0x19b) [0x55a4333aca0b]
20: (BlueStore::_fsck(BlueStore::FSCKDepth, bool)+0x1e1) [0x55a4333fe0a1]
21: (main()+0x136e) [0x55a4332f916e]
22: (__libc_start_main()+0xf3) [0x7fc417a176a3]
23: (_start()+0x2e) [0x55a43331b65e]
NOTE: a copy of the executable, or `objdump -rdS <executable>` is needed to interpret this.
-447> 2020-12-16T13:24:20.065+0000 7fc4240cc240 -1 bluestore(/var/lib/ceph/osd/ceph-1) allocate_bluefs_freespace failed to allocate on 0x3d1b0000 min_size 0x43d0000 > allocated total 0x300000 bluefs_shared_alloc_size 0x10000 allocated 0x300000 available 0x b019c8000
-446> 2020-12-16T13:24:20.065+0000 7fc4240cc240 -1 bluefs _allocate failed to expand slow device to fit +0x43ce0aa
-445> 2020-12-16T13:24:20.065+0000 7fc4240cc240 -1 bluefs _flush_range allocated: 0x0 offset: 0x0 length: 0x43ce0aa
-444> 2020-12-16T13:24:20.069+0000 7fc4240cc240 -1 /home/jenkins-build/build/workspace/ceph-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos8/DIST/centos8/MACHINE_SIZE/gigantic/release/15.2.6/rpm/el8/BUILD/ceph-15.2.6/src/os/bluestore/BlueFS.cc: In function 'int BlueFS::_flush_range(BlueFS::FileWriter*, uint64_t, uint64_t)' thread 7fc4240cc240 time 2020-12-16T13:24:20.069595+0000
/home/jenkins-build/build/workspace/ceph-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos8/DIST/centos8/MACHINE_SIZE/gigantic/release/15.2.6/rpm/el8/BUILD/ceph-15.2.6/src/os/bluestore/BlueFS.cc: 2721: ceph_abort_msg("bluefs enospc")
ceph version 15.2.6 (cb8c61a60551b72614257d632a574d420064c17a) octopus (stable)
1: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)+0xe5) [0x7fc41a1e82cc]
2: (BlueFS::_flush_range(BlueFS::FileWriter*, unsigned long, unsigned long)+0x1a43) [0x55a433337073]
3: (BlueFS::_flush(BlueFS::FileWriter*, bool, bool*)+0x110) [0x55a433337760]
4: (BlueFS::_flush(BlueFS::FileWriter*, bool, std::unique_lock<std::mutex>&)+0x32) [0x55a4333484f2]
5: (BlueRocksWritableFile::Flush()+0x72) [0x55a4335204a2]
6: (rocksdb::WritableFileWriter::Flush()+0x1fc) [0x55a43374c95c]
7: (rocksdb::WritableFileWriter::Sync(bool)+0x1f) [0x55a43374cadf]
8: (rocksdb::BuildTable(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, rocksdb::Env*, rocksdb::ImmutableCFOptions const&, rocksdb::MutableCFOptions const&, rocksdb::EnvOptions const&, rocksdb::TableCache*, rocksdb::InternalIteratorBase<rocksdb::Slice>*, std::vector<std::unique_ptr<rocksdb::FragmentedRangeTombstoneIterator, std::default_delete<rocksdb::FragmentedRangeTombstoneIterator> >, std::allocator<std::unique_ptr<rocksdb::FragmentedRangeTombstoneIterator, std::default_delete<rocksdb::FragmentedRangeTombstoneIterator> > > >, rocksdb::FileMetaData*, rocksdb::InternalKeyComparator const&, std::vector<std::unique_ptr<rocksdb::IntTblPropCollectorFactory, std::default_delete<rocksdb::IntTblPropCollectorFactory> >, std::allocator<std::unique_ptr<rocksdb::IntTblPropCollectorFactory, std::default_delete<rocksdb::IntTblPropCollectorFactory> > > > const*, unsigned int, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::vector<unsigned long, std::allocator<unsigned long> >, unsigned long, rocksdb::SnapshotChecker*, rocksdb::CompressionType, unsigned long, rocksdb::CompressionOptions const&, bool, rocksdb::InternalStats*, rocksdb::TableFileCreationReason, rocksdb::EventLogger*, int, rocksdb::Env::IOPriority, rocksdb::TableProperties*, int, unsigned long, unsigned long, rocksdb::Env::WriteLifeTimeHint)+0x2be5) [0x55a433774085]
9: (rocksdb::DBImpl::WriteLevel0TableForRecovery(int, rocksdb::ColumnFamilyData*, rocksdb::MemTable*, rocksdb::VersionEdit*)+0xc07) [0x55a4336217c7]
10: (rocksdb::DBImpl::RecoverLogFiles(std::vector<unsigned long, std::allocator<unsigned long> > const&, unsigned long*, bool)+0x12dd) [0x55a4336232cd]
11: (rocksdb::DBImpl::Recover(std::vector<rocksdb::ColumnFamilyDescriptor, std::allocator<rocksdb::ColumnFamilyDescriptor> > const&, bool, bool, bool)+0xb09) [0x55a433624789]
12: (rocksdb::DBImpl::Open(rocksdb::DBOptions const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::vector<rocksdb::ColumnFamilyDescriptor, std::allocator<rocksdb::ColumnFamilyDescriptor> > const&, std::vector<rocksdb::ColumnFamilyHandle*, std::allocator<rocksdb::ColumnFamilyHandle*> >*, rocksdb::DB**, bool, bool)+0x9f5) [0x55a43361f025]
13: (rocksdb::DB::Open(rocksdb::DBOptions const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::vector<rocksdb::ColumnFamilyDescriptor, std::allocator<rocksdb::ColumnFamilyDescriptor> > const&, std::vector<rocksdb::ColumnFamilyHandle*, std::allocator<rocksdb::ColumnFamilyHandle*> >*, rocksdb::DB**)+0x15) [0x55a433620375]
14: (RocksDBStore::do_open(std::ostream&, bool, bool, std::vector<KeyValueDB::ColumnFamily, std::allocator<KeyValueDB::ColumnFamily> > const*)+0x24e4) [0x55a4335af714]
15: (BlueStore::_open_db(bool, bool, bool)+0x1239) [0x55a433395989]
16: (BlueStore::_open_db_and_around(bool)+0x19b) [0x55a4333aca0b]
17: (BlueStore::_fsck(BlueStore::FSCKDepth, bool)+0x1e1) [0x55a4333fe0a1]
18: (main()+0x136e) [0x55a4332f916e]
19: (__libc_start_main()+0xf3) [0x7fc417a176a3]
20: (_start()+0x2e) [0x55a43331b65e]
-443> 2020-12-16T13:24:20.069+0000 7fc4240cc240 -1 *** Caught signal (Aborted) **
in thread 7fc4240cc240 thread_name:ceph-bluestore-
ceph version 15.2.6 (cb8c61a60551b72614257d632a574d420064c17a) octopus (stable)
1: (()+0x12dd0) [0x7fc4193e6dd0]
2: (gsignal()+0x10f) [0x7fc417a2b70f]
3: (abort()+0x127) [0x7fc417a15b25]
4: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)+0x1b6) [0x7fc41a1e839d]
5: (BlueFS::_flush_range(BlueFS::FileWriter*, unsigned long, unsigned long)+0x1a43) [0x55a433337073]
6: (BlueFS::_flush(BlueFS::FileWriter*, bool, bool*)+0x110) [0x55a433337760]
7: (BlueFS::_flush(BlueFS::FileWriter*, bool, std::unique_lock<std::mutex>&)+0x32) [0x55a4333484f2]
8: (BlueRocksWritableFile::Flush()+0x72) [0x55a4335204a2]
9: (rocksdb::WritableFileWriter::Flush()+0x1fc) [0x55a43374c95c]
10: (rocksdb::WritableFileWriter::Sync(bool)+0x1f) [0x55a43374cadf]
11: (rocksdb::BuildTable(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, rocksdb::Env*, rocksdb::ImmutableCFOptions const&, rocksdb::MutableCFOptions const&, rocksdb::EnvOptions const&, rocksdb::TableCache*, rocksdb::InternalIteratorBase<rocksdb::Slice>*, std::vector<std::unique_ptr<rocksdb::FragmentedRangeTombstoneIterator, std::default_delete<rocksdb::FragmentedRangeTombstoneIterator> >, std::allocator<std::unique_ptr<rocksdb::FragmentedRangeTombstoneIterator, std::default_delete<rocksdb::FragmentedRangeTombstoneIterator> > > >, rocksdb::FileMetaData*, rocksdb::InternalKeyComparator const&, std::vector<std::unique_ptr<rocksdb::IntTblPropCollectorFactory, std::default_delete<rocksdb::IntTblPropCollectorFactory> >, std::allocator<std::unique_ptr<rocksdb::IntTblPropCollectorFactory, std::default_delete<rocksdb::IntTblPropCollectorFactory> > > > const*, unsigned int, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::vector<unsigned long, std::allocator<unsigned long> >, unsigned long, rocksdb::SnapshotChecker*, rocksdb::CompressionType, unsigned long, rocksdb::CompressionOptions const&, bool, rocksdb::InternalStats*, rocksdb::TableFileCreationReason, rocksdb::EventLogger*, int, rocksdb::Env::IOPriority, rocksdb::TableProperties*, int, unsigned long, unsigned long, rocksdb::Env::WriteLifeTimeHint)+0x2be5) [0x55a433774085]
12: (rocksdb::DBImpl::WriteLevel0TableForRecovery(int, rocksdb::ColumnFamilyData*, rocksdb::MemTable*, rocksdb::VersionEdit*)+0xc07) [0x55a4336217c7]
13: (rocksdb::DBImpl::RecoverLogFiles(std::vector<unsigned long, std::allocator<unsigned long> > const&, unsigned long*, bool)+0x12dd) [0x55a4336232cd]
14: (rocksdb::DBImpl::Recover(std::vector<rocksdb::ColumnFamilyDescriptor, std::allocator<rocksdb::ColumnFamilyDescriptor> > const&, bool, bool, bool)+0xb09) [0x55a433624789]
15: (rocksdb::DBImpl::Open(rocksdb::DBOptions const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::vector<rocksdb::ColumnFamilyDescriptor, std::allocator<rocksdb::ColumnFamilyDescriptor> > const&, std::vector<rocksdb::ColumnFamilyHandle*, std::allocator<rocksdb::ColumnFamilyHandle*> >*, rocksdb::DB**, bool, bool)+0x9f5) [0x55a43361f025]
16: (rocksdb::DB::Open(rocksdb::DBOptions const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::vector<rocksdb::ColumnFamilyDescriptor, std::allocator<rocksdb::ColumnFamilyDescriptor> > const&, std::vector<rocksdb::ColumnFamilyHandle*, std::allocator<rocksdb::ColumnFamilyHandle*> >*, rocksdb::DB**)+0x15) [0x55a433620375]
17: (RocksDBStore::do_open(std::ostream&, bool, bool, std::vector<KeyValueDB::ColumnFamily, std::allocator<KeyValueDB::ColumnFamily> > const*)+0x24e4) [0x55a4335af714]
18: (BlueStore::_open_db(bool, bool, bool)+0x1239) [0x55a433395989]
19: (BlueStore::_open_db_and_around(bool)+0x19b) [0x55a4333aca0b]
20: (BlueStore::_fsck(BlueStore::FSCKDepth, bool)+0x1e1) [0x55a4333fe0a1]
21: (main()+0x136e) [0x55a4332f916e]
22: (__libc_start_main()+0xf3) [0x7fc417a176a3]
23: (_start()+0x2e) [0x55a43331b65e]
NOTE: a copy of the executable, or `objdump -rdS <executable>` is needed to interpret this.
-4> 2020-12-16T13:24:20.065+0000 7fc4240cc240 -1 bluestore(/var/lib/ceph/osd/ceph-1) allocate_bluefs_freespace failed to allocate on 0x3d1b0000 min_size 0x43d0000 > allocated total 0x300000 bluefs_shared_alloc_size 0x10000 allocated 0x300000 available 0x b019c8000
-3> 2020-12-16T13:24:20.065+0000 7fc4240cc240 -1 bluefs _allocate failed to expand slow device to fit +0x43ce0aa
-2> 2020-12-16T13:24:20.065+0000 7fc4240cc240 -1 bluefs _flush_range allocated: 0x0 offset: 0x0 length: 0x43ce0aa
-1> 2020-12-16T13:24:20.069+0000 7fc4240cc240 -1 /home/jenkins-build/build/workspace/ceph-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos8/DIST/centos8/MACHINE_SIZE/gigantic/release/15.2.6/rpm/el8/BUILD/ceph-15.2.6/src/os/bluestore/BlueFS.cc: In function 'int BlueFS::_flush_range(BlueFS::FileWriter*, uint64_t, uint64_t)' thread 7fc4240cc240 time 2020-12-16T13:24:20.069595+0000
/home/jenkins-build/build/workspace/ceph-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos8/DIST/centos8/MACHINE_SIZE/gigantic/release/15.2.6/rpm/el8/BUILD/ceph-15.2.6/src/os/bluestore/BlueFS.cc: 2721: ceph_abort_msg("bluefs enospc")
ceph version 15.2.6 (cb8c61a60551b72614257d632a574d420064c17a) octopus (stable)
1: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)+0xe5) [0x7fc41a1e82cc]
2: (BlueFS::_flush_range(BlueFS::FileWriter*, unsigned long, unsigned long)+0x1a43) [0x55a433337073]
3: (BlueFS::_flush(BlueFS::FileWriter*, bool, bool*)+0x110) [0x55a433337760]
4: (BlueFS::_flush(BlueFS::FileWriter*, bool, std::unique_lock<std::mutex>&)+0x32) [0x55a4333484f2]
5: (BlueRocksWritableFile::Flush()+0x72) [0x55a4335204a2]
6: (rocksdb::WritableFileWriter::Flush()+0x1fc) [0x55a43374c95c]
7: (rocksdb::WritableFileWriter::Sync(bool)+0x1f) [0x55a43374cadf]
8: (rocksdb::BuildTable(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, rocksdb::Env*, rocksdb::ImmutableCFOptions const&, rocksdb::MutableCFOptions const&, rocksdb::EnvOptions const&, rocksdb::TableCache*, rocksdb::InternalIteratorBase<rocksdb::Slice>*, std::vector<std::unique_ptr<rocksdb::FragmentedRangeTombstoneIterator, std::default_delete<rocksdb::FragmentedRangeTombstoneIterator> >, std::allocator<std::unique_ptr<rocksdb::FragmentedRangeTombstoneIterator, std::default_delete<rocksdb::FragmentedRangeTombstoneIterator> > > >, rocksdb::FileMetaData*, rocksdb::InternalKeyComparator const&, std::vector<std::unique_ptr<rocksdb::IntTblPropCollectorFactory, std::default_delete<rocksdb::IntTblPropCollectorFactory> >, std::allocator<std::unique_ptr<rocksdb::IntTblPropCollectorFactory, std::default_delete<rocksdb::IntTblPropCollectorFactory> > > > const*, unsigned int, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::vector<unsigned long, std::allocator<unsigned long> >, unsigned long, rocksdb::SnapshotChecker*, rocksdb::CompressionType, unsigned long, rocksdb::CompressionOptions const&, bool, rocksdb::InternalStats*, rocksdb::TableFileCreationReason, rocksdb::EventLogger*, int, rocksdb::Env::IOPriority, rocksdb::TableProperties*, int, unsigned long, unsigned long, rocksdb::Env::WriteLifeTimeHint)+0x2be5) [0x55a433774085]
9: (rocksdb::DBImpl::WriteLevel0TableForRecovery(int, rocksdb::ColumnFamilyData*, rocksdb::MemTable*, rocksdb::VersionEdit*)+0xc07) [0x55a4336217c7]
10: (rocksdb::DBImpl::RecoverLogFiles(std::vector<unsigned long, std::allocator<unsigned long> > const&, unsigned long*, bool)+0x12dd) [0x55a4336232cd]
11: (rocksdb::DBImpl::Recover(std::vector<rocksdb::ColumnFamilyDescriptor, std::allocator<rocksdb::ColumnFamilyDescriptor> > const&, bool, bool, bool)+0xb09) [0x55a433624789]
12: (rocksdb::DBImpl::Open(rocksdb::DBOptions const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::vector<rocksdb::ColumnFamilyDescriptor, std::allocator<rocksdb::ColumnFamilyDescriptor> > const&, std::vector<rocksdb::ColumnFamilyHandle*, std::allocator<rocksdb::ColumnFamilyHandle*> >*, rocksdb::DB**, bool, bool)+0x9f5) [0x55a43361f025]
13: (rocksdb::DB::Open(rocksdb::DBOptions const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::vector<rocksdb::ColumnFamilyDescriptor, std::allocator<rocksdb::ColumnFamilyDescriptor> > const&, std::vector<rocksdb::ColumnFamilyHandle*, std::allocator<rocksdb::ColumnFamilyHandle*> >*, rocksdb::DB**)+0x15) [0x55a433620375]
14: (RocksDBStore::do_open(std::ostream&, bool, bool, std::vector<KeyValueDB::ColumnFamily, std::allocator<KeyValueDB::ColumnFamily> > const*)+0x24e4) [0x55a4335af714]
15: (BlueStore::_open_db(bool, bool, bool)+0x1239) [0x55a433395989]
16: (BlueStore::_open_db_and_around(bool)+0x19b) [0x55a4333aca0b]
17: (BlueStore::_fsck(BlueStore::FSCKDepth, bool)+0x1e1) [0x55a4333fe0a1]
18: (main()+0x136e) [0x55a4332f916e]
19: (__libc_start_main()+0xf3) [0x7fc417a176a3]
20: (_start()+0x2e) [0x55a43331b65e]
0> 2020-12-16T13:24:20.069+0000 7fc4240cc240 -1 *** Caught signal (Aborted) **
in thread 7fc4240cc240 thread_name:ceph-bluestore-
ceph version 15.2.6 (cb8c61a60551b72614257d632a574d420064c17a) octopus (stable)
1: (()+0x12dd0) [0x7fc4193e6dd0]
2: (gsignal()+0x10f) [0x7fc417a2b70f]
3: (abort()+0x127) [0x7fc417a15b25]
4: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)+0x1b6) [0x7fc41a1e839d]
5: (BlueFS::_flush_range(BlueFS::FileWriter*, unsigned long, unsigned long)+0x1a43) [0x55a433337073]
6: (BlueFS::_flush(BlueFS::FileWriter*, bool, bool*)+0x110) [0x55a433337760]
7: (BlueFS::_flush(BlueFS::FileWriter*, bool, std::unique_lock<std::mutex>&)+0x32) [0x55a4333484f2]
8: (BlueRocksWritableFile::Flush()+0x72) [0x55a4335204a2]
9: (rocksdb::WritableFileWriter::Flush()+0x1fc) [0x55a43374c95c]
10: (rocksdb::WritableFileWriter::Sync(bool)+0x1f) [0x55a43374cadf]
11: (rocksdb::BuildTable(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, rocksdb::Env*, rocksdb::ImmutableCFOptions const&, rocksdb::MutableCFOptions const&, rocksdb::EnvOptions const&, rocksdb::TableCache*, rocksdb::InternalIteratorBase<rocksdb::Slice>*, std::vector<std::unique_ptr<rocksdb::FragmentedRangeTombstoneIterator, std::default_delete<rocksdb::FragmentedRangeTombstoneIterator> >, std::allocator<std::unique_ptr<rocksdb::FragmentedRangeTombstoneIterator, std::default_delete<rocksdb::FragmentedRangeTombstoneIterator> > > >, rocksdb::FileMetaData*, rocksdb::InternalKeyComparator const&, std::vector<std::unique_ptr<rocksdb::IntTblPropCollectorFactory, std::default_delete<rocksdb::IntTblPropCollectorFactory> >, std::allocator<std::unique_ptr<rocksdb::IntTblPropCollectorFactory, std::default_delete<rocksdb::IntTblPropCollectorFactory> > > > const*, unsigned int, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::vector<unsigned long, std::allocator<unsigned long> >, unsigned long, rocksdb::SnapshotChecker*, rocksdb::CompressionType, unsigned long, rocksdb::CompressionOptions const&, bool, rocksdb::InternalStats*, rocksdb::TableFileCreationReason, rocksdb::EventLogger*, int, rocksdb::Env::IOPriority, rocksdb::TableProperties*, int, unsigned long, unsigned long, rocksdb::Env::WriteLifeTimeHint)+0x2be5) [0x55a433774085]
12: (rocksdb::DBImpl::WriteLevel0TableForRecovery(int, rocksdb::ColumnFamilyData*, rocksdb::MemTable*, rocksdb::VersionEdit*)+0xc07) [0x55a4336217c7]
13: (rocksdb::DBImpl::RecoverLogFiles(std::vector<unsigned long, std::allocator<unsigned long> > const&, unsigned long*, bool)+0x12dd) [0x55a4336232cd]
14: (rocksdb::DBImpl::Recover(std::vector<rocksdb::ColumnFamilyDescriptor, std::allocator<rocksdb::ColumnFamilyDescriptor> > const&, bool, bool, bool)+0xb09) [0x55a433624789]
15: (rocksdb::DBImpl::Open(rocksdb::DBOptions const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::vector<rocksdb::ColumnFamilyDescriptor, std::allocator<rocksdb::ColumnFamilyDescriptor> > const&, std::vector<rocksdb::ColumnFamilyHandle*, std::allocator<rocksdb::ColumnFamilyHandle*> >*, rocksdb::DB**, bool, bool)+0x9f5) [0x55a43361f025]
16: (rocksdb::DB::Open(rocksdb::DBOptions const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::vector<rocksdb::ColumnFamilyDescriptor, std::allocator<rocksdb::ColumnFamilyDescriptor> > const&, std::vector<rocksdb::ColumnFamilyHandle*, std::allocator<rocksdb::ColumnFamilyHandle*> >*, rocksdb::DB**)+0x15) [0x55a433620375]
17: (RocksDBStore::do_open(std::ostream&, bool, bool, std::vector<KeyValueDB::ColumnFamily, std::allocator<KeyValueDB::ColumnFamily> > const*)+0x24e4) [0x55a4335af714]
18: (BlueStore::_open_db(bool, bool, bool)+0x1239) [0x55a433395989]
19: (BlueStore::_open_db_and_around(bool)+0x19b) [0x55a4333aca0b]
20: (BlueStore::_fsck(BlueStore::FSCKDepth, bool)+0x1e1) [0x55a4333fe0a1]
21: (main()+0x136e) [0x55a4332f916e]
22: (__libc_start_main()+0xf3) [0x7fc417a176a3]
23: (_start()+0x2e) [0x55a43331b65e]
NOTE: a copy of the executable, or `objdump -rdS <executable>` is needed to interpret this.
Aborted (core dumped)
_______________________________________________
ceph-users mailing list -- ceph-users@xxxxxxx
To unsubscribe send an email to ceph-users-leave@xxxxxxx
