Hi,
Most of my osd's use's part of slow storage for RocksDB, but some is
not. I investigated this and think because is most oldest of Bluestore
osd's in this cluster.
I figure out this by /var/lib/osd/<osd.id> creation date, don't know is
possible or not to determine real osd creation date from osd metadata.
In my case is Dec 2017, so this is a Luminous 12.2.2 - this osd's keep
all RocksDB on NVMe storage. Another osd's was created on 12.2.4 or 12.2.5.
I was trying to compact object store omap (`ceph tell <osd.id>
compact`), this is helps, but after some period of time slow is grow's
again. Graph: [1]
BlueFS of two osd's (33 is old osd, slow is not used): [2], [3]
[root@ceph-osd5 /]# ceph daemon osd.33 perf dump | jq .bluefs
{
"gift_bytes": 0,
"reclaim_bytes": 0,
"db_total_bytes": 32212246528,
"db_used_bytes": 6128926720,
"wal_total_bytes": 1073737728,
"wal_used_bytes": 528482304,
"slow_total_bytes": 240043163648,
"slow_used_bytes": 0, <----------------------------------------------
"num_files": 109,
"log_bytes": 12476416,
"log_compactions": 23,
"logged_bytes": 518922240,
"files_written_wal": 2,
"files_written_sst": 26235,
"bytes_written_wal": 1480717702902,
"bytes_written_sst": 1382491287991
}
[root@ceph-osd5 /]# ceph daemon osd.73 perf dump | jq .bluefs
{
"gift_bytes": 0,
"reclaim_bytes": 0,
"db_total_bytes": 30064762880,
"db_used_bytes": 1874853888,
"wal_total_bytes": 1073737728,
"wal_used_bytes": 529530880,
"slow_total_bytes": 240043163648,
"slow_used_bytes": 4678746112, <----------------------------------
"num_files": 116,
"log_bytes": 9302016,
"log_compactions": 23,
"logged_bytes": 573952000,
"files_written_wal": 2,
"files_written_sst": 19755,
"bytes_written_wal": 1401924883737,
"bytes_written_sst": 980145086778
}
And osd metadata for this osd's:
{
"id": 33,
"arch": "x86_64",
"back_addr": "10.10.10.6:6812/175022",
"back_iface": "vlan3",
"bluefs": "1",
"bluefs_db_access_mode": "blk",
"bluefs_db_block_size": "4096",
"bluefs_db_dev": "259:22",
"bluefs_db_dev_node": "nvme2n1",
"bluefs_db_driver": "KernelDevice",
"bluefs_db_model": "INTEL SSDPEDMD400G4 ",
"bluefs_db_partition_path": "/dev/nvme2n1p7",
"bluefs_db_rotational": "0",
"bluefs_db_serial": "CVFT4324002Q400BGN ",
"bluefs_db_size": "32212254720",
"bluefs_db_type": "nvme",
"bluefs_single_shared_device": "0",
"bluefs_slow_access_mode": "blk",
"bluefs_slow_block_size": "4096",
"bluefs_slow_dev": "8:112",
"bluefs_slow_dev_node": "sdh",
"bluefs_slow_driver": "KernelDevice",
"bluefs_slow_model": "TOSHIBA HDWE160 ",
"bluefs_slow_partition_path": "/dev/sdh2",
"bluefs_slow_rotational": "1",
"bluefs_slow_size": "6001069199360",
"bluefs_slow_type": "hdd",
"bluefs_wal_access_mode": "blk",
"bluefs_wal_block_size": "4096",
"bluefs_wal_dev": "259:22",
"bluefs_wal_dev_node": "nvme2n1",
"bluefs_wal_driver": "KernelDevice",
"bluefs_wal_model": "INTEL SSDPEDMD400G4 ",
"bluefs_wal_partition_path": "/dev/nvme2n1p8",
"bluefs_wal_rotational": "0",
"bluefs_wal_serial": "CVFT4324002Q400BGN ",
"bluefs_wal_size": "1073741824",
"bluefs_wal_type": "nvme",
"bluestore_bdev_access_mode": "blk",
"bluestore_bdev_block_size": "4096",
"bluestore_bdev_dev": "8:112",
"bluestore_bdev_dev_node": "sdh",
"bluestore_bdev_driver": "KernelDevice",
"bluestore_bdev_model": "TOSHIBA HDWE160 ",
"bluestore_bdev_partition_path": "/dev/sdh2",
"bluestore_bdev_rotational": "1",
"bluestore_bdev_size": "6001069199360",
"bluestore_bdev_type": "hdd",
"ceph_version": "ceph version 12.2.10
(177915764b752804194937482a39e95e0ca3de94) luminous (stable)",
"cpu": "Intel(R) Xeon(R) CPU E5-2609 v4 @ 1.70GHz",
"default_device_class": "hdd",
"distro": "centos",
"distro_description": "CentOS Linux 7 (Core)",
"distro_version": "7",
"front_addr": "172.16.16.16:6812/175022",
"front_iface": "vlan4",
"hb_back_addr": "10.10.10.6:6813/175022",
"hb_front_addr": "172.16.16.16:6813/175022",
"hostname": "ceph-osd5",
"journal_rotational": "0",
"kernel_description": "#1 SMP Tue Aug 14 21:49:04 UTC 2018",
"kernel_version": "3.10.0-862.11.6.el7.x86_64",
"mem_swap_kb": "0",
"mem_total_kb": "65724256",
"os": "Linux",
"osd_data": "/var/lib/ceph/osd/ceph-33",
"osd_objectstore": "bluestore",
"rotational": "1"
}
[root@ceph-mon0 /]# ceph osd metadata osd.37
{
"id": 37,
"arch": "x86_64",
"back_addr": "10.10.10.4:6808/164908",
"back_iface": "vlan3",
"bluefs": "1",
"bluefs_db_access_mode": "blk",
"bluefs_db_block_size": "4096",
"bluefs_db_dev": "259:22",
"bluefs_db_dev_node": "nvme2n1",
"bluefs_db_driver": "KernelDevice",
"bluefs_db_model": "INTEL SSDPEDMD400G4 ",
"bluefs_db_partition_path": "/dev/nvme2n1p7",
"bluefs_db_rotational": "0",
"bluefs_db_serial": "CVFT5334007G400BGN ",
"bluefs_db_size": "32212254720",
"bluefs_db_type": "nvme",
"bluefs_single_shared_device": "0",
"bluefs_slow_access_mode": "blk",
"bluefs_slow_block_size": "4096",
"bluefs_slow_dev": "8:112",
"bluefs_slow_dev_node": "sdh",
"bluefs_slow_driver": "KernelDevice",
"bluefs_slow_model": "TOSHIBA HDWE160 ",
"bluefs_slow_partition_path": "/dev/sdh2",
"bluefs_slow_rotational": "1",
"bluefs_slow_size": "6001069199360",
"bluefs_slow_type": "hdd",
"bluefs_wal_access_mode": "blk",
"bluefs_wal_block_size": "4096",
"bluefs_wal_dev": "259:22",
"bluefs_wal_dev_node": "nvme2n1",
"bluefs_wal_driver": "KernelDevice",
"bluefs_wal_model": "INTEL SSDPEDMD400G4 ",
"bluefs_wal_partition_path": "/dev/nvme2n1p8",
"bluefs_wal_rotational": "0",
"bluefs_wal_serial": "CVFT5334007G400BGN ",
"bluefs_wal_size": "1073741824",
"bluefs_wal_type": "nvme",
"bluestore_bdev_access_mode": "blk",
"bluestore_bdev_block_size": "4096",
"bluestore_bdev_dev": "8:112",
"bluestore_bdev_dev_node": "sdh",
"bluestore_bdev_driver": "KernelDevice",
"bluestore_bdev_model": "TOSHIBA HDWE160 ",
"bluestore_bdev_partition_path": "/dev/sdh2",
"bluestore_bdev_rotational": "1",
"bluestore_bdev_size": "6001069199360",
"bluestore_bdev_type": "hdd",
"ceph_version": "ceph version 12.2.10
(177915764b752804194937482a39e95e0ca3de94) luminous (stable)",
"cpu": "Intel(R) Xeon(R) CPU E5-2609 v4 @ 1.70GHz",
"default_device_class": "hdd",
"distro": "centos",
"distro_description": "CentOS Linux 7 (Core)",
"distro_version": "7",
"front_addr": "172.16.16.14:6807/164908",
"front_iface": "vlan4",
"hb_back_addr": "10.10.10.4:6809/164908",
"hb_front_addr": "172.16.16.14:6808/164908",
"hostname": "ceph-osd3",
"journal_rotational": "0",
"kernel_description": "#1 SMP Tue Aug 14 21:49:04 UTC 2018",
"kernel_version": "3.10.0-862.11.6.el7.x86_64",
"mem_swap_kb": "0",
"mem_total_kb": "65724256",
"os": "Linux",
"osd_data": "/var/lib/ceph/osd/ceph-37",
"osd_objectstore": "bluestore",
"rotational": "1"
}
[1] https://ibb.co/ZcyTbhB
[2] https://ibb.co/MMpSvXc
[3] https://ibb.co/jVr0LK4
Thanks,
k
_______________________________________________
ceph-users mailing list
ceph-users@xxxxxxxxxxxxxx
http://lists.ceph.com/listinfo.cgi/ceph-users-ceph.com
