dd is only using one thread, and I'm not sure it's deep enough. Try using fio with similar amounts of threads.
----------------
Robert LeBlanc
PGP Fingerprint 79A2 9CA4 6CC4 45DD A904 C70E E654 3BB2 FA62 B9F1
Robert LeBlanc
PGP Fingerprint 79A2 9CA4 6CC4 45DD A904 C70E E654 3BB2 FA62 B9F1
On Wed, Aug 14, 2019 at 8:26 AM Olivier AUDRY <olivier@xxxxxxx> wrote:
helloI got a huge difference of read/write performance on my ceph cluster when I use rbd.rbd bench reach the limit of my cluster (1gbps network) when the performance in mapped rbd is very low. 30MB/s.ceph version : ceph version 14.2.2 (4f8fa0a0024755aae7d95567c63f11d6862d55be) nautilus (stable)kernel version : 4.19.0-5-amd64ceph -scluster:id: fdf6eba0-cde9-401c-8901-751eb1e8b7d2health: HEALTH_OKservices:mon: 3 daemons, quorum gra3-k8s001,sbg2-k8s001,rbx4-k8s001 (age 41h)mgr: sbg2-k8s001(active, since 46h), standbys: gra3-k8s001, rbx4-k8s001osd: 10 osds: 10 up (since 41h), 10 in (since 41h)rgw: 3 daemons active (gra3-k8s001, rbx4-k8s001, sbg2-k8s001)data:pools: 8 pools, 240 pgsobjects: 109.14k objects, 229 GiBusage: 694 GiB used, 3.0 TiB / 3.7 TiB availpgs: 240 active+cleanhere the benchrbd create --size=10G kube/benchrbd feature disable kube/bench object-map fast-diff deep-flattenrbd -p kube bench kube/bench --io-type write --io-size 8192 --io-threads 256 --io-total 10G --io-pattern seqelapsed: 110 ops: 1310720 ops/sec: 11884.39 bytes/sec: 97356926.64this reach the 1gbps limit of my network. Looks good.let's test random writerbd -p kube bench kube/bench --io-type write --io-size 8192 --io-threads 256 --io-total 10G --io-pattern randelapsed: 125 ops: 1310720 ops/sec: 10416.31 bytes/sec: 85330446.58this reach the 1gbps limit of my network. Looks good.rbd -p kube bench kube/bench --io-type read --io-size 8192 --io-threads 256 --io-total 10G --io-pattern seqelapsed: 40 ops: 1310720 ops/sec: 32712.03 bytes/sec: 267976974.35this is great perf !let's try randrbd -p kube bench kube/bench --io-type read --io-size 8192 --io-threads 256 --io-total 10G --io-pattern randelapsed: 29 ops: 1310720 ops/sec: 43859.78 bytes/sec: 359299337.92this is great perf !But when I mount the image :rbd map bench --pool kube --name client.admin/sbin/mkfs.ext4 /dev/rbd/kube/benchmount /dev/rbd/kube/bench /mnt/cd /mnt/dd if=/dev/zero of=test bs=8192k count=100 oflag=direct838860800 bytes (839 MB, 800 MiB) copied, 24.6185 s, 34.1 MB/s34.1MB/s vs 85MB/s ....here the full config{"name": "mon.sbg2-k8s001","cluster": "ceph","admin_socket": "/var/run/ceph/ceph-mon.sbg2-k8s001.asok","admin_socket_mode": "","auth_client_required": "cephx","auth_cluster_required": "cephx","auth_debug": "false","auth_mon_ticket_ttl": "43200.000000","auth_service_required": "cephx","auth_service_ticket_ttl": "3600.000000","auth_supported": "","bdev_aio": "true","bdev_aio_max_queue_depth": "1024","bdev_aio_poll_ms": "250","bdev_aio_reap_max": "16","bdev_async_discard": "false","bdev_block_size": "4096","bdev_debug_aio": "false","bdev_debug_aio_log_age": "5.000000","bdev_debug_aio_suicide_timeout": "60.000000","bdev_debug_inflight_ios": "false","bdev_enable_discard": "false","bdev_inject_crash": "0","bdev_inject_crash_flush_delay": "2","bdev_nvme_retry_count": "-1","bdev_nvme_unbind_from_kernel": "false","bluefs_alloc_size": "1048576","bluefs_allocator": "bitmap","bluefs_buffered_io": "true","bluefs_compact_log_sync": "false","bluefs_log_compact_min_ratio": "5.000000","bluefs_log_compact_min_size": "16777216","bluefs_max_log_runway": "4194304","bluefs_max_prefetch": "1048576","bluefs_min_flush_size": "524288","bluefs_min_log_runway": "1048576","bluefs_preextend_wal_files": "false","bluefs_sync_write": "false","bluestore_2q_cache_kin_ratio": "0.500000","bluestore_2q_cache_kout_ratio": "0.500000","bluestore_allocator": "bitmap","bluestore_bitmapallocator_blocks_per_zone": "1024","bluestore_bitmapallocator_span_size": "1024","bluestore_blobid_prealloc": "10240","bluestore_block_create": "true","bluestore_block_db_create": "false","bluestore_block_db_path": "","bluestore_block_db_size": "0","bluestore_block_path": "","bluestore_block_preallocate_file": "false","bluestore_block_size": "10737418240","bluestore_block_wal_create": "false","bluestore_block_wal_path": "","bluestore_block_wal_size": "100663296","bluestore_bluefs": "true","bluestore_bluefs_alloc_failure_dump_interval": "0.000000","bluestore_bluefs_balance_interval": "1.000000","bluestore_bluefs_db_compatibility": "true","bluestore_bluefs_env_mirror": "false","bluestore_bluefs_gift_ratio": "0.020000","bluestore_bluefs_max_ratio": "0.900000","bluestore_bluefs_min": "1073741824","bluestore_bluefs_min_free": "1073741824","bluestore_bluefs_min_ratio": "0.020000","bluestore_bluefs_reclaim_ratio": "0.200000","bluestore_cache_autotune": "true","bluestore_cache_autotune_interval": "5.000000","bluestore_cache_kv_ratio": "0.400000","bluestore_cache_meta_ratio": "0.400000","bluestore_cache_size": "0","bluestore_cache_size_hdd": "1073741824","bluestore_cache_size_ssd": "3221225472","bluestore_cache_trim_interval": "0.050000","bluestore_cache_trim_max_skip_pinned": "64","bluestore_cache_type": "2q","bluestore_clone_cow": "true","bluestore_compression_algorithm": "snappy","bluestore_compression_max_blob_size": "0","bluestore_compression_max_blob_size_hdd": "524288","bluestore_compression_max_blob_size_ssd": "65536","bluestore_compression_min_blob_size": "0","bluestore_compression_min_blob_size_hdd": "131072","bluestore_compression_min_blob_size_ssd": "8192","bluestore_compression_mode": "none","bluestore_compression_required_ratio": "0.875000","bluestore_csum_type": "crc32c","bluestore_debug_freelist": "false","bluestore_debug_fsck_abort": "false","bluestore_debug_inject_bug21040": "false","bluestore_debug_inject_csum_err_probability": "0.000000","bluestore_debug_inject_read_err": "false","bluestore_debug_misc": "false","bluestore_debug_no_reuse_blocks": "false","bluestore_debug_omit_block_device_write": "false","bluestore_debug_omit_kv_commit": "false","bluestore_debug_permit_any_bdev_label": "false","bluestore_debug_prefill": "0.000000","bluestore_debug_prefragment_max": "1048576","bluestore_debug_random_read_err": "0.000000","bluestore_debug_randomize_serial_transaction": "0","bluestore_debug_small_allocations": "0","bluestore_default_buffered_read": "true","bluestore_default_buffered_write": "false","bluestore_deferred_batch_ops": "0","bluestore_deferred_batch_ops_hdd": "64","bluestore_deferred_batch_ops_ssd": "16","bluestore_extent_map_inline_shard_prealloc_size": "256","bluestore_extent_map_shard_max_size": "1200","bluestore_extent_map_shard_min_size": "150","bluestore_extent_map_shard_target_size": "500","bluestore_extent_map_shard_target_size_slop": "0.200000","bluestore_freelist_blocks_per_key": "128","bluestore_fsck_on_mkfs": "true","bluestore_fsck_on_mkfs_deep": "false","bluestore_fsck_on_mount": "false","bluestore_fsck_on_mount_deep": "true","bluestore_fsck_on_umount": "false","bluestore_fsck_on_umount_deep": "true","bluestore_fsck_read_bytes_cap": "67108864","bluestore_gc_enable_blob_threshold": "0","bluestore_gc_enable_total_threshold": "0","bluestore_ignore_data_csum": "false","bluestore_kvbackend": "rocksdb","bluestore_log_omap_iterator_age": "1.000000","bluestore_log_op_age": "5.000000","bluestore_max_alloc_size": "0","bluestore_max_blob_size": "0","bluestore_max_blob_size_hdd": "524288","bluestore_max_blob_size_ssd": "65536","bluestore_max_deferred_txc": "32","bluestore_min_alloc_size": "0","bluestore_min_alloc_size_hdd": "65536","bluestore_min_alloc_size_ssd": "16384","bluestore_nid_prealloc": "1024","bluestore_no_per_pool_stats_tolerance": "until_repair","bluestore_prefer_deferred_size": "0","bluestore_prefer_deferred_size_hdd": "32768","bluestore_prefer_deferred_size_ssd": "0","bluestore_retry_disk_reads": "3","bluestore_rocksdb_cf": "false","bluestore_rocksdb_cfs": "M= P= L=","bluestore_rocksdb_options": "compression=kNoCompression,max_write_buffer_number=4,min_write_buffer_number_to_merge=1,recycle_log_file_num=4,write_buffer_size=268435456,writable_file_max_buffer_size=0,compaction_readahead_size=2097152","bluestore_spdk_coremask": "0x1","bluestore_spdk_io_sleep": "5","bluestore_spdk_max_io_completion": "0","bluestore_spdk_mem": "512","bluestore_sync_submit_transaction": "false","bluestore_throttle_bytes": "67108864","bluestore_throttle_cost_per_io": "0","bluestore_throttle_cost_per_io_hdd": "670000","bluestore_throttle_cost_per_io_ssd": "4000","bluestore_throttle_deferred_bytes": "134217728","bluestore_warn_on_bluefs_spillover": "true","bluestore_warn_on_legacy_statfs": "true","cephx_cluster_require_signatures": "false","cephx_cluster_require_version": "1","cephx_require_signatures": "false","cephx_require_version": "1","cephx_service_require_signatures": "false","cephx_service_require_version": "1","cephx_sign_messages": "true","chdir": "","client_acl_type": "","client_cache_mid": "0.750000","client_cache_size": "16384","client_caps_release_delay": "5","client_check_pool_perm": "true","client_debug_force_sync_read": "false","client_debug_getattr_caps": "false","client_debug_inject_tick_delay": "0","client_die_on_failed_dentry_invalidate": "true","client_die_on_failed_remount": "false","client_dirsize_rbytes": "true","client_force_lazyio": "false","client_inject_fixed_oldest_tid": "false","client_inject_release_failure": "false","client_max_inline_size": "4096","client_mds_namespace": "","client_metadata": "","client_mount_gid": "-1","client_mount_timeout": "300.000000","client_mount_uid": "-1","client_mountpoint": "/","client_notify_timeout": "10","client_oc": "true","client_oc_max_dirty": "104857600","client_oc_max_dirty_age": "5.000000","client_oc_max_objects": "1000","client_oc_size": "209715200","client_oc_target_dirty": "8388608","client_permissions": "true","client_quota_df": "true","client_readahead_max_bytes": "0","client_readahead_max_periods": "4","client_readahead_min": "131072","client_reconnect_stale": "false","client_snapdir": ".snap","client_tick_interval": "1.000000","client_trace": "","client_try_dentry_invalidate": "false","client_use_faked_inos": "false","client_use_random_mds": "false","clog_to_graylog": "false","clog_to_graylog_host": "127.0.0.1","clog_to_graylog_port": "12201","clog_to_monitors": "default=true","clog_to_syslog": "false","clog_to_syslog_facility": "default=daemon audit=local0","clog_to_syslog_level": "info","cluster_addr": "-","cluster_network": "192.168.212.0/24","cluster_network_interface": "","compressor_zlib_isal": "false","compressor_zlib_level": "5","crash_dir": "/var/lib/ceph/crash","crush_location": "","crush_location_hook": "","crush_location_hook_timeout": "10","daemonize": "false","debug_allow_any_pool_priority": "false","debug_asok": "1/5","debug_asok_assert_abort": "false","debug_asserts_on_shutdown": "false","debug_auth": "1/5","debug_bdev": "1/3","debug_bluefs": "1/5","debug_bluestore": "1/5","debug_buffer": "0/1","debug_civetweb": "1/10","debug_client": "0/5","debug_compressor": "1/5","debug_context": "0/1","debug_crush": "1/1","debug_crypto": "1/5","debug_deliberately_leak_memory": "false","debug_dpdk": "1/5","debug_eventtrace": "1/5","debug_filer": "0/1","debug_filestore": "1/3","debug_finisher": "1/1","debug_fuse": "1/5","debug_heartbeatmap": "1/5","debug_javaclient": "1/5","debug_journal": "1/3","debug_journaler": "0/5","debug_kinetic": "1/5","debug_kstore": "1/5","debug_leveldb": "4/5","debug_lockdep": "0/1","debug_mds": "1/5","debug_mds_balancer": "1/5","debug_mds_locker": "1/5","debug_mds_log": "1/5","debug_mds_log_expire": "1/5","debug_mds_migrator": "1/5","debug_memdb": "4/5","debug_mgr": "1/5","debug_mgrc": "1/5","debug_mon": "1/5","debug_monc": "0/10","debug_ms": "0/0","debug_none": "0/5","debug_objclass": "0/5","debug_objectcacher": "0/5","debug_objecter": "0/1","debug_optracker": "0/5","debug_osd": "1/5","debug_paxos": "1/5","debug_perfcounter": "1/5","debug_rados": "0/5","debug_rbd": "0/5","debug_rbd_mirror": "0/5","debug_rbd_replay": "0/5","debug_refs": "0/0","debug_reserver": "1/1","debug_rgw": "1/5","debug_rgw_sync": "1/5","debug_rocksdb": "4/5","debug_striper": "0/1","debug_throttle": "1/1","debug_timer": "0/1","debug_tp": "0/5","debug_xio": "1/5","device_failure_prediction_mode": "none","enable_experimental_unrecoverable_data_corrupting_features": "","erasure_code_dir": "/usr/lib/ceph/erasure-code","err_to_graylog": "false","err_to_stderr": "true","err_to_syslog": "false","event_tracing": "false","fake_statfs_for_testing": "0","fatal_signal_handlers": "true","filer_max_purge_ops": "10","filer_max_truncate_ops": "128","filestore_apply_finisher_threads": "1","filestore_blackhole": "false","filestore_btrfs_clone_range": "true","filestore_btrfs_snap": "true","filestore_caller_concurrency": "10","filestore_collect_device_partition_information": "true","filestore_commit_timeout": "600.000000","filestore_debug_inject_read_err": "false","filestore_debug_omap_check": "false","filestore_debug_random_read_err": "0.000000","filestore_debug_verify_split": "false","filestore_dump_file": "","filestore_expected_throughput_bytes": "209715200.000000","filestore_expected_throughput_ops": "200.000000","filestore_fadvise": "true","filestore_fail_eio": "true","filestore_fd_cache_shards": "16","filestore_fd_cache_size": "128","filestore_fiemap": "false","filestore_fiemap_threshold": "4096","filestore_fsync_flushes_journal_data": "false","filestore_index_retry_probability": "0.000000","filestore_inject_stall": "0","filestore_journal_parallel": "false","filestore_journal_trailing": "false","filestore_journal_writeahead": "false","filestore_kill_at": "0","filestore_max_alloc_hint_size": "1048576","filestore_max_inline_xattr_size": "0","filestore_max_inline_xattr_size_btrfs": "2048","filestore_max_inline_xattr_size_other": "512","filestore_max_inline_xattr_size_xfs": "65536","filestore_max_inline_xattrs": "0","filestore_max_inline_xattrs_btrfs": "10","filestore_max_inline_xattrs_other": "2","filestore_max_inline_xattrs_xfs": "10","filestore_max_sync_interval": "5.000000","filestore_max_xattr_value_size": "0","filestore_max_xattr_value_size_btrfs": "65536","filestore_max_xattr_value_size_other": "1024","filestore_max_xattr_value_size_xfs": "65536","filestore_merge_threshold": "-10","filestore_min_sync_interval": "0.010000","filestore_odsync_write": "false","filestore_omap_backend": "rocksdb","filestore_omap_backend_path": "","filestore_omap_header_cache_size": "1024","filestore_ondisk_finisher_threads": "1","filestore_op_thread_suicide_timeout": "180","filestore_op_thread_timeout": "60","filestore_op_threads": "2","filestore_punch_hole": "false","filestore_queue_high_delay_multiple": "0.000000","filestore_queue_high_delay_multiple_bytes": "0.000000","filestore_queue_high_delay_multiple_ops": "0.000000","filestore_queue_high_threshhold": "0.900000","filestore_queue_low_threshhold": "0.300000","filestore_queue_max_bytes": "104857600","filestore_queue_max_delay_multiple": "0.000000","filestore_queue_max_delay_multiple_bytes": "0.000000","filestore_queue_max_delay_multiple_ops": "0.000000","filestore_queue_max_ops": "50","filestore_rocksdb_options": "max_background_jobs=10,compaction_readahead_size=2097152,compression=kNoCompression","filestore_seek_data_hole": "false","filestore_sloppy_crc": "false","filestore_sloppy_crc_block_size": "65536","filestore_splice": "false","filestore_split_multiple": "2","filestore_split_rand_factor": "20","filestore_update_to": "1000","filestore_wbthrottle_btrfs_bytes_hard_limit": "419430400","filestore_wbthrottle_btrfs_bytes_start_flusher": "41943040","filestore_wbthrottle_btrfs_inodes_hard_limit": "5000","filestore_wbthrottle_btrfs_inodes_start_flusher": "500","filestore_wbthrottle_btrfs_ios_hard_limit": "5000","filestore_wbthrottle_btrfs_ios_start_flusher": "500","filestore_wbthrottle_enable": "true","filestore_wbthrottle_xfs_bytes_hard_limit": "419430400","filestore_wbthrottle_xfs_bytes_start_flusher": "41943040","filestore_wbthrottle_xfs_inodes_hard_limit": "5000","filestore_wbthrottle_xfs_inodes_start_flusher": "500","filestore_wbthrottle_xfs_ios_hard_limit": "5000","filestore_wbthrottle_xfs_ios_start_flusher": "500","filestore_xfs_extsize": "false","filestore_zfs_snap": "false","fio_dir": "/tmp/fio","fsid": "fdf6eba0-cde9-401c-8901-751eb1e8b7d2","fuse_allow_other": "true","fuse_atomic_o_trunc": "true","fuse_big_writes": "false","fuse_debug": "false","fuse_default_permissions": "false","fuse_disable_pagecache": "false","fuse_max_write": "0","fuse_multithreaded": "true","fuse_require_active_mds": "true","fuse_set_user_groups": "true","fuse_syncfs_on_mksnap": "true","fuse_use_invalidate_cb": "true","gss_ktab_client_file": "/var/lib/ceph/mon.sbg2-k8s001/gss_client_mon.sbg2-k8s001.ktab","gss_target_name": "ceph","heartbeat_file": "","heartbeat_inject_failure": "0","heartbeat_interval": "5","host": "","inject_early_sigterm": "false","journal_aio": "true","journal_align_min_size": "65536","journal_block_align": "true","journal_block_size": "4096","journal_dio": "true","journal_discard": "false","journal_force_aio": "false","journal_ignore_corruption": "false","journal_max_write_bytes": "1073714824","journal_max_write_entries": "10000","journal_replay_from": "0","journal_throttle_high_multiple": "0.000000","journal_throttle_high_threshhold": "0.900000","journal_throttle_low_threshhold": "0.600000","journal_throttle_max_multiple": "0.000000","journal_write_header_frequency": "0","journal_zero_on_create": "false","journaler_prefetch_periods": "10","journaler_prezero_periods": "5","journaler_write_head_interval": "15","key": "","keyfile": "","keyring": "/var/lib/ceph/mon/ceph-sbg2-k8s001/keyring","kinetic_hmac_key": "asdfasdf","kinetic_host": "","kinetic_port": "8123","kinetic_use_ssl": "false","kinetic_user_id": "1","kstore_backend": "rocksdb","kstore_default_stripe_size": "65536","kstore_fsck_on_mount": "false","kstore_fsck_on_mount_deep": "true","kstore_max_bytes": "67108864","kstore_max_ops": "512","kstore_nid_prealloc": "1024","kstore_onode_map_size": "1024","kstore_rocksdb_options": "compression=kNoCompression","kstore_sync_submit_transaction": "false","kstore_sync_transaction": "false","leveldb_block_size": "65536","leveldb_bloom_size": "0","leveldb_cache_size": "536870912","leveldb_compact_on_mount": "false","leveldb_compression": "false","leveldb_log": "","leveldb_log_to_ceph_log": "true","leveldb_max_open_files": "0","leveldb_paranoid": "false","leveldb_write_buffer_size": "33554432","lockdep": "false","lockdep_force_backtrace": "false","log_coarse_timestamps": "true","log_file": "/var/log/ceph/ceph-mon.sbg2-k8s001.log","log_flush_on_exit": "false","log_graylog_host": "127.0.0.1","log_graylog_port": "12201","log_max_new": "1000","log_max_recent": "10000","log_stderr_prefix": "","log_stop_at_utilization": "0.970000","log_to_file": "true","log_to_graylog": "false","log_to_stderr": "false","log_to_syslog": "false","max_rotating_auth_attempts": "10","mds_action_on_write_error": "1","mds_bal_export_pin": "true","mds_bal_fragment_dirs": "true","mds_bal_fragment_fast_factor": "1.500000","mds_bal_fragment_interval": "5","mds_bal_fragment_size_max": "100000","mds_bal_idle_threshold": "0.000000","mds_bal_interval": "10","mds_bal_max": "-1","mds_bal_max_until": "-1","mds_bal_merge_size": "50","mds_bal_midchunk": "0.300000","mds_bal_min_rebalance": "0.100000","mds_bal_min_start": "0.200000","mds_bal_minchunk": "0.001000","mds_bal_mode": "0","mds_bal_need_max": "1.200000","mds_bal_need_min": "0.800000","mds_bal_replicate_threshold": "8000.000000","mds_bal_sample_interval": "3.000000","mds_bal_split_bits": "3","mds_bal_split_rd": "25000.000000","mds_bal_split_size": "10000","mds_bal_split_wr": "10000.000000","mds_bal_target_decay": "10.000000","mds_bal_unreplicate_threshold": "0.000000","mds_beacon_grace": "15.000000","mds_beacon_interval": "4.000000","mds_cache_memory_limit": "1073741824","mds_cache_mid": "0.700000","mds_cache_reservation": "0.050000","mds_cache_size": "0","mds_cache_trim_decay_rate": "1.000000","mds_cache_trim_threshold": "65536","mds_cap_revoke_eviction_timeout": "0.000000","mds_client_prealloc_inos": "1000","mds_client_writeable_range_max_inc_objs": "1024","mds_damage_table_max_entries": "10000","mds_data": "/var/lib/ceph/mds/ceph-sbg2-k8s001","mds_debug_auth_pins": "false","mds_debug_frag": "false","mds_debug_scatterstat": "false","mds_debug_subtrees": "false","mds_decay_halflife": "5.000000","mds_default_dir_hash": "2","mds_dir_keys_per_op": "16384","mds_dir_max_commit_size": "10","mds_dirstat_min_interval": "1.000000","mds_dump_cache_after_rejoin": "false","mds_dump_cache_on_map": "false","mds_dump_cache_threshold_file": "0","mds_dump_cache_threshold_formatter": "1073741824","mds_early_reply": "true","mds_enable_op_tracker": "true","mds_enforce_unique_name": "true","mds_freeze_tree_timeout": "30.000000","mds_hack_allow_loading_invalid_metadata": "false","mds_health_cache_threshold": "1.500000","mds_health_summarize_threshold": "10","mds_heartbeat_grace": "15.000000","mds_inject_migrator_session_race": "false","mds_inject_traceless_reply_probability": "0.000000","mds_journal_format": "1","mds_kill_create_at": "0","mds_kill_export_at": "0","mds_kill_import_at": "0","mds_kill_journal_at": "0","mds_kill_journal_expire_at": "0","mds_kill_journal_replay_at": "0","mds_kill_link_at": "0","mds_kill_mdstable_at": "0","mds_kill_openc_at": "0","mds_kill_rename_at": "0","mds_log_events_per_segment": "1024","mds_log_max_events": "-1","mds_log_max_segments": "128","mds_log_pause": "false","mds_log_segment_size": "0","mds_log_skip_corrupt_events": "false","mds_max_caps_per_client": "1048576","mds_max_completed_flushes": "100000","mds_max_completed_requests": "100000","mds_max_export_size": "20971520","mds_max_file_recover": "32","mds_max_purge_files": "64","mds_max_purge_ops": "8192","mds_max_purge_ops_per_pg": "0.500000","mds_max_retries_on_remount_failure": "5","mds_max_scrub_ops_in_progress": "5","mds_max_xattr_pairs_size": "65536","mds_min_caps_per_client": "100","mds_mon_shutdown_timeout": "5.000000","mds_op_complaint_time": "30.000000","mds_op_history_duration": "600","mds_op_history_size": "20","mds_op_log_threshold": "5","mds_purge_queue_busy_flush_period": "1.000000","mds_recall_global_max_decay_threshold": "65536","mds_recall_max_caps": "5000","mds_recall_max_decay_rate": "2.500000","mds_recall_max_decay_threshold": "16384","mds_recall_warning_decay_rate": "60.000000","mds_recall_warning_threshold": "32768","mds_reconnect_timeout": "45.000000","mds_replay_interval": "1.000000","mds_request_load_average_decay_rate": "60.000000","mds_root_ino_gid": "0","mds_root_ino_uid": "0","mds_scatter_nudge_interval": "5.000000","mds_session_blacklist_on_evict": "true","mds_session_blacklist_on_timeout": "true","mds_sessionmap_keys_per_op": "1024","mds_shutdown_check": "0","mds_skip_ino": "0","mds_snap_max_uid": "4294967294","mds_snap_min_uid": "0","mds_snap_rstat": "false","mds_thrash_exports": "0","mds_thrash_fragments": "0","mds_tick_interval": "5.000000","mds_verify_backtrace": "1","mds_verify_scatter": "false","mds_wipe_ino_prealloc": "false","mds_wipe_sessions": "false","mempool_debug": "false","memstore_device_bytes": "1073741824","memstore_page_set": "false","memstore_page_size": "65536","mgr_client_bytes": "134217728","mgr_client_messages": "512","mgr_client_service_daemon_unregister_timeout": "1.000000","mgr_connect_retry_interval": "1.000000","mgr_data": "/var/lib/ceph/mgr/ceph-sbg2-k8s001","mgr_debug_aggressive_pg_num_changes": "false","mgr_initial_modules": "restful iostat","mgr_mds_bytes": "134217728","mgr_mds_messages": "128","mgr_module_path": "/usr/share/ceph/mgr","mgr_mon_bytes": "134217728","mgr_mon_messages": "128","mgr_osd_bytes": "536870912","mgr_osd_messages": "8192","mgr_service_beacon_grace": "60.000000","mgr_stats_period": "5","mgr_stats_threshold": "5","mgr_tick_period": "2","mon_accept_timeout_factor": "2.000000","mon_allow_pool_delete": "false","mon_cache_target_full_warn_ratio": "0.660000","mon_client_bytes": "104857600","mon_client_hunt_interval": "3.000000","mon_client_hunt_interval_backoff": "1.500000","mon_client_hunt_interval_max_multiple": "10.000000","mon_client_hunt_interval_min_multiple": "1.000000","mon_client_hunt_parallel": "3","mon_client_max_log_entries_per_message": "1000","mon_client_ping_interval": "10.000000","mon_client_ping_timeout": "30.000000","mon_clock_drift_allowed": "0.050000","mon_clock_drift_warn_backoff": "5.000000","mon_cluster_log_file": "default=/var/log/ceph/ceph.$channel.log cluster=/var/log/ceph/ceph.log","mon_cluster_log_file_level": "debug","mon_cluster_log_to_file": "true","mon_cluster_log_to_graylog": "false","mon_cluster_log_to_graylog_host": "127.0.0.1","mon_cluster_log_to_graylog_port": "12201","mon_cluster_log_to_stderr": "false","mon_cluster_log_to_syslog": "default=false","mon_cluster_log_to_syslog_facility": "daemon","mon_cluster_log_to_syslog_level": "info","mon_compact_on_bootstrap": "false","mon_compact_on_start": "false","mon_compact_on_trim": "true","mon_config_key_max_entry_size": "65536","mon_cpu_threads": "4","mon_crush_min_required_version": "hammer","mon_daemon_bytes": "419430400","mon_data": "/var/lib/ceph/mon/ceph-sbg2-k8s001","mon_data_avail_crit": "5","mon_data_avail_warn": "30","mon_data_size_warn": "16106127360","mon_debug_block_osdmap_trim": "false","mon_debug_deprecated_as_obsolete": "false","mon_debug_dump_json": "false","mon_debug_dump_location": "/var/log/ceph/ceph-mon.sbg2-k8s001.tdump","mon_debug_dump_transactions": "false","mon_debug_extra_checks": "false","mon_debug_no_initial_persistent_features": "false","mon_debug_no_require_bluestore_for_ec_overwrites": "false","mon_debug_no_require_mimic": "false","mon_debug_no_require_nautilus": "false","mon_debug_unsafe_allow_tier_with_nonempty_snaps": "false","mon_delta_reset_interval": "10.000000","mon_dns_srv_name": "ceph-mon","mon_election_timeout": "5.000000","mon_enable_op_tracker": "true","mon_fake_pool_delete": "false","mon_force_quorum_join": "false","mon_globalid_prealloc": "10000","mon_health_log_update_period": "5","mon_health_max_detail": "50","mon_health_to_clog": "true","mon_health_to_clog_interval": "3600","mon_health_to_clog_tick_interval": "60.000000","mon_host": "192.168.211.14","mon_initial_members": "rbx4-k8s001","mon_inject_pg_merge_bounce_probability": "0.000000","mon_inject_sync_get_chunk_delay": "0.000000","mon_inject_transaction_delay_max": "10.000000","mon_inject_transaction_delay_probability": "0.000000","mon_keyvaluedb": "rocksdb","mon_lease": "5.000000","mon_lease_ack_timeout_factor": "2.000000","mon_lease_renew_interval_factor": "0.600000","mon_log_max_summary": "50","mon_max_log_entries_per_event": "4096","mon_max_log_epochs": "500","mon_max_mdsmap_epochs": "500","mon_max_mgrmap_epochs": "500","mon_max_osd": "10000","mon_max_pg_per_osd": "250","mon_max_pool_pg_num": "65536","mon_max_snap_prune_per_epoch": "100","mon_mds_blacklist_interval": "86400.000000","mon_mds_force_trim_to": "0","mon_mds_skip_sanity": "false","mon_mgr_beacon_grace": "30","mon_mgr_digest_period": "5","mon_mgr_inactive_grace": "60","mon_mgr_mkfs_grace": "120","mon_mgr_proxy_client_bytes_ratio": "0.300000","mon_min_osdmap_epochs": "500","mon_op_complaint_time": "30","mon_op_history_duration": "600","mon_op_history_size": "20","mon_op_history_slow_op_size": "20","mon_op_history_slow_op_threshold": "10","mon_op_log_threshold": "5","mon_osd_adjust_down_out_interval": "true","mon_osd_adjust_heartbeat_grace": "true","mon_osd_auto_mark_auto_out_in": "true","mon_osd_auto_mark_in": "false","mon_osd_auto_mark_new_in": "true","mon_osd_backfillfull_ratio": "0.900000","mon_osd_blacklist_default_expire": "3600.000000","mon_osd_cache_size": "500","mon_osd_crush_smoke_test": "true","mon_osd_destroyed_out_interval": "600","mon_osd_down_out_interval": "600","mon_osd_down_out_subtree_limit": "rack","mon_osd_err_op_age_ratio": "128.000000","mon_osd_force_trim_to": "0","mon_osd_full_ratio": "0.950000","mon_osd_initial_require_min_compat_client": "jewel","mon_osd_laggy_halflife": "3600","mon_osd_laggy_max_interval": "300","mon_osd_laggy_weight": "0.300000","mon_osd_mapping_pgs_per_chunk": "4096","mon_osd_max_creating_pgs": "1024","mon_osd_max_initial_pgs": "1024","mon_osd_min_down_reporters": "2","mon_osd_min_in_ratio": "0.750000","mon_osd_min_up_ratio": "0.300000","mon_osd_nearfull_ratio": "0.850000","mon_osd_prime_pg_temp": "true","mon_osd_prime_pg_temp_max_estimate": "0.250000","mon_osd_prime_pg_temp_max_time": "0.500000","mon_osd_report_timeout": "900","mon_osd_reporter_subtree_level": "host","mon_osd_snap_trim_queue_warn_on": "32768","mon_osd_warn_op_age": "32.000000","mon_osdmap_full_prune_enabled": "true","mon_osdmap_full_prune_interval": "10","mon_osdmap_full_prune_min": "10000","mon_osdmap_full_prune_txsize": "100","mon_pg_check_down_all_threshold": "0.500000","mon_pg_stuck_threshold": "60","mon_pg_warn_max_object_skew": "10.000000","mon_pg_warn_min_objects": "10000","mon_pg_warn_min_per_osd": "30","mon_pg_warn_min_pool_objects": "1000","mon_pool_quota_crit_threshold": "0","mon_pool_quota_warn_threshold": "0","mon_probe_timeout": "2.000000","mon_reweight_max_change": "0.050000","mon_reweight_max_osds": "4","mon_reweight_min_bytes_per_osd": "104857600","mon_reweight_min_pgs_per_osd": "10","mon_rocksdb_options": "write_buffer_size=33554432,compression=kNoCompression,level_compaction_dynamic_level_bytes=true","mon_scrub_inject_crc_mismatch": "0.000000","mon_scrub_inject_missing_keys": "0.000000","mon_scrub_interval": "86400","mon_scrub_max_keys": "100","mon_scrub_timeout": "300","mon_session_timeout": "300","mon_smart_report_timeout": "5","mon_stat_smooth_intervals": "6","mon_subscribe_interval": "86400.000000","mon_sync_debug": "false","mon_sync_max_payload_size": "1048576","mon_sync_provider_kill_at": "0","mon_sync_requester_kill_at": "0","mon_sync_timeout": "60.000000","mon_target_pg_per_osd": "100","mon_tick_interval": "5","mon_timecheck_interval": "300.000000","mon_timecheck_skew_interval": "30.000000","mon_warn_on_cache_pools_without_hit_sets": "true","mon_warn_on_crush_straw_calc_version_zero": "true","mon_warn_on_legacy_crush_tunables": "true","mon_warn_on_misplaced": "false","mon_warn_on_msgr2_not_enabled": "true","mon_warn_on_osd_down_out_interval_zero": "true","mon_warn_on_pool_no_app": "true","mon_warn_pg_not_deep_scrubbed_ratio": "0.750000","mon_warn_pg_not_scrubbed_ratio": "0.500000","monmap": "","ms_async_max_op_threads": "5","ms_async_op_threads": "3","ms_async_rdma_buffer_size": "131072","ms_async_rdma_cm": "false","ms_async_rdma_device_name": "","ms_async_rdma_dscp": "96","ms_async_rdma_enable_hugepage": "false","ms_async_rdma_local_gid": "","ms_async_rdma_polling_us": "1000","ms_async_rdma_port_num": "1","ms_async_rdma_receive_buffers": "32768","ms_async_rdma_receive_queue_len": "4096","ms_async_rdma_roce_ver": "1","ms_async_rdma_send_buffers": "1024","ms_async_rdma_sl": "3","ms_async_rdma_support_srq": "true","ms_async_rdma_type": "ib","ms_bind_before_connect": "false","ms_bind_ipv4": "true","ms_bind_ipv6": "false","ms_bind_msgr1": "true","ms_bind_msgr2": "true","ms_bind_port_max": "7300","ms_bind_port_min": "6800","ms_bind_prefer_ipv4": "false","ms_bind_retry_count": "3","ms_bind_retry_delay": "5","ms_client_mode": "crc","ms_cluster_mode": "crc","ms_cluster_type": "","ms_connection_idle_timeout": "900","ms_connection_ready_timeout": "10","ms_crc_data": "true","ms_crc_header": "true","ms_die_on_bad_msg": "false","ms_die_on_bug": "false","ms_die_on_old_message": "false","ms_die_on_skipped_message": "false","ms_die_on_unhandled_msg": "false","ms_dispatch_throttle_bytes": "104857600","ms_dpdk_coremask": "0xF","ms_dpdk_debug_allow_loopback": "false","ms_dpdk_gateway_ipv4_addr": "","ms_dpdk_host_ipv4_addr": "","ms_dpdk_hugepages": "","ms_dpdk_hw_flow_control": "true","ms_dpdk_hw_queue_weight": "1.000000","ms_dpdk_lro": "true","ms_dpdk_memory_channel": "4","ms_dpdk_netmask_ipv4_addr": "","ms_dpdk_pmd": "","ms_dpdk_port_id": "0","ms_dpdk_rx_buffer_count_per_core": "8192","ms_dump_corrupt_message_level": "1","ms_dump_on_send": "false","ms_initial_backoff": "0.200000","ms_inject_delay_max": "1.000000","ms_inject_delay_msg_type": "","ms_inject_delay_probability": "0.000000","ms_inject_delay_type": "","ms_inject_internal_delays": "0.000000","ms_inject_socket_failures": "0","ms_learn_addr_from_peer": "true","ms_max_accept_failures": "4","ms_max_backoff": "15.000000","ms_mon_client_mode": "crc","ms_mon_cluster_mode": "crc","ms_mon_service_mode": "crc","ms_msgr2_encrypt_messages": "false","ms_msgr2_sign_messages": "false","ms_pq_max_tokens_per_priority": "16777216","ms_pq_min_cost": "65536","ms_public_type": "","ms_rwthread_stack_bytes": "1048576","ms_service_mode": "crc","ms_tcp_listen_backlog": "512","ms_tcp_nodelay": "true","ms_tcp_prefetch_max_size": "4096","ms_tcp_rcvbuf": "0","ms_type": "async+posix","nss_db_path": "","objecter_completion_locks_per_session": "32","objecter_debug_inject_relock_delay": "false","objecter_inflight_op_bytes": "104857600","objecter_inflight_ops": "1024","objecter_inject_no_watch_ping": "false","objecter_retry_writes_after_first_reply": "false","objecter_tick_interval": "5.000000","objecter_timeout": "10.000000","objectstore_blackhole": "false","osd_agent_delay_time": "5.000000","osd_agent_hist_halflife": "1000","osd_agent_max_low_ops": "2","osd_agent_max_ops": "4","osd_agent_min_evict_effort": "0.100000","osd_agent_quantize_effort": "0.100000","osd_agent_slop": "0.020000","osd_allow_recovery_below_min_size": "true","osd_async_recovery_min_cost": "100","osd_auto_mark_unfound_lost": "false","osd_backfill_retry_interval": "30.000000","osd_backfill_scan_max": "512","osd_backfill_scan_min": "64","osd_backoff_on_degraded": "false","osd_backoff_on_peering": "false","osd_backoff_on_unfound": "true","osd_beacon_report_interval": "300","osd_bench_duration": "30","osd_bench_large_size_max_throughput": "104857600","osd_bench_max_block_size": "67108864","osd_bench_small_size_max_iops": "100","osd_blkin_trace_all": "false","osd_calc_pg_upmaps_aggressively": "true","osd_calc_pg_upmaps_local_fallback_retries": "100","osd_calc_pg_upmaps_max_stddev": "1.000000","osd_check_for_log_corruption": "false","osd_check_max_object_name_len_on_startup": "true","osd_class_default_list": "cephfs hello journal lock log numops otp rbd refcount rgw timeindex user version cas","osd_class_dir": "/usr/lib/rados-classes","osd_class_load_list": "cephfs hello journal lock log numops otp rbd refcount rgw timeindex user version cas","osd_class_update_on_start": "true","osd_client_message_cap": "100","osd_client_message_size_cap": "2147483648","osd_client_op_priority": "63","osd_client_watch_timeout": "30","osd_command_max_records": "256","osd_command_thread_suicide_timeout": "900","osd_command_thread_timeout": "600","osd_copyfrom_max_chunk": "8388608","osd_crush_chooseleaf_type": "1","osd_crush_initial_weight": "-1.000000","osd_crush_update_on_start": "true","osd_crush_update_weight_set": "true","osd_data": "/var/lib/ceph/osd/ceph-sbg2-k8s001","osd_debug_crash_on_ignored_backoff": "false","osd_debug_deep_scrub_sleep": "0.000000","osd_debug_drop_ping_duration": "0","osd_debug_drop_ping_probability": "0.000000","osd_debug_feed_pullee": "-1","osd_debug_inject_copyfrom_error": "false","osd_debug_inject_dispatch_delay_duration": "0.100000","osd_debug_inject_dispatch_delay_probability": "0.000000","osd_debug_misdirected_ops": "false","osd_debug_no_acting_change": "false","osd_debug_no_purge_strays": "false","osd_debug_op_order": "false","osd_debug_pg_log_writeout": "false","osd_debug_random_push_read_error": "0.000000","osd_debug_reject_backfill_probability": "0.000000","osd_debug_shutdown": "false","osd_debug_skip_full_check_in_backfill_reservation": "false","osd_debug_skip_full_check_in_recovery": "false","osd_debug_verify_cached_snaps": "false","osd_debug_verify_missing_on_start": "false","osd_debug_verify_snaps": "false","osd_debug_verify_stray_on_activate": "false","osd_deep_scrub_interval": "604800.000000","osd_deep_scrub_keys": "1024","osd_deep_scrub_large_omap_object_key_threshold": "2000000","osd_deep_scrub_large_omap_object_value_sum_threshold": "1073741824","osd_deep_scrub_randomize_ratio": "0.150000","osd_deep_scrub_stride": "131072","osd_deep_scrub_update_digest_min_age": "7200","osd_default_data_pool_replay_window": "45","osd_default_notify_timeout": "30","osd_delete_sleep": "0.000000","osd_delete_sleep_hdd": "5.000000","osd_delete_sleep_hybrid": "2.000000","osd_delete_sleep_ssd": "0.000000","osd_discard_disconnected_ops": "true","osd_enable_op_tracker": "true","osd_erasure_code_plugins": "jerasure lrc isa","osd_failsafe_full_ratio": "0.970000","osd_fast_fail_on_connection_refused": "true","osd_fast_info": "true","osd_find_best_info_ignore_history_les": "false","osd_force_auth_primary_missing_objects": "100","osd_force_recovery_pg_log_entries_factor": "1.300000","osd_function_tracing": "false","osd_heartbeat_grace": "20","osd_heartbeat_interval": "6","osd_heartbeat_min_healthy_ratio": "0.330000","osd_heartbeat_min_peers": "10","osd_heartbeat_min_size": "2000","osd_heartbeat_use_min_delay_socket": "false","osd_hit_set_max_size": "100000","osd_hit_set_min_size": "1000","osd_hit_set_namespace": ".ceph-internal","osd_ignore_stale_divergent_priors": "false","osd_inject_bad_map_crc_probability": "0.000000","osd_inject_failure_on_pg_removal": "false","osd_journal": "/var/lib/ceph/osd/ceph-sbg2-k8s001/journal","osd_journal_flush_on_shutdown": "true","osd_journal_size": "5120","osd_kill_backfill_at": "0","osd_loop_before_reset_tphandle": "64","osd_map_cache_size": "1024","osd_map_dedup": "true","osd_map_message_max": "40","osd_map_message_max_bytes": "10485760","osd_map_share_max_epochs": "40","osd_max_attr_name_len": "100","osd_max_attr_size": "0","osd_max_backfills": "4","osd_max_markdown_count": "5","osd_max_markdown_period": "600","osd_max_object_name_len": "2048","osd_max_object_namespace_len": "256","osd_max_object_size": "134217728","osd_max_omap_bytes_per_request": "1073741824","osd_max_omap_entries_per_request": "131072","osd_max_pg_blocked_by": "16","osd_max_pg_log_entries": "3000","osd_max_pg_per_osd_hard_ratio": "3.000000","osd_max_pgls": "1024","osd_max_push_cost": "8388608","osd_max_push_objects": "10","osd_max_scrubs": "1","osd_max_snap_prune_intervals_per_epoch": "512","osd_max_trimming_pgs": "2","osd_max_write_size": "512","osd_memory_base": "805306368","osd_memory_cache_min": "134217728","osd_memory_cache_resize_interval": "1.000000","osd_memory_expected_fragmentation": "0.150000","osd_memory_target": "4294967296","osd_memory_target_cgroup_limit_ratio": "0.800000","osd_min_pg_log_entries": "3000","osd_min_recovery_priority": "0","osd_mon_ack_timeout": "30.000000","osd_mon_heartbeat_interval": "30","osd_mon_report_interval": "5","osd_mon_report_max_in_flight": "2","osd_mon_shutdown_timeout": "5.000000","osd_num_op_tracker_shard": "32","osd_numa_auto_affinity": "true","osd_numa_node": "-1","osd_numa_prefer_iface": "true","osd_objecter_finishers": "1","osd_objectstore": "bluestore","osd_objectstore_fuse": "false","osd_objectstore_tracing": "false","osd_op_complaint_time": "30.000000","osd_op_history_duration": "600","osd_op_history_size": "20","osd_op_history_slow_op_size": "20","osd_op_history_slow_op_threshold": "10.000000","osd_op_log_threshold": "5","osd_op_num_shards": "0","osd_op_num_shards_hdd": "5","osd_op_num_shards_ssd": "8","osd_op_num_threads_per_shard": "0","osd_op_num_threads_per_shard_hdd": "1","osd_op_num_threads_per_shard_ssd": "2","osd_op_pq_max_tokens_per_priority": "4194304","osd_op_pq_min_cost": "65536","osd_op_queue": "wpq","osd_op_queue_cut_off": "low","osd_op_queue_mclock_anticipation_timeout": "0.000000","osd_op_queue_mclock_client_op_lim": "0.000000","osd_op_queue_mclock_client_op_res": "1000.000000","osd_op_queue_mclock_client_op_wgt": "500.000000","osd_op_queue_mclock_osd_rep_op_lim": "0.000000","osd_op_queue_mclock_osd_rep_op_res": "1000.000000","osd_op_queue_mclock_osd_rep_op_wgt": "500.000000","osd_op_queue_mclock_peering_event_lim": "0.001000","osd_op_queue_mclock_peering_event_res": "0.000000","osd_op_queue_mclock_peering_event_wgt": "1.000000","osd_op_queue_mclock_pg_delete_lim": "0.001000","osd_op_queue_mclock_pg_delete_res": "0.000000","osd_op_queue_mclock_pg_delete_wgt": "1.000000","osd_op_queue_mclock_recov_lim": "0.001000","osd_op_queue_mclock_recov_res": "0.000000","osd_op_queue_mclock_recov_wgt": "1.000000","osd_op_queue_mclock_scrub_lim": "0.001000","osd_op_queue_mclock_scrub_res": "0.000000","osd_op_queue_mclock_scrub_wgt": "1.000000","osd_op_queue_mclock_snap_lim": "0.001000","osd_op_queue_mclock_snap_res": "0.000000","osd_op_queue_mclock_snap_wgt": "1.000000","osd_op_thread_suicide_timeout": "150","osd_op_thread_timeout": "15","osd_open_classes_on_start": "true","osd_os_flags": "0","osd_peering_op_priority": "255","osd_pg_delete_cost": "1048576","osd_pg_delete_priority": "5","osd_pg_epoch_max_lag_factor": "2.000000","osd_pg_epoch_persisted_max_stale": "40","osd_pg_log_dups_tracked": "3000","osd_pg_log_trim_max": "10000","osd_pg_log_trim_min": "100","osd_pg_max_concurrent_snap_trims": "2","osd_pg_object_context_cache_count": "64","osd_pg_stat_report_interval_max": "500","osd_pool_default_cache_max_evict_check_size": "10","osd_pool_default_cache_min_evict_age": "0","osd_pool_default_cache_min_flush_age": "0","osd_pool_default_cache_target_dirty_high_ratio": "0.600000","osd_pool_default_cache_target_dirty_ratio": "0.400000","osd_pool_default_cache_target_full_ratio": "0.800000","osd_pool_default_crush_rule": "-1","osd_pool_default_ec_fast_read": "false","osd_pool_default_erasure_code_profile": "plugin=jerasure technique=reed_sol_van k=2 m=1","osd_pool_default_flag_hashpspool": "true","osd_pool_default_flag_nodelete": "false","osd_pool_default_flag_nopgchange": "false","osd_pool_default_flag_nosizechange": "false","osd_pool_default_flags": "0","osd_pool_default_hit_set_bloom_fpp": "0.050000","osd_pool_default_min_size": "2","osd_pool_default_pg_autoscale_mode": "on","osd_pool_default_pg_num": "8","osd_pool_default_pgp_num": "0","osd_pool_default_size": "3","osd_pool_default_type": "replicated","osd_pool_erasure_code_stripe_unit": "4096","osd_pool_use_gmt_hitset": "true","osd_push_per_object_cost": "1000","osd_read_ec_check_for_errors": "false","osd_recover_clone_overlap": "true","osd_recover_clone_overlap_limit": "10","osd_recovery_cost": "20971520","osd_recovery_delay_start": "0.000000","osd_recovery_max_active": "10","osd_recovery_max_chunk": "8388608","osd_recovery_max_omap_entries_per_chunk": "8096","osd_recovery_max_single_start": "1","osd_recovery_op_priority": "4","osd_recovery_op_warn_multiple": "16","osd_recovery_priority": "5","osd_recovery_retry_interval": "30.000000","osd_recovery_sleep": "0.000000","osd_recovery_sleep_hdd": "0.100000","osd_recovery_sleep_hybrid": "0.025000","osd_recovery_sleep_ssd": "0.000000","osd_requested_scrub_priority": "120","osd_rollback_to_cluster_snap": "","osd_scrub_auto_repair": "false","osd_scrub_auto_repair_num_errors": "5","osd_scrub_backoff_ratio": "0.660000","osd_scrub_begin_hour": "0","osd_scrub_begin_week_day": "0","osd_scrub_chunk_max": "25","osd_scrub_chunk_min": "5","osd_scrub_cost": "52428800","osd_scrub_during_recovery": "false","osd_scrub_end_hour": "24","osd_scrub_end_week_day": "7","osd_scrub_interval_randomize_ratio": "0.500000","osd_scrub_invalid_stats": "true","osd_scrub_load_threshold": "0.500000","osd_scrub_max_interval": "604800.000000","osd_scrub_max_preemptions": "5","osd_scrub_min_interval": "86400.000000","osd_scrub_priority": "5","osd_scrub_sleep": "0.000000","osd_shutdown_pgref_assert": "false","osd_skip_data_digest": "false","osd_smart_report_timeout": "5","osd_snap_trim_cost": "1048576","osd_snap_trim_priority": "5","osd_snap_trim_sleep": "0.000000","osd_stats_ack_timeout_decay": "0.900000","osd_stats_ack_timeout_factor": "2.000000","osd_target_transaction_size": "30","osd_tier_default_cache_hit_set_count": "4","osd_tier_default_cache_hit_set_grade_decay_rate": "20","osd_tier_default_cache_hit_set_period": "1200","osd_tier_default_cache_hit_set_search_last_n": "1","osd_tier_default_cache_hit_set_type": "bloom","osd_tier_default_cache_min_read_recency_for_promote": "1","osd_tier_default_cache_min_write_recency_for_promote": "1","osd_tier_default_cache_mode": "writeback","osd_tier_promote_max_bytes_sec": "5242880","osd_tier_promote_max_objects_sec": "25","osd_tracing": "false","osd_use_stale_snap": "false","osd_uuid": "00000000-0000-0000-0000-000000000000","osd_verify_sparse_read_holes": "false","osdc_blkin_trace_all": "false","paxos_kill_at": "0","paxos_max_join_drift": "10","paxos_min": "500","paxos_min_wait": "0.050000","paxos_propose_interval": "1.000000","paxos_service_trim_max": "500","paxos_service_trim_min": "250","paxos_stash_full_interval": "25","paxos_trim_max": "500","paxos_trim_min": "250","perf": "true","pid_file": "","plugin_crypto_accelerator": "crypto_isal","plugin_dir": "/usr/lib/ceph","public_addr": "v2:192.168.211.12:0/0","public_bind_addr": "-","public_network": "192.168.211.0/24","public_network_interface": "","qat_compressor_enabled": "false","rados_mon_op_timeout": "0.000000","rados_osd_op_timeout": "0.000000","rados_tracing": "false","rbd_atime_update_interval": "60","rbd_auto_exclusive_lock_until_manual_request": "true","rbd_balance_parent_reads": "false","rbd_balance_snap_reads": "false","rbd_blacklist_expire_seconds": "0","rbd_blacklist_on_break_lock": "true","rbd_blkin_trace_all": "false","rbd_cache": "true","rbd_cache_block_writes_upfront": "false","rbd_cache_max_dirty": "25165824","rbd_cache_max_dirty_age": "1.000000","rbd_cache_max_dirty_object": "0","rbd_cache_size": "33554432","rbd_cache_target_dirty": "16777216","rbd_cache_writethrough_until_flush": "true","rbd_clone_copy_on_read": "false","rbd_concurrent_management_ops": "10","rbd_default_clone_format": "auto","rbd_default_data_pool": "","rbd_default_features": "61","rbd_default_format": "2","rbd_default_map_options": "","rbd_default_order": "22","rbd_default_pool": "rbd","rbd_default_stripe_count": "0","rbd_default_stripe_unit": "0","rbd_discard_granularity_bytes": "65536","rbd_discard_on_zeroed_write_same": "true","rbd_enable_alloc_hint": "true","rbd_journal_commit_age": "5.000000","rbd_journal_max_concurrent_object_sets": "0","rbd_journal_max_payload_bytes": "16384","rbd_journal_object_flush_age": "0.000000","rbd_journal_object_flush_bytes": "0","rbd_journal_object_flush_interval": "0","rbd_journal_object_max_in_flight_appends": "0","rbd_journal_order": "24","rbd_journal_pool": "","rbd_journal_splay_width": "4","rbd_localize_parent_reads": "false","rbd_localize_snap_reads": "false","rbd_mirror_concurrent_image_deletions": "1","rbd_mirror_concurrent_image_syncs": "5","rbd_mirror_delete_retry_interval": "30.000000","rbd_mirror_image_policy_migration_throttle": "300","rbd_mirror_image_policy_rebalance_timeout": "0.000000","rbd_mirror_image_policy_type": "simple","rbd_mirror_image_policy_update_throttle_interval": "1.000000","rbd_mirror_image_state_check_interval": "30","rbd_mirror_journal_commit_age": "5.000000","rbd_mirror_journal_max_fetch_bytes": "32768","rbd_mirror_journal_poll_age": "5.000000","rbd_mirror_leader_heartbeat_interval": "5","rbd_mirror_leader_max_acquire_attempts_before_break": "3","rbd_mirror_leader_max_missed_heartbeats": "2","rbd_mirror_perf_stats_prio": "5","rbd_mirror_pool_replayers_refresh_interval": "30","rbd_mirror_sync_point_update_age": "30.000000","rbd_mirroring_delete_delay": "0","rbd_mirroring_replay_delay": "0","rbd_mirroring_resync_after_disconnect": "false","rbd_move_to_trash_on_remove": "false","rbd_move_to_trash_on_remove_expire_seconds": "0","rbd_mtime_update_interval": "60","rbd_non_blocking_aio": "true","rbd_op_thread_timeout": "60","rbd_op_threads": "1","rbd_qos_bps_burst": "0","rbd_qos_bps_limit": "0","rbd_qos_iops_burst": "0","rbd_qos_iops_limit": "0","rbd_qos_read_bps_burst": "0","rbd_qos_read_bps_limit": "0","rbd_qos_read_iops_burst": "0","rbd_qos_read_iops_limit": "0","rbd_qos_schedule_tick_min": "50","rbd_qos_write_bps_burst": "0","rbd_qos_write_bps_limit": "0","rbd_qos_write_iops_burst": "0","rbd_qos_write_iops_limit": "0","rbd_readahead_disable_after_bytes": "52428800","rbd_readahead_max_bytes": "524288","rbd_readahead_trigger_requests": "10","rbd_request_timed_out_seconds": "30","rbd_skip_partial_discard": "true","rbd_sparse_read_threshold_bytes": "65536","rbd_tracing": "false","rbd_validate_names": "true","rbd_validate_pool": "true","restapi_base_url": "","restapi_log_level": "","rgw_acl_grants_max_num": "100","rgw_admin_entry": "admin","rgw_barbican_url": "","rgw_bucket_default_quota_max_objects": "-1","rgw_bucket_default_quota_max_size": "-1","rgw_bucket_index_max_aio": "8","rgw_bucket_quota_cache_size": "10000","rgw_bucket_quota_soft_threshold": "0.950000","rgw_bucket_quota_ttl": "600","rgw_cache_enabled": "true","rgw_cache_expiry_interval": "900","rgw_cache_lru_size": "10000","rgw_content_length_compat": "false","rgw_copy_obj_progress": "true","rgw_copy_obj_progress_every_bytes": "1048576","rgw_cors_rules_max_num": "100","rgw_cross_domain_policy": "<allow-access-from domain=\"*\" secure=\"false\" />","rgw_crypt_default_encryption_key": "","rgw_crypt_require_ssl": "true","rgw_crypt_s3_kms_encryption_keys": "","rgw_crypt_suppress_logs": "true","rgw_curl_low_speed_limit": "1024","rgw_curl_low_speed_time": "300","rgw_curl_wait_timeout_ms": "1000","rgw_data": "/var/lib/ceph/radosgw/ceph-sbg2-k8s001","rgw_data_log_changes_size": "1000","rgw_data_log_num_shards": "128","rgw_data_log_obj_prefix": "data_log","rgw_data_log_window": "30","rgw_data_notify_interval_msec": "200","rgw_default_realm_info_oid": "default.realm","rgw_default_region_info_oid": "default.region","rgw_default_zone_info_oid": "default.zone","rgw_default_zonegroup_info_oid": "default.zonegroup","rgw_defer_to_bucket_acls": "","rgw_delete_multi_obj_max_num": "1000","rgw_dmclock_admin_lim": "0.000000","rgw_dmclock_admin_res": "100.000000","rgw_dmclock_admin_wgt": "100.000000","rgw_dmclock_auth_lim": "0.000000","rgw_dmclock_auth_res": "200.000000","rgw_dmclock_auth_wgt": "100.000000","rgw_dmclock_data_lim": "0.000000","rgw_dmclock_data_res": "500.000000","rgw_dmclock_data_wgt": "500.000000","rgw_dmclock_metadata_lim": "0.000000","rgw_dmclock_metadata_res": "500.000000","rgw_dmclock_metadata_wgt": "500.000000","rgw_dns_name": "","rgw_dns_s3website_name": "","rgw_dynamic_resharding": "true","rgw_enable_apis": "s3, s3website, swift, swift_auth, admin, sts","rgw_enable_gc_threads": "true","rgw_enable_lc_threads": "true","rgw_enable_ops_log": "false","rgw_enable_quota_threads": "true","rgw_enable_static_website": "false","rgw_enable_usage_log": "false","rgw_enforce_swift_acls": "true","rgw_exit_timeout_secs": "120","rgw_expose_bucket": "false","rgw_extended_http_attrs": "","rgw_fcgi_socket_backlog": "1024","rgw_frontends": "beast port=7480","rgw_gc_max_concurrent_io": "10","rgw_gc_max_objs": "32","rgw_gc_max_trim_chunk": "16","rgw_gc_obj_min_wait": "7200","rgw_gc_processor_max_time": "3600","rgw_gc_processor_period": "3600","rgw_get_obj_max_req_size": "4194304","rgw_get_obj_window_size": "16777216","rgw_healthcheck_disabling_path": "","rgw_host": "","rgw_ignore_get_invalid_range": "false","rgw_init_timeout": "300","rgw_inject_notify_timeout_probability": "0.000000","rgw_keystone_accepted_admin_roles": "","rgw_keystone_accepted_roles": "Member, admin","rgw_keystone_admin_domain": "","rgw_keystone_admin_password": "","rgw_keystone_admin_password_path": "","rgw_keystone_admin_project": "","rgw_keystone_admin_tenant": "","rgw_keystone_admin_token": "","rgw_keystone_admin_token_path": "","rgw_keystone_admin_user": "","rgw_keystone_api_version": "2","rgw_keystone_barbican_domain": "","rgw_keystone_barbican_password": "","rgw_keystone_barbican_project": "","rgw_keystone_barbican_tenant": "","rgw_keystone_barbican_user": "","rgw_keystone_implicit_tenants": "false","rgw_keystone_revocation_interval": "900","rgw_keystone_token_cache_size": "10000","rgw_keystone_url": "","rgw_keystone_verify_ssl": "true","rgw_lc_debug_interval": "-1","rgw_lc_lock_max_time": "60","rgw_lc_max_objs": "32","rgw_lc_max_rules": "1000","rgw_lc_thread_delay": "0","rgw_ldap_binddn": "uid=admin,cn=users,dc=example,dc=com","rgw_ldap_dnattr": "uid","rgw_ldap_searchdn": "cn=users,cn=accounts,dc=example,dc=com","rgw_ldap_searchfilter": "","rgw_ldap_secret": "/etc/openldap/secret","rgw_ldap_uri": "ldaps://<ldap.your.domain>","rgw_lifecycle_work_time": "00:00-06:00","rgw_list_bucket_min_readahead": "1000","rg
_______________________________________________ ceph-users mailing list -- ceph-users@xxxxxxx To unsubscribe send an email to ceph-users-leave@xxxxxxx
