Hi, Last night I upgraded my cluster from Centos 6.5 -> Centos 7.1 and in the process upgraded from Emperor -> Firefly -> Hammer When I finished I changed the crush tunables from ceph osd crush tunables legacy -> ceph osd crush tunables optimal I knew this would cause data movement. But the IO for my clients is unacceptable. Can any please tell what the best settings are for my configuration. I have 2 Dell R720 Servers and 2 Dell R730 servers. I have 36 1TB SATA SSD Drives in my cluster. The servers have 128 GB of RAM. Below is some detail the might help. According to my calculations the rebalance will take over a day. I would greatly appreciate some help on this. Thank you, Joe ----- BEGIN -------------------------------------------------------------- NODE: gold.sys.cu.cait.org CMD : free -m total used free shared buff/cache available Mem: 128726 71316 2077 20 55332 56767 Swap: 0 0 0 END -------------------------------------------------------------- BEGIN -------------------------------------------------------------- NODE: gallo.sys.cu.cait.org CMD : free -m total used free shared buff/cache available Mem: 128726 79489 462 36 48774 48547 Swap: 8191 0 8191 END -------------------------------------------------------------- BEGIN -------------------------------------------------------------- NODE: hamms.sys.cu.cait.org CMD : free -m total used free shared buff/cache available Mem: 128536 69412 659 19 58464 58342 Swap: 16383 0 16383 END -------------------------------------------------------------- BEGIN -------------------------------------------------------------- NODE: helm.sys.cu.cait.org CMD : free -m total used free shared buff/cache available Mem: 128536 66216 8799 26 53520 61603 Swap: 16383 1739 14644 END -------------------------------------------------------------- ceph osd tree ID WEIGHT TYPE NAME UP/DOWN REWEIGHT PRIMARY-AFFINITY -1 36.00000 root default -3 36.00000 rack curack -10 9.00000 host helm 0 1.00000 osd.0 up 1.00000 1.00000 1 1.00000 osd.1 up 1.00000 1.00000 2 1.00000 osd.2 up 1.00000 1.00000 3 1.00000 osd.3 up 1.00000 1.00000 4 1.00000 osd.4 up 1.00000 1.00000 5 1.00000 osd.5 up 1.00000 1.00000 6 1.00000 osd.6 up 1.00000 1.00000 7 1.00000 osd.7 up 1.00000 1.00000 8 1.00000 osd.8 up 1.00000 1.00000 -7 9.00000 host gold 16 1.00000 osd.16 up 1.00000 1.00000 17 1.00000 osd.17 up 1.00000 1.00000 18 1.00000 osd.18 up 1.00000 1.00000 19 1.00000 osd.19 up 1.00000 1.00000 20 1.00000 osd.20 up 1.00000 1.00000 21 1.00000 osd.21 up 1.00000 1.00000 9 1.00000 osd.9 up 1.00000 1.00000 10 1.00000 osd.10 up 1.00000 1.00000 34 1.00000 osd.34 up 1.00000 1.00000 -8 9.00000 host gallo 22 1.00000 osd.22 up 1.00000 1.00000 23 1.00000 osd.23 up 1.00000 1.00000 24 1.00000 osd.24 up 1.00000 1.00000 25 1.00000 osd.25 up 1.00000 1.00000 26 1.00000 osd.26 up 1.00000 1.00000 27 1.00000 osd.27 up 1.00000 1.00000 11 1.00000 osd.11 up 1.00000 1.00000 12 1.00000 osd.12 up 1.00000 1.00000 35 1.00000 osd.35 up 1.00000 1.00000 -9 9.00000 host hamms 13 1.00000 osd.13 up 1.00000 1.00000 14 1.00000 osd.14 up 1.00000 1.00000 15 1.00000 osd.15 up 1.00000 1.00000 28 1.00000 osd.28 up 1.00000 1.00000 29 1.00000 osd.29 up 1.00000 1.00000 30 1.00000 osd.30 up 1.00000 1.00000 31 1.00000 osd.31 up 1.00000 1.00000 32 1.00000 osd.32 up 1.00000 1.00000 33 1.00000 osd.33 up 1.00000 1.00000 ceph -s cluster 0c4fe157-88d3-4e83-8603-4dae31f63af6 health HEALTH_WARN 2767 pgs backfill 7 pgs backfilling 2774 pgs stuck unclean recovery 328/4281405 objects degraded (0.008%) recovery 2476616/4281405 objects misplaced (57.846%) monmap e34: 3 mons at {gold=10.66.4.7:6789/0,hamms=10.66.4.9:6789/0,helm=10.66.4.4:6789/0} election epoch 824, quorum 0,1,2 helm,gold,hamms mdsmap e2870: 0/0/1 up osdmap e116330: 36 osds: 36 up, 36 in; 2774 remapped pgs pgmap v73418676: 9166 pgs, 7 pools, 3865 GB data, 990 kobjects 11682 GB used, 19765 GB / 31448 GB avail 328/4281405 objects degraded (0.008%) 2476616/4281405 objects misplaced (57.846%) 6392 active+clean 2767 active+remapped+wait_backfill 7 active+remapped+backfilling recovery io 77061 kB/s, 19 objects/s client io 118 kB/s rd, 956 kB/s wr, 265 op/s ------ ceph --admin-daemon /var/run/ceph/ceph-osd.0.asok config show { "name": "osd.0", "cluster": "ceph", "debug_none": "0\/5", "debug_lockdep": "0\/1", "debug_context": "0\/1", "debug_crush": "1\/1", "debug_mds": "1\/5", "debug_mds_balancer": "1\/5", "debug_mds_locker": "1\/5", "debug_mds_log": "1\/5", "debug_mds_log_expire": "1\/5", "debug_mds_migrator": "1\/5", "debug_buffer": "0\/1", "debug_timer": "0\/1", "debug_filer": "0\/1", "debug_striper": "0\/1", "debug_objecter": "0\/1", "debug_rados": "0\/5", "debug_rbd": "0\/5", "debug_rbd_replay": "0\/5", "debug_journaler": "0\/5", "debug_objectcacher": "0\/5", "debug_client": "0\/5", "debug_osd": "0\/5", "debug_optracker": "0\/5", "debug_objclass": "0\/5", "debug_filestore": "1\/3", "debug_keyvaluestore": "1\/3", "debug_journal": "1\/3", "debug_ms": "0\/5", "debug_mon": "1\/5", "debug_monc": "0\/10", "debug_paxos": "1\/5", "debug_tp": "0\/5", "debug_auth": "1\/5", "debug_crypto": "1\/5", "debug_finisher": "1\/1", "debug_heartbeatmap": "1\/5", "debug_perfcounter": "1\/5", "debug_rgw": "1\/5", "debug_civetweb": "1\/10", "debug_javaclient": "1\/5", "debug_asok": "1\/5", "debug_throttle": "1\/1", "debug_refs": "0\/0", "debug_xio": "1\/5", "host": "helm", "fsid": "00000000-0000-0000-0000-000000000000", "public_addr": "10.66.4.4:0\/0", "cluster_addr": "10.66.4.4:0\/0", "public_network": "10.66.4.0\/24", "cluster_network": "10.66.4.0\/24", "num_client": "1", "monmap": "", "mon_host": "", "lockdep": "false", "lockdep_force_backtrace": "false", "run_dir": "\/var\/run\/ceph", "admin_socket": "\/var\/run\/ceph\/ceph-osd.0.asok", "daemonize": "false", "pid_file": "\/var\/run\/ceph\/osd.0.pid", "chdir": "\/", "max_open_files": "0", "restapi_log_level": "", "restapi_base_url": "", "fatal_signal_handlers": "true", "log_file": "\/var\/log\/ceph\/ceph-osd.0.log", "log_max_new": "1000", "log_max_recent": "10000", "log_to_stderr": "false", "err_to_stderr": "true", "log_to_syslog": "false", "err_to_syslog": "false", "log_flush_on_exit": "true", "log_stop_at_utilization": "0.97", "clog_to_monitors": "default=true", "clog_to_syslog": "false", "clog_to_syslog_level": "info", "clog_to_syslog_facility": "default=daemon audit=local0", "mon_cluster_log_to_syslog": "default=false", "mon_cluster_log_to_syslog_level": "info", "mon_cluster_log_to_syslog_facility": "daemon", "mon_cluster_log_file": "default=\/var\/log\/ceph\/ceph.$channel.log cluster=\/var\/log\/ceph\/ceph.log", "mon_cluster_log_file_level": "info", "enable_experimental_unrecoverable_data_corrupting_features": "", "xio_trace_mempool": "false", "xio_trace_msgcnt": "false", "xio_trace_xcon": "false", "xio_queue_depth": "512", "xio_mp_min": "128", "xio_mp_max_64": "65536", "xio_mp_max_256": "8192", "xio_mp_max_1k": "8192", "xio_mp_max_page": "4096", "xio_mp_max_hint": "4096", "xio_portal_threads": "2", "key": "", "keyfile": "", "keyring": "\/var\/lib\/ceph\/osd\/ceph-0\/keyring", "heartbeat_interval": "5", "heartbeat_file": "", "heartbeat_inject_failure": "0", "perf": "true", "ms_type": "simple", "ms_tcp_nodelay": "true", "ms_tcp_rcvbuf": "0", "ms_tcp_prefetch_max_size": "4096", "ms_initial_backoff": "0.2", "ms_max_backoff": "15", "ms_crc_data": "true", "ms_crc_header": "true", "ms_die_on_bad_msg": "false", "ms_die_on_unhandled_msg": "false", "ms_die_on_old_message": "false", "ms_die_on_skipped_message": "false", "ms_dispatch_throttle_bytes": "104857600", "ms_bind_ipv6": "false", "ms_bind_port_min": "6800", "ms_bind_port_max": "7300", "ms_bind_retry_count": "3", "ms_bind_retry_delay": "5", "ms_rwthread_stack_bytes": "1048576", "ms_tcp_read_timeout": "900", "ms_pq_max_tokens_per_priority": "16777216", "ms_pq_min_cost": "65536", "ms_inject_socket_failures": "0", "ms_inject_delay_type": "", "ms_inject_delay_msg_type": "", "ms_inject_delay_max": "1", "ms_inject_delay_probability": "0", "ms_inject_internal_delays": "0", "ms_dump_on_send": "false", "ms_dump_corrupt_message_level": "1", "ms_async_op_threads": "2", "ms_async_set_affinity": "true", "ms_async_affinity_cores": "", "inject_early_sigterm": "false", "mon_data": "\/var\/lib\/ceph\/mon\/ceph-0", "mon_initial_members": "", "mon_sync_fs_threshold": "5", "mon_compact_on_start": "false", "mon_compact_on_bootstrap": "false", "mon_compact_on_trim": "true", "mon_osd_cache_size": "10", "mon_tick_interval": "5", "mon_subscribe_interval": "300", "mon_delta_reset_interval": "10", "mon_osd_laggy_halflife": "3600", "mon_osd_laggy_weight": "0.3", "mon_osd_adjust_heartbeat_grace": "true", "mon_osd_adjust_down_out_interval": "true", "mon_osd_auto_mark_in": "false", "mon_osd_auto_mark_auto_out_in": "true", "mon_osd_auto_mark_new_in": "true", "mon_osd_down_out_interval": "300", "mon_osd_down_out_subtree_limit": "rack", "mon_osd_min_up_ratio": "0.3", "mon_osd_min_in_ratio": "0.3", "mon_osd_max_op_age": "32", "mon_osd_max_split_count": "32", "mon_osd_allow_primary_temp": "false", "mon_osd_allow_primary_affinity": "false", "mon_stat_smooth_intervals": "2", "mon_lease": "5", "mon_lease_renew_interval": "3", "mon_lease_ack_timeout": "10", "mon_clock_drift_allowed": "0.05", "mon_clock_drift_warn_backoff": "5", "mon_timecheck_interval": "300", "mon_accept_timeout": "10", "mon_pg_create_interval": "30", "mon_pg_stuck_threshold": "300", "mon_pg_warn_min_per_osd": "30", "mon_pg_warn_max_per_osd": "300", "mon_pg_warn_max_object_skew": "10", "mon_pg_warn_min_objects": "10000", "mon_pg_warn_min_pool_objects": "1000", "mon_cache_target_full_warn_ratio": "0.66", "mon_osd_full_ratio": "0.98", "mon_osd_nearfull_ratio": "0.94", "mon_allow_pool_delete": "true", "mon_globalid_prealloc": "10000", "mon_osd_report_timeout": "900", "mon_force_standby_active": "true", "mon_warn_on_old_mons": "true", "mon_warn_on_legacy_crush_tunables": "true", "mon_warn_on_osd_down_out_interval_zero": "true", "mon_warn_on_cache_pools_without_hit_sets": "true", "mon_min_osdmap_epochs": "500", "mon_max_pgmap_epochs": "500", "mon_max_log_epochs": "500", "mon_max_mdsmap_epochs": "500", "mon_max_osd": "10000", "mon_probe_timeout": "2", "mon_slurp_timeout": "10", "mon_slurp_bytes": "262144", "mon_client_bytes": "104857600", "mon_daemon_bytes": "419430400", "mon_max_log_entries_per_event": "4096", "mon_reweight_min_pgs_per_osd": "10", "mon_reweight_min_bytes_per_osd": "104857600", "mon_health_data_update_interval": "60", "mon_health_to_clog": "true", "mon_health_to_clog_interval": "3600", "mon_health_to_clog_tick_interval": "60", "mon_data_avail_crit": "5", "mon_data_avail_warn": "30", "mon_data_size_warn": "16106127360", "mon_config_key_max_entry_size": "4096", "mon_sync_timeout": "60", "mon_sync_max_payload_size": "1048576", "mon_sync_debug": "false", "mon_sync_debug_leader": "-1", "mon_sync_debug_provider": "-1", "mon_sync_debug_provider_fallback": "-1", "mon_inject_sync_get_chunk_delay": "0", "mon_osd_min_down_reporters": "1", "mon_osd_min_down_reports": "3", "mon_osd_force_trim_to": "0", "mon_mds_force_trim_to": "0", "crushtool": "crushtool", "mon_debug_dump_transactions": "false", "mon_debug_dump_location": "\/var\/log\/ceph\/ceph-osd.0.tdump", "mon_inject_transaction_delay_max": "10", "mon_inject_transaction_delay_probability": "0", "mon_sync_provider_kill_at": "0", "mon_sync_requester_kill_at": "0", "mon_force_quorum_join": "false", "mon_keyvaluedb": "leveldb", "mon_debug_unsafe_allow_tier_with_nonempty_snaps": "false", "paxos_stash_full_interval": "25", "paxos_max_join_drift": "10", "paxos_propose_interval": "1", "paxos_min_wait": "0.05", "paxos_min": "500", "paxos_trim_min": "250", "paxos_trim_max": "500", "paxos_service_trim_min": "250", "paxos_service_trim_max": "500", "paxos_kill_at": "0", "clock_offset": "0", "auth_cluster_required": "cephx", "auth_service_required": "cephx", "auth_client_required": "cephx", "auth_supported": "", "cephx_require_signatures": "false", "cephx_cluster_require_signatures": "false", "cephx_service_require_signatures": "false", "cephx_sign_messages": "true", "auth_mon_ticket_ttl": "43200", "auth_service_ticket_ttl": "3600", "auth_debug": "false", "mon_client_hunt_interval": "3", "mon_client_ping_interval": "10", "mon_client_ping_timeout": "30", "mon_client_hunt_interval_backoff": "2", "mon_client_hunt_interval_max_multiple": "10", "mon_client_max_log_entries_per_message": "1000", "mon_max_pool_pg_num": "65536", "mon_pool_quota_warn_threshold": "0", "mon_pool_quota_crit_threshold": "0", "client_cache_size": "16384", "client_cache_mid": "0.75", "client_use_random_mds": "false", "client_mount_timeout": "300", "client_tick_interval": "1", "client_trace": "", "client_readahead_min": "131072", "client_readahead_max_bytes": "0", "client_readahead_max_periods": "4", "client_snapdir": ".snap", "client_mountpoint": "\/", "client_notify_timeout": "10", "osd_client_watch_timeout": "30", "client_caps_release_delay": "5", "client_quota": "false", "client_oc": "true", "client_oc_size": "209715200", "client_oc_max_dirty": "104857600", "client_oc_target_dirty": "8388608", "client_oc_max_dirty_age": "5", "client_oc_max_objects": "1000", "client_debug_force_sync_read": "false", "client_debug_inject_tick_delay": "0", "client_max_inline_size": "4096", "client_inject_release_failure": "false", "fuse_use_invalidate_cb": "false", "fuse_allow_other": "true", "fuse_default_permissions": "true", "fuse_big_writes": "true", "fuse_atomic_o_trunc": "true", "fuse_debug": "false", "fuse_multithreaded": "true", "client_try_dentry_invalidate": "true", "client_die_on_failed_remount": "true", "client_check_pool_perm": "true", "crush_location": "", "objecter_tick_interval": "5", "objecter_timeout": "10", "objecter_inflight_op_bytes": "104857600", "objecter_inflight_ops": "1024", "objecter_completion_locks_per_session": "32", "objecter_inject_no_watch_ping": "false", "journaler_allow_split_entries": "true", "journaler_write_head_interval": "15", "journaler_prefetch_periods": "10", "journaler_prezero_periods": "5", "journaler_batch_interval": "0.001", "journaler_batch_max": "0", "mds_data": "\/var\/lib\/ceph\/mds\/ceph-0", "mds_max_file_size": "1099511627776", "mds_cache_size": "100000", "mds_cache_mid": "0.7", "mds_max_file_recover": "32", "mds_mem_max": "1048576", "mds_dir_max_commit_size": "10", "mds_decay_halflife": "5", "mds_beacon_interval": "4", "mds_beacon_grace": "15", "mds_enforce_unique_name": "true", "mds_blacklist_interval": "1440", "mds_session_timeout": "60", "mds_revoke_cap_timeout": "60", "mds_recall_state_timeout": "60", "mds_freeze_tree_timeout": "30", "mds_session_autoclose": "300", "mds_health_summarize_threshold": "10", "mds_reconnect_timeout": "45", "mds_tick_interval": "5", "mds_dirstat_min_interval": "1", "mds_scatter_nudge_interval": "5", "mds_client_prealloc_inos": "1000", "mds_early_reply": "true", "mds_default_dir_hash": "2", "mds_log": "true", "mds_log_skip_corrupt_events": "false", "mds_log_max_events": "-1", "mds_log_events_per_segment": "1024", "mds_log_segment_size": "0", "mds_log_max_segments": "30", "mds_log_max_expiring": "20", "mds_bal_sample_interval": "3", "mds_bal_replicate_threshold": "8000", "mds_bal_unreplicate_threshold": "0", "mds_bal_frag": "false", "mds_bal_split_size": "10000", "mds_bal_split_rd": "25000", "mds_bal_split_wr": "10000", "mds_bal_split_bits": "3", "mds_bal_merge_size": "50", "mds_bal_merge_rd": "1000", "mds_bal_merge_wr": "1000", "mds_bal_interval": "10", "mds_bal_fragment_interval": "5", "mds_bal_idle_threshold": "0", "mds_bal_max": "-1", "mds_bal_max_until": "-1", "mds_bal_mode": "0", "mds_bal_min_rebalance": "0.1", "mds_bal_min_start": "0.2", "mds_bal_need_min": "0.8", "mds_bal_need_max": "1.2", "mds_bal_midchunk": "0.3", "mds_bal_minchunk": "0.001", "mds_bal_target_removal_min": "5", "mds_bal_target_removal_max": "10", "mds_replay_interval": "1", "mds_shutdown_check": "0", "mds_thrash_exports": "0", "mds_thrash_fragments": "0", "mds_dump_cache_on_map": "false", "mds_dump_cache_after_rejoin": "false", "mds_verify_scatter": "false", "mds_debug_scatterstat": "false", "mds_debug_frag": "false", "mds_debug_auth_pins": "false", "mds_debug_subtrees": "false", "mds_kill_mdstable_at": "0", "mds_kill_export_at": "0", "mds_kill_import_at": "0", "mds_kill_link_at": "0", "mds_kill_rename_at": "0", "mds_kill_openc_at": "0", "mds_kill_journal_at": "0", "mds_kill_journal_expire_at": "0", "mds_kill_journal_replay_at": "0", "mds_journal_format": "1", "mds_kill_create_at": "0", "mds_inject_traceless_reply_probability": "0", "mds_wipe_sessions": "false", "mds_wipe_ino_prealloc": "false", "mds_skip_ino": "0", "max_mds": "1", "mds_standby_for_name": "", "mds_standby_for_rank": "-1", "mds_standby_replay": "false", "mds_enable_op_tracker": "true", "mds_op_history_size": "20", "mds_op_history_duration": "600", "mds_op_complaint_time": "30", "mds_op_log_threshold": "5", "mds_snap_min_uid": "0", "mds_snap_max_uid": "65536", "mds_verify_backtrace": "1", "mds_action_on_write_error": "1", "osd_compact_leveldb_on_mount": "false", "osd_max_backfills": "1", "osd_min_recovery_priority": "0", "osd_backfill_full_ratio": "0.93", "osd_backfill_retry_interval": "10", "osd_agent_max_ops": "4", "osd_agent_min_evict_effort": "0.1", "osd_agent_quantize_effort": "0.1", "osd_agent_delay_time": "5", "osd_find_best_info_ignore_history_les": "false", "osd_agent_hist_halflife": "1000", "osd_agent_slop": "0.02", "osd_uuid": "00000000-0000-0000-0000-000000000000", "osd_data": "\/var\/lib\/ceph\/osd\/ceph-0", "osd_journal": "\/dev\/vg-osd-0\/lv-journal", "osd_journal_size": "5000", "osd_max_write_size": "90", "osd_max_pgls": "1024", "osd_client_message_size_cap": "524288000", "osd_client_message_cap": "100", "osd_pg_bits": "6", "osd_pgp_bits": "6", "osd_crush_chooseleaf_type": "1", "osd_pool_default_crush_rule": "-1", "osd_pool_default_crush_replicated_ruleset": "0", "osd_pool_erasure_code_stripe_width": "4096", "osd_pool_default_size": "3", "osd_pool_default_min_size": "0", "osd_pool_default_pg_num": "8", "osd_pool_default_pgp_num": "8", "osd_pool_default_erasure_code_directory": "\/usr\/lib64\/ceph\/erasure-code", "osd_pool_default_erasure_code_profile": "plugin=jerasure technique=reed_sol_van k=2 m=1 ", "osd_erasure_code_plugins": "jerasure lrc isa", "osd_allow_recovery_below_min_size": "true", "osd_pool_default_flags": "0", "osd_pool_default_flag_hashpspool": "true", "osd_pool_default_flag_nodelete": "false", "osd_pool_default_flag_nopgchange": "false", "osd_pool_default_flag_nosizechange": "false", "osd_pool_default_hit_set_bloom_fpp": "0.05", "osd_pool_default_cache_target_dirty_ratio": "0.4", "osd_pool_default_cache_target_full_ratio": "0.8", "osd_pool_default_cache_min_flush_age": "0", "osd_pool_default_cache_min_evict_age": "0", "osd_hit_set_min_size": "1000", "osd_hit_set_max_size": "100000", "osd_hit_set_namespace": ".ceph-internal", "osd_tier_default_cache_mode": "writeback", "osd_tier_default_cache_hit_set_count": "4", "osd_tier_default_cache_hit_set_period": "1200", "osd_tier_default_cache_hit_set_type": "bloom", "osd_tier_default_cache_min_read_recency_for_promote": "1", "osd_map_dedup": "true", "osd_map_max_advance": "200", "osd_map_cache_size": "500", "osd_map_message_max": "100", "osd_map_share_max_epochs": "100", "osd_inject_bad_map_crc_probability": "0", "osd_inject_failure_on_pg_removal": "false", "osd_op_threads": "2", "osd_peering_wq_batch_size": "20", "osd_op_pq_max_tokens_per_priority": "4194304", "osd_op_pq_min_cost": "65536", "osd_disk_threads": "1", "osd_disk_thread_ioprio_class": "", "osd_disk_thread_ioprio_priority": "-1", "osd_recovery_threads": "1", "osd_recover_clone_overlap": "true", "osd_op_num_threads_per_shard": "2", "osd_op_num_shards": "5", "osd_read_eio_on_bad_digest": "true", "osd_recover_clone_overlap_limit": "10", "osd_backfill_scan_min": "64", "osd_backfill_scan_max": "512", "osd_op_thread_timeout": "15", "osd_op_thread_suicide_timeout": "150", "osd_recovery_thread_timeout": "30", "osd_recovery_thread_suicide_timeout": "300", "osd_snap_trim_thread_timeout": "3600", "osd_snap_trim_thread_suicide_timeout": "36000", "osd_snap_trim_sleep": "0", "osd_scrub_thread_timeout": "60", "osd_scrub_thread_suicide_timeout": "60", "osd_scrub_finalize_thread_timeout": "600", "osd_scrub_invalid_stats": "true", "osd_remove_thread_timeout": "3600", "osd_remove_thread_suicide_timeout": "36000", "osd_command_thread_timeout": "600", "osd_age": "0.8", "osd_age_time": "0", "osd_command_thread_suicide_timeout": "900", "osd_heartbeat_addr": ":\/0", "osd_heartbeat_interval": "6", "osd_heartbeat_grace": "20", "osd_heartbeat_min_peers": "10", "osd_heartbeat_use_min_delay_socket": "false", "osd_pg_max_concurrent_snap_trims": "2", "osd_heartbeat_min_healthy_ratio": "0.33", "osd_mon_heartbeat_interval": "30", "osd_mon_report_interval_max": "120", "osd_mon_report_interval_min": "5", "osd_pg_stat_report_interval_max": "500", "osd_mon_ack_timeout": "30", "osd_default_data_pool_replay_window": "45", "osd_preserve_trimmed_log": "false", "osd_auto_mark_unfound_lost": "false", "osd_recovery_delay_start": "0", "osd_recovery_max_active": "1", "osd_recovery_max_single_start": "5", "osd_recovery_max_chunk": "8388608", "osd_copyfrom_max_chunk": "8388608", "osd_push_per_object_cost": "1000", "osd_max_push_cost": "8388608", "osd_max_push_objects": "10", "osd_recovery_forget_lost_objects": "false", "osd_max_scrubs": "1", "osd_scrub_begin_hour": "0", "osd_scrub_end_hour": "24", "osd_scrub_load_threshold": "0.5", "osd_scrub_min_interval": "86400", "osd_scrub_max_interval": "604800", "osd_scrub_chunk_min": "5", "osd_scrub_chunk_max": "25", "osd_scrub_sleep": "0", "osd_deep_scrub_interval": "604800", "osd_deep_scrub_stride": "524288", "osd_deep_scrub_update_digest_min_age": "7200", "osd_scan_list_ping_tp_interval": "100", "osd_auto_weight": "false", "osd_class_dir": "\/usr\/lib64\/rados-classes", "osd_open_classes_on_start": "true", "osd_check_for_log_corruption": "false", "osd_use_stale_snap": "false", "osd_rollback_to_cluster_snap": "", "osd_default_notify_timeout": "30", "osd_kill_backfill_at": "0", "osd_pg_epoch_persisted_max_stale": "200", "osd_min_pg_log_entries": "3000", "osd_max_pg_log_entries": "10000", "osd_pg_log_trim_min": "100", "osd_op_complaint_time": "30", "osd_command_max_records": "256", "osd_max_pg_blocked_by": "16", "osd_op_log_threshold": "5", "osd_verify_sparse_read_holes": "false", "osd_debug_drop_ping_probability": "0", "osd_debug_drop_ping_duration": "0", "osd_debug_drop_pg_create_probability": "0", "osd_debug_drop_pg_create_duration": "1", "osd_debug_drop_op_probability": "0", "osd_debug_op_order": "false", "osd_debug_scrub_chance_rewrite_digest": "0", "osd_debug_verify_snaps_on_info": "false", "osd_debug_verify_stray_on_activate": "false", "osd_debug_skip_full_check_in_backfill_reservation": "false", "osd_debug_reject_backfill_probability": "0", "osd_debug_inject_copyfrom_error": "false", "osd_enable_op_tracker": "true", "osd_num_op_tracker_shard": "32", "osd_op_history_size": "20", "osd_op_history_duration": "600", "osd_target_transaction_size": "30", "osd_failsafe_full_ratio": "0.97", "osd_failsafe_nearfull_ratio": "0.9", "osd_pg_object_context_cache_count": "64", "osd_debug_pg_log_writeout": "false", "threadpool_default_timeout": "60", "threadpool_empty_queue_max_wait": "2", "leveldb_write_buffer_size": "8388608", "leveldb_cache_size": "134217728", "leveldb_block_size": "0", "leveldb_bloom_size": "0", "leveldb_max_open_files": "0", "leveldb_compression": "true", "leveldb_paranoid": "false", "leveldb_log": "", "leveldb_compact_on_mount": "false", "kinetic_host": "", "kinetic_port": "8123", "kinetic_user_id": "1", "kinetic_hmac_key": "asdfasdf", "kinetic_use_ssl": "false", "rocksdb_compact_on_mount": "false", "rocksdb_write_buffer_size": "0", "rocksdb_target_file_size_base": "0", "rocksdb_cache_size": "0", "rocksdb_block_size": "0", "rocksdb_bloom_size": "0", "rocksdb_write_buffer_num": "0", "rocksdb_background_compactions": "0", "rocksdb_background_flushes": "0", "rocksdb_max_open_files": "0", "rocksdb_compression": "", "rocksdb_paranoid": "false", "rocksdb_log": "\/dev\/null", "rocksdb_level0_file_num_compaction_trigger": "0", "rocksdb_level0_slowdown_writes_trigger": "0", "rocksdb_level0_stop_writes_trigger": "0", "rocksdb_disableDataSync": "true", "rocksdb_disableWAL": "false", "rocksdb_num_levels": "0", "rocksdb_wal_dir": "", "rocksdb_info_log_level": "info", "osd_client_op_priority": "63", "osd_recovery_op_priority": "1", "osd_recovery_op_warn_multiple": "16", "osd_mon_shutdown_timeout": "5", "osd_max_object_size": "107374182400", "osd_max_object_name_len": "2048", "osd_max_attr_name_len": "100", "osd_max_attr_size": "0", "osd_objectstore": "filestore", "osd_debug_override_acting_compat": "false", "osd_bench_small_size_max_iops": "100", "osd_bench_large_size_max_throughput": "104857600", "osd_bench_max_block_size": "67108864", "osd_bench_duration": "30", "memstore_device_bytes": "1073741824", "filestore_omap_backend": "leveldb", "filestore_debug_disable_sharded_check": "false", "filestore_wbthrottle_enable": "true", "filestore_wbthrottle_btrfs_bytes_start_flusher": "41943040", "filestore_wbthrottle_btrfs_bytes_hard_limit": "419430400", "filestore_wbthrottle_btrfs_ios_start_flusher": "500", "filestore_wbthrottle_btrfs_ios_hard_limit": "5000", "filestore_wbthrottle_btrfs_inodes_start_flusher": "500", "filestore_wbthrottle_xfs_bytes_start_flusher": "41943040", "filestore_wbthrottle_xfs_bytes_hard_limit": "419430400", "filestore_wbthrottle_xfs_ios_start_flusher": "500", "filestore_wbthrottle_xfs_ios_hard_limit": "5000", "filestore_wbthrottle_xfs_inodes_start_flusher": "500", "filestore_wbthrottle_btrfs_inodes_hard_limit": "5000", "filestore_wbthrottle_xfs_inodes_hard_limit": "5000", "filestore_index_retry_probability": "0", "filestore_debug_inject_read_err": "false", "filestore_debug_omap_check": "false", "filestore_omap_header_cache_size": "1024", "filestore_max_inline_xattr_size": "0", "filestore_max_inline_xattr_size_xfs": "65536", "filestore_max_inline_xattr_size_btrfs": "2048", "filestore_max_inline_xattr_size_other": "512", "filestore_max_inline_xattrs": "0", "filestore_max_inline_xattrs_xfs": "10", "filestore_max_inline_xattrs_btrfs": "10", "filestore_max_inline_xattrs_other": "2", "filestore_sloppy_crc": "false", "filestore_sloppy_crc_block_size": "65536", "filestore_max_alloc_hint_size": "1048576", "filestore_max_sync_interval": "5", "filestore_min_sync_interval": "0.01", "filestore_btrfs_snap": "true", "filestore_btrfs_clone_range": "true", "filestore_zfs_snap": "false", "filestore_fsync_flushes_journal_data": "false", "filestore_fiemap": "false", "filestore_fadvise": "true", "filestore_xfs_extsize": "true", "filestore_journal_parallel": "false", "filestore_journal_writeahead": "false", "filestore_journal_trailing": "false", "filestore_queue_max_ops": "50", "filestore_queue_max_bytes": "104857600", "filestore_queue_committing_max_ops": "500", "filestore_queue_committing_max_bytes": "104857600", "filestore_op_threads": "2", "filestore_op_thread_timeout": "60", "filestore_op_thread_suicide_timeout": "180", "filestore_commit_timeout": "600", "filestore_fiemap_threshold": "4096", "filestore_merge_threshold": "10", "filestore_split_multiple": "2", "filestore_update_to": "1000", "filestore_blackhole": "false", "filestore_fd_cache_size": "128", "filestore_fd_cache_shards": "16", "filestore_dump_file": "", "filestore_kill_at": "0", "filestore_inject_stall": "0", "filestore_fail_eio": "true", "filestore_debug_verify_split": "false", "journal_dio": "true", "journal_aio": "true", "journal_force_aio": "false", "keyvaluestore_queue_max_ops": "50", "keyvaluestore_queue_max_bytes": "104857600", "keyvaluestore_debug_check_backend": "false", "keyvaluestore_op_threads": "2", "keyvaluestore_op_thread_timeout": "60", "keyvaluestore_op_thread_suicide_timeout": "180", "keyvaluestore_default_strip_size": "4096", "keyvaluestore_max_expected_write_size": "16777216", "keyvaluestore_header_cache_size": "4096", "keyvaluestore_backend": "leveldb", "journal_max_corrupt_search": "10485760", "journal_block_align": "true", "journal_write_header_frequency": "0", "journal_max_write_bytes": "10485760", "journal_max_write_entries": "100", "journal_queue_max_ops": "300", "journal_queue_max_bytes": "33554432", "journal_align_min_size": "65536", "journal_replay_from": "0", "journal_zero_on_create": "false", "journal_ignore_corruption": "false", "journal_discard": "false", "rados_mon_op_timeout": "0", "rados_osd_op_timeout": "0", "rbd_op_threads": "1", "rbd_op_thread_timeout": "60", "rbd_non_blocking_aio": "true", "rbd_cache": "true", "rbd_cache_writethrough_until_flush": "true", "rbd_cache_size": "33554432", "rbd_cache_max_dirty": "25165824", "rbd_cache_target_dirty": "16777216", "rbd_cache_max_dirty_age": "1", "rbd_cache_max_dirty_object": "0", "rbd_cache_block_writes_upfront": "false", "rbd_concurrent_management_ops": "10", "rbd_balance_snap_reads": "false", "rbd_localize_snap_reads": "false", "rbd_balance_parent_reads": "false", "rbd_localize_parent_reads": "true", "rbd_readahead_trigger_requests": "10", "rbd_readahead_max_bytes": "524288", "rbd_readahead_disable_after_bytes": "52428800", "rbd_clone_copy_on_read": "false", "rbd_blacklist_on_break_lock": "true", "rbd_blacklist_expire_seconds": "0", "rbd_request_timed_out_seconds": "30", "rbd_default_format": "1", "rbd_default_order": "22", "rbd_default_stripe_count": "0", "rbd_default_stripe_unit": "0", "rbd_default_features": "3", "nss_db_path": "", "rgw_max_chunk_size": "524288", "rgw_override_bucket_index_max_shards": "0", "rgw_bucket_index_max_aio": "8", "rgw_enable_quota_threads": "true", "rgw_enable_gc_threads": "true", "rgw_data": "\/var\/lib\/ceph\/radosgw\/ceph-0", "rgw_enable_apis": "s3, swift, swift_auth, admin", "rgw_cache_enabled": "true", "rgw_cache_lru_size": "10000", "rgw_socket_path": "", "rgw_host": "", "rgw_port": "", "rgw_dns_name": "", "rgw_content_length_compat": "false", "rgw_script_uri": "", "rgw_request_uri": "", "rgw_swift_url": "", "rgw_swift_url_prefix": "swift", "rgw_swift_auth_url": "", "rgw_swift_auth_entry": "auth", "rgw_swift_tenant_name": "", "rgw_swift_enforce_content_length": "false", "rgw_keystone_url": "", "rgw_keystone_admin_token": "", "rgw_keystone_admin_user": "", "rgw_keystone_admin_password": "", "rgw_keystone_admin_tenant": "", "rgw_keystone_accepted_roles": "Member, admin", "rgw_keystone_token_cache_size": "10000", "rgw_keystone_revocation_interval": "900", "rgw_s3_auth_use_rados": "true", "rgw_s3_auth_use_keystone": "false", "rgw_admin_entry": "admin", "rgw_enforce_swift_acls": "true", "rgw_swift_token_expiration": "86400", "rgw_print_continue": "true", "rgw_remote_addr_param": "REMOTE_ADDR", "rgw_op_thread_timeout": "600", "rgw_op_thread_suicide_timeout": "0", "rgw_thread_pool_size": "100", "rgw_num_control_oids": "8", "rgw_num_rados_handles": "1", "rgw_zone": "", "rgw_zone_root_pool": ".rgw.root", "rgw_region": "", "rgw_region_root_pool": ".rgw.root", "rgw_default_region_info_oid": "default.region", "rgw_log_nonexistent_bucket": "false", "rgw_log_object_name": "%Y-%m-%d-%H-%i-%n", "rgw_log_object_name_utc": "false", "rgw_usage_max_shards": "32", "rgw_usage_max_user_shards": "1", "rgw_enable_ops_log": "false", "rgw_enable_usage_log": "false", "rgw_ops_log_rados": "true", "rgw_ops_log_socket_path": "", "rgw_ops_log_data_backlog": "5242880", "rgw_usage_log_flush_threshold": "1024", "rgw_usage_log_tick_interval": "30", "rgw_intent_log_object_name": "%Y-%m-%d-%i-%n", "rgw_intent_log_object_name_utc": "false", "rgw_init_timeout": "300", "rgw_mime_types_file": "\/etc\/mime.types", "rgw_gc_max_objs": "32", "rgw_gc_obj_min_wait": "7200", "rgw_gc_processor_max_time": "3600", "rgw_gc_processor_period": "3600", "rgw_s3_success_create_obj_status": "0", "rgw_resolve_cname": "false", "rgw_obj_stripe_size": "4194304", "rgw_extended_http_attrs": "", "rgw_exit_timeout_secs": "120", "rgw_get_obj_window_size": "16777216", "rgw_get_obj_max_req_size": "4194304", "rgw_relaxed_s3_bucket_names": "false", "rgw_defer_to_bucket_acls": "", "rgw_list_buckets_max_chunk": "1000", "rgw_md_log_max_shards": "64", "rgw_num_zone_opstate_shards": "128", "rgw_opstate_ratelimit_sec": "30", "rgw_curl_wait_timeout_ms": "1000", "rgw_copy_obj_progress": "true", "rgw_copy_obj_progress_every_bytes": "1048576", "rgw_data_log_window": "30", "rgw_data_log_changes_size": "1000", "rgw_data_log_num_shards": "128", "rgw_data_log_obj_prefix": "data_log", "rgw_replica_log_obj_prefix": "replica_log", "rgw_bucket_quota_ttl": "600", "rgw_bucket_quota_soft_threshold": "0.95", "rgw_bucket_quota_cache_size": "10000", "rgw_expose_bucket": "false", "rgw_frontends": "fastcgi, civetweb port=7480", "rgw_user_quota_bucket_sync_interval": "180", "rgw_user_quota_sync_interval": "86400", "rgw_user_quota_sync_idle_users": "false", "rgw_user_quota_sync_wait_time": "86400", "rgw_multipart_min_part_size": "5242880", "rgw_olh_pending_timeout_sec": "3600", "rgw_user_max_buckets": "1000", "mutex_perf_counter": "false", "throttler_perf_counter": "true", "internal_safe_to_start_threads": "true" } ceph.conf cat /etc/ceph/ceph.conf [client] rbd cache = true # 256 MB of cache rbd cache size = 268435456 # 64 MB Max Dirty rbd cache max dirty = 67108864 # 32 MB Target Dirty rbd cache target dirty = 33554432 # 2 Seconds worth can be dirty rbd cache max dirty age = 2.0 [global] public network = 10.66.4.0/24 cluster network = 10.66.4.0/24 # For version 0.55 and beyond, you must explicitly enable # or disable authentication with "auth" entries in [global]. auth cluster required = cephx auth service required = cephx auth client required = cephx mon osd full ratio = .98 mon osd nearfull ratio = .94 mon_pg_warn_max_per_osd = 0 [osd] osd journal size = 5000 journal aio = true # See https://ceph.com/docs/master/rados/configuration/osd-config-ref/ osd client op priority = 90 osd recovery op priority = 10 osd recovery max active = 1 osd max backfills = 1 osd backfill full ratio = 0.93 #The following assumes ext4 filesystem. # filestore xattr use omap = true # For Bobtail (v 0.56) and subsequent versions, you may # add settings for mkcephfs so that it will create and mount # the file system on a particular OSD for you. Remove the comment `#` # character for the following settings and replace the values # in braces with appropriate values, or leave the following settings # commented out to accept the default values. You must specify the # --mkfs option with mkcephfs in order for the deployment script to # utilize the following settings, and you must define the 'devs' # option for each osd instance; see below. osd mkfs type = btrfs osd mkfs options btrfs = -L $name # xfs options by Dell RAID CARD specifies 64k stripe size(su) and each osd is a single drive(1) for (sw) # see : http://www.mysqlperformanceblog.com/2011/12/16/setting-up-xfs-the-simple-edition/ # also see : https://raid.wiki.kernel.org/index.php/RAID_setup#XFS # mkfs -t xfs -f -L ceph-xx -d su=64k,sw=1 /dev/sdX1 osd mkfs options xfs = -L -d su=64k,sw=1 $name osd mount options btrfs = rw,noatime osd mount options xfs = rw,noatime,nodiratime # mount -o rw,noatime,nodiratime /dev/disk/by-label/ceph-X /var/lib/ceph/osd/ceph-X # For example, for ext4, the mount option might look like this: #osd mkfs options ext4 = user_xattr,rw,noatime # Execute $ hostname to retrieve the name of your host, # and replace {hostname} with the name of your host. # For the monitor, replace {ip-address} with the IP # address of your host. # CAIT -- Manual commands to make and mount file system # -- Make xfs file system # mkfs -t xfs -f -L ceph-X -d su=64k,sw=1 /dev/sdX1 # -- Rescan Parition Label # partprobe /dev/sdX1 # -- Mount ceph file system # mount -o rw,noatime,nodiratime /dev/disk/by-label/ceph-X /var/lib/ceph/osd/ceph-X [mon] mon debug dump transactions = false # mon warn on legacy crush tunables = false # [mon.splinter] # host = splinter # mon addr = 10.66.4.2:6789 [mon.helm] host = helm mon addr = 10.66.4.4:6789 [mon.hamms] host = hamms mon addr = 10.66.4.9:6789 [mon.gold] host = gold mon addr = 10.66.4.7:6789 [osd.0] host = helm devs = /dev/vg-osd-0/lv-osd osd mkfs type = xfs osd journal = /dev/vg-osd-0/lv-journal [osd.1] host = helm devs = /dev/vg-osd-1/lv-osd osd mkfs type = xfs osd journal = /dev/vg-osd-1/lv-journal [osd.2] host = helm devs = /dev/vg-osd-2/lv-osd osd mkfs type = xfs osd journal = /dev/vg-osd-2/lv-journal [osd.3] host = helm devs = /dev/vg-osd-3/lv-osd osd mkfs type = xfs osd journal = /dev/vg-osd-3/lv-journal [osd.4] host = helm devs = /dev/vg-osd-4/lv-osd osd mkfs type = xfs osd journal = /dev/vg-osd-4/lv-journal [osd.5] host = helm devs = /dev/vg-osd-5/lv-osd osd mkfs type = xfs osd journal = /dev/vg-osd-5/lv-journal [osd.6] host = helm devs = /dev/vg-osd-6/lv-osd osd mkfs type = xfs osd journal = /dev/vg-osd-6/lv-journal [osd.7] host = helm devs = /dev/vg-osd-7/lv-osd osd mkfs type = xfs osd journal = /dev/vg-osd-7/lv-journal [osd.8] host = helm devs = /dev/vg-osd-8/lv-osd osd mkfs type = xfs osd journal = /dev/vg-osd-8/lv-journal [osd.16] host = gold devs = /dev/disk/by-label/ceph-16 osd mkfs type = xfs [osd.17] host = gold devs = /dev/disk/by-label/ceph-17 osd mkfs type = xfs [osd.18] host = gold devs = /dev/disk/by-label/ceph-18 osd mkfs type = xfs [osd.19] host = gold devs = /dev/disk/by-label/ceph-19 osd mkfs type = xfs [osd.20] host = gold devs = /dev/disk/by-label/ceph-20 osd mkfs type = xfs [osd.21] host = gold devs = /dev/disk/by-label/ceph-21 osd mkfs type = xfs [osd.9] host = gold devs = /dev/vg-osd-9/lv-osd osd mkfs type = xfs osd journal = /dev/vg-osd-9/lv-journal [osd.10] host = gold devs = /dev/vg-osd-10/lv-osd osd mkfs type = xfs osd journal = /dev/vg-osd-10/lv-journal [osd.34] host = gold devs = /dev/vg-osd-34/lv-osd osd mkfs type = xfs osd journal = /dev/vg-osd-34/lv-journal [osd.22] host = gallo devs = /dev/disk/by-label/ceph-22 osd mkfs type = xfs [osd.23] host = gallo devs = /dev/disk/by-label/ceph-23 osd mkfs type = xfs [osd.24] host = gallo devs = /dev/disk/by-label/ceph-24 osd mkfs type = xfs [osd.25] host = gallo devs = /dev/disk/by-label/ceph-25 osd mkfs type = xfs [osd.26] host = gallo devs = /dev/disk/by-label/ceph-26 osd mkfs type = xfs [osd.27] host = gallo devs = /dev/disk/by-label/ceph-27 osd mkfs type = xfs [osd.11] host = gallo devs = /dev/vg-osd-11/lv-osd osd mkfs type = xfs osd journal = /dev/vg-osd-11/lv-journal [osd.12] host = gallo devs = /dev/vg-osd-12/lv-osd osd mkfs type = xfs osd journal = /dev/vg-osd-12/lv-journal [osd.35] host = gallo devs = /dev/vg-osd-35/lv-osd osd mkfs type = xfs osd journal = /dev/vg-osd-35/lv-journal [osd.13] host = hamms devs = /dev/vg-osd-13/lv-osd osd mkfs type = xfs osd journal = /dev/vg-osd-13/lv-journal [osd.14] host = hamms devs = /dev/vg-osd-14/lv-osd osd mkfs type = xfs osd journal = /dev/vg-osd-14/lv-journal [osd.15] host = hamms devs = /dev/vg-osd-15/lv-osd osd mkfs type = xfs osd journal = /dev/vg-osd-15/lv-journal [osd.28] host = hamms devs = /dev/vg-osd-28/lv-osd osd mkfs type = xfs osd journal = /dev/vg-osd-28/lv-journal [osd.29] host = hamms devs = /dev/vg-osd-29/lv-osd osd mkfs type = xfs osd journal = /dev/vg-osd-29/lv-journal [osd.30] host = hamms devs = /dev/vg-osd-30/lv-osd osd mkfs type = xfs osd journal = /dev/vg-osd-30/lv-journal [osd.31] host = hamms devs = /dev/vg-osd-31/lv-osd osd mkfs type = xfs osd journal = /dev/vg-osd-31/lv-journal [osd.32] host = hamms devs = /dev/vg-osd-32/lv-osd osd mkfs type = xfs osd journal = /dev/vg-osd-32/lv-journal [osd.33] host = hamms devs = /dev/vg-osd-33/lv-osd osd mkfs type = xfs osd journal = /dev/vg-osd-33/lv-journal -- Joe Ryner Center for the Application of Information Technologies (CAIT) Production Coordinator P: (309) 298-1804 F: (309) 298-2806 _______________________________________________ ceph-users mailing list ceph-users@xxxxxxxxxxxxxx http://lists.ceph.com/listinfo.cgi/ceph-users-ceph.com