You upgraded (and restarted as appropriate) all the clients first, right?
Warren Wang
On 11/24/15, 10:52 AM, "Joe Ryner" <jryner@xxxxxxxx> wrote:
>Hi,
>
>Last night I upgraded my cluster from Centos 6.5 -> Centos 7.1 and in the
>process upgraded from Emperor -> Firefly -> Hammer
>
>When I finished I changed the crush tunables from
>ceph osd crush tunables legacy -> ceph osd crush tunables optimal
>
>I knew this would cause data movement. But the IO for my clients is
>unacceptable. Can any please tell what the best settings are for my
>configuration. I have 2 Dell R720 Servers and 2 Dell R730 servers. I
>have 36 1TB SATA SSD Drives in my cluster. The servers have 128 GB of
>RAM.
>
>Below is some detail the might help. According to my calculations the
>rebalance will take over a day.
>
>I would greatly appreciate some help on this.
>
>Thank you,
>
>Joe
>
>-----
>BEGIN --------------------------------------------------------------
>NODE: gold.sys.cu.cait.org
>CMD : free -m
> total used free shared buff/cache
>available
>Mem: 128726 71316 2077 20 55332
>56767
>Swap: 0 0 0
>END --------------------------------------------------------------
>BEGIN --------------------------------------------------------------
>NODE: gallo.sys.cu.cait.org
>CMD : free -m
> total used free shared buff/cache
>available
>Mem: 128726 79489 462 36 48774
>48547
>Swap: 8191 0 8191
>END --------------------------------------------------------------
>BEGIN --------------------------------------------------------------
>NODE: hamms.sys.cu.cait.org
>CMD : free -m
> total used free shared buff/cache
>available
>Mem: 128536 69412 659 19 58464
>58342
>Swap: 16383 0 16383
>END --------------------------------------------------------------
>BEGIN --------------------------------------------------------------
>NODE: helm.sys.cu.cait.org
>CMD : free -m
> total used free shared buff/cache
>available
>Mem: 128536 66216 8799 26 53520
>61603
>Swap: 16383 1739 14644
>END --------------------------------------------------------------
>
>ceph osd tree
>ID WEIGHT TYPE NAME UP/DOWN REWEIGHT PRIMARY-AFFINITY
> -1 36.00000 root default
> -3 36.00000 rack curack
>-10 9.00000 host helm
> 0 1.00000 osd.0 up 1.00000 1.00000
> 1 1.00000 osd.1 up 1.00000 1.00000
> 2 1.00000 osd.2 up 1.00000 1.00000
> 3 1.00000 osd.3 up 1.00000 1.00000
> 4 1.00000 osd.4 up 1.00000 1.00000
> 5 1.00000 osd.5 up 1.00000 1.00000
> 6 1.00000 osd.6 up 1.00000 1.00000
> 7 1.00000 osd.7 up 1.00000 1.00000
> 8 1.00000 osd.8 up 1.00000 1.00000
> -7 9.00000 host gold
> 16 1.00000 osd.16 up 1.00000 1.00000
> 17 1.00000 osd.17 up 1.00000 1.00000
> 18 1.00000 osd.18 up 1.00000 1.00000
> 19 1.00000 osd.19 up 1.00000 1.00000
> 20 1.00000 osd.20 up 1.00000 1.00000
> 21 1.00000 osd.21 up 1.00000 1.00000
> 9 1.00000 osd.9 up 1.00000 1.00000
> 10 1.00000 osd.10 up 1.00000 1.00000
> 34 1.00000 osd.34 up 1.00000 1.00000
> -8 9.00000 host gallo
> 22 1.00000 osd.22 up 1.00000 1.00000
> 23 1.00000 osd.23 up 1.00000 1.00000
> 24 1.00000 osd.24 up 1.00000 1.00000
> 25 1.00000 osd.25 up 1.00000 1.00000
> 26 1.00000 osd.26 up 1.00000 1.00000
> 27 1.00000 osd.27 up 1.00000 1.00000
> 11 1.00000 osd.11 up 1.00000 1.00000
> 12 1.00000 osd.12 up 1.00000 1.00000
> 35 1.00000 osd.35 up 1.00000 1.00000
> -9 9.00000 host hamms
> 13 1.00000 osd.13 up 1.00000 1.00000
> 14 1.00000 osd.14 up 1.00000 1.00000
> 15 1.00000 osd.15 up 1.00000 1.00000
> 28 1.00000 osd.28 up 1.00000 1.00000
> 29 1.00000 osd.29 up 1.00000 1.00000
> 30 1.00000 osd.30 up 1.00000 1.00000
> 31 1.00000 osd.31 up 1.00000 1.00000
> 32 1.00000 osd.32 up 1.00000 1.00000
> 33 1.00000 osd.33 up 1.00000 1.00000
>
>ceph -s
> cluster 0c4fe157-88d3-4e83-8603-4dae31f63af6
> health HEALTH_WARN
> 2767 pgs backfill
> 7 pgs backfilling
> 2774 pgs stuck unclean
> recovery 328/4281405 objects degraded (0.008%)
> recovery 2476616/4281405 objects misplaced (57.846%)
> monmap e34: 3 mons at
>{gold=10.66.4.7:6789/0,hamms=10.66.4.9:6789/0,helm=10.66.4.4:6789/0}
> election epoch 824, quorum 0,1,2 helm,gold,hamms
> mdsmap e2870: 0/0/1 up
> osdmap e116330: 36 osds: 36 up, 36 in; 2774 remapped pgs
> pgmap v73418676: 9166 pgs, 7 pools, 3865 GB data, 990 kobjects
> 11682 GB used, 19765 GB / 31448 GB avail
> 328/4281405 objects degraded (0.008%)
> 2476616/4281405 objects misplaced (57.846%)
> 6392 active+clean
> 2767 active+remapped+wait_backfill
> 7 active+remapped+backfilling
>recovery io 77061 kB/s, 19 objects/s
> client io 118 kB/s rd, 956 kB/s wr, 265 op/s
>
>
>------
>ceph --admin-daemon /var/run/ceph/ceph-osd.0.asok config show
>{
> "name": "osd.0",
> "cluster": "ceph",
> "debug_none": "0\/5",
> "debug_lockdep": "0\/1",
> "debug_context": "0\/1",
> "debug_crush": "1\/1",
> "debug_mds": "1\/5",
> "debug_mds_balancer": "1\/5",
> "debug_mds_locker": "1\/5",
> "debug_mds_log": "1\/5",
> "debug_mds_log_expire": "1\/5",
> "debug_mds_migrator": "1\/5",
> "debug_buffer": "0\/1",
> "debug_timer": "0\/1",
> "debug_filer": "0\/1",
> "debug_striper": "0\/1",
> "debug_objecter": "0\/1",
> "debug_rados": "0\/5",
> "debug_rbd": "0\/5",
> "debug_rbd_replay": "0\/5",
> "debug_journaler": "0\/5",
> "debug_objectcacher": "0\/5",
> "debug_client": "0\/5",
> "debug_osd": "0\/5",
> "debug_optracker": "0\/5",
> "debug_objclass": "0\/5",
> "debug_filestore": "1\/3",
> "debug_keyvaluestore": "1\/3",
> "debug_journal": "1\/3",
> "debug_ms": "0\/5",
> "debug_mon": "1\/5",
> "debug_monc": "0\/10",
> "debug_paxos": "1\/5",
> "debug_tp": "0\/5",
> "debug_auth": "1\/5",
> "debug_crypto": "1\/5",
> "debug_finisher": "1\/1",
> "debug_heartbeatmap": "1\/5",
> "debug_perfcounter": "1\/5",
> "debug_rgw": "1\/5",
> "debug_civetweb": "1\/10",
> "debug_javaclient": "1\/5",
> "debug_asok": "1\/5",
> "debug_throttle": "1\/1",
> "debug_refs": "0\/0",
> "debug_xio": "1\/5",
> "host": "helm",
> "fsid": "00000000-0000-0000-0000-000000000000",
> "public_addr": "10.66.4.4:0\/0",
> "cluster_addr": "10.66.4.4:0\/0",
> "public_network": "10.66.4.0\/24",
> "cluster_network": "10.66.4.0\/24",
> "num_client": "1",
> "monmap": "",
> "mon_host": "",
> "lockdep": "false",
> "lockdep_force_backtrace": "false",
> "run_dir": "\/var\/run\/ceph",
> "admin_socket": "\/var\/run\/ceph\/ceph-osd.0.asok",
> "daemonize": "false",
> "pid_file": "\/var\/run\/ceph\/osd.0.pid",
> "chdir": "\/",
> "max_open_files": "0",
> "restapi_log_level": "",
> "restapi_base_url": "",
> "fatal_signal_handlers": "true",
> "log_file": "\/var\/log\/ceph\/ceph-osd.0.log",
> "log_max_new": "1000",
> "log_max_recent": "10000",
> "log_to_stderr": "false",
> "err_to_stderr": "true",
> "log_to_syslog": "false",
> "err_to_syslog": "false",
> "log_flush_on_exit": "true",
> "log_stop_at_utilization": "0.97",
> "clog_to_monitors": "default=true",
> "clog_to_syslog": "false",
> "clog_to_syslog_level": "info",
> "clog_to_syslog_facility": "default=daemon audit=local0",
> "mon_cluster_log_to_syslog": "default=false",
> "mon_cluster_log_to_syslog_level": "info",
> "mon_cluster_log_to_syslog_facility": "daemon",
> "mon_cluster_log_file": "default=\/var\/log\/ceph\/ceph.$channel.log
>cluster=\/var\/log\/ceph\/ceph.log",
> "mon_cluster_log_file_level": "info",
> "enable_experimental_unrecoverable_data_corrupting_features": "",
> "xio_trace_mempool": "false",
> "xio_trace_msgcnt": "false",
> "xio_trace_xcon": "false",
> "xio_queue_depth": "512",
> "xio_mp_min": "128",
> "xio_mp_max_64": "65536",
> "xio_mp_max_256": "8192",
> "xio_mp_max_1k": "8192",
> "xio_mp_max_page": "4096",
> "xio_mp_max_hint": "4096",
> "xio_portal_threads": "2",
> "key": "",
> "keyfile": "",
> "keyring": "\/var\/lib\/ceph\/osd\/ceph-0\/keyring",
> "heartbeat_interval": "5",
> "heartbeat_file": "",
> "heartbeat_inject_failure": "0",
> "perf": "true",
> "ms_type": "simple",
> "ms_tcp_nodelay": "true",
> "ms_tcp_rcvbuf": "0",
> "ms_tcp_prefetch_max_size": "4096",
> "ms_initial_backoff": "0.2",
> "ms_max_backoff": "15",
> "ms_crc_data": "true",
> "ms_crc_header": "true",
> "ms_die_on_bad_msg": "false",
> "ms_die_on_unhandled_msg": "false",
> "ms_die_on_old_message": "false",
> "ms_die_on_skipped_message": "false",
> "ms_dispatch_throttle_bytes": "104857600",
> "ms_bind_ipv6": "false",
> "ms_bind_port_min": "6800",
> "ms_bind_port_max": "7300",
> "ms_bind_retry_count": "3",
> "ms_bind_retry_delay": "5",
> "ms_rwthread_stack_bytes": "1048576",
> "ms_tcp_read_timeout": "900",
> "ms_pq_max_tokens_per_priority": "16777216",
> "ms_pq_min_cost": "65536",
> "ms_inject_socket_failures": "0",
> "ms_inject_delay_type": "",
> "ms_inject_delay_msg_type": "",
> "ms_inject_delay_max": "1",
> "ms_inject_delay_probability": "0",
> "ms_inject_internal_delays": "0",
> "ms_dump_on_send": "false",
> "ms_dump_corrupt_message_level": "1",
> "ms_async_op_threads": "2",
> "ms_async_set_affinity": "true",
> "ms_async_affinity_cores": "",
> "inject_early_sigterm": "false",
> "mon_data": "\/var\/lib\/ceph\/mon\/ceph-0",
> "mon_initial_members": "",
> "mon_sync_fs_threshold": "5",
> "mon_compact_on_start": "false",
> "mon_compact_on_bootstrap": "false",
> "mon_compact_on_trim": "true",
> "mon_osd_cache_size": "10",
> "mon_tick_interval": "5",
> "mon_subscribe_interval": "300",
> "mon_delta_reset_interval": "10",
> "mon_osd_laggy_halflife": "3600",
> "mon_osd_laggy_weight": "0.3",
> "mon_osd_adjust_heartbeat_grace": "true",
> "mon_osd_adjust_down_out_interval": "true",
> "mon_osd_auto_mark_in": "false",
> "mon_osd_auto_mark_auto_out_in": "true",
> "mon_osd_auto_mark_new_in": "true",
> "mon_osd_down_out_interval": "300",
> "mon_osd_down_out_subtree_limit": "rack",
> "mon_osd_min_up_ratio": "0.3",
> "mon_osd_min_in_ratio": "0.3",
> "mon_osd_max_op_age": "32",
> "mon_osd_max_split_count": "32",
> "mon_osd_allow_primary_temp": "false",
> "mon_osd_allow_primary_affinity": "false",
> "mon_stat_smooth_intervals": "2",
> "mon_lease": "5",
> "mon_lease_renew_interval": "3",
> "mon_lease_ack_timeout": "10",
> "mon_clock_drift_allowed": "0.05",
> "mon_clock_drift_warn_backoff": "5",
> "mon_timecheck_interval": "300",
> "mon_accept_timeout": "10",
> "mon_pg_create_interval": "30",
> "mon_pg_stuck_threshold": "300",
> "mon_pg_warn_min_per_osd": "30",
> "mon_pg_warn_max_per_osd": "300",
> "mon_pg_warn_max_object_skew": "10",
> "mon_pg_warn_min_objects": "10000",
> "mon_pg_warn_min_pool_objects": "1000",
> "mon_cache_target_full_warn_ratio": "0.66",
> "mon_osd_full_ratio": "0.98",
> "mon_osd_nearfull_ratio": "0.94",
> "mon_allow_pool_delete": "true",
> "mon_globalid_prealloc": "10000",
> "mon_osd_report_timeout": "900",
> "mon_force_standby_active": "true",
> "mon_warn_on_old_mons": "true",
> "mon_warn_on_legacy_crush_tunables": "true",
> "mon_warn_on_osd_down_out_interval_zero": "true",
> "mon_warn_on_cache_pools_without_hit_sets": "true",
> "mon_min_osdmap_epochs": "500",
> "mon_max_pgmap_epochs": "500",
> "mon_max_log_epochs": "500",
> "mon_max_mdsmap_epochs": "500",
> "mon_max_osd": "10000",
> "mon_probe_timeout": "2",
> "mon_slurp_timeout": "10",
> "mon_slurp_bytes": "262144",
> "mon_client_bytes": "104857600",
> "mon_daemon_bytes": "419430400",
> "mon_max_log_entries_per_event": "4096",
> "mon_reweight_min_pgs_per_osd": "10",
> "mon_reweight_min_bytes_per_osd": "104857600",
> "mon_health_data_update_interval": "60",
> "mon_health_to_clog": "true",
> "mon_health_to_clog_interval": "3600",
> "mon_health_to_clog_tick_interval": "60",
> "mon_data_avail_crit": "5",
> "mon_data_avail_warn": "30",
> "mon_data_size_warn": "16106127360",
> "mon_config_key_max_entry_size": "4096",
> "mon_sync_timeout": "60",
> "mon_sync_max_payload_size": "1048576",
> "mon_sync_debug": "false",
> "mon_sync_debug_leader": "-1",
> "mon_sync_debug_provider": "-1",
> "mon_sync_debug_provider_fallback": "-1",
> "mon_inject_sync_get_chunk_delay": "0",
> "mon_osd_min_down_reporters": "1",
> "mon_osd_min_down_reports": "3",
> "mon_osd_force_trim_to": "0",
> "mon_mds_force_trim_to": "0",
> "crushtool": "crushtool",
> "mon_debug_dump_transactions": "false",
> "mon_debug_dump_location": "\/var\/log\/ceph\/ceph-osd.0.tdump",
> "mon_inject_transaction_delay_max": "10",
> "mon_inject_transaction_delay_probability": "0",
> "mon_sync_provider_kill_at": "0",
> "mon_sync_requester_kill_at": "0",
> "mon_force_quorum_join": "false",
> "mon_keyvaluedb": "leveldb",
> "mon_debug_unsafe_allow_tier_with_nonempty_snaps": "false",
> "paxos_stash_full_interval": "25",
> "paxos_max_join_drift": "10",
> "paxos_propose_interval": "1",
> "paxos_min_wait": "0.05",
> "paxos_min": "500",
> "paxos_trim_min": "250",
> "paxos_trim_max": "500",
> "paxos_service_trim_min": "250",
> "paxos_service_trim_max": "500",
> "paxos_kill_at": "0",
> "clock_offset": "0",
> "auth_cluster_required": "cephx",
> "auth_service_required": "cephx",
> "auth_client_required": "cephx",
> "auth_supported": "",
> "cephx_require_signatures": "false",
> "cephx_cluster_require_signatures": "false",
> "cephx_service_require_signatures": "false",
> "cephx_sign_messages": "true",
> "auth_mon_ticket_ttl": "43200",
> "auth_service_ticket_ttl": "3600",
> "auth_debug": "false",
> "mon_client_hunt_interval": "3",
> "mon_client_ping_interval": "10",
> "mon_client_ping_timeout": "30",
> "mon_client_hunt_interval_backoff": "2",
> "mon_client_hunt_interval_max_multiple": "10",
> "mon_client_max_log_entries_per_message": "1000",
> "mon_max_pool_pg_num": "65536",
> "mon_pool_quota_warn_threshold": "0",
> "mon_pool_quota_crit_threshold": "0",
> "client_cache_size": "16384",
> "client_cache_mid": "0.75",
> "client_use_random_mds": "false",
> "client_mount_timeout": "300",
> "client_tick_interval": "1",
> "client_trace": "",
> "client_readahead_min": "131072",
> "client_readahead_max_bytes": "0",
> "client_readahead_max_periods": "4",
> "client_snapdir": ".snap",
> "client_mountpoint": "\/",
> "client_notify_timeout": "10",
> "osd_client_watch_timeout": "30",
> "client_caps_release_delay": "5",
> "client_quota": "false",
> "client_oc": "true",
> "client_oc_size": "209715200",
> "client_oc_max_dirty": "104857600",
> "client_oc_target_dirty": "8388608",
> "client_oc_max_dirty_age": "5",
> "client_oc_max_objects": "1000",
> "client_debug_force_sync_read": "false",
> "client_debug_inject_tick_delay": "0",
> "client_max_inline_size": "4096",
> "client_inject_release_failure": "false",
> "fuse_use_invalidate_cb": "false",
> "fuse_allow_other": "true",
> "fuse_default_permissions": "true",
> "fuse_big_writes": "true",
> "fuse_atomic_o_trunc": "true",
> "fuse_debug": "false",
> "fuse_multithreaded": "true",
> "client_try_dentry_invalidate": "true",
> "client_die_on_failed_remount": "true",
> "client_check_pool_perm": "true",
> "crush_location": "",
> "objecter_tick_interval": "5",
> "objecter_timeout": "10",
> "objecter_inflight_op_bytes": "104857600",
> "objecter_inflight_ops": "1024",
> "objecter_completion_locks_per_session": "32",
> "objecter_inject_no_watch_ping": "false",
> "journaler_allow_split_entries": "true",
> "journaler_write_head_interval": "15",
> "journaler_prefetch_periods": "10",
> "journaler_prezero_periods": "5",
> "journaler_batch_interval": "0.001",
> "journaler_batch_max": "0",
> "mds_data": "\/var\/lib\/ceph\/mds\/ceph-0",
> "mds_max_file_size": "1099511627776",
> "mds_cache_size": "100000",
> "mds_cache_mid": "0.7",
> "mds_max_file_recover": "32",
> "mds_mem_max": "1048576",
> "mds_dir_max_commit_size": "10",
> "mds_decay_halflife": "5",
> "mds_beacon_interval": "4",
> "mds_beacon_grace": "15",
> "mds_enforce_unique_name": "true",
> "mds_blacklist_interval": "1440",
> "mds_session_timeout": "60",
> "mds_revoke_cap_timeout": "60",
> "mds_recall_state_timeout": "60",
> "mds_freeze_tree_timeout": "30",
> "mds_session_autoclose": "300",
> "mds_health_summarize_threshold": "10",
> "mds_reconnect_timeout": "45",
> "mds_tick_interval": "5",
> "mds_dirstat_min_interval": "1",
> "mds_scatter_nudge_interval": "5",
> "mds_client_prealloc_inos": "1000",
> "mds_early_reply": "true",
> "mds_default_dir_hash": "2",
> "mds_log": "true",
> "mds_log_skip_corrupt_events": "false",
> "mds_log_max_events": "-1",
> "mds_log_events_per_segment": "1024",
> "mds_log_segment_size": "0",
> "mds_log_max_segments": "30",
> "mds_log_max_expiring": "20",
> "mds_bal_sample_interval": "3",
> "mds_bal_replicate_threshold": "8000",
> "mds_bal_unreplicate_threshold": "0",
> "mds_bal_frag": "false",
> "mds_bal_split_size": "10000",
> "mds_bal_split_rd": "25000",
> "mds_bal_split_wr": "10000",
> "mds_bal_split_bits": "3",
> "mds_bal_merge_size": "50",
> "mds_bal_merge_rd": "1000",
> "mds_bal_merge_wr": "1000",
> "mds_bal_interval": "10",
> "mds_bal_fragment_interval": "5",
> "mds_bal_idle_threshold": "0",
> "mds_bal_max": "-1",
> "mds_bal_max_until": "-1",
> "mds_bal_mode": "0",
> "mds_bal_min_rebalance": "0.1",
> "mds_bal_min_start": "0.2",
> "mds_bal_need_min": "0.8",
> "mds_bal_need_max": "1.2",
> "mds_bal_midchunk": "0.3",
> "mds_bal_minchunk": "0.001",
> "mds_bal_target_removal_min": "5",
> "mds_bal_target_removal_max": "10",
> "mds_replay_interval": "1",
> "mds_shutdown_check": "0",
> "mds_thrash_exports": "0",
> "mds_thrash_fragments": "0",
> "mds_dump_cache_on_map": "false",
> "mds_dump_cache_after_rejoin": "false",
> "mds_verify_scatter": "false",
> "mds_debug_scatterstat": "false",
> "mds_debug_frag": "false",
> "mds_debug_auth_pins": "false",
> "mds_debug_subtrees": "false",
> "mds_kill_mdstable_at": "0",
> "mds_kill_export_at": "0",
> "mds_kill_import_at": "0",
> "mds_kill_link_at": "0",
> "mds_kill_rename_at": "0",
> "mds_kill_openc_at": "0",
> "mds_kill_journal_at": "0",
> "mds_kill_journal_expire_at": "0",
> "mds_kill_journal_replay_at": "0",
> "mds_journal_format": "1",
> "mds_kill_create_at": "0",
> "mds_inject_traceless_reply_probability": "0",
> "mds_wipe_sessions": "false",
> "mds_wipe_ino_prealloc": "false",
> "mds_skip_ino": "0",
> "max_mds": "1",
> "mds_standby_for_name": "",
> "mds_standby_for_rank": "-1",
> "mds_standby_replay": "false",
> "mds_enable_op_tracker": "true",
> "mds_op_history_size": "20",
> "mds_op_history_duration": "600",
> "mds_op_complaint_time": "30",
> "mds_op_log_threshold": "5",
> "mds_snap_min_uid": "0",
> "mds_snap_max_uid": "65536",
> "mds_verify_backtrace": "1",
> "mds_action_on_write_error": "1",
> "osd_compact_leveldb_on_mount": "false",
> "osd_max_backfills": "1",
> "osd_min_recovery_priority": "0",
> "osd_backfill_full_ratio": "0.93",
> "osd_backfill_retry_interval": "10",
> "osd_agent_max_ops": "4",
> "osd_agent_min_evict_effort": "0.1",
> "osd_agent_quantize_effort": "0.1",
> "osd_agent_delay_time": "5",
> "osd_find_best_info_ignore_history_les": "false",
> "osd_agent_hist_halflife": "1000",
> "osd_agent_slop": "0.02",
> "osd_uuid": "00000000-0000-0000-0000-000000000000",
> "osd_data": "\/var\/lib\/ceph\/osd\/ceph-0",
> "osd_journal": "\/dev\/vg-osd-0\/lv-journal",
> "osd_journal_size": "5000",
> "osd_max_write_size": "90",
> "osd_max_pgls": "1024",
> "osd_client_message_size_cap": "524288000",
> "osd_client_message_cap": "100",
> "osd_pg_bits": "6",
> "osd_pgp_bits": "6",
> "osd_crush_chooseleaf_type": "1",
> "osd_pool_default_crush_rule": "-1",
> "osd_pool_default_crush_replicated_ruleset": "0",
> "osd_pool_erasure_code_stripe_width": "4096",
> "osd_pool_default_size": "3",
> "osd_pool_default_min_size": "0",
> "osd_pool_default_pg_num": "8",
> "osd_pool_default_pgp_num": "8",
> "osd_pool_default_erasure_code_directory":
>"\/usr\/lib64\/ceph\/erasure-code",
> "osd_pool_default_erasure_code_profile": "plugin=jerasure
>technique=reed_sol_van k=2 m=1 ",
> "osd_erasure_code_plugins": "jerasure lrc isa",
> "osd_allow_recovery_below_min_size": "true",
> "osd_pool_default_flags": "0",
> "osd_pool_default_flag_hashpspool": "true",
> "osd_pool_default_flag_nodelete": "false",
> "osd_pool_default_flag_nopgchange": "false",
> "osd_pool_default_flag_nosizechange": "false",
> "osd_pool_default_hit_set_bloom_fpp": "0.05",
> "osd_pool_default_cache_target_dirty_ratio": "0.4",
> "osd_pool_default_cache_target_full_ratio": "0.8",
> "osd_pool_default_cache_min_flush_age": "0",
> "osd_pool_default_cache_min_evict_age": "0",
> "osd_hit_set_min_size": "1000",
> "osd_hit_set_max_size": "100000",
> "osd_hit_set_namespace": ".ceph-internal",
> "osd_tier_default_cache_mode": "writeback",
> "osd_tier_default_cache_hit_set_count": "4",
> "osd_tier_default_cache_hit_set_period": "1200",
> "osd_tier_default_cache_hit_set_type": "bloom",
> "osd_tier_default_cache_min_read_recency_for_promote": "1",
> "osd_map_dedup": "true",
> "osd_map_max_advance": "200",
> "osd_map_cache_size": "500",
> "osd_map_message_max": "100",
> "osd_map_share_max_epochs": "100",
> "osd_inject_bad_map_crc_probability": "0",
> "osd_inject_failure_on_pg_removal": "false",
> "osd_op_threads": "2",
> "osd_peering_wq_batch_size": "20",
> "osd_op_pq_max_tokens_per_priority": "4194304",
> "osd_op_pq_min_cost": "65536",
> "osd_disk_threads": "1",
> "osd_disk_thread_ioprio_class": "",
> "osd_disk_thread_ioprio_priority": "-1",
> "osd_recovery_threads": "1",
> "osd_recover_clone_overlap": "true",
> "osd_op_num_threads_per_shard": "2",
> "osd_op_num_shards": "5",
> "osd_read_eio_on_bad_digest": "true",
> "osd_recover_clone_overlap_limit": "10",
> "osd_backfill_scan_min": "64",
> "osd_backfill_scan_max": "512",
> "osd_op_thread_timeout": "15",
> "osd_op_thread_suicide_timeout": "150",
> "osd_recovery_thread_timeout": "30",
> "osd_recovery_thread_suicide_timeout": "300",
> "osd_snap_trim_thread_timeout": "3600",
> "osd_snap_trim_thread_suicide_timeout": "36000",
> "osd_snap_trim_sleep": "0",
> "osd_scrub_thread_timeout": "60",
> "osd_scrub_thread_suicide_timeout": "60",
> "osd_scrub_finalize_thread_timeout": "600",
> "osd_scrub_invalid_stats": "true",
> "osd_remove_thread_timeout": "3600",
> "osd_remove_thread_suicide_timeout": "36000",
> "osd_command_thread_timeout": "600",
> "osd_age": "0.8",
> "osd_age_time": "0",
> "osd_command_thread_suicide_timeout": "900",
> "osd_heartbeat_addr": ":\/0",
> "osd_heartbeat_interval": "6",
> "osd_heartbeat_grace": "20",
> "osd_heartbeat_min_peers": "10",
> "osd_heartbeat_use_min_delay_socket": "false",
> "osd_pg_max_concurrent_snap_trims": "2",
> "osd_heartbeat_min_healthy_ratio": "0.33",
> "osd_mon_heartbeat_interval": "30",
> "osd_mon_report_interval_max": "120",
> "osd_mon_report_interval_min": "5",
> "osd_pg_stat_report_interval_max": "500",
> "osd_mon_ack_timeout": "30",
> "osd_default_data_pool_replay_window": "45",
> "osd_preserve_trimmed_log": "false",
> "osd_auto_mark_unfound_lost": "false",
> "osd_recovery_delay_start": "0",
> "osd_recovery_max_active": "1",
> "osd_recovery_max_single_start": "5",
> "osd_recovery_max_chunk": "8388608",
> "osd_copyfrom_max_chunk": "8388608",
> "osd_push_per_object_cost": "1000",
> "osd_max_push_cost": "8388608",
> "osd_max_push_objects": "10",
> "osd_recovery_forget_lost_objects": "false",
> "osd_max_scrubs": "1",
> "osd_scrub_begin_hour": "0",
> "osd_scrub_end_hour": "24",
> "osd_scrub_load_threshold": "0.5",
> "osd_scrub_min_interval": "86400",
> "osd_scrub_max_interval": "604800",
> "osd_scrub_chunk_min": "5",
> "osd_scrub_chunk_max": "25",
> "osd_scrub_sleep": "0",
> "osd_deep_scrub_interval": "604800",
> "osd_deep_scrub_stride": "524288",
> "osd_deep_scrub_update_digest_min_age": "7200",
> "osd_scan_list_ping_tp_interval": "100",
> "osd_auto_weight": "false",
> "osd_class_dir": "\/usr\/lib64\/rados-classes",
> "osd_open_classes_on_start": "true",
> "osd_check_for_log_corruption": "false",
> "osd_use_stale_snap": "false",
> "osd_rollback_to_cluster_snap": "",
> "osd_default_notify_timeout": "30",
> "osd_kill_backfill_at": "0",
> "osd_pg_epoch_persisted_max_stale": "200",
> "osd_min_pg_log_entries": "3000",
> "osd_max_pg_log_entries": "10000",
> "osd_pg_log_trim_min": "100",
> "osd_op_complaint_time": "30",
> "osd_command_max_records": "256",
> "osd_max_pg_blocked_by": "16",
> "osd_op_log_threshold": "5",
> "osd_verify_sparse_read_holes": "false",
> "osd_debug_drop_ping_probability": "0",
> "osd_debug_drop_ping_duration": "0",
> "osd_debug_drop_pg_create_probability": "0",
> "osd_debug_drop_pg_create_duration": "1",
> "osd_debug_drop_op_probability": "0",
> "osd_debug_op_order": "false",
> "osd_debug_scrub_chance_rewrite_digest": "0",
> "osd_debug_verify_snaps_on_info": "false",
> "osd_debug_verify_stray_on_activate": "false",
> "osd_debug_skip_full_check_in_backfill_reservation": "false",
> "osd_debug_reject_backfill_probability": "0",
> "osd_debug_inject_copyfrom_error": "false",
> "osd_enable_op_tracker": "true",
> "osd_num_op_tracker_shard": "32",
> "osd_op_history_size": "20",
> "osd_op_history_duration": "600",
> "osd_target_transaction_size": "30",
> "osd_failsafe_full_ratio": "0.97",
> "osd_failsafe_nearfull_ratio": "0.9",
> "osd_pg_object_context_cache_count": "64",
> "osd_debug_pg_log_writeout": "false",
> "threadpool_default_timeout": "60",
> "threadpool_empty_queue_max_wait": "2",
> "leveldb_write_buffer_size": "8388608",
> "leveldb_cache_size": "134217728",
> "leveldb_block_size": "0",
> "leveldb_bloom_size": "0",
> "leveldb_max_open_files": "0",
> "leveldb_compression": "true",
> "leveldb_paranoid": "false",
> "leveldb_log": "",
> "leveldb_compact_on_mount": "false",
> "kinetic_host": "",
> "kinetic_port": "8123",
> "kinetic_user_id": "1",
> "kinetic_hmac_key": "asdfasdf",
> "kinetic_use_ssl": "false",
> "rocksdb_compact_on_mount": "false",
> "rocksdb_write_buffer_size": "0",
> "rocksdb_target_file_size_base": "0",
> "rocksdb_cache_size": "0",
> "rocksdb_block_size": "0",
> "rocksdb_bloom_size": "0",
> "rocksdb_write_buffer_num": "0",
> "rocksdb_background_compactions": "0",
> "rocksdb_background_flushes": "0",
> "rocksdb_max_open_files": "0",
> "rocksdb_compression": "",
> "rocksdb_paranoid": "false",
> "rocksdb_log": "\/dev\/null",
> "rocksdb_level0_file_num_compaction_trigger": "0",
> "rocksdb_level0_slowdown_writes_trigger": "0",
> "rocksdb_level0_stop_writes_trigger": "0",
> "rocksdb_disableDataSync": "true",
> "rocksdb_disableWAL": "false",
> "rocksdb_num_levels": "0",
> "rocksdb_wal_dir": "",
> "rocksdb_info_log_level": "info",
> "osd_client_op_priority": "63",
> "osd_recovery_op_priority": "1",
> "osd_recovery_op_warn_multiple": "16",
> "osd_mon_shutdown_timeout": "5",
> "osd_max_object_size": "107374182400",
> "osd_max_object_name_len": "2048",
> "osd_max_attr_name_len": "100",
> "osd_max_attr_size": "0",
> "osd_objectstore": "filestore",
> "osd_debug_override_acting_compat": "false",
> "osd_bench_small_size_max_iops": "100",
> "osd_bench_large_size_max_throughput": "104857600",
> "osd_bench_max_block_size": "67108864",
> "osd_bench_duration": "30",
> "memstore_device_bytes": "1073741824",
> "filestore_omap_backend": "leveldb",
> "filestore_debug_disable_sharded_check": "false",
> "filestore_wbthrottle_enable": "true",
> "filestore_wbthrottle_btrfs_bytes_start_flusher": "41943040",
> "filestore_wbthrottle_btrfs_bytes_hard_limit": "419430400",
> "filestore_wbthrottle_btrfs_ios_start_flusher": "500",
> "filestore_wbthrottle_btrfs_ios_hard_limit": "5000",
> "filestore_wbthrottle_btrfs_inodes_start_flusher": "500",
> "filestore_wbthrottle_xfs_bytes_start_flusher": "41943040",
> "filestore_wbthrottle_xfs_bytes_hard_limit": "419430400",
> "filestore_wbthrottle_xfs_ios_start_flusher": "500",
> "filestore_wbthrottle_xfs_ios_hard_limit": "5000",
> "filestore_wbthrottle_xfs_inodes_start_flusher": "500",
> "filestore_wbthrottle_btrfs_inodes_hard_limit": "5000",
> "filestore_wbthrottle_xfs_inodes_hard_limit": "5000",
> "filestore_index_retry_probability": "0",
> "filestore_debug_inject_read_err": "false",
> "filestore_debug_omap_check": "false",
> "filestore_omap_header_cache_size": "1024",
> "filestore_max_inline_xattr_size": "0",
> "filestore_max_inline_xattr_size_xfs": "65536",
> "filestore_max_inline_xattr_size_btrfs": "2048",
> "filestore_max_inline_xattr_size_other": "512",
> "filestore_max_inline_xattrs": "0",
> "filestore_max_inline_xattrs_xfs": "10",
> "filestore_max_inline_xattrs_btrfs": "10",
> "filestore_max_inline_xattrs_other": "2",
> "filestore_sloppy_crc": "false",
> "filestore_sloppy_crc_block_size": "65536",
> "filestore_max_alloc_hint_size": "1048576",
> "filestore_max_sync_interval": "5",
> "filestore_min_sync_interval": "0.01",
> "filestore_btrfs_snap": "true",
> "filestore_btrfs_clone_range": "true",
> "filestore_zfs_snap": "false",
> "filestore_fsync_flushes_journal_data": "false",
> "filestore_fiemap": "false",
> "filestore_fadvise": "true",
> "filestore_xfs_extsize": "true",
> "filestore_journal_parallel": "false",
> "filestore_journal_writeahead": "false",
> "filestore_journal_trailing": "false",
> "filestore_queue_max_ops": "50",
> "filestore_queue_max_bytes": "104857600",
> "filestore_queue_committing_max_ops": "500",
> "filestore_queue_committing_max_bytes": "104857600",
> "filestore_op_threads": "2",
> "filestore_op_thread_timeout": "60",
> "filestore_op_thread_suicide_timeout": "180",
> "filestore_commit_timeout": "600",
> "filestore_fiemap_threshold": "4096",
> "filestore_merge_threshold": "10",
> "filestore_split_multiple": "2",
> "filestore_update_to": "1000",
> "filestore_blackhole": "false",
> "filestore_fd_cache_size": "128",
> "filestore_fd_cache_shards": "16",
> "filestore_dump_file": "",
> "filestore_kill_at": "0",
> "filestore_inject_stall": "0",
> "filestore_fail_eio": "true",
> "filestore_debug_verify_split": "false",
> "journal_dio": "true",
> "journal_aio": "true",
> "journal_force_aio": "false",
> "keyvaluestore_queue_max_ops": "50",
> "keyvaluestore_queue_max_bytes": "104857600",
> "keyvaluestore_debug_check_backend": "false",
> "keyvaluestore_op_threads": "2",
> "keyvaluestore_op_thread_timeout": "60",
> "keyvaluestore_op_thread_suicide_timeout": "180",
> "keyvaluestore_default_strip_size": "4096",
> "keyvaluestore_max_expected_write_size": "16777216",
> "keyvaluestore_header_cache_size": "4096",
> "keyvaluestore_backend": "leveldb",
> "journal_max_corrupt_search": "10485760",
> "journal_block_align": "true",
> "journal_write_header_frequency": "0",
> "journal_max_write_bytes": "10485760",
> "journal_max_write_entries": "100",
> "journal_queue_max_ops": "300",
> "journal_queue_max_bytes": "33554432",
> "journal_align_min_size": "65536",
> "journal_replay_from": "0",
> "journal_zero_on_create": "false",
> "journal_ignore_corruption": "false",
> "journal_discard": "false",
> "rados_mon_op_timeout": "0",
> "rados_osd_op_timeout": "0",
> "rbd_op_threads": "1",
> "rbd_op_thread_timeout": "60",
> "rbd_non_blocking_aio": "true",
> "rbd_cache": "true",
> "rbd_cache_writethrough_until_flush": "true",
> "rbd_cache_size": "33554432",
> "rbd_cache_max_dirty": "25165824",
> "rbd_cache_target_dirty": "16777216",
> "rbd_cache_max_dirty_age": "1",
> "rbd_cache_max_dirty_object": "0",
> "rbd_cache_block_writes_upfront": "false",
> "rbd_concurrent_management_ops": "10",
> "rbd_balance_snap_reads": "false",
> "rbd_localize_snap_reads": "false",
> "rbd_balance_parent_reads": "false",
> "rbd_localize_parent_reads": "true",
> "rbd_readahead_trigger_requests": "10",
> "rbd_readahead_max_bytes": "524288",
> "rbd_readahead_disable_after_bytes": "52428800",
> "rbd_clone_copy_on_read": "false",
> "rbd_blacklist_on_break_lock": "true",
> "rbd_blacklist_expire_seconds": "0",
> "rbd_request_timed_out_seconds": "30",
> "rbd_default_format": "1",
> "rbd_default_order": "22",
> "rbd_default_stripe_count": "0",
> "rbd_default_stripe_unit": "0",
> "rbd_default_features": "3",
> "nss_db_path": "",
> "rgw_max_chunk_size": "524288",
> "rgw_override_bucket_index_max_shards": "0",
> "rgw_bucket_index_max_aio": "8",
> "rgw_enable_quota_threads": "true",
> "rgw_enable_gc_threads": "true",
> "rgw_data": "\/var\/lib\/ceph\/radosgw\/ceph-0",
> "rgw_enable_apis": "s3, swift, swift_auth, admin",
> "rgw_cache_enabled": "true",
> "rgw_cache_lru_size": "10000",
> "rgw_socket_path": "",
> "rgw_host": "",
> "rgw_port": "",
> "rgw_dns_name": "",
> "rgw_content_length_compat": "false",
> "rgw_script_uri": "",
> "rgw_request_uri": "",
> "rgw_swift_url": "",
> "rgw_swift_url_prefix": "swift",
> "rgw_swift_auth_url": "",
> "rgw_swift_auth_entry": "auth",
> "rgw_swift_tenant_name": "",
> "rgw_swift_enforce_content_length": "false",
> "rgw_keystone_url": "",
> "rgw_keystone_admin_token": "",
> "rgw_keystone_admin_user": "",
> "rgw_keystone_admin_password": "",
> "rgw_keystone_admin_tenant": "",
> "rgw_keystone_accepted_roles": "Member, admin",
> "rgw_keystone_token_cache_size": "10000",
> "rgw_keystone_revocation_interval": "900",
> "rgw_s3_auth_use_rados": "true",
> "rgw_s3_auth_use_keystone": "false",
> "rgw_admin_entry": "admin",
> "rgw_enforce_swift_acls": "true",
> "rgw_swift_token_expiration": "86400",
> "rgw_print_continue": "true",
> "rgw_remote_addr_param": "REMOTE_ADDR",
> "rgw_op_thread_timeout": "600",
> "rgw_op_thread_suicide_timeout": "0",
> "rgw_thread_pool_size": "100",
> "rgw_num_control_oids": "8",
> "rgw_num_rados_handles": "1",
> "rgw_zone": "",
> "rgw_zone_root_pool": ".rgw.root",
> "rgw_region": "",
> "rgw_region_root_pool": ".rgw.root",
> "rgw_default_region_info_oid": "default.region",
> "rgw_log_nonexistent_bucket": "false",
> "rgw_log_object_name": "%Y-%m-%d-%H-%i-%n",
> "rgw_log_object_name_utc": "false",
> "rgw_usage_max_shards": "32",
> "rgw_usage_max_user_shards": "1",
> "rgw_enable_ops_log": "false",
> "rgw_enable_usage_log": "false",
> "rgw_ops_log_rados": "true",
> "rgw_ops_log_socket_path": "",
> "rgw_ops_log_data_backlog": "5242880",
> "rgw_usage_log_flush_threshold": "1024",
> "rgw_usage_log_tick_interval": "30",
> "rgw_intent_log_object_name": "%Y-%m-%d-%i-%n",
> "rgw_intent_log_object_name_utc": "false",
> "rgw_init_timeout": "300",
> "rgw_mime_types_file": "\/etc\/mime.types",
> "rgw_gc_max_objs": "32",
> "rgw_gc_obj_min_wait": "7200",
> "rgw_gc_processor_max_time": "3600",
> "rgw_gc_processor_period": "3600",
> "rgw_s3_success_create_obj_status": "0",
> "rgw_resolve_cname": "false",
> "rgw_obj_stripe_size": "4194304",
> "rgw_extended_http_attrs": "",
> "rgw_exit_timeout_secs": "120",
> "rgw_get_obj_window_size": "16777216",
> "rgw_get_obj_max_req_size": "4194304",
> "rgw_relaxed_s3_bucket_names": "false",
> "rgw_defer_to_bucket_acls": "",
> "rgw_list_buckets_max_chunk": "1000",
> "rgw_md_log_max_shards": "64",
> "rgw_num_zone_opstate_shards": "128",
> "rgw_opstate_ratelimit_sec": "30",
> "rgw_curl_wait_timeout_ms": "1000",
> "rgw_copy_obj_progress": "true",
> "rgw_copy_obj_progress_every_bytes": "1048576",
> "rgw_data_log_window": "30",
> "rgw_data_log_changes_size": "1000",
> "rgw_data_log_num_shards": "128",
> "rgw_data_log_obj_prefix": "data_log",
> "rgw_replica_log_obj_prefix": "replica_log",
> "rgw_bucket_quota_ttl": "600",
> "rgw_bucket_quota_soft_threshold": "0.95",
> "rgw_bucket_quota_cache_size": "10000",
> "rgw_expose_bucket": "false",
> "rgw_frontends": "fastcgi, civetweb port=7480",
> "rgw_user_quota_bucket_sync_interval": "180",
> "rgw_user_quota_sync_interval": "86400",
> "rgw_user_quota_sync_idle_users": "false",
> "rgw_user_quota_sync_wait_time": "86400",
> "rgw_multipart_min_part_size": "5242880",
> "rgw_olh_pending_timeout_sec": "3600",
> "rgw_user_max_buckets": "1000",
> "mutex_perf_counter": "false",
> "throttler_perf_counter": "true",
> "internal_safe_to_start_threads": "true"
>}
>
>ceph.conf
>
>cat /etc/ceph/ceph.conf
>[client]
>rbd cache = true
> # 256 MB of cache
> rbd cache size = 268435456
> # 64 MB Max Dirty
> rbd cache max dirty = 67108864
> # 32 MB Target Dirty
> rbd cache target dirty = 33554432
> # 2 Seconds worth can be dirty
> rbd cache max dirty age = 2.0
>[global]
> public network = 10.66.4.0/24
> cluster network = 10.66.4.0/24
>
> # For version 0.55 and beyond, you must explicitly enable
> # or disable authentication with "auth" entries in [global].
>
> auth cluster required = cephx
> auth service required = cephx
> auth client required = cephx
>
> mon osd full ratio = .98
> mon osd nearfull ratio = .94
>
>
> mon_pg_warn_max_per_osd = 0
>
>[osd]
> osd journal size = 5000
> journal aio = true
>
> # See
>https://ceph.com/docs/master/rados/configuration/osd-config-ref/
> osd client op priority = 90
> osd recovery op priority = 10
> osd recovery max active = 1
> osd max backfills = 1
> osd backfill full ratio = 0.93
>
> #The following assumes ext4 filesystem.
> # filestore xattr use omap = true
>
>
> # For Bobtail (v 0.56) and subsequent versions, you may
> # add settings for mkcephfs so that it will create and mount
> # the file system on a particular OSD for you. Remove the comment `#`
> # character for the following settings and replace the values
> # in braces with appropriate values, or leave the following settings
> # commented out to accept the default values. You must specify the
> # --mkfs option with mkcephfs in order for the deployment script to
> # utilize the following settings, and you must define the 'devs'
> # option for each osd instance; see below.
>
> osd mkfs type = btrfs
> osd mkfs options btrfs = -L $name
> # xfs options by Dell RAID CARD specifies 64k stripe size(su) and
>each osd is a single drive(1) for (sw)
> # see :
>http://www.mysqlperformanceblog.com/2011/12/16/setting-up-xfs-the-simple-e
>dition/
> # also see : https://raid.wiki.kernel.org/index.php/RAID_setup#XFS
> # mkfs -t xfs -f -L ceph-xx -d su=64k,sw=1 /dev/sdX1
> osd mkfs options xfs = -L -d su=64k,sw=1 $name
> osd mount options btrfs = rw,noatime
> osd mount options xfs = rw,noatime,nodiratime
> # mount -o rw,noatime,nodiratime /dev/disk/by-label/ceph-X
>/var/lib/ceph/osd/ceph-X
>
> # For example, for ext4, the mount option might look like this:
>
> #osd mkfs options ext4 = user_xattr,rw,noatime
>
> # Execute $ hostname to retrieve the name of your host,
> # and replace {hostname} with the name of your host.
> # For the monitor, replace {ip-address} with the IP
> # address of your host.
>
>
> # CAIT -- Manual commands to make and mount file system
>
> # -- Make xfs file system
> # mkfs -t xfs -f -L ceph-X -d su=64k,sw=1 /dev/sdX1
>
> # -- Rescan Parition Label
> # partprobe /dev/sdX1
>
> # -- Mount ceph file system
> # mount -o rw,noatime,nodiratime /dev/disk/by-label/ceph-X
>/var/lib/ceph/osd/ceph-X
>
>
>
>
>
>[mon]
> mon debug dump transactions = false
> # mon warn on legacy crush tunables = false
>
># [mon.splinter]
> # host = splinter
> # mon addr = 10.66.4.2:6789
>
>[mon.helm]
> host = helm
> mon addr = 10.66.4.4:6789
>
>[mon.hamms]
> host = hamms
> mon addr = 10.66.4.9:6789
>
>[mon.gold]
> host = gold
> mon addr = 10.66.4.7:6789
>
>[osd.0]
> host = helm
> devs = /dev/vg-osd-0/lv-osd
> osd mkfs type = xfs
> osd journal = /dev/vg-osd-0/lv-journal
>
>[osd.1]
> host = helm
> devs = /dev/vg-osd-1/lv-osd
> osd mkfs type = xfs
> osd journal = /dev/vg-osd-1/lv-journal
>
>[osd.2]
> host = helm
> devs = /dev/vg-osd-2/lv-osd
> osd mkfs type = xfs
> osd journal = /dev/vg-osd-2/lv-journal
>
>[osd.3]
> host = helm
> devs = /dev/vg-osd-3/lv-osd
> osd mkfs type = xfs
> osd journal = /dev/vg-osd-3/lv-journal
>
>[osd.4]
> host = helm
> devs = /dev/vg-osd-4/lv-osd
> osd mkfs type = xfs
> osd journal = /dev/vg-osd-4/lv-journal
>
>[osd.5]
> host = helm
> devs = /dev/vg-osd-5/lv-osd
> osd mkfs type = xfs
> osd journal = /dev/vg-osd-5/lv-journal
>
>[osd.6]
> host = helm
> devs = /dev/vg-osd-6/lv-osd
> osd mkfs type = xfs
> osd journal = /dev/vg-osd-6/lv-journal
>
>[osd.7]
> host = helm
> devs = /dev/vg-osd-7/lv-osd
> osd mkfs type = xfs
> osd journal = /dev/vg-osd-7/lv-journal
>
>[osd.8]
> host = helm
> devs = /dev/vg-osd-8/lv-osd
> osd mkfs type = xfs
> osd journal = /dev/vg-osd-8/lv-journal
>
>[osd.16]
> host = gold
> devs = /dev/disk/by-label/ceph-16
> osd mkfs type = xfs
>
>[osd.17]
> host = gold
> devs = /dev/disk/by-label/ceph-17
> osd mkfs type = xfs
>
>[osd.18]
> host = gold
> devs = /dev/disk/by-label/ceph-18
> osd mkfs type = xfs
>
>[osd.19]
> host = gold
> devs = /dev/disk/by-label/ceph-19
> osd mkfs type = xfs
>
>[osd.20]
> host = gold
> devs = /dev/disk/by-label/ceph-20
> osd mkfs type = xfs
>
>[osd.21]
> host = gold
> devs = /dev/disk/by-label/ceph-21
> osd mkfs type = xfs
>
>[osd.9]
> host = gold
> devs = /dev/vg-osd-9/lv-osd
> osd mkfs type = xfs
> osd journal = /dev/vg-osd-9/lv-journal
>
>[osd.10]
> host = gold
> devs = /dev/vg-osd-10/lv-osd
> osd mkfs type = xfs
> osd journal = /dev/vg-osd-10/lv-journal
>
>[osd.34]
> host = gold
> devs = /dev/vg-osd-34/lv-osd
> osd mkfs type = xfs
> osd journal = /dev/vg-osd-34/lv-journal
>
>[osd.22]
> host = gallo
> devs = /dev/disk/by-label/ceph-22
> osd mkfs type = xfs
>
>[osd.23]
> host = gallo
> devs = /dev/disk/by-label/ceph-23
> osd mkfs type = xfs
>
>[osd.24]
> host = gallo
> devs = /dev/disk/by-label/ceph-24
> osd mkfs type = xfs
>
>[osd.25]
> host = gallo
> devs = /dev/disk/by-label/ceph-25
> osd mkfs type = xfs
>
>[osd.26]
> host = gallo
> devs = /dev/disk/by-label/ceph-26
> osd mkfs type = xfs
>
>[osd.27]
> host = gallo
> devs = /dev/disk/by-label/ceph-27
> osd mkfs type = xfs
>
>[osd.11]
> host = gallo
> devs = /dev/vg-osd-11/lv-osd
> osd mkfs type = xfs
> osd journal = /dev/vg-osd-11/lv-journal
>
>[osd.12]
> host = gallo
> devs = /dev/vg-osd-12/lv-osd
> osd mkfs type = xfs
> osd journal = /dev/vg-osd-12/lv-journal
>
>[osd.35]
> host = gallo
> devs = /dev/vg-osd-35/lv-osd
> osd mkfs type = xfs
> osd journal = /dev/vg-osd-35/lv-journal
>
>[osd.13]
> host = hamms
> devs = /dev/vg-osd-13/lv-osd
> osd mkfs type = xfs
> osd journal = /dev/vg-osd-13/lv-journal
>
>[osd.14]
> host = hamms
> devs = /dev/vg-osd-14/lv-osd
> osd mkfs type = xfs
> osd journal = /dev/vg-osd-14/lv-journal
>
>[osd.15]
> host = hamms
> devs = /dev/vg-osd-15/lv-osd
> osd mkfs type = xfs
> osd journal = /dev/vg-osd-15/lv-journal
>
>[osd.28]
> host = hamms
> devs = /dev/vg-osd-28/lv-osd
> osd mkfs type = xfs
> osd journal = /dev/vg-osd-28/lv-journal
>
>[osd.29]
> host = hamms
> devs = /dev/vg-osd-29/lv-osd
> osd mkfs type = xfs
> osd journal = /dev/vg-osd-29/lv-journal
>
>[osd.30]
> host = hamms
> devs = /dev/vg-osd-30/lv-osd
> osd mkfs type = xfs
> osd journal = /dev/vg-osd-30/lv-journal
>
>[osd.31]
> host = hamms
> devs = /dev/vg-osd-31/lv-osd
> osd mkfs type = xfs
> osd journal = /dev/vg-osd-31/lv-journal
>
>[osd.32]
> host = hamms
> devs = /dev/vg-osd-32/lv-osd
> osd mkfs type = xfs
> osd journal = /dev/vg-osd-32/lv-journal
>
>[osd.33]
> host = hamms
> devs = /dev/vg-osd-33/lv-osd
> osd mkfs type = xfs
> osd journal = /dev/vg-osd-33/lv-journal
>
>
>--
>Joe Ryner
>Center for the Application of Information Technologies (CAIT)
>Production Coordinator
>P: (309) 298-1804
>F: (309) 298-2806
>_______________________________________________
>ceph-users mailing list
>ceph-users@xxxxxxxxxxxxxx
>http://lists.ceph.com/listinfo.cgi/ceph-users-ceph.com
This email and any files transmitted with it are confidential and intended solely for the individual or entity to whom they are addressed. If you have received this email in error destroy it immediately. *** Walmart Confidential ***
_______________________________________________
ceph-users mailing list
ceph-users@xxxxxxxxxxxxxx
http://lists.ceph.com/listinfo.cgi/ceph-users-ceph.com
