On Mon, Mar 23, 2015 at 3:04 PM, Josef Johansson <josef86@xxxxxxxxx> wrote:
>
> On 23 Mar 2015, at 03:58, Haomai Wang <haomaiwang@xxxxxxxxx> wrote:
>
> On Mon, Mar 23, 2015 at 2:53 AM, Josef Johansson <josef86@xxxxxxxxx> wrote:
>
> Hi all!
>
> Trying to figure out how much my journals are used, using SSDs as journals
> and SATA-drives as storage, I dive into perf dump.
> But I can’t figure out why journal_queue_bytes is at constant 0. The only
> thing that differs is dirtied in WBThrottle.
>
>
> journal_queue_bytes means how much journal data in the queue and is
> waiting for Journal Thread to be processed.
>
> Still now osd can't tell you how much data in the journal waiting for
> writeback and sync.
>
> Hm, who knows that then?
> Is this the WBThrottle value?
WBThrottle only will tell you the dirtied data in the system buffer
cache. It's not directly related to journal data.
>
> No way of knowing how much journal is used at all?
>
> Maybe I thought of this wrong so if I understand you correctly
>
> Data is written to OSD
> The journal saves it to the queue
> Waits for others to sync the requests as well
> Sends a ACK to the client
> Starts writing to the filestore buffer
> filestore buffer commits when limits and met (inodes/ios-dirtied,
> filestore_sync_max_interval)
>
> So if I’m meeting latency and want to see if my journals are lazy, I should
> indeed look at journal_queue_bytes, if that’s zero, it’s behaving well.
yes or not, journal_queue_bytes is related to latency but not always
the key to see whether the journal is lazy.
>
> Thanks,
> Josef
>
>
> Maybe I’ve disable that when setting the in-memory debug variables to 0/0?
>
> Thanks,
> Josef
>
> # ceph --version
> ceph version 0.67.7 (d7ab4244396b57aac8b7e80812115bbd079e6b73)
>
> # ceph --admin-daemon /var/run/ceph/ceph-osd.0.asok config show | grep
> journal
> "journaler": "0\/0",
> "journal": "0\/0",
> "journaler_allow_split_entries": "true",
> "journaler_write_head_interval": "15",
> "journaler_prefetch_periods": "10",
> "journaler_prezero_periods": "5",
> "journaler_batch_interval": "0.001",
> "journaler_batch_max": "0",
> "mds_kill_journal_at": "0",
> "mds_kill_journal_expire_at": "0",
> "mds_kill_journal_replay_at": "0",
> "osd_journal": "\/var\/lib\/ceph\/osd\/ceph-0\/journal",
> "osd_journal_size": "25600",
> "filestore_fsync_flushes_journal_data": "false",
> "filestore_journal_parallel": "false",
> "filestore_journal_writeahead": "false",
> "filestore_journal_trailing": "false",
> "journal_dio": "true",
> "journal_aio": "true",
> "journal_force_aio": "false",
> "journal_max_corrupt_search": "10485760",
> "journal_block_align": "true",
> "journal_write_header_frequency": "0",
> "journal_max_write_bytes": "10485760",
> "journal_max_write_entries": "100",
> "journal_queue_max_ops": "300",
> "journal_queue_max_bytes": "33554432",
> "journal_align_min_size": "65536",
> "journal_replay_from": "0",
> "journal_zero_on_create": "false",
> "journal_ignore_corruption": "false",
>
> # ceph --admin-daemon /var/run/ceph/ceph-osd.0.asok perf dump
> { "WBThrottle": { "bytes_dirtied": 32137216,
> "bytes_wb": 0,
> "ios_dirtied": 1445,
> "ios_wb": 0,
> "inodes_dirtied": 491,
> "inodes_wb": 0},
> "filestore": { "journal_queue_max_ops": 300,
> "journal_queue_ops": 0,
> "journal_ops": 116105073,
> "journal_queue_max_bytes": 33554432,
> "journal_queue_bytes": 0,
> "journal_bytes": 3160504432839,
> "journal_latency": { "avgcount": 116105073,
> "sum": 64951.260611000},
> "journal_wr": 112261141,
> "journal_wr_bytes": { "avgcount": 112261141,
> "sum": 3426141528064},
> "op_queue_max_ops": 50,
> "op_queue_ops": 0,
> "ops": 116105073,
> "op_queue_max_bytes": 104857600,
> "op_queue_bytes": 0,
> "bytes": 3159111228243,
> "apply_latency": { "avgcount": 116105073,
> "sum": 247410.066048000},
> "committing": 0,
> "commitcycle": 267176,
> "commitcycle_interval": { "avgcount": 267176,
> "sum": 1873193.631124000},
> "commitcycle_latency": { "avgcount": 267176,
> "sum": 390421.062990000},
> "journal_full": 0,
> "queue_transaction_latency_avg": { "avgcount": 116105073,
> "sum": 378.948923000}},
> "leveldb": { "leveldb_get": 699871216,
> "leveldb_transaction": 522440246,
> "leveldb_compact": 0,
> "leveldb_compact_range": 0,
> "leveldb_compact_queue_merge": 0,
> "leveldb_compact_queue_len": 0},
> "mutex-FileJournal::completions_lock": { "wait": { "avgcount": 0,
> "sum": 0.000000000}},
> "mutex-FileJournal::finisher_lock": { "wait": { "avgcount": 0,
> "sum": 0.000000000}},
> "mutex-FileJournal::write_lock": { "wait": { "avgcount": 0,
> "sum": 0.000000000}},
> "mutex-FileJournal::writeq_lock": { "wait": { "avgcount": 0,
> "sum": 0.000000000}},
> "mutex-JOS::ApplyManager::apply_lock": { "wait": { "avgcount": 0,
> "sum": 0.000000000}},
> "mutex-JOS::ApplyManager::com_lock": { "wait": { "avgcount": 0,
> "sum": 0.000000000}},
> "mutex-JOS::SubmitManager::lock": { "wait": { "avgcount": 0,
> "sum": 0.000000000}},
> "mutex-WBThrottle::lock": { "wait": { "avgcount": 0,
> "sum": 0.000000000}},
> "osd": { "opq": 0,
> "op_wip": 0,
> "op": 83920139,
> "op_in_bytes": 1075345387581,
> "op_out_bytes": 954428806331,
> "op_latency": { "avgcount": 83920139,
> "sum": 1279934.620502000},
> "op_r": 32399024,
> "op_r_out_bytes": 953657617715,
> "op_r_latency": { "avgcount": 32399024,
> "sum": 238792.729743000},
> "op_w": 3321731,
> "op_w_in_bytes": 52637941027,
> "op_w_rlat": { "avgcount": 3321731,
> "sum": 15577.620040000},
> "op_w_latency": { "avgcount": 3321731,
> "sum": 62541.746123000},
> "op_rw": 48199384,
> "op_rw_in_bytes": 1022707446554,
> "op_rw_out_bytes": 771188616,
> "op_rw_rlat": { "avgcount": 48199384,
> "sum": 169776.087496000},
> "op_rw_latency": { "avgcount": 48199384,
> "sum": 978600.144636000},
> "subop": 73746080,
> "subop_in_bytes": 2008774955062,
> "subop_latency": { "avgcount": 73746080,
> "sum": 346096.627047000},
> "subop_w": 0,
> "subop_w_in_bytes": 2008774955062,
> "subop_w_latency": { "avgcount": 73746080,
> "sum": 346096.627047000},
> "subop_pull": 0,
> "subop_pull_latency": { "avgcount": 0,
> "sum": 0.000000000},
> "subop_push": 0,
> "subop_push_in_bytes": 0,
> "subop_push_latency": { "avgcount": 0,
> "sum": 0.000000000},
> "pull": 0,
> "push": 8255,
> "push_out_bytes": 24792364384,
> "push_in": 0,
> "push_in_bytes": 0,
> "recovery_ops": 8126,
> "loadavg": 585,
> "buffer_bytes": 0,
> "numpg": 402,
> "numpg_primary": 161,
> "numpg_replica": 241,
> "numpg_stray": 0,
> "heartbeat_to_peers": 38,
> "heartbeat_from_peers": 0,
> "map_messages": 15262,
> "map_message_epochs": 70160,
> "map_message_epoch_dups": 116821,
> "messages_delayed_for_map": 105,
> "peering_latency": { "avgcount": 628,
> "sum": 28166.821628000}},
> "throttle-filestore_bytes": { "val": 0,
> "max": 33554432,
> "get": 0,
> "get_sum": 0,
> "get_or_fail_fail": 0,
> "get_or_fail_success": 0,
> "take": 116105073,
> "take_sum": 3160504432839,
> "put": 112261141,
> "put_sum": 3160504432839,
> "wait": { "avgcount": 12,
> "sum": 0.097590000}},
> "throttle-filestore_ops": { "val": 0,
> "max": 300,
> "get": 0,
> "get_sum": 0,
> "get_or_fail_fail": 0,
> "get_or_fail_success": 0,
> "take": 116105073,
> "take_sum": 116105073,
> "put": 112261141,
> "put_sum": 116105073,
> "wait": { "avgcount": 0,
> "sum": 0.000000000}},
> "throttle-msgr_dispatch_throttler-client": { "val": 0,
> "max": 104857600,
> "get": 75187594,
> "get_sum": 978560902803,
> "get_or_fail_fail": 0,
> "get_or_fail_success": 0,
> "take": 0,
> "take_sum": 0,
> "put": 75187594,
> "put_sum": 978560902803,
> "wait": { "avgcount": 0,
> "sum": 0.000000000}},
> "throttle-msgr_dispatch_throttler-cluster": { "val": 0,
> "max": 104857600,
> "get": 159487799,
> "get_sum": 2119870334346,
> "get_or_fail_fail": 0,
> "get_or_fail_success": 0,
> "take": 0,
> "take_sum": 0,
> "put": 159487799,
> "put_sum": 2119870334346,
> "wait": { "avgcount": 0,
> "sum": 0.000000000}},
> "throttle-msgr_dispatch_throttler-hb_back_server": { "val": 0,
> "max": 104857600,
> "get": 19958614,
> "get_sum": 938054858,
> "get_or_fail_fail": 0,
> "get_or_fail_success": 0,
> "take": 0,
> "take_sum": 0,
> "put": 19958614,
> "put_sum": 938054858,
> "wait": { "avgcount": 0,
> "sum": 0.000000000}},
> "throttle-msgr_dispatch_throttler-hb_front_server": { "val": 0,
> "max": 104857600,
> "get": 19958614,
> "get_sum": 938054858,
> "get_or_fail_fail": 0,
> "get_or_fail_success": 0,
> "take": 0,
> "take_sum": 0,
> "put": 19958614,
> "put_sum": 938054858,
> "wait": { "avgcount": 0,
> "sum": 0.000000000}},
> "throttle-msgr_dispatch_throttler-hbclient": { "val": 0,
> "max": 104857600,
> "get": 44608572,
> "get_sum": 2096602884,
> "get_or_fail_fail": 0,
> "get_or_fail_success": 0,
> "take": 0,
> "take_sum": 0,
> "put": 44608572,
> "put_sum": 2096602884,
> "wait": { "avgcount": 0,
> "sum": 0.000000000}},
> "throttle-osd_client_bytes": { "val": 0,
> "max": 524288000,
> "get": 74786210,
> "get_sum": 978238232914,
> "get_or_fail_fail": 0,
> "get_or_fail_success": 0,
> "take": 0,
> "take_sum": 0,
> "put": 117138891,
> "put_sum": 978238232914,
> "wait": { "avgcount": 0,
> "sum": 0.000000000}},
> "throttle-osd_client_messages": { "val": 17,
> "max": 100,
> "get": 75909689,
> "get_sum": 75909689,
> "get_or_fail_fail": 0,
> "get_or_fail_success": 0,
> "take": 0,
> "take_sum": 0,
> "put": 75909672,
> "put_sum": 75909672,
> "wait": { "avgcount": 478562,
> "sum": 5665.119305000}}}
> _______________________________________________
> ceph-users mailing list
> ceph-users@xxxxxxxxxxxxxx
> http://lists.ceph.com/listinfo.cgi/ceph-users-ceph.com
>
>
>
>
> --
> Best Regards,
>
> Wheat
>
>
--
Best Regards,
Wheat
_______________________________________________
ceph-users mailing list
ceph-users@xxxxxxxxxxxxxx
http://lists.ceph.com/listinfo.cgi/ceph-users-ceph.com
