On Mon, Mar 23, 2015 at 2:53 AM, Josef Johansson <josef86@xxxxxxxxx> wrote:Hi all!
Trying to figure out how much my journals are used, using SSDs as journals and SATA-drives as storage, I dive into perf dump.
But I can’t figure out why journal_queue_bytes is at constant 0. The only thing that differs is dirtied in WBThrottle.
journal_queue_bytes means how much journal data in the queue and iswaiting for Journal Thread to be processed.Still now osd can't tell you how much data in the journal waiting forwriteback and sync.
Hm, who knows that then? Is this the WBThrottle value?
No way of knowing how much journal is used at all?
Maybe I thought of this wrong so if I understand you correctly
Data is written to OSD The journal saves it to the queue Waits for others to sync the requests as well Sends a ACK to the client Starts writing to the filestore buffer filestore buffer commits when limits and met (inodes/ios-dirtied, filestore_sync_max_interval)
So if I’m meeting latency and want to see if my journals are lazy, I should indeed look at journal_queue_bytes, if that’s zero, it’s behaving well.
Thanks, Josef
Maybe I’ve disable that when setting the in-memory debug variables to 0/0?
Thanks,
Josef
# ceph --version
ceph version 0.67.7 (d7ab4244396b57aac8b7e80812115bbd079e6b73)
# ceph --admin-daemon /var/run/ceph/ceph-osd.0.asok config show | grep journal
"journaler": "0\/0",
"journal": "0\/0",
"journaler_allow_split_entries": "true",
"journaler_write_head_interval": "15",
"journaler_prefetch_periods": "10",
"journaler_prezero_periods": "5",
"journaler_batch_interval": "0.001",
"journaler_batch_max": "0",
"mds_kill_journal_at": "0",
"mds_kill_journal_expire_at": "0",
"mds_kill_journal_replay_at": "0",
"osd_journal": "\/var\/lib\/ceph\/osd\/ceph-0\/journal",
"osd_journal_size": "25600",
"filestore_fsync_flushes_journal_data": "false",
"filestore_journal_parallel": "false",
"filestore_journal_writeahead": "false",
"filestore_journal_trailing": "false",
"journal_dio": "true",
"journal_aio": "true",
"journal_force_aio": "false",
"journal_max_corrupt_search": "10485760",
"journal_block_align": "true",
"journal_write_header_frequency": "0",
"journal_max_write_bytes": "10485760",
"journal_max_write_entries": "100",
"journal_queue_max_ops": "300",
"journal_queue_max_bytes": "33554432",
"journal_align_min_size": "65536",
"journal_replay_from": "0",
"journal_zero_on_create": "false",
"journal_ignore_corruption": "false",
# ceph --admin-daemon /var/run/ceph/ceph-osd.0.asok perf dump
{ "WBThrottle": { "bytes_dirtied": 32137216,
"bytes_wb": 0,
"ios_dirtied": 1445,
"ios_wb": 0,
"inodes_dirtied": 491,
"inodes_wb": 0},
"filestore": { "journal_queue_max_ops": 300,
"journal_queue_ops": 0,
"journal_ops": 116105073,
"journal_queue_max_bytes": 33554432,
"journal_queue_bytes": 0,
"journal_bytes": 3160504432839,
"journal_latency": { "avgcount": 116105073,
"sum": 64951.260611000},
"journal_wr": 112261141,
"journal_wr_bytes": { "avgcount": 112261141,
"sum": 3426141528064},
"op_queue_max_ops": 50,
"op_queue_ops": 0,
"ops": 116105073,
"op_queue_max_bytes": 104857600,
"op_queue_bytes": 0,
"bytes": 3159111228243,
"apply_latency": { "avgcount": 116105073,
"sum": 247410.066048000},
"committing": 0,
"commitcycle": 267176,
"commitcycle_interval": { "avgcount": 267176,
"sum": 1873193.631124000},
"commitcycle_latency": { "avgcount": 267176,
"sum": 390421.062990000},
"journal_full": 0,
"queue_transaction_latency_avg": { "avgcount": 116105073,
"sum": 378.948923000}},
"leveldb": { "leveldb_get": 699871216,
"leveldb_transaction": 522440246,
"leveldb_compact": 0,
"leveldb_compact_range": 0,
"leveldb_compact_queue_merge": 0,
"leveldb_compact_queue_len": 0},
"mutex-FileJournal::completions_lock": { "wait": { "avgcount": 0,
"sum": 0.000000000}},
"mutex-FileJournal::finisher_lock": { "wait": { "avgcount": 0,
"sum": 0.000000000}},
"mutex-FileJournal::write_lock": { "wait": { "avgcount": 0,
"sum": 0.000000000}},
"mutex-FileJournal::writeq_lock": { "wait": { "avgcount": 0,
"sum": 0.000000000}},
"mutex-JOS::ApplyManager::apply_lock": { "wait": { "avgcount": 0,
"sum": 0.000000000}},
"mutex-JOS::ApplyManager::com_lock": { "wait": { "avgcount": 0,
"sum": 0.000000000}},
"mutex-JOS::SubmitManager::lock": { "wait": { "avgcount": 0,
"sum": 0.000000000}},
"mutex-WBThrottle::lock": { "wait": { "avgcount": 0,
"sum": 0.000000000}},
"osd": { "opq": 0,
"op_wip": 0,
"op": 83920139,
"op_in_bytes": 1075345387581,
"op_out_bytes": 954428806331,
"op_latency": { "avgcount": 83920139,
"sum": 1279934.620502000},
"op_r": 32399024,
"op_r_out_bytes": 953657617715,
"op_r_latency": { "avgcount": 32399024,
"sum": 238792.729743000},
"op_w": 3321731,
"op_w_in_bytes": 52637941027,
"op_w_rlat": { "avgcount": 3321731,
"sum": 15577.620040000},
"op_w_latency": { "avgcount": 3321731,
"sum": 62541.746123000},
"op_rw": 48199384,
"op_rw_in_bytes": 1022707446554,
"op_rw_out_bytes": 771188616,
"op_rw_rlat": { "avgcount": 48199384,
"sum": 169776.087496000},
"op_rw_latency": { "avgcount": 48199384,
"sum": 978600.144636000},
"subop": 73746080,
"subop_in_bytes": 2008774955062,
"subop_latency": { "avgcount": 73746080,
"sum": 346096.627047000},
"subop_w": 0,
"subop_w_in_bytes": 2008774955062,
"subop_w_latency": { "avgcount": 73746080,
"sum": 346096.627047000},
"subop_pull": 0,
"subop_pull_latency": { "avgcount": 0,
"sum": 0.000000000},
"subop_push": 0,
"subop_push_in_bytes": 0,
"subop_push_latency": { "avgcount": 0,
"sum": 0.000000000},
"pull": 0,
"push": 8255,
"push_out_bytes": 24792364384,
"push_in": 0,
"push_in_bytes": 0,
"recovery_ops": 8126,
"loadavg": 585,
"buffer_bytes": 0,
"numpg": 402,
"numpg_primary": 161,
"numpg_replica": 241,
"numpg_stray": 0,
"heartbeat_to_peers": 38,
"heartbeat_from_peers": 0,
"map_messages": 15262,
"map_message_epochs": 70160,
"map_message_epoch_dups": 116821,
"messages_delayed_for_map": 105,
"peering_latency": { "avgcount": 628,
"sum": 28166.821628000}},
"throttle-filestore_bytes": { "val": 0,
"max": 33554432,
"get": 0,
"get_sum": 0,
"get_or_fail_fail": 0,
"get_or_fail_success": 0,
"take": 116105073,
"take_sum": 3160504432839,
"put": 112261141,
"put_sum": 3160504432839,
"wait": { "avgcount": 12,
"sum": 0.097590000}},
"throttle-filestore_ops": { "val": 0,
"max": 300,
"get": 0,
"get_sum": 0,
"get_or_fail_fail": 0,
"get_or_fail_success": 0,
"take": 116105073,
"take_sum": 116105073,
"put": 112261141,
"put_sum": 116105073,
"wait": { "avgcount": 0,
"sum": 0.000000000}},
"throttle-msgr_dispatch_throttler-client": { "val": 0,
"max": 104857600,
"get": 75187594,
"get_sum": 978560902803,
"get_or_fail_fail": 0,
"get_or_fail_success": 0,
"take": 0,
"take_sum": 0,
"put": 75187594,
"put_sum": 978560902803,
"wait": { "avgcount": 0,
"sum": 0.000000000}},
"throttle-msgr_dispatch_throttler-cluster": { "val": 0,
"max": 104857600,
"get": 159487799,
"get_sum": 2119870334346,
"get_or_fail_fail": 0,
"get_or_fail_success": 0,
"take": 0,
"take_sum": 0,
"put": 159487799,
"put_sum": 2119870334346,
"wait": { "avgcount": 0,
"sum": 0.000000000}},
"throttle-msgr_dispatch_throttler-hb_back_server": { "val": 0,
"max": 104857600,
"get": 19958614,
"get_sum": 938054858,
"get_or_fail_fail": 0,
"get_or_fail_success": 0,
"take": 0,
"take_sum": 0,
"put": 19958614,
"put_sum": 938054858,
"wait": { "avgcount": 0,
"sum": 0.000000000}},
"throttle-msgr_dispatch_throttler-hb_front_server": { "val": 0,
"max": 104857600,
"get": 19958614,
"get_sum": 938054858,
"get_or_fail_fail": 0,
"get_or_fail_success": 0,
"take": 0,
"take_sum": 0,
"put": 19958614,
"put_sum": 938054858,
"wait": { "avgcount": 0,
"sum": 0.000000000}},
"throttle-msgr_dispatch_throttler-hbclient": { "val": 0,
"max": 104857600,
"get": 44608572,
"get_sum": 2096602884,
"get_or_fail_fail": 0,
"get_or_fail_success": 0,
"take": 0,
"take_sum": 0,
"put": 44608572,
"put_sum": 2096602884,
"wait": { "avgcount": 0,
"sum": 0.000000000}},
"throttle-osd_client_bytes": { "val": 0,
"max": 524288000,
"get": 74786210,
"get_sum": 978238232914,
"get_or_fail_fail": 0,
"get_or_fail_success": 0,
"take": 0,
"take_sum": 0,
"put": 117138891,
"put_sum": 978238232914,
"wait": { "avgcount": 0,
"sum": 0.000000000}},
"throttle-osd_client_messages": { "val": 17,
"max": 100,
"get": 75909689,
"get_sum": 75909689,
"get_or_fail_fail": 0,
"get_or_fail_success": 0,
"take": 0,
"take_sum": 0,
"put": 75909672,
"put_sum": 75909672,
"wait": { "avgcount": 478562,
"sum": 5665.119305000}}}
_______________________________________________
ceph-users mailing list
ceph-users@xxxxxxxxxxxxxx
http://lists.ceph.com/listinfo.cgi/ceph-users-ceph.com
-- Best Regards,Wheat
|
