From: Xiubo Li <xiubli@xxxxxxxxxx> The logic is the same with osdc/Objecter.cc in ceph in user space. URL: https://tracker.ceph.com/issues/48053 Signed-off-by: Xiubo Li <xiubli@xxxxxxxxxx> --- V3: - typo fixing about oring the _WRITE include/linux/ceph/osd_client.h | 9 ++++++ net/ceph/debugfs.c | 13 ++++++++ net/ceph/osd_client.c | 56 +++++++++++++++++++++++++++++++++ 3 files changed, 78 insertions(+) diff --git a/include/linux/ceph/osd_client.h b/include/linux/ceph/osd_client.h index 83fa08a06507..24301513b186 100644 --- a/include/linux/ceph/osd_client.h +++ b/include/linux/ceph/osd_client.h @@ -339,6 +339,13 @@ struct ceph_osd_backoff { struct ceph_hobject_id *end; }; +struct ceph_osd_metric { + struct percpu_counter op_ops; + struct percpu_counter op_rmw; + struct percpu_counter op_r; + struct percpu_counter op_w; +}; + #define CEPH_LINGER_ID_START 0xffff000000000000ULL struct ceph_osd_client { @@ -371,6 +378,8 @@ struct ceph_osd_client { struct ceph_msgpool msgpool_op; struct ceph_msgpool msgpool_op_reply; + struct ceph_osd_metric metric; + struct workqueue_struct *notify_wq; struct workqueue_struct *completion_wq; }; diff --git a/net/ceph/debugfs.c b/net/ceph/debugfs.c index 2110439f8a24..af90019386ab 100644 --- a/net/ceph/debugfs.c +++ b/net/ceph/debugfs.c @@ -339,6 +339,16 @@ static void dump_backoffs(struct seq_file *s, struct ceph_osd *osd) mutex_unlock(&osd->lock); } +static void dump_op_metric(struct seq_file *s, struct ceph_osd_client *osdc) +{ + struct ceph_osd_metric *m = &osdc->metric; + + seq_printf(s, " op_ops: %lld\n", percpu_counter_sum(&m->op_ops)); + seq_printf(s, " op_rmw: %lld\n", percpu_counter_sum(&m->op_rmw)); + seq_printf(s, " op_r: %lld\n", percpu_counter_sum(&m->op_r)); + seq_printf(s, " op_w: %lld\n", percpu_counter_sum(&m->op_w)); +} + static int osdc_show(struct seq_file *s, void *pp) { struct ceph_client *client = s->private; @@ -372,6 +382,9 @@ static int osdc_show(struct seq_file *s, void *pp) } up_read(&osdc->lock); + + seq_puts(s, "OP METRIC:\n"); + dump_op_metric(s, osdc); return 0; } diff --git a/net/ceph/osd_client.c b/net/ceph/osd_client.c index 7901ab6c79fd..0e6e127dd669 100644 --- a/net/ceph/osd_client.c +++ b/net/ceph/osd_client.c @@ -2424,6 +2424,21 @@ static void __submit_request(struct ceph_osd_request *req, bool wrlocked) goto again; } +static void osd_acount_op_metric(struct ceph_osd_request *req) +{ + struct ceph_osd_metric *m = &req->r_osdc->metric; + + percpu_counter_inc(&m->op_ops); + + if ((req->r_flags & (CEPH_OSD_FLAG_READ | CEPH_OSD_FLAG_WRITE)) + == (CEPH_OSD_FLAG_READ | CEPH_OSD_FLAG_WRITE)) + percpu_counter_inc(&m->op_rmw); + if (req->r_flags & CEPH_OSD_FLAG_READ) + percpu_counter_inc(&m->op_r); + else if (req->r_flags & CEPH_OSD_FLAG_WRITE) + percpu_counter_inc(&m->op_w); +} + static void account_request(struct ceph_osd_request *req) { WARN_ON(req->r_flags & (CEPH_OSD_FLAG_ACK | CEPH_OSD_FLAG_ONDISK)); @@ -2434,6 +2449,8 @@ static void account_request(struct ceph_osd_request *req) req->r_start_stamp = jiffies; req->r_start_latency = ktime_get(); + + osd_acount_op_metric(req); } static void submit_request(struct ceph_osd_request *req, bool wrlocked) @@ -5205,6 +5222,39 @@ void ceph_osdc_reopen_osds(struct ceph_osd_client *osdc) up_write(&osdc->lock); } +static void ceph_metric_destroy(struct ceph_osd_metric *m) +{ + percpu_counter_destroy(&m->op_ops); + percpu_counter_destroy(&m->op_rmw); + percpu_counter_destroy(&m->op_r); + percpu_counter_destroy(&m->op_w); +} + +static int ceph_metric_init(struct ceph_osd_metric *m) +{ + int ret; + + memset(m, 0, sizeof(*m)); + + ret = percpu_counter_init(&m->op_ops, 0, GFP_NOIO); + if (ret) + return ret; + ret = percpu_counter_init(&m->op_rmw, 0, GFP_NOIO); + if (ret) + goto err; + ret = percpu_counter_init(&m->op_r, 0, GFP_NOIO); + if (ret) + goto err; + ret = percpu_counter_init(&m->op_w, 0, GFP_NOIO); + if (ret) + goto err; + return 0; + +err: + ceph_metric_destroy(m); + return ret; +} + /* * init, shutdown */ @@ -5257,6 +5307,9 @@ int ceph_osdc_init(struct ceph_osd_client *osdc, struct ceph_client *client) if (!osdc->completion_wq) goto out_notify_wq; + if (ceph_metric_init(&osdc->metric) < 0) + goto out_completion_wq; + schedule_delayed_work(&osdc->timeout_work, osdc->client->options->osd_keepalive_timeout); schedule_delayed_work(&osdc->osds_timeout_work, @@ -5264,6 +5317,8 @@ int ceph_osdc_init(struct ceph_osd_client *osdc, struct ceph_client *client) return 0; +out_completion_wq: + destroy_workqueue(osdc->completion_wq); out_notify_wq: destroy_workqueue(osdc->notify_wq); out_msgpool_reply: @@ -5302,6 +5357,7 @@ void ceph_osdc_stop(struct ceph_osd_client *osdc) WARN_ON(atomic_read(&osdc->num_requests)); WARN_ON(atomic_read(&osdc->num_homeless)); + ceph_metric_destroy(&osdc->metric); ceph_osdmap_destroy(osdc->osdmap); mempool_destroy(osdc->req_mempool); ceph_msgpool_destroy(&osdc->msgpool_op); -- 2.27.0
