libceph: add osd op counter metric support

The logic is the same with osdc/Objecter.cc in ceph in user space.

URL: https://tracker.ceph.com/issues/48053
Signed-off-by: Xiubo Li <xiubli@redhat.com>
Signed-off-by: Jeff Layton <jlayton@kernel.org>

diff --git a/include/linux/ceph/osd_client.h b/include/linux/ceph/osd_client.h
index 83fa08a0650710880a86cd3bf10fd53da2e7f1c2..24301513b186accb1d4ab0ebd6b24b4fff2614c1 100644 (file)
--- a/include/linux/ceph/osd_client.h
+++ b/include/linux/ceph/osd_client.h
@@ -339,6 +339,13 @@ struct ceph_osd_backoff {
        struct ceph_hobject_id *end;
 };
 
+struct ceph_osd_metric {
+       struct percpu_counter op_ops;
+       struct percpu_counter op_rmw;
+       struct percpu_counter op_r;
+       struct percpu_counter op_w;
+};
+
 #define CEPH_LINGER_ID_START   0xffff000000000000ULL
 
 struct ceph_osd_client {
@@ -371,6 +378,8 @@ struct ceph_osd_client {
        struct ceph_msgpool     msgpool_op;
        struct ceph_msgpool     msgpool_op_reply;
 
+       struct ceph_osd_metric  metric;
+
        struct workqueue_struct *notify_wq;
        struct workqueue_struct *completion_wq;
 };

diff --git a/net/ceph/debugfs.c b/net/ceph/debugfs.c
index 2110439f8a247cac8efa533958a2e1d1ccc1d673..af90019386abe844a263e389ed7f1764e9fa45eb 100644 (file)
--- a/net/ceph/debugfs.c
+++ b/net/ceph/debugfs.c
@@ -339,6 +339,16 @@ static void dump_backoffs(struct seq_file *s, struct ceph_osd *osd)
        mutex_unlock(&osd->lock);
 }
 
+static void dump_op_metric(struct seq_file *s, struct ceph_osd_client *osdc)
+{
+       struct ceph_osd_metric *m = &osdc->metric;
+
+       seq_printf(s, "  op_ops: %lld\n", percpu_counter_sum(&m->op_ops));
+       seq_printf(s, "  op_rmw: %lld\n", percpu_counter_sum(&m->op_rmw));
+       seq_printf(s, "  op_r:   %lld\n", percpu_counter_sum(&m->op_r));
+       seq_printf(s, "  op_w:   %lld\n", percpu_counter_sum(&m->op_w));
+}
+
 static int osdc_show(struct seq_file *s, void *pp)
 {
        struct ceph_client *client = s->private;
@@ -372,6 +382,9 @@ static int osdc_show(struct seq_file *s, void *pp)
        }
 
        up_read(&osdc->lock);
+
+       seq_puts(s, "OP METRIC:\n");
+       dump_op_metric(s, osdc);
        return 0;
 }
 

diff --git a/net/ceph/osd_client.c b/net/ceph/osd_client.c
index ff8624a7c96438df6b691cdc536a7b196919df95..6bf7c981874c76cdc98d33985fc4d61b78fdd76f 100644 (file)
--- a/net/ceph/osd_client.c
+++ b/net/ceph/osd_client.c
@@ -2424,6 +2424,21 @@ promote:
        goto again;
 }
 
+static void osd_acount_op_metric(struct ceph_osd_request *req)
+{
+       struct ceph_osd_metric *m = &req->r_osdc->metric;
+
+       percpu_counter_inc(&m->op_ops);
+
+       if ((req->r_flags & (CEPH_OSD_FLAG_READ | CEPH_OSD_FLAG_WRITE))
+           == (CEPH_OSD_FLAG_READ | CEPH_OSD_FLAG_WRITE))
+               percpu_counter_inc(&m->op_rmw);
+       if (req->r_flags & CEPH_OSD_FLAG_READ)
+               percpu_counter_inc(&m->op_r);
+       else if (req->r_flags & CEPH_OSD_FLAG_WRITE)
+               percpu_counter_inc(&m->op_w);
+}
+
 static void account_request(struct ceph_osd_request *req)
 {
        WARN_ON(req->r_flags & (CEPH_OSD_FLAG_ACK | CEPH_OSD_FLAG_ONDISK));
@@ -2434,6 +2449,8 @@ static void account_request(struct ceph_osd_request *req)
 
        req->r_start_stamp = jiffies;
        req->r_start_latency = ktime_get();
+
+       osd_acount_op_metric(req);
 }
 
 static void submit_request(struct ceph_osd_request *req, bool wrlocked)
@@ -5207,6 +5224,39 @@ void ceph_osdc_reopen_osds(struct ceph_osd_client *osdc)
        up_write(&osdc->lock);
 }
 
+static void ceph_metric_destroy(struct ceph_osd_metric *m)
+{
+       percpu_counter_destroy(&m->op_ops);
+       percpu_counter_destroy(&m->op_rmw);
+       percpu_counter_destroy(&m->op_r);
+       percpu_counter_destroy(&m->op_w);
+}
+
+static int ceph_metric_init(struct ceph_osd_metric *m)
+{
+       int ret;
+
+       memset(m, 0, sizeof(*m));
+
+       ret = percpu_counter_init(&m->op_ops, 0, GFP_NOIO);
+       if (ret)
+               return ret;
+       ret = percpu_counter_init(&m->op_rmw, 0, GFP_NOIO);
+       if (ret)
+               goto err;
+       ret = percpu_counter_init(&m->op_r, 0, GFP_NOIO);
+       if (ret)
+               goto err;
+       ret = percpu_counter_init(&m->op_w, 0, GFP_NOIO);
+       if (ret)
+               goto err;
+       return 0;
+
+err:
+       ceph_metric_destroy(m);
+       return ret;
+}
+
 /*
  * init, shutdown
  */
@@ -5259,6 +5309,9 @@ int ceph_osdc_init(struct ceph_osd_client *osdc, struct ceph_client *client)
        if (!osdc->completion_wq)
                goto out_notify_wq;
 
+       if (ceph_metric_init(&osdc->metric) < 0)
+               goto out_completion_wq;
+
        schedule_delayed_work(&osdc->timeout_work,
                              osdc->client->options->osd_keepalive_timeout);
        schedule_delayed_work(&osdc->osds_timeout_work,
@@ -5266,6 +5319,8 @@ int ceph_osdc_init(struct ceph_osd_client *osdc, struct ceph_client *client)
 
        return 0;
 
+out_completion_wq:
+       destroy_workqueue(osdc->completion_wq);
 out_notify_wq:
        destroy_workqueue(osdc->notify_wq);
 out_msgpool_reply:
@@ -5304,6 +5359,7 @@ void ceph_osdc_stop(struct ceph_osd_client *osdc)
        WARN_ON(atomic_read(&osdc->num_requests));
        WARN_ON(atomic_read(&osdc->num_homeless));
 
+       ceph_metric_destroy(&osdc->metric);
        ceph_osdmap_destroy(osdc->osdmap);
        mempool_destroy(osdc->req_mempool);
        ceph_msgpool_destroy(&osdc->msgpool_op);