On Tue, Mar 3, 2020 at 12:29 PM Leon Romanovsky [off-list ref] wrote:

On Fri, Feb 21, 2020 at 11:47:03AM +0100, Jack Wang wrote:

quoted

From: Jack Wang <jinpu.wang@cloud.ionos.com>

This introduces set of functions used on client side to account
statistics of RDMA data sent/received, amount of IOs inflight,
latency, cpu migrations, etc.  Almost all statistics are collected
using percpu variables.

Signed-off-by: Danil Kipnis <redacted>
Signed-off-by: Jack Wang <jinpu.wang@cloud.ionos.com>
---
 drivers/infiniband/ulp/rtrs/rtrs-clt-stats.c | 205 +++++++++++++++++++
 1 file changed, 205 insertions(+)
 create mode 100644 drivers/infiniband/ulp/rtrs/rtrs-clt-stats.c

diff --git a/drivers/infiniband/ulp/rtrs/rtrs-clt-stats.c b/drivers/infiniband/ulp/rtrs/rtrs-clt-stats.c
new file mode 100644
index 000000000000..3f556b884a4e
--- /dev/null
+++ b/drivers/infiniband/ulp/rtrs/rtrs-clt-stats.c

@@ -0,0 +1,205 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * RDMA Transport Layer
+ *
+ * Copyright (c) 2014 - 2018 ProfitBricks GmbH. All rights reserved.
+ * Copyright (c) 2018 - 2019 1&1 IONOS Cloud GmbH. All rights reserved.
+ * Copyright (c) 2019 - 2020 1&1 IONOS SE. All rights reserved.
+ */
+#undef pr_fmt
+#define pr_fmt(fmt) KBUILD_MODNAME " L" __stringify(__LINE__) ": " fmt
+
+#include "rtrs-clt.h"
+
+void rtrs_clt_update_wc_stats(struct rtrs_clt_con *con)
+{
+     struct rtrs_clt_sess *sess = to_clt_sess(con->c.sess);
+     struct rtrs_clt_stats *stats = &sess->stats;
+     struct rtrs_clt_stats_pcpu *s;
+     int cpu;
+
+     cpu = raw_smp_processor_id();
+     s = this_cpu_ptr(stats->pcpu_stats);
+     if (unlikely(con->cpu != cpu)) {
+             s->cpu_migr.to++;
+
+             /* Careful here, override s pointer */
+             s = per_cpu_ptr(stats->pcpu_stats, con->cpu);
+             atomic_inc(&s->cpu_migr.from);
+     }
+}
+
+void rtrs_clt_inc_failover_cnt(struct rtrs_clt_stats *stats)
+{
+     struct rtrs_clt_stats_pcpu *s;
+
+     s = this_cpu_ptr(stats->pcpu_stats);
+     s->rdma.failover_cnt++;
+}
+
+int rtrs_clt_stats_migration_cnt_to_str(struct rtrs_clt_stats *stats,
+                                      char *buf, size_t len)
+{
+     struct rtrs_clt_stats_pcpu *s;
+
+     size_t used;
+     int cpu;
+
+     used = scnprintf(buf, len, "    ");
+     for_each_possible_cpu(cpu)
+             used += scnprintf(buf + used, len - used, " CPU%u", cpu);
+
+     used += scnprintf(buf + used, len - used, "\nfrom:");
+     for_each_possible_cpu(cpu) {
+             s = per_cpu_ptr(stats->pcpu_stats, cpu);
+             used += scnprintf(buf + used, len - used, " %d",
+                               atomic_read(&s->cpu_migr.from));
+     }
+
+     used += scnprintf(buf + used, len - used, "\nto  :");
+     for_each_possible_cpu(cpu) {
+             s = per_cpu_ptr(stats->pcpu_stats, cpu);
+             used += scnprintf(buf + used, len - used, " %d",
+                               s->cpu_migr.to);
+     }
+     used += scnprintf(buf + used, len - used, "\n");
+
+     return used;
+}
+
+int rtrs_clt_stats_reconnects_to_str(struct rtrs_clt_stats *stats, char *buf,
+                                   size_t len)
+{
+     return scnprintf(buf, len, "%d %d\n",
+                      stats->reconnects.successful_cnt,
+                      stats->reconnects.fail_cnt);

How will user know that first value is successful_cnt and second fail_cnt?

quoted

+}
+
+ssize_t rtrs_clt_stats_rdma_to_str(struct rtrs_clt_stats *stats,
+                                 char *page, size_t len)
+{
+     struct rtrs_clt_stats_rdma sum;
+     struct rtrs_clt_stats_rdma *r;
+     int cpu;
+
+     memset(&sum, 0, sizeof(sum));
+
+     for_each_possible_cpu(cpu) {
+             r = &per_cpu_ptr(stats->pcpu_stats, cpu)->rdma;
+
+             sum.dir[READ].cnt         += r->dir[READ].cnt;
+             sum.dir[READ].size_total  += r->dir[READ].size_total;
+             sum.dir[WRITE].cnt        += r->dir[WRITE].cnt;
+             sum.dir[WRITE].size_total += r->dir[WRITE].size_total;
+             sum.failover_cnt          += r->failover_cnt;
+     }
+
+     return scnprintf(page, len, "%llu %llu %llu %llu %u %llu\n",
+                      sum.dir[READ].cnt, sum.dir[READ].size_total,
+                      sum.dir[WRITE].cnt, sum.dir[WRITE].size_total,
+                      atomic_read(&stats->inflight), sum.failover_cnt);

Same question.

Thanks

Hi Leon,

For all the sysfs entries, we documented in Documentation, in your
cases, it's in Documentation/ABI/testing/sysfs-class-rtrs-client,
in Patch 14.

Thanks!