>-----Original Message----- >From: linux-rdma-owner@xxxxxxxxxxxxxxx [mailto:linux-rdma- >owner@xxxxxxxxxxxxxxx] On Behalf Of ira.weiny@xxxxxxxxx >Sent: Monday, February 11, 2019 10:15 AM >To: linux-rdma@xxxxxxxxxxxxxxx >Cc: Hal Rosenstock <hal@xxxxxxxxxxxxxxxxxx>; Doug Ledford ><dledford@xxxxxxxxxx>; Jason Gunthorpe <jgg@xxxxxxxx>; Steven Rostedt ><rostedt@xxxxxxxxxxx>; Ingo Molnar <mingo@xxxxxxxxxx>; Alexei >Starovoitov <ast@xxxxxxxxxx>; Daniel Borkmann <daniel@xxxxxxxxxxxxx>; >Leon Romanovsky <leon@xxxxxxxxxx>; Weiny, Ira <ira.weiny@xxxxxxxxx> >Subject: [PATCH for-next V4 1/6] IB/MAD: Add send path trace points > >From: Ira Weiny <ira.weiny@xxxxxxxxx> > >Use the standard Linux trace mechanism to trace MADs being sent. 4 trace >points are added, when the MAD is posted to the qp, when the MAD is >completed, if a MAD is resent, and when the MAD completes in error. > >Signed-off-by: Ira Weiny <ira.weiny@xxxxxxxxx> > >--- > >Changes since v3: > Change dev_name to dev_index > >Changes since v2: > Change license text to SPDX tag > Change dev_name to string from array > Reorder fields for more compact ring buffer utilization > Use a defined roce address type for safer memcpy/memset > >Changes since v1: > Update MAINTAINERS with tracing file > > MAINTAINERS | 1 + > drivers/infiniband/core/mad.c | 73 ++++++++++++++- > include/trace/events/ib_mad.h | 164 >++++++++++++++++++++++++++++++++++ > 3 files changed, 237 insertions(+), 1 deletion(-) > create mode 100644 include/trace/events/ib_mad.h > >diff --git a/MAINTAINERS b/MAINTAINERS >index 8c68de3cfd80..9254bd40f1ae 100644 >--- a/MAINTAINERS >+++ b/MAINTAINERS >@@ -7491,6 +7491,7 @@ F: drivers/infiniband/ > F: include/uapi/linux/if_infiniband.h > F: include/uapi/rdma/ > F: include/rdma/ >+F: include/trace/events/ib_mad.h > > INGENIC JZ4780 DMA Driver > M: Zubair Lutfullah Kakakhel <Zubair.Kakakhel@xxxxxxxxxx> >diff --git a/drivers/infiniband/core/mad.c b/drivers/infiniband/core/mad.c >index 7870823bac47..e166d584b80c 100644 >--- a/drivers/infiniband/core/mad.c >+++ b/drivers/infiniband/core/mad.c >@@ -3,7 +3,7 @@ > * Copyright (c) 2005 Intel Corporation. All rights reserved. > * Copyright (c) 2005 Mellanox Technologies Ltd. All rights reserved. > * Copyright (c) 2009 HNR Consulting. All rights reserved. >- * Copyright (c) 2014 Intel Corporation. All rights reserved. >+ * Copyright (c) 2014,2018 Intel Corporation. All rights reserved. > * > * This software is available to you under a choice of one of two > * licenses. You may choose to be licensed under the terms of the GNU >@@ -51,6 +51,51 @@ > #include "opa_smi.h" > #include "agent.h" > >+struct rdma_mad_trace_addr { >+ u32 dlid; >+ struct roce_ah_attr roce_addr; >+ u8 sl; >+ u16 pkey; >+ u32 rqpn; >+ u32 rqkey; >+}; >+static void trace_create_mad_addr(struct ib_device *dev, u8 pnum, >+ struct ib_ud_wr *wr, >+ struct rdma_mad_trace_addr *addr) >+{ >+ struct rdma_ah_attr attr; >+ >+ memset(&attr, 0, sizeof(attr)); >+ rdma_query_ah(wr->ah, &attr); >+ >+ /* These are common */ >+ addr->sl = attr.sl; >+ ib_query_pkey(dev, pnum, wr->pkey_index, &addr->pkey); >+ addr->rqpn = wr->remote_qpn; >+ addr->rqkey = wr->remote_qkey; >+ >+ switch (attr.type) { >+ case RDMA_AH_ATTR_TYPE_IB: >+ addr->dlid = attr.ib.dlid; >+ memset(&addr->roce_addr, 0, sizeof(addr->roce_addr)); >+ break; >+ case RDMA_AH_ATTR_TYPE_OPA: >+ addr->dlid = attr.opa.dlid; >+ memset(&addr->roce_addr, 0, sizeof(addr->roce_addr)); >+ break; >+ case RDMA_AH_ATTR_TYPE_ROCE: >+ addr->dlid = 0; >+ memcpy(&addr->roce_addr, &attr.roce, sizeof(addr- >>roce_addr)); >+ break; >+ case RDMA_AH_ATTR_TYPE_UNDEFINED: >+ addr->dlid = 0; >+ memset(&addr->roce_addr, 0, sizeof(addr->roce_addr)); >+ break; >+ } >+} >+#define CREATE_TRACE_POINTS >+#include <trace/events/ib_mad.h> >+ > static int mad_sendq_size = IB_MAD_QP_SEND_SIZE; > static int mad_recvq_size = IB_MAD_QP_RECV_SIZE; > >@@ -1223,6 +1268,14 @@ int ib_send_mad(struct ib_mad_send_wr_private >*mad_send_wr) > > spin_lock_irqsave(&qp_info->send_queue.lock, flags); > if (qp_info->send_queue.count < qp_info->send_queue.max_active) >{ >+ if (trace_ib_mad_ib_send_mad_enabled()) { >+ struct rdma_mad_trace_addr addr; >+ >+ trace_create_mad_addr(qp_info->port_priv->device, >+ qp_info->port_priv->port_num, >+ &mad_send_wr->send_wr, >&addr); >+ trace_ib_mad_ib_send_mad(mad_send_wr, &addr); >+ } > ret = ib_post_send(mad_agent->qp, &mad_send_wr- >>send_wr.wr, > NULL); > list = &qp_info->send_queue.list; Ira, Are the trace_ib_mad_ib_send_mad_enabled() (and the other locations) actually necessary? When you populate the TP_STRUCT_entry values, you can call a function to get information (i.e. see the hfi1/trac_ibhdrs.h usage of hfi1_trace_packet_hdr_len(). You may be able to call the trace_create_mad_addr() there (will probably need to pass qp_info into trac_ib_mad_ib_send_mad()) rather than having to have this enable check in the code. Mike >@@ -2496,6 +2549,8 @@ static void ib_mad_send_done(struct ib_cq *cq, >struct ib_wc *wc) > send_queue = mad_list->mad_queue; > qp_info = send_queue->qp_info; > >+ trace_ib_mad_send_done_handler(mad_send_wr, wc); >+ > retry: > ib_dma_unmap_single(mad_send_wr->send_buf.mad_agent- >>device, > mad_send_wr->header_mapping, >@@ -2527,6 +2582,14 @@ static void ib_mad_send_done(struct ib_cq *cq, >struct ib_wc *wc) > ib_mad_complete_send_wr(mad_send_wr, &mad_send_wc); > > if (queued_send_wr) { >+ if (trace_ib_mad_send_done_resend_enabled()) { >+ struct rdma_mad_trace_addr addr; >+ >+ trace_create_mad_addr(qp_info->port_priv->device, >+ qp_info->port_priv->port_num, >+ &mad_send_wr->send_wr, >&addr); >+ > trace_ib_mad_send_done_resend(queued_send_wr, &addr); >+ } > ret = ib_post_send(qp_info->qp, &queued_send_wr- >>send_wr.wr, > NULL); > if (ret) { >@@ -2574,6 +2637,14 @@ static bool ib_mad_send_error(struct >ib_mad_port_private *port_priv, > if (mad_send_wr->retry) { > /* Repost send */ > mad_send_wr->retry = 0; >+ if (trace_ib_mad_error_handler_enabled()) { >+ struct rdma_mad_trace_addr addr; >+ >+ trace_create_mad_addr(qp_info->port_priv- >>device, >+ qp_info->port_priv- >>port_num, >+ &mad_send_wr->send_wr, >&addr); >+ trace_ib_mad_error_handler(mad_send_wr, >&addr); >+ } > ret = ib_post_send(qp_info->qp, &mad_send_wr- >>send_wr.wr, > NULL); > if (!ret) >diff --git a/include/trace/events/ib_mad.h b/include/trace/events/ib_mad.h >new file mode 100644 >index 000000000000..30f72a29cccb >--- /dev/null >+++ b/include/trace/events/ib_mad.h >@@ -0,0 +1,164 @@ >+// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB >+ >+/* >+ * Copyright (c) 2018 Intel Corporation. All rights reserved. >+ */ >+ >+#undef TRACE_SYSTEM >+#define TRACE_SYSTEM ib_mad >+ >+#if !defined(_TRACE_IB_MAD_H) || defined(TRACE_HEADER_MULTI_READ) >+#define _TRACE_IB_MAD_H >+ >+#include <linux/tracepoint.h> >+#include <rdma/ib_mad.h> >+ >+DECLARE_EVENT_CLASS(ib_mad_send_template, >+ TP_PROTO(struct ib_mad_send_wr_private *wr, struct >rdma_mad_trace_addr *addr), >+ TP_ARGS(wr, addr), >+ >+ TP_STRUCT__entry( >+ __field(u8, base_version) >+ __field(u8, mgmt_class) >+ __field(u8, class_version) >+ __field(u8, port_num) >+ __field(u32, qp_num) >+ __field(u8, method) >+ __field(u8, sl) >+ __field(u16, attr_id) >+ __field(u32, attr_mod) >+ __field(u64, wrtid) >+ __field(u64, tid) >+ __field(u16, status) >+ __field(u16, class_specific) >+ __field(u32, length) >+ __field(u32, dlid) >+ __field(u32, rqpn) >+ __field(u32, rqkey) >+ __field(u32, dev_index) >+ __field(void *, agent_priv) >+ __field(unsigned long, timeout) >+ __field(int, retries_left) >+ __field(int, max_retries) >+ __field(int, retry) >+ __field(u16, pkey) >+ ), >+ >+ TP_fast_assign( >+ __entry->dev_index = wr->mad_agent_priv->agent.device- >>index; >+ __entry->port_num = wr->mad_agent_priv- >>agent.port_num; >+ __entry->qp_num = wr->mad_agent_priv->qp_info->qp- >>qp_num; >+ __entry->agent_priv = wr->mad_agent_priv; >+ __entry->wrtid = wr->tid; >+ __entry->max_retries = wr->max_retries; >+ __entry->retries_left = wr->retries_left; >+ __entry->retry = wr->retry; >+ __entry->timeout = wr->timeout; >+ __entry->length = wr->send_buf.hdr_len + >+ wr->send_buf.data_len; >+ __entry->base_version = ((struct ib_mad_hdr *)wr- >>send_buf.mad)->base_version; >+ __entry->mgmt_class = ((struct ib_mad_hdr *)wr- >>send_buf.mad)->mgmt_class; >+ __entry->class_version = ((struct ib_mad_hdr *)wr- >>send_buf.mad)->class_version; >+ __entry->method = ((struct ib_mad_hdr *)wr- >>send_buf.mad)->method; >+ __entry->status = ((struct ib_mad_hdr *)wr- >>send_buf.mad)->status; >+ __entry->class_specific = ((struct ib_mad_hdr *)wr- >>send_buf.mad)->class_specific; >+ __entry->tid = ((struct ib_mad_hdr *)wr->send_buf.mad)- >>tid; >+ __entry->attr_id = ((struct ib_mad_hdr *)wr- >>send_buf.mad)->attr_id; >+ __entry->attr_mod = ((struct ib_mad_hdr *)wr- >>send_buf.mad)->attr_mod; >+ __entry->dlid = addr->dlid; >+ __entry->sl = addr->sl; >+ __entry->pkey = addr->pkey; >+ __entry->rqpn = addr->rqpn; >+ __entry->rqkey = addr->rqkey; >+ ), >+ >+ TP_printk("%d:%d QP%d agent %p: " \ >+ "wrtid 0x%llx; %d/%d retries(%d); timeout %lu length %d : >hdr : " \ >+ "base_ver 0x%x class 0x%x class_ver 0x%x method 0x%x " \ >+ "status 0x%x class_specific 0x%x tid 0x%llx attr_id 0x%x >attr_mod 0x%x " \ >+ " => dlid 0x%08x sl %d pkey 0x%x rpqn 0x%x rqpkey 0x%x", >+ __entry->dev_index, __entry->port_num, __entry- >>qp_num, >+ __entry->agent_priv, be64_to_cpu(__entry->wrtid), >+ __entry->retries_left, __entry->max_retries, >+ __entry->retry, __entry->timeout, __entry->length, >+ __entry->base_version, __entry->mgmt_class, __entry- >>class_version, >+ __entry->method, be16_to_cpu(__entry->status), >+ be16_to_cpu(__entry->class_specific), >+ be64_to_cpu(__entry->tid), be16_to_cpu(__entry->attr_id), >+ be32_to_cpu(__entry->attr_mod), >+ be32_to_cpu(__entry->dlid), __entry->sl, __entry->pkey, >__entry->rqpn, >+ __entry->rqkey >+ ) >+); >+ >+DEFINE_EVENT(ib_mad_send_template, ib_mad_error_handler, >+ TP_PROTO(struct ib_mad_send_wr_private *wr, struct >rdma_mad_trace_addr *addr), >+ TP_ARGS(wr, addr)); >+DEFINE_EVENT(ib_mad_send_template, ib_mad_ib_send_mad, >+ TP_PROTO(struct ib_mad_send_wr_private *wr, struct >rdma_mad_trace_addr *addr), >+ TP_ARGS(wr, addr)); >+DEFINE_EVENT(ib_mad_send_template, ib_mad_send_done_resend, >+ TP_PROTO(struct ib_mad_send_wr_private *wr, struct >rdma_mad_trace_addr *addr), >+ TP_ARGS(wr, addr)); >+ >+TRACE_EVENT(ib_mad_send_done_handler, >+ TP_PROTO(struct ib_mad_send_wr_private *wr, struct ib_wc *wc), >+ TP_ARGS(wr, wc), >+ >+ TP_STRUCT__entry( >+ __field(u8, port_num) >+ __field(u8, base_version) >+ __field(u8, mgmt_class) >+ __field(u8, class_version) >+ __field(u32, qp_num) >+ __field(u64, wrtid) >+ __field(u16, status) >+ __field(u16, wc_status) >+ __field(u32, length) >+ __field(void *, agent_priv) >+ __field(unsigned long, timeout) >+ __field(u32, dev_index) >+ __field(int, retries_left) >+ __field(int, max_retries) >+ __field(int, retry) >+ __field(u8, method) >+ ), >+ >+ TP_fast_assign( >+ __entry->dev_index = wr->mad_agent_priv->agent.device- >>index; >+ __entry->port_num = wr->mad_agent_priv- >>agent.port_num; >+ __entry->qp_num = wr->mad_agent_priv->qp_info->qp- >>qp_num; >+ __entry->agent_priv = wr->mad_agent_priv; >+ __entry->wrtid = wr->tid; >+ __entry->max_retries = wr->max_retries; >+ __entry->retries_left = wr->retries_left; >+ __entry->retry = wr->retry; >+ __entry->timeout = wr->timeout; >+ __entry->base_version = ((struct ib_mad_hdr *)wr- >>send_buf.mad)->base_version; >+ __entry->mgmt_class = ((struct ib_mad_hdr *)wr- >>send_buf.mad)->mgmt_class; >+ __entry->class_version = ((struct ib_mad_hdr *)wr- >>send_buf.mad)->class_version; >+ __entry->method = ((struct ib_mad_hdr *)wr- >>send_buf.mad)->method; >+ __entry->status = ((struct ib_mad_hdr *)wr- >>send_buf.mad)->status; >+ __entry->wc_status = wc->status; >+ __entry->length = wc->byte_len; >+ ), >+ >+ TP_printk("%d:%d QP%d : SEND WC Status %d : agent %p: " \ >+ "wrtid 0x%llx %d/%d retries(%d) timeout %lu length %d: hdr : >" \ >+ "base_ver 0x%x class 0x%x class_ver 0x%x method 0x%x " \ >+ "status 0x%x", >+ __entry->dev_index, __entry->port_num, __entry- >>qp_num, >+ __entry->wc_status, >+ __entry->agent_priv, be64_to_cpu(__entry->wrtid), >+ __entry->retries_left, __entry->max_retries, >+ __entry->retry, __entry->timeout, >+ __entry->length, >+ __entry->base_version, __entry->mgmt_class, __entry- >>class_version, >+ __entry->method, be16_to_cpu(__entry->status) >+ ) >+); >+ >+ >+#endif /* _TRACE_IB_MAD_H */ >+ >+#include <trace/define_trace.h> >-- >2.20.1
