On 7/4/2018 8:57 AM, Leon Romanovsky wrote:
> From: Jason Gunthorpe <jgg@xxxxxxxxxxxx>
>
> The internal flag IP_BASED_GIDS was added to a field that was being used
> to hold the port Info CapabilityMask without considering the effects
> this will have. Since most drivers just use the value from the HW MAD
> it means IP_BASED_GIDS will also become set on any HW that sets the IBA
> flag IsOtherLocalChangesNoticeSupported - which is not intended.
>
> Fix this by keeping port_cap_flags only for the IBA CapabilityMask value
> and store unrelated flags externally. Move the bit definitions for this to
> ib_mad.h to make it clear what is happening.
>
> To keep the uAPI unchanged define a new set of flags in the uapi header
> that are only used by ib_uverbs_query_port_resp.port_cap_flags which match
> the current flags supported in rdma-core, and the values exposed by the
> current kernel.
>
> Fixes: b4a26a27287a ("IB: Report using RoCE IP based gids in port caps")
> Signed-off-by: Jason Gunthorpe <jgg@xxxxxxxxxxxx>
> Signed-off-by: Artemy Kovalyov <artemyko@xxxxxxxxxxxx>
> Signed-off-by: Leon Romanovsky <leonro@xxxxxxxxxxxx>
> ---
> drivers/infiniband/core/uverbs_cmd.c | 23 ++++++++++++++-
> drivers/infiniband/hw/bnxt_re/ib_verbs.c | 4 +--
> drivers/infiniband/hw/mlx4/main.c | 3 +-
> drivers/infiniband/hw/mlx5/main.c | 4 +--
> drivers/infiniband/hw/ocrdma/ocrdma_verbs.c | 9 +++---
> drivers/infiniband/hw/qedr/verbs.c | 2 +-
> drivers/infiniband/hw/qib/qib_verbs.h | 3 --
> drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.c | 3 +-
> include/rdma/ib_mad.h | 33 +++++++++++++++++++++
> include/rdma/ib_verbs.h | 31 ++------------------
> include/uapi/rdma/ib_user_ioctl_verbs.h | 38 +++++++++++++++++++++++++
> include/uapi/rdma/ib_user_verbs.h | 2 +-
> 12 files changed, 110 insertions(+), 45 deletions(-)
>
> diff --git a/drivers/infiniband/core/uverbs_cmd.c b/drivers/infiniband/core/uverbs_cmd.c
> index 60e6ee5bb5eb..a5e31af634f3 100644
> --- a/drivers/infiniband/core/uverbs_cmd.c
> +++ b/drivers/infiniband/core/uverbs_cmd.c
> @@ -241,6 +241,27 @@ ssize_t ib_uverbs_query_device(struct ib_uverbs_file *file,
> return in_len;
> }
>
> +/*
> + * ib_uverbs_query_port_resp.port_cap_flags started out as just a copy of the
> + * PortInfo CapabilitMask, but was extended with unique bits.
typo: CapabilitMask -> CapabilityMask
> + */
> +static u32 make_port_cap_flags(const struct ib_port_attr *attr)
> +{
> + u32 res;
> +
> + /* All IBA CapabilityMask bits are passed through here, except bit 26,
> + * which is overridden with IP_BASED_GIDS. This is due to a historical
> + * mistake in the implementation of IP_BASED_GIDS. Otherwise all other
> + * bits match the IBA definition across all kernel versions.
> + */
> + res = attr->port_cap_flags & ~(u32)IB_UVERBS_PCF_IP_BASED_GIDS;
> +
> + if (attr->ip_gids)
> + res |= IB_UVERBS_PCF_IP_BASED_GIDS;
> +
> + return res;
> +}
> +
> ssize_t ib_uverbs_query_port(struct ib_uverbs_file *file,
> struct ib_device *ib_dev,
> const char __user *buf,
> @@ -267,7 +288,7 @@ ssize_t ib_uverbs_query_port(struct ib_uverbs_file *file,
> resp.max_mtu = attr.max_mtu;
> resp.active_mtu = attr.active_mtu;
> resp.gid_tbl_len = attr.gid_tbl_len;
> - resp.port_cap_flags = attr.port_cap_flags;
> + resp.port_cap_flags = make_port_cap_flags(&attr);
> resp.max_msg_sz = attr.max_msg_sz;
> resp.bad_pkey_cntr = attr.bad_pkey_cntr;
> resp.qkey_viol_cntr = attr.qkey_viol_cntr;
> diff --git a/drivers/infiniband/hw/bnxt_re/ib_verbs.c b/drivers/infiniband/hw/bnxt_re/ib_verbs.c
> index 6c0c6d3426e0..492c750f7ed6 100644
> --- a/drivers/infiniband/hw/bnxt_re/ib_verbs.c
> +++ b/drivers/infiniband/hw/bnxt_re/ib_verbs.c
> @@ -244,8 +244,8 @@ int bnxt_re_query_port(struct ib_device *ibdev, u8 port_num,
> port_attr->gid_tbl_len = dev_attr->max_sgid;
> port_attr->port_cap_flags = IB_PORT_CM_SUP | IB_PORT_REINIT_SUP |
> IB_PORT_DEVICE_MGMT_SUP |
> - IB_PORT_VENDOR_CLASS_SUP |
> - IB_PORT_IP_BASED_GIDS;
> + IB_PORT_VENDOR_CLASS_SUP;
> + port_attr->ip_gids = true;
>
> port_attr->max_msg_sz = (u32)BNXT_RE_MAX_MR_SIZE_LOW;
> port_attr->bad_pkey_cntr = 0;
> diff --git a/drivers/infiniband/hw/mlx4/main.c b/drivers/infiniband/hw/mlx4/main.c
> index 5bc522ca9431..ca0f1ee26091 100644
> --- a/drivers/infiniband/hw/mlx4/main.c
> +++ b/drivers/infiniband/hw/mlx4/main.c
> @@ -762,7 +762,8 @@ static int eth_link_query_port(struct ib_device *ibdev, u8 port,
> IB_WIDTH_4X : IB_WIDTH_1X;
> props->active_speed = (((u8 *)mailbox->buf)[5] == 0x20 /*56Gb*/) ?
> IB_SPEED_FDR : IB_SPEED_QDR;
> - props->port_cap_flags = IB_PORT_CM_SUP | IB_PORT_IP_BASED_GIDS;
> + props->port_cap_flags = IB_PORT_CM_SUP;
> + props->ip_gids = true;
> props->gid_tbl_len = mdev->dev->caps.gid_table_len[port];
> props->max_msg_sz = mdev->dev->caps.max_msg_sz;
> props->pkey_tbl_len = 1;
> diff --git a/drivers/infiniband/hw/mlx5/main.c b/drivers/infiniband/hw/mlx5/main.c
> index e874a07f6ab9..d178d80155cc 100644
> --- a/drivers/infiniband/hw/mlx5/main.c
> +++ b/drivers/infiniband/hw/mlx5/main.c
> @@ -419,8 +419,8 @@ static int mlx5_query_port_roce(struct ib_device *device, u8 port_num,
> translate_eth_proto_oper(eth_prot_oper, &props->active_speed,
> &props->active_width);
>
> - props->port_cap_flags |= IB_PORT_CM_SUP;
> - props->port_cap_flags |= IB_PORT_IP_BASED_GIDS;
> + props->port_cap_flags |= IB_PORT_CM_SUP;
> + props->ip_gids = true;
>
> props->gid_tbl_len = MLX5_CAP_ROCE(dev->mdev,
> roce_address_table_size);
> diff --git a/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c b/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c
> index 1f057fdb3a8c..9d0431e01dce 100644
> --- a/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c
> +++ b/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c
> @@ -197,11 +197,10 @@ int ocrdma_query_port(struct ib_device *ibdev,
> props->sm_lid = 0;
> props->sm_sl = 0;
> props->state = port_state;
> - props->port_cap_flags =
> - IB_PORT_CM_SUP |
> - IB_PORT_REINIT_SUP |
> - IB_PORT_DEVICE_MGMT_SUP | IB_PORT_VENDOR_CLASS_SUP |
> - IB_PORT_IP_BASED_GIDS;
> + props->port_cap_flags = IB_PORT_CM_SUP | IB_PORT_REINIT_SUP |
> + IB_PORT_DEVICE_MGMT_SUP |
> + IB_PORT_VENDOR_CLASS_SUP;
> + props->ip_gids = true;
> props->gid_tbl_len = OCRDMA_MAX_SGID;
> props->pkey_tbl_len = 1;
> props->bad_pkey_cntr = 0;
> diff --git a/drivers/infiniband/hw/qedr/verbs.c b/drivers/infiniband/hw/qedr/verbs.c
> index 0c41d54f586b..b82c5d5fb0e3 100644
> --- a/drivers/infiniband/hw/qedr/verbs.c
> +++ b/drivers/infiniband/hw/qedr/verbs.c
> @@ -225,7 +225,7 @@ int qedr_query_port(struct ib_device *ibdev, u8 port, struct ib_port_attr *attr)
> attr->lmc = 0;
> attr->sm_lid = 0;
> attr->sm_sl = 0;
> - attr->port_cap_flags = IB_PORT_IP_BASED_GIDS;
> + attr->ip_gids = true;
> if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
> attr->gid_tbl_len = 1;
> attr->pkey_tbl_len = 1;
> diff --git a/drivers/infiniband/hw/qib/qib_verbs.h b/drivers/infiniband/hw/qib/qib_verbs.h
> index f9a46768a19a..e72562a8959a 100644
> --- a/drivers/infiniband/hw/qib/qib_verbs.h
> +++ b/drivers/infiniband/hw/qib/qib_verbs.h
> @@ -78,9 +78,6 @@ struct qib_verbs_txreq;
>
> #define QIB_VENDOR_IPG cpu_to_be16(0xFFA0)
>
> -/* XXX Should be defined in ib_verbs.h enum ib_port_cap_flags */
> -#define IB_PORT_OTHER_LOCAL_CHANGES_SUP (1 << 26)
> -
> #define IB_DEFAULT_GID_PREFIX cpu_to_be64(0xfe80000000000000ULL)
>
> /* Values for set/get portinfo VLCap OperationalVLs */
> diff --git a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.c b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.c
> index 816cc285daf6..b65d10b0a875 100644
> --- a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.c
> +++ b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.c
> @@ -155,7 +155,8 @@ int pvrdma_query_port(struct ib_device *ibdev, u8 port,
> props->gid_tbl_len = resp->attrs.gid_tbl_len;
> props->port_cap_flags =
> pvrdma_port_cap_flags_to_ib(resp->attrs.port_cap_flags);
> - props->port_cap_flags |= IB_PORT_CM_SUP | IB_PORT_IP_BASED_GIDS;
> + props->port_cap_flags |= IB_PORT_CM_SUP;
> + props->ip_gids = true;
> props->max_msg_sz = resp->attrs.max_msg_sz;
> props->bad_pkey_cntr = resp->attrs.bad_pkey_cntr;
> props->qkey_viol_cntr = resp->attrs.qkey_viol_cntr;
> diff --git a/include/rdma/ib_mad.h b/include/rdma/ib_mad.h
> index 2f4f1768ded4..f6ba366051c7 100644
> --- a/include/rdma/ib_mad.h
> +++ b/include/rdma/ib_mad.h
> @@ -262,6 +262,39 @@ struct ib_class_port_info {
> __be32 trap_qkey;
> };
>
> +/* PortInfo CapabilityMask */
> +enum ib_port_capability_mask_bits {
> + IB_PORT_SM = 1 << 1,
> + IB_PORT_NOTICE_SUP = 1 << 2,
> + IB_PORT_TRAP_SUP = 1 << 3,
> + IB_PORT_OPT_IPD_SUP = 1 << 4,
> + IB_PORT_AUTO_MIGR_SUP = 1 << 5,
> + IB_PORT_SL_MAP_SUP = 1 << 6,
> + IB_PORT_MKEY_NVRAM = 1 << 7,
> + IB_PORT_PKEY_NVRAM = 1 << 8,
> + IB_PORT_LED_INFO_SUP = 1 << 9,
> + IB_PORT_SM_DISABLED = 1 << 10,
> + IB_PORT_SYS_IMAGE_GUID_SUP = 1 << 11,
> + IB_PORT_PKEY_SW_EXT_PORT_TRAP_SUP = 1 << 12,
> + IB_PORT_EXTENDED_SPEEDS_SUP = 1 << 14,
> + IB_PORT_CM_SUP = 1 << 16,
> + IB_PORT_SNMP_TUNNEL_SUP = 1 << 17,
> + IB_PORT_REINIT_SUP = 1 << 18,
> + IB_PORT_DEVICE_MGMT_SUP = 1 << 19,
> + IB_PORT_VENDOR_CLASS_SUP = 1 << 20,
> + IB_PORT_DR_NOTICE_SUP = 1 << 21,
> + IB_PORT_CAP_MASK_NOTICE_SUP = 1 << 22,
> + IB_PORT_BOOT_MGMT_SUP = 1 << 23,
> + IB_PORT_LINK_LATENCY_SUP = 1 << 24,
> + IB_PORT_CLIENT_REG_SUP = 1 << 25,
> + IB_PORT_OTHER_LOCAL_CHANGES_SUP = 1 << 26,
> + IB_PORT_LINK_SPEED_WIDTH_TABLE_SUP = 1 << 27,
> + IB_PORT_VENDOR_SPECIFIC_MADS_TABLE_SUP = 1 << 28,
> + IB_PORT_MCAST_PKEY_TRAP_SUPPRESSION_SUP = 1 << 29,
> + IB_PORT_MCAST_FDB_TOP_SUP = 1 << 30,
> + IB_PORT_HIERARCHY_INFO_SUP = 1ULL << 31,
> +};
> +
> #define OPA_CLASS_PORT_INFO_PR_SUPPORT BIT(26)
>
> struct opa_class_port_info {
> diff --git a/include/rdma/ib_verbs.h b/include/rdma/ib_verbs.h
> index d249cba3c5cc..86e65d3d9784 100644
> --- a/include/rdma/ib_verbs.h
> +++ b/include/rdma/ib_verbs.h
> @@ -432,33 +432,6 @@ enum ib_port_state {
> IB_PORT_ACTIVE_DEFER = 5
> };
>
> -enum ib_port_cap_flags {
> - IB_PORT_SM = 1 << 1,
> - IB_PORT_NOTICE_SUP = 1 << 2,
> - IB_PORT_TRAP_SUP = 1 << 3,
> - IB_PORT_OPT_IPD_SUP = 1 << 4,
> - IB_PORT_AUTO_MIGR_SUP = 1 << 5,
> - IB_PORT_SL_MAP_SUP = 1 << 6,
> - IB_PORT_MKEY_NVRAM = 1 << 7,
> - IB_PORT_PKEY_NVRAM = 1 << 8,
> - IB_PORT_LED_INFO_SUP = 1 << 9,
> - IB_PORT_SM_DISABLED = 1 << 10,
> - IB_PORT_SYS_IMAGE_GUID_SUP = 1 << 11,
> - IB_PORT_PKEY_SW_EXT_PORT_TRAP_SUP = 1 << 12,
> - IB_PORT_EXTENDED_SPEEDS_SUP = 1 << 14,
> - IB_PORT_CM_SUP = 1 << 16,
> - IB_PORT_SNMP_TUNNEL_SUP = 1 << 17,
> - IB_PORT_REINIT_SUP = 1 << 18,
> - IB_PORT_DEVICE_MGMT_SUP = 1 << 19,
> - IB_PORT_VENDOR_CLASS_SUP = 1 << 20,
> - IB_PORT_DR_NOTICE_SUP = 1 << 21,
> - IB_PORT_CAP_MASK_NOTICE_SUP = 1 << 22,
> - IB_PORT_BOOT_MGMT_SUP = 1 << 23,
> - IB_PORT_LINK_LATENCY_SUP = 1 << 24,
> - IB_PORT_CLIENT_REG_SUP = 1 << 25,
> - IB_PORT_IP_BASED_GIDS = 1 << 26,
> -};
> -
> enum ib_port_width {
> IB_WIDTH_1X = 1,
> IB_WIDTH_4X = 2,
> @@ -597,6 +570,9 @@ struct ib_port_attr {
> enum ib_mtu max_mtu;
> enum ib_mtu active_mtu;
> int gid_tbl_len;
> + unsigned int grh_required:1;
> + unsigned int ip_gids:1;
> + /* This is the value from PortInfo CapabilityMask, defined by IBA */
> u32 port_cap_flags;
> u32 max_msg_sz;
> u32 bad_pkey_cntr;
> @@ -612,7 +588,6 @@ struct ib_port_attr {
> u8 active_width;
> u8 active_speed;
> u8 phys_state;
> - bool grh_required;
> };
>
> enum ib_device_modify_flags {
> diff --git a/include/uapi/rdma/ib_user_ioctl_verbs.h b/include/uapi/rdma/ib_user_ioctl_verbs.h
> index 625545d862d7..1220f1df3ded 100644
> --- a/include/uapi/rdma/ib_user_ioctl_verbs.h
> +++ b/include/uapi/rdma/ib_user_ioctl_verbs.h
> @@ -40,6 +40,44 @@
> #define RDMA_UAPI_PTR(_type, _name) __aligned_u64 _name
> #endif
>
> +enum ib_uverbs_query_port_cap_flags {
> + IB_UVERBS_PCF_SM = 1 << 1,
> + IB_UVERBS_PCF_NOTICE_SUP = 1 << 2,
> + IB_UVERBS_PCF_TRAP_SUP = 1 << 3,
> + IB_UVERBS_PCF_OPT_IPD_SUP = 1 << 4,
> + IB_UVERBS_PCF_AUTO_MIGR_SUP = 1 << 5,
> + IB_UVERBS_PCF_SL_MAP_SUP = 1 << 6,
> + IB_UVERBS_PCF_MKEY_NVRAM = 1 << 7,
> + IB_UVERBS_PCF_PKEY_NVRAM = 1 << 8,
> + IB_UVERBS_PCF_LED_INFO_SUP = 1 << 9,
> + IB_UVERBS_PCF_SM_DISABLED = 1 << 10,
> + IB_UVERBS_PCF_SYS_IMAGE_GUID_SUP = 1 << 11,
> + IB_UVERBS_PCF_PKEY_SW_EXT_PORT_TRAP_SUP = 1 << 12,
> + IB_UVERBS_PCF_EXTENDED_SPEEDS_SUP = 1 << 14,
> + IB_UVERBS_PCF_CM_SUP = 1 << 16,
> + IB_UVERBS_PCF_SNMP_TUNNEL_SUP = 1 << 17,
> + IB_UVERBS_PCF_REINIT_SUP = 1 << 18,
> + IB_UVERBS_PCF_DEVICE_MGMT_SUP = 1 << 19,
> + IB_UVERBS_PCF_VENDOR_CLASS_SUP = 1 << 20,
> + IB_UVERBS_PCF_DR_NOTICE_SUP = 1 << 21,
> + IB_UVERBS_PCF_CAP_MASK_NOTICE_SUP = 1 << 22,
> + IB_UVERBS_PCF_BOOT_MGMT_SUP = 1 << 23,
> + IB_UVERBS_PCF_LINK_LATENCY_SUP = 1 << 24,
> + IB_UVERBS_PCF_CLIENT_REG_SUP = 1 << 25,
> + /*
> + * IsOtherLocalChangesNoticeSupported is aliased by IP_BASED_GIDS and
> + * is inaccessible
> + */
> + IB_UVERBS_PCF_LINK_SPEED_WIDTH_TABLE_SUP = 1 << 27,
> + IB_UVERBS_PCF_VENDOR_SPECIFIC_MADS_TABLE_SUP = 1 << 28,
> + IB_UVERBS_PCF_MCAST_PKEY_TRAP_SUPPRESSION_SUP = 1 << 29,
> + IB_UVERBS_PCF_MCAST_FDB_TOP_SUP = 1 << 30,
> + IB_UVERBS_PCF_HIERARCHY_INFO_SUP = 1ULL << 31,
> +
> + /* NOTE this is an internal flag, not an IBA flag */
> + IB_UVERBS_PCF_IP_BASED_GIDS = 1 << 26,
> +};
> +
> enum ib_uverbs_flow_action_esp_keymat {
> IB_UVERBS_FLOW_ACTION_ESP_KEYMAT_AES_GCM,
> };
> diff --git a/include/uapi/rdma/ib_user_verbs.h b/include/uapi/rdma/ib_user_verbs.h
> index 4f9991de8e3a..0a9070abb3f8 100644
> --- a/include/uapi/rdma/ib_user_verbs.h
> +++ b/include/uapi/rdma/ib_user_verbs.h
> @@ -279,7 +279,7 @@ struct ib_uverbs_query_port {
> };
>
> struct ib_uverbs_query_port_resp {
> - __u32 port_cap_flags;
> + __u32 port_cap_flags; /* see ib_uverbs_query_port_cap_flags */
> __u32 max_msg_sz;
> __u32 bad_pkey_cntr;
> __u32 qkey_viol_cntr;
>
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo@xxxxxxxxxxxxxxx
More majordomo info at http://vger.kernel.org/majordomo-info.html
