Hi Sagi,
My apologies for not replaying sooner. This works except I think we need this additional change:
diff -Nupr old/iscsi_iser.c new/iscsi_iser.c
--- old/iscsi_iser.c 2018-06-26 13:26:12.540014585 -0400
+++ new/iscsi_iser.c 2018-06-26 13:26:39.499089983 -0400
@@ -680,7 +680,7 @@ iscsi_iser_session_create(struct iscsi_e
shost->max_sectors);
cls_session = iscsi_session_setup(&iscsi_iser_transport, shost,
- cmds_max, 0,
+ shost->can_queue, 0,
sizeof(struct iscsi_iser_task),
initial_cmdsn, 0);
if (!cls_session)
I do not know if this change makes sense or not, however, if I do not make this then the system locks up with this:
[ 1309.840980] iser: iser_connect: connecting to: 10.10.2.1:3260
[ 1309.841046] iser: iser_cma_handler: address resolved (0): status 0 conn 0000000014fdd3ef id 00000000ec15dc14
[ 1309.841049] iser: iser_assign_reg_ops: FastReg supported, using FastReg for registration
[ 1309.841053] iser: iser_create_device_ib_res: using 40 CQs, device mlx5_0 supports 40 vectors max_cqe 41048
[ 1309.913764] iser: iscsi_iser_ep_poll: iser conn 0000000014fdd3ef rc = 0
[ 1309.914031] iser: iser_cma_handler: route resolved (2): status 0 conn 0000000014fdd3ef id 00000000ec15dc14
[ 1309.914033] iser: iser_create_ib_conn_res: cq index 0 used for ib_conn 0000000014fdd3ef
[ 1309.914910] iser: iser_create_ib_conn_res: setting conn 0000000014fdd3ef cma_id 00000000ec15dc14 qp 000000004bb4c424
[ 1309.928794] iser: iser_cma_handler: established (9): status 0 conn 0000000014fdd3ef id 00000000ec15dc14
[ 1309.928979] iser: iser_connected_handler: remote qpn:a43 my qpn:6a
[ 1309.928981] iser: iser_connected_handler: conn 0000000014fdd3ef: negotiated local invalidation
[ 1310.913851] iser: iscsi_iser_ep_poll: iser conn 0000000014fdd3ef rc = 1
[ 1310.914130] scsi host12: iSCSI Initiator over iSER
[ 1310.915358] iser: iscsi_iser_session_create: iser_conn 0000000014fdd3ef, sg_tablesize 128, max_sectors 1016
[ 1311.099609] iser: iscsi_iser_conn_bind: binding iscsi conn 0000000096323413 to iser_conn 0000000014fdd3ef
[ 1311.099977] iser: iscsi_iser_mtask_xmit: mtask xmit [cid 0 itt 0x0]
[ 1311.099980] iser: iser_send_control: op 43 dsl 1dc, posting login rx buffer
[ 1311.099983] iser: iser_post_rx_bufs: req op 43 flags 87
[ 1311.099985] iser: iser_post_rx_bufs: Normal session, posting batch of RX 256 buffers
[ 1311.100005] general protection fault: 0000 [#1] SMP PTI
[ 1311.100054] Modules linked in: rpcrdma sunrpc ib_isert iscsi_target_mod ib_iser(E) libiscsi scsi_transport_iscsi ib_srpt target_core_mod ib_srp scsi_transport_srp ib_ipoib rdma_ucm ib_ucm ib_umad
rdma_cm ib_cm iw_cm mlx5_ib ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 xt_conntrack ip_set nfnetlink ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6
ip6table_mangle ip6table_raw ip6table_security iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack libcrc32c iptable_mangle iptable_raw iptable_security ebtable_filter
ebtables ip6table_filter ip6_tables ib_uverbs ib_core intel_rapl sb_edac x86_pkg_temp_thermal intel_powerclamp coretemp kvm_intel kvm mlx5_core irqbypass crct10dif_pclmul crc32_pclmul ipmi_ssif
crc32c_intel mlxfw ghash_clmulni_intel
[ 1311.100542] devlink tg3 intel_cstate ptp intel_uncore iTCO_wdt ipmi_si pps_core iTCO_vendor_support mxm_wmi pcspkr intel_rapl_perf ipmi_devintf lpc_ich ipmi_msghandler mei_me mei shpchp wmi
acpi_power_meter mgag200 i2c_algo_bit drm_kms_helper ttm drm uas usb_storage megaraid_sas
[ 1311.100721] CPU: 5 PID: 1810 Comm: iscsid Tainted: G E 4.17.0-karan #1
[ 1311.100772] Hardware name: /04N3DF, BIOS 2.0.1 02/12/2016
[ 1311.100817] RIP: 0010:__x86_indirect_thunk_rax+0x10/0x20
[ 1311.100854] RSP: 0018:ffffb7ab0441baa0 EFLAGS: 00010206
[ 1311.100891] RAX: 3d7ab000ffff8a4e RBX: ffff8a4e6ccb6898 RCX: 0000000000000001
[ 1311.100938] RDX: 000000000000004c RSI: 000000086ccb6898 RDI: ffff8a4e5a8c3078
[ 1311.100985] RBP: 0000000000000000 R08: 0001000000000000 R09: 0800000000607a3d
[ 1311.101032] R10: 0000000000607a3d R11: 0000000000000001 R12: ffff8a4e5a8c2000
[ 1311.101079] R13: 0000000000000001 R14: ffff8a4e5a8c2000 R15: ffff8a4e6ccb43f8
[ 1311.101128] FS: 00007fd8274b8880(0000) GS:ffff8a4e6f080000(0000) knlGS:0000000000000000
[ 1311.101181] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[ 1311.101220] CR2: 00007fd81561b010 CR3: 0000000843c5c006 CR4: 00000000003606e0
[ 1311.101268] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
[ 1311.101315] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
[ 1311.101361] Call Trace:
[ 1311.101393] ? iser_post_send+0xa4/0x150 [ib_iser]
[ 1311.101431] ? iser_send_control+0x140/0x220 [ib_iser]
[ 1311.101471] ? iscsi_iser_task_xmit+0x145/0x170 [ib_iser]
[ 1311.101512] ? iscsi_iser_task_init+0x1b/0x52 [ib_iser]
[ 1311.101553] ? __iscsi_conn_send_pdu+0x322/0x400 [libiscsi]
[ 1311.101594] ? iscsi_conn_send_pdu+0x3e/0x60 [libiscsi]
[ 1311.101642] ? iscsi_if_rx+0xe4c/0x12f0 [scsi_transport_iscsi]
[ 1311.101686] ? __netlink_lookup+0xd0/0x130
[ 1311.101719] ? netlink_unicast+0x19e/0x260
[ 1311.101750] ? netlink_sendmsg+0x1ff/0x3c0
[ 1311.101783] ? sock_sendmsg+0x36/0x40
[ 1311.101812] ? ___sys_sendmsg+0x295/0x2f0
[ 1311.101843] ? ___sys_recvmsg+0x15e/0x1e0
[ 1311.101880] ? show_conn_param_ISCSI_PARAM_PING_TMO+0x1e/0x30 [scsi_transport_iscsi]
[ 1311.101935] ? dev_attr_show+0x1c/0x40
[ 1311.101966] ? __check_object_size+0x9c/0x171
[ 1311.102001] ? seq_read+0x1ef/0x460
[ 1311.102034] ? _cond_resched+0x15/0x30
[ 1311.102064] ? __sys_sendmsg+0x57/0xa0
[ 1311.102096] ? do_syscall_64+0x5b/0x160
[ 1311.102127] ? entry_SYSCALL_64_after_hwframe+0x44/0xa9
[ 1311.102163] Code: 90 90 90 90 90 90 90 90 90 90 90 90 90 90 90 90 90 90 90 90 90 90 90 90 90 90 90 e8 07 00 00 00 f3 90 0f ae e8 eb f9 48 89 04 24 <c3> 0f 1f 44 00 00 66 2e 0f 1f 84 00 00 00 00 00
e8 07 00 00 00
[ 1311.102338] RIP: __x86_indirect_thunk_rax+0x10/0x20 RSP: ffffb7ab0441baa0
[ 1311.102422] ---[ end trace 55b2693837002964 ]---
Thanks
-Karan
On Tue, 2018-06-19 at 17:59 +0300, Sagi Grimberg wrote:
> > Hi,
>
> Hi Karan,
>
> > I have run in to a problem where it seems like an ISER initiator cannot
> > submit more than 128 commands to the target (per scsi host). I tried to
> > debug this and found that when iscsi_host_add() is called,
> > Scsi_Host>can_queue is unset. This causes iscsi_host_add() to use the
> > default qdepth (128) when creating the scsi host's blk-mq in
> > scsi_add_host(). After the call to iscsi_host_add() shost->can_queue is
> > set to the correct value, however by then the blk-mq is already created
> > with the wrong nuber of tags.
> >
> > This causes the queue depth not to exceed 128.
> >
> > Please advise if this is the expected behaviour. Have I missed
> > something? It appears that other drivers that call scsi_add_host() set
> > can_queue before the function call.
>
> Thanks for reporting, you are correct, this can be easily fixed, does
> the below patch fix your issue:
> --
> diff --git a/drivers/infiniband/ulp/iser/iscsi_iser.c
> b/drivers/infiniband/ulp/iser/iscsi_iser.c
> index 0336643c2ed6..434ae88ff623 100644
> --- a/drivers/infiniband/ulp/iser/iscsi_iser.c
> +++ b/drivers/infiniband/ulp/iser/iscsi_iser.c
> @@ -633,8 +633,8 @@ iscsi_iser_session_create(struct iscsi_endpoint *ep,
> */
> if (ep) {
> iser_conn = ep->dd_data;
> - max_cmds = iser_conn->max_cmds;
> shost->sg_tablesize = iser_conn->scsi_sg_tablesize;
> + shost->can_queue = min_t(u16, cmds_max,
> iser_conn->max_cmds);
>
> mutex_lock(&iser_conn->state_mutex);
> if (iser_conn->state != ISER_CONN_UP) {
> @@ -660,6 +660,7 @@ iscsi_iser_session_create(struct iscsi_endpoint *ep,
> }
> mutex_unlock(&iser_conn->state_mutex);
> } else {
> + shost->can_queue = min_t(u16, cmds_max,
> ISER_DEF_XMIT_CMDS_MAX);
> max_cmds = ISER_DEF_XMIT_CMDS_MAX;
> if (iscsi_host_add(shost, NULL))
> goto free_host;
> @@ -678,12 +679,6 @@ iscsi_iser_session_create(struct iscsi_endpoint *ep,
> iser_conn, shost->sg_tablesize,
> shost->max_sectors);
>
> - if (cmds_max > max_cmds) {
> - iser_info("cmds_max changed from %u to %u\n",
> - cmds_max, max_cmds);
> - cmds_max = max_cmds;
> - }
> -
> cls_session = iscsi_session_setup(&iscsi_iser_transport, shost,
> cmds_max, 0,
> sizeof(struct iscsi_iser_task),
> @@ -692,7 +687,6 @@ iscsi_iser_session_create(struct iscsi_endpoint *ep,
> goto remove_host;
> session = cls_session->dd_data;
>
> - shost->can_queue = session->scsi_cmds_max;
> return cls_session;
>
> remove_host:
> --
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo@xxxxxxxxxxxxxxx
More majordomo info at http://vger.kernel.org/majordomo-info.html
