I had to implement some similar infrastucture recently, so a few
late comments:
> The reason your hitting this is because FCoE target doesn't use
> target_sess_cmd_list_set_waiting() + target_wait_for_sess_cmds() during
> se_session shutdown, which target_submit_cmd() + target_submit_tmr()
> depends on for checking se_session->sess_tearing_down, and why
> target_release_cmd_kref() does complete(&se_cmd->cmd_wait_comp);
Needing an explicit shutdown and drain call during shutdown seems
like an unfortunate design choice. I think everyone would be
better off having an implicit wait that ensures no commands are
outstanding during shutdown.
> So really, this needs to be a BUG_ON(!list_empty()) for sess_cmd_list +
> sess_wait_list in transport_free_session(), and FCoE target needs to be
> converted to use these primitives following tcm_qla2xxx, ib_isert, and
> ib_srpt function.
>
> Adding a hack for FCoE target is not going to cut it.
>
> > @@ -2524,6 +2531,8 @@ static void target_release_cmd_kref(struct kref *kref)
> > return;
> > }
> > list_del(&se_cmd->se_cmd_list);
> > + if (list_empty(&se_cmd->se_cmd_list))
> > + wake_up(&se_sess->cmd_list_wq);
> > spin_unlock_irqrestore(&se_sess->sess_cmd_lock, flags);
> >
> > se_cmd->se_tfo->release_cmd(se_cmd);
>
> Unnecessary fast-path wake-up for every I/O. No thanks.
I think this should be a list_empty(&se_sess->sess_cmd_list),
which would not be once per I/O.
Otherwise I agree there should just be one place to wait for shutdown,
but even if that once place is target_wait_for_sess_cmds it would
benefit from using the more efficient algorithm in Bart's patch.
Something like the untested patch below (on top of this series):
diff --git a/drivers/target/target_core_transport.c b/drivers/target/target_core_transport.c
index 8b0ea56..11eba9f 100644
--- a/drivers/target/target_core_transport.c
+++ b/drivers/target/target_core_transport.c
@@ -237,7 +237,6 @@ struct se_session *transport_init_session(enum target_prot_op sup_prot_ops)
INIT_LIST_HEAD(&se_sess->sess_list);
INIT_LIST_HEAD(&se_sess->sess_acl_list);
INIT_LIST_HEAD(&se_sess->sess_cmd_list);
- INIT_LIST_HEAD(&se_sess->sess_wait_list);
spin_lock_init(&se_sess->sess_cmd_lock);
init_waitqueue_head(&se_sess->cmd_list_wq);
kref_init(&se_sess->sess_kref);
@@ -465,12 +464,6 @@ EXPORT_SYMBOL(transport_deregister_session_configfs);
void transport_free_session(struct se_session *se_sess)
{
- spin_lock_irq(&se_sess->sess_cmd_lock);
- wait_event_lock_irq(se_sess->cmd_list_wq,
- list_empty(&se_sess->sess_cmd_list),
- se_sess->sess_cmd_lock);
- spin_unlock_irq(&se_sess->sess_cmd_lock);
-
if (se_sess->sess_cmd_map) {
percpu_ida_destroy(&se_sess->sess_tag_pool);
kvfree(se_sess->sess_cmd_map);
@@ -1165,7 +1158,6 @@ void transport_init_se_cmd(
INIT_LIST_HEAD(&cmd->se_qf_node);
INIT_LIST_HEAD(&cmd->se_cmd_list);
INIT_LIST_HEAD(&cmd->state_list);
- init_completion(&cmd->cmd_wait_comp);
init_completion(&cmd->finished);
spin_lock_init(&cmd->t_state_lock);
kref_init(&cmd->cmd_kref);
@@ -2458,19 +2450,11 @@ static void target_release_cmd_kref(struct kref *kref)
unsigned long flags;
spin_lock_irqsave(&se_sess->sess_cmd_lock, flags);
- if (list_empty(&se_cmd->se_cmd_list)) {
- spin_unlock_irqrestore(&se_sess->sess_cmd_lock, flags);
- se_cmd->se_tfo->release_cmd(se_cmd);
- return;
- }
- if (se_sess->sess_tearing_down && se_cmd->cmd_wait_set) {
- spin_unlock_irqrestore(&se_sess->sess_cmd_lock, flags);
- complete(&se_cmd->cmd_wait_comp);
- return;
+ if (!list_empty(&se_cmd->se_cmd_list)) {
+ list_del(&se_cmd->se_cmd_list);
+ if (list_empty(&se_sess->sess_cmd_list))
+ wake_up(&se_sess->cmd_list_wq);
}
- list_del(&se_cmd->se_cmd_list);
- if (list_empty(&se_cmd->se_cmd_list))
- wake_up(&se_sess->cmd_list_wq);
spin_unlock_irqrestore(&se_sess->sess_cmd_lock, flags);
se_cmd->se_tfo->release_cmd(se_cmd);
@@ -2491,59 +2475,35 @@ int target_put_sess_cmd(struct se_cmd *se_cmd)
}
EXPORT_SYMBOL(target_put_sess_cmd);
-/* target_sess_cmd_list_set_waiting - Flag all commands in
- * sess_cmd_list to complete cmd_wait_comp. Set
- * sess_tearing_down so no more commands are queued.
+/**
+ * target_sess_cmd_list_set_waiting - stop processing of new commands
* @se_sess: session to flag
+ *
+ * Prevent new commands from being queued on this session. The caller should
+ * then use target_wait_for_sess_cmds to wait for all outstanding commands to
+ * be processed.
*/
void target_sess_cmd_list_set_waiting(struct se_session *se_sess)
{
- struct se_cmd *se_cmd;
unsigned long flags;
spin_lock_irqsave(&se_sess->sess_cmd_lock, flags);
- if (se_sess->sess_tearing_down) {
- spin_unlock_irqrestore(&se_sess->sess_cmd_lock, flags);
- return;
- }
se_sess->sess_tearing_down = 1;
- list_splice_init(&se_sess->sess_cmd_list, &se_sess->sess_wait_list);
-
- list_for_each_entry(se_cmd, &se_sess->sess_wait_list, se_cmd_list)
- se_cmd->cmd_wait_set = 1;
-
spin_unlock_irqrestore(&se_sess->sess_cmd_lock, flags);
}
EXPORT_SYMBOL(target_sess_cmd_list_set_waiting);
-/* target_wait_for_sess_cmds - Wait for outstanding descriptors
+/**
+ * target_wait_for_sess_cmds - Wait for outstanding descriptors
* @se_sess: session to wait for active I/O
*/
void target_wait_for_sess_cmds(struct se_session *se_sess)
{
- struct se_cmd *se_cmd, *tmp_cmd;
- unsigned long flags;
-
- list_for_each_entry_safe(se_cmd, tmp_cmd,
- &se_sess->sess_wait_list, se_cmd_list) {
- list_del(&se_cmd->se_cmd_list);
-
- pr_debug("Waiting for se_cmd: %p t_state: %d, fabric state:"
- " %d\n", se_cmd, se_cmd->t_state,
- se_cmd->se_tfo->get_cmd_state(se_cmd));
-
- wait_for_completion(&se_cmd->cmd_wait_comp);
- pr_debug("After cmd_wait_comp: se_cmd: %p t_state: %d"
- " fabric state: %d\n", se_cmd, se_cmd->t_state,
- se_cmd->se_tfo->get_cmd_state(se_cmd));
-
- se_cmd->se_tfo->release_cmd(se_cmd);
- }
-
- spin_lock_irqsave(&se_sess->sess_cmd_lock, flags);
- WARN_ON(!list_empty(&se_sess->sess_cmd_list));
- spin_unlock_irqrestore(&se_sess->sess_cmd_lock, flags);
-
+ spin_lock_irq(&se_sess->sess_cmd_lock);
+ wait_event_lock_irq(se_sess->cmd_list_wq,
+ list_empty(&se_sess->sess_cmd_list),
+ se_sess->sess_cmd_lock);
+ spin_unlock_irq(&se_sess->sess_cmd_lock);
}
EXPORT_SYMBOL(target_wait_for_sess_cmds);
diff --git a/include/target/target_core_base.h b/include/target/target_core_base.h
index 94d9697..d44551a 100644
--- a/include/target/target_core_base.h
+++ b/include/target/target_core_base.h
@@ -438,7 +438,6 @@ struct se_cmd {
u8 scsi_asc;
u8 scsi_ascq;
u16 scsi_sense_length;
- unsigned cmd_wait_set:1;
unsigned unknown_data_length:1;
bool state_active:1;
unsigned send_abort_response:1;
@@ -471,7 +470,6 @@ struct se_cmd {
struct se_session *se_sess;
struct se_tmr_req *se_tmr_req;
struct list_head se_cmd_list;
- struct completion cmd_wait_comp;
const struct target_core_fabric_ops *se_tfo;
sense_reason_t (*execute_cmd)(struct se_cmd *);
sense_reason_t (*transport_complete_callback)(struct se_cmd *, bool);
@@ -597,7 +595,6 @@ struct se_session {
struct list_head sess_list;
struct list_head sess_acl_list;
struct list_head sess_cmd_list;
- struct list_head sess_wait_list;
spinlock_t sess_cmd_lock;
wait_queue_head_t cmd_list_wq;
struct kref sess_kref;
--
To unsubscribe from this list: send the line "unsubscribe target-devel" in
the body of a message to majordomo@xxxxxxxxxxxxxxx
More majordomo info at http://vger.kernel.org/majordomo-info.html
