If IORING_URING_CMD_DIRECT flag is set, get the registered qid and
- submit the io_uring command via mq_ops->queue_uring_cmd.
- complete it via mq_ops->poll_uring_cmd.
If the command could not be submitted this way due to any reason,
abstract it and fallback to old way of submission.
This keeps IORING_URING_CMD_DIRECT flag advisory.
Signed-off-by: Kanchan Joshi <joshi.k@xxxxxxxxxxx>
---
drivers/nvme/host/ioctl.c | 105 +++++++++++++++++++++++++++++++++-----
drivers/nvme/host/nvme.h | 4 ++
2 files changed, 97 insertions(+), 12 deletions(-)
diff --git a/drivers/nvme/host/ioctl.c b/drivers/nvme/host/ioctl.c
index 18f4f20f5e76..df86fb4f132b 100644
--- a/drivers/nvme/host/ioctl.c
+++ b/drivers/nvme/host/ioctl.c
@@ -653,6 +653,23 @@ static bool is_ctrl_ioctl(unsigned int cmd)
return false;
}
+static int nvme_uring_cmd_io_direct(struct nvme_ctrl *ctrl, struct nvme_ns *ns,
+ struct io_uring_cmd *ioucmd, unsigned int issue_flags)
+{
+ struct request_queue *q = ns ? ns->queue : ctrl->admin_q;
+ int qid = io_uring_cmd_import_qid(ioucmd);
+ struct nvme_uring_direct_pdu *pdu =
+ (struct nvme_uring_direct_pdu *)&ioucmd->pdu;
+
+ if ((issue_flags & IO_URING_F_IOPOLL) != IO_URING_F_IOPOLL)
+ return -EOPNOTSUPP;
+
+ pdu->ns = ns;
+ if (q->mq_ops && q->mq_ops->queue_uring_cmd)
+ return q->mq_ops->queue_uring_cmd(ioucmd, qid);
+ return -EOPNOTSUPP;
+}
+
static int nvme_ctrl_ioctl(struct nvme_ctrl *ctrl, unsigned int cmd,
void __user *argp, fmode_t mode)
{
@@ -763,6 +780,14 @@ static int nvme_ns_uring_cmd(struct nvme_ns *ns, struct io_uring_cmd *ioucmd,
switch (ioucmd->cmd_op) {
case NVME_URING_CMD_IO:
+ if (ioucmd->flags & IORING_URING_CMD_DIRECT) {
+ ret = nvme_uring_cmd_io_direct(ctrl, ns, ioucmd,
+ issue_flags);
+ if (ret == -EIOCBQUEUED)
+ return ret;
+ /* in case of any error, just fallback */
+ ioucmd->flags &= ~(IORING_URING_CMD_DIRECT);
+ }
ret = nvme_uring_cmd_io(ctrl, ns, ioucmd, issue_flags, false);
break;
case NVME_URING_CMD_IO_VEC:
@@ -783,6 +808,38 @@ int nvme_ns_chr_uring_cmd(struct io_uring_cmd *ioucmd, unsigned int issue_flags)
return nvme_ns_uring_cmd(ns, ioucmd, issue_flags);
}
+/* similar to blk_mq_poll; may be possible to unify */
+int nvme_uring_cmd_iopoll_qid(struct request_queue *q,
+ struct io_uring_cmd *ioucmd, int qid,
+ struct io_comp_batch *iob,
+ unsigned int flags)
+{
+ long state = get_current_state();
+ int ret;
+
+ if (!(q->mq_ops && q->mq_ops->poll_uring_cmd))
+ return 0;
+ do {
+ ret = q->mq_ops->poll_uring_cmd(ioucmd, qid, iob);
+ if (ret > 0) {
+ __set_current_state(TASK_RUNNING);
+ return ret;
+ }
+ if (signal_pending_state(state, current))
+ __set_current_state(TASK_RUNNING);
+ if (task_is_running(current))
+ return 1;
+
+ if (ret < 0 || (flags & BLK_POLL_ONESHOT))
+ break;
+ cpu_relax();
+
+ } while (!need_resched());
+
+ __set_current_state(TASK_RUNNING);
+ return 0;
+}
+
int nvme_ns_chr_uring_cmd_iopoll(struct io_uring_cmd *ioucmd,
struct io_comp_batch *iob,
unsigned int poll_flags)
@@ -792,14 +849,26 @@ int nvme_ns_chr_uring_cmd_iopoll(struct io_uring_cmd *ioucmd,
struct nvme_ns *ns;
struct request_queue *q;
- rcu_read_lock();
- bio = READ_ONCE(ioucmd->cookie);
ns = container_of(file_inode(ioucmd->file)->i_cdev,
struct nvme_ns, cdev);
q = ns->queue;
- if (test_bit(QUEUE_FLAG_POLL, &q->queue_flags) && bio && bio->bi_bdev)
- ret = bio_poll(bio, iob, poll_flags);
- rcu_read_unlock();
+ if (!(ioucmd->flags & IORING_URING_CMD_DIRECT)) {
+ rcu_read_lock();
+ bio = READ_ONCE(ioucmd->cookie);
+ if (test_bit(QUEUE_FLAG_POLL, &q->queue_flags) && bio && bio->bi_bdev)
+ ret = bio_poll(bio, iob, poll_flags);
+
+ rcu_read_unlock();
+ } else {
+ int qid = io_uring_cmd_import_qid(ioucmd);
+
+ if (qid <= 0)
+ return 0;
+ if (!percpu_ref_tryget(&q->q_usage_counter))
+ return 0;
+ ret = nvme_uring_cmd_iopoll_qid(q, ioucmd, qid, iob, poll_flags);
+ percpu_ref_put(&q->q_usage_counter);
+ }
return ret;
}
#ifdef CONFIG_NVME_MULTIPATH
@@ -952,13 +1021,25 @@ int nvme_ns_head_chr_uring_cmd_iopoll(struct io_uring_cmd *ioucmd,
struct request_queue *q;
if (ns) {
- rcu_read_lock();
- bio = READ_ONCE(ioucmd->cookie);
- q = ns->queue;
- if (test_bit(QUEUE_FLAG_POLL, &q->queue_flags) && bio
- && bio->bi_bdev)
- ret = bio_poll(bio, iob, poll_flags);
- rcu_read_unlock();
+ if (!(ioucmd->flags & IORING_URING_CMD_DIRECT)) {
+ rcu_read_lock();
+ bio = READ_ONCE(ioucmd->cookie);
+ q = ns->queue;
+ if (test_bit(QUEUE_FLAG_POLL, &q->queue_flags) && bio
+ && bio->bi_bdev)
+ ret = bio_poll(bio, iob, poll_flags);
+ rcu_read_unlock();
+ } else {
+ int qid = io_uring_cmd_import_qid(ioucmd);
+
+ if (qid <= 0)
+ return 0;
+ if (!percpu_ref_tryget(&q->q_usage_counter))
+ return 0;
+ ret = nvme_uring_cmd_iopoll_qid(q, ioucmd, qid, iob,
+ poll_flags);
+ percpu_ref_put(&q->q_usage_counter);
+ }
}
srcu_read_unlock(&head->srcu, srcu_idx);
return ret;
diff --git a/drivers/nvme/host/nvme.h b/drivers/nvme/host/nvme.h
index 4eb45afc9484..2fd4432fbe12 100644
--- a/drivers/nvme/host/nvme.h
+++ b/drivers/nvme/host/nvme.h
@@ -176,6 +176,10 @@ struct nvme_uring_data {
__u32 timeout_ms;
};
+struct nvme_uring_direct_pdu {
+ struct nvme_ns *ns;
+};
+
/*
* Mark a bio as coming in through the mpath node.
*/
--
2.25.1
