On Thu, Oct 14, 2021 at 10:13 AM Pavel Begunkov <asml.silence@xxxxxxxxx> wrote:
>
> Combine force_nonblock branches (which is already optimised by
> compiler), flip branches so the most hot/common path is the first, e.g.
> as with non on-stack iov setup, and add extra likely/unlikely
> attributions for errror paths.
>
> Signed-off-by: Pavel Begunkov <asml.silence@xxxxxxxxx>
> ---
> fs/io_uring.c | 75 +++++++++++++++++++++++++--------------------------
> 1 file changed, 37 insertions(+), 38 deletions(-)
>
> diff --git a/fs/io_uring.c b/fs/io_uring.c
> index f9af54b10238..8bbbe7ccad54 100644
> --- a/fs/io_uring.c
> +++ b/fs/io_uring.c
> @@ -3395,7 +3395,7 @@ static bool io_rw_should_retry(struct io_kiocb *req)
>
> static inline int io_iter_do_read(struct io_kiocb *req, struct iov_iter *iter)
> {
> - if (req->file->f_op->read_iter)
> + if (likely(req->file->f_op->read_iter))
> return call_read_iter(req->file, &req->rw.kiocb, iter);
> else if (req->file->f_op->read)
> return loop_rw_iter(READ, req, iter);
> @@ -3411,14 +3411,18 @@ static bool need_read_all(struct io_kiocb *req)
>
> static int io_read(struct io_kiocb *req, unsigned int issue_flags)
> {
> - struct io_rw_state __s, *s;
> + struct io_rw_state __s, *s = &__s;
> struct iovec *iovec;
> struct kiocb *kiocb = &req->rw.kiocb;
> bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
> struct io_async_rw *rw;
> ssize_t ret, ret2;
>
> - if (req_has_async_data(req)) {
> + if (!req_has_async_data(req)) {
> + ret = io_import_iovec(READ, req, &iovec, s, issue_flags);
> + if (unlikely(ret < 0))
> + return ret;
> + } else {
> rw = req->async_data;
> s = &rw->s;
> /*
> @@ -3428,24 +3432,19 @@ static int io_read(struct io_kiocb *req, unsigned int issue_flags)
> */
> iov_iter_restore(&s->iter, &s->iter_state);
> iovec = NULL;
> - } else {
> - s = &__s;
> - ret = io_import_iovec(READ, req, &iovec, s, issue_flags);
> - if (unlikely(ret < 0))
> - return ret;
> }
> req->result = iov_iter_count(&s->iter);
>
> - /* Ensure we clear previously set non-block flag */
> - if (!force_nonblock)
> - kiocb->ki_flags &= ~IOCB_NOWAIT;
> - else
> + if (force_nonblock) {
> + /* If the file doesn't support async, just async punt */
> + if (unlikely(!io_file_supports_nowait(req, READ))) {
> + ret = io_setup_async_rw(req, iovec, s, true);
> + return ret ?: -EAGAIN;
> + }
> kiocb->ki_flags |= IOCB_NOWAIT;
> -
> - /* If the file doesn't support async, just async punt */
> - if (force_nonblock && !io_file_supports_nowait(req, READ)) {
> - ret = io_setup_async_rw(req, iovec, s, true);
> - return ret ?: -EAGAIN;
> + } else {
> + /* Ensure we clear previously set non-block flag */
> + kiocb->ki_flags &= ~IOCB_NOWAIT;
> }
>
> ret = rw_verify_area(READ, req->file, io_kiocb_ppos(kiocb), req->result);
> @@ -3541,40 +3540,40 @@ static int io_write_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
>
> static int io_write(struct io_kiocb *req, unsigned int issue_flags)
> {
> - struct io_rw_state __s, *s;
> - struct io_async_rw *rw;
> + struct io_rw_state __s, *s = &__s;
> struct iovec *iovec;
> struct kiocb *kiocb = &req->rw.kiocb;
> bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
> ssize_t ret, ret2;
>
> - if (req_has_async_data(req)) {
> - rw = req->async_data;
> - s = &rw->s;
> - iov_iter_restore(&s->iter, &s->iter_state);
> - iovec = NULL;
> - } else {
> - s = &__s;
> + if (!req_has_async_data(req)) {
> ret = io_import_iovec(WRITE, req, &iovec, s, issue_flags);
> if (unlikely(ret < 0))
> return ret;
> + } else {
> + struct io_async_rw *rw = req->async_data;
> +
> + s = &rw->s;
> + iov_iter_restore(&s->iter, &s->iter_state);
> + iovec = NULL;
> }
> req->result = iov_iter_count(&s->iter);
>
> - /* Ensure we clear previously set non-block flag */
> - if (!force_nonblock)
> - kiocb->ki_flags &= ~IOCB_NOWAIT;
> - else
> - kiocb->ki_flags |= IOCB_NOWAIT;
> + if (force_nonblock) {
> + /* If the file doesn't support async, just async punt */
> + if (unlikely(!io_file_supports_nowait(req, WRITE)))
> + goto copy_iov;
>
> - /* If the file doesn't support async, just async punt */
> - if (force_nonblock && !io_file_supports_nowait(req, WRITE))
> - goto copy_iov;
> + /* file path doesn't support NOWAIT for non-direct_IO */
> + if (force_nonblock && !(kiocb->ki_flags & IOCB_DIRECT) &&
You can drop this 'force_nonblock' no?
> + (req->flags & REQ_F_ISREG))
> + goto copy_iov;
>
> - /* file path doesn't support NOWAIT for non-direct_IO */
> - if (force_nonblock && !(kiocb->ki_flags & IOCB_DIRECT) &&
> - (req->flags & REQ_F_ISREG))
> - goto copy_iov;
> + kiocb->ki_flags |= IOCB_NOWAIT;
> + } else {
> + /* Ensure we clear previously set non-block flag */
> + kiocb->ki_flags &= ~IOCB_NOWAIT;
> + }
>
> ret = rw_verify_area(WRITE, req->file, io_kiocb_ppos(kiocb), req->result);
> if (unlikely(ret))
...
What swapping order of conditions below:
if ((req->ctx->flags & IORING_SETUP_IOPOLL) && ret2 == -EAGAIN)
The ret2 check will almost certainly be faster than 2x deref.
> --
> 2.33.0
>
