The array of sg_io_v4 objects given for share variable blocking
multiple requests (svb mrq) is pre-scanned to check for errors and
to apply certain fix-ups. Add further code to determine whether the
READs and WRITEs are of the same size and that no use is made of
SGV4_FLAG_DOUT_OFFSET. Also require that SGV4_FLAG_NO_DXFER is used
on the READs. If those requirements are met then term the svb mrq
as 'uniform svb' and set the SG_FFD_CAN_REUSE_BIO bit flag. To see
the benefit from this, the number of commands given to the uniform
svb needs to be greater than SG_MAX_RSV_REQS (currently 8).
Preferably two or more times that number.
As part of the above, divide the per sg_request bitmap (formerly:
frq_bm) into two bitmaps: frq_lt_bm and frq_pc_bm. The "lt" group
are long term, potentially spanning many SCSI commands that use a
sg_request object. The "pc" group are per command and pertain to
the current SCSI command being processed by the sg_request object.
Rework the sg_rq_map_kern() function.
Signed-off-by: Douglas Gilbert <dgilbert@xxxxxxxxxxxx>
---
drivers/scsi/sg.c | 370 +++++++++++++++++++++++++++-------------------
1 file changed, 222 insertions(+), 148 deletions(-)
diff --git a/drivers/scsi/sg.c b/drivers/scsi/sg.c
index ac7321ffbd05..0a0b40a8ab65 100644
--- a/drivers/scsi/sg.c
+++ b/drivers/scsi/sg.c
@@ -129,19 +129,22 @@ enum sg_shr_var {
#define SG_ADD_RQ_MAX_RETRIES 40 /* to stop infinite _trylock(s) */
#define SG_DEF_BLK_POLL_LOOP_COUNT 1000 /* may allow user to tweak this */
-/* Bit positions (flags) for sg_request::frq_bm bitmask follow */
-#define SG_FRQ_IS_V4I 0 /* true (set) when is v4 interface */
-#define SG_FRQ_IS_ORPHAN 1 /* owner of request gone */
-#define SG_FRQ_SYNC_INVOC 2 /* synchronous (blocking) invocation */
-#define SG_FRQ_US_XFER 3 /* kernel<-->user_space data transfer */
-#define SG_FRQ_ABORTING 4 /* in process of aborting this cmd */
-#define SG_FRQ_DEACT_ORPHAN 5 /* not keeping orphan so de-activate */
-#define SG_FRQ_RECEIVING 6 /* guard against multiple receivers */
-#define SG_FRQ_FOR_MMAP 7 /* request needs PAGE_SIZE elements */
-#define SG_FRQ_COUNT_ACTIVE 8 /* sfp->submitted + waiting active */
-#define SG_FRQ_ISSUED 9 /* blk_execute_rq_nowait() finished */
+/* Bit positions (flags) for sg_request::frq_lt_bm bitmask, lt: long term */
+#define SG_FRQ_LT_RESERVED 0 /* marks a reserved request */
+#define SG_FRQ_LT_REUSE_BIO 1 /* srp->bio primed for re-use */
+
+/* Bit positions (flags) for sg_request::frq_pc_bm bitmask. pc: per command */
+#define SG_FRQ_PC_IS_V4I 0 /* true (set) when is v4 interface */
+#define SG_FRQ_PC_IS_ORPHAN 1 /* owner of request gone */
+#define SG_FRQ_PC_SYNC_INVOC 2 /* synchronous (blocking) invocation */
+#define SG_FRQ_PC_US_XFER 3 /* kernel<-->user_space data transfer */
+#define SG_FRQ_PC_ABORTING 4 /* in process of aborting this cmd */
+#define SG_FRQ_PC_DEACT_ORPHAN 5 /* not keeping orphan so de-activate */
+#define SG_FRQ_PC_RECEIVING 6 /* guard against multiple receivers */
+#define SG_FRQ_PC_FOR_MMAP 7 /* request needs PAGE_SIZE elements */
+#define SG_FRQ_PC_COUNT_ACTIVE 8 /* sfp->submitted + waiting active */
+#define SG_FRQ_PC_ISSUED 9 /* blk_execute_rq_nowait() finished */
#define SG_FRQ_POLL_SLEPT 10 /* stop re-entry of hybrid_sleep() */
-#define SG_FRQ_RESERVED 11 /* marks a reserved request */
/* Bit positions (flags) for sg_fd::ffd_bm bitmask follow */
#define SG_FFD_FORCE_PACKID 0 /* receive only given pack_id/tag */
@@ -159,6 +162,7 @@ enum sg_shr_var {
#define SG_FFD_EXCL_WAITQ 12 /* append _exclusive to wait_event */
#define SG_FFD_SVB_ACTIVE 13 /* shared variable blocking active */
#define SG_FFD_RESHARE 14 /* reshare limits to single rsv req */
+#define SG_FFD_CAN_REUSE_BIO 15 /* uniform svb --> can re-use bio_s */
/* Bit positions (flags) for sg_device::fdev_bm bitmask follow */
#define SG_FDEV_EXCLUDE 0 /* have fd open with O_EXCL */
@@ -185,6 +189,7 @@ static int sg_allow_dio = SG_ALLOW_DIO_DEF; /* ignored by code */
static int scatter_elem_sz = SG_SCATTER_SZ;
static bool no_attach_msg;
+static atomic_t sg_tmp_count_reused_bios;
#define SG_DEF_SECTOR_SZ 512
@@ -264,7 +269,8 @@ struct sg_request { /* active SCSI command or inactive request */
int tag; /* block layer identifier of request */
blk_qc_t cookie; /* ids 1 or more queues for blk_poll() */
u64 start_ns; /* starting point of command duration calc */
- unsigned long frq_bm[1]; /* see SG_FRQ_* defines above */
+ unsigned long frq_lt_bm[1]; /* see SG_FRQ_LT_* defines above */
+ unsigned long frq_pc_bm[1]; /* see SG_FRQ_PC_* defines above */
u8 *sense_bp; /* mempool alloc-ed sense buffer, as needed */
struct sg_fd *parentfp; /* pointer to owning fd, even when on fl */
struct request *rqq; /* released in sg_rq_end_io(), bio kept */
@@ -327,8 +333,8 @@ struct sg_comm_wr_t { /* arguments to sg_common_write() */
int rsv_idx; /* wanted rsv_arr index, def: -1 (anyone) */
int dlen; /* dout or din length in bytes */
int wr_offset; /* non-zero if v4 and DOUT_OFFSET set */
- unsigned long frq_bm[1]; /* see SG_FRQ_* defines above */
- union { /* selector is frq_bm.SG_FRQ_IS_V4I */
+ unsigned long frq_pc_bm[1]; /* see SG_FRQ_PC_* defines above */
+ union { /* selector is frq_pc_bm.SG_FRQ_IS_V4I */
struct sg_io_hdr *h3p;
struct sg_io_v4 *h4p;
};
@@ -422,7 +428,7 @@ static void sg_take_snap(struct sg_fd *sfp, bool clear_first);
#define SG_HAVE_EXCLUDE(sdp) test_bit(SG_FDEV_EXCLUDE, (sdp)->fdev_bm)
#define SG_IS_O_NONBLOCK(sfp) (!!((sfp)->filp->f_flags & O_NONBLOCK))
#define SG_RQ_ACTIVE(srp) (atomic_read(&(srp)->rq_st) != SG_RQ_INACTIVE)
-#define SG_IS_V4I(srp) test_bit(SG_FRQ_IS_V4I, (srp)->frq_bm)
+#define SG_IS_V4I(srp) test_bit(SG_FRQ_PC_IS_V4I, (srp)->frq_pc_bm)
/*
* Kernel needs to be built with CONFIG_SCSI_LOGGING to see log messages. 'depth' is a number
@@ -486,7 +492,8 @@ static void sg_take_snap(struct sg_fd *sfp, bool clear_first);
* SG_IOABORT: no match on pack_id or tag; mrq: no active reqs
* ENODEV target (SCSI) device associated with the fd has "disappeared"
* ENOMEM obvious; could be some pre-allocated cache that is exhausted
- * ENOMSG data transfer setup needed or (direction) disallowed (sharing)
+ * ENOMSG data transfer setup needed or (direction) disallowed (sharing);
+ * inconsistency in share settings (mrq)
* ENOSTR write-side request abandoned due to read-side error or state
* ENOTSOCK sharing: file descriptor for sharing unassociated with sg driver
* ENXIO 'no such device or address' SCSI mid-level processing errors
@@ -762,7 +769,7 @@ static inline void
sg_comm_wr_init(struct sg_comm_wr_t *cwrp)
{
memset(cwrp, 0, sizeof(*cwrp));
- WRITE_ONCE(cwrp->frq_bm[0], 0);
+ /* WRITE_ONCE(cwrp->frq_pc_bm[0], 0); */
cwrp->rsv_idx = -1;
}
@@ -979,7 +986,7 @@ sg_submit_v3(struct sg_fd *sfp, struct sg_io_hdr *hp, bool sync, struct sg_reque
clear_bit(SG_FFD_NO_CMD_Q, sfp->ffd_bm);
ul_timeout = msecs_to_jiffies(hp->timeout);
sg_comm_wr_init(&cwr);
- __assign_bit(SG_FRQ_SYNC_INVOC, cwr.frq_bm, (int)sync);
+ __assign_bit(SG_FRQ_PC_SYNC_INVOC, cwr.frq_pc_bm, (int)sync);
cwr.h3p = hp;
cwr.dlen = hp->dxfer_len;
cwr.timeout = min_t(unsigned long, ul_timeout, INT_MAX);
@@ -1256,7 +1263,7 @@ sg_srp_hybrid_sleep(struct sg_request *srp)
enum hrtimer_mode mode;
ktime_t kt = ns_to_ktime(5000);
- if (test_and_set_bit(SG_FRQ_POLL_SLEPT, srp->frq_bm))
+ if (test_and_set_bit(SG_FRQ_POLL_SLEPT, srp->frq_pc_bm))
return false;
if (kt == 0)
return false;
@@ -1303,7 +1310,7 @@ sg_wait_poll_for_given_srp(struct sg_fd *sfp, struct sg_request *srp, bool do_po
/* N.B. The SG_FFD_EXCL_WAITQ flag is ignored here. */
res = __wait_event_interruptible(sfp->cmpl_wait, sg_rq_landed(sdp, srp));
if (unlikely(res)) { /* -ERESTARTSYS because signal hit thread */
- set_bit(SG_FRQ_IS_ORPHAN, srp->frq_bm);
+ set_bit(SG_FRQ_PC_IS_ORPHAN, srp->frq_pc_bm);
/* orphans harvested when sfp->keep_orphan is false */
sg_rq_chg_state_force(srp, SG_RQ_INFLIGHT);
SG_LOG(1, sfp, "%s: wait_event_interruptible(): %s[%d]\n", __func__,
@@ -1355,7 +1362,7 @@ sg_wait_poll_for_given_srp(struct sg_fd *sfp, struct sg_request *srp, bool do_po
return sg_rq_chg_state(srp, sr_st, SG_RQ_BUSY);
}
if (atomic_read_acquire(&srp->rq_st) != SG_RQ_AWAIT_RCV)
- return (test_bit(SG_FRQ_COUNT_ACTIVE, srp->frq_bm) &&
+ return (test_bit(SG_FRQ_PC_COUNT_ACTIVE, srp->frq_pc_bm) &&
atomic_read(&sfp->submitted) < 1) ? -ENODATA : 0;
return unlikely(sg_rq_chg_state(srp, SG_RQ_AWAIT_RCV, SG_RQ_BUSY)) ? -EPROTO : 0;
@@ -1483,13 +1490,15 @@ sg_mrq_complets(struct sg_mrq_hold *mhp, struct sg_fd *sfp, struct sg_fd *sec_sf
* a strict READ (like) thence WRITE (like) sequence on all data carrying commands; also
* a dangling READ is not allowed at the end of a scb request array.
*/
-static bool
-sg_mrq_sanity(struct sg_mrq_hold *mhp, bool is_svb)
+static int
+sg_mrq_prepare(struct sg_mrq_hold *mhp, bool is_svb)
{
bool last_is_keep_share = false;
bool expect_wr = false;
+ bool uniform_svb = true; /* no dxfr to user space, all data moves same size */
bool share, have_mrq_sense, have_file_share;
- int k;
+ int k, dlen;
+ int prev_dlen = 0;
struct sg_io_v4 *cop = mhp->cwrp->h4p;
u32 cdb_alen = cop->request_len;
u32 cdb_mxlen = cdb_alen / mhp->tot_reqs;
@@ -1504,7 +1513,7 @@ sg_mrq_sanity(struct sg_mrq_hold *mhp, bool is_svb)
have_file_share = sg_fd_is_shared(sfp);
if (is_svb && unlikely(!have_file_share)) {
SG_LOG(1, sfp, "%s: share variable blocking (svb) needs file share\n", __func__);
- return false;
+ return -ENOMSG;
}
/* Pre-check each request for anomalies, plus some preparation */
for (k = 0, hp = a_hds; k < mhp->tot_reqs; ++k, ++hp) {
@@ -1512,11 +1521,11 @@ sg_mrq_sanity(struct sg_mrq_hold *mhp, bool is_svb)
sg_v4h_partial_zero(hp);
if (unlikely(hp->guard != 'Q' || hp->protocol != 0 || hp->subprotocol != 0)) {
SG_LOG(1, sfp, "%s: req index %u: bad guard or protocol\n", __func__, k);
- return false;
+ return -EPERM;
}
if (unlikely(flags & SGV4_FLAG_MULTIPLE_REQS)) {
SG_LOG(1, sfp, "%s: %s %u: no nested multi-reqs\n", __func__, rip, k);
- return false;
+ return -ERANGE;
}
share = !!(flags & SGV4_FLAG_SHARE);
last_is_keep_share = !!(flags & SGV4_FLAG_KEEP_SHARE);
@@ -1524,27 +1533,27 @@ sg_mrq_sanity(struct sg_mrq_hold *mhp, bool is_svb)
unlikely(flags & (SGV4_FLAG_DO_ON_OTHER | SGV4_FLAG_COMPLETE_B4))) {
SG_LOG(1, sfp, "%s: %s %u, no IMMED with ON_OTHER or COMPLETE_B4\n",
__func__, rip, k);
- return false;
+ return -ERANGE;
}
if (mhp->immed && unlikely(share)) {
SG_LOG(1, sfp, "%s: %s %u, no IMMED with FLAG_SHARE\n", __func__, rip, k);
- return false;
+ return -ENOMSG;
}
if (mhp->co_mmap && (flags & SGV4_FLAG_MMAP_IO)) {
SG_LOG(1, sfp, "%s: %s %u, MMAP in co AND here\n", __func__, rip, k);
- return false;
+ return -ERANGE;
}
if (unlikely(!have_file_share && share)) {
SG_LOG(1, sfp, "%s: %s %u, no file share\n", __func__, rip, k);
- return false;
+ return -ENOMSG;
}
if (unlikely(!have_file_share && !!(flags & SGV4_FLAG_DO_ON_OTHER))) {
SG_LOG(1, sfp, "%s: %s %u, no other fd to do on\n", __func__, rip, k);
- return false;
+ return -ENOMSG;
}
if (cdb_ap && unlikely(hp->request_len > cdb_mxlen)) {
SG_LOG(1, sfp, "%s: %s %u, cdb too long\n", __func__, rip, k);
- return false;
+ return -ERANGE;
}
if (have_mrq_sense && hp->response == 0 && hp->max_response_len == 0) {
hp->response = cop->response;
@@ -1558,43 +1567,66 @@ sg_mrq_sanity(struct sg_mrq_hold *mhp, bool is_svb)
/* mrq share variable blocking (svb) additional constraints checked here */
if (unlikely(flags & (SGV4_FLAG_COMPLETE_B4 | SGV4_FLAG_KEEP_SHARE))) {
SG_LOG(1, sfp, "%s: %s %u: no KEEP_SHARE with svb\n", __func__, rip, k);
- return false;
+ return -ENOMSG;
}
+ dlen = 0;
if (!expect_wr) {
+ dlen = hp->din_xfer_len;
if (hp->dout_xfer_len > 0)
goto bad_svb;
- if (hp->din_xfer_len > 0) {
+ if (dlen > 0) {
if (!(flags & SGV4_FLAG_SHARE))
goto bad_svb;
if (flags & SGV4_FLAG_DO_ON_OTHER)
goto bad_svb;
expect_wr = true;
+ if (!(flags & SGV4_FLAG_NO_DXFER))
+ uniform_svb = false;
}
/* allowing commands with no dxfer (in both cases) */
} else { /* checking write side */
- if (hp->dout_xfer_len > 0) {
+ dlen = hp->dout_xfer_len;
+ if (dlen > 0) {
if (unlikely(~flags & (SGV4_FLAG_NO_DXFER | SGV4_FLAG_SHARE |
SGV4_FLAG_DO_ON_OTHER)))
goto bad_svb;
expect_wr = false;
+ if (unlikely(flags & SGV4_FLAG_DOUT_OFFSET))
+ uniform_svb = false;
} else if (unlikely(hp->din_xfer_len > 0)) {
goto bad_svb;
}
}
+ if (!uniform_svb)
+ continue;
+ if (prev_dlen == 0)
+ prev_dlen = dlen;
+ else if (dlen != prev_dlen)
+ uniform_svb = false;
} /* end of request array iterating loop */
if (last_is_keep_share) {
SG_LOG(1, sfp, "%s: Can't set SGV4_FLAG_KEEP_SHARE on last mrq req\n", __func__);
- return false;
+ return -ENOMSG;
}
if (is_svb && expect_wr) {
SG_LOG(1, sfp, "%s: svb: unpaired READ at end of request array\n", __func__);
- return false;
+ return -ENOMSG;
}
- return true;
+ if (is_svb) {
+ bool cur_uniform_svb = test_bit(SG_FFD_CAN_REUSE_BIO, sfp->ffd_bm);
+
+ if (uniform_svb != cur_uniform_svb) {
+ if (uniform_svb)
+ set_bit(SG_FFD_CAN_REUSE_BIO, sfp->ffd_bm);
+ else
+ clear_bit(SG_FFD_CAN_REUSE_BIO, sfp->ffd_bm);
+ }
+ }
+ return 0;
bad_svb:
SG_LOG(1, sfp, "%s: %s %u: svb alternating read-then-write or flags bad\n", __func__,
rip, k);
- return false;
+ return -ENOMSG;
}
/* rsv_idx>=0 only when this request is the write-side of a request share */
@@ -1615,8 +1647,8 @@ sg_mrq_submit(struct sg_fd *rq_sfp, struct sg_mrq_hold *mhp, int pos_in_rq_arr,
r_cwrp->cmd_len = hp->request_len;
r_cwrp->rsv_idx = rsv_idx;
ul_timeout = msecs_to_jiffies(hp->timeout);
- __assign_bit(SG_FRQ_SYNC_INVOC, r_cwrp->frq_bm, (int)mhp->from_sg_io);
- __set_bit(SG_FRQ_IS_V4I, r_cwrp->frq_bm);
+ __assign_bit(SG_FRQ_PC_SYNC_INVOC, r_cwrp->frq_pc_bm, (int)mhp->from_sg_io);
+ __set_bit(SG_FRQ_PC_IS_V4I, r_cwrp->frq_pc_bm);
r_cwrp->h4p = hp;
r_cwrp->dlen = hp->din_xfer_len ? hp->din_xfer_len : hp->dout_xfer_len;
r_cwrp->timeout = min_t(unsigned long, ul_timeout, INT_MAX);
@@ -1985,6 +2017,21 @@ sg_svb_mrq_ordered(struct sg_fd *fp, struct sg_fd *o_sfp, struct sg_mrq_hold *mh
return 0;
}
+static void
+sg_svb_cleanup(struct sg_fd *sfp)
+{
+ unsigned long idx;
+ struct xarray *xafp = &sfp->srp_arr;
+ struct sg_request *srp;
+
+ xa_for_each_marked(xafp, idx, srp, SG_XA_RQ_INACTIVE) {
+ if (test_and_clear_bit(SG_FRQ_LT_REUSE_BIO, srp->frq_lt_bm)) {
+ bio_put(srp->bio); /* _get() near end of sg_start_req() */
+ srp->bio = NULL;
+ }
+ }
+}
+
/*
* Processes shared variable blocking (svb) method for multiple requests (mrq). There are two
* variants: unordered write-side requests; and ordered write-side requests. The read-side requests
@@ -2040,12 +2087,15 @@ sg_process_svb_mrq(struct sg_fd *fp, struct sg_fd *o_sfp, struct sg_mrq_hold *mh
}
if (mhp->id_of_mrq) /* can no longer do a mrq abort */
atomic_set(&fp->mrq_id_abort, 0);
+ if (test_and_clear_bit(SG_FFD_CAN_REUSE_BIO, fp->ffd_bm))
+ sg_svb_cleanup(fp);
return res;
}
#if IS_ENABLED(SG_LOG_ACTIVE)
+/* Returns a descriptive string for the different mrq varieties */
static const char *
-sg_mrq_name(bool from_sg_io, u32 flags)
+sg_mrq_var_str(bool from_sg_io, u32 flags)
{
if (!(flags & SGV4_FLAG_MULTIPLE_REQS))
return "_not_ multiple requests control object";
@@ -2063,7 +2113,8 @@ sg_mrq_name(bool from_sg_io, u32 flags)
* Implements the multiple request functionality. When from_sg_io is true invocation was via
* ioctl(SG_IO), otherwise it was via ioctl(SG_IOSUBMIT). Submit non-blocking if IMMED flag given
* or when ioctl(SG_IOSUBMIT) is used with O_NONBLOCK set on its file descriptor. Hipri
- * non-blocking is when the HIPRI flag is given.
+ * non-blocking is when the HIPRI flag is given. Note that on this fd, svb cannot be started
+ * if any mrq is in progress and no mrq can be started if svb is in progress.
*/
static int
sg_do_multi_req(struct sg_comm_wr_t *cwrp, bool from_sg_io)
@@ -2085,13 +2136,13 @@ sg_do_multi_req(struct sg_comm_wr_t *cwrp, bool from_sg_io)
struct sg_mrq_hold mh;
struct sg_mrq_hold *mhp = &mh;
#if IS_ENABLED(SG_LOG_ACTIVE)
- const char *mrq_name;
+ const char *mrq_vs;
#endif
mhp->cwrp = cwrp;
mhp->from_sg_io = from_sg_io; /* false if from SG_IOSUBMIT */
#if IS_ENABLED(SG_LOG_ACTIVE)
- mrq_name = sg_mrq_name(from_sg_io, cop->flags);
+ mrq_vs = sg_mrq_var_str(from_sg_io, cop->flags);
#endif
f_non_block = !!(fp->filp->f_flags & O_NONBLOCK);
is_svb = !!(cop->flags & SGV4_FLAG_SHARE); /* via ioctl(SG_IOSUBMIT) only */
@@ -2135,7 +2186,7 @@ sg_do_multi_req(struct sg_comm_wr_t *cwrp, bool from_sg_io)
}
if (!mhp->immed && f_non_block)
mhp->immed = true; /* hmm, think about this */
- SG_LOG(3, fp, "%s: %s, tot_reqs=%u, id_of_mrq=%d\n", __func__, mrq_name, tot_reqs,
+ SG_LOG(3, fp, "%s: %s, tot_reqs=%u, id_of_mrq=%d\n", __func__, mrq_vs, tot_reqs,
mhp->id_of_mrq);
sg_v4h_partial_zero(cop);
@@ -2177,8 +2228,13 @@ sg_do_multi_req(struct sg_comm_wr_t *cwrp, bool from_sg_io)
if (SG_IS_DETACHING(sdp) || (o_sfp && SG_IS_DETACHING(o_sfp->parentdp)))
return -ENODEV;
- if (is_svb && unlikely(test_and_set_bit(SG_FFD_SVB_ACTIVE, fp->ffd_bm))) {
- SG_LOG(1, fp, "%s: %s already active\n", __func__, mrq_name);
+ if (is_svb) {
+ if (unlikely(test_and_set_bit(SG_FFD_SVB_ACTIVE, fp->ffd_bm))) {
+ SG_LOG(1, fp, "%s: %s already active\n", __func__, mrq_vs);
+ return -EBUSY;
+ }
+ } else if (unlikely(test_bit(SG_FFD_SVB_ACTIVE, fp->ffd_bm))) {
+ SG_LOG(1, fp, "%s: %s disallowed with existing svb\n", __func__, mrq_vs);
return -EBUSY;
}
a_hds = kcalloc(tot_reqs, SZ_SG_IO_V4, GFP_KERNEL | __GFP_NOWARN);
@@ -2205,11 +2261,10 @@ sg_do_multi_req(struct sg_comm_wr_t *cwrp, bool from_sg_io)
mhp->cdb_ap = cdb_ap;
mhp->a_hds = a_hds;
mhp->cdb_mxlen = cdb_mxlen;
- /* do sanity checks on all requests before starting */
- if (unlikely(!sg_mrq_sanity(mhp, is_svb))) {
- res = -ERANGE;
+ /* do pre-scan on mrq array for sanity and fix-ups */
+ res = sg_mrq_prepare(mhp, is_svb);
+ if (unlikely(res))
goto fini;
- }
/* override cmd queuing setting to allow */
clear_bit(SG_FFD_NO_CMD_Q, fp->ffd_bm);
@@ -2276,8 +2331,8 @@ sg_submit_v4(struct sg_fd *sfp, void __user *p, struct sg_io_v4 *h4p, bool from_
clear_bit(SG_FFD_NO_CMD_Q, sfp->ffd_bm);
ul_timeout = msecs_to_jiffies(h4p->timeout);
cwr.sfp = sfp;
- __assign_bit(SG_FRQ_SYNC_INVOC, cwr.frq_bm, (int)from_sg_io);
- __set_bit(SG_FRQ_IS_V4I, cwr.frq_bm);
+ __assign_bit(SG_FRQ_PC_SYNC_INVOC, cwr.frq_pc_bm, (int)from_sg_io);
+ __set_bit(SG_FRQ_PC_IS_V4I, cwr.frq_pc_bm);
cwr.h4p = h4p;
cwr.timeout = min_t(unsigned long, ul_timeout, INT_MAX);
cwr.cmd_len = h4p->request_len;
@@ -2596,11 +2651,11 @@ sg_get_probable_read_side(struct sg_fd *sfp)
struct sg_request *rs_srp;
struct sg_request *rs_inactive_srp = NULL;
- for (rapp = sfp->rsv_arr; rapp < rapp + SG_MAX_RSV_REQS; ++rapp) {
+ for (rapp = sfp->rsv_arr; rapp < sfp->rsv_arr + SG_MAX_RSV_REQS; ++rapp) {
rs_srp = *rapp;
if (IS_ERR_OR_NULL(rs_srp) || rs_srp->sh_srp)
continue;
- switch (atomic_read_acquire(&rs_srp->rq_st)) {
+ switch (atomic_read(&rs_srp->rq_st)) {
case SG_RQ_INFLIGHT:
case SG_RQ_AWAIT_RCV:
case SG_RQ_BUSY:
@@ -2685,7 +2740,7 @@ sg_execute_cmd(struct sg_fd *sfp, struct sg_request *srp)
srp->start_ns = ktime_get_boottime_ns();/* assume always > 0 */
srp->duration = 0;
- if (!test_bit(SG_FRQ_IS_V4I, srp->frq_bm) && srp->s_hdr3.interface_id == '\0')
+ if (!test_bit(SG_FRQ_PC_IS_V4I, srp->frq_pc_bm) && srp->s_hdr3.interface_id == '\0')
at_head = true; /* backward compatibility for v1+v2 interfaces */
else if (test_bit(SG_FFD_Q_AT_TAIL, sfp->ffd_bm))
/* cmd flags can override sfd setting */
@@ -2696,16 +2751,16 @@ sg_execute_cmd(struct sg_fd *sfp, struct sg_request *srp)
kref_get(&sfp->f_ref); /* put usually in: sg_rq_end_io() */
sg_rq_chg_state_force(srp, SG_RQ_INFLIGHT);
/* >>>>>>> send cmd/req off to other levels <<<<<<<< */
- if (!test_bit(SG_FRQ_SYNC_INVOC, srp->frq_bm)) {
+ if (!test_bit(SG_FRQ_PC_SYNC_INVOC, srp->frq_pc_bm)) {
atomic_inc(&sfp->submitted);
- set_bit(SG_FRQ_COUNT_ACTIVE, srp->frq_bm);
+ set_bit(SG_FRQ_PC_COUNT_ACTIVE, srp->frq_pc_bm);
}
if (srp->rq_flags & SGV4_FLAG_HIPRI) {
rqq->cmd_flags |= REQ_HIPRI;
srp->cookie = request_to_qc_t(rqq->mq_hctx, rqq);
}
blk_execute_rq_nowait(sdp->disk, rqq, (int)at_head, sg_rq_end_io);
- set_bit(SG_FRQ_ISSUED, srp->frq_bm);
+ set_bit(SG_FRQ_PC_ISSUED, srp->frq_pc_bm);
}
/*
@@ -2729,7 +2784,7 @@ sg_common_write(struct sg_comm_wr_t *cwrp)
struct sg_io_hdr *hi_p;
struct sg_io_v4 *h4p;
- if (likely(test_bit(SG_FRQ_IS_V4I, cwrp->frq_bm))) {
+ if (likely(test_bit(SG_FRQ_PC_IS_V4I, cwrp->frq_pc_bm))) {
h4p = cwrp->h4p;
hi_p = NULL;
dir = SG_DXFER_NONE;
@@ -2830,7 +2885,7 @@ sg_get_ready_srp(struct sg_fd *sfp, struct sg_request **srpp, int id, bool is_ta
/* Returns number of bytes copied to user space provided sense buffer or negated errno value. */
static int
-sg_copy_sense(struct sg_request *srp, bool v4_active)
+sg_copy_sense(struct sg_request *srp)
{
int sb_len_ret = 0;
int scsi_stat;
@@ -2845,7 +2900,7 @@ sg_copy_sense(struct sg_request *srp, bool v4_active)
void __user *up;
srp->sense_bp = NULL;
- if (v4_active) {
+ if (SG_IS_V4I(srp)) {
up = uptr64(srp->s_hdr4.sbp);
mx_sb_len = srp->s_hdr4.max_sb_len;
} else {
@@ -2866,7 +2921,7 @@ sg_copy_sense(struct sg_request *srp, bool v4_active)
}
static int
-sg_rec_state_v3v4(struct sg_fd *sfp, struct sg_request *srp, bool v4_active)
+sg_rec_state_v3v4(struct sg_fd *sfp, struct sg_request *srp)
{
int err = 0;
u32 rq_res = srp->rq_result;
@@ -2877,13 +2932,13 @@ sg_rec_state_v3v4(struct sg_fd *sfp, struct sg_request *srp, bool v4_active)
if (unlikely(!sg_result_is_good(rq_res))) {
srp->rq_info |= SG_INFO_CHECK;
if (!scsi_status_is_good(rq_res)) {
- int sb_len_wr = sg_copy_sense(srp, v4_active);
+ int sb_len_wr = sg_copy_sense(srp);
if (unlikely(sb_len_wr < 0))
return sb_len_wr;
}
}
- if (unlikely(test_bit(SG_FRQ_ABORTING, srp->frq_bm)))
+ if (unlikely(test_bit(SG_FRQ_PC_ABORTING, srp->frq_pc_bm)))
srp->rq_info |= SG_INFO_ABORTED;
if (sh_var == SG_SHR_WS_RQ && sg_fd_is_shared(sfp)) {
@@ -2930,7 +2985,7 @@ sg_rec_state_v3v4(struct sg_fd *sfp, struct sg_request *srp, bool v4_active)
rs_srp->sh_var = SG_SHR_NONE;
sg_rq_chg_state_force(rs_srp, SG_RQ_INACTIVE);
atomic_inc(&rs_srp->parentfp->inactives);
- rs_srp->frq_bm[0] &= (1 << SG_FRQ_RESERVED);
+ rs_srp->frq_pc_bm[0] = 0;
rs_srp->in_resid = 0;
rs_srp->rq_info = 0;
rs_srp->sense_len = 0;
@@ -3008,7 +3063,7 @@ sg_receive_v4(struct sg_fd *sfp, struct sg_request *srp, void __user *p, struct
SG_LOG(3, sfp, "%s: p=%s, h4p=%s\n", __func__, (p ? "given" : "NULL"),
(h4p ? "given" : "NULL"));
- err = sg_rec_state_v3v4(sfp, srp, true);
+ err = sg_rec_state_v3v4(sfp, srp);
h4p->guard = 'Q';
h4p->protocol = 0;
h4p->subprotocol = 0;
@@ -3155,7 +3210,7 @@ sg_mrq_ioreceive(struct sg_fd *sfp, struct sg_io_v4 *cop, void __user *p, bool n
return res;
}
-// sg_wait_id_event
+/* Either wait for command completion matching id ('-1': any); or poll for it if do_poll==true */
static int
sg_wait_poll_by_id(struct sg_fd *sfp, struct sg_request **srpp, int id,
bool is_tag, int do_poll)
@@ -3248,7 +3303,7 @@ sg_ctl_ioreceive(struct sg_fd *sfp, void __user *p)
if (unlikely(res))
return res; /* signal --> -ERESTARTSYS */
}
- if (test_and_set_bit(SG_FRQ_RECEIVING, srp->frq_bm)) {
+ if (test_and_set_bit(SG_FRQ_PC_RECEIVING, srp->frq_pc_bm)) {
cpu_relax();
goto try_again;
}
@@ -3302,7 +3357,7 @@ sg_ctl_ioreceive_v3(struct sg_fd *sfp, void __user *p)
if (IS_ERR(srp))
return PTR_ERR(srp);
}
- if (test_and_set_bit(SG_FRQ_RECEIVING, srp->frq_bm)) {
+ if (test_and_set_bit(SG_FRQ_PC_RECEIVING, srp->frq_pc_bm)) {
cpu_relax();
goto try_again;
}
@@ -3475,7 +3530,7 @@ sg_read(struct file *filp, char __user *p, size_t count, loff_t *ppos)
if (IS_ERR(srp))
return PTR_ERR(srp);
}
- if (test_and_set_bit(SG_FRQ_RECEIVING, srp->frq_bm)) {
+ if (test_and_set_bit(SG_FRQ_PC_RECEIVING, srp->frq_pc_bm)) {
cpu_relax();
goto try_again;
}
@@ -3511,7 +3566,7 @@ sg_receive_v3(struct sg_fd *sfp, struct sg_request *srp, void __user *p)
SG_LOG(3, sfp, "%s: sh_var: %s srp=0x%pK\n", __func__, sg_shr_str(srp->sh_var, false),
srp);
- err = sg_rec_state_v3v4(sfp, srp, false);
+ err = sg_rec_state_v3v4(sfp, srp);
memset(hp, 0, sizeof(*hp));
memcpy(hp, &srp->s_hdr3, sizeof(srp->s_hdr3));
hp->sb_len_wr = srp->sense_len;
@@ -3645,7 +3700,6 @@ sg_finish_rs_rq(struct sg_fd *sfp, struct sg_request *rs_srp, bool even_if_in_ws
atomic_inc(&rs_sfp->inactives);
rs_rsv_srp->tag = SG_TAG_WILDCARD;
rs_rsv_srp->sh_var = SG_SHR_NONE;
- set_bit(SG_FRQ_RESERVED, rs_rsv_srp->frq_bm);
rs_rsv_srp->in_resid = 0;
rs_rsv_srp->rq_info = 0;
rs_rsv_srp->sense_len = 0;
@@ -3758,7 +3812,7 @@ sg_remove_sfp_share(struct sg_fd *sfp, bool is_rd_side)
rsv_srp = *rapp;
if (IS_ERR_OR_NULL(rsv_srp) || rsv_srp->sh_var != SG_SHR_RS_RQ)
continue;
- sr_st = atomic_read_acquire(&rsv_srp->rq_st);
+ sr_st = atomic_read(&rsv_srp->rq_st);
switch (sr_st) {
case SG_RQ_SHR_SWAP:
set_inactive = true;
@@ -3949,8 +4003,8 @@ sg_fill_request_element(struct sg_fd *sfp, struct sg_request *srp, struct sg_req
rip->duration = sg_get_dur(srp, NULL, test_bit(SG_FFD_TIME_IN_NS, sfp->ffd_bm), NULL);
if (rip->duration == U32_MAX)
rip->duration = 0;
- rip->orphan = test_bit(SG_FRQ_IS_ORPHAN, srp->frq_bm);
- rip->sg_io_owned = test_bit(SG_FRQ_SYNC_INVOC, srp->frq_bm);
+ rip->orphan = test_bit(SG_FRQ_PC_IS_ORPHAN, srp->frq_pc_bm);
+ rip->sg_io_owned = test_bit(SG_FRQ_PC_SYNC_INVOC, srp->frq_pc_bm);
rip->problem = !sg_result_is_good(srp->rq_result);
rip->pack_id = test_bit(SG_FFD_PREFER_TAG, sfp->ffd_bm) ? srp->tag : srp->pack_id;
rip->usr_ptr = SG_IS_V4I(srp) ? uptr64(srp->s_hdr4.usr_ptr) : srp->s_hdr3.usr_ptr;
@@ -4095,7 +4149,7 @@ sg_abort_req(struct sg_fd *sfp, struct sg_request *srp)
enum sg_rq_state rq_st;
struct request *rqq;
- if (test_and_set_bit(SG_FRQ_ABORTING, srp->frq_bm)) {
+ if (test_and_set_bit(SG_FRQ_PC_ABORTING, srp->frq_pc_bm)) {
SG_LOG(1, sfp, "%s: already aborting req pack_id/tag=%d/%d\n", __func__,
srp->pack_id, srp->tag);
goto fini; /* skip quietly if already aborted */
@@ -4105,16 +4159,16 @@ sg_abort_req(struct sg_fd *sfp, struct sg_request *srp)
sg_rq_st_str(rq_st, false));
switch (rq_st) {
case SG_RQ_BUSY:
- clear_bit(SG_FRQ_ABORTING, srp->frq_bm);
+ clear_bit(SG_FRQ_PC_ABORTING, srp->frq_pc_bm);
res = -EBUSY; /* should not occur often */
break;
case SG_RQ_INACTIVE: /* perhaps done already */
- clear_bit(SG_FRQ_ABORTING, srp->frq_bm);
+ clear_bit(SG_FRQ_PC_ABORTING, srp->frq_pc_bm);
break;
case SG_RQ_AWAIT_RCV: /* user should still do completion */
case SG_RQ_SHR_SWAP:
case SG_RQ_SHR_IN_WS:
- clear_bit(SG_FRQ_ABORTING, srp->frq_bm);
+ clear_bit(SG_FRQ_PC_ABORTING, srp->frq_pc_bm);
break; /* nothing to do here, return 0 */
case SG_RQ_INFLIGHT: /* only attempt abort if inflight */
srp->rq_result |= (DRIVER_SOFT << 24);
@@ -4125,7 +4179,7 @@ sg_abort_req(struct sg_fd *sfp, struct sg_request *srp)
}
break;
default:
- clear_bit(SG_FRQ_ABORTING, srp->frq_bm);
+ clear_bit(SG_FRQ_PC_ABORTING, srp->frq_pc_bm);
break;
}
fini:
@@ -4523,10 +4577,13 @@ sg_set_reserved_sz(struct sg_fd *sfp, int want_rsv_sz)
use_new_srp = true;
xa_for_each_marked(xafp, idx, t_srp, SG_XA_RQ_INACTIVE) {
if (t_srp != o_srp && new_sz <= t_srp->sgatp->buflen) {
+ bool is_reuse_bio = test_bit(SG_FRQ_LT_REUSE_BIO,
+ o_srp->frq_lt_bm);
use_new_srp = false;
xa_lock_irqsave(xafp, iflags);
- __clear_bit(SG_FRQ_RESERVED, o_srp->frq_bm);
- __set_bit(SG_FRQ_RESERVED, t_srp->frq_bm);
+ __clear_bit(SG_FRQ_LT_RESERVED, o_srp->frq_lt_bm);
+ __set_bit(SG_FRQ_LT_RESERVED, t_srp->frq_lt_bm);
+ __assign_bit(SG_FRQ_LT_REUSE_BIO, t_srp->frq_lt_bm, is_reuse_bio);
*rapp = t_srp;
xa_unlock_irqrestore(xafp, iflags);
sg_remove_srp(n_srp);
@@ -4543,8 +4600,10 @@ sg_set_reserved_sz(struct sg_fd *sfp, int want_rsv_sz)
idx = o_srp->rq_idx;
cxc_srp = __xa_cmpxchg(xafp, idx, o_srp, n_srp, GFP_ATOMIC);
if (o_srp == cxc_srp) {
- __assign_bit(SG_FRQ_RESERVED, n_srp->frq_bm,
- test_bit(SG_FRQ_RESERVED, o_srp->frq_bm));
+ __assign_bit(SG_FRQ_LT_RESERVED, n_srp->frq_lt_bm,
+ test_bit(SG_FRQ_LT_RESERVED, o_srp->frq_lt_bm));
+ __assign_bit(SG_FRQ_LT_REUSE_BIO, n_srp->frq_lt_bm,
+ test_bit(SG_FRQ_LT_REUSE_BIO, o_srp->frq_lt_bm));
*rapp = n_srp;
sg_rq_chg_state_force_ulck(n_srp, SG_RQ_INACTIVE);
/* no bump of sfp->inactives since replacement */
@@ -4608,7 +4667,7 @@ sg_any_persistent_orphans(struct sg_fd *sfp)
if (sg_num_waiting_maybe_acquire(sfp) < 1)
return false;
xa_for_each_marked(xafp, idx, srp, SG_XA_RQ_AWAIT) {
- if (test_bit(SG_FRQ_IS_ORPHAN, srp->frq_bm))
+ if (test_bit(SG_FRQ_PC_IS_ORPHAN, srp->frq_pc_bm))
return true;
}
}
@@ -4771,7 +4830,7 @@ sg_extended_bool_flags(struct sg_fd *sfp, struct sg_extended_info *seip)
if (rs_sfp && !IS_ERR_OR_NULL(rs_sfp->rsv_arr[0])) {
struct sg_request *res_srp = rs_sfp->rsv_arr[0];
- if (atomic_read_acquire(&res_srp->rq_st) == SG_RQ_SHR_SWAP)
+ if (atomic_read(&res_srp->rq_st) == SG_RQ_SHR_SWAP)
c_flgs_val_out |= SG_CTL_FLAGM_READ_SIDE_FINI;
else
c_flgs_val_out &= ~SG_CTL_FLAGM_READ_SIDE_FINI;
@@ -5215,14 +5274,14 @@ sg_ioctl_common(struct file *filp, struct sg_device *sdp, struct sg_fd *sfp, uns
val = -1;
if (test_bit(SG_FFD_PREFER_TAG, sfp->ffd_bm)) {
xa_for_each_marked(&sfp->srp_arr, idx, srp, SG_XA_RQ_AWAIT) {
- if (!test_bit(SG_FRQ_SYNC_INVOC, srp->frq_bm)) {
+ if (!test_bit(SG_FRQ_PC_SYNC_INVOC, srp->frq_pc_bm)) {
val = srp->tag;
break;
}
}
} else {
xa_for_each_marked(&sfp->srp_arr, idx, srp, SG_XA_RQ_AWAIT) {
- if (!test_bit(SG_FRQ_SYNC_INVOC, srp->frq_bm)) {
+ if (!test_bit(SG_FRQ_PC_SYNC_INVOC, srp->frq_pc_bm)) {
val = srp->pack_id;
break;
}
@@ -5471,9 +5530,9 @@ sg_sfp_blk_poll(struct sg_fd *sfp, int loop_count)
xa_lock_irqsave(xafp, iflags);
xa_for_each(xafp, idx, srp) {
if ((srp->rq_flags & SGV4_FLAG_HIPRI) &&
- !test_bit(SG_FRQ_SYNC_INVOC, srp->frq_bm) &&
+ !test_bit(SG_FRQ_PC_SYNC_INVOC, srp->frq_pc_bm) &&
atomic_read(&srp->rq_st) == SG_RQ_INFLIGHT &&
- test_bit(SG_FRQ_ISSUED, srp->frq_bm)) {
+ test_bit(SG_FRQ_PC_ISSUED, srp->frq_pc_bm)) {
xa_unlock_irqrestore(xafp, iflags);
n = sg_srp_q_blk_poll(srp, q, loop_count);
if (n == -ENODATA)
@@ -5655,7 +5714,7 @@ sg_mmap(struct file *filp, struct vm_area_struct *vma)
}
if (srp->sgat_h.page_order > 0 || req_sz > (unsigned long)srp->sgat_h.buflen) {
sg_remove_srp(srp);
- set_bit(SG_FRQ_FOR_MMAP, srp->frq_bm);
+ set_bit(SG_FRQ_PC_FOR_MMAP, srp->frq_pc_bm);
res = sg_mk_sgat(srp, sfp, req_sz);
if (res) {
SG_LOG(1, sfp, "%s: sg_mk_sgat failed, wanted=%lu\n", __func__, req_sz);
@@ -5688,7 +5747,7 @@ sg_uc_rq_end_io_orphaned(struct work_struct *work)
return;
}
SG_LOG(3, sfp, "%s: srp=0x%pK\n", __func__, srp);
- if (test_bit(SG_FRQ_DEACT_ORPHAN, srp->frq_bm)) {
+ if (test_bit(SG_FRQ_PC_DEACT_ORPHAN, srp->frq_pc_bm)) {
sg_finish_scsi_blk_rq(srp); /* clean up orphan case */
sg_deact_request(sfp, srp);
}
@@ -5730,7 +5789,7 @@ sg_rq_end_io(struct request *rqq, blk_status_t status)
srp->in_resid = a_resid;
}
}
- if (unlikely(test_bit(SG_FRQ_ABORTING, srp->frq_bm)) && sg_result_is_good(rq_result))
+ if (unlikely(test_bit(SG_FRQ_PC_ABORTING, srp->frq_pc_bm)) && sg_result_is_good(rq_result))
srp->rq_result |= (DRIVER_HARD << 24);
SG_LOG(6, sfp, "%s: pack/tag_id=%d/%d, cmd=0x%x, res=0x%x\n", __func__, srp->pack_id,
@@ -5764,19 +5823,19 @@ sg_rq_end_io(struct request *rqq, blk_status_t status)
}
}
srp->sense_len = slen;
- if (unlikely(test_bit(SG_FRQ_IS_ORPHAN, srp->frq_bm))) {
+ if (unlikely(test_bit(SG_FRQ_PC_IS_ORPHAN, srp->frq_pc_bm))) {
if (test_bit(SG_FFD_KEEP_ORPHAN, sfp->ffd_bm)) {
- __clear_bit(SG_FRQ_SYNC_INVOC, srp->frq_bm);
+ __clear_bit(SG_FRQ_PC_SYNC_INVOC, srp->frq_pc_bm);
} else {
rqq_state = SG_RQ_BUSY;
- __set_bit(SG_FRQ_DEACT_ORPHAN, srp->frq_bm);
+ __set_bit(SG_FRQ_PC_DEACT_ORPHAN, srp->frq_pc_bm);
}
}
xa_lock_irqsave(&sfp->srp_arr, iflags);
- __set_bit(SG_FRQ_ISSUED, srp->frq_bm);
+ __set_bit(SG_FRQ_PC_ISSUED, srp->frq_pc_bm);
sg_rq_chg_state_force_ulck(srp, rqq_state); /* normally --> SG_RQ_AWAIT_RCV */
WRITE_ONCE(srp->rqq, NULL);
- if (test_bit(SG_FRQ_COUNT_ACTIVE, srp->frq_bm)) {
+ if (test_bit(SG_FRQ_PC_COUNT_ACTIVE, srp->frq_pc_bm)) {
int num = atomic_inc_return(&sfp->waiting);
if (num < 2) {
@@ -6200,6 +6259,7 @@ sg_rq_map_kern(struct sg_request *srp, struct request_queue *q, struct request *
static int
sg_start_req(struct sg_request *srp, struct sg_comm_wr_t *cwrp, int dxfer_dir)
{
+ bool bump_bio_get = false;
bool no_dxfer, us_xfer;
int res = 0;
int dlen = cwrp->dlen;
@@ -6286,7 +6346,7 @@ sg_start_req(struct sg_request *srp, struct sg_comm_wr_t *cwrp, int dxfer_dir)
srp->cmd_opcode = scsi_rp->cmd[0];
no_dxfer = dlen <= 0 || dxfer_dir == SG_DXFER_NONE;
us_xfer = !(rq_flags & (SG_FLAG_NO_DXFER | SG_FLAG_MMAP_IO));
- __assign_bit(SG_FRQ_US_XFER, srp->frq_bm, !no_dxfer && us_xfer);
+ __assign_bit(SG_FRQ_PC_US_XFER, srp->frq_pc_bm, !no_dxfer && us_xfer);
rqq->end_io_data = srp;
scsi_rp->retries = SG_DEFAULT_RETRIES;
req_schp = srp->sgatp;
@@ -6301,7 +6361,24 @@ sg_start_req(struct sg_request *srp, struct sg_comm_wr_t *cwrp, int dxfer_dir)
md = NULL;
if (IS_ENABLED(CONFIG_SCSI_PROC_FS))
cp = "direct_io, ";
- } else { /* normal IO and failed conditions for dio path */
+ } else if (test_bit(SG_FFD_CAN_REUSE_BIO, sfp->ffd_bm)) {
+ if (test_bit(SG_FRQ_LT_REUSE_BIO, srp->frq_lt_bm)) {
+ if (srp->bio) {
+ res = blk_rq_append_bio(rqq, &srp->bio);
+ if (res)
+ SG_LOG(1, sfp, "%s: blk_rq_append_bio err=%d\n", __func__,
+ res);
+ md = NULL;
+ atomic_inc(&sg_tmp_count_reused_bios);
+ } else {
+ res = -EPROTO;
+ }
+ goto fini;
+ } else { /* first use of bio, almost normal setup */
+ md = &map_data;
+ bump_bio_get = true;
+ }
+ } else { /* normal indirect IO */
md = &map_data;
}
@@ -6355,6 +6432,12 @@ sg_start_req(struct sg_request *srp, struct sg_comm_wr_t *cwrp, int dxfer_dir)
#endif
} else { /* transfer data to/from kernel buffers */
res = sg_rq_map_kern(srp, q, rqq, r0w);
+ if (res)
+ goto fini;
+ if (bump_bio_get) { /* keep bio alive to re-use next time */
+ set_bit(SG_FRQ_LT_REUSE_BIO, srp->frq_lt_bm);
+ bio_get(rqq->bio); /* _put() in sg_svb_cleanup() */
+ }
}
fini:
if (unlikely(res)) { /* failure, free up resources */
@@ -6385,11 +6468,12 @@ sg_finish_scsi_blk_rq(struct sg_request *srp)
int ret;
struct sg_fd *sfp = srp->parentfp;
struct request *rqq = READ_ONCE(srp->rqq);
+ struct bio *bio;
__maybe_unused char b[32];
SG_LOG(4, sfp, "%s: srp=0x%pK%s\n", __func__, srp, sg_get_rsv_str_lck(srp, " ", "",
sizeof(b), b));
- if (test_and_clear_bit(SG_FRQ_COUNT_ACTIVE, srp->frq_bm)) {
+ if (test_and_clear_bit(SG_FRQ_PC_COUNT_ACTIVE, srp->frq_pc_bm)) {
if (atomic_dec_and_test(&sfp->submitted))
clear_bit(SG_FFD_HIPRI_SEEN, sfp->ffd_bm);
atomic_dec_return_release(&sfp->waiting);
@@ -6402,16 +6486,18 @@ sg_finish_scsi_blk_rq(struct sg_request *srp)
scsi_req_free_cmd(scsi_req(rqq));
blk_put_request(rqq);
}
- if (likely(srp->bio)) {
- bool us_xfer = test_bit(SG_FRQ_US_XFER, srp->frq_bm);
- struct bio *bio = srp->bio;
+ bio = srp->bio;
+ if (likely(bio)) {
+ bool us_xfer = test_bit(SG_FRQ_PC_US_XFER, srp->frq_pc_bm);
- srp->bio = NULL;
- if (us_xfer && bio) {
+ if (us_xfer) {
ret = blk_rq_unmap_user(bio);
if (unlikely(ret)) /* -EINTR (-4) can be ignored */
SG_LOG(6, sfp, "%s: blk_rq_unmap_user() --> %d\n", __func__, ret);
- }
+ srp->bio = NULL;
+ } else if (!test_bit(SG_FRQ_LT_REUSE_BIO, srp->frq_lt_bm)) {
+ srp->bio = NULL;
+ } /* else may be able to re-use this bio [mrq, uniform svb] */
}
/* In worst case, READ data returned to user space by this point */
}
@@ -6444,7 +6530,8 @@ sg_mk_sgat(struct sg_request *srp, struct sg_fd *sfp, int minlen)
return -ENOMEM;
/* elem_sz must be power of 2 and >= PAGE_SIZE */
- elem_sz = test_bit(SG_FRQ_FOR_MMAP, srp->frq_bm) ? (int)PAGE_SIZE : sfp->sgat_elem_sz;
+ elem_sz = test_bit(SG_FRQ_PC_FOR_MMAP, srp->frq_pc_bm) ? (int)PAGE_SIZE :
+ sfp->sgat_elem_sz;
if (sdp && unlikely(sdp->device->host->unchecked_isa_dma))
mask_ap |= GFP_DMA;
o_order = get_order(elem_sz);
@@ -6606,7 +6693,7 @@ sg_find_srp_by_id(struct sg_fd *sfp, int id, bool is_tag)
for (srp = xa_find(xafp, &idx, end_idx, SG_XA_RQ_AWAIT);
srp;
srp = xa_find_after(xafp, &idx, end_idx, SG_XA_RQ_AWAIT)) {
- if (test_bit(SG_FRQ_SYNC_INVOC, srp->frq_bm))
+ if (test_bit(SG_FRQ_PC_SYNC_INVOC, srp->frq_pc_bm))
continue;
if (unlikely(is_tag)) {
if (srp->tag != id)
@@ -6639,7 +6726,7 @@ sg_find_srp_by_id(struct sg_fd *sfp, int id, bool is_tag)
for (srp = xa_find(xafp, &idx, end_idx, SG_XA_RQ_AWAIT);
srp;
srp = xa_find_after(xafp, &idx, end_idx, SG_XA_RQ_AWAIT)) {
- if (test_bit(SG_FRQ_SYNC_INVOC, srp->frq_bm))
+ if (test_bit(SG_FRQ_PC_SYNC_INVOC, srp->frq_pc_bm))
continue;
res = sg_rq_chg_state(srp, SG_RQ_AWAIT_RCV, SG_RQ_BUSY);
if (likely(res == 0)) {
@@ -6849,7 +6936,7 @@ sg_setup_req_new_srp(struct sg_comm_wr_t *cwrp, bool new_rsv_srp, bool no_reqs,
r_srp);
if (new_rsv_srp) {
fp->rsv_arr[ra_idx] = r_srp;
- set_bit(SG_FRQ_RESERVED, r_srp->frq_bm);
+ set_bit(SG_FRQ_LT_RESERVED, r_srp->frq_lt_bm);
r_srp->sh_srp = NULL;
}
xa_lock_irqsave(xafp, iflags);
@@ -6883,7 +6970,7 @@ sg_setup_req(struct sg_comm_wr_t *cwrp, enum sg_shr_var sh_var)
bool no_reqs = false;
bool ws_rq = false;
bool try_harder = false;
- bool keep_frq_bm = false;
+ bool keep_frq_pc_bm = false;
bool second = false;
int res, ra_idx, l_used_idx;
int dlen = cwrp->dlen;
@@ -6906,7 +6993,7 @@ sg_setup_req(struct sg_comm_wr_t *cwrp, enum sg_shr_var sh_var)
r_srp = NULL;
} else {
atomic_dec(&fp->inactives);
- keep_frq_bm = true;
+ keep_frq_pc_bm = true;
r_srp->sh_srp = NULL;
goto final_setup;
}
@@ -6917,7 +7004,7 @@ sg_setup_req(struct sg_comm_wr_t *cwrp, enum sg_shr_var sh_var)
goto maybe_new;
}
r_srp = fp->rsv_arr[ra_idx];
- sr_st = atomic_read_acquire(&r_srp->rq_st);
+ sr_st = atomic_read(&r_srp->rq_st);
if (sr_st == SG_RQ_INACTIVE) {
res = sg_rq_chg_state(r_srp, sr_st, SG_RQ_BUSY);
if (unlikely(res)) {
@@ -6955,13 +7042,13 @@ sg_setup_req(struct sg_comm_wr_t *cwrp, enum sg_shr_var sh_var)
ws_rq = true;
r_srp = cwrp->possible_srp;
if (r_srp) {
- sr_st = atomic_read_acquire(&r_srp->rq_st);
+ sr_st = atomic_read(&r_srp->rq_st);
if (sr_st == SG_RQ_INACTIVE && dlen <= r_srp->sgat_h.buflen) {
res = sg_rq_chg_state(r_srp, sr_st, SG_RQ_BUSY);
if (likely(res == 0)) {
/* possible_srp bypasses loop to find candidate */
mk_new_srp = false;
- keep_frq_bm = true;
+ keep_frq_pc_bm = true;
goto final_setup;
}
}
@@ -6990,7 +7077,8 @@ sg_setup_req(struct sg_comm_wr_t *cwrp, enum sg_shr_var sh_var)
s_idx = (l_used_idx < 0) ? 0 : l_used_idx;
if (l_used_idx >= 0 && xa_get_mark(xafp, s_idx, SG_XA_RQ_INACTIVE)) {
r_srp = xa_load(xafp, s_idx);
- if (r_srp && (allow_rsv || !test_bit(SG_FRQ_RESERVED, r_srp->frq_bm))) {
+ if (r_srp &&
+ (allow_rsv || !test_bit(SG_FRQ_LT_RESERVED, r_srp->frq_lt_bm))) {
if (r_srp->sgat_h.buflen <= SG_DEF_SECTOR_SZ) {
if (sg_rq_chg_state(r_srp, SG_RQ_INACTIVE,
SG_RQ_BUSY) == 0) {
@@ -7002,7 +7090,7 @@ sg_setup_req(struct sg_comm_wr_t *cwrp, enum sg_shr_var sh_var)
}
}
xa_for_each_marked(xafp, idx, r_srp, SG_XA_RQ_INACTIVE) {
- if (allow_rsv || !test_bit(SG_FRQ_RESERVED, r_srp->frq_bm)) {
+ if (allow_rsv || !test_bit(SG_FRQ_LT_RESERVED, r_srp->frq_lt_bm)) {
if (r_srp->sgat_h.buflen <= SG_DEF_SECTOR_SZ) {
if (sg_rq_chg_state(r_srp, SG_RQ_INACTIVE, SG_RQ_BUSY))
continue;
@@ -7056,7 +7144,7 @@ sg_setup_req(struct sg_comm_wr_t *cwrp, enum sg_shr_var sh_var)
r_srp;
r_srp = xa_find_after(xafp, &idx, end_idx, SG_XA_RQ_INACTIVE)) {
if (dlen <= r_srp->sgat_h.buflen &&
- !test_bit(SG_FRQ_RESERVED, r_srp->frq_bm)) {
+ !test_bit(SG_FRQ_LT_RESERVED, r_srp->frq_lt_bm)) {
if (sg_rq_chg_state(r_srp, SG_RQ_INACTIVE, SG_RQ_BUSY))
continue;
atomic_dec(&fp->inactives);
@@ -7090,19 +7178,8 @@ sg_setup_req(struct sg_comm_wr_t *cwrp, enum sg_shr_var sh_var)
goto start_again;
}
final_setup:
- if (!keep_frq_bm) {
- /* keep SG_FRQ_RESERVED setting from prior/new r_srp; clear rest */
- bool is_rsv = test_bit(SG_FRQ_RESERVED, r_srp->frq_bm);
-
- r_srp->frq_bm[0] = 0;
- if (is_rsv)
- set_bit(SG_FRQ_RESERVED, r_srp->frq_bm);
- /* r_srp inherits these flags from cwrp->frq_bm */
- if (test_bit(SG_FRQ_IS_V4I, cwrp->frq_bm))
- set_bit(SG_FRQ_IS_V4I, r_srp->frq_bm);
- if (test_bit(SG_FRQ_SYNC_INVOC, cwrp->frq_bm))
- set_bit(SG_FRQ_SYNC_INVOC, r_srp->frq_bm);
- }
+ if (!keep_frq_pc_bm)
+ r_srp->frq_pc_bm[0] = cwrp->frq_pc_bm[0];
r_srp->sgatp->dlen = dlen; /* must be <= r_srp->sgat_h.buflen */
r_srp->sh_var = sh_var;
r_srp->cmd_opcode = 0xff; /* set invalid opcode (VS), 0x0 is TUR */
@@ -7143,7 +7220,6 @@ sg_setup_req(struct sg_comm_wr_t *cwrp, enum sg_shr_var sh_var)
static void
sg_deact_request(struct sg_fd *sfp, struct sg_request *srp)
{
- bool is_rsv;
enum sg_rq_state sr_st;
u8 *sbp;
@@ -7152,15 +7228,12 @@ sg_deact_request(struct sg_fd *sfp, struct sg_request *srp)
SG_LOG(3, sfp, "%s: srp=%pK\n", __func__, srp);
sbp = srp->sense_bp;
srp->sense_bp = NULL;
- sr_st = atomic_read_acquire(&srp->rq_st);
+ sr_st = atomic_read(&srp->rq_st);
if (sr_st != SG_RQ_SHR_SWAP) {
/* Called from many contexts, don't know whether xa locks held. So assume not. */
sg_rq_chg_state_force(srp, SG_RQ_INACTIVE);
atomic_inc(&sfp->inactives);
- is_rsv = test_bit(SG_FRQ_RESERVED, srp->frq_bm);
- WRITE_ONCE(srp->frq_bm[0], 0);
- if (is_rsv)
- __set_bit(SG_FRQ_RESERVED, srp->frq_bm);
+ WRITE_ONCE(srp->frq_pc_bm[0], 0);
srp->tag = SG_TAG_WILDCARD;
srp->in_resid = 0;
srp->rq_info = 0;
@@ -7250,7 +7323,7 @@ sg_add_sfp(struct sg_device *sdp, struct file *filp)
}
srp->rq_idx = idx;
srp->parentfp = sfp;
- __set_bit(SG_FRQ_RESERVED, srp->frq_bm);
+ __set_bit(SG_FRQ_LT_RESERVED, srp->frq_lt_bm);
sg_rq_chg_state_force_ulck(srp, SG_RQ_INACTIVE);
atomic_inc(&sfp->inactives);
xa_unlock_irqrestore(xafp, iflags);
@@ -7747,8 +7820,8 @@ sg_proc_debug_fd(struct sg_fd *fp, char *obp, int len, unsigned long idx, bool r
if (xa_get_mark(&fp->srp_arr, idx, SG_XA_RQ_INACTIVE))
continue;
if (set_debug)
- n += scnprintf(obp + n, len - n, " rq_bm=0x%lx", srp->frq_bm[0]);
- else if (test_bit(SG_FRQ_ABORTING, srp->frq_bm))
+ n += scnprintf(obp + n, len - n, " rq_pc_bm=0x%lx", srp->frq_pc_bm[0]);
+ else if (test_bit(SG_FRQ_PC_ABORTING, srp->frq_pc_bm))
n += scnprintf(obp + n, len - n, " abort>> ");
n += sg_proc_debug_sreq(srp, fp->timeout, t_in_ns, false, obp + n, len - n);
++k;
@@ -7765,7 +7838,7 @@ sg_proc_debug_fd(struct sg_fd *fp, char *obp, int len, unsigned long idx, bool r
if (k == 0)
n += scnprintf(obp + n, len - n, " Inactives:\n");
if (set_debug)
- n += scnprintf(obp + n, len - n, " rq_bm=0x%lx", srp->frq_bm[0]);
+ n += scnprintf(obp + n, len - n, " rq_lt_bm=0x%lx", srp->frq_lt_bm[0]);
n += sg_proc_debug_sreq(srp, fp->timeout, t_in_ns, true, obp + n, len - n);
++k;
if ((k % 8) == 0) { /* don't hold up things too long */
@@ -7826,8 +7899,9 @@ sg_proc_seq_show_debug(struct seq_file *s, void *v, bool reduced)
b1[0] = '\0';
if (it && it->index == 0)
- seq_printf(s, "max_active_device=%d def_reserved_size=%d\n", (int)it->max,
- def_reserved_size);
+ seq_printf(s, "max_active_device=%d def_reserved_size=%d num_reused_bios=%d\n",
+ (int)it->max, def_reserved_size,
+ atomic_read(&sg_tmp_count_reused_bios));
fdi_p = it ? &it->fd_index : &k;
bp = kzalloc(bp_len, __GFP_NOWARN | GFP_KERNEL);
if (unlikely(!bp)) {
--
2.25.1
