There are several variants here to cope with. The SGV4_FLAG_HIPRI
flag may be set on the control object and/or set on any number of
requests in the req_arr. To simplify things, if that flag is
set on the control object, then its is set internally on each
request in req_arr. That leaves the case were this flag is cleared
on the control object, but set on some or all of the requests in
req_arr. That will work but it may not be the best approach, see
the next paragraph.
For the shared variable blocking mrq method, if the SGV4_FLAG_HIPRI
flag is set on the control object, then an interleaved blk_poll()
algorithm is performed on inflight requests. This should give
better results than doing blk_poll() on each request until it is
completed before moving onto the next inflight request.
Signed-off-by: Douglas Gilbert <dgilbert@xxxxxxxxxxxx>
---
drivers/scsi/sg.c | 449 ++++++++++++++++++++++++++--------------------
1 file changed, 253 insertions(+), 196 deletions(-)
diff --git a/drivers/scsi/sg.c b/drivers/scsi/sg.c
index 26047a8ff1e2..773843a14038 100644
--- a/drivers/scsi/sg.c
+++ b/drivers/scsi/sg.c
@@ -332,7 +332,6 @@ struct sg_device { /* holds the state of each scsi generic device */
};
struct sg_comm_wr_t { /* arguments to sg_common_write() */
- bool keep_share;
int timeout;
int cmd_len;
int rsv_idx; /* wanted rsv_arr index, def: -1 (anyone) */
@@ -353,10 +352,10 @@ struct sg_mrq_hold { /* for passing context between multiple requests (mrq) func
unsigned from_sg_io:1;
unsigned chk_abort:1;
unsigned immed:1;
- unsigned hipri:1;
unsigned stop_if:1;
unsigned co_mmap:1;
unsigned ordered_wr:1;
+ unsigned hipri:1;
int id_of_mrq;
int s_res; /* secondary error: some-good-then-error; in co.spare_out */
int dtd_errs; /* incremented for each driver/transport/device error */
@@ -394,7 +393,7 @@ static void sg_remove_srp(struct sg_request *srp);
static struct sg_fd *sg_add_sfp(struct sg_device *sdp, struct file *filp);
static void sg_remove_sfp(struct kref *);
static void sg_remove_sfp_share(struct sg_fd *sfp, bool is_rd_side);
-static struct sg_request *sg_find_srp_by_id(struct sg_fd *sfp, int id, bool is_tag);
+static struct sg_request *sg_get_srp_by_id(struct sg_fd *sfp, int id, bool is_tag, bool part_mrq);
static struct sg_request *sg_setup_req(struct sg_comm_wr_t *cwrp, enum sg_shr_var sh_var);
static void sg_deact_request(struct sg_fd *sfp, struct sg_request *srp);
static struct sg_device *sg_get_dev(int min_dev);
@@ -1163,11 +1162,11 @@ sg_num_waiting_maybe_acquire(struct sg_fd *sfp)
}
/*
- * Returns true if a request is ready and its srp is written to *srpp . If nothing can be found
- * returns false and NULL --> *srpp . If device is detaching, returns true and NULL --> *srpp .
+ * Looks for request in SG_RQ_AWAIT_RCV state on given fd that matches part_mrq. The first one
+ * found is placed in SG_RQ_BUSY state and its address is returned. If none found returns NULL.
*/
-static bool
-sg_mrq_get_ready_srp(struct sg_fd *sfp, struct sg_request **srpp)
+static struct sg_request *
+sg_get_any_srp(struct sg_fd *sfp, bool part_mrq)
{
bool second = false;
int l_await_idx = READ_ONCE(sfp->low_await_idx);
@@ -1175,25 +1174,18 @@ sg_mrq_get_ready_srp(struct sg_fd *sfp, struct sg_request **srpp)
struct sg_request *srp;
struct xarray *xafp = &sfp->srp_arr;
- if (SG_IS_DETACHING(sfp->parentdp)) {
- *srpp = ERR_PTR(-ENODEV);
- return true;
- }
- if (sg_num_waiting_maybe_acquire(sfp) < 1)
- goto fini;
-
s_idx = (l_await_idx < 0) ? 0 : l_await_idx;
idx = s_idx;
end_idx = ULONG_MAX;
-
second_time:
for (srp = xa_find(xafp, &idx, end_idx, SG_XA_RQ_AWAIT);
srp;
srp = xa_find_after(xafp, &idx, end_idx, SG_XA_RQ_AWAIT)) {
+ if (part_mrq != test_bit(SG_FRQ_PC_PART_MRQ, srp->frq_pc_bm))
+ continue;
if (likely(sg_rq_chg_state(srp, SG_RQ_AWAIT_RCV, SG_RQ_BUSY) == 0)) {
- *srpp = srp;
WRITE_ONCE(sfp->low_await_idx, idx + 1);
- return true;
+ return srp;
}
}
/* If not found so far, need to wrap around and search [0 ... s_idx) */
@@ -1204,9 +1196,33 @@ sg_mrq_get_ready_srp(struct sg_fd *sfp, struct sg_request **srpp)
second = true;
goto second_time;
}
-fini:
- *srpp = NULL;
- return false;
+ return NULL;
+}
+
+/*
+ * Returns true if a request is ready and its srp is written to *srpp . If nothing can be found
+ * returns false and NULL --> *srpp . If an error is detected returns true with IS_ERR(*srpp)
+ * also being true.
+ */
+static bool
+sg_mrq_get_ready_srp(struct sg_fd *sfp, struct sg_request **srpp)
+{
+ if (SG_IS_DETACHING(sfp->parentdp)) {
+ *srpp = ERR_PTR(-ENODEV);
+ return true;
+ }
+ if (sg_num_waiting_maybe_acquire(sfp) < 1) {
+ if (test_bit(SG_FFD_HIPRI_SEEN, sfp->ffd_bm)) {
+ int res = sg_sfp_blk_poll(sfp, 1);
+
+ if (res < 0) {
+ *srpp = ERR_PTR(res);
+ return true;
+ }
+ }
+ }
+ *srpp = sg_get_any_srp(sfp, true);
+ return !!*srpp;
}
/* N.B. After this function is completed what srp points to should be considered invalid. */
@@ -1256,84 +1272,67 @@ sg_mrq_1complet(struct sg_mrq_hold *mhp, struct sg_request *srp)
}
static int
-sg_wait_any_mrq(struct sg_fd *sfp, struct sg_request **srpp)
+sg_wait_any_mrq(struct sg_fd *sfp, struct sg_mrq_hold *mhp, struct sg_request **srpp)
{
+ bool hipri = mhp->hipri || test_bit(SG_FFD_HIPRI_SEEN, sfp->ffd_bm);
+
+ if (hipri) {
+ long state = current->state;
+ struct sg_request *srp;
+
+ do {
+ if (hipri) {
+ int res = sg_sfp_blk_poll(sfp, SG_DEF_BLK_POLL_LOOP_COUNT);
+
+ if (res < 0)
+ return res;
+ }
+ srp = sg_get_any_srp(sfp, true);
+ if (IS_ERR(srp))
+ return PTR_ERR(srp);
+ if (srp) {
+ __set_current_state(TASK_RUNNING);
+ break;
+ }
+ if (SG_IS_DETACHING(sfp->parentdp)) {
+ __set_current_state(TASK_RUNNING);
+ return -ENODEV;
+ }
+ if (signal_pending_state(state, current)) {
+ __set_current_state(TASK_RUNNING);
+ return -ERESTARTSYS;
+ }
+ cpu_relax();
+ } while (true);
+ *srpp = srp;
+ return 0;
+ }
if (test_bit(SG_FFD_EXCL_WAITQ, sfp->ffd_bm))
return __wait_event_interruptible_exclusive(sfp->cmpl_wait,
sg_mrq_get_ready_srp(sfp, srpp));
return __wait_event_interruptible(sfp->cmpl_wait, sg_mrq_get_ready_srp(sfp, srpp));
}
-static bool
-sg_srp_hybrid_sleep(struct sg_request *srp)
-{
- struct hrtimer_sleeper hs;
- enum hrtimer_mode mode;
- ktime_t kt = ns_to_ktime(5000);
-
- if (test_and_set_bit(SG_FRQ_POLL_SLEPT, srp->frq_pc_bm))
- return false;
- if (kt == 0)
- return false;
-
- mode = HRTIMER_MODE_REL;
- hrtimer_init_sleeper_on_stack(&hs, CLOCK_MONOTONIC, mode);
- hrtimer_set_expires(&hs.timer, kt);
-
- do {
- if (atomic_read(&srp->rq_st) != SG_RQ_INFLIGHT)
- break;
- set_current_state(TASK_UNINTERRUPTIBLE);
- hrtimer_sleeper_start_expires(&hs, mode);
- if (hs.task)
- io_schedule();
- hrtimer_cancel(&hs.timer);
- mode = HRTIMER_MODE_ABS;
- } while (hs.task && !signal_pending(current));
-
- __set_current_state(TASK_RUNNING);
- destroy_hrtimer_on_stack(&hs.timer);
- return true;
-}
-
static inline bool
sg_rq_landed(struct sg_device *sdp, struct sg_request *srp)
{
return atomic_read_acquire(&srp->rq_st) != SG_RQ_INFLIGHT || SG_IS_DETACHING(sdp);
}
-/* This is a blocking wait (or poll) for a given srp. */
+/*
+ * This is a blocking wait (or poll) for a given srp to reach completion. If
+ * SGV4_FLAG_HIPRI is set this functions goes into a polling loop.
+ */
static int
-sg_wait_poll_for_given_srp(struct sg_fd *sfp, struct sg_request *srp, bool do_poll)
+sg_poll_wait4_given_srp(struct sg_fd *sfp, struct sg_request *srp)
{
int res;
struct sg_device *sdp = sfp->parentdp;
- SG_LOG(3, sfp, "%s: do_poll=%d\n", __func__, (int)do_poll);
- if (do_poll || (srp->rq_flags & SGV4_FLAG_HIPRI))
- goto poll_loop;
-
- if (atomic_read(&srp->rq_st) != SG_RQ_INFLIGHT)
- goto skip_wait; /* and skip _acquire() */
- /* N.B. The SG_FFD_EXCL_WAITQ flag is ignored here. */
- res = __wait_event_interruptible(sfp->cmpl_wait, sg_rq_landed(sdp, srp));
- if (unlikely(res)) { /* -ERESTARTSYS because signal hit thread */
- set_bit(SG_FRQ_PC_IS_ORPHAN, srp->frq_pc_bm);
- /* orphans harvested when sfp->keep_orphan is false */
- sg_rq_chg_state_force(srp, SG_RQ_INFLIGHT);
- SG_LOG(1, sfp, "%s: wait_event_interruptible(): %s[%d]\n", __func__,
- (res == -ERESTARTSYS ? "ERESTARTSYS" : ""), res);
- return res;
- }
-skip_wait:
- if (SG_IS_DETACHING(sdp))
- goto detaching;
- return sg_rq_chg_state(srp, SG_RQ_AWAIT_RCV, SG_RQ_BUSY);
-
-poll_loop:
if (srp->rq_flags & SGV4_FLAG_HIPRI) {
long state = current->state;
+ SG_LOG(3, sfp, "%s: polling\n", __func__);
do {
res = sg_srp_q_blk_poll(srp, sdp->device->request_queue,
SG_DEF_BLK_POLL_LOOP_COUNT);
@@ -1356,18 +1355,17 @@ sg_wait_poll_for_given_srp(struct sg_fd *sfp, struct sg_request *srp, bool do_po
cpu_relax();
} while (true);
} else {
- enum sg_rq_state sr_st;
-
- if (!sg_srp_hybrid_sleep(srp))
- return -EINVAL;
- if (signal_pending(current))
- return -ERESTARTSYS;
- if (SG_IS_DETACHING(sdp))
- goto detaching;
- sr_st = atomic_read(&srp->rq_st);
- if (unlikely(sr_st != SG_RQ_AWAIT_RCV))
- return -EPROTO; /* Logic error */
- return sg_rq_chg_state(srp, sr_st, SG_RQ_BUSY);
+ SG_LOG(3, sfp, "%s: wait_event\n", __func__);
+ /* N.B. The SG_FFD_EXCL_WAITQ flag is ignored here. */
+ res = __wait_event_interruptible(sfp->cmpl_wait, sg_rq_landed(sdp, srp));
+ if (unlikely(res)) { /* -ERESTARTSYS because signal hit thread */
+ set_bit(SG_FRQ_PC_IS_ORPHAN, srp->frq_pc_bm);
+ /* orphans harvested when sfp->keep_orphan is false */
+ sg_rq_chg_state_force(srp, SG_RQ_INFLIGHT);
+ SG_LOG(1, sfp, "%s: wait_event_interruptible(): %s[%d]\n", __func__,
+ (res == -ERESTARTSYS ? "ERESTARTSYS" : ""), res);
+ return res;
+ }
}
if (atomic_read_acquire(&srp->rq_st) != SG_RQ_AWAIT_RCV)
return (test_bit(SG_FRQ_PC_COUNT_ACTIVE, srp->frq_pc_bm) &&
@@ -1444,7 +1442,7 @@ sg_mrq_complets(struct sg_mrq_hold *mhp, struct sg_fd *sfp, struct sg_fd *sec_sf
if (res)
break;
if (mreqs > 0) {
- res = sg_wait_any_mrq(sfp, &srp);
+ res = sg_wait_any_mrq(sfp, mhp, &srp);
if (unlikely(res))
return res; /* signal --> -ERESTARTSYS */
if (IS_ERR(srp)) {
@@ -1457,7 +1455,7 @@ sg_mrq_complets(struct sg_mrq_hold *mhp, struct sg_fd *sfp, struct sg_fd *sec_sf
}
}
if (sec_reqs > 0) {
- res = sg_wait_any_mrq(sec_sfp, &srp);
+ res = sg_wait_any_mrq(sec_sfp, mhp, &srp);
if (unlikely(res))
return res; /* signal --> -ERESTARTSYS */
if (IS_ERR(srp)) {
@@ -1551,13 +1549,16 @@ sg_mrq_prepare(struct sg_mrq_hold *mhp, bool is_svb)
SG_LOG(1, sfp, "%s: %s %u, MMAP in co AND here\n", __func__, rip, k);
return -ERANGE;
}
- if (unlikely(!have_file_share && share)) {
- SG_LOG(1, sfp, "%s: %s %u, no file share\n", __func__, rip, k);
- return -ENOMSG;
- }
- if (unlikely(!have_file_share && !!(flags & SGV4_FLAG_DO_ON_OTHER))) {
- SG_LOG(1, sfp, "%s: %s %u, no other fd to do on\n", __func__, rip, k);
- return -ENOMSG;
+ if (!have_file_share) {
+ if (unlikely(share || (flags & SGV4_FLAG_DO_ON_OTHER))) {
+ if (share)
+ SG_LOG(1, sfp, "%s: %s %u, no file share\n", __func__,
+ rip, k);
+ else
+ SG_LOG(1, sfp, "%s: %s %u, no other fd to do on\n",
+ __func__, rip, k);
+ return -ENOMSG;
+ }
}
if (cdb_ap && unlikely(hp->request_len > cdb_mxlen)) {
SG_LOG(1, sfp, "%s: %s %u, cdb too long\n", __func__, rip, k);
@@ -1567,10 +1568,16 @@ sg_mrq_prepare(struct sg_mrq_hold *mhp, bool is_svb)
hp->response = cop->response;
hp->max_response_len = cop->max_response_len;
}
+ if (mhp->hipri) {
+ if (!(hp->flags & SGV4_FLAG_HIPRI))
+ hp->flags |= SGV4_FLAG_HIPRI;
+ } /* HIPRI may be set on hp->flags and _not_ on the control object */
if (!is_svb) {
- if (cop->flags & SGV4_FLAG_REC_ORDER)
- hp->flags |= SGV4_FLAG_REC_ORDER;
- continue;
+ if (cop->flags & SGV4_FLAG_REC_ORDER) {
+ if (!(hp->flags & SGV4_FLAG_REC_ORDER))
+ hp->flags |= SGV4_FLAG_REC_ORDER;
+ }
+ continue; /* <<< non-svb skip rest of loop */
}
/* mrq share variable blocking (svb) additional constraints checked here */
if (unlikely(flags & (SGV4_FLAG_COMPLETE_B4 | SGV4_FLAG_KEEP_SHARE))) {
@@ -1713,7 +1720,7 @@ sg_process_most_mrq(struct sg_fd *fp, struct sg_fd *o_sfp, struct sg_mrq_hold *m
++other_fp_sent;
continue; /* defer completion until all submitted */
}
- res = sg_wait_poll_for_given_srp(rq_sfp, srp, mhp->hipri);
+ res = sg_poll_wait4_given_srp(rq_sfp, srp);
if (unlikely(res)) {
mhp->s_res = res;
if (res == -ERESTARTSYS || res == -ENODEV)
@@ -1915,7 +1922,7 @@ sg_svb_mrq_first_come(struct sg_fd *fp, struct sg_fd *o_sfp, struct sg_mrq_hold
goto oth_first;
this_second:
if (this_fp_sent > 0) {
- res = sg_wait_any_mrq(fp, &srp);
+ res = sg_wait_any_mrq(fp, mhp, &srp);
if (unlikely(res))
mhp->s_res = res;
else if (IS_ERR(srp))
@@ -1927,7 +1934,7 @@ sg_svb_mrq_first_come(struct sg_fd *fp, struct sg_fd *o_sfp, struct sg_mrq_hold
continue;
oth_first:
if (other_fp_sent > 0) {
- res = sg_wait_any_mrq(o_sfp, &srp);
+ res = sg_wait_any_mrq(o_sfp, mhp, &srp);
if (unlikely(res))
mhp->s_res = res;
else if (IS_ERR(srp))
@@ -2006,7 +2013,7 @@ sg_svb_mrq_ordered(struct sg_fd *fp, struct sg_fd *o_sfp, struct sg_mrq_hold *mh
rs_srp = svb_arr[m].rs_srp;
if (!rs_srp)
continue;
- res = sg_wait_poll_for_given_srp(fp, rs_srp, mhp->hipri);
+ res = sg_poll_wait4_given_srp(fp, rs_srp);
if (unlikely(res))
mhp->s_res = res;
--this_fp_sent;
@@ -2050,7 +2057,7 @@ sg_svb_mrq_ordered(struct sg_fd *fp, struct sg_fd *o_sfp, struct sg_mrq_hold *mh
}
while (this_fp_sent > 0) { /* non-data requests */
--this_fp_sent;
- res = sg_wait_any_mrq(fp, &srp);
+ res = sg_wait_any_mrq(fp, mhp, &srp);
if (unlikely(res)) {
mhp->s_res = res;
continue;
@@ -2066,7 +2073,7 @@ sg_svb_mrq_ordered(struct sg_fd *fp, struct sg_fd *o_sfp, struct sg_mrq_hold *mh
}
while (other_fp_sent > 0) {
--other_fp_sent;
- res = sg_wait_any_mrq(o_sfp, &srp);
+ res = sg_wait_any_mrq(o_sfp, mhp, &srp);
if (unlikely(res)) {
mhp->s_res = res;
continue;
@@ -2199,7 +2206,6 @@ sg_do_multi_req(struct sg_comm_wr_t *cwrp, bool from_sg_io)
{
bool f_non_block, is_svb;
int res = 0;
- int existing_id;
u32 cdb_mxlen;
struct sg_io_v4 *cop = cwrp->h4p; /* controlling object */
u32 dout_len = cop->dout_xfer_len;
@@ -2210,13 +2216,33 @@ sg_do_multi_req(struct sg_comm_wr_t *cwrp, bool from_sg_io)
struct sg_io_v4 *a_hds; /* array of request objects */
struct sg_fd *fp = cwrp->sfp;
struct sg_fd *o_sfp = sg_fd_share_ptr(fp);
- struct sg_device *sdp = fp->parentdp;
struct sg_mrq_hold mh;
struct sg_mrq_hold *mhp = &mh;
#if IS_ENABLED(SG_LOG_ACTIVE)
const char *mrq_vs;
#endif
+ if (unlikely(SG_IS_DETACHING(fp->parentdp) || (o_sfp && SG_IS_DETACHING(o_sfp->parentdp))))
+ return -ENODEV;
+ if (unlikely(tot_reqs == 0))
+ return 0;
+ if (unlikely(tot_reqs > U16_MAX))
+ return -E2BIG;
+ if (unlikely(!!cdb_alen != !!cop->request))
+ return -ERANGE; /* both must be zero or both non-zero */
+ if (cdb_alen) {
+ if (unlikely(cdb_alen > SG_MAX_MULTI_REQ_SZ))
+ return -E2BIG;
+ if (unlikely(cdb_alen % tot_reqs))
+ return -ERANGE;
+ cdb_mxlen = cdb_alen / tot_reqs;
+ if (unlikely(cdb_mxlen < 6))
+ return -ERANGE; /* too short for SCSI cdbs */
+ } else {
+ cdb_mxlen = 0;
+ }
+ if (unlikely(din_len > SG_MAX_MULTI_REQ_SZ || dout_len > SG_MAX_MULTI_REQ_SZ))
+ return -E2BIG;
mhp->cwrp = cwrp;
mhp->from_sg_io = from_sg_io; /* false if from SG_IOSUBMIT */
#if IS_ENABLED(SG_LOG_ACTIVE)
@@ -2227,7 +2253,10 @@ sg_do_multi_req(struct sg_comm_wr_t *cwrp, bool from_sg_io)
mhp->immed = !!(cop->flags & SGV4_FLAG_IMMED);
mhp->hipri = !!(cop->flags & SGV4_FLAG_HIPRI);
mhp->stop_if = !!(cop->flags & SGV4_FLAG_STOP_IF);
+ if (unlikely(mhp->immed && mhp->stop_if))
+ return -ERANGE;
mhp->ordered_wr = !!(cop->flags & SGV4_FLAG_ORDERED_WR);
+ mhp->hipri = !!(cop->flags & SGV4_FLAG_HIPRI);
mhp->co_mmap = !!(cop->flags & SGV4_FLAG_MMAP_IO);
if (mhp->co_mmap)
mhp->co_mmap_sgatp = fp->rsv_arr[0]->sgatp;
@@ -2236,7 +2265,8 @@ sg_do_multi_req(struct sg_comm_wr_t *cwrp, bool from_sg_io)
mhp->s_res = 0;
mhp->dtd_errs = 0;
if (mhp->id_of_mrq) {
- existing_id = atomic_cmpxchg(&fp->mrq_id_abort, 0, mhp->id_of_mrq);
+ int existing_id = atomic_cmpxchg(&fp->mrq_id_abort, 0, mhp->id_of_mrq);
+
if (existing_id && existing_id != mhp->id_of_mrq) {
SG_LOG(1, fp, "%s: existing id=%d id_of_mrq=%d\n", __func__, existing_id,
mhp->id_of_mrq);
@@ -2284,42 +2314,13 @@ sg_do_multi_req(struct sg_comm_wr_t *cwrp, bool from_sg_io)
else
return -EPROTO;
}
- if (din_len > 0) {
- if (unlikely(din_len > SG_MAX_MULTI_REQ_SZ))
- return -E2BIG;
- } else if (dout_len > 0) {
- if (unlikely(dout_len > SG_MAX_MULTI_REQ_SZ))
- return -E2BIG;
- }
- if (unlikely(tot_reqs > U16_MAX)) {
- return -ERANGE;
- } else if (unlikely(mhp->immed && mhp->stop_if)) {
- return -ERANGE;
- } else if (unlikely(tot_reqs == 0)) {
- return 0;
- } else if (unlikely(!!cdb_alen != !!cop->request)) {
- return -ERANGE; /* both must be zero or both non-zero */
- } else if (cdb_alen) {
- if (unlikely(cdb_alen > SG_MAX_MULTI_REQ_SZ))
- return -E2BIG;
- if (unlikely(cdb_alen % tot_reqs))
- return -ERANGE;
- cdb_mxlen = cdb_alen / tot_reqs;
- if (unlikely(cdb_mxlen < 6))
- return -ERANGE; /* too short for SCSI cdbs */
- } else {
- cdb_mxlen = 0;
- }
-
- if (SG_IS_DETACHING(sdp) || (o_sfp && SG_IS_DETACHING(o_sfp->parentdp)))
- return -ENODEV;
if (is_svb) {
if (unlikely(test_and_set_bit(SG_FFD_SVB_ACTIVE, fp->ffd_bm))) {
SG_LOG(1, fp, "%s: %s already active\n", __func__, mrq_vs);
return -EBUSY;
}
} else if (unlikely(test_bit(SG_FFD_SVB_ACTIVE, fp->ffd_bm))) {
- SG_LOG(1, fp, "%s: %s disallowed with existing svb\n", __func__, mrq_vs);
+ SG_LOG(1, fp, "%s: svb active on this fd so %s disallowed\n", __func__, mrq_vs);
return -EBUSY;
}
a_hds = kcalloc(tot_reqs, SZ_SG_IO_V4, GFP_KERNEL | __GFP_NOWARN);
@@ -2887,7 +2888,6 @@ sg_common_write(struct sg_comm_wr_t *cwrp)
res = sg_share_chk_flags(fp, rq_flags, dlen, dir, &sh_var);
if (unlikely(res < 0))
return ERR_PTR(res);
- cwrp->keep_share = !!(rq_flags & SGV4_FLAG_KEEP_SHARE);
} else {
sh_var = SG_SHR_NONE;
if (unlikely(rq_flags & SGV4_FLAG_SHARE))
@@ -2962,8 +2962,21 @@ sg_get_ready_srp(struct sg_fd *sfp, struct sg_request **srpp, int id, bool is_ta
*srpp = ERR_PTR(-ENODEV);
return true;
}
- srp = sg_find_srp_by_id(sfp, id, is_tag);
- *srpp = srp;
+ srp = sg_get_srp_by_id(sfp, id, is_tag, false);
+ *srpp = srp; /* Warning: IS_ERR(srp) may be true */
+ return !!srp;
+}
+
+static inline bool
+sg_get_any_ready_srp(struct sg_fd *sfp, struct sg_request **srpp)
+{
+ struct sg_request *srp;
+
+ if (SG_IS_DETACHING(sfp->parentdp))
+ srp = ERR_PTR(-ENODEV);
+ else
+ srp = sg_get_any_srp(sfp, false);
+ *srpp = srp; /* Warning: IS_ERR(srp) may be true */
return !!srp;
}
@@ -3191,17 +3204,19 @@ sg_receive_v4(struct sg_fd *sfp, struct sg_request *srp, void __user *p, struct
* of elements written to rsp_arr, which may be 0 if mrqs submitted but none waiting
*/
static int
-sg_mrq_iorec_complets(struct sg_fd *sfp, bool non_block, int max_rcv, int num_rsp_arr,
- struct sg_io_v4 *rsp_arr)
+sg_mrq_iorec_complets(struct sg_fd *sfp, struct sg_mrq_hold *mhp, bool non_block, int max_rcv)
{
int k, idx;
int res = 0;
struct sg_request *srp;
+ struct sg_io_v4 *rsp_arr = mhp->a_hds;
- SG_LOG(3, sfp, "%s: num_rsp_arr=%d, max_rcv=%d", __func__, num_rsp_arr, max_rcv);
- if (max_rcv == 0 || max_rcv > num_rsp_arr)
- max_rcv = num_rsp_arr;
+ SG_LOG(3, sfp, "%s: num_responses=%d, max_rcv=%d, hipri=%u\n", __func__,
+ mhp->tot_reqs, max_rcv, mhp->hipri);
+ if (max_rcv == 0 || max_rcv > mhp->tot_reqs)
+ max_rcv = mhp->tot_reqs;
k = 0;
+recheck:
for ( ; k < max_rcv; ++k) {
if (!sg_mrq_get_ready_srp(sfp, &srp))
break;
@@ -3209,7 +3224,7 @@ sg_mrq_iorec_complets(struct sg_fd *sfp, bool non_block, int max_rcv, int num_rs
return k ? k /* some but not all */ : PTR_ERR(srp);
if (srp->rq_flags & SGV4_FLAG_REC_ORDER) {
idx = srp->s_hdr4.mrq_ind;
- if (idx >= num_rsp_arr)
+ if (idx >= mhp->tot_reqs)
idx = 0; /* overwrite index 0 when trouble */
} else {
idx = k; /* completion order */
@@ -3221,16 +3236,24 @@ sg_mrq_iorec_complets(struct sg_fd *sfp, bool non_block, int max_rcv, int num_rs
}
if (non_block || k >= max_rcv)
return k;
+ if (mhp->hipri) {
+ if (SG_IS_DETACHING(sfp->parentdp))
+ return -ENODEV;
+ if (signal_pending(current))
+ return -ERESTARTSYS;
+ cpu_relax();
+ goto recheck;
+ }
SG_LOG(6, sfp, "%s: received=%d, max=%d\n", __func__, k, max_rcv);
for ( ; k < max_rcv; ++k) {
- res = sg_wait_any_mrq(sfp, &srp);
+ res = sg_wait_any_mrq(sfp, mhp, &srp);
if (unlikely(res))
return res; /* signal --> -ERESTARTSYS */
if (IS_ERR(srp))
return k ? k : PTR_ERR(srp);
if (srp->rq_flags & SGV4_FLAG_REC_ORDER) {
idx = srp->s_hdr4.mrq_ind;
- if (idx >= num_rsp_arr)
+ if (idx >= mhp->tot_reqs)
idx = 0;
} else {
idx = k;
@@ -3256,6 +3279,8 @@ sg_mrq_ioreceive(struct sg_fd *sfp, struct sg_io_v4 *cop, void __user *p, bool n
u32 len, n;
struct sg_io_v4 *rsp_v4_arr;
void __user *pp;
+ struct sg_mrq_hold mh;
+ struct sg_mrq_hold *mhp = &mh;
SG_LOG(3, sfp, "%s: non_block=%d\n", __func__, !!non_block);
n = cop->din_xfer_len;
@@ -3263,9 +3288,12 @@ sg_mrq_ioreceive(struct sg_fd *sfp, struct sg_io_v4 *cop, void __user *p, bool n
return -E2BIG;
if (unlikely(!cop->din_xferp || n < SZ_SG_IO_V4 || (n % SZ_SG_IO_V4)))
return -ERANGE;
+ memset(mhp, 0, sizeof(*mhp));
n /= SZ_SG_IO_V4;
+ mhp->tot_reqs = n;
len = n * SZ_SG_IO_V4;
max_rcv = cop->din_iovec_count;
+ mhp->hipri = !!(cop->flags & SGV4_FLAG_HIPRI);
SG_LOG(3, sfp, "%s: %s, num_reqs=%u, max_rcv=%d\n", __func__,
(non_block ? "IMMED" : "blocking"), n, max_rcv);
rsp_v4_arr = kcalloc(n, SZ_SG_IO_V4, GFP_KERNEL);
@@ -3274,7 +3302,8 @@ sg_mrq_ioreceive(struct sg_fd *sfp, struct sg_io_v4 *cop, void __user *p, bool n
sg_v4h_partial_zero(cop);
cop->din_resid = n;
- res = sg_mrq_iorec_complets(sfp, non_block, max_rcv, n, rsp_v4_arr);
+ mhp->a_hds = rsp_v4_arr;
+ res = sg_mrq_iorec_complets(sfp, mhp, non_block, max_rcv);
if (unlikely(res < 0))
goto fini;
cop->din_resid -= res;
@@ -3294,41 +3323,70 @@ sg_mrq_ioreceive(struct sg_fd *sfp, struct sg_io_v4 *cop, void __user *p, bool n
return res;
}
-/* Either wait for command completion matching id ('-1': any); or poll for it if do_poll==true */
+static struct sg_request *
+sg_poll_wait4_srp(struct sg_fd *sfp, int id, bool is_tag, bool part_mrq)
+{
+ long state = current->state;
+ struct sg_request *srp;
+
+ do {
+ if (test_bit(SG_FFD_HIPRI_SEEN, sfp->ffd_bm)) {
+ int res = sg_sfp_blk_poll(sfp, SG_DEF_BLK_POLL_LOOP_COUNT);
+
+ if (res < 0)
+ return ERR_PTR(res);
+ }
+ if (id == -1)
+ srp = sg_get_any_srp(sfp, part_mrq);
+ else
+ srp = sg_get_srp_by_id(sfp, id, is_tag, part_mrq);
+ if (IS_ERR(srp))
+ return srp;
+ if (srp) {
+ __set_current_state(TASK_RUNNING);
+ return srp;
+ }
+ if (SG_IS_DETACHING(sfp->parentdp)) {
+ __set_current_state(TASK_RUNNING);
+ return ERR_PTR(-ENODEV);
+ }
+ if (signal_pending_state(state, current)) {
+ __set_current_state(TASK_RUNNING);
+ return ERR_PTR(-ERESTARTSYS);
+ }
+ cpu_relax();
+ } while (true);
+}
+
+/*
+ * Called from read(), ioctl(SG_IORECEIVE) or ioctl(SG_IORECEIVE_V3). Either wait event for
+ * command completion matching id ('-1': any); or poll for it if do_poll==true
+ */
static int
sg_wait_poll_by_id(struct sg_fd *sfp, struct sg_request **srpp, int id,
bool is_tag, int do_poll)
{
- if (do_poll)
- goto poll_loop;
+ if (do_poll) {
+ struct sg_request *srp = sg_poll_wait4_srp(sfp, id, is_tag, false);
- if (test_bit(SG_FFD_EXCL_WAITQ, sfp->ffd_bm))
- return __wait_event_interruptible_exclusive
+ if (IS_ERR(srp))
+ return PTR_ERR(srp);
+ *srpp = srp;
+ return 0;
+ }
+ if (test_bit(SG_FFD_EXCL_WAITQ, sfp->ffd_bm)) {
+ if (id == -1)
+ return __wait_event_interruptible_exclusive
+ (sfp->cmpl_wait, sg_get_any_ready_srp(sfp, srpp));
+ else
+ return __wait_event_interruptible_exclusive
(sfp->cmpl_wait, sg_get_ready_srp(sfp, srpp, id, is_tag));
- return __wait_event_interruptible(sfp->cmpl_wait, sg_get_ready_srp(sfp, srpp, id, is_tag));
-poll_loop:
- {
- long state = current->state;
- struct sg_request *srp;
-
- do {
- srp = sg_find_srp_by_id(sfp, id, is_tag);
- if (srp) {
- __set_current_state(TASK_RUNNING);
- *srpp = srp;
- return 0;
- }
- if (SG_IS_DETACHING(sfp->parentdp)) {
- __set_current_state(TASK_RUNNING);
- return -ENODEV;
- }
- if (signal_pending_state(state, current)) {
- __set_current_state(TASK_RUNNING);
- return -ERESTARTSYS;
- }
- cpu_relax();
- } while (true);
}
+ if (id == -1)
+ return __wait_event_interruptible(sfp->cmpl_wait, sg_get_any_ready_srp(sfp, srpp));
+ else
+ return __wait_event_interruptible(sfp->cmpl_wait, sg_get_ready_srp(sfp, srpp, id,
+ is_tag));
}
/*
@@ -3376,7 +3434,7 @@ sg_ctl_ioreceive(struct sg_fd *sfp, void __user *p)
id = use_tag ? tag : pack_id;
try_again:
if (non_block) {
- srp = sg_find_srp_by_id(sfp, id, use_tag);
+ srp = sg_get_srp_by_id(sfp, id, use_tag, false /* part_mrq */);
if (!srp)
return SG_IS_DETACHING(sdp) ? -ENODEV : -EAGAIN;
} else {
@@ -3430,7 +3488,7 @@ sg_ctl_ioreceive_v3(struct sg_fd *sfp, void __user *p)
pack_id = h3p->pack_id;
try_again:
if (non_block) {
- srp = sg_find_srp_by_id(sfp, pack_id, false);
+ srp = sg_get_srp_by_id(sfp, pack_id, false, false);
if (!srp)
return SG_IS_DETACHING(sdp) ? -ENODEV : -EAGAIN;
} else {
@@ -3604,11 +3662,11 @@ sg_read(struct file *filp, char __user *p, size_t count, loff_t *ppos)
}
try_again:
if (non_block) {
- srp = sg_find_srp_by_id(sfp, want_id, false);
+ srp = sg_get_srp_by_id(sfp, want_id, false, false /* part_mrq */);
if (!srp)
return SG_IS_DETACHING(sdp) ? -ENODEV : -EAGAIN;
} else {
- ret = sg_wait_poll_by_id(sfp, &srp, want_id, false, false);
+ ret = sg_wait_poll_by_id(sfp, &srp, want_id, false, false /* do_poll */);
if (unlikely(ret))
return ret; /* signal --> -ERESTARTSYS */
if (IS_ERR(srp))
@@ -3715,8 +3773,7 @@ sg_calc_sgat_param(struct sg_device *sdp)
/*
* Only valid for shared file descriptors. Designed to be called after a read-side request has
- * successfully completed leaving valid data in a reserve request buffer. May also be called after
- * a write-side request that has the SGV4_FLAG_KEEP_SHARE flag set. If rs_srp is NULL, acts
+ * successfully completed leaving valid data in a reserve request buffer. If rs_srp is NULL, acts
* on first reserve request in SG_RQ_SHR_SWAP state, making it inactive and returning 0. If rs_srp
* is non-NULL and is a reserve request and is in SG_RQ_SHR_SWAP state, makes it busy then
* inactive and returns 0. Otherwise -EINVAL is returned, unless write-side is in progress in
@@ -4110,7 +4167,7 @@ sg_fill_request_element(struct sg_fd *sfp, struct sg_request *srp, struct sg_req
static int
sg_ctl_sg_io(struct sg_device *sdp, struct sg_fd *sfp, void __user *p)
{
- bool is_v4, hipri;
+ bool is_v4;
int res;
struct sg_request *srp = NULL;
u8 hu8arr[SZ_SG_IO_V4];
@@ -4139,11 +4196,9 @@ sg_ctl_sg_io(struct sg_device *sdp, struct sg_fd *sfp, void __user *p)
SZ_SG_IO_V4 - v3_len))
return -EFAULT;
is_v4 = true;
- hipri = !!(h4p->flags & SGV4_FLAG_HIPRI);
res = sg_submit_v4(sfp, p, h4p, true, &srp);
} else if (h3p->interface_id == 'S') {
is_v4 = false;
- hipri = !!(h3p->flags & SGV4_FLAG_HIPRI);
res = sg_submit_v3(sfp, h3p, true, &srp);
} else {
pr_info_once("sg: %s: v3 or v4 interface only here\n", __func__);
@@ -4153,7 +4208,7 @@ sg_ctl_sg_io(struct sg_device *sdp, struct sg_fd *sfp, void __user *p)
return res;
if (!srp) /* mrq case: already processed all responses */
return res;
- res = sg_wait_poll_for_given_srp(sfp, srp, hipri);
+ res = sg_poll_wait4_given_srp(sfp, srp);
#if IS_ENABLED(SG_LOG_ACTIVE)
if (unlikely(res))
SG_LOG(1, sfp, "%s: unexpected srp=0x%pK state: %s, share: %s\n", __func__,
@@ -6779,7 +6834,7 @@ sg_read_append(struct sg_request *srp, void __user *outp, int num_xfer)
* from SG_RQ_AWAIT_RCV to SG_RQ_BUSY.
*/
static struct sg_request *
-sg_find_srp_by_id(struct sg_fd *sfp, int id, bool is_tag)
+sg_get_srp_by_id(struct sg_fd *sfp, int id, bool is_tag, bool part_mrq)
{
__maybe_unused bool is_bad_st = false;
bool search_for_1 = (id != SG_TAG_WILDCARD);
@@ -6802,11 +6857,13 @@ sg_find_srp_by_id(struct sg_fd *sfp, int id, bool is_tag)
s_idx = (l_await_idx < 0) ? 0 : l_await_idx;
idx = s_idx;
if (unlikely(search_for_1)) {
-second_time:
+second_time_for_1:
for (srp = xa_find(xafp, &idx, end_idx, SG_XA_RQ_AWAIT);
srp;
srp = xa_find_after(xafp, &idx, end_idx, SG_XA_RQ_AWAIT)) {
- if (test_bit(SG_FRQ_PC_SYNC_INVOC, srp->frq_pc_bm))
+ if (part_mrq != test_bit(SG_FRQ_PC_PART_MRQ, srp->frq_pc_bm))
+ continue;
+ if (!part_mrq && test_bit(SG_FRQ_PC_SYNC_INVOC, srp->frq_pc_bm))
continue;
if (unlikely(is_tag)) {
if (srp->tag != id)
@@ -6825,7 +6882,7 @@ sg_find_srp_by_id(struct sg_fd *sfp, int id, bool is_tag)
s_idx = 0;
idx = s_idx;
second = true;
- goto second_time;
+ goto second_time_for_1;
}
} else {
/*
@@ -6835,7 +6892,7 @@ sg_find_srp_by_id(struct sg_fd *sfp, int id, bool is_tag)
* is ready. If there is no queuing and the "last used" has been re-used then the
* first (relative) position will be the request we want.
*/
-second_time2:
+second_time_for_any:
for (srp = xa_find(xafp, &idx, end_idx, SG_XA_RQ_AWAIT);
srp;
srp = xa_find_after(xafp, &idx, end_idx, SG_XA_RQ_AWAIT)) {
@@ -6855,7 +6912,7 @@ sg_find_srp_by_id(struct sg_fd *sfp, int id, bool is_tag)
s_idx = 0;
idx = s_idx;
second = true;
- goto second_time2;
+ goto second_time_for_any;
}
}
return NULL;
--
2.25.1
