Instead of cloning the request we should pass it down directly.
This saves a memory allocation and avoid the usage of the 'special'
pointer.
Signed-off-by: Hannes Reinecke <hare@xxxxxxx>
---
drivers/md/dm.c | 273 ++++++++++++--------------------------------------------
1 file changed, 55 insertions(+), 218 deletions(-)
diff --git a/drivers/md/dm.c b/drivers/md/dm.c
index 4da9941..5b0dc00 100644
--- a/drivers/md/dm.c
+++ b/drivers/md/dm.c
@@ -75,25 +75,13 @@ struct dm_io {
struct dm_rq_target_io {
struct mapped_device *md;
struct dm_target *ti;
- struct request *orig, clone;
+ struct request_queue *q;
+ rq_end_io_fn *end_io;
+ void *end_io_data;
int error;
union map_info info;
};
-/*
- * For request-based dm - the bio clones we allocate are embedded in these
- * structs.
- *
- * We allocate these with bio_alloc_bioset, using the front_pad parameter when
- * the bioset is created - this means the bio has to come at the end of the
- * struct.
- */
-struct dm_rq_clone_bio_info {
- struct bio *orig;
- struct dm_rq_target_io *tio;
- struct bio clone;
-};
-
union map_info *dm_get_rq_mapinfo(struct request *rq)
{
if (rq && rq->end_io_data)
@@ -788,57 +776,6 @@ static void clone_endio(struct bio *bio, int error)
}
/*
- * Partial completion handling for request-based dm
- */
-static void end_clone_bio(struct bio *clone, int error)
-{
- struct dm_rq_clone_bio_info *info =
- container_of(clone, struct dm_rq_clone_bio_info, clone);
- struct dm_rq_target_io *tio = info->tio;
- struct bio *bio = info->orig;
- unsigned int nr_bytes = info->orig->bi_iter.bi_size;
-
- bio_put(clone);
-
- if (tio->error)
- /*
- * An error has already been detected on the request.
- * Once error occurred, just let clone->end_io() handle
- * the remainder.
- */
- return;
- else if (error) {
- /*
- * Don't notice the error to the upper layer yet.
- * The error handling decision is made by the target driver,
- * when the request is completed.
- */
- tio->error = error;
- return;
- }
-
- /*
- * I/O for the bio successfully completed.
- * Notice the data completion to the upper layer.
- */
-
- /*
- * bios are processed from the head of the list.
- * So the completing bio should always be rq->bio.
- * If it's not, something wrong is happening.
- */
- if (tio->orig->bio != bio)
- DMERR("bio completion is going in the middle of the request");
-
- /*
- * Update the original request.
- * Do not use blk_end_request() here, because it may complete
- * the original request before the clone, and break the ordering.
- */
- blk_update_request(tio->orig, 0, nr_bytes);
-}
-
-/*
* Don't touch any member of the md after calling this function because
* the md may be freed in dm_put() at the end of this function.
* Or do dm_get() before calling this function and dm_put() later.
@@ -867,35 +804,6 @@ static void rq_completed(struct mapped_device *md, int rw, int run_queue)
}
/*
- * Complete the clone and the original request.
- * Must be called without queue lock.
- */
-static void dm_end_request(struct request *clone, struct request *rq,
- int error)
-{
- if (rq->cmd_type == REQ_TYPE_BLOCK_PC) {
- rq->errors = clone->errors;
- rq->resid_len = clone->resid_len;
-
- if (rq->sense)
- /*
- * We are using the sense buffer of the original
- * request.
- * So setting the length of the sense data is enough.
- */
- rq->sense_len = clone->sense_len;
- }
-
- blk_end_request_all(rq, error);
-}
-
-static void dm_unprep_request(struct request *rq)
-{
- rq->special = NULL;
- rq->cmd_flags &= ~REQ_DONTPREP;
-}
-
-/*
* Requeue the original request of a clone.
*/
void dm_requeue_unmapped_request(struct request *rq)
@@ -903,8 +811,6 @@ void dm_requeue_unmapped_request(struct request *rq)
struct request_queue *q = rq->q;
unsigned long flags;
- dm_unprep_request(rq);
-
spin_lock_irqsave(q->queue_lock, flags);
blk_requeue_request(q, rq);
spin_unlock_irqrestore(q->queue_lock, flags);
@@ -940,32 +846,29 @@ static void start_queue(struct request_queue *q)
spin_unlock_irqrestore(q->queue_lock, flags);
}
-static void dm_done(struct request *clone, struct dm_rq_target_io *tio,
+static void dm_done(struct request *rq, struct dm_rq_target_io *tio,
bool mapped)
{
int r = tio->error;
dm_request_endio_fn rq_end_io = NULL;
struct mapped_device *md = tio->md;
- struct request *rq = tio->orig;
- int rw = rq_data_dir(clone);
+ int rw = rq_data_dir(rq);
if (tio->ti) {
rq_end_io = tio->ti->type->rq_end_io;
if (mapped && rq_end_io)
- r = rq_end_io(tio->ti, clone, tio->error, &tio->info);
+ r = rq_end_io(tio->ti, rq, tio->error, &tio->info);
}
free_rq_tio(tio);
if (r <= 0) {
/* The target wants to complete the I/O */
- dm_end_request(clone, rq, r);
- blk_rq_unprep_clone(clone);
+ blk_end_request_all(rq, r);
rq_completed(md, rw, true);
} else if (r == DM_ENDIO_REQUEUE) {
/* The target wants to requeue the I/O */
dm_requeue_unmapped_request(rq);
- blk_rq_unprep_clone(clone);
rq_completed(md, rw, false);
} else {
DMWARN("unimplemented target endio return value: %d", r);
@@ -979,54 +882,38 @@ static void dm_done(struct request *clone, struct dm_rq_target_io *tio,
static void dm_softirq_done(struct request *rq)
{
bool mapped = true;
- struct request *clone = rq->completion_data;
- struct dm_rq_target_io *tio = clone->end_io_data;
+ struct dm_rq_target_io *tio = rq->completion_data;
- if (rq->cmd_flags & REQ_FAILED)
+ if (rq->cmd_flags & REQ_FAILED) {
mapped = false;
+ rq->cmd_flags &= ~REQ_FAILED;
+ }
- dm_done(clone, tio, mapped);
+ dm_done(rq, tio, mapped);
}
/*
- * Complete the clone and the original request with the error status
+ * Complete the remapped request with the error status
* through softirq context.
+ *
+ * Called with the queue lock held
*/
-static void dm_complete_request(struct request *clone, int error)
+static void dm_complete_request(struct request *rq, int error)
{
- struct dm_rq_target_io *tio = clone->end_io_data;
- struct request *rq = tio->orig;
+ struct dm_rq_target_io *tio = rq->end_io_data;
tio->error = error;
- rq->completion_data = clone;
+ rq->completion_data = tio;
+ rq->end_io = tio->end_io;
+ rq->end_io_data = tio->end_io_data;
+ rq->q = tio->q;
+ tio->end_io = NULL;
+ tio->end_io_data = NULL;
+ tio->q = NULL;
blk_complete_request(rq);
}
/*
- * Called with the queue lock held
- */
-static void end_clone_request(struct request *clone, int error)
-{
- /*
- * For just cleaning up the information of the queue in which
- * the clone was dispatched.
- * The clone is *NOT* freed actually here because it is alloced from
- * dm own mempool and REQ_ALLOCED isn't set in clone->cmd_flags.
- */
- __blk_put_request(clone->q, clone);
-
- /*
- * Actual request completion is done in a softirq context which doesn't
- * hold the queue lock. Otherwise, deadlock could occur because:
- * - another request may be submitted by the upper level driver
- * of the stacking during the completion
- * - the submission which requires queue lock may be done
- * against this queue
- */
- dm_complete_request(clone, error);
-}
-
-/*
* Return maximum size of I/O possible at the supplied sector up to the current
* target boundary.
*/
@@ -1482,64 +1369,27 @@ void dm_dispatch_request(struct request *rq)
}
EXPORT_SYMBOL_GPL(dm_dispatch_request);
-static int dm_rq_bio_constructor(struct bio *bio, struct bio *bio_orig,
- void *data)
+static int setup_rq(struct request *rq, struct mapped_device *md,
+ gfp_t gfp_mask)
{
- struct dm_rq_target_io *tio = data;
- struct dm_rq_clone_bio_info *info =
- container_of(bio, struct dm_rq_clone_bio_info, clone);
-
- info->orig = bio_orig;
- info->tio = tio;
- bio->bi_end_io = end_clone_bio;
-
- return 0;
-}
-
-static int setup_clone(struct request *clone, struct request *rq,
- struct dm_rq_target_io *tio)
-{
- int r;
-
- r = blk_rq_prep_clone(clone, rq, tio->md->bs, GFP_ATOMIC,
- dm_rq_bio_constructor, tio);
- if (r)
- return r;
-
- clone->cmd = rq->cmd;
- clone->cmd_len = rq->cmd_len;
- clone->sense = rq->sense;
- clone->buffer = rq->buffer;
- clone->end_io = end_clone_request;
- clone->end_io_data = tio;
-
- return 0;
-}
-
-static struct request *clone_rq(struct request *rq, struct mapped_device *md,
- gfp_t gfp_mask)
-{
- struct request *clone;
struct dm_rq_target_io *tio;
tio = alloc_rq_tio(md, gfp_mask);
if (!tio)
- return NULL;
+ return -ENOMEM;
tio->md = md;
tio->ti = NULL;
- tio->orig = rq;
+ tio->q = rq->q;
+ tio->end_io = rq->end_io;
+ tio->end_io_data = rq->end_io_data;
tio->error = 0;
memset(&tio->info, 0, sizeof(tio->info));
- clone = &tio->clone;
- if (setup_clone(clone, rq, tio)) {
- /* -ENOMEM */
- free_rq_tio(tio);
- return NULL;
- }
+ rq->end_io = dm_complete_request;
+ rq->end_io_data = tio;
- return clone;
+ return 0;
}
/*
@@ -1548,18 +1398,10 @@ static struct request *clone_rq(struct request *rq, struct mapped_device *md,
static int dm_prep_fn(struct request_queue *q, struct request *rq)
{
struct mapped_device *md = q->queuedata;
- struct request *clone;
- if (unlikely(rq->special)) {
- DMWARN("Already has something in rq->special.");
- return BLKPREP_KILL;
- }
-
- clone = clone_rq(rq, md, GFP_ATOMIC);
- if (!clone)
+ if (setup_rq(rq, md, GFP_ATOMIC) < 0)
return BLKPREP_DEFER;
- rq->special = clone;
rq->cmd_flags |= REQ_DONTPREP;
return BLKPREP_OK;
@@ -1570,30 +1412,28 @@ static int dm_prep_fn(struct request_queue *q, struct request *rq)
* 0 : the request has been processed (not requeued)
* !0 : the request has been requeued
*/
-static int map_request(struct dm_target *ti, struct request *clone,
+static int map_request(struct dm_target *ti, struct request *rq,
struct mapped_device *md)
{
int r, requeued = 0;
- struct dm_rq_target_io *tio = clone->end_io_data;
- struct request *rq = tio->orig;
- int rw = rq_data_dir(clone);
+ struct dm_rq_target_io *tio = rq->end_io_data;
+ int rw = rq_data_dir(rq);
tio->ti = ti;
- r = ti->type->map_rq(ti, clone, &tio->info);
+ r = ti->type->map_rq(ti, rq, &tio->info);
switch (r) {
case DM_MAPIO_SUBMITTED:
/* The target has taken the I/O to submit by itself later */
break;
case DM_MAPIO_REMAPPED:
/* The target has remapped the I/O so dispatch it */
- trace_block_rq_remap(clone->q, clone, disk_devt(dm_disk(md)),
+ trace_block_rq_remap(rq->q, rq, disk_devt(dm_disk(md)),
blk_rq_pos(rq));
- dm_dispatch_request(clone);
+ dm_dispatch_request(rq);
break;
case DM_MAPIO_REQUEUE:
/* The target wants to requeue the I/O */
free_rq_tio(tio);
- blk_rq_unprep_clone(clone);
dm_requeue_unmapped_request(rq);
rq_completed(md, rw, false);
requeued = 1;
@@ -1606,20 +1446,17 @@ static int map_request(struct dm_target *ti, struct request *clone,
/* The target wants to complete the I/O */
rq->cmd_flags |= REQ_FAILED;
- dm_complete_request(clone, r);
+ dm_complete_request(rq, r);
break;
}
return requeued;
}
-static struct request *dm_start_request(struct mapped_device *md, struct request *orig)
+static void dm_start_request(struct mapped_device *md, struct request *rq)
{
- struct request *clone;
-
- blk_start_request(orig);
- clone = orig->special;
- atomic_inc(&md->pending[rq_data_dir(clone)]);
+ blk_start_request(rq);
+ atomic_inc(&md->pending[rq_data_dir(rq)]);
/*
* Hold the md reference here for the in-flight I/O.
@@ -1629,8 +1466,6 @@ static struct request *dm_start_request(struct mapped_device *md, struct request
* See the comment in rq_completed() too.
*/
dm_get(md);
-
- return clone;
}
/*
@@ -1643,7 +1478,7 @@ static void dm_request_fn(struct request_queue *q)
int srcu_idx;
struct dm_table *map = dm_get_live_table(md, &srcu_idx);
struct dm_target *ti;
- struct request *rq, *clone;
+ struct request *rq;
sector_t pos;
/*
@@ -1669,19 +1504,19 @@ static void dm_request_fn(struct request_queue *q)
* before calling dm_complete_request
*/
DMERR_LIMIT("request attempted access beyond the end of device");
- clone = dm_start_request(md, rq);
+ dm_start_request(md, rq);
rq->cmd_flags |= REQ_FAILED;
- dm_complete_request(clone, -EIO);
+ dm_complete_request(rq, -EIO);
continue;
}
if (ti->type->busy && ti->type->busy(ti))
goto delay_and_out;
- clone = dm_start_request(md, rq);
+ dm_start_request(md, rq);
spin_unlock(q->queue_lock);
- if (map_request(ti, clone, md))
+ if (map_request(ti, rq, md))
goto requeued;
BUG_ON(!irqs_disabled());
@@ -2794,7 +2629,7 @@ struct dm_md_mempools *dm_alloc_md_mempools(unsigned type, unsigned integrity, u
} else if (type == DM_TYPE_REQUEST_BASED) {
cachep = _rq_tio_cache;
pool_size = dm_get_reserved_rq_based_ios();
- front_pad = offsetof(struct dm_rq_clone_bio_info, clone);
+ front_pad = 0;
/* per_bio_data_size is not used. See __bind_mempools(). */
WARN_ON(per_bio_data_size != 0);
} else
@@ -2804,9 +2639,11 @@ struct dm_md_mempools *dm_alloc_md_mempools(unsigned type, unsigned integrity, u
if (!pools->io_pool)
goto out;
- pools->bs = bioset_create(pool_size, front_pad);
- if (!pools->bs)
- goto out;
+ if (front_pad) {
+ pools->bs = bioset_create(pool_size, front_pad);
+ if (!pools->bs)
+ goto out;
+ }
if (integrity && bioset_integrity_create(pools->bs, pool_size))
goto out;
--
1.7.12.4
--
dm-devel mailing list
dm-devel@xxxxxxxxxx
https://www.redhat.com/mailman/listinfo/dm-devel
