Call verify callback same as bio integrity.
If verify fail, drivers like MD will try other mirrors until get a correct
one or return failure after all mirrors are tried.
The MD driver already works like this, so no extra changed.
Todo:
- union with "struct bio_integrity_payload *bi_integrity" to save bio space.
Signed-off-by: Bob Liu <bob.liu@xxxxxxxxxx>
---
block/bio-integrity.c | 45 +++++++++++++++++++++++++++++++++++++++
block/bio.c | 3 +++
block/blk-core.c | 4 ++++
block/blk.h | 8 +++++++
block/bounce.c | 1 +
drivers/md/raid1.c | 1 +
drivers/md/raid5-ppl.c | 1 +
include/linux/blk_types.h | 5 +++++
8 files changed, 68 insertions(+)
diff --git a/block/bio-integrity.c b/block/bio-integrity.c
index 1b633a3526d4..90a47ad31dbf 100644
--- a/block/bio-integrity.c
+++ b/block/bio-integrity.c
@@ -372,6 +372,51 @@ bool __bio_integrity_endio(struct bio *bio)
return true;
}
+/**
+ * bio_verify_fn - Verify I/O completion worker
+ * @work: Work struct stored in bio to be verified
+ *
+ * Description: This workqueue function is called to complete a READ
+ * request. The function call verifier callack that fs pass down
+ * and then calls the original bio end_io function.
+ */
+static void bio_verify_fn(struct work_struct *work)
+{
+ struct bio *bio =
+ container_of(work, struct bio, bi_work);
+
+ bio->bi_status = bio->bi_verifier(bio);
+ /* Clear flag if verify succeed to avoid verifing
+ * it unnecessary by parent bio
+ */
+ if (!bio->bi_status)
+ bio->bi_opf &= ~REQ_VERIFY;
+ bio_endio(bio);
+}
+
+/**
+ * __bio_verify_endio - Verify I/O completion function
+ * @bio: Protected bio
+ *
+ * Description: Completion for verify I/O
+ *
+ * Normally I/O completion is done in interrupt context. However,
+ * verifying I/O is a time-consuming task which must be run
+ * in process context. This function postpones completion
+ * accordingly.
+ */
+bool __bio_verify_endio(struct bio *bio)
+{
+ if (bio_op(bio) == REQ_OP_READ && !bio->bi_status &&
+ (bio->bi_opf & REQ_VERIFY) && bio->bi_verifier) {
+ INIT_WORK(&bio->bi_work, bio_verify_fn);
+ queue_work(kintegrityd_wq, &bio->bi_work);
+ return false;
+ }
+
+ return true;
+}
+
/**
* bio_integrity_advance - Advance integrity vector
* @bio: bio whose integrity vector to update
diff --git a/block/bio.c b/block/bio.c
index 4db1008309ed..8928806acda6 100644
--- a/block/bio.c
+++ b/block/bio.c
@@ -608,6 +608,7 @@ void __bio_clone_fast(struct bio *bio, struct bio *bio_src)
bio->bi_write_hint = bio_src->bi_write_hint;
bio->bi_iter = bio_src->bi_iter;
bio->bi_io_vec = bio_src->bi_io_vec;
+ bio->bi_verifier = bio_src->bi_verifier;
bio_clone_blkg_association(bio, bio_src);
blkcg_bio_issue_init(bio);
@@ -1763,6 +1764,8 @@ void bio_endio(struct bio *bio)
return;
if (!bio_integrity_endio(bio))
return;
+ if (!bio_verify_endio(bio))
+ return;
if (bio->bi_disk)
rq_qos_done_bio(bio->bi_disk->queue, bio);
diff --git a/block/blk-core.c b/block/blk-core.c
index d265d2924c32..cbec80f2d73a 100644
--- a/block/blk-core.c
+++ b/block/blk-core.c
@@ -1167,6 +1167,10 @@ EXPORT_SYMBOL_GPL(direct_make_request);
blk_qc_t submit_bio_verify(struct bio *bio,
int (*verifier_cb_func)(struct bio *))
{
+ if (verifier_cb_func) {
+ bio->bi_verifier = verifier_cb_func;
+ bio->bi_opf |= REQ_VERIFY;
+ }
/*
* If it's a regular read/write or a barrier with data attached,
* go through the normal accounting stuff before submission.
diff --git a/block/blk.h b/block/blk.h
index 848278c52030..cdf30c65d4a8 100644
--- a/block/blk.h
+++ b/block/blk.h
@@ -151,6 +151,14 @@ static inline bool bio_integrity_endio(struct bio *bio)
}
#endif /* CONFIG_BLK_DEV_INTEGRITY */
+bool __bio_verify_endio(struct bio *);
+static inline bool bio_verify_endio(struct bio *bio)
+{
+ if (bio->bi_opf & REQ_VERIFY)
+ return __bio_verify_endio(bio);
+ return true;
+}
+
unsigned long blk_rq_timeout(unsigned long timeout);
void blk_add_timer(struct request *req);
diff --git a/block/bounce.c b/block/bounce.c
index ffb9e9ecfa7e..7a2c3f536030 100644
--- a/block/bounce.c
+++ b/block/bounce.c
@@ -252,6 +252,7 @@ static struct bio *bounce_clone_bio(struct bio *bio_src, gfp_t gfp_mask,
bio->bi_write_hint = bio_src->bi_write_hint;
bio->bi_iter.bi_sector = bio_src->bi_iter.bi_sector;
bio->bi_iter.bi_size = bio_src->bi_iter.bi_size;
+ bio->bi_verifier = bio_src->bi_verifier;
switch (bio_op(bio)) {
case REQ_OP_DISCARD:
diff --git a/drivers/md/raid1.c b/drivers/md/raid1.c
index 1d54109071cc..11b29a3831e1 100644
--- a/drivers/md/raid1.c
+++ b/drivers/md/raid1.c
@@ -1103,6 +1103,7 @@ static void alloc_behind_master_bio(struct r1bio *r1_bio,
}
behind_bio->bi_write_hint = bio->bi_write_hint;
+ behind_bio->bi_verifier = bio->bi_verifier;
while (i < vcnt && size) {
struct page *page;
diff --git a/drivers/md/raid5-ppl.c b/drivers/md/raid5-ppl.c
index 3a7c36326589..4cdaa5dabfbe 100644
--- a/drivers/md/raid5-ppl.c
+++ b/drivers/md/raid5-ppl.c
@@ -505,6 +505,7 @@ static void ppl_submit_iounit(struct ppl_io_unit *io)
bio = bio_alloc_bioset(GFP_NOIO, BIO_MAX_PAGES,
&ppl_conf->bs);
bio->bi_opf = prev->bi_opf;
+ bio->bi_verifier = prev->bi_verifier;
bio_copy_dev(bio, prev);
bio->bi_iter.bi_sector = bio_end_sector(prev);
bio_add_page(bio, sh->ppl_page, PAGE_SIZE, 0);
diff --git a/include/linux/blk_types.h b/include/linux/blk_types.h
index d66bf5f32610..e9f25f162138 100644
--- a/include/linux/blk_types.h
+++ b/include/linux/blk_types.h
@@ -18,6 +18,7 @@ struct block_device;
struct io_context;
struct cgroup_subsys_state;
typedef void (bio_end_io_t) (struct bio *);
+typedef int (bio_verifier_t) (struct bio *);
/*
* Block error status values. See block/blk-core:blk_errors for the details.
@@ -187,6 +188,8 @@ struct bio {
struct bio_integrity_payload *bi_integrity; /* data integrity */
#endif
};
+ bio_verifier_t *bi_verifier; /* verify callback when endio */
+ struct work_struct bi_work; /* I/O completion */
unsigned short bi_vcnt; /* how many bio_vec's */
@@ -329,6 +332,7 @@ enum req_flag_bits {
/* for driver use */
__REQ_DRV,
__REQ_SWAP, /* swapping request. */
+ __REQ_VERIFY, /* verify IO when endio is called */
__REQ_NR_BITS, /* stops here */
};
@@ -351,6 +355,7 @@ enum req_flag_bits {
#define REQ_DRV (1ULL << __REQ_DRV)
#define REQ_SWAP (1ULL << __REQ_SWAP)
+#define REQ_VERIFY (1ULL << __REQ_VERIFY)
#define REQ_FAILFAST_MASK \
(REQ_FAILFAST_DEV | REQ_FAILFAST_TRANSPORT | REQ_FAILFAST_DRIVER)
--
2.17.1
