From: Keith Busch <kbusch@xxxxxxxxxx>
User space passthrough commands that utilize metadata currently need to
bounce the "integrity" buffer through the kernel. This adds unnecessary
overhead and memory pressure.
Add support for mapping user space directly so that we can avoid this
costly copy. This is similiar to how the bio payload utilizes user
addresses with bio_map_user_iov().
Signed-off-by: Keith Busch <kbusch@xxxxxxxxxx>
---
block/bio-integrity.c | 67 +++++++++++++++++++++++++++++++++++++++++++
include/linux/bio.h | 8 ++++++
2 files changed, 75 insertions(+)
diff --git a/block/bio-integrity.c b/block/bio-integrity.c
index ec8ac8cf6e1b9..08f70b837a29b 100644
--- a/block/bio-integrity.c
+++ b/block/bio-integrity.c
@@ -91,6 +91,19 @@ struct bio_integrity_payload *bio_integrity_alloc(struct bio *bio,
}
EXPORT_SYMBOL(bio_integrity_alloc);
+static void bio_integrity_unmap_user(struct bio_integrity_payload *bip)
+{
+ bool dirty = bio_data_dir(bip->bip_bio) == READ;
+ struct bvec_iter iter;
+ struct bio_vec bv;
+
+ bip_for_each_vec(bv, bip, iter) {
+ if (dirty && !PageCompound(bv.bv_page))
+ set_page_dirty_lock(bv.bv_page);
+ unpin_user_page(bv.bv_page);
+ }
+}
+
/**
* bio_integrity_free - Free bio integrity payload
* @bio: bio containing bip to be freed
@@ -105,6 +118,8 @@ void bio_integrity_free(struct bio *bio)
if (bip->bip_flags & BIP_BLOCK_INTEGRITY)
kfree(bvec_virt(bip->bip_vec));
+ else if (bip->bip_flags & BIP_INTEGRITY_USER)
+ bio_integrity_unmap_user(bip);;
__bio_integrity_free(bs, bip);
bio->bi_integrity = NULL;
@@ -160,6 +175,58 @@ int bio_integrity_add_page(struct bio *bio, struct page *page,
}
EXPORT_SYMBOL(bio_integrity_add_page);
+int bio_integrity_map_user(struct bio *bio, void __user *ubuf, unsigned int len,
+ u32 seed, u32 maxvecs)
+{
+ struct request_queue *q = bdev_get_queue(bio->bi_bdev);
+ unsigned long align = q->dma_pad_mask | queue_dma_alignment(q);
+ struct page *stack_pages[UIO_FASTIOV];
+ size_t offset = offset_in_page(ubuf);
+ unsigned long ptr = (uintptr_t)ubuf;
+ struct page **pages = stack_pages;
+ struct bio_integrity_payload *bip;
+ int npages, ret, i;
+
+ if (bio_integrity(bio) || ptr & align || maxvecs > UIO_FASTIOV)
+ return -EINVAL;
+
+ bip = bio_integrity_alloc(bio, GFP_KERNEL, maxvecs);
+ if (IS_ERR(bip))
+ return PTR_ERR(bip);
+
+ ret = pin_user_pages_fast(ptr, UIO_FASTIOV, FOLL_WRITE, pages);
+ if (unlikely(ret < 0))
+ goto free_bip;
+
+ npages = ret;
+ for (i = 0; i < npages; i++) {
+ u32 bytes = min_t(u32, len, PAGE_SIZE - offset);
+ ret = bio_integrity_add_page(bio, pages[i], bytes, offset);
+ if (ret != bytes) {
+ ret = -EINVAL;
+ goto release_pages;
+ }
+ len -= ret;
+ offset = 0;
+ }
+
+ if (len) {
+ ret = -EINVAL;
+ goto release_pages;
+ }
+
+ bip->bip_iter.bi_sector = seed;
+ bip->bip_flags |= BIP_INTEGRITY_USER;
+ return 0;
+
+release_pages:
+ unpin_user_pages(pages, npages);
+free_bip:
+ bio_integrity_free(bio);
+ return ret;
+}
+EXPORT_SYMBOL(bio_integrity_map_user);
+
/**
* bio_integrity_process - Process integrity metadata for a bio
* @bio: bio to generate/verify integrity metadata for
diff --git a/include/linux/bio.h b/include/linux/bio.h
index 41d417ee13499..144cc280b6ad3 100644
--- a/include/linux/bio.h
+++ b/include/linux/bio.h
@@ -324,6 +324,7 @@ enum bip_flags {
BIP_CTRL_NOCHECK = 1 << 2, /* disable HBA integrity checking */
BIP_DISK_NOCHECK = 1 << 3, /* disable disk integrity checking */
BIP_IP_CHECKSUM = 1 << 4, /* IP checksum */
+ BIP_INTEGRITY_USER = 1 << 5, /* Integrity payload is user address */
};
/*
@@ -720,6 +721,7 @@ static inline bool bioset_initialized(struct bio_set *bs)
extern struct bio_integrity_payload *bio_integrity_alloc(struct bio *, gfp_t, unsigned int);
extern int bio_integrity_add_page(struct bio *, struct page *, unsigned int, unsigned int);
+extern int bio_integrity_map_user(struct bio *, void __user *, unsigned int, u32, u32);
extern bool bio_integrity_prep(struct bio *);
extern void bio_integrity_advance(struct bio *, unsigned int);
extern void bio_integrity_trim(struct bio *);
@@ -789,6 +791,12 @@ static inline int bio_integrity_add_page(struct bio *bio, struct page *page,
return 0;
}
+static inline int bio_integrity_map_user(struct bio *bio, void __user *ubuf,
+ unsigned int len, u32 seed, u32 maxvecs)
+{
+ return -EINVAL
+}
+
#endif /* CONFIG_BLK_DEV_INTEGRITY */
/*
--
2.34.1
