No commit message...
And this is a 8600+ lines patch.
Can you split this into manageable pieces ?
I do not think anybody will review such a huge patch.
On 2020/10/21 18:05, Sergei Shtepa wrote:
> Signed-off-by: Sergei Shtepa <sergei.shtepa@xxxxxxxxx>
> ---
> drivers/block/Kconfig | 2 +
> drivers/block/Makefile | 1 +
> drivers/block/blk-snap/Kconfig | 24 +
> drivers/block/blk-snap/Makefile | 28 +
> drivers/block/blk-snap/big_buffer.c | 193 ++++
> drivers/block/blk-snap/big_buffer.h | 24 +
> drivers/block/blk-snap/blk-snap-ctl.h | 190 ++++
> drivers/block/blk-snap/blk_deferred.c | 566 +++++++++++
> drivers/block/blk-snap/blk_deferred.h | 67 ++
> drivers/block/blk-snap/blk_descr_file.c | 82 ++
> drivers/block/blk-snap/blk_descr_file.h | 26 +
> drivers/block/blk-snap/blk_descr_mem.c | 66 ++
> drivers/block/blk-snap/blk_descr_mem.h | 14 +
> drivers/block/blk-snap/blk_descr_multidev.c | 86 ++
> drivers/block/blk-snap/blk_descr_multidev.h | 25 +
> drivers/block/blk-snap/blk_descr_pool.c | 190 ++++
> drivers/block/blk-snap/blk_descr_pool.h | 38 +
> drivers/block/blk-snap/blk_redirect.c | 507 ++++++++++
> drivers/block/blk-snap/blk_redirect.h | 73 ++
> drivers/block/blk-snap/blk_util.c | 33 +
> drivers/block/blk-snap/blk_util.h | 33 +
> drivers/block/blk-snap/cbt_map.c | 210 +++++
> drivers/block/blk-snap/cbt_map.h | 62 ++
> drivers/block/blk-snap/common.h | 31 +
> drivers/block/blk-snap/ctrl_fops.c | 691 ++++++++++++++
> drivers/block/blk-snap/ctrl_fops.h | 19 +
> drivers/block/blk-snap/ctrl_pipe.c | 562 +++++++++++
> drivers/block/blk-snap/ctrl_pipe.h | 34 +
> drivers/block/blk-snap/ctrl_sysfs.c | 73 ++
> drivers/block/blk-snap/ctrl_sysfs.h | 5 +
> drivers/block/blk-snap/defer_io.c | 397 ++++++++
> drivers/block/blk-snap/defer_io.h | 39 +
> drivers/block/blk-snap/main.c | 82 ++
> drivers/block/blk-snap/params.c | 58 ++
> drivers/block/blk-snap/params.h | 29 +
> drivers/block/blk-snap/rangevector.c | 85 ++
> drivers/block/blk-snap/rangevector.h | 31 +
> drivers/block/blk-snap/snapimage.c | 982 ++++++++++++++++++++
> drivers/block/blk-snap/snapimage.h | 16 +
> drivers/block/blk-snap/snapshot.c | 225 +++++
> drivers/block/blk-snap/snapshot.h | 17 +
> drivers/block/blk-snap/snapstore.c | 929 ++++++++++++++++++
> drivers/block/blk-snap/snapstore.h | 68 ++
> drivers/block/blk-snap/snapstore_device.c | 532 +++++++++++
> drivers/block/blk-snap/snapstore_device.h | 63 ++
> drivers/block/blk-snap/snapstore_file.c | 52 ++
> drivers/block/blk-snap/snapstore_file.h | 15 +
> drivers/block/blk-snap/snapstore_mem.c | 91 ++
> drivers/block/blk-snap/snapstore_mem.h | 20 +
> drivers/block/blk-snap/snapstore_multidev.c | 118 +++
> drivers/block/blk-snap/snapstore_multidev.h | 22 +
> drivers/block/blk-snap/tracker.c | 449 +++++++++
> drivers/block/blk-snap/tracker.h | 38 +
> drivers/block/blk-snap/tracking.c | 270 ++++++
> drivers/block/blk-snap/tracking.h | 13 +
> drivers/block/blk-snap/version.h | 7 +
> 56 files changed, 8603 insertions(+)
> create mode 100644 drivers/block/blk-snap/Kconfig
> create mode 100644 drivers/block/blk-snap/Makefile
> create mode 100644 drivers/block/blk-snap/big_buffer.c
> create mode 100644 drivers/block/blk-snap/big_buffer.h
> create mode 100644 drivers/block/blk-snap/blk-snap-ctl.h
> create mode 100644 drivers/block/blk-snap/blk_deferred.c
> create mode 100644 drivers/block/blk-snap/blk_deferred.h
> create mode 100644 drivers/block/blk-snap/blk_descr_file.c
> create mode 100644 drivers/block/blk-snap/blk_descr_file.h
> create mode 100644 drivers/block/blk-snap/blk_descr_mem.c
> create mode 100644 drivers/block/blk-snap/blk_descr_mem.h
> create mode 100644 drivers/block/blk-snap/blk_descr_multidev.c
> create mode 100644 drivers/block/blk-snap/blk_descr_multidev.h
> create mode 100644 drivers/block/blk-snap/blk_descr_pool.c
> create mode 100644 drivers/block/blk-snap/blk_descr_pool.h
> create mode 100644 drivers/block/blk-snap/blk_redirect.c
> create mode 100644 drivers/block/blk-snap/blk_redirect.h
> create mode 100644 drivers/block/blk-snap/blk_util.c
> create mode 100644 drivers/block/blk-snap/blk_util.h
> create mode 100644 drivers/block/blk-snap/cbt_map.c
> create mode 100644 drivers/block/blk-snap/cbt_map.h
> create mode 100644 drivers/block/blk-snap/common.h
> create mode 100644 drivers/block/blk-snap/ctrl_fops.c
> create mode 100644 drivers/block/blk-snap/ctrl_fops.h
> create mode 100644 drivers/block/blk-snap/ctrl_pipe.c
> create mode 100644 drivers/block/blk-snap/ctrl_pipe.h
> create mode 100644 drivers/block/blk-snap/ctrl_sysfs.c
> create mode 100644 drivers/block/blk-snap/ctrl_sysfs.h
> create mode 100644 drivers/block/blk-snap/defer_io.c
> create mode 100644 drivers/block/blk-snap/defer_io.h
> create mode 100644 drivers/block/blk-snap/main.c
> create mode 100644 drivers/block/blk-snap/params.c
> create mode 100644 drivers/block/blk-snap/params.h
> create mode 100644 drivers/block/blk-snap/rangevector.c
> create mode 100644 drivers/block/blk-snap/rangevector.h
> create mode 100644 drivers/block/blk-snap/snapimage.c
> create mode 100644 drivers/block/blk-snap/snapimage.h
> create mode 100644 drivers/block/blk-snap/snapshot.c
> create mode 100644 drivers/block/blk-snap/snapshot.h
> create mode 100644 drivers/block/blk-snap/snapstore.c
> create mode 100644 drivers/block/blk-snap/snapstore.h
> create mode 100644 drivers/block/blk-snap/snapstore_device.c
> create mode 100644 drivers/block/blk-snap/snapstore_device.h
> create mode 100644 drivers/block/blk-snap/snapstore_file.c
> create mode 100644 drivers/block/blk-snap/snapstore_file.h
> create mode 100644 drivers/block/blk-snap/snapstore_mem.c
> create mode 100644 drivers/block/blk-snap/snapstore_mem.h
> create mode 100644 drivers/block/blk-snap/snapstore_multidev.c
> create mode 100644 drivers/block/blk-snap/snapstore_multidev.h
> create mode 100644 drivers/block/blk-snap/tracker.c
> create mode 100644 drivers/block/blk-snap/tracker.h
> create mode 100644 drivers/block/blk-snap/tracking.c
> create mode 100644 drivers/block/blk-snap/tracking.h
> create mode 100644 drivers/block/blk-snap/version.h
>
> diff --git a/drivers/block/Kconfig b/drivers/block/Kconfig
> index ecceaaa1a66f..c53ef661110f 100644
> --- a/drivers/block/Kconfig
> +++ b/drivers/block/Kconfig
> @@ -460,4 +460,6 @@ config BLK_DEV_RSXX
>
> source "drivers/block/rnbd/Kconfig"
>
> +source "drivers/block/blk-snap/Kconfig"
> +
> endif # BLK_DEV
> diff --git a/drivers/block/Makefile b/drivers/block/Makefile
> index e1f63117ee94..312000598944 100644
> --- a/drivers/block/Makefile
> +++ b/drivers/block/Makefile
> @@ -40,6 +40,7 @@ obj-$(CONFIG_BLK_DEV_PCIESSD_MTIP32XX) += mtip32xx/
> obj-$(CONFIG_BLK_DEV_RSXX) += rsxx/
> obj-$(CONFIG_ZRAM) += zram/
> obj-$(CONFIG_BLK_DEV_RNBD) += rnbd/
> +obj-$(CONFIG_BLK_SNAP) += blk-snap/
>
> obj-$(CONFIG_BLK_DEV_NULL_BLK) += null_blk.o
> null_blk-objs := null_blk_main.o
> diff --git a/drivers/block/blk-snap/Kconfig b/drivers/block/blk-snap/Kconfig
> new file mode 100644
> index 000000000000..7a2db99a80dd
> --- /dev/null
> +++ b/drivers/block/blk-snap/Kconfig
> @@ -0,0 +1,24 @@
> +# SPDX-License-Identifier: GPL-2.0
> +#
> +# blk-snap block io layer filter module configuration
> +#
> +#
> +#select BLK_FILTER
> +
> +config BLK_SNAP
> + tristate "Block device snapshot filter"
> + depends on BLK_FILTER
> + help
> +
> + Allow to create snapshots and track block changes for a block
> + devices. Designed for creating backups for any block devices
> + (without device mapper). Snapshots are temporary and are released
> + then backup is completed. Change block tracking allows you to
> + create incremental or differential backups.
> +
> +config BLK_SNAP_SNAPSTORE_MULTIDEV
> + bool "Multi device snapstore configuration support"
> + depends on BLK_SNAP
> + help
> +
> + Allow to create snapstore on multiple block devices.
> diff --git a/drivers/block/blk-snap/Makefile b/drivers/block/blk-snap/Makefile
> new file mode 100644
> index 000000000000..1d628e8e1862
> --- /dev/null
> +++ b/drivers/block/blk-snap/Makefile
> @@ -0,0 +1,28 @@
> +# SPDX-License-Identifier: GPL-2.0
> +blk-snap-y += blk_deferred.o
> +blk-snap-y += blk_descr_file.o
> +blk-snap-y += blk_descr_mem.o
> +blk-snap-y += blk_descr_multidev.o
> +blk-snap-y += blk_descr_pool.o
> +blk-snap-y += blk_redirect.o
> +blk-snap-y += blk_util.o
> +blk-snap-y += cbt_map.o
> +blk-snap-y += ctrl_fops.o
> +blk-snap-y += ctrl_pipe.o
> +blk-snap-y += ctrl_sysfs.o
> +blk-snap-y += defer_io.o
> +blk-snap-y += main.o
> +blk-snap-y += params.o
> +blk-snap-y += big_buffer.o
> +blk-snap-y += rangevector.o
> +blk-snap-y += snapimage.o
> +blk-snap-y += snapshot.o
> +blk-snap-y += snapstore.o
> +blk-snap-y += snapstore_device.o
> +blk-snap-y += snapstore_file.o
> +blk-snap-y += snapstore_mem.o
> +blk-snap-y += snapstore_multidev.o
> +blk-snap-y += tracker.o
> +blk-snap-y += tracking.o
> +
> +obj-$(CONFIG_BLK_SNAP) += blk-snap.o
> diff --git a/drivers/block/blk-snap/big_buffer.c b/drivers/block/blk-snap/big_buffer.c
> new file mode 100644
> index 000000000000..c0a75255a807
> --- /dev/null
> +++ b/drivers/block/blk-snap/big_buffer.c
> @@ -0,0 +1,193 @@
> +// SPDX-License-Identifier: GPL-2.0
> +#include "common.h"
> +#include <linux/mm.h>
> +#include "big_buffer.h"
> +
> +static inline size_t page_count_calc(size_t buffer_size)
> +{
> + size_t page_count = buffer_size / PAGE_SIZE;
> +
> + if (buffer_size & (PAGE_SIZE - 1))
> + page_count += 1;
> + return page_count;
> +}
> +
> +struct big_buffer *big_buffer_alloc(size_t buffer_size, int gfp_opt)
> +{
> + int res = SUCCESS;
> + struct big_buffer *bbuff;
> + size_t count;
> + size_t inx;
> +
> + count = page_count_calc(buffer_size);
> +
> + bbuff = kzalloc(sizeof(struct big_buffer) + count * sizeof(void *), gfp_opt);
> + if (bbuff == NULL)
> + return NULL;
> +
> + bbuff->pg_cnt = count;
> + for (inx = 0; inx < bbuff->pg_cnt; ++inx) {
> + struct page *pg = alloc_page(gfp_opt);
> +
> + if (!pg) {
> + res = -ENOMEM;
> + break;
> + }
> + bbuff->pg[inx] = page_address(pg);
> + }
> +
> + if (res != SUCCESS) {
> + big_buffer_free(bbuff);
> + return NULL;
> + }
> +
> + return bbuff;
> +}
> +
> +void big_buffer_free(struct big_buffer *bbuff)
> +{
> + size_t inx;
> + size_t count = bbuff->pg_cnt;
> +
> + if (bbuff == NULL)
> + return;
> +
> + for (inx = 0; inx < count; ++inx)
> + if (bbuff->pg[inx] != NULL)
> + free_page((unsigned long)bbuff->pg[inx]);
> +
> + kfree(bbuff);
> +}
> +
> +size_t big_buffer_copy_to_user(char __user *dst_user, size_t offset, struct big_buffer *bbuff,
> + size_t length)
> +{
> + size_t left_data_length;
> + int page_inx = offset / PAGE_SIZE;
> + size_t processed_len = 0;
> + size_t unordered = offset & (PAGE_SIZE - 1);
> +
> + if (unordered != 0) { //first
> + size_t page_len = min_t(size_t, (PAGE_SIZE - unordered), length);
> +
> + left_data_length = copy_to_user(dst_user + processed_len,
> + bbuff->pg[page_inx] + unordered, page_len);
> + if (left_data_length != 0) {
> + pr_err("Failed to copy data from big_buffer to user buffer\n");
> + return processed_len;
> + }
> +
> + ++page_inx;
> + processed_len += page_len;
> + }
> +
> + while ((processed_len < length) && (page_inx < bbuff->pg_cnt)) {
> + size_t page_len = min_t(size_t, PAGE_SIZE, (length - processed_len));
> +
> + left_data_length =
> + copy_to_user(dst_user + processed_len, bbuff->pg[page_inx], page_len);
> + if (left_data_length != 0) {
> + pr_err("Failed to copy data from big_buffer to user buffer\n");
> + break;
> + }
> +
> + ++page_inx;
> + processed_len += page_len;
> + }
> +
> + return processed_len;
> +}
> +
> +size_t big_buffer_copy_from_user(const char __user *src_user, size_t offset,
> + struct big_buffer *bbuff, size_t length)
> +{
> + size_t left_data_length;
> + int page_inx = offset / PAGE_SIZE;
> + size_t processed_len = 0;
> + size_t unordered = offset & (PAGE_SIZE - 1);
> +
> + if (unordered != 0) { //first
> + size_t page_len = min_t(size_t, (PAGE_SIZE - unordered), length);
> +
> + left_data_length = copy_from_user(bbuff->pg[page_inx] + unordered,
> + src_user + processed_len, page_len);
> + if (left_data_length != 0) {
> + pr_err("Failed to copy data from user buffer to big_buffer\n");
> + return processed_len;
> + }
> +
> + ++page_inx;
> + processed_len += page_len;
> + }
> +
> + while ((processed_len < length) && (page_inx < bbuff->pg_cnt)) {
> + size_t page_len = min_t(size_t, PAGE_SIZE, (length - processed_len));
> +
> + left_data_length =
> + copy_from_user(bbuff->pg[page_inx], src_user + processed_len, page_len);
> + if (left_data_length != 0) {
> + pr_err("Failed to copy data from user buffer to big_buffer\n");
> + break;
> + }
> +
> + ++page_inx;
> + processed_len += page_len;
> + }
> +
> + return processed_len;
> +}
> +
> +void *big_buffer_get_element(struct big_buffer *bbuff, size_t index, size_t sizeof_element)
> +{
> + size_t elements_in_page = PAGE_SIZE / sizeof_element;
> + size_t pg_inx = index / elements_in_page;
> + size_t pg_ofs = (index - (pg_inx * elements_in_page)) * sizeof_element;
> +
> + if (pg_inx >= bbuff->pg_cnt)
> + return NULL;
> +
> + return bbuff->pg[pg_inx] + pg_ofs;
> +}
> +
> +void big_buffer_memset(struct big_buffer *bbuff, int value)
> +{
> + size_t inx;
> +
> + for (inx = 0; inx < bbuff->pg_cnt; ++inx)
> + memset(bbuff->pg[inx], value, PAGE_SIZE);
> +}
> +
> +void big_buffer_memcpy(struct big_buffer *dst, struct big_buffer *src)
> +{
> + size_t inx;
> + size_t count = min_t(size_t, dst->pg_cnt, src->pg_cnt);
> +
> + for (inx = 0; inx < count; ++inx)
> + memcpy(dst->pg[inx], src->pg[inx], PAGE_SIZE);
> +}
> +
> +int big_buffer_byte_get(struct big_buffer *bbuff, size_t inx, u8 *value)
> +{
> + size_t page_inx = inx >> PAGE_SHIFT;
> + size_t byte_pos = inx & (PAGE_SIZE - 1);
> +
> + if (page_inx >= bbuff->pg_cnt)
> + return -ENODATA;
> +
> + *value = bbuff->pg[page_inx][byte_pos];
> +
> + return SUCCESS;
> +}
> +
> +int big_buffer_byte_set(struct big_buffer *bbuff, size_t inx, u8 value)
> +{
> + size_t page_inx = inx >> PAGE_SHIFT;
> + size_t byte_pos = inx & (PAGE_SIZE - 1);
> +
> + if (page_inx >= bbuff->pg_cnt)
> + return -ENODATA;
> +
> + bbuff->pg[page_inx][byte_pos] = value;
> +
> + return SUCCESS;
> +}
> diff --git a/drivers/block/blk-snap/big_buffer.h b/drivers/block/blk-snap/big_buffer.h
> new file mode 100644
> index 000000000000..f38ab5288b05
> --- /dev/null
> +++ b/drivers/block/blk-snap/big_buffer.h
> @@ -0,0 +1,24 @@
> +/* SPDX-License-Identifier: GPL-2.0-or-later */
> +#pragma once
> +
> +struct big_buffer {
> + size_t pg_cnt;
> + u8 *pg[0];
> +};
> +
> +struct big_buffer *big_buffer_alloc(size_t count, int gfp_opt);
> +void big_buffer_free(struct big_buffer *bbuff);
> +
> +size_t big_buffer_copy_to_user(char __user *dst_user_buffer, size_t offset,
> + struct big_buffer *bbuff, size_t length);
> +size_t big_buffer_copy_from_user(const char __user *src_user_buffer, size_t offset,
> + struct big_buffer *bbuff, size_t length);
> +
> +void *big_buffer_get_element(struct big_buffer *bbuff, size_t index, size_t sizeof_element);
> +
> +void big_buffer_memset(struct big_buffer *bbuff, int value);
> +void big_buffer_memcpy(struct big_buffer *dst, struct big_buffer *src);
> +
> +//byte access
> +int big_buffer_byte_get(struct big_buffer *bbuff, size_t inx, u8 *value);
> +int big_buffer_byte_set(struct big_buffer *bbuff, size_t inx, u8 value);
> diff --git a/drivers/block/blk-snap/blk-snap-ctl.h b/drivers/block/blk-snap/blk-snap-ctl.h
> new file mode 100644
> index 000000000000..4ffd836836b1
> --- /dev/null
> +++ b/drivers/block/blk-snap/blk-snap-ctl.h
> @@ -0,0 +1,190 @@
> +/* SPDX-License-Identifier: GPL-2.0-or-later */
> +#pragma once
> +
> +#define MODULE_NAME "blk-snap"
> +#define SNAP_IMAGE_NAME "blk-snap-image"
> +
> +#define SUCCESS 0
> +
> +#define MAX_TRACKING_DEVICE_COUNT 256
> +
> +#define BLK_SNAP 'V'
> +
> +#pragma pack(push, 1)
> +//////////////////////////////////////////////////////////////////////////
> +// version
> +
> +#define BLK_SNAP_COMPATIBILITY_SNAPSTORE 0x0000000000000001ull /* rudiment */
> +//#define BLK_SNAP_COMPATIBILITY_BTRFS 0x0000000000000002ull /* rudiment */
> +#define BLK_SNAP_COMPATIBILITY_MULTIDEV 0x0000000000000004ull
> +
> +struct ioctl_compatibility_flags_s {
> + unsigned long long flags;
> +};
> +#define IOCTL_COMPATIBILITY_FLAGS _IOW(BLK_SNAP, 0, struct ioctl_compatibility_flags_s)
> +
> +struct ioctl_getversion_s {
> + unsigned short major;
> + unsigned short minor;
> + unsigned short revision;
> + unsigned short build;
> +};
> +#define IOCTL_GETVERSION _IOW(BLK_SNAP, 1, struct ioctl_getversion_s)
> +
> +//////////////////////////////////////////////////////////////////////////
> +// tracking
> +struct ioctl_dev_id_s {
> + int major;
> + int minor;
> +};
> +#define IOCTL_TRACKING_ADD _IOW(BLK_SNAP, 2, struct ioctl_dev_id_s)
> +
> +#define IOCTL_TRACKING_REMOVE _IOW(BLK_SNAP, 3, struct ioctl_dev_id_s)
> +
> +struct cbt_info_s {
> + struct ioctl_dev_id_s dev_id;
> + unsigned long long dev_capacity;
> + unsigned int cbt_map_size;
> + unsigned char snap_number;
> + unsigned char generationId[16];
> +};
> +struct ioctl_tracking_collect_s {
> + unsigned int count;
> + union {
> + struct cbt_info_s *p_cbt_info;
> + unsigned long long ull_cbt_info;
> + };
> +};
> +#define IOCTL_TRACKING_COLLECT _IOW(BLK_SNAP, 4, struct ioctl_tracking_collect_s)
> +
> +#define IOCTL_TRACKING_BLOCK_SIZE _IOW(BLK_SNAP, 5, unsigned int)
> +
> +struct ioctl_tracking_read_cbt_bitmap_s {
> + struct ioctl_dev_id_s dev_id;
> + unsigned int offset;
> + unsigned int length;
> + union {
> + unsigned char *buff;
> + unsigned long long ull_buff;
> + };
> +};
> +#define IOCTL_TRACKING_READ_CBT_BITMAP _IOR(BLK_SNAP, 6, struct ioctl_tracking_read_cbt_bitmap_s)
> +
> +struct block_range_s {
> + unsigned long long ofs; //sectors
> + unsigned long long cnt; //sectors
> +};
> +
> +struct ioctl_tracking_mark_dirty_blocks_s {
> + struct ioctl_dev_id_s image_dev_id;
> + unsigned int count;
> + union {
> + struct block_range_s *p_dirty_blocks;
> + unsigned long long ull_dirty_blocks;
> + };
> +};
> +#define IOCTL_TRACKING_MARK_DIRTY_BLOCKS \
> + _IOR(BLK_SNAP, 7, struct ioctl_tracking_mark_dirty_blocks_s)
> +//////////////////////////////////////////////////////////////////////////
> +// snapshot
> +
> +struct ioctl_snapshot_create_s {
> + unsigned long long snapshot_id;
> + unsigned int count;
> + union {
> + struct ioctl_dev_id_s *p_dev_id;
> + unsigned long long ull_dev_id;
> + };
> +};
> +#define IOCTL_SNAPSHOT_CREATE _IOW(BLK_SNAP, 0x10, struct ioctl_snapshot_create_s)
> +
> +#define IOCTL_SNAPSHOT_DESTROY _IOR(BLK_SNAP, 0x11, unsigned long long)
> +
> +struct ioctl_snapshot_errno_s {
> + struct ioctl_dev_id_s dev_id;
> + int err_code;
> +};
> +#define IOCTL_SNAPSHOT_ERRNO _IOW(BLK_SNAP, 0x12, struct ioctl_snapshot_errno_s)
> +
> +struct ioctl_range_s {
> + unsigned long long left;
> + unsigned long long right;
> +};
> +
> +//////////////////////////////////////////////////////////////////////////
> +// snapstore
> +struct ioctl_snapstore_create_s {
> + unsigned char id[16];
> + struct ioctl_dev_id_s snapstore_dev_id;
> + unsigned int count;
> + union {
> + struct ioctl_dev_id_s *p_dev_id;
> + unsigned long long ull_dev_id;
> + };
> +};
> +#define IOCTL_SNAPSTORE_CREATE _IOR(BLK_SNAP, 0x28, struct ioctl_snapstore_create_s)
> +
> +struct ioctl_snapstore_file_add_s {
> + unsigned char id[16];
> + unsigned int range_count;
> + union {
> + struct ioctl_range_s *ranges;
> + unsigned long long ull_ranges;
> + };
> +};
> +#define IOCTL_SNAPSTORE_FILE _IOR(BLK_SNAP, 0x29, struct ioctl_snapstore_file_add_s)
> +
> +struct ioctl_snapstore_memory_limit_s {
> + unsigned char id[16];
> + unsigned long long size;
> +};
> +#define IOCTL_SNAPSTORE_MEMORY _IOR(BLK_SNAP, 0x2A, struct ioctl_snapstore_memory_limit_s)
> +
> +struct ioctl_snapstore_cleanup_s {
> + unsigned char id[16];
> + unsigned long long filled_bytes;
> +};
> +#define IOCTL_SNAPSTORE_CLEANUP _IOW(BLK_SNAP, 0x2B, struct ioctl_snapstore_cleanup_s)
> +
> +struct ioctl_snapstore_file_add_multidev_s {
> + unsigned char id[16];
> + struct ioctl_dev_id_s dev_id;
> + unsigned int range_count;
> + union {
> + struct ioctl_range_s *ranges;
> + unsigned long long ull_ranges;
> + };
> +};
> +#define IOCTL_SNAPSTORE_FILE_MULTIDEV \
> + _IOR(BLK_SNAP, 0x2C, struct ioctl_snapstore_file_add_multidev_s)
> +//////////////////////////////////////////////////////////////////////////
> +// collect snapshot images
> +
> +struct image_info_s {
> + struct ioctl_dev_id_s original_dev_id;
> + struct ioctl_dev_id_s snapshot_dev_id;
> +};
> +
> +struct ioctl_collect_snapshot_images_s {
> + int count; //
> + union {
> + struct image_info_s *p_image_info;
> + unsigned long long ull_image_info;
> + };
> +};
> +#define IOCTL_COLLECT_SNAPSHOT_IMAGES _IOW(BLK_SNAP, 0x30, struct ioctl_collect_snapshot_images_s)
> +
> +#pragma pack(pop)
> +
> +// commands for character device interface
> +#define BLK_SNAP_CHARCMD_UNDEFINED 0x00
> +#define BLK_SNAP_CHARCMD_ACKNOWLEDGE 0x01
> +#define BLK_SNAP_CHARCMD_INVALID 0xFF
> +// to module commands
> +#define BLK_SNAP_CHARCMD_INITIATE 0x21
> +#define BLK_SNAP_CHARCMD_NEXT_PORTION 0x22
> +#define BLK_SNAP_CHARCMD_NEXT_PORTION_MULTIDEV 0x23
> +// from module commands
> +#define BLK_SNAP_CHARCMD_HALFFILL 0x41
> +#define BLK_SNAP_CHARCMD_OVERFLOW 0x42
> +#define BLK_SNAP_CHARCMD_TERMINATE 0x43
> diff --git a/drivers/block/blk-snap/blk_deferred.c b/drivers/block/blk-snap/blk_deferred.c
> new file mode 100644
> index 000000000000..1d0b7d2c4d71
> --- /dev/null
> +++ b/drivers/block/blk-snap/blk_deferred.c
> @@ -0,0 +1,566 @@
> +// SPDX-License-Identifier: GPL-2.0
> +#define BLK_SNAP_SECTION "-deferred"
> +#include "common.h"
> +
> +#include "blk_deferred.h"
> +#include "blk_util.h"
> +#include "snapstore.h"
> +#include "params.h"
> +
> +struct bio_set blk_deferred_bioset = { 0 };
> +
> +struct dio_bio_complete {
> + struct blk_deferred_request *dio_req;
> + sector_t bio_sect_len;
> +};
> +
> +struct dio_deadlocked_list {
> + struct list_head link;
> +
> + struct blk_deferred_request *dio_req;
> +};
> +
> +LIST_HEAD(dio_deadlocked_list);
> +DEFINE_RWLOCK(dio_deadlocked_list_lock);
> +
> +atomic64_t dio_alloc_count = ATOMIC64_INIT(0);
> +atomic64_t dio_free_count = ATOMIC64_INIT(0);
> +
> +void blk_deferred_done(void)
> +{
> + struct dio_deadlocked_list *dio_locked;
> +
> + do {
> + dio_locked = NULL;
> +
> + write_lock(&dio_deadlocked_list_lock);
> + if (!list_empty(&dio_deadlocked_list)) {
> + dio_locked = list_entry(dio_deadlocked_list.next,
> + struct dio_deadlocked_list, link);
> +
> + list_del(&dio_locked->link);
> + }
> + write_unlock(&dio_deadlocked_list_lock);
> +
> + if (dio_locked) {
> + if (dio_locked->dio_req->sect_len ==
> + atomic64_read(&dio_locked->dio_req->sect_processed))
> + blk_deferred_request_free(dio_locked->dio_req);
> + else
> + pr_err("Locked defer IO is still in memory\n");
> +
> + kfree(dio_locked);
> + }
> + } while (dio_locked);
> +}
> +
> +void blk_deferred_request_deadlocked(struct blk_deferred_request *dio_req)
> +{
> + struct dio_deadlocked_list *dio_locked =
> + kzalloc(sizeof(struct dio_deadlocked_list), GFP_KERNEL);
> +
> + dio_locked->dio_req = dio_req;
> +
> + write_lock(&dio_deadlocked_list_lock);
> + list_add_tail(&dio_locked->link, &dio_deadlocked_list);
> + write_unlock(&dio_deadlocked_list_lock);
> +
> + pr_warn("Deadlock with defer IO\n");
> +}
> +
> +void blk_deferred_free(struct blk_deferred_io *dio)
> +{
> + size_t inx = 0;
> +
> + if (dio->page_array != NULL) {
> + while (dio->page_array[inx] != NULL) {
> + __free_page(dio->page_array[inx]);
> + dio->page_array[inx] = NULL;
> +
> + ++inx;
> + }
> +
> + kfree(dio->page_array);
> + dio->page_array = NULL;
> + }
> + kfree(dio);
> +}
> +
> +struct blk_deferred_io *blk_deferred_alloc(unsigned long block_index,
> + union blk_descr_unify blk_descr)
> +{
> + size_t inx;
> + size_t page_count;
> + struct blk_deferred_io *dio = kmalloc(sizeof(struct blk_deferred_io), GFP_NOIO);
> +
> + if (dio == NULL)
> + return NULL;
> +
> + INIT_LIST_HEAD(&dio->link);
> +
> + dio->blk_descr = blk_descr;
> + dio->blk_index = block_index;
> +
> + dio->sect.ofs = block_index << snapstore_block_shift();
> + dio->sect.cnt = snapstore_block_size();
> +
> + page_count = snapstore_block_size() / (PAGE_SIZE / SECTOR_SIZE);
> + /*
> + * empty pointer on the end
> + */
> + dio->page_array = kzalloc((page_count + 1) * sizeof(struct page *), GFP_NOIO);
> + if (dio->page_array == NULL) {
> + blk_deferred_free(dio);
> + return NULL;
> + }
> +
> + for (inx = 0; inx < page_count; inx++) {
> + dio->page_array[inx] = alloc_page(GFP_NOIO);
> + if (dio->page_array[inx] == NULL) {
> + pr_err("Failed to allocate page\n");
> + blk_deferred_free(dio);
> + return NULL;
> + }
> + }
> +
> + return dio;
> +}
> +
> +int blk_deferred_bioset_create(void)
> +{
> + return bioset_init(&blk_deferred_bioset, 64, sizeof(struct dio_bio_complete),
> + BIOSET_NEED_BVECS | BIOSET_NEED_RESCUER);
> +}
> +
> +void blk_deferred_bioset_free(void)
> +{
> + bioset_exit(&blk_deferred_bioset);
> +}
> +
> +struct bio *_blk_deferred_bio_alloc(int nr_iovecs)
> +{
> + struct bio *new_bio = bio_alloc_bioset(GFP_NOIO, nr_iovecs, &blk_deferred_bioset);
> +
> + if (new_bio == NULL)
> + return NULL;
> +
> + new_bio->bi_end_io = blk_deferred_bio_endio;
> + new_bio->bi_private = ((void *)new_bio) - sizeof(struct dio_bio_complete);
> +
> + return new_bio;
> +}
> +
> +static void blk_deferred_complete(struct blk_deferred_request *dio_req, sector_t portion_sect_cnt,
> + int result)
> +{
> + atomic64_add(portion_sect_cnt, &dio_req->sect_processed);
> +
> + if (dio_req->sect_len == atomic64_read(&dio_req->sect_processed))
> + complete(&dio_req->complete);
> +
> + if (result != SUCCESS) {
> + dio_req->result = result;
> + pr_err("Failed to process defer IO request. errno=%d\n", result);
> + }
> +}
> +
> +void blk_deferred_bio_endio(struct bio *bio)
> +{
> + int local_err;
> + struct dio_bio_complete *complete_param = (struct dio_bio_complete *)bio->bi_private;
> +
> + if (complete_param == NULL) {
> + //bio already complete
> + } else {
> + if (bio->bi_status != BLK_STS_OK)
> + local_err = -EIO;
> + else
> + local_err = SUCCESS;
> +
> + blk_deferred_complete(complete_param->dio_req, complete_param->bio_sect_len,
> + local_err);
> + bio->bi_private = NULL;
> + }
> +
> + bio_put(bio);
> +}
> +
> +static inline size_t _page_count_calculate(sector_t size_sector)
> +{
> + size_t page_count = size_sector / (PAGE_SIZE / SECTOR_SIZE);
> +
> + if (unlikely(size_sector & ((PAGE_SIZE / SECTOR_SIZE) - 1)))
> + page_count += 1;
> +
> + return page_count;
> +}
> +
> +sector_t _blk_deferred_submit_pages(struct block_device *blk_dev,
> + struct blk_deferred_request *dio_req, int direction,
> + sector_t arr_ofs, struct page **page_array, sector_t ofs_sector,
> + sector_t size_sector)
> +{
> + struct bio *bio = NULL;
> + int nr_iovecs;
> + int page_inx = arr_ofs >> (PAGE_SHIFT - SECTOR_SHIFT);
> + sector_t process_sect = 0;
> +
> + nr_iovecs = _page_count_calculate(size_sector);
> +
> + while (NULL == (bio = _blk_deferred_bio_alloc(nr_iovecs))) {
> + size_sector = (size_sector >> 1) & ~((PAGE_SIZE / SECTOR_SIZE) - 1);
> + if (size_sector == 0)
> + return 0;
> +
> + nr_iovecs = _page_count_calculate(size_sector);
> + }
> +
> + bio_set_dev(bio, blk_dev);
> +
> + if (direction == READ)
> + bio_set_op_attrs(bio, REQ_OP_READ, 0);
> + else
> + bio_set_op_attrs(bio, REQ_OP_WRITE, 0);
> +
> + bio->bi_iter.bi_sector = ofs_sector;
> +
> + { //add first
> + sector_t unordered = arr_ofs & ((PAGE_SIZE / SECTOR_SIZE) - 1);
> + sector_t bvec_len_sect =
> + min_t(sector_t, ((PAGE_SIZE / SECTOR_SIZE) - unordered), size_sector);
> + struct page *page = page_array[page_inx];
> + unsigned int len = (unsigned int)from_sectors(bvec_len_sect);
> + unsigned int offset = (unsigned int)from_sectors(unordered);
> +
> + if (unlikely(page == NULL)) {
> + pr_err("NULL found in page array");
> + bio_put(bio);
> + return 0;
> + }
> + if (unlikely(bio_add_page(bio, page, len, offset) != len)) {
> + bio_put(bio);
> + return 0;
> + }
> + ++page_inx;
> + process_sect += bvec_len_sect;
> + }
> +
> + while (process_sect < size_sector) {
> + sector_t bvec_len_sect =
> + min_t(sector_t, (PAGE_SIZE / SECTOR_SIZE), (size_sector - process_sect));
> + struct page *page = page_array[page_inx];
> + unsigned int len = (unsigned int)from_sectors(bvec_len_sect);
> +
> +
> + if (unlikely(page == NULL)) {
> + pr_err("NULL found in page array");
> + break;
> + }
> + if (unlikely(bio_add_page(bio, page, len, 0) != len))
> + break;
> +
> + ++page_inx;
> + process_sect += bvec_len_sect;
> + }
> +
> + ((struct dio_bio_complete *)bio->bi_private)->dio_req = dio_req;
> + ((struct dio_bio_complete *)bio->bi_private)->bio_sect_len = process_sect;
> +
> + submit_bio_direct(bio);
> +
> + return process_sect;
> +}
> +
> +sector_t blk_deferred_submit_pages(struct block_device *blk_dev,
> + struct blk_deferred_request *dio_req, int direction,
> + sector_t arr_ofs, struct page **page_array, sector_t ofs_sector,
> + sector_t size_sector)
> +{
> + sector_t process_sect = 0;
> +
> + do {
> + sector_t portion_sect = _blk_deferred_submit_pages(
> + blk_dev, dio_req, direction, arr_ofs + process_sect, page_array,
> + ofs_sector + process_sect, size_sector - process_sect);
> + if (portion_sect == 0) {
> + pr_err("Failed to submit defer IO pages. Only [%lld] sectors processed\n",
> + process_sect);
> + break;
> + }
> + process_sect += portion_sect;
> + } while (process_sect < size_sector);
> +
> + return process_sect;
> +}
> +
> +struct blk_deferred_request *blk_deferred_request_new(void)
> +{
> + struct blk_deferred_request *dio_req = NULL;
> +
> + dio_req = kzalloc(sizeof(struct blk_deferred_request), GFP_NOIO);
> + if (dio_req == NULL)
> + return NULL;
> +
> + INIT_LIST_HEAD(&dio_req->dios);
> +
> + dio_req->result = SUCCESS;
> + atomic64_set(&dio_req->sect_processed, 0);
> + dio_req->sect_len = 0;
> + init_completion(&dio_req->complete);
> +
> + return dio_req;
> +}
> +
> +bool blk_deferred_request_already_added(struct blk_deferred_request *dio_req,
> + unsigned long block_index)
> +{
> + bool result = false;
> + struct list_head *_list_head;
> +
> + if (list_empty(&dio_req->dios))
> + return result;
> +
> + list_for_each(_list_head, &dio_req->dios) {
> + struct blk_deferred_io *dio = list_entry(_list_head, struct blk_deferred_io, link);
> +
> + if (dio->blk_index == block_index) {
> + result = true;
> + break;
> + }
> + }
> +
> + return result;
> +}
> +
> +int blk_deferred_request_add(struct blk_deferred_request *dio_req, struct blk_deferred_io *dio)
> +{
> + list_add_tail(&dio->link, &dio_req->dios);
> + dio_req->sect_len += dio->sect.cnt;
> +
> + return SUCCESS;
> +}
> +
> +void blk_deferred_request_free(struct blk_deferred_request *dio_req)
> +{
> + if (dio_req != NULL) {
> + while (!list_empty(&dio_req->dios)) {
> + struct blk_deferred_io *dio =
> + list_entry(dio_req->dios.next, struct blk_deferred_io, link);
> +
> + list_del(&dio->link);
> +
> + blk_deferred_free(dio);
> + }
> + kfree(dio_req);
> + }
> +}
> +
> +void blk_deferred_request_waiting_skip(struct blk_deferred_request *dio_req)
> +{
> + init_completion(&dio_req->complete);
> + atomic64_set(&dio_req->sect_processed, 0);
> +}
> +
> +int blk_deferred_request_wait(struct blk_deferred_request *dio_req)
> +{
> + u64 start_jiffies = get_jiffies_64();
> + u64 current_jiffies;
> +
> + while (wait_for_completion_timeout(&dio_req->complete, (HZ * 1)) == 0) {
> + current_jiffies = get_jiffies_64();
> + if (jiffies_to_msecs(current_jiffies - start_jiffies) > 60 * 1000) {
> + pr_warn("Defer IO request timeout\n");
> + return -EDEADLK;
> + }
> + }
> +
> + return dio_req->result;
> +}
> +
> +int blk_deferred_request_read_original(struct block_device *original_blk_dev,
> + struct blk_deferred_request *dio_copy_req)
> +{
> + int res = -ENODATA;
> + struct list_head *_list_head;
> +
> + blk_deferred_request_waiting_skip(dio_copy_req);
> +
> + if (list_empty(&dio_copy_req->dios))
> + return res;
> +
> + list_for_each(_list_head, &dio_copy_req->dios) {
> + struct blk_deferred_io *dio = list_entry(_list_head, struct blk_deferred_io, link);
> +
> + sector_t ofs = dio->sect.ofs;
> + sector_t cnt = dio->sect.cnt;
> +
> + if (cnt != blk_deferred_submit_pages(original_blk_dev, dio_copy_req, READ, 0,
> + dio->page_array, ofs, cnt)) {
> + pr_err("Failed to submit reading defer IO request. offset=%lld\n",
> + dio->sect.ofs);
> + res = -EIO;
> + break;
> + }
> + res = SUCCESS;
> + }
> +
> + if (res == SUCCESS)
> + res = blk_deferred_request_wait(dio_copy_req);
> +
> + return res;
> +}
> +
> +
> +static int _store_file(struct block_device *blk_dev, struct blk_deferred_request *dio_copy_req,
> + struct blk_descr_file *blk_descr, struct page **page_array)
> +{
> + struct list_head *_rangelist_head;
> + sector_t page_array_ofs = 0;
> +
> + if (unlikely(list_empty(&blk_descr->rangelist))) {
> + pr_err("Invalid block descriptor");
> + return -EINVAL;
> + }
> + list_for_each(_rangelist_head, &blk_descr->rangelist) {
> + struct blk_range_link *range_link;
> + sector_t process_sect;
> +
> + range_link = list_entry(_rangelist_head, struct blk_range_link, link);
> + process_sect = blk_deferred_submit_pages(blk_dev, dio_copy_req, WRITE,
> + page_array_ofs, page_array,
> + range_link->rg.ofs, range_link->rg.cnt);
> + if (range_link->rg.cnt != process_sect) {
> + pr_err("Failed to submit defer IO request for storing\n");
> + return -EIO;
> + }
> + page_array_ofs += range_link->rg.cnt;
> + }
> + return SUCCESS;
> +}
> +
> +int blk_deferred_request_store_file(struct block_device *blk_dev,
> + struct blk_deferred_request *dio_copy_req)
> +{
> + struct list_head *_dio_list_head;
> +
> + blk_deferred_request_waiting_skip(dio_copy_req);
> +
> + if (unlikely(list_empty(&dio_copy_req->dios))) {
> + pr_err("Invalid deferred io request");
> + return -EINVAL;
> + }
> + list_for_each(_dio_list_head, &dio_copy_req->dios) {
> + int res;
> + struct blk_deferred_io *dio;
> +
> + dio = list_entry(_dio_list_head, struct blk_deferred_io, link);
> + res = _store_file(blk_dev, dio_copy_req, dio->blk_descr.file, dio->page_array);
> + if (res != SUCCESS)
> + return res;
> + }
> +
> + return blk_deferred_request_wait(dio_copy_req);
> +}
> +
> +#ifdef CONFIG_BLK_SNAP_SNAPSTORE_MULTIDEV
> +
> +static int _store_multidev(struct blk_deferred_request *dio_copy_req,
> + struct blk_descr_multidev *blk_descr, struct page **page_array)
> +{
> + struct list_head *_ranges_list_head;
> + sector_t page_array_ofs = 0;
> +
> + if (unlikely(list_empty(&blk_descr->rangelist))) {
> + pr_err("Invalid block descriptor");
> + return -EINVAL;
> + }
> + list_for_each(_ranges_list_head, &blk_descr->rangelist) {
> + sector_t process_sect;
> + struct blk_range_link_ex *range_link;
> +
> + range_link = list_entry(_ranges_list_head, struct blk_range_link_ex, link);
> + process_sect = blk_deferred_submit_pages(range_link->blk_dev, dio_copy_req, WRITE,
> + page_array_ofs, page_array,
> + range_link->rg.ofs, range_link->rg.cnt);
> + if (range_link->rg.cnt != process_sect) {
> + pr_err("Failed to submit defer IO request for storing\n");
> + return -EIO;
> + }
> +
> + page_array_ofs += range_link->rg.cnt;
> + }
> +
> + return SUCCESS;
> +}
> +
> +int blk_deferred_request_store_multidev(struct blk_deferred_request *dio_copy_req)
> +{
> + struct list_head *_dio_list_head;
> +
> + blk_deferred_request_waiting_skip(dio_copy_req);
> +
> + if (unlikely(list_empty(&dio_copy_req->dios))) {
> + pr_err("Invalid deferred io request");
> + return -EINVAL;
> + }
> + list_for_each(_dio_list_head, &dio_copy_req->dios) {
> + int res;
> + struct blk_deferred_io *dio;
> +
> + dio = list_entry(_dio_list_head, struct blk_deferred_io, link);
> + res = _store_multidev(dio_copy_req, dio->blk_descr.multidev, dio->page_array);
> + if (res != SUCCESS)
> + return res;
> + }
> +
> + return blk_deferred_request_wait(dio_copy_req);
> +}
> +#endif
> +
> +static size_t _store_pages(void *dst, struct page **page_array, size_t length)
> +{
> + size_t page_inx = 0;
> + size_t processed_len = 0;
> +
> + while ((processed_len < length) && (page_array[page_inx] != NULL)) {
> + void *src;
> + size_t page_len = min_t(size_t, PAGE_SIZE, (length - processed_len));
> +
> + src = page_address(page_array[page_inx]);
> + memcpy(dst + processed_len, src, page_len);
> +
> + ++page_inx;
> + processed_len += page_len;
> + }
> +
> + return processed_len;
> +}
> +
> +int blk_deferred_request_store_mem(struct blk_deferred_request *dio_copy_req)
> +{
> + int res = SUCCESS;
> + sector_t processed = 0;
> +
> + if (!list_empty(&dio_copy_req->dios)) {
> + struct list_head *_list_head;
> +
> + list_for_each(_list_head, &dio_copy_req->dios) {
> + size_t length;
> + size_t portion;
> + struct blk_deferred_io *dio;
> +
> + dio = list_entry(_list_head, struct blk_deferred_io, link);
> + length = snapstore_block_size() * SECTOR_SIZE;
> +
> + portion = _store_pages(dio->blk_descr.mem->buff, dio->page_array, length);
> + if (unlikely(portion != length)) {
> + res = -EIO;
> + break;
> + }
> + processed += (sector_t)to_sectors(portion);
> + }
> + }
> +
> + blk_deferred_complete(dio_copy_req, processed, res);
> + return res;
> +}
> diff --git a/drivers/block/blk-snap/blk_deferred.h b/drivers/block/blk-snap/blk_deferred.h
> new file mode 100644
> index 000000000000..3c516a835c25
> --- /dev/null
> +++ b/drivers/block/blk-snap/blk_deferred.h
> @@ -0,0 +1,67 @@
> +/* SPDX-License-Identifier: GPL-2.0-or-later */
> +#pragma once
> +
> +#include "blk_descr_file.h"
> +#include "blk_descr_mem.h"
> +#include "blk_descr_multidev.h"
> +
> +#define DEFER_IO_DIO_REQUEST_LENGTH 250
> +#define DEFER_IO_DIO_REQUEST_SECTORS_COUNT (10 * 1024 * 1024 / SECTOR_SIZE)
> +
> +struct blk_deferred_io {
> + struct list_head link;
> +
> + unsigned long blk_index;
> + union blk_descr_unify blk_descr;
> +
> + struct blk_range sect;
> +
> + struct page **page_array; //null pointer on tail
> +};
> +
> +struct blk_deferred_request {
> + struct completion complete;
> + sector_t sect_len;
> + atomic64_t sect_processed;
> + int result;
> +
> + struct list_head dios;
> +};
> +
> +void blk_deferred_done(void);
> +
> +struct blk_deferred_io *blk_deferred_alloc(unsigned long block_index,
> + union blk_descr_unify blk_descr);
> +void blk_deferred_free(struct blk_deferred_io *dio);
> +
> +void blk_deferred_bio_endio(struct bio *bio);
> +
> +sector_t blk_deferred_submit_pages(struct block_device *blk_dev,
> + struct blk_deferred_request *dio_req, int direction,
> + sector_t arr_ofs, struct page **page_array, sector_t ofs_sector,
> + sector_t size_sector);
> +
> +struct blk_deferred_request *blk_deferred_request_new(void);
> +
> +bool blk_deferred_request_already_added(struct blk_deferred_request *dio_req,
> + unsigned long block_index);
> +
> +int blk_deferred_request_add(struct blk_deferred_request *dio_req, struct blk_deferred_io *dio);
> +void blk_deferred_request_free(struct blk_deferred_request *dio_req);
> +void blk_deferred_request_deadlocked(struct blk_deferred_request *dio_req);
> +
> +void blk_deferred_request_waiting_skip(struct blk_deferred_request *dio_req);
> +int blk_deferred_request_wait(struct blk_deferred_request *dio_req);
> +
> +int blk_deferred_bioset_create(void);
> +void blk_deferred_bioset_free(void);
> +
> +int blk_deferred_request_read_original(struct block_device *original_blk_dev,
> + struct blk_deferred_request *dio_copy_req);
> +
> +int blk_deferred_request_store_file(struct block_device *blk_dev,
> + struct blk_deferred_request *dio_copy_req);
> +#ifdef CONFIG_BLK_SNAP_SNAPSTORE_MULTIDEV
> +int blk_deferred_request_store_multidev(struct blk_deferred_request *dio_copy_req);
> +#endif
> +int blk_deferred_request_store_mem(struct blk_deferred_request *dio_copy_req);
> diff --git a/drivers/block/blk-snap/blk_descr_file.c b/drivers/block/blk-snap/blk_descr_file.c
> new file mode 100644
> index 000000000000..fca298d35744
> --- /dev/null
> +++ b/drivers/block/blk-snap/blk_descr_file.c
> @@ -0,0 +1,82 @@
> +// SPDX-License-Identifier: GPL-2.0
> +#define BLK_SNAP_SECTION "-blk_descr"
> +#include "common.h"
> +
> +#include "blk_descr_file.h"
> +
> +static inline void list_assign(struct list_head *dst, struct list_head *src)
> +{
> + dst->next = src->next;
> + dst->prev = src->prev;
> +
> + src->next->prev = dst;
> + src->prev->next = dst;
> +}
> +
> +static inline void blk_descr_file_init(struct blk_descr_file *blk_descr,
> + struct list_head *rangelist)
> +{
> + list_assign(&blk_descr->rangelist, rangelist);
> +}
> +
> +static inline void blk_descr_file_done(struct blk_descr_file *blk_descr)
> +{
> + struct blk_range_link *range_link;
> +
> + while (!list_empty(&blk_descr->rangelist)) {
> + range_link = list_entry(blk_descr->rangelist.next, struct blk_range_link, link);
> +
> + list_del(&range_link->link);
> + kfree(range_link);
> + }
> +}
> +
> +void blk_descr_file_pool_init(struct blk_descr_pool *pool)
> +{
> + blk_descr_pool_init(pool, 0);
> +}
> +
> +void _blk_descr_file_cleanup(void *descr_array, size_t count)
> +{
> + size_t inx;
> + struct blk_descr_file *file_blocks = descr_array;
> +
> + for (inx = 0; inx < count; ++inx)
> + blk_descr_file_done(file_blocks + inx);
> +}
> +
> +void blk_descr_file_pool_done(struct blk_descr_pool *pool)
> +{
> + blk_descr_pool_done(pool, _blk_descr_file_cleanup);
> +}
> +
> +static union blk_descr_unify _blk_descr_file_allocate(void *descr_array, size_t index, void *arg)
> +{
> + union blk_descr_unify blk_descr;
> + struct blk_descr_file *file_blocks = descr_array;
> +
> + blk_descr.file = &file_blocks[index];
> +
> + blk_descr_file_init(blk_descr.file, (struct list_head *)arg);
> +
> + return blk_descr;
> +}
> +
> +int blk_descr_file_pool_add(struct blk_descr_pool *pool, struct list_head *rangelist)
> +{
> + union blk_descr_unify blk_descr;
> +
> + blk_descr = blk_descr_pool_alloc(pool, sizeof(struct blk_descr_file),
> + _blk_descr_file_allocate, (void *)rangelist);
> + if (blk_descr.ptr == NULL) {
> + pr_err("Failed to allocate block descriptor\n");
> + return -ENOMEM;
> + }
> +
> + return SUCCESS;
> +}
> +
> +union blk_descr_unify blk_descr_file_pool_take(struct blk_descr_pool *pool)
> +{
> + return blk_descr_pool_take(pool, sizeof(struct blk_descr_file));
> +}
> diff --git a/drivers/block/blk-snap/blk_descr_file.h b/drivers/block/blk-snap/blk_descr_file.h
> new file mode 100644
> index 000000000000..0e9a5e3efdea
> --- /dev/null
> +++ b/drivers/block/blk-snap/blk_descr_file.h
> @@ -0,0 +1,26 @@
> +/* SPDX-License-Identifier: GPL-2.0-or-later */
> +#pragma once
> +
> +#include "blk_descr_pool.h"
> +
> +struct blk_descr_file {
> + struct list_head rangelist;
> +};
> +
> +struct blk_range_link {
> + struct list_head link;
> + struct blk_range rg;
> +};
> +
> +void blk_descr_file_pool_init(struct blk_descr_pool *pool);
> +void blk_descr_file_pool_done(struct blk_descr_pool *pool);
> +
> +/*
> + * allocate new empty block in pool
> + */
> +int blk_descr_file_pool_add(struct blk_descr_pool *pool, struct list_head *rangelist);
> +
> +/*
> + * take empty block from pool
> + */
> +union blk_descr_unify blk_descr_file_pool_take(struct blk_descr_pool *pool);
> diff --git a/drivers/block/blk-snap/blk_descr_mem.c b/drivers/block/blk-snap/blk_descr_mem.c
> new file mode 100644
> index 000000000000..cd326ac150b6
> --- /dev/null
> +++ b/drivers/block/blk-snap/blk_descr_mem.c
> @@ -0,0 +1,66 @@
> +// SPDX-License-Identifier: GPL-2.0
> +#define BLK_SNAP_SECTION "-blk_descr"
> +#include "common.h"
> +#include "blk_descr_mem.h"
> +
> +#define SECTION "blk_descr "
> +
> +static inline void blk_descr_mem_init(struct blk_descr_mem *blk_descr, void *ptr)
> +{
> + blk_descr->buff = ptr;
> +}
> +
> +static inline void blk_descr_mem_done(struct blk_descr_mem *blk_descr)
> +{
> + blk_descr->buff = NULL;
> +}
> +
> +void blk_descr_mem_pool_init(struct blk_descr_pool *pool, size_t available_blocks)
> +{
> + blk_descr_pool_init(pool, available_blocks);
> +}
> +
> +void blk_descr_mem_cleanup(void *descr_array, size_t count)
> +{
> + size_t inx;
> + struct blk_descr_mem *mem_blocks = descr_array;
> +
> + for (inx = 0; inx < count; ++inx)
> + blk_descr_mem_done(mem_blocks + inx);
> +}
> +
> +void blk_descr_mem_pool_done(struct blk_descr_pool *pool)
> +{
> + blk_descr_pool_done(pool, blk_descr_mem_cleanup);
> +}
> +
> +static union blk_descr_unify blk_descr_mem_alloc(void *descr_array, size_t index, void *arg)
> +{
> + union blk_descr_unify blk_descr;
> + struct blk_descr_mem *mem_blocks = descr_array;
> +
> + blk_descr.mem = &mem_blocks[index];
> +
> + blk_descr_mem_init(blk_descr.mem, (void *)arg);
> +
> + return blk_descr;
> +}
> +
> +int blk_descr_mem_pool_add(struct blk_descr_pool *pool, void *buffer)
> +{
> + union blk_descr_unify blk_descr;
> +
> + blk_descr = blk_descr_pool_alloc(pool, sizeof(struct blk_descr_mem),
> + blk_descr_mem_alloc, buffer);
> + if (blk_descr.ptr == NULL) {
> + pr_err("Failed to allocate block descriptor\n");
> + return -ENOMEM;
> + }
> +
> + return SUCCESS;
> +}
> +
> +union blk_descr_unify blk_descr_mem_pool_take(struct blk_descr_pool *pool)
> +{
> + return blk_descr_pool_take(pool, sizeof(struct blk_descr_mem));
> +}
> diff --git a/drivers/block/blk-snap/blk_descr_mem.h b/drivers/block/blk-snap/blk_descr_mem.h
> new file mode 100644
> index 000000000000..43e8de76c07c
> --- /dev/null
> +++ b/drivers/block/blk-snap/blk_descr_mem.h
> @@ -0,0 +1,14 @@
> +/* SPDX-License-Identifier: GPL-2.0-or-later */
> +#pragma once
> +
> +#include "blk_descr_pool.h"
> +
> +struct blk_descr_mem {
> + void *buff; //pointer to snapstore block in memory
> +};
> +
> +void blk_descr_mem_pool_init(struct blk_descr_pool *pool, size_t available_blocks);
> +void blk_descr_mem_pool_done(struct blk_descr_pool *pool);
> +
> +int blk_descr_mem_pool_add(struct blk_descr_pool *pool, void *buffer);
> +union blk_descr_unify blk_descr_mem_pool_take(struct blk_descr_pool *pool);
> diff --git a/drivers/block/blk-snap/blk_descr_multidev.c b/drivers/block/blk-snap/blk_descr_multidev.c
> new file mode 100644
> index 000000000000..cf5e0ed6f781
> --- /dev/null
> +++ b/drivers/block/blk-snap/blk_descr_multidev.c
> @@ -0,0 +1,86 @@
> +// SPDX-License-Identifier: GPL-2.0
> +#define BLK_SNAP_SECTION "-blk_descr"
> +#include "common.h"
> +
> +#ifdef CONFIG_BLK_SNAP_SNAPSTORE_MULTIDEV
> +#include "blk_descr_multidev.h"
> +
> +static inline void list_assign(struct list_head *dst, struct list_head *src)
> +{
> + dst->next = src->next;
> + dst->prev = src->prev;
> +
> + src->next->prev = dst;
> + src->prev->next = dst;
> +}
> +
> +static inline void blk_descr_multidev_init(struct blk_descr_multidev *blk_descr,
> + struct list_head *rangelist)
> +{
> + list_assign(&blk_descr->rangelist, rangelist);
> +}
> +
> +static inline void blk_descr_multidev_done(struct blk_descr_multidev *blk_descr)
> +{
> + struct blk_range_link_ex *rangelist;
> +
> + while (!list_empty(&blk_descr->rangelist)) {
> + rangelist = list_entry(blk_descr->rangelist.next,
> + struct blk_range_link_ex, link);
> +
> + list_del(&rangelist->link);
> + kfree(rangelist);
> + }
> +}
> +
> +void blk_descr_multidev_pool_init(struct blk_descr_pool *pool)
> +{
> + blk_descr_pool_init(pool, 0);
> +}
> +
> +static void blk_descr_multidev_cleanup(void *descr_array, size_t count)
> +{
> + size_t inx;
> + struct blk_descr_multidev *descr_multidev = descr_array;
> +
> + for (inx = 0; inx < count; ++inx)
> + blk_descr_multidev_done(descr_multidev + inx);
> +}
> +
> +void blk_descr_multidev_pool_done(struct blk_descr_pool *pool)
> +{
> + blk_descr_pool_done(pool, blk_descr_multidev_cleanup);
> +}
> +
> +static union blk_descr_unify blk_descr_multidev_allocate(void *descr_array, size_t index, void *arg)
> +{
> + union blk_descr_unify blk_descr;
> + struct blk_descr_multidev *multidev_blocks = descr_array;
> +
> + blk_descr.multidev = &multidev_blocks[index];
> +
> + blk_descr_multidev_init(blk_descr.multidev, (struct list_head *)arg);
> +
> + return blk_descr;
> +}
> +
> +int blk_descr_multidev_pool_add(struct blk_descr_pool *pool, struct list_head *rangelist)
> +{
> + union blk_descr_unify blk_descr;
> +
> + blk_descr = blk_descr_pool_alloc(pool, sizeof(struct blk_descr_multidev),
> + blk_descr_multidev_allocate, (void *)rangelist);
> + if (blk_descr.ptr == NULL) {
> + pr_err("Failed to allocate block descriptor\n");
> + return -ENOMEM;
> + }
> +
> + return SUCCESS;
> +}
> +
> +union blk_descr_unify blk_descr_multidev_pool_take(struct blk_descr_pool *pool)
> +{
> + return blk_descr_pool_take(pool, sizeof(struct blk_descr_multidev));
> +}
> +
> +#endif //CONFIG_BLK_SNAP_SNAPSTORE_MULTIDEV
> diff --git a/drivers/block/blk-snap/blk_descr_multidev.h b/drivers/block/blk-snap/blk_descr_multidev.h
> new file mode 100644
> index 000000000000..0145b0d78b10
> --- /dev/null
> +++ b/drivers/block/blk-snap/blk_descr_multidev.h
> @@ -0,0 +1,25 @@
> +/* SPDX-License-Identifier: GPL-2.0-or-later */
> +#pragma once
> +
> +#ifdef CONFIG_BLK_SNAP_SNAPSTORE_MULTIDEV
> +
> +#include "blk_descr_pool.h"
> +
> +struct blk_descr_multidev {
> + struct list_head rangelist;
> +};
> +
> +struct blk_range_link_ex {
> + struct list_head link;
> + struct blk_range rg;
> + struct block_device *blk_dev;
> +};
> +
> +void blk_descr_multidev_pool_init(struct blk_descr_pool *pool);
> +void blk_descr_multidev_pool_done(struct blk_descr_pool *pool);
> +
> +int blk_descr_multidev_pool_add(struct blk_descr_pool *pool,
> + struct list_head *rangelist); //allocate new empty block
> +union blk_descr_unify blk_descr_multidev_pool_take(struct blk_descr_pool *pool); //take empty
> +
> +#endif //CONFIG_BLK_SNAP_SNAPSTORE_MULTIDEV
> diff --git a/drivers/block/blk-snap/blk_descr_pool.c b/drivers/block/blk-snap/blk_descr_pool.c
> new file mode 100644
> index 000000000000..b1fe2ba9c2d0
> --- /dev/null
> +++ b/drivers/block/blk-snap/blk_descr_pool.c
> @@ -0,0 +1,190 @@
> +// SPDX-License-Identifier: GPL-2.0
> +#define BLK_SNAP_SECTION "-blk_descr"
> +#include "common.h"
> +#include "blk_descr_pool.h"
> +#include "params.h"
> +
> +struct pool_el {
> + struct list_head link;
> +
> + size_t used_cnt; // used blocks
> + size_t capacity; // blocks array capacity
> +
> + u8 descr_array[0];
> +};
> +
> +static void *kmalloc_huge(size_t max_size, size_t min_size, gfp_t flags, size_t *p_allocated_size)
> +{
> + void *ptr = NULL;
> +
> + do {
> + ptr = kmalloc(max_size, flags | __GFP_NOWARN | __GFP_RETRY_MAYFAIL);
> +
> + if (ptr != NULL) {
> + *p_allocated_size = max_size;
> + return ptr;
> + }
> + pr_err("Failed to allocate buffer size=%zu\n", max_size);
> + max_size = max_size >> 1;
> + } while (max_size >= min_size);
> +
> + pr_err("Failed to allocate buffer.");
> + return NULL;
> +}
> +
> +static struct pool_el *pool_el_alloc(size_t blk_descr_size)
> +{
> + size_t el_size;
> + struct pool_el *el;
> +
> + el = kmalloc_huge(8 * PAGE_SIZE, PAGE_SIZE, GFP_NOIO, &el_size);
> + if (el == NULL)
> + return NULL;
> +
> + el->capacity = (el_size - sizeof(struct pool_el)) / blk_descr_size;
> + el->used_cnt = 0;
> +
> + INIT_LIST_HEAD(&el->link);
> +
> + return el;
> +}
> +
> +static void _pool_el_free(struct pool_el *el)
> +{
> + if (el != NULL)
> + kfree(el);
> +}
> +
> +void blk_descr_pool_init(struct blk_descr_pool *pool, size_t available_blocks)
> +{
> + mutex_init(&pool->lock);
> +
> + INIT_LIST_HEAD(&pool->head);
> +
> + pool->blocks_cnt = 0;
> +
> + pool->total_cnt = available_blocks;
> + pool->take_cnt = 0;
> +}
> +
> +void blk_descr_pool_done(struct blk_descr_pool *pool,
> + void (*blocks_cleanup_cb)(void *descr_array, size_t count))
> +{
> + mutex_lock(&pool->lock);
> + while (!list_empty(&pool->head)) {
> + struct pool_el *el;
> +
> + el = list_entry(pool->head.next, struct pool_el, link);
> + if (el == NULL)
> + break;
> +
> + list_del(&el->link);
> + --pool->blocks_cnt;
> +
> + pool->total_cnt -= el->used_cnt;
> +
> + blocks_cleanup_cb(el->descr_array, el->used_cnt);
> +
> + _pool_el_free(el);
> + }
> + mutex_unlock(&pool->lock);
> +}
> +
> +union blk_descr_unify blk_descr_pool_alloc(
> + struct blk_descr_pool *pool, size_t blk_descr_size,
> + union blk_descr_unify (*block_alloc_cb)(void *descr_array, size_t index, void *arg),
> + void *arg)
> +{
> + union blk_descr_unify blk_descr = { NULL };
> +
> + mutex_lock(&pool->lock);
> + do {
> + struct pool_el *el = NULL;
> +
> + if (!list_empty(&pool->head)) {
> + el = list_entry(pool->head.prev, struct pool_el, link);
> + if (el->used_cnt == el->capacity)
> + el = NULL;
> + }
> +
> + if (el == NULL) {
> + el = pool_el_alloc(blk_descr_size);
> + if (el == NULL)
> + break;
> +
> + list_add_tail(&el->link, &pool->head);
> +
> + ++pool->blocks_cnt;
> + }
> +
> + blk_descr = block_alloc_cb(el->descr_array, el->used_cnt, arg);
> +
> + ++el->used_cnt;
> + ++pool->total_cnt;
> +
> + } while (false);
> + mutex_unlock(&pool->lock);
> +
> + return blk_descr;
> +}
> +
> +static union blk_descr_unify __blk_descr_pool_at(struct blk_descr_pool *pool, size_t blk_descr_size,
> + size_t index)
> +{
> + union blk_descr_unify bkl_descr = { NULL };
> + size_t curr_inx = 0;
> + struct pool_el *el;
> + struct list_head *_list_head;
> +
> + if (list_empty(&(pool)->head))
> + return bkl_descr;
> +
> + list_for_each(_list_head, &(pool)->head) {
> + el = list_entry(_list_head, struct pool_el, link);
> +
> + if ((index >= curr_inx) && (index < (curr_inx + el->used_cnt))) {
> + bkl_descr.ptr = el->descr_array + (index - curr_inx) * blk_descr_size;
> + break;
> + }
> + curr_inx += el->used_cnt;
> + }
> +
> + return bkl_descr;
> +}
> +
> +union blk_descr_unify blk_descr_pool_take(struct blk_descr_pool *pool, size_t blk_descr_size)
> +{
> + union blk_descr_unify result = { NULL };
> +
> + mutex_lock(&pool->lock);
> + do {
> + if (pool->take_cnt >= pool->total_cnt) {
> + pr_err("Unable to get block descriptor: ");
> + pr_err("not enough descriptors, already took %zu, total %zu\n",
> + pool->take_cnt, pool->total_cnt);
> + break;
> + }
> +
> + result = __blk_descr_pool_at(pool, blk_descr_size, pool->take_cnt);
> + if (result.ptr == NULL) {
> + pr_err("Unable to get block descriptor: ");
> + pr_err("not enough descriptors, already took %zu, total %zu\n",
> + pool->take_cnt, pool->total_cnt);
> + break;
> + }
> +
> + ++pool->take_cnt;
> + } while (false);
> + mutex_unlock(&pool->lock);
> + return result;
> +}
> +
> +bool blk_descr_pool_check_halffill(struct blk_descr_pool *pool, sector_t empty_limit,
> + sector_t *fill_status)
> +{
> + size_t empty_blocks = (pool->total_cnt - pool->take_cnt);
> +
> + *fill_status = (sector_t)(pool->take_cnt) << snapstore_block_shift();
> +
> + return (empty_blocks < (size_t)(empty_limit >> snapstore_block_shift()));
> +}
> diff --git a/drivers/block/blk-snap/blk_descr_pool.h b/drivers/block/blk-snap/blk_descr_pool.h
> new file mode 100644
> index 000000000000..32f8b8c4103e
> --- /dev/null
> +++ b/drivers/block/blk-snap/blk_descr_pool.h
> @@ -0,0 +1,38 @@
> +/* SPDX-License-Identifier: GPL-2.0-or-later */
> +#pragma once
> +
> +struct blk_descr_mem;
> +struct blk_descr_file;
> +struct blk_descr_multidev;
> +
> +union blk_descr_unify {
> + void *ptr;
> + struct blk_descr_mem *mem;
> + struct blk_descr_file *file;
> + struct blk_descr_multidev *multidev;
> +};
> +
> +struct blk_descr_pool {
> + struct list_head head;
> + struct mutex lock;
> +
> + size_t blocks_cnt; // count of struct pool_el
> +
> + size_t total_cnt; // total count of block descriptors
> + size_t take_cnt; // take count of block descriptors
> +};
> +
> +void blk_descr_pool_init(struct blk_descr_pool *pool, size_t available_blocks);
> +
> +void blk_descr_pool_done(struct blk_descr_pool *pool,
> + void (*blocks_cleanup_cb)(void *descr_array, size_t count));
> +
> +union blk_descr_unify blk_descr_pool_alloc(
> + struct blk_descr_pool *pool, size_t blk_descr_size,
> + union blk_descr_unify (*block_alloc_cb)(void *descr_array, size_t index, void *arg),
> + void *arg);
> +
> +union blk_descr_unify blk_descr_pool_take(struct blk_descr_pool *pool, size_t blk_descr_size);
> +
> +bool blk_descr_pool_check_halffill(struct blk_descr_pool *pool, sector_t empty_limit,
> + sector_t *fill_status);
> diff --git a/drivers/block/blk-snap/blk_redirect.c b/drivers/block/blk-snap/blk_redirect.c
> new file mode 100644
> index 000000000000..4c28a8cb4275
> --- /dev/null
> +++ b/drivers/block/blk-snap/blk_redirect.c
> @@ -0,0 +1,507 @@
> +// SPDX-License-Identifier: GPL-2.0
> +#define BLK_SNAP_SECTION "-redirect"
> +#include "common.h"
> +#include "blk_util.h"
> +#include "blk_redirect.h"
> +
> +#define bio_vec_sectors(bv) (bv.bv_len >> SECTOR_SHIFT)
> +
> +struct bio_set blk_redirect_bioset = { 0 };
> +
> +int blk_redirect_bioset_create(void)
> +{
> + return bioset_init(&blk_redirect_bioset, 64, 0, BIOSET_NEED_BVECS | BIOSET_NEED_RESCUER);
> +}
> +
> +void blk_redirect_bioset_free(void)
> +{
> + bioset_exit(&blk_redirect_bioset);
> +}
> +
> +void blk_redirect_bio_endio(struct bio *bb)
> +{
> + struct blk_redirect_bio *rq_redir = (struct blk_redirect_bio *)bb->bi_private;
> +
> + if (rq_redir != NULL) {
> + int err = SUCCESS;
> +
> + if (bb->bi_status != BLK_STS_OK)
> + err = -EIO;
> +
> + if (err != SUCCESS) {
> + pr_err("Failed to process redirect IO request. errno=%d\n", 0 - err);
> +
> + if (rq_redir->err == SUCCESS)
> + rq_redir->err = err;
> + }
> +
> + if (atomic64_dec_and_test(&rq_redir->bio_count))
> + blk_redirect_complete(rq_redir, rq_redir->err);
> + }
> + bio_put(bb);
> +}
> +
> +struct bio *_blk_dev_redirect_bio_alloc(int nr_iovecs, void *bi_private)
> +{
> + struct bio *new_bio;
> +
> + new_bio = bio_alloc_bioset(GFP_NOIO, nr_iovecs, &blk_redirect_bioset);
> + if (new_bio == NULL)
> + return NULL;
> +
> + new_bio->bi_end_io = blk_redirect_bio_endio;
> + new_bio->bi_private = bi_private;
> +
> + return new_bio;
> +}
> +
> +struct blk_redirect_bio_list *_redirect_bio_allocate_list(struct bio *new_bio)
> +{
> + struct blk_redirect_bio_list *next;
> +
> + next = kzalloc(sizeof(struct blk_redirect_bio_list), GFP_NOIO);
> + if (next == NULL)
> + return NULL;
> +
> + next->next = NULL;
> + next->this = new_bio;
> +
> + return next;
> +}
> +
> +int bio_endio_list_push(struct blk_redirect_bio *rq_redir, struct bio *new_bio)
> +{
> + struct blk_redirect_bio_list *head;
> +
> + if (rq_redir->bio_list_head == NULL) {
> + //list is empty, add first bio
> + rq_redir->bio_list_head = _redirect_bio_allocate_list(new_bio);
> + if (rq_redir->bio_list_head == NULL)
> + return -ENOMEM;
> + return SUCCESS;
> + }
> +
> + // seek end of list
> + head = rq_redir->bio_list_head;
> + while (head->next != NULL)
> + head = head->next;
> +
> + //append new bio to the end of list
> + head->next = _redirect_bio_allocate_list(new_bio);
> + if (head->next == NULL)
> + return -ENOMEM;
> +
> + return SUCCESS;
> +}
> +
> +void bio_endio_list_cleanup(struct blk_redirect_bio_list *curr)
> +{
> + while (curr != NULL) {
> + struct blk_redirect_bio_list *next;
> +
> + next = curr->next;
> + kfree(curr);
> + curr = next;
> + }
> +}
> +
> +static unsigned int get_max_sect(struct block_device *blk_dev)
> +{
> + struct request_queue *q = bdev_get_queue(blk_dev);
> +
> + return min((unsigned int)(BIO_MAX_PAGES << (PAGE_SHIFT - SECTOR_SHIFT)),
> + q->limits.max_sectors);
> +}
> +
> +static int _blk_dev_redirect_part_fast(struct blk_redirect_bio *rq_redir, int direction,
> + struct block_device *blk_dev, sector_t target_pos,
> + sector_t rq_ofs, sector_t rq_count)
> +{
> + __label__ _fail_out;
> + __label__ _reprocess_bv;
> +
> + int res = SUCCESS;
> +
> + struct bio_vec bvec;
> + struct bvec_iter iter;
> +
> + struct bio *new_bio = NULL;
> +
> + sector_t sect_ofs = 0;
> + sector_t processed_sectors = 0;
> + int nr_iovecs;
> + struct blk_redirect_bio_list *bio_endio_rec;
> +
> + nr_iovecs = get_max_sect(blk_dev) >> (PAGE_SHIFT - SECTOR_SHIFT);
> +
> + bio_for_each_segment(bvec, rq_redir->bio, iter) {
> + sector_t bvec_ofs;
> + sector_t bvec_sectors;
> +
> + unsigned int len;
> + unsigned int offset;
> +
> + if ((sect_ofs + bio_vec_sectors(bvec)) <= rq_ofs) {
> + sect_ofs += bio_vec_sectors(bvec);
> + continue;
> + }
> + if (sect_ofs >= (rq_ofs + rq_count))
> + break;
> +
> + bvec_ofs = 0;
> + if (sect_ofs < rq_ofs)
> + bvec_ofs = rq_ofs - sect_ofs;
> +
> + bvec_sectors = bio_vec_sectors(bvec) - bvec_ofs;
> + if (bvec_sectors > (rq_count - processed_sectors))
> + bvec_sectors = rq_count - processed_sectors;
> +
> + if (bvec_sectors == 0) {
> + res = -EIO;
> + goto _fail_out;
> + }
> +
> +_reprocess_bv:
> + if (new_bio == NULL) {
> + new_bio = _blk_dev_redirect_bio_alloc(nr_iovecs, rq_redir);
> + while (new_bio == NULL) {
> + pr_err("Unable to allocate new bio for redirect IO.\n");
> + res = -ENOMEM;
> + goto _fail_out;
> + }
> +
> + bio_set_dev(new_bio, blk_dev);
> +
> + if (direction == READ)
> + bio_set_op_attrs(new_bio, REQ_OP_READ, 0);
> +
> + if (direction == WRITE)
> + bio_set_op_attrs(new_bio, REQ_OP_WRITE, 0);
> +
> + new_bio->bi_iter.bi_sector = target_pos + processed_sectors;
> + }
> +
> + len = (unsigned int)from_sectors(bvec_sectors);
> + offset = bvec.bv_offset + (unsigned int)from_sectors(bvec_ofs);
> + if (unlikely(bio_add_page(new_bio, bvec.bv_page, len, offset) != len)) {
> + if (bio_sectors(new_bio) == 0) {
> + res = -EIO;
> + goto _fail_out;
> + }
> +
> + res = bio_endio_list_push(rq_redir, new_bio);
> + if (res != SUCCESS) {
> + pr_err("Failed to add bio into bio_endio_list\n");
> + goto _fail_out;
> + }
> +
> + atomic64_inc(&rq_redir->bio_count);
> + new_bio = NULL;
> +
> + goto _reprocess_bv;
> + }
> + processed_sectors += bvec_sectors;
> +
> + sect_ofs += bio_vec_sectors(bvec);
> + }
> +
> + if (new_bio != NULL) {
> + res = bio_endio_list_push(rq_redir, new_bio);
> + if (res != SUCCESS) {
> + pr_err("Failed to add bio into bio_endio_list\n");
> + goto _fail_out;
> + }
> +
> + atomic64_inc(&rq_redir->bio_count);
> + new_bio = NULL;
> + }
> +
> + return SUCCESS;
> +
> +_fail_out:
> + bio_endio_rec = rq_redir->bio_list_head;
> + while (bio_endio_rec != NULL) {
> + if (bio_endio_rec->this != NULL)
> + bio_put(bio_endio_rec->this);
> +
> + bio_endio_rec = bio_endio_rec->next;
> + }
> +
> + bio_endio_list_cleanup(bio_endio_rec);
> +
> + pr_err("Failed to process part of redirect IO request. rq_ofs=%lld, rq_count=%lld\n",
> + rq_ofs, rq_count);
> + return res;
> +}
> +
> +int blk_dev_redirect_part(struct blk_redirect_bio *rq_redir, int direction,
> + struct block_device *blk_dev, sector_t target_pos, sector_t rq_ofs,
> + sector_t rq_count)
> +{
> + struct request_queue *q = bdev_get_queue(blk_dev);
> + sector_t logical_block_size_mask =
> + (sector_t)((q->limits.logical_block_size >> SECTOR_SHIFT) - 1);
> +
> + if (likely(logical_block_size_mask == 0))
> + return _blk_dev_redirect_part_fast(rq_redir, direction, blk_dev, target_pos, rq_ofs,
> + rq_count);
> +
> + if (likely((0 == (target_pos & logical_block_size_mask)) &&
> + (0 == (rq_count & logical_block_size_mask))))
> + return _blk_dev_redirect_part_fast(rq_redir, direction, blk_dev, target_pos, rq_ofs,
> + rq_count);
> +
> + return -EFAULT;
> +}
> +
> +void blk_dev_redirect_submit(struct blk_redirect_bio *rq_redir)
> +{
> + struct blk_redirect_bio_list *head;
> + struct blk_redirect_bio_list *curr;
> +
> + head = curr = rq_redir->bio_list_head;
> + rq_redir->bio_list_head = NULL;
> +
> + while (curr != NULL) {
> + submit_bio_direct(curr->this);
> +
> + curr = curr->next;
> + }
> +
> + bio_endio_list_cleanup(head);
> +}
> +
> +int blk_dev_redirect_memcpy_part(struct blk_redirect_bio *rq_redir, int direction, void *buff,
> + sector_t rq_ofs, sector_t rq_count)
> +{
> + struct bio_vec bvec;
> + struct bvec_iter iter;
> +
> + sector_t sect_ofs = 0;
> + sector_t processed_sectors = 0;
> +
> + bio_for_each_segment(bvec, rq_redir->bio, iter) {
> + void *mem;
> + sector_t bvec_ofs;
> + sector_t bvec_sectors;
> +
> + if ((sect_ofs + bio_vec_sectors(bvec)) <= rq_ofs) {
> + sect_ofs += bio_vec_sectors(bvec);
> + continue;
> + }
> +
> + if (sect_ofs >= (rq_ofs + rq_count))
> + break;
> +
> + bvec_ofs = 0;
> + if (sect_ofs < rq_ofs)
> + bvec_ofs = rq_ofs - sect_ofs;
> +
> + bvec_sectors = bio_vec_sectors(bvec) - bvec_ofs;
> + if (bvec_sectors > (rq_count - processed_sectors))
> + bvec_sectors = rq_count - processed_sectors;
> +
> + mem = kmap_atomic(bvec.bv_page);
> + if (direction == READ) {
> + memcpy(mem + bvec.bv_offset + (unsigned int)from_sectors(bvec_ofs),
> + buff + (unsigned int)from_sectors(processed_sectors),
> + (unsigned int)from_sectors(bvec_sectors));
> + } else {
> + memcpy(buff + (unsigned int)from_sectors(processed_sectors),
> + mem + bvec.bv_offset + (unsigned int)from_sectors(bvec_ofs),
> + (unsigned int)from_sectors(bvec_sectors));
> + }
> + kunmap_atomic(mem);
> +
> + processed_sectors += bvec_sectors;
> +
> + sect_ofs += bio_vec_sectors(bvec);
> + }
> +
> + return SUCCESS;
> +}
> +
> +int blk_dev_redirect_zeroed_part(struct blk_redirect_bio *rq_redir, sector_t rq_ofs,
> + sector_t rq_count)
> +{
> + struct bio_vec bvec;
> + struct bvec_iter iter;
> +
> + sector_t sect_ofs = 0;
> + sector_t processed_sectors = 0;
> +
> + bio_for_each_segment(bvec, rq_redir->bio, iter) {
> + void *mem;
> + sector_t bvec_ofs;
> + sector_t bvec_sectors;
> +
> + if ((sect_ofs + bio_vec_sectors(bvec)) <= rq_ofs) {
> + sect_ofs += bio_vec_sectors(bvec);
> + continue;
> + }
> +
> + if (sect_ofs >= (rq_ofs + rq_count))
> + break;
> +
> + bvec_ofs = 0;
> + if (sect_ofs < rq_ofs)
> + bvec_ofs = rq_ofs - sect_ofs;
> +
> + bvec_sectors = bio_vec_sectors(bvec) - bvec_ofs;
> + if (bvec_sectors > (rq_count - processed_sectors))
> + bvec_sectors = rq_count - processed_sectors;
> +
> + mem = kmap_atomic(bvec.bv_page);
> + memset(mem + bvec.bv_offset + (unsigned int)from_sectors(bvec_ofs), 0,
> + (unsigned int)from_sectors(bvec_sectors));
> + kunmap_atomic(mem);
> +
> + processed_sectors += bvec_sectors;
> +
> + sect_ofs += bio_vec_sectors(bvec);
> + }
> +
> + return SUCCESS;
> +}
> +
> +int blk_dev_redirect_read_zeroed(struct blk_redirect_bio *rq_redir, struct block_device *blk_dev,
> + sector_t rq_pos, sector_t blk_ofs_start, sector_t blk_ofs_count,
> + struct rangevector *zero_sectors)
> +{
> + int res = SUCCESS;
> + struct blk_range_tree_node *range_node;
> +
> + sector_t ofs = 0;
> +
> + sector_t from = rq_pos + blk_ofs_start;
> + sector_t to = rq_pos + blk_ofs_start + blk_ofs_count - 1;
> +
> + down_read(&zero_sectors->lock);
> + range_node = blk_range_rb_iter_first(&zero_sectors->root, from, to);
> + while (range_node) {
> + struct blk_range *zero_range = &range_node->range;
> + sector_t current_portion;
> +
> + if (zero_range->ofs > rq_pos + blk_ofs_start + ofs) {
> + sector_t pre_zero_cnt = zero_range->ofs - (rq_pos + blk_ofs_start + ofs);
> +
> + res = blk_dev_redirect_part(rq_redir, READ, blk_dev,
> + rq_pos + blk_ofs_start + ofs,
> + blk_ofs_start + ofs, pre_zero_cnt);
> + if (res != SUCCESS)
> + break;
> +
> + ofs += pre_zero_cnt;
> + }
> +
> + current_portion = min_t(sector_t, zero_range->cnt, blk_ofs_count - ofs);
> +
> + res = blk_dev_redirect_zeroed_part(rq_redir, blk_ofs_start + ofs, current_portion);
> + if (res != SUCCESS)
> + break;
> +
> + ofs += current_portion;
> +
> + range_node = blk_range_rb_iter_next(range_node, from, to);
> + }
> + up_read(&zero_sectors->lock);
> +
> + if (res == SUCCESS)
> + if ((blk_ofs_count - ofs) > 0)
> + res = blk_dev_redirect_part(rq_redir, READ, blk_dev,
> + rq_pos + blk_ofs_start + ofs,
> + blk_ofs_start + ofs, blk_ofs_count - ofs);
> +
> + return res;
> +}
> +void blk_redirect_complete(struct blk_redirect_bio *rq_redir, int res)
> +{
> + rq_redir->complete_cb(rq_redir->complete_param, rq_redir->bio, res);
> + redirect_bio_queue_free(rq_redir);
> +}
> +
> +void redirect_bio_queue_init(struct redirect_bio_queue *queue)
> +{
> + INIT_LIST_HEAD(&queue->list);
> +
> + spin_lock_init(&queue->lock);
> +
> + atomic_set(&queue->in_queue_cnt, 0);
> + atomic_set(&queue->alloc_cnt, 0);
> +
> + atomic_set(&queue->active_state, true);
> +}
> +
> +struct blk_redirect_bio *redirect_bio_queue_new(struct redirect_bio_queue *queue)
> +{
> + struct blk_redirect_bio *rq_redir = kzalloc(sizeof(struct blk_redirect_bio), GFP_NOIO);
> +
> + if (rq_redir == NULL)
> + return NULL;
> +
> + atomic_inc(&queue->alloc_cnt);
> +
> + INIT_LIST_HEAD(&rq_redir->link);
> + rq_redir->queue = queue;
> +
> + return rq_redir;
> +}
> +
> +void redirect_bio_queue_free(struct blk_redirect_bio *rq_redir)
> +{
> + if (rq_redir) {
> + if (rq_redir->queue)
> + atomic_dec(&rq_redir->queue->alloc_cnt);
> +
> + kfree(rq_redir);
> + }
> +}
> +
> +int redirect_bio_queue_push_back(struct redirect_bio_queue *queue,
> + struct blk_redirect_bio *rq_redir)
> +{
> + int res = SUCCESS;
> +
> + spin_lock(&queue->lock);
> +
> + if (atomic_read(&queue->active_state)) {
> + INIT_LIST_HEAD(&rq_redir->link);
> + list_add_tail(&rq_redir->link, &queue->list);
> + atomic_inc(&queue->in_queue_cnt);
> + } else
> + res = -EACCES;
> +
> + spin_unlock(&queue->lock);
> + return res;
> +}
> +
> +struct blk_redirect_bio *redirect_bio_queue_get_first(struct redirect_bio_queue *queue)
> +{
> + struct blk_redirect_bio *rq_redir = NULL;
> +
> + spin_lock(&queue->lock);
> +
> + if (!list_empty(&queue->list)) {
> + rq_redir = list_entry(queue->list.next, struct blk_redirect_bio, link);
> + list_del(&rq_redir->link);
> + atomic_dec(&queue->in_queue_cnt);
> + }
> +
> + spin_unlock(&queue->lock);
> +
> + return rq_redir;
> +}
> +
> +bool redirect_bio_queue_active(struct redirect_bio_queue *queue, bool state)
> +{
> + bool prev_state;
> +
> + spin_lock(&queue->lock);
> +
> + prev_state = atomic_read(&queue->active_state);
> + atomic_set(&queue->active_state, state);
> +
> + spin_unlock(&queue->lock);
> +
> + return prev_state;
> +}
> diff --git a/drivers/block/blk-snap/blk_redirect.h b/drivers/block/blk-snap/blk_redirect.h
> new file mode 100644
> index 000000000000..aae23e78ebe2
> --- /dev/null
> +++ b/drivers/block/blk-snap/blk_redirect.h
> @@ -0,0 +1,73 @@
> +/* SPDX-License-Identifier: GPL-2.0-or-later */
> +#pragma once
> +
> +#include "rangevector.h"
> +
> +int blk_redirect_bioset_create(void);
> +void blk_redirect_bioset_free(void);
> +
> +void blk_redirect_bio_endio(struct bio *bb);
> +
> +struct blk_redirect_bio_list {
> + struct blk_redirect_bio_list *next;
> + struct bio *this;
> +};
> +
> +struct redirect_bio_queue {
> + struct list_head list;
> + spinlock_t lock;
> +
> + atomic_t active_state;
> + atomic_t in_queue_cnt;
> + atomic_t alloc_cnt;
> +};
> +
> +struct blk_redirect_bio {
> + struct list_head link;
> + struct redirect_bio_queue *queue;
> +
> + struct bio *bio;
> + int err;
> + struct blk_redirect_bio_list *bio_list_head; //list of created bios
> + atomic64_t bio_count;
> +
> + void *complete_param;
> + void (*complete_cb)(void *complete_param, struct bio *rq, int err);
> +};
> +
> +int blk_dev_redirect_part(struct blk_redirect_bio *rq_redir, int direction,
> + struct block_device *blk_dev, sector_t target_pos, sector_t rq_ofs,
> + sector_t rq_count);
> +
> +void blk_dev_redirect_submit(struct blk_redirect_bio *rq_redir);
> +
> +int blk_dev_redirect_memcpy_part(struct blk_redirect_bio *rq_redir, int direction, void *src_buff,
> + sector_t rq_ofs, sector_t rq_count);
> +
> +int blk_dev_redirect_zeroed_part(struct blk_redirect_bio *rq_redir, sector_t rq_ofs,
> + sector_t rq_count);
> +
> +int blk_dev_redirect_read_zeroed(struct blk_redirect_bio *rq_redir, struct block_device *blk_dev,
> + sector_t rq_pos, sector_t blk_ofs_start, sector_t blk_ofs_count,
> + struct rangevector *zero_sectors);
> +
> +void blk_redirect_complete(struct blk_redirect_bio *rq_redir, int res);
> +
> +void redirect_bio_queue_init(struct redirect_bio_queue *queue);
> +
> +struct blk_redirect_bio *redirect_bio_queue_new(struct redirect_bio_queue *queue);
> +
> +void redirect_bio_queue_free(struct blk_redirect_bio *rq_redir);
> +
> +int redirect_bio_queue_push_back(struct redirect_bio_queue *queue,
> + struct blk_redirect_bio *rq_redir);
> +
> +struct blk_redirect_bio *redirect_bio_queue_get_first(struct redirect_bio_queue *queue);
> +
> +bool redirect_bio_queue_active(struct redirect_bio_queue *queue, bool state);
> +
> +#define redirect_bio_queue_empty(queue) (atomic_read(&(queue).in_queue_cnt) == 0)
> +
> +#define redirect_bio_queue_unactive(queue) \
> + ((atomic_read(&((queue).active_state)) == false) && \
> + (atomic_read(&((queue).alloc_cnt)) == 0))
> diff --git a/drivers/block/blk-snap/blk_util.c b/drivers/block/blk-snap/blk_util.c
> new file mode 100644
> index 000000000000..57db70b86516
> --- /dev/null
> +++ b/drivers/block/blk-snap/blk_util.c
> @@ -0,0 +1,33 @@
> +// SPDX-License-Identifier: GPL-2.0
> +#include "common.h"
> +#include "blk_util.h"
> +
> +int blk_dev_open(dev_t dev_id, struct block_device **p_blk_dev)
> +{
> + int result = SUCCESS;
> + struct block_device *blk_dev;
> + int refCount;
> +
> + blk_dev = bdget(dev_id);
> + if (blk_dev == NULL) {
> + pr_err("Unable to open device [%d:%d]: bdget return NULL\n", MAJOR(dev_id),
> + MINOR(dev_id));
> + return -ENODEV;
> + }
> +
> + refCount = blkdev_get(blk_dev, FMODE_READ | FMODE_WRITE, NULL);
> + if (refCount < 0) {
> + pr_err("Unable to open device [%d:%d]: blkdev_get return error code %d\n",
> + MAJOR(dev_id), MINOR(dev_id), 0 - refCount);
> + result = refCount;
> + }
> +
> + if (result == SUCCESS)
> + *p_blk_dev = blk_dev;
> + return result;
> +}
> +
> +void blk_dev_close(struct block_device *blk_dev)
> +{
> + blkdev_put(blk_dev, FMODE_READ);
> +}
> diff --git a/drivers/block/blk-snap/blk_util.h b/drivers/block/blk-snap/blk_util.h
> new file mode 100644
> index 000000000000..0776f2faa668
> --- /dev/null
> +++ b/drivers/block/blk-snap/blk_util.h
> @@ -0,0 +1,33 @@
> +/* SPDX-License-Identifier: GPL-2.0-or-later */
> +#pragma once
> +
> +int blk_dev_open(dev_t dev_id, struct block_device **p_blk_dev);
> +void blk_dev_close(struct block_device *blk_dev);
> +
> +/*
> + * this function was copied from block/blk.h
> + */
> +static inline sector_t part_nr_sects_read(struct hd_struct *part)
> +{
> +#if (BITS_PER_LONG == 32) && defined(CONFIG_SMP)
> + sector_t nr_sects;
> + unsigned int seq;
> +
> + do {
> + seq = read_seqcount_begin(&part->nr_sects_seq);
> + nr_sects = part->nr_sects;
> + } while (read_seqcount_retry(&part->nr_sects_seq, seq));
> +
> + return nr_sects;
> +#elif (BITS_PER_LONG == 32) && defined(CONFIG_PREEMPTION)
> + sector_t nr_sects;
> +
> + preempt_disable();
> + nr_sects = part->nr_sects;
> + preempt_enable();
> +
> + return nr_sects;
> +#else
> + return part->nr_sects;
> +#endif
> +}
> diff --git a/drivers/block/blk-snap/cbt_map.c b/drivers/block/blk-snap/cbt_map.c
> new file mode 100644
> index 000000000000..e913069d1a57
> --- /dev/null
> +++ b/drivers/block/blk-snap/cbt_map.c
> @@ -0,0 +1,210 @@
> +// SPDX-License-Identifier: GPL-2.0
> +#define BLK_SNAP_SECTION "-cbt_map"
> +#include "common.h"
> +#include "cbt_map.h"
> +
> +int cbt_map_allocate(struct cbt_map *cbt_map, unsigned int cbt_sect_in_block_degree,
> + sector_t device_capacity)
> +{
> + sector_t size_mod;
> +
> + cbt_map->sect_in_block_degree = cbt_sect_in_block_degree;
> + cbt_map->device_capacity = device_capacity;
> + cbt_map->map_size = (device_capacity >> (sector_t)cbt_sect_in_block_degree);
> +
> + pr_info("Allocate CBT map of %zu\n", cbt_map->map_size);
> +
> + size_mod = (device_capacity & ((sector_t)(1 << cbt_sect_in_block_degree) - 1));
> + if (size_mod)
> + cbt_map->map_size++;
> +
> + cbt_map->read_map = big_buffer_alloc(cbt_map->map_size, GFP_KERNEL);
> + if (cbt_map->read_map != NULL)
> + big_buffer_memset(cbt_map->read_map, 0);
> +
> + cbt_map->write_map = big_buffer_alloc(cbt_map->map_size, GFP_KERNEL);
> + if (cbt_map->write_map != NULL)
> + big_buffer_memset(cbt_map->write_map, 0);
> +
> + if ((cbt_map->read_map == NULL) || (cbt_map->write_map == NULL)) {
> + pr_err("Cannot allocate CBT map. map_size=%zu\n", cbt_map->map_size);
> + return -ENOMEM;
> + }
> +
> + cbt_map->snap_number_previous = 0;
> + cbt_map->snap_number_active = 1;
> + generate_random_uuid(cbt_map->generationId.b);
> + cbt_map->active = true;
> +
> + cbt_map->state_changed_sectors = 0;
> + cbt_map->state_dirty_sectors = 0;
> +
> + return SUCCESS;
> +}
> +
> +void cbt_map_deallocate(struct cbt_map *cbt_map)
> +{
> + if (cbt_map->read_map != NULL) {
> + big_buffer_free(cbt_map->read_map);
> + cbt_map->read_map = NULL;
> + }
> +
> + if (cbt_map->write_map != NULL) {
> + big_buffer_free(cbt_map->write_map);
> + cbt_map->write_map = NULL;
> + }
> +
> + cbt_map->active = false;
> +}
> +
> +static void cbt_map_destroy(struct cbt_map *cbt_map)
> +{
> + pr_info("CBT map destroy\n");
> + if (cbt_map != NULL) {
> + cbt_map_deallocate(cbt_map);
> +
> + kfree(cbt_map);
> + }
> +}
> +
> +struct cbt_map *cbt_map_create(unsigned int cbt_sect_in_block_degree, sector_t device_capacity)
> +{
> + struct cbt_map *cbt_map = NULL;
> +
> + pr_info("CBT map create\n");
> +
> + cbt_map = kzalloc(sizeof(struct cbt_map), GFP_KERNEL);
> + if (cbt_map == NULL)
> + return NULL;
> +
> + if (cbt_map_allocate(cbt_map, cbt_sect_in_block_degree, device_capacity) != SUCCESS) {
> + cbt_map_destroy(cbt_map);
> + return NULL;
> + }
> +
> + spin_lock_init(&cbt_map->locker);
> + init_rwsem(&cbt_map->rw_lock);
> + kref_init(&cbt_map->refcount);
> +
> + return cbt_map;
> +}
> +
> +void cbt_map_destroy_cb(struct kref *kref)
> +{
> + cbt_map_destroy(container_of(kref, struct cbt_map, refcount));
> +}
> +
> +struct cbt_map *cbt_map_get_resource(struct cbt_map *cbt_map)
> +{
> + if (cbt_map)
> + kref_get(&cbt_map->refcount);
> +
> + return cbt_map;
> +}
> +
> +void cbt_map_put_resource(struct cbt_map *cbt_map)
> +{
> + if (cbt_map)
> + kref_put(&cbt_map->refcount, cbt_map_destroy_cb);
> +}
> +
> +void cbt_map_switch(struct cbt_map *cbt_map)
> +{
> + pr_info("CBT map switch\n");
> + spin_lock(&cbt_map->locker);
> +
> + big_buffer_memcpy(cbt_map->read_map, cbt_map->write_map);
> +
> + cbt_map->snap_number_previous = cbt_map->snap_number_active;
> + ++cbt_map->snap_number_active;
> + if (cbt_map->snap_number_active == 256) {
> + cbt_map->snap_number_active = 1;
> +
> + big_buffer_memset(cbt_map->write_map, 0);
> +
> + generate_random_uuid(cbt_map->generationId.b);
> +
> + pr_info("CBT reset\n");
> + }
> + spin_unlock(&cbt_map->locker);
> +}
> +
> +int _cbt_map_set(struct cbt_map *cbt_map, sector_t sector_start, sector_t sector_cnt,
> + u8 snap_number, struct big_buffer *map)
> +{
> + int res = SUCCESS;
> + size_t cbt_block;
> + size_t cbt_block_first = (size_t)(sector_start >> cbt_map->sect_in_block_degree);
> + size_t cbt_block_last = (size_t)((sector_start + sector_cnt - 1) >>
> + cbt_map->sect_in_block_degree); //inclusive
> +
> + for (cbt_block = cbt_block_first; cbt_block <= cbt_block_last; ++cbt_block) {
> + if (cbt_block < cbt_map->map_size) {
> + u8 num;
> +
> + res = big_buffer_byte_get(map, cbt_block, &num);
> + if (res == SUCCESS)
> + if (num < snap_number)
> + res = big_buffer_byte_set(map, cbt_block, snap_number);
> + } else
> + res = -EINVAL;
> +
> + if (res != SUCCESS) {
> + pr_err("Block index is too large. #%zu was demanded, map size %zu\n",
> + cbt_block, cbt_map->map_size);
> + break;
> + }
> + }
> + return res;
> +}
> +
> +int cbt_map_set(struct cbt_map *cbt_map, sector_t sector_start, sector_t sector_cnt)
> +{
> + int res = SUCCESS;
> +
> + spin_lock(&cbt_map->locker);
> +
> + res = _cbt_map_set(cbt_map, sector_start, sector_cnt, (u8)cbt_map->snap_number_active,
> + cbt_map->write_map);
> + cbt_map->state_changed_sectors += sector_cnt;
> +
> + spin_unlock(&cbt_map->locker);
> + return res;
> +}
> +
> +int cbt_map_set_both(struct cbt_map *cbt_map, sector_t sector_start, sector_t sector_cnt)
> +{
> + int res = SUCCESS;
> +
> + spin_lock(&cbt_map->locker);
> +
> + res = _cbt_map_set(cbt_map, sector_start, sector_cnt,
> + (u8)cbt_map->snap_number_active, cbt_map->write_map);
> + if (res == SUCCESS)
> + res = _cbt_map_set(cbt_map, sector_start, sector_cnt,
> + (u8)cbt_map->snap_number_previous, cbt_map->read_map);
> + cbt_map->state_dirty_sectors += sector_cnt;
> +
> + spin_unlock(&cbt_map->locker);
> + return res;
> +}
> +
> +size_t cbt_map_read_to_user(struct cbt_map *cbt_map, void __user *user_buff, size_t offset,
> + size_t size)
> +{
> + size_t readed = 0;
> + size_t left_size;
> + size_t real_size = min((cbt_map->map_size - offset), size);
> +
> + left_size = real_size -
> + big_buffer_copy_to_user(user_buff, offset, cbt_map->read_map, real_size);
> +
> + if (left_size == 0)
> + readed = real_size;
> + else {
> + pr_err("Not all CBT data was read. Left [%zu] bytes\n", left_size);
> + readed = real_size - left_size;
> + }
> +
> + return readed;
> +}
> diff --git a/drivers/block/blk-snap/cbt_map.h b/drivers/block/blk-snap/cbt_map.h
> new file mode 100644
> index 000000000000..cb52b09531fe
> --- /dev/null
> +++ b/drivers/block/blk-snap/cbt_map.h
> @@ -0,0 +1,62 @@
> +/* SPDX-License-Identifier: GPL-2.0-or-later */
> +#pragma once
> +
> +#include "big_buffer.h"
> +#include <linux/kref.h>
> +#include <linux/uuid.h>
> +
> +struct cbt_map {
> + struct kref refcount;
> +
> + spinlock_t locker;
> +
> + size_t sect_in_block_degree;
> + sector_t device_capacity;
> + size_t map_size;
> +
> + struct big_buffer *read_map;
> + struct big_buffer *write_map;
> +
> + unsigned long snap_number_active;
> + unsigned long snap_number_previous;
> + uuid_t generationId;
> +
> + bool active;
> +
> + struct rw_semaphore rw_lock;
> +
> + sector_t state_changed_sectors;
> + sector_t state_dirty_sectors;
> +};
> +
> +struct cbt_map *cbt_map_create(unsigned int cbt_sect_in_block_degree, sector_t device_capacity);
> +
> +struct cbt_map *cbt_map_get_resource(struct cbt_map *cbt_map);
> +void cbt_map_put_resource(struct cbt_map *cbt_map);
> +
> +void cbt_map_switch(struct cbt_map *cbt_map);
> +int cbt_map_set(struct cbt_map *cbt_map, sector_t sector_start, sector_t sector_cnt);
> +int cbt_map_set_both(struct cbt_map *cbt_map, sector_t sector_start, sector_t sector_cnt);
> +
> +size_t cbt_map_read_to_user(struct cbt_map *cbt_map, void __user *user_buffer, size_t offset,
> + size_t size);
> +
> +static inline void cbt_map_read_lock(struct cbt_map *cbt_map)
> +{
> + down_read(&cbt_map->rw_lock);
> +};
> +
> +static inline void cbt_map_read_unlock(struct cbt_map *cbt_map)
> +{
> + up_read(&cbt_map->rw_lock);
> +};
> +
> +static inline void cbt_map_write_lock(struct cbt_map *cbt_map)
> +{
> + down_write(&cbt_map->rw_lock);
> +};
> +
> +static inline void cbt_map_write_unlock(struct cbt_map *cbt_map)
> +{
> + up_write(&cbt_map->rw_lock);
> +};
> diff --git a/drivers/block/blk-snap/common.h b/drivers/block/blk-snap/common.h
> new file mode 100644
> index 000000000000..bbd5e98ab2a6
> --- /dev/null
> +++ b/drivers/block/blk-snap/common.h
> @@ -0,0 +1,31 @@
> +/* SPDX-License-Identifier: GPL-2.0-or-later */
> +#pragma once
> +
> +#ifndef BLK_SNAP_SECTION
> +#define BLK_SNAP_SECTION ""
> +#endif
> +#define pr_fmt(fmt) KBUILD_MODNAME BLK_SNAP_SECTION ": " fmt
> +
> +#include <linux/version.h> /*rudiment - needed for using KERNEL_VERSION */
> +
> +#include <linux/types.h>
> +#include <linux/errno.h>
> +#include <linux/mutex.h>
> +#include <linux/rwsem.h>
> +#include <linux/spinlock.h>
> +#include <linux/slab.h>
> +#include <linux/list.h>
> +#include <linux/atomic.h>
> +#include <linux/blkdev.h>
> +
> +#define from_sectors(_sectors) (_sectors << SECTOR_SHIFT)
> +#define to_sectors(_byte_size) (_byte_size >> SECTOR_SHIFT)
> +
> +struct blk_range {
> + sector_t ofs;
> + blkcnt_t cnt;
> +};
> +
> +#ifndef SUCCESS
> +#define SUCCESS 0
> +#endif
> diff --git a/drivers/block/blk-snap/ctrl_fops.c b/drivers/block/blk-snap/ctrl_fops.c
> new file mode 100644
> index 000000000000..b7b18539ee96
> --- /dev/null
> +++ b/drivers/block/blk-snap/ctrl_fops.c
> @@ -0,0 +1,691 @@
> +// SPDX-License-Identifier: GPL-2.0
> +#define BLK_SNAP_SECTION "-ctrl"
> +#include "common.h"
> +#include "blk-snap-ctl.h"
> +#include "ctrl_fops.h"
> +#include "version.h"
> +#include "tracking.h"
> +#include "snapshot.h"
> +#include "snapstore.h"
> +#include "snapimage.h"
> +#include "tracker.h"
> +#include "blk_deferred.h"
> +#include "big_buffer.h"
> +#include "params.h"
> +
> +#include <linux/module.h>
> +#include <linux/poll.h>
> +#include <linux/uaccess.h>
> +
> +int blk_snap_major; //zero by default
> +
> +const struct file_operations ctrl_fops = { .owner = THIS_MODULE,
> + .read = ctrl_read,
> + .write = ctrl_write,
> + .open = ctrl_open,
> + .release = ctrl_release,
> + .poll = ctrl_poll,
> + .unlocked_ioctl = ctrl_unlocked_ioctl };
> +
> +atomic_t dev_open_cnt = ATOMIC_INIT(0);
> +
> +const struct ioctl_getversion_s version = { .major = FILEVER_MAJOR,
> + .minor = FILEVER_MINOR,
> + .revision = FILEVER_REVISION,
> + .build = 0 };
> +
> +int get_blk_snap_major(void)
> +{
> + return blk_snap_major;
> +}
> +
> +int ctrl_init(void)
> +{
> + int ret;
> +
> + ret = register_chrdev(0, MODULE_NAME, &ctrl_fops);
> + if (ret < 0) {
> + pr_err("Failed to register a character device. errno=%d\n", blk_snap_major);
> + return ret;
> + }
> +
> + blk_snap_major = ret;
> + pr_info("Module major [%d]\n", blk_snap_major);
> + return SUCCESS;
> +}
> +
> +void ctrl_done(void)
> +{
> + unregister_chrdev(blk_snap_major, MODULE_NAME);
> + ctrl_pipe_done();
> +}
> +
> +ssize_t ctrl_read(struct file *fl, char __user *buffer, size_t length, loff_t *offset)
> +{
> + ssize_t bytes_read = 0;
> + struct ctrl_pipe *pipe = (struct ctrl_pipe *)fl->private_data;
> +
> + if (pipe == NULL) {
> + pr_err("Unable to read from pipe: invalid pipe pointer\n");
> + return -EINVAL;
> + }
> +
> + bytes_read = ctrl_pipe_read(pipe, buffer, length);
> + if (bytes_read == 0)
> + if (fl->f_flags & O_NONBLOCK)
> + bytes_read = -EAGAIN;
> +
> + return bytes_read;
> +}
> +
> +ssize_t ctrl_write(struct file *fl, const char __user *buffer, size_t length, loff_t *offset)
> +{
> + struct ctrl_pipe *pipe = (struct ctrl_pipe *)fl->private_data;
> +
> + if (pipe == NULL) {
> + pr_err("Unable to write into pipe: invalid pipe pointer\n");
> + return -EINVAL;
> + }
> +
> + return ctrl_pipe_write(pipe, buffer, length);
> +}
> +
> +unsigned int ctrl_poll(struct file *fl, struct poll_table_struct *wait)
> +{
> + struct ctrl_pipe *pipe = (struct ctrl_pipe *)fl->private_data;
> +
> + if (pipe == NULL) {
> + pr_err("Unable to poll pipe: invalid pipe pointer\n");
> + return -EINVAL;
> + }
> +
> + return ctrl_pipe_poll(pipe);
> +}
> +
> +int ctrl_open(struct inode *inode, struct file *fl)
> +{
> + fl->f_pos = 0;
> +
> + if (false == try_module_get(THIS_MODULE))
> + return -EINVAL;
> +
> + fl->private_data = (void *)ctrl_pipe_new();
> + if (fl->private_data == NULL) {
> + pr_err("Failed to open ctrl file\n");
> + return -ENOMEM;
> + }
> +
> + atomic_inc(&dev_open_cnt);
> +
> + return SUCCESS;
> +}
> +
> +int ctrl_release(struct inode *inode, struct file *fl)
> +{
> + int result = SUCCESS;
> +
> + if (atomic_read(&dev_open_cnt) > 0) {
> + module_put(THIS_MODULE);
> + ctrl_pipe_put_resource((struct ctrl_pipe *)fl->private_data);
> +
> + atomic_dec(&dev_open_cnt);
> + } else {
> + pr_err("Unable to close ctrl file: the file is already closed\n");
> + result = -EALREADY;
> + }
> +
> + return result;
> +}
> +
> +int ioctl_compatibility_flags(unsigned long arg)
> +{
> + unsigned long len;
> + struct ioctl_compatibility_flags_s param;
> +
> + param.flags = 0;
> + param.flags |= BLK_SNAP_COMPATIBILITY_SNAPSTORE;
> +#ifdef CONFIG_BLK_SNAP_SNAPSTORE_MULTIDEV
> + param.flags |= BLK_SNAP_COMPATIBILITY_MULTIDEV;
> +#endif
> + len = copy_to_user((void *)arg, ¶m, sizeof(struct ioctl_compatibility_flags_s));
> + if (len != 0) {
> + pr_err("Unable to get compatibility flags: invalid user buffer\n");
> + return -EINVAL;
> + }
> +
> + return SUCCESS;
> +}
> +
> +int ioctl_get_version(unsigned long arg)
> +{
> + unsigned long len;
> +
> + pr_info("Get version\n");
> +
> + len = copy_to_user((void *)arg, &version, sizeof(struct ioctl_getversion_s));
> + if (len != 0) {
> + pr_err("Unable to get version: invalid user buffer\n");
> + return -ENODATA;
> + }
> +
> + return SUCCESS;
> +}
> +
> +int ioctl_tracking_add(unsigned long arg)
> +{
> + unsigned long len;
> + struct ioctl_dev_id_s dev;
> +
> + len = copy_from_user(&dev, (void *)arg, sizeof(struct ioctl_dev_id_s));
> + if (len != 0) {
> + pr_err("Unable to add device under tracking: invalid user buffer\n");
> + return -ENODATA;
> + }
> +
> + return tracking_add(MKDEV(dev.major, dev.minor), 0ull);
> +}
> +
> +int ioctl_tracking_remove(unsigned long arg)
> +{
> + struct ioctl_dev_id_s dev;
> +
> + if (copy_from_user(&dev, (void *)arg, sizeof(struct ioctl_dev_id_s)) != 0) {
> + pr_err("Unable to remove device from tracking: invalid user buffer\n");
> + return -ENODATA;
> + }
> + return tracking_remove(MKDEV(dev.major, dev.minor));
> + ;
> +}
> +
> +int ioctl_tracking_collect(unsigned long arg)
> +{
> + unsigned long len;
> + int res;
> + struct ioctl_tracking_collect_s get;
> +
> + pr_info("Collecting tracking devices:\n");
> +
> + len = copy_from_user(&get, (void *)arg, sizeof(struct ioctl_tracking_collect_s));
> + if (len != 0) {
> + pr_err("Unable to collect tracking devices: invalid user buffer\n");
> + return -ENODATA;
> + }
> +
> + if (get.p_cbt_info == NULL) {
> + res = tracking_collect(0x7FFFffff, NULL, &get.count);
> + if (res == SUCCESS) {
> + len = copy_to_user((void *)arg, (void *)&get,
> + sizeof(struct ioctl_tracking_collect_s));
> + if (len != 0) {
> + pr_err("Unable to collect tracking devices: invalid user buffer for arguments\n");
> + res = -ENODATA;
> + }
> + } else {
> + pr_err("Failed to execute tracking_collect. errno=%d\n", res);
> + }
> + } else {
> + struct cbt_info_s *p_cbt_info = NULL;
> +
> + p_cbt_info = kcalloc(get.count, sizeof(struct cbt_info_s), GFP_KERNEL);
> + if (p_cbt_info == NULL)
> + return -ENOMEM;
> +
> + do {
> + res = tracking_collect(get.count, p_cbt_info, &get.count);
> + if (res != SUCCESS) {
> + pr_err("Failed to execute tracking_collect. errno=%d\n", res);
> + break;
> + }
> + len = copy_to_user(get.p_cbt_info, p_cbt_info,
> + get.count * sizeof(struct cbt_info_s));
> + if (len != 0) {
> + pr_err("Unable to collect tracking devices: invalid user buffer for CBT info\n");
> + res = -ENODATA;
> + break;
> + }
> +
> + len = copy_to_user((void *)arg, (void *)&get,
> + sizeof(struct ioctl_tracking_collect_s));
> + if (len != 0) {
> + pr_err("Unable to collect tracking devices: invalid user buffer for arguments\n");
> + res = -ENODATA;
> + break;
> + }
> +
> + } while (false);
> +
> + kfree(p_cbt_info);
> + p_cbt_info = NULL;
> + }
> + return res;
> +}
> +
> +int ioctl_tracking_block_size(unsigned long arg)
> +{
> + unsigned long len;
> + unsigned int blk_sz = change_tracking_block_size();
> +
> + len = copy_to_user((void *)arg, &blk_sz, sizeof(unsigned int));
> + if (len != 0) {
> + pr_err("Unable to get tracking block size: invalid user buffer for arguments\n");
> + return -ENODATA;
> + }
> + return SUCCESS;
> +}
> +
> +int ioctl_tracking_read_cbt_map(unsigned long arg)
> +{
> + dev_t dev_id;
> + unsigned long len;
> + struct ioctl_tracking_read_cbt_bitmap_s readbitmap;
> +
> + len = copy_from_user(&readbitmap, (void *)arg,
> + sizeof(struct ioctl_tracking_read_cbt_bitmap_s));
> + if (len != 0) {
> + pr_err("Unable to read CBT map: invalid user buffer\n");
> + return -ENODATA;
> + }
> +
> + dev_id = MKDEV(readbitmap.dev_id.major, readbitmap.dev_id.minor);
> + return tracking_read_cbt_bitmap(dev_id, readbitmap.offset, readbitmap.length,
> + (void *)readbitmap.buff);
> +}
> +
> +int ioctl_tracking_mark_dirty_blocks(unsigned long arg)
> +{
> + unsigned long len;
> + struct ioctl_tracking_mark_dirty_blocks_s param;
> + struct block_range_s *p_dirty_blocks;
> + size_t buffer_size;
> + int result = SUCCESS;
> +
> + len = copy_from_user(¶m, (void *)arg,
> + sizeof(struct ioctl_tracking_mark_dirty_blocks_s));
> + if (len != 0) {
> + pr_err("Unable to mark dirty blocks: invalid user buffer\n");
> + return -ENODATA;
> + }
> +
> + buffer_size = param.count * sizeof(struct block_range_s);
> + p_dirty_blocks = kzalloc(buffer_size, GFP_KERNEL);
> + if (p_dirty_blocks == NULL) {
> + pr_err("Unable to mark dirty blocks: cannot allocate [%zu] bytes\n", buffer_size);
> + return -ENOMEM;
> + }
> +
> + do {
> + dev_t image_dev_id;
> +
> + len = copy_from_user(p_dirty_blocks, (void *)param.p_dirty_blocks, buffer_size);
> + if (len != 0) {
> + pr_err("Unable to mark dirty blocks: invalid user buffer\n");
> + result = -ENODATA;
> + break;
> + }
> +
> + image_dev_id = MKDEV(param.image_dev_id.major, param.image_dev_id.minor);
> + result = snapimage_mark_dirty_blocks(image_dev_id, p_dirty_blocks, param.count);
> + } while (false);
> + kfree(p_dirty_blocks);
> +
> + return result;
> +}
> +
> +int ioctl_snapshot_create(unsigned long arg)
> +{
> + unsigned long len;
> + size_t dev_id_buffer_size;
> + int status;
> + struct ioctl_snapshot_create_s param;
> + struct ioctl_dev_id_s *pk_dev_id = NULL;
> +
> + len = copy_from_user(¶m, (void *)arg, sizeof(struct ioctl_snapshot_create_s));
> + if (len != 0) {
> + pr_err("Unable to create snapshot: invalid user buffer\n");
> + return -ENODATA;
> + }
> +
> + dev_id_buffer_size = sizeof(struct ioctl_dev_id_s) * param.count;
> + pk_dev_id = kzalloc(dev_id_buffer_size, GFP_KERNEL);
> + if (pk_dev_id == NULL) {
> + pr_err("Unable to create snapshot: cannot allocate [%zu] bytes\n",
> + dev_id_buffer_size);
> + return -ENOMEM;
> + }
> +
> + do {
> + size_t dev_buffer_size;
> + dev_t *p_dev = NULL;
> + int inx = 0;
> +
> + len = copy_from_user(pk_dev_id, (void *)param.p_dev_id,
> + param.count * sizeof(struct ioctl_dev_id_s));
> + if (len != 0) {
> + pr_err("Unable to create snapshot: invalid user buffer for parameters\n");
> + status = -ENODATA;
> + break;
> + }
> +
> + dev_buffer_size = sizeof(dev_t) * param.count;
> + p_dev = kzalloc(dev_buffer_size, GFP_KERNEL);
> + if (p_dev == NULL) {
> + pr_err("Unable to create snapshot: cannot allocate [%zu] bytes\n",
> + dev_buffer_size);
> + status = -ENOMEM;
> + break;
> + }
> +
> + for (inx = 0; inx < param.count; ++inx)
> + p_dev[inx] = MKDEV(pk_dev_id[inx].major, pk_dev_id[inx].minor);
> +
> + status = snapshot_create(p_dev, param.count, ¶m.snapshot_id);
> +
> + kfree(p_dev);
> + p_dev = NULL;
> +
> + } while (false);
> + kfree(pk_dev_id);
> + pk_dev_id = NULL;
> +
> + if (status == SUCCESS) {
> + len = copy_to_user((void *)arg, ¶m, sizeof(struct ioctl_snapshot_create_s));
> + if (len != 0) {
> + pr_err("Unable to create snapshot: invalid user buffer\n");
> + status = -ENODATA;
> + }
> + }
> +
> + return status;
> +}
> +
> +int ioctl_snapshot_destroy(unsigned long arg)
> +{
> + unsigned long len;
> + unsigned long long param;
> +
> + len = copy_from_user(¶m, (void *)arg, sizeof(unsigned long long));
> + if (len != 0) {
> + pr_err("Unable to destroy snapshot: invalid user buffer\n");
> + return -ENODATA;
> + }
> +
> + return snapshot_destroy(param);
> +}
> +
> +static inline dev_t _snapstore_dev(struct ioctl_dev_id_s *dev_id)
> +{
> + if ((dev_id->major == 0) && (dev_id->minor == 0))
> + return 0; //memory snapstore
> +
> + if ((dev_id->major == -1) && (dev_id->minor == -1))
> + return 0xFFFFffff; //multidevice snapstore
> +
> + return MKDEV(dev_id->major, dev_id->minor);
> +}
> +
> +int ioctl_snapstore_create(unsigned long arg)
> +{
> + unsigned long len;
> + int res = SUCCESS;
> + struct ioctl_snapstore_create_s param;
> + size_t inx = 0;
> + dev_t *dev_id_set = NULL;
> +
> + len = copy_from_user(¶m, (void *)arg, sizeof(struct ioctl_snapstore_create_s));
> + if (len != 0) {
> + pr_err("Unable to create snapstore: invalid user buffer\n");
> + return -EINVAL;
> + }
> +
> + dev_id_set = kcalloc(param.count, sizeof(dev_t), GFP_KERNEL);
> + if (dev_id_set == NULL)
> + return -ENOMEM;
> +
> + for (inx = 0; inx < param.count; ++inx) {
> + struct ioctl_dev_id_s dev_id;
> +
> + len = copy_from_user(&dev_id, param.p_dev_id + inx, sizeof(struct ioctl_dev_id_s));
> + if (len != 0) {
> + pr_err("Unable to create snapstore: ");
> + pr_err("invalid user buffer for parameters\n");
> +
> + res = -ENODATA;
> + break;
> + }
> +
> + dev_id_set[inx] = MKDEV(dev_id.major, dev_id.minor);
> + }
> +
> + if (res == SUCCESS)
> + res = snapstore_create((uuid_t *)param.id, _snapstore_dev(¶m.snapstore_dev_id),
> + dev_id_set, (size_t)param.count);
> +
> + kfree(dev_id_set);
> +
> + return res;
> +}
> +
> +int ioctl_snapstore_file(unsigned long arg)
> +{
> + unsigned long len;
> + int res = SUCCESS;
> + struct ioctl_snapstore_file_add_s param;
> + struct big_buffer *ranges = NULL;
> + size_t ranges_buffer_size;
> +
> + len = copy_from_user(¶m, (void *)arg, sizeof(struct ioctl_snapstore_file_add_s));
> + if (len != 0) {
> + pr_err("Unable to add file to snapstore: invalid user buffer\n");
> + return -EINVAL;
> + }
> +
> + ranges_buffer_size = sizeof(struct ioctl_range_s) * param.range_count;
> +
> + ranges = big_buffer_alloc(ranges_buffer_size, GFP_KERNEL);
> + if (ranges == NULL) {
> + pr_err("Unable to add file to snapstore: cannot allocate [%zu] bytes\n",
> + ranges_buffer_size);
> + return -ENOMEM;
> + }
> +
> + if (big_buffer_copy_from_user((void *)param.ranges, 0, ranges, ranges_buffer_size)
> + != ranges_buffer_size) {
> +
> + pr_err("Unable to add file to snapstore: invalid user buffer for parameters\n");
> + res = -ENODATA;
> + } else
> + res = snapstore_add_file((uuid_t *)(param.id), ranges, (size_t)param.range_count);
> +
> + big_buffer_free(ranges);
> +
> + return res;
> +}
> +
> +int ioctl_snapstore_memory(unsigned long arg)
> +{
> + unsigned long len;
> + int res = SUCCESS;
> + struct ioctl_snapstore_memory_limit_s param;
> +
> + len = copy_from_user(¶m, (void *)arg, sizeof(struct ioctl_snapstore_memory_limit_s));
> + if (len != 0) {
> + pr_err("Unable to add memory block to snapstore: invalid user buffer\n");
> + return -EINVAL;
> + }
> +
> + res = snapstore_add_memory((uuid_t *)param.id, param.size);
> +
> + return res;
> +}
> +int ioctl_snapstore_cleanup(unsigned long arg)
> +{
> + unsigned long len;
> + int res = SUCCESS;
> + struct ioctl_snapstore_cleanup_s param;
> +
> + len = copy_from_user(¶m, (void *)arg, sizeof(struct ioctl_snapstore_cleanup_s));
> + if (len != 0) {
> + pr_err("Unable to perform snapstore cleanup: invalid user buffer\n");
> + return -EINVAL;
> + }
> +
> + pr_info("Cleanup snapstore %pUB\n", (uuid_t *)param.id);
> + res = snapstore_cleanup((uuid_t *)param.id, ¶m.filled_bytes);
> +
> + if (res == SUCCESS) {
> + if (0 !=
> + copy_to_user((void *)arg, ¶m, sizeof(struct ioctl_snapstore_cleanup_s))) {
> + pr_err("Unable to perform snapstore cleanup: invalid user buffer\n");
> + res = -ENODATA;
> + }
> + }
> +
> + return res;
> +}
> +
> +#ifdef CONFIG_BLK_SNAP_SNAPSTORE_MULTIDEV
> +int ioctl_snapstore_file_multidev(unsigned long arg)
> +{
> + unsigned long len;
> + int res = SUCCESS;
> + struct ioctl_snapstore_file_add_multidev_s param;
> + struct big_buffer *ranges = NULL; //struct ioctl_range_s* ranges = NULL;
> + size_t ranges_buffer_size;
> +
> + len = copy_from_user(¶m, (void *)arg,
> + sizeof(struct ioctl_snapstore_file_add_multidev_s));
> + if (len != 0) {
> + pr_err("Unable to add file to multidev snapstore: invalid user buffer\n");
> + return -EINVAL;
> + }
> +
> + ranges_buffer_size = sizeof(struct ioctl_range_s) * param.range_count;
> +
> + ranges = big_buffer_alloc(ranges_buffer_size, GFP_KERNEL);
> + if (ranges == NULL) {
> + pr_err("Unable to add file to multidev snapstore: cannot allocate [%zu] bytes\n",
> + ranges_buffer_size);
> + return -ENOMEM;
> + }
> +
> + do {
> + uuid_t *id = (uuid_t *)(param.id);
> + dev_t snapstore_device = MKDEV(param.dev_id.major, param.dev_id.minor);
> + size_t ranges_cnt = (size_t)param.range_count;
> +
> + if (ranges_buffer_size != big_buffer_copy_from_user((void *)param.ranges, 0, ranges,
> + ranges_buffer_size)) {
> + pr_err("Unable to add file to snapstore: invalid user buffer for parameters\n");
> + res = -ENODATA;
> + break;
> + }
> +
> + res = snapstore_add_multidev(id, snapstore_device, ranges, ranges_cnt);
> + } while (false);
> + big_buffer_free(ranges);
> +
> + return res;
> +}
> +
> +#endif
> +//////////////////////////////////////////////////////////////////////////
> +
> +/*
> + * Snapshot get errno for device
> + */
> +int ioctl_snapshot_errno(unsigned long arg)
> +{
> + unsigned long len;
> + int res;
> + struct ioctl_snapshot_errno_s param;
> +
> + len = copy_from_user(¶m, (void *)arg, sizeof(struct ioctl_dev_id_s));
> + if (len != 0) {
> + pr_err("Unable failed to get snapstore error code: invalid user buffer\n");
> + return -EINVAL;
> + }
> +
> + res = snapstore_device_errno(MKDEV(param.dev_id.major, param.dev_id.minor),
> + ¶m.err_code);
> +
> + if (res != SUCCESS)
> + return res;
> +
> + len = copy_to_user((void *)arg, ¶m, sizeof(struct ioctl_snapshot_errno_s));
> + if (len != 0) {
> + pr_err("Unable to get snapstore error code: invalid user buffer\n");
> + return -EINVAL;
> + }
> +
> + return SUCCESS;
> +}
> +
> +int ioctl_collect_snapimages(unsigned long arg)
> +{
> + unsigned long len;
> + int status = SUCCESS;
> + struct ioctl_collect_snapshot_images_s param;
> +
> + len = copy_from_user(¶m, (void *)arg, sizeof(struct ioctl_collect_snapshot_images_s));
> + if (len != 0) {
> + pr_err("Unable to collect snapshot images: invalid user buffer\n");
> + return -ENODATA;
> + }
> +
> + status = snapimage_collect_images(param.count, param.p_image_info, ¶m.count);
> +
> + len = copy_to_user((void *)arg, ¶m, sizeof(struct ioctl_collect_snapshot_images_s));
> + if (len != 0) {
> + pr_err("Unable to collect snapshot images: invalid user buffer\n");
> + return -ENODATA;
> + }
> +
> + return status;
> +}
> +
> +struct blk_snap_ioctl_table {
> + unsigned int cmd;
> + int (*fn)(unsigned long arg);
> +};
> +
> +static struct blk_snap_ioctl_table blk_snap_ioctl_table[] = {
> + { (IOCTL_COMPATIBILITY_FLAGS), ioctl_compatibility_flags },
> + { (IOCTL_GETVERSION), ioctl_get_version },
> +
> + { (IOCTL_TRACKING_ADD), ioctl_tracking_add },
> + { (IOCTL_TRACKING_REMOVE), ioctl_tracking_remove },
> + { (IOCTL_TRACKING_COLLECT), ioctl_tracking_collect },
> + { (IOCTL_TRACKING_BLOCK_SIZE), ioctl_tracking_block_size },
> + { (IOCTL_TRACKING_READ_CBT_BITMAP), ioctl_tracking_read_cbt_map },
> + { (IOCTL_TRACKING_MARK_DIRTY_BLOCKS), ioctl_tracking_mark_dirty_blocks },
> +
> + { (IOCTL_SNAPSHOT_CREATE), ioctl_snapshot_create },
> + { (IOCTL_SNAPSHOT_DESTROY), ioctl_snapshot_destroy },
> + { (IOCTL_SNAPSHOT_ERRNO), ioctl_snapshot_errno },
> +
> + { (IOCTL_SNAPSTORE_CREATE), ioctl_snapstore_create },
> + { (IOCTL_SNAPSTORE_FILE), ioctl_snapstore_file },
> + { (IOCTL_SNAPSTORE_MEMORY), ioctl_snapstore_memory },
> + { (IOCTL_SNAPSTORE_CLEANUP), ioctl_snapstore_cleanup },
> +#ifdef CONFIG_BLK_SNAP_SNAPSTORE_MULTIDEV
> + { (IOCTL_SNAPSTORE_FILE_MULTIDEV), ioctl_snapstore_file_multidev },
> +#endif
> + { (IOCTL_COLLECT_SNAPSHOT_IMAGES), ioctl_collect_snapimages },
> + { 0, NULL }
> +};
> +
> +long ctrl_unlocked_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
> +{
> + long status = -ENOTTY;
> + size_t inx = 0;
> +
> + while (blk_snap_ioctl_table[inx].cmd != 0) {
> + if (blk_snap_ioctl_table[inx].cmd == cmd) {
> + status = blk_snap_ioctl_table[inx].fn(arg);
> + break;
> + }
> + ++inx;
> + }
> +
> + return status;
> +}
> diff --git a/drivers/block/blk-snap/ctrl_fops.h b/drivers/block/blk-snap/ctrl_fops.h
> new file mode 100644
> index 000000000000..98072b61aa96
> --- /dev/null
> +++ b/drivers/block/blk-snap/ctrl_fops.h
> @@ -0,0 +1,19 @@
> +/* SPDX-License-Identifier: GPL-2.0-or-later */
> +#pragma once
> +
> +#include <linux/fs.h>
> +
> +int get_blk_snap_major(void);
> +
> +int ctrl_init(void);
> +void ctrl_done(void);
> +
> +int ctrl_open(struct inode *inode, struct file *file);
> +int ctrl_release(struct inode *inode, struct file *file);
> +
> +ssize_t ctrl_read(struct file *filp, char __user *buffer, size_t length, loff_t *offset);
> +ssize_t ctrl_write(struct file *filp, const char __user *buffer, size_t length, loff_t *offset);
> +
> +unsigned int ctrl_poll(struct file *filp, struct poll_table_struct *wait);
> +
> +long ctrl_unlocked_ioctl(struct file *file, unsigned int cmd, unsigned long arg);
> diff --git a/drivers/block/blk-snap/ctrl_pipe.c b/drivers/block/blk-snap/ctrl_pipe.c
> new file mode 100644
> index 000000000000..73cfbca93487
> --- /dev/null
> +++ b/drivers/block/blk-snap/ctrl_pipe.c
> @@ -0,0 +1,562 @@
> +// SPDX-License-Identifier: GPL-2.0
> +#define BLK_SNAP_SECTION "-ctrl"
> +#include "common.h"
> +#include "ctrl_pipe.h"
> +#include "version.h"
> +#include "blk-snap-ctl.h"
> +#include "snapstore.h"
> +#include "big_buffer.h"
> +
> +#include <linux/poll.h>
> +#include <linux/uuid.h>
> +
> +#define CMD_TO_USER_FIFO_SIZE 1024
> +
> +LIST_HEAD(ctl_pipes);
> +DECLARE_RWSEM(ctl_pipes_lock);
> +
> +
> +static void ctrl_pipe_push_request(struct ctrl_pipe *pipe, unsigned int *cmd, size_t cmd_len)
> +{
> + kfifo_in_spinlocked(&pipe->cmd_to_user, cmd, (cmd_len * sizeof(unsigned int)),
> + &pipe->cmd_to_user_lock);
> +
> + wake_up(&pipe->readq);
> +}
> +
> +static void ctrl_pipe_request_acknowledge(struct ctrl_pipe *pipe, unsigned int result)
> +{
> + unsigned int cmd[2];
> +
> + cmd[0] = BLK_SNAP_CHARCMD_ACKNOWLEDGE;
> + cmd[1] = result;
> +
> + ctrl_pipe_push_request(pipe, cmd, 2);
> +}
> +
> +static inline dev_t _snapstore_dev(struct ioctl_dev_id_s *dev_id)
> +{
> + if ((dev_id->major == 0) && (dev_id->minor == 0))
> + return 0; //memory snapstore
> +
> + if ((dev_id->major == -1) && (dev_id->minor == -1))
> + return 0xFFFFffff; //multidevice snapstore
> +
> + return MKDEV(dev_id->major, dev_id->minor);
> +}
> +
> +static ssize_t ctrl_pipe_command_initiate(struct ctrl_pipe *pipe, const char __user *buffer,
> + size_t length)
> +{
> + unsigned long len;
> + int result = SUCCESS;
> + ssize_t processed = 0;
> + char *kernel_buffer;
> +
> + kernel_buffer = kmalloc(length, GFP_KERNEL);
> + if (kernel_buffer == NULL)
> + return -ENOMEM;
> +
> + len = copy_from_user(kernel_buffer, buffer, length);
> + if (len != 0) {
> + kfree(kernel_buffer);
> + pr_err("Unable to write to pipe: invalid user buffer\n");
> + return -EINVAL;
> + }
> +
> + do {
> + u64 stretch_empty_limit;
> + unsigned int dev_id_list_length;
> + uuid_t *unique_id;
> + struct ioctl_dev_id_s *snapstore_dev_id;
> + struct ioctl_dev_id_s *dev_id_list;
> +
> + //get snapstore uuid
> + if ((length - processed) < 16) {
> + pr_err("Unable to get snapstore uuid: invalid ctrl pipe initiate command. length=%zu\n",
> + length);
> + break;
> + }
> + unique_id = (uuid_t *)(kernel_buffer + processed);
> + processed += 16;
> +
> + //get snapstore empty limit
> + if ((length - processed) < sizeof(u64)) {
> + pr_err("Unable to get stretch snapstore limit: invalid ctrl pipe initiate command. length=%zu\n",
> + length);
> + break;
> + }
> + stretch_empty_limit = *(u64 *)(kernel_buffer + processed);
> + processed += sizeof(u64);
> +
> + //get snapstore device id
> + if ((length - processed) < sizeof(struct ioctl_dev_id_s)) {
> + pr_err("Unable to get snapstore device id: invalid ctrl pipe initiate command. length=%zu\n",
> + length);
> + break;
> + }
> + snapstore_dev_id = (struct ioctl_dev_id_s *)(kernel_buffer + processed);
> + processed += sizeof(struct ioctl_dev_id_s);
> +
> + //get device id list length
> + if ((length - processed) < 4) {
> + pr_err("Unable to get device id list length: ivalid ctrl pipe initiate command. length=%zu\n",
> + length);
> + break;
> + }
> + dev_id_list_length = *(unsigned int *)(kernel_buffer + processed);
> + processed += sizeof(unsigned int);
> +
> + //get devices id list
> + if ((length - processed) < (dev_id_list_length * sizeof(struct ioctl_dev_id_s))) {
> + pr_err("Unable to get all devices from device id list: invalid ctrl pipe initiate command. length=%zu\n",
> + length);
> + break;
> + }
> + dev_id_list = (struct ioctl_dev_id_s *)(kernel_buffer + processed);
> + processed += (dev_id_list_length * sizeof(struct ioctl_dev_id_s));
> +
> + {
> + size_t inx;
> + dev_t *dev_set;
> + size_t dev_id_set_length = (size_t)dev_id_list_length;
> +
> + dev_set = kcalloc(dev_id_set_length, sizeof(dev_t), GFP_KERNEL);
> + if (dev_set == NULL) {
> + result = -ENOMEM;
> + break;
> + }
> +
> + for (inx = 0; inx < dev_id_set_length; ++inx)
> + dev_set[inx] =
> + MKDEV(dev_id_list[inx].major, dev_id_list[inx].minor);
> +
> + result = snapstore_create(unique_id, _snapstore_dev(snapstore_dev_id),
> + dev_set, dev_id_set_length);
> + kfree(dev_set);
> + if (result != SUCCESS) {
> + pr_err("Failed to create snapstore\n");
> + break;
> + }
> +
> + result = snapstore_stretch_initiate(
> + unique_id, pipe, (sector_t)to_sectors(stretch_empty_limit));
> + if (result != SUCCESS) {
> + pr_err("Failed to initiate stretch snapstore %pUB\n", unique_id);
> + break;
> + }
> + }
> + } while (false);
> + kfree(kernel_buffer);
> + ctrl_pipe_request_acknowledge(pipe, result);
> +
> + if (result == SUCCESS)
> + return processed;
> + return result;
> +}
> +
> +static ssize_t ctrl_pipe_command_next_portion(struct ctrl_pipe *pipe, const char __user *buffer,
> + size_t length)
> +{
> + unsigned long len;
> + int result = SUCCESS;
> + ssize_t processed = 0;
> + struct big_buffer *ranges = NULL;
> +
> + do {
> + uuid_t unique_id;
> + unsigned int ranges_length;
> + size_t ranges_buffer_size;
> +
> + //get snapstore id
> + if ((length - processed) < 16) {
> + pr_err("Unable to get snapstore id: ");
> + pr_err("invalid ctrl pipe next portion command. length=%zu\n",
> + length);
> + break;
> + }
> + len = copy_from_user(&unique_id, buffer + processed, sizeof(uuid_t));
> + if (len != 0) {
> + pr_err("Unable to write to pipe: invalid user buffer\n");
> + processed = -EINVAL;
> + break;
> + }
> + processed += 16;
> +
> + //get ranges length
> + if ((length - processed) < 4) {
> + pr_err("Unable to get device id list length: ");
> + pr_err("invalid ctrl pipe next portion command. length=%zu\n",
> + length);
> + break;
> + }
> + len = copy_from_user(&ranges_length, buffer + processed, sizeof(unsigned int));
> + if (len != 0) {
> + pr_err("Unable to write to pipe: invalid user buffer\n");
> + processed = -EINVAL;
> + break;
> + }
> + processed += sizeof(unsigned int);
> +
> + ranges_buffer_size = ranges_length * sizeof(struct ioctl_range_s);
> +
> + // ranges
> + if ((length - processed) < (ranges_buffer_size)) {
> + pr_err("Unable to get all ranges: ");
> + pr_err("invalid ctrl pipe next portion command. length=%zu\n",
> + length);
> + break;
> + }
> + ranges = big_buffer_alloc(ranges_buffer_size, GFP_KERNEL);
> + if (ranges == NULL) {
> + pr_err("Unable to allocate page array buffer: ");
> + pr_err("failed to process next portion command\n");
> + processed = -ENOMEM;
> + break;
> + }
> + if (ranges_buffer_size !=
> + big_buffer_copy_from_user(buffer + processed, 0, ranges, ranges_buffer_size)) {
> + pr_err("Unable to process next portion command: ");
> + pr_err("invalid user buffer for parameters\n");
> + processed = -EINVAL;
> + break;
> + }
> + processed += ranges_buffer_size;
> +
> + {
> + result = snapstore_add_file(&unique_id, ranges, ranges_length);
> +
> + if (result != SUCCESS) {
> + pr_err("Failed to add file to snapstore\n");
> + result = -ENODEV;
> + break;
> + }
> + }
> + } while (false);
> + if (ranges)
> + big_buffer_free(ranges);
> +
> + if (result == SUCCESS)
> + return processed;
> + return result;
> +}
> +
> +#ifdef CONFIG_BLK_SNAP_SNAPSTORE_MULTIDEV
> +static ssize_t ctrl_pipe_command_next_portion_multidev(struct ctrl_pipe *pipe,
> + const char __user *buffer, size_t length)
> +{
> + unsigned long len;
> + int result = SUCCESS;
> + ssize_t processed = 0;
> + struct big_buffer *ranges = NULL;
> +
> + do {
> + uuid_t unique_id;
> + int snapstore_major;
> + int snapstore_minor;
> + unsigned int ranges_length;
> + size_t ranges_buffer_size;
> +
> + //get snapstore id
> + if ((length - processed) < 16) {
> + pr_err("Unable to get snapstore id: ");
> + pr_err("invalid ctrl pipe next portion command. length=%zu\n",
> + length);
> + break;
> + }
> + len = copy_from_user(&unique_id, buffer + processed, sizeof(uuid_t));
> + if (len != 0) {
> + pr_err("Unable to write to pipe: invalid user buffer\n");
> + processed = -EINVAL;
> + break;
> + }
> + processed += 16;
> +
> + //get device id
> + if ((length - processed) < 8) {
> + pr_err("Unable to get device id list length: ");
> + pr_err("invalid ctrl pipe next portion command. length=%zu\n", length);
> + break;
> + }
> + len = copy_from_user(&snapstore_major, buffer + processed, sizeof(unsigned int));
> + if (len != 0) {
> + pr_err("Unable to write to pipe: invalid user buffer\n");
> + processed = -EINVAL;
> + break;
> + }
> + processed += sizeof(unsigned int);
> +
> + len = copy_from_user(&snapstore_minor, buffer + processed, sizeof(unsigned int));
> + if (len != 0) {
> + pr_err("Unable to write to pipe: invalid user buffer\n");
> + processed = -EINVAL;
> + break;
> + }
> + processed += sizeof(unsigned int);
> +
> + //get ranges length
> + if ((length - processed) < 4) {
> + pr_err("Unable to get device id list length: ");
> + pr_err("invalid ctrl pipe next portion command. length=%zu\n",
> + length);
> + break;
> + }
> + len = copy_from_user(&ranges_length, buffer + processed, sizeof(unsigned int));
> + if (len != 0) {
> + pr_err("Unable to write to pipe: invalid user buffer\n");
> + processed = -EINVAL;
> + break;
> + }
> + processed += sizeof(unsigned int);
> +
> + ranges_buffer_size = ranges_length * sizeof(struct ioctl_range_s);
> +
> + // ranges
> + if ((length - processed) < (ranges_buffer_size)) {
> + pr_err("Unable to get all ranges: ");
> + pr_err("invalid ctrl pipe next portion command. length=%zu\n",
> + length);
> + break;
> + }
> + ranges = big_buffer_alloc(ranges_buffer_size, GFP_KERNEL);
> + if (ranges == NULL) {
> + pr_err("Unable to process next portion command: ");
> + pr_err("failed to allocate page array buffer\n");
> + processed = -ENOMEM;
> + break;
> + }
> + if (ranges_buffer_size !=
> + big_buffer_copy_from_user(buffer + processed, 0, ranges, ranges_buffer_size)) {
> + pr_err("Unable to process next portion command: ");
> + pr_err("invalid user buffer from parameters\n");
> + processed = -EINVAL;
> + break;
> + }
> + processed += ranges_buffer_size;
> +
> + {
> + result = snapstore_add_multidev(&unique_id,
> + MKDEV(snapstore_major, snapstore_minor),
> + ranges, ranges_length);
> +
> + if (result != SUCCESS) {
> + pr_err("Failed to add file to snapstore\n");
> + result = -ENODEV;
> + break;
> + }
> + }
> + } while (false);
> + if (ranges)
> + big_buffer_free(ranges);
> +
> + if (result == SUCCESS)
> + return processed;
> +
> + return result;
> +}
> +#endif
> +
> +static void ctrl_pipe_release_cb(struct kref *kref)
> +{
> + struct ctrl_pipe *pipe = container_of(kref, struct ctrl_pipe, refcount);
> +
> + down_write(&ctl_pipes_lock);
> + list_del(&pipe->link);
> + up_write(&ctl_pipes_lock);
> +
> + kfifo_free(&pipe->cmd_to_user);
> +
> + kfree(pipe);
> +}
> +
> +struct ctrl_pipe *ctrl_pipe_get_resource(struct ctrl_pipe *pipe)
> +{
> + if (pipe)
> + kref_get(&pipe->refcount);
> +
> + return pipe;
> +}
> +
> +void ctrl_pipe_put_resource(struct ctrl_pipe *pipe)
> +{
> + if (pipe)
> + kref_put(&pipe->refcount, ctrl_pipe_release_cb);
> +}
> +
> +void ctrl_pipe_done(void)
> +{
> + bool is_empty;
> +
> + pr_info("Ctrl pipes - done\n");
> +
> + down_write(&ctl_pipes_lock);
> + is_empty = list_empty(&ctl_pipes);
> + up_write(&ctl_pipes_lock);
> +
> + if (!is_empty)
> + pr_err("Unable to perform ctrl pipes cleanup: container is not empty\n");
> +}
> +
> +struct ctrl_pipe *ctrl_pipe_new(void)
> +{
> + int ret;
> + struct ctrl_pipe *pipe;
> +
> + pipe = kzalloc(sizeof(struct ctrl_pipe), GFP_KERNEL);
> + if (pipe == NULL)
> + return NULL;
> +
> + INIT_LIST_HEAD(&pipe->link);
> +
> + ret = kfifo_alloc(&pipe->cmd_to_user, CMD_TO_USER_FIFO_SIZE, GFP_KERNEL);
> + if (ret) {
> + pr_err("Failed to allocate fifo. errno=%d.\n", ret);
> + kfree(pipe);
> + return NULL;
> + }
> + spin_lock_init(&pipe->cmd_to_user_lock);
> +
> + kref_init(&pipe->refcount);
> +
> + init_waitqueue_head(&pipe->readq);
> +
> + down_write(&ctl_pipes_lock);
> + list_add_tail(&pipe->link, &ctl_pipes);
> + up_write(&ctl_pipes_lock);
> +
> + return pipe;
> +}
> +
> +ssize_t ctrl_pipe_read(struct ctrl_pipe *pipe, char __user *buffer, size_t length)
> +{
> + int ret;
> + unsigned int processed = 0;
> +
> + if (kfifo_is_empty_spinlocked(&pipe->cmd_to_user, &pipe->cmd_to_user_lock)) {
> + //nothing to read
> + ret = wait_event_interruptible(pipe->readq,
> + !kfifo_is_empty_spinlocked(&pipe->cmd_to_user,
> + &pipe->cmd_to_user_lock));
> + if (ret) {
> + pr_err("Unable to wait for pipe read queue: interrupt signal was received\n");
> + return -ERESTARTSYS;
> + }
> + }
> +
> + ret = kfifo_to_user(&pipe->cmd_to_user, buffer, length, &processed);
> + if (ret) {
> + pr_err("Failed to read command from ctrl pipe\n");
> + return ret;
> + }
> +
> + return (ssize_t)processed;
> +}
> +
> +ssize_t ctrl_pipe_write(struct ctrl_pipe *pipe, const char __user *buffer, size_t length)
> +{
> + ssize_t processed = 0;
> +
> + do {
> + unsigned long len;
> + unsigned int command;
> +
> + if ((length - processed) < 4) {
> + pr_err("Unable to write command to ctrl pipe: invalid command length=%zu\n",
> + length);
> + break;
> + }
> + len = copy_from_user(&command, buffer + processed, sizeof(unsigned int));
> + if (len != 0) {
> + pr_err("Unable to write to pipe: invalid user buffer\n");
> + processed = -EINVAL;
> + break;
> + }
> + processed += sizeof(unsigned int);
> + //+4
> + switch (command) {
> + case BLK_SNAP_CHARCMD_INITIATE: {
> + ssize_t res = ctrl_pipe_command_initiate(pipe, buffer + processed,
> + length - processed);
> + if (res >= 0)
> + processed += res;
> + else
> + processed = res;
> + } break;
> + case BLK_SNAP_CHARCMD_NEXT_PORTION: {
> + ssize_t res = ctrl_pipe_command_next_portion(pipe, buffer + processed,
> + length - processed);
> + if (res >= 0)
> + processed += res;
> + else
> + processed = res;
> + } break;
> +#ifdef CONFIG_BLK_SNAP_SNAPSTORE_MULTIDEV
> + case BLK_SNAP_CHARCMD_NEXT_PORTION_MULTIDEV: {
> + ssize_t res = ctrl_pipe_command_next_portion_multidev(
> + pipe, buffer + processed, length - processed);
> + if (res >= 0)
> + processed += res;
> + else
> + processed = res;
> + } break;
> +#endif
> + default:
> + pr_err("Ctrl pipe write error: invalid command [0x%x] received\n", command);
> + break;
> + }
> + } while (false);
> + return processed;
> +}
> +
> +unsigned int ctrl_pipe_poll(struct ctrl_pipe *pipe)
> +{
> + unsigned int mask = 0;
> +
> + if (!kfifo_is_empty_spinlocked(&pipe->cmd_to_user, &pipe->cmd_to_user_lock))
> + mask |= (POLLIN | POLLRDNORM); /* readable */
> +
> + mask |= (POLLOUT | POLLWRNORM); /* writable */
> +
> + return mask;
> +}
> +
> +void ctrl_pipe_request_halffill(struct ctrl_pipe *pipe, unsigned long long filled_status)
> +{
> + unsigned int cmd[3];
> +
> + pr_info("Snapstore is half-full\n");
> +
> + cmd[0] = (unsigned int)BLK_SNAP_CHARCMD_HALFFILL;
> + cmd[1] = (unsigned int)(filled_status & 0xFFFFffff); //lo
> + cmd[2] = (unsigned int)(filled_status >> 32);
> +
> + ctrl_pipe_push_request(pipe, cmd, 3);
> +}
> +
> +void ctrl_pipe_request_overflow(struct ctrl_pipe *pipe, unsigned int error_code,
> + unsigned long long filled_status)
> +{
> + unsigned int cmd[4];
> +
> + pr_info("Snapstore overflow\n");
> +
> + cmd[0] = (unsigned int)BLK_SNAP_CHARCMD_OVERFLOW;
> + cmd[1] = error_code;
> + cmd[2] = (unsigned int)(filled_status & 0xFFFFffff); //lo
> + cmd[3] = (unsigned int)(filled_status >> 32);
> +
> + ctrl_pipe_push_request(pipe, cmd, 4);
> +}
> +
> +void ctrl_pipe_request_terminate(struct ctrl_pipe *pipe, unsigned long long filled_status)
> +{
> + unsigned int cmd[3];
> +
> + pr_info("Snapstore termination\n");
> +
> + cmd[0] = (unsigned int)BLK_SNAP_CHARCMD_TERMINATE;
> + cmd[1] = (unsigned int)(filled_status & 0xFFFFffff); //lo
> + cmd[2] = (unsigned int)(filled_status >> 32);
> +
> + ctrl_pipe_push_request(pipe, cmd, 3);
> +}
> diff --git a/drivers/block/blk-snap/ctrl_pipe.h b/drivers/block/blk-snap/ctrl_pipe.h
> new file mode 100644
> index 000000000000..1aa1099eec25
> --- /dev/null
> +++ b/drivers/block/blk-snap/ctrl_pipe.h
> @@ -0,0 +1,34 @@
> +/* SPDX-License-Identifier: GPL-2.0-or-later */
> +#pragma once
> +
> +#include <linux/kref.h>
> +#include <linux/wait.h>
> +#include <linux/kfifo.h>
> +
> +struct ctrl_pipe {
> + struct list_head link;
> +
> + struct kref refcount;
> +
> + wait_queue_head_t readq;
> +
> + struct kfifo cmd_to_user;
> + spinlock_t cmd_to_user_lock;
> +};
> +
> +struct ctrl_pipe *ctrl_pipe_get_resource(struct ctrl_pipe *pipe);
> +void ctrl_pipe_put_resource(struct ctrl_pipe *pipe);
> +
> +void ctrl_pipe_done(void);
> +
> +struct ctrl_pipe *ctrl_pipe_new(void);
> +
> +ssize_t ctrl_pipe_read(struct ctrl_pipe *pipe, char __user *buffer, size_t length);
> +ssize_t ctrl_pipe_write(struct ctrl_pipe *pipe, const char __user *buffer, size_t length);
> +
> +unsigned int ctrl_pipe_poll(struct ctrl_pipe *pipe);
> +
> +void ctrl_pipe_request_halffill(struct ctrl_pipe *pipe, unsigned long long filled_status);
> +void ctrl_pipe_request_overflow(struct ctrl_pipe *pipe, unsigned int error_code,
> + unsigned long long filled_status);
> +void ctrl_pipe_request_terminate(struct ctrl_pipe *pipe, unsigned long long filled_status);
> diff --git a/drivers/block/blk-snap/ctrl_sysfs.c b/drivers/block/blk-snap/ctrl_sysfs.c
> new file mode 100644
> index 000000000000..4ec78e85b510
> --- /dev/null
> +++ b/drivers/block/blk-snap/ctrl_sysfs.c
> @@ -0,0 +1,73 @@
> +// SPDX-License-Identifier: GPL-2.0
> +#define BLK_SNAP_SECTION "-ctrl"
> +#include "common.h"
> +#include "ctrl_sysfs.h"
> +#include "ctrl_fops.h"
> +#include "blk-snap-ctl.h"
> +
> +#include <linux/blkdev.h>
> +#include <linux/sysfs.h>
> +
> +static ssize_t major_show(struct class *class, struct class_attribute *attr, char *buf)
> +{
> + sprintf(buf, "%d", get_blk_snap_major());
> + return strlen(buf);
> +}
> +
> +CLASS_ATTR_RO(major); // declare class_attr_major
> +static struct class *blk_snap_class;
> +
> +static struct device *blk_snap_device;
> +
> +int ctrl_sysfs_init(void)
> +{
> + struct device *dev;
> + int res;
> +
> + blk_snap_class = class_create(THIS_MODULE, MODULE_NAME);
> + if (IS_ERR(blk_snap_class)) {
> + res = PTR_ERR(blk_snap_class);
> +
> + pr_err("Bad class create. errno=%d\n", 0 - res);
> + return res;
> + }
> +
> + pr_info("Create 'major' sysfs attribute\n");
> + res = class_create_file(blk_snap_class, &class_attr_major);
> + if (res != SUCCESS) {
> + pr_err("Failed to create 'major' sysfs file\n");
> +
> + class_destroy(blk_snap_class);
> + blk_snap_class = NULL;
> + return res;
> + }
> +
> + dev = device_create(blk_snap_class, NULL, MKDEV(get_blk_snap_major(), 0), NULL,
> + MODULE_NAME);
> + if (IS_ERR(dev)) {
> + res = PTR_ERR(dev);
> + pr_err("Failed to create device, errno=%d\n", res);
> +
> + class_remove_file(blk_snap_class, &class_attr_major);
> + class_destroy(blk_snap_class);
> + blk_snap_class = NULL;
> + return res;
> + }
> +
> + blk_snap_device = dev;
> + return res;
> +}
> +
> +void ctrl_sysfs_done(void)
> +{
> + if (blk_snap_device) {
> + device_unregister(blk_snap_device);
> + blk_snap_device = NULL;
> + }
> +
> + if (blk_snap_class != NULL) {
> + class_remove_file(blk_snap_class, &class_attr_major);
> + class_destroy(blk_snap_class);
> + blk_snap_class = NULL;
> + }
> +}
> diff --git a/drivers/block/blk-snap/ctrl_sysfs.h b/drivers/block/blk-snap/ctrl_sysfs.h
> new file mode 100644
> index 000000000000..27a2a4d3da4c
> --- /dev/null
> +++ b/drivers/block/blk-snap/ctrl_sysfs.h
> @@ -0,0 +1,5 @@
> +/* SPDX-License-Identifier: GPL-2.0-or-later */
> +#pragma once
> +
> +int ctrl_sysfs_init(void);
> +void ctrl_sysfs_done(void);
> diff --git a/drivers/block/blk-snap/defer_io.c b/drivers/block/blk-snap/defer_io.c
> new file mode 100644
> index 000000000000..309216fe7319
> --- /dev/null
> +++ b/drivers/block/blk-snap/defer_io.c
> @@ -0,0 +1,397 @@
> +// SPDX-License-Identifier: GPL-2.0
> +#define BLK_SNAP_SECTION "-defer_io"
> +#include "common.h"
> +#include "defer_io.h"
> +#include "blk_deferred.h"
> +#include "tracker.h"
> +#include "blk_util.h"
> +
> +#include <linux/kthread.h>
> +
> +#define BLK_IMAGE_THROTTLE_TIMEOUT (1 * HZ) //delay 1 sec
> +//#define BLK_IMAGE_THROTTLE_TIMEOUT ( HZ/1000 * 10 ) //delay 10 ms
> +
> +
> +
> +struct defer_io_orig_rq {
> + struct list_head link;
> + struct defer_io_queue *queue;
> +
> + struct bio *bio;
> + struct tracker *tracker;
> +};
> +
> +static inline void defer_io_queue_init(struct defer_io_queue *queue)
> +{
> + INIT_LIST_HEAD(&queue->list);
> +
> + spin_lock_init(&queue->lock);
> +
> + atomic_set(&queue->in_queue_cnt, 0);
> + atomic_set(&queue->active_state, true);
> +}
> +
> +static inline struct defer_io_orig_rq *defer_io_queue_new(struct defer_io_queue *queue, struct bio *bio)
> +{
> + struct defer_io_orig_rq *dio_rq;
> +
> + dio_rq = kzalloc(sizeof(struct defer_io_orig_rq), GFP_NOIO);
> + if (dio_rq == NULL)
> + return NULL;
> +
> + dio_rq->bio = bio;
> + bio_get(dio_rq->bio);
> +
> + INIT_LIST_HEAD(&dio_rq->link);
> + dio_rq->queue = queue;
> +
> + return dio_rq;
> +}
> +
> +static inline void defer_io_queue_free(struct defer_io_orig_rq *dio_rq)
> +{
> + if (likely(dio_rq)) {
> + if (likely(dio_rq->bio)) {
> + bio_put(dio_rq->bio);
> + dio_rq->bio = NULL;
> + }
> + kfree(dio_rq);
> + }
> +}
> +
> +static int defer_io_queue_push_back(struct defer_io_queue *queue, struct defer_io_orig_rq *dio_rq)
> +{
> + int res = SUCCESS;
> +
> + spin_lock(&queue->lock);
> +
> + if (atomic_read(&queue->active_state)) {
> + list_add_tail(&dio_rq->link, &queue->list);
> + atomic_inc(&queue->in_queue_cnt);
> + } else
> + res = -EACCES;
> +
> + spin_unlock(&queue->lock);
> + return res;
> +}
> +
> +static struct defer_io_orig_rq *defer_io_queue_get_first(struct defer_io_queue *queue)
> +{
> + struct defer_io_orig_rq *dio_rq = NULL;
> +
> + spin_lock(&queue->lock);
> +
> + if (!list_empty(&queue->list)) {
> + dio_rq = list_entry(queue->list.next, struct defer_io_orig_rq, link);
> + list_del(&dio_rq->link);
> + atomic_dec(&queue->in_queue_cnt);
> + }
> +
> + spin_unlock(&queue->lock);
> +
> + return dio_rq;
> +}
> +
> +static bool defer_io_queue_active(struct defer_io_queue *queue, bool state)
> +{
> + bool prev_state;
> +
> + spin_lock(&queue->lock);
> +
> + prev_state = atomic_read(&queue->active_state);
> + atomic_set(&queue->active_state, state);
> +
> + spin_unlock(&queue->lock);
> +
> + return prev_state;
> +}
> +
> +#define defer_io_queue_empty(queue) (atomic_read(&(queue).in_queue_cnt) == 0)
> +
> +static void _defer_io_finish(struct defer_io *defer_io, struct defer_io_queue *queue_in_progress)
> +{
> + while (!defer_io_queue_empty(*queue_in_progress)) {
> + struct tracker *tracker = NULL;
> + bool cbt_locked = false;
> + bool is_write_bio;
> + sector_t sectCount = 0;
> +
> + struct defer_io_orig_rq *orig_req = defer_io_queue_get_first(queue_in_progress);
> +
> + is_write_bio = bio_data_dir(orig_req->bio) && bio_has_data(orig_req->bio);
> +
> + if (orig_req->tracker && is_write_bio) {
> + tracker = orig_req->tracker;
> + cbt_locked = tracker_cbt_bitmap_lock(tracker);
> + if (cbt_locked) {
> + sectCount = bio_sectors(orig_req->bio);
> + tracker_cbt_bitmap_set(tracker, orig_req->bio->bi_iter.bi_sector,
> + sectCount);
> + }
> + }
> +
> + submit_bio_direct(orig_req->bio);
> +
> + if (cbt_locked)
> + tracker_cbt_bitmap_unlock(tracker);
> +
> + defer_io_queue_free(orig_req);
> + }
> +}
> +
> +static int _defer_io_copy_prepare(struct defer_io *defer_io,
> + struct defer_io_queue *queue_in_process,
> + struct blk_deferred_request **dio_copy_req)
> +{
> + int res = SUCCESS;
> + int dios_count = 0;
> + sector_t dios_sectors_count = 0;
> +
> + //fill copy_request set
> + while (!defer_io_queue_empty(defer_io->dio_queue) &&
> + (dios_count < DEFER_IO_DIO_REQUEST_LENGTH) &&
> + (dios_sectors_count < DEFER_IO_DIO_REQUEST_SECTORS_COUNT)) {
> + struct defer_io_orig_rq *dio_orig_req =
> + (struct defer_io_orig_rq *)defer_io_queue_get_first(&defer_io->dio_queue);
> + atomic_dec(&defer_io->queue_filling_count);
> +
> + defer_io_queue_push_back(queue_in_process, dio_orig_req);
> +
> + if (!kthread_should_stop() &&
> + !snapstore_device_is_corrupted(defer_io->snapstore_device)) {
> + if (bio_data_dir(dio_orig_req->bio) && bio_has_data(dio_orig_req->bio)) {
> + struct blk_range copy_range;
> +
> + copy_range.ofs = dio_orig_req->bio->bi_iter.bi_sector;
> + copy_range.cnt = bio_sectors(dio_orig_req->bio);
> + res = snapstore_device_prepare_requests(defer_io->snapstore_device,
> + ©_range, dio_copy_req);
> + if (res != SUCCESS) {
> + pr_err("Unable to execute Copy On Write algorithm: failed to add ranges to copy to snapstore request. errno=%d\n",
> + res);
> + break;
> + }
> +
> + dios_sectors_count += copy_range.cnt;
> + }
> + }
> + ++dios_count;
> + }
> + return res;
> +}
> +
> +static int defer_io_work_thread(void *p)
> +{
> + struct defer_io_queue queue_in_process = { 0 };
> + struct defer_io *defer_io = NULL;
> +
> + //set_user_nice( current, -20 ); //MIN_NICE
> + defer_io_queue_init(&queue_in_process);
> +
> + defer_io = defer_io_get_resource((struct defer_io *)p);
> + pr_info("Defer IO thread for original device [%d:%d] started\n",
> + MAJOR(defer_io->original_dev_id), MINOR(defer_io->original_dev_id));
> +
> + while (!kthread_should_stop() || !defer_io_queue_empty(defer_io->dio_queue)) {
> + if (defer_io_queue_empty(defer_io->dio_queue)) {
> + int res = wait_event_interruptible_timeout(
> + defer_io->queue_add_event,
> + (!defer_io_queue_empty(defer_io->dio_queue)),
> + BLK_IMAGE_THROTTLE_TIMEOUT);
> + if (-ERESTARTSYS == res)
> + pr_err("Signal received in defer IO thread. Waiting for completion with code ERESTARTSYS\n");
> + }
> +
> + if (!defer_io_queue_empty(defer_io->dio_queue)) {
> + int dio_copy_result = SUCCESS;
> + struct blk_deferred_request *dio_copy_req = NULL;
> +
> + mutex_lock(&defer_io->snapstore_device->store_block_map_locker);
> + do {
> + dio_copy_result = _defer_io_copy_prepare(
> + defer_io, &queue_in_process, &dio_copy_req);
> + if (dio_copy_result != SUCCESS) {
> + pr_err("Unable to process defer IO request: failed to prepare copy request. erro=%d\n",
> + dio_copy_result);
> + break;
> + }
> + if (dio_copy_req == NULL)
> + break; //nothing to copy
> +
> + dio_copy_result = blk_deferred_request_read_original(
> + defer_io->original_blk_dev, dio_copy_req);
> + if (dio_copy_result != SUCCESS) {
> + pr_err("Unable to process defer IO request: failed to read data to copy request. errno=%d\n",
> + dio_copy_result);
> + break;
> + }
> + dio_copy_result = snapstore_device_store(defer_io->snapstore_device,
> + dio_copy_req);
> + if (dio_copy_result != SUCCESS) {
> + pr_err("Unable to process defer IO request: failed to write data from copy request. errno=%d\n",
> + dio_copy_result);
> + break;
> + }
> +
> + } while (false);
> + _defer_io_finish(defer_io, &queue_in_process);
> + mutex_unlock(&defer_io->snapstore_device->store_block_map_locker);
> +
> + if (dio_copy_req) {
> + if (dio_copy_result == -EDEADLK)
> + blk_deferred_request_deadlocked(dio_copy_req);
> + else
> + blk_deferred_request_free(dio_copy_req);
> + }
> + }
> +
> + //wake up snapimage if defer io queue empty
> + if (defer_io_queue_empty(defer_io->dio_queue))
> + wake_up_interruptible(&defer_io->queue_throttle_waiter);
> + }
> + defer_io_queue_active(&defer_io->dio_queue, false);
> +
> + //waiting for all sent request complete
> + _defer_io_finish(defer_io, &defer_io->dio_queue);
> +
> + pr_info("Defer IO thread for original device [%d:%d] completed\n",
> + MAJOR(defer_io->original_dev_id), MINOR(defer_io->original_dev_id));
> + defer_io_put_resource(defer_io);
> + return SUCCESS;
> +}
> +
> +static void _defer_io_destroy(struct defer_io *defer_io)
> +{
> + if (defer_io == NULL)
> + return;
> +
> + if (defer_io->dio_thread)
> + defer_io_stop(defer_io);
> +
> + if (defer_io->snapstore_device)
> + snapstore_device_put_resource(defer_io->snapstore_device);
> +
> + kfree(defer_io);
> + pr_info("Defer IO processor was destroyed\n");
> +}
> +
> +static void defer_io_destroy_cb(struct kref *kref)
> +{
> + _defer_io_destroy(container_of(kref, struct defer_io, refcount));
> +}
> +
> +struct defer_io *defer_io_get_resource(struct defer_io *defer_io)
> +{
> + if (defer_io)
> + kref_get(&defer_io->refcount);
> +
> + return defer_io;
> +}
> +
> +void defer_io_put_resource(struct defer_io *defer_io)
> +{
> + if (defer_io)
> + kref_put(&defer_io->refcount, defer_io_destroy_cb);
> +}
> +
> +int defer_io_create(dev_t dev_id, struct block_device *blk_dev, struct defer_io **pp_defer_io)
> +{
> + int res = SUCCESS;
> + struct defer_io *defer_io = NULL;
> + struct snapstore_device *snapstore_device;
> +
> + pr_info("Defer IO processor was created for device [%d:%d]\n", MAJOR(dev_id),
> + MINOR(dev_id));
> +
> + defer_io = kzalloc(sizeof(struct defer_io), GFP_KERNEL);
> + if (defer_io == NULL)
> + return -ENOMEM;
> +
> + snapstore_device = snapstore_device_find_by_dev_id(dev_id);
> + if (snapstore_device == NULL) {
> + pr_err("Unable to create defer IO processor: failed to initialize snapshot data for device [%d:%d]\n",
> + MAJOR(dev_id), MINOR(dev_id));
> +
> + kfree(defer_io);
> + return -ENODATA;
> + }
> +
> + defer_io->snapstore_device = snapstore_device_get_resource(snapstore_device);
> + defer_io->original_dev_id = dev_id;
> + defer_io->original_blk_dev = blk_dev;
> +
> + kref_init(&defer_io->refcount);
> +
> + defer_io_queue_init(&defer_io->dio_queue);
> +
> + init_waitqueue_head(&defer_io->queue_add_event);
> +
> + atomic_set(&defer_io->queue_filling_count, 0);
> +
> + init_waitqueue_head(&defer_io->queue_throttle_waiter);
> +
> + defer_io->dio_thread = kthread_create(defer_io_work_thread, (void *)defer_io,
> + "blksnapdeferio%d:%d", MAJOR(dev_id), MINOR(dev_id));
> + if (IS_ERR(defer_io->dio_thread)) {
> + res = PTR_ERR(defer_io->dio_thread);
> + pr_err("Unable to create defer IO processor: failed to create thread. errno=%d\n",
> + res);
> +
> + _defer_io_destroy(defer_io);
> + defer_io = NULL;
> + *pp_defer_io = NULL;
> +
> + return res;
> + }
> +
> + wake_up_process(defer_io->dio_thread);
> +
> + *pp_defer_io = defer_io;
> + pr_info("Defer IO processor was created\n");
> +
> + return SUCCESS;
> +}
> +
> +int defer_io_stop(struct defer_io *defer_io)
> +{
> + int res = SUCCESS;
> +
> + pr_info("Defer IO thread for the device stopped [%d:%d]\n",
> + MAJOR(defer_io->original_dev_id), MINOR(defer_io->original_dev_id));
> +
> + if (defer_io->dio_thread != NULL) {
> + struct task_struct *dio_thread = defer_io->dio_thread;
> +
> + defer_io->dio_thread = NULL;
> + res = kthread_stop(dio_thread); //stopping and waiting.
> + if (res != SUCCESS)
> + pr_err("Failed to stop defer IO thread. errno=%d\n", res);
> + }
> +
> + return res;
> +}
> +
> +int defer_io_redirect_bio(struct defer_io *defer_io, struct bio *bio, void *tracker)
> +{
> + struct defer_io_orig_rq *dio_orig_req;
> +
> + if (snapstore_device_is_corrupted(defer_io->snapstore_device))
> + return -ENODATA;
> +
> + dio_orig_req = defer_io_queue_new(&defer_io->dio_queue, bio);
> + if (dio_orig_req == NULL)
> + return -ENOMEM;
> +
> + dio_orig_req->tracker = (struct tracker *)tracker;
> +
> + if (defer_io_queue_push_back(&defer_io->dio_queue, dio_orig_req) != SUCCESS) {
> + defer_io_queue_free(dio_orig_req);
> + return -EFAULT;
> + }
> +
> + atomic_inc(&defer_io->queue_filling_count);
> +
> + wake_up_interruptible(&defer_io->queue_add_event);
> +
> + return SUCCESS;
> +}
> diff --git a/drivers/block/blk-snap/defer_io.h b/drivers/block/blk-snap/defer_io.h
> new file mode 100644
> index 000000000000..27c3bb03241f
> --- /dev/null
> +++ b/drivers/block/blk-snap/defer_io.h
> @@ -0,0 +1,39 @@
> +/* SPDX-License-Identifier: GPL-2.0-or-later */
> +#pragma once
> +
> +#include <linux/kref.h>
> +#include "snapstore_device.h"
> +
> +struct defer_io_queue {
> + struct list_head list;
> + spinlock_t lock;
> +
> + atomic_t active_state;
> + atomic_t in_queue_cnt;
> +};
> +
> +struct defer_io {
> + struct kref refcount;
> +
> + wait_queue_head_t queue_add_event;
> +
> + atomic_t queue_filling_count;
> + wait_queue_head_t queue_throttle_waiter;
> +
> + dev_t original_dev_id;
> + struct block_device *original_blk_dev;
> +
> + struct snapstore_device *snapstore_device;
> +
> + struct task_struct *dio_thread;
> +
> + struct defer_io_queue dio_queue;
> +};
> +
> +int defer_io_create(dev_t dev_id, struct block_device *blk_dev, struct defer_io **pp_defer_io);
> +int defer_io_stop(struct defer_io *defer_io);
> +
> +struct defer_io *defer_io_get_resource(struct defer_io *defer_io);
> +void defer_io_put_resource(struct defer_io *defer_io);
> +
> +int defer_io_redirect_bio(struct defer_io *defer_io, struct bio *bio, void *tracker);
> diff --git a/drivers/block/blk-snap/main.c b/drivers/block/blk-snap/main.c
> new file mode 100644
> index 000000000000..d1d4e08a4890
> --- /dev/null
> +++ b/drivers/block/blk-snap/main.c
> @@ -0,0 +1,82 @@
> +// SPDX-License-Identifier: GPL-2.0
> +#include "common.h"
> +#include "version.h"
> +#include "blk-snap-ctl.h"
> +#include "params.h"
> +#include "ctrl_fops.h"
> +#include "ctrl_pipe.h"
> +#include "ctrl_sysfs.h"
> +#include "snapimage.h"
> +#include "snapstore.h"
> +#include "snapstore_device.h"
> +#include "snapshot.h"
> +#include "tracker.h"
> +#include "tracking.h"
> +#include <linux/module.h>
> +
> +int __init blk_snap_init(void)
> +{
> + int result = SUCCESS;
> +
> + pr_info("Loading\n");
> +
> + params_check();
> +
> + result = ctrl_init();
> + if (result != SUCCESS)
> + return result;
> +
> + result = blk_redirect_bioset_create();
> + if (result != SUCCESS)
> + return result;
> +
> + result = blk_deferred_bioset_create();
> + if (result != SUCCESS)
> + return result;
> +
> + result = snapimage_init();
> + if (result != SUCCESS)
> + return result;
> +
> + result = ctrl_sysfs_init();
> + if (result != SUCCESS)
> + return result;
> +
> + result = tracking_init();
> + if (result != SUCCESS)
> + return result;
> +
> + return result;
> +}
> +
> +void __exit blk_snap_exit(void)
> +{
> + pr_info("Unloading module\n");
> +
> + ctrl_sysfs_done();
> +
> + snapshot_done();
> +
> + snapstore_device_done();
> + snapstore_done();
> +
> + tracker_done();
> + tracking_done();
> +
> + snapimage_done();
> +
> + blk_deferred_bioset_free();
> + blk_deferred_done();
> +
> + blk_redirect_bioset_free();
> +
> + ctrl_done();
> +}
> +
> +module_init(blk_snap_init);
> +module_exit(blk_snap_exit);
> +
> +MODULE_DESCRIPTION("Block Layer Snapshot Kernel Module");
> +MODULE_VERSION(FILEVER_STR);
> +MODULE_AUTHOR("Veeam Software Group GmbH");
> +MODULE_LICENSE("GPL");
> diff --git a/drivers/block/blk-snap/params.c b/drivers/block/blk-snap/params.c
> new file mode 100644
> index 000000000000..7eba3c8bf395
> --- /dev/null
> +++ b/drivers/block/blk-snap/params.c
> @@ -0,0 +1,58 @@
> +// SPDX-License-Identifier: GPL-2.0
> +#include "common.h"
> +#include "params.h"
> +#include <linux/module.h>
> +
> +int snapstore_block_size_pow = 14;
> +int change_tracking_block_size_pow = 18;
> +
> +int get_snapstore_block_size_pow(void)
> +{
> + return snapstore_block_size_pow;
> +}
> +
> +int inc_snapstore_block_size_pow(void)
> +{
> + if (snapstore_block_size_pow > 30)
> + return -EFAULT;
> +
> + ++snapstore_block_size_pow;
> + return SUCCESS;
> +}
> +
> +int get_change_tracking_block_size_pow(void)
> +{
> + return change_tracking_block_size_pow;
> +}
> +
> +void params_check(void)
> +{
> + pr_info("snapstore_block_size_pow: %d\n", snapstore_block_size_pow);
> + pr_info("change_tracking_block_size_pow: %d\n", change_tracking_block_size_pow);
> +
> + if (snapstore_block_size_pow > 23) {
> + snapstore_block_size_pow = 23;
> + pr_info("Limited snapstore_block_size_pow: %d\n", snapstore_block_size_pow);
> + } else if (snapstore_block_size_pow < 12) {
> + snapstore_block_size_pow = 12;
> + pr_info("Limited snapstore_block_size_pow: %d\n", snapstore_block_size_pow);
> + }
> +
> + if (change_tracking_block_size_pow > 23) {
> + change_tracking_block_size_pow = 23;
> + pr_info("Limited change_tracking_block_size_pow: %d\n",
> + change_tracking_block_size_pow);
> + } else if (change_tracking_block_size_pow < 12) {
> + change_tracking_block_size_pow = 12;
> + pr_info("Limited change_tracking_block_size_pow: %d\n",
> + change_tracking_block_size_pow);
> + }
> +}
> +
> +module_param_named(snapstore_block_size_pow, snapstore_block_size_pow, int, 0644);
> +MODULE_PARM_DESC(snapstore_block_size_pow,
> + "Snapstore block size binary pow. 20 for 1MiB block size");
> +
> +module_param_named(change_tracking_block_size_pow, change_tracking_block_size_pow, int, 0644);
> +MODULE_PARM_DESC(change_tracking_block_size_pow,
> + "Change-tracking block size binary pow. 18 for 256 KiB block size");
> diff --git a/drivers/block/blk-snap/params.h b/drivers/block/blk-snap/params.h
> new file mode 100644
> index 000000000000..c1b853a1363b
> --- /dev/null
> +++ b/drivers/block/blk-snap/params.h
> @@ -0,0 +1,29 @@
> +/* SPDX-License-Identifier: GPL-2.0-or-later */
> +#pragma once
> +
> +int get_snapstore_block_size_pow(void);
> +int inc_snapstore_block_size_pow(void);
> +
> +static inline sector_t snapstore_block_shift(void)
> +{
> + return get_snapstore_block_size_pow() - SECTOR_SHIFT;
> +};
> +
> +static inline sector_t snapstore_block_size(void)
> +{
> + return 1ull << snapstore_block_shift();
> +};
> +
> +static inline sector_t snapstore_block_mask(void)
> +{
> + return snapstore_block_size() - 1ull;
> +};
> +
> +int get_change_tracking_block_size_pow(void);
> +
> +static inline unsigned int change_tracking_block_size(void)
> +{
> + return 1 << get_change_tracking_block_size_pow();
> +};
> +
> +void params_check(void);
> diff --git a/drivers/block/blk-snap/rangevector.c b/drivers/block/blk-snap/rangevector.c
> new file mode 100644
> index 000000000000..49fe4589b6f7
> --- /dev/null
> +++ b/drivers/block/blk-snap/rangevector.c
> @@ -0,0 +1,85 @@
> +// SPDX-License-Identifier: GPL-2.0
> +#include "common.h"
> +#include "rangevector.h"
> +
> +#define SECTION "ranges "
> +
> +static inline sector_t range_node_start(struct blk_range_tree_node *range_node)
> +{
> + return range_node->range.ofs;
> +}
> +
> +static inline sector_t range_node_last(struct blk_range_tree_node *range_node)
> +{
> + return range_node->range.ofs + range_node->range.cnt - 1;
> +}
> +
> +#ifndef INTERVAL_TREE_DEFINE
> +#pragma message("INTERVAL_TREE_DEFINE is undefined")
> +#endif
> +INTERVAL_TREE_DEFINE(struct blk_range_tree_node, _node, sector_t, _subtree_last,
> + range_node_start, range_node_last, static inline, _blk_range_rb)
> +
> +void blk_range_rb_insert(struct blk_range_tree_node *node, struct rb_root_cached *root)
> +{
> + _blk_range_rb_insert(node, root);
> +}
> +
> +void blk_range_rb_remove(struct blk_range_tree_node *node, struct rb_root_cached *root)
> +{
> + _blk_range_rb_remove(node, root);
> +}
> +
> +struct blk_range_tree_node *blk_range_rb_iter_first(struct rb_root_cached *root, sector_t start,
> + sector_t last)
> +{
> + return _blk_range_rb_iter_first(root, start, last);
> +}
> +
> +struct blk_range_tree_node *blk_range_rb_iter_next(struct blk_range_tree_node *node, sector_t start,
> + sector_t last)
> +{
> + return _blk_range_rb_iter_next(node, start, last);
> +}
> +
> +void rangevector_init(struct rangevector *rangevector)
> +{
> + init_rwsem(&rangevector->lock);
> +
> + rangevector->root = RB_ROOT_CACHED;
> +}
> +
> +void rangevector_done(struct rangevector *rangevector)
> +{
> + struct rb_node *rb_node = NULL;
> +
> + down_write(&rangevector->lock);
> + rb_node = rb_first_cached(&rangevector->root);
> + while (rb_node) {
> + struct blk_range_tree_node *range_node = (struct blk_range_tree_node *)
> + rb_node; //container_of(rb_node, struct blk_range_tree_node, node);
> +
> + blk_range_rb_remove(range_node, &rangevector->root);
> + kfree(range_node);
> +
> + rb_node = rb_first_cached(&rangevector->root);
> + }
> + up_write(&rangevector->lock);
> +}
> +
> +int rangevector_add(struct rangevector *rangevector, struct blk_range *rg)
> +{
> + struct blk_range_tree_node *range_node;
> +
> + range_node = kzalloc(sizeof(struct blk_range_tree_node), GFP_KERNEL);
> + if (range_node)
> + return -ENOMEM;
> +
> + range_node->range = *rg;
> +
> + down_write(&rangevector->lock);
> + blk_range_rb_insert(range_node, &rangevector->root);
> + up_write(&rangevector->lock);
> +
> + return SUCCESS;
> +}
> diff --git a/drivers/block/blk-snap/rangevector.h b/drivers/block/blk-snap/rangevector.h
> new file mode 100644
> index 000000000000..5ff439423178
> --- /dev/null
> +++ b/drivers/block/blk-snap/rangevector.h
> @@ -0,0 +1,31 @@
> +/* SPDX-License-Identifier: GPL-2.0 */
> +#pragma once
> +
> +#include <linux/interval_tree_generic.h>
> +
> +struct blk_range_tree_node {
> + struct rb_node _node;
> + struct blk_range range;
> + sector_t _subtree_last;
> +};
> +
> +void blk_range_rb_insert(struct blk_range_tree_node *node, struct rb_root_cached *root);
> +
> +void blk_range_rb_remove(struct blk_range_tree_node *node, struct rb_root_cached *root);
> +
> +struct blk_range_tree_node *blk_range_rb_iter_first(struct rb_root_cached *root, sector_t start,
> + sector_t last);
> +
> +struct blk_range_tree_node *blk_range_rb_iter_next(struct blk_range_tree_node *node, sector_t start,
> + sector_t last);
> +
> +struct rangevector {
> + struct rb_root_cached root;
> + struct rw_semaphore lock;
> +};
> +
> +void rangevector_init(struct rangevector *rangevector);
> +
> +void rangevector_done(struct rangevector *rangevector);
> +
> +int rangevector_add(struct rangevector *rangevector, struct blk_range *rg);
> diff --git a/drivers/block/blk-snap/snapimage.c b/drivers/block/blk-snap/snapimage.c
> new file mode 100644
> index 000000000000..da971486cbef
> --- /dev/null
> +++ b/drivers/block/blk-snap/snapimage.c
> @@ -0,0 +1,982 @@
> +// SPDX-License-Identifier: GPL-2.0
> +#define BLK_SNAP_SECTION "-snapimage"
> +#include "common.h"
> +#include "snapimage.h"
> +#include "blk_util.h"
> +#include "defer_io.h"
> +#include "cbt_map.h"
> +#include "tracker.h"
> +
> +#include <asm/div64.h>
> +#include <linux/cdrom.h>
> +#include <linux/blk-mq.h>
> +#include <linux/hdreg.h>
> +#include <linux/kthread.h>
> +
> +#define SNAPIMAGE_MAX_DEVICES 2048
> +
> +int snapimage_major;
> +unsigned long *snapimage_minors;
> +DEFINE_SPINLOCK(snapimage_minors_lock);
> +
> +LIST_HEAD(snap_images);
> +DECLARE_RWSEM(snap_images_lock);
> +
> +DECLARE_RWSEM(snap_image_destroy_lock);
> +
> +struct snapimage {
> + struct list_head link;
> +
> + sector_t capacity;
> + dev_t original_dev;
> +
> + struct defer_io *defer_io;
> + struct cbt_map *cbt_map;
> +
> + dev_t image_dev;
> +
> + struct request_queue *queue;
> + struct gendisk *disk;
> +
> + atomic_t own_cnt;
> +
> + struct redirect_bio_queue image_queue;
> +
> + struct task_struct *rq_processor;
> +
> + wait_queue_head_t rq_proc_event;
> + wait_queue_head_t rq_complete_event;
> +
> + struct mutex open_locker;
> + struct block_device *open_bdev;
> +
> + size_t open_cnt;
> +};
> +
> +int _snapimage_open(struct block_device *bdev, fmode_t mode)
> +{
> + int res = SUCCESS;
> +
> + if (bdev->bd_disk == NULL) {
> + pr_err("Unable to open snapshot image: bd_disk is NULL. Device [%d:%d]\n",
> + MAJOR(bdev->bd_dev), MINOR(bdev->bd_dev));
> + pr_err("Block device object %p\n", bdev);
> + return -ENODEV;
> + }
> +
> + down_read(&snap_image_destroy_lock);
> + do {
> + struct snapimage *image = bdev->bd_disk->private_data;
> +
> + if (image == NULL) {
> + pr_err("Unable to open snapshot image: private data is not initialized. Block device object %p\n",
> + bdev);
> + res = -ENODEV;
> + break;
> + }
> +
> + mutex_lock(&image->open_locker);
> + {
> + if (image->open_cnt == 0)
> + image->open_bdev = bdev;
> +
> + image->open_cnt++;
> + }
> + mutex_unlock(&image->open_locker);
> + } while (false);
> + up_read(&snap_image_destroy_lock);
> + return res;
> +}
> +
> +static inline uint64_t do_div_inline(uint64_t division, uint32_t divisor)
> +{
> + do_div(division, divisor);
> + return division;
> +}
> +
> +int _snapimage_getgeo(struct block_device *bdev, struct hd_geometry *geo)
> +{
> + int res = SUCCESS;
> + sector_t quotient;
> +
> + down_read(&snap_image_destroy_lock);
> + do {
> + struct snapimage *image = bdev->bd_disk->private_data;
> +
> + if (image == NULL) {
> + pr_err("Unable to open snapshot image: private data is not initialized. Block device object %p\n",
> + bdev);
> + res = -ENODEV;
> + break;
> + }
> +
> + pr_info("Getting geo for snapshot image device [%d:%d]\n", MAJOR(image->image_dev),
> + MINOR(image->image_dev));
> +
> + geo->start = 0;
> + if (image->capacity > 63) {
> + geo->sectors = 63;
> + quotient = do_div_inline(image->capacity + (63 - 1), 63);
> +
> + if (quotient > 255ULL) {
> + geo->heads = 255;
> + geo->cylinders =
> + (unsigned short)do_div_inline(quotient + (255 - 1), 255);
> + } else {
> + geo->heads = (unsigned char)quotient;
> + geo->cylinders = 1;
> + }
> + } else {
> + geo->sectors = (unsigned char)image->capacity;
> + geo->cylinders = 1;
> + geo->heads = 1;
> + }
> +
> + pr_info("Image device geo: capacity=%lld, heads=%d, cylinders=%d, sectors=%d\n",
> + image->capacity, geo->heads, geo->cylinders, geo->sectors);
> + } while (false);
> + up_read(&snap_image_destroy_lock);
> +
> + return res;
> +}
> +
> +void _snapimage_close(struct gendisk *disk, fmode_t mode)
> +{
> + if (disk->private_data != NULL) {
> + down_read(&snap_image_destroy_lock);
> + do {
> + struct snapimage *image = disk->private_data;
> +
> + mutex_lock(&image->open_locker);
> + {
> + if (image->open_cnt > 0)
> + image->open_cnt--;
> +
> + if (image->open_cnt == 0)
> + image->open_bdev = NULL;
> + }
> + mutex_unlock(&image->open_locker);
> + } while (false);
> + up_read(&snap_image_destroy_lock);
> + } else
> + pr_err("Unable to close snapshot image: private data is not initialized\n");
> +}
> +
> +int _snapimage_ioctl(struct block_device *bdev, fmode_t mode, unsigned int cmd, unsigned long arg)
> +{
> + int res = -ENOTTY;
> +
> + down_read(&snap_image_destroy_lock);
> + {
> + struct snapimage *image = bdev->bd_disk->private_data;
> +
> + switch (cmd) {
> + /*
> + * The only command we need to interpret is HDIO_GETGEO, since
> + * we can't partition the drive otherwise. We have no real
> + * geometry, of course, so make something up.
> + */
> + case HDIO_GETGEO: {
> + unsigned long len;
> + struct hd_geometry geo;
> +
> + res = _snapimage_getgeo(bdev, &geo);
> +
> + len = copy_to_user((void *)arg, &geo, sizeof(geo));
> + if (len != 0)
> + res = -EFAULT;
> + else
> + res = SUCCESS;
> + } break;
> + case CDROM_GET_CAPABILITY: //0x5331 / * get capabilities * /
> + {
> + struct gendisk *disk = bdev->bd_disk;
> +
> + if (bdev->bd_disk && (disk->flags & GENHD_FL_CD))
> + res = SUCCESS;
> + else
> + res = -EINVAL;
> + } break;
> +
> + default:
> + pr_info("Snapshot image ioctl receive unsupported command\n");
> + pr_info("Device [%d:%d], command 0x%x, arg 0x%lx\n",
> + MAJOR(image->image_dev), MINOR(image->image_dev), cmd, arg);
> +
> + res = -ENOTTY; /* unknown command */
> + }
> + }
> + up_read(&snap_image_destroy_lock);
> + return res;
> +}
> +
> +blk_qc_t _snapimage_submit_bio(struct bio *bio);
> +
> +const struct block_device_operations snapimage_ops = {
> + .owner = THIS_MODULE,
> + .submit_bio = _snapimage_submit_bio,
> + .open = _snapimage_open,
> + .ioctl = _snapimage_ioctl,
> + .release = _snapimage_close,
> +};
> +
> +static inline int _snapimage_request_read(struct snapimage *image,
> + struct blk_redirect_bio *rq_redir)
> +{
> + struct snapstore_device *snapstore_device = image->defer_io->snapstore_device;
> +
> + return snapstore_device_read(snapstore_device, rq_redir);
> +}
> +
> +int _snapimage_request_write(struct snapimage *image, struct blk_redirect_bio *rq_redir)
> +{
> + struct snapstore_device *snapstore_device;
> + struct cbt_map *cbt_map;
> + int res = SUCCESS;
> +
> + if (unlikely((image->defer_io == NULL) || (image->cbt_map == NULL))) {
> + pr_err("Invalid snapshot image structure");
> + return -EINVAL;
> + }
> +
> +
> + snapstore_device = image->defer_io->snapstore_device;
> + cbt_map = image->cbt_map;
> +
> + if (snapstore_device_is_corrupted(snapstore_device))
> + return -ENODATA;
> +
> + if (!bio_has_data(rq_redir->bio)) {
> + pr_warn("Snapshot image receive empty block IO. flags=%u\n",
> + rq_redir->bio->bi_flags);
> +
> + blk_redirect_complete(rq_redir, SUCCESS);
> + return SUCCESS;
> + }
> +
> + if (cbt_map != NULL) {
> + sector_t ofs = rq_redir->bio->bi_iter.bi_sector;
> + sector_t cnt = bio_sectors(rq_redir->bio);
> +
> + res = cbt_map_set_both(cbt_map, ofs, cnt);
> + if (res != SUCCESS)
> + pr_err("Unable to write data to snapshot image: failed to set CBT map. errno=%d\n",
> + res);
> + }
> +
> + res = snapstore_device_write(snapstore_device, rq_redir);
> +
> + if (res != SUCCESS) {
> + pr_err("Failed to write data to snapshot image\n");
> + return res;
> + }
> +
> + return res;
> +}
> +
> +void _snapimage_processing(struct snapimage *image)
> +{
> + int res = SUCCESS;
> + struct blk_redirect_bio *rq_redir;
> +
> + rq_redir = redirect_bio_queue_get_first(&image->image_queue);
> +
> + if (bio_data_dir(rq_redir->bio) == READ) {
> + res = _snapimage_request_read(image, rq_redir);
> + if (res != SUCCESS)
> + pr_err("Failed to read data from snapshot image. errno=%d\n", res);
> +
> + } else {
> + res = _snapimage_request_write(image, rq_redir);
> + if (res != SUCCESS)
> + pr_err("Failed to write data to snapshot image. errno=%d\n", res);
> + }
> +
> + if (res != SUCCESS)
> + blk_redirect_complete(rq_redir, res);
> +}
> +
> +int snapimage_processor_waiting(struct snapimage *image)
> +{
> + int res = SUCCESS;
> +
> + if (redirect_bio_queue_empty(image->image_queue)) {
> + res = wait_event_interruptible_timeout(
> + image->rq_proc_event,
> + (!redirect_bio_queue_empty(image->image_queue) || kthread_should_stop()),
> + 5 * HZ);
> + if (res > 0)
> + res = SUCCESS;
> + else if (res == 0)
> + res = -ETIME;
> + }
> + return res;
> +}
> +
> +int snapimage_processor_thread(void *data)
> +{
> + struct snapimage *image = data;
> +
> + pr_info("Snapshot image thread for device [%d:%d] start\n", MAJOR(image->image_dev),
> + MINOR(image->image_dev));
> +
> + add_disk(image->disk);
> +
> + //priority
> + set_user_nice(current, -20); //MIN_NICE
> +
> + while (!kthread_should_stop()) {
> + int res = snapimage_processor_waiting(image);
> +
> + if (res == SUCCESS) {
> + if (!redirect_bio_queue_empty(image->image_queue))
> + _snapimage_processing(image);
> + } else if (res != -ETIME) {
> + pr_err("Failed to wait snapshot image thread queue. errno=%d\n", res);
> + return res;
> + }
> + schedule();
> + }
> + pr_info("Snapshot image disk delete\n");
> + del_gendisk(image->disk);
> +
> + while (!redirect_bio_queue_empty(image->image_queue))
> + _snapimage_processing(image);
> +
> + pr_info("Snapshot image thread for device [%d:%d] complete", MAJOR(image->image_dev),
> + MINOR(image->image_dev));
> + return 0;
> +}
> +
> +static inline void _snapimage_bio_complete(struct bio *bio, int err)
> +{
> + if (err == SUCCESS)
> + bio->bi_status = BLK_STS_OK;
> + else
> + bio->bi_status = BLK_STS_IOERR;
> +
> + bio_endio(bio);
> +}
> +
> +void _snapimage_bio_complete_cb(void *complete_param, struct bio *bio, int err)
> +{
> + struct snapimage *image = (struct snapimage *)complete_param;
> +
> + _snapimage_bio_complete(bio, err);
> +
> + if (redirect_bio_queue_unactive(image->image_queue))
> + wake_up_interruptible(&image->rq_complete_event);
> +
> + atomic_dec(&image->own_cnt);
> +}
> +
> +int _snapimage_throttling(struct defer_io *defer_io)
> +{
> + return wait_event_interruptible(defer_io->queue_throttle_waiter,
> + redirect_bio_queue_empty(defer_io->dio_queue));
> +}
> +
> +blk_qc_t _snapimage_submit_bio(struct bio *bio)
> +{
> + blk_qc_t result = SUCCESS;
> + struct request_queue *q = bio->bi_disk->queue;
> + struct snapimage *image = q->queuedata;
> +
> + if (unlikely(blk_mq_queue_stopped(q))) {
> + pr_info("Failed to make snapshot image request. Queue already is not active.");
> + pr_info("Queue flags=%lx\n", q->queue_flags);
> +
> + _snapimage_bio_complete(bio, -ENODEV);
> +
> + return result;
> + }
> +
> + atomic_inc(&image->own_cnt);
> + do {
> + int res;
> + struct blk_redirect_bio *rq_redir;
> +
> + if (false == atomic_read(&(image->image_queue.active_state))) {
> + _snapimage_bio_complete(bio, -ENODEV);
> + break;
> + }
> +
> + if (snapstore_device_is_corrupted(image->defer_io->snapstore_device)) {
> + _snapimage_bio_complete(bio, -ENODATA);
> + break;
> + }
> +
> + res = _snapimage_throttling(image->defer_io);
> + if (res != SUCCESS) {
> + pr_err("Failed to throttle snapshot image device. errno=%d\n", res);
> + _snapimage_bio_complete(bio, res);
> + break;
> + }
> +
> + rq_redir = redirect_bio_queue_new(&image->image_queue);
> + if (rq_redir == NULL) {
> + pr_err("Unable to make snapshot image request: failed to allocate redirect bio structure\n");
> + _snapimage_bio_complete(bio, -ENOMEM);
> + break;
> + }
> + rq_redir->bio = bio;
> + rq_redir->complete_cb = _snapimage_bio_complete_cb;
> + rq_redir->complete_param = (void *)image;
> + atomic_inc(&image->own_cnt);
> +
> + res = redirect_bio_queue_push_back(&image->image_queue, rq_redir);
> + if (res == SUCCESS)
> + wake_up(&image->rq_proc_event);
> + else {
> + redirect_bio_queue_free(rq_redir);
> + _snapimage_bio_complete(bio, -EIO);
> +
> + if (redirect_bio_queue_unactive(image->image_queue))
> + wake_up_interruptible(&image->rq_complete_event);
> + }
> +
> + } while (false);
> + atomic_dec(&image->own_cnt);
> +
> + return result;
> +}
> +
> +struct blk_dev_info {
> + size_t blk_size;
> + sector_t start_sect;
> + sector_t count_sect;
> +
> + unsigned int io_min;
> + unsigned int physical_block_size;
> + unsigned short logical_block_size;
> +};
> +
> +static int _blk_dev_get_info(struct block_device *blk_dev, struct blk_dev_info *pdev_info)
> +{
> + sector_t SectorStart;
> + sector_t SectorsCapacity;
> +
> + if (blk_dev->bd_part)
> + SectorsCapacity = blk_dev->bd_part->nr_sects;
> + else if (blk_dev->bd_disk)
> + SectorsCapacity = get_capacity(blk_dev->bd_disk);
> + else
> + return -EINVAL;
> +
> + SectorStart = get_start_sect(blk_dev);
> +
> + pdev_info->physical_block_size = blk_dev->bd_disk->queue->limits.physical_block_size;
> + pdev_info->logical_block_size = blk_dev->bd_disk->queue->limits.logical_block_size;
> + pdev_info->io_min = blk_dev->bd_disk->queue->limits.io_min;
> +
> + pdev_info->blk_size = block_size(blk_dev);
> + pdev_info->start_sect = SectorStart;
> + pdev_info->count_sect = SectorsCapacity;
> + return SUCCESS;
> +}
> +
> +static int blk_dev_get_info(dev_t dev_id, struct blk_dev_info *pdev_info)
> +{
> + int result = SUCCESS;
> + struct block_device *blk_dev;
> +
> + result = blk_dev_open(dev_id, &blk_dev);
> + if (result != SUCCESS) {
> + pr_err("Failed to open device [%d:%d]\n", MAJOR(dev_id), MINOR(dev_id));
> + return result;
> + }
> +
> + result = _blk_dev_get_info(blk_dev, pdev_info);
> + if (result != SUCCESS)
> + pr_err("Failed to identify block device [%d:%d]\n", MAJOR(dev_id), MINOR(dev_id));
> +
> + blk_dev_close(blk_dev);
> +
> + return result;
> +}
> +
> +static inline void _snapimage_free(struct snapimage *image)
> +{
> + defer_io_put_resource(image->defer_io);
> + cbt_map_put_resource(image->cbt_map);
> + image->defer_io = NULL;
> +}
> +
> +static void _snapimage_stop(struct snapimage *image)
> +{
> + if (image->rq_processor != NULL) {
> + if (redirect_bio_queue_active(&image->image_queue, false)) {
> + struct request_queue *q = image->queue;
> +
> + pr_info("Snapshot image request processing stop\n");
> +
> + if (!blk_queue_stopped(q)) {
> + blk_sync_queue(q);
> + blk_mq_stop_hw_queues(q);
> + }
> + }
> +
> + pr_info("Snapshot image thread stop\n");
> + kthread_stop(image->rq_processor);
> + image->rq_processor = NULL;
> +
> + while (!redirect_bio_queue_unactive(image->image_queue))
> + wait_event_interruptible(image->rq_complete_event,
> + redirect_bio_queue_unactive(image->image_queue));
> + }
> +}
> +
> +static void _snapimage_destroy(struct snapimage *image)
> +{
> + if (image->rq_processor != NULL)
> + _snapimage_stop(image);
> +
> + if (image->queue) {
> + pr_info("Snapshot image queue cleanup\n");
> + blk_cleanup_queue(image->queue);
> + image->queue = NULL;
> + }
> +
> + if (image->disk != NULL) {
> + struct gendisk *disk;
> +
> + disk = image->disk;
> + image->disk = NULL;
> +
> + pr_info("Snapshot image disk structure release\n");
> +
> + disk->private_data = NULL;
> + put_disk(disk);
> + }
> +
> + spin_lock(&snapimage_minors_lock);
> + bitmap_clear(snapimage_minors, MINOR(image->image_dev), 1u);
> + spin_unlock(&snapimage_minors_lock);
> +}
> +
> +int snapimage_create(dev_t original_dev)
> +{
> + int res = SUCCESS;
> + struct tracker *tracker = NULL;
> + struct snapimage *image = NULL;
> + struct gendisk *disk = NULL;
> + int minor;
> + struct blk_dev_info original_dev_info;
> +
> + pr_info("Create snapshot image for device [%d:%d]\n", MAJOR(original_dev),
> + MINOR(original_dev));
> +
> + res = blk_dev_get_info(original_dev, &original_dev_info);
> + if (res != SUCCESS) {
> + pr_err("Failed to obtain original device info\n");
> + return res;
> + }
> +
> + res = tracker_find_by_dev_id(original_dev, &tracker);
> + if (res != SUCCESS) {
> + pr_err("Unable to create snapshot image: cannot find tracker for device [%d:%d]\n",
> + MAJOR(original_dev), MINOR(original_dev));
> + return res;
> + }
> +
> + image = kzalloc(sizeof(struct snapimage), GFP_KERNEL);
> + if (image == NULL)
> + return -ENOMEM;
> +
> + INIT_LIST_HEAD(&image->link);
> +
> + do {
> + spin_lock(&snapimage_minors_lock);
> + minor = bitmap_find_free_region(snapimage_minors, SNAPIMAGE_MAX_DEVICES, 0);
> + spin_unlock(&snapimage_minors_lock);
> +
> + if (minor < SUCCESS) {
> + pr_err("Failed to allocate minor for snapshot image device. errno=%d\n",
> + 0 - minor);
> + break;
> + }
> +
> + image->rq_processor = NULL;
> +
> + image->capacity = original_dev_info.count_sect;
> +
> + image->defer_io = defer_io_get_resource(tracker->defer_io);
> + image->cbt_map = cbt_map_get_resource(tracker->cbt_map);
> + image->original_dev = original_dev;
> +
> + image->image_dev = MKDEV(snapimage_major, minor);
> + pr_info("Snapshot image device id [%d:%d]\n", MAJOR(image->image_dev),
> + MINOR(image->image_dev));
> +
> + atomic_set(&image->own_cnt, 0);
> +
> + mutex_init(&image->open_locker);
> + image->open_bdev = NULL;
> + image->open_cnt = 0;
> +
> + image->queue = blk_alloc_queue(NUMA_NO_NODE);
> + if (image->queue == NULL) {
> + res = -ENOMEM;
> + break;
> + }
> + image->queue->queuedata = image;
> +
> + blk_queue_max_segment_size(image->queue, 1024 * PAGE_SIZE);
> +
> + {
> + unsigned int physical_block_size = original_dev_info.physical_block_size;
> + unsigned short logical_block_size = original_dev_info.logical_block_size;
> +
> + pr_info("Snapshot image physical block size %d\n", physical_block_size);
> + pr_info("Snapshot image logical block size %d\n", logical_block_size);
> +
> + blk_queue_physical_block_size(image->queue, physical_block_size);
> + blk_queue_logical_block_size(image->queue, logical_block_size);
> + }
> + disk = alloc_disk(1); //only one partition on disk
> + if (disk == NULL) {
> + pr_err("Failed to allocate disk for snapshot image device\n");
> + res = -ENOMEM;
> + break;
> + }
> + image->disk = disk;
> +
> + if (snprintf(disk->disk_name, DISK_NAME_LEN, "%s%d", SNAP_IMAGE_NAME, minor) < 0) {
> + pr_err("Unable to set disk name for snapshot image device: invalid minor %d\n",
> + minor);
> + res = -EINVAL;
> + break;
> + }
> +
> + pr_info("Snapshot image disk name [%s]", disk->disk_name);
> +
> + disk->flags |= GENHD_FL_NO_PART_SCAN;
> + disk->flags |= GENHD_FL_REMOVABLE;
> +
> + disk->major = snapimage_major;
> + disk->minors = 1; // one disk have only one partition.
> + disk->first_minor = minor;
> +
> + disk->private_data = image;
> +
> + disk->fops = &snapimage_ops;
> + disk->queue = image->queue;
> +
> + set_capacity(disk, image->capacity);
> + pr_info("Snapshot image device capacity %lld bytes",
> + (u64)from_sectors(image->capacity));
> +
> + //res = -ENOMEM;
> + redirect_bio_queue_init(&image->image_queue);
> +
> + {
> + struct task_struct *task =
> + kthread_create(snapimage_processor_thread, image, disk->disk_name);
> + if (IS_ERR(task)) {
> + res = PTR_ERR(task);
> + pr_err("Failed to create request processing thread for snapshot image device. errno=%d\n",
> + res);
> + break;
> + }
> + image->rq_processor = task;
> + }
> + init_waitqueue_head(&image->rq_complete_event);
> +
> + init_waitqueue_head(&image->rq_proc_event);
> + wake_up_process(image->rq_processor);
> + } while (false);
> +
> + if (res == SUCCESS) {
> + down_write(&snap_images_lock);
> + list_add_tail(&image->link, &snap_images);
> + up_write(&snap_images_lock);
> + } else {
> + _snapimage_destroy(image);
> + _snapimage_free(image);
> +
> + kfree(image);
> + image = NULL;
> + }
> + return res;
> +}
> +
> +static struct snapimage *snapimage_find(dev_t original_dev)
> +{
> + struct snapimage *image = NULL;
> +
> + down_read(&snap_images_lock);
> + if (!list_empty(&snap_images)) {
> + struct list_head *_list_head;
> +
> + list_for_each(_list_head, &snap_images) {
> + struct snapimage *_image = list_entry(_list_head, struct snapimage, link);
> +
> + if (_image->original_dev == original_dev) {
> + image = _image;
> + break;
> + }
> + }
> + }
> + up_read(&snap_images_lock);
> +
> + return image;
> +}
> +
> +void snapimage_stop(dev_t original_dev)
> +{
> + struct snapimage *image;
> +
> + pr_info("Snapshot image processing stop for original device [%d:%d]\n", MAJOR(original_dev),
> + MINOR(original_dev));
> +
> + down_read(&snap_image_destroy_lock);
> +
> + image = snapimage_find(original_dev);
> + if (image != NULL)
> + _snapimage_stop(image);
> + else
> + pr_err("Snapshot image [%d:%d] not found\n", MAJOR(original_dev),
> + MINOR(original_dev));
> +
> + up_read(&snap_image_destroy_lock);
> +}
> +
> +void snapimage_destroy(dev_t original_dev)
> +{
> + struct snapimage *image = NULL;
> +
> + pr_info("Destroy snapshot image for device [%d:%d]\n", MAJOR(original_dev),
> + MINOR(original_dev));
> +
> + down_write(&snap_images_lock);
> + if (!list_empty(&snap_images)) {
> + struct list_head *_list_head;
> +
> + list_for_each(_list_head, &snap_images) {
> + struct snapimage *_image = list_entry(_list_head, struct snapimage, link);
> +
> + if (_image->original_dev == original_dev) {
> + image = _image;
> + list_del(&image->link);
> + break;
> + }
> + }
> + }
> + up_write(&snap_images_lock);
> +
> + if (image != NULL) {
> + down_write(&snap_image_destroy_lock);
> +
> + _snapimage_destroy(image);
> + _snapimage_free(image);
> +
> + kfree(image);
> + image = NULL;
> +
> + up_write(&snap_image_destroy_lock);
> + } else
> + pr_err("Snapshot image [%d:%d] not found\n", MAJOR(original_dev),
> + MINOR(original_dev));
> +}
> +
> +void snapimage_destroy_for(dev_t *p_dev, int count)
> +{
> + int inx = 0;
> +
> + for (; inx < count; ++inx)
> + snapimage_destroy(p_dev[inx]);
> +}
> +
> +int snapimage_create_for(dev_t *p_dev, int count)
> +{
> + int res = SUCCESS;
> + int inx = 0;
> +
> + for (; inx < count; ++inx) {
> + res = snapimage_create(p_dev[inx]);
> + if (res != SUCCESS) {
> + pr_err("Failed to create snapshot image for original device [%d:%d]\n",
> + MAJOR(p_dev[inx]), MINOR(p_dev[inx]));
> + break;
> + }
> + }
> + if (res != SUCCESS)
> + if (inx > 0)
> + snapimage_destroy_for(p_dev, inx - 1);
> + return res;
> +}
> +
> +int snapimage_init(void)
> +{
> + int res = SUCCESS;
> +
> + res = register_blkdev(snapimage_major, SNAP_IMAGE_NAME);
> + if (res >= SUCCESS) {
> + snapimage_major = res;
> + pr_info("Snapshot image block device major %d was registered\n", snapimage_major);
> + res = SUCCESS;
> +
> + spin_lock(&snapimage_minors_lock);
> + snapimage_minors = bitmap_zalloc(SNAPIMAGE_MAX_DEVICES, GFP_KERNEL);
> + spin_unlock(&snapimage_minors_lock);
> +
> + if (snapimage_minors == NULL)
> + pr_err("Failed to initialize bitmap of minors\n");
> + } else
> + pr_err("Failed to register snapshot image block device. errno=%d\n", res);
> +
> + return res;
> +}
> +
> +void snapimage_done(void)
> +{
> + down_write(&snap_image_destroy_lock);
> + while (true) {
> + struct snapimage *image = NULL;
> +
> + down_write(&snap_images_lock);
> + if (!list_empty(&snap_images)) {
> + image = list_entry(snap_images.next, struct snapimage, link);
> +
> + list_del(&image->link);
> + }
> + up_write(&snap_images_lock);
> +
> + if (image == NULL)
> + break;
> +
> + pr_err("Snapshot image for device was unexpectedly removed [%d:%d]\n",
> + MAJOR(image->original_dev), MINOR(image->original_dev));
> +
> + _snapimage_destroy(image);
> + _snapimage_free(image);
> +
> + kfree(image);
> + image = NULL;
> + }
> +
> + spin_lock(&snapimage_minors_lock);
> + bitmap_free(snapimage_minors);
> + snapimage_minors = NULL;
> + spin_unlock(&snapimage_minors_lock);
> +
> + if (!list_empty(&snap_images))
> + pr_err("Failed to release snapshot images container\n");
> +
> + unregister_blkdev(snapimage_major, SNAP_IMAGE_NAME);
> + pr_info("Snapshot image block device [%d] was unregistered\n", snapimage_major);
> +
> + up_write(&snap_image_destroy_lock);
> +}
> +
> +int snapimage_collect_images(int count, struct image_info_s *p_user_image_info, int *p_real_count)
> +{
> + int res = SUCCESS;
> + int real_count = 0;
> +
> + down_read(&snap_images_lock);
> + if (!list_empty(&snap_images)) {
> + struct list_head *_list_head;
> +
> + list_for_each(_list_head, &snap_images)
> + real_count++;
> + }
> + up_read(&snap_images_lock);
> + *p_real_count = real_count;
> +
> + if (count < real_count)
> + res = -ENODATA;
> +
> + real_count = min(count, real_count);
> + if (real_count > 0) {
> + unsigned long len;
> + struct image_info_s *p_kernel_image_info = NULL;
> + size_t buff_size;
> +
> + buff_size = sizeof(struct image_info_s) * real_count;
> + p_kernel_image_info = kzalloc(buff_size, GFP_KERNEL);
> + if (p_kernel_image_info == NULL) {
> + pr_err("Unable to collect snapshot images: not enough memory. size=%zu\n",
> + buff_size);
> + return res = -ENOMEM;
> + }
> +
> + down_read(&snap_image_destroy_lock);
> + down_read(&snap_images_lock);
> +
> + if (!list_empty(&snap_images)) {
> + size_t inx = 0;
> + struct list_head *_list_head;
> +
> + list_for_each(_list_head, &snap_images) {
> + struct snapimage *img =
> + list_entry(_list_head, struct snapimage, link);
> +
> + real_count++;
> +
> + p_kernel_image_info[inx].original_dev_id.major =
> + MAJOR(img->original_dev);
> + p_kernel_image_info[inx].original_dev_id.minor =
> + MINOR(img->original_dev);
> +
> + p_kernel_image_info[inx].snapshot_dev_id.major =
> + MAJOR(img->image_dev);
> + p_kernel_image_info[inx].snapshot_dev_id.minor =
> + MINOR(img->image_dev);
> +
> + ++inx;
> + if (inx > real_count)
> + break;
> + }
> + }
> +
> + up_read(&snap_images_lock);
> + up_read(&snap_image_destroy_lock);
> +
> + len = copy_to_user(p_user_image_info, p_kernel_image_info, buff_size);
> + if (len != 0) {
> + pr_err("Unable to collect snapshot images: failed to copy data to user buffer\n");
> + res = -ENODATA;
> + }
> +
> + kfree(p_kernel_image_info);
> + }
> +
> + return res;
> +}
> +
> +int snapimage_mark_dirty_blocks(dev_t image_dev_id, struct block_range_s *block_ranges,
> + unsigned int count)
> +{
> + size_t inx = 0;
> + int res = SUCCESS;
> +
> + pr_info("Marking [%d] dirty blocks for image device [%d:%d]\n", count, MAJOR(image_dev_id),
> + MINOR(image_dev_id));
> +
> + down_read(&snap_image_destroy_lock);
> + do {
> + struct snapimage *image = snapimage_find(image_dev_id);
> +
> + if (image == NULL) {
> + pr_err("Cannot find device [%d:%d]\n", MAJOR(image_dev_id),
> + MINOR(image_dev_id));
> + res = -ENODEV;
> + break;
> + }
> +
> + for (inx = 0; inx < count; ++inx) {
> + sector_t ofs = (sector_t)block_ranges[inx].ofs;
> + sector_t cnt = (sector_t)block_ranges[inx].cnt;
> +
> + res = cbt_map_set_both(image->cbt_map, ofs, cnt);
> + if (res != SUCCESS) {
> + pr_err("Failed to set CBT table. errno=%d\n", res);
> + break;
> + }
> + }
> + } while (false);
> + up_read(&snap_image_destroy_lock);
> +
> + return res;
> +}
> diff --git a/drivers/block/blk-snap/snapimage.h b/drivers/block/blk-snap/snapimage.h
> new file mode 100644
> index 000000000000..67995c321496
> --- /dev/null
> +++ b/drivers/block/blk-snap/snapimage.h
> @@ -0,0 +1,16 @@
> +/* SPDX-License-Identifier: GPL-2.0 */
> +#pragma once
> +
> +#include "blk-snap-ctl.h"
> +
> +int snapimage_init(void);
> +void snapimage_done(void);
> +int snapimage_create_for(dev_t *p_dev, int count);
> +
> +void snapimage_stop(dev_t original_dev);
> +void snapimage_destroy(dev_t original_dev);
> +
> +int snapimage_collect_images(int count, struct image_info_s *p_user_image_info, int *p_real_count);
> +
> +int snapimage_mark_dirty_blocks(dev_t image_dev_id, struct block_range_s *block_ranges,
> + unsigned int count);
> diff --git a/drivers/block/blk-snap/snapshot.c b/drivers/block/blk-snap/snapshot.c
> new file mode 100644
> index 000000000000..fdef713103d2
> --- /dev/null
> +++ b/drivers/block/blk-snap/snapshot.c
> @@ -0,0 +1,225 @@
> +// SPDX-License-Identifier: GPL-2.0
> +#define BLK_SNAP_SECTION "-snapshot"
> +#include "common.h"
> +#include "snapshot.h"
> +#include "tracker.h"
> +#include "snapimage.h"
> +#include "tracking.h"
> +
> +LIST_HEAD(snapshots);
> +DECLARE_RWSEM(snapshots_lock);
> +
> +
> +static int _snapshot_remove_device(dev_t dev_id)
> +{
> + int result;
> + struct tracker *tracker = NULL;
> +
> + result = tracker_find_by_dev_id(dev_id, &tracker);
> + if (result != SUCCESS) {
> + if (result == -ENODEV)
> + pr_err("Cannot find device by device id=[%d:%d]\n", MAJOR(dev_id),
> + MINOR(dev_id));
> + else
> + pr_err("Failed to find device by device id=[%d:%d]\n", MAJOR(dev_id),
> + MINOR(dev_id));
> + return SUCCESS;
> + }
> +
> + if (result != SUCCESS)
> + return result;
> +
> + tracker->snapshot_id = 0ull;
> +
> + pr_info("Device [%d:%d] successfully removed from snapshot\n", MAJOR(dev_id),
> + MINOR(dev_id));
> + return SUCCESS;
> +}
> +
> +static void _snapshot_cleanup(struct snapshot *snapshot)
> +{
> + int inx;
> +
> + for (inx = 0; inx < snapshot->dev_id_set_size; ++inx) {
> +
> + if (_snapshot_remove_device(snapshot->dev_id_set[inx]) != SUCCESS)
> + pr_err("Failed to remove device [%d:%d] from snapshot\n",
> + MAJOR(snapshot->dev_id_set[inx]), MINOR(snapshot->dev_id_set[inx]));
> + }
> +
> + if (snapshot->dev_id_set != NULL)
> + kfree(snapshot->dev_id_set);
> + kfree(snapshot);
> +}
> +
> +static void _snapshot_destroy(struct snapshot *snapshot)
> +{
> + size_t inx;
> +
> + for (inx = 0; inx < snapshot->dev_id_set_size; ++inx)
> + snapimage_stop(snapshot->dev_id_set[inx]);
> +
> + pr_info("Release snapshot [0x%llx]\n", snapshot->id);
> +
> + tracker_release_snapshot(snapshot->dev_id_set, snapshot->dev_id_set_size);
> +
> + for (inx = 0; inx < snapshot->dev_id_set_size; ++inx)
> + snapimage_destroy(snapshot->dev_id_set[inx]);
> +
> + _snapshot_cleanup(snapshot);
> +}
> +
> +
> +static int _snapshot_new(dev_t *p_dev, int count, struct snapshot **pp_snapshot)
> +{
> + struct snapshot *p_snapshot = NULL;
> + dev_t *snap_set = NULL;
> +
> + p_snapshot = kzalloc(sizeof(struct snapshot), GFP_KERNEL);
> + if (p_snapshot == NULL)
> + return -ENOMEM;
> +
> + INIT_LIST_HEAD(&p_snapshot->link);
> +
> + p_snapshot->id = (unsigned long)(p_snapshot);
> +
> + snap_set = kcalloc(count, sizeof(dev_t), GFP_KERNEL);
> + if (snap_set == NULL) {
> + kfree(p_snapshot);
> +
> + pr_err("Unable to create snapshot: faile to allocate memory for snapshot map\n");
> + return -ENOMEM;
> + }
> + memcpy(snap_set, p_dev, sizeof(dev_t) * count);
> +
> + p_snapshot->dev_id_set_size = count;
> + p_snapshot->dev_id_set = snap_set;
> +
> + down_write(&snapshots_lock);
> + list_add_tail(&snapshots, &p_snapshot->link);
> + up_write(&snapshots_lock);
> +
> + *pp_snapshot = p_snapshot;
> +
> + return SUCCESS;
> +}
> +
> +void snapshot_done(void)
> +{
> + struct snapshot *snap;
> +
> + pr_info("Removing all snapshots\n");
> + do {
> + snap = NULL;
> + down_write(&snapshots_lock);
> + if (!list_empty(&snapshots)) {
> + struct snapshot *snap = list_entry(snapshots.next, struct snapshot, link);
> +
> + list_del(&snap->link);
> + }
> + up_write(&snapshots_lock);
> +
> + if (snap)
> + _snapshot_destroy(snap);
> +
> + } while (snap);
> +}
> +
> +int snapshot_create(dev_t *dev_id_set, unsigned int dev_id_set_size,
> + unsigned long long *p_snapshot_id)
> +{
> + struct snapshot *snapshot = NULL;
> + int result = SUCCESS;
> + unsigned int inx;
> +
> + pr_info("Create snapshot for devices:\n");
> + for (inx = 0; inx < dev_id_set_size; ++inx)
> + pr_info("\t%d:%d\n", MAJOR(dev_id_set[inx]), MINOR(dev_id_set[inx]));
> +
> + result = _snapshot_new(dev_id_set, dev_id_set_size, &snapshot);
> + if (result != SUCCESS) {
> + pr_err("Unable to create snapshot: failed to allocate snapshot structure\n");
> + return result;
> + }
> +
> + do {
> + result = -ENODEV;
> + for (inx = 0; inx < snapshot->dev_id_set_size; ++inx) {
> + dev_t dev_id = snapshot->dev_id_set[inx];
> +
> + result = tracking_add(dev_id, snapshot->id);
> + if (result == -EALREADY)
> + result = SUCCESS;
> + else if (result != SUCCESS) {
> + pr_err("Unable to create snapshot\n");
> + pr_err("Failed to add device [%d:%d] to snapshot tracking\n",
> + MAJOR(dev_id), MINOR(dev_id));
> + break;
> + }
> + }
> + if (result != SUCCESS)
> + break;
> +
> + result = tracker_capture_snapshot(snapshot->dev_id_set, snapshot->dev_id_set_size);
> + if (result != SUCCESS) {
> + pr_err("Unable to create snapshot: failed to capture snapshot [0x%llx]\n",
> + snapshot->id);
> + break;
> + }
> +
> + result = snapimage_create_for(snapshot->dev_id_set, snapshot->dev_id_set_size);
> + if (result != SUCCESS) {
> + pr_err("Unable to create snapshot\n");
> + pr_err("Failed to create snapshot image devices\n");
> +
> + tracker_release_snapshot(snapshot->dev_id_set, snapshot->dev_id_set_size);
> + break;
> + }
> +
> + *p_snapshot_id = snapshot->id;
> + pr_info("Snapshot [0x%llx] was created\n", snapshot->id);
> + } while (false);
> +
> + if (result != SUCCESS) {
> + pr_info("Snapshot [0x%llx] cleanup\n", snapshot->id);
> +
> + down_write(&snapshots_lock);
> + list_del(&snapshot->link);
> + up_write(&snapshots_lock);
> +
> + _snapshot_cleanup(snapshot);
> + }
> + return result;
> +}
> +
> +int snapshot_destroy(unsigned long long snapshot_id)
> +{
> + struct snapshot *snapshot = NULL;
> +
> + pr_info("Destroy snapshot [0x%llx]\n", snapshot_id);
> +
> + down_read(&snapshots_lock);
> + if (!list_empty(&snapshots)) {
> + struct list_head *_head;
> +
> + list_for_each(_head, &snapshots) {
> + struct snapshot *_snap = list_entry(_head, struct snapshot, link);
> +
> + if (_snap->id == snapshot_id) {
> + snapshot = _snap;
> + list_del(&snapshot->link);
> + break;
> + }
> + }
> + }
> + up_read(&snapshots_lock);
> +
> + if (snapshot == NULL) {
> + pr_err("Unable to destroy snapshot [0x%llx]: cannot find snapshot by id\n",
> + snapshot_id);
> + return -ENODEV;
> + }
> +
> + _snapshot_destroy(snapshot);
> + return SUCCESS;
> +}
> diff --git a/drivers/block/blk-snap/snapshot.h b/drivers/block/blk-snap/snapshot.h
> new file mode 100644
> index 000000000000..59fb4dba0241
> --- /dev/null
> +++ b/drivers/block/blk-snap/snapshot.h
> @@ -0,0 +1,17 @@
> +/* SPDX-License-Identifier: GPL-2.0 */
> +#pragma once
> +
> +struct snapshot {
> + struct list_head link;
> + unsigned long long id;
> +
> + dev_t *dev_id_set; //array of assigned devices
> + int dev_id_set_size;
> +};
> +
> +void snapshot_done(void);
> +
> +int snapshot_create(dev_t *dev_id_set, unsigned int dev_id_set_size,
> + unsigned long long *p_snapshot_id);
> +
> +int snapshot_destroy(unsigned long long snapshot_id);
> diff --git a/drivers/block/blk-snap/snapstore.c b/drivers/block/blk-snap/snapstore.c
> new file mode 100644
> index 000000000000..0bedeaeec021
> --- /dev/null
> +++ b/drivers/block/blk-snap/snapstore.c
> @@ -0,0 +1,929 @@
> +// SPDX-License-Identifier: GPL-2.0
> +#define BLK_SNAP_SECTION "-snapstore"
> +#include "common.h"
> +#include "snapstore.h"
> +#include "snapstore_device.h"
> +#include "big_buffer.h"
> +#include "params.h"
> +
> +LIST_HEAD(snapstores);
> +DECLARE_RWSEM(snapstores_lock);
> +
> +bool _snapstore_check_halffill(struct snapstore *snapstore, sector_t *fill_status)
> +{
> + struct blk_descr_pool *pool = NULL;
> +
> + if (snapstore->file)
> + pool = &snapstore->file->pool;
> +#ifdef CONFIG_BLK_SNAP_SNAPSTORE_MULTIDEV
> + else if (snapstore->multidev)
> + pool = &snapstore->multidev->pool;
> +#endif
> + else if (snapstore->mem)
> + pool = &snapstore->mem->pool;
> +
> + if (pool)
> + return blk_descr_pool_check_halffill(pool, snapstore->empty_limit, fill_status);
> +
> + return false;
> +}
> +
> +void _snapstore_destroy(struct snapstore *snapstore)
> +{
> + sector_t fill_status;
> +
> + pr_info("Destroy snapstore with id %pUB\n", &snapstore->id);
> +
> + _snapstore_check_halffill(snapstore, &fill_status);
> +
> + down_write(&snapstores_lock);
> + list_del(&snapstore->link);
> + up_write(&snapstores_lock);
> +
> + if (snapstore->mem != NULL)
> + snapstore_mem_destroy(snapstore->mem);
> + if (snapstore->multidev != NULL)
> + snapstore_multidev_destroy(snapstore->multidev);
> + if (snapstore->file != NULL)
> + snapstore_file_destroy(snapstore->file);
> +
> + if (snapstore->ctrl_pipe) {
> + struct ctrl_pipe *pipe;
> +
> + pipe = snapstore->ctrl_pipe;
> + snapstore->ctrl_pipe = NULL;
> +
> + ctrl_pipe_request_terminate(pipe, fill_status);
> +
> + ctrl_pipe_put_resource(pipe);
> + }
> +
> + kfree(snapstore);
> +}
> +
> +static void _snapstore_destroy_cb(struct kref *kref)
> +{
> + struct snapstore *snapstore = container_of(kref, struct snapstore, refcount);
> +
> + _snapstore_destroy(snapstore);
> +}
> +
> +struct snapstore *snapstore_get(struct snapstore *snapstore)
> +{
> + if (snapstore)
> + kref_get(&snapstore->refcount);
> +
> + return snapstore;
> +}
> +
> +void snapstore_put(struct snapstore *snapstore)
> +{
> + if (snapstore)
> + kref_put(&snapstore->refcount, _snapstore_destroy_cb);
> +}
> +
> +void snapstore_done(void)
> +{
> + bool is_empty;
> +
> + down_read(&snapstores_lock);
> + is_empty = list_empty(&snapstores);
> + up_read(&snapstores_lock);
> +
> + if (!is_empty)
> + pr_err("Unable to perform snapstore cleanup: container is not empty\n");
> +}
> +
> +int snapstore_create(uuid_t *id, dev_t snapstore_dev_id, dev_t *dev_id_set,
> + size_t dev_id_set_length)
> +{
> + int res = SUCCESS;
> + size_t dev_id_inx;
> + struct snapstore *snapstore = NULL;
> +
> + if (dev_id_set_length == 0)
> + return -EINVAL;
> +
> + snapstore = kzalloc(sizeof(struct snapstore), GFP_KERNEL);
> + if (snapstore == NULL)
> + return -ENOMEM;
> +
> + INIT_LIST_HEAD(&snapstore->link);
> + uuid_copy(&snapstore->id, id);
> +
> + pr_info("Create snapstore with id %pUB\n", &snapstore->id);
> +
> + snapstore->mem = NULL;
> + snapstore->multidev = NULL;
> + snapstore->file = NULL;
> +
> + snapstore->ctrl_pipe = NULL;
> + snapstore->empty_limit = (sector_t)(64 * (1024 * 1024 / SECTOR_SIZE)); //by default value
> + snapstore->halffilled = false;
> + snapstore->overflowed = false;
> +
> + if (snapstore_dev_id == 0)
> + pr_info("Memory snapstore create\n");
> +
> +#ifdef CONFIG_BLK_SNAP_SNAPSTORE_MULTIDEV
> + else if (snapstore_dev_id == 0xFFFFffff) {
> + struct snapstore_multidev *multidev = NULL;
> +
> + res = snapstore_multidev_create(&multidev);
> + if (res != SUCCESS) {
> + kfree(snapstore);
> +
> + pr_err("Failed to create multidevice snapstore %pUB\n", id);
> + return res;
> + }
> + snapstore->multidev = multidev;
> + }
> +#endif
> + else {
> + struct snapstore_file *file = NULL;
> +
> + res = snapstore_file_create(snapstore_dev_id, &file);
> + if (res != SUCCESS) {
> + kfree(snapstore);
> +
> + pr_err("Failed to create snapstore file for snapstore %pUB\n", id);
> + return res;
> + }
> + snapstore->file = file;
> + }
> +
> + down_write(&snapstores_lock);
> + list_add_tail(&snapstores, &snapstore->link);
> + up_write(&snapstores_lock);
> +
> + kref_init(&snapstore->refcount);
> +
> + for (dev_id_inx = 0; dev_id_inx < dev_id_set_length; ++dev_id_inx) {
> + res = snapstore_device_create(dev_id_set[dev_id_inx], snapstore);
> + if (res != SUCCESS)
> + break;
> + }
> +
> + if (res != SUCCESS)
> + snapstore_device_cleanup(id);
> +
> + snapstore_put(snapstore);
> + return res;
> +}
> +
> +#ifdef CONFIG_BLK_SNAP_SNAPSTORE_MULTIDEV
> +int snapstore_create_multidev(uuid_t *id, dev_t *dev_id_set, size_t dev_id_set_length)
> +{
> + int res = SUCCESS;
> + size_t dev_id_inx;
> + struct snapstore *snapstore = NULL;
> + struct snapstore_multidev *multidev = NULL;
> +
> + if (dev_id_set_length == 0)
> + return -EINVAL;
> +
> + snapstore = kzalloc(sizeof(struct snapstore), GFP_KERNEL);
> + if (snapstore == NULL)
> + return -ENOMEM;
> +
> + INIT_LIST_HEAD(&snapstore->link);
> +
> + uuid_copy(&snapstore->id, id);
> +
> + pr_info("Create snapstore with id %pUB\n", &snapstore->id);
> +
> + snapstore->mem = NULL;
> + snapstore->file = NULL;
> + snapstore->multidev = NULL;
> +
> + snapstore->ctrl_pipe = NULL;
> + snapstore->empty_limit = (sector_t)(64 * (1024 * 1024 / SECTOR_SIZE)); //by default value
> + snapstore->halffilled = false;
> + snapstore->overflowed = false;
> +
> + res = snapstore_multidev_create(&multidev);
> + if (res != SUCCESS) {
> + kfree(snapstore);
> +
> + pr_err("Failed to create snapstore file for snapstore %pUB\n", id);
> + return res;
> + }
> + snapstore->multidev = multidev;
> +
> + down_write(&snapstores_lock);
> + list_add_tail(&snapstore->link, &snapstores);
> + up_write(&snapstores_lock);
> +
> + kref_init(&snapstore->refcount);
> +
> + for (dev_id_inx = 0; dev_id_inx < dev_id_set_length; ++dev_id_inx) {
> + res = snapstore_device_create(dev_id_set[dev_id_inx], snapstore);
> + if (res != SUCCESS)
> + break;
> + }
> +
> + if (res != SUCCESS)
> + snapstore_device_cleanup(id);
> +
> + snapstore_put(snapstore);
> + return res;
> +}
> +#endif
> +
> +int snapstore_cleanup(uuid_t *id, u64 *filled_bytes)
> +{
> + int res;
> + sector_t filled;
> +
> + res = snapstore_check_halffill(id, &filled);
> + if (res == SUCCESS) {
> + *filled_bytes = (u64)from_sectors(filled);
> +
> + pr_info("Snapstore fill size: %lld MiB\n", (*filled_bytes >> 20));
> + } else {
> + *filled_bytes = -1;
> + pr_err("Failed to obtain snapstore data filled size\n");
> + }
> +
> + return snapstore_device_cleanup(id);
> +}
> +
> +struct snapstore *_snapstore_find(uuid_t *id)
> +{
> + struct snapstore *result = NULL;
> +
> + down_read(&snapstores_lock);
> + if (!list_empty(&snapstores)) {
> + struct list_head *_head;
> +
> + list_for_each(_head, &snapstores) {
> + struct snapstore *snapstore = list_entry(_head, struct snapstore, link);
> +
> + if (uuid_equal(&snapstore->id, id)) {
> + result = snapstore;
> + break;
> + }
> + }
> + }
> + up_read(&snapstores_lock);
> +
> + return result;
> +}
> +
> +int snapstore_stretch_initiate(uuid_t *unique_id, struct ctrl_pipe *ctrl_pipe, sector_t empty_limit)
> +{
> + struct snapstore *snapstore;
> +
> + snapstore = _snapstore_find(unique_id);
> + if (snapstore == NULL) {
> + pr_err("Unable to initiate stretch snapstore: ");
> + pr_err("cannot find snapstore by uuid %pUB\n", unique_id);
> + return -ENODATA;
> + }
> +
> + snapstore->ctrl_pipe = ctrl_pipe_get_resource(ctrl_pipe);
> + snapstore->empty_limit = empty_limit;
> +
> + return SUCCESS;
> +}
> +
> +int snapstore_add_memory(uuid_t *id, unsigned long long sz)
> +{
> + int res = SUCCESS;
> + struct snapstore *snapstore = NULL;
> + size_t available_blocks = (size_t)(sz >> (snapstore_block_shift() + SECTOR_SHIFT));
> + size_t current_block = 0;
> +
> + pr_info("Adding %lld bytes to the snapstore\n", sz);
> +
> + snapstore = _snapstore_find(id);
> + if (snapstore == NULL) {
> + pr_err("Unable to add memory block to the snapstore: ");
> + pr_err("cannot found snapstore by id %pUB\n", id);
> + return -ENODATA;
> + }
> +
> + if (snapstore->file != NULL) {
> + pr_err("Unable to add memory block to the snapstore: ");
> + pr_err("snapstore file is already created\n");
> + return -EINVAL;
> + }
> +#ifdef CONFIG_BLK_SNAP_SNAPSTORE_MULTIDEV
> + if (snapstore->multidev != NULL) {
> + pr_err("Unable to add memory block to the snapstore: ");
> + pr_err("snapstore multidevice is already created\n");
> + return -EINVAL;
> + }
> +#endif
> + if (snapstore->mem != NULL) {
> + pr_err("Unable to add memory block to the snapstore: ");
> + pr_err("snapstore memory buffer is already created\n");
> + return -EINVAL;
> + }
> +
> + snapstore->mem = snapstore_mem_create(available_blocks);
> + for (current_block = 0; current_block < available_blocks; ++current_block) {
> + void *buffer = snapstore_mem_get_block(snapstore->mem);
> +
> + if (buffer == NULL) {
> + pr_err("Unable to add memory block to the snapstore: ");
> + pr_err("not enough memory\n");
> + res = -ENOMEM;
> + break;
> + }
> +
> + res = blk_descr_mem_pool_add(&snapstore->mem->pool, buffer);
> + if (res != SUCCESS) {
> + pr_err("Unable to add memory block to the snapstore: ");
> + pr_err("failed to initialize new block\n");
> + break;
> + }
> + }
> + if (res != SUCCESS) {
> + snapstore_mem_destroy(snapstore->mem);
> + snapstore->mem = NULL;
> + }
> +
> + return res;
> +}
> +
> +int rangelist_add(struct list_head *rglist, struct blk_range *rg)
> +{
> + struct blk_range_link *range_link;
> +
> + range_link = kzalloc(sizeof(struct blk_range_link), GFP_KERNEL);
> + if (range_link == NULL)
> + return -ENOMEM;
> +
> + INIT_LIST_HEAD(&range_link->link);
> +
> + range_link->rg.ofs = rg->ofs;
> + range_link->rg.cnt = rg->cnt;
> +
> + list_add_tail(&range_link->link, rglist);
> +
> + return SUCCESS;
> +}
> +
> +int snapstore_add_file(uuid_t *id, struct big_buffer *ranges, size_t ranges_cnt)
> +{
> + int res = SUCCESS;
> + struct snapstore *snapstore = NULL;
> + struct snapstore_device *snapstore_device = NULL;
> + sector_t current_blk_size = 0;
> + LIST_HEAD(blk_rangelist);
> + size_t inx;
> +
> + pr_info("Snapstore add %zu ranges\n", ranges_cnt);
> +
> + if ((ranges_cnt == 0) || (ranges == NULL))
> + return -EINVAL;
> +
> + snapstore = _snapstore_find(id);
> + if (snapstore == NULL) {
> + pr_err("Unable to add file to snapstore: ");
> + pr_err("cannot find snapstore by id %pUB\n", id);
> + return -ENODATA;
> + }
> +
> + if (snapstore->file == NULL) {
> + pr_err("Unable to add file to snapstore: ");
> + pr_err("snapstore file was not initialized\n");
> + return -EFAULT;
> + }
> +
> + snapstore_device =
> + snapstore_device_find_by_dev_id(snapstore->file->blk_dev_id); //for zeroed
> +
> + for (inx = 0; inx < ranges_cnt; ++inx) {
> + size_t blocks_count = 0;
> + sector_t range_offset = 0;
> +
> + struct blk_range range;
> + struct ioctl_range_s *ioctl_range;
> +
> + ioctl_range = big_buffer_get_element(ranges, inx, sizeof(struct ioctl_range_s));
> + if (ioctl_range == NULL) {
> + pr_err("Invalid count of ranges\n");
> + res = -ENODATA;
> + break;
> + }
> +
> + range.ofs = (sector_t)to_sectors(ioctl_range->left);
> + range.cnt = (blkcnt_t)to_sectors(ioctl_range->right) - range.ofs;
> +
> + while (range_offset < range.cnt) {
> + struct blk_range rg;
> +
> + rg.ofs = range.ofs + range_offset;
> + rg.cnt = min_t(sector_t, (range.cnt - range_offset),
> + (snapstore_block_size() - current_blk_size));
> +
> + range_offset += rg.cnt;
> +
> + res = rangelist_add(&blk_rangelist, &rg);
> + if (res != SUCCESS) {
> + pr_err("Unable to add file to snapstore: ");
> + pr_err("cannot add range to rangelist\n");
> + break;
> + }
> +
> + //zero sectors logic
> + if (snapstore_device != NULL) {
> + res = rangevector_add(&snapstore_device->zero_sectors, &rg);
> + if (res != SUCCESS) {
> + pr_err("Unable to add file to snapstore: ");
> + pr_err("cannot add range to zero_sectors tree\n");
> + break;
> + }
> + }
> +
> + current_blk_size += rg.cnt;
> +
> + if (current_blk_size == snapstore_block_size()) { //allocate block
> + res = blk_descr_file_pool_add(&snapstore->file->pool,
> + &blk_rangelist);
> + if (res != SUCCESS) {
> + pr_err("Unable to add file to snapstore: ");
> + pr_err("cannot initialize new block\n");
> + break;
> + }
> +
> + snapstore->halffilled = false;
> +
> + current_blk_size = 0;
> + INIT_LIST_HEAD(&blk_rangelist); //renew list
> + ++blocks_count;
> + }
> + }
> + if (res != SUCCESS)
> + break;
> + }
> +
> + if ((res == SUCCESS) && (current_blk_size != 0))
> + pr_warn("Snapstore portion was not ordered by Copy-on-Write block size\n");
> +
> + return res;
> +}
> +
> +#ifdef CONFIG_BLK_SNAP_SNAPSTORE_MULTIDEV
> +static int rangelist_ex_add(struct list_head *list, struct blk_range *rg,
> + struct block_device *blk_dev)
> +{
> + struct blk_range_link_ex *range_link =
> + kzalloc(sizeof(struct blk_range_link_ex), GFP_KERNEL);
> + if (range_link == NULL)
> + return -ENOMEM;
> +
> + INIT_LIST_HEAD(&range_link->link);
> +
> + range_link->rg.ofs = rg->ofs;
> + range_link->rg.cnt = rg->cnt;
> + range_link->blk_dev = blk_dev;
> +
> + list_add_tail(&range_link->link, list);
> +
> + return SUCCESS;
> +}
> +
> +int snapstore_add_multidev(uuid_t *id, dev_t dev_id, struct big_buffer *ranges, size_t ranges_cnt)
> +{
> + int res = SUCCESS;
> + struct snapstore *snapstore = NULL;
> + sector_t current_blk_size = 0;
> + size_t inx;
> + LIST_HEAD(blk_rangelist);
> +
> + pr_info("Snapstore add %zu ranges for device [%d:%d]\n", ranges_cnt, MAJOR(dev_id),
> + MINOR(dev_id));
> +
> + if ((ranges_cnt == 0) || (ranges == NULL))
> + return -EINVAL;
> +
> + snapstore = _snapstore_find(id);
> + if (snapstore == NULL) {
> + pr_err("Unable to add file to multidevice snapstore: ");
> + pr_err("cannot find snapstore by id %pUB\n", id);
> + return -ENODATA;
> + }
> +
> + if (snapstore->multidev == NULL) {
> + pr_err("Unable to add file to multidevice snapstore: ");
> + pr_err("it was not initialized\n");
> + return -EFAULT;
> + }
> +
> + for (inx = 0; inx < ranges_cnt; ++inx) {
> + size_t blocks_count = 0;
> + sector_t range_offset = 0;
> + struct blk_range range;
> + struct ioctl_range_s *data;
> +
> + data = big_buffer_get_element(ranges, inx, sizeof(struct ioctl_range_s));
> + if (data == NULL) {
> + pr_err("Invalid count of ranges\n");
> + res = -ENODATA;
> + break;
> + }
> +
> + range.ofs = (sector_t)to_sectors(data->left);
> + range.cnt = (blkcnt_t)to_sectors(data->right) - range.ofs;
> +
> + while (range_offset < range.cnt) {
> + struct blk_range rg;
> + struct block_device *blk_dev = NULL;
> +
> + rg.ofs = range.ofs + range_offset;
> + rg.cnt = min_t(sector_t,
> + range.cnt - range_offset,
> + snapstore_block_size() - current_blk_size);
> +
> + range_offset += rg.cnt;
> +
> + blk_dev = snapstore_multidev_get_device(snapstore->multidev, dev_id);
> + if (blk_dev == NULL) {
> + pr_err("Cannot find or open device [%d:%d] for multidevice snapstore\n",
> + MAJOR(dev_id), MINOR(dev_id));
> + res = -ENODEV;
> + break;
> + }
> +
> + res = rangelist_ex_add(&blk_rangelist, &rg, blk_dev);
> + if (res != SUCCESS) {
> + pr_err("Unable to add file to multidevice snapstore: ");
> + pr_err("failed to add range to rangelist\n");
> + break;
> + }
> +
> + /*
> + * zero sectors logic is not implemented for multidevice snapstore
> + */
> +
> + current_blk_size += rg.cnt;
> +
> + if (current_blk_size == snapstore_block_size()) { //allocate block
> + res = blk_descr_multidev_pool_add(&snapstore->multidev->pool,
> + &blk_rangelist);
> + if (res != SUCCESS) {
> + pr_err("Unable to add file to multidevice snapstore: ");
> + pr_err("failed to initialize new block\n");
> + break;
> + }
> +
> + snapstore->halffilled = false;
> +
> + current_blk_size = 0;
> + INIT_LIST_HEAD(&blk_rangelist);
> + ++blocks_count;
> + }
> + }
> + if (res != SUCCESS)
> + break;
> + }
> +
> + if ((res == SUCCESS) && (current_blk_size != 0))
> + pr_warn("Snapstore portion was not ordered by Copy-on-Write block size\n");
> +
> + return res;
> +}
> +#endif
> +
> +void snapstore_order_border(struct blk_range *in, struct blk_range *out)
> +{
> + struct blk_range unorder;
> +
> + unorder.ofs = in->ofs & snapstore_block_mask();
> + out->ofs = in->ofs & ~snapstore_block_mask();
> + out->cnt = in->cnt + unorder.ofs;
> +
> + unorder.cnt = out->cnt & snapstore_block_mask();
> + if (unorder.cnt != 0)
> + out->cnt += (snapstore_block_size() - unorder.cnt);
> +}
> +
> +union blk_descr_unify snapstore_get_empty_block(struct snapstore *snapstore)
> +{
> + union blk_descr_unify result = { NULL };
> +
> + if (snapstore->overflowed)
> + return result;
> +
> + if (snapstore->file != NULL)
> + result = blk_descr_file_pool_take(&snapstore->file->pool);
> + else if (snapstore->multidev != NULL)
> + result = blk_descr_multidev_pool_take(&snapstore->multidev->pool);
> + else if (snapstore->mem != NULL)
> + result = blk_descr_mem_pool_take(&snapstore->mem->pool);
> +
> + if (result.ptr == NULL) {
> + if (snapstore->ctrl_pipe) {
> + sector_t fill_status;
> +
> + _snapstore_check_halffill(snapstore, &fill_status);
> + ctrl_pipe_request_overflow(snapstore->ctrl_pipe, -EINVAL,
> + (u64)from_sectors(fill_status));
> + }
> + snapstore->overflowed = true;
> + }
> +
> + return result;
> +}
> +
> +int snapstore_check_halffill(uuid_t *unique_id, sector_t *fill_status)
> +{
> + struct snapstore *snapstore;
> +
> + snapstore = _snapstore_find(unique_id);
> + if (snapstore == NULL) {
> + pr_err("Cannot find snapstore by uuid %pUB\n", unique_id);
> + return -ENODATA;
> + }
> +
> + _snapstore_check_halffill(snapstore, fill_status);
> +
> + return SUCCESS;
> +}
> +
> +int snapstore_request_store(struct snapstore *snapstore, struct blk_deferred_request *dio_copy_req)
> +{
> + int res = SUCCESS;
> +
> + if (snapstore->ctrl_pipe) {
> + if (!snapstore->halffilled) {
> + sector_t fill_status = 0;
> +
> + if (_snapstore_check_halffill(snapstore, &fill_status)) {
> + snapstore->halffilled = true;
> + ctrl_pipe_request_halffill(snapstore->ctrl_pipe,
> + (u64)from_sectors(fill_status));
> + }
> + }
> + }
> +
> + if (snapstore->file)
> + res = blk_deferred_request_store_file(snapstore->file->blk_dev, dio_copy_req);
> +#ifdef CONFIG_BLK_SNAP_SNAPSTORE_MULTIDEV
> + else if (snapstore->multidev)
> + res = blk_deferred_request_store_multidev(dio_copy_req);
> +#endif
> + else if (snapstore->mem)
> + res = blk_deferred_request_store_mem(dio_copy_req);
> + else
> + res = -EINVAL;
> +
> + return res;
> +}
> +
> +static int _snapstore_redirect_read_file(struct blk_redirect_bio *rq_redir,
> + struct block_device *snapstore_blk_dev,
> + struct blk_descr_file *file,
> + sector_t block_ofs,
> + sector_t rq_ofs, sector_t rq_count)
> +{
> + int res = SUCCESS;
> + sector_t current_ofs = 0;
> + struct list_head *_list_head;
> +
> + if (unlikely(list_empty(&file->rangelist))) {
> + pr_err("Invalid file block descriptor");
> + return -EINVAL;
> + }
> +
> + list_for_each(_list_head, &file->rangelist) {
> + struct blk_range_link *range_link;
> +
> + range_link = list_entry(_list_head, struct blk_range_link, link);
> + if (current_ofs >= rq_count)
> + break;
> +
> + if (range_link->rg.cnt > block_ofs) {
> + sector_t pos = range_link->rg.ofs + block_ofs;
> + sector_t len = min_t(sector_t,
> + range_link->rg.cnt - block_ofs,
> + rq_count - current_ofs);
> +
> + res = blk_dev_redirect_part(rq_redir, READ, snapstore_blk_dev, pos,
> + rq_ofs + current_ofs, len);
> + if (res != SUCCESS) {
> + pr_err("Failed to read from snapstore file. Sector #%lld\n",
> + pos);
> + break;
> + }
> +
> + current_ofs += len;
> + block_ofs = 0;
> + } else
> + block_ofs -= range_link->rg.cnt;
> + }
> +
> + if (res != SUCCESS)
> + pr_err("Failed to read from file snapstore\n");
> + return res;
> +}
> +
> +#ifdef CONFIG_BLK_SNAP_SNAPSTORE_MULTIDEV
> +static int _snapstore_redirect_read_multidev(struct blk_redirect_bio *rq_redir,
> + struct blk_descr_multidev *multidev,
> + sector_t block_ofs,
> + sector_t rq_ofs, sector_t rq_count)
> +{
> + int res = SUCCESS;
> + sector_t current_ofs = 0;
> + struct list_head *_list_head;
> +
> + if (unlikely(list_empty(&multidev->rangelist))) {
> + pr_err("Invalid multidev block descriptor");
> + return -EINVAL;
> + }
> +
> + list_for_each(_list_head, &multidev->rangelist) {
> + struct blk_range_link_ex *range_link =
> + list_entry(_list_head, struct blk_range_link_ex, link);
> +
> + if (current_ofs >= rq_count)
> + break;
> +
> + if (range_link->rg.cnt > block_ofs) {
> + sector_t pos = range_link->rg.ofs + block_ofs;
> + sector_t len = min_t(sector_t,
> + range_link->rg.cnt - block_ofs,
> + rq_count - current_ofs);
> +
> + res = blk_dev_redirect_part(rq_redir, READ, range_link->blk_dev, pos,
> + rq_ofs + current_ofs, len);
> +
> + if (res != SUCCESS) {
> + pr_err("Failed to read from snapstore file. Sector #%lld\n", pos);
> + break;
> + }
> +
> + current_ofs += len;
> + block_ofs = 0;
> + } else
> + block_ofs -= range_link->rg.cnt;
> + }
> +
> + if (res != SUCCESS)
> + pr_err("Failed to read from multidev snapstore\n");
> + return res;
> +}
> +#endif
> +
> +int snapstore_redirect_read(struct blk_redirect_bio *rq_redir, struct snapstore *snapstore,
> + union blk_descr_unify blk_descr, sector_t target_pos, sector_t rq_ofs,
> + sector_t rq_count)
> +{
> + int res = SUCCESS;
> + sector_t block_ofs = target_pos & snapstore_block_mask();
> +
> + if (snapstore->file)
> + res = _snapstore_redirect_read_file(rq_redir, snapstore->file->blk_dev,
> + blk_descr.file, block_ofs, rq_ofs, rq_count);
> +#ifdef CONFIG_BLK_SNAP_SNAPSTORE_MULTIDEV
> + else if (snapstore->multidev)
> + res = _snapstore_redirect_read_multidev(rq_redir, blk_descr.multidev, block_ofs,
> + rq_ofs, rq_count);
> +#endif
> + else if (snapstore->mem) {
> + res = blk_dev_redirect_memcpy_part(
> + rq_redir, READ, blk_descr.mem->buff + (size_t)from_sectors(block_ofs),
> + rq_ofs, rq_count);
> +
> + if (res != SUCCESS)
> + pr_err("Failed to read from snapstore memory\n");
> + } else
> + res = -EINVAL;
> +
> + if (res != SUCCESS)
> + pr_err("Failed to read from snapstore. Offset %lld sector\n", target_pos);
> + return res;
> +}
> +
> +static int _snapstore_redirect_write_file(struct blk_redirect_bio *rq_redir,
> + struct block_device *snapstore_blk_dev,
> + struct blk_descr_file *file,
> + sector_t block_ofs,
> + sector_t rq_ofs, sector_t rq_count)
> +{
> + int res = SUCCESS;
> + sector_t current_ofs = 0;
> + struct list_head *_list_head;
> +
> + if (unlikely(list_empty(&file->rangelist))) {
> + pr_err("Invalid file block descriptor");
> + return -EINVAL;
> + }
> +
> + list_for_each(_list_head, &file->rangelist) {
> + struct blk_range_link *range_link;
> +
> + range_link = list_entry(_list_head, struct blk_range_link, link);
> + if (current_ofs >= rq_count)
> + break;
> +
> + if (range_link->rg.cnt > block_ofs) {
> + sector_t pos = range_link->rg.ofs + block_ofs;
> + sector_t len = min_t(sector_t,
> + range_link->rg.cnt - block_ofs,
> + rq_count - current_ofs);
> +
> + res = blk_dev_redirect_part(rq_redir, WRITE, snapstore_blk_dev, pos,
> + rq_ofs + current_ofs, len);
> +
> + if (res != SUCCESS) {
> + pr_err("Failed to write to snapstore file. Sector #%lld\n",
> + pos);
> + break;
> + }
> +
> + current_ofs += len;
> + block_ofs = 0;
> + } else
> + block_ofs -= range_link->rg.cnt;
> + }
> + if (res != SUCCESS)
> + pr_err("Failed to write to file snapstore\n");
> + return res;
> +}
> +
> +#ifdef CONFIG_BLK_SNAP_SNAPSTORE_MULTIDEV
> +static int _snapstore_redirect_write_multidev(struct blk_redirect_bio *rq_redir,
> + struct blk_descr_multidev *multidev,
> + sector_t block_ofs,
> + sector_t rq_ofs, sector_t rq_count)
> +{
> + int res = SUCCESS;
> + sector_t current_ofs = 0;
> + struct list_head *_list_head;
> +
> + if (unlikely(list_empty(&multidev->rangelist))) {
> + pr_err("Invalid multidev block descriptor");
> + return -EINVAL;
> + }
> +
> + list_for_each(_list_head, &multidev->rangelist) {
> + struct blk_range_link_ex *range_link;
> +
> + range_link = list_entry(_list_head, struct blk_range_link_ex, link);
> + if (current_ofs >= rq_count)
> + break;
> +
> + if (range_link->rg.cnt > block_ofs) {
> + sector_t pos = range_link->rg.ofs + block_ofs;
> + sector_t len = min_t(sector_t,
> + range_link->rg.cnt - block_ofs,
> + rq_count - current_ofs);
> +
> + res = blk_dev_redirect_part(rq_redir, WRITE, range_link->blk_dev, pos,
> + rq_ofs + current_ofs, len);
> +
> + if (res != SUCCESS) {
> + pr_err("Failed to write to snapstore file. Sector #%lld\n",
> + pos);
> + break;
> + }
> +
> + current_ofs += len;
> + block_ofs = 0;
> + } else
> + block_ofs -= range_link->rg.cnt;
> + }
> +
> + if (res != SUCCESS)
> + pr_err("Failed to write to multidevice snapstore\n");
> + return res;
> +}
> +#endif
> +
> +int snapstore_redirect_write(struct blk_redirect_bio *rq_redir, struct snapstore *snapstore,
> + union blk_descr_unify blk_descr, sector_t target_pos, sector_t rq_ofs,
> + sector_t rq_count)
> +{
> + int res = SUCCESS;
> + sector_t block_ofs = target_pos & snapstore_block_mask();
> +
> + if (snapstore->file)
> + res = _snapstore_redirect_write_file(rq_redir, snapstore->file->blk_dev,
> + blk_descr.file, block_ofs, rq_ofs, rq_count);
> +
> +#ifdef CONFIG_BLK_SNAP_SNAPSTORE_MULTIDEV
> + else if (snapstore->multidev)
> + res = _snapstore_redirect_write_multidev(rq_redir, blk_descr.multidev,
> + block_ofs, rq_ofs, rq_count);
> +#endif
> + else if (snapstore->mem) {
> + res = blk_dev_redirect_memcpy_part(
> + rq_redir, WRITE, blk_descr.mem->buff + (size_t)from_sectors(block_ofs),
> + rq_ofs, rq_count);
> +
> + if (res != SUCCESS)
> + pr_err("Failed to write to memory snapstore\n");
> + } else {
> + pr_err("Unable to write to snapstore: invalid type of snapstore device\n");
> + res = -EINVAL;
> + }
> +
> + if (res != SUCCESS)
> + pr_err("Failed to write to snapstore. Offset %lld sector\n", target_pos);
> + return res;
> +}
> diff --git a/drivers/block/blk-snap/snapstore.h b/drivers/block/blk-snap/snapstore.h
> new file mode 100644
> index 000000000000..db34ad2e2c58
> --- /dev/null
> +++ b/drivers/block/blk-snap/snapstore.h
> @@ -0,0 +1,68 @@
> +/* SPDX-License-Identifier: GPL-2.0 */
> +#pragma once
> +
> +#include <linux/uuid.h>
> +#include <linux/kref.h>
> +#include "blk-snap-ctl.h"
> +#include "rangevector.h"
> +#include "snapstore_mem.h"
> +#include "snapstore_file.h"
> +#include "snapstore_multidev.h"
> +#include "blk_redirect.h"
> +#include "ctrl_pipe.h"
> +#include "big_buffer.h"
> +
> +struct snapstore {
> + struct list_head link;
> + struct kref refcount;
> +
> + uuid_t id;
> +
> + struct snapstore_mem *mem;
> + struct snapstore_file *file;
> +#ifdef CONFIG_BLK_SNAP_SNAPSTORE_MULTIDEV
> + struct snapstore_multidev *multidev;
> +#endif
> +
> + struct ctrl_pipe *ctrl_pipe;
> + sector_t empty_limit;
> +
> + bool halffilled;
> + bool overflowed;
> +};
> +
> +void snapstore_done(void);
> +
> +int snapstore_create(uuid_t *id, dev_t snapstore_dev_id, dev_t *dev_id_set,
> + size_t dev_id_set_length);
> +#ifdef CONFIG_BLK_SNAP_SNAPSTORE_MULTIDEV
> +int snapstore_create_multidev(uuid_t *id, dev_t *dev_id_set, size_t dev_id_set_length);
> +#endif
> +int snapstore_cleanup(uuid_t *id, u64 *filled_bytes);
> +
> +struct snapstore *snapstore_get(struct snapstore *snapstore);
> +void snapstore_put(struct snapstore *snapstore);
> +
> +int snapstore_stretch_initiate(uuid_t *unique_id, struct ctrl_pipe *ctrl_pipe,
> + sector_t empty_limit);
> +
> +int snapstore_add_memory(uuid_t *id, unsigned long long sz);
> +int snapstore_add_file(uuid_t *id, struct big_buffer *ranges, size_t ranges_cnt);
> +#ifdef CONFIG_BLK_SNAP_SNAPSTORE_MULTIDEV
> +int snapstore_add_multidev(uuid_t *id, dev_t dev_id, struct big_buffer *ranges, size_t ranges_cnt);
> +#endif
> +
> +void snapstore_order_border(struct blk_range *in, struct blk_range *out);
> +
> +union blk_descr_unify snapstore_get_empty_block(struct snapstore *snapstore);
> +
> +int snapstore_request_store(struct snapstore *snapstore, struct blk_deferred_request *dio_copy_req);
> +
> +int snapstore_redirect_read(struct blk_redirect_bio *rq_redir, struct snapstore *snapstore,
> + union blk_descr_unify blk_descr, sector_t target_pos, sector_t rq_ofs,
> + sector_t rq_count);
> +int snapstore_redirect_write(struct blk_redirect_bio *rq_redir, struct snapstore *snapstore,
> + union blk_descr_unify blk_descr, sector_t target_pos, sector_t rq_ofs,
> + sector_t rq_count);
> +
> +int snapstore_check_halffill(uuid_t *unique_id, sector_t *fill_status);
> diff --git a/drivers/block/blk-snap/snapstore_device.c b/drivers/block/blk-snap/snapstore_device.c
> new file mode 100644
> index 000000000000..6fdeebacce22
> --- /dev/null
> +++ b/drivers/block/blk-snap/snapstore_device.c
> @@ -0,0 +1,532 @@
> +// SPDX-License-Identifier: GPL-2.0
> +#define BLK_SNAP_SECTION "-snapstore"
> +#include "common.h"
> +#include "snapstore_device.h"
> +#include "snapstore.h"
> +#include "params.h"
> +#include "blk_util.h"
> +
> +LIST_HEAD(snapstore_devices);
> +DECLARE_RWSEM(snapstore_devices_lock);
> +
> +static inline void _snapstore_device_descr_write_lock(struct snapstore_device *snapstore_device)
> +{
> + mutex_lock(&snapstore_device->store_block_map_locker);
> +}
> +static inline void _snapstore_device_descr_write_unlock(struct snapstore_device *snapstore_device)
> +{
> + mutex_unlock(&snapstore_device->store_block_map_locker);
> +}
> +
> +void snapstore_device_done(void)
> +{
> + struct snapstore_device *snapstore_device = NULL;
> +
> + do {
> + down_write(&snapstore_devices_lock);
> + if (!list_empty(&snapstore_devices)) {
> + snapstore_device =
> + list_entry(snapstore_devices.next, struct snapstore_device, link);
> + list_del(&snapstore_device->link);
> + }
> + up_write(&snapstore_devices_lock);
> +
> + if (snapstore_device)
> + snapstore_device_put_resource(snapstore_device);
> + } while (snapstore_device);
> +}
> +
> +struct snapstore_device *snapstore_device_find_by_dev_id(dev_t dev_id)
> +{
> + struct snapstore_device *result = NULL;
> +
> + down_read(&snapstore_devices_lock);
> + if (!list_empty(&snapstore_devices)) {
> + struct list_head *_head;
> +
> + list_for_each(_head, &snapstore_devices) {
> + struct snapstore_device *snapstore_device =
> + list_entry(_head, struct snapstore_device, link);
> +
> + if (dev_id == snapstore_device->dev_id) {
> + result = snapstore_device;
> + break;
> + }
> + }
> + }
> + up_read(&snapstore_devices_lock);
> +
> + return result;
> +}
> +
> +struct snapstore_device *_snapstore_device_get_by_snapstore_id(uuid_t *id)
> +{
> + struct snapstore_device *result = NULL;
> +
> + down_write(&snapstore_devices_lock);
> + if (!list_empty(&snapstore_devices)) {
> + struct list_head *_head;
> +
> + list_for_each(_head, &snapstore_devices) {
> + struct snapstore_device *snapstore_device =
> + list_entry(_head, struct snapstore_device, link);
> +
> + if (uuid_equal(id, &snapstore_device->snapstore->id)) {
> + result = snapstore_device;
> + list_del(&snapstore_device->link);
> + break;
> + }
> + }
> + }
> + up_write(&snapstore_devices_lock);
> +
> + return result;
> +}
> +
> +static void _snapstore_device_destroy(struct snapstore_device *snapstore_device)
> +{
> + pr_info("Destroy snapstore device\n");
> +
> + xa_destroy(&snapstore_device->store_block_map);
> +
> + if (snapstore_device->orig_blk_dev != NULL)
> + blk_dev_close(snapstore_device->orig_blk_dev);
> +
> + rangevector_done(&snapstore_device->zero_sectors);
> +
> + if (snapstore_device->snapstore) {
> + pr_info("Snapstore uuid %pUB\n", &snapstore_device->snapstore->id);
> +
> + snapstore_put(snapstore_device->snapstore);
> + snapstore_device->snapstore = NULL;
> + }
> +
> + kfree(snapstore_device);
> +}
> +
> +static void snapstore_device_free_cb(struct kref *kref)
> +{
> + struct snapstore_device *snapstore_device =
> + container_of(kref, struct snapstore_device, refcount);
> +
> + _snapstore_device_destroy(snapstore_device);
> +}
> +
> +struct snapstore_device *snapstore_device_get_resource(struct snapstore_device *snapstore_device)
> +{
> + if (snapstore_device)
> + kref_get(&snapstore_device->refcount);
> +
> + return snapstore_device;
> +};
> +
> +void snapstore_device_put_resource(struct snapstore_device *snapstore_device)
> +{
> + if (snapstore_device)
> + kref_put(&snapstore_device->refcount, snapstore_device_free_cb);
> +};
> +
> +int snapstore_device_cleanup(uuid_t *id)
> +{
> + int result = SUCCESS;
> + struct snapstore_device *snapstore_device = NULL;
> +
> + while (NULL != (snapstore_device = _snapstore_device_get_by_snapstore_id(id))) {
> + pr_info("Cleanup snapstore device for device [%d:%d]\n",
> + MAJOR(snapstore_device->dev_id), MINOR(snapstore_device->dev_id));
> +
> + snapstore_device_put_resource(snapstore_device);
> + }
> + return result;
> +}
> +
> +int snapstore_device_create(dev_t dev_id, struct snapstore *snapstore)
> +{
> + int res = SUCCESS;
> + struct snapstore_device *snapstore_device =
> + kzalloc(sizeof(struct snapstore_device), GFP_KERNEL);
> +
> + if (snapstore_device == NULL)
> + return -ENOMEM;
> +
> + INIT_LIST_HEAD(&snapstore_device->link);
> + snapstore_device->dev_id = dev_id;
> +
> + res = blk_dev_open(dev_id, &snapstore_device->orig_blk_dev);
> + if (res != SUCCESS) {
> + kfree(snapstore_device);
> +
> + pr_err("Unable to create snapstore device: failed to open original device [%d:%d]\n",
> + MAJOR(dev_id), MINOR(dev_id));
> + return res;
> + }
> +
> + kref_init(&snapstore_device->refcount);
> +
> + snapstore_device->snapstore = NULL;
> + snapstore_device->err_code = SUCCESS;
> + snapstore_device->corrupted = false;
> + atomic_set(&snapstore_device->req_failed_cnt, 0);
> +
> + mutex_init(&snapstore_device->store_block_map_locker);
> +
> + rangevector_init(&snapstore_device->zero_sectors);
> +
> + xa_init(&snapstore_device->store_block_map);
> +
> + snapstore_device->snapstore = snapstore_get(snapstore);
> +
> + down_write(&snapstore_devices_lock);
> + list_add_tail(&snapstore_device->link, &snapstore_devices);
> + up_write(&snapstore_devices_lock);
> +
> + return SUCCESS;
> +}
> +
> +int snapstore_device_add_request(struct snapstore_device *snapstore_device,
> + unsigned long block_index,
> + struct blk_deferred_request **dio_copy_req)
> +{
> + int res = SUCCESS;
> + union blk_descr_unify blk_descr = { NULL };
> + struct blk_deferred_io *dio = NULL;
> + bool req_new = false;
> +
> + blk_descr = snapstore_get_empty_block(snapstore_device->snapstore);
> + if (blk_descr.ptr == NULL) {
> + pr_err("Unable to add block to defer IO request: failed to allocate next block\n");
> + return -ENODATA;
> + }
> +
> + res = xa_err(
> + xa_store(&snapstore_device->store_block_map, block_index, blk_descr.ptr, GFP_NOIO));
> + if (res != SUCCESS) {
> + pr_err("Unable to add block to defer IO request: failed to set block descriptor to descriptors array. errno=%d\n",
> + res);
> + return res;
> + }
> +
> + if (*dio_copy_req == NULL) {
> + *dio_copy_req = blk_deferred_request_new();
> + if (*dio_copy_req == NULL) {
> + pr_err("Unable to add block to defer IO request: failed to allocate defer IO request\n");
> + return -ENOMEM;
> + }
> + req_new = true;
> + }
> +
> + do {
> + dio = blk_deferred_alloc(block_index, blk_descr);
> + if (dio == NULL) {
> + pr_err("Unabled to add block to defer IO request: failed to allocate defer IO\n");
> + res = -ENOMEM;
> + break;
> + }
> +
> + res = blk_deferred_request_add(*dio_copy_req, dio);
> + if (res != SUCCESS)
> + pr_err("Unable to add block to defer IO request: failed to add defer IO to request\n");
> + } while (false);
> +
> + if (res != SUCCESS) {
> + if (dio != NULL) {
> + blk_deferred_free(dio);
> + dio = NULL;
> + }
> + if (req_new) {
> + blk_deferred_request_free(*dio_copy_req);
> + *dio_copy_req = NULL;
> + }
> + }
> +
> + return res;
> +}
> +
> +int snapstore_device_prepare_requests(struct snapstore_device *snapstore_device,
> + struct blk_range *copy_range,
> + struct blk_deferred_request **dio_copy_req)
> +{
> + int res = SUCCESS;
> + unsigned long inx = 0;
> + unsigned long first = (unsigned long)(copy_range->ofs >> snapstore_block_shift());
> + unsigned long last =
> + (unsigned long)((copy_range->ofs + copy_range->cnt - 1) >> snapstore_block_shift());
> +
> + for (inx = first; inx <= last; inx++) {
> + if (xa_load(&snapstore_device->store_block_map, inx) == NULL) {
> + res = snapstore_device_add_request(snapstore_device, inx, dio_copy_req);
> + if (res != SUCCESS) {
> + pr_err("Failed to create copy defer IO request. errno=%d\n", res);
> + break;
> + }
> + }
> + /*
> + * If xa_load() return not NULL, then block already stored.
> + */
> + }
> + if (res != SUCCESS)
> + snapstore_device_set_corrupted(snapstore_device, res);
> +
> + return res;
> +}
> +
> +int snapstore_device_store(struct snapstore_device *snapstore_device,
> + struct blk_deferred_request *dio_copy_req)
> +{
> + int res;
> +
> + res = snapstore_request_store(snapstore_device->snapstore, dio_copy_req);
> + if (res != SUCCESS)
> + snapstore_device_set_corrupted(snapstore_device, res);
> +
> + return res;
> +}
> +
> +int snapstore_device_read(struct snapstore_device *snapstore_device,
> + struct blk_redirect_bio *rq_redir)
> +{
> + int res = SUCCESS;
> +
> + unsigned long block_index;
> + unsigned long block_index_last;
> + unsigned long block_index_first;
> +
> + sector_t blk_ofs_start = 0; //device range start
> + sector_t blk_ofs_count = 0; //device range length
> +
> + struct blk_range rq_range;
> + struct rangevector *zero_sectors = &snapstore_device->zero_sectors;
> +
> + if (snapstore_device_is_corrupted(snapstore_device))
> + return -ENODATA;
> +
> + rq_range.cnt = bio_sectors(rq_redir->bio);
> + rq_range.ofs = rq_redir->bio->bi_iter.bi_sector;
> +
> + if (!bio_has_data(rq_redir->bio)) {
> + pr_warn("Empty bio was found during reading from snapstore device. flags=%u\n",
> + rq_redir->bio->bi_flags);
> +
> + blk_redirect_complete(rq_redir, SUCCESS);
> + return SUCCESS;
> + }
> +
> + block_index_first = (unsigned long)(rq_range.ofs >> snapstore_block_shift());
> + block_index_last =
> + (unsigned long)((rq_range.ofs + rq_range.cnt - 1) >> snapstore_block_shift());
> +
> + _snapstore_device_descr_write_lock(snapstore_device);
> + for (block_index = block_index_first; block_index <= block_index_last; ++block_index) {
> + union blk_descr_unify blk_descr;
> +
> + blk_ofs_count = min_t(sector_t,
> + (((sector_t)(block_index + 1)) << snapstore_block_shift()) -
> + (rq_range.ofs + blk_ofs_start),
> + rq_range.cnt - blk_ofs_start);
> +
> + blk_descr = (union blk_descr_unify)xa_load(&snapstore_device->store_block_map,
> + block_index);
> + if (blk_descr.ptr) {
> + //push snapstore read
> + res = snapstore_redirect_read(rq_redir, snapstore_device->snapstore,
> + blk_descr, rq_range.ofs + blk_ofs_start,
> + blk_ofs_start, blk_ofs_count);
> + if (res != SUCCESS) {
> + pr_err("Failed to read from snapstore device\n");
> + break;
> + }
> + } else {
> + //device read with zeroing
> + if (zero_sectors)
> + res = blk_dev_redirect_read_zeroed(rq_redir,
> + snapstore_device->orig_blk_dev,
> + rq_range.ofs, blk_ofs_start,
> + blk_ofs_count, zero_sectors);
> + else
> + res = blk_dev_redirect_part(rq_redir, READ,
> + snapstore_device->orig_blk_dev,
> + rq_range.ofs + blk_ofs_start,
> + blk_ofs_start, blk_ofs_count);
> +
> + if (res != SUCCESS) {
> + pr_err("Failed to redirect read request to the original device [%d:%d]\n",
> + MAJOR(snapstore_device->dev_id),
> + MINOR(snapstore_device->dev_id));
> + break;
> + }
> + }
> +
> + blk_ofs_start += blk_ofs_count;
> + }
> +
> + if (res == SUCCESS) {
> + if (atomic64_read(&rq_redir->bio_count) > 0ll) //async direct access needed
> + blk_dev_redirect_submit(rq_redir);
> + else
> + blk_redirect_complete(rq_redir, res);
> + } else {
> + pr_err("Failed to read from snapstore device. errno=%d\n", res);
> + pr_err("Position %lld sector, length %lld sectors\n", rq_range.ofs, rq_range.cnt);
> + }
> + _snapstore_device_descr_write_unlock(snapstore_device);
> +
> + return res;
> +}
> +
> +int _snapstore_device_copy_on_write(struct snapstore_device *snapstore_device,
> + struct blk_range *rq_range)
> +{
> + int res = SUCCESS;
> + struct blk_deferred_request *dio_copy_req = NULL;
> +
> + mutex_lock(&snapstore_device->store_block_map_locker);
> + do {
> + res = snapstore_device_prepare_requests(snapstore_device, rq_range, &dio_copy_req);
> + if (res != SUCCESS) {
> + pr_err("Failed to create defer IO request for range. errno=%d\n", res);
> + break;
> + }
> +
> + if (dio_copy_req == NULL)
> + break; //nothing to copy
> +
> + res = blk_deferred_request_read_original(snapstore_device->orig_blk_dev,
> + dio_copy_req);
> + if (res != SUCCESS) {
> + pr_err("Failed to read data from the original device. errno=%d\n", res);
> + break;
> + }
> +
> + res = snapstore_device_store(snapstore_device, dio_copy_req);
> + if (res != SUCCESS) {
> + pr_err("Failed to write data to snapstore. errno=%d\n", res);
> + break;
> + }
> + } while (false);
> + mutex_unlock(&snapstore_device->store_block_map_locker);
> +
> + if (dio_copy_req) {
> + if (res == -EDEADLK)
> + blk_deferred_request_deadlocked(dio_copy_req);
> + else
> + blk_deferred_request_free(dio_copy_req);
> + }
> +
> + return res;
> +}
> +
> +int snapstore_device_write(struct snapstore_device *snapstore_device,
> + struct blk_redirect_bio *rq_redir)
> +{
> + int res = SUCCESS;
> + unsigned long block_index;
> + unsigned long block_index_last;
> + unsigned long block_index_first;
> + sector_t blk_ofs_start = 0; //device range start
> + sector_t blk_ofs_count = 0; //device range length
> + struct blk_range rq_range;
> +
> + if (snapstore_device_is_corrupted(snapstore_device))
> + return -ENODATA;
> +
> + rq_range.cnt = bio_sectors(rq_redir->bio);
> + rq_range.ofs = rq_redir->bio->bi_iter.bi_sector;
> +
> + if (!bio_has_data(rq_redir->bio)) {
> + pr_warn("Empty bio was found during reading from snapstore device. flags=%u\n",
> + rq_redir->bio->bi_flags);
> +
> + blk_redirect_complete(rq_redir, SUCCESS);
> + return SUCCESS;
> + }
> +
> + // do copy to snapstore previously
> + res = _snapstore_device_copy_on_write(snapstore_device, &rq_range);
> +
> + block_index_first = (unsigned long)(rq_range.ofs >> snapstore_block_shift());
> + block_index_last =
> + (unsigned long)((rq_range.ofs + rq_range.cnt - 1) >> snapstore_block_shift());
> +
> + _snapstore_device_descr_write_lock(snapstore_device);
> + for (block_index = block_index_first; block_index <= block_index_last; ++block_index) {
> + union blk_descr_unify blk_descr;
> +
> + blk_ofs_count = min_t(sector_t,
> + (((sector_t)(block_index + 1)) << snapstore_block_shift()) -
> + (rq_range.ofs + blk_ofs_start),
> + rq_range.cnt - blk_ofs_start);
> +
> + blk_descr = (union blk_descr_unify)xa_load(&snapstore_device->store_block_map,
> + block_index);
> + if (blk_descr.ptr == NULL) {
> + pr_err("Unable to write from snapstore device: invalid snapstore block descriptor\n");
> + res = -EIO;
> + break;
> + }
> +
> + res = snapstore_redirect_write(rq_redir, snapstore_device->snapstore, blk_descr,
> + rq_range.ofs + blk_ofs_start, blk_ofs_start,
> + blk_ofs_count);
> + if (res != SUCCESS) {
> + pr_err("Unable to write from snapstore device: failed to redirect write request to snapstore\n");
> + break;
> + }
> +
> + blk_ofs_start += blk_ofs_count;
> + }
> + if (res == SUCCESS) {
> + if (atomic64_read(&rq_redir->bio_count) > 0) { //async direct access needed
> + blk_dev_redirect_submit(rq_redir);
> + } else {
> + blk_redirect_complete(rq_redir, res);
> + }
> + } else {
> + pr_err("Failed to write from snapstore device. errno=%d\n", res);
> + pr_err("Position %lld sector, length %lld sectors\n", rq_range.ofs, rq_range.cnt);
> +
> + snapstore_device_set_corrupted(snapstore_device, res);
> + }
> + _snapstore_device_descr_write_unlock(snapstore_device);
> + return res;
> +}
> +
> +bool snapstore_device_is_corrupted(struct snapstore_device *snapstore_device)
> +{
> + if (snapstore_device == NULL)
> + return true;
> +
> + if (snapstore_device->corrupted) {
> + if (atomic_read(&snapstore_device->req_failed_cnt) == 0)
> + pr_err("Snapshot device is corrupted for [%d:%d]\n",
> + MAJOR(snapstore_device->dev_id), MINOR(snapstore_device->dev_id));
> +
> + atomic_inc(&snapstore_device->req_failed_cnt);
> + return true;
> + }
> +
> + return false;
> +}
> +
> +void snapstore_device_set_corrupted(struct snapstore_device *snapstore_device, int err_code)
> +{
> + if (!snapstore_device->corrupted) {
> + atomic_set(&snapstore_device->req_failed_cnt, 0);
> + snapstore_device->corrupted = true;
> + snapstore_device->err_code = abs(err_code);
> +
> + pr_err("Set snapshot device is corrupted for [%d:%d]\n",
> + MAJOR(snapstore_device->dev_id), MINOR(snapstore_device->dev_id));
> + }
> +}
> +
> +int snapstore_device_errno(dev_t dev_id, int *p_err_code)
> +{
> + struct snapstore_device *snapstore_device;
> +
> + snapstore_device = snapstore_device_find_by_dev_id(dev_id);
> + if (snapstore_device == NULL)
> + return -ENODATA;
> +
> + *p_err_code = snapstore_device->err_code;
> + return SUCCESS;
> +}
> diff --git a/drivers/block/blk-snap/snapstore_device.h b/drivers/block/blk-snap/snapstore_device.h
> new file mode 100644
> index 000000000000..729b3c05ef70
> --- /dev/null
> +++ b/drivers/block/blk-snap/snapstore_device.h
> @@ -0,0 +1,63 @@
> +/* SPDX-License-Identifier: GPL-2.0 */
> +#pragma once
> +
> +#include "rangevector.h"
> +#include "blk_deferred.h"
> +#include "blk_redirect.h"
> +#include "snapstore.h"
> +#include <linux/xarray.h>
> +#include <linux/kref.h>
> +
> +struct snapstore_device {
> + struct list_head link;
> + struct kref refcount;
> +
> + dev_t dev_id;
> + struct snapstore *snapstore;
> +
> + struct block_device *orig_blk_dev;
> +
> + struct xarray store_block_map; // map block index to read block offset
> + struct mutex store_block_map_locker;
> +
> + struct rangevector zero_sectors;
> +
> + atomic_t req_failed_cnt;
> + int err_code;
> + bool corrupted;
> +};
> +
> +void snapstore_device_done(void);
> +
> +struct snapstore_device *snapstore_device_get_resource(struct snapstore_device *snapstore_device);
> +void snapstore_device_put_resource(struct snapstore_device *snapstore_device);
> +
> +struct snapstore_device *snapstore_device_find_by_dev_id(dev_t dev_id);
> +
> +int snapstore_device_create(dev_t dev_id, struct snapstore *snapstore);
> +
> +int snapstore_device_cleanup(uuid_t *id);
> +
> +int snapstore_device_prepare_requests(struct snapstore_device *snapstore_device,
> + struct blk_range *copy_range,
> + struct blk_deferred_request **dio_copy_req);
> +int snapstore_device_store(struct snapstore_device *snapstore_device,
> + struct blk_deferred_request *dio_copy_req);
> +
> +int snapstore_device_read(struct snapstore_device *snapstore_device,
> + struct blk_redirect_bio *rq_redir); //request from image
> +int snapstore_device_write(struct snapstore_device *snapstore_device,
> + struct blk_redirect_bio *rq_redir); //request from image
> +
> +bool snapstore_device_is_corrupted(struct snapstore_device *snapstore_device);
> +void snapstore_device_set_corrupted(struct snapstore_device *snapstore_device, int err_code);
> +int snapstore_device_errno(dev_t dev_id, int *p_err_code);
> +
> +static inline void _snapstore_device_descr_read_lock(struct snapstore_device *snapstore_device)
> +{
> + mutex_lock(&snapstore_device->store_block_map_locker);
> +}
> +static inline void _snapstore_device_descr_read_unlock(struct snapstore_device *snapstore_device)
> +{
> + mutex_unlock(&snapstore_device->store_block_map_locker);
> +}
> diff --git a/drivers/block/blk-snap/snapstore_file.c b/drivers/block/blk-snap/snapstore_file.c
> new file mode 100644
> index 000000000000..a5c959a8070c
> --- /dev/null
> +++ b/drivers/block/blk-snap/snapstore_file.c
> @@ -0,0 +1,52 @@
> +// SPDX-License-Identifier: GPL-2.0
> +#define BLK_SNAP_SECTION "-snapstore"
> +#include "common.h"
> +#include "snapstore_file.h"
> +#include "blk_util.h"
> +
> +int snapstore_file_create(dev_t dev_id, struct snapstore_file **pfile)
> +{
> + int res = SUCCESS;
> + struct snapstore_file *file;
> +
> + pr_info("Single device file snapstore was created on device [%d:%d]\n", MAJOR(dev_id),
> + MINOR(dev_id));
> +
> + file = kzalloc(sizeof(struct snapstore_file), GFP_KERNEL);
> + if (file == NULL)
> + return -ENOMEM;
> +
> + res = blk_dev_open(dev_id, &file->blk_dev);
> + if (res != SUCCESS) {
> + kfree(file);
> + pr_err("Unable to create snapstore file: failed to open device [%d:%d]. errno=%d",
> + MAJOR(dev_id), MINOR(dev_id), res);
> + return res;
> + }
> + {
> + struct request_queue *q = bdev_get_queue(file->blk_dev);
> +
> + pr_info("snapstore device logical block size %d\n", q->limits.logical_block_size);
> + pr_info("snapstore device physical block size %d\n", q->limits.physical_block_size);
> + }
> +
> + file->blk_dev_id = dev_id;
> + blk_descr_file_pool_init(&file->pool);
> +
> + *pfile = file;
> + return res;
> +}
> +
> +void snapstore_file_destroy(struct snapstore_file *file)
> +{
> + if (file) {
> + blk_descr_file_pool_done(&file->pool);
> +
> + if (file->blk_dev != NULL) {
> + blk_dev_close(file->blk_dev);
> + file->blk_dev = NULL;
> + }
> +
> + kfree(file);
> + }
> +}
> diff --git a/drivers/block/blk-snap/snapstore_file.h b/drivers/block/blk-snap/snapstore_file.h
> new file mode 100644
> index 000000000000..effd9d888781
> --- /dev/null
> +++ b/drivers/block/blk-snap/snapstore_file.h
> @@ -0,0 +1,15 @@
> +/* SPDX-License-Identifier: GPL-2.0 */
> +#pragma once
> +
> +#include "blk_deferred.h"
> +
> +struct snapstore_file {
> + dev_t blk_dev_id;
> + struct block_device *blk_dev;
> +
> + struct blk_descr_pool pool;
> +};
> +
> +int snapstore_file_create(dev_t dev_id, struct snapstore_file **pfile);
> +
> +void snapstore_file_destroy(struct snapstore_file *file);
> diff --git a/drivers/block/blk-snap/snapstore_mem.c b/drivers/block/blk-snap/snapstore_mem.c
> new file mode 100644
> index 000000000000..29a607617d99
> --- /dev/null
> +++ b/drivers/block/blk-snap/snapstore_mem.c
> @@ -0,0 +1,91 @@
> +// SPDX-License-Identifier: GPL-2.0
> +#define BLK_SNAP_SECTION "-snapstore"
> +#include "common.h"
> +#include "snapstore_mem.h"
> +#include "params.h"
> +
> +#include <linux/vmalloc.h>
> +
> +struct buffer_el {
> + struct list_head link;
> + void *buff;
> +};
> +
> +struct snapstore_mem *snapstore_mem_create(size_t available_blocks)
> +{
> + struct snapstore_mem *mem = kzalloc(sizeof(struct snapstore_mem), GFP_KERNEL);
> +
> + if (mem == NULL)
> + return NULL;
> +
> + blk_descr_mem_pool_init(&mem->pool, available_blocks);
> +
> + mem->blocks_limit = available_blocks;
> +
> + INIT_LIST_HEAD(&mem->blocks);
> + mutex_init(&mem->blocks_lock);
> +
> + return mem;
> +}
> +
> +void snapstore_mem_destroy(struct snapstore_mem *mem)
> +{
> + struct buffer_el *buffer_el;
> +
> + if (mem == NULL)
> + return;
> +
> + do {
> + buffer_el = NULL;
> +
> + mutex_lock(&mem->blocks_lock);
> + if (!list_empty(&mem->blocks)) {
> + buffer_el = list_entry(mem->blocks.next, struct buffer_el, link);
> +
> + list_del(&buffer_el->link);
> + }
> + mutex_unlock(&mem->blocks_lock);
> +
> + if (buffer_el) {
> + vfree(buffer_el->buff);
> + kfree(buffer_el);
> + }
> + } while (buffer_el);
> +
> + blk_descr_mem_pool_done(&mem->pool);
> +
> + kfree(mem);
> +}
> +
> +void *snapstore_mem_get_block(struct snapstore_mem *mem)
> +{
> + struct buffer_el *buffer_el;
> +
> + if (mem->blocks_allocated >= mem->blocks_limit) {
> + pr_err("Unable to get block from snapstore in memory\n");
> + pr_err("Block limit is reached, allocated %zu, limit %zu\n", mem->blocks_allocated,
> + mem->blocks_limit);
> + return NULL;
> + }
> +
> + buffer_el = kzalloc(sizeof(struct buffer_el), GFP_KERNEL);
> + if (buffer_el == NULL)
> + return NULL;
> + INIT_LIST_HEAD(&buffer_el->link);
> +
> + buffer_el->buff = vmalloc(snapstore_block_size() * SECTOR_SIZE);
> + if (buffer_el->buff == NULL) {
> + kfree(buffer_el);
> + return NULL;
> + }
> +
> + ++mem->blocks_allocated;
> + if (0 == (mem->blocks_allocated & 0x7F))
> + pr_info("%zu MiB was allocated\n", mem->blocks_allocated);
> +
> + mutex_lock(&mem->blocks_lock);
> + list_add_tail(&buffer_el->link, &mem->blocks);
> + mutex_unlock(&mem->blocks_lock);
> +
> + return buffer_el->buff;
> +}
> diff --git a/drivers/block/blk-snap/snapstore_mem.h b/drivers/block/blk-snap/snapstore_mem.h
> new file mode 100644
> index 000000000000..9044a6525966
> --- /dev/null
> +++ b/drivers/block/blk-snap/snapstore_mem.h
> @@ -0,0 +1,20 @@
> +/* SPDX-License-Identifier: GPL-2.0 */
> +#pragma once
> +
> +#include "blk_descr_mem.h"
> +
> +struct snapstore_mem {
> + struct list_head blocks;
> + struct mutex blocks_lock;
> +
> + size_t blocks_limit;
> + size_t blocks_allocated;
> +
> + struct blk_descr_pool pool;
> +};
> +
> +struct snapstore_mem *snapstore_mem_create(size_t available_blocks);
> +
> +void snapstore_mem_destroy(struct snapstore_mem *mem);
> +
> +void *snapstore_mem_get_block(struct snapstore_mem *mem);
> diff --git a/drivers/block/blk-snap/snapstore_multidev.c b/drivers/block/blk-snap/snapstore_multidev.c
> new file mode 100644
> index 000000000000..bb6bfefa68d7
> --- /dev/null
> +++ b/drivers/block/blk-snap/snapstore_multidev.c
> @@ -0,0 +1,118 @@
> +// SPDX-License-Identifier: GPL-2.0
> +#define BLK_SNAP_SECTION "-snapstore"
> +#include "common.h"
> +
> +#ifdef CONFIG_BLK_SNAP_SNAPSTORE_MULTIDEV
> +
> +#include "snapstore_multidev.h"
> +#include "blk_util.h"
> +
> +struct multidev_el {
> + struct list_head link;
> +
> + dev_t dev_id;
> + struct block_device *blk_dev;
> +};
> +
> +int snapstore_multidev_create(struct snapstore_multidev **p_multidev)
> +{
> + int res = SUCCESS;
> + struct snapstore_multidev *multidev;
> +
> + pr_info("Multidevice file snapstore create\n");
> +
> + multidev = kzalloc(sizeof(struct snapstore_multidev), GFP_KERNEL);
> + if (multidev == NULL)
> + return -ENOMEM;
> +
> + INIT_LIST_HEAD(&multidev->devicelist);
> + spin_lock_init(&multidev->devicelist_lock);
> +
> + blk_descr_multidev_pool_init(&multidev->pool);
> +
> + *p_multidev = multidev;
> + return res;
> +}
> +
> +void snapstore_multidev_destroy(struct snapstore_multidev *multidev)
> +{
> + struct multidev_el *el;
> +
> + blk_descr_multidev_pool_done(&multidev->pool);
> +
> + do {
> + el = NULL;
> + spin_lock(&multidev->devicelist_lock);
> + if (!list_empty(&multidev->devicelist)) {
> + el = list_entry(multidev->devicelist.next, struct multidev_el, link);
> +
> + list_del(&el->link);
> + }
> + spin_unlock(&multidev->devicelist_lock);
> +
> + if (el) {
> + blk_dev_close(el->blk_dev);
> +
> + pr_info("Close device for multidevice snapstore [%d:%d]\n",
> + MAJOR(el->dev_id), MINOR(el->dev_id));
> +
> + kfree(el);
> + }
> + } while (el);
> +
> + kfree(multidev);
> +}
> +
> +struct multidev_el *snapstore_multidev_find(struct snapstore_multidev *multidev, dev_t dev_id)
> +{
> + struct multidev_el *el = NULL;
> +
> + spin_lock(&multidev->devicelist_lock);
> + if (!list_empty(&multidev->devicelist)) {
> + struct list_head *_head;
> +
> + list_for_each(_head, &multidev->devicelist) {
> + struct multidev_el *_el = list_entry(_head, struct multidev_el, link);
> +
> + if (_el->dev_id == dev_id) {
> + el = _el;
> + break;
> + }
> + }
> + }
> + spin_unlock(&multidev->devicelist_lock);
> +
> + return el;
> +}
> +
> +struct block_device *snapstore_multidev_get_device(struct snapstore_multidev *multidev,
> + dev_t dev_id)
> +{
> + int res;
> + struct block_device *blk_dev = NULL;
> + struct multidev_el *el = snapstore_multidev_find(multidev, dev_id);
> +
> + if (el)
> + return el->blk_dev;
> +
> + res = blk_dev_open(dev_id, &blk_dev);
> + if (res != SUCCESS) {
> + pr_err("Unable to add device to snapstore multidevice file\n");
> + pr_err("Failed to open [%d:%d]. errno=%d", MAJOR(dev_id), MINOR(dev_id), res);
> + return NULL;
> + }
> +
> + el = kzalloc(sizeof(struct multidev_el), GFP_KERNEL);
> + INIT_LIST_HEAD(&el->link);
> +
> + el->blk_dev = blk_dev;
> + el->dev_id = dev_id;
> +
> + spin_lock(&multidev->devicelist_lock);
> + list_add_tail(&el->link, &multidev->devicelist);
> + spin_unlock(&multidev->devicelist_lock);
> +
> + return el->blk_dev;
> +}
> +
> +#endif
> diff --git a/drivers/block/blk-snap/snapstore_multidev.h b/drivers/block/blk-snap/snapstore_multidev.h
> new file mode 100644
> index 000000000000..40c1c3a41b08
> --- /dev/null
> +++ b/drivers/block/blk-snap/snapstore_multidev.h
> @@ -0,0 +1,22 @@
> +/* SPDX-License-Identifier: GPL-2.0 */
> +#pragma once
> +
> +#ifdef CONFIG_BLK_SNAP_SNAPSTORE_MULTIDEV
> +
> +#include "blk_deferred.h"
> +#include "blk_descr_multidev.h"
> +
> +struct snapstore_multidev {
> + struct list_head devicelist; //for mapping device id to opened device struct pointer
> + spinlock_t devicelist_lock;
> +
> + struct blk_descr_pool pool;
> +};
> +
> +int snapstore_multidev_create(struct snapstore_multidev **p_file);
> +
> +void snapstore_multidev_destroy(struct snapstore_multidev *file);
> +
> +struct block_device *snapstore_multidev_get_device(struct snapstore_multidev *multidev,
> + dev_t dev_id);
> +#endif
> diff --git a/drivers/block/blk-snap/tracker.c b/drivers/block/blk-snap/tracker.c
> new file mode 100644
> index 000000000000..3cda996d3f0a
> --- /dev/null
> +++ b/drivers/block/blk-snap/tracker.c
> @@ -0,0 +1,449 @@
> +// SPDX-License-Identifier: GPL-2.0
> +#define BLK_SNAP_SECTION "-tracker"
> +#include "common.h"
> +#include "tracker.h"
> +#include "blk_util.h"
> +#include "params.h"
> +
> +LIST_HEAD(trackers);
> +DEFINE_RWLOCK(trackers_lock);
> +
> +void tracker_done(void)
> +{
> + tracker_remove_all();
> +}
> +
> +int tracker_find_by_bio(struct bio *bio, struct tracker **ptracker)
> +{
> + int result = -ENODATA;
> +
> + read_lock(&trackers_lock);
> + if (!list_empty(&trackers)) {
> + struct list_head *_head;
> +
> + list_for_each(_head, &trackers) {
> + struct tracker *_tracker = list_entry(_head, struct tracker, link);
> +
> + if ((bio->bi_disk == _tracker->target_dev->bd_disk) &&
> + (bio->bi_partno == _tracker->target_dev->bd_partno)) {
> + if (ptracker != NULL)
> + *ptracker = _tracker;
> +
> + result = SUCCESS;
> + break;
> + }
> + }
> + }
> + read_unlock(&trackers_lock);
> +
> + return result;
> +}
> +
> +int tracker_find_by_dev_id(dev_t dev_id, struct tracker **ptracker)
> +{
> + int result = -ENODATA;
> +
> + read_lock(&trackers_lock);
> + if (!list_empty(&trackers)) {
> + struct list_head *_head;
> +
> + list_for_each(_head, &trackers) {
> + struct tracker *_tracker = list_entry(_head, struct tracker, link);
> +
> + if (_tracker->original_dev_id == dev_id) {
> + if (ptracker != NULL)
> + *ptracker = _tracker;
> +
> + result = SUCCESS;
> + break;
> + }
> + }
> + }
> + read_unlock(&trackers_lock);
> +
> + return result;
> +}
> +
> +int tracker_enum_cbt_info(int max_count, struct cbt_info_s *p_cbt_info, int *p_count)
> +{
> + int result = SUCCESS;
> + int count = 0;
> +
> + read_lock(&trackers_lock);
> + if (!list_empty(&trackers)) {
> + struct list_head *_head;
> +
> + list_for_each(_head, &trackers) {
> + struct tracker *tracker = list_entry(_head, struct tracker, link);
> +
> + if (count >= max_count) {
> + result = -ENOBUFS;
> + break; //don`t continue
> + }
> +
> + if (p_cbt_info != NULL) {
> + p_cbt_info[count].dev_id.major = MAJOR(tracker->original_dev_id);
> + p_cbt_info[count].dev_id.minor = MINOR(tracker->original_dev_id);
> +
> + if (tracker->cbt_map) {
> + p_cbt_info[count].cbt_map_size = tracker->cbt_map->map_size;
> + p_cbt_info[count].snap_number =
> + (unsigned char)
> + tracker->cbt_map->snap_number_previous;
> + uuid_copy((uuid_t *)(p_cbt_info[count].generationId),
> + &tracker->cbt_map->generationId);
> + } else {
> + p_cbt_info[count].cbt_map_size = 0;
> + p_cbt_info[count].snap_number = 0;
> + }
> +
> + p_cbt_info[count].dev_capacity = (u64)from_sectors(
> + part_nr_sects_read(tracker->target_dev->bd_part));
> + }
> +
> + ++count;
> + }
> + }
> + read_unlock(&trackers_lock);
> +
> + if (result == SUCCESS)
> + if (count == 0)
> + result = -ENODATA;
> +
> + *p_count = count;
> + return result;
> +}
> +
> +static void blk_thaw_bdev(dev_t dev_id, struct block_device *device,
> + struct super_block *superblock)
> +{
> + if (superblock == NULL)
> + return;
> +
> + if (thaw_bdev(device, superblock) == SUCCESS)
> + pr_info("Device [%d:%d] was unfrozen\n", MAJOR(dev_id), MINOR(dev_id));
> + else
> + pr_err("Failed to unfreeze device [%d:%d]\n", MAJOR(dev_id), MINOR(dev_id));
> +}
> +
> +static int blk_freeze_bdev(dev_t dev_id, struct block_device *device,
> + struct super_block **psuperblock)
> +{
> + struct super_block *superblock;
> +
> + if (device->bd_super == NULL) {
> + pr_warn("Unable to freeze device [%d:%d]: no superblock was found\n",
> + MAJOR(dev_id), MINOR(dev_id));
> + return SUCCESS;
> + }
> +
> + superblock = freeze_bdev(device);
> + if (IS_ERR_OR_NULL(superblock)) {
> + int result;
> +
> + pr_err("Failed to freeze device [%d:%d]\n", MAJOR(dev_id), MINOR(dev_id));
> +
> + if (superblock == NULL)
> + result = -ENODEV;
> + else {
> + result = PTR_ERR(superblock);
> + pr_err("Error code: %d\n", result);
> + }
> + return result;
> + }
> +
> + pr_info("Device [%d:%d] was frozen\n", MAJOR(dev_id), MINOR(dev_id));
> + *psuperblock = superblock;
> +
> + return SUCCESS;
> +}
> +
> +int _tracker_create(struct tracker *tracker, void *filter, bool attach_filter)
> +{
> + int result = SUCCESS;
> + unsigned int sect_in_block_degree;
> + sector_t capacity;
> + struct super_block *superblock = NULL;
> +
> + result = blk_dev_open(tracker->original_dev_id, &tracker->target_dev);
> + if (result != SUCCESS)
> + return ENODEV;
> +
> + pr_info("Create tracker for device [%d:%d]. Capacity 0x%llx sectors\n",
> + MAJOR(tracker->original_dev_id), MINOR(tracker->original_dev_id),
> + (unsigned long long)part_nr_sects_read(tracker->target_dev->bd_part));
> +
> + sect_in_block_degree = get_change_tracking_block_size_pow() - SECTOR_SHIFT;
> + capacity = part_nr_sects_read(tracker->target_dev->bd_part);
> +
> + tracker->cbt_map = cbt_map_create(sect_in_block_degree, capacity);
> + if (tracker->cbt_map == NULL) {
> + pr_err("Failed to create tracker for device [%d:%d]\n",
> + MAJOR(tracker->original_dev_id), MINOR(tracker->original_dev_id));
> + tracker_remove(tracker);
> + return -ENOMEM;
> + }
> +
> + tracker->snapshot_id = 0ull;
> +
> + if (attach_filter) {
> + blk_freeze_bdev(tracker->original_dev_id, tracker->target_dev, &superblock);
> +
> + blk_filter_attach(tracker->original_dev_id, filter, tracker);
> +
> + blk_thaw_bdev(tracker->original_dev_id, tracker->target_dev, superblock);
> + }
> +
> + return SUCCESS;
> +}
> +
> +int tracker_create(dev_t dev_id, void *filter, struct tracker **ptracker)
> +{
> + int ret;
> + struct tracker *tracker = NULL;
> +
> + *ptracker = NULL;
> +
> + tracker = kzalloc(sizeof(struct tracker), GFP_KERNEL);
> + if (tracker == NULL)
> + return -ENOMEM;
> +
> + INIT_LIST_HEAD(&tracker->link);
> + atomic_set(&tracker->is_captured, false);
> + tracker->original_dev_id = dev_id;
> +
> + write_lock(&trackers_lock);
> + list_add_tail(&tracker->link, &trackers);
> + write_unlock(&trackers_lock);
> +
> + ret = _tracker_create(tracker, filter, true);
> + if (ret < 0) {
> + tracker_remove(tracker);
> + return ret;
> + }
> +
> + *ptracker = tracker;
> + if (ret == ENODEV)
> + pr_info("Cannot attach to unknown device [%d:%d]",
> + MAJOR(tracker->original_dev_id), MINOR(tracker->original_dev_id));
> +
> + return ret;
> +}
> +
> +void _tracker_remove(struct tracker *tracker, bool detach_filter)
> +{
> + struct super_block *superblock = NULL;
> +
> + if (tracker->target_dev != NULL) {
> + if (detach_filter) {
> + blk_freeze_bdev(tracker->original_dev_id, tracker->target_dev, &superblock);
> +
> + blk_filter_detach(tracker->original_dev_id);
> +
> + blk_thaw_bdev(tracker->original_dev_id, tracker->target_dev, superblock);
> + }
> +
> + blk_dev_close(tracker->target_dev);
> + tracker->target_dev = NULL;
> + }
> +
> + if (tracker->cbt_map != NULL) {
> + cbt_map_put_resource(tracker->cbt_map);
> + tracker->cbt_map = NULL;
> + }
> +}
> +
> +void tracker_remove(struct tracker *tracker)
> +{
> + _tracker_remove(tracker, true);
> +
> + write_lock(&trackers_lock);
> + list_del(&tracker->link);
> + write_unlock(&trackers_lock);
> +
> + kfree(tracker);
> +}
> +
> +void tracker_remove_all(void)
> +{
> + struct tracker *tracker;
> +
> + pr_info("Removing all devices from tracking\n");
> +
> + do {
> + tracker = NULL;
> +
> + write_lock(&trackers_lock);
> + if (!list_empty(&trackers)) {
> + tracker = list_entry(trackers.next, struct tracker, link);
> +
> + list_del(&tracker->link);
> + }
> + write_unlock(&trackers_lock);
> +
> + if (tracker) {
> + _tracker_remove(tracker, true);
> + kfree(tracker);
> + }
> + } while (tracker);
> +}
> +
> +void tracker_cbt_bitmap_set(struct tracker *tracker, sector_t sector, sector_t sector_cnt)
> +{
> + if (tracker->cbt_map == NULL)
> + return;
> +
> + if (tracker->cbt_map->device_capacity != part_nr_sects_read(tracker->target_dev->bd_part)) {
> + pr_warn("Device resize detected\n");
> + tracker->cbt_map->active = false;
> + return;
> + }
> +
> + if (cbt_map_set(tracker->cbt_map, sector, sector_cnt) != SUCCESS) { //cbt corrupt
> + pr_warn("CBT fault detected\n");
> + tracker->cbt_map->active = false;
> + return;
> + }
> +}
> +
> +bool tracker_cbt_bitmap_lock(struct tracker *tracker)
> +{
> + if (tracker->cbt_map == NULL)
> + return false;
> +
> + cbt_map_read_lock(tracker->cbt_map);
> + if (!tracker->cbt_map->active) {
> + cbt_map_read_unlock(tracker->cbt_map);
> + return false;
> + }
> +
> + return true;
> +}
> +
> +void tracker_cbt_bitmap_unlock(struct tracker *tracker)
> +{
> + if (tracker->cbt_map)
> + cbt_map_read_unlock(tracker->cbt_map);
> +}
> +
> +int _tracker_capture_snapshot(struct tracker *tracker)
> +{
> + int result = SUCCESS;
> +
> + result = defer_io_create(tracker->original_dev_id, tracker->target_dev, &tracker->defer_io);
> + if (result != SUCCESS) {
> + pr_err("Failed to create defer IO processor\n");
> + return result;
> + }
> +
> + atomic_set(&tracker->is_captured, true);
> +
> + if (tracker->cbt_map != NULL) {
> + cbt_map_write_lock(tracker->cbt_map);
> + cbt_map_switch(tracker->cbt_map);
> + cbt_map_write_unlock(tracker->cbt_map);
> +
> + pr_info("Snapshot captured for device [%d:%d]. New snap number %ld\n",
> + MAJOR(tracker->original_dev_id), MINOR(tracker->original_dev_id),
> + tracker->cbt_map->snap_number_active);
> + }
> +
> + return result;
> +}
> +
> +int tracker_capture_snapshot(dev_t *dev_id_set, int dev_id_set_size)
> +{
> + int result = SUCCESS;
> + int inx = 0;
> +
> + for (inx = 0; inx < dev_id_set_size; ++inx) {
> + struct super_block *superblock = NULL;
> + struct tracker *tracker = NULL;
> + dev_t dev_id = dev_id_set[inx];
> +
> + result = tracker_find_by_dev_id(dev_id, &tracker);
> + if (result != SUCCESS) {
> + pr_err("Unable to capture snapshot: cannot find device [%d:%d]\n",
> + MAJOR(dev_id), MINOR(dev_id));
> + break;
> + }
> +
> +
> + blk_freeze_bdev(tracker->original_dev_id, tracker->target_dev, &superblock);
> + blk_filter_freeze(tracker->target_dev);
> +
> + result = _tracker_capture_snapshot(tracker);
> + if (result != SUCCESS)
> + pr_err("Failed to capture snapshot for device [%d:%d]\n",
> + MAJOR(dev_id), MINOR(dev_id));
> +
> + blk_filter_thaw(tracker->target_dev);
> + blk_thaw_bdev(tracker->original_dev_id, tracker->target_dev, superblock);
> + }
> + if (result != SUCCESS)
> + return result;
> +
> + for (inx = 0; inx < dev_id_set_size; ++inx) {
> + struct tracker *tracker = NULL;
> + dev_t dev_id = dev_id_set[inx];
> +
> + result = tracker_find_by_dev_id(dev_id, &tracker);
> + if (result != SUCCESS) {
> + pr_err("Unable to capture snapshot: cannot find device [%d:%d]\n",
> + MAJOR(dev_id), MINOR(dev_id));
> + continue;
> + }
> +
> + if (snapstore_device_is_corrupted(tracker->defer_io->snapstore_device)) {
> + pr_err("Unable to freeze devices [%d:%d]: snapshot data is corrupted\n",
> + MAJOR(dev_id), MINOR(dev_id));
> + result = -EDEADLK;
> + break;
> + }
> + }
> +
> + if (result != SUCCESS) {
> + pr_err("Failed to capture snapshot. errno=%d\n", result);
> +
> + tracker_release_snapshot(dev_id_set, dev_id_set_size);
> + }
> + return result;
> +}
> +
> +void _tracker_release_snapshot(struct tracker *tracker)
> +{
> + struct super_block *superblock = NULL;
> + struct defer_io *defer_io = tracker->defer_io;
> +
> + blk_freeze_bdev(tracker->original_dev_id, tracker->target_dev, &superblock);
> + blk_filter_freeze(tracker->target_dev);
> + { //locked region
> + atomic_set(&tracker->is_captured, false); //clear freeze flag
> +
> + tracker->defer_io = NULL;
> + }
> + blk_filter_thaw(tracker->target_dev);
> +
> + blk_thaw_bdev(tracker->original_dev_id, tracker->target_dev, superblock);
> +
> + defer_io_stop(defer_io);
> + defer_io_put_resource(defer_io);
> +}
> +
> +void tracker_release_snapshot(dev_t *dev_id_set, int dev_id_set_size)
> +{
> + int inx = 0;
> +
> + for (; inx < dev_id_set_size; ++inx) {
> + int status;
> + struct tracker *p_tracker = NULL;
> + dev_t dev = dev_id_set[inx];
> +
> + status = tracker_find_by_dev_id(dev, &p_tracker);
> + if (status == SUCCESS)
> + _tracker_release_snapshot(p_tracker);
> + else
> + pr_err("Unable to release snapshot: cannot find tracker for device [%d:%d]\n",
> + MAJOR(dev), MINOR(dev));
> + }
> +}
> diff --git a/drivers/block/blk-snap/tracker.h b/drivers/block/blk-snap/tracker.h
> new file mode 100644
> index 000000000000..9fff7c0942c3
> --- /dev/null
> +++ b/drivers/block/blk-snap/tracker.h
> @@ -0,0 +1,38 @@
> +/* SPDX-License-Identifier: GPL-2.0 */
> +#pragma once
> +#include "cbt_map.h"
> +#include "defer_io.h"
> +#include "blk-snap-ctl.h"
> +#include "snapshot.h"
> +
> +struct tracker {
> + struct list_head link;
> + dev_t original_dev_id;
> + struct block_device *target_dev;
> + struct cbt_map *cbt_map;
> + atomic_t is_captured;
> + struct defer_io *defer_io;
> + unsigned long long snapshot_id; // current snapshot for this device
> +};
> +
> +void tracker_done(void);
> +
> +int tracker_find_by_bio(struct bio *bio, struct tracker **ptracker);
> +int tracker_find_by_dev_id(dev_t dev_id, struct tracker **ptracker);
> +
> +int tracker_enum_cbt_info(int max_count, struct cbt_info_s *p_cbt_info, int *p_count);
> +
> +int tracker_capture_snapshot(dev_t *dev_id_set, int dev_id_set_size);
> +void tracker_release_snapshot(dev_t *dev_id_set, int dev_id_set_size);
> +
> +int _tracker_create(struct tracker *tracker, void *filter, bool attach_filter);
> +int tracker_create(dev_t dev_id, void *filter, struct tracker **ptracker);
> +
> +void _tracker_remove(struct tracker *tracker, bool detach_filter);
> +void tracker_remove(struct tracker *tracker);
> +void tracker_remove_all(void);
> +
> +void tracker_cbt_bitmap_set(struct tracker *tracker, sector_t sector, sector_t sector_cnt);
> +
> +bool tracker_cbt_bitmap_lock(struct tracker *tracker);
> +void tracker_cbt_bitmap_unlock(struct tracker *tracker);
> diff --git a/drivers/block/blk-snap/tracking.c b/drivers/block/blk-snap/tracking.c
> new file mode 100644
> index 000000000000..55e18891bb96
> --- /dev/null
> +++ b/drivers/block/blk-snap/tracking.c
> @@ -0,0 +1,270 @@
> +// SPDX-License-Identifier: GPL-2.0
> +#define BLK_SNAP_SECTION "-tracking"
> +#include "common.h"
> +#include "tracking.h"
> +#include "tracker.h"
> +#include "blk_util.h"
> +#include "defer_io.h"
> +#include "params.h"
> +
> +#include <linux/blk-filter.h>
> +
> +/* pointer to block layer filter */
> +void *filter;
> +
> +/*
> + * _tracking_submit_bio() - Intercept bio by block io layer filter
> + */
> +static bool _tracking_submit_bio(struct bio *bio, void *filter_data)
> +{
> + int res;
> + bool cbt_locked = false;
> + struct tracker *tracker = filter_data;
> +
> + if (!tracker)
> + return false;
> +
> + //intercepting
> + if (atomic_read(&tracker->is_captured)) {
> + //snapshot is captured, call bio redirect algorithm
> +
> + res = defer_io_redirect_bio(tracker->defer_io, bio, tracker);
> + if (res == SUCCESS)
> + return true;
> + }
> +
> + cbt_locked = false;
> + if (tracker && bio_data_dir(bio) && bio_has_data(bio)) {
> + //call CBT algorithm
> + cbt_locked = tracker_cbt_bitmap_lock(tracker);
> + if (cbt_locked) {
> + sector_t sectStart = bio->bi_iter.bi_sector;
> + sector_t sectCount = bio_sectors(bio);
> +
> + tracker_cbt_bitmap_set(tracker, sectStart, sectCount);
> + }
> + }
> + if (cbt_locked)
> + tracker_cbt_bitmap_unlock(tracker);
> +
> + return false;
> +}
> +
> +static bool _tracking_part_add(dev_t devt, void **p_filter_data)
> +{
> + int result;
> + struct tracker *tracker = NULL;
> +
> + pr_info("new block device [%d:%d] in system\n", MAJOR(devt), MINOR(devt));
> +
> + result = tracker_find_by_dev_id(devt, &tracker);
> + if (result != SUCCESS)
> + return false; /*do not track this device*/
> +
> + if (_tracker_create(tracker, filter, false)) {
> + pr_err("Failed to attach new device to tracker. errno=%d\n", result);
> + return false; /*failed to attach new device to tracker*/
> + }
> +
> + *p_filter_data = tracker;
> + return true;
> +}
> +
> +static void _tracking_part_del(void *private_data)
> +{
> + struct tracker *tracker = private_data;
> +
> + if (!tracker)
> + return;
> +
> + pr_info("delete block device [%d:%d] from system\n",
> + MAJOR(tracker->original_dev_id), MINOR(tracker->original_dev_id));
> +
> + _tracker_remove(tracker, false);
> +}
> +
> +struct blk_filter_ops filter_ops = {
> + .filter_bio = _tracking_submit_bio,
> + .part_add = _tracking_part_add,
> + .part_del = _tracking_part_del };
> +
> +
> +
> +int tracking_init(void)
> +{
> + filter = blk_filter_register(&filter_ops);
> + if (!filter)
> + return -ENOMEM;
> + return SUCCESS;
> +}
> +
> +void tracking_done(void)
> +{
> + if (filter) {
> + blk_filter_unregister(filter);
> + filter = NULL;
> + }
> +}
> +
> +static int _add_already_tracked(dev_t dev_id, unsigned long long snapshot_id,
> + struct tracker *tracker)
> +{
> + int result = SUCCESS;
> + bool cbt_reset_needed = false;
> +
> + if ((snapshot_id != 0ull) && (tracker->snapshot_id == 0ull))
> + tracker->snapshot_id = snapshot_id; // set new snapshot id
> +
> + if (tracker->cbt_map == NULL) {
> + unsigned int sect_in_block_degree =
> + get_change_tracking_block_size_pow() - SECTOR_SHIFT;
> + tracker->cbt_map = cbt_map_create(sect_in_block_degree - SECTOR_SHIFT,
> + part_nr_sects_read(tracker->target_dev->bd_part));
> + if (tracker->cbt_map == NULL)
> + return -ENOMEM;
> +
> + // skip snapshot id
> + tracker->snapshot_id = snapshot_id;
> + return SUCCESS;
> + }
> +
> + if (!tracker->cbt_map->active) {
> + cbt_reset_needed = true;
> + pr_warn("Nonactive CBT table detected. CBT fault\n");
> + }
> +
> + if (tracker->cbt_map->device_capacity != part_nr_sects_read(tracker->target_dev->bd_part)) {
> + cbt_reset_needed = true;
> + pr_warn("Device resize detected. CBT fault\n");
> + }
> +
> + if (!cbt_reset_needed)
> + return SUCCESS;
> +
> + _tracker_remove(tracker, true);
> +
> + result = _tracker_create(tracker, filter, true);
> + if (result != SUCCESS) {
> + pr_err("Failed to create tracker. errno=%d\n", result);
> + return result;
> + }
> +
> + tracker->snapshot_id = snapshot_id;
> +
> + return SUCCESS;
> +}
> +
> +static int _create_new_tracker(dev_t dev_id, unsigned long long snapshot_id)
> +{
> + int result;
> + struct tracker *tracker = NULL;
> +
> + result = tracker_create(dev_id, filter, &tracker);
> + if (result != SUCCESS) {
> + pr_err("Failed to create tracker. errno=%d\n", result);
> + return result;
> + }
> +
> + tracker->snapshot_id = snapshot_id;
> +
> + return SUCCESS;
> +}
> +
> +
> +int tracking_add(dev_t dev_id, unsigned long long snapshot_id)
> +{
> + int result;
> + struct tracker *tracker = NULL;
> +
> + pr_info("Adding device [%d:%d] under tracking\n", MAJOR(dev_id), MINOR(dev_id));
> +
> + result = tracker_find_by_dev_id(dev_id, &tracker);
> + if (result == SUCCESS) {
> + //pr_info("Device [%d:%d] is already tracked\n", MAJOR(dev_id), MINOR(dev_id));
> + result = _add_already_tracked(dev_id, snapshot_id, tracker);
> + if (result == SUCCESS)
> + result = -EALREADY;
> + } else if (-ENODATA == result)
> + result = _create_new_tracker(dev_id, snapshot_id);
> + else {
> + pr_err("Unable to add device [%d:%d] under tracking\n", MAJOR(dev_id),
> + MINOR(dev_id));
> + pr_err("Invalid trackers container. errno=%d\n", result);
> + }
> +
> + return result;
> +}
> +
> +int tracking_remove(dev_t dev_id)
> +{
> + int result;
> + struct tracker *tracker = NULL;
> +
> + pr_info("Removing device [%d:%d] from tracking\n", MAJOR(dev_id), MINOR(dev_id));
> +
> + result = tracker_find_by_dev_id(dev_id, &tracker);
> + if (result != SUCCESS) {
> + pr_err("Unable to remove device [%d:%d] from tracking: ",
> + MAJOR(dev_id), MINOR(dev_id));
> +
> + if (-ENODATA == result)
> + pr_err("tracker not found\n");
> + else
> + pr_err("tracker container failed. errno=%d\n", result);
> +
> + return result;
> + }
> +
> + if (tracker->snapshot_id != 0ull) {
> + pr_err("Unable to remove device [%d:%d] from tracking: ",
> + MAJOR(dev_id), MINOR(dev_id));
> + pr_err("snapshot [0x%llx] already exist\n", tracker->snapshot_id);
> + return -EBUSY;
> + }
> +
> + tracker_remove(tracker);
> +
> + return SUCCESS;
> +}
> +
> +int tracking_collect(int max_count, struct cbt_info_s *p_cbt_info, int *p_count)
> +{
> + int res = tracker_enum_cbt_info(max_count, p_cbt_info, p_count);
> +
> + if (res == SUCCESS)
> + pr_info("%d devices found under tracking\n", *p_count);
> + else if (res == -ENODATA) {
> + pr_info("There are no devices under tracking\n");
> + *p_count = 0;
> + res = SUCCESS;
> + } else
> + pr_err("Failed to collect devices under tracking. errno=%d", res);
> +
> + return res;
> +}
> +
> +int tracking_read_cbt_bitmap(dev_t dev_id, unsigned int offset, size_t length,
> + void __user *user_buff)
> +{
> + int result = SUCCESS;
> + struct tracker *tracker = NULL;
> +
> + result = tracker_find_by_dev_id(dev_id, &tracker);
> + if (result == SUCCESS) {
> + if (atomic_read(&tracker->is_captured))
> + result = cbt_map_read_to_user(tracker->cbt_map, user_buff, offset, length);
> + else {
> + pr_err("Unable to read CBT bitmap for device [%d:%d]: ", MAJOR(dev_id),
> + MINOR(dev_id));
> + pr_err("device is not captured by snapshot\n");
> + result = -EPERM;
> + }
> + } else if (-ENODATA == result) {
> + pr_err("Unable to read CBT bitmap for device [%d:%d]: ", MAJOR(dev_id),
> + MINOR(dev_id));
> + pr_err("device not found\n");
> + } else
> + pr_err("Failed to find devices under tracking. errno=%d", result);
> +
> + return result;
> +}
> diff --git a/drivers/block/blk-snap/tracking.h b/drivers/block/blk-snap/tracking.h
> new file mode 100644
> index 000000000000..22bd5ba54963
> --- /dev/null
> +++ b/drivers/block/blk-snap/tracking.h
> @@ -0,0 +1,13 @@
> +/* SPDX-License-Identifier: GPL-2.0 */
> +#pragma once
> +#include "blk-snap-ctl.h"
> +#include <linux/bio.h>
> +
> +int tracking_init(void);
> +void tracking_done(void);
> +
> +int tracking_add(dev_t dev_id, unsigned long long snapshot_id);
> +int tracking_remove(dev_t dev_id);
> +int tracking_collect(int max_count, struct cbt_info_s *p_cbt_info, int *p_count);
> +int tracking_read_cbt_bitmap(dev_t dev_id, unsigned int offset, size_t length,
> + void __user *user_buff);
> diff --git a/drivers/block/blk-snap/version.h b/drivers/block/blk-snap/version.h
> new file mode 100644
> index 000000000000..a4431da73611
> --- /dev/null
> +++ b/drivers/block/blk-snap/version.h
> @@ -0,0 +1,7 @@
> +/* SPDX-License-Identifier: GPL-2.0 */
> +#pragma once
> +
> +#define FILEVER_MAJOR 5
> +#define FILEVER_MINOR 0
> +#define FILEVER_REVISION 0
> +#define FILEVER_STR "5.0.0"
>
--
Damien Le Moal
Western Digital Research
