Signed-off-by: Sergei Shtepa <sergei.shtepa@xxxxxxxxx>
---
drivers/block/Kconfig | 2 +
drivers/block/Makefile | 1 +
drivers/block/blk-snap/Kconfig | 24 +
drivers/block/blk-snap/Makefile | 28 +
drivers/block/blk-snap/big_buffer.c | 193 ++++
drivers/block/blk-snap/big_buffer.h | 24 +
drivers/block/blk-snap/blk-snap-ctl.h | 190 ++++
drivers/block/blk-snap/blk_deferred.c | 566 +++++++++++
drivers/block/blk-snap/blk_deferred.h | 67 ++
drivers/block/blk-snap/blk_descr_file.c | 82 ++
drivers/block/blk-snap/blk_descr_file.h | 26 +
drivers/block/blk-snap/blk_descr_mem.c | 66 ++
drivers/block/blk-snap/blk_descr_mem.h | 14 +
drivers/block/blk-snap/blk_descr_multidev.c | 86 ++
drivers/block/blk-snap/blk_descr_multidev.h | 25 +
drivers/block/blk-snap/blk_descr_pool.c | 190 ++++
drivers/block/blk-snap/blk_descr_pool.h | 38 +
drivers/block/blk-snap/blk_redirect.c | 507 ++++++++++
drivers/block/blk-snap/blk_redirect.h | 73 ++
drivers/block/blk-snap/blk_util.c | 33 +
drivers/block/blk-snap/blk_util.h | 33 +
drivers/block/blk-snap/cbt_map.c | 210 +++++
drivers/block/blk-snap/cbt_map.h | 62 ++
drivers/block/blk-snap/common.h | 31 +
drivers/block/blk-snap/ctrl_fops.c | 691 ++++++++++++++
drivers/block/blk-snap/ctrl_fops.h | 19 +
drivers/block/blk-snap/ctrl_pipe.c | 562 +++++++++++
drivers/block/blk-snap/ctrl_pipe.h | 34 +
drivers/block/blk-snap/ctrl_sysfs.c | 73 ++
drivers/block/blk-snap/ctrl_sysfs.h | 5 +
drivers/block/blk-snap/defer_io.c | 397 ++++++++
drivers/block/blk-snap/defer_io.h | 39 +
drivers/block/blk-snap/main.c | 82 ++
drivers/block/blk-snap/params.c | 58 ++
drivers/block/blk-snap/params.h | 29 +
drivers/block/blk-snap/rangevector.c | 85 ++
drivers/block/blk-snap/rangevector.h | 31 +
drivers/block/blk-snap/snapimage.c | 982 ++++++++++++++++++++
drivers/block/blk-snap/snapimage.h | 16 +
drivers/block/blk-snap/snapshot.c | 225 +++++
drivers/block/blk-snap/snapshot.h | 17 +
drivers/block/blk-snap/snapstore.c | 929 ++++++++++++++++++
drivers/block/blk-snap/snapstore.h | 68 ++
drivers/block/blk-snap/snapstore_device.c | 532 +++++++++++
drivers/block/blk-snap/snapstore_device.h | 63 ++
drivers/block/blk-snap/snapstore_file.c | 52 ++
drivers/block/blk-snap/snapstore_file.h | 15 +
drivers/block/blk-snap/snapstore_mem.c | 91 ++
drivers/block/blk-snap/snapstore_mem.h | 20 +
drivers/block/blk-snap/snapstore_multidev.c | 118 +++
drivers/block/blk-snap/snapstore_multidev.h | 22 +
drivers/block/blk-snap/tracker.c | 449 +++++++++
drivers/block/blk-snap/tracker.h | 38 +
drivers/block/blk-snap/tracking.c | 270 ++++++
drivers/block/blk-snap/tracking.h | 13 +
drivers/block/blk-snap/version.h | 7 +
56 files changed, 8603 insertions(+)
create mode 100644 drivers/block/blk-snap/Kconfig
create mode 100644 drivers/block/blk-snap/Makefile
create mode 100644 drivers/block/blk-snap/big_buffer.c
create mode 100644 drivers/block/blk-snap/big_buffer.h
create mode 100644 drivers/block/blk-snap/blk-snap-ctl.h
create mode 100644 drivers/block/blk-snap/blk_deferred.c
create mode 100644 drivers/block/blk-snap/blk_deferred.h
create mode 100644 drivers/block/blk-snap/blk_descr_file.c
create mode 100644 drivers/block/blk-snap/blk_descr_file.h
create mode 100644 drivers/block/blk-snap/blk_descr_mem.c
create mode 100644 drivers/block/blk-snap/blk_descr_mem.h
create mode 100644 drivers/block/blk-snap/blk_descr_multidev.c
create mode 100644 drivers/block/blk-snap/blk_descr_multidev.h
create mode 100644 drivers/block/blk-snap/blk_descr_pool.c
create mode 100644 drivers/block/blk-snap/blk_descr_pool.h
create mode 100644 drivers/block/blk-snap/blk_redirect.c
create mode 100644 drivers/block/blk-snap/blk_redirect.h
create mode 100644 drivers/block/blk-snap/blk_util.c
create mode 100644 drivers/block/blk-snap/blk_util.h
create mode 100644 drivers/block/blk-snap/cbt_map.c
create mode 100644 drivers/block/blk-snap/cbt_map.h
create mode 100644 drivers/block/blk-snap/common.h
create mode 100644 drivers/block/blk-snap/ctrl_fops.c
create mode 100644 drivers/block/blk-snap/ctrl_fops.h
create mode 100644 drivers/block/blk-snap/ctrl_pipe.c
create mode 100644 drivers/block/blk-snap/ctrl_pipe.h
create mode 100644 drivers/block/blk-snap/ctrl_sysfs.c
create mode 100644 drivers/block/blk-snap/ctrl_sysfs.h
create mode 100644 drivers/block/blk-snap/defer_io.c
create mode 100644 drivers/block/blk-snap/defer_io.h
create mode 100644 drivers/block/blk-snap/main.c
create mode 100644 drivers/block/blk-snap/params.c
create mode 100644 drivers/block/blk-snap/params.h
create mode 100644 drivers/block/blk-snap/rangevector.c
create mode 100644 drivers/block/blk-snap/rangevector.h
create mode 100644 drivers/block/blk-snap/snapimage.c
create mode 100644 drivers/block/blk-snap/snapimage.h
create mode 100644 drivers/block/blk-snap/snapshot.c
create mode 100644 drivers/block/blk-snap/snapshot.h
create mode 100644 drivers/block/blk-snap/snapstore.c
create mode 100644 drivers/block/blk-snap/snapstore.h
create mode 100644 drivers/block/blk-snap/snapstore_device.c
create mode 100644 drivers/block/blk-snap/snapstore_device.h
create mode 100644 drivers/block/blk-snap/snapstore_file.c
create mode 100644 drivers/block/blk-snap/snapstore_file.h
create mode 100644 drivers/block/blk-snap/snapstore_mem.c
create mode 100644 drivers/block/blk-snap/snapstore_mem.h
create mode 100644 drivers/block/blk-snap/snapstore_multidev.c
create mode 100644 drivers/block/blk-snap/snapstore_multidev.h
create mode 100644 drivers/block/blk-snap/tracker.c
create mode 100644 drivers/block/blk-snap/tracker.h
create mode 100644 drivers/block/blk-snap/tracking.c
create mode 100644 drivers/block/blk-snap/tracking.h
create mode 100644 drivers/block/blk-snap/version.h
diff --git a/drivers/block/Kconfig b/drivers/block/Kconfig
index ecceaaa1a66f..c53ef661110f 100644
--- a/drivers/block/Kconfig
+++ b/drivers/block/Kconfig
@@ -460,4 +460,6 @@ config BLK_DEV_RSXX
source "drivers/block/rnbd/Kconfig"
+source "drivers/block/blk-snap/Kconfig"
+
endif # BLK_DEV
diff --git a/drivers/block/Makefile b/drivers/block/Makefile
index e1f63117ee94..312000598944 100644
--- a/drivers/block/Makefile
+++ b/drivers/block/Makefile
@@ -40,6 +40,7 @@ obj-$(CONFIG_BLK_DEV_PCIESSD_MTIP32XX) += mtip32xx/
obj-$(CONFIG_BLK_DEV_RSXX) += rsxx/
obj-$(CONFIG_ZRAM) += zram/
obj-$(CONFIG_BLK_DEV_RNBD) += rnbd/
+obj-$(CONFIG_BLK_SNAP) += blk-snap/
obj-$(CONFIG_BLK_DEV_NULL_BLK) += null_blk.o
null_blk-objs := null_blk_main.o
diff --git a/drivers/block/blk-snap/Kconfig b/drivers/block/blk-snap/Kconfig
new file mode 100644
index 000000000000..7a2db99a80dd
--- /dev/null
+++ b/drivers/block/blk-snap/Kconfig
@@ -0,0 +1,24 @@
+# SPDX-License-Identifier: GPL-2.0
+#
+# blk-snap block io layer filter module configuration
+#
+#
+#select BLK_FILTER
+
+config BLK_SNAP
+ tristate "Block device snapshot filter"
+ depends on BLK_FILTER
+ help
+
+ Allow to create snapshots and track block changes for a block
+ devices. Designed for creating backups for any block devices
+ (without device mapper). Snapshots are temporary and are released
+ then backup is completed. Change block tracking allows you to
+ create incremental or differential backups.
+
+config BLK_SNAP_SNAPSTORE_MULTIDEV
+ bool "Multi device snapstore configuration support"
+ depends on BLK_SNAP
+ help
+
+ Allow to create snapstore on multiple block devices.
diff --git a/drivers/block/blk-snap/Makefile b/drivers/block/blk-snap/Makefile
new file mode 100644
index 000000000000..1d628e8e1862
--- /dev/null
+++ b/drivers/block/blk-snap/Makefile
@@ -0,0 +1,28 @@
+# SPDX-License-Identifier: GPL-2.0
+blk-snap-y += blk_deferred.o
+blk-snap-y += blk_descr_file.o
+blk-snap-y += blk_descr_mem.o
+blk-snap-y += blk_descr_multidev.o
+blk-snap-y += blk_descr_pool.o
+blk-snap-y += blk_redirect.o
+blk-snap-y += blk_util.o
+blk-snap-y += cbt_map.o
+blk-snap-y += ctrl_fops.o
+blk-snap-y += ctrl_pipe.o
+blk-snap-y += ctrl_sysfs.o
+blk-snap-y += defer_io.o
+blk-snap-y += main.o
+blk-snap-y += params.o
+blk-snap-y += big_buffer.o
+blk-snap-y += rangevector.o
+blk-snap-y += snapimage.o
+blk-snap-y += snapshot.o
+blk-snap-y += snapstore.o
+blk-snap-y += snapstore_device.o
+blk-snap-y += snapstore_file.o
+blk-snap-y += snapstore_mem.o
+blk-snap-y += snapstore_multidev.o
+blk-snap-y += tracker.o
+blk-snap-y += tracking.o
+
+obj-$(CONFIG_BLK_SNAP) += blk-snap.o
diff --git a/drivers/block/blk-snap/big_buffer.c b/drivers/block/blk-snap/big_buffer.c
new file mode 100644
index 000000000000..c0a75255a807
--- /dev/null
+++ b/drivers/block/blk-snap/big_buffer.c
@@ -0,0 +1,193 @@
+// SPDX-License-Identifier: GPL-2.0
+#include "common.h"
+#include <linux/mm.h>
+#include "big_buffer.h"
+
+static inline size_t page_count_calc(size_t buffer_size)
+{
+ size_t page_count = buffer_size / PAGE_SIZE;
+
+ if (buffer_size & (PAGE_SIZE - 1))
+ page_count += 1;
+ return page_count;
+}
+
+struct big_buffer *big_buffer_alloc(size_t buffer_size, int gfp_opt)
+{
+ int res = SUCCESS;
+ struct big_buffer *bbuff;
+ size_t count;
+ size_t inx;
+
+ count = page_count_calc(buffer_size);
+
+ bbuff = kzalloc(sizeof(struct big_buffer) + count * sizeof(void *), gfp_opt);
+ if (bbuff == NULL)
+ return NULL;
+
+ bbuff->pg_cnt = count;
+ for (inx = 0; inx < bbuff->pg_cnt; ++inx) {
+ struct page *pg = alloc_page(gfp_opt);
+
+ if (!pg) {
+ res = -ENOMEM;
+ break;
+ }
+ bbuff->pg[inx] = page_address(pg);
+ }
+
+ if (res != SUCCESS) {
+ big_buffer_free(bbuff);
+ return NULL;
+ }
+
+ return bbuff;
+}
+
+void big_buffer_free(struct big_buffer *bbuff)
+{
+ size_t inx;
+ size_t count = bbuff->pg_cnt;
+
+ if (bbuff == NULL)
+ return;
+
+ for (inx = 0; inx < count; ++inx)
+ if (bbuff->pg[inx] != NULL)
+ free_page((unsigned long)bbuff->pg[inx]);
+
+ kfree(bbuff);
+}
+
+size_t big_buffer_copy_to_user(char __user *dst_user, size_t offset, struct big_buffer *bbuff,
+ size_t length)
+{
+ size_t left_data_length;
+ int page_inx = offset / PAGE_SIZE;
+ size_t processed_len = 0;
+ size_t unordered = offset & (PAGE_SIZE - 1);
+
+ if (unordered != 0) { //first
+ size_t page_len = min_t(size_t, (PAGE_SIZE - unordered), length);
+
+ left_data_length = copy_to_user(dst_user + processed_len,
+ bbuff->pg[page_inx] + unordered, page_len);
+ if (left_data_length != 0) {
+ pr_err("Failed to copy data from big_buffer to user buffer\n");
+ return processed_len;
+ }
+
+ ++page_inx;
+ processed_len += page_len;
+ }
+
+ while ((processed_len < length) && (page_inx < bbuff->pg_cnt)) {
+ size_t page_len = min_t(size_t, PAGE_SIZE, (length - processed_len));
+
+ left_data_length =
+ copy_to_user(dst_user + processed_len, bbuff->pg[page_inx], page_len);
+ if (left_data_length != 0) {
+ pr_err("Failed to copy data from big_buffer to user buffer\n");
+ break;
+ }
+
+ ++page_inx;
+ processed_len += page_len;
+ }
+
+ return processed_len;
+}
+
+size_t big_buffer_copy_from_user(const char __user *src_user, size_t offset,
+ struct big_buffer *bbuff, size_t length)
+{
+ size_t left_data_length;
+ int page_inx = offset / PAGE_SIZE;
+ size_t processed_len = 0;
+ size_t unordered = offset & (PAGE_SIZE - 1);
+
+ if (unordered != 0) { //first
+ size_t page_len = min_t(size_t, (PAGE_SIZE - unordered), length);
+
+ left_data_length = copy_from_user(bbuff->pg[page_inx] + unordered,
+ src_user + processed_len, page_len);
+ if (left_data_length != 0) {
+ pr_err("Failed to copy data from user buffer to big_buffer\n");
+ return processed_len;
+ }
+
+ ++page_inx;
+ processed_len += page_len;
+ }
+
+ while ((processed_len < length) && (page_inx < bbuff->pg_cnt)) {
+ size_t page_len = min_t(size_t, PAGE_SIZE, (length - processed_len));
+
+ left_data_length =
+ copy_from_user(bbuff->pg[page_inx], src_user + processed_len, page_len);
+ if (left_data_length != 0) {
+ pr_err("Failed to copy data from user buffer to big_buffer\n");
+ break;
+ }
+
+ ++page_inx;
+ processed_len += page_len;
+ }
+
+ return processed_len;
+}
+
+void *big_buffer_get_element(struct big_buffer *bbuff, size_t index, size_t sizeof_element)
+{
+ size_t elements_in_page = PAGE_SIZE / sizeof_element;
+ size_t pg_inx = index / elements_in_page;
+ size_t pg_ofs = (index - (pg_inx * elements_in_page)) * sizeof_element;
+
+ if (pg_inx >= bbuff->pg_cnt)
+ return NULL;
+
+ return bbuff->pg[pg_inx] + pg_ofs;
+}
+
+void big_buffer_memset(struct big_buffer *bbuff, int value)
+{
+ size_t inx;
+
+ for (inx = 0; inx < bbuff->pg_cnt; ++inx)
+ memset(bbuff->pg[inx], value, PAGE_SIZE);
+}
+
+void big_buffer_memcpy(struct big_buffer *dst, struct big_buffer *src)
+{
+ size_t inx;
+ size_t count = min_t(size_t, dst->pg_cnt, src->pg_cnt);
+
+ for (inx = 0; inx < count; ++inx)
+ memcpy(dst->pg[inx], src->pg[inx], PAGE_SIZE);
+}
+
+int big_buffer_byte_get(struct big_buffer *bbuff, size_t inx, u8 *value)
+{
+ size_t page_inx = inx >> PAGE_SHIFT;
+ size_t byte_pos = inx & (PAGE_SIZE - 1);
+
+ if (page_inx >= bbuff->pg_cnt)
+ return -ENODATA;
+
+ *value = bbuff->pg[page_inx][byte_pos];
+
+ return SUCCESS;
+}
+
+int big_buffer_byte_set(struct big_buffer *bbuff, size_t inx, u8 value)
+{
+ size_t page_inx = inx >> PAGE_SHIFT;
+ size_t byte_pos = inx & (PAGE_SIZE - 1);
+
+ if (page_inx >= bbuff->pg_cnt)
+ return -ENODATA;
+
+ bbuff->pg[page_inx][byte_pos] = value;
+
+ return SUCCESS;
+}
diff --git a/drivers/block/blk-snap/big_buffer.h b/drivers/block/blk-snap/big_buffer.h
new file mode 100644
index 000000000000..f38ab5288b05
--- /dev/null
+++ b/drivers/block/blk-snap/big_buffer.h
@@ -0,0 +1,24 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+#pragma once
+
+struct big_buffer {
+ size_t pg_cnt;
+ u8 *pg[0];
+};
+
+struct big_buffer *big_buffer_alloc(size_t count, int gfp_opt);
+void big_buffer_free(struct big_buffer *bbuff);
+
+size_t big_buffer_copy_to_user(char __user *dst_user_buffer, size_t offset,
+ struct big_buffer *bbuff, size_t length);
+size_t big_buffer_copy_from_user(const char __user *src_user_buffer, size_t offset,
+ struct big_buffer *bbuff, size_t length);
+
+void *big_buffer_get_element(struct big_buffer *bbuff, size_t index, size_t sizeof_element);
+
+void big_buffer_memset(struct big_buffer *bbuff, int value);
+void big_buffer_memcpy(struct big_buffer *dst, struct big_buffer *src);
+
+//byte access
+int big_buffer_byte_get(struct big_buffer *bbuff, size_t inx, u8 *value);
+int big_buffer_byte_set(struct big_buffer *bbuff, size_t inx, u8 value);
diff --git a/drivers/block/blk-snap/blk-snap-ctl.h b/drivers/block/blk-snap/blk-snap-ctl.h
new file mode 100644
index 000000000000..4ffd836836b1
--- /dev/null
+++ b/drivers/block/blk-snap/blk-snap-ctl.h
@@ -0,0 +1,190 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+#pragma once
+
+#define MODULE_NAME "blk-snap"
+#define SNAP_IMAGE_NAME "blk-snap-image"
+
+#define SUCCESS 0
+
+#define MAX_TRACKING_DEVICE_COUNT 256
+
+#define BLK_SNAP 'V'
+
+#pragma pack(push, 1)
+//////////////////////////////////////////////////////////////////////////
+// version
+
+#define BLK_SNAP_COMPATIBILITY_SNAPSTORE 0x0000000000000001ull /* rudiment */
+//#define BLK_SNAP_COMPATIBILITY_BTRFS 0x0000000000000002ull /* rudiment */
+#define BLK_SNAP_COMPATIBILITY_MULTIDEV 0x0000000000000004ull
+
+struct ioctl_compatibility_flags_s {
+ unsigned long long flags;
+};
+#define IOCTL_COMPATIBILITY_FLAGS _IOW(BLK_SNAP, 0, struct ioctl_compatibility_flags_s)
+
+struct ioctl_getversion_s {
+ unsigned short major;
+ unsigned short minor;
+ unsigned short revision;
+ unsigned short build;
+};
+#define IOCTL_GETVERSION _IOW(BLK_SNAP, 1, struct ioctl_getversion_s)
+
+//////////////////////////////////////////////////////////////////////////
+// tracking
+struct ioctl_dev_id_s {
+ int major;
+ int minor;
+};
+#define IOCTL_TRACKING_ADD _IOW(BLK_SNAP, 2, struct ioctl_dev_id_s)
+
+#define IOCTL_TRACKING_REMOVE _IOW(BLK_SNAP, 3, struct ioctl_dev_id_s)
+
+struct cbt_info_s {
+ struct ioctl_dev_id_s dev_id;
+ unsigned long long dev_capacity;
+ unsigned int cbt_map_size;
+ unsigned char snap_number;
+ unsigned char generationId[16];
+};
+struct ioctl_tracking_collect_s {
+ unsigned int count;
+ union {
+ struct cbt_info_s *p_cbt_info;
+ unsigned long long ull_cbt_info;
+ };
+};
+#define IOCTL_TRACKING_COLLECT _IOW(BLK_SNAP, 4, struct ioctl_tracking_collect_s)
+
+#define IOCTL_TRACKING_BLOCK_SIZE _IOW(BLK_SNAP, 5, unsigned int)
+
+struct ioctl_tracking_read_cbt_bitmap_s {
+ struct ioctl_dev_id_s dev_id;
+ unsigned int offset;
+ unsigned int length;
+ union {
+ unsigned char *buff;
+ unsigned long long ull_buff;
+ };
+};
+#define IOCTL_TRACKING_READ_CBT_BITMAP _IOR(BLK_SNAP, 6, struct ioctl_tracking_read_cbt_bitmap_s)
+
+struct block_range_s {
+ unsigned long long ofs; //sectors
+ unsigned long long cnt; //sectors
+};
+
+struct ioctl_tracking_mark_dirty_blocks_s {
+ struct ioctl_dev_id_s image_dev_id;
+ unsigned int count;
+ union {
+ struct block_range_s *p_dirty_blocks;
+ unsigned long long ull_dirty_blocks;
+ };
+};
+#define IOCTL_TRACKING_MARK_DIRTY_BLOCKS \
+ _IOR(BLK_SNAP, 7, struct ioctl_tracking_mark_dirty_blocks_s)
+//////////////////////////////////////////////////////////////////////////
+// snapshot
+
+struct ioctl_snapshot_create_s {
+ unsigned long long snapshot_id;
+ unsigned int count;
+ union {
+ struct ioctl_dev_id_s *p_dev_id;
+ unsigned long long ull_dev_id;
+ };
+};
+#define IOCTL_SNAPSHOT_CREATE _IOW(BLK_SNAP, 0x10, struct ioctl_snapshot_create_s)
+
+#define IOCTL_SNAPSHOT_DESTROY _IOR(BLK_SNAP, 0x11, unsigned long long)
+
+struct ioctl_snapshot_errno_s {
+ struct ioctl_dev_id_s dev_id;
+ int err_code;
+};
+#define IOCTL_SNAPSHOT_ERRNO _IOW(BLK_SNAP, 0x12, struct ioctl_snapshot_errno_s)
+
+struct ioctl_range_s {
+ unsigned long long left;
+ unsigned long long right;
+};
+
+//////////////////////////////////////////////////////////////////////////
+// snapstore
+struct ioctl_snapstore_create_s {
+ unsigned char id[16];
+ struct ioctl_dev_id_s snapstore_dev_id;
+ unsigned int count;
+ union {
+ struct ioctl_dev_id_s *p_dev_id;
+ unsigned long long ull_dev_id;
+ };
+};
+#define IOCTL_SNAPSTORE_CREATE _IOR(BLK_SNAP, 0x28, struct ioctl_snapstore_create_s)
+
+struct ioctl_snapstore_file_add_s {
+ unsigned char id[16];
+ unsigned int range_count;
+ union {
+ struct ioctl_range_s *ranges;
+ unsigned long long ull_ranges;
+ };
+};
+#define IOCTL_SNAPSTORE_FILE _IOR(BLK_SNAP, 0x29, struct ioctl_snapstore_file_add_s)
+
+struct ioctl_snapstore_memory_limit_s {
+ unsigned char id[16];
+ unsigned long long size;
+};
+#define IOCTL_SNAPSTORE_MEMORY _IOR(BLK_SNAP, 0x2A, struct ioctl_snapstore_memory_limit_s)
+
+struct ioctl_snapstore_cleanup_s {
+ unsigned char id[16];
+ unsigned long long filled_bytes;
+};
+#define IOCTL_SNAPSTORE_CLEANUP _IOW(BLK_SNAP, 0x2B, struct ioctl_snapstore_cleanup_s)
+
+struct ioctl_snapstore_file_add_multidev_s {
+ unsigned char id[16];
+ struct ioctl_dev_id_s dev_id;
+ unsigned int range_count;
+ union {
+ struct ioctl_range_s *ranges;
+ unsigned long long ull_ranges;
+ };
+};
+#define IOCTL_SNAPSTORE_FILE_MULTIDEV \
+ _IOR(BLK_SNAP, 0x2C, struct ioctl_snapstore_file_add_multidev_s)
+//////////////////////////////////////////////////////////////////////////
+// collect snapshot images
+
+struct image_info_s {
+ struct ioctl_dev_id_s original_dev_id;
+ struct ioctl_dev_id_s snapshot_dev_id;
+};
+
+struct ioctl_collect_snapshot_images_s {
+ int count; //
+ union {
+ struct image_info_s *p_image_info;
+ unsigned long long ull_image_info;
+ };
+};
+#define IOCTL_COLLECT_SNAPSHOT_IMAGES _IOW(BLK_SNAP, 0x30, struct ioctl_collect_snapshot_images_s)
+
+#pragma pack(pop)
+
+// commands for character device interface
+#define BLK_SNAP_CHARCMD_UNDEFINED 0x00
+#define BLK_SNAP_CHARCMD_ACKNOWLEDGE 0x01
+#define BLK_SNAP_CHARCMD_INVALID 0xFF
+// to module commands
+#define BLK_SNAP_CHARCMD_INITIATE 0x21
+#define BLK_SNAP_CHARCMD_NEXT_PORTION 0x22
+#define BLK_SNAP_CHARCMD_NEXT_PORTION_MULTIDEV 0x23
+// from module commands
+#define BLK_SNAP_CHARCMD_HALFFILL 0x41
+#define BLK_SNAP_CHARCMD_OVERFLOW 0x42
+#define BLK_SNAP_CHARCMD_TERMINATE 0x43
diff --git a/drivers/block/blk-snap/blk_deferred.c b/drivers/block/blk-snap/blk_deferred.c
new file mode 100644
index 000000000000..1d0b7d2c4d71
--- /dev/null
+++ b/drivers/block/blk-snap/blk_deferred.c
@@ -0,0 +1,566 @@
+// SPDX-License-Identifier: GPL-2.0
+#define BLK_SNAP_SECTION "-deferred"
+#include "common.h"
+
+#include "blk_deferred.h"
+#include "blk_util.h"
+#include "snapstore.h"
+#include "params.h"
+
+struct bio_set blk_deferred_bioset = { 0 };
+
+struct dio_bio_complete {
+ struct blk_deferred_request *dio_req;
+ sector_t bio_sect_len;
+};
+
+struct dio_deadlocked_list {
+ struct list_head link;
+
+ struct blk_deferred_request *dio_req;
+};
+
+LIST_HEAD(dio_deadlocked_list);
+DEFINE_RWLOCK(dio_deadlocked_list_lock);
+
+atomic64_t dio_alloc_count = ATOMIC64_INIT(0);
+atomic64_t dio_free_count = ATOMIC64_INIT(0);
+
+void blk_deferred_done(void)
+{
+ struct dio_deadlocked_list *dio_locked;
+
+ do {
+ dio_locked = NULL;
+
+ write_lock(&dio_deadlocked_list_lock);
+ if (!list_empty(&dio_deadlocked_list)) {
+ dio_locked = list_entry(dio_deadlocked_list.next,
+ struct dio_deadlocked_list, link);
+
+ list_del(&dio_locked->link);
+ }
+ write_unlock(&dio_deadlocked_list_lock);
+
+ if (dio_locked) {
+ if (dio_locked->dio_req->sect_len ==
+ atomic64_read(&dio_locked->dio_req->sect_processed))
+ blk_deferred_request_free(dio_locked->dio_req);
+ else
+ pr_err("Locked defer IO is still in memory\n");
+
+ kfree(dio_locked);
+ }
+ } while (dio_locked);
+}
+
+void blk_deferred_request_deadlocked(struct blk_deferred_request *dio_req)
+{
+ struct dio_deadlocked_list *dio_locked =
+ kzalloc(sizeof(struct dio_deadlocked_list), GFP_KERNEL);
+
+ dio_locked->dio_req = dio_req;
+
+ write_lock(&dio_deadlocked_list_lock);
+ list_add_tail(&dio_locked->link, &dio_deadlocked_list);
+ write_unlock(&dio_deadlocked_list_lock);
+
+ pr_warn("Deadlock with defer IO\n");
+}
+
+void blk_deferred_free(struct blk_deferred_io *dio)
+{
+ size_t inx = 0;
+
+ if (dio->page_array != NULL) {
+ while (dio->page_array[inx] != NULL) {
+ __free_page(dio->page_array[inx]);
+ dio->page_array[inx] = NULL;
+
+ ++inx;
+ }
+
+ kfree(dio->page_array);
+ dio->page_array = NULL;
+ }
+ kfree(dio);
+}
+
+struct blk_deferred_io *blk_deferred_alloc(unsigned long block_index,
+ union blk_descr_unify blk_descr)
+{
+ size_t inx;
+ size_t page_count;
+ struct blk_deferred_io *dio = kmalloc(sizeof(struct blk_deferred_io), GFP_NOIO);
+
+ if (dio == NULL)
+ return NULL;
+
+ INIT_LIST_HEAD(&dio->link);
+
+ dio->blk_descr = blk_descr;
+ dio->blk_index = block_index;
+
+ dio->sect.ofs = block_index << snapstore_block_shift();
+ dio->sect.cnt = snapstore_block_size();
+
+ page_count = snapstore_block_size() / (PAGE_SIZE / SECTOR_SIZE);
+ /*
+ * empty pointer on the end
+ */
+ dio->page_array = kzalloc((page_count + 1) * sizeof(struct page *), GFP_NOIO);
+ if (dio->page_array == NULL) {
+ blk_deferred_free(dio);
+ return NULL;
+ }
+
+ for (inx = 0; inx < page_count; inx++) {
+ dio->page_array[inx] = alloc_page(GFP_NOIO);
+ if (dio->page_array[inx] == NULL) {
+ pr_err("Failed to allocate page\n");
+ blk_deferred_free(dio);
+ return NULL;
+ }
+ }
+
+ return dio;
+}
+
+int blk_deferred_bioset_create(void)
+{
+ return bioset_init(&blk_deferred_bioset, 64, sizeof(struct dio_bio_complete),
+ BIOSET_NEED_BVECS | BIOSET_NEED_RESCUER);
+}
+
+void blk_deferred_bioset_free(void)
+{
+ bioset_exit(&blk_deferred_bioset);
+}
+
+struct bio *_blk_deferred_bio_alloc(int nr_iovecs)
+{
+ struct bio *new_bio = bio_alloc_bioset(GFP_NOIO, nr_iovecs, &blk_deferred_bioset);
+
+ if (new_bio == NULL)
+ return NULL;
+
+ new_bio->bi_end_io = blk_deferred_bio_endio;
+ new_bio->bi_private = ((void *)new_bio) - sizeof(struct dio_bio_complete);
+
+ return new_bio;
+}
+
+static void blk_deferred_complete(struct blk_deferred_request *dio_req, sector_t portion_sect_cnt,
+ int result)
+{
+ atomic64_add(portion_sect_cnt, &dio_req->sect_processed);
+
+ if (dio_req->sect_len == atomic64_read(&dio_req->sect_processed))
+ complete(&dio_req->complete);
+
+ if (result != SUCCESS) {
+ dio_req->result = result;
+ pr_err("Failed to process defer IO request. errno=%d\n", result);
+ }
+}
+
+void blk_deferred_bio_endio(struct bio *bio)
+{
+ int local_err;
+ struct dio_bio_complete *complete_param = (struct dio_bio_complete *)bio->bi_private;
+
+ if (complete_param == NULL) {
+ //bio already complete
+ } else {
+ if (bio->bi_status != BLK_STS_OK)
+ local_err = -EIO;
+ else
+ local_err = SUCCESS;
+
+ blk_deferred_complete(complete_param->dio_req, complete_param->bio_sect_len,
+ local_err);
+ bio->bi_private = NULL;
+ }
+
+ bio_put(bio);
+}
+
+static inline size_t _page_count_calculate(sector_t size_sector)
+{
+ size_t page_count = size_sector / (PAGE_SIZE / SECTOR_SIZE);
+
+ if (unlikely(size_sector & ((PAGE_SIZE / SECTOR_SIZE) - 1)))
+ page_count += 1;
+
+ return page_count;
+}
+
+sector_t _blk_deferred_submit_pages(struct block_device *blk_dev,
+ struct blk_deferred_request *dio_req, int direction,
+ sector_t arr_ofs, struct page **page_array, sector_t ofs_sector,
+ sector_t size_sector)
+{
+ struct bio *bio = NULL;
+ int nr_iovecs;
+ int page_inx = arr_ofs >> (PAGE_SHIFT - SECTOR_SHIFT);
+ sector_t process_sect = 0;
+
+ nr_iovecs = _page_count_calculate(size_sector);
+
+ while (NULL == (bio = _blk_deferred_bio_alloc(nr_iovecs))) {
+ size_sector = (size_sector >> 1) & ~((PAGE_SIZE / SECTOR_SIZE) - 1);
+ if (size_sector == 0)
+ return 0;
+
+ nr_iovecs = _page_count_calculate(size_sector);
+ }
+
+ bio_set_dev(bio, blk_dev);
+
+ if (direction == READ)
+ bio_set_op_attrs(bio, REQ_OP_READ, 0);
+ else
+ bio_set_op_attrs(bio, REQ_OP_WRITE, 0);
+
+ bio->bi_iter.bi_sector = ofs_sector;
+
+ { //add first
+ sector_t unordered = arr_ofs & ((PAGE_SIZE / SECTOR_SIZE) - 1);
+ sector_t bvec_len_sect =
+ min_t(sector_t, ((PAGE_SIZE / SECTOR_SIZE) - unordered), size_sector);
+ struct page *page = page_array[page_inx];
+ unsigned int len = (unsigned int)from_sectors(bvec_len_sect);
+ unsigned int offset = (unsigned int)from_sectors(unordered);
+
+ if (unlikely(page == NULL)) {
+ pr_err("NULL found in page array");
+ bio_put(bio);
+ return 0;
+ }
+ if (unlikely(bio_add_page(bio, page, len, offset) != len)) {
+ bio_put(bio);
+ return 0;
+ }
+ ++page_inx;
+ process_sect += bvec_len_sect;
+ }
+
+ while (process_sect < size_sector) {
+ sector_t bvec_len_sect =
+ min_t(sector_t, (PAGE_SIZE / SECTOR_SIZE), (size_sector - process_sect));
+ struct page *page = page_array[page_inx];
+ unsigned int len = (unsigned int)from_sectors(bvec_len_sect);
+
+
+ if (unlikely(page == NULL)) {
+ pr_err("NULL found in page array");
+ break;
+ }
+ if (unlikely(bio_add_page(bio, page, len, 0) != len))
+ break;
+
+ ++page_inx;
+ process_sect += bvec_len_sect;
+ }
+
+ ((struct dio_bio_complete *)bio->bi_private)->dio_req = dio_req;
+ ((struct dio_bio_complete *)bio->bi_private)->bio_sect_len = process_sect;
+
+ submit_bio_direct(bio);
+
+ return process_sect;
+}
+
+sector_t blk_deferred_submit_pages(struct block_device *blk_dev,
+ struct blk_deferred_request *dio_req, int direction,
+ sector_t arr_ofs, struct page **page_array, sector_t ofs_sector,
+ sector_t size_sector)
+{
+ sector_t process_sect = 0;
+
+ do {
+ sector_t portion_sect = _blk_deferred_submit_pages(
+ blk_dev, dio_req, direction, arr_ofs + process_sect, page_array,
+ ofs_sector + process_sect, size_sector - process_sect);
+ if (portion_sect == 0) {
+ pr_err("Failed to submit defer IO pages. Only [%lld] sectors processed\n",
+ process_sect);
+ break;
+ }
+ process_sect += portion_sect;
+ } while (process_sect < size_sector);
+
+ return process_sect;
+}
+
+struct blk_deferred_request *blk_deferred_request_new(void)
+{
+ struct blk_deferred_request *dio_req = NULL;
+
+ dio_req = kzalloc(sizeof(struct blk_deferred_request), GFP_NOIO);
+ if (dio_req == NULL)
+ return NULL;
+
+ INIT_LIST_HEAD(&dio_req->dios);
+
+ dio_req->result = SUCCESS;
+ atomic64_set(&dio_req->sect_processed, 0);
+ dio_req->sect_len = 0;
+ init_completion(&dio_req->complete);
+
+ return dio_req;
+}
+
+bool blk_deferred_request_already_added(struct blk_deferred_request *dio_req,
+ unsigned long block_index)
+{
+ bool result = false;
+ struct list_head *_list_head;
+
+ if (list_empty(&dio_req->dios))
+ return result;
+
+ list_for_each(_list_head, &dio_req->dios) {
+ struct blk_deferred_io *dio = list_entry(_list_head, struct blk_deferred_io, link);
+
+ if (dio->blk_index == block_index) {
+ result = true;
+ break;
+ }
+ }
+
+ return result;
+}
+
+int blk_deferred_request_add(struct blk_deferred_request *dio_req, struct blk_deferred_io *dio)
+{
+ list_add_tail(&dio->link, &dio_req->dios);
+ dio_req->sect_len += dio->sect.cnt;
+
+ return SUCCESS;
+}
+
+void blk_deferred_request_free(struct blk_deferred_request *dio_req)
+{
+ if (dio_req != NULL) {
+ while (!list_empty(&dio_req->dios)) {
+ struct blk_deferred_io *dio =
+ list_entry(dio_req->dios.next, struct blk_deferred_io, link);
+
+ list_del(&dio->link);
+
+ blk_deferred_free(dio);
+ }
+ kfree(dio_req);
+ }
+}
+
+void blk_deferred_request_waiting_skip(struct blk_deferred_request *dio_req)
+{
+ init_completion(&dio_req->complete);
+ atomic64_set(&dio_req->sect_processed, 0);
+}
+
+int blk_deferred_request_wait(struct blk_deferred_request *dio_req)
+{
+ u64 start_jiffies = get_jiffies_64();
+ u64 current_jiffies;
+
+ while (wait_for_completion_timeout(&dio_req->complete, (HZ * 1)) == 0) {
+ current_jiffies = get_jiffies_64();
+ if (jiffies_to_msecs(current_jiffies - start_jiffies) > 60 * 1000) {
+ pr_warn("Defer IO request timeout\n");
+ return -EDEADLK;
+ }
+ }
+
+ return dio_req->result;
+}
+
+int blk_deferred_request_read_original(struct block_device *original_blk_dev,
+ struct blk_deferred_request *dio_copy_req)
+{
+ int res = -ENODATA;
+ struct list_head *_list_head;
+
+ blk_deferred_request_waiting_skip(dio_copy_req);
+
+ if (list_empty(&dio_copy_req->dios))
+ return res;
+
+ list_for_each(_list_head, &dio_copy_req->dios) {
+ struct blk_deferred_io *dio = list_entry(_list_head, struct blk_deferred_io, link);
+
+ sector_t ofs = dio->sect.ofs;
+ sector_t cnt = dio->sect.cnt;
+
+ if (cnt != blk_deferred_submit_pages(original_blk_dev, dio_copy_req, READ, 0,
+ dio->page_array, ofs, cnt)) {
+ pr_err("Failed to submit reading defer IO request. offset=%lld\n",
+ dio->sect.ofs);
+ res = -EIO;
+ break;
+ }
+ res = SUCCESS;
+ }
+
+ if (res == SUCCESS)
+ res = blk_deferred_request_wait(dio_copy_req);
+
+ return res;
+}
+
+
+static int _store_file(struct block_device *blk_dev, struct blk_deferred_request *dio_copy_req,
+ struct blk_descr_file *blk_descr, struct page **page_array)
+{
+ struct list_head *_rangelist_head;
+ sector_t page_array_ofs = 0;
+
+ if (unlikely(list_empty(&blk_descr->rangelist))) {
+ pr_err("Invalid block descriptor");
+ return -EINVAL;
+ }
+ list_for_each(_rangelist_head, &blk_descr->rangelist) {
+ struct blk_range_link *range_link;
+ sector_t process_sect;
+
+ range_link = list_entry(_rangelist_head, struct blk_range_link, link);
+ process_sect = blk_deferred_submit_pages(blk_dev, dio_copy_req, WRITE,
+ page_array_ofs, page_array,
+ range_link->rg.ofs, range_link->rg.cnt);
+ if (range_link->rg.cnt != process_sect) {
+ pr_err("Failed to submit defer IO request for storing\n");
+ return -EIO;
+ }
+ page_array_ofs += range_link->rg.cnt;
+ }
+ return SUCCESS;
+}
+
+int blk_deferred_request_store_file(struct block_device *blk_dev,
+ struct blk_deferred_request *dio_copy_req)
+{
+ struct list_head *_dio_list_head;
+
+ blk_deferred_request_waiting_skip(dio_copy_req);
+
+ if (unlikely(list_empty(&dio_copy_req->dios))) {
+ pr_err("Invalid deferred io request");
+ return -EINVAL;
+ }
+ list_for_each(_dio_list_head, &dio_copy_req->dios) {
+ int res;
+ struct blk_deferred_io *dio;
+
+ dio = list_entry(_dio_list_head, struct blk_deferred_io, link);
+ res = _store_file(blk_dev, dio_copy_req, dio->blk_descr.file, dio->page_array);
+ if (res != SUCCESS)
+ return res;
+ }
+
+ return blk_deferred_request_wait(dio_copy_req);
+}
+
+#ifdef CONFIG_BLK_SNAP_SNAPSTORE_MULTIDEV
+
+static int _store_multidev(struct blk_deferred_request *dio_copy_req,
+ struct blk_descr_multidev *blk_descr, struct page **page_array)
+{
+ struct list_head *_ranges_list_head;
+ sector_t page_array_ofs = 0;
+
+ if (unlikely(list_empty(&blk_descr->rangelist))) {
+ pr_err("Invalid block descriptor");
+ return -EINVAL;
+ }
+ list_for_each(_ranges_list_head, &blk_descr->rangelist) {
+ sector_t process_sect;
+ struct blk_range_link_ex *range_link;
+
+ range_link = list_entry(_ranges_list_head, struct blk_range_link_ex, link);
+ process_sect = blk_deferred_submit_pages(range_link->blk_dev, dio_copy_req, WRITE,
+ page_array_ofs, page_array,
+ range_link->rg.ofs, range_link->rg.cnt);
+ if (range_link->rg.cnt != process_sect) {
+ pr_err("Failed to submit defer IO request for storing\n");
+ return -EIO;
+ }
+
+ page_array_ofs += range_link->rg.cnt;
+ }
+
+ return SUCCESS;
+}
+
+int blk_deferred_request_store_multidev(struct blk_deferred_request *dio_copy_req)
+{
+ struct list_head *_dio_list_head;
+
+ blk_deferred_request_waiting_skip(dio_copy_req);
+
+ if (unlikely(list_empty(&dio_copy_req->dios))) {
+ pr_err("Invalid deferred io request");
+ return -EINVAL;
+ }
+ list_for_each(_dio_list_head, &dio_copy_req->dios) {
+ int res;
+ struct blk_deferred_io *dio;
+
+ dio = list_entry(_dio_list_head, struct blk_deferred_io, link);
+ res = _store_multidev(dio_copy_req, dio->blk_descr.multidev, dio->page_array);
+ if (res != SUCCESS)
+ return res;
+ }
+
+ return blk_deferred_request_wait(dio_copy_req);
+}
+#endif
+
+static size_t _store_pages(void *dst, struct page **page_array, size_t length)
+{
+ size_t page_inx = 0;
+ size_t processed_len = 0;
+
+ while ((processed_len < length) && (page_array[page_inx] != NULL)) {
+ void *src;
+ size_t page_len = min_t(size_t, PAGE_SIZE, (length - processed_len));
+
+ src = page_address(page_array[page_inx]);
+ memcpy(dst + processed_len, src, page_len);
+
+ ++page_inx;
+ processed_len += page_len;
+ }
+
+ return processed_len;
+}
+
+int blk_deferred_request_store_mem(struct blk_deferred_request *dio_copy_req)
+{
+ int res = SUCCESS;
+ sector_t processed = 0;
+
+ if (!list_empty(&dio_copy_req->dios)) {
+ struct list_head *_list_head;
+
+ list_for_each(_list_head, &dio_copy_req->dios) {
+ size_t length;
+ size_t portion;
+ struct blk_deferred_io *dio;
+
+ dio = list_entry(_list_head, struct blk_deferred_io, link);
+ length = snapstore_block_size() * SECTOR_SIZE;
+
+ portion = _store_pages(dio->blk_descr.mem->buff, dio->page_array, length);
+ if (unlikely(portion != length)) {
+ res = -EIO;
+ break;
+ }
+ processed += (sector_t)to_sectors(portion);
+ }
+ }
+
+ blk_deferred_complete(dio_copy_req, processed, res);
+ return res;
+}
diff --git a/drivers/block/blk-snap/blk_deferred.h b/drivers/block/blk-snap/blk_deferred.h
new file mode 100644
index 000000000000..3c516a835c25
--- /dev/null
+++ b/drivers/block/blk-snap/blk_deferred.h
@@ -0,0 +1,67 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+#pragma once
+
+#include "blk_descr_file.h"
+#include "blk_descr_mem.h"
+#include "blk_descr_multidev.h"
+
+#define DEFER_IO_DIO_REQUEST_LENGTH 250
+#define DEFER_IO_DIO_REQUEST_SECTORS_COUNT (10 * 1024 * 1024 / SECTOR_SIZE)
+
+struct blk_deferred_io {
+ struct list_head link;
+
+ unsigned long blk_index;
+ union blk_descr_unify blk_descr;
+
+ struct blk_range sect;
+
+ struct page **page_array; //null pointer on tail
+};
+
+struct blk_deferred_request {
+ struct completion complete;
+ sector_t sect_len;
+ atomic64_t sect_processed;
+ int result;
+
+ struct list_head dios;
+};
+
+void blk_deferred_done(void);
+
+struct blk_deferred_io *blk_deferred_alloc(unsigned long block_index,
+ union blk_descr_unify blk_descr);
+void blk_deferred_free(struct blk_deferred_io *dio);
+
+void blk_deferred_bio_endio(struct bio *bio);
+
+sector_t blk_deferred_submit_pages(struct block_device *blk_dev,
+ struct blk_deferred_request *dio_req, int direction,
+ sector_t arr_ofs, struct page **page_array, sector_t ofs_sector,
+ sector_t size_sector);
+
+struct blk_deferred_request *blk_deferred_request_new(void);
+
+bool blk_deferred_request_already_added(struct blk_deferred_request *dio_req,
+ unsigned long block_index);
+
+int blk_deferred_request_add(struct blk_deferred_request *dio_req, struct blk_deferred_io *dio);
+void blk_deferred_request_free(struct blk_deferred_request *dio_req);
+void blk_deferred_request_deadlocked(struct blk_deferred_request *dio_req);
+
+void blk_deferred_request_waiting_skip(struct blk_deferred_request *dio_req);
+int blk_deferred_request_wait(struct blk_deferred_request *dio_req);
+
+int blk_deferred_bioset_create(void);
+void blk_deferred_bioset_free(void);
+
+int blk_deferred_request_read_original(struct block_device *original_blk_dev,
+ struct blk_deferred_request *dio_copy_req);
+
+int blk_deferred_request_store_file(struct block_device *blk_dev,
+ struct blk_deferred_request *dio_copy_req);
+#ifdef CONFIG_BLK_SNAP_SNAPSTORE_MULTIDEV
+int blk_deferred_request_store_multidev(struct blk_deferred_request *dio_copy_req);
+#endif
+int blk_deferred_request_store_mem(struct blk_deferred_request *dio_copy_req);
diff --git a/drivers/block/blk-snap/blk_descr_file.c b/drivers/block/blk-snap/blk_descr_file.c
new file mode 100644
index 000000000000..fca298d35744
--- /dev/null
+++ b/drivers/block/blk-snap/blk_descr_file.c
@@ -0,0 +1,82 @@
+// SPDX-License-Identifier: GPL-2.0
+#define BLK_SNAP_SECTION "-blk_descr"
+#include "common.h"
+
+#include "blk_descr_file.h"
+
+static inline void list_assign(struct list_head *dst, struct list_head *src)
+{
+ dst->next = src->next;
+ dst->prev = src->prev;
+
+ src->next->prev = dst;
+ src->prev->next = dst;
+}
+
+static inline void blk_descr_file_init(struct blk_descr_file *blk_descr,
+ struct list_head *rangelist)
+{
+ list_assign(&blk_descr->rangelist, rangelist);
+}
+
+static inline void blk_descr_file_done(struct blk_descr_file *blk_descr)
+{
+ struct blk_range_link *range_link;
+
+ while (!list_empty(&blk_descr->rangelist)) {
+ range_link = list_entry(blk_descr->rangelist.next, struct blk_range_link, link);
+
+ list_del(&range_link->link);
+ kfree(range_link);
+ }
+}
+
+void blk_descr_file_pool_init(struct blk_descr_pool *pool)
+{
+ blk_descr_pool_init(pool, 0);
+}
+
+void _blk_descr_file_cleanup(void *descr_array, size_t count)
+{
+ size_t inx;
+ struct blk_descr_file *file_blocks = descr_array;
+
+ for (inx = 0; inx < count; ++inx)
+ blk_descr_file_done(file_blocks + inx);
+}
+
+void blk_descr_file_pool_done(struct blk_descr_pool *pool)
+{
+ blk_descr_pool_done(pool, _blk_descr_file_cleanup);
+}
+
+static union blk_descr_unify _blk_descr_file_allocate(void *descr_array, size_t index, void *arg)
+{
+ union blk_descr_unify blk_descr;
+ struct blk_descr_file *file_blocks = descr_array;
+
+ blk_descr.file = &file_blocks[index];
+
+ blk_descr_file_init(blk_descr.file, (struct list_head *)arg);
+
+ return blk_descr;
+}
+
+int blk_descr_file_pool_add(struct blk_descr_pool *pool, struct list_head *rangelist)
+{
+ union blk_descr_unify blk_descr;
+
+ blk_descr = blk_descr_pool_alloc(pool, sizeof(struct blk_descr_file),
+ _blk_descr_file_allocate, (void *)rangelist);
+ if (blk_descr.ptr == NULL) {
+ pr_err("Failed to allocate block descriptor\n");
+ return -ENOMEM;
+ }
+
+ return SUCCESS;
+}
+
+union blk_descr_unify blk_descr_file_pool_take(struct blk_descr_pool *pool)
+{
+ return blk_descr_pool_take(pool, sizeof(struct blk_descr_file));
+}
diff --git a/drivers/block/blk-snap/blk_descr_file.h b/drivers/block/blk-snap/blk_descr_file.h
new file mode 100644
index 000000000000..0e9a5e3efdea
--- /dev/null
+++ b/drivers/block/blk-snap/blk_descr_file.h
@@ -0,0 +1,26 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+#pragma once
+
+#include "blk_descr_pool.h"
+
+struct blk_descr_file {
+ struct list_head rangelist;
+};
+
+struct blk_range_link {
+ struct list_head link;
+ struct blk_range rg;
+};
+
+void blk_descr_file_pool_init(struct blk_descr_pool *pool);
+void blk_descr_file_pool_done(struct blk_descr_pool *pool);
+
+/*
+ * allocate new empty block in pool
+ */
+int blk_descr_file_pool_add(struct blk_descr_pool *pool, struct list_head *rangelist);
+
+/*
+ * take empty block from pool
+ */
+union blk_descr_unify blk_descr_file_pool_take(struct blk_descr_pool *pool);
diff --git a/drivers/block/blk-snap/blk_descr_mem.c b/drivers/block/blk-snap/blk_descr_mem.c
new file mode 100644
index 000000000000..cd326ac150b6
--- /dev/null
+++ b/drivers/block/blk-snap/blk_descr_mem.c
@@ -0,0 +1,66 @@
+// SPDX-License-Identifier: GPL-2.0
+#define BLK_SNAP_SECTION "-blk_descr"
+#include "common.h"
+#include "blk_descr_mem.h"
+
+#define SECTION "blk_descr "
+
+static inline void blk_descr_mem_init(struct blk_descr_mem *blk_descr, void *ptr)
+{
+ blk_descr->buff = ptr;
+}
+
+static inline void blk_descr_mem_done(struct blk_descr_mem *blk_descr)
+{
+ blk_descr->buff = NULL;
+}
+
+void blk_descr_mem_pool_init(struct blk_descr_pool *pool, size_t available_blocks)
+{
+ blk_descr_pool_init(pool, available_blocks);
+}
+
+void blk_descr_mem_cleanup(void *descr_array, size_t count)
+{
+ size_t inx;
+ struct blk_descr_mem *mem_blocks = descr_array;
+
+ for (inx = 0; inx < count; ++inx)
+ blk_descr_mem_done(mem_blocks + inx);
+}
+
+void blk_descr_mem_pool_done(struct blk_descr_pool *pool)
+{
+ blk_descr_pool_done(pool, blk_descr_mem_cleanup);
+}
+
+static union blk_descr_unify blk_descr_mem_alloc(void *descr_array, size_t index, void *arg)
+{
+ union blk_descr_unify blk_descr;
+ struct blk_descr_mem *mem_blocks = descr_array;
+
+ blk_descr.mem = &mem_blocks[index];
+
+ blk_descr_mem_init(blk_descr.mem, (void *)arg);
+
+ return blk_descr;
+}
+
+int blk_descr_mem_pool_add(struct blk_descr_pool *pool, void *buffer)
+{
+ union blk_descr_unify blk_descr;
+
+ blk_descr = blk_descr_pool_alloc(pool, sizeof(struct blk_descr_mem),
+ blk_descr_mem_alloc, buffer);
+ if (blk_descr.ptr == NULL) {
+ pr_err("Failed to allocate block descriptor\n");
+ return -ENOMEM;
+ }
+
+ return SUCCESS;
+}
+
+union blk_descr_unify blk_descr_mem_pool_take(struct blk_descr_pool *pool)
+{
+ return blk_descr_pool_take(pool, sizeof(struct blk_descr_mem));
+}
diff --git a/drivers/block/blk-snap/blk_descr_mem.h b/drivers/block/blk-snap/blk_descr_mem.h
new file mode 100644
index 000000000000..43e8de76c07c
--- /dev/null
+++ b/drivers/block/blk-snap/blk_descr_mem.h
@@ -0,0 +1,14 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+#pragma once
+
+#include "blk_descr_pool.h"
+
+struct blk_descr_mem {
+ void *buff; //pointer to snapstore block in memory
+};
+
+void blk_descr_mem_pool_init(struct blk_descr_pool *pool, size_t available_blocks);
+void blk_descr_mem_pool_done(struct blk_descr_pool *pool);
+
+int blk_descr_mem_pool_add(struct blk_descr_pool *pool, void *buffer);
+union blk_descr_unify blk_descr_mem_pool_take(struct blk_descr_pool *pool);
diff --git a/drivers/block/blk-snap/blk_descr_multidev.c b/drivers/block/blk-snap/blk_descr_multidev.c
new file mode 100644
index 000000000000..cf5e0ed6f781
--- /dev/null
+++ b/drivers/block/blk-snap/blk_descr_multidev.c
@@ -0,0 +1,86 @@
+// SPDX-License-Identifier: GPL-2.0
+#define BLK_SNAP_SECTION "-blk_descr"
+#include "common.h"
+
+#ifdef CONFIG_BLK_SNAP_SNAPSTORE_MULTIDEV
+#include "blk_descr_multidev.h"
+
+static inline void list_assign(struct list_head *dst, struct list_head *src)
+{
+ dst->next = src->next;
+ dst->prev = src->prev;
+
+ src->next->prev = dst;
+ src->prev->next = dst;
+}
+
+static inline void blk_descr_multidev_init(struct blk_descr_multidev *blk_descr,
+ struct list_head *rangelist)
+{
+ list_assign(&blk_descr->rangelist, rangelist);
+}
+
+static inline void blk_descr_multidev_done(struct blk_descr_multidev *blk_descr)
+{
+ struct blk_range_link_ex *rangelist;
+
+ while (!list_empty(&blk_descr->rangelist)) {
+ rangelist = list_entry(blk_descr->rangelist.next,
+ struct blk_range_link_ex, link);
+
+ list_del(&rangelist->link);
+ kfree(rangelist);
+ }
+}
+
+void blk_descr_multidev_pool_init(struct blk_descr_pool *pool)
+{
+ blk_descr_pool_init(pool, 0);
+}
+
+static void blk_descr_multidev_cleanup(void *descr_array, size_t count)
+{
+ size_t inx;
+ struct blk_descr_multidev *descr_multidev = descr_array;
+
+ for (inx = 0; inx < count; ++inx)
+ blk_descr_multidev_done(descr_multidev + inx);
+}
+
+void blk_descr_multidev_pool_done(struct blk_descr_pool *pool)
+{
+ blk_descr_pool_done(pool, blk_descr_multidev_cleanup);
+}
+
+static union blk_descr_unify blk_descr_multidev_allocate(void *descr_array, size_t index, void *arg)
+{
+ union blk_descr_unify blk_descr;
+ struct blk_descr_multidev *multidev_blocks = descr_array;
+
+ blk_descr.multidev = &multidev_blocks[index];
+
+ blk_descr_multidev_init(blk_descr.multidev, (struct list_head *)arg);
+
+ return blk_descr;
+}
+
+int blk_descr_multidev_pool_add(struct blk_descr_pool *pool, struct list_head *rangelist)
+{
+ union blk_descr_unify blk_descr;
+
+ blk_descr = blk_descr_pool_alloc(pool, sizeof(struct blk_descr_multidev),
+ blk_descr_multidev_allocate, (void *)rangelist);
+ if (blk_descr.ptr == NULL) {
+ pr_err("Failed to allocate block descriptor\n");
+ return -ENOMEM;
+ }
+
+ return SUCCESS;
+}
+
+union blk_descr_unify blk_descr_multidev_pool_take(struct blk_descr_pool *pool)
+{
+ return blk_descr_pool_take(pool, sizeof(struct blk_descr_multidev));
+}
+
+#endif //CONFIG_BLK_SNAP_SNAPSTORE_MULTIDEV
diff --git a/drivers/block/blk-snap/blk_descr_multidev.h b/drivers/block/blk-snap/blk_descr_multidev.h
new file mode 100644
index 000000000000..0145b0d78b10
--- /dev/null
+++ b/drivers/block/blk-snap/blk_descr_multidev.h
@@ -0,0 +1,25 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+#pragma once
+
+#ifdef CONFIG_BLK_SNAP_SNAPSTORE_MULTIDEV
+
+#include "blk_descr_pool.h"
+
+struct blk_descr_multidev {
+ struct list_head rangelist;
+};
+
+struct blk_range_link_ex {
+ struct list_head link;
+ struct blk_range rg;
+ struct block_device *blk_dev;
+};
+
+void blk_descr_multidev_pool_init(struct blk_descr_pool *pool);
+void blk_descr_multidev_pool_done(struct blk_descr_pool *pool);
+
+int blk_descr_multidev_pool_add(struct blk_descr_pool *pool,
+ struct list_head *rangelist); //allocate new empty block
+union blk_descr_unify blk_descr_multidev_pool_take(struct blk_descr_pool *pool); //take empty
+
+#endif //CONFIG_BLK_SNAP_SNAPSTORE_MULTIDEV
diff --git a/drivers/block/blk-snap/blk_descr_pool.c b/drivers/block/blk-snap/blk_descr_pool.c
new file mode 100644
index 000000000000..b1fe2ba9c2d0
--- /dev/null
+++ b/drivers/block/blk-snap/blk_descr_pool.c
@@ -0,0 +1,190 @@
+// SPDX-License-Identifier: GPL-2.0
+#define BLK_SNAP_SECTION "-blk_descr"
+#include "common.h"
+#include "blk_descr_pool.h"
+#include "params.h"
+
+struct pool_el {
+ struct list_head link;
+
+ size_t used_cnt; // used blocks
+ size_t capacity; // blocks array capacity
+
+ u8 descr_array[0];
+};
+
+static void *kmalloc_huge(size_t max_size, size_t min_size, gfp_t flags, size_t *p_allocated_size)
+{
+ void *ptr = NULL;
+
+ do {
+ ptr = kmalloc(max_size, flags | __GFP_NOWARN | __GFP_RETRY_MAYFAIL);
+
+ if (ptr != NULL) {
+ *p_allocated_size = max_size;
+ return ptr;
+ }
+ pr_err("Failed to allocate buffer size=%zu\n", max_size);
+ max_size = max_size >> 1;
+ } while (max_size >= min_size);
+
+ pr_err("Failed to allocate buffer.");
+ return NULL;
+}
+
+static struct pool_el *pool_el_alloc(size_t blk_descr_size)
+{
+ size_t el_size;
+ struct pool_el *el;
+
+ el = kmalloc_huge(8 * PAGE_SIZE, PAGE_SIZE, GFP_NOIO, &el_size);
+ if (el == NULL)
+ return NULL;
+
+ el->capacity = (el_size - sizeof(struct pool_el)) / blk_descr_size;
+ el->used_cnt = 0;
+
+ INIT_LIST_HEAD(&el->link);
+
+ return el;
+}
+
+static void _pool_el_free(struct pool_el *el)
+{
+ if (el != NULL)
+ kfree(el);
+}
+
+void blk_descr_pool_init(struct blk_descr_pool *pool, size_t available_blocks)
+{
+ mutex_init(&pool->lock);
+
+ INIT_LIST_HEAD(&pool->head);
+
+ pool->blocks_cnt = 0;
+
+ pool->total_cnt = available_blocks;
+ pool->take_cnt = 0;
+}
+
+void blk_descr_pool_done(struct blk_descr_pool *pool,
+ void (*blocks_cleanup_cb)(void *descr_array, size_t count))
+{
+ mutex_lock(&pool->lock);
+ while (!list_empty(&pool->head)) {
+ struct pool_el *el;
+
+ el = list_entry(pool->head.next, struct pool_el, link);
+ if (el == NULL)
+ break;
+
+ list_del(&el->link);
+ --pool->blocks_cnt;
+
+ pool->total_cnt -= el->used_cnt;
+
+ blocks_cleanup_cb(el->descr_array, el->used_cnt);
+
+ _pool_el_free(el);
+ }
+ mutex_unlock(&pool->lock);
+}
+
+union blk_descr_unify blk_descr_pool_alloc(
+ struct blk_descr_pool *pool, size_t blk_descr_size,
+ union blk_descr_unify (*block_alloc_cb)(void *descr_array, size_t index, void *arg),
+ void *arg)
+{
+ union blk_descr_unify blk_descr = { NULL };
+
+ mutex_lock(&pool->lock);
+ do {
+ struct pool_el *el = NULL;
+
+ if (!list_empty(&pool->head)) {
+ el = list_entry(pool->head.prev, struct pool_el, link);
+ if (el->used_cnt == el->capacity)
+ el = NULL;
+ }
+
+ if (el == NULL) {
+ el = pool_el_alloc(blk_descr_size);
+ if (el == NULL)
+ break;
+
+ list_add_tail(&el->link, &pool->head);
+
+ ++pool->blocks_cnt;
+ }
+
+ blk_descr = block_alloc_cb(el->descr_array, el->used_cnt, arg);
+
+ ++el->used_cnt;
+ ++pool->total_cnt;
+
+ } while (false);
+ mutex_unlock(&pool->lock);
+
+ return blk_descr;
+}
+
+static union blk_descr_unify __blk_descr_pool_at(struct blk_descr_pool *pool, size_t blk_descr_size,
+ size_t index)
+{
+ union blk_descr_unify bkl_descr = { NULL };
+ size_t curr_inx = 0;
+ struct pool_el *el;
+ struct list_head *_list_head;
+
+ if (list_empty(&(pool)->head))
+ return bkl_descr;
+
+ list_for_each(_list_head, &(pool)->head) {
+ el = list_entry(_list_head, struct pool_el, link);
+
+ if ((index >= curr_inx) && (index < (curr_inx + el->used_cnt))) {
+ bkl_descr.ptr = el->descr_array + (index - curr_inx) * blk_descr_size;
+ break;
+ }
+ curr_inx += el->used_cnt;
+ }
+
+ return bkl_descr;
+}
+
+union blk_descr_unify blk_descr_pool_take(struct blk_descr_pool *pool, size_t blk_descr_size)
+{
+ union blk_descr_unify result = { NULL };
+
+ mutex_lock(&pool->lock);
+ do {
+ if (pool->take_cnt >= pool->total_cnt) {
+ pr_err("Unable to get block descriptor: ");
+ pr_err("not enough descriptors, already took %zu, total %zu\n",
+ pool->take_cnt, pool->total_cnt);
+ break;
+ }
+
+ result = __blk_descr_pool_at(pool, blk_descr_size, pool->take_cnt);
+ if (result.ptr == NULL) {
+ pr_err("Unable to get block descriptor: ");
+ pr_err("not enough descriptors, already took %zu, total %zu\n",
+ pool->take_cnt, pool->total_cnt);
+ break;
+ }
+
+ ++pool->take_cnt;
+ } while (false);
+ mutex_unlock(&pool->lock);
+ return result;
+}
+
+bool blk_descr_pool_check_halffill(struct blk_descr_pool *pool, sector_t empty_limit,
+ sector_t *fill_status)
+{
+ size_t empty_blocks = (pool->total_cnt - pool->take_cnt);
+
+ *fill_status = (sector_t)(pool->take_cnt) << snapstore_block_shift();
+
+ return (empty_blocks < (size_t)(empty_limit >> snapstore_block_shift()));
+}
diff --git a/drivers/block/blk-snap/blk_descr_pool.h b/drivers/block/blk-snap/blk_descr_pool.h
new file mode 100644
index 000000000000..32f8b8c4103e
--- /dev/null
+++ b/drivers/block/blk-snap/blk_descr_pool.h
@@ -0,0 +1,38 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+#pragma once
+
+struct blk_descr_mem;
+struct blk_descr_file;
+struct blk_descr_multidev;
+
+union blk_descr_unify {
+ void *ptr;
+ struct blk_descr_mem *mem;
+ struct blk_descr_file *file;
+ struct blk_descr_multidev *multidev;
+};
+
+struct blk_descr_pool {
+ struct list_head head;
+ struct mutex lock;
+
+ size_t blocks_cnt; // count of struct pool_el
+
+ size_t total_cnt; // total count of block descriptors
+ size_t take_cnt; // take count of block descriptors
+};
+
+void blk_descr_pool_init(struct blk_descr_pool *pool, size_t available_blocks);
+
+void blk_descr_pool_done(struct blk_descr_pool *pool,
+ void (*blocks_cleanup_cb)(void *descr_array, size_t count));
+
+union blk_descr_unify blk_descr_pool_alloc(
+ struct blk_descr_pool *pool, size_t blk_descr_size,
+ union blk_descr_unify (*block_alloc_cb)(void *descr_array, size_t index, void *arg),
+ void *arg);
+
+union blk_descr_unify blk_descr_pool_take(struct blk_descr_pool *pool, size_t blk_descr_size);
+
+bool blk_descr_pool_check_halffill(struct blk_descr_pool *pool, sector_t empty_limit,
+ sector_t *fill_status);
diff --git a/drivers/block/blk-snap/blk_redirect.c b/drivers/block/blk-snap/blk_redirect.c
new file mode 100644
index 000000000000..4c28a8cb4275
--- /dev/null
+++ b/drivers/block/blk-snap/blk_redirect.c
@@ -0,0 +1,507 @@
+// SPDX-License-Identifier: GPL-2.0
+#define BLK_SNAP_SECTION "-redirect"
+#include "common.h"
+#include "blk_util.h"
+#include "blk_redirect.h"
+
+#define bio_vec_sectors(bv) (bv.bv_len >> SECTOR_SHIFT)
+
+struct bio_set blk_redirect_bioset = { 0 };
+
+int blk_redirect_bioset_create(void)
+{
+ return bioset_init(&blk_redirect_bioset, 64, 0, BIOSET_NEED_BVECS | BIOSET_NEED_RESCUER);
+}
+
+void blk_redirect_bioset_free(void)
+{
+ bioset_exit(&blk_redirect_bioset);
+}
+
+void blk_redirect_bio_endio(struct bio *bb)
+{
+ struct blk_redirect_bio *rq_redir = (struct blk_redirect_bio *)bb->bi_private;
+
+ if (rq_redir != NULL) {
+ int err = SUCCESS;
+
+ if (bb->bi_status != BLK_STS_OK)
+ err = -EIO;
+
+ if (err != SUCCESS) {
+ pr_err("Failed to process redirect IO request. errno=%d\n", 0 - err);
+
+ if (rq_redir->err == SUCCESS)
+ rq_redir->err = err;
+ }
+
+ if (atomic64_dec_and_test(&rq_redir->bio_count))
+ blk_redirect_complete(rq_redir, rq_redir->err);
+ }
+ bio_put(bb);
+}
+
+struct bio *_blk_dev_redirect_bio_alloc(int nr_iovecs, void *bi_private)
+{
+ struct bio *new_bio;
+
+ new_bio = bio_alloc_bioset(GFP_NOIO, nr_iovecs, &blk_redirect_bioset);
+ if (new_bio == NULL)
+ return NULL;
+
+ new_bio->bi_end_io = blk_redirect_bio_endio;
+ new_bio->bi_private = bi_private;
+
+ return new_bio;
+}
+
+struct blk_redirect_bio_list *_redirect_bio_allocate_list(struct bio *new_bio)
+{
+ struct blk_redirect_bio_list *next;
+
+ next = kzalloc(sizeof(struct blk_redirect_bio_list), GFP_NOIO);
+ if (next == NULL)
+ return NULL;
+
+ next->next = NULL;
+ next->this = new_bio;
+
+ return next;
+}
+
+int bio_endio_list_push(struct blk_redirect_bio *rq_redir, struct bio *new_bio)
+{
+ struct blk_redirect_bio_list *head;
+
+ if (rq_redir->bio_list_head == NULL) {
+ //list is empty, add first bio
+ rq_redir->bio_list_head = _redirect_bio_allocate_list(new_bio);
+ if (rq_redir->bio_list_head == NULL)
+ return -ENOMEM;
+ return SUCCESS;
+ }
+
+ // seek end of list
+ head = rq_redir->bio_list_head;
+ while (head->next != NULL)
+ head = head->next;
+
+ //append new bio to the end of list
+ head->next = _redirect_bio_allocate_list(new_bio);
+ if (head->next == NULL)
+ return -ENOMEM;
+
+ return SUCCESS;
+}
+
+void bio_endio_list_cleanup(struct blk_redirect_bio_list *curr)
+{
+ while (curr != NULL) {
+ struct blk_redirect_bio_list *next;
+
+ next = curr->next;
+ kfree(curr);
+ curr = next;
+ }
+}
+
+static unsigned int get_max_sect(struct block_device *blk_dev)
+{
+ struct request_queue *q = bdev_get_queue(blk_dev);
+
+ return min((unsigned int)(BIO_MAX_PAGES << (PAGE_SHIFT - SECTOR_SHIFT)),
+ q->limits.max_sectors);
+}
+
+static int _blk_dev_redirect_part_fast(struct blk_redirect_bio *rq_redir, int direction,
+ struct block_device *blk_dev, sector_t target_pos,
+ sector_t rq_ofs, sector_t rq_count)
+{
+ __label__ _fail_out;
+ __label__ _reprocess_bv;
+
+ int res = SUCCESS;
+
+ struct bio_vec bvec;
+ struct bvec_iter iter;
+
+ struct bio *new_bio = NULL;
+
+ sector_t sect_ofs = 0;
+ sector_t processed_sectors = 0;
+ int nr_iovecs;
+ struct blk_redirect_bio_list *bio_endio_rec;
+
+ nr_iovecs = get_max_sect(blk_dev) >> (PAGE_SHIFT - SECTOR_SHIFT);
+
+ bio_for_each_segment(bvec, rq_redir->bio, iter) {
+ sector_t bvec_ofs;
+ sector_t bvec_sectors;
+
+ unsigned int len;
+ unsigned int offset;
+
+ if ((sect_ofs + bio_vec_sectors(bvec)) <= rq_ofs) {
+ sect_ofs += bio_vec_sectors(bvec);
+ continue;
+ }
+ if (sect_ofs >= (rq_ofs + rq_count))
+ break;
+
+ bvec_ofs = 0;
+ if (sect_ofs < rq_ofs)
+ bvec_ofs = rq_ofs - sect_ofs;
+
+ bvec_sectors = bio_vec_sectors(bvec) - bvec_ofs;
+ if (bvec_sectors > (rq_count - processed_sectors))
+ bvec_sectors = rq_count - processed_sectors;
+
+ if (bvec_sectors == 0) {
+ res = -EIO;
+ goto _fail_out;
+ }
+
+_reprocess_bv:
+ if (new_bio == NULL) {
+ new_bio = _blk_dev_redirect_bio_alloc(nr_iovecs, rq_redir);
+ while (new_bio == NULL) {
+ pr_err("Unable to allocate new bio for redirect IO.\n");
+ res = -ENOMEM;
+ goto _fail_out;
+ }
+
+ bio_set_dev(new_bio, blk_dev);
+
+ if (direction == READ)
+ bio_set_op_attrs(new_bio, REQ_OP_READ, 0);
+
+ if (direction == WRITE)
+ bio_set_op_attrs(new_bio, REQ_OP_WRITE, 0);
+
+ new_bio->bi_iter.bi_sector = target_pos + processed_sectors;
+ }
+
+ len = (unsigned int)from_sectors(bvec_sectors);
+ offset = bvec.bv_offset + (unsigned int)from_sectors(bvec_ofs);
+ if (unlikely(bio_add_page(new_bio, bvec.bv_page, len, offset) != len)) {
+ if (bio_sectors(new_bio) == 0) {
+ res = -EIO;
+ goto _fail_out;
+ }
+
+ res = bio_endio_list_push(rq_redir, new_bio);
+ if (res != SUCCESS) {
+ pr_err("Failed to add bio into bio_endio_list\n");
+ goto _fail_out;
+ }
+
+ atomic64_inc(&rq_redir->bio_count);
+ new_bio = NULL;
+
+ goto _reprocess_bv;
+ }
+ processed_sectors += bvec_sectors;
+
+ sect_ofs += bio_vec_sectors(bvec);
+ }
+
+ if (new_bio != NULL) {
+ res = bio_endio_list_push(rq_redir, new_bio);
+ if (res != SUCCESS) {
+ pr_err("Failed to add bio into bio_endio_list\n");
+ goto _fail_out;
+ }
+
+ atomic64_inc(&rq_redir->bio_count);
+ new_bio = NULL;
+ }
+
+ return SUCCESS;
+
+_fail_out:
+ bio_endio_rec = rq_redir->bio_list_head;
+ while (bio_endio_rec != NULL) {
+ if (bio_endio_rec->this != NULL)
+ bio_put(bio_endio_rec->this);
+
+ bio_endio_rec = bio_endio_rec->next;
+ }
+
+ bio_endio_list_cleanup(bio_endio_rec);
+
+ pr_err("Failed to process part of redirect IO request. rq_ofs=%lld, rq_count=%lld\n",
+ rq_ofs, rq_count);
+ return res;
+}
+
+int blk_dev_redirect_part(struct blk_redirect_bio *rq_redir, int direction,
+ struct block_device *blk_dev, sector_t target_pos, sector_t rq_ofs,
+ sector_t rq_count)
+{
+ struct request_queue *q = bdev_get_queue(blk_dev);
+ sector_t logical_block_size_mask =
+ (sector_t)((q->limits.logical_block_size >> SECTOR_SHIFT) - 1);
+
+ if (likely(logical_block_size_mask == 0))
+ return _blk_dev_redirect_part_fast(rq_redir, direction, blk_dev, target_pos, rq_ofs,
+ rq_count);
+
+ if (likely((0 == (target_pos & logical_block_size_mask)) &&
+ (0 == (rq_count & logical_block_size_mask))))
+ return _blk_dev_redirect_part_fast(rq_redir, direction, blk_dev, target_pos, rq_ofs,
+ rq_count);
+
+ return -EFAULT;
+}
+
+void blk_dev_redirect_submit(struct blk_redirect_bio *rq_redir)
+{
+ struct blk_redirect_bio_list *head;
+ struct blk_redirect_bio_list *curr;
+
+ head = curr = rq_redir->bio_list_head;
+ rq_redir->bio_list_head = NULL;
+
+ while (curr != NULL) {
+ submit_bio_direct(curr->this);
+
+ curr = curr->next;
+ }
+
+ bio_endio_list_cleanup(head);
+}
+
+int blk_dev_redirect_memcpy_part(struct blk_redirect_bio *rq_redir, int direction, void *buff,
+ sector_t rq_ofs, sector_t rq_count)
+{
+ struct bio_vec bvec;
+ struct bvec_iter iter;
+
+ sector_t sect_ofs = 0;
+ sector_t processed_sectors = 0;
+
+ bio_for_each_segment(bvec, rq_redir->bio, iter) {
+ void *mem;
+ sector_t bvec_ofs;
+ sector_t bvec_sectors;
+
+ if ((sect_ofs + bio_vec_sectors(bvec)) <= rq_ofs) {
+ sect_ofs += bio_vec_sectors(bvec);
+ continue;
+ }
+
+ if (sect_ofs >= (rq_ofs + rq_count))
+ break;
+
+ bvec_ofs = 0;
+ if (sect_ofs < rq_ofs)
+ bvec_ofs = rq_ofs - sect_ofs;
+
+ bvec_sectors = bio_vec_sectors(bvec) - bvec_ofs;
+ if (bvec_sectors > (rq_count - processed_sectors))
+ bvec_sectors = rq_count - processed_sectors;
+
+ mem = kmap_atomic(bvec.bv_page);
+ if (direction == READ) {
+ memcpy(mem + bvec.bv_offset + (unsigned int)from_sectors(bvec_ofs),
+ buff + (unsigned int)from_sectors(processed_sectors),
+ (unsigned int)from_sectors(bvec_sectors));
+ } else {
+ memcpy(buff + (unsigned int)from_sectors(processed_sectors),
+ mem + bvec.bv_offset + (unsigned int)from_sectors(bvec_ofs),
+ (unsigned int)from_sectors(bvec_sectors));
+ }
+ kunmap_atomic(mem);
+
+ processed_sectors += bvec_sectors;
+
+ sect_ofs += bio_vec_sectors(bvec);
+ }
+
+ return SUCCESS;
+}
+
+int blk_dev_redirect_zeroed_part(struct blk_redirect_bio *rq_redir, sector_t rq_ofs,
+ sector_t rq_count)
+{
+ struct bio_vec bvec;
+ struct bvec_iter iter;
+
+ sector_t sect_ofs = 0;
+ sector_t processed_sectors = 0;
+
+ bio_for_each_segment(bvec, rq_redir->bio, iter) {
+ void *mem;
+ sector_t bvec_ofs;
+ sector_t bvec_sectors;
+
+ if ((sect_ofs + bio_vec_sectors(bvec)) <= rq_ofs) {
+ sect_ofs += bio_vec_sectors(bvec);
+ continue;
+ }
+
+ if (sect_ofs >= (rq_ofs + rq_count))
+ break;
+
+ bvec_ofs = 0;
+ if (sect_ofs < rq_ofs)
+ bvec_ofs = rq_ofs - sect_ofs;
+
+ bvec_sectors = bio_vec_sectors(bvec) - bvec_ofs;
+ if (bvec_sectors > (rq_count - processed_sectors))
+ bvec_sectors = rq_count - processed_sectors;
+
+ mem = kmap_atomic(bvec.bv_page);
+ memset(mem + bvec.bv_offset + (unsigned int)from_sectors(bvec_ofs), 0,
+ (unsigned int)from_sectors(bvec_sectors));
+ kunmap_atomic(mem);
+
+ processed_sectors += bvec_sectors;
+
+ sect_ofs += bio_vec_sectors(bvec);
+ }
+
+ return SUCCESS;
+}
+
+int blk_dev_redirect_read_zeroed(struct blk_redirect_bio *rq_redir, struct block_device *blk_dev,
+ sector_t rq_pos, sector_t blk_ofs_start, sector_t blk_ofs_count,
+ struct rangevector *zero_sectors)
+{
+ int res = SUCCESS;
+ struct blk_range_tree_node *range_node;
+
+ sector_t ofs = 0;
+
+ sector_t from = rq_pos + blk_ofs_start;
+ sector_t to = rq_pos + blk_ofs_start + blk_ofs_count - 1;
+
+ down_read(&zero_sectors->lock);
+ range_node = blk_range_rb_iter_first(&zero_sectors->root, from, to);
+ while (range_node) {
+ struct blk_range *zero_range = &range_node->range;
+ sector_t current_portion;
+
+ if (zero_range->ofs > rq_pos + blk_ofs_start + ofs) {
+ sector_t pre_zero_cnt = zero_range->ofs - (rq_pos + blk_ofs_start + ofs);
+
+ res = blk_dev_redirect_part(rq_redir, READ, blk_dev,
+ rq_pos + blk_ofs_start + ofs,
+ blk_ofs_start + ofs, pre_zero_cnt);
+ if (res != SUCCESS)
+ break;
+
+ ofs += pre_zero_cnt;
+ }
+
+ current_portion = min_t(sector_t, zero_range->cnt, blk_ofs_count - ofs);
+
+ res = blk_dev_redirect_zeroed_part(rq_redir, blk_ofs_start + ofs, current_portion);
+ if (res != SUCCESS)
+ break;
+
+ ofs += current_portion;
+
+ range_node = blk_range_rb_iter_next(range_node, from, to);
+ }
+ up_read(&zero_sectors->lock);
+
+ if (res == SUCCESS)
+ if ((blk_ofs_count - ofs) > 0)
+ res = blk_dev_redirect_part(rq_redir, READ, blk_dev,
+ rq_pos + blk_ofs_start + ofs,
+ blk_ofs_start + ofs, blk_ofs_count - ofs);
+
+ return res;
+}
+void blk_redirect_complete(struct blk_redirect_bio *rq_redir, int res)
+{
+ rq_redir->complete_cb(rq_redir->complete_param, rq_redir->bio, res);
+ redirect_bio_queue_free(rq_redir);
+}
+
+void redirect_bio_queue_init(struct redirect_bio_queue *queue)
+{
+ INIT_LIST_HEAD(&queue->list);
+
+ spin_lock_init(&queue->lock);
+
+ atomic_set(&queue->in_queue_cnt, 0);
+ atomic_set(&queue->alloc_cnt, 0);
+
+ atomic_set(&queue->active_state, true);
+}
+
+struct blk_redirect_bio *redirect_bio_queue_new(struct redirect_bio_queue *queue)
+{
+ struct blk_redirect_bio *rq_redir = kzalloc(sizeof(struct blk_redirect_bio), GFP_NOIO);
+
+ if (rq_redir == NULL)
+ return NULL;
+
+ atomic_inc(&queue->alloc_cnt);
+
+ INIT_LIST_HEAD(&rq_redir->link);
+ rq_redir->queue = queue;
+
+ return rq_redir;
+}
+
+void redirect_bio_queue_free(struct blk_redirect_bio *rq_redir)
+{
+ if (rq_redir) {
+ if (rq_redir->queue)
+ atomic_dec(&rq_redir->queue->alloc_cnt);
+
+ kfree(rq_redir);
+ }
+}
+
+int redirect_bio_queue_push_back(struct redirect_bio_queue *queue,
+ struct blk_redirect_bio *rq_redir)
+{
+ int res = SUCCESS;
+
+ spin_lock(&queue->lock);
+
+ if (atomic_read(&queue->active_state)) {
+ INIT_LIST_HEAD(&rq_redir->link);
+ list_add_tail(&rq_redir->link, &queue->list);
+ atomic_inc(&queue->in_queue_cnt);
+ } else
+ res = -EACCES;
+
+ spin_unlock(&queue->lock);
+ return res;
+}
+
+struct blk_redirect_bio *redirect_bio_queue_get_first(struct redirect_bio_queue *queue)
+{
+ struct blk_redirect_bio *rq_redir = NULL;
+
+ spin_lock(&queue->lock);
+
+ if (!list_empty(&queue->list)) {
+ rq_redir = list_entry(queue->list.next, struct blk_redirect_bio, link);
+ list_del(&rq_redir->link);
+ atomic_dec(&queue->in_queue_cnt);
+ }
+
+ spin_unlock(&queue->lock);
+
+ return rq_redir;
+}
+
+bool redirect_bio_queue_active(struct redirect_bio_queue *queue, bool state)
+{
+ bool prev_state;
+
+ spin_lock(&queue->lock);
+
+ prev_state = atomic_read(&queue->active_state);
+ atomic_set(&queue->active_state, state);
+
+ spin_unlock(&queue->lock);
+
+ return prev_state;
+}
diff --git a/drivers/block/blk-snap/blk_redirect.h b/drivers/block/blk-snap/blk_redirect.h
new file mode 100644
index 000000000000..aae23e78ebe2
--- /dev/null
+++ b/drivers/block/blk-snap/blk_redirect.h
@@ -0,0 +1,73 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+#pragma once
+
+#include "rangevector.h"
+
+int blk_redirect_bioset_create(void);
+void blk_redirect_bioset_free(void);
+
+void blk_redirect_bio_endio(struct bio *bb);
+
+struct blk_redirect_bio_list {
+ struct blk_redirect_bio_list *next;
+ struct bio *this;
+};
+
+struct redirect_bio_queue {
+ struct list_head list;
+ spinlock_t lock;
+
+ atomic_t active_state;
+ atomic_t in_queue_cnt;
+ atomic_t alloc_cnt;
+};
+
+struct blk_redirect_bio {
+ struct list_head link;
+ struct redirect_bio_queue *queue;
+
+ struct bio *bio;
+ int err;
+ struct blk_redirect_bio_list *bio_list_head; //list of created bios
+ atomic64_t bio_count;
+
+ void *complete_param;
+ void (*complete_cb)(void *complete_param, struct bio *rq, int err);
+};
+
+int blk_dev_redirect_part(struct blk_redirect_bio *rq_redir, int direction,
+ struct block_device *blk_dev, sector_t target_pos, sector_t rq_ofs,
+ sector_t rq_count);
+
+void blk_dev_redirect_submit(struct blk_redirect_bio *rq_redir);
+
+int blk_dev_redirect_memcpy_part(struct blk_redirect_bio *rq_redir, int direction, void *src_buff,
+ sector_t rq_ofs, sector_t rq_count);
+
+int blk_dev_redirect_zeroed_part(struct blk_redirect_bio *rq_redir, sector_t rq_ofs,
+ sector_t rq_count);
+
+int blk_dev_redirect_read_zeroed(struct blk_redirect_bio *rq_redir, struct block_device *blk_dev,
+ sector_t rq_pos, sector_t blk_ofs_start, sector_t blk_ofs_count,
+ struct rangevector *zero_sectors);
+
+void blk_redirect_complete(struct blk_redirect_bio *rq_redir, int res);
+
+void redirect_bio_queue_init(struct redirect_bio_queue *queue);
+
+struct blk_redirect_bio *redirect_bio_queue_new(struct redirect_bio_queue *queue);
+
+void redirect_bio_queue_free(struct blk_redirect_bio *rq_redir);
+
+int redirect_bio_queue_push_back(struct redirect_bio_queue *queue,
+ struct blk_redirect_bio *rq_redir);
+
+struct blk_redirect_bio *redirect_bio_queue_get_first(struct redirect_bio_queue *queue);
+
+bool redirect_bio_queue_active(struct redirect_bio_queue *queue, bool state);
+
+#define redirect_bio_queue_empty(queue) (atomic_read(&(queue).in_queue_cnt) == 0)
+
+#define redirect_bio_queue_unactive(queue) \
+ ((atomic_read(&((queue).active_state)) == false) && \
+ (atomic_read(&((queue).alloc_cnt)) == 0))
diff --git a/drivers/block/blk-snap/blk_util.c b/drivers/block/blk-snap/blk_util.c
new file mode 100644
index 000000000000..57db70b86516
--- /dev/null
+++ b/drivers/block/blk-snap/blk_util.c
@@ -0,0 +1,33 @@
+// SPDX-License-Identifier: GPL-2.0
+#include "common.h"
+#include "blk_util.h"
+
+int blk_dev_open(dev_t dev_id, struct block_device **p_blk_dev)
+{
+ int result = SUCCESS;
+ struct block_device *blk_dev;
+ int refCount;
+
+ blk_dev = bdget(dev_id);
+ if (blk_dev == NULL) {
+ pr_err("Unable to open device [%d:%d]: bdget return NULL\n", MAJOR(dev_id),
+ MINOR(dev_id));
+ return -ENODEV;
+ }
+
+ refCount = blkdev_get(blk_dev, FMODE_READ | FMODE_WRITE, NULL);
+ if (refCount < 0) {
+ pr_err("Unable to open device [%d:%d]: blkdev_get return error code %d\n",
+ MAJOR(dev_id), MINOR(dev_id), 0 - refCount);
+ result = refCount;
+ }
+
+ if (result == SUCCESS)
+ *p_blk_dev = blk_dev;
+ return result;
+}
+
+void blk_dev_close(struct block_device *blk_dev)
+{
+ blkdev_put(blk_dev, FMODE_READ);
+}
diff --git a/drivers/block/blk-snap/blk_util.h b/drivers/block/blk-snap/blk_util.h
new file mode 100644
index 000000000000..0776f2faa668
--- /dev/null
+++ b/drivers/block/blk-snap/blk_util.h
@@ -0,0 +1,33 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+#pragma once
+
+int blk_dev_open(dev_t dev_id, struct block_device **p_blk_dev);
+void blk_dev_close(struct block_device *blk_dev);
+
+/*
+ * this function was copied from block/blk.h
+ */
+static inline sector_t part_nr_sects_read(struct hd_struct *part)
+{
+#if (BITS_PER_LONG == 32) && defined(CONFIG_SMP)
+ sector_t nr_sects;
+ unsigned int seq;
+
+ do {
+ seq = read_seqcount_begin(&part->nr_sects_seq);
+ nr_sects = part->nr_sects;
+ } while (read_seqcount_retry(&part->nr_sects_seq, seq));
+
+ return nr_sects;
+#elif (BITS_PER_LONG == 32) && defined(CONFIG_PREEMPTION)
+ sector_t nr_sects;
+
+ preempt_disable();
+ nr_sects = part->nr_sects;
+ preempt_enable();
+
+ return nr_sects;
+#else
+ return part->nr_sects;
+#endif
+}
diff --git a/drivers/block/blk-snap/cbt_map.c b/drivers/block/blk-snap/cbt_map.c
new file mode 100644
index 000000000000..e913069d1a57
--- /dev/null
+++ b/drivers/block/blk-snap/cbt_map.c
@@ -0,0 +1,210 @@
+// SPDX-License-Identifier: GPL-2.0
+#define BLK_SNAP_SECTION "-cbt_map"
+#include "common.h"
+#include "cbt_map.h"
+
+int cbt_map_allocate(struct cbt_map *cbt_map, unsigned int cbt_sect_in_block_degree,
+ sector_t device_capacity)
+{
+ sector_t size_mod;
+
+ cbt_map->sect_in_block_degree = cbt_sect_in_block_degree;
+ cbt_map->device_capacity = device_capacity;
+ cbt_map->map_size = (device_capacity >> (sector_t)cbt_sect_in_block_degree);
+
+ pr_info("Allocate CBT map of %zu\n", cbt_map->map_size);
+
+ size_mod = (device_capacity & ((sector_t)(1 << cbt_sect_in_block_degree) - 1));
+ if (size_mod)
+ cbt_map->map_size++;
+
+ cbt_map->read_map = big_buffer_alloc(cbt_map->map_size, GFP_KERNEL);
+ if (cbt_map->read_map != NULL)
+ big_buffer_memset(cbt_map->read_map, 0);
+
+ cbt_map->write_map = big_buffer_alloc(cbt_map->map_size, GFP_KERNEL);
+ if (cbt_map->write_map != NULL)
+ big_buffer_memset(cbt_map->write_map, 0);
+
+ if ((cbt_map->read_map == NULL) || (cbt_map->write_map == NULL)) {
+ pr_err("Cannot allocate CBT map. map_size=%zu\n", cbt_map->map_size);
+ return -ENOMEM;
+ }
+
+ cbt_map->snap_number_previous = 0;
+ cbt_map->snap_number_active = 1;
+ generate_random_uuid(cbt_map->generationId.b);
+ cbt_map->active = true;
+
+ cbt_map->state_changed_sectors = 0;
+ cbt_map->state_dirty_sectors = 0;
+
+ return SUCCESS;
+}
+
+void cbt_map_deallocate(struct cbt_map *cbt_map)
+{
+ if (cbt_map->read_map != NULL) {
+ big_buffer_free(cbt_map->read_map);
+ cbt_map->read_map = NULL;
+ }
+
+ if (cbt_map->write_map != NULL) {
+ big_buffer_free(cbt_map->write_map);
+ cbt_map->write_map = NULL;
+ }
+
+ cbt_map->active = false;
+}
+
+static void cbt_map_destroy(struct cbt_map *cbt_map)
+{
+ pr_info("CBT map destroy\n");
+ if (cbt_map != NULL) {
+ cbt_map_deallocate(cbt_map);
+
+ kfree(cbt_map);
+ }
+}
+
+struct cbt_map *cbt_map_create(unsigned int cbt_sect_in_block_degree, sector_t device_capacity)
+{
+ struct cbt_map *cbt_map = NULL;
+
+ pr_info("CBT map create\n");
+
+ cbt_map = kzalloc(sizeof(struct cbt_map), GFP_KERNEL);
+ if (cbt_map == NULL)
+ return NULL;
+
+ if (cbt_map_allocate(cbt_map, cbt_sect_in_block_degree, device_capacity) != SUCCESS) {
+ cbt_map_destroy(cbt_map);
+ return NULL;
+ }
+
+ spin_lock_init(&cbt_map->locker);
+ init_rwsem(&cbt_map->rw_lock);
+ kref_init(&cbt_map->refcount);
+
+ return cbt_map;
+}
+
+void cbt_map_destroy_cb(struct kref *kref)
+{
+ cbt_map_destroy(container_of(kref, struct cbt_map, refcount));
+}
+
+struct cbt_map *cbt_map_get_resource(struct cbt_map *cbt_map)
+{
+ if (cbt_map)
+ kref_get(&cbt_map->refcount);
+
+ return cbt_map;
+}
+
+void cbt_map_put_resource(struct cbt_map *cbt_map)
+{
+ if (cbt_map)
+ kref_put(&cbt_map->refcount, cbt_map_destroy_cb);
+}
+
+void cbt_map_switch(struct cbt_map *cbt_map)
+{
+ pr_info("CBT map switch\n");
+ spin_lock(&cbt_map->locker);
+
+ big_buffer_memcpy(cbt_map->read_map, cbt_map->write_map);
+
+ cbt_map->snap_number_previous = cbt_map->snap_number_active;
+ ++cbt_map->snap_number_active;
+ if (cbt_map->snap_number_active == 256) {
+ cbt_map->snap_number_active = 1;
+
+ big_buffer_memset(cbt_map->write_map, 0);
+
+ generate_random_uuid(cbt_map->generationId.b);
+
+ pr_info("CBT reset\n");
+ }
+ spin_unlock(&cbt_map->locker);
+}
+
+int _cbt_map_set(struct cbt_map *cbt_map, sector_t sector_start, sector_t sector_cnt,
+ u8 snap_number, struct big_buffer *map)
+{
+ int res = SUCCESS;
+ size_t cbt_block;
+ size_t cbt_block_first = (size_t)(sector_start >> cbt_map->sect_in_block_degree);
+ size_t cbt_block_last = (size_t)((sector_start + sector_cnt - 1) >>
+ cbt_map->sect_in_block_degree); //inclusive
+
+ for (cbt_block = cbt_block_first; cbt_block <= cbt_block_last; ++cbt_block) {
+ if (cbt_block < cbt_map->map_size) {
+ u8 num;
+
+ res = big_buffer_byte_get(map, cbt_block, &num);
+ if (res == SUCCESS)
+ if (num < snap_number)
+ res = big_buffer_byte_set(map, cbt_block, snap_number);
+ } else
+ res = -EINVAL;
+
+ if (res != SUCCESS) {
+ pr_err("Block index is too large. #%zu was demanded, map size %zu\n",
+ cbt_block, cbt_map->map_size);
+ break;
+ }
+ }
+ return res;
+}
+
+int cbt_map_set(struct cbt_map *cbt_map, sector_t sector_start, sector_t sector_cnt)
+{
+ int res = SUCCESS;
+
+ spin_lock(&cbt_map->locker);
+
+ res = _cbt_map_set(cbt_map, sector_start, sector_cnt, (u8)cbt_map->snap_number_active,
+ cbt_map->write_map);
+ cbt_map->state_changed_sectors += sector_cnt;
+
+ spin_unlock(&cbt_map->locker);
+ return res;
+}
+
+int cbt_map_set_both(struct cbt_map *cbt_map, sector_t sector_start, sector_t sector_cnt)
+{
+ int res = SUCCESS;
+
+ spin_lock(&cbt_map->locker);
+
+ res = _cbt_map_set(cbt_map, sector_start, sector_cnt,
+ (u8)cbt_map->snap_number_active, cbt_map->write_map);
+ if (res == SUCCESS)
+ res = _cbt_map_set(cbt_map, sector_start, sector_cnt,
+ (u8)cbt_map->snap_number_previous, cbt_map->read_map);
+ cbt_map->state_dirty_sectors += sector_cnt;
+
+ spin_unlock(&cbt_map->locker);
+ return res;
+}
+
+size_t cbt_map_read_to_user(struct cbt_map *cbt_map, void __user *user_buff, size_t offset,
+ size_t size)
+{
+ size_t readed = 0;
+ size_t left_size;
+ size_t real_size = min((cbt_map->map_size - offset), size);
+
+ left_size = real_size -
+ big_buffer_copy_to_user(user_buff, offset, cbt_map->read_map, real_size);
+
+ if (left_size == 0)
+ readed = real_size;
+ else {
+ pr_err("Not all CBT data was read. Left [%zu] bytes\n", left_size);
+ readed = real_size - left_size;
+ }
+
+ return readed;
+}
diff --git a/drivers/block/blk-snap/cbt_map.h b/drivers/block/blk-snap/cbt_map.h
new file mode 100644
index 000000000000..cb52b09531fe
--- /dev/null
+++ b/drivers/block/blk-snap/cbt_map.h
@@ -0,0 +1,62 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+#pragma once
+
+#include "big_buffer.h"
+#include <linux/kref.h>
+#include <linux/uuid.h>
+
+struct cbt_map {
+ struct kref refcount;
+
+ spinlock_t locker;
+
+ size_t sect_in_block_degree;
+ sector_t device_capacity;
+ size_t map_size;
+
+ struct big_buffer *read_map;
+ struct big_buffer *write_map;
+
+ unsigned long snap_number_active;
+ unsigned long snap_number_previous;
+ uuid_t generationId;
+
+ bool active;
+
+ struct rw_semaphore rw_lock;
+
+ sector_t state_changed_sectors;
+ sector_t state_dirty_sectors;
+};
+
+struct cbt_map *cbt_map_create(unsigned int cbt_sect_in_block_degree, sector_t device_capacity);
+
+struct cbt_map *cbt_map_get_resource(struct cbt_map *cbt_map);
+void cbt_map_put_resource(struct cbt_map *cbt_map);
+
+void cbt_map_switch(struct cbt_map *cbt_map);
+int cbt_map_set(struct cbt_map *cbt_map, sector_t sector_start, sector_t sector_cnt);
+int cbt_map_set_both(struct cbt_map *cbt_map, sector_t sector_start, sector_t sector_cnt);
+
+size_t cbt_map_read_to_user(struct cbt_map *cbt_map, void __user *user_buffer, size_t offset,
+ size_t size);
+
+static inline void cbt_map_read_lock(struct cbt_map *cbt_map)
+{
+ down_read(&cbt_map->rw_lock);
+};
+
+static inline void cbt_map_read_unlock(struct cbt_map *cbt_map)
+{
+ up_read(&cbt_map->rw_lock);
+};
+
+static inline void cbt_map_write_lock(struct cbt_map *cbt_map)
+{
+ down_write(&cbt_map->rw_lock);
+};
+
+static inline void cbt_map_write_unlock(struct cbt_map *cbt_map)
+{
+ up_write(&cbt_map->rw_lock);
+};
diff --git a/drivers/block/blk-snap/common.h b/drivers/block/blk-snap/common.h
new file mode 100644
index 000000000000..bbd5e98ab2a6
--- /dev/null
+++ b/drivers/block/blk-snap/common.h
@@ -0,0 +1,31 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+#pragma once
+
+#ifndef BLK_SNAP_SECTION
+#define BLK_SNAP_SECTION ""
+#endif
+#define pr_fmt(fmt) KBUILD_MODNAME BLK_SNAP_SECTION ": " fmt
+
+#include <linux/version.h> /*rudiment - needed for using KERNEL_VERSION */
+
+#include <linux/types.h>
+#include <linux/errno.h>
+#include <linux/mutex.h>
+#include <linux/rwsem.h>
+#include <linux/spinlock.h>
+#include <linux/slab.h>
+#include <linux/list.h>
+#include <linux/atomic.h>
+#include <linux/blkdev.h>
+
+#define from_sectors(_sectors) (_sectors << SECTOR_SHIFT)
+#define to_sectors(_byte_size) (_byte_size >> SECTOR_SHIFT)
+
+struct blk_range {
+ sector_t ofs;
+ blkcnt_t cnt;
+};
+
+#ifndef SUCCESS
+#define SUCCESS 0
+#endif
diff --git a/drivers/block/blk-snap/ctrl_fops.c b/drivers/block/blk-snap/ctrl_fops.c
new file mode 100644
index 000000000000..b7b18539ee96
--- /dev/null
+++ b/drivers/block/blk-snap/ctrl_fops.c
@@ -0,0 +1,691 @@
+// SPDX-License-Identifier: GPL-2.0
+#define BLK_SNAP_SECTION "-ctrl"
+#include "common.h"
+#include "blk-snap-ctl.h"
+#include "ctrl_fops.h"
+#include "version.h"
+#include "tracking.h"
+#include "snapshot.h"
+#include "snapstore.h"
+#include "snapimage.h"
+#include "tracker.h"
+#include "blk_deferred.h"
+#include "big_buffer.h"
+#include "params.h"
+
+#include <linux/module.h>
+#include <linux/poll.h>
+#include <linux/uaccess.h>
+
+int blk_snap_major; //zero by default
+
+const struct file_operations ctrl_fops = { .owner = THIS_MODULE,
+ .read = ctrl_read,
+ .write = ctrl_write,
+ .open = ctrl_open,
+ .release = ctrl_release,
+ .poll = ctrl_poll,
+ .unlocked_ioctl = ctrl_unlocked_ioctl };
+
+atomic_t dev_open_cnt = ATOMIC_INIT(0);
+
+const struct ioctl_getversion_s version = { .major = FILEVER_MAJOR,
+ .minor = FILEVER_MINOR,
+ .revision = FILEVER_REVISION,
+ .build = 0 };
+
+int get_blk_snap_major(void)
+{
+ return blk_snap_major;
+}
+
+int ctrl_init(void)
+{
+ int ret;
+
+ ret = register_chrdev(0, MODULE_NAME, &ctrl_fops);
+ if (ret < 0) {
+ pr_err("Failed to register a character device. errno=%d\n", blk_snap_major);
+ return ret;
+ }
+
+ blk_snap_major = ret;
+ pr_info("Module major [%d]\n", blk_snap_major);
+ return SUCCESS;
+}
+
+void ctrl_done(void)
+{
+ unregister_chrdev(blk_snap_major, MODULE_NAME);
+ ctrl_pipe_done();
+}
+
+ssize_t ctrl_read(struct file *fl, char __user *buffer, size_t length, loff_t *offset)
+{
+ ssize_t bytes_read = 0;
+ struct ctrl_pipe *pipe = (struct ctrl_pipe *)fl->private_data;
+
+ if (pipe == NULL) {
+ pr_err("Unable to read from pipe: invalid pipe pointer\n");
+ return -EINVAL;
+ }
+
+ bytes_read = ctrl_pipe_read(pipe, buffer, length);
+ if (bytes_read == 0)
+ if (fl->f_flags & O_NONBLOCK)
+ bytes_read = -EAGAIN;
+
+ return bytes_read;
+}
+
+ssize_t ctrl_write(struct file *fl, const char __user *buffer, size_t length, loff_t *offset)
+{
+ struct ctrl_pipe *pipe = (struct ctrl_pipe *)fl->private_data;
+
+ if (pipe == NULL) {
+ pr_err("Unable to write into pipe: invalid pipe pointer\n");
+ return -EINVAL;
+ }
+
+ return ctrl_pipe_write(pipe, buffer, length);
+}
+
+unsigned int ctrl_poll(struct file *fl, struct poll_table_struct *wait)
+{
+ struct ctrl_pipe *pipe = (struct ctrl_pipe *)fl->private_data;
+
+ if (pipe == NULL) {
+ pr_err("Unable to poll pipe: invalid pipe pointer\n");
+ return -EINVAL;
+ }
+
+ return ctrl_pipe_poll(pipe);
+}
+
+int ctrl_open(struct inode *inode, struct file *fl)
+{
+ fl->f_pos = 0;
+
+ if (false == try_module_get(THIS_MODULE))
+ return -EINVAL;
+
+ fl->private_data = (void *)ctrl_pipe_new();
+ if (fl->private_data == NULL) {
+ pr_err("Failed to open ctrl file\n");
+ return -ENOMEM;
+ }
+
+ atomic_inc(&dev_open_cnt);
+
+ return SUCCESS;
+}
+
+int ctrl_release(struct inode *inode, struct file *fl)
+{
+ int result = SUCCESS;
+
+ if (atomic_read(&dev_open_cnt) > 0) {
+ module_put(THIS_MODULE);
+ ctrl_pipe_put_resource((struct ctrl_pipe *)fl->private_data);
+
+ atomic_dec(&dev_open_cnt);
+ } else {
+ pr_err("Unable to close ctrl file: the file is already closed\n");
+ result = -EALREADY;
+ }
+
+ return result;
+}
+
+int ioctl_compatibility_flags(unsigned long arg)
+{
+ unsigned long len;
+ struct ioctl_compatibility_flags_s param;
+
+ param.flags = 0;
+ param.flags |= BLK_SNAP_COMPATIBILITY_SNAPSTORE;
+#ifdef CONFIG_BLK_SNAP_SNAPSTORE_MULTIDEV
+ param.flags |= BLK_SNAP_COMPATIBILITY_MULTIDEV;
+#endif
+ len = copy_to_user((void *)arg, ¶m, sizeof(struct ioctl_compatibility_flags_s));
+ if (len != 0) {
+ pr_err("Unable to get compatibility flags: invalid user buffer\n");
+ return -EINVAL;
+ }
+
+ return SUCCESS;
+}
+
+int ioctl_get_version(unsigned long arg)
+{
+ unsigned long len;
+
+ pr_info("Get version\n");
+
+ len = copy_to_user((void *)arg, &version, sizeof(struct ioctl_getversion_s));
+ if (len != 0) {
+ pr_err("Unable to get version: invalid user buffer\n");
+ return -ENODATA;
+ }
+
+ return SUCCESS;
+}
+
+int ioctl_tracking_add(unsigned long arg)
+{
+ unsigned long len;
+ struct ioctl_dev_id_s dev;
+
+ len = copy_from_user(&dev, (void *)arg, sizeof(struct ioctl_dev_id_s));
+ if (len != 0) {
+ pr_err("Unable to add device under tracking: invalid user buffer\n");
+ return -ENODATA;
+ }
+
+ return tracking_add(MKDEV(dev.major, dev.minor), 0ull);
+}
+
+int ioctl_tracking_remove(unsigned long arg)
+{
+ struct ioctl_dev_id_s dev;
+
+ if (copy_from_user(&dev, (void *)arg, sizeof(struct ioctl_dev_id_s)) != 0) {
+ pr_err("Unable to remove device from tracking: invalid user buffer\n");
+ return -ENODATA;
+ }
+ return tracking_remove(MKDEV(dev.major, dev.minor));
+ ;
+}
+
+int ioctl_tracking_collect(unsigned long arg)
+{
+ unsigned long len;
+ int res;
+ struct ioctl_tracking_collect_s get;
+
+ pr_info("Collecting tracking devices:\n");
+
+ len = copy_from_user(&get, (void *)arg, sizeof(struct ioctl_tracking_collect_s));
+ if (len != 0) {
+ pr_err("Unable to collect tracking devices: invalid user buffer\n");
+ return -ENODATA;
+ }
+
+ if (get.p_cbt_info == NULL) {
+ res = tracking_collect(0x7FFFffff, NULL, &get.count);
+ if (res == SUCCESS) {
+ len = copy_to_user((void *)arg, (void *)&get,
+ sizeof(struct ioctl_tracking_collect_s));
+ if (len != 0) {
+ pr_err("Unable to collect tracking devices: invalid user buffer for arguments\n");
+ res = -ENODATA;
+ }
+ } else {
+ pr_err("Failed to execute tracking_collect. errno=%d\n", res);
+ }
+ } else {
+ struct cbt_info_s *p_cbt_info = NULL;
+
+ p_cbt_info = kcalloc(get.count, sizeof(struct cbt_info_s), GFP_KERNEL);
+ if (p_cbt_info == NULL)
+ return -ENOMEM;
+
+ do {
+ res = tracking_collect(get.count, p_cbt_info, &get.count);
+ if (res != SUCCESS) {
+ pr_err("Failed to execute tracking_collect. errno=%d\n", res);
+ break;
+ }
+ len = copy_to_user(get.p_cbt_info, p_cbt_info,
+ get.count * sizeof(struct cbt_info_s));
+ if (len != 0) {
+ pr_err("Unable to collect tracking devices: invalid user buffer for CBT info\n");
+ res = -ENODATA;
+ break;
+ }
+
+ len = copy_to_user((void *)arg, (void *)&get,
+ sizeof(struct ioctl_tracking_collect_s));
+ if (len != 0) {
+ pr_err("Unable to collect tracking devices: invalid user buffer for arguments\n");
+ res = -ENODATA;
+ break;
+ }
+
+ } while (false);
+
+ kfree(p_cbt_info);
+ p_cbt_info = NULL;
+ }
+ return res;
+}
+
+int ioctl_tracking_block_size(unsigned long arg)
+{
+ unsigned long len;
+ unsigned int blk_sz = change_tracking_block_size();
+
+ len = copy_to_user((void *)arg, &blk_sz, sizeof(unsigned int));
+ if (len != 0) {
+ pr_err("Unable to get tracking block size: invalid user buffer for arguments\n");
+ return -ENODATA;
+ }
+ return SUCCESS;
+}
+
+int ioctl_tracking_read_cbt_map(unsigned long arg)
+{
+ dev_t dev_id;
+ unsigned long len;
+ struct ioctl_tracking_read_cbt_bitmap_s readbitmap;
+
+ len = copy_from_user(&readbitmap, (void *)arg,
+ sizeof(struct ioctl_tracking_read_cbt_bitmap_s));
+ if (len != 0) {
+ pr_err("Unable to read CBT map: invalid user buffer\n");
+ return -ENODATA;
+ }
+
+ dev_id = MKDEV(readbitmap.dev_id.major, readbitmap.dev_id.minor);
+ return tracking_read_cbt_bitmap(dev_id, readbitmap.offset, readbitmap.length,
+ (void *)readbitmap.buff);
+}
+
+int ioctl_tracking_mark_dirty_blocks(unsigned long arg)
+{
+ unsigned long len;
+ struct ioctl_tracking_mark_dirty_blocks_s param;
+ struct block_range_s *p_dirty_blocks;
+ size_t buffer_size;
+ int result = SUCCESS;
+
+ len = copy_from_user(¶m, (void *)arg,
+ sizeof(struct ioctl_tracking_mark_dirty_blocks_s));
+ if (len != 0) {
+ pr_err("Unable to mark dirty blocks: invalid user buffer\n");
+ return -ENODATA;
+ }
+
+ buffer_size = param.count * sizeof(struct block_range_s);
+ p_dirty_blocks = kzalloc(buffer_size, GFP_KERNEL);
+ if (p_dirty_blocks == NULL) {
+ pr_err("Unable to mark dirty blocks: cannot allocate [%zu] bytes\n", buffer_size);
+ return -ENOMEM;
+ }
+
+ do {
+ dev_t image_dev_id;
+
+ len = copy_from_user(p_dirty_blocks, (void *)param.p_dirty_blocks, buffer_size);
+ if (len != 0) {
+ pr_err("Unable to mark dirty blocks: invalid user buffer\n");
+ result = -ENODATA;
+ break;
+ }
+
+ image_dev_id = MKDEV(param.image_dev_id.major, param.image_dev_id.minor);
+ result = snapimage_mark_dirty_blocks(image_dev_id, p_dirty_blocks, param.count);
+ } while (false);
+ kfree(p_dirty_blocks);
+
+ return result;
+}
+
+int ioctl_snapshot_create(unsigned long arg)
+{
+ unsigned long len;
+ size_t dev_id_buffer_size;
+ int status;
+ struct ioctl_snapshot_create_s param;
+ struct ioctl_dev_id_s *pk_dev_id = NULL;
+
+ len = copy_from_user(¶m, (void *)arg, sizeof(struct ioctl_snapshot_create_s));
+ if (len != 0) {
+ pr_err("Unable to create snapshot: invalid user buffer\n");
+ return -ENODATA;
+ }
+
+ dev_id_buffer_size = sizeof(struct ioctl_dev_id_s) * param.count;
+ pk_dev_id = kzalloc(dev_id_buffer_size, GFP_KERNEL);
+ if (pk_dev_id == NULL) {
+ pr_err("Unable to create snapshot: cannot allocate [%zu] bytes\n",
+ dev_id_buffer_size);
+ return -ENOMEM;
+ }
+
+ do {
+ size_t dev_buffer_size;
+ dev_t *p_dev = NULL;
+ int inx = 0;
+
+ len = copy_from_user(pk_dev_id, (void *)param.p_dev_id,
+ param.count * sizeof(struct ioctl_dev_id_s));
+ if (len != 0) {
+ pr_err("Unable to create snapshot: invalid user buffer for parameters\n");
+ status = -ENODATA;
+ break;
+ }
+
+ dev_buffer_size = sizeof(dev_t) * param.count;
+ p_dev = kzalloc(dev_buffer_size, GFP_KERNEL);
+ if (p_dev == NULL) {
+ pr_err("Unable to create snapshot: cannot allocate [%zu] bytes\n",
+ dev_buffer_size);
+ status = -ENOMEM;
+ break;
+ }
+
+ for (inx = 0; inx < param.count; ++inx)
+ p_dev[inx] = MKDEV(pk_dev_id[inx].major, pk_dev_id[inx].minor);
+
+ status = snapshot_create(p_dev, param.count, ¶m.snapshot_id);
+
+ kfree(p_dev);
+ p_dev = NULL;
+
+ } while (false);
+ kfree(pk_dev_id);
+ pk_dev_id = NULL;
+
+ if (status == SUCCESS) {
+ len = copy_to_user((void *)arg, ¶m, sizeof(struct ioctl_snapshot_create_s));
+ if (len != 0) {
+ pr_err("Unable to create snapshot: invalid user buffer\n");
+ status = -ENODATA;
+ }
+ }
+
+ return status;
+}
+
+int ioctl_snapshot_destroy(unsigned long arg)
+{
+ unsigned long len;
+ unsigned long long param;
+
+ len = copy_from_user(¶m, (void *)arg, sizeof(unsigned long long));
+ if (len != 0) {
+ pr_err("Unable to destroy snapshot: invalid user buffer\n");
+ return -ENODATA;
+ }
+
+ return snapshot_destroy(param);
+}
+
+static inline dev_t _snapstore_dev(struct ioctl_dev_id_s *dev_id)
+{
+ if ((dev_id->major == 0) && (dev_id->minor == 0))
+ return 0; //memory snapstore
+
+ if ((dev_id->major == -1) && (dev_id->minor == -1))
+ return 0xFFFFffff; //multidevice snapstore
+
+ return MKDEV(dev_id->major, dev_id->minor);
+}
+
+int ioctl_snapstore_create(unsigned long arg)
+{
+ unsigned long len;
+ int res = SUCCESS;
+ struct ioctl_snapstore_create_s param;
+ size_t inx = 0;
+ dev_t *dev_id_set = NULL;
+
+ len = copy_from_user(¶m, (void *)arg, sizeof(struct ioctl_snapstore_create_s));
+ if (len != 0) {
+ pr_err("Unable to create snapstore: invalid user buffer\n");
+ return -EINVAL;
+ }
+
+ dev_id_set = kcalloc(param.count, sizeof(dev_t), GFP_KERNEL);
+ if (dev_id_set == NULL)
+ return -ENOMEM;
+
+ for (inx = 0; inx < param.count; ++inx) {
+ struct ioctl_dev_id_s dev_id;
+
+ len = copy_from_user(&dev_id, param.p_dev_id + inx, sizeof(struct ioctl_dev_id_s));
+ if (len != 0) {
+ pr_err("Unable to create snapstore: ");
+ pr_err("invalid user buffer for parameters\n");
+
+ res = -ENODATA;
+ break;
+ }
+
+ dev_id_set[inx] = MKDEV(dev_id.major, dev_id.minor);
+ }
+
+ if (res == SUCCESS)
+ res = snapstore_create((uuid_t *)param.id, _snapstore_dev(¶m.snapstore_dev_id),
+ dev_id_set, (size_t)param.count);
+
+ kfree(dev_id_set);
+
+ return res;
+}
+
+int ioctl_snapstore_file(unsigned long arg)
+{
+ unsigned long len;
+ int res = SUCCESS;
+ struct ioctl_snapstore_file_add_s param;
+ struct big_buffer *ranges = NULL;
+ size_t ranges_buffer_size;
+
+ len = copy_from_user(¶m, (void *)arg, sizeof(struct ioctl_snapstore_file_add_s));
+ if (len != 0) {
+ pr_err("Unable to add file to snapstore: invalid user buffer\n");
+ return -EINVAL;
+ }
+
+ ranges_buffer_size = sizeof(struct ioctl_range_s) * param.range_count;
+
+ ranges = big_buffer_alloc(ranges_buffer_size, GFP_KERNEL);
+ if (ranges == NULL) {
+ pr_err("Unable to add file to snapstore: cannot allocate [%zu] bytes\n",
+ ranges_buffer_size);
+ return -ENOMEM;
+ }
+
+ if (big_buffer_copy_from_user((void *)param.ranges, 0, ranges, ranges_buffer_size)
+ != ranges_buffer_size) {
+
+ pr_err("Unable to add file to snapstore: invalid user buffer for parameters\n");
+ res = -ENODATA;
+ } else
+ res = snapstore_add_file((uuid_t *)(param.id), ranges, (size_t)param.range_count);
+
+ big_buffer_free(ranges);
+
+ return res;
+}
+
+int ioctl_snapstore_memory(unsigned long arg)
+{
+ unsigned long len;
+ int res = SUCCESS;
+ struct ioctl_snapstore_memory_limit_s param;
+
+ len = copy_from_user(¶m, (void *)arg, sizeof(struct ioctl_snapstore_memory_limit_s));
+ if (len != 0) {
+ pr_err("Unable to add memory block to snapstore: invalid user buffer\n");
+ return -EINVAL;
+ }
+
+ res = snapstore_add_memory((uuid_t *)param.id, param.size);
+
+ return res;
+}
+int ioctl_snapstore_cleanup(unsigned long arg)
+{
+ unsigned long len;
+ int res = SUCCESS;
+ struct ioctl_snapstore_cleanup_s param;
+
+ len = copy_from_user(¶m, (void *)arg, sizeof(struct ioctl_snapstore_cleanup_s));
+ if (len != 0) {
+ pr_err("Unable to perform snapstore cleanup: invalid user buffer\n");
+ return -EINVAL;
+ }
+
+ pr_info("Cleanup snapstore %pUB\n", (uuid_t *)param.id);
+ res = snapstore_cleanup((uuid_t *)param.id, ¶m.filled_bytes);
+
+ if (res == SUCCESS) {
+ if (0 !=
+ copy_to_user((void *)arg, ¶m, sizeof(struct ioctl_snapstore_cleanup_s))) {
+ pr_err("Unable to perform snapstore cleanup: invalid user buffer\n");
+ res = -ENODATA;
+ }
+ }
+
+ return res;
+}
+
+#ifdef CONFIG_BLK_SNAP_SNAPSTORE_MULTIDEV
+int ioctl_snapstore_file_multidev(unsigned long arg)
+{
+ unsigned long len;
+ int res = SUCCESS;
+ struct ioctl_snapstore_file_add_multidev_s param;
+ struct big_buffer *ranges = NULL; //struct ioctl_range_s* ranges = NULL;
+ size_t ranges_buffer_size;
+
+ len = copy_from_user(¶m, (void *)arg,
+ sizeof(struct ioctl_snapstore_file_add_multidev_s));
+ if (len != 0) {
+ pr_err("Unable to add file to multidev snapstore: invalid user buffer\n");
+ return -EINVAL;
+ }
+
+ ranges_buffer_size = sizeof(struct ioctl_range_s) * param.range_count;
+
+ ranges = big_buffer_alloc(ranges_buffer_size, GFP_KERNEL);
+ if (ranges == NULL) {
+ pr_err("Unable to add file to multidev snapstore: cannot allocate [%zu] bytes\n",
+ ranges_buffer_size);
+ return -ENOMEM;
+ }
+
+ do {
+ uuid_t *id = (uuid_t *)(param.id);
+ dev_t snapstore_device = MKDEV(param.dev_id.major, param.dev_id.minor);
+ size_t ranges_cnt = (size_t)param.range_count;
+
+ if (ranges_buffer_size != big_buffer_copy_from_user((void *)param.ranges, 0, ranges,
+ ranges_buffer_size)) {
+ pr_err("Unable to add file to snapstore: invalid user buffer for parameters\n");
+ res = -ENODATA;
+ break;
+ }
+
+ res = snapstore_add_multidev(id, snapstore_device, ranges, ranges_cnt);
+ } while (false);
+ big_buffer_free(ranges);
+
+ return res;
+}
+
+#endif
+//////////////////////////////////////////////////////////////////////////
+
+/*
+ * Snapshot get errno for device
+ */
+int ioctl_snapshot_errno(unsigned long arg)
+{
+ unsigned long len;
+ int res;
+ struct ioctl_snapshot_errno_s param;
+
+ len = copy_from_user(¶m, (void *)arg, sizeof(struct ioctl_dev_id_s));
+ if (len != 0) {
+ pr_err("Unable failed to get snapstore error code: invalid user buffer\n");
+ return -EINVAL;
+ }
+
+ res = snapstore_device_errno(MKDEV(param.dev_id.major, param.dev_id.minor),
+ ¶m.err_code);
+
+ if (res != SUCCESS)
+ return res;
+
+ len = copy_to_user((void *)arg, ¶m, sizeof(struct ioctl_snapshot_errno_s));
+ if (len != 0) {
+ pr_err("Unable to get snapstore error code: invalid user buffer\n");
+ return -EINVAL;
+ }
+
+ return SUCCESS;
+}
+
+int ioctl_collect_snapimages(unsigned long arg)
+{
+ unsigned long len;
+ int status = SUCCESS;
+ struct ioctl_collect_snapshot_images_s param;
+
+ len = copy_from_user(¶m, (void *)arg, sizeof(struct ioctl_collect_snapshot_images_s));
+ if (len != 0) {
+ pr_err("Unable to collect snapshot images: invalid user buffer\n");
+ return -ENODATA;
+ }
+
+ status = snapimage_collect_images(param.count, param.p_image_info, ¶m.count);
+
+ len = copy_to_user((void *)arg, ¶m, sizeof(struct ioctl_collect_snapshot_images_s));
+ if (len != 0) {
+ pr_err("Unable to collect snapshot images: invalid user buffer\n");
+ return -ENODATA;
+ }
+
+ return status;
+}
+
+struct blk_snap_ioctl_table {
+ unsigned int cmd;
+ int (*fn)(unsigned long arg);
+};
+
+static struct blk_snap_ioctl_table blk_snap_ioctl_table[] = {
+ { (IOCTL_COMPATIBILITY_FLAGS), ioctl_compatibility_flags },
+ { (IOCTL_GETVERSION), ioctl_get_version },
+
+ { (IOCTL_TRACKING_ADD), ioctl_tracking_add },
+ { (IOCTL_TRACKING_REMOVE), ioctl_tracking_remove },
+ { (IOCTL_TRACKING_COLLECT), ioctl_tracking_collect },
+ { (IOCTL_TRACKING_BLOCK_SIZE), ioctl_tracking_block_size },
+ { (IOCTL_TRACKING_READ_CBT_BITMAP), ioctl_tracking_read_cbt_map },
+ { (IOCTL_TRACKING_MARK_DIRTY_BLOCKS), ioctl_tracking_mark_dirty_blocks },
+
+ { (IOCTL_SNAPSHOT_CREATE), ioctl_snapshot_create },
+ { (IOCTL_SNAPSHOT_DESTROY), ioctl_snapshot_destroy },
+ { (IOCTL_SNAPSHOT_ERRNO), ioctl_snapshot_errno },
+
+ { (IOCTL_SNAPSTORE_CREATE), ioctl_snapstore_create },
+ { (IOCTL_SNAPSTORE_FILE), ioctl_snapstore_file },
+ { (IOCTL_SNAPSTORE_MEMORY), ioctl_snapstore_memory },
+ { (IOCTL_SNAPSTORE_CLEANUP), ioctl_snapstore_cleanup },
+#ifdef CONFIG_BLK_SNAP_SNAPSTORE_MULTIDEV
+ { (IOCTL_SNAPSTORE_FILE_MULTIDEV), ioctl_snapstore_file_multidev },
+#endif
+ { (IOCTL_COLLECT_SNAPSHOT_IMAGES), ioctl_collect_snapimages },
+ { 0, NULL }
+};
+
+long ctrl_unlocked_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
+{
+ long status = -ENOTTY;
+ size_t inx = 0;
+
+ while (blk_snap_ioctl_table[inx].cmd != 0) {
+ if (blk_snap_ioctl_table[inx].cmd == cmd) {
+ status = blk_snap_ioctl_table[inx].fn(arg);
+ break;
+ }
+ ++inx;
+ }
+
+ return status;
+}
diff --git a/drivers/block/blk-snap/ctrl_fops.h b/drivers/block/blk-snap/ctrl_fops.h
new file mode 100644
index 000000000000..98072b61aa96
--- /dev/null
+++ b/drivers/block/blk-snap/ctrl_fops.h
@@ -0,0 +1,19 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+#pragma once
+
+#include <linux/fs.h>
+
+int get_blk_snap_major(void);
+
+int ctrl_init(void);
+void ctrl_done(void);
+
+int ctrl_open(struct inode *inode, struct file *file);
+int ctrl_release(struct inode *inode, struct file *file);
+
+ssize_t ctrl_read(struct file *filp, char __user *buffer, size_t length, loff_t *offset);
+ssize_t ctrl_write(struct file *filp, const char __user *buffer, size_t length, loff_t *offset);
+
+unsigned int ctrl_poll(struct file *filp, struct poll_table_struct *wait);
+
+long ctrl_unlocked_ioctl(struct file *file, unsigned int cmd, unsigned long arg);
diff --git a/drivers/block/blk-snap/ctrl_pipe.c b/drivers/block/blk-snap/ctrl_pipe.c
new file mode 100644
index 000000000000..73cfbca93487
--- /dev/null
+++ b/drivers/block/blk-snap/ctrl_pipe.c
@@ -0,0 +1,562 @@
+// SPDX-License-Identifier: GPL-2.0
+#define BLK_SNAP_SECTION "-ctrl"
+#include "common.h"
+#include "ctrl_pipe.h"
+#include "version.h"
+#include "blk-snap-ctl.h"
+#include "snapstore.h"
+#include "big_buffer.h"
+
+#include <linux/poll.h>
+#include <linux/uuid.h>
+
+#define CMD_TO_USER_FIFO_SIZE 1024
+
+LIST_HEAD(ctl_pipes);
+DECLARE_RWSEM(ctl_pipes_lock);
+
+
+static void ctrl_pipe_push_request(struct ctrl_pipe *pipe, unsigned int *cmd, size_t cmd_len)
+{
+ kfifo_in_spinlocked(&pipe->cmd_to_user, cmd, (cmd_len * sizeof(unsigned int)),
+ &pipe->cmd_to_user_lock);
+
+ wake_up(&pipe->readq);
+}
+
+static void ctrl_pipe_request_acknowledge(struct ctrl_pipe *pipe, unsigned int result)
+{
+ unsigned int cmd[2];
+
+ cmd[0] = BLK_SNAP_CHARCMD_ACKNOWLEDGE;
+ cmd[1] = result;
+
+ ctrl_pipe_push_request(pipe, cmd, 2);
+}
+
+static inline dev_t _snapstore_dev(struct ioctl_dev_id_s *dev_id)
+{
+ if ((dev_id->major == 0) && (dev_id->minor == 0))
+ return 0; //memory snapstore
+
+ if ((dev_id->major == -1) && (dev_id->minor == -1))
+ return 0xFFFFffff; //multidevice snapstore
+
+ return MKDEV(dev_id->major, dev_id->minor);
+}
+
+static ssize_t ctrl_pipe_command_initiate(struct ctrl_pipe *pipe, const char __user *buffer,
+ size_t length)
+{
+ unsigned long len;
+ int result = SUCCESS;
+ ssize_t processed = 0;
+ char *kernel_buffer;
+
+ kernel_buffer = kmalloc(length, GFP_KERNEL);
+ if (kernel_buffer == NULL)
+ return -ENOMEM;
+
+ len = copy_from_user(kernel_buffer, buffer, length);
+ if (len != 0) {
+ kfree(kernel_buffer);
+ pr_err("Unable to write to pipe: invalid user buffer\n");
+ return -EINVAL;
+ }
+
+ do {
+ u64 stretch_empty_limit;
+ unsigned int dev_id_list_length;
+ uuid_t *unique_id;
+ struct ioctl_dev_id_s *snapstore_dev_id;
+ struct ioctl_dev_id_s *dev_id_list;
+
+ //get snapstore uuid
+ if ((length - processed) < 16) {
+ pr_err("Unable to get snapstore uuid: invalid ctrl pipe initiate command. length=%zu\n",
+ length);
+ break;
+ }
+ unique_id = (uuid_t *)(kernel_buffer + processed);
+ processed += 16;
+
+ //get snapstore empty limit
+ if ((length - processed) < sizeof(u64)) {
+ pr_err("Unable to get stretch snapstore limit: invalid ctrl pipe initiate command. length=%zu\n",
+ length);
+ break;
+ }
+ stretch_empty_limit = *(u64 *)(kernel_buffer + processed);
+ processed += sizeof(u64);
+
+ //get snapstore device id
+ if ((length - processed) < sizeof(struct ioctl_dev_id_s)) {
+ pr_err("Unable to get snapstore device id: invalid ctrl pipe initiate command. length=%zu\n",
+ length);
+ break;
+ }
+ snapstore_dev_id = (struct ioctl_dev_id_s *)(kernel_buffer + processed);
+ processed += sizeof(struct ioctl_dev_id_s);
+
+ //get device id list length
+ if ((length - processed) < 4) {
+ pr_err("Unable to get device id list length: ivalid ctrl pipe initiate command. length=%zu\n",
+ length);
+ break;
+ }
+ dev_id_list_length = *(unsigned int *)(kernel_buffer + processed);
+ processed += sizeof(unsigned int);
+
+ //get devices id list
+ if ((length - processed) < (dev_id_list_length * sizeof(struct ioctl_dev_id_s))) {
+ pr_err("Unable to get all devices from device id list: invalid ctrl pipe initiate command. length=%zu\n",
+ length);
+ break;
+ }
+ dev_id_list = (struct ioctl_dev_id_s *)(kernel_buffer + processed);
+ processed += (dev_id_list_length * sizeof(struct ioctl_dev_id_s));
+
+ {
+ size_t inx;
+ dev_t *dev_set;
+ size_t dev_id_set_length = (size_t)dev_id_list_length;
+
+ dev_set = kcalloc(dev_id_set_length, sizeof(dev_t), GFP_KERNEL);
+ if (dev_set == NULL) {
+ result = -ENOMEM;
+ break;
+ }
+
+ for (inx = 0; inx < dev_id_set_length; ++inx)
+ dev_set[inx] =
+ MKDEV(dev_id_list[inx].major, dev_id_list[inx].minor);
+
+ result = snapstore_create(unique_id, _snapstore_dev(snapstore_dev_id),
+ dev_set, dev_id_set_length);
+ kfree(dev_set);
+ if (result != SUCCESS) {
+ pr_err("Failed to create snapstore\n");
+ break;
+ }
+
+ result = snapstore_stretch_initiate(
+ unique_id, pipe, (sector_t)to_sectors(stretch_empty_limit));
+ if (result != SUCCESS) {
+ pr_err("Failed to initiate stretch snapstore %pUB\n", unique_id);
+ break;
+ }
+ }
+ } while (false);
+ kfree(kernel_buffer);
+ ctrl_pipe_request_acknowledge(pipe, result);
+
+ if (result == SUCCESS)
+ return processed;
+ return result;
+}
+
+static ssize_t ctrl_pipe_command_next_portion(struct ctrl_pipe *pipe, const char __user *buffer,
+ size_t length)
+{
+ unsigned long len;
+ int result = SUCCESS;
+ ssize_t processed = 0;
+ struct big_buffer *ranges = NULL;
+
+ do {
+ uuid_t unique_id;
+ unsigned int ranges_length;
+ size_t ranges_buffer_size;
+
+ //get snapstore id
+ if ((length - processed) < 16) {
+ pr_err("Unable to get snapstore id: ");
+ pr_err("invalid ctrl pipe next portion command. length=%zu\n",
+ length);
+ break;
+ }
+ len = copy_from_user(&unique_id, buffer + processed, sizeof(uuid_t));
+ if (len != 0) {
+ pr_err("Unable to write to pipe: invalid user buffer\n");
+ processed = -EINVAL;
+ break;
+ }
+ processed += 16;
+
+ //get ranges length
+ if ((length - processed) < 4) {
+ pr_err("Unable to get device id list length: ");
+ pr_err("invalid ctrl pipe next portion command. length=%zu\n",
+ length);
+ break;
+ }
+ len = copy_from_user(&ranges_length, buffer + processed, sizeof(unsigned int));
+ if (len != 0) {
+ pr_err("Unable to write to pipe: invalid user buffer\n");
+ processed = -EINVAL;
+ break;
+ }
+ processed += sizeof(unsigned int);
+
+ ranges_buffer_size = ranges_length * sizeof(struct ioctl_range_s);
+
+ // ranges
+ if ((length - processed) < (ranges_buffer_size)) {
+ pr_err("Unable to get all ranges: ");
+ pr_err("invalid ctrl pipe next portion command. length=%zu\n",
+ length);
+ break;
+ }
+ ranges = big_buffer_alloc(ranges_buffer_size, GFP_KERNEL);
+ if (ranges == NULL) {
+ pr_err("Unable to allocate page array buffer: ");
+ pr_err("failed to process next portion command\n");
+ processed = -ENOMEM;
+ break;
+ }
+ if (ranges_buffer_size !=
+ big_buffer_copy_from_user(buffer + processed, 0, ranges, ranges_buffer_size)) {
+ pr_err("Unable to process next portion command: ");
+ pr_err("invalid user buffer for parameters\n");
+ processed = -EINVAL;
+ break;
+ }
+ processed += ranges_buffer_size;
+
+ {
+ result = snapstore_add_file(&unique_id, ranges, ranges_length);
+
+ if (result != SUCCESS) {
+ pr_err("Failed to add file to snapstore\n");
+ result = -ENODEV;
+ break;
+ }
+ }
+ } while (false);
+ if (ranges)
+ big_buffer_free(ranges);
+
+ if (result == SUCCESS)
+ return processed;
+ return result;
+}
+
+#ifdef CONFIG_BLK_SNAP_SNAPSTORE_MULTIDEV
+static ssize_t ctrl_pipe_command_next_portion_multidev(struct ctrl_pipe *pipe,
+ const char __user *buffer, size_t length)
+{
+ unsigned long len;
+ int result = SUCCESS;
+ ssize_t processed = 0;
+ struct big_buffer *ranges = NULL;
+
+ do {
+ uuid_t unique_id;
+ int snapstore_major;
+ int snapstore_minor;
+ unsigned int ranges_length;
+ size_t ranges_buffer_size;
+
+ //get snapstore id
+ if ((length - processed) < 16) {
+ pr_err("Unable to get snapstore id: ");
+ pr_err("invalid ctrl pipe next portion command. length=%zu\n",
+ length);
+ break;
+ }
+ len = copy_from_user(&unique_id, buffer + processed, sizeof(uuid_t));
+ if (len != 0) {
+ pr_err("Unable to write to pipe: invalid user buffer\n");
+ processed = -EINVAL;
+ break;
+ }
+ processed += 16;
+
+ //get device id
+ if ((length - processed) < 8) {
+ pr_err("Unable to get device id list length: ");
+ pr_err("invalid ctrl pipe next portion command. length=%zu\n", length);
+ break;
+ }
+ len = copy_from_user(&snapstore_major, buffer + processed, sizeof(unsigned int));
+ if (len != 0) {
+ pr_err("Unable to write to pipe: invalid user buffer\n");
+ processed = -EINVAL;
+ break;
+ }
+ processed += sizeof(unsigned int);
+
+ len = copy_from_user(&snapstore_minor, buffer + processed, sizeof(unsigned int));
+ if (len != 0) {
+ pr_err("Unable to write to pipe: invalid user buffer\n");
+ processed = -EINVAL;
+ break;
+ }
+ processed += sizeof(unsigned int);
+
+ //get ranges length
+ if ((length - processed) < 4) {
+ pr_err("Unable to get device id list length: ");
+ pr_err("invalid ctrl pipe next portion command. length=%zu\n",
+ length);
+ break;
+ }
+ len = copy_from_user(&ranges_length, buffer + processed, sizeof(unsigned int));
+ if (len != 0) {
+ pr_err("Unable to write to pipe: invalid user buffer\n");
+ processed = -EINVAL;
+ break;
+ }
+ processed += sizeof(unsigned int);
+
+ ranges_buffer_size = ranges_length * sizeof(struct ioctl_range_s);
+
+ // ranges
+ if ((length - processed) < (ranges_buffer_size)) {
+ pr_err("Unable to get all ranges: ");
+ pr_err("invalid ctrl pipe next portion command. length=%zu\n",
+ length);
+ break;
+ }
+ ranges = big_buffer_alloc(ranges_buffer_size, GFP_KERNEL);
+ if (ranges == NULL) {
+ pr_err("Unable to process next portion command: ");
+ pr_err("failed to allocate page array buffer\n");
+ processed = -ENOMEM;
+ break;
+ }
+ if (ranges_buffer_size !=
+ big_buffer_copy_from_user(buffer + processed, 0, ranges, ranges_buffer_size)) {
+ pr_err("Unable to process next portion command: ");
+ pr_err("invalid user buffer from parameters\n");
+ processed = -EINVAL;
+ break;
+ }
+ processed += ranges_buffer_size;
+
+ {
+ result = snapstore_add_multidev(&unique_id,
+ MKDEV(snapstore_major, snapstore_minor),
+ ranges, ranges_length);
+
+ if (result != SUCCESS) {
+ pr_err("Failed to add file to snapstore\n");
+ result = -ENODEV;
+ break;
+ }
+ }
+ } while (false);
+ if (ranges)
+ big_buffer_free(ranges);
+
+ if (result == SUCCESS)
+ return processed;
+
+ return result;
+}
+#endif
+
+static void ctrl_pipe_release_cb(struct kref *kref)
+{
+ struct ctrl_pipe *pipe = container_of(kref, struct ctrl_pipe, refcount);
+
+ down_write(&ctl_pipes_lock);
+ list_del(&pipe->link);
+ up_write(&ctl_pipes_lock);
+
+ kfifo_free(&pipe->cmd_to_user);
+
+ kfree(pipe);
+}
+
+struct ctrl_pipe *ctrl_pipe_get_resource(struct ctrl_pipe *pipe)
+{
+ if (pipe)
+ kref_get(&pipe->refcount);
+
+ return pipe;
+}
+
+void ctrl_pipe_put_resource(struct ctrl_pipe *pipe)
+{
+ if (pipe)
+ kref_put(&pipe->refcount, ctrl_pipe_release_cb);
+}
+
+void ctrl_pipe_done(void)
+{
+ bool is_empty;
+
+ pr_info("Ctrl pipes - done\n");
+
+ down_write(&ctl_pipes_lock);
+ is_empty = list_empty(&ctl_pipes);
+ up_write(&ctl_pipes_lock);
+
+ if (!is_empty)
+ pr_err("Unable to perform ctrl pipes cleanup: container is not empty\n");
+}
+
+struct ctrl_pipe *ctrl_pipe_new(void)
+{
+ int ret;
+ struct ctrl_pipe *pipe;
+
+ pipe = kzalloc(sizeof(struct ctrl_pipe), GFP_KERNEL);
+ if (pipe == NULL)
+ return NULL;
+
+ INIT_LIST_HEAD(&pipe->link);
+
+ ret = kfifo_alloc(&pipe->cmd_to_user, CMD_TO_USER_FIFO_SIZE, GFP_KERNEL);
+ if (ret) {
+ pr_err("Failed to allocate fifo. errno=%d.\n", ret);
+ kfree(pipe);
+ return NULL;
+ }
+ spin_lock_init(&pipe->cmd_to_user_lock);
+
+ kref_init(&pipe->refcount);
+
+ init_waitqueue_head(&pipe->readq);
+
+ down_write(&ctl_pipes_lock);
+ list_add_tail(&pipe->link, &ctl_pipes);
+ up_write(&ctl_pipes_lock);
+
+ return pipe;
+}
+
+ssize_t ctrl_pipe_read(struct ctrl_pipe *pipe, char __user *buffer, size_t length)
+{
+ int ret;
+ unsigned int processed = 0;
+
+ if (kfifo_is_empty_spinlocked(&pipe->cmd_to_user, &pipe->cmd_to_user_lock)) {
+ //nothing to read
+ ret = wait_event_interruptible(pipe->readq,
+ !kfifo_is_empty_spinlocked(&pipe->cmd_to_user,
+ &pipe->cmd_to_user_lock));
+ if (ret) {
+ pr_err("Unable to wait for pipe read queue: interrupt signal was received\n");
+ return -ERESTARTSYS;
+ }
+ }
+
+ ret = kfifo_to_user(&pipe->cmd_to_user, buffer, length, &processed);
+ if (ret) {
+ pr_err("Failed to read command from ctrl pipe\n");
+ return ret;
+ }
+
+ return (ssize_t)processed;
+}
+
+ssize_t ctrl_pipe_write(struct ctrl_pipe *pipe, const char __user *buffer, size_t length)
+{
+ ssize_t processed = 0;
+
+ do {
+ unsigned long len;
+ unsigned int command;
+
+ if ((length - processed) < 4) {
+ pr_err("Unable to write command to ctrl pipe: invalid command length=%zu\n",
+ length);
+ break;
+ }
+ len = copy_from_user(&command, buffer + processed, sizeof(unsigned int));
+ if (len != 0) {
+ pr_err("Unable to write to pipe: invalid user buffer\n");
+ processed = -EINVAL;
+ break;
+ }
+ processed += sizeof(unsigned int);
+ //+4
+ switch (command) {
+ case BLK_SNAP_CHARCMD_INITIATE: {
+ ssize_t res = ctrl_pipe_command_initiate(pipe, buffer + processed,
+ length - processed);
+ if (res >= 0)
+ processed += res;
+ else
+ processed = res;
+ } break;
+ case BLK_SNAP_CHARCMD_NEXT_PORTION: {
+ ssize_t res = ctrl_pipe_command_next_portion(pipe, buffer + processed,
+ length - processed);
+ if (res >= 0)
+ processed += res;
+ else
+ processed = res;
+ } break;
+#ifdef CONFIG_BLK_SNAP_SNAPSTORE_MULTIDEV
+ case BLK_SNAP_CHARCMD_NEXT_PORTION_MULTIDEV: {
+ ssize_t res = ctrl_pipe_command_next_portion_multidev(
+ pipe, buffer + processed, length - processed);
+ if (res >= 0)
+ processed += res;
+ else
+ processed = res;
+ } break;
+#endif
+ default:
+ pr_err("Ctrl pipe write error: invalid command [0x%x] received\n", command);
+ break;
+ }
+ } while (false);
+ return processed;
+}
+
+unsigned int ctrl_pipe_poll(struct ctrl_pipe *pipe)
+{
+ unsigned int mask = 0;
+
+ if (!kfifo_is_empty_spinlocked(&pipe->cmd_to_user, &pipe->cmd_to_user_lock))
+ mask |= (POLLIN | POLLRDNORM); /* readable */
+
+ mask |= (POLLOUT | POLLWRNORM); /* writable */
+
+ return mask;
+}
+
+void ctrl_pipe_request_halffill(struct ctrl_pipe *pipe, unsigned long long filled_status)
+{
+ unsigned int cmd[3];
+
+ pr_info("Snapstore is half-full\n");
+
+ cmd[0] = (unsigned int)BLK_SNAP_CHARCMD_HALFFILL;
+ cmd[1] = (unsigned int)(filled_status & 0xFFFFffff); //lo
+ cmd[2] = (unsigned int)(filled_status >> 32);
+
+ ctrl_pipe_push_request(pipe, cmd, 3);
+}
+
+void ctrl_pipe_request_overflow(struct ctrl_pipe *pipe, unsigned int error_code,
+ unsigned long long filled_status)
+{
+ unsigned int cmd[4];
+
+ pr_info("Snapstore overflow\n");
+
+ cmd[0] = (unsigned int)BLK_SNAP_CHARCMD_OVERFLOW;
+ cmd[1] = error_code;
+ cmd[2] = (unsigned int)(filled_status & 0xFFFFffff); //lo
+ cmd[3] = (unsigned int)(filled_status >> 32);
+
+ ctrl_pipe_push_request(pipe, cmd, 4);
+}
+
+void ctrl_pipe_request_terminate(struct ctrl_pipe *pipe, unsigned long long filled_status)
+{
+ unsigned int cmd[3];
+
+ pr_info("Snapstore termination\n");
+
+ cmd[0] = (unsigned int)BLK_SNAP_CHARCMD_TERMINATE;
+ cmd[1] = (unsigned int)(filled_status & 0xFFFFffff); //lo
+ cmd[2] = (unsigned int)(filled_status >> 32);
+
+ ctrl_pipe_push_request(pipe, cmd, 3);
+}
diff --git a/drivers/block/blk-snap/ctrl_pipe.h b/drivers/block/blk-snap/ctrl_pipe.h
new file mode 100644
index 000000000000..1aa1099eec25
--- /dev/null
+++ b/drivers/block/blk-snap/ctrl_pipe.h
@@ -0,0 +1,34 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+#pragma once
+
+#include <linux/kref.h>
+#include <linux/wait.h>
+#include <linux/kfifo.h>
+
+struct ctrl_pipe {
+ struct list_head link;
+
+ struct kref refcount;
+
+ wait_queue_head_t readq;
+
+ struct kfifo cmd_to_user;
+ spinlock_t cmd_to_user_lock;
+};
+
+struct ctrl_pipe *ctrl_pipe_get_resource(struct ctrl_pipe *pipe);
+void ctrl_pipe_put_resource(struct ctrl_pipe *pipe);
+
+void ctrl_pipe_done(void);
+
+struct ctrl_pipe *ctrl_pipe_new(void);
+
+ssize_t ctrl_pipe_read(struct ctrl_pipe *pipe, char __user *buffer, size_t length);
+ssize_t ctrl_pipe_write(struct ctrl_pipe *pipe, const char __user *buffer, size_t length);
+
+unsigned int ctrl_pipe_poll(struct ctrl_pipe *pipe);
+
+void ctrl_pipe_request_halffill(struct ctrl_pipe *pipe, unsigned long long filled_status);
+void ctrl_pipe_request_overflow(struct ctrl_pipe *pipe, unsigned int error_code,
+ unsigned long long filled_status);
+void ctrl_pipe_request_terminate(struct ctrl_pipe *pipe, unsigned long long filled_status);
diff --git a/drivers/block/blk-snap/ctrl_sysfs.c b/drivers/block/blk-snap/ctrl_sysfs.c
new file mode 100644
index 000000000000..4ec78e85b510
--- /dev/null
+++ b/drivers/block/blk-snap/ctrl_sysfs.c
@@ -0,0 +1,73 @@
+// SPDX-License-Identifier: GPL-2.0
+#define BLK_SNAP_SECTION "-ctrl"
+#include "common.h"
+#include "ctrl_sysfs.h"
+#include "ctrl_fops.h"
+#include "blk-snap-ctl.h"
+
+#include <linux/blkdev.h>
+#include <linux/sysfs.h>
+
+static ssize_t major_show(struct class *class, struct class_attribute *attr, char *buf)
+{
+ sprintf(buf, "%d", get_blk_snap_major());
+ return strlen(buf);
+}
+
+CLASS_ATTR_RO(major); // declare class_attr_major
+static struct class *blk_snap_class;
+
+static struct device *blk_snap_device;
+
+int ctrl_sysfs_init(void)
+{
+ struct device *dev;
+ int res;
+
+ blk_snap_class = class_create(THIS_MODULE, MODULE_NAME);
+ if (IS_ERR(blk_snap_class)) {
+ res = PTR_ERR(blk_snap_class);
+
+ pr_err("Bad class create. errno=%d\n", 0 - res);
+ return res;
+ }
+
+ pr_info("Create 'major' sysfs attribute\n");
+ res = class_create_file(blk_snap_class, &class_attr_major);
+ if (res != SUCCESS) {
+ pr_err("Failed to create 'major' sysfs file\n");
+
+ class_destroy(blk_snap_class);
+ blk_snap_class = NULL;
+ return res;
+ }
+
+ dev = device_create(blk_snap_class, NULL, MKDEV(get_blk_snap_major(), 0), NULL,
+ MODULE_NAME);
+ if (IS_ERR(dev)) {
+ res = PTR_ERR(dev);
+ pr_err("Failed to create device, errno=%d\n", res);
+
+ class_remove_file(blk_snap_class, &class_attr_major);
+ class_destroy(blk_snap_class);
+ blk_snap_class = NULL;
+ return res;
+ }
+
+ blk_snap_device = dev;
+ return res;
+}
+
+void ctrl_sysfs_done(void)
+{
+ if (blk_snap_device) {
+ device_unregister(blk_snap_device);
+ blk_snap_device = NULL;
+ }
+
+ if (blk_snap_class != NULL) {
+ class_remove_file(blk_snap_class, &class_attr_major);
+ class_destroy(blk_snap_class);
+ blk_snap_class = NULL;
+ }
+}
diff --git a/drivers/block/blk-snap/ctrl_sysfs.h b/drivers/block/blk-snap/ctrl_sysfs.h
new file mode 100644
index 000000000000..27a2a4d3da4c
--- /dev/null
+++ b/drivers/block/blk-snap/ctrl_sysfs.h
@@ -0,0 +1,5 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+#pragma once
+
+int ctrl_sysfs_init(void);
+void ctrl_sysfs_done(void);
diff --git a/drivers/block/blk-snap/defer_io.c b/drivers/block/blk-snap/defer_io.c
new file mode 100644
index 000000000000..309216fe7319
--- /dev/null
+++ b/drivers/block/blk-snap/defer_io.c
@@ -0,0 +1,397 @@
+// SPDX-License-Identifier: GPL-2.0
+#define BLK_SNAP_SECTION "-defer_io"
+#include "common.h"
+#include "defer_io.h"
+#include "blk_deferred.h"
+#include "tracker.h"
+#include "blk_util.h"
+
+#include <linux/kthread.h>
+
+#define BLK_IMAGE_THROTTLE_TIMEOUT (1 * HZ) //delay 1 sec
+//#define BLK_IMAGE_THROTTLE_TIMEOUT ( HZ/1000 * 10 ) //delay 10 ms
+
+
+
+struct defer_io_orig_rq {
+ struct list_head link;
+ struct defer_io_queue *queue;
+
+ struct bio *bio;
+ struct tracker *tracker;
+};
+
+static inline void defer_io_queue_init(struct defer_io_queue *queue)
+{
+ INIT_LIST_HEAD(&queue->list);
+
+ spin_lock_init(&queue->lock);
+
+ atomic_set(&queue->in_queue_cnt, 0);
+ atomic_set(&queue->active_state, true);
+}
+
+static inline struct defer_io_orig_rq *defer_io_queue_new(struct defer_io_queue *queue, struct bio *bio)
+{
+ struct defer_io_orig_rq *dio_rq;
+
+ dio_rq = kzalloc(sizeof(struct defer_io_orig_rq), GFP_NOIO);
+ if (dio_rq == NULL)
+ return NULL;
+
+ dio_rq->bio = bio;
+ bio_get(dio_rq->bio);
+
+ INIT_LIST_HEAD(&dio_rq->link);
+ dio_rq->queue = queue;
+
+ return dio_rq;
+}
+
+static inline void defer_io_queue_free(struct defer_io_orig_rq *dio_rq)
+{
+ if (likely(dio_rq)) {
+ if (likely(dio_rq->bio)) {
+ bio_put(dio_rq->bio);
+ dio_rq->bio = NULL;
+ }
+ kfree(dio_rq);
+ }
+}
+
+static int defer_io_queue_push_back(struct defer_io_queue *queue, struct defer_io_orig_rq *dio_rq)
+{
+ int res = SUCCESS;
+
+ spin_lock(&queue->lock);
+
+ if (atomic_read(&queue->active_state)) {
+ list_add_tail(&dio_rq->link, &queue->list);
+ atomic_inc(&queue->in_queue_cnt);
+ } else
+ res = -EACCES;
+
+ spin_unlock(&queue->lock);
+ return res;
+}
+
+static struct defer_io_orig_rq *defer_io_queue_get_first(struct defer_io_queue *queue)
+{
+ struct defer_io_orig_rq *dio_rq = NULL;
+
+ spin_lock(&queue->lock);
+
+ if (!list_empty(&queue->list)) {
+ dio_rq = list_entry(queue->list.next, struct defer_io_orig_rq, link);
+ list_del(&dio_rq->link);
+ atomic_dec(&queue->in_queue_cnt);
+ }
+
+ spin_unlock(&queue->lock);
+
+ return dio_rq;
+}
+
+static bool defer_io_queue_active(struct defer_io_queue *queue, bool state)
+{
+ bool prev_state;
+
+ spin_lock(&queue->lock);
+
+ prev_state = atomic_read(&queue->active_state);
+ atomic_set(&queue->active_state, state);
+
+ spin_unlock(&queue->lock);
+
+ return prev_state;
+}
+
+#define defer_io_queue_empty(queue) (atomic_read(&(queue).in_queue_cnt) == 0)
+
+static void _defer_io_finish(struct defer_io *defer_io, struct defer_io_queue *queue_in_progress)
+{
+ while (!defer_io_queue_empty(*queue_in_progress)) {
+ struct tracker *tracker = NULL;
+ bool cbt_locked = false;
+ bool is_write_bio;
+ sector_t sectCount = 0;
+
+ struct defer_io_orig_rq *orig_req = defer_io_queue_get_first(queue_in_progress);
+
+ is_write_bio = bio_data_dir(orig_req->bio) && bio_has_data(orig_req->bio);
+
+ if (orig_req->tracker && is_write_bio) {
+ tracker = orig_req->tracker;
+ cbt_locked = tracker_cbt_bitmap_lock(tracker);
+ if (cbt_locked) {
+ sectCount = bio_sectors(orig_req->bio);
+ tracker_cbt_bitmap_set(tracker, orig_req->bio->bi_iter.bi_sector,
+ sectCount);
+ }
+ }
+
+ submit_bio_direct(orig_req->bio);
+
+ if (cbt_locked)
+ tracker_cbt_bitmap_unlock(tracker);
+
+ defer_io_queue_free(orig_req);
+ }
+}
+
+static int _defer_io_copy_prepare(struct defer_io *defer_io,
+ struct defer_io_queue *queue_in_process,
+ struct blk_deferred_request **dio_copy_req)
+{
+ int res = SUCCESS;
+ int dios_count = 0;
+ sector_t dios_sectors_count = 0;
+
+ //fill copy_request set
+ while (!defer_io_queue_empty(defer_io->dio_queue) &&
+ (dios_count < DEFER_IO_DIO_REQUEST_LENGTH) &&
+ (dios_sectors_count < DEFER_IO_DIO_REQUEST_SECTORS_COUNT)) {
+ struct defer_io_orig_rq *dio_orig_req =
+ (struct defer_io_orig_rq *)defer_io_queue_get_first(&defer_io->dio_queue);
+ atomic_dec(&defer_io->queue_filling_count);
+
+ defer_io_queue_push_back(queue_in_process, dio_orig_req);
+
+ if (!kthread_should_stop() &&
+ !snapstore_device_is_corrupted(defer_io->snapstore_device)) {
+ if (bio_data_dir(dio_orig_req->bio) && bio_has_data(dio_orig_req->bio)) {
+ struct blk_range copy_range;
+
+ copy_range.ofs = dio_orig_req->bio->bi_iter.bi_sector;
+ copy_range.cnt = bio_sectors(dio_orig_req->bio);
+ res = snapstore_device_prepare_requests(defer_io->snapstore_device,
+ ©_range, dio_copy_req);
+ if (res != SUCCESS) {
+ pr_err("Unable to execute Copy On Write algorithm: failed to add ranges to copy to snapstore request. errno=%d\n",
+ res);
+ break;
+ }
+
+ dios_sectors_count += copy_range.cnt;
+ }
+ }
+ ++dios_count;
+ }
+ return res;
+}
+
+static int defer_io_work_thread(void *p)
+{
+ struct defer_io_queue queue_in_process = { 0 };
+ struct defer_io *defer_io = NULL;
+
+ //set_user_nice( current, -20 ); //MIN_NICE
+ defer_io_queue_init(&queue_in_process);
+
+ defer_io = defer_io_get_resource((struct defer_io *)p);
+ pr_info("Defer IO thread for original device [%d:%d] started\n",
+ MAJOR(defer_io->original_dev_id), MINOR(defer_io->original_dev_id));
+
+ while (!kthread_should_stop() || !defer_io_queue_empty(defer_io->dio_queue)) {
+ if (defer_io_queue_empty(defer_io->dio_queue)) {
+ int res = wait_event_interruptible_timeout(
+ defer_io->queue_add_event,
+ (!defer_io_queue_empty(defer_io->dio_queue)),
+ BLK_IMAGE_THROTTLE_TIMEOUT);
+ if (-ERESTARTSYS == res)
+ pr_err("Signal received in defer IO thread. Waiting for completion with code ERESTARTSYS\n");
+ }
+
+ if (!defer_io_queue_empty(defer_io->dio_queue)) {
+ int dio_copy_result = SUCCESS;
+ struct blk_deferred_request *dio_copy_req = NULL;
+
+ mutex_lock(&defer_io->snapstore_device->store_block_map_locker);
+ do {
+ dio_copy_result = _defer_io_copy_prepare(
+ defer_io, &queue_in_process, &dio_copy_req);
+ if (dio_copy_result != SUCCESS) {
+ pr_err("Unable to process defer IO request: failed to prepare copy request. erro=%d\n",
+ dio_copy_result);
+ break;
+ }
+ if (dio_copy_req == NULL)
+ break; //nothing to copy
+
+ dio_copy_result = blk_deferred_request_read_original(
+ defer_io->original_blk_dev, dio_copy_req);
+ if (dio_copy_result != SUCCESS) {
+ pr_err("Unable to process defer IO request: failed to read data to copy request. errno=%d\n",
+ dio_copy_result);
+ break;
+ }
+ dio_copy_result = snapstore_device_store(defer_io->snapstore_device,
+ dio_copy_req);
+ if (dio_copy_result != SUCCESS) {
+ pr_err("Unable to process defer IO request: failed to write data from copy request. errno=%d\n",
+ dio_copy_result);
+ break;
+ }
+
+ } while (false);
+ _defer_io_finish(defer_io, &queue_in_process);
+ mutex_unlock(&defer_io->snapstore_device->store_block_map_locker);
+
+ if (dio_copy_req) {
+ if (dio_copy_result == -EDEADLK)
+ blk_deferred_request_deadlocked(dio_copy_req);
+ else
+ blk_deferred_request_free(dio_copy_req);
+ }
+ }
+
+ //wake up snapimage if defer io queue empty
+ if (defer_io_queue_empty(defer_io->dio_queue))
+ wake_up_interruptible(&defer_io->queue_throttle_waiter);
+ }
+ defer_io_queue_active(&defer_io->dio_queue, false);
+
+ //waiting for all sent request complete
+ _defer_io_finish(defer_io, &defer_io->dio_queue);
+
+ pr_info("Defer IO thread for original device [%d:%d] completed\n",
+ MAJOR(defer_io->original_dev_id), MINOR(defer_io->original_dev_id));
+ defer_io_put_resource(defer_io);
+ return SUCCESS;
+}
+
+static void _defer_io_destroy(struct defer_io *defer_io)
+{
+ if (defer_io == NULL)
+ return;
+
+ if (defer_io->dio_thread)
+ defer_io_stop(defer_io);
+
+ if (defer_io->snapstore_device)
+ snapstore_device_put_resource(defer_io->snapstore_device);
+
+ kfree(defer_io);
+ pr_info("Defer IO processor was destroyed\n");
+}
+
+static void defer_io_destroy_cb(struct kref *kref)
+{
+ _defer_io_destroy(container_of(kref, struct defer_io, refcount));
+}
+
+struct defer_io *defer_io_get_resource(struct defer_io *defer_io)
+{
+ if (defer_io)
+ kref_get(&defer_io->refcount);
+
+ return defer_io;
+}
+
+void defer_io_put_resource(struct defer_io *defer_io)
+{
+ if (defer_io)
+ kref_put(&defer_io->refcount, defer_io_destroy_cb);
+}
+
+int defer_io_create(dev_t dev_id, struct block_device *blk_dev, struct defer_io **pp_defer_io)
+{
+ int res = SUCCESS;
+ struct defer_io *defer_io = NULL;
+ struct snapstore_device *snapstore_device;
+
+ pr_info("Defer IO processor was created for device [%d:%d]\n", MAJOR(dev_id),
+ MINOR(dev_id));
+
+ defer_io = kzalloc(sizeof(struct defer_io), GFP_KERNEL);
+ if (defer_io == NULL)
+ return -ENOMEM;
+
+ snapstore_device = snapstore_device_find_by_dev_id(dev_id);
+ if (snapstore_device == NULL) {
+ pr_err("Unable to create defer IO processor: failed to initialize snapshot data for device [%d:%d]\n",
+ MAJOR(dev_id), MINOR(dev_id));
+
+ kfree(defer_io);
+ return -ENODATA;
+ }
+
+ defer_io->snapstore_device = snapstore_device_get_resource(snapstore_device);
+ defer_io->original_dev_id = dev_id;
+ defer_io->original_blk_dev = blk_dev;
+
+ kref_init(&defer_io->refcount);
+
+ defer_io_queue_init(&defer_io->dio_queue);
+
+ init_waitqueue_head(&defer_io->queue_add_event);
+
+ atomic_set(&defer_io->queue_filling_count, 0);
+
+ init_waitqueue_head(&defer_io->queue_throttle_waiter);
+
+ defer_io->dio_thread = kthread_create(defer_io_work_thread, (void *)defer_io,
+ "blksnapdeferio%d:%d", MAJOR(dev_id), MINOR(dev_id));
+ if (IS_ERR(defer_io->dio_thread)) {
+ res = PTR_ERR(defer_io->dio_thread);
+ pr_err("Unable to create defer IO processor: failed to create thread. errno=%d\n",
+ res);
+
+ _defer_io_destroy(defer_io);
+ defer_io = NULL;
+ *pp_defer_io = NULL;
+
+ return res;
+ }
+
+ wake_up_process(defer_io->dio_thread);
+
+ *pp_defer_io = defer_io;
+ pr_info("Defer IO processor was created\n");
+
+ return SUCCESS;
+}
+
+int defer_io_stop(struct defer_io *defer_io)
+{
+ int res = SUCCESS;
+
+ pr_info("Defer IO thread for the device stopped [%d:%d]\n",
+ MAJOR(defer_io->original_dev_id), MINOR(defer_io->original_dev_id));
+
+ if (defer_io->dio_thread != NULL) {
+ struct task_struct *dio_thread = defer_io->dio_thread;
+
+ defer_io->dio_thread = NULL;
+ res = kthread_stop(dio_thread); //stopping and waiting.
+ if (res != SUCCESS)
+ pr_err("Failed to stop defer IO thread. errno=%d\n", res);
+ }
+
+ return res;
+}
+
+int defer_io_redirect_bio(struct defer_io *defer_io, struct bio *bio, void *tracker)
+{
+ struct defer_io_orig_rq *dio_orig_req;
+
+ if (snapstore_device_is_corrupted(defer_io->snapstore_device))
+ return -ENODATA;
+
+ dio_orig_req = defer_io_queue_new(&defer_io->dio_queue, bio);
+ if (dio_orig_req == NULL)
+ return -ENOMEM;
+
+ dio_orig_req->tracker = (struct tracker *)tracker;
+
+ if (defer_io_queue_push_back(&defer_io->dio_queue, dio_orig_req) != SUCCESS) {
+ defer_io_queue_free(dio_orig_req);
+ return -EFAULT;
+ }
+
+ atomic_inc(&defer_io->queue_filling_count);
+
+ wake_up_interruptible(&defer_io->queue_add_event);
+
+ return SUCCESS;
+}
diff --git a/drivers/block/blk-snap/defer_io.h b/drivers/block/blk-snap/defer_io.h
new file mode 100644
index 000000000000..27c3bb03241f
--- /dev/null
+++ b/drivers/block/blk-snap/defer_io.h
@@ -0,0 +1,39 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+#pragma once
+
+#include <linux/kref.h>
+#include "snapstore_device.h"
+
+struct defer_io_queue {
+ struct list_head list;
+ spinlock_t lock;
+
+ atomic_t active_state;
+ atomic_t in_queue_cnt;
+};
+
+struct defer_io {
+ struct kref refcount;
+
+ wait_queue_head_t queue_add_event;
+
+ atomic_t queue_filling_count;
+ wait_queue_head_t queue_throttle_waiter;
+
+ dev_t original_dev_id;
+ struct block_device *original_blk_dev;
+
+ struct snapstore_device *snapstore_device;
+
+ struct task_struct *dio_thread;
+
+ struct defer_io_queue dio_queue;
+};
+
+int defer_io_create(dev_t dev_id, struct block_device *blk_dev, struct defer_io **pp_defer_io);
+int defer_io_stop(struct defer_io *defer_io);
+
+struct defer_io *defer_io_get_resource(struct defer_io *defer_io);
+void defer_io_put_resource(struct defer_io *defer_io);
+
+int defer_io_redirect_bio(struct defer_io *defer_io, struct bio *bio, void *tracker);
diff --git a/drivers/block/blk-snap/main.c b/drivers/block/blk-snap/main.c
new file mode 100644
index 000000000000..d1d4e08a4890
--- /dev/null
+++ b/drivers/block/blk-snap/main.c
@@ -0,0 +1,82 @@
+// SPDX-License-Identifier: GPL-2.0
+#include "common.h"
+#include "version.h"
+#include "blk-snap-ctl.h"
+#include "params.h"
+#include "ctrl_fops.h"
+#include "ctrl_pipe.h"
+#include "ctrl_sysfs.h"
+#include "snapimage.h"
+#include "snapstore.h"
+#include "snapstore_device.h"
+#include "snapshot.h"
+#include "tracker.h"
+#include "tracking.h"
+#include <linux/module.h>
+
+int __init blk_snap_init(void)
+{
+ int result = SUCCESS;
+
+ pr_info("Loading\n");
+
+ params_check();
+
+ result = ctrl_init();
+ if (result != SUCCESS)
+ return result;
+
+ result = blk_redirect_bioset_create();
+ if (result != SUCCESS)
+ return result;
+
+ result = blk_deferred_bioset_create();
+ if (result != SUCCESS)
+ return result;
+
+ result = snapimage_init();
+ if (result != SUCCESS)
+ return result;
+
+ result = ctrl_sysfs_init();
+ if (result != SUCCESS)
+ return result;
+
+ result = tracking_init();
+ if (result != SUCCESS)
+ return result;
+
+ return result;
+}
+
+void __exit blk_snap_exit(void)
+{
+ pr_info("Unloading module\n");
+
+ ctrl_sysfs_done();
+
+ snapshot_done();
+
+ snapstore_device_done();
+ snapstore_done();
+
+ tracker_done();
+ tracking_done();
+
+ snapimage_done();
+
+ blk_deferred_bioset_free();
+ blk_deferred_done();
+
+ blk_redirect_bioset_free();
+
+ ctrl_done();
+}
+
+module_init(blk_snap_init);
+module_exit(blk_snap_exit);
+
+MODULE_DESCRIPTION("Block Layer Snapshot Kernel Module");
+MODULE_VERSION(FILEVER_STR);
+MODULE_AUTHOR("Veeam Software Group GmbH");
+MODULE_LICENSE("GPL");
diff --git a/drivers/block/blk-snap/params.c b/drivers/block/blk-snap/params.c
new file mode 100644
index 000000000000..7eba3c8bf395
--- /dev/null
+++ b/drivers/block/blk-snap/params.c
@@ -0,0 +1,58 @@
+// SPDX-License-Identifier: GPL-2.0
+#include "common.h"
+#include "params.h"
+#include <linux/module.h>
+
+int snapstore_block_size_pow = 14;
+int change_tracking_block_size_pow = 18;
+
+int get_snapstore_block_size_pow(void)
+{
+ return snapstore_block_size_pow;
+}
+
+int inc_snapstore_block_size_pow(void)
+{
+ if (snapstore_block_size_pow > 30)
+ return -EFAULT;
+
+ ++snapstore_block_size_pow;
+ return SUCCESS;
+}
+
+int get_change_tracking_block_size_pow(void)
+{
+ return change_tracking_block_size_pow;
+}
+
+void params_check(void)
+{
+ pr_info("snapstore_block_size_pow: %d\n", snapstore_block_size_pow);
+ pr_info("change_tracking_block_size_pow: %d\n", change_tracking_block_size_pow);
+
+ if (snapstore_block_size_pow > 23) {
+ snapstore_block_size_pow = 23;
+ pr_info("Limited snapstore_block_size_pow: %d\n", snapstore_block_size_pow);
+ } else if (snapstore_block_size_pow < 12) {
+ snapstore_block_size_pow = 12;
+ pr_info("Limited snapstore_block_size_pow: %d\n", snapstore_block_size_pow);
+ }
+
+ if (change_tracking_block_size_pow > 23) {
+ change_tracking_block_size_pow = 23;
+ pr_info("Limited change_tracking_block_size_pow: %d\n",
+ change_tracking_block_size_pow);
+ } else if (change_tracking_block_size_pow < 12) {
+ change_tracking_block_size_pow = 12;
+ pr_info("Limited change_tracking_block_size_pow: %d\n",
+ change_tracking_block_size_pow);
+ }
+}
+
+module_param_named(snapstore_block_size_pow, snapstore_block_size_pow, int, 0644);
+MODULE_PARM_DESC(snapstore_block_size_pow,
+ "Snapstore block size binary pow. 20 for 1MiB block size");
+
+module_param_named(change_tracking_block_size_pow, change_tracking_block_size_pow, int, 0644);
+MODULE_PARM_DESC(change_tracking_block_size_pow,
+ "Change-tracking block size binary pow. 18 for 256 KiB block size");
diff --git a/drivers/block/blk-snap/params.h b/drivers/block/blk-snap/params.h
new file mode 100644
index 000000000000..c1b853a1363b
--- /dev/null
+++ b/drivers/block/blk-snap/params.h
@@ -0,0 +1,29 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+#pragma once
+
+int get_snapstore_block_size_pow(void);
+int inc_snapstore_block_size_pow(void);
+
+static inline sector_t snapstore_block_shift(void)
+{
+ return get_snapstore_block_size_pow() - SECTOR_SHIFT;
+};
+
+static inline sector_t snapstore_block_size(void)
+{
+ return 1ull << snapstore_block_shift();
+};
+
+static inline sector_t snapstore_block_mask(void)
+{
+ return snapstore_block_size() - 1ull;
+};
+
+int get_change_tracking_block_size_pow(void);
+
+static inline unsigned int change_tracking_block_size(void)
+{
+ return 1 << get_change_tracking_block_size_pow();
+};
+
+void params_check(void);
diff --git a/drivers/block/blk-snap/rangevector.c b/drivers/block/blk-snap/rangevector.c
new file mode 100644
index 000000000000..49fe4589b6f7
--- /dev/null
+++ b/drivers/block/blk-snap/rangevector.c
@@ -0,0 +1,85 @@
+// SPDX-License-Identifier: GPL-2.0
+#include "common.h"
+#include "rangevector.h"
+
+#define SECTION "ranges "
+
+static inline sector_t range_node_start(struct blk_range_tree_node *range_node)
+{
+ return range_node->range.ofs;
+}
+
+static inline sector_t range_node_last(struct blk_range_tree_node *range_node)
+{
+ return range_node->range.ofs + range_node->range.cnt - 1;
+}
+
+#ifndef INTERVAL_TREE_DEFINE
+#pragma message("INTERVAL_TREE_DEFINE is undefined")
+#endif
+INTERVAL_TREE_DEFINE(struct blk_range_tree_node, _node, sector_t, _subtree_last,
+ range_node_start, range_node_last, static inline, _blk_range_rb)
+
+void blk_range_rb_insert(struct blk_range_tree_node *node, struct rb_root_cached *root)
+{
+ _blk_range_rb_insert(node, root);
+}
+
+void blk_range_rb_remove(struct blk_range_tree_node *node, struct rb_root_cached *root)
+{
+ _blk_range_rb_remove(node, root);
+}
+
+struct blk_range_tree_node *blk_range_rb_iter_first(struct rb_root_cached *root, sector_t start,
+ sector_t last)
+{
+ return _blk_range_rb_iter_first(root, start, last);
+}
+
+struct blk_range_tree_node *blk_range_rb_iter_next(struct blk_range_tree_node *node, sector_t start,
+ sector_t last)
+{
+ return _blk_range_rb_iter_next(node, start, last);
+}
+
+void rangevector_init(struct rangevector *rangevector)
+{
+ init_rwsem(&rangevector->lock);
+
+ rangevector->root = RB_ROOT_CACHED;
+}
+
+void rangevector_done(struct rangevector *rangevector)
+{
+ struct rb_node *rb_node = NULL;
+
+ down_write(&rangevector->lock);
+ rb_node = rb_first_cached(&rangevector->root);
+ while (rb_node) {
+ struct blk_range_tree_node *range_node = (struct blk_range_tree_node *)
+ rb_node; //container_of(rb_node, struct blk_range_tree_node, node);
+
+ blk_range_rb_remove(range_node, &rangevector->root);
+ kfree(range_node);
+
+ rb_node = rb_first_cached(&rangevector->root);
+ }
+ up_write(&rangevector->lock);
+}
+
+int rangevector_add(struct rangevector *rangevector, struct blk_range *rg)
+{
+ struct blk_range_tree_node *range_node;
+
+ range_node = kzalloc(sizeof(struct blk_range_tree_node), GFP_KERNEL);
+ if (range_node)
+ return -ENOMEM;
+
+ range_node->range = *rg;
+
+ down_write(&rangevector->lock);
+ blk_range_rb_insert(range_node, &rangevector->root);
+ up_write(&rangevector->lock);
+
+ return SUCCESS;
+}
diff --git a/drivers/block/blk-snap/rangevector.h b/drivers/block/blk-snap/rangevector.h
new file mode 100644
index 000000000000..5ff439423178
--- /dev/null
+++ b/drivers/block/blk-snap/rangevector.h
@@ -0,0 +1,31 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#pragma once
+
+#include <linux/interval_tree_generic.h>
+
+struct blk_range_tree_node {
+ struct rb_node _node;
+ struct blk_range range;
+ sector_t _subtree_last;
+};
+
+void blk_range_rb_insert(struct blk_range_tree_node *node, struct rb_root_cached *root);
+
+void blk_range_rb_remove(struct blk_range_tree_node *node, struct rb_root_cached *root);
+
+struct blk_range_tree_node *blk_range_rb_iter_first(struct rb_root_cached *root, sector_t start,
+ sector_t last);
+
+struct blk_range_tree_node *blk_range_rb_iter_next(struct blk_range_tree_node *node, sector_t start,
+ sector_t last);
+
+struct rangevector {
+ struct rb_root_cached root;
+ struct rw_semaphore lock;
+};
+
+void rangevector_init(struct rangevector *rangevector);
+
+void rangevector_done(struct rangevector *rangevector);
+
+int rangevector_add(struct rangevector *rangevector, struct blk_range *rg);
diff --git a/drivers/block/blk-snap/snapimage.c b/drivers/block/blk-snap/snapimage.c
new file mode 100644
index 000000000000..da971486cbef
--- /dev/null
+++ b/drivers/block/blk-snap/snapimage.c
@@ -0,0 +1,982 @@
+// SPDX-License-Identifier: GPL-2.0
+#define BLK_SNAP_SECTION "-snapimage"
+#include "common.h"
+#include "snapimage.h"
+#include "blk_util.h"
+#include "defer_io.h"
+#include "cbt_map.h"
+#include "tracker.h"
+
+#include <asm/div64.h>
+#include <linux/cdrom.h>
+#include <linux/blk-mq.h>
+#include <linux/hdreg.h>
+#include <linux/kthread.h>
+
+#define SNAPIMAGE_MAX_DEVICES 2048
+
+int snapimage_major;
+unsigned long *snapimage_minors;
+DEFINE_SPINLOCK(snapimage_minors_lock);
+
+LIST_HEAD(snap_images);
+DECLARE_RWSEM(snap_images_lock);
+
+DECLARE_RWSEM(snap_image_destroy_lock);
+
+struct snapimage {
+ struct list_head link;
+
+ sector_t capacity;
+ dev_t original_dev;
+
+ struct defer_io *defer_io;
+ struct cbt_map *cbt_map;
+
+ dev_t image_dev;
+
+ struct request_queue *queue;
+ struct gendisk *disk;
+
+ atomic_t own_cnt;
+
+ struct redirect_bio_queue image_queue;
+
+ struct task_struct *rq_processor;
+
+ wait_queue_head_t rq_proc_event;
+ wait_queue_head_t rq_complete_event;
+
+ struct mutex open_locker;
+ struct block_device *open_bdev;
+
+ size_t open_cnt;
+};
+
+int _snapimage_open(struct block_device *bdev, fmode_t mode)
+{
+ int res = SUCCESS;
+
+ if (bdev->bd_disk == NULL) {
+ pr_err("Unable to open snapshot image: bd_disk is NULL. Device [%d:%d]\n",
+ MAJOR(bdev->bd_dev), MINOR(bdev->bd_dev));
+ pr_err("Block device object %p\n", bdev);
+ return -ENODEV;
+ }
+
+ down_read(&snap_image_destroy_lock);
+ do {
+ struct snapimage *image = bdev->bd_disk->private_data;
+
+ if (image == NULL) {
+ pr_err("Unable to open snapshot image: private data is not initialized. Block device object %p\n",
+ bdev);
+ res = -ENODEV;
+ break;
+ }
+
+ mutex_lock(&image->open_locker);
+ {
+ if (image->open_cnt == 0)
+ image->open_bdev = bdev;
+
+ image->open_cnt++;
+ }
+ mutex_unlock(&image->open_locker);
+ } while (false);
+ up_read(&snap_image_destroy_lock);
+ return res;
+}
+
+static inline uint64_t do_div_inline(uint64_t division, uint32_t divisor)
+{
+ do_div(division, divisor);
+ return division;
+}
+
+int _snapimage_getgeo(struct block_device *bdev, struct hd_geometry *geo)
+{
+ int res = SUCCESS;
+ sector_t quotient;
+
+ down_read(&snap_image_destroy_lock);
+ do {
+ struct snapimage *image = bdev->bd_disk->private_data;
+
+ if (image == NULL) {
+ pr_err("Unable to open snapshot image: private data is not initialized. Block device object %p\n",
+ bdev);
+ res = -ENODEV;
+ break;
+ }
+
+ pr_info("Getting geo for snapshot image device [%d:%d]\n", MAJOR(image->image_dev),
+ MINOR(image->image_dev));
+
+ geo->start = 0;
+ if (image->capacity > 63) {
+ geo->sectors = 63;
+ quotient = do_div_inline(image->capacity + (63 - 1), 63);
+
+ if (quotient > 255ULL) {
+ geo->heads = 255;
+ geo->cylinders =
+ (unsigned short)do_div_inline(quotient + (255 - 1), 255);
+ } else {
+ geo->heads = (unsigned char)quotient;
+ geo->cylinders = 1;
+ }
+ } else {
+ geo->sectors = (unsigned char)image->capacity;
+ geo->cylinders = 1;
+ geo->heads = 1;
+ }
+
+ pr_info("Image device geo: capacity=%lld, heads=%d, cylinders=%d, sectors=%d\n",
+ image->capacity, geo->heads, geo->cylinders, geo->sectors);
+ } while (false);
+ up_read(&snap_image_destroy_lock);
+
+ return res;
+}
+
+void _snapimage_close(struct gendisk *disk, fmode_t mode)
+{
+ if (disk->private_data != NULL) {
+ down_read(&snap_image_destroy_lock);
+ do {
+ struct snapimage *image = disk->private_data;
+
+ mutex_lock(&image->open_locker);
+ {
+ if (image->open_cnt > 0)
+ image->open_cnt--;
+
+ if (image->open_cnt == 0)
+ image->open_bdev = NULL;
+ }
+ mutex_unlock(&image->open_locker);
+ } while (false);
+ up_read(&snap_image_destroy_lock);
+ } else
+ pr_err("Unable to close snapshot image: private data is not initialized\n");
+}
+
+int _snapimage_ioctl(struct block_device *bdev, fmode_t mode, unsigned int cmd, unsigned long arg)
+{
+ int res = -ENOTTY;
+
+ down_read(&snap_image_destroy_lock);
+ {
+ struct snapimage *image = bdev->bd_disk->private_data;
+
+ switch (cmd) {
+ /*
+ * The only command we need to interpret is HDIO_GETGEO, since
+ * we can't partition the drive otherwise. We have no real
+ * geometry, of course, so make something up.
+ */
+ case HDIO_GETGEO: {
+ unsigned long len;
+ struct hd_geometry geo;
+
+ res = _snapimage_getgeo(bdev, &geo);
+
+ len = copy_to_user((void *)arg, &geo, sizeof(geo));
+ if (len != 0)
+ res = -EFAULT;
+ else
+ res = SUCCESS;
+ } break;
+ case CDROM_GET_CAPABILITY: //0x5331 / * get capabilities * /
+ {
+ struct gendisk *disk = bdev->bd_disk;
+
+ if (bdev->bd_disk && (disk->flags & GENHD_FL_CD))
+ res = SUCCESS;
+ else
+ res = -EINVAL;
+ } break;
+
+ default:
+ pr_info("Snapshot image ioctl receive unsupported command\n");
+ pr_info("Device [%d:%d], command 0x%x, arg 0x%lx\n",
+ MAJOR(image->image_dev), MINOR(image->image_dev), cmd, arg);
+
+ res = -ENOTTY; /* unknown command */
+ }
+ }
+ up_read(&snap_image_destroy_lock);
+ return res;
+}
+
+blk_qc_t _snapimage_submit_bio(struct bio *bio);
+
+const struct block_device_operations snapimage_ops = {
+ .owner = THIS_MODULE,
+ .submit_bio = _snapimage_submit_bio,
+ .open = _snapimage_open,
+ .ioctl = _snapimage_ioctl,
+ .release = _snapimage_close,
+};
+
+static inline int _snapimage_request_read(struct snapimage *image,
+ struct blk_redirect_bio *rq_redir)
+{
+ struct snapstore_device *snapstore_device = image->defer_io->snapstore_device;
+
+ return snapstore_device_read(snapstore_device, rq_redir);
+}
+
+int _snapimage_request_write(struct snapimage *image, struct blk_redirect_bio *rq_redir)
+{
+ struct snapstore_device *snapstore_device;
+ struct cbt_map *cbt_map;
+ int res = SUCCESS;
+
+ if (unlikely((image->defer_io == NULL) || (image->cbt_map == NULL))) {
+ pr_err("Invalid snapshot image structure");
+ return -EINVAL;
+ }
+
+
+ snapstore_device = image->defer_io->snapstore_device;
+ cbt_map = image->cbt_map;
+
+ if (snapstore_device_is_corrupted(snapstore_device))
+ return -ENODATA;
+
+ if (!bio_has_data(rq_redir->bio)) {
+ pr_warn("Snapshot image receive empty block IO. flags=%u\n",
+ rq_redir->bio->bi_flags);
+
+ blk_redirect_complete(rq_redir, SUCCESS);
+ return SUCCESS;
+ }
+
+ if (cbt_map != NULL) {
+ sector_t ofs = rq_redir->bio->bi_iter.bi_sector;
+ sector_t cnt = bio_sectors(rq_redir->bio);
+
+ res = cbt_map_set_both(cbt_map, ofs, cnt);
+ if (res != SUCCESS)
+ pr_err("Unable to write data to snapshot image: failed to set CBT map. errno=%d\n",
+ res);
+ }
+
+ res = snapstore_device_write(snapstore_device, rq_redir);
+
+ if (res != SUCCESS) {
+ pr_err("Failed to write data to snapshot image\n");
+ return res;
+ }
+
+ return res;
+}
+
+void _snapimage_processing(struct snapimage *image)
+{
+ int res = SUCCESS;
+ struct blk_redirect_bio *rq_redir;
+
+ rq_redir = redirect_bio_queue_get_first(&image->image_queue);
+
+ if (bio_data_dir(rq_redir->bio) == READ) {
+ res = _snapimage_request_read(image, rq_redir);
+ if (res != SUCCESS)
+ pr_err("Failed to read data from snapshot image. errno=%d\n", res);
+
+ } else {
+ res = _snapimage_request_write(image, rq_redir);
+ if (res != SUCCESS)
+ pr_err("Failed to write data to snapshot image. errno=%d\n", res);
+ }
+
+ if (res != SUCCESS)
+ blk_redirect_complete(rq_redir, res);
+}
+
+int snapimage_processor_waiting(struct snapimage *image)
+{
+ int res = SUCCESS;
+
+ if (redirect_bio_queue_empty(image->image_queue)) {
+ res = wait_event_interruptible_timeout(
+ image->rq_proc_event,
+ (!redirect_bio_queue_empty(image->image_queue) || kthread_should_stop()),
+ 5 * HZ);
+ if (res > 0)
+ res = SUCCESS;
+ else if (res == 0)
+ res = -ETIME;
+ }
+ return res;
+}
+
+int snapimage_processor_thread(void *data)
+{
+ struct snapimage *image = data;
+
+ pr_info("Snapshot image thread for device [%d:%d] start\n", MAJOR(image->image_dev),
+ MINOR(image->image_dev));
+
+ add_disk(image->disk);
+
+ //priority
+ set_user_nice(current, -20); //MIN_NICE
+
+ while (!kthread_should_stop()) {
+ int res = snapimage_processor_waiting(image);
+
+ if (res == SUCCESS) {
+ if (!redirect_bio_queue_empty(image->image_queue))
+ _snapimage_processing(image);
+ } else if (res != -ETIME) {
+ pr_err("Failed to wait snapshot image thread queue. errno=%d\n", res);
+ return res;
+ }
+ schedule();
+ }
+ pr_info("Snapshot image disk delete\n");
+ del_gendisk(image->disk);
+
+ while (!redirect_bio_queue_empty(image->image_queue))
+ _snapimage_processing(image);
+
+ pr_info("Snapshot image thread for device [%d:%d] complete", MAJOR(image->image_dev),
+ MINOR(image->image_dev));
+ return 0;
+}
+
+static inline void _snapimage_bio_complete(struct bio *bio, int err)
+{
+ if (err == SUCCESS)
+ bio->bi_status = BLK_STS_OK;
+ else
+ bio->bi_status = BLK_STS_IOERR;
+
+ bio_endio(bio);
+}
+
+void _snapimage_bio_complete_cb(void *complete_param, struct bio *bio, int err)
+{
+ struct snapimage *image = (struct snapimage *)complete_param;
+
+ _snapimage_bio_complete(bio, err);
+
+ if (redirect_bio_queue_unactive(image->image_queue))
+ wake_up_interruptible(&image->rq_complete_event);
+
+ atomic_dec(&image->own_cnt);
+}
+
+int _snapimage_throttling(struct defer_io *defer_io)
+{
+ return wait_event_interruptible(defer_io->queue_throttle_waiter,
+ redirect_bio_queue_empty(defer_io->dio_queue));
+}
+
+blk_qc_t _snapimage_submit_bio(struct bio *bio)
+{
+ blk_qc_t result = SUCCESS;
+ struct request_queue *q = bio->bi_disk->queue;
+ struct snapimage *image = q->queuedata;
+
+ if (unlikely(blk_mq_queue_stopped(q))) {
+ pr_info("Failed to make snapshot image request. Queue already is not active.");
+ pr_info("Queue flags=%lx\n", q->queue_flags);
+
+ _snapimage_bio_complete(bio, -ENODEV);
+
+ return result;
+ }
+
+ atomic_inc(&image->own_cnt);
+ do {
+ int res;
+ struct blk_redirect_bio *rq_redir;
+
+ if (false == atomic_read(&(image->image_queue.active_state))) {
+ _snapimage_bio_complete(bio, -ENODEV);
+ break;
+ }
+
+ if (snapstore_device_is_corrupted(image->defer_io->snapstore_device)) {
+ _snapimage_bio_complete(bio, -ENODATA);
+ break;
+ }
+
+ res = _snapimage_throttling(image->defer_io);
+ if (res != SUCCESS) {
+ pr_err("Failed to throttle snapshot image device. errno=%d\n", res);
+ _snapimage_bio_complete(bio, res);
+ break;
+ }
+
+ rq_redir = redirect_bio_queue_new(&image->image_queue);
+ if (rq_redir == NULL) {
+ pr_err("Unable to make snapshot image request: failed to allocate redirect bio structure\n");
+ _snapimage_bio_complete(bio, -ENOMEM);
+ break;
+ }
+ rq_redir->bio = bio;
+ rq_redir->complete_cb = _snapimage_bio_complete_cb;
+ rq_redir->complete_param = (void *)image;
+ atomic_inc(&image->own_cnt);
+
+ res = redirect_bio_queue_push_back(&image->image_queue, rq_redir);
+ if (res == SUCCESS)
+ wake_up(&image->rq_proc_event);
+ else {
+ redirect_bio_queue_free(rq_redir);
+ _snapimage_bio_complete(bio, -EIO);
+
+ if (redirect_bio_queue_unactive(image->image_queue))
+ wake_up_interruptible(&image->rq_complete_event);
+ }
+
+ } while (false);
+ atomic_dec(&image->own_cnt);
+
+ return result;
+}
+
+struct blk_dev_info {
+ size_t blk_size;
+ sector_t start_sect;
+ sector_t count_sect;
+
+ unsigned int io_min;
+ unsigned int physical_block_size;
+ unsigned short logical_block_size;
+};
+
+static int _blk_dev_get_info(struct block_device *blk_dev, struct blk_dev_info *pdev_info)
+{
+ sector_t SectorStart;
+ sector_t SectorsCapacity;
+
+ if (blk_dev->bd_part)
+ SectorsCapacity = blk_dev->bd_part->nr_sects;
+ else if (blk_dev->bd_disk)
+ SectorsCapacity = get_capacity(blk_dev->bd_disk);
+ else
+ return -EINVAL;
+
+ SectorStart = get_start_sect(blk_dev);
+
+ pdev_info->physical_block_size = blk_dev->bd_disk->queue->limits.physical_block_size;
+ pdev_info->logical_block_size = blk_dev->bd_disk->queue->limits.logical_block_size;
+ pdev_info->io_min = blk_dev->bd_disk->queue->limits.io_min;
+
+ pdev_info->blk_size = block_size(blk_dev);
+ pdev_info->start_sect = SectorStart;
+ pdev_info->count_sect = SectorsCapacity;
+ return SUCCESS;
+}
+
+static int blk_dev_get_info(dev_t dev_id, struct blk_dev_info *pdev_info)
+{
+ int result = SUCCESS;
+ struct block_device *blk_dev;
+
+ result = blk_dev_open(dev_id, &blk_dev);
+ if (result != SUCCESS) {
+ pr_err("Failed to open device [%d:%d]\n", MAJOR(dev_id), MINOR(dev_id));
+ return result;
+ }
+
+ result = _blk_dev_get_info(blk_dev, pdev_info);
+ if (result != SUCCESS)
+ pr_err("Failed to identify block device [%d:%d]\n", MAJOR(dev_id), MINOR(dev_id));
+
+ blk_dev_close(blk_dev);
+
+ return result;
+}
+
+static inline void _snapimage_free(struct snapimage *image)
+{
+ defer_io_put_resource(image->defer_io);
+ cbt_map_put_resource(image->cbt_map);
+ image->defer_io = NULL;
+}
+
+static void _snapimage_stop(struct snapimage *image)
+{
+ if (image->rq_processor != NULL) {
+ if (redirect_bio_queue_active(&image->image_queue, false)) {
+ struct request_queue *q = image->queue;
+
+ pr_info("Snapshot image request processing stop\n");
+
+ if (!blk_queue_stopped(q)) {
+ blk_sync_queue(q);
+ blk_mq_stop_hw_queues(q);
+ }
+ }
+
+ pr_info("Snapshot image thread stop\n");
+ kthread_stop(image->rq_processor);
+ image->rq_processor = NULL;
+
+ while (!redirect_bio_queue_unactive(image->image_queue))
+ wait_event_interruptible(image->rq_complete_event,
+ redirect_bio_queue_unactive(image->image_queue));
+ }
+}
+
+static void _snapimage_destroy(struct snapimage *image)
+{
+ if (image->rq_processor != NULL)
+ _snapimage_stop(image);
+
+ if (image->queue) {
+ pr_info("Snapshot image queue cleanup\n");
+ blk_cleanup_queue(image->queue);
+ image->queue = NULL;
+ }
+
+ if (image->disk != NULL) {
+ struct gendisk *disk;
+
+ disk = image->disk;
+ image->disk = NULL;
+
+ pr_info("Snapshot image disk structure release\n");
+
+ disk->private_data = NULL;
+ put_disk(disk);
+ }
+
+ spin_lock(&snapimage_minors_lock);
+ bitmap_clear(snapimage_minors, MINOR(image->image_dev), 1u);
+ spin_unlock(&snapimage_minors_lock);
+}
+
+int snapimage_create(dev_t original_dev)
+{
+ int res = SUCCESS;
+ struct tracker *tracker = NULL;
+ struct snapimage *image = NULL;
+ struct gendisk *disk = NULL;
+ int minor;
+ struct blk_dev_info original_dev_info;
+
+ pr_info("Create snapshot image for device [%d:%d]\n", MAJOR(original_dev),
+ MINOR(original_dev));
+
+ res = blk_dev_get_info(original_dev, &original_dev_info);
+ if (res != SUCCESS) {
+ pr_err("Failed to obtain original device info\n");
+ return res;
+ }
+
+ res = tracker_find_by_dev_id(original_dev, &tracker);
+ if (res != SUCCESS) {
+ pr_err("Unable to create snapshot image: cannot find tracker for device [%d:%d]\n",
+ MAJOR(original_dev), MINOR(original_dev));
+ return res;
+ }
+
+ image = kzalloc(sizeof(struct snapimage), GFP_KERNEL);
+ if (image == NULL)
+ return -ENOMEM;
+
+ INIT_LIST_HEAD(&image->link);
+
+ do {
+ spin_lock(&snapimage_minors_lock);
+ minor = bitmap_find_free_region(snapimage_minors, SNAPIMAGE_MAX_DEVICES, 0);
+ spin_unlock(&snapimage_minors_lock);
+
+ if (minor < SUCCESS) {
+ pr_err("Failed to allocate minor for snapshot image device. errno=%d\n",
+ 0 - minor);
+ break;
+ }
+
+ image->rq_processor = NULL;
+
+ image->capacity = original_dev_info.count_sect;
+
+ image->defer_io = defer_io_get_resource(tracker->defer_io);
+ image->cbt_map = cbt_map_get_resource(tracker->cbt_map);
+ image->original_dev = original_dev;
+
+ image->image_dev = MKDEV(snapimage_major, minor);
+ pr_info("Snapshot image device id [%d:%d]\n", MAJOR(image->image_dev),
+ MINOR(image->image_dev));
+
+ atomic_set(&image->own_cnt, 0);
+
+ mutex_init(&image->open_locker);
+ image->open_bdev = NULL;
+ image->open_cnt = 0;
+
+ image->queue = blk_alloc_queue(NUMA_NO_NODE);
+ if (image->queue == NULL) {
+ res = -ENOMEM;
+ break;
+ }
+ image->queue->queuedata = image;
+
+ blk_queue_max_segment_size(image->queue, 1024 * PAGE_SIZE);
+
+ {
+ unsigned int physical_block_size = original_dev_info.physical_block_size;
+ unsigned short logical_block_size = original_dev_info.logical_block_size;
+
+ pr_info("Snapshot image physical block size %d\n", physical_block_size);
+ pr_info("Snapshot image logical block size %d\n", logical_block_size);
+
+ blk_queue_physical_block_size(image->queue, physical_block_size);
+ blk_queue_logical_block_size(image->queue, logical_block_size);
+ }
+ disk = alloc_disk(1); //only one partition on disk
+ if (disk == NULL) {
+ pr_err("Failed to allocate disk for snapshot image device\n");
+ res = -ENOMEM;
+ break;
+ }
+ image->disk = disk;
+
+ if (snprintf(disk->disk_name, DISK_NAME_LEN, "%s%d", SNAP_IMAGE_NAME, minor) < 0) {
+ pr_err("Unable to set disk name for snapshot image device: invalid minor %d\n",
+ minor);
+ res = -EINVAL;
+ break;
+ }
+
+ pr_info("Snapshot image disk name [%s]", disk->disk_name);
+
+ disk->flags |= GENHD_FL_NO_PART_SCAN;
+ disk->flags |= GENHD_FL_REMOVABLE;
+
+ disk->major = snapimage_major;
+ disk->minors = 1; // one disk have only one partition.
+ disk->first_minor = minor;
+
+ disk->private_data = image;
+
+ disk->fops = &snapimage_ops;
+ disk->queue = image->queue;
+
+ set_capacity(disk, image->capacity);
+ pr_info("Snapshot image device capacity %lld bytes",
+ (u64)from_sectors(image->capacity));
+
+ //res = -ENOMEM;
+ redirect_bio_queue_init(&image->image_queue);
+
+ {
+ struct task_struct *task =
+ kthread_create(snapimage_processor_thread, image, disk->disk_name);
+ if (IS_ERR(task)) {
+ res = PTR_ERR(task);
+ pr_err("Failed to create request processing thread for snapshot image device. errno=%d\n",
+ res);
+ break;
+ }
+ image->rq_processor = task;
+ }
+ init_waitqueue_head(&image->rq_complete_event);
+
+ init_waitqueue_head(&image->rq_proc_event);
+ wake_up_process(image->rq_processor);
+ } while (false);
+
+ if (res == SUCCESS) {
+ down_write(&snap_images_lock);
+ list_add_tail(&image->link, &snap_images);
+ up_write(&snap_images_lock);
+ } else {
+ _snapimage_destroy(image);
+ _snapimage_free(image);
+
+ kfree(image);
+ image = NULL;
+ }
+ return res;
+}
+
+static struct snapimage *snapimage_find(dev_t original_dev)
+{
+ struct snapimage *image = NULL;
+
+ down_read(&snap_images_lock);
+ if (!list_empty(&snap_images)) {
+ struct list_head *_list_head;
+
+ list_for_each(_list_head, &snap_images) {
+ struct snapimage *_image = list_entry(_list_head, struct snapimage, link);
+
+ if (_image->original_dev == original_dev) {
+ image = _image;
+ break;
+ }
+ }
+ }
+ up_read(&snap_images_lock);
+
+ return image;
+}
+
+void snapimage_stop(dev_t original_dev)
+{
+ struct snapimage *image;
+
+ pr_info("Snapshot image processing stop for original device [%d:%d]\n", MAJOR(original_dev),
+ MINOR(original_dev));
+
+ down_read(&snap_image_destroy_lock);
+
+ image = snapimage_find(original_dev);
+ if (image != NULL)
+ _snapimage_stop(image);
+ else
+ pr_err("Snapshot image [%d:%d] not found\n", MAJOR(original_dev),
+ MINOR(original_dev));
+
+ up_read(&snap_image_destroy_lock);
+}
+
+void snapimage_destroy(dev_t original_dev)
+{
+ struct snapimage *image = NULL;
+
+ pr_info("Destroy snapshot image for device [%d:%d]\n", MAJOR(original_dev),
+ MINOR(original_dev));
+
+ down_write(&snap_images_lock);
+ if (!list_empty(&snap_images)) {
+ struct list_head *_list_head;
+
+ list_for_each(_list_head, &snap_images) {
+ struct snapimage *_image = list_entry(_list_head, struct snapimage, link);
+
+ if (_image->original_dev == original_dev) {
+ image = _image;
+ list_del(&image->link);
+ break;
+ }
+ }
+ }
+ up_write(&snap_images_lock);
+
+ if (image != NULL) {
+ down_write(&snap_image_destroy_lock);
+
+ _snapimage_destroy(image);
+ _snapimage_free(image);
+
+ kfree(image);
+ image = NULL;
+
+ up_write(&snap_image_destroy_lock);
+ } else
+ pr_err("Snapshot image [%d:%d] not found\n", MAJOR(original_dev),
+ MINOR(original_dev));
+}
+
+void snapimage_destroy_for(dev_t *p_dev, int count)
+{
+ int inx = 0;
+
+ for (; inx < count; ++inx)
+ snapimage_destroy(p_dev[inx]);
+}
+
+int snapimage_create_for(dev_t *p_dev, int count)
+{
+ int res = SUCCESS;
+ int inx = 0;
+
+ for (; inx < count; ++inx) {
+ res = snapimage_create(p_dev[inx]);
+ if (res != SUCCESS) {
+ pr_err("Failed to create snapshot image for original device [%d:%d]\n",
+ MAJOR(p_dev[inx]), MINOR(p_dev[inx]));
+ break;
+ }
+ }
+ if (res != SUCCESS)
+ if (inx > 0)
+ snapimage_destroy_for(p_dev, inx - 1);
+ return res;
+}
+
+int snapimage_init(void)
+{
+ int res = SUCCESS;
+
+ res = register_blkdev(snapimage_major, SNAP_IMAGE_NAME);
+ if (res >= SUCCESS) {
+ snapimage_major = res;
+ pr_info("Snapshot image block device major %d was registered\n", snapimage_major);
+ res = SUCCESS;
+
+ spin_lock(&snapimage_minors_lock);
+ snapimage_minors = bitmap_zalloc(SNAPIMAGE_MAX_DEVICES, GFP_KERNEL);
+ spin_unlock(&snapimage_minors_lock);
+
+ if (snapimage_minors == NULL)
+ pr_err("Failed to initialize bitmap of minors\n");
+ } else
+ pr_err("Failed to register snapshot image block device. errno=%d\n", res);
+
+ return res;
+}
+
+void snapimage_done(void)
+{
+ down_write(&snap_image_destroy_lock);
+ while (true) {
+ struct snapimage *image = NULL;
+
+ down_write(&snap_images_lock);
+ if (!list_empty(&snap_images)) {
+ image = list_entry(snap_images.next, struct snapimage, link);
+
+ list_del(&image->link);
+ }
+ up_write(&snap_images_lock);
+
+ if (image == NULL)
+ break;
+
+ pr_err("Snapshot image for device was unexpectedly removed [%d:%d]\n",
+ MAJOR(image->original_dev), MINOR(image->original_dev));
+
+ _snapimage_destroy(image);
+ _snapimage_free(image);
+
+ kfree(image);
+ image = NULL;
+ }
+
+ spin_lock(&snapimage_minors_lock);
+ bitmap_free(snapimage_minors);
+ snapimage_minors = NULL;
+ spin_unlock(&snapimage_minors_lock);
+
+ if (!list_empty(&snap_images))
+ pr_err("Failed to release snapshot images container\n");
+
+ unregister_blkdev(snapimage_major, SNAP_IMAGE_NAME);
+ pr_info("Snapshot image block device [%d] was unregistered\n", snapimage_major);
+
+ up_write(&snap_image_destroy_lock);
+}
+
+int snapimage_collect_images(int count, struct image_info_s *p_user_image_info, int *p_real_count)
+{
+ int res = SUCCESS;
+ int real_count = 0;
+
+ down_read(&snap_images_lock);
+ if (!list_empty(&snap_images)) {
+ struct list_head *_list_head;
+
+ list_for_each(_list_head, &snap_images)
+ real_count++;
+ }
+ up_read(&snap_images_lock);
+ *p_real_count = real_count;
+
+ if (count < real_count)
+ res = -ENODATA;
+
+ real_count = min(count, real_count);
+ if (real_count > 0) {
+ unsigned long len;
+ struct image_info_s *p_kernel_image_info = NULL;
+ size_t buff_size;
+
+ buff_size = sizeof(struct image_info_s) * real_count;
+ p_kernel_image_info = kzalloc(buff_size, GFP_KERNEL);
+ if (p_kernel_image_info == NULL) {
+ pr_err("Unable to collect snapshot images: not enough memory. size=%zu\n",
+ buff_size);
+ return res = -ENOMEM;
+ }
+
+ down_read(&snap_image_destroy_lock);
+ down_read(&snap_images_lock);
+
+ if (!list_empty(&snap_images)) {
+ size_t inx = 0;
+ struct list_head *_list_head;
+
+ list_for_each(_list_head, &snap_images) {
+ struct snapimage *img =
+ list_entry(_list_head, struct snapimage, link);
+
+ real_count++;
+
+ p_kernel_image_info[inx].original_dev_id.major =
+ MAJOR(img->original_dev);
+ p_kernel_image_info[inx].original_dev_id.minor =
+ MINOR(img->original_dev);
+
+ p_kernel_image_info[inx].snapshot_dev_id.major =
+ MAJOR(img->image_dev);
+ p_kernel_image_info[inx].snapshot_dev_id.minor =
+ MINOR(img->image_dev);
+
+ ++inx;
+ if (inx > real_count)
+ break;
+ }
+ }
+
+ up_read(&snap_images_lock);
+ up_read(&snap_image_destroy_lock);
+
+ len = copy_to_user(p_user_image_info, p_kernel_image_info, buff_size);
+ if (len != 0) {
+ pr_err("Unable to collect snapshot images: failed to copy data to user buffer\n");
+ res = -ENODATA;
+ }
+
+ kfree(p_kernel_image_info);
+ }
+
+ return res;
+}
+
+int snapimage_mark_dirty_blocks(dev_t image_dev_id, struct block_range_s *block_ranges,
+ unsigned int count)
+{
+ size_t inx = 0;
+ int res = SUCCESS;
+
+ pr_info("Marking [%d] dirty blocks for image device [%d:%d]\n", count, MAJOR(image_dev_id),
+ MINOR(image_dev_id));
+
+ down_read(&snap_image_destroy_lock);
+ do {
+ struct snapimage *image = snapimage_find(image_dev_id);
+
+ if (image == NULL) {
+ pr_err("Cannot find device [%d:%d]\n", MAJOR(image_dev_id),
+ MINOR(image_dev_id));
+ res = -ENODEV;
+ break;
+ }
+
+ for (inx = 0; inx < count; ++inx) {
+ sector_t ofs = (sector_t)block_ranges[inx].ofs;
+ sector_t cnt = (sector_t)block_ranges[inx].cnt;
+
+ res = cbt_map_set_both(image->cbt_map, ofs, cnt);
+ if (res != SUCCESS) {
+ pr_err("Failed to set CBT table. errno=%d\n", res);
+ break;
+ }
+ }
+ } while (false);
+ up_read(&snap_image_destroy_lock);
+
+ return res;
+}
diff --git a/drivers/block/blk-snap/snapimage.h b/drivers/block/blk-snap/snapimage.h
new file mode 100644
index 000000000000..67995c321496
--- /dev/null
+++ b/drivers/block/blk-snap/snapimage.h
@@ -0,0 +1,16 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#pragma once
+
+#include "blk-snap-ctl.h"
+
+int snapimage_init(void);
+void snapimage_done(void);
+int snapimage_create_for(dev_t *p_dev, int count);
+
+void snapimage_stop(dev_t original_dev);
+void snapimage_destroy(dev_t original_dev);
+
+int snapimage_collect_images(int count, struct image_info_s *p_user_image_info, int *p_real_count);
+
+int snapimage_mark_dirty_blocks(dev_t image_dev_id, struct block_range_s *block_ranges,
+ unsigned int count);
diff --git a/drivers/block/blk-snap/snapshot.c b/drivers/block/blk-snap/snapshot.c
new file mode 100644
index 000000000000..fdef713103d2
--- /dev/null
+++ b/drivers/block/blk-snap/snapshot.c
@@ -0,0 +1,225 @@
+// SPDX-License-Identifier: GPL-2.0
+#define BLK_SNAP_SECTION "-snapshot"
+#include "common.h"
+#include "snapshot.h"
+#include "tracker.h"
+#include "snapimage.h"
+#include "tracking.h"
+
+LIST_HEAD(snapshots);
+DECLARE_RWSEM(snapshots_lock);
+
+
+static int _snapshot_remove_device(dev_t dev_id)
+{
+ int result;
+ struct tracker *tracker = NULL;
+
+ result = tracker_find_by_dev_id(dev_id, &tracker);
+ if (result != SUCCESS) {
+ if (result == -ENODEV)
+ pr_err("Cannot find device by device id=[%d:%d]\n", MAJOR(dev_id),
+ MINOR(dev_id));
+ else
+ pr_err("Failed to find device by device id=[%d:%d]\n", MAJOR(dev_id),
+ MINOR(dev_id));
+ return SUCCESS;
+ }
+
+ if (result != SUCCESS)
+ return result;
+
+ tracker->snapshot_id = 0ull;
+
+ pr_info("Device [%d:%d] successfully removed from snapshot\n", MAJOR(dev_id),
+ MINOR(dev_id));
+ return SUCCESS;
+}
+
+static void _snapshot_cleanup(struct snapshot *snapshot)
+{
+ int inx;
+
+ for (inx = 0; inx < snapshot->dev_id_set_size; ++inx) {
+
+ if (_snapshot_remove_device(snapshot->dev_id_set[inx]) != SUCCESS)
+ pr_err("Failed to remove device [%d:%d] from snapshot\n",
+ MAJOR(snapshot->dev_id_set[inx]), MINOR(snapshot->dev_id_set[inx]));
+ }
+
+ if (snapshot->dev_id_set != NULL)
+ kfree(snapshot->dev_id_set);
+ kfree(snapshot);
+}
+
+static void _snapshot_destroy(struct snapshot *snapshot)
+{
+ size_t inx;
+
+ for (inx = 0; inx < snapshot->dev_id_set_size; ++inx)
+ snapimage_stop(snapshot->dev_id_set[inx]);
+
+ pr_info("Release snapshot [0x%llx]\n", snapshot->id);
+
+ tracker_release_snapshot(snapshot->dev_id_set, snapshot->dev_id_set_size);
+
+ for (inx = 0; inx < snapshot->dev_id_set_size; ++inx)
+ snapimage_destroy(snapshot->dev_id_set[inx]);
+
+ _snapshot_cleanup(snapshot);
+}
+
+
+static int _snapshot_new(dev_t *p_dev, int count, struct snapshot **pp_snapshot)
+{
+ struct snapshot *p_snapshot = NULL;
+ dev_t *snap_set = NULL;
+
+ p_snapshot = kzalloc(sizeof(struct snapshot), GFP_KERNEL);
+ if (p_snapshot == NULL)
+ return -ENOMEM;
+
+ INIT_LIST_HEAD(&p_snapshot->link);
+
+ p_snapshot->id = (unsigned long)(p_snapshot);
+
+ snap_set = kcalloc(count, sizeof(dev_t), GFP_KERNEL);
+ if (snap_set == NULL) {
+ kfree(p_snapshot);
+
+ pr_err("Unable to create snapshot: faile to allocate memory for snapshot map\n");
+ return -ENOMEM;
+ }
+ memcpy(snap_set, p_dev, sizeof(dev_t) * count);
+
+ p_snapshot->dev_id_set_size = count;
+ p_snapshot->dev_id_set = snap_set;
+
+ down_write(&snapshots_lock);
+ list_add_tail(&snapshots, &p_snapshot->link);
+ up_write(&snapshots_lock);
+
+ *pp_snapshot = p_snapshot;
+
+ return SUCCESS;
+}
+
+void snapshot_done(void)
+{
+ struct snapshot *snap;
+
+ pr_info("Removing all snapshots\n");
+ do {
+ snap = NULL;
+ down_write(&snapshots_lock);
+ if (!list_empty(&snapshots)) {
+ struct snapshot *snap = list_entry(snapshots.next, struct snapshot, link);
+
+ list_del(&snap->link);
+ }
+ up_write(&snapshots_lock);
+
+ if (snap)
+ _snapshot_destroy(snap);
+
+ } while (snap);
+}
+
+int snapshot_create(dev_t *dev_id_set, unsigned int dev_id_set_size,
+ unsigned long long *p_snapshot_id)
+{
+ struct snapshot *snapshot = NULL;
+ int result = SUCCESS;
+ unsigned int inx;
+
+ pr_info("Create snapshot for devices:\n");
+ for (inx = 0; inx < dev_id_set_size; ++inx)
+ pr_info("\t%d:%d\n", MAJOR(dev_id_set[inx]), MINOR(dev_id_set[inx]));
+
+ result = _snapshot_new(dev_id_set, dev_id_set_size, &snapshot);
+ if (result != SUCCESS) {
+ pr_err("Unable to create snapshot: failed to allocate snapshot structure\n");
+ return result;
+ }
+
+ do {
+ result = -ENODEV;
+ for (inx = 0; inx < snapshot->dev_id_set_size; ++inx) {
+ dev_t dev_id = snapshot->dev_id_set[inx];
+
+ result = tracking_add(dev_id, snapshot->id);
+ if (result == -EALREADY)
+ result = SUCCESS;
+ else if (result != SUCCESS) {
+ pr_err("Unable to create snapshot\n");
+ pr_err("Failed to add device [%d:%d] to snapshot tracking\n",
+ MAJOR(dev_id), MINOR(dev_id));
+ break;
+ }
+ }
+ if (result != SUCCESS)
+ break;
+
+ result = tracker_capture_snapshot(snapshot->dev_id_set, snapshot->dev_id_set_size);
+ if (result != SUCCESS) {
+ pr_err("Unable to create snapshot: failed to capture snapshot [0x%llx]\n",
+ snapshot->id);
+ break;
+ }
+
+ result = snapimage_create_for(snapshot->dev_id_set, snapshot->dev_id_set_size);
+ if (result != SUCCESS) {
+ pr_err("Unable to create snapshot\n");
+ pr_err("Failed to create snapshot image devices\n");
+
+ tracker_release_snapshot(snapshot->dev_id_set, snapshot->dev_id_set_size);
+ break;
+ }
+
+ *p_snapshot_id = snapshot->id;
+ pr_info("Snapshot [0x%llx] was created\n", snapshot->id);
+ } while (false);
+
+ if (result != SUCCESS) {
+ pr_info("Snapshot [0x%llx] cleanup\n", snapshot->id);
+
+ down_write(&snapshots_lock);
+ list_del(&snapshot->link);
+ up_write(&snapshots_lock);
+
+ _snapshot_cleanup(snapshot);
+ }
+ return result;
+}
+
+int snapshot_destroy(unsigned long long snapshot_id)
+{
+ struct snapshot *snapshot = NULL;
+
+ pr_info("Destroy snapshot [0x%llx]\n", snapshot_id);
+
+ down_read(&snapshots_lock);
+ if (!list_empty(&snapshots)) {
+ struct list_head *_head;
+
+ list_for_each(_head, &snapshots) {
+ struct snapshot *_snap = list_entry(_head, struct snapshot, link);
+
+ if (_snap->id == snapshot_id) {
+ snapshot = _snap;
+ list_del(&snapshot->link);
+ break;
+ }
+ }
+ }
+ up_read(&snapshots_lock);
+
+ if (snapshot == NULL) {
+ pr_err("Unable to destroy snapshot [0x%llx]: cannot find snapshot by id\n",
+ snapshot_id);
+ return -ENODEV;
+ }
+
+ _snapshot_destroy(snapshot);
+ return SUCCESS;
+}
diff --git a/drivers/block/blk-snap/snapshot.h b/drivers/block/blk-snap/snapshot.h
new file mode 100644
index 000000000000..59fb4dba0241
--- /dev/null
+++ b/drivers/block/blk-snap/snapshot.h
@@ -0,0 +1,17 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#pragma once
+
+struct snapshot {
+ struct list_head link;
+ unsigned long long id;
+
+ dev_t *dev_id_set; //array of assigned devices
+ int dev_id_set_size;
+};
+
+void snapshot_done(void);
+
+int snapshot_create(dev_t *dev_id_set, unsigned int dev_id_set_size,
+ unsigned long long *p_snapshot_id);
+
+int snapshot_destroy(unsigned long long snapshot_id);
diff --git a/drivers/block/blk-snap/snapstore.c b/drivers/block/blk-snap/snapstore.c
new file mode 100644
index 000000000000..0bedeaeec021
--- /dev/null
+++ b/drivers/block/blk-snap/snapstore.c
@@ -0,0 +1,929 @@
+// SPDX-License-Identifier: GPL-2.0
+#define BLK_SNAP_SECTION "-snapstore"
+#include "common.h"
+#include "snapstore.h"
+#include "snapstore_device.h"
+#include "big_buffer.h"
+#include "params.h"
+
+LIST_HEAD(snapstores);
+DECLARE_RWSEM(snapstores_lock);
+
+bool _snapstore_check_halffill(struct snapstore *snapstore, sector_t *fill_status)
+{
+ struct blk_descr_pool *pool = NULL;
+
+ if (snapstore->file)
+ pool = &snapstore->file->pool;
+#ifdef CONFIG_BLK_SNAP_SNAPSTORE_MULTIDEV
+ else if (snapstore->multidev)
+ pool = &snapstore->multidev->pool;
+#endif
+ else if (snapstore->mem)
+ pool = &snapstore->mem->pool;
+
+ if (pool)
+ return blk_descr_pool_check_halffill(pool, snapstore->empty_limit, fill_status);
+
+ return false;
+}
+
+void _snapstore_destroy(struct snapstore *snapstore)
+{
+ sector_t fill_status;
+
+ pr_info("Destroy snapstore with id %pUB\n", &snapstore->id);
+
+ _snapstore_check_halffill(snapstore, &fill_status);
+
+ down_write(&snapstores_lock);
+ list_del(&snapstore->link);
+ up_write(&snapstores_lock);
+
+ if (snapstore->mem != NULL)
+ snapstore_mem_destroy(snapstore->mem);
+ if (snapstore->multidev != NULL)
+ snapstore_multidev_destroy(snapstore->multidev);
+ if (snapstore->file != NULL)
+ snapstore_file_destroy(snapstore->file);
+
+ if (snapstore->ctrl_pipe) {
+ struct ctrl_pipe *pipe;
+
+ pipe = snapstore->ctrl_pipe;
+ snapstore->ctrl_pipe = NULL;
+
+ ctrl_pipe_request_terminate(pipe, fill_status);
+
+ ctrl_pipe_put_resource(pipe);
+ }
+
+ kfree(snapstore);
+}
+
+static void _snapstore_destroy_cb(struct kref *kref)
+{
+ struct snapstore *snapstore = container_of(kref, struct snapstore, refcount);
+
+ _snapstore_destroy(snapstore);
+}
+
+struct snapstore *snapstore_get(struct snapstore *snapstore)
+{
+ if (snapstore)
+ kref_get(&snapstore->refcount);
+
+ return snapstore;
+}
+
+void snapstore_put(struct snapstore *snapstore)
+{
+ if (snapstore)
+ kref_put(&snapstore->refcount, _snapstore_destroy_cb);
+}
+
+void snapstore_done(void)
+{
+ bool is_empty;
+
+ down_read(&snapstores_lock);
+ is_empty = list_empty(&snapstores);
+ up_read(&snapstores_lock);
+
+ if (!is_empty)
+ pr_err("Unable to perform snapstore cleanup: container is not empty\n");
+}
+
+int snapstore_create(uuid_t *id, dev_t snapstore_dev_id, dev_t *dev_id_set,
+ size_t dev_id_set_length)
+{
+ int res = SUCCESS;
+ size_t dev_id_inx;
+ struct snapstore *snapstore = NULL;
+
+ if (dev_id_set_length == 0)
+ return -EINVAL;
+
+ snapstore = kzalloc(sizeof(struct snapstore), GFP_KERNEL);
+ if (snapstore == NULL)
+ return -ENOMEM;
+
+ INIT_LIST_HEAD(&snapstore->link);
+ uuid_copy(&snapstore->id, id);
+
+ pr_info("Create snapstore with id %pUB\n", &snapstore->id);
+
+ snapstore->mem = NULL;
+ snapstore->multidev = NULL;
+ snapstore->file = NULL;
+
+ snapstore->ctrl_pipe = NULL;
+ snapstore->empty_limit = (sector_t)(64 * (1024 * 1024 / SECTOR_SIZE)); //by default value
+ snapstore->halffilled = false;
+ snapstore->overflowed = false;
+
+ if (snapstore_dev_id == 0)
+ pr_info("Memory snapstore create\n");
+
+#ifdef CONFIG_BLK_SNAP_SNAPSTORE_MULTIDEV
+ else if (snapstore_dev_id == 0xFFFFffff) {
+ struct snapstore_multidev *multidev = NULL;
+
+ res = snapstore_multidev_create(&multidev);
+ if (res != SUCCESS) {
+ kfree(snapstore);
+
+ pr_err("Failed to create multidevice snapstore %pUB\n", id);
+ return res;
+ }
+ snapstore->multidev = multidev;
+ }
+#endif
+ else {
+ struct snapstore_file *file = NULL;
+
+ res = snapstore_file_create(snapstore_dev_id, &file);
+ if (res != SUCCESS) {
+ kfree(snapstore);
+
+ pr_err("Failed to create snapstore file for snapstore %pUB\n", id);
+ return res;
+ }
+ snapstore->file = file;
+ }
+
+ down_write(&snapstores_lock);
+ list_add_tail(&snapstores, &snapstore->link);
+ up_write(&snapstores_lock);
+
+ kref_init(&snapstore->refcount);
+
+ for (dev_id_inx = 0; dev_id_inx < dev_id_set_length; ++dev_id_inx) {
+ res = snapstore_device_create(dev_id_set[dev_id_inx], snapstore);
+ if (res != SUCCESS)
+ break;
+ }
+
+ if (res != SUCCESS)
+ snapstore_device_cleanup(id);
+
+ snapstore_put(snapstore);
+ return res;
+}
+
+#ifdef CONFIG_BLK_SNAP_SNAPSTORE_MULTIDEV
+int snapstore_create_multidev(uuid_t *id, dev_t *dev_id_set, size_t dev_id_set_length)
+{
+ int res = SUCCESS;
+ size_t dev_id_inx;
+ struct snapstore *snapstore = NULL;
+ struct snapstore_multidev *multidev = NULL;
+
+ if (dev_id_set_length == 0)
+ return -EINVAL;
+
+ snapstore = kzalloc(sizeof(struct snapstore), GFP_KERNEL);
+ if (snapstore == NULL)
+ return -ENOMEM;
+
+ INIT_LIST_HEAD(&snapstore->link);
+
+ uuid_copy(&snapstore->id, id);
+
+ pr_info("Create snapstore with id %pUB\n", &snapstore->id);
+
+ snapstore->mem = NULL;
+ snapstore->file = NULL;
+ snapstore->multidev = NULL;
+
+ snapstore->ctrl_pipe = NULL;
+ snapstore->empty_limit = (sector_t)(64 * (1024 * 1024 / SECTOR_SIZE)); //by default value
+ snapstore->halffilled = false;
+ snapstore->overflowed = false;
+
+ res = snapstore_multidev_create(&multidev);
+ if (res != SUCCESS) {
+ kfree(snapstore);
+
+ pr_err("Failed to create snapstore file for snapstore %pUB\n", id);
+ return res;
+ }
+ snapstore->multidev = multidev;
+
+ down_write(&snapstores_lock);
+ list_add_tail(&snapstore->link, &snapstores);
+ up_write(&snapstores_lock);
+
+ kref_init(&snapstore->refcount);
+
+ for (dev_id_inx = 0; dev_id_inx < dev_id_set_length; ++dev_id_inx) {
+ res = snapstore_device_create(dev_id_set[dev_id_inx], snapstore);
+ if (res != SUCCESS)
+ break;
+ }
+
+ if (res != SUCCESS)
+ snapstore_device_cleanup(id);
+
+ snapstore_put(snapstore);
+ return res;
+}
+#endif
+
+int snapstore_cleanup(uuid_t *id, u64 *filled_bytes)
+{
+ int res;
+ sector_t filled;
+
+ res = snapstore_check_halffill(id, &filled);
+ if (res == SUCCESS) {
+ *filled_bytes = (u64)from_sectors(filled);
+
+ pr_info("Snapstore fill size: %lld MiB\n", (*filled_bytes >> 20));
+ } else {
+ *filled_bytes = -1;
+ pr_err("Failed to obtain snapstore data filled size\n");
+ }
+
+ return snapstore_device_cleanup(id);
+}
+
+struct snapstore *_snapstore_find(uuid_t *id)
+{
+ struct snapstore *result = NULL;
+
+ down_read(&snapstores_lock);
+ if (!list_empty(&snapstores)) {
+ struct list_head *_head;
+
+ list_for_each(_head, &snapstores) {
+ struct snapstore *snapstore = list_entry(_head, struct snapstore, link);
+
+ if (uuid_equal(&snapstore->id, id)) {
+ result = snapstore;
+ break;
+ }
+ }
+ }
+ up_read(&snapstores_lock);
+
+ return result;
+}
+
+int snapstore_stretch_initiate(uuid_t *unique_id, struct ctrl_pipe *ctrl_pipe, sector_t empty_limit)
+{
+ struct snapstore *snapstore;
+
+ snapstore = _snapstore_find(unique_id);
+ if (snapstore == NULL) {
+ pr_err("Unable to initiate stretch snapstore: ");
+ pr_err("cannot find snapstore by uuid %pUB\n", unique_id);
+ return -ENODATA;
+ }
+
+ snapstore->ctrl_pipe = ctrl_pipe_get_resource(ctrl_pipe);
+ snapstore->empty_limit = empty_limit;
+
+ return SUCCESS;
+}
+
+int snapstore_add_memory(uuid_t *id, unsigned long long sz)
+{
+ int res = SUCCESS;
+ struct snapstore *snapstore = NULL;
+ size_t available_blocks = (size_t)(sz >> (snapstore_block_shift() + SECTOR_SHIFT));
+ size_t current_block = 0;
+
+ pr_info("Adding %lld bytes to the snapstore\n", sz);
+
+ snapstore = _snapstore_find(id);
+ if (snapstore == NULL) {
+ pr_err("Unable to add memory block to the snapstore: ");
+ pr_err("cannot found snapstore by id %pUB\n", id);
+ return -ENODATA;
+ }
+
+ if (snapstore->file != NULL) {
+ pr_err("Unable to add memory block to the snapstore: ");
+ pr_err("snapstore file is already created\n");
+ return -EINVAL;
+ }
+#ifdef CONFIG_BLK_SNAP_SNAPSTORE_MULTIDEV
+ if (snapstore->multidev != NULL) {
+ pr_err("Unable to add memory block to the snapstore: ");
+ pr_err("snapstore multidevice is already created\n");
+ return -EINVAL;
+ }
+#endif
+ if (snapstore->mem != NULL) {
+ pr_err("Unable to add memory block to the snapstore: ");
+ pr_err("snapstore memory buffer is already created\n");
+ return -EINVAL;
+ }
+
+ snapstore->mem = snapstore_mem_create(available_blocks);
+ for (current_block = 0; current_block < available_blocks; ++current_block) {
+ void *buffer = snapstore_mem_get_block(snapstore->mem);
+
+ if (buffer == NULL) {
+ pr_err("Unable to add memory block to the snapstore: ");
+ pr_err("not enough memory\n");
+ res = -ENOMEM;
+ break;
+ }
+
+ res = blk_descr_mem_pool_add(&snapstore->mem->pool, buffer);
+ if (res != SUCCESS) {
+ pr_err("Unable to add memory block to the snapstore: ");
+ pr_err("failed to initialize new block\n");
+ break;
+ }
+ }
+ if (res != SUCCESS) {
+ snapstore_mem_destroy(snapstore->mem);
+ snapstore->mem = NULL;
+ }
+
+ return res;
+}
+
+int rangelist_add(struct list_head *rglist, struct blk_range *rg)
+{
+ struct blk_range_link *range_link;
+
+ range_link = kzalloc(sizeof(struct blk_range_link), GFP_KERNEL);
+ if (range_link == NULL)
+ return -ENOMEM;
+
+ INIT_LIST_HEAD(&range_link->link);
+
+ range_link->rg.ofs = rg->ofs;
+ range_link->rg.cnt = rg->cnt;
+
+ list_add_tail(&range_link->link, rglist);
+
+ return SUCCESS;
+}
+
+int snapstore_add_file(uuid_t *id, struct big_buffer *ranges, size_t ranges_cnt)
+{
+ int res = SUCCESS;
+ struct snapstore *snapstore = NULL;
+ struct snapstore_device *snapstore_device = NULL;
+ sector_t current_blk_size = 0;
+ LIST_HEAD(blk_rangelist);
+ size_t inx;
+
+ pr_info("Snapstore add %zu ranges\n", ranges_cnt);
+
+ if ((ranges_cnt == 0) || (ranges == NULL))
+ return -EINVAL;
+
+ snapstore = _snapstore_find(id);
+ if (snapstore == NULL) {
+ pr_err("Unable to add file to snapstore: ");
+ pr_err("cannot find snapstore by id %pUB\n", id);
+ return -ENODATA;
+ }
+
+ if (snapstore->file == NULL) {
+ pr_err("Unable to add file to snapstore: ");
+ pr_err("snapstore file was not initialized\n");
+ return -EFAULT;
+ }
+
+ snapstore_device =
+ snapstore_device_find_by_dev_id(snapstore->file->blk_dev_id); //for zeroed
+
+ for (inx = 0; inx < ranges_cnt; ++inx) {
+ size_t blocks_count = 0;
+ sector_t range_offset = 0;
+
+ struct blk_range range;
+ struct ioctl_range_s *ioctl_range;
+
+ ioctl_range = big_buffer_get_element(ranges, inx, sizeof(struct ioctl_range_s));
+ if (ioctl_range == NULL) {
+ pr_err("Invalid count of ranges\n");
+ res = -ENODATA;
+ break;
+ }
+
+ range.ofs = (sector_t)to_sectors(ioctl_range->left);
+ range.cnt = (blkcnt_t)to_sectors(ioctl_range->right) - range.ofs;
+
+ while (range_offset < range.cnt) {
+ struct blk_range rg;
+
+ rg.ofs = range.ofs + range_offset;
+ rg.cnt = min_t(sector_t, (range.cnt - range_offset),
+ (snapstore_block_size() - current_blk_size));
+
+ range_offset += rg.cnt;
+
+ res = rangelist_add(&blk_rangelist, &rg);
+ if (res != SUCCESS) {
+ pr_err("Unable to add file to snapstore: ");
+ pr_err("cannot add range to rangelist\n");
+ break;
+ }
+
+ //zero sectors logic
+ if (snapstore_device != NULL) {
+ res = rangevector_add(&snapstore_device->zero_sectors, &rg);
+ if (res != SUCCESS) {
+ pr_err("Unable to add file to snapstore: ");
+ pr_err("cannot add range to zero_sectors tree\n");
+ break;
+ }
+ }
+
+ current_blk_size += rg.cnt;
+
+ if (current_blk_size == snapstore_block_size()) { //allocate block
+ res = blk_descr_file_pool_add(&snapstore->file->pool,
+ &blk_rangelist);
+ if (res != SUCCESS) {
+ pr_err("Unable to add file to snapstore: ");
+ pr_err("cannot initialize new block\n");
+ break;
+ }
+
+ snapstore->halffilled = false;
+
+ current_blk_size = 0;
+ INIT_LIST_HEAD(&blk_rangelist); //renew list
+ ++blocks_count;
+ }
+ }
+ if (res != SUCCESS)
+ break;
+ }
+
+ if ((res == SUCCESS) && (current_blk_size != 0))
+ pr_warn("Snapstore portion was not ordered by Copy-on-Write block size\n");
+
+ return res;
+}
+
+#ifdef CONFIG_BLK_SNAP_SNAPSTORE_MULTIDEV
+static int rangelist_ex_add(struct list_head *list, struct blk_range *rg,
+ struct block_device *blk_dev)
+{
+ struct blk_range_link_ex *range_link =
+ kzalloc(sizeof(struct blk_range_link_ex), GFP_KERNEL);
+ if (range_link == NULL)
+ return -ENOMEM;
+
+ INIT_LIST_HEAD(&range_link->link);
+
+ range_link->rg.ofs = rg->ofs;
+ range_link->rg.cnt = rg->cnt;
+ range_link->blk_dev = blk_dev;
+
+ list_add_tail(&range_link->link, list);
+
+ return SUCCESS;
+}
+
+int snapstore_add_multidev(uuid_t *id, dev_t dev_id, struct big_buffer *ranges, size_t ranges_cnt)
+{
+ int res = SUCCESS;
+ struct snapstore *snapstore = NULL;
+ sector_t current_blk_size = 0;
+ size_t inx;
+ LIST_HEAD(blk_rangelist);
+
+ pr_info("Snapstore add %zu ranges for device [%d:%d]\n", ranges_cnt, MAJOR(dev_id),
+ MINOR(dev_id));
+
+ if ((ranges_cnt == 0) || (ranges == NULL))
+ return -EINVAL;
+
+ snapstore = _snapstore_find(id);
+ if (snapstore == NULL) {
+ pr_err("Unable to add file to multidevice snapstore: ");
+ pr_err("cannot find snapstore by id %pUB\n", id);
+ return -ENODATA;
+ }
+
+ if (snapstore->multidev == NULL) {
+ pr_err("Unable to add file to multidevice snapstore: ");
+ pr_err("it was not initialized\n");
+ return -EFAULT;
+ }
+
+ for (inx = 0; inx < ranges_cnt; ++inx) {
+ size_t blocks_count = 0;
+ sector_t range_offset = 0;
+ struct blk_range range;
+ struct ioctl_range_s *data;
+
+ data = big_buffer_get_element(ranges, inx, sizeof(struct ioctl_range_s));
+ if (data == NULL) {
+ pr_err("Invalid count of ranges\n");
+ res = -ENODATA;
+ break;
+ }
+
+ range.ofs = (sector_t)to_sectors(data->left);
+ range.cnt = (blkcnt_t)to_sectors(data->right) - range.ofs;
+
+ while (range_offset < range.cnt) {
+ struct blk_range rg;
+ struct block_device *blk_dev = NULL;
+
+ rg.ofs = range.ofs + range_offset;
+ rg.cnt = min_t(sector_t,
+ range.cnt - range_offset,
+ snapstore_block_size() - current_blk_size);
+
+ range_offset += rg.cnt;
+
+ blk_dev = snapstore_multidev_get_device(snapstore->multidev, dev_id);
+ if (blk_dev == NULL) {
+ pr_err("Cannot find or open device [%d:%d] for multidevice snapstore\n",
+ MAJOR(dev_id), MINOR(dev_id));
+ res = -ENODEV;
+ break;
+ }
+
+ res = rangelist_ex_add(&blk_rangelist, &rg, blk_dev);
+ if (res != SUCCESS) {
+ pr_err("Unable to add file to multidevice snapstore: ");
+ pr_err("failed to add range to rangelist\n");
+ break;
+ }
+
+ /*
+ * zero sectors logic is not implemented for multidevice snapstore
+ */
+
+ current_blk_size += rg.cnt;
+
+ if (current_blk_size == snapstore_block_size()) { //allocate block
+ res = blk_descr_multidev_pool_add(&snapstore->multidev->pool,
+ &blk_rangelist);
+ if (res != SUCCESS) {
+ pr_err("Unable to add file to multidevice snapstore: ");
+ pr_err("failed to initialize new block\n");
+ break;
+ }
+
+ snapstore->halffilled = false;
+
+ current_blk_size = 0;
+ INIT_LIST_HEAD(&blk_rangelist);
+ ++blocks_count;
+ }
+ }
+ if (res != SUCCESS)
+ break;
+ }
+
+ if ((res == SUCCESS) && (current_blk_size != 0))
+ pr_warn("Snapstore portion was not ordered by Copy-on-Write block size\n");
+
+ return res;
+}
+#endif
+
+void snapstore_order_border(struct blk_range *in, struct blk_range *out)
+{
+ struct blk_range unorder;
+
+ unorder.ofs = in->ofs & snapstore_block_mask();
+ out->ofs = in->ofs & ~snapstore_block_mask();
+ out->cnt = in->cnt + unorder.ofs;
+
+ unorder.cnt = out->cnt & snapstore_block_mask();
+ if (unorder.cnt != 0)
+ out->cnt += (snapstore_block_size() - unorder.cnt);
+}
+
+union blk_descr_unify snapstore_get_empty_block(struct snapstore *snapstore)
+{
+ union blk_descr_unify result = { NULL };
+
+ if (snapstore->overflowed)
+ return result;
+
+ if (snapstore->file != NULL)
+ result = blk_descr_file_pool_take(&snapstore->file->pool);
+ else if (snapstore->multidev != NULL)
+ result = blk_descr_multidev_pool_take(&snapstore->multidev->pool);
+ else if (snapstore->mem != NULL)
+ result = blk_descr_mem_pool_take(&snapstore->mem->pool);
+
+ if (result.ptr == NULL) {
+ if (snapstore->ctrl_pipe) {
+ sector_t fill_status;
+
+ _snapstore_check_halffill(snapstore, &fill_status);
+ ctrl_pipe_request_overflow(snapstore->ctrl_pipe, -EINVAL,
+ (u64)from_sectors(fill_status));
+ }
+ snapstore->overflowed = true;
+ }
+
+ return result;
+}
+
+int snapstore_check_halffill(uuid_t *unique_id, sector_t *fill_status)
+{
+ struct snapstore *snapstore;
+
+ snapstore = _snapstore_find(unique_id);
+ if (snapstore == NULL) {
+ pr_err("Cannot find snapstore by uuid %pUB\n", unique_id);
+ return -ENODATA;
+ }
+
+ _snapstore_check_halffill(snapstore, fill_status);
+
+ return SUCCESS;
+}
+
+int snapstore_request_store(struct snapstore *snapstore, struct blk_deferred_request *dio_copy_req)
+{
+ int res = SUCCESS;
+
+ if (snapstore->ctrl_pipe) {
+ if (!snapstore->halffilled) {
+ sector_t fill_status = 0;
+
+ if (_snapstore_check_halffill(snapstore, &fill_status)) {
+ snapstore->halffilled = true;
+ ctrl_pipe_request_halffill(snapstore->ctrl_pipe,
+ (u64)from_sectors(fill_status));
+ }
+ }
+ }
+
+ if (snapstore->file)
+ res = blk_deferred_request_store_file(snapstore->file->blk_dev, dio_copy_req);
+#ifdef CONFIG_BLK_SNAP_SNAPSTORE_MULTIDEV
+ else if (snapstore->multidev)
+ res = blk_deferred_request_store_multidev(dio_copy_req);
+#endif
+ else if (snapstore->mem)
+ res = blk_deferred_request_store_mem(dio_copy_req);
+ else
+ res = -EINVAL;
+
+ return res;
+}
+
+static int _snapstore_redirect_read_file(struct blk_redirect_bio *rq_redir,
+ struct block_device *snapstore_blk_dev,
+ struct blk_descr_file *file,
+ sector_t block_ofs,
+ sector_t rq_ofs, sector_t rq_count)
+{
+ int res = SUCCESS;
+ sector_t current_ofs = 0;
+ struct list_head *_list_head;
+
+ if (unlikely(list_empty(&file->rangelist))) {
+ pr_err("Invalid file block descriptor");
+ return -EINVAL;
+ }
+
+ list_for_each(_list_head, &file->rangelist) {
+ struct blk_range_link *range_link;
+
+ range_link = list_entry(_list_head, struct blk_range_link, link);
+ if (current_ofs >= rq_count)
+ break;
+
+ if (range_link->rg.cnt > block_ofs) {
+ sector_t pos = range_link->rg.ofs + block_ofs;
+ sector_t len = min_t(sector_t,
+ range_link->rg.cnt - block_ofs,
+ rq_count - current_ofs);
+
+ res = blk_dev_redirect_part(rq_redir, READ, snapstore_blk_dev, pos,
+ rq_ofs + current_ofs, len);
+ if (res != SUCCESS) {
+ pr_err("Failed to read from snapstore file. Sector #%lld\n",
+ pos);
+ break;
+ }
+
+ current_ofs += len;
+ block_ofs = 0;
+ } else
+ block_ofs -= range_link->rg.cnt;
+ }
+
+ if (res != SUCCESS)
+ pr_err("Failed to read from file snapstore\n");
+ return res;
+}
+
+#ifdef CONFIG_BLK_SNAP_SNAPSTORE_MULTIDEV
+static int _snapstore_redirect_read_multidev(struct blk_redirect_bio *rq_redir,
+ struct blk_descr_multidev *multidev,
+ sector_t block_ofs,
+ sector_t rq_ofs, sector_t rq_count)
+{
+ int res = SUCCESS;
+ sector_t current_ofs = 0;
+ struct list_head *_list_head;
+
+ if (unlikely(list_empty(&multidev->rangelist))) {
+ pr_err("Invalid multidev block descriptor");
+ return -EINVAL;
+ }
+
+ list_for_each(_list_head, &multidev->rangelist) {
+ struct blk_range_link_ex *range_link =
+ list_entry(_list_head, struct blk_range_link_ex, link);
+
+ if (current_ofs >= rq_count)
+ break;
+
+ if (range_link->rg.cnt > block_ofs) {
+ sector_t pos = range_link->rg.ofs + block_ofs;
+ sector_t len = min_t(sector_t,
+ range_link->rg.cnt - block_ofs,
+ rq_count - current_ofs);
+
+ res = blk_dev_redirect_part(rq_redir, READ, range_link->blk_dev, pos,
+ rq_ofs + current_ofs, len);
+
+ if (res != SUCCESS) {
+ pr_err("Failed to read from snapstore file. Sector #%lld\n", pos);
+ break;
+ }
+
+ current_ofs += len;
+ block_ofs = 0;
+ } else
+ block_ofs -= range_link->rg.cnt;
+ }
+
+ if (res != SUCCESS)
+ pr_err("Failed to read from multidev snapstore\n");
+ return res;
+}
+#endif
+
+int snapstore_redirect_read(struct blk_redirect_bio *rq_redir, struct snapstore *snapstore,
+ union blk_descr_unify blk_descr, sector_t target_pos, sector_t rq_ofs,
+ sector_t rq_count)
+{
+ int res = SUCCESS;
+ sector_t block_ofs = target_pos & snapstore_block_mask();
+
+ if (snapstore->file)
+ res = _snapstore_redirect_read_file(rq_redir, snapstore->file->blk_dev,
+ blk_descr.file, block_ofs, rq_ofs, rq_count);
+#ifdef CONFIG_BLK_SNAP_SNAPSTORE_MULTIDEV
+ else if (snapstore->multidev)
+ res = _snapstore_redirect_read_multidev(rq_redir, blk_descr.multidev, block_ofs,
+ rq_ofs, rq_count);
+#endif
+ else if (snapstore->mem) {
+ res = blk_dev_redirect_memcpy_part(
+ rq_redir, READ, blk_descr.mem->buff + (size_t)from_sectors(block_ofs),
+ rq_ofs, rq_count);
+
+ if (res != SUCCESS)
+ pr_err("Failed to read from snapstore memory\n");
+ } else
+ res = -EINVAL;
+
+ if (res != SUCCESS)
+ pr_err("Failed to read from snapstore. Offset %lld sector\n", target_pos);
+ return res;
+}
+
+static int _snapstore_redirect_write_file(struct blk_redirect_bio *rq_redir,
+ struct block_device *snapstore_blk_dev,
+ struct blk_descr_file *file,
+ sector_t block_ofs,
+ sector_t rq_ofs, sector_t rq_count)
+{
+ int res = SUCCESS;
+ sector_t current_ofs = 0;
+ struct list_head *_list_head;
+
+ if (unlikely(list_empty(&file->rangelist))) {
+ pr_err("Invalid file block descriptor");
+ return -EINVAL;
+ }
+
+ list_for_each(_list_head, &file->rangelist) {
+ struct blk_range_link *range_link;
+
+ range_link = list_entry(_list_head, struct blk_range_link, link);
+ if (current_ofs >= rq_count)
+ break;
+
+ if (range_link->rg.cnt > block_ofs) {
+ sector_t pos = range_link->rg.ofs + block_ofs;
+ sector_t len = min_t(sector_t,
+ range_link->rg.cnt - block_ofs,
+ rq_count - current_ofs);
+
+ res = blk_dev_redirect_part(rq_redir, WRITE, snapstore_blk_dev, pos,
+ rq_ofs + current_ofs, len);
+
+ if (res != SUCCESS) {
+ pr_err("Failed to write to snapstore file. Sector #%lld\n",
+ pos);
+ break;
+ }
+
+ current_ofs += len;
+ block_ofs = 0;
+ } else
+ block_ofs -= range_link->rg.cnt;
+ }
+ if (res != SUCCESS)
+ pr_err("Failed to write to file snapstore\n");
+ return res;
+}
+
+#ifdef CONFIG_BLK_SNAP_SNAPSTORE_MULTIDEV
+static int _snapstore_redirect_write_multidev(struct blk_redirect_bio *rq_redir,
+ struct blk_descr_multidev *multidev,
+ sector_t block_ofs,
+ sector_t rq_ofs, sector_t rq_count)
+{
+ int res = SUCCESS;
+ sector_t current_ofs = 0;
+ struct list_head *_list_head;
+
+ if (unlikely(list_empty(&multidev->rangelist))) {
+ pr_err("Invalid multidev block descriptor");
+ return -EINVAL;
+ }
+
+ list_for_each(_list_head, &multidev->rangelist) {
+ struct blk_range_link_ex *range_link;
+
+ range_link = list_entry(_list_head, struct blk_range_link_ex, link);
+ if (current_ofs >= rq_count)
+ break;
+
+ if (range_link->rg.cnt > block_ofs) {
+ sector_t pos = range_link->rg.ofs + block_ofs;
+ sector_t len = min_t(sector_t,
+ range_link->rg.cnt - block_ofs,
+ rq_count - current_ofs);
+
+ res = blk_dev_redirect_part(rq_redir, WRITE, range_link->blk_dev, pos,
+ rq_ofs + current_ofs, len);
+
+ if (res != SUCCESS) {
+ pr_err("Failed to write to snapstore file. Sector #%lld\n",
+ pos);
+ break;
+ }
+
+ current_ofs += len;
+ block_ofs = 0;
+ } else
+ block_ofs -= range_link->rg.cnt;
+ }
+
+ if (res != SUCCESS)
+ pr_err("Failed to write to multidevice snapstore\n");
+ return res;
+}
+#endif
+
+int snapstore_redirect_write(struct blk_redirect_bio *rq_redir, struct snapstore *snapstore,
+ union blk_descr_unify blk_descr, sector_t target_pos, sector_t rq_ofs,
+ sector_t rq_count)
+{
+ int res = SUCCESS;
+ sector_t block_ofs = target_pos & snapstore_block_mask();
+
+ if (snapstore->file)
+ res = _snapstore_redirect_write_file(rq_redir, snapstore->file->blk_dev,
+ blk_descr.file, block_ofs, rq_ofs, rq_count);
+
+#ifdef CONFIG_BLK_SNAP_SNAPSTORE_MULTIDEV
+ else if (snapstore->multidev)
+ res = _snapstore_redirect_write_multidev(rq_redir, blk_descr.multidev,
+ block_ofs, rq_ofs, rq_count);
+#endif
+ else if (snapstore->mem) {
+ res = blk_dev_redirect_memcpy_part(
+ rq_redir, WRITE, blk_descr.mem->buff + (size_t)from_sectors(block_ofs),
+ rq_ofs, rq_count);
+
+ if (res != SUCCESS)
+ pr_err("Failed to write to memory snapstore\n");
+ } else {
+ pr_err("Unable to write to snapstore: invalid type of snapstore device\n");
+ res = -EINVAL;
+ }
+
+ if (res != SUCCESS)
+ pr_err("Failed to write to snapstore. Offset %lld sector\n", target_pos);
+ return res;
+}
diff --git a/drivers/block/blk-snap/snapstore.h b/drivers/block/blk-snap/snapstore.h
new file mode 100644
index 000000000000..db34ad2e2c58
--- /dev/null
+++ b/drivers/block/blk-snap/snapstore.h
@@ -0,0 +1,68 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#pragma once
+
+#include <linux/uuid.h>
+#include <linux/kref.h>
+#include "blk-snap-ctl.h"
+#include "rangevector.h"
+#include "snapstore_mem.h"
+#include "snapstore_file.h"
+#include "snapstore_multidev.h"
+#include "blk_redirect.h"
+#include "ctrl_pipe.h"
+#include "big_buffer.h"
+
+struct snapstore {
+ struct list_head link;
+ struct kref refcount;
+
+ uuid_t id;
+
+ struct snapstore_mem *mem;
+ struct snapstore_file *file;
+#ifdef CONFIG_BLK_SNAP_SNAPSTORE_MULTIDEV
+ struct snapstore_multidev *multidev;
+#endif
+
+ struct ctrl_pipe *ctrl_pipe;
+ sector_t empty_limit;
+
+ bool halffilled;
+ bool overflowed;
+};
+
+void snapstore_done(void);
+
+int snapstore_create(uuid_t *id, dev_t snapstore_dev_id, dev_t *dev_id_set,
+ size_t dev_id_set_length);
+#ifdef CONFIG_BLK_SNAP_SNAPSTORE_MULTIDEV
+int snapstore_create_multidev(uuid_t *id, dev_t *dev_id_set, size_t dev_id_set_length);
+#endif
+int snapstore_cleanup(uuid_t *id, u64 *filled_bytes);
+
+struct snapstore *snapstore_get(struct snapstore *snapstore);
+void snapstore_put(struct snapstore *snapstore);
+
+int snapstore_stretch_initiate(uuid_t *unique_id, struct ctrl_pipe *ctrl_pipe,
+ sector_t empty_limit);
+
+int snapstore_add_memory(uuid_t *id, unsigned long long sz);
+int snapstore_add_file(uuid_t *id, struct big_buffer *ranges, size_t ranges_cnt);
+#ifdef CONFIG_BLK_SNAP_SNAPSTORE_MULTIDEV
+int snapstore_add_multidev(uuid_t *id, dev_t dev_id, struct big_buffer *ranges, size_t ranges_cnt);
+#endif
+
+void snapstore_order_border(struct blk_range *in, struct blk_range *out);
+
+union blk_descr_unify snapstore_get_empty_block(struct snapstore *snapstore);
+
+int snapstore_request_store(struct snapstore *snapstore, struct blk_deferred_request *dio_copy_req);
+
+int snapstore_redirect_read(struct blk_redirect_bio *rq_redir, struct snapstore *snapstore,
+ union blk_descr_unify blk_descr, sector_t target_pos, sector_t rq_ofs,
+ sector_t rq_count);
+int snapstore_redirect_write(struct blk_redirect_bio *rq_redir, struct snapstore *snapstore,
+ union blk_descr_unify blk_descr, sector_t target_pos, sector_t rq_ofs,
+ sector_t rq_count);
+
+int snapstore_check_halffill(uuid_t *unique_id, sector_t *fill_status);
diff --git a/drivers/block/blk-snap/snapstore_device.c b/drivers/block/blk-snap/snapstore_device.c
new file mode 100644
index 000000000000..6fdeebacce22
--- /dev/null
+++ b/drivers/block/blk-snap/snapstore_device.c
@@ -0,0 +1,532 @@
+// SPDX-License-Identifier: GPL-2.0
+#define BLK_SNAP_SECTION "-snapstore"
+#include "common.h"
+#include "snapstore_device.h"
+#include "snapstore.h"
+#include "params.h"
+#include "blk_util.h"
+
+LIST_HEAD(snapstore_devices);
+DECLARE_RWSEM(snapstore_devices_lock);
+
+static inline void _snapstore_device_descr_write_lock(struct snapstore_device *snapstore_device)
+{
+ mutex_lock(&snapstore_device->store_block_map_locker);
+}
+static inline void _snapstore_device_descr_write_unlock(struct snapstore_device *snapstore_device)
+{
+ mutex_unlock(&snapstore_device->store_block_map_locker);
+}
+
+void snapstore_device_done(void)
+{
+ struct snapstore_device *snapstore_device = NULL;
+
+ do {
+ down_write(&snapstore_devices_lock);
+ if (!list_empty(&snapstore_devices)) {
+ snapstore_device =
+ list_entry(snapstore_devices.next, struct snapstore_device, link);
+ list_del(&snapstore_device->link);
+ }
+ up_write(&snapstore_devices_lock);
+
+ if (snapstore_device)
+ snapstore_device_put_resource(snapstore_device);
+ } while (snapstore_device);
+}
+
+struct snapstore_device *snapstore_device_find_by_dev_id(dev_t dev_id)
+{
+ struct snapstore_device *result = NULL;
+
+ down_read(&snapstore_devices_lock);
+ if (!list_empty(&snapstore_devices)) {
+ struct list_head *_head;
+
+ list_for_each(_head, &snapstore_devices) {
+ struct snapstore_device *snapstore_device =
+ list_entry(_head, struct snapstore_device, link);
+
+ if (dev_id == snapstore_device->dev_id) {
+ result = snapstore_device;
+ break;
+ }
+ }
+ }
+ up_read(&snapstore_devices_lock);
+
+ return result;
+}
+
+struct snapstore_device *_snapstore_device_get_by_snapstore_id(uuid_t *id)
+{
+ struct snapstore_device *result = NULL;
+
+ down_write(&snapstore_devices_lock);
+ if (!list_empty(&snapstore_devices)) {
+ struct list_head *_head;
+
+ list_for_each(_head, &snapstore_devices) {
+ struct snapstore_device *snapstore_device =
+ list_entry(_head, struct snapstore_device, link);
+
+ if (uuid_equal(id, &snapstore_device->snapstore->id)) {
+ result = snapstore_device;
+ list_del(&snapstore_device->link);
+ break;
+ }
+ }
+ }
+ up_write(&snapstore_devices_lock);
+
+ return result;
+}
+
+static void _snapstore_device_destroy(struct snapstore_device *snapstore_device)
+{
+ pr_info("Destroy snapstore device\n");
+
+ xa_destroy(&snapstore_device->store_block_map);
+
+ if (snapstore_device->orig_blk_dev != NULL)
+ blk_dev_close(snapstore_device->orig_blk_dev);
+
+ rangevector_done(&snapstore_device->zero_sectors);
+
+ if (snapstore_device->snapstore) {
+ pr_info("Snapstore uuid %pUB\n", &snapstore_device->snapstore->id);
+
+ snapstore_put(snapstore_device->snapstore);
+ snapstore_device->snapstore = NULL;
+ }
+
+ kfree(snapstore_device);
+}
+
+static void snapstore_device_free_cb(struct kref *kref)
+{
+ struct snapstore_device *snapstore_device =
+ container_of(kref, struct snapstore_device, refcount);
+
+ _snapstore_device_destroy(snapstore_device);
+}
+
+struct snapstore_device *snapstore_device_get_resource(struct snapstore_device *snapstore_device)
+{
+ if (snapstore_device)
+ kref_get(&snapstore_device->refcount);
+
+ return snapstore_device;
+};
+
+void snapstore_device_put_resource(struct snapstore_device *snapstore_device)
+{
+ if (snapstore_device)
+ kref_put(&snapstore_device->refcount, snapstore_device_free_cb);
+};
+
+int snapstore_device_cleanup(uuid_t *id)
+{
+ int result = SUCCESS;
+ struct snapstore_device *snapstore_device = NULL;
+
+ while (NULL != (snapstore_device = _snapstore_device_get_by_snapstore_id(id))) {
+ pr_info("Cleanup snapstore device for device [%d:%d]\n",
+ MAJOR(snapstore_device->dev_id), MINOR(snapstore_device->dev_id));
+
+ snapstore_device_put_resource(snapstore_device);
+ }
+ return result;
+}
+
+int snapstore_device_create(dev_t dev_id, struct snapstore *snapstore)
+{
+ int res = SUCCESS;
+ struct snapstore_device *snapstore_device =
+ kzalloc(sizeof(struct snapstore_device), GFP_KERNEL);
+
+ if (snapstore_device == NULL)
+ return -ENOMEM;
+
+ INIT_LIST_HEAD(&snapstore_device->link);
+ snapstore_device->dev_id = dev_id;
+
+ res = blk_dev_open(dev_id, &snapstore_device->orig_blk_dev);
+ if (res != SUCCESS) {
+ kfree(snapstore_device);
+
+ pr_err("Unable to create snapstore device: failed to open original device [%d:%d]\n",
+ MAJOR(dev_id), MINOR(dev_id));
+ return res;
+ }
+
+ kref_init(&snapstore_device->refcount);
+
+ snapstore_device->snapstore = NULL;
+ snapstore_device->err_code = SUCCESS;
+ snapstore_device->corrupted = false;
+ atomic_set(&snapstore_device->req_failed_cnt, 0);
+
+ mutex_init(&snapstore_device->store_block_map_locker);
+
+ rangevector_init(&snapstore_device->zero_sectors);
+
+ xa_init(&snapstore_device->store_block_map);
+
+ snapstore_device->snapstore = snapstore_get(snapstore);
+
+ down_write(&snapstore_devices_lock);
+ list_add_tail(&snapstore_device->link, &snapstore_devices);
+ up_write(&snapstore_devices_lock);
+
+ return SUCCESS;
+}
+
+int snapstore_device_add_request(struct snapstore_device *snapstore_device,
+ unsigned long block_index,
+ struct blk_deferred_request **dio_copy_req)
+{
+ int res = SUCCESS;
+ union blk_descr_unify blk_descr = { NULL };
+ struct blk_deferred_io *dio = NULL;
+ bool req_new = false;
+
+ blk_descr = snapstore_get_empty_block(snapstore_device->snapstore);
+ if (blk_descr.ptr == NULL) {
+ pr_err("Unable to add block to defer IO request: failed to allocate next block\n");
+ return -ENODATA;
+ }
+
+ res = xa_err(
+ xa_store(&snapstore_device->store_block_map, block_index, blk_descr.ptr, GFP_NOIO));
+ if (res != SUCCESS) {
+ pr_err("Unable to add block to defer IO request: failed to set block descriptor to descriptors array. errno=%d\n",
+ res);
+ return res;
+ }
+
+ if (*dio_copy_req == NULL) {
+ *dio_copy_req = blk_deferred_request_new();
+ if (*dio_copy_req == NULL) {
+ pr_err("Unable to add block to defer IO request: failed to allocate defer IO request\n");
+ return -ENOMEM;
+ }
+ req_new = true;
+ }
+
+ do {
+ dio = blk_deferred_alloc(block_index, blk_descr);
+ if (dio == NULL) {
+ pr_err("Unabled to add block to defer IO request: failed to allocate defer IO\n");
+ res = -ENOMEM;
+ break;
+ }
+
+ res = blk_deferred_request_add(*dio_copy_req, dio);
+ if (res != SUCCESS)
+ pr_err("Unable to add block to defer IO request: failed to add defer IO to request\n");
+ } while (false);
+
+ if (res != SUCCESS) {
+ if (dio != NULL) {
+ blk_deferred_free(dio);
+ dio = NULL;
+ }
+ if (req_new) {
+ blk_deferred_request_free(*dio_copy_req);
+ *dio_copy_req = NULL;
+ }
+ }
+
+ return res;
+}
+
+int snapstore_device_prepare_requests(struct snapstore_device *snapstore_device,
+ struct blk_range *copy_range,
+ struct blk_deferred_request **dio_copy_req)
+{
+ int res = SUCCESS;
+ unsigned long inx = 0;
+ unsigned long first = (unsigned long)(copy_range->ofs >> snapstore_block_shift());
+ unsigned long last =
+ (unsigned long)((copy_range->ofs + copy_range->cnt - 1) >> snapstore_block_shift());
+
+ for (inx = first; inx <= last; inx++) {
+ if (xa_load(&snapstore_device->store_block_map, inx) == NULL) {
+ res = snapstore_device_add_request(snapstore_device, inx, dio_copy_req);
+ if (res != SUCCESS) {
+ pr_err("Failed to create copy defer IO request. errno=%d\n", res);
+ break;
+ }
+ }
+ /*
+ * If xa_load() return not NULL, then block already stored.
+ */
+ }
+ if (res != SUCCESS)
+ snapstore_device_set_corrupted(snapstore_device, res);
+
+ return res;
+}
+
+int snapstore_device_store(struct snapstore_device *snapstore_device,
+ struct blk_deferred_request *dio_copy_req)
+{
+ int res;
+
+ res = snapstore_request_store(snapstore_device->snapstore, dio_copy_req);
+ if (res != SUCCESS)
+ snapstore_device_set_corrupted(snapstore_device, res);
+
+ return res;
+}
+
+int snapstore_device_read(struct snapstore_device *snapstore_device,
+ struct blk_redirect_bio *rq_redir)
+{
+ int res = SUCCESS;
+
+ unsigned long block_index;
+ unsigned long block_index_last;
+ unsigned long block_index_first;
+
+ sector_t blk_ofs_start = 0; //device range start
+ sector_t blk_ofs_count = 0; //device range length
+
+ struct blk_range rq_range;
+ struct rangevector *zero_sectors = &snapstore_device->zero_sectors;
+
+ if (snapstore_device_is_corrupted(snapstore_device))
+ return -ENODATA;
+
+ rq_range.cnt = bio_sectors(rq_redir->bio);
+ rq_range.ofs = rq_redir->bio->bi_iter.bi_sector;
+
+ if (!bio_has_data(rq_redir->bio)) {
+ pr_warn("Empty bio was found during reading from snapstore device. flags=%u\n",
+ rq_redir->bio->bi_flags);
+
+ blk_redirect_complete(rq_redir, SUCCESS);
+ return SUCCESS;
+ }
+
+ block_index_first = (unsigned long)(rq_range.ofs >> snapstore_block_shift());
+ block_index_last =
+ (unsigned long)((rq_range.ofs + rq_range.cnt - 1) >> snapstore_block_shift());
+
+ _snapstore_device_descr_write_lock(snapstore_device);
+ for (block_index = block_index_first; block_index <= block_index_last; ++block_index) {
+ union blk_descr_unify blk_descr;
+
+ blk_ofs_count = min_t(sector_t,
+ (((sector_t)(block_index + 1)) << snapstore_block_shift()) -
+ (rq_range.ofs + blk_ofs_start),
+ rq_range.cnt - blk_ofs_start);
+
+ blk_descr = (union blk_descr_unify)xa_load(&snapstore_device->store_block_map,
+ block_index);
+ if (blk_descr.ptr) {
+ //push snapstore read
+ res = snapstore_redirect_read(rq_redir, snapstore_device->snapstore,
+ blk_descr, rq_range.ofs + blk_ofs_start,
+ blk_ofs_start, blk_ofs_count);
+ if (res != SUCCESS) {
+ pr_err("Failed to read from snapstore device\n");
+ break;
+ }
+ } else {
+ //device read with zeroing
+ if (zero_sectors)
+ res = blk_dev_redirect_read_zeroed(rq_redir,
+ snapstore_device->orig_blk_dev,
+ rq_range.ofs, blk_ofs_start,
+ blk_ofs_count, zero_sectors);
+ else
+ res = blk_dev_redirect_part(rq_redir, READ,
+ snapstore_device->orig_blk_dev,
+ rq_range.ofs + blk_ofs_start,
+ blk_ofs_start, blk_ofs_count);
+
+ if (res != SUCCESS) {
+ pr_err("Failed to redirect read request to the original device [%d:%d]\n",
+ MAJOR(snapstore_device->dev_id),
+ MINOR(snapstore_device->dev_id));
+ break;
+ }
+ }
+
+ blk_ofs_start += blk_ofs_count;
+ }
+
+ if (res == SUCCESS) {
+ if (atomic64_read(&rq_redir->bio_count) > 0ll) //async direct access needed
+ blk_dev_redirect_submit(rq_redir);
+ else
+ blk_redirect_complete(rq_redir, res);
+ } else {
+ pr_err("Failed to read from snapstore device. errno=%d\n", res);
+ pr_err("Position %lld sector, length %lld sectors\n", rq_range.ofs, rq_range.cnt);
+ }
+ _snapstore_device_descr_write_unlock(snapstore_device);
+
+ return res;
+}
+
+int _snapstore_device_copy_on_write(struct snapstore_device *snapstore_device,
+ struct blk_range *rq_range)
+{
+ int res = SUCCESS;
+ struct blk_deferred_request *dio_copy_req = NULL;
+
+ mutex_lock(&snapstore_device->store_block_map_locker);
+ do {
+ res = snapstore_device_prepare_requests(snapstore_device, rq_range, &dio_copy_req);
+ if (res != SUCCESS) {
+ pr_err("Failed to create defer IO request for range. errno=%d\n", res);
+ break;
+ }
+
+ if (dio_copy_req == NULL)
+ break; //nothing to copy
+
+ res = blk_deferred_request_read_original(snapstore_device->orig_blk_dev,
+ dio_copy_req);
+ if (res != SUCCESS) {
+ pr_err("Failed to read data from the original device. errno=%d\n", res);
+ break;
+ }
+
+ res = snapstore_device_store(snapstore_device, dio_copy_req);
+ if (res != SUCCESS) {
+ pr_err("Failed to write data to snapstore. errno=%d\n", res);
+ break;
+ }
+ } while (false);
+ mutex_unlock(&snapstore_device->store_block_map_locker);
+
+ if (dio_copy_req) {
+ if (res == -EDEADLK)
+ blk_deferred_request_deadlocked(dio_copy_req);
+ else
+ blk_deferred_request_free(dio_copy_req);
+ }
+
+ return res;
+}
+
+int snapstore_device_write(struct snapstore_device *snapstore_device,
+ struct blk_redirect_bio *rq_redir)
+{
+ int res = SUCCESS;
+ unsigned long block_index;
+ unsigned long block_index_last;
+ unsigned long block_index_first;
+ sector_t blk_ofs_start = 0; //device range start
+ sector_t blk_ofs_count = 0; //device range length
+ struct blk_range rq_range;
+
+ if (snapstore_device_is_corrupted(snapstore_device))
+ return -ENODATA;
+
+ rq_range.cnt = bio_sectors(rq_redir->bio);
+ rq_range.ofs = rq_redir->bio->bi_iter.bi_sector;
+
+ if (!bio_has_data(rq_redir->bio)) {
+ pr_warn("Empty bio was found during reading from snapstore device. flags=%u\n",
+ rq_redir->bio->bi_flags);
+
+ blk_redirect_complete(rq_redir, SUCCESS);
+ return SUCCESS;
+ }
+
+ // do copy to snapstore previously
+ res = _snapstore_device_copy_on_write(snapstore_device, &rq_range);
+
+ block_index_first = (unsigned long)(rq_range.ofs >> snapstore_block_shift());
+ block_index_last =
+ (unsigned long)((rq_range.ofs + rq_range.cnt - 1) >> snapstore_block_shift());
+
+ _snapstore_device_descr_write_lock(snapstore_device);
+ for (block_index = block_index_first; block_index <= block_index_last; ++block_index) {
+ union blk_descr_unify blk_descr;
+
+ blk_ofs_count = min_t(sector_t,
+ (((sector_t)(block_index + 1)) << snapstore_block_shift()) -
+ (rq_range.ofs + blk_ofs_start),
+ rq_range.cnt - blk_ofs_start);
+
+ blk_descr = (union blk_descr_unify)xa_load(&snapstore_device->store_block_map,
+ block_index);
+ if (blk_descr.ptr == NULL) {
+ pr_err("Unable to write from snapstore device: invalid snapstore block descriptor\n");
+ res = -EIO;
+ break;
+ }
+
+ res = snapstore_redirect_write(rq_redir, snapstore_device->snapstore, blk_descr,
+ rq_range.ofs + blk_ofs_start, blk_ofs_start,
+ blk_ofs_count);
+ if (res != SUCCESS) {
+ pr_err("Unable to write from snapstore device: failed to redirect write request to snapstore\n");
+ break;
+ }
+
+ blk_ofs_start += blk_ofs_count;
+ }
+ if (res == SUCCESS) {
+ if (atomic64_read(&rq_redir->bio_count) > 0) { //async direct access needed
+ blk_dev_redirect_submit(rq_redir);
+ } else {
+ blk_redirect_complete(rq_redir, res);
+ }
+ } else {
+ pr_err("Failed to write from snapstore device. errno=%d\n", res);
+ pr_err("Position %lld sector, length %lld sectors\n", rq_range.ofs, rq_range.cnt);
+
+ snapstore_device_set_corrupted(snapstore_device, res);
+ }
+ _snapstore_device_descr_write_unlock(snapstore_device);
+ return res;
+}
+
+bool snapstore_device_is_corrupted(struct snapstore_device *snapstore_device)
+{
+ if (snapstore_device == NULL)
+ return true;
+
+ if (snapstore_device->corrupted) {
+ if (atomic_read(&snapstore_device->req_failed_cnt) == 0)
+ pr_err("Snapshot device is corrupted for [%d:%d]\n",
+ MAJOR(snapstore_device->dev_id), MINOR(snapstore_device->dev_id));
+
+ atomic_inc(&snapstore_device->req_failed_cnt);
+ return true;
+ }
+
+ return false;
+}
+
+void snapstore_device_set_corrupted(struct snapstore_device *snapstore_device, int err_code)
+{
+ if (!snapstore_device->corrupted) {
+ atomic_set(&snapstore_device->req_failed_cnt, 0);
+ snapstore_device->corrupted = true;
+ snapstore_device->err_code = abs(err_code);
+
+ pr_err("Set snapshot device is corrupted for [%d:%d]\n",
+ MAJOR(snapstore_device->dev_id), MINOR(snapstore_device->dev_id));
+ }
+}
+
+int snapstore_device_errno(dev_t dev_id, int *p_err_code)
+{
+ struct snapstore_device *snapstore_device;
+
+ snapstore_device = snapstore_device_find_by_dev_id(dev_id);
+ if (snapstore_device == NULL)
+ return -ENODATA;
+
+ *p_err_code = snapstore_device->err_code;
+ return SUCCESS;
+}
diff --git a/drivers/block/blk-snap/snapstore_device.h b/drivers/block/blk-snap/snapstore_device.h
new file mode 100644
index 000000000000..729b3c05ef70
--- /dev/null
+++ b/drivers/block/blk-snap/snapstore_device.h
@@ -0,0 +1,63 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#pragma once
+
+#include "rangevector.h"
+#include "blk_deferred.h"
+#include "blk_redirect.h"
+#include "snapstore.h"
+#include <linux/xarray.h>
+#include <linux/kref.h>
+
+struct snapstore_device {
+ struct list_head link;
+ struct kref refcount;
+
+ dev_t dev_id;
+ struct snapstore *snapstore;
+
+ struct block_device *orig_blk_dev;
+
+ struct xarray store_block_map; // map block index to read block offset
+ struct mutex store_block_map_locker;
+
+ struct rangevector zero_sectors;
+
+ atomic_t req_failed_cnt;
+ int err_code;
+ bool corrupted;
+};
+
+void snapstore_device_done(void);
+
+struct snapstore_device *snapstore_device_get_resource(struct snapstore_device *snapstore_device);
+void snapstore_device_put_resource(struct snapstore_device *snapstore_device);
+
+struct snapstore_device *snapstore_device_find_by_dev_id(dev_t dev_id);
+
+int snapstore_device_create(dev_t dev_id, struct snapstore *snapstore);
+
+int snapstore_device_cleanup(uuid_t *id);
+
+int snapstore_device_prepare_requests(struct snapstore_device *snapstore_device,
+ struct blk_range *copy_range,
+ struct blk_deferred_request **dio_copy_req);
+int snapstore_device_store(struct snapstore_device *snapstore_device,
+ struct blk_deferred_request *dio_copy_req);
+
+int snapstore_device_read(struct snapstore_device *snapstore_device,
+ struct blk_redirect_bio *rq_redir); //request from image
+int snapstore_device_write(struct snapstore_device *snapstore_device,
+ struct blk_redirect_bio *rq_redir); //request from image
+
+bool snapstore_device_is_corrupted(struct snapstore_device *snapstore_device);
+void snapstore_device_set_corrupted(struct snapstore_device *snapstore_device, int err_code);
+int snapstore_device_errno(dev_t dev_id, int *p_err_code);
+
+static inline void _snapstore_device_descr_read_lock(struct snapstore_device *snapstore_device)
+{
+ mutex_lock(&snapstore_device->store_block_map_locker);
+}
+static inline void _snapstore_device_descr_read_unlock(struct snapstore_device *snapstore_device)
+{
+ mutex_unlock(&snapstore_device->store_block_map_locker);
+}
diff --git a/drivers/block/blk-snap/snapstore_file.c b/drivers/block/blk-snap/snapstore_file.c
new file mode 100644
index 000000000000..a5c959a8070c
--- /dev/null
+++ b/drivers/block/blk-snap/snapstore_file.c
@@ -0,0 +1,52 @@
+// SPDX-License-Identifier: GPL-2.0
+#define BLK_SNAP_SECTION "-snapstore"
+#include "common.h"
+#include "snapstore_file.h"
+#include "blk_util.h"
+
+int snapstore_file_create(dev_t dev_id, struct snapstore_file **pfile)
+{
+ int res = SUCCESS;
+ struct snapstore_file *file;
+
+ pr_info("Single device file snapstore was created on device [%d:%d]\n", MAJOR(dev_id),
+ MINOR(dev_id));
+
+ file = kzalloc(sizeof(struct snapstore_file), GFP_KERNEL);
+ if (file == NULL)
+ return -ENOMEM;
+
+ res = blk_dev_open(dev_id, &file->blk_dev);
+ if (res != SUCCESS) {
+ kfree(file);
+ pr_err("Unable to create snapstore file: failed to open device [%d:%d]. errno=%d",
+ MAJOR(dev_id), MINOR(dev_id), res);
+ return res;
+ }
+ {
+ struct request_queue *q = bdev_get_queue(file->blk_dev);
+
+ pr_info("snapstore device logical block size %d\n", q->limits.logical_block_size);
+ pr_info("snapstore device physical block size %d\n", q->limits.physical_block_size);
+ }
+
+ file->blk_dev_id = dev_id;
+ blk_descr_file_pool_init(&file->pool);
+
+ *pfile = file;
+ return res;
+}
+
+void snapstore_file_destroy(struct snapstore_file *file)
+{
+ if (file) {
+ blk_descr_file_pool_done(&file->pool);
+
+ if (file->blk_dev != NULL) {
+ blk_dev_close(file->blk_dev);
+ file->blk_dev = NULL;
+ }
+
+ kfree(file);
+ }
+}
diff --git a/drivers/block/blk-snap/snapstore_file.h b/drivers/block/blk-snap/snapstore_file.h
new file mode 100644
index 000000000000..effd9d888781
--- /dev/null
+++ b/drivers/block/blk-snap/snapstore_file.h
@@ -0,0 +1,15 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#pragma once
+
+#include "blk_deferred.h"
+
+struct snapstore_file {
+ dev_t blk_dev_id;
+ struct block_device *blk_dev;
+
+ struct blk_descr_pool pool;
+};
+
+int snapstore_file_create(dev_t dev_id, struct snapstore_file **pfile);
+
+void snapstore_file_destroy(struct snapstore_file *file);
diff --git a/drivers/block/blk-snap/snapstore_mem.c b/drivers/block/blk-snap/snapstore_mem.c
new file mode 100644
index 000000000000..29a607617d99
--- /dev/null
+++ b/drivers/block/blk-snap/snapstore_mem.c
@@ -0,0 +1,91 @@
+// SPDX-License-Identifier: GPL-2.0
+#define BLK_SNAP_SECTION "-snapstore"
+#include "common.h"
+#include "snapstore_mem.h"
+#include "params.h"
+
+#include <linux/vmalloc.h>
+
+struct buffer_el {
+ struct list_head link;
+ void *buff;
+};
+
+struct snapstore_mem *snapstore_mem_create(size_t available_blocks)
+{
+ struct snapstore_mem *mem = kzalloc(sizeof(struct snapstore_mem), GFP_KERNEL);
+
+ if (mem == NULL)
+ return NULL;
+
+ blk_descr_mem_pool_init(&mem->pool, available_blocks);
+
+ mem->blocks_limit = available_blocks;
+
+ INIT_LIST_HEAD(&mem->blocks);
+ mutex_init(&mem->blocks_lock);
+
+ return mem;
+}
+
+void snapstore_mem_destroy(struct snapstore_mem *mem)
+{
+ struct buffer_el *buffer_el;
+
+ if (mem == NULL)
+ return;
+
+ do {
+ buffer_el = NULL;
+
+ mutex_lock(&mem->blocks_lock);
+ if (!list_empty(&mem->blocks)) {
+ buffer_el = list_entry(mem->blocks.next, struct buffer_el, link);
+
+ list_del(&buffer_el->link);
+ }
+ mutex_unlock(&mem->blocks_lock);
+
+ if (buffer_el) {
+ vfree(buffer_el->buff);
+ kfree(buffer_el);
+ }
+ } while (buffer_el);
+
+ blk_descr_mem_pool_done(&mem->pool);
+
+ kfree(mem);
+}
+
+void *snapstore_mem_get_block(struct snapstore_mem *mem)
+{
+ struct buffer_el *buffer_el;
+
+ if (mem->blocks_allocated >= mem->blocks_limit) {
+ pr_err("Unable to get block from snapstore in memory\n");
+ pr_err("Block limit is reached, allocated %zu, limit %zu\n", mem->blocks_allocated,
+ mem->blocks_limit);
+ return NULL;
+ }
+
+ buffer_el = kzalloc(sizeof(struct buffer_el), GFP_KERNEL);
+ if (buffer_el == NULL)
+ return NULL;
+ INIT_LIST_HEAD(&buffer_el->link);
+
+ buffer_el->buff = vmalloc(snapstore_block_size() * SECTOR_SIZE);
+ if (buffer_el->buff == NULL) {
+ kfree(buffer_el);
+ return NULL;
+ }
+
+ ++mem->blocks_allocated;
+ if (0 == (mem->blocks_allocated & 0x7F))
+ pr_info("%zu MiB was allocated\n", mem->blocks_allocated);
+
+ mutex_lock(&mem->blocks_lock);
+ list_add_tail(&buffer_el->link, &mem->blocks);
+ mutex_unlock(&mem->blocks_lock);
+
+ return buffer_el->buff;
+}
diff --git a/drivers/block/blk-snap/snapstore_mem.h b/drivers/block/blk-snap/snapstore_mem.h
new file mode 100644
index 000000000000..9044a6525966
--- /dev/null
+++ b/drivers/block/blk-snap/snapstore_mem.h
@@ -0,0 +1,20 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#pragma once
+
+#include "blk_descr_mem.h"
+
+struct snapstore_mem {
+ struct list_head blocks;
+ struct mutex blocks_lock;
+
+ size_t blocks_limit;
+ size_t blocks_allocated;
+
+ struct blk_descr_pool pool;
+};
+
+struct snapstore_mem *snapstore_mem_create(size_t available_blocks);
+
+void snapstore_mem_destroy(struct snapstore_mem *mem);
+
+void *snapstore_mem_get_block(struct snapstore_mem *mem);
diff --git a/drivers/block/blk-snap/snapstore_multidev.c b/drivers/block/blk-snap/snapstore_multidev.c
new file mode 100644
index 000000000000..bb6bfefa68d7
--- /dev/null
+++ b/drivers/block/blk-snap/snapstore_multidev.c
@@ -0,0 +1,118 @@
+// SPDX-License-Identifier: GPL-2.0
+#define BLK_SNAP_SECTION "-snapstore"
+#include "common.h"
+
+#ifdef CONFIG_BLK_SNAP_SNAPSTORE_MULTIDEV
+
+#include "snapstore_multidev.h"
+#include "blk_util.h"
+
+struct multidev_el {
+ struct list_head link;
+
+ dev_t dev_id;
+ struct block_device *blk_dev;
+};
+
+int snapstore_multidev_create(struct snapstore_multidev **p_multidev)
+{
+ int res = SUCCESS;
+ struct snapstore_multidev *multidev;
+
+ pr_info("Multidevice file snapstore create\n");
+
+ multidev = kzalloc(sizeof(struct snapstore_multidev), GFP_KERNEL);
+ if (multidev == NULL)
+ return -ENOMEM;
+
+ INIT_LIST_HEAD(&multidev->devicelist);
+ spin_lock_init(&multidev->devicelist_lock);
+
+ blk_descr_multidev_pool_init(&multidev->pool);
+
+ *p_multidev = multidev;
+ return res;
+}
+
+void snapstore_multidev_destroy(struct snapstore_multidev *multidev)
+{
+ struct multidev_el *el;
+
+ blk_descr_multidev_pool_done(&multidev->pool);
+
+ do {
+ el = NULL;
+ spin_lock(&multidev->devicelist_lock);
+ if (!list_empty(&multidev->devicelist)) {
+ el = list_entry(multidev->devicelist.next, struct multidev_el, link);
+
+ list_del(&el->link);
+ }
+ spin_unlock(&multidev->devicelist_lock);
+
+ if (el) {
+ blk_dev_close(el->blk_dev);
+
+ pr_info("Close device for multidevice snapstore [%d:%d]\n",
+ MAJOR(el->dev_id), MINOR(el->dev_id));
+
+ kfree(el);
+ }
+ } while (el);
+
+ kfree(multidev);
+}
+
+struct multidev_el *snapstore_multidev_find(struct snapstore_multidev *multidev, dev_t dev_id)
+{
+ struct multidev_el *el = NULL;
+
+ spin_lock(&multidev->devicelist_lock);
+ if (!list_empty(&multidev->devicelist)) {
+ struct list_head *_head;
+
+ list_for_each(_head, &multidev->devicelist) {
+ struct multidev_el *_el = list_entry(_head, struct multidev_el, link);
+
+ if (_el->dev_id == dev_id) {
+ el = _el;
+ break;
+ }
+ }
+ }
+ spin_unlock(&multidev->devicelist_lock);
+
+ return el;
+}
+
+struct block_device *snapstore_multidev_get_device(struct snapstore_multidev *multidev,
+ dev_t dev_id)
+{
+ int res;
+ struct block_device *blk_dev = NULL;
+ struct multidev_el *el = snapstore_multidev_find(multidev, dev_id);
+
+ if (el)
+ return el->blk_dev;
+
+ res = blk_dev_open(dev_id, &blk_dev);
+ if (res != SUCCESS) {
+ pr_err("Unable to add device to snapstore multidevice file\n");
+ pr_err("Failed to open [%d:%d]. errno=%d", MAJOR(dev_id), MINOR(dev_id), res);
+ return NULL;
+ }
+
+ el = kzalloc(sizeof(struct multidev_el), GFP_KERNEL);
+ INIT_LIST_HEAD(&el->link);
+
+ el->blk_dev = blk_dev;
+ el->dev_id = dev_id;
+
+ spin_lock(&multidev->devicelist_lock);
+ list_add_tail(&el->link, &multidev->devicelist);
+ spin_unlock(&multidev->devicelist_lock);
+
+ return el->blk_dev;
+}
+
+#endif
diff --git a/drivers/block/blk-snap/snapstore_multidev.h b/drivers/block/blk-snap/snapstore_multidev.h
new file mode 100644
index 000000000000..40c1c3a41b08
--- /dev/null
+++ b/drivers/block/blk-snap/snapstore_multidev.h
@@ -0,0 +1,22 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#pragma once
+
+#ifdef CONFIG_BLK_SNAP_SNAPSTORE_MULTIDEV
+
+#include "blk_deferred.h"
+#include "blk_descr_multidev.h"
+
+struct snapstore_multidev {
+ struct list_head devicelist; //for mapping device id to opened device struct pointer
+ spinlock_t devicelist_lock;
+
+ struct blk_descr_pool pool;
+};
+
+int snapstore_multidev_create(struct snapstore_multidev **p_file);
+
+void snapstore_multidev_destroy(struct snapstore_multidev *file);
+
+struct block_device *snapstore_multidev_get_device(struct snapstore_multidev *multidev,
+ dev_t dev_id);
+#endif
diff --git a/drivers/block/blk-snap/tracker.c b/drivers/block/blk-snap/tracker.c
new file mode 100644
index 000000000000..3cda996d3f0a
--- /dev/null
+++ b/drivers/block/blk-snap/tracker.c
@@ -0,0 +1,449 @@
+// SPDX-License-Identifier: GPL-2.0
+#define BLK_SNAP_SECTION "-tracker"
+#include "common.h"
+#include "tracker.h"
+#include "blk_util.h"
+#include "params.h"
+
+LIST_HEAD(trackers);
+DEFINE_RWLOCK(trackers_lock);
+
+void tracker_done(void)
+{
+ tracker_remove_all();
+}
+
+int tracker_find_by_bio(struct bio *bio, struct tracker **ptracker)
+{
+ int result = -ENODATA;
+
+ read_lock(&trackers_lock);
+ if (!list_empty(&trackers)) {
+ struct list_head *_head;
+
+ list_for_each(_head, &trackers) {
+ struct tracker *_tracker = list_entry(_head, struct tracker, link);
+
+ if ((bio->bi_disk == _tracker->target_dev->bd_disk) &&
+ (bio->bi_partno == _tracker->target_dev->bd_partno)) {
+ if (ptracker != NULL)
+ *ptracker = _tracker;
+
+ result = SUCCESS;
+ break;
+ }
+ }
+ }
+ read_unlock(&trackers_lock);
+
+ return result;
+}
+
+int tracker_find_by_dev_id(dev_t dev_id, struct tracker **ptracker)
+{
+ int result = -ENODATA;
+
+ read_lock(&trackers_lock);
+ if (!list_empty(&trackers)) {
+ struct list_head *_head;
+
+ list_for_each(_head, &trackers) {
+ struct tracker *_tracker = list_entry(_head, struct tracker, link);
+
+ if (_tracker->original_dev_id == dev_id) {
+ if (ptracker != NULL)
+ *ptracker = _tracker;
+
+ result = SUCCESS;
+ break;
+ }
+ }
+ }
+ read_unlock(&trackers_lock);
+
+ return result;
+}
+
+int tracker_enum_cbt_info(int max_count, struct cbt_info_s *p_cbt_info, int *p_count)
+{
+ int result = SUCCESS;
+ int count = 0;
+
+ read_lock(&trackers_lock);
+ if (!list_empty(&trackers)) {
+ struct list_head *_head;
+
+ list_for_each(_head, &trackers) {
+ struct tracker *tracker = list_entry(_head, struct tracker, link);
+
+ if (count >= max_count) {
+ result = -ENOBUFS;
+ break; //don`t continue
+ }
+
+ if (p_cbt_info != NULL) {
+ p_cbt_info[count].dev_id.major = MAJOR(tracker->original_dev_id);
+ p_cbt_info[count].dev_id.minor = MINOR(tracker->original_dev_id);
+
+ if (tracker->cbt_map) {
+ p_cbt_info[count].cbt_map_size = tracker->cbt_map->map_size;
+ p_cbt_info[count].snap_number =
+ (unsigned char)
+ tracker->cbt_map->snap_number_previous;
+ uuid_copy((uuid_t *)(p_cbt_info[count].generationId),
+ &tracker->cbt_map->generationId);
+ } else {
+ p_cbt_info[count].cbt_map_size = 0;
+ p_cbt_info[count].snap_number = 0;
+ }
+
+ p_cbt_info[count].dev_capacity = (u64)from_sectors(
+ part_nr_sects_read(tracker->target_dev->bd_part));
+ }
+
+ ++count;
+ }
+ }
+ read_unlock(&trackers_lock);
+
+ if (result == SUCCESS)
+ if (count == 0)
+ result = -ENODATA;
+
+ *p_count = count;
+ return result;
+}
+
+static void blk_thaw_bdev(dev_t dev_id, struct block_device *device,
+ struct super_block *superblock)
+{
+ if (superblock == NULL)
+ return;
+
+ if (thaw_bdev(device, superblock) == SUCCESS)
+ pr_info("Device [%d:%d] was unfrozen\n", MAJOR(dev_id), MINOR(dev_id));
+ else
+ pr_err("Failed to unfreeze device [%d:%d]\n", MAJOR(dev_id), MINOR(dev_id));
+}
+
+static int blk_freeze_bdev(dev_t dev_id, struct block_device *device,
+ struct super_block **psuperblock)
+{
+ struct super_block *superblock;
+
+ if (device->bd_super == NULL) {
+ pr_warn("Unable to freeze device [%d:%d]: no superblock was found\n",
+ MAJOR(dev_id), MINOR(dev_id));
+ return SUCCESS;
+ }
+
+ superblock = freeze_bdev(device);
+ if (IS_ERR_OR_NULL(superblock)) {
+ int result;
+
+ pr_err("Failed to freeze device [%d:%d]\n", MAJOR(dev_id), MINOR(dev_id));
+
+ if (superblock == NULL)
+ result = -ENODEV;
+ else {
+ result = PTR_ERR(superblock);
+ pr_err("Error code: %d\n", result);
+ }
+ return result;
+ }
+
+ pr_info("Device [%d:%d] was frozen\n", MAJOR(dev_id), MINOR(dev_id));
+ *psuperblock = superblock;
+
+ return SUCCESS;
+}
+
+int _tracker_create(struct tracker *tracker, void *filter, bool attach_filter)
+{
+ int result = SUCCESS;
+ unsigned int sect_in_block_degree;
+ sector_t capacity;
+ struct super_block *superblock = NULL;
+
+ result = blk_dev_open(tracker->original_dev_id, &tracker->target_dev);
+ if (result != SUCCESS)
+ return ENODEV;
+
+ pr_info("Create tracker for device [%d:%d]. Capacity 0x%llx sectors\n",
+ MAJOR(tracker->original_dev_id), MINOR(tracker->original_dev_id),
+ (unsigned long long)part_nr_sects_read(tracker->target_dev->bd_part));
+
+ sect_in_block_degree = get_change_tracking_block_size_pow() - SECTOR_SHIFT;
+ capacity = part_nr_sects_read(tracker->target_dev->bd_part);
+
+ tracker->cbt_map = cbt_map_create(sect_in_block_degree, capacity);
+ if (tracker->cbt_map == NULL) {
+ pr_err("Failed to create tracker for device [%d:%d]\n",
+ MAJOR(tracker->original_dev_id), MINOR(tracker->original_dev_id));
+ tracker_remove(tracker);
+ return -ENOMEM;
+ }
+
+ tracker->snapshot_id = 0ull;
+
+ if (attach_filter) {
+ blk_freeze_bdev(tracker->original_dev_id, tracker->target_dev, &superblock);
+
+ blk_filter_attach(tracker->original_dev_id, filter, tracker);
+
+ blk_thaw_bdev(tracker->original_dev_id, tracker->target_dev, superblock);
+ }
+
+ return SUCCESS;
+}
+
+int tracker_create(dev_t dev_id, void *filter, struct tracker **ptracker)
+{
+ int ret;
+ struct tracker *tracker = NULL;
+
+ *ptracker = NULL;
+
+ tracker = kzalloc(sizeof(struct tracker), GFP_KERNEL);
+ if (tracker == NULL)
+ return -ENOMEM;
+
+ INIT_LIST_HEAD(&tracker->link);
+ atomic_set(&tracker->is_captured, false);
+ tracker->original_dev_id = dev_id;
+
+ write_lock(&trackers_lock);
+ list_add_tail(&tracker->link, &trackers);
+ write_unlock(&trackers_lock);
+
+ ret = _tracker_create(tracker, filter, true);
+ if (ret < 0) {
+ tracker_remove(tracker);
+ return ret;
+ }
+
+ *ptracker = tracker;
+ if (ret == ENODEV)
+ pr_info("Cannot attach to unknown device [%d:%d]",
+ MAJOR(tracker->original_dev_id), MINOR(tracker->original_dev_id));
+
+ return ret;
+}
+
+void _tracker_remove(struct tracker *tracker, bool detach_filter)
+{
+ struct super_block *superblock = NULL;
+
+ if (tracker->target_dev != NULL) {
+ if (detach_filter) {
+ blk_freeze_bdev(tracker->original_dev_id, tracker->target_dev, &superblock);
+
+ blk_filter_detach(tracker->original_dev_id);
+
+ blk_thaw_bdev(tracker->original_dev_id, tracker->target_dev, superblock);
+ }
+
+ blk_dev_close(tracker->target_dev);
+ tracker->target_dev = NULL;
+ }
+
+ if (tracker->cbt_map != NULL) {
+ cbt_map_put_resource(tracker->cbt_map);
+ tracker->cbt_map = NULL;
+ }
+}
+
+void tracker_remove(struct tracker *tracker)
+{
+ _tracker_remove(tracker, true);
+
+ write_lock(&trackers_lock);
+ list_del(&tracker->link);
+ write_unlock(&trackers_lock);
+
+ kfree(tracker);
+}
+
+void tracker_remove_all(void)
+{
+ struct tracker *tracker;
+
+ pr_info("Removing all devices from tracking\n");
+
+ do {
+ tracker = NULL;
+
+ write_lock(&trackers_lock);
+ if (!list_empty(&trackers)) {
+ tracker = list_entry(trackers.next, struct tracker, link);
+
+ list_del(&tracker->link);
+ }
+ write_unlock(&trackers_lock);
+
+ if (tracker) {
+ _tracker_remove(tracker, true);
+ kfree(tracker);
+ }
+ } while (tracker);
+}
+
+void tracker_cbt_bitmap_set(struct tracker *tracker, sector_t sector, sector_t sector_cnt)
+{
+ if (tracker->cbt_map == NULL)
+ return;
+
+ if (tracker->cbt_map->device_capacity != part_nr_sects_read(tracker->target_dev->bd_part)) {
+ pr_warn("Device resize detected\n");
+ tracker->cbt_map->active = false;
+ return;
+ }
+
+ if (cbt_map_set(tracker->cbt_map, sector, sector_cnt) != SUCCESS) { //cbt corrupt
+ pr_warn("CBT fault detected\n");
+ tracker->cbt_map->active = false;
+ return;
+ }
+}
+
+bool tracker_cbt_bitmap_lock(struct tracker *tracker)
+{
+ if (tracker->cbt_map == NULL)
+ return false;
+
+ cbt_map_read_lock(tracker->cbt_map);
+ if (!tracker->cbt_map->active) {
+ cbt_map_read_unlock(tracker->cbt_map);
+ return false;
+ }
+
+ return true;
+}
+
+void tracker_cbt_bitmap_unlock(struct tracker *tracker)
+{
+ if (tracker->cbt_map)
+ cbt_map_read_unlock(tracker->cbt_map);
+}
+
+int _tracker_capture_snapshot(struct tracker *tracker)
+{
+ int result = SUCCESS;
+
+ result = defer_io_create(tracker->original_dev_id, tracker->target_dev, &tracker->defer_io);
+ if (result != SUCCESS) {
+ pr_err("Failed to create defer IO processor\n");
+ return result;
+ }
+
+ atomic_set(&tracker->is_captured, true);
+
+ if (tracker->cbt_map != NULL) {
+ cbt_map_write_lock(tracker->cbt_map);
+ cbt_map_switch(tracker->cbt_map);
+ cbt_map_write_unlock(tracker->cbt_map);
+
+ pr_info("Snapshot captured for device [%d:%d]. New snap number %ld\n",
+ MAJOR(tracker->original_dev_id), MINOR(tracker->original_dev_id),
+ tracker->cbt_map->snap_number_active);
+ }
+
+ return result;
+}
+
+int tracker_capture_snapshot(dev_t *dev_id_set, int dev_id_set_size)
+{
+ int result = SUCCESS;
+ int inx = 0;
+
+ for (inx = 0; inx < dev_id_set_size; ++inx) {
+ struct super_block *superblock = NULL;
+ struct tracker *tracker = NULL;
+ dev_t dev_id = dev_id_set[inx];
+
+ result = tracker_find_by_dev_id(dev_id, &tracker);
+ if (result != SUCCESS) {
+ pr_err("Unable to capture snapshot: cannot find device [%d:%d]\n",
+ MAJOR(dev_id), MINOR(dev_id));
+ break;
+ }
+
+
+ blk_freeze_bdev(tracker->original_dev_id, tracker->target_dev, &superblock);
+ blk_filter_freeze(tracker->target_dev);
+
+ result = _tracker_capture_snapshot(tracker);
+ if (result != SUCCESS)
+ pr_err("Failed to capture snapshot for device [%d:%d]\n",
+ MAJOR(dev_id), MINOR(dev_id));
+
+ blk_filter_thaw(tracker->target_dev);
+ blk_thaw_bdev(tracker->original_dev_id, tracker->target_dev, superblock);
+ }
+ if (result != SUCCESS)
+ return result;
+
+ for (inx = 0; inx < dev_id_set_size; ++inx) {
+ struct tracker *tracker = NULL;
+ dev_t dev_id = dev_id_set[inx];
+
+ result = tracker_find_by_dev_id(dev_id, &tracker);
+ if (result != SUCCESS) {
+ pr_err("Unable to capture snapshot: cannot find device [%d:%d]\n",
+ MAJOR(dev_id), MINOR(dev_id));
+ continue;
+ }
+
+ if (snapstore_device_is_corrupted(tracker->defer_io->snapstore_device)) {
+ pr_err("Unable to freeze devices [%d:%d]: snapshot data is corrupted\n",
+ MAJOR(dev_id), MINOR(dev_id));
+ result = -EDEADLK;
+ break;
+ }
+ }
+
+ if (result != SUCCESS) {
+ pr_err("Failed to capture snapshot. errno=%d\n", result);
+
+ tracker_release_snapshot(dev_id_set, dev_id_set_size);
+ }
+ return result;
+}
+
+void _tracker_release_snapshot(struct tracker *tracker)
+{
+ struct super_block *superblock = NULL;
+ struct defer_io *defer_io = tracker->defer_io;
+
+ blk_freeze_bdev(tracker->original_dev_id, tracker->target_dev, &superblock);
+ blk_filter_freeze(tracker->target_dev);
+ { //locked region
+ atomic_set(&tracker->is_captured, false); //clear freeze flag
+
+ tracker->defer_io = NULL;
+ }
+ blk_filter_thaw(tracker->target_dev);
+
+ blk_thaw_bdev(tracker->original_dev_id, tracker->target_dev, superblock);
+
+ defer_io_stop(defer_io);
+ defer_io_put_resource(defer_io);
+}
+
+void tracker_release_snapshot(dev_t *dev_id_set, int dev_id_set_size)
+{
+ int inx = 0;
+
+ for (; inx < dev_id_set_size; ++inx) {
+ int status;
+ struct tracker *p_tracker = NULL;
+ dev_t dev = dev_id_set[inx];
+
+ status = tracker_find_by_dev_id(dev, &p_tracker);
+ if (status == SUCCESS)
+ _tracker_release_snapshot(p_tracker);
+ else
+ pr_err("Unable to release snapshot: cannot find tracker for device [%d:%d]\n",
+ MAJOR(dev), MINOR(dev));
+ }
+}
diff --git a/drivers/block/blk-snap/tracker.h b/drivers/block/blk-snap/tracker.h
new file mode 100644
index 000000000000..9fff7c0942c3
--- /dev/null
+++ b/drivers/block/blk-snap/tracker.h
@@ -0,0 +1,38 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#pragma once
+#include "cbt_map.h"
+#include "defer_io.h"
+#include "blk-snap-ctl.h"
+#include "snapshot.h"
+
+struct tracker {
+ struct list_head link;
+ dev_t original_dev_id;
+ struct block_device *target_dev;
+ struct cbt_map *cbt_map;
+ atomic_t is_captured;
+ struct defer_io *defer_io;
+ unsigned long long snapshot_id; // current snapshot for this device
+};
+
+void tracker_done(void);
+
+int tracker_find_by_bio(struct bio *bio, struct tracker **ptracker);
+int tracker_find_by_dev_id(dev_t dev_id, struct tracker **ptracker);
+
+int tracker_enum_cbt_info(int max_count, struct cbt_info_s *p_cbt_info, int *p_count);
+
+int tracker_capture_snapshot(dev_t *dev_id_set, int dev_id_set_size);
+void tracker_release_snapshot(dev_t *dev_id_set, int dev_id_set_size);
+
+int _tracker_create(struct tracker *tracker, void *filter, bool attach_filter);
+int tracker_create(dev_t dev_id, void *filter, struct tracker **ptracker);
+
+void _tracker_remove(struct tracker *tracker, bool detach_filter);
+void tracker_remove(struct tracker *tracker);
+void tracker_remove_all(void);
+
+void tracker_cbt_bitmap_set(struct tracker *tracker, sector_t sector, sector_t sector_cnt);
+
+bool tracker_cbt_bitmap_lock(struct tracker *tracker);
+void tracker_cbt_bitmap_unlock(struct tracker *tracker);
diff --git a/drivers/block/blk-snap/tracking.c b/drivers/block/blk-snap/tracking.c
new file mode 100644
index 000000000000..55e18891bb96
--- /dev/null
+++ b/drivers/block/blk-snap/tracking.c
@@ -0,0 +1,270 @@
+// SPDX-License-Identifier: GPL-2.0
+#define BLK_SNAP_SECTION "-tracking"
+#include "common.h"
+#include "tracking.h"
+#include "tracker.h"
+#include "blk_util.h"
+#include "defer_io.h"
+#include "params.h"
+
+#include <linux/blk-filter.h>
+
+/* pointer to block layer filter */
+void *filter;
+
+/*
+ * _tracking_submit_bio() - Intercept bio by block io layer filter
+ */
+static bool _tracking_submit_bio(struct bio *bio, void *filter_data)
+{
+ int res;
+ bool cbt_locked = false;
+ struct tracker *tracker = filter_data;
+
+ if (!tracker)
+ return false;
+
+ //intercepting
+ if (atomic_read(&tracker->is_captured)) {
+ //snapshot is captured, call bio redirect algorithm
+
+ res = defer_io_redirect_bio(tracker->defer_io, bio, tracker);
+ if (res == SUCCESS)
+ return true;
+ }
+
+ cbt_locked = false;
+ if (tracker && bio_data_dir(bio) && bio_has_data(bio)) {
+ //call CBT algorithm
+ cbt_locked = tracker_cbt_bitmap_lock(tracker);
+ if (cbt_locked) {
+ sector_t sectStart = bio->bi_iter.bi_sector;
+ sector_t sectCount = bio_sectors(bio);
+
+ tracker_cbt_bitmap_set(tracker, sectStart, sectCount);
+ }
+ }
+ if (cbt_locked)
+ tracker_cbt_bitmap_unlock(tracker);
+
+ return false;
+}
+
+static bool _tracking_part_add(dev_t devt, void **p_filter_data)
+{
+ int result;
+ struct tracker *tracker = NULL;
+
+ pr_info("new block device [%d:%d] in system\n", MAJOR(devt), MINOR(devt));
+
+ result = tracker_find_by_dev_id(devt, &tracker);
+ if (result != SUCCESS)
+ return false; /*do not track this device*/
+
+ if (_tracker_create(tracker, filter, false)) {
+ pr_err("Failed to attach new device to tracker. errno=%d\n", result);
+ return false; /*failed to attach new device to tracker*/
+ }
+
+ *p_filter_data = tracker;
+ return true;
+}
+
+static void _tracking_part_del(void *private_data)
+{
+ struct tracker *tracker = private_data;
+
+ if (!tracker)
+ return;
+
+ pr_info("delete block device [%d:%d] from system\n",
+ MAJOR(tracker->original_dev_id), MINOR(tracker->original_dev_id));
+
+ _tracker_remove(tracker, false);
+}
+
+struct blk_filter_ops filter_ops = {
+ .filter_bio = _tracking_submit_bio,
+ .part_add = _tracking_part_add,
+ .part_del = _tracking_part_del };
+
+
+
+int tracking_init(void)
+{
+ filter = blk_filter_register(&filter_ops);
+ if (!filter)
+ return -ENOMEM;
+ return SUCCESS;
+}
+
+void tracking_done(void)
+{
+ if (filter) {
+ blk_filter_unregister(filter);
+ filter = NULL;
+ }
+}
+
+static int _add_already_tracked(dev_t dev_id, unsigned long long snapshot_id,
+ struct tracker *tracker)
+{
+ int result = SUCCESS;
+ bool cbt_reset_needed = false;
+
+ if ((snapshot_id != 0ull) && (tracker->snapshot_id == 0ull))
+ tracker->snapshot_id = snapshot_id; // set new snapshot id
+
+ if (tracker->cbt_map == NULL) {
+ unsigned int sect_in_block_degree =
+ get_change_tracking_block_size_pow() - SECTOR_SHIFT;
+ tracker->cbt_map = cbt_map_create(sect_in_block_degree - SECTOR_SHIFT,
+ part_nr_sects_read(tracker->target_dev->bd_part));
+ if (tracker->cbt_map == NULL)
+ return -ENOMEM;
+
+ // skip snapshot id
+ tracker->snapshot_id = snapshot_id;
+ return SUCCESS;
+ }
+
+ if (!tracker->cbt_map->active) {
+ cbt_reset_needed = true;
+ pr_warn("Nonactive CBT table detected. CBT fault\n");
+ }
+
+ if (tracker->cbt_map->device_capacity != part_nr_sects_read(tracker->target_dev->bd_part)) {
+ cbt_reset_needed = true;
+ pr_warn("Device resize detected. CBT fault\n");
+ }
+
+ if (!cbt_reset_needed)
+ return SUCCESS;
+
+ _tracker_remove(tracker, true);
+
+ result = _tracker_create(tracker, filter, true);
+ if (result != SUCCESS) {
+ pr_err("Failed to create tracker. errno=%d\n", result);
+ return result;
+ }
+
+ tracker->snapshot_id = snapshot_id;
+
+ return SUCCESS;
+}
+
+static int _create_new_tracker(dev_t dev_id, unsigned long long snapshot_id)
+{
+ int result;
+ struct tracker *tracker = NULL;
+
+ result = tracker_create(dev_id, filter, &tracker);
+ if (result != SUCCESS) {
+ pr_err("Failed to create tracker. errno=%d\n", result);
+ return result;
+ }
+
+ tracker->snapshot_id = snapshot_id;
+
+ return SUCCESS;
+}
+
+
+int tracking_add(dev_t dev_id, unsigned long long snapshot_id)
+{
+ int result;
+ struct tracker *tracker = NULL;
+
+ pr_info("Adding device [%d:%d] under tracking\n", MAJOR(dev_id), MINOR(dev_id));
+
+ result = tracker_find_by_dev_id(dev_id, &tracker);
+ if (result == SUCCESS) {
+ //pr_info("Device [%d:%d] is already tracked\n", MAJOR(dev_id), MINOR(dev_id));
+ result = _add_already_tracked(dev_id, snapshot_id, tracker);
+ if (result == SUCCESS)
+ result = -EALREADY;
+ } else if (-ENODATA == result)
+ result = _create_new_tracker(dev_id, snapshot_id);
+ else {
+ pr_err("Unable to add device [%d:%d] under tracking\n", MAJOR(dev_id),
+ MINOR(dev_id));
+ pr_err("Invalid trackers container. errno=%d\n", result);
+ }
+
+ return result;
+}
+
+int tracking_remove(dev_t dev_id)
+{
+ int result;
+ struct tracker *tracker = NULL;
+
+ pr_info("Removing device [%d:%d] from tracking\n", MAJOR(dev_id), MINOR(dev_id));
+
+ result = tracker_find_by_dev_id(dev_id, &tracker);
+ if (result != SUCCESS) {
+ pr_err("Unable to remove device [%d:%d] from tracking: ",
+ MAJOR(dev_id), MINOR(dev_id));
+
+ if (-ENODATA == result)
+ pr_err("tracker not found\n");
+ else
+ pr_err("tracker container failed. errno=%d\n", result);
+
+ return result;
+ }
+
+ if (tracker->snapshot_id != 0ull) {
+ pr_err("Unable to remove device [%d:%d] from tracking: ",
+ MAJOR(dev_id), MINOR(dev_id));
+ pr_err("snapshot [0x%llx] already exist\n", tracker->snapshot_id);
+ return -EBUSY;
+ }
+
+ tracker_remove(tracker);
+
+ return SUCCESS;
+}
+
+int tracking_collect(int max_count, struct cbt_info_s *p_cbt_info, int *p_count)
+{
+ int res = tracker_enum_cbt_info(max_count, p_cbt_info, p_count);
+
+ if (res == SUCCESS)
+ pr_info("%d devices found under tracking\n", *p_count);
+ else if (res == -ENODATA) {
+ pr_info("There are no devices under tracking\n");
+ *p_count = 0;
+ res = SUCCESS;
+ } else
+ pr_err("Failed to collect devices under tracking. errno=%d", res);
+
+ return res;
+}
+
+int tracking_read_cbt_bitmap(dev_t dev_id, unsigned int offset, size_t length,
+ void __user *user_buff)
+{
+ int result = SUCCESS;
+ struct tracker *tracker = NULL;
+
+ result = tracker_find_by_dev_id(dev_id, &tracker);
+ if (result == SUCCESS) {
+ if (atomic_read(&tracker->is_captured))
+ result = cbt_map_read_to_user(tracker->cbt_map, user_buff, offset, length);
+ else {
+ pr_err("Unable to read CBT bitmap for device [%d:%d]: ", MAJOR(dev_id),
+ MINOR(dev_id));
+ pr_err("device is not captured by snapshot\n");
+ result = -EPERM;
+ }
+ } else if (-ENODATA == result) {
+ pr_err("Unable to read CBT bitmap for device [%d:%d]: ", MAJOR(dev_id),
+ MINOR(dev_id));
+ pr_err("device not found\n");
+ } else
+ pr_err("Failed to find devices under tracking. errno=%d", result);
+
+ return result;
+}
diff --git a/drivers/block/blk-snap/tracking.h b/drivers/block/blk-snap/tracking.h
new file mode 100644
index 000000000000..22bd5ba54963
--- /dev/null
+++ b/drivers/block/blk-snap/tracking.h
@@ -0,0 +1,13 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#pragma once
+#include "blk-snap-ctl.h"
+#include <linux/bio.h>
+
+int tracking_init(void);
+void tracking_done(void);
+
+int tracking_add(dev_t dev_id, unsigned long long snapshot_id);
+int tracking_remove(dev_t dev_id);
+int tracking_collect(int max_count, struct cbt_info_s *p_cbt_info, int *p_count);
+int tracking_read_cbt_bitmap(dev_t dev_id, unsigned int offset, size_t length,
+ void __user *user_buff);
diff --git a/drivers/block/blk-snap/version.h b/drivers/block/blk-snap/version.h
new file mode 100644
index 000000000000..a4431da73611
--- /dev/null
+++ b/drivers/block/blk-snap/version.h
@@ -0,0 +1,7 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#pragma once
+
+#define FILEVER_MAJOR 5
+#define FILEVER_MINOR 0
+#define FILEVER_REVISION 0
+#define FILEVER_STR "5.0.0"
--
2.20.1
