From: Boaz Harrosh <boazh@xxxxxxxxxx>
zuf-core established the communication channels with the ZUS
User Mode Server.
In this patch we have the core communication mechanics.
Which is the Novelty of this project.
(See previous submitted documentation for more info)
Users will come later in the patchset
Signed-off-by: Boaz Harrosh <boazh@xxxxxxxxxx>
---
fs/zuf/_extern.h | 22 +
fs/zuf/_pr.h | 4 +
fs/zuf/relay.h | 88 ++++
fs/zuf/zuf-core.c | 1016 ++++++++++++++++++++++++++++++++++++++++++++-
fs/zuf/zuf-root.c | 7 +
fs/zuf/zuf.h | 46 ++
fs/zuf/zus_api.h | 185 +++++++++
7 files changed, 1367 insertions(+), 1 deletion(-)
create mode 100644 fs/zuf/relay.h
diff --git a/fs/zuf/_extern.h b/fs/zuf/_extern.h
index 3bb9f1d9acf6..52bb6b9deafe 100644
--- a/fs/zuf/_extern.h
+++ b/fs/zuf/_extern.h
@@ -28,10 +28,32 @@ struct dentry *zuf_mount(struct file_system_type *fs_type, int flags,
const char *dev_name, void *data);
/* zuf-core.c */
+int zufc_zts_init(struct zuf_root_info *zri); /* Some private types in core */
+void zufc_zts_fini(struct zuf_root_info *zri);
+
long zufc_ioctl(struct file *filp, unsigned int cmd, ulong arg);
int zufc_release(struct inode *inode, struct file *file);
int zufc_mmap(struct file *file, struct vm_area_struct *vma);
+int __zufc_dispatch_mount(struct zuf_root_info *zri,
+ enum e_mount_operation op,
+ struct zufs_ioc_mount *zim);
+int zufc_dispatch_mount(struct zuf_root_info *zri, struct zus_fs_info *zus_zfi,
+ enum e_mount_operation operation,
+ struct zufs_ioc_mount *zim);
+
+const char *zuf_op_name(enum e_zufs_operation op);
+int __zufc_dispatch(struct zuf_root_info *zri, struct zuf_dispatch_op *zdo);
+static inline
+int zufc_dispatch(struct zuf_root_info *zri, struct zufs_ioc_hdr *hdr,
+ struct page **pages, uint nump)
+{
+ struct zuf_dispatch_op zdo;
+
+ zuf_dispatch_init(&zdo, hdr, pages, nump);
+ return __zufc_dispatch(zri, &zdo);
+}
+
/* zuf-root.c */
int zufr_register_fs(struct super_block *sb, struct zufs_ioc_register_fs *rfs);
diff --git a/fs/zuf/_pr.h b/fs/zuf/_pr.h
index 30b8cf912c1f..dc9f85453890 100644
--- a/fs/zuf/_pr.h
+++ b/fs/zuf/_pr.h
@@ -39,5 +39,9 @@
/* ~~~ channel prints ~~~ */
#define zuf_dbg_err(s, args ...) zuf_chan_debug("error", s, ##args)
+#define zuf_dbg_vfs(s, args ...) zuf_chan_debug("vfs ", s, ##args)
+#define zuf_dbg_core(s, args ...) zuf_chan_debug("core ", s, ##args)
+#define zuf_dbg_zus(s, args ...) zuf_chan_debug("zusdg", s, ##args)
+#define zuf_dbg_verbose(s, args ...) zuf_chan_debug("d-oto", s, ##args)
#endif /* define __ZUF_PR_H__ */
diff --git a/fs/zuf/relay.h b/fs/zuf/relay.h
new file mode 100644
index 000000000000..a17d242b313a
--- /dev/null
+++ b/fs/zuf/relay.h
@@ -0,0 +1,88 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Relay scheduler-object Header file.
+ *
+ * Copyright (c) 2018 NetApp Inc. All rights reserved.
+ *
+ * ZUFS-License: GPL-2.0. See module.c for LICENSE details.
+ *
+ * Authors:
+ * Boaz Harrosh <boazh@xxxxxxxxxx>
+ */
+
+#ifndef __RELAY_H__
+#define __RELAY_H__
+
+/* ~~~~ Relay ~~~~ */
+struct relay {
+ wait_queue_head_t fss_wq;
+ bool fss_wakeup;
+ bool fss_waiting;
+
+ wait_queue_head_t app_wq;
+ bool app_wakeup;
+ bool app_waiting;
+
+ cpumask_t cpus_allowed;
+};
+
+static inline void relay_init(struct relay *relay)
+{
+ init_waitqueue_head(&relay->fss_wq);
+ init_waitqueue_head(&relay->app_wq);
+}
+
+static inline bool relay_is_app_waiting(struct relay *relay)
+{
+ return relay->app_waiting;
+}
+
+static inline void relay_app_wakeup(struct relay *relay)
+{
+ relay->app_waiting = false;
+
+ relay->app_wakeup = true;
+ wake_up(&relay->app_wq);
+}
+
+static inline int relay_fss_wait(struct relay *relay)
+{
+ int err;
+
+ relay->fss_waiting = true;
+ relay->fss_wakeup = false;
+ err = wait_event_interruptible(relay->fss_wq, relay->fss_wakeup);
+
+ return err;
+}
+
+static inline bool relay_is_fss_waiting_grab(struct relay *relay)
+{
+ if (relay->fss_waiting) {
+ relay->fss_waiting = false;
+ return true;
+ }
+ return false;
+}
+
+static inline void relay_fss_wakeup(struct relay *relay)
+{
+ relay->fss_wakeup = true;
+ wake_up(&relay->fss_wq);
+}
+
+static inline void relay_fss_wakeup_app_wait(struct relay *relay,
+ spinlock_t *spinlock)
+{
+ relay->app_waiting = true;
+
+ relay_fss_wakeup(relay);
+
+ relay->app_wakeup = false;
+ if (spinlock)
+ spin_unlock(spinlock);
+
+ wait_event(relay->app_wq, relay->app_wakeup);
+}
+
+#endif /* ifndef __RELAY_H__ */
diff --git a/fs/zuf/zuf-core.c b/fs/zuf/zuf-core.c
index e12cae584f8a..95582c0a4ba5 100644
--- a/fs/zuf/zuf-core.c
+++ b/fs/zuf/zuf-core.c
@@ -18,14 +18,820 @@
#include <linux/delay.h>
#include <linux/pfn_t.h>
#include <linux/sched/signal.h>
+#include <linux/uaccess.h>
#include "zuf.h"
+struct zufc_thread {
+ struct zuf_special_file hdr;
+ struct relay relay;
+ struct vm_area_struct *vma;
+ int no;
+ int chan;
+
+ /* Kernel side allocated IOCTL buffer */
+ struct vm_area_struct *opt_buff_vma;
+ void *opt_buff;
+ ulong max_zt_command;
+
+ /* Next operation*/
+ struct zuf_dispatch_op *zdo;
+};
+
+enum { INITIAL_ZT_CHANNELS = 3 };
+
+struct zuf_threads_pool {
+ uint _max_zts;
+ uint _max_channels;
+ /* array of pcp_arrays */
+ struct zufc_thread *_all_zt[ZUFS_MAX_ZT_CHANNELS];
+};
+
+static int _alloc_zts_channel(struct zuf_root_info *zri, int channel)
+{
+ zri->_ztp->_all_zt[channel] = alloc_percpu(struct zufc_thread);
+ if (unlikely(!zri->_ztp->_all_zt[channel])) {
+ zuf_err("!!! alloc_percpu channel=%d failed\n", channel);
+ return -ENOMEM;
+ }
+ return 0;
+}
+
+static inline ulong _zt_pr_no(struct zufc_thread *zt)
+{
+ /* So in hex it will be channel as first nibble and cpu as 3rd and on */
+ return ((ulong)zt->no << 8) | zt->chan;
+}
+
+int zufc_zts_init(struct zuf_root_info *zri)
+{
+ int c;
+
+ zri->_ztp = kcalloc(1, sizeof(struct zuf_threads_pool), GFP_KERNEL);
+ if (unlikely(!zri->_ztp))
+ return -ENOMEM;
+
+ zri->_ztp->_max_zts = num_online_cpus();
+ zri->_ztp->_max_channels = INITIAL_ZT_CHANNELS;
+
+ for (c = 0; c < INITIAL_ZT_CHANNELS; ++c) {
+ int err = _alloc_zts_channel(zri, c);
+
+ if (unlikely(err))
+ return err;
+ }
+
+ return 0;
+}
+
+void zufc_zts_fini(struct zuf_root_info *zri)
+{
+ int c;
+
+ /* Always safe/must call zufc_zts_fini */
+ if (!zri->_ztp)
+ return;
+
+ for (c = 0; c < zri->_ztp->_max_channels; ++c) {
+ if (zri->_ztp->_all_zt[c])
+ free_percpu(zri->_ztp->_all_zt[c]);
+ }
+ kfree(zri->_ztp);
+ zri->_ztp = NULL;
+}
+
+static struct zufc_thread *_zt_from_cpu(struct zuf_root_info *zri,
+ int cpu, uint chan)
+{
+ return per_cpu_ptr(zri->_ztp->_all_zt[chan], cpu);
+}
+
+static int _zt_from_f(struct file *filp, int cpu, uint chan,
+ struct zufc_thread **ztp)
+{
+ *ztp = _zt_from_cpu(ZRI(filp->f_inode->i_sb), cpu, chan);
+ if (unlikely(!*ztp))
+ return -ERANGE;
+ return 0;
+}
+
+static int _zu_register_fs(struct file *file, void *parg)
+{
+ struct zufs_ioc_register_fs rfs;
+ int err;
+
+ err = copy_from_user(&rfs, parg, sizeof(rfs));
+ if (unlikely(err)) {
+ zuf_err("=>%d\n", err);
+ return err;
+ }
+
+ err = zufr_register_fs(file->f_inode->i_sb, &rfs);
+ if (err)
+ zuf_err("=>%d\n", err);
+ err = put_user(err, (int *)parg);
+ return err;
+}
+
+/* ~~~~ mounting ~~~~*/
+int __zufc_dispatch_mount(struct zuf_root_info *zri,
+ enum e_mount_operation operation,
+ struct zufs_ioc_mount *zim)
+{
+ zim->hdr.operation = operation;
+
+ for (;;) {
+ bool fss_waiting;
+
+ spin_lock(&zri->mount.lock);
+
+ if (unlikely(!zri->mount.zsf.file)) {
+ spin_unlock(&zri->mount.lock);
+ zuf_err("Server not up\n");
+ zim->hdr.err = -EIO;
+ return zim->hdr.err;
+ }
+
+ fss_waiting = relay_is_fss_waiting_grab(&zri->mount.relay);
+ if (fss_waiting)
+ break;
+ /* in case of break above spin_unlock is done inside
+ * relay_fss_wakeup_app_wait
+ */
+
+ spin_unlock(&zri->mount.lock);
+
+ /* It is OK to wait if user storms mounts */
+ zuf_dbg_verbose("waiting\n");
+ msleep(100);
+ }
+
+ zri->mount.zim = zim;
+ relay_fss_wakeup_app_wait(&zri->mount.relay, &zri->mount.lock);
+
+ return zim->hdr.err;
+}
+
+int zufc_dispatch_mount(struct zuf_root_info *zri, struct zus_fs_info *zus_zfi,
+ enum e_mount_operation operation,
+ struct zufs_ioc_mount *zim)
+{
+ zim->hdr.out_len = sizeof(*zim);
+ zim->hdr.in_len = sizeof(*zim);
+ if (operation == ZUFS_M_MOUNT || operation == ZUFS_M_REMOUNT)
+ zim->hdr.in_len += zim->zmi.po.mount_options_len;
+ zim->zmi.zus_zfi = zus_zfi;
+ zim->zmi.num_cpu = zri->_ztp->_max_zts;
+ zim->zmi.num_channels = zri->_ztp->_max_channels;
+
+ return __zufc_dispatch_mount(zri, operation, zim);
+}
+
+static int _zu_mount(struct file *file, void *parg)
+{
+ struct super_block *sb = file->f_inode->i_sb;
+ struct zuf_root_info *zri = ZRI(sb);
+ bool waiting_for_reply;
+ struct zufs_ioc_mount *zim;
+ ulong cp_ret;
+ int err;
+
+ spin_lock(&zri->mount.lock);
+
+ if (unlikely(!file->private_data)) {
+ /* First time register this file as the mount-thread owner */
+ zri->mount.zsf.type = zlfs_e_mout_thread;
+ zri->mount.zsf.file = file;
+ file->private_data = &zri->mount.zsf;
+ } else if (unlikely(file->private_data != &zri->mount)) {
+ spin_unlock(&zri->mount.lock);
+ zuf_err("Say what?? %p != %p\n",
+ file->private_data, &zri->mount);
+ return -EIO;
+ }
+
+ zim = zri->mount.zim;
+ zri->mount.zim = NULL;
+ waiting_for_reply = zim && relay_is_app_waiting(&zri->mount.relay);
+
+ spin_unlock(&zri->mount.lock);
+
+ if (waiting_for_reply) {
+ cp_ret = copy_from_user(zim, parg, zim->hdr.out_len);
+ if (unlikely(cp_ret)) {
+ zuf_err("copy_from_user => %ld\n", cp_ret);
+ zim->hdr.err = -EFAULT;
+ }
+
+ relay_app_wakeup(&zri->mount.relay);
+ }
+
+ /* This gets to sleep until a mount comes */
+ err = relay_fss_wait(&zri->mount.relay);
+ if (unlikely(err || !zri->mount.zim)) {
+ struct zufs_ioc_hdr *hdr = parg;
+
+ /* Released by _zu_break INTER or crash */
+ zuf_dbg_zus("_zu_break? %p => %d\n", zri->mount.zim, err);
+ put_user(ZUFS_OP_BREAK, &hdr->operation);
+ put_user(EIO, &hdr->err);
+ return err;
+ }
+
+ zim = zri->mount.zim;
+ cp_ret = copy_to_user(parg, zim, zim->hdr.in_len);
+ if (unlikely(cp_ret)) {
+ err = -EFAULT;
+ zuf_err("copy_to_user =>%ld\n", cp_ret);
+ }
+ return err;
+}
+
+static void zufc_mounter_release(struct file *file)
+{
+ struct zuf_root_info *zri = ZRI(file->f_inode->i_sb);
+
+ zuf_dbg_zus("closed fu=%d au=%d fw=%d aw=%d\n",
+ zri->mount.relay.fss_wakeup, zri->mount.relay.app_wakeup,
+ zri->mount.relay.fss_waiting, zri->mount.relay.app_waiting);
+
+ spin_lock(&zri->mount.lock);
+ zri->mount.zsf.file = NULL;
+ if (relay_is_app_waiting(&zri->mount.relay)) {
+ zuf_err("server emergency exit while IO\n");
+
+ if (zri->mount.zim)
+ zri->mount.zim->hdr.err = -EIO;
+ spin_unlock(&zri->mount.lock);
+
+ relay_app_wakeup(&zri->mount.relay);
+ msleep(1000); /* crap */
+ } else {
+ if (zri->mount.zim)
+ zri->mount.zim->hdr.err = 0;
+ spin_unlock(&zri->mount.lock);
+ }
+}
+
+/* ~~~~ ZU_IOC_NUMA_MAP ~~~~ */
+static int _zu_numa_map(struct file *file, void *parg)
+{
+ struct zufs_ioc_numa_map *numa_map;
+ int n_nodes = num_online_nodes();
+ int n_cpus = num_online_cpus();
+ uint *nodes_cpu_count;
+ uint max_cpu_per_node = 0;
+ uint alloc_size;
+ int cpu, i, err;
+
+ alloc_size = sizeof(*numa_map) + n_cpus; /* char per cpu */
+
+ if ((n_nodes > 255) || (alloc_size > PAGE_SIZE)) {
+ zuf_warn("!!!unexpected big machine with %d nodes alloc_size=0x%x\n",
+ n_nodes, alloc_size);
+ return -ENOTSUPP;
+ }
+
+ nodes_cpu_count = kcalloc(n_nodes, sizeof(uint), GFP_KERNEL);
+ if (unlikely(!nodes_cpu_count))
+ return -ENOMEM;
+
+ numa_map = kzalloc(alloc_size, GFP_KERNEL);
+ if (unlikely(!numa_map)) {
+ err = -ENOMEM;
+ goto out;
+ }
+
+ numa_map->possible_nodes = num_possible_nodes();
+ numa_map->possible_cpus = num_possible_cpus();
+
+ numa_map->online_nodes = n_nodes;
+ numa_map->online_cpus = n_cpus;
+
+ for_each_cpu(cpu, cpu_online_mask) {
+ uint ctn = cpu_to_node(cpu);
+ uint ncc = ++nodes_cpu_count[ctn];
+
+ numa_map->cpu_to_node[cpu] = ctn;
+ max_cpu_per_node = max(max_cpu_per_node, ncc);
+ }
+
+ for (i = 1; i < n_nodes; ++i) {
+ if (nodes_cpu_count[i] != nodes_cpu_count[0]) {
+ zuf_info("@[%d]=%d Unbalanced CPU sockets @[0]=%d\n",
+ i, nodes_cpu_count[i], nodes_cpu_count[0]);
+ numa_map->nodes_not_symmetrical = true;
+ break;
+ }
+ }
+
+ numa_map->max_cpu_per_node = max_cpu_per_node;
+
+ zuf_dbg_verbose(
+ "possible_nodes=%d possible_cpus=%d online_nodes=%d online_cpus=%d\n",
+ numa_map->possible_nodes, numa_map->possible_cpus,
+ n_nodes, n_cpus);
+
+ err = copy_to_user(parg, numa_map, alloc_size);
+ kfree(numa_map);
+out:
+ kfree(nodes_cpu_count);
+ return err;
+}
+
+static int _map_pages(struct zufc_thread *zt, struct page **pages, uint nump,
+ bool map_readonly)
+{
+ int p, err;
+
+ if (!(zt->vma && pages && nump))
+ return 0;
+
+ for (p = 0; p < nump; ++p) {
+ ulong zt_addr = zt->vma->vm_start + p * PAGE_SIZE;
+ ulong pfn = page_to_pfn(pages[p]);
+ pfn_t pfnt = phys_to_pfn_t(PFN_PHYS(pfn), PFN_MAP | PFN_DEV);
+ vm_fault_t flt;
+
+ if (map_readonly)
+ flt = vmf_insert_mixed(zt->vma, zt_addr, pfnt);
+ else
+ flt = vmf_insert_mixed_mkwrite(zt->vma, zt_addr, pfnt);
+ err = zuf_flt_to_err(flt);
+ if (unlikely(err)) {
+ zuf_err("zuf: remap_pfn_range => %d p=0x%x start=0x%lx\n",
+ err, p, zt->vma->vm_start);
+ return err;
+ }
+ }
+ return 0;
+}
+
+static void _unmap_pages(struct zufc_thread *zt, struct page **pages, uint nump)
+{
+ if (!(zt->vma && zt->zdo && pages && nump))
+ return;
+
+ zt->zdo->pages = NULL;
+ zt->zdo->nump = 0;
+
+ zap_vma_ptes(zt->vma, zt->vma->vm_start, nump * PAGE_SIZE);
+}
+
+static void _fill_buff(ulong *buff, uint size)
+{
+ ulong *buff_end = buff + size;
+ ulong val = 0;
+
+ for (; buff < buff_end; ++buff, ++val)
+ *buff = val;
+}
+
+static int _zu_init(struct file *file, void *parg)
+{
+ struct zufc_thread *zt;
+ int cpu = smp_processor_id();
+ struct zufs_ioc_init zi_init;
+ int err;
+
+ err = copy_from_user(&zi_init, parg, sizeof(zi_init));
+ if (unlikely(err)) {
+ zuf_err("=>%d\n", err);
+ return err;
+ }
+ if (unlikely(zi_init.channel_no >= ZUFS_MAX_ZT_CHANNELS)) {
+ zuf_err("[%d] channel_no=%d\n", cpu, zi_init.channel_no);
+ return -EINVAL;
+ }
+
+ zuf_dbg_zus("[%d] aff=0x%lx channel=%d\n",
+ cpu, zi_init.affinity, zi_init.channel_no);
+
+ zi_init.hdr.err = _zt_from_f(file, cpu, zi_init.channel_no, &zt);
+ if (unlikely(zi_init.hdr.err)) {
+ zuf_err("=>%d\n", err);
+ goto out;
+ }
+
+ if (unlikely(zt->hdr.file)) {
+ zi_init.hdr.err = -EINVAL;
+ zuf_err("[%d] !!! thread already set\n", cpu);
+ goto out;
+ }
+
+ relay_init(&zt->relay);
+ zt->hdr.type = zlfs_e_zt;
+ zt->hdr.file = file;
+ zt->no = cpu;
+ zt->chan = zi_init.channel_no;
+
+ zt->max_zt_command = zi_init.max_command;
+ zt->opt_buff = vmalloc(zi_init.max_command);
+ if (unlikely(!zt->opt_buff)) {
+ zi_init.hdr.err = -ENOMEM;
+ goto out;
+ }
+ _fill_buff(zt->opt_buff, zi_init.max_command / sizeof(ulong));
+
+ file->private_data = &zt->hdr;
+out:
+ err = copy_to_user(parg, &zi_init, sizeof(zi_init));
+ if (err)
+ zuf_err("=>%d\n", err);
+ return err;
+}
+
+struct zufc_thread *_zt_from_f_private(struct file *file)
+{
+ struct zuf_special_file *zsf = file->private_data;
+
+ WARN_ON(zsf->type != zlfs_e_zt);
+ return container_of(zsf, struct zufc_thread, hdr);
+}
+
+/* Caller checks that file->private_data != NULL */
+static void zufc_zt_release(struct file *file)
+{
+ struct zufc_thread *zt = _zt_from_f_private(file);
+
+ if (unlikely(zt->hdr.file != file))
+ zuf_err("What happened zt->file(%p) != file(%p)\n",
+ zt->hdr.file, file);
+
+ zuf_dbg_zus("[%d] closed fu=%d au=%d fw=%d aw=%d\n",
+ zt->no, zt->relay.fss_wakeup, zt->relay.app_wakeup,
+ zt->relay.fss_waiting, zt->relay.app_waiting);
+
+ if (relay_is_app_waiting(&zt->relay)) {
+ zuf_err("server emergency exit while IO\n");
+
+ /* NOTE: Do not call _unmap_pages the vma is gone */
+ zt->hdr.file = NULL;
+
+ relay_app_wakeup(&zt->relay);
+ msleep(1000); /* crap */
+ }
+
+ vfree(zt->opt_buff);
+ memset(zt, 0, sizeof(*zt));
+}
+
+static int _copy_outputs(struct zufc_thread *zt, void *arg)
+{
+ struct zufs_ioc_hdr *hdr = zt->zdo->hdr;
+ struct zufs_ioc_hdr *user_hdr = zt->opt_buff;
+
+ if (zt->opt_buff_vma->vm_start != (ulong)arg) {
+ zuf_err("malicious Server\n");
+ return -EINVAL;
+ }
+
+ /* Update on the user out_len and return-code */
+ hdr->err = user_hdr->err;
+ hdr->out_len = user_hdr->out_len;
+
+ if (!hdr->out_len)
+ return 0;
+
+ if ((hdr->err == -EZUFS_RETRY) || (hdr->out_max < hdr->out_len)) {
+ if (WARN_ON(!zt->zdo->oh)) {
+ zuf_err("Trouble op(%s) out_max=%d out_len=%d\n",
+ zuf_op_name(hdr->operation),
+ hdr->out_max, hdr->out_len);
+ return -EFAULT;
+ }
+ zuf_dbg_zus("[%s] %d %d => %d\n",
+ zuf_op_name(hdr->operation),
+ hdr->out_max, hdr->out_len, hdr->err);
+ return zt->zdo->oh(zt->zdo, zt->opt_buff, zt->max_zt_command);
+ } else {
+ void *rply = (void *)hdr + hdr->out_start;
+ void *from = zt->opt_buff + hdr->out_start;
+
+ memcpy(rply, from, hdr->out_len);
+ return 0;
+ }
+}
+
+static int _zu_wait(struct file *file, void *parg)
+{
+ struct zufc_thread *zt;
+ int err;
+
+ zt = _zt_from_f_private(file);
+ if (unlikely(!zt)) {
+ zuf_err("Unexpected ZT state\n");
+ err = -ERANGE;
+ goto err;
+ }
+
+ if (!zt->hdr.file || file != zt->hdr.file) {
+ zuf_err("fatal\n");
+ err = -E2BIG;
+ goto err;
+ }
+ if (unlikely((ulong)parg != zt->opt_buff_vma->vm_start)) {
+ zuf_err("fatal 2\n");
+ err = -EINVAL;
+ goto err;
+ }
+
+ if (relay_is_app_waiting(&zt->relay)) {
+ if (unlikely(!zt->zdo)) {
+ zuf_err("User has gone...\n");
+ err = -E2BIG;
+ goto err;
+ } else {
+ /* overflow_handler might decide to execute the
+ *parg here at zus context and return to server
+ * If it also has an error to report to zus it
+ * will set zdo->hdr->err.
+ * EZUS_RETRY_DONE is when that happens.
+ * In this case pages stay mapped in zt->vma
+ */
+ err = _copy_outputs(zt, parg);
+ if (err == EZUF_RETRY_DONE) {
+ put_user(zt->zdo->hdr->err, (int *)parg);
+ return 0;
+ }
+
+ _unmap_pages(zt, zt->zdo->pages, zt->zdo->nump);
+ zt->zdo = NULL;
+ if (unlikely(err)) /* _copy_outputs returned an err */
+ goto err;
+ }
+ relay_app_wakeup(&zt->relay);
+ }
+
+ err = relay_fss_wait(&zt->relay);
+ if (err)
+ zuf_dbg_err("[%d] relay error: %d\n", zt->no, err);
+
+ if (zt->zdo && zt->zdo->hdr &&
+ zt->zdo->hdr->operation < ZUFS_OP_BREAK) {
+ /* call map here at the zuf thread so we need no locks
+ * TODO: Currently only ZUFS_OP_WRITE protects user-buffers
+ * we should have a bit set in zt->zdo->hdr set per operation.
+ * TODO: Why this does not work?
+ */
+ _map_pages(zt, zt->zdo->pages, zt->zdo->nump, 0);
+ memcpy(zt->opt_buff, zt->zdo->hdr, zt->zdo->hdr->in_len);
+ } else {
+ struct zufs_ioc_hdr *hdr = zt->opt_buff;
+
+ /* This Means we were released by _zu_break */
+ zuf_dbg_zus("_zu_break? => %d\n", err);
+ hdr->operation = ZUFS_OP_BREAK;
+ hdr->err = err;
+ }
+
+ return err;
+
+err:
+ put_user(err, (int *)parg);
+ return err;
+}
+
+static int _try_grab_zt_channel(struct zuf_root_info *zri, int cpu,
+ struct zufc_thread **ztp)
+{
+ struct zufc_thread *zt;
+ int c;
+
+ for (c = 0; ; ++c) {
+ zt = _zt_from_cpu(zri, cpu, c);
+ if (unlikely(!zt || !zt->hdr.file))
+ break;
+
+ if (relay_is_fss_waiting_grab(&zt->relay)) {
+ *ztp = zt;
+ return true;
+ }
+ }
+
+ *ztp = _zt_from_cpu(zri, cpu, 0);
+ return false;
+}
+
+#define _zuf_get_cpu() get_cpu()
+#define _zuf_put_cpu() put_cpu()
+
+#ifdef CONFIG_ZUF_DEBUG
+static
+int _r_zufs_dispatch(struct zuf_root_info *zri, struct zuf_dispatch_op *zdo)
+#else
+int __zufc_dispatch(struct zuf_root_info *zri, struct zuf_dispatch_op *zdo)
+#endif
+{
+ struct task_struct *app = get_current();
+ struct zufs_ioc_hdr *hdr = zdo->hdr;
+ int cpu, cpu2;
+ struct zufc_thread *zt;
+
+ if (unlikely(hdr->out_len && !hdr->out_max)) {
+ /* TODO: Complain here and let caller code do this proper */
+ hdr->out_max = hdr->out_len;
+ }
+
+channel_busy:
+ cpu = _zuf_get_cpu();
+
+ if (!_try_grab_zt_channel(zri, cpu, &zt)) {
+ _zuf_put_cpu();
+
+ /* If channel was grabbed then maybe a break_all is in progress
+ * on a different CPU make sure zt->file on this core is
+ * updated
+ */
+ mb();
+ if (unlikely(!zt->hdr.file)) {
+ zuf_err("[%d] !zt->file\n", cpu);
+ return -EIO;
+ }
+ zuf_dbg_err("[%d] can this be\n", cpu);
+ /* FIXME: Do something much smarter */
+ msleep(10);
+ if (signal_pending(get_current())) {
+ zuf_dbg_err("[%d] => EINTR\n", cpu);
+ return -EINTR;
+ }
+ goto channel_busy;
+ }
+
+ /* lock app to this cpu while waiting */
+ cpumask_copy(&zt->relay.cpus_allowed, &app->cpus_allowed);
+ cpumask_copy(&app->cpus_allowed, cpumask_of(smp_processor_id()));
+
+ zt->zdo = zdo;
+
+ _zuf_put_cpu();
+
+ relay_fss_wakeup_app_wait(&zt->relay, NULL);
+
+ /* restore cpu affinity after wakeup */
+ cpumask_copy(&app->cpus_allowed, &zt->relay.cpus_allowed);
+
+cpu2 = smp_processor_id();
+if (cpu2 != cpu)
+ zuf_warn("App switched cpu1=%u cpu2=%u\n", cpu, cpu2);
+
+ return zt->hdr.file ? hdr->err : -EIO;
+}
+
+const char *zuf_op_name(enum e_zufs_operation op)
+{
+#define CASE_ENUM_NAME(e) case e: return #e
+ switch (op) {
+ CASE_ENUM_NAME(ZUFS_OP_BREAK );
+ default:
+ return "UNKNOWN";
+ }
+}
+
+#ifdef CONFIG_ZUF_DEBUG
+
+#define MAX_ZT_SEC 5
+int __zufc_dispatch(struct zuf_root_info *zri, struct zuf_dispatch_op *zdo)
+{
+ u64 t1, t2;
+ int err;
+
+ t1 = ktime_get_ns();
+ err = _r_zufs_dispatch(zri, zdo);
+ t2 = ktime_get_ns();
+
+ if ((t2 - t1) > MAX_ZT_SEC * NSEC_PER_SEC)
+ zuf_err("zufc_dispatch(%s, [0x%x-0x%x]) took %lld sec\n",
+ zuf_op_name(zdo->hdr->operation), zdo->hdr->offset,
+ zdo->hdr->len,
+ (t2 - t1) / NSEC_PER_SEC);
+
+ return err;
+}
+#endif /* def CONFIG_ZUF_DEBUG */
+
+/* ~~~ iomap_exec && exec_buffer allocation ~~~ */
+struct zu_exec_buff {
+ struct zuf_special_file hdr;
+ struct vm_area_struct *vma;
+ void *opt_buff;
+ ulong alloc_size;
+};
+
+/* Do some common checks and conversions */
+static inline struct zu_exec_buff *_ebuff_from_file(struct file *file)
+{
+ struct zu_exec_buff *ebuff = file->private_data;
+
+ if (WARN_ON_ONCE(ebuff->hdr.type != zlfs_e_dpp_buff)) {
+ zuf_err("Must call ZU_IOC_ALLOC_BUFFER first\n");
+ return NULL;
+ }
+
+ if (WARN_ON_ONCE(ebuff->hdr.file != file))
+ return NULL;
+
+ return ebuff;
+}
+
+static int _zu_ebuff_alloc(struct file *file, void *arg)
+{
+ struct zufs_ioc_alloc_buffer ioc_alloc;
+ struct zu_exec_buff *ebuff;
+ int err;
+
+ err = copy_from_user(&ioc_alloc, arg, sizeof(ioc_alloc));
+ if (unlikely(err)) {
+ zuf_err("=>%d\n", err);
+ return err;
+ }
+
+ if (ioc_alloc.init_size > ioc_alloc.max_size)
+ return -EINVAL;
+
+ /* TODO: Easily Support growing */
+ /* TODO: Support global pools, also easy */
+ if (ioc_alloc.pool_no || ioc_alloc.init_size != ioc_alloc.max_size)
+ return -ENOTSUPP;
+
+ ebuff = kzalloc(sizeof(*ebuff), GFP_KERNEL);
+ if (unlikely(!ebuff))
+ return -ENOMEM;
+
+ ebuff->hdr.type = zlfs_e_dpp_buff;
+ ebuff->hdr.file = file;
+ i_size_write(file->f_inode, ioc_alloc.max_size);
+ ebuff->alloc_size = ioc_alloc.init_size;
+ ebuff->opt_buff = vmalloc(ioc_alloc.init_size);
+ if (unlikely(!ebuff->opt_buff)) {
+ kfree(ebuff);
+ return -ENOMEM;
+ }
+ _fill_buff(ebuff->opt_buff, ioc_alloc.init_size / sizeof(ulong));
+
+ file->private_data = &ebuff->hdr;
+ return 0;
+}
+
+static void zufc_ebuff_release(struct file *file)
+{
+ struct zu_exec_buff *ebuff = _ebuff_from_file(file);
+
+ if (unlikely(!ebuff))
+ return;
+
+ vfree(ebuff->opt_buff);
+ ebuff->hdr.type = 0;
+ ebuff->hdr.file = NULL; /* for none-dbg Kernels && use-after-free */
+ kfree(ebuff);
+}
+
+static int _zu_break(struct file *filp, void *parg)
+{
+ struct zuf_root_info *zri = ZRI(filp->f_inode->i_sb);
+ int i, c;
+
+ zuf_dbg_core("enter\n");
+ mb(); /* TODO how to schedule on all CPU's */
+
+ for (i = 0; i < zri->_ztp->_max_zts; ++i) {
+ for (c = 0; c < zri->_ztp->_max_channels; ++c) {
+ struct zufc_thread *zt = _zt_from_cpu(zri, i, c);
+
+ if (unlikely(!(zt && zt->hdr.file)))
+ continue;
+ relay_fss_wakeup(&zt->relay);
+ }
+ }
+
+ if (zri->mount.zsf.file)
+ relay_fss_wakeup(&zri->mount.relay);
+
+ zuf_dbg_core("exit\n");
+ return 0;
+}
+
long zufc_ioctl(struct file *file, unsigned int cmd, ulong arg)
{
+ void __user *parg = (void __user *)arg;
+
switch (cmd) {
+ case ZU_IOC_REGISTER_FS:
+ return _zu_register_fs(file, parg);
+ case ZU_IOC_MOUNT:
+ return _zu_mount(file, parg);
+ case ZU_IOC_NUMA_MAP:
+ return _zu_numa_map(file, parg);
+ case ZU_IOC_INIT_THREAD:
+ return _zu_init(file, parg);
+ case ZU_IOC_WAIT_OPT:
+ return _zu_wait(file, parg);
+ case ZU_IOC_ALLOC_BUFFER:
+ return _zu_ebuff_alloc(file, parg);
+ case ZU_IOC_BREAK_ALL:
+ return _zu_break(file, parg);
default:
- zuf_err("%d\n", cmd);
+ zuf_err("%d %ld\n", cmd, ZU_IOC_WAIT_OPT);
return -ENOTTY;
}
}
@@ -38,11 +844,215 @@ int zufc_release(struct inode *inode, struct file *file)
return 0;
switch (zsf->type) {
+ case zlfs_e_zt:
+ zufc_zt_release(file);
+ return 0;
+ case zlfs_e_mout_thread:
+ zufc_mounter_release(file);
+ return 0;
+ case zlfs_e_pmem:
+ /* NOTHING to clean for pmem file yet */
+ /* zuf_pmem_release(file);*/
+ return 0;
+ case zlfs_e_dpp_buff:
+ zufc_ebuff_release(file);
+ return 0;
default:
return 0;
}
}
+/* ~~~~ mmap area of app buffers into server ~~~~ */
+
+static int zuf_zt_fault(struct vm_fault *vmf)
+{
+ zuf_err("should not fault\n");
+ return VM_FAULT_SIGBUS;
+}
+
+static const struct vm_operations_struct zuf_vm_ops = {
+ .fault = zuf_zt_fault,
+};
+
+static int _zufc_zt_mmap(struct file *file, struct vm_area_struct *vma,
+ struct zufc_thread *zt)
+{
+ /* Tell Kernel We will only access on a single core */
+ vma->vm_flags |= VM_MIXEDMAP;
+ vma->vm_ops = &zuf_vm_ops;
+
+ zt->vma = vma;
+
+ zuf_dbg_core(
+ "[0x%lx] start=0x%lx end=0x%lx flags=0x%lx file-start=0x%lx\n",
+ _zt_pr_no(zt), vma->vm_start, vma->vm_end, vma->vm_flags,
+ vma->vm_pgoff);
+
+ return 0;
+}
+
+/* ~~~~ mmap the Kernel allocated IOCTL buffer per ZT ~~~~ */
+static int _opt_buff_mmap(struct vm_area_struct *vma, void *opt_buff,
+ ulong opt_size)
+{
+ ulong offset;
+
+ if (!opt_buff)
+ return -ENOMEM;
+
+ for (offset = 0; offset < opt_size; offset += PAGE_SIZE) {
+ ulong addr = vma->vm_start + offset;
+ ulong pfn = vmalloc_to_pfn(opt_buff + offset);
+ pfn_t pfnt = phys_to_pfn_t(PFN_PHYS(pfn), PFN_MAP | PFN_DEV);
+ int err;
+
+ zuf_dbg_verbose("[0x%lx] pfn-0x%lx addr=0x%lx buff=0x%lx\n",
+ offset, pfn, addr, (ulong)opt_buff + offset);
+
+ err = zuf_flt_to_err(vmf_insert_mixed_mkwrite(vma, addr, pfnt));
+ if (unlikely(err)) {
+ zuf_err("zuf: zuf_insert_mixed_mkwrite => %d offset=0x%lx addr=0x%lx\n",
+ err, offset, addr);
+ return err;
+ }
+ }
+
+ return 0;
+}
+
+static int zuf_obuff_fault(struct vm_fault *vmf)
+{
+ struct vm_area_struct *vma = vmf->vma;
+ struct zufc_thread *zt = _zt_from_f_private(vma->vm_file);
+ long offset = (vmf->pgoff << PAGE_SHIFT) - ZUS_API_MAP_MAX_SIZE;
+ int err;
+
+ zuf_dbg_core(
+ "[0x%lx] start=0x%lx end=0x%lx file-start=0x%lx offset=0x%lx\n",
+ _zt_pr_no(zt), vma->vm_start, vma->vm_end, vma->vm_pgoff,
+ offset);
+
+ /* if Server overruns its buffer crash it dead */
+ if (unlikely((offset < 0) || (zt->max_zt_command < offset))) {
+ zuf_err("[0x%lx] start=0x%lx end=0x%lx file-start=0x%lx offset=0x%lx\n",
+ _zt_pr_no(zt), vma->vm_start,
+ vma->vm_end, vma->vm_pgoff, offset);
+ return VM_FAULT_SIGBUS;
+ }
+
+ /* We never released a zus-core.c that does not fault the
+ * first page first. I want to see if this happens
+ */
+ if (unlikely(offset))
+ zuf_warn("Suspicious server activity\n");
+
+ /* This faults only once at very first access */
+ err = _opt_buff_mmap(vma, zt->opt_buff, zt->max_zt_command);
+ if (unlikely(err))
+ return VM_FAULT_SIGBUS;
+
+ return VM_FAULT_NOPAGE;
+}
+
+static const struct vm_operations_struct zuf_obuff_ops = {
+ .fault = zuf_obuff_fault,
+};
+
+static int _zufc_obuff_mmap(struct file *file, struct vm_area_struct *vma,
+ struct zufc_thread *zt)
+{
+ vma->vm_flags |= VM_MIXEDMAP;
+ vma->vm_ops = &zuf_obuff_ops;
+
+ zt->opt_buff_vma = vma;
+
+ zuf_dbg_core(
+ "[0x%lx] start=0x%lx end=0x%lx flags=0x%lx file-start=0x%lx\n",
+ _zt_pr_no(zt), vma->vm_start, vma->vm_end, vma->vm_flags,
+ vma->vm_pgoff);
+
+ return 0;
+}
+
+/* ~~~ */
+
+static int zufc_zt_mmap(struct file *file, struct vm_area_struct *vma)
+{
+ struct zufc_thread *zt = _zt_from_f_private(file);
+
+ /* We have two areas of mmap in this special file.
+ * 0 to ZUS_API_MAP_MAX_SIZE:
+ * The first part where app pages are mapped
+ * into server per operation.
+ * ZUS_API_MAP_MAX_SIZE of size zuf_root_info->max_zt_command
+ * Is where we map the per ZT ioctl-buffer, later passed
+ * to the zus_ioc_wait IOCTL call
+ */
+ if (vma->vm_pgoff == ZUS_API_MAP_MAX_SIZE / PAGE_SIZE)
+ return _zufc_obuff_mmap(file, vma, zt);
+
+ /* zuf ZT API is very particular about where in its
+ * special file we communicate
+ */
+ if (unlikely(vma->vm_pgoff))
+ return -EINVAL;
+
+ return _zufc_zt_mmap(file, vma, zt);
+}
+
+/* ~~~~ Implementation of the ZU_IOC_ALLOC_BUFFER mmap facility ~~~~ */
+
+static int zuf_ebuff_fault(struct vm_fault *vmf)
+{
+ struct vm_area_struct *vma = vmf->vma;
+ struct zu_exec_buff *ebuff = _ebuff_from_file(vma->vm_file);
+ long offset = (vmf->pgoff << PAGE_SHIFT);
+ int err;
+
+ zuf_dbg_core("start=0x%lx end=0x%lx file-start=0x%lx file-off=0x%lx\n",
+ vma->vm_start, vma->vm_end, vma->vm_pgoff, offset);
+
+ /* if Server overruns its buffer crash it dead */
+ if (unlikely((offset < 0) || (ebuff->alloc_size < offset))) {
+ zuf_err("start=0x%lx end=0x%lx file-start=0x%lx file-off=0x%lx\n",
+ vma->vm_start, vma->vm_end, vma->vm_pgoff,
+ offset);
+ return VM_FAULT_SIGBUS;
+ }
+
+ /* We never released a zus-core.c that does not fault the
+ * first page first. I want to see if this happens
+ */
+ if (unlikely(offset))
+ zuf_warn("Suspicious server activity\n");
+
+ /* This faults only once at very first access */
+ err = _opt_buff_mmap(vma, ebuff->opt_buff, ebuff->alloc_size);
+ if (unlikely(err))
+ return VM_FAULT_SIGBUS;
+
+ return VM_FAULT_NOPAGE;
+}
+
+static const struct vm_operations_struct zuf_ebuff_ops = {
+ .fault = zuf_ebuff_fault,
+};
+
+static int zufc_ebuff_mmap(struct file *file, struct vm_area_struct *vma)
+{
+ struct zu_exec_buff *ebuff = _ebuff_from_file(vma->vm_file);
+
+ vma->vm_flags |= VM_MIXEDMAP;
+ vma->vm_ops = &zuf_ebuff_ops;
+
+ ebuff->vma = vma;
+
+ zuf_dbg_core("start=0x%lx end=0x%lx flags=0x%lx file-start=0x%lx\n",
+ vma->vm_start, vma->vm_end, vma->vm_flags, vma->vm_pgoff);
+
+ return 0;
+}
+
int zufc_mmap(struct file *file, struct vm_area_struct *vma)
{
struct zuf_special_file *zsf = file->private_data;
@@ -53,6 +1063,10 @@ int zufc_mmap(struct file *file, struct vm_area_struct *vma)
}
switch (zsf->type) {
+ case zlfs_e_zt:
+ return zufc_zt_mmap(file, vma);
+ case zlfs_e_dpp_buff:
+ return zufc_ebuff_mmap(file, vma);
default:
zuf_err("type=%d\n", zsf->type);
return -ENOTTY;
diff --git a/fs/zuf/zuf-root.c b/fs/zuf/zuf-root.c
index 55a839dbc854..37b70ca33d3c 100644
--- a/fs/zuf/zuf-root.c
+++ b/fs/zuf/zuf-root.c
@@ -227,6 +227,7 @@ static void zufr_put_super(struct super_block *sb)
{
struct zuf_root_info *zri = ZRI(sb);
+ zufc_zts_fini(zri);
_unregister_all_fses(zri);
zuf_info("zuf_root umount\n");
@@ -282,10 +283,16 @@ static int zufr_fill_super(struct super_block *sb, void *data, int silent)
root_i->i_fop = &zufr_file_dir_operations;
root_i->i_op = &zufr_inode_operations;
+ spin_lock_init(&zri->mount.lock);
mutex_init(&zri->sbl_lock);
+ relay_init(&zri->mount.relay);
INIT_LIST_HEAD(&zri->fst_list);
INIT_LIST_HEAD(&zri->pmem_list);
+ err = zufc_zts_init(zri);
+ if (unlikely(err))
+ return err; /* put will be called we have a root */
+
return 0;
}
diff --git a/fs/zuf/zuf.h b/fs/zuf/zuf.h
index f979d8cbe60c..a33f5908155d 100644
--- a/fs/zuf/zuf.h
+++ b/fs/zuf/zuf.h
@@ -23,9 +23,11 @@
#include <linux/xattr.h>
#include <linux/exportfs.h>
#include <linux/page_ref.h>
+#include <linux/mm.h>
#include "zus_api.h"
+#include "relay.h"
#include "_pr.h"
enum zlfs_e_special_file {
@@ -44,6 +46,8 @@ struct zuf_special_file {
struct zuf_root_info {
struct __mount_thread_info {
struct zuf_special_file zsf;
+ spinlock_t lock;
+ struct relay relay;
struct zufs_ioc_mount *zim;
} mount;
@@ -102,6 +106,48 @@ static inline struct zuf_inode_info *ZUII(struct inode *inode)
return container_of(inode, struct zuf_inode_info, vfs_inode);
}
+static inline struct zuf_fs_type *ZUF_FST(struct file_system_type *fs_type)
+{
+ return container_of(fs_type, struct zuf_fs_type, vfs_fst);
+}
+
+static inline struct zuf_fs_type *zuf_fst(struct super_block *sb)
+{
+ return ZUF_FST(sb->s_type);
+}
+
+struct zuf_dispatch_op;
+typedef int (*overflow_handler)(struct zuf_dispatch_op *zdo, void *parg,
+ ulong zt_max_bytes);
+struct zuf_dispatch_op {
+ struct zufs_ioc_hdr *hdr;
+ struct page **pages;
+ uint nump;
+ overflow_handler oh;
+ struct super_block *sb;
+ struct inode *inode;
+};
+
+static inline void
+zuf_dispatch_init(struct zuf_dispatch_op *zdo, struct zufs_ioc_hdr *hdr,
+ struct page **pages, uint nump)
+{
+ memset(zdo, 0, sizeof(*zdo));
+ zdo->hdr = hdr;
+ zdo->pages = pages; zdo->nump = nump;
+}
+
+static inline int zuf_flt_to_err(vm_fault_t flt)
+{
+ if (likely(flt == VM_FAULT_NOPAGE))
+ return 0;
+
+ if (flt == VM_FAULT_OOM)
+ return -ENOMEM;
+
+ return -EACCES;
+}
+
/* Keep this include last thing in file */
#include "_extern.h"
diff --git a/fs/zuf/zus_api.h b/fs/zuf/zus_api.h
index 34e3e1a9a107..3319a70b5ccc 100644
--- a/fs/zuf/zus_api.h
+++ b/fs/zuf/zus_api.h
@@ -66,6 +66,47 @@
#endif /* ndef __KERNEL__ */
+/* first available error code after include/linux/errno.h */
+#define EZUFS_RETRY 531
+
+/* The below is private to zuf Kernel only. Is not exposed to VFS nor zus
+ * (defined here to allocate the constant)
+ */
+#define EZUF_RETRY_DONE 540
+
+/**
+ * zufs dual port memory
+ * This is a special type of offset to either memory or persistent-memory,
+ * that is designed to be used in the interface mechanism between userspace
+ * and kernel, and can be accessed by both.
+ * 3 first bits denote a mem-pool:
+ * 0 - pmem pool
+ * 1-6 - established shared pool by a call to zufs_ioc_create_mempool (below)
+ * 7 - offset into app memory
+ */
+typedef __u64 __bitwise zu_dpp_t;
+
+static inline uint zu_dpp_t_pool(zu_dpp_t t)
+{
+ return t & 0x7;
+}
+
+static inline ulong zu_dpp_t_val(zu_dpp_t t)
+{
+ return t & ~0x7;
+}
+
+static inline zu_dpp_t enc_zu_dpp_t(ulong v, uint pool)
+{
+ return v | pool;
+}
+
+/* ~~~~~ ZUFS API ioctl commands ~~~~~ */
+enum {
+ ZUS_API_MAP_MAX_PAGES = 1024,
+ ZUS_API_MAP_MAX_SIZE = ZUS_API_MAP_MAX_PAGES * PAGE_SIZE,
+};
+
struct zufs_ioc_hdr {
__u32 err; /* IN/OUT must be first */
__u16 in_len; /* How much to be copied *to* zus */
@@ -102,4 +143,148 @@ struct zufs_ioc_register_fs {
};
#define ZU_IOC_REGISTER_FS _IOWR('Z', 10, struct zufs_ioc_register_fs)
+/* A cookie from user-mode returned by mount */
+struct zus_sb_info;
+
+/* zus cookie per inode */
+struct zus_inode_info;
+
+enum ZUFS_M_FLAGS {
+ ZUFS_M_PEDANTIC = 0x00000001,
+ ZUFS_M_EPHEMERAL = 0x00000002,
+ ZUFS_M_SILENT = 0x00000004,
+};
+
+struct zufs_parse_options {
+ __u32 mount_options_len;
+ __u32 pedantic;
+ __u64 mount_flags;
+ char mount_options[0];
+};
+
+enum e_mount_operation {
+ ZUFS_M_MOUNT = 1,
+ ZUFS_M_UMOUNT,
+ ZUFS_M_REMOUNT,
+ ZUFS_M_DDBG_RD,
+ ZUFS_M_DDBG_WR,
+};
+
+struct zufs_mount_info {
+ /* IN */
+ struct zus_fs_info *zus_zfi;
+ __u16 num_cpu;
+ __u16 num_channels;
+ __u32 pmem_kern_id;
+ __u64 sb_id;
+
+ /* OUT */
+ struct zus_sb_info *zus_sbi;
+ /* mount is also iget of root */
+ struct zus_inode_info *zus_ii;
+ zu_dpp_t _zi;
+ __u64 old_mount_opt;
+ __u64 remount_flags;
+
+ /* More FS specific info */
+ __u32 s_blocksize_bits;
+ __u8 acl_on;
+ struct zufs_parse_options po;
+};
+
+/* mount / umount */
+struct zufs_ioc_mount {
+ struct zufs_ioc_hdr hdr;
+ struct zufs_mount_info zmi;
+};
+#define ZU_IOC_MOUNT _IOWR('Z', 11, struct zufs_ioc_mount)
+
+/* pmem */
+struct zufs_ioc_numa_map {
+ /* Set by zus */
+ struct zufs_ioc_hdr hdr;
+
+ __u32 possible_nodes;
+ __u32 possible_cpus;
+ __u32 online_nodes;
+ __u32 online_cpus;
+
+ __u32 max_cpu_per_node;
+
+ /* This indicates that NOT all nodes have @max_cpu_per_node cpus */
+ bool nodes_not_symmetrical;
+
+ /* Variable size must keep last
+ * size @online_cpus
+ */
+ __u8 cpu_to_node[];
+};
+#define ZU_IOC_NUMA_MAP _IOWR('Z', 12, struct zufs_ioc_numa_map)
+
+/* ZT init */
+enum { ZUFS_MAX_ZT_CHANNELS = 64 };
+
+struct zufs_ioc_init {
+ struct zufs_ioc_hdr hdr;
+ ulong affinity; /* IN */
+ uint channel_no;
+ uint max_command;
+};
+#define ZU_IOC_INIT_THREAD _IOWR('Z', 14, struct zufs_ioc_init)
+
+/* break_all (Server telling kernel to clean) */
+struct zufs_ioc_break_all {
+ struct zufs_ioc_hdr hdr;
+};
+#define ZU_IOC_BREAK_ALL _IOWR('Z', 15, struct zufs_ioc_break_all)
+
+/* ~~~ zufs_ioc_wait_operation ~~~ */
+struct zufs_ioc_wait_operation {
+ struct zufs_ioc_hdr hdr;
+ /* maximum size is governed by zufs_ioc_init->max_command */
+ char opt_buff[];
+};
+#define ZU_IOC_WAIT_OPT _IOWR('Z', 16, struct zufs_ioc_wait_operation)
+
+/* These are the possible operations sent from Kernel to the Server in the
+ * return of the ZU_IOC_WAIT_OPT.
+ */
+enum e_zufs_operation {
+ ZUFS_OP_NULL = 0,
+
+ ZUFS_OP_BREAK, /* Kernel telling Server to exit */
+ ZUFS_OP_MAX_OPT,
+};
+
+/* Allocate a special_file that will be a dual-port communication buffer with
+ * user mode.
+ * Server will access the buffer via the mmap of this file.
+ * Kernel will access the file via the valloc() pointer
+ *
+ * Some IOCTLs below demand use of this kind of buffer for communication
+ * TODO:
+ * pool_no is if we want to associate this buffer onto the 6 possible
+ * mem-pools per zuf_sbi. So anywhere we have a zu_dpp_t it will mean
+ * access from this pool.
+ * If pool_no is zero then it is private to only this file. In this case
+ * sb_id && zus_sbi are ignored / not needed.
+ */
+struct zufs_ioc_alloc_buffer {
+ struct zufs_ioc_hdr hdr;
+ /* The ID of the super block received in mount */
+ __u64 sb_id;
+ /* We verify the sb_id validity against zus_sbi */
+ struct zus_sb_info *zus_sbi;
+ /* max size of buffer allowed (size of mmap) */
+ __u32 max_size;
+ /* allocate this much on initial call and set into vma */
+ __u32 init_size;
+
+ /* TODO: These below are now set to ZERO. Need implementation */
+ __u16 pool_no;
+ __u16 flags;
+ __u32 reserved;
+};
+#define ZU_IOC_ALLOC_BUFFER _IOWR('Z', 17, struct zufs_ioc_init)
+
#endif /* _LINUX_ZUFS_API_H */
--
2.20.1
