Re: [RFC 0/2] TCM: A userspace backend

[Shaohua Li <shli@xxxxxxxxxx>]

On Sat, Nov 16, 2013 at 09:22:50AM +0800, Shaohua Li wrote:
> Hi,
> 
> This is a userspace backend, which allows handling requests in application. It
> would be useful if complex data transformation is required. Patch 2 description
> has more details about it. Attached is the simple test application I used.
> Comment is welcome!

forgot the test app, here it is.

#define _LARGEFILE64_SOURCE
#include <sys/types.h>
#include <poll.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <unistd.h>
#include <stdio.h>
#include <sys/mman.h>
#include <signal.h>
#include <stdint.h>
#include <stdlib.h>
#include <string.h>
#include "/home/shli/linux/drivers/target/target_core_user.h"

#define MAX_ENTRIES 4086
#define PAGE_SIZE 4096
#define ROUND_UP_PAGES(a) ((a + PAGE_SIZE - 1) / PAGE_SIZE)
#define DEV_FILE "test.img"

static int dev_fd, signal_fd;
static struct tcmu_mailbox *mb;

static int handle_one_request(struct tcmu_cmd_entry *cmd)
{
	uint64_t head = mb->cpl_head;
	struct tcmu_cpl_entry *cpl = (struct tcmu_cpl_entry *)mb->cpl_head;
	uint16_t iov_cnt = 0;
	uint64_t repeat;

	switch (cmd->opcode) {
	case TCMU_OP_READ:
		iov_cnt = cmd->iov_cnt;
		preadv(dev_fd, cmd->iov, cmd->iov_cnt, cmd->offset);
		break;
	case TCMU_OP_WRITE:
	case TCMU_OP_WRITE_FUA:
		iov_cnt = cmd->iov_cnt;
		pwritev(dev_fd, cmd->iov, cmd->iov_cnt, cmd->offset);
		break;
	case TCMU_OP_WRITE_SAME:
		iov_cnt = 1;
		for (repeat = 0; repeat < cmd->writesame_repeat; repeat++)
			pwritev(dev_fd, cmd->iov, cmd->iov_cnt,
				cmd->offset + repeat * cmd->iov[0].iov_len);
		break;
	case TCMU_OP_SYNC: /* ignore for now */
		break;
	case TCMU_OP_UNMAP:
		break;
	}

	cpl->cmd_id = cmd->cmd_id;
	cpl->result = 0;

	head += sizeof(struct tcmu_cpl_entry);
	if (head >= mb->cplr_addr + mb->cplr_size)
		head = mb->cplr_addr;

	mb->cpl_head = head;

	return sizeof(*cmd) + sizeof(struct iovec) * iov_cnt;
}

static void handle_request(void)
{
	struct tcmu_cmd_entry *cmd;
	char *tail, *head;
	uint32_t event = TCMU_EVT_USER_CMDR_NOT_FULL|TCMU_EVT_USER_CPL_PENDING;

	tail = (char *)mb->cmd_tail;
	head = (char *)mb->cmd_head;
	while (tail != head) {
		cmd = (struct tcmu_cmd_entry *)tail;
		if (cmd->opcode == TCMU_OP_PAD)
			tail += TCMU_OP_PAD_SIZE;
		else
			tail += handle_one_request(cmd);
		if ((unsigned long)tail >= mb->cmdr_addr + mb->cmdr_size)
			tail = (char *)mb->cmdr_addr;
	}
	mb->cmd_tail = (uint64_t)head;

	write(signal_fd, &event, sizeof(event));
}

static void sighand(int sig)
{
}

#define NAA_STR "naa.60014051153c198a"
#define VPD_STR "b9802841-1d9a-4e08-a8a0-d4161221"
static void write_file(char *name, char *data, ssize_t len)
{
	int fd = open(name, O_WRONLY|O_CREAT|O_TRUNC, 0666);
	write(fd, data, len);
	close(fd);
}

int main(void)
{
	char *addr;
	struct stat stat;
	char buf[100];
	int cnt;
	struct pollfd pollfd;
	int cmd_pages = ROUND_UP_PAGES(MAX_ENTRIES * (sizeof(struct tcmu_cmd_entry) +
			32 * sizeof(struct iovec)));
	int cpl_pages = ROUND_UP_PAGES(MAX_ENTRIES * sizeof(struct tcmu_cpl_entry));
	int pages = cmd_pages + cpl_pages + 1 + MAX_ENTRIES;

	addr = mmap(NULL, pages * PAGE_SIZE, PROT_READ|PROT_WRITE,
		MAP_PRIVATE|MAP_ANONYMOUS, 0, 0);
	if (!addr) {
		perror("mmap failed\n");
		exit(1);
	}

	mb = (struct tcmu_mailbox *)addr;
	mb->version = TCMU_MAILBOX_VERSION;
	mb->cmdr_addr = (uint64_t)(addr + PAGE_SIZE);
	mb->cmdr_size = cmd_pages * PAGE_SIZE;
	mb->cplr_addr = mb->cmdr_addr + mb->cmdr_size;
	mb->cplr_size = cpl_pages * PAGE_SIZE;
	mb->data_addr = mb->cplr_addr + mb->cplr_size;
	mb->data_size = MAX_ENTRIES * PAGE_SIZE;

	mb->cmd_head = mb->cmd_tail = mb->cmdr_addr;
	mb->cpl_head = mb->cpl_tail = mb->cplr_addr;

	dev_fd = open(DEV_FILE, O_LARGEFILE|O_RDWR);
	if (dev_fd < 0) {
		perror("open fd failed\n");
		exit(1);
	}

	if (fstat(dev_fd, &stat) < 0) {
		perror("stat file failed\n");
		exit(1);
	}
	
	mkdir("/sys/kernel/config/target/loopback/naa.60014059436855c1", 0777);
	mkdir("/sys/kernel/config/target/loopback/naa.60014059436855c1/tpgt_1", 0777);
	write_file("/sys/kernel/config/target/loopback/naa.60014059436855c1/tpgt_1/nexus",
		NAA_STR, strlen(NAA_STR));

	mkdir("/sys/kernel/config/target/core/tcmu_0", 0777);
	mkdir("/sys/kernel/config/target/core/tcmu_0/test", 0777);
	cnt = sprintf(buf, "mailbox_addr=%llu,mailbox_size=%u,dev_size=%llu",
		addr, pages * PAGE_SIZE, stat.st_size);
	write_file("/sys/kernel/config/target/core/tcmu_0/test/control",
		buf, cnt);
	write_file("/sys/kernel/config/target/core/tcmu_0/test/udev_path",
		"test.img", 8);
	write_file("/sys/kernel/config/target/core/tcmu_0/test/enable",
		"1\n", 2);
	write_file("/sys/kernel/config/target/core/tcmu_0/test/wwn/vpd_unit_serial",
		VPD_STR, strlen(VPD_STR));

	mkdir("/sys/kernel/config/target/loopback/naa.60014059436855c1/tpgt_1/lun/lun_0", 0777);
	symlink("/sys/kernel/config/target/core/tcmu_0/test",
	 "/sys/kernel/config/target/loopback/naa.60014059436855c1/tpgt_1/lun/lun_0/c68d79910b");

	signal(SIGINT, sighand);

	signal_fd = mb->signal_fd;

	pollfd.fd = signal_fd;
	pollfd.events = POLLIN|POLLHUP;

	while (1) {
		uint32_t event;

		if (poll(&pollfd, 1, -1) < 0)
			break;
		if (pollfd.revents & POLLHUP)
			break;

		read(signal_fd, &event, sizeof(event));
		handle_request();
	}

	unlink("/sys/kernel/config/target/loopback/naa.60014059436855c1/tpgt_1/lun/lun_0/c68d79910b");
	rmdir("/sys/kernel/config/target/loopback/naa.60014059436855c1/tpgt_1/lun/lun_0");

	rmdir("/sys/kernel/config/target/core/tcmu_0/test");
	rmdir("/sys/kernel/config/target/core/tcmu_0");

	rmdir("/sys/kernel/config/target/loopback/naa.60014059436855c1/tpgt_1");
	rmdir("/sys/kernel/config/target/loopback/naa.60014059436855c1");

	close(signal_fd);
	close(dev_fd);
	munmap(addr, pages * PAGE_SIZE);
	return 0;
}