This patch adds the first new back-end and makes some changes to the way
nodes are added, to make the invariants of storage_node more sensible.
The filesystem back-end itself is not intended for production use,
so it makes no attempt to run any asynchronous transfers.
We also add a test. Note that this differs from the preliminary versions
of this patch. We used to add both chunk and fs back-ends, so that tabled
replicates to both. This makes sense as a test of store path, but on
retrieval tabled selects any one of available storage nodes with the
object, randomly. It creates gaps in test coverage in any given run.
Therefore, we test two back-end types sequentially now.
Signed-off-by: Pete Zaitcev <zaitcev@xxxxxxxxxx>
---
server/Makefile.am | 2
server/stor_chunk.c | 21 -
server/stor_fs.c | 498 +++++++++++++++++++++++++++++++++++++++++
server/storage.c | 157 ++++++++++--
server/storparse.c | 97 +++++++
server/tabled.h | 31 ++
test/Makefile.am | 3
test/be_fs-test.conf | 5
test/combo-redux | 74 ++++++
test/prep-db | 4
test/start-daemon | 1
test/stop-daemon | 9
12 files changed, 835 insertions(+), 67 deletions(-)
commit bccedeedabbe713e4053afa185314b3f57f3d204
Author: Pete Zaitcev <zaitcev@xxxxxxxxx>
Date: Sun Nov 28 17:58:05 2010 -0700
Add fs back-end, with a test.
diff --git a/server/Makefile.am b/server/Makefile.am
index 52beec4..71bcb35 100644
--- a/server/Makefile.am
+++ b/server/Makefile.am
@@ -6,7 +6,7 @@ sbin_PROGRAMS = tabled tdbadm
tabled_SOURCES = tabled.h \
bucket.c cldu.c config.c metarep.c object.c replica.c \
server.c status.c storage.c storparse.c \
- stor_chunk.c util.c
+ stor_chunk.c stor_fs.c util.c
tabled_LDADD = ../lib/libtdb.a \
@HAIL_LIBS@ @PCRE_LIBS@ @GLIB_LIBS@ \
@CRYPTO_LIBS@ @DB4_LIBS@ @EVENT_LIBS@ @SSL_LIBS@
diff --git a/server/stor_chunk.c b/server/stor_chunk.c
index 815adcf..7462a9c 100644
--- a/server/stor_chunk.c
+++ b/server/stor_chunk.c
@@ -31,8 +31,7 @@
#include <netdb.h>
#include "tabled.h"
-static const char stor_key_fmt[] = "%016llx";
-#define STOR_KEY_SLEN 16
+static const char stor_key_fmt[] = STOR_KEY_FMT;
static int stor_new_stc(struct storage_node *stn, struct st_client **stcp)
{
@@ -66,24 +65,6 @@ static int stor_new_stc(struct storage_node *stn, struct st_client **stcp)
return 0;
}
-static void stor_read_event(int fd, short events, void *userdata)
-{
- struct open_chunk *cep = userdata;
-
- cep->r_armed = false; /* no EV_PERSIST */
- if (cep->ocb)
- (*cep->ocb)(cep);
-}
-
-static void stor_write_event(int fd, short events, void *userdata)
-{
- struct open_chunk *cep = userdata;
-
- cep->w_armed = false; /* no EV_PERSIST */
- if (cep->ocb)
- (*cep->ocb)(cep);
-}
-
/*
* Open *cep using stn, set up chunk session if needed.
*/
diff --git a/server/stor_fs.c b/server/stor_fs.c
new file mode 100644
index 0000000..b433a67
--- /dev/null
+++ b/server/stor_fs.c
@@ -0,0 +1,498 @@
+
+/*
+ * Copyright 2010 Red Hat, Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; see the file COPYING. If not, write to
+ * the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ */
+
+#define _GNU_SOURCE
+#include "tabled-config.h"
+
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <syslog.h>
+#include <string.h>
+#include <glib.h>
+#include <event.h>
+#include "tabled.h"
+
+static const char stor_key_fmt[] = STOR_KEY_FMT;
+
+static char *fs_obj_pathname(const char *base, uint64_t key)
+{
+ enum { PREFIX_LEN = 3 };
+ char prefix[PREFIX_LEN + 1];
+ char stckey[STOR_KEY_SLEN+1];
+ char *s;
+ int rc;
+
+ /* we know that stckey is going to be longer than PREFIX_LEN */
+ sprintf(stckey, stor_key_fmt, (unsigned long long) key);
+ memcpy(prefix, stckey, PREFIX_LEN);
+ prefix[PREFIX_LEN] = 0;
+
+ rc = asprintf(&s, "%s/%s/%s", base, prefix, stckey + PREFIX_LEN);
+ if (rc < 0)
+ goto err_out;
+
+ return s;
+
+err_out:
+ return NULL;
+}
+
+static char *fs_ctl_pathname(const char *base, const char *file)
+{
+ char *s;
+ int rc;
+
+ rc = asprintf(&s, "%s/%s", base, file);
+ if (rc < 0)
+ return NULL;
+ return s;
+}
+
+static int fs_obj_mkpath(const char *path)
+{
+ struct stat statb;
+ char *s;
+ int rc;
+
+ /* one dir is enough */
+ /* not using dirname because on some platforms it modifies its arg. */
+ s = strrchr(path, '/');
+ if (s == NULL)
+ return -EINVAL;
+ s = strndup(path, s-path);
+ if (!s)
+ return -ENOMEM;
+
+ /* create subdir on the fly, if not already exists */
+ if (stat(s, &statb) < 0) {
+ rc = errno;
+ if (rc != ENOENT)
+ goto err_out;
+ if (mkdir(s, 0777) < 0) {
+ rc = errno;
+ /*
+ * Directory already exists, perhaps
+ * because we raced with another thread.
+ */
+ if (rc != EEXIST)
+ goto err_out;
+ }
+ } else {
+ if (!S_ISDIR(statb.st_mode)) {
+ rc = EINVAL;
+ goto err_out;
+ }
+ }
+
+ free(s);
+ return 0;
+
+err_out:
+ free(s);
+ return -rc;
+}
+
+static int fs_open(struct open_chunk *cep, struct storage_node *stn,
+ struct event_base *ev_base)
+{
+ if (cep->node)
+ return -EBUSY;
+
+ if (!stn->basepath) {
+ applog(LOG_WARNING,
+ "No base path for Posix chunk, nid %u", stn->id);
+ return -EINVAL;
+ }
+
+ cep->evbase = ev_base;
+ cep->node = stor_node_get(stn);
+ cep->pfd = -1;
+
+ return 0;
+}
+
+static int fs_open_read(struct open_chunk *cep,
+ void (*cb)(struct open_chunk *),
+ uint64_t key, uint64_t *psize)
+{
+ char *objpath;
+ struct stat statb;
+ uint64_t size;
+ int rc;
+
+ if (!cep->node || cep->key)
+ return -EBUSY;
+
+ objpath = fs_obj_pathname(cep->node->basepath, key);
+ if (!objpath) {
+ applog(LOG_WARNING, "No core");
+ return -ENOMEM;
+ }
+
+ rc = open(objpath, O_RDONLY);
+ if (rc == -1) {
+ rc = errno;
+ applog(LOG_WARNING, "Cannot open file %s oid %llX: %s",
+ objpath, (long long) key, strerror(rc));
+ free(objpath);
+ return -rc;
+ }
+ cep->pfd = rc;
+
+ if (fstat(cep->pfd, &statb) < 0) {
+ rc = errno;
+ applog(LOG_WARNING, "Cannot stat file %s: %s",
+ objpath, strerror(rc));
+ close(cep->pfd);
+ cep->pfd = -1;
+ free(objpath);
+ return -rc;
+ }
+ size = statb.st_size;
+
+ *psize = size;
+ cep->size = size;
+ cep->done = 0;
+ cep->key = key;
+ cep->ocb = cb;
+
+ /*
+ * We cannot call cep->ocb directly. Instead, we steal the
+ * arm-disarm mechanism from chunk. This works because in Linux
+ * regular files can be polled and always return ready.
+ */
+ event_set(&cep->revt, cep->pfd, EV_READ, stor_read_event, cep);
+ event_base_set(cep->evbase, &cep->revt);
+
+ free(objpath);
+ return 0;
+}
+
+static void fs_close(struct open_chunk *cep)
+{
+ if (cep->node) {
+ stor_node_put(cep->node);
+ cep->node = NULL;
+ if (cep->pfd != -1) {
+ close(cep->pfd);
+ cep->pfd = -1;
+ }
+ }
+
+ cep->done = 0;
+ cep->size = 0;
+
+ if (cep->r_armed) {
+ event_del(&cep->revt);
+ cep->r_armed = false;
+ }
+
+ if (cep->w_armed) {
+ event_del(&cep->wevt);
+ cep->w_armed = false;
+ }
+
+ cep->key = 0;
+}
+
+static void fs_abort(struct open_chunk *cep)
+{
+ if (cep->r_armed) {
+ event_del(&cep->revt);
+ cep->r_armed = false;
+ }
+ if (cep->w_armed) {
+ event_del(&cep->wevt);
+ cep->w_armed = false;
+ }
+ /* XXX delete the unfinished object under write */
+ cep->key = 0;
+}
+
+static int fs_put_start(struct open_chunk *cep,
+ void (*cb)(struct open_chunk *),
+ uint64_t key, uint64_t size)
+{
+ char *objpath;
+ int rc;
+
+ if (!cep->node || cep->key)
+ return -EBUSY;
+
+ objpath = fs_obj_pathname(cep->node->basepath, key);
+ if (!objpath) {
+ applog(LOG_WARNING, "No core");
+ return -ENOMEM;
+ }
+
+ rc = fs_obj_mkpath(objpath);
+ if (rc) {
+ applog(LOG_WARNING, "Cannot create a directory for %s: %s",
+ objpath, strerror(-rc));
+ free(objpath);
+ return rc;
+ }
+
+ rc = open(objpath, O_WRONLY|O_TRUNC|O_CREAT, 0666);
+ if (rc == -1) {
+ rc = errno;
+ applog(LOG_WARNING, "Cannot create file %s: %s",
+ objpath, strerror(rc));
+ free(objpath);
+ return -rc;
+ }
+ cep->pfd = rc;
+
+ cep->size = size;
+ cep->done = 0;
+ cep->key = key;
+ cep->ocb = cb;
+ event_set(&cep->wevt, cep->pfd, EV_WRITE, stor_write_event, cep);
+ event_base_set(cep->evbase, &cep->wevt);
+
+ free(objpath);
+ return 0;
+}
+
+static ssize_t fs_put_buf(struct open_chunk *cep, void *data, size_t len)
+{
+ ssize_t rcs;
+
+ if (len == 0) {
+ applog(LOG_ERR, "Put length zero remaining %ld",
+ cep->size - cep->done);
+ return -EIO; /* will spin otherwise, better error */
+ }
+
+ if (cep->done + len > cep->size) {
+ /* P3 */ applog(LOG_ERR, "Put length %ld remaining %ld",
+ (long) len, (long) (cep->size - cep->done));
+ if (cep->done == cep->size)
+ return -EIO; /* will spin otherwise, better error */
+ len = cep->size - cep->done;
+ }
+
+ if (!cep->node)
+ return -EPIPE;
+
+ rcs = write(cep->pfd, data, len);
+ if (rcs < 0)
+ return -errno;
+
+ if (!cep->w_armed) {
+ event_add(&cep->wevt, NULL);
+ cep->w_armed = true;
+ }
+
+ cep->done += rcs;
+ if (rcs < len)
+ return rcs;
+
+ return len;
+}
+
+static bool fs_put_end(struct open_chunk *cep)
+{
+ if (!cep->node)
+ return true;
+ if (cep->pfd != -1) {
+ if (fdatasync(cep->pfd) != 0)
+ return false;
+ close(cep->pfd);
+ cep->pfd = -1;
+ }
+ if (cep->w_armed) {
+ event_del(&cep->wevt);
+ cep->w_armed = false;
+ }
+ return true;
+}
+
+static ssize_t fs_get_buf(struct open_chunk *cep, void *data, size_t req_len)
+{
+ size_t xfer_len;
+ ssize_t rcs;
+
+ if (!cep->node)
+ return -EDOM;
+
+ if (cep->done + req_len < cep->done) /* wrap */
+ return -EINVAL;
+ if (cep->done + req_len > cep->size)
+ xfer_len = cep->size - cep->done;
+ else
+ xfer_len = req_len;
+ if (xfer_len == 0)
+ return 0;
+
+ rcs = read(cep->pfd, data, xfer_len);
+ if (rcs < 0)
+ return -errno;
+
+ cep->done += rcs;
+ if (cep->done == cep->size) {
+ cep->done = 0;
+ cep->size = 0;
+ close(cep->pfd);
+ cep->pfd = -1;
+ if (cep->r_armed) {
+ event_del(&cep->revt);
+ cep->r_armed = false;
+ }
+ return rcs;
+ }
+
+ if (xfer_len != rcs && cep->size && !cep->r_armed) {
+ cep->r_armed = true;
+ if (event_add(&cep->revt, NULL))
+ cep->r_armed = false;
+ }
+
+ return rcs;
+}
+
+static int fs_obj_del(struct storage_node *stn, uint64_t key)
+{
+ char *objpath;
+ int rc;
+
+ objpath = fs_obj_pathname(stn->basepath, key);
+ if (!objpath) {
+ applog(LOG_WARNING, "No core");
+ return -ENOMEM;
+ }
+
+ if (unlink(objpath) != 0) {
+ rc = errno;
+ applog(LOG_WARNING, "Cannot unlink oid %llX file %s: %s",
+ (long long)key, objpath, strerror(rc));
+ free(objpath);
+ return -rc;
+ }
+
+ free(objpath);
+ return 0;
+}
+
+static bool fs_obj_test(struct open_chunk *cep, uint64_t key)
+{
+ struct stat statb;
+ char *objpath;
+
+ objpath = fs_obj_pathname(cep->node->basepath, key);
+ if (!objpath) {
+ applog(LOG_WARNING, "No core");
+ return false;
+ }
+
+ if (stat(objpath, &statb) != 0) {
+ applog(LOG_WARNING, "Cannot stat oid %llX file %s: %s",
+ (long long)key, objpath, strerror(errno));
+ free(objpath);
+ return false;
+ }
+
+ free(objpath);
+ return true;
+}
+
+static long stor_readnid(const char *fname)
+{
+ enum { LEN = 45 };
+ char buf[LEN+1];
+ long num;
+ int fd;
+ int rc;
+
+ if ((fd = open(fname, O_RDONLY)) == -1)
+ return -errno;
+ rc = read(fd, buf, LEN);
+ close(fd);
+ if (rc < 0)
+ return -errno;
+ if (rc == 0)
+ return -EPIPE;
+ buf[rc] = 0;
+
+ num = strtol(buf, NULL, 10);
+ if (num < INT_MIN || num > INT_MAX)
+ return -ERANGE;
+
+ return num;
+}
+
+static int fs_node_check(struct storage_node *stn)
+{
+ char *objpath;
+ long rcl;
+
+ if (!stn->basepath)
+ return -1;
+
+ objpath = fs_ctl_pathname(stn->basepath, "NID");
+ if (!objpath) {
+ applog(LOG_WARNING, "No core");
+ return -1;
+ }
+
+ rcl = stor_readnid(objpath);
+ if (rcl < 0) {
+ if (!stn->reported) {
+ applog(LOG_ERR, "Cannot verify nid %u path %s: %s",
+ stn->id, objpath, strerror(-rcl));
+ stn->reported = true;
+ }
+ free(objpath);
+ return -1;
+ }
+
+ /*
+ * This prevents a catastrophy of two entries in CLD pointing
+ * to the same directory. Happens way easier than one expects.
+ */
+ if (stn->id != rcl) {
+ if (!stn->reported) {
+ applog(LOG_ERR, "Mismatch nid %u fetched %u",
+ stn->id, rcl);
+ stn->reported = true;
+ }
+ free(objpath);
+ return -1;
+ }
+
+ free(objpath);
+ return 0;
+}
+
+struct st_node_ops stor_ops_posix = {
+ .open = fs_open,
+ .open_read = fs_open_read,
+ .close = fs_close,
+ .abort = fs_abort,
+ .put_start = fs_put_start,
+ .put_buf = fs_put_buf,
+ .put_end = fs_put_end,
+ .get_buf = fs_get_buf,
+ .obj_del = fs_obj_del,
+ .obj_test = fs_obj_test,
+ .node_check = fs_node_check,
+};
+
diff --git a/server/storage.c b/server/storage.c
index b0278a0..97a4ca2 100644
--- a/server/storage.c
+++ b/server/storage.c
@@ -50,6 +50,24 @@ void stor_node_put(struct storage_node *sn)
--sn->ref;
}
+void stor_read_event(int fd, short events, void *userdata)
+{
+ struct open_chunk *cep = userdata;
+
+ cep->r_armed = false; /* no EV_PERSIST */
+ if (cep->ocb)
+ (*cep->ocb)(cep);
+}
+
+void stor_write_event(int fd, short events, void *userdata)
+{
+ struct open_chunk *cep = userdata;
+
+ cep->w_armed = false; /* no EV_PERSIST */
+ if (cep->ocb)
+ (*cep->ocb)(cep);
+}
+
static struct storage_node *_stor_node_by_nid(uint32_t nid)
{
struct storage_node *sn;
@@ -80,6 +98,16 @@ static int stor_add_node_addr(struct storage_node *sn,
struct addrinfo *res, *res0;
int rc;
+ if (sn->hostname == NULL || strcmp(sn->hostname, hostname) != 0) {
+ free(sn->hostname);
+ sn->hostname = strdup(hostname);
+ if (!sn->hostname) {
+ applog(LOG_WARNING, "No core");
+ return -1;
+ }
+ sn->reported = false;
+ }
+
memset(&hints, 0, sizeof(struct addrinfo));
hints.ai_family = PF_UNSPEC;
hints.ai_socktype = SOCK_DGRAM;
@@ -114,7 +142,39 @@ static int stor_add_node_addr(struct storage_node *sn,
return -1;
}
-void stor_add_node(uint32_t nid, const char *hostname, const char *portstr,
+static int stor_add_node_base(struct storage_node *sn, const char *base)
+{
+ if (sn->basepath == NULL || strcmp(sn->basepath, base) != 0) {
+ free(sn->basepath);
+ sn->basepath = strdup(base);
+ if (!sn->basepath) {
+ applog(LOG_WARNING, "No core");
+ return -1;
+ }
+ sn->reported = false;
+ }
+ return 0;
+}
+
+static int stor_add_node_this(struct storage_node *sn,
+ enum storage_type type, const char *base,
+ const char *hostname, const char *portstr)
+{
+ sn->type = type;
+ switch (type) {
+ case STT_POSIX:
+ sn->ops = &stor_ops_posix;
+ return stor_add_node_base(sn, base);
+ case STT_SWIFT:
+ return -1;
+ default:
+ sn->ops = &stor_ops_chunk;
+ return stor_add_node_addr(sn, hostname, portstr);
+ }
+}
+
+void stor_add_node(uint32_t nid, enum storage_type type, const char *base,
+ const char *hostname, const char *portstr,
struct geo *locp)
{
struct storage_node *sn;
@@ -122,7 +182,7 @@ void stor_add_node(uint32_t nid, const char *hostname, const char *portstr,
g_mutex_lock(tabled_srv.bigmutex);
sn = _stor_node_by_nid(nid);
if (sn) {
- stor_add_node_addr(sn, hostname, portstr);
+ stor_add_node_this(sn, type, base, hostname, portstr);
} else {
if ((sn = malloc(sizeof(struct storage_node))) == NULL) {
applog(LOG_WARNING, "No core (%ld)",
@@ -132,17 +192,10 @@ void stor_add_node(uint32_t nid, const char *hostname, const char *portstr,
}
memset(sn, 0, sizeof(struct storage_node));
sn->id = nid;
- sn->ops = &stor_ops_chunk;
-
- if ((sn->hostname = strdup(hostname)) == NULL) {
- applog(LOG_WARNING, "No core");
- free(sn);
- g_mutex_unlock(tabled_srv.bigmutex);
- return;
- }
- if (stor_add_node_addr(sn, hostname, portstr)) {
+ if (stor_add_node_this(sn, type, base, hostname, portstr)) {
free(sn->hostname);
+ free(sn->basepath);
free(sn);
g_mutex_unlock(tabled_srv.bigmutex);
return;
@@ -165,16 +218,37 @@ void stor_stats()
now = time(NULL);
list_for_each_entry(sn, &tabled_srv.all_stor, all_link) {
if (sn->last_up) {
- applog(LOG_INFO,
- "SN: nid %u %s ref %d name %s last %lu (+ %ld)",
- sn->id, sn->up? "up": "down",
- sn->ref, sn->hostname,
- (long) sn->last_up, (long) (now - sn->last_up));
+ switch (sn->type) {
+ case STT_POSIX:
+ applog(LOG_INFO, "SN: nid %u %s ref %d"
+ " path %s last %lu (+ %ld)",
+ sn->id, sn->up? "up": "down",
+ sn->ref, sn->basepath,
+ (long) sn->last_up,
+ (long) (now - sn->last_up));
+ break;
+ default:
+ applog(LOG_INFO, "SN: nid %u %s ref %d"
+ " name %s last %lu (+ %ld)",
+ sn->id, sn->up? "up": "down",
+ sn->ref, sn->hostname,
+ (long) sn->last_up,
+ (long) (now - sn->last_up));
+ }
} else {
- applog(LOG_INFO,
- "SN: nid %u %s ref %d name %s",
- sn->id, sn->up? "up": "down",
- sn->ref, sn->hostname);
+ switch (sn->type) {
+ case STT_POSIX:
+ applog(LOG_INFO,
+ "SN: nid %u %s ref %d path %s",
+ sn->id, sn->up? "up": "down",
+ sn->ref, sn->basepath);
+ break;
+ default:
+ applog(LOG_INFO,
+ "SN: nid %u %s ref %d name %s",
+ sn->id, sn->up? "up": "down",
+ sn->ref, sn->hostname);
+ }
}
}
g_mutex_unlock(tabled_srv.bigmutex);
@@ -193,18 +267,39 @@ bool stor_status(struct client *cli, GList *content)
now = time(NULL);
list_for_each_entry(sn, &tabled_srv.all_stor, all_link) {
if (sn->last_up) {
- rc = asprintf(&str,
- "SN: nid %u %s ref %d name %s"
- " last %lu (+ %ld)<br />\r\n",
- sn->id, sn->up? "up": tag_down,
- sn->ref, sn->hostname,
- (long) sn->last_up,
- (long) (now - sn->last_up));
+ switch (sn->type) {
+ case STT_POSIX:
+ rc = asprintf(&str,
+ "SN: nid %u %s ref %d path %s"
+ " last %lu (+ %ld)<br />\r\n",
+ sn->id, sn->up? "up": tag_down,
+ sn->ref, sn->basepath,
+ (long) sn->last_up,
+ (long) (now - sn->last_up));
+ break;
+ default:
+ rc = asprintf(&str,
+ "SN: nid %u %s ref %d name %s"
+ " last %lu (+ %ld)<br />\r\n",
+ sn->id, sn->up? "up": tag_down,
+ sn->ref, sn->hostname,
+ (long) sn->last_up,
+ (long) (now - sn->last_up));
+ }
} else {
- rc = asprintf(&str,
- "SN: nid %u %s ref %d name %s<br />\r\n",
- sn->id, sn->up? "up": tag_down,
- sn->ref, sn->hostname);
+ switch (sn->type) {
+ case STT_POSIX:
+ rc = asprintf(&str, "SN: nid %u %s ref %d"
+ "path %s<br />\r\n",
+ sn->id, sn->up? "up": tag_down,
+ sn->ref, sn->basepath);
+ break;
+ default:
+ rc = asprintf(&str, "SN: nid %u %s ref %d"
+ "name %s<br />\r\n",
+ sn->id, sn->up? "up": tag_down,
+ sn->ref, sn->hostname);
+ }
}
if (rc < 0)
break;
diff --git a/server/storparse.c b/server/storparse.c
index cbffe0f..feec74e 100644
--- a/server/storparse.c
+++ b/server/storparse.c
@@ -1,6 +1,6 @@
/*
- * Copyright 2009 Red Hat, Inc.
+ * Copyright 2009,2010 Red Hat, Inc.
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
@@ -28,6 +28,14 @@
#include <ctype.h>
#include "tabled.h"
+enum config_type {
+ CFG_TYPE_NONE = 0, /* Omitted <Type>, assuming Chunk */
+ CFG_TYPE_UNKNOWN, /* Incompatible, like "swift-ssl" or "nfs4" */
+ CFG_TYPE_CHUNK, /* Explicit Chunk */
+ CFG_TYPE_FS,
+ CFG_TYPE_SWIFT,
+};
+
struct config_context {
char *text;
@@ -49,6 +57,8 @@ struct config_context {
struct geo loc;
unsigned int nid;
+ enum config_type stor_type;
+ char *stor_base;
};
static void cfg_elm_start (GMarkupParseContext *context,
@@ -315,6 +325,37 @@ static void cfg_elm_end (GMarkupParseContext *context,
}
}
+ else if (!strcmp(element_name, "Base")) {
+ if (!cc->text) {
+ applog(LOG_WARNING, "%s: Base element empty",
+ cc->fname);
+ return;
+ }
+ free(cc->stor_base);
+ cc->stor_base = cc->text;
+ cc->text = NULL;
+ }
+
+ else if (!strcmp(element_name, "Type")) {
+ if (!cc->text) {
+ applog(LOG_WARNING, "%s: Type element empty",
+ cc->fname);
+ return;
+ }
+ if (!strcmp(cc->text, "chunk")) {
+ cc->stor_type = CFG_TYPE_CHUNK;
+ } else if (!strcmp(cc->text, "fs")) {
+ cc->stor_type = CFG_TYPE_FS;
+ } else if (!strcmp(cc->text, "swift")) {
+ /* "cf" type reserved for "old" Rackspace auth. */
+ cc->stor_type = CFG_TYPE_SWIFT;
+ } else {
+ cc->stor_type = CFG_TYPE_UNKNOWN;
+ }
+ free(cc->text);
+ cc->text = NULL;
+ }
+
else {
applog(LOG_WARNING, "%s: Unknown element \"%s\"",
cc->fname, element_name);
@@ -350,6 +391,32 @@ static void cfg_elm_text (GMarkupParseContext *context,
cc->text = g_strndup(text, text_len);
}
+static bool stor_verify_chunk(char *fname, struct config_context *cc)
+{
+ if (!cc->nid) {
+ applog(LOG_WARNING, "%s: No NID", fname);
+ return false;
+ }
+ if (!cc->stor_ok) {
+ applog(LOG_WARNING, "%s: No useable Socket clause", fname);
+ return false;
+ }
+ return true;
+}
+
+static bool stor_verify_fs(char *fname, struct config_context *cc)
+{
+ if (!cc->nid) {
+ applog(LOG_WARNING, "%s: No NID", fname);
+ return false;
+ }
+ if (!cc->stor_base) {
+ applog(LOG_WARNING, "%s: No base directory", fname);
+ return false;
+ }
+ return true;
+}
+
static const GMarkupParser cfg_parse_ops = {
.start_element = cfg_elm_start,
.end_element = cfg_elm_end,
@@ -378,15 +445,25 @@ void stor_parse(char *fname, const char *text, size_t len)
g_markup_parse_context_free(parser);
- if (!ctx.nid) {
- applog(LOG_WARNING, "%s: No NID\n", fname);
- goto out_free_all;
+ switch (ctx.stor_type) {
+ case CFG_TYPE_FS:
+ if (!stor_verify_fs(fname, &ctx))
+ goto out_free_all;
+ stor_add_node(ctx.nid, STT_POSIX, ctx.stor_base,
+ NULL, NULL, &ctx.loc);
+ break;
+ case CFG_TYPE_SWIFT:
+ case CFG_TYPE_UNKNOWN:
+ if (debugging)
+ applog(LOG_DEBUG, "%s: Unknown storage type", fname);
+ break;
+ case CFG_TYPE_CHUNK:
+ default:
+ if (!stor_verify_chunk(fname, &ctx))
+ goto out_free_all;
+ stor_add_node(ctx.nid, STT_CHUNK, NULL,
+ ctx.stor_ok_host, ctx.stor_ok_port, &ctx.loc);
}
- if (!ctx.stor_ok) {
- applog(LOG_WARNING, "%s: No useable Socket clause", fname);
- goto out_free_all;
- }
- stor_add_node(ctx.nid, ctx.stor_ok_host, ctx.stor_ok_port, &ctx.loc);
out_free_all:
free(ctx.text);
@@ -400,5 +477,7 @@ out_free_all:
free(ctx.loc.area);
free(ctx.loc.zone);
free(ctx.loc.rack);
+
+ free(ctx.stor_base);
return;
}
diff --git a/server/tabled.h b/server/tabled.h
index 27bc7c1..0bbe527 100644
--- a/server/tabled.h
+++ b/server/tabled.h
@@ -24,6 +24,7 @@
#include <time.h>
#include <netinet/in.h>
#include <openssl/md5.h>
+#include <curl/curl.h>
#include <glib.h>
#include <pcre.h>
#include <event.h>
@@ -90,6 +91,12 @@ struct geo {
char *rack;
};
+enum storage_type {
+ STT_CHUNK, /* Hail chunkserver */
+ STT_POSIX, /* UNIX filesystem interface, may be NFS */
+ STT_SWIFT /* OpenStack storage (evolved CloudFiles) */
+};
+
struct storage_node;
struct open_chunk;
@@ -115,15 +122,21 @@ struct st_node_ops {
struct storage_node {
struct list_head all_link;
uint32_t id;
+ bool reported;
bool up;
time_t last_up;
+ enum storage_type type;
struct st_node_ops *ops;
int ref; /* number of open_chunk or other */
+ /* chunk */
unsigned alen;
struct sockaddr_in6 addr;
char *hostname;
+
+ /* file */
+ char *basepath;
};
typedef bool (*cli_evt_func)(struct client *, unsigned int,
@@ -154,6 +167,13 @@ struct open_chunk {
int rfd;
bool r_armed;
struct event revt;
+
+ /* posix */
+ /*
+ * Don't merge fd, rfd, wfd prematurely. A future backend may need
+ * several fds and then what? It's an implementation detail.
+ */
+ int pfd;
};
/* internal client socket state */
@@ -297,6 +317,9 @@ struct server {
struct server_stats stats; /* global statistics */
};
+#define STOR_KEY_SLEN 16
+#define STOR_KEY_FMT "%016llx"
+
/*
* Low-level channel, for both sides.
*
@@ -429,6 +452,8 @@ extern void read_config(void);
/* storage.c */
extern struct storage_node *stor_node_get(struct storage_node *stn);
extern void stor_node_put(struct storage_node *stn);
+extern void stor_read_event(int fd, short events, void *userdata);
+extern void stor_write_event(int fd, short events, void *userdata);
static inline int stor_open(struct open_chunk *cep, struct storage_node *stn,
struct event_base *ev_base)
{
@@ -488,7 +513,8 @@ static inline int stor_node_check(struct storage_node *stn)
return stn->ops->node_check(stn);
}
extern struct storage_node *stor_node_by_nid(uint32_t nid);
-extern void stor_add_node(uint32_t nid,
+extern void stor_add_node(uint32_t nid, enum storage_type type,
+ const char *base,
const char *hostname, const char *portstr,
struct geo *locp);
extern void stor_stats(void);
@@ -500,6 +526,9 @@ extern void stor_parse(char *fname, const char *text, size_t len);
/* stor_chunk.c */
extern struct st_node_ops stor_ops_chunk;
+/* stor_fs.c */
+extern struct st_node_ops stor_ops_posix;
+
/* replica.c */
extern void rep_init(struct event_base *ev_base);
extern void rep_start(void);
diff --git a/test/Makefile.am b/test/Makefile.am
index cc4e6fe..60d7b66 100644
--- a/test/Makefile.am
+++ b/test/Makefile.am
@@ -6,11 +6,13 @@ EXTRA_DIST = \
test.h \
users.data \
chunkd-test.conf \
+ be_fs-test.conf \
tabled-test.conf \
prep-db \
start-daemon \
pid-exists \
daemon-running \
+ combo-redux \
stop-daemon \
clean-db
@@ -27,6 +29,7 @@ TESTS = \
hdr-content-type \
hdr-meta \
list-keys \
+ combo-redux \
stop-daemon \
clean-db
diff --git a/test/be_fs-test.conf b/test/be_fs-test.conf
new file mode 100644
index 0000000..09b868c
--- /dev/null
+++ b/test/be_fs-test.conf
@@ -0,0 +1,5 @@
+<Chunk>
+ <NID>256</NID>
+ <Type>fs</Type>
+ <Base>data/be_fs</Base>
+</Chunk>
diff --git a/test/combo-redux b/test/combo-redux
new file mode 100755
index 0000000..2743a55
--- /dev/null
+++ b/test/combo-redux
@@ -0,0 +1,74 @@
+#!/bin/sh
+#
+# We stop tabled because it makes it easier to know what went wrong.
+# We could just swap the back-end on the fly, but then replication
+# confuses matters.
+#
+
+killpid () {
+ pidfile=$1
+ kill $(cat $pidfile)
+
+ for n in 0 1 2 3 4 5 6 7 8 9
+ do
+ if [ ! -f $pidfile ]
+ then
+ return 0
+ fi
+
+ sleep 1
+ done
+
+ echo "PID file $pidfile not removed, after signal sent." >&2
+ rm -f $pidfile
+ return 1
+}
+
+#
+# Step 1. Kill tabled and chunkd.
+#
+rm -f tabled.acc
+if [ ! -f tabled.pid ]
+then
+ # Just a warning. Previous test somehow made the daemon to die.
+ echo "No tabled PID file found." >&2
+else
+ killpid tabled.pid || exit 1
+fi
+
+if [ ! -f chunkd.pid ]
+then
+ echo "No chunkd PID file found." >&2
+else
+ killpid chunkd.pid || exit 1
+fi
+
+#
+# Step 2. Swap chunk back-end for fs.
+# We remove the chunk's <InfoPath> just to be neat.
+#
+if [ \! -s cld.port ]; then
+ echo "cld.port is not available" >&2
+ exit 1
+fi
+echo + "cldcli -h localhost:"`cat cld.port`
+cldcli -h localhost:`cat cld.port` <<EOF
+rm /chunk-default/19690720
+cpin $top_srcdir/test/be_fs-test.conf /chunk-default/256
+EOF
+
+#
+# Step 3. Restart tabled.
+#
+../server/tabled -C $top_srcdir/test/tabled-test.conf -E
+./wait-for-listen || exit 1
+
+#
+# Step 4. Run some of the tests that framework has done before.
+#
+./basic-object || exit 1
+echo "PASS redux: basic-object"
+./large-object || exit 1
+echo "PASS redux: large-object"
+
+exit 0
diff --git a/test/prep-db b/test/prep-db
index ef90e1c..b98fc63 100755
--- a/test/prep-db
+++ b/test/prep-db
@@ -4,10 +4,14 @@ DATADIR=data
TDBDIR=$DATADIR/tdb
CLDDIR=$DATADIR/cld
CHUNKDIR=$DATADIR/chunk
+BEFSDIR=$DATADIR/be_fs
mkdir -p $TDBDIR
mkdir -p $CLDDIR
mkdir -p $CHUNKDIR
+mkdir -p $BEFSDIR
+
+echo 256 > $BEFSDIR/NID
cat $top_srcdir/test/users.data | ../server/tdbadm -u -C $top_srcdir/test/tabled-test.conf
diff --git a/test/start-daemon b/test/start-daemon
index 6985468..9eb45aa 100755
--- a/test/start-daemon
+++ b/test/start-daemon
@@ -12,6 +12,7 @@ do
fi
done
+rm -f cld.port
# May be different on Solaris... like /usr/libexec or such.
cld -d data/cld -P cld.pid -p auto --port-file=cld.port -E
diff --git a/test/stop-daemon b/test/stop-daemon
index de5db0c..2370ec2 100755
--- a/test/stop-daemon
+++ b/test/stop-daemon
@@ -25,22 +25,21 @@ ret=0
if [ ! -f tabled.pid ]
then
- # Just a warning. Previous test somehow made the daemon to die.
echo "No tabled PID file found." >&2
else
killpid tabled.pid || ret=1
fi
-if [ ! -f chunkd.pid ]
+# The combo-redux kills chunkd, kill quietly if anything slipped through.
+if [ -f chunkd.pid ]
then
- echo "No chunkd PID file found." >&2
-else
killpid chunkd.pid || ret=1
fi
if [ ! -f cld.pid ]
then
- echo "No cld PID file found." >&2
+ # Just a warning. Previous test somehow made the daemon to die.
+ echo "No cld.pid file found." >&2
else
killpid cld.pid || ret=1
fi
--
To unsubscribe from this list: send the line "unsubscribe hail-devel" in
the body of a message to majordomo@xxxxxxxxxxxxxxx
More majordomo info at http://vger.kernel.org/majordomo-info.html
