This patch helps us to build inside Fedora build system where port
conflicts from simultaneous builds are an issue. Most of the code is
taken from a similar patch in CLD, although there are differences.
In particular, we use the configuration file instead of command line.
The whole deal is left underdocumented.
Also, this patch changes the invocation of cld in start-daemon,
and therefore requires changes in the installed cld. Thus we have
a one-way flag day.
Signed-off-by: Pete Zaitcev <zaitcev@xxxxxxxxxx>
diff --git a/include/chunkc.h b/include/chunkc.h
index 49c52ae..8b7df15 100644
--- a/include/chunkc.h
+++ b/include/chunkc.h
@@ -63,4 +63,6 @@ extern bool stc_ping(struct st_client *stc);
extern struct st_keylist *stc_keys(struct st_client *stc);
+extern int stc_readport(const char *fname);
+
#endif /* __STC_H__ */
diff --git a/lib/chunkdc.c b/lib/chunkdc.c
index e8c6391..121b589 100644
--- a/lib/chunkdc.c
+++ b/lib/chunkdc.c
@@ -6,6 +6,7 @@
#include <sys/ioctl.h>
#include <stdlib.h>
#include <unistd.h>
+#include <fcntl.h>
#include <string.h>
#include <netinet/in.h>
#include <netinet/tcp.h>
@@ -864,3 +865,32 @@ void stc_init(void)
srand(time(NULL) ^ getpid()); // for __cld_rand64 et.al.
}
+/*
+ * Read a port number from a port file, return the value or negative error.
+ * A 100% copy of cld_readport for now, but permits us not to link libcldc.
+ */
+int stc_readport(const char *fname)
+{
+ enum { LEN = 11 };
+ char buf[LEN+1];
+ long port;
+ int fd;
+ int rc;
+
+ if ((fd = open(fname, O_RDONLY)) == -1)
+ return -errno;
+ rc = read(fd, buf, LEN);
+ close(fd);
+ if (rc < 0)
+ return -errno;
+ if (rc == 0)
+ return -EPIPE;
+ buf[rc] = 0;
+
+ port = strtol(buf, NULL, 10);
+ if (port <= 0 || port >= 65636)
+ return -EDOM;
+
+ return (int)port;
+}
+
diff --git a/server/chunkd.h b/server/chunkd.h
index 754c250..b7ad09d 100644
--- a/server/chunkd.h
+++ b/server/chunkd.h
@@ -115,6 +115,7 @@ enum st_cld {
struct listen_cfg {
char *node;
char *port;
+ char *port_file;
bool encrypt;
};
diff --git a/server/config.c b/server/config.c
index 6218dd2..fa909d5 100644
--- a/server/config.c
+++ b/server/config.c
@@ -17,6 +17,7 @@
#include <stdio.h>
#include <openssl/hmac.h>
#include <glib.h>
+#include <cldc.h>
#include "chunkd.h"
struct config_context {
@@ -33,6 +34,7 @@ struct config_context {
bool in_cld;
unsigned short cld_port;
char *cld_host;
+ char *cld_port_file;
struct listen_cfg tmp_listen;
};
@@ -129,25 +131,32 @@ static void cfg_elm_end_listen(struct config_context *cc)
return;
}
- if (!cc->tmp_listen.port) {
- free(cc->tmp_listen.node);
- memset(&cc->tmp_listen, 0, sizeof(struct listen_cfg));
- applog(LOG_WARNING, "cfgfile: TCP port not specified in Listen");
- return;
+ if (cc->tmp_listen.port && cc->tmp_listen.port_file) {
+ applog(LOG_ERR, "cfgfile: Listen with both Port and PortFile");
+ goto err;
+ }
+
+ if (!cc->tmp_listen.port && !cc->tmp_listen.port_file) {
+ applog(LOG_ERR, "cfgfile: Listen with no Port or PortFile");
+ goto err;
}
cfg = malloc(sizeof(*cfg));
if (!cfg) {
- free(cc->tmp_listen.node);
- free(cc->tmp_listen.port);
- memset(&cc->tmp_listen, 0, sizeof(struct listen_cfg));
applog(LOG_ERR, "OOM");
- return;
+ goto err;
}
memcpy(cfg, &cc->tmp_listen, sizeof(*cfg));
chunkd_srv.listeners = g_list_append(chunkd_srv.listeners, cfg);
memset(&cc->tmp_listen, 0, sizeof(struct listen_cfg));
+ return;
+
+ err:
+ free(cc->tmp_listen.node);
+ free(cc->tmp_listen.port);
+ free(cc->tmp_listen.port_file);
+ memset(&cc->tmp_listen, 0, sizeof(struct listen_cfg));
}
static void cfg_elm_end_geo(struct config_context *cc)
@@ -195,11 +204,31 @@ static void cfg_elm_end_cld(struct config_context *cc)
applog(LOG_WARNING, "No host for CLD element");
goto end;
}
- if (!cc->cld_port) {
- applog(LOG_WARNING, "No port for CLD element");
+
+ if (!cc->cld_port && !cc->cld_port_file) {
+ applog(LOG_WARNING, "No Port nor PortFile for CLD element");
goto end;
}
+ /*
+ * Waiting here is disadvantageous, because it defeats testing
+ * of bootstrap robustness for Chunk as a client of CLD.
+ * But it's the most direct way to give us variable ports.
+ * Also, no mysterious sleep commands in start-daemon script.
+ */
+ if (cc->cld_port_file) {
+ int port;
+ if ((port = cld_readport(cc->cld_port_file)) <= 0) {
+ applog(LOG_INFO, "Waiting for CLD PortFile %s",
+ cc->cld_port_file);
+ sleep(2);
+ while ((port = cld_readport(cc->cld_port_file)) <= 0)
+ sleep(3);
+ applog(LOG_INFO, "Using CLD port %u", port);
+ }
+ cc->cld_port = port;
+ }
+
cldu_add_host(cc->cld_host, cc->cld_port);
end:
@@ -375,6 +404,26 @@ static void cfg_elm_end (GMarkupParseContext *context,
cc->in_geo = false;
}
+ else if (!strcmp(element_name, "PortFile")) {
+ if (!cc->text) {
+ applog(LOG_WARNING, "PortFile element empty");
+ return;
+ }
+
+ if (cc->in_listen) {
+ free(cc->tmp_listen.port_file);
+ cc->tmp_listen.port_file = cc->text;
+ } else if (cc->in_cld) {
+ free(cc->cld_port_file);
+ cc->cld_port_file = cc->text;
+ } else {
+ applog(LOG_WARNING,
+ "PortFile element not in Listen or CLD");
+ free(cc->text);
+ }
+ cc->text = NULL;
+ }
+
else {
applog(LOG_WARNING, "Unknown element \"%s\"", element_name);
}
diff --git a/server/server.c b/server/server.c
index 44d97c9..9b63497 100644
--- a/server/server.c
+++ b/server/server.c
@@ -43,8 +43,6 @@ enum {
};
static struct argp_option options[] = {
- { "config", 'f', "FILE", 0,
- "Read master configuration from FILE (deprecated)" },
{ "config", 'C', "FILE", 0,
"Read master configuration from FILE" },
{ "debug", 'D', NULL, 0,
@@ -147,7 +145,6 @@ void applog(int prio, const char *fmt, ...)
static error_t parse_opt (int key, char *arg, struct argp_state *state)
{
switch(key) {
- case 'f':
case 'C':
chunkd_srv.config = arg;
break;
@@ -1021,7 +1018,149 @@ err_out:
cli_free(cli);
}
-static int net_open(const struct listen_cfg *cfg)
+static int net_open_socket(const struct listen_cfg *cfg,
+ int addr_fam, int sock_type, int sock_prot,
+ int addr_len, void *addr_ptr)
+{
+ struct server_socket *sock;
+ int fd, on;
+ int rc;
+
+ fd = socket(addr_fam, sock_type, sock_prot);
+ if (fd < 0) {
+ rc = errno;
+ syslogerr("tcp socket");
+ return -rc;
+ }
+
+ on = 1;
+ if (setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &on, sizeof(on)) < 0) {
+ rc = errno;
+ syslogerr("setsockopt(SO_REUSEADDR)");
+ close(fd);
+ return -rc;
+ }
+
+ if (bind(fd, addr_ptr, addr_len) < 0) {
+ rc = errno;
+ syslogerr("tcp bind");
+ close(fd);
+ return -rc;
+ }
+
+ if (listen(fd, 100) < 0) {
+ rc = errno;
+ syslogerr("tcp listen");
+ close(fd);
+ return -rc;
+ }
+
+ rc = fsetflags("tcp server", fd, O_NONBLOCK);
+ if (rc) {
+ close(fd);
+ return rc;
+ }
+
+ sock = calloc(1, sizeof(*sock));
+ if (!sock) {
+ close(fd);
+ return -ENOMEM;
+ }
+
+ sock->fd = fd;
+ sock->cfg = cfg;
+
+ event_set(&sock->ev, fd, EV_READ | EV_PERSIST, tcp_srv_event, sock);
+
+ if (event_add(&sock->ev, NULL) < 0) {
+ applog(LOG_WARNING, "tcp socket event_add");
+ free(sock);
+ close(fd);
+ return -EIO;
+ }
+
+ chunkd_srv.sockets = g_list_append(chunkd_srv.sockets, sock);
+ return fd;
+}
+
+/*
+ * This, annoyingly, has to have a side effect: it fills out cfg->port,
+ * so that we can later export it into CLD.
+ */
+static int net_open_any(struct listen_cfg *cfg)
+{
+ char *portfile = cfg->port_file;
+ struct sockaddr_in addr4;
+ struct sockaddr_in6 addr6;
+ FILE *portf;
+ int fd4, fd6;
+ socklen_t addr_len;
+ unsigned short port;
+ int rc;
+
+ port = 0;
+
+ /* Thanks to Linux, IPv6 must be bound first. */
+ memset(&addr6, 0, sizeof(addr6));
+ addr6.sin6_family = AF_INET6;
+ memcpy(&addr6.sin6_addr, &in6addr_any, sizeof(struct in6_addr));
+ fd6 = net_open_socket(cfg,
+ AF_INET6, SOCK_STREAM, 0, sizeof(addr6), &addr6);
+
+ if (fd6 >= 0) {
+ addr_len = sizeof(addr6);
+ if (getsockname(fd6, &addr6, &addr_len) != 0) {
+ rc = errno;
+ applog(LOG_ERR, "getsockname failed: %s", strerror(rc));
+ return -rc;
+ }
+ port = ntohs(addr6.sin6_port);
+ }
+
+ memset(&addr4, 0, sizeof(addr4));
+ addr4.sin_family = AF_INET;
+ addr4.sin_addr.s_addr = htonl(INADDR_ANY);
+ /* If IPv6 worked, we must use the same port number for IPv4 */
+ if (port)
+ addr4.sin_port = port;
+ fd4 = net_open_socket(cfg,
+ AF_INET, SOCK_STREAM, 0, sizeof(addr4), &addr4);
+
+ if (!port) {
+ if (fd4 < 0)
+ return fd4;
+
+ addr_len = sizeof(addr4);
+ if (getsockname(fd4, &addr4, &addr_len) != 0) {
+ rc = errno;
+ applog(LOG_ERR, "getsockname failed: %s", strerror(rc));
+ return -rc;
+ }
+ port = ntohs(addr4.sin_port);
+ }
+
+ applog(LOG_INFO, "Listening on port %u file %s", port, portfile);
+
+ rc = asprintf(&cfg->port, "%u", port);
+ if (rc < 0) {
+ applog(LOG_ERR, "OOM");
+ return -ENOMEM;
+ }
+
+ portf = fopen(portfile, "w");
+ if (portf == NULL) {
+ rc = errno;
+ applog(LOG_INFO, "Cannot create port file %s: %s",
+ portfile, strerror(rc));
+ return -rc;
+ }
+ fprintf(portf, "%u\n", port);
+ fclose(portf);
+
+ return 0;
+}
+
+static int net_open_known(const struct listen_cfg *cfg)
{
int ipv6_found = 0;
int rc;
@@ -1058,71 +1197,16 @@ static int net_open(const struct listen_cfg *cfg)
#endif
for (res = res0; res; res = res->ai_next) {
- struct server_socket *sock;
- int fd, on;
char listen_host[65], listen_serv[65];
if (ipv6_found && res->ai_family == PF_INET)
continue;
- fd = socket(res->ai_family, res->ai_socktype, res->ai_protocol);
- if (fd < 0) {
- syslogerr("tcp socket");
- return -errno;
- }
-
- on = 1;
- if (setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &on,
- sizeof(on)) < 0) {
- syslogerr("setsockopt(SO_REUSEADDR)");
- rc = -errno;
- goto err_out;
- }
-
- if (bind(fd, res->ai_addr, res->ai_addrlen) < 0) {
- /* sigh... */
- if (errno == EADDRINUSE && res->ai_family == PF_INET) {
- if (debugging)
- applog(LOG_INFO, "already bound to socket, ignoring");
- close(fd);
- continue;
- }
-
- syslogerr("tcp bind");
- rc = -errno;
- goto err_out;
- }
-
- if (listen(fd, 100) < 0) {
- syslogerr("tcp listen");
- rc = -errno;
- goto err_out;
- }
-
- rc = fsetflags("tcp server", fd, O_NONBLOCK);
- if (rc)
- goto err_out;
-
- sock = calloc(1, sizeof(*sock));
- if (!sock) {
- rc = -ENOMEM;
- goto err_out;
- }
-
- sock->fd = fd;
- sock->cfg = cfg;
-
- event_set(&sock->ev, fd, EV_READ | EV_PERSIST,
- tcp_srv_event, sock);
-
- if (event_add(&sock->ev, NULL) < 0) {
- applog(LOG_WARNING, "tcp socket event_add");
- rc = -EIO;
+ rc = net_open_socket(cfg, res->ai_family, res->ai_socktype,
+ res->ai_protocol,
+ res->ai_addrlen, res->ai_addr);
+ if (rc < 0)
goto err_out;
- }
-
- chunkd_srv.sockets =
- g_list_append(chunkd_srv.sockets, sock);
getnameinfo(res->ai_addr, res->ai_addrlen,
listen_host, sizeof(listen_host),
@@ -1141,6 +1225,14 @@ err_out:
return rc;
}
+static int net_open(struct listen_cfg *cfg)
+{
+ if (cfg->port_file)
+ return net_open_any(cfg);
+ else
+ return net_open_known(cfg);
+}
+
int main (int argc, char *argv[])
{
error_t aprc;
@@ -1230,13 +1322,10 @@ int main (int argc, char *argv[])
}
/* set up server networking */
- tmpl = chunkd_srv.listeners;
- while (tmpl) {
+ for (tmpl = chunkd_srv.listeners; tmpl; tmpl = tmpl->next) {
rc = net_open(tmpl->data);
if (rc)
goto err_out_listen;
-
- tmpl = tmpl->next;
}
applog(LOG_INFO, "initialized");
diff --git a/server/util.c b/server/util.c
index dd31ddd..76dea5b 100644
--- a/server/util.c
+++ b/server/util.c
@@ -133,8 +133,9 @@ int fsetflags(const char *prefix, int fd, int or_flags)
/* get current flags */
old_flags = fcntl(fd, F_GETFL);
if (old_flags < 0) {
- applog(LOG_ERR, "%s F_GETFL: %s", prefix, strerror(errno));
- return -errno;
+ rc = errno;
+ applog(LOG_ERR, "%s F_GETFL: %s", prefix, strerror(rc));
+ return -rc;
}
/* add or_flags */
@@ -144,8 +145,9 @@ int fsetflags(const char *prefix, int fd, int or_flags)
/* set new flags */
if (flags != old_flags)
if (fcntl(fd, F_SETFL, flags) < 0) {
- applog(LOG_ERR, "%s F_SETFL: %s", prefix, strerror(errno));
- rc = -errno;
+ rc = errno;
+ applog(LOG_ERR, "%s F_SETFL: %s", prefix, strerror(rc));
+ rc = -rc;
}
return rc;
diff --git a/test/auth.c b/test/auth.c
index 1a8dab1..107b988 100644
--- a/test/auth.c
+++ b/test/auth.c
@@ -14,6 +14,7 @@ static void test(bool encrypt)
struct st_object *obj;
struct st_keylist *klist;
struct st_client *stc1, *stc2;
+ int port;
bool rcb;
char val1[] = "my first value";
char val2[] = "my second value";
@@ -22,12 +23,13 @@ static void test(bool encrypt)
size_t len = 0;
void *mem;
- stc1 = stc_new(TEST_HOST, encrypt ? TEST_SSL_PORT : TEST_PORT,
- TEST_USER, TEST_USER_KEY, encrypt);
+ port = stc_readport(encrypt ? TEST_PORTFILE_SSL : TEST_PORTFILE);
+ OK(port > 0);
+
+ stc1 = stc_new(TEST_HOST, port, TEST_USER, TEST_USER_KEY, encrypt);
OK(stc1);
- stc2 = stc_new(TEST_HOST, encrypt ? TEST_SSL_PORT : TEST_PORT,
- TEST_USER2, TEST_USER2_KEY, encrypt);
+ stc2 = stc_new(TEST_HOST, port, TEST_USER2, TEST_USER2_KEY, encrypt);
OK(stc2);
/* store object 1 */
diff --git a/test/basic-object.c b/test/basic-object.c
index ee4e7c5..7abb1d7 100644
--- a/test/basic-object.c
+++ b/test/basic-object.c
@@ -14,14 +14,17 @@ static void test(bool encrypt)
struct st_object *obj;
struct st_keylist *klist;
struct st_client *stc;
+ int port;
bool rcb;
char val[] = "my first value";
char key[64] = "deadbeef";
size_t len = 0;
void *mem;
- stc = stc_new(TEST_HOST, encrypt ? TEST_SSL_PORT : TEST_PORT,
- TEST_USER, TEST_USER_KEY, encrypt);
+ port = stc_readport(encrypt ? TEST_PORTFILE_SSL : TEST_PORTFILE);
+ OK(port > 0);
+
+ stc = stc_new(TEST_HOST, port, TEST_USER, TEST_USER_KEY, encrypt);
OK(stc);
/* store object */
diff --git a/test/it-works.c b/test/it-works.c
index cb53ed1..ef923d8 100644
--- a/test/it-works.c
+++ b/test/it-works.c
@@ -12,10 +12,13 @@
static void test(bool ssl)
{
struct st_client *stc;
+ int port;
bool rcb;
- stc = stc_new(TEST_HOST, ssl ? TEST_SSL_PORT : TEST_PORT,
- TEST_USER, TEST_USER_KEY, ssl);
+ port = stc_readport(ssl ? TEST_PORTFILE_SSL : TEST_PORTFILE);
+ OK(port > 0);
+
+ stc = stc_new(TEST_HOST, port, TEST_USER, TEST_USER_KEY, ssl);
OK(stc);
rcb = stc_ping(stc);
diff --git a/test/large-object.c b/test/large-object.c
index ed8d73a..d97ce21 100644
--- a/test/large-object.c
+++ b/test/large-object.c
@@ -75,6 +75,7 @@ static void test(bool encrypt)
struct st_object *obj;
struct st_keylist *klist;
struct st_client *stc;
+ int port;
bool rcb;
char key[64] = "deadbeef";
uint64_t len = 0;
@@ -86,8 +87,10 @@ static void test(bool encrypt)
memset(data, 0xdeadbeef, sizeof(data));
- stc = stc_new(TEST_HOST, encrypt ? TEST_SSL_PORT : TEST_PORT,
- TEST_USER, TEST_USER_KEY, encrypt);
+ port = stc_readport(encrypt ? TEST_PORTFILE_SSL : TEST_PORTFILE);
+ OK(port > 0);
+
+ stc = stc_new(TEST_HOST, port, TEST_USER, TEST_USER_KEY, encrypt);
OK(stc);
sync();
diff --git a/test/lotsa-objects.c b/test/lotsa-objects.c
index 9bcb01c..a26ecc0 100644
--- a/test/lotsa-objects.c
+++ b/test/lotsa-objects.c
@@ -18,6 +18,7 @@ static void test(int n_objects, bool encrypt)
{
struct st_keylist *klist;
struct st_client *stc;
+ int port;
bool rcb;
char val[] = "my first value";
char key[64] = "";
@@ -26,8 +27,10 @@ static void test(int n_objects, bool encrypt)
char *k;
struct timeval ta, tb;
- stc = stc_new(TEST_HOST, encrypt ? TEST_SSL_PORT : TEST_PORT,
- TEST_USER, TEST_USER_KEY, encrypt);
+ port = stc_readport(encrypt ? TEST_PORTFILE_SSL : TEST_PORTFILE);
+ OK(port > 0);
+
+ stc = stc_new(TEST_HOST, port, TEST_USER, TEST_USER_KEY, encrypt);
OK(stc);
fprintf(stderr, " lotsa-objects syncing...\n");
diff --git a/test/nop.c b/test/nop.c
index 7ef11a1..c683dc4 100644
--- a/test/nop.c
+++ b/test/nop.c
@@ -17,12 +17,15 @@ enum {
static void test(int n_nops, bool encrypt)
{
struct st_client *stc;
+ int port;
bool rcb;
int i;
struct timeval ta, tb;
- stc = stc_new(TEST_HOST, encrypt ? TEST_SSL_PORT : TEST_PORT,
- TEST_USER, TEST_USER_KEY, encrypt);
+ port = stc_readport(encrypt ? TEST_PORTFILE_SSL : TEST_PORTFILE);
+ OK(port > 0);
+
+ stc = stc_new(TEST_HOST, port, TEST_USER, TEST_USER_KEY, encrypt);
OK(stc);
gettimeofday(&ta, NULL);
diff --git a/test/server-test.cfg b/test/server-test.cfg
index 02c39d4..034f677 100644
--- a/test/server-test.cfg
+++ b/test/server-test.cfg
@@ -5,11 +5,11 @@
</SSL>
<Listen>
- <Port>18082</Port>
+ <PortFile>chunkd.port</PortFile>
</Listen>
<Listen>
- <Port>28082</Port>
+ <PortFile>chunkd-ssl.port</PortFile>
<Encrypt>true</Encrypt>
</Listen>
@@ -20,7 +20,7 @@
<NID>1</NID>
<CLD>
- <Port>18081</Port>
+ <PortFile>cld.port</PortFile>
<Host>localhost</Host>
</CLD>
diff --git a/test/start-daemon.real.in b/test/start-daemon.real.in
index 052360a..cd23a2f 100644
--- a/test/start-daemon.real.in
+++ b/test/start-daemon.real.in
@@ -11,7 +11,7 @@ then
exit 1
fi
-@CLDC_CMD@ -d data/cld -P cld.pid -p 18081 -E
+@CLDC_CMD@ -d data/cld -P cld.pid --port-file=cld.port -E
../server/chunkd -C $top_srcdir/test/server-test.cfg -E $*
diff --git a/test/stop-daemon b/test/stop-daemon
index df14776..87eaf8b 100755
--- a/test/stop-daemon
+++ b/test/stop-daemon
@@ -35,4 +35,6 @@ else
killpid cld.pid || ret=1
fi
+rm -f chunkd.port chunkd-ssl.port cld.port
+
exit "$ret"
diff --git a/test/test.h b/test/test.h
index d1bf601..dd99843 100644
--- a/test/test.h
+++ b/test/test.h
@@ -14,8 +14,9 @@
#define TEST_USER2 "testuser2"
#define TEST_USER2_KEY "testuser2"
-#define TEST_PORT 18082
-#define TEST_SSL_PORT 28082
+#define TEST_PORTFILE_CLD "cld.port"
+#define TEST_PORTFILE "chunkd.port"
+#define TEST_PORTFILE_SSL "chunkd-ssl.port"
#define OK(expr) \
do { \
--
To unsubscribe from this list: send the line "unsubscribe hail-devel" in
the body of a message to majordomo@xxxxxxxxxxxxxxx
More majordomo info at http://vger.kernel.org/majordomo-info.html
