On Fri, Apr 12 2013, Jens Axboe wrote:
> But sounds like it would be a good idea to turn zlib into a soft
> requirement.
The below does that, I have committed it. Please test.
diff --git a/client.c b/client.c
index eb695ff..13e84f6 100644
--- a/client.c
+++ b/client.c
@@ -14,7 +14,9 @@
#include <arpa/inet.h>
#include <netdb.h>
#include <signal.h>
+#ifdef CONFIG_ZLIB
#include <zlib.h>
+#endif
#include "fio.h"
#include "client.h"
@@ -296,9 +298,18 @@ int fio_client_add(struct client_ops *ops, const char *hostname, void **cookie)
static void probe_client(struct fio_client *client)
{
+ struct cmd_client_probe_pdu pdu;
+ uint64_t tag;
+
dprint(FD_NET, "client: send probe\n");
- fio_net_send_simple_cmd(client->fd, FIO_NET_CMD_PROBE, 0, &client->cmd_list);
+#ifdef CONFIG_ZLIB
+ pdu.flags = __le64_to_cpu(FIO_PROBE_FLAG_ZLIB);
+#else
+ pdu.flags = 0;
+#endif
+
+ fio_net_send_cmd(client->fd, FIO_NET_CMD_PROBE, &pdu, sizeof(pdu), &tag, &client->cmd_list);
}
static int fio_client_connect_ip(struct fio_client *client)
@@ -998,7 +1009,7 @@ static void handle_eta(struct fio_client *client, struct fio_net_cmd *cmd)
static void handle_probe(struct fio_client *client, struct fio_net_cmd *cmd)
{
- struct cmd_probe_pdu *probe = (struct cmd_probe_pdu *) cmd->payload;
+ struct cmd_probe_reply_pdu *probe = (struct cmd_probe_reply_pdu *) cmd->payload;
const char *os, *arch;
char bit[16];
@@ -1011,10 +1022,11 @@ static void handle_probe(struct fio_client *client, struct fio_net_cmd *cmd)
os = "unknown";
sprintf(bit, "%d-bit", probe->bpp * 8);
+ probe->flags = le64_to_cpu(probe->flags);
- log_info("hostname=%s, be=%u, %s, os=%s, arch=%s, fio=%s\n",
+ log_info("hostname=%s, be=%u, %s, os=%s, arch=%s, fio=%s, flags=%lx\n",
probe->hostname, probe->bigendian, bit, os, arch,
- probe->fio_version);
+ probe->fio_version, (unsigned long) probe->flags);
if (!client->name)
client->name = strdup((char *) probe->hostname);
@@ -1057,19 +1069,15 @@ static void convert_text(struct fio_net_cmd *cmd)
pdu->log_usec = le64_to_cpu(pdu->log_usec);
}
-/*
- * This has been compressed on the server side, since it can be big.
- * Uncompress here.
- */
-static struct cmd_iolog_pdu *convert_iolog(struct fio_net_cmd *cmd)
+static struct cmd_iolog_pdu *convert_iolog_gz(struct fio_net_cmd *cmd,
+ struct cmd_iolog_pdu *pdu)
{
- struct cmd_iolog_pdu *pdu = (struct cmd_iolog_pdu *) cmd->payload;
+#ifdef CONFIG_ZLIB
struct cmd_iolog_pdu *ret;
- uint32_t nr_samples;
- unsigned long total;
z_stream stream;
+ uint32_t nr_samples;
+ size_t total;
void *p;
- int i;
stream.zalloc = Z_NULL;
stream.zfree = Z_NULL;
@@ -1087,10 +1095,9 @@ static struct cmd_iolog_pdu *convert_iolog(struct fio_net_cmd *cmd)
total = nr_samples * sizeof(struct io_sample);
ret = malloc(total + sizeof(*pdu));
- ret->thread_number = le32_to_cpu(pdu->thread_number);
ret->nr_samples = nr_samples;
- ret->log_type = le32_to_cpu(pdu->log_type);
- strcpy((char *) ret->name, (char *) pdu->name);
+
+ memcpy(ret, pdu, sizeof(*pdu));
p = (void *) ret + sizeof(*pdu);
@@ -1112,7 +1119,7 @@ static struct cmd_iolog_pdu *convert_iolog(struct fio_net_cmd *cmd)
log_err("fio: inflate error %d\n", err);
free(ret);
ret = NULL;
- goto out;
+ goto err;
}
this_len = this_chunk - stream.avail_out;
@@ -1120,6 +1127,46 @@ static struct cmd_iolog_pdu *convert_iolog(struct fio_net_cmd *cmd)
total -= this_len;
}
+err:
+ inflateEnd(&stream);
+ return ret;
+#else
+ return NULL;
+#endif
+}
+
+/*
+ * This has been compressed on the server side, since it can be big.
+ * Uncompress here.
+ */
+static struct cmd_iolog_pdu *convert_iolog(struct fio_net_cmd *cmd)
+{
+ struct cmd_iolog_pdu *pdu = (struct cmd_iolog_pdu *) cmd->payload;
+ struct cmd_iolog_pdu *ret;
+ int i;
+
+ /*
+ * Convert if compressed and we support it. If it's not
+ * compressed, we need not do anything.
+ */
+ if (le32_to_cpu(pdu->compressed)) {
+#ifndef CONFIG_ZLIB
+ log_err("fio: server sent compressed data by mistake\n");
+ return NULL;
+#endif
+ ret = convert_iolog_gz(cmd, pdu);
+ if (!ret) {
+ log_err("fio: failed decompressing log\n");
+ return NULL;
+ }
+ } else
+ ret = pdu;
+
+ ret->thread_number = le32_to_cpu(ret->thread_number);
+ ret->nr_samples = le32_to_cpu(ret->nr_samples);
+ ret->log_type = le32_to_cpu(ret->log_type);
+ ret->compressed = le32_to_cpu(ret->compressed);
+
for (i = 0; i < ret->nr_samples; i++) {
struct io_sample *s = &ret->samples[i];
@@ -1129,8 +1176,6 @@ static struct cmd_iolog_pdu *convert_iolog(struct fio_net_cmd *cmd)
s->bs = le32_to_cpu(s->bs);
}
-out:
- inflateEnd(&stream);
return ret;
}
diff --git a/configure b/configure
index 87d659c..7be8789 100755
--- a/configure
+++ b/configure
@@ -415,9 +415,6 @@ EOF
if compile_prog "" "-lz" "zlib" ; then
zlib=yes
LIBS="-lz $LIBS"
-else
- feature_not_found "zlib" "zlib development files"
- zlib=no
fi
echo "zlib $zlib"
@@ -1083,6 +1080,9 @@ if test "$bigendian" = "yes" ; then
else
output_sym "CONFIG_LITTLE_ENDIAN"
fi
+if test "$zlib" = "yes" ; then
+ output_sym "CONFIG_ZLIB"
+fi
if test "$libaio" = "yes" ; then
output_sym "CONFIG_LIBAIO"
fi
diff --git a/server.c b/server.c
index 8a6275d..020d1d7 100644
--- a/server.c
+++ b/server.c
@@ -16,7 +16,9 @@
#include <netdb.h>
#include <syslog.h>
#include <signal.h>
+#ifdef CONFIG_ZLIB
#include <zlib.h>
+#endif
#include "fio.h"
#include "server.h"
@@ -33,6 +35,12 @@ static char *bind_sock;
static struct sockaddr_in saddr_in;
static struct sockaddr_in6 saddr_in6;
static int use_ipv6;
+#ifdef CONFIG_ZLIB
+static unsigned int has_zlib = 1;
+#else
+static unsigned int has_zlib = 0;
+#endif
+static unsigned int use_zlib;
struct fio_fork_item {
struct flist_head list;
@@ -616,7 +624,8 @@ static int handle_jobline_cmd(struct fio_net_cmd *cmd)
static int handle_probe_cmd(struct fio_net_cmd *cmd)
{
- struct cmd_probe_pdu probe;
+ struct cmd_client_probe_pdu *pdu = (struct cmd_client_probe_pdu *) cmd->payload;
+ struct cmd_probe_reply_pdu probe;
uint64_t tag = cmd->tag;
dprint(FD_NET, "server: sending probe reply\n");
@@ -632,7 +641,17 @@ static int handle_probe_cmd(struct fio_net_cmd *cmd)
probe.arch = FIO_ARCH;
probe.bpp = sizeof(void *);
probe.cpus = __cpu_to_le32(cpus_online());
- probe.flags = 0;
+
+ /*
+ * If the client supports compression and we do too, then enable it
+ */
+ if (has_zlib && le64_to_cpu(pdu->flags) & FIO_PROBE_FLAG_ZLIB) {
+ probe.flags = __cpu_to_le64(FIO_PROBE_FLAG_ZLIB);
+ use_zlib = 1;
+ } else {
+ probe.flags = 0;
+ use_zlib = 0;
+ }
return fio_net_send_cmd(server_fd, FIO_NET_CMD_PROBE, &probe, sizeof(probe), &tag, NULL);
}
@@ -1117,26 +1136,12 @@ static int fio_send_cmd_ext_pdu(int sk, uint16_t opcode, const void *buf,
return fio_sendv_data(sk, iov, 2);
}
-int fio_send_iolog(struct thread_data *td, struct io_log *log, const char *name)
+static int fio_send_iolog_gz(struct cmd_iolog_pdu *pdu, struct io_log *log)
{
- struct cmd_iolog_pdu pdu;
+ int ret = 0;
+#ifdef CONFIG_ZLIB
z_stream stream;
void *out_pdu;
- int i, ret = 0;
-
- pdu.thread_number = cpu_to_le32(td->thread_number);
- pdu.nr_samples = __cpu_to_le32(log->nr_samples);
- pdu.log_type = cpu_to_le32(log->log_type);
- strcpy((char *) pdu.name, name);
-
- for (i = 0; i < log->nr_samples; i++) {
- struct io_sample *s = &log->log[i];
-
- s->time = cpu_to_le64(s->time);
- s->val = cpu_to_le64(s->val);
- s->ddir = cpu_to_le32(s->ddir);
- s->bs = cpu_to_le32(s->bs);
- }
/*
* Dirty - since the log is potentially huge, compress it into
@@ -1154,14 +1159,6 @@ int fio_send_iolog(struct thread_data *td, struct io_log *log, const char *name)
goto err;
}
- /*
- * Send header first, it's not compressed.
- */
- ret = fio_send_cmd_ext_pdu(server_fd, FIO_NET_CMD_IOLOG, &pdu,
- sizeof(pdu), 0, FIO_NET_CMD_F_MORE);
- if (ret)
- goto err_zlib;
-
stream.next_in = (void *) log->log;
stream.avail_in = log->nr_samples * sizeof(struct io_sample);
@@ -1191,9 +1188,48 @@ err_zlib:
deflateEnd(&stream);
err:
free(out_pdu);
+#endif
return ret;
}
+int fio_send_iolog(struct thread_data *td, struct io_log *log, const char *name)
+{
+ struct cmd_iolog_pdu pdu;
+ int i, ret = 0;
+
+ pdu.thread_number = cpu_to_le32(td->thread_number);
+ pdu.nr_samples = __cpu_to_le32(log->nr_samples);
+ pdu.log_type = cpu_to_le32(log->log_type);
+ pdu.compressed = cpu_to_le32(use_zlib);
+ strcpy((char *) pdu.name, name);
+
+ for (i = 0; i < log->nr_samples; i++) {
+ struct io_sample *s = &log->log[i];
+
+ s->time = cpu_to_le64(s->time);
+ s->val = cpu_to_le64(s->val);
+ s->ddir = cpu_to_le32(s->ddir);
+ s->bs = cpu_to_le32(s->bs);
+ }
+
+ /*
+ * Send header first, it's not compressed.
+ */
+ ret = fio_send_cmd_ext_pdu(server_fd, FIO_NET_CMD_IOLOG, &pdu,
+ sizeof(pdu), 0, FIO_NET_CMD_F_MORE);
+ if (ret)
+ return ret;
+
+ /*
+ * Now send actual log, compress if we can, otherwise just plain
+ */
+ if (use_zlib)
+ return fio_send_iolog_gz(&pdu, log);
+
+ return fio_send_cmd_ext_pdu(server_fd, FIO_NET_CMD_IOLOG, log->log,
+ log->nr_samples * sizeof(struct io_sample), 0, 0);
+}
+
void fio_server_send_add_job(struct thread_data *td)
{
struct cmd_add_job_pdu pdu;
diff --git a/server.h b/server.h
index 067cdc0..478c283 100644
--- a/server.h
+++ b/server.h
@@ -38,7 +38,7 @@ struct fio_net_cmd_reply {
};
enum {
- FIO_SERVER_VER = 21,
+ FIO_SERVER_VER = 22,
FIO_SERVER_MAX_FRAGMENT_PDU = 1024,
@@ -71,6 +71,8 @@ enum {
FIO_NET_NAME_MAX = 256,
FIO_NET_CLIENT_TIMEOUT = 5000,
+
+ FIO_PROBE_FLAG_ZLIB = 1UL << 0,
};
struct cmd_ts_pdu {
@@ -83,7 +85,11 @@ struct cmd_du_pdu {
struct disk_util_agg agg;
};
-struct cmd_probe_pdu {
+struct cmd_client_probe_pdu {
+ uint64_t flags;
+};
+
+struct cmd_probe_reply_pdu {
uint8_t hostname[64];
uint8_t bigendian;
uint8_t fio_version[32];
@@ -139,6 +145,7 @@ struct cmd_iolog_pdu {
uint32_t thread_number;
uint32_t nr_samples;
uint32_t log_type;
+ uint32_t compressed;
uint8_t name[FIO_NET_NAME_MAX];
struct io_sample samples[0];
};
--
Jens Axboe
--
To unsubscribe from this list: send the line "unsubscribe fio" in
the body of a message to majordomo@xxxxxxxxxxxxxxx
More majordomo info at http://vger.kernel.org/majordomo-info.html
