The following changes since commit dc632d12d74bab6a439aaf8c317d250d3a8def5c:
add example job file for the RBD engine (2014-02-18 11:22:16 -0800)
are available in the git repository at:
git://git.kernel.dk/fio.git master
for you to fetch changes up to 65f21d61d5d0796335ceb3320b8846e4d6d30ac7:
xxhash: dos2unix'ize (2014-02-20 13:29:42 -0800)
----------------------------------------------------------------
Christian Ehrhardt (6):
fio: fix job clone mem leak
fio: allow general repeatability
fio: allow milliseconds on all time specifiers
fio: provide an option for a startdelay range
fio: add multi directory support
fio: allow to combine terse output with any selected output type
Jens Axboe (4):
server: bump protocol version
Add support for the Google xxhash checksumming function
Add option group/category to allrandrepeat
xxhash: dos2unix'ize
Peter Oberparleiter (2):
fio: flush log files on test end
fio: allow 0 as compress percentage
HOWTO | 4 +
backend.c | 5 +-
cconv.c | 4 +
crc/test.c | 30 ++++
crc/xxhash.c | 421 ++++++++++++++++++++++++++++++++++++++++++++++++++++++
crc/xxhash.h | 177 +++++++++++++++++++++++
engines/net.c | 2 +-
engines/rbd.c | 2 +-
eta.c | 14 +-
file.h | 8 +-
filesetup.c | 70 ++++++++-
fio.1 | 50 +++++--
fio.h | 6 +
init.c | 64 ++++++++-
iolog.c | 2 +-
iolog.h | 1 +
options.c | 91 +++++++++---
options.h | 2 +
parse.c | 48 +++++--
server.h | 2 +-
stat.c | 83 +++++++----
thread_options.h | 4 +
time.c | 2 +-
verify.c | 46 ++++++
verify.h | 4 +
25 files changed, 1045 insertions(+), 97 deletions(-)
create mode 100644 crc/xxhash.c
create mode 100644 crc/xxhash.h
---
Diff of recent changes:
diff --git a/HOWTO b/HOWTO
index 9830fa1..bafad93 100644
--- a/HOWTO
+++ b/HOWTO
@@ -1105,6 +1105,10 @@ verify=str If writing to a file, fio can verify the file contents
crc7 Use a crc7 sum of the data area and store
it in the header of each block.
+ xxhash Use xxhash as the checksum function. Generally
+ the fastest software checksum that fio
+ supports.
+
sha512 Use sha512 as the checksum function.
sha256 Use sha256 as the checksum function.
diff --git a/backend.c b/backend.c
index 32bc265..87aec87 100644
--- a/backend.c
+++ b/backend.c
@@ -346,7 +346,7 @@ static inline int runtime_exceeded(struct thread_data *td, struct timeval *t)
return 0;
if (!td->o.timeout)
return 0;
- if (mtime_since(&td->epoch, t) >= td->o.timeout * 1000)
+ if (mtime_since(&td->epoch, t) >= td->o.timeout )
return 1;
return 0;
@@ -1461,6 +1461,7 @@ static void *thread_main(void *data)
fio_unpin_memory(td);
fio_mutex_down(writeout_mutex);
+ finalize_logs(td);
if (td->bw_log) {
if (o->bw_log_file) {
finish_log_named(td, td->bw_log,
@@ -1783,7 +1784,7 @@ static void run_threads(void)
if (td->o.start_delay) {
spent = mtime_since_genesis();
- if (td->o.start_delay * 1000 > spent)
+ if (td->o.start_delay > spent)
continue;
}
diff --git a/cconv.c b/cconv.c
index b7d469e..fd8d0ad 100644
--- a/cconv.c
+++ b/cconv.c
@@ -142,6 +142,7 @@ void convert_thread_options_to_cpu(struct thread_options *o,
o->do_disk_util = le32_to_cpu(top->do_disk_util);
o->override_sync = le32_to_cpu(top->override_sync);
o->rand_repeatable = le32_to_cpu(top->rand_repeatable);
+ o->allrand_repeatable = le32_to_cpu(top->allrand_repeatable);
o->rand_seed = le64_to_cpu(top->rand_seed);
o->use_os_rand = le32_to_cpu(top->use_os_rand);
o->log_avg_msec = le32_to_cpu(top->log_avg_msec);
@@ -165,6 +166,7 @@ void convert_thread_options_to_cpu(struct thread_options *o,
o->verify_backlog = le64_to_cpu(top->verify_backlog);
o->start_delay = le64_to_cpu(top->start_delay);
+ o->start_delay_high = le64_to_cpu(top->start_delay_high);
o->timeout = le64_to_cpu(top->timeout);
o->ramp_time = le64_to_cpu(top->ramp_time);
o->zone_range = le64_to_cpu(top->zone_range);
@@ -308,6 +310,7 @@ void convert_thread_options_to_net(struct thread_options_pack *top,
top->do_disk_util = cpu_to_le32(o->do_disk_util);
top->override_sync = cpu_to_le32(o->override_sync);
top->rand_repeatable = cpu_to_le32(o->rand_repeatable);
+ top->allrand_repeatable = cpu_to_le32(o->allrand_repeatable);
top->rand_seed = __cpu_to_le64(o->rand_seed);
top->use_os_rand = cpu_to_le32(o->use_os_rand);
top->log_avg_msec = cpu_to_le32(o->log_avg_msec);
@@ -420,6 +423,7 @@ void convert_thread_options_to_net(struct thread_options_pack *top,
top->size = __cpu_to_le64(o->size);
top->verify_backlog = __cpu_to_le64(o->verify_backlog);
top->start_delay = __cpu_to_le64(o->start_delay);
+ top->start_delay_high = __cpu_to_le64(o->start_delay_high);
top->timeout = __cpu_to_le64(o->timeout);
top->ramp_time = __cpu_to_le64(o->ramp_time);
top->zone_range = __cpu_to_le64(o->zone_range);
diff --git a/crc/test.c b/crc/test.c
index 2f6d9ee..174bea3 100644
--- a/crc/test.c
+++ b/crc/test.c
@@ -16,6 +16,7 @@
#include "../crc/sha1.h"
#include "../crc/sha256.h"
#include "../crc/sha512.h"
+#include "../crc/xxhash.h"
#define CHUNK 131072U
#define NR_CHUNKS 2048U
@@ -36,6 +37,7 @@ enum {
T_SHA1 = 1U << 6,
T_SHA256 = 1U << 7,
T_SHA512 = 1U << 8,
+ T_XXHASH = 1U << 9,
};
static void randomize_buf(void *buf, unsigned int size, int seed)
@@ -233,6 +235,29 @@ static uint64_t t_sha512(void)
return ret;
}
+static uint64_t t_xxhash(void)
+{
+ void *state;
+ struct timeval s;
+ uint64_t ret;
+ void *buf;
+ int i;
+
+ state = XXH32_init(0x8989);
+
+ buf = malloc(CHUNK);
+ randomize_buf(buf, CHUNK, 0x8989);
+
+ fio_gettime(&s, NULL);
+ for (i = 0; i < NR_CHUNKS; i++)
+ XXH32_update(state, buf, CHUNK);
+
+ XXH32_digest(state);
+ ret = utime_since_now(&s);
+ free(buf);
+ return ret;
+}
+
static struct test_type t[] = {
{
.name = "md5",
@@ -280,6 +305,11 @@ static struct test_type t[] = {
.fn = t_sha512,
},
{
+ .name = "xxhash",
+ .mask = T_XXHASH,
+ .fn = t_xxhash,
+ },
+ {
.name = NULL,
},
};
diff --git a/crc/xxhash.c b/crc/xxhash.c
new file mode 100644
index 0000000..5190330
--- /dev/null
+++ b/crc/xxhash.c
@@ -0,0 +1,421 @@
+/*
+xxHash - Fast Hash algorithm
+Copyright (C) 2012-2014, Yann Collet.
+BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+* Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+* Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions and the following disclaimer
+in the documentation and/or other materials provided with the
+distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+You can contact the author at :
+- xxHash source repository : http://code.google.com/p/xxhash/
+*/
+
+
+//**************************************
+// Tuning parameters
+//**************************************
+// Unaligned memory access is automatically enabled for "common" CPU, such as x86.
+// For others CPU, the compiler will be more cautious, and insert extra code to ensure aligned access is respected.
+// If you know your target CPU supports unaligned memory access, you want to force this option manually to improve performance.
+// You can also enable this parameter if you know your input data will always be aligned (boundaries of 4, for uint32_t).
+#if defined(__ARM_FEATURE_UNALIGNED) || defined(__i386) || defined(_M_IX86) || defined(__x86_64__) || defined(_M_X64)
+# define XXH_USE_UNALIGNED_ACCESS 1
+#endif
+
+// XXH_ACCEPT_NULL_INPUT_POINTER :
+// If the input pointer is a null pointer, xxHash default behavior is to trigger a memory access error, since it is a bad pointer.
+// When this option is enabled, xxHash output for null input pointers will be the same as a null-length input.
+// This option has a very small performance cost (only measurable on small inputs).
+// By default, this option is disabled. To enable it, uncomment below define :
+//#define XXH_ACCEPT_NULL_INPUT_POINTER 1
+
+// XXH_FORCE_NATIVE_FORMAT :
+// By default, xxHash library provides endian-independant Hash values, based on little-endian convention.
+// Results are therefore identical for little-endian and big-endian CPU.
+// This comes at a performance cost for big-endian CPU, since some swapping is required to emulate little-endian format.
+// Should endian-independance be of no importance for your application, you may set the #define below to 1.
+// It will improve speed for Big-endian CPU.
+// This option has no impact on Little_Endian CPU.
+#define XXH_FORCE_NATIVE_FORMAT 0
+
+
+//**************************************
+// Includes & Memory related functions
+//**************************************
+#include "xxhash.h"
+#include <stdlib.h>
+#include <string.h>
+
+
+#if defined(__GNUC__) && !defined(XXH_USE_UNALIGNED_ACCESS)
+# define _PACKED __attribute__ ((packed))
+#else
+# define _PACKED
+#endif
+
+#if !defined(XXH_USE_UNALIGNED_ACCESS) && !defined(__GNUC__)
+# ifdef __IBMC__
+# pragma pack(1)
+# else
+# pragma pack(push, 1)
+# endif
+#endif
+
+typedef struct _uint32_t_S { uint32_t v; } _PACKED uint32_t_S;
+
+#if !defined(XXH_USE_UNALIGNED_ACCESS) && !defined(__GNUC__)
+# pragma pack(pop)
+#endif
+
+#define A32(x) (((uint32_t_S *)(x))->v)
+
+
+//***************************************
+// Compiler-specific Functions and Macros
+//***************************************
+#define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
+
+// Note : although _rotl exists for minGW (GCC under windows), performance seems poor
+#if defined(_MSC_VER)
+# define XXH_rotl32(x,r) _rotl(x,r)
+#else
+# define XXH_rotl32(x,r) ((x << r) | (x >> (32 - r)))
+#endif
+
+#if defined(_MSC_VER) // Visual Studio
+# define XXH_swap32 _byteswap_ulong
+#elif GCC_VERSION >= 403
+# define XXH_swap32 __builtin_bswap32
+#else
+static inline uint32_t XXH_swap32 (uint32_t x)
+{
+ return ((x << 24) & 0xff000000 ) |
+ ((x << 8) & 0x00ff0000 ) |
+ ((x >> 8) & 0x0000ff00 ) |
+ ((x >> 24) & 0x000000ff );
+}
+#endif
+
+
+//**************************************
+// Constants
+//**************************************
+#define PRIME32_1 2654435761U
+#define PRIME32_2 2246822519U
+#define PRIME32_3 3266489917U
+#define PRIME32_4 668265263U
+#define PRIME32_5 374761393U
+
+
+//**************************************
+// Architecture Macros
+//**************************************
+typedef enum { XXH_bigEndian=0, XXH_littleEndian=1 } XXH_endianess;
+#ifndef XXH_CPU_LITTLE_ENDIAN // It is possible to define XXH_CPU_LITTLE_ENDIAN externally, for example using a compiler switch
+ static const int one = 1;
+# define XXH_CPU_LITTLE_ENDIAN (*(char*)(&one))
+#endif
+
+
+//**************************************
+// Macros
+//**************************************
+#define XXH_STATIC_ASSERT(c) { enum { XXH_static_assert = 1/(!!(c)) }; } // use only *after* variable declarations
+
+
+//****************************
+// Memory reads
+//****************************
+typedef enum { XXH_aligned, XXH_unaligned } XXH_alignment;
+
+uint32_t XXH_readLE32_align(const uint32_t* ptr, XXH_endianess endian, XXH_alignment align)
+{
+ if (align==XXH_unaligned)
+ return endian==XXH_littleEndian ? A32(ptr) : XXH_swap32(A32(ptr));
+ else
+ return endian==XXH_littleEndian ? *ptr : XXH_swap32(*ptr);
+}
+
+uint32_t XXH_readLE32(const uint32_t* ptr, XXH_endianess endian) { return XXH_readLE32_align(ptr, endian, XXH_unaligned); }
+
+
+//****************************
+// Simple Hash Functions
+//****************************
+uint32_t XXH32_endian_align(const void* input, int len, uint32_t seed, XXH_endianess endian, XXH_alignment align)
+{
+ const uint8_t *p = (const uint8_t *)input;
+ const uint8_t * const bEnd = p + len;
+ uint32_t h32;
+
+#ifdef XXH_ACCEPT_NULL_INPUT_POINTER
+ if (p==NULL) { len=0; p=(const uint8_t *)(size_t)16; }
+#endif
+
+ if (len>=16)
+ {
+ const uint8_t * const limit = bEnd - 16;
+ uint32_t v1 = seed + PRIME32_1 + PRIME32_2;
+ uint32_t v2 = seed + PRIME32_2;
+ uint32_t v3 = seed + 0;
+ uint32_t v4 = seed - PRIME32_1;
+
+ do
+ {
+ v1 += XXH_readLE32_align((const uint32_t*)p, endian, align) * PRIME32_2; v1 = XXH_rotl32(v1, 13); v1 *= PRIME32_1; p+=4;
+ v2 += XXH_readLE32_align((const uint32_t*)p, endian, align) * PRIME32_2; v2 = XXH_rotl32(v2, 13); v2 *= PRIME32_1; p+=4;
+ v3 += XXH_readLE32_align((const uint32_t*)p, endian, align) * PRIME32_2; v3 = XXH_rotl32(v3, 13); v3 *= PRIME32_1; p+=4;
+ v4 += XXH_readLE32_align((const uint32_t*)p, endian, align) * PRIME32_2; v4 = XXH_rotl32(v4, 13); v4 *= PRIME32_1; p+=4;
+ } while (p<=limit);
+
+ h32 = XXH_rotl32(v1, 1) + XXH_rotl32(v2, 7) + XXH_rotl32(v3, 12) + XXH_rotl32(v4, 18);
+ }
+ else
+ {
+ h32 = seed + PRIME32_5;
+ }
+
+ h32 += (uint32_t) len;
+
+ while (p<=bEnd-4)
+ {
+ h32 += XXH_readLE32_align((const uint32_t*)p, endian, align) * PRIME32_3;
+ h32 = XXH_rotl32(h32, 17) * PRIME32_4 ;
+ p+=4;
+ }
+
+ while (p<bEnd)
+ {
+ h32 += (*p) * PRIME32_5;
+ h32 = XXH_rotl32(h32, 11) * PRIME32_1 ;
+ p++;
+ }
+
+ h32 ^= h32 >> 15;
+ h32 *= PRIME32_2;
+ h32 ^= h32 >> 13;
+ h32 *= PRIME32_3;
+ h32 ^= h32 >> 16;
+
+ return h32;
+}
+
+
+uint32_t XXH32(const void* input, int len, uint32_t seed)
+{
+#if 0
+ // Simple version, good for code maintenance, but unfortunately slow for small inputs
+ void* state = XXH32_init(seed);
+ XXH32_update(state, input, len);
+ return XXH32_digest(state);
+#else
+ XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN;
+
+# if !defined(XXH_USE_UNALIGNED_ACCESS)
+ if ((((size_t)input) & 3)) // Input is aligned, let's leverage the speed advantage
+ {
+ if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
+ return XXH32_endian_align(input, len, seed, XXH_littleEndian, XXH_aligned);
+ else
+ return XXH32_endian_align(input, len, seed, XXH_bigEndian, XXH_aligned);
+ }
+# endif
+
+ if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
+ return XXH32_endian_align(input, len, seed, XXH_littleEndian, XXH_unaligned);
+ else
+ return XXH32_endian_align(input, len, seed, XXH_bigEndian, XXH_unaligned);
+#endif
+}
+
+
+//****************************
+// Advanced Hash Functions
+//****************************
+
+int XXH32_sizeofState()
+{
+ XXH_STATIC_ASSERT(XXH32_SIZEOFSTATE >= sizeof(struct XXH_state32_t)); // A compilation error here means XXH32_SIZEOFSTATE is not large enough
+ return sizeof(struct XXH_state32_t);
+}
+
+
+XXH_errorcode XXH32_resetState(void* state_in, uint32_t seed)
+{
+ struct XXH_state32_t * state = (struct XXH_state32_t *) state_in;
+ state->seed = seed;
+ state->v1 = seed + PRIME32_1 + PRIME32_2;
+ state->v2 = seed + PRIME32_2;
+ state->v3 = seed + 0;
+ state->v4 = seed - PRIME32_1;
+ state->total_len = 0;
+ state->memsize = 0;
+ return XXH_OK;
+}
+
+
+void* XXH32_init (uint32_t seed)
+{
+ void *state = malloc (sizeof(struct XXH_state32_t));
+ XXH32_resetState(state, seed);
+ return state;
+}
+
+
+XXH_errorcode XXH32_update_endian (void* state_in, const void* input, int len, XXH_endianess endian)
+{
+ struct XXH_state32_t * state = (struct XXH_state32_t *) state_in;
+ const uint8_t *p = (const uint8_t *)input;
+ const uint8_t * const bEnd = p + len;
+
+#ifdef XXH_ACCEPT_NULL_INPUT_POINTER
+ if (input==NULL) return XXH_ERROR;
+#endif
+
+ state->total_len += len;
+
+ if (state->memsize + len < 16) // fill in tmp buffer
+ {
+ memcpy(state->memory + state->memsize, input, len);
+ state->memsize += len;
+ return XXH_OK;
+ }
+
+ if (state->memsize) // some data left from previous update
+ {
+ memcpy(state->memory + state->memsize, input, 16-state->memsize);
+ {
+ const uint32_t* p32 = (const uint32_t*)state->memory;
+ state->v1 += XXH_readLE32(p32, endian) * PRIME32_2; state->v1 = XXH_rotl32(state->v1, 13); state->v1 *= PRIME32_1; p32++;
+ state->v2 += XXH_readLE32(p32, endian) * PRIME32_2; state->v2 = XXH_rotl32(state->v2, 13); state->v2 *= PRIME32_1; p32++;
+ state->v3 += XXH_readLE32(p32, endian) * PRIME32_2; state->v3 = XXH_rotl32(state->v3, 13); state->v3 *= PRIME32_1; p32++;
+ state->v4 += XXH_readLE32(p32, endian) * PRIME32_2; state->v4 = XXH_rotl32(state->v4, 13); state->v4 *= PRIME32_1; p32++;
+ }
+ p += 16-state->memsize;
+ state->memsize = 0;
+ }
+
+ if (p <= bEnd-16)
+ {
+ const uint8_t * const limit = bEnd - 16;
+ uint32_t v1 = state->v1;
+ uint32_t v2 = state->v2;
+ uint32_t v3 = state->v3;
+ uint32_t v4 = state->v4;
+
+ do
+ {
+ v1 += XXH_readLE32((const uint32_t*)p, endian) * PRIME32_2; v1 = XXH_rotl32(v1, 13); v1 *= PRIME32_1; p+=4;
+ v2 += XXH_readLE32((const uint32_t*)p, endian) * PRIME32_2; v2 = XXH_rotl32(v2, 13); v2 *= PRIME32_1; p+=4;
+ v3 += XXH_readLE32((const uint32_t*)p, endian) * PRIME32_2; v3 = XXH_rotl32(v3, 13); v3 *= PRIME32_1; p+=4;
+ v4 += XXH_readLE32((const uint32_t*)p, endian) * PRIME32_2; v4 = XXH_rotl32(v4, 13); v4 *= PRIME32_1; p+=4;
+ } while (p<=limit);
+
+ state->v1 = v1;
+ state->v2 = v2;
+ state->v3 = v3;
+ state->v4 = v4;
+ }
+
+ if (p < bEnd)
+ {
+ memcpy(state->memory, p, bEnd-p);
+ state->memsize = (int)(bEnd-p);
+ }
+
+ return XXH_OK;
+}
+
+XXH_errorcode XXH32_update (void* state_in, const void* input, int len)
+{
+ XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN;
+
+ if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
+ return XXH32_update_endian(state_in, input, len, XXH_littleEndian);
+ else
+ return XXH32_update_endian(state_in, input, len, XXH_bigEndian);
+}
+
+
+
+uint32_t XXH32_intermediateDigest_endian (void* state_in, XXH_endianess endian)
+{
+ struct XXH_state32_t * state = (struct XXH_state32_t *) state_in;
+ const uint8_t *p = (const uint8_t *)state->memory;
+ uint8_t * bEnd = (uint8_t *)state->memory + state->memsize;
+ uint32_t h32;
+
+ if (state->total_len >= 16)
+ {
+ h32 = XXH_rotl32(state->v1, 1) + XXH_rotl32(state->v2, 7) + XXH_rotl32(state->v3, 12) + XXH_rotl32(state->v4, 18);
+ }
+ else
+ {
+ h32 = state->seed + PRIME32_5;
+ }
+
+ h32 += (uint32_t) state->total_len;
+
+ while (p<=bEnd-4)
+ {
+ h32 += XXH_readLE32((const uint32_t*)p, endian) * PRIME32_3;
+ h32 = XXH_rotl32(h32, 17) * PRIME32_4;
+ p+=4;
+ }
+
+ while (p<bEnd)
+ {
+ h32 += (*p) * PRIME32_5;
+ h32 = XXH_rotl32(h32, 11) * PRIME32_1;
+ p++;
+ }
+
+ h32 ^= h32 >> 15;
+ h32 *= PRIME32_2;
+ h32 ^= h32 >> 13;
+ h32 *= PRIME32_3;
+ h32 ^= h32 >> 16;
+
+ return h32;
+}
+
+
+uint32_t XXH32_intermediateDigest (void* state_in)
+{
+ XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN;
+
+ if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
+ return XXH32_intermediateDigest_endian(state_in, XXH_littleEndian);
+ else
+ return XXH32_intermediateDigest_endian(state_in, XXH_bigEndian);
+}
+
+
+uint32_t XXH32_digest (void* state_in)
+{
+ uint32_t h32 = XXH32_intermediateDigest(state_in);
+
+ free(state_in);
+
+ return h32;
+}
diff --git a/crc/xxhash.h b/crc/xxhash.h
new file mode 100644
index 0000000..bcd060e
--- /dev/null
+++ b/crc/xxhash.h
@@ -0,0 +1,177 @@
+/*
+ xxHash - Fast Hash algorithm
+ Header File
+ Copyright (C) 2012-2014, Yann Collet.
+ BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions are
+ met:
+
+ * Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ * Redistributions in binary form must reproduce the above
+ copyright notice, this list of conditions and the following disclaimer
+ in the documentation and/or other materials provided with the
+ distribution.
+
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+ You can contact the author at :
+ - xxHash source repository : http://code.google.com/p/xxhash/
+*/
+
+/* Notice extracted from xxHash homepage :
+
+xxHash is an extremely fast Hash algorithm, running at RAM speed limits.
+It also successfully passes all tests from the SMHasher suite.
+
+Comparison (single thread, Windows Seven 32 bits, using SMHasher on a Core 2 Duo @3GHz)
+
+Name Speed Q.Score Author
+xxHash 5.4 GB/s 10
+CrapWow 3.2 GB/s 2 Andrew
+MumurHash 3a 2.7 GB/s 10 Austin Appleby
+SpookyHash 2.0 GB/s 10 Bob Jenkins
+SBox 1.4 GB/s 9 Bret Mulvey
+Lookup3 1.2 GB/s 9 Bob Jenkins
+SuperFastHash 1.2 GB/s 1 Paul Hsieh
+CityHash64 1.05 GB/s 10 Pike & Alakuijala
+FNV 0.55 GB/s 5 Fowler, Noll, Vo
+CRC32 0.43 GB/s 9
+MD5-32 0.33 GB/s 10 Ronald L. Rivest
+SHA1-32 0.28 GB/s 10
+
+Q.Score is a measure of quality of the hash function.
+It depends on successfully passing SMHasher test set.
+10 is a perfect score.
+*/
+
+#pragma once
+
+#if defined (__cplusplus)
+extern "C" {
+#endif
+
+#include <inttypes.h>
+
+struct XXH_state32_t
+{
+ uint64_t total_len;
+ uint32_t seed;
+ uint32_t v1;
+ uint32_t v2;
+ uint32_t v3;
+ uint32_t v4;
+ int memsize;
+ char memory[16];
+};
+
+//****************************
+// Type
+//****************************
+typedef enum { XXH_OK=0, XXH_ERROR } XXH_errorcode;
+
+
+
+//****************************
+// Simple Hash Functions
+//****************************
+
+unsigned int XXH32 (const void* input, int len, unsigned int seed);
+
+/*
+XXH32() :
+ Calculate the 32-bits hash of sequence of length "len" stored at memory address "input".
+ The memory between input & input+len must be valid (allocated and read-accessible).
+ "seed" can be used to alter the result predictably.
+ This function successfully passes all SMHasher tests.
+ Speed on Core 2 Duo @ 3 GHz (single thread, SMHasher benchmark) : 5.4 GB/s
+ Note that "len" is type "int", which means it is limited to 2^31-1.
+ If your data is larger, use the advanced functions below.
+*/
+
+
+
+//****************************
+// Advanced Hash Functions
+//****************************
+
+void* XXH32_init (unsigned int seed);
+XXH_errorcode XXH32_update (void* state, const void* input, int len);
+unsigned int XXH32_digest (void* state);
+
+/*
+These functions calculate the xxhash of an input provided in several small packets,
+as opposed to an input provided as a single block.
+
+It must be started with :
+void* XXH32_init()
+The function returns a pointer which holds the state of calculation.
+
+This pointer must be provided as "void* state" parameter for XXH32_update().
+XXH32_update() can be called as many times as necessary.
+The user must provide a valid (allocated) input.
+The function returns an error code, with 0 meaning OK, and any other value meaning there is an error.
+Note that "len" is type "int", which means it is limited to 2^31-1.
+If your data is larger, it is recommended to chunk your data into blocks
+of size for example 2^30 (1GB) to avoid any "int" overflow issue.
+
+Finally, you can end the calculation anytime, by using XXH32_digest().
+This function returns the final 32-bits hash.
+You must provide the same "void* state" parameter created by XXH32_init().
+Memory will be freed by XXH32_digest().
+*/
+
+
+int XXH32_sizeofState();
+XXH_errorcode XXH32_resetState(void* state, unsigned int seed);
+
+#define XXH32_SIZEOFSTATE 48
+typedef struct { long long ll[(XXH32_SIZEOFSTATE+(sizeof(long long)-1))/sizeof(long long)]; } XXH32_stateSpace_t;
+/*
+These functions allow user application to make its own allocation for state.
+
+XXH32_sizeofState() is used to know how much space must be allocated for the xxHash 32-bits state.
+Note that the state must be aligned to access 'long long' fields. Memory must be allocated and referenced by a pointer.
+This pointer must then be provided as 'state' into XXH32_resetState(), which initializes the state.
+
+For static allocation purposes (such as allocation on stack, or freestanding systems without malloc()),
+use the structure XXH32_stateSpace_t, which will ensure that memory space is large enough and correctly aligned to access 'long long' fields.
+*/
+
+
+unsigned int XXH32_intermediateDigest (void* state);
+/*
+This function does the same as XXH32_digest(), generating a 32-bit hash,
+but preserve memory context.
+This way, it becomes possible to generate intermediate hashes, and then continue feeding data with XXH32_update().
+To free memory context, use XXH32_digest(), or free().
+*/
+
+
+
+//****************************
+// Deprecated function names
+//****************************
+// The following translations are provided to ease code transition
+// You are encouraged to no longer this function names
+#define XXH32_feed XXH32_update
+#define XXH32_result XXH32_digest
+#define XXH32_getIntermediateResult XXH32_intermediateDigest
+
+
+
+#if defined (__cplusplus)
+}
+#endif
diff --git a/engines/net.c b/engines/net.c
index 1dc55d5..8b85a88 100644
--- a/engines/net.c
+++ b/engines/net.c
@@ -1194,7 +1194,7 @@ static int fio_netio_setup(struct thread_data *td)
struct netio_data *nd;
if (!td->files_index) {
- add_file(td, td->o.filename ?: "net");
+ add_file(td, td->o.filename ?: "net", 0);
td->o.nr_files = td->o.nr_files ?: 1;
}
diff --git a/engines/rbd.c b/engines/rbd.c
index d089a41..39fa0ce 100644
--- a/engines/rbd.c
+++ b/engines/rbd.c
@@ -377,7 +377,7 @@ static int fio_rbd_setup(struct thread_data *td)
* The size of the RBD is set instead of a artificial file.
*/
if (!td->files_index) {
- add_file(td, td->o.filename ? : "rbd");
+ add_file(td, td->o.filename ? : "rbd", 0);
td->o.nr_files = td->o.nr_files ? : 1;
}
f = td->files[0];
diff --git a/eta.c b/eta.c
index 9fc6e27..e12e4fc 100644
--- a/eta.c
+++ b/eta.c
@@ -174,7 +174,8 @@ static int thread_eta(struct thread_data *td)
perc = 1.0;
if (td->o.time_based) {
- perc_t = (double) elapsed / (double) td->o.timeout;
+ perc_t = (double) elapsed /
+ (double) (td->o.timeout / 1000);
if (perc_t < perc)
perc = perc_t;
}
@@ -182,8 +183,9 @@ static int thread_eta(struct thread_data *td)
eta_sec = (unsigned long) (elapsed * (1.0 / perc)) - elapsed;
if (td->o.timeout &&
- eta_sec > (td->o.timeout + done_secs - elapsed))
- eta_sec = td->o.timeout + done_secs - elapsed;
+ eta_sec > ( (td->o.timeout / 1000) + done_secs - elapsed))
+ eta_sec = (td->o.timeout / 1000) + done_secs
+ - elapsed;
} else if (td->runstate == TD_NOT_CREATED || td->runstate == TD_CREATED
|| td->runstate == TD_INITIALIZED
|| td->runstate == TD_SETTING_UP
@@ -197,8 +199,8 @@ static int thread_eta(struct thread_data *td)
* if given, otherwise assume it'll run at the specified rate.
*/
if (td->o.timeout) {
- t_eta = td->o.timeout + td->o.start_delay +
- td->o.ramp_time;
+ t_eta = (td->o.timeout + td->o.start_delay +
+ td->o.ramp_time ) / 1000;
if (in_ramp_time(td)) {
unsigned long ramp_left;
@@ -212,7 +214,7 @@ static int thread_eta(struct thread_data *td)
rate_bytes = ddir_rw_sum(td->o.rate);
if (rate_bytes) {
r_eta = (bytes_total / 1024) / rate_bytes;
- r_eta += td->o.start_delay;
+ r_eta += td->o.start_delay / 1000;
}
if (r_eta && t_eta)
diff --git a/file.h b/file.h
index c1d02a5..d065a25 100644
--- a/file.h
+++ b/file.h
@@ -125,6 +125,11 @@ struct fio_file {
struct disk_util *du;
};
+struct file_name {
+ struct flist_head list;
+ char *filename;
+};
+
#define FILE_FLAG_FNS(name) \
static inline void fio_file_set_##name(struct fio_file *f) \
{ \
@@ -162,7 +167,7 @@ extern int __must_check generic_close_file(struct thread_data *, struct fio_file
extern int __must_check generic_get_file_size(struct thread_data *, struct fio_file *);
extern int __must_check file_lookup_open(struct fio_file *f, int flags);
extern int __must_check pre_read_files(struct thread_data *);
-extern int add_file(struct thread_data *, const char *);
+extern int add_file(struct thread_data *, const char *, int);
extern int add_file_exclusive(struct thread_data *, const char *);
extern void get_file(struct fio_file *);
extern int __must_check put_file(struct thread_data *, struct fio_file *);
@@ -175,6 +180,7 @@ extern int init_random_map(struct thread_data *);
extern void dup_files(struct thread_data *, struct thread_data *);
extern int get_fileno(struct thread_data *, const char *);
extern void free_release_files(struct thread_data *);
+extern void filesetup_mem_free(void);
void fio_file_reset(struct thread_data *, struct fio_file *);
int fio_files_done(struct thread_data *);
diff --git a/filesetup.c b/filesetup.c
index 544ecb1..f0e3b34 100644
--- a/filesetup.c
+++ b/filesetup.c
@@ -11,6 +11,7 @@
#include "fio.h"
#include "smalloc.h"
#include "filehash.h"
+#include "options.h"
#include "os/os.h"
#include "hash.h"
#include "lib/axmap.h"
@@ -21,6 +22,8 @@
static int root_warn;
+static FLIST_HEAD(filename_list);
+
static inline void clear_error(struct thread_data *td)
{
td->error = 0;
@@ -1101,7 +1104,48 @@ static void get_file_type(struct fio_file *f)
}
}
-int add_file(struct thread_data *td, const char *fname)
+static void set_already_allocated(const char *fname) {
+ struct file_name *fn;
+
+ fn = malloc(sizeof(struct file_name));
+ fn->filename = strdup(fname);
+ flist_add_tail(&fn->list, &filename_list);
+}
+
+static int is_already_allocated(const char *fname)
+{
+ struct flist_head *entry;
+ char *filename;
+
+ if (!flist_empty(&filename_list))
+ {
+ flist_for_each(entry, &filename_list) {
+ filename = flist_entry(entry, struct file_name, list)->filename;
+
+ if (strcmp(filename, fname) == 0)
+ return 1;
+ }
+ }
+
+ return 0;
+}
+
+static void free_already_allocated() {
+ struct flist_head *entry, *tmp;
+ struct file_name *fn;
+
+ if (!flist_empty(&filename_list))
+ {
+ flist_for_each_safe(entry, tmp, &filename_list) {
+ fn = flist_entry(entry, struct file_name, list);
+ free(fn->filename);
+ flist_del(&fn->list);
+ free(fn);
+ }
+ }
+}
+
+int add_file(struct thread_data *td, const char *fname, int numjob)
{
int cur_files = td->files_index;
char file_name[PATH_MAX];
@@ -1110,6 +1154,15 @@ int add_file(struct thread_data *td, const char *fname)
dprint(FD_FILE, "add file %s\n", fname);
+ if (td->o.directory)
+ len = set_name_idx(file_name, td->o.directory, numjob);
+
+ sprintf(file_name + len, "%s", fname);
+
+ /* clean cloned siblings using existing files */
+ if (numjob && is_already_allocated(file_name))
+ return 0;
+
f = smalloc(sizeof(*f));
if (!f) {
log_err("fio: smalloc OOM\n");
@@ -1149,10 +1202,6 @@ int add_file(struct thread_data *td, const char *fname)
if (td->io_ops && (td->io_ops->flags & FIO_DISKLESSIO))
f->real_file_size = -1ULL;
- if (td->o.directory)
- len = sprintf(file_name, "%s/", td->o.directory);
-
- sprintf(file_name + len, "%s", fname);
f->file_name = smalloc_strdup(file_name);
if (!f->file_name) {
log_err("fio: smalloc OOM\n");
@@ -1179,6 +1228,8 @@ int add_file(struct thread_data *td, const char *fname)
if (f->filetype == FIO_TYPE_FILE)
td->nr_normal_files++;
+ set_already_allocated(file_name);
+
dprint(FD_FILE, "file %p \"%s\" added at %d\n", f, f->file_name,
cur_files);
@@ -1195,7 +1246,7 @@ int add_file_exclusive(struct thread_data *td, const char *fname)
return i;
}
- return add_file(td, fname);
+ return add_file(td, fname, 0);
}
void get_file(struct fio_file *f)
@@ -1304,7 +1355,7 @@ static int recurse_dir(struct thread_data *td, const char *dirname)
}
if (S_ISREG(sb.st_mode)) {
- add_file(td, full_path);
+ add_file(td, full_path, 0);
td->o.nr_files++;
continue;
}
@@ -1421,3 +1472,8 @@ int fio_files_done(struct thread_data *td)
return 1;
}
+
+/* free memory used in initialization phase only */
+void filesetup_mem_free() {
+ free_already_allocated();
+}
diff --git a/fio.1 b/fio.1
index 1df1cd1..81b03f7 100644
--- a/fio.1
+++ b/fio.1
@@ -32,6 +32,9 @@ Generate per-job bandwidth logs.
.B \-\-minimal
Print statistics in a terse, semicolon-delimited format.
.TP
+.B \-\-append-terse
+Print statistics in selected mode AND terse, semicolon-delimited format.
+.TP
.B \-\-version
Display version information and exit.
.TP
@@ -121,12 +124,16 @@ String: a sequence of alphanumeric characters.
SI integer: a whole number, possibly containing a suffix denoting the base unit
of the value. Accepted suffixes are `k', 'M', 'G', 'T', and 'P', denoting
kilo (1024), mega (1024^2), giga (1024^3), tera (1024^4), and peta (1024^5)
-respectively. The suffix is not case sensitive. If prefixed with '0x', the
-value is assumed to be base 16 (hexadecimal). A suffix may include a trailing 'b',
-for instance 'kb' is identical to 'k'. You can specify a base 10 value
-by using 'KiB', 'MiB', 'GiB', etc. This is useful for disk drives where
-values are often given in base 10 values. Specifying '30GiB' will get you
-30*1000^3 bytes.
+respectively. If prefixed with '0x', the value is assumed to be base 16
+(hexadecimal). A suffix may include a trailing 'b', for instance 'kb' is
+identical to 'k'. You can specify a base 10 value by using 'KiB', 'MiB','GiB',
+etc. This is useful for disk drives where values are often given in base 10
+values. Specifying '30GiB' will get you 30*1000^3 bytes.
+When specifying times the default suffix meaning changes, still denoting the
+base unit of the value, but accepted suffixes are 'D' (days), 'H' (hours), 'M'
+(minutes), 'S' Seconds, 'ms' milli seconds. Time values without a unit specify
+seconds.
+The suffixes are not case sensitive.
.TP
.I bool
Boolean: a true or false value. `0' denotes false, `1' denotes true.
@@ -154,6 +161,12 @@ otherwise has no special purpose.
.BI directory \fR=\fPstr
Prefix filenames with this directory. Used to place files in a location other
than `./'.
+You can specify a number of directories by separating the names with a ':'
+character. These directories will be assigned equally distributed to job clones
+creates with \fInumjobs\fR as long as they are using generated filenames.
+If specific \fIfilename(s)\fR are set fio will use the first listed directory,
+and thereby matching the \fIfilename\fR semantic which generates a file each
+clone if not specified, but let all clones use the same if set.
.TP
.BI filename \fR=\fPstr
.B fio
@@ -293,8 +306,12 @@ read, write, and trim are accounted and reported separately. If this option is
set, the fio will sum the results and report them as "mixed" instead.
.TP
.BI randrepeat \fR=\fPbool
-Seed the random number generator in a predictable way so results are repeatable
-across runs. Default: true.
+Seed the random number generator used for random I/O patterns in a predictable
+way so the pattern is repeatable across runs. Default: true.
+.TP
+.BI allrandrepeat \fR=\fPbool
+Seed all random number generators in a predictable way so results are
+repeatable across runs. Default: false.
.TP
.BI randseed \fR=\fPint
Seed the random number generators based on this seed value, to be able to
@@ -838,8 +855,12 @@ needed to be specified. For \fBprefer\fR, only one node is
allowed. For \fBbind\fR and \fBinterleave\fR, \fBnodelist\fR allows
comma delimited list of numbers, A-B ranges, or 'all'.
.TP
-.BI startdelay \fR=\fPint
-Delay start of job for the specified number of seconds.
+.BI startdelay \fR=\fPirange
+Delay start of job for the specified number of seconds. Supports all time
+suffixes to allow specification of hours, minutes, seconds and
+milliseconds - seconds are the default if a unit is ommited.
+Can be given as a range which causes each thread to choose randomly out of the
+range.
.TP
.BI runtime \fR=\fPint
Terminate processing after the specified number of seconds.
@@ -964,7 +985,7 @@ values are:
.RS
.RS
.TP
-.B md5 crc16 crc32 crc32c crc32c-intel crc64 crc7 sha256 sha512 sha1
+.B md5 crc16 crc32 crc32c crc32c-intel crc64 crc7 sha256 sha512 sha1 xxhash
Store appropriate checksum in the header of each block. crc32c-intel is
hardware accelerated SSE4.2 driven, falls back to regular crc32c if
not supported by the system.
@@ -1560,9 +1581,10 @@ It is also possible to get fio to dump the current output while it is
running, without terminating the job. To do that, send fio the \fBUSR1\fR
signal.
.SH TERSE OUTPUT
-If the \fB\-\-minimal\fR option is given, the results will be printed in a
-semicolon-delimited format suitable for scripted use - a job description
-(if provided) follows on a new line. Note that the first
+If the \fB\-\-minimal\fR / \fB\-\-append-terse\fR options are given, the
+results will be printed/appended in a semicolon-delimited format suitable for
+scripted use.
+A job description (if provided) follows on a new line. Note that the first
number in the line is the version number. If the output has to be changed
for some reason, this number will be incremented by 1 to signify that
change. The fields are:
diff --git a/fio.h b/fio.h
index cda4668..d1180cd 100644
--- a/fio.h
+++ b/fio.h
@@ -85,6 +85,7 @@ enum {
FIO_RAND_SEQ_RAND_READ_OFF,
FIO_RAND_SEQ_RAND_WRITE_OFF,
FIO_RAND_SEQ_RAND_TRIM_OFF,
+ FIO_RAND_START_DELAY,
FIO_RAND_NR_OFFS,
};
@@ -164,6 +165,10 @@ struct thread_data {
os_random_state_t trim_state;
struct frand_state __trim_state;
};
+ union {
+ os_random_state_t delay_state;
+ struct frand_state __delay_state;
+ };
struct frand_state buf_state;
@@ -368,6 +373,7 @@ extern unsigned int stat_number;
extern int shm_id;
extern int groupid;
extern int output_format;
+extern int append_terse_output;
extern int temp_stall_ts;
extern uintptr_t page_mask, page_size;
extern int read_only;
diff --git a/init.c b/init.c
index fa1df8e..5fc3b22 100644
--- a/init.c
+++ b/init.c
@@ -43,6 +43,7 @@ struct thread_data *threads = NULL;
int exitall_on_terminate = 0;
int output_format = FIO_OUTPUT_NORMAL;
+int append_terse_output = 0;
int eta_print = FIO_ETA_AUTO;
int eta_new_line = 0;
FILE *f_out = NULL;
@@ -109,6 +110,11 @@ static struct option l_opts[FIO_NR_OPTIONS] = {
.val = 'F' | FIO_CLIENT_FLAG,
},
{
+ .name = (char *) "append-terse",
+ .has_arg = optional_argument,
+ .val = 'f',
+ },
+ {
.name = (char *) "version",
.has_arg = no_argument,
.val = 'v' | FIO_CLIENT_FLAG,
@@ -412,6 +418,26 @@ static int fixed_block_size(struct thread_options *o)
o->min_bs[DDIR_READ] == o->min_bs[DDIR_TRIM];
}
+
+static unsigned long long get_rand_start_delay(struct thread_data *td)
+{
+ unsigned long long delayrange;
+ unsigned long r;
+
+ delayrange = td->o.start_delay_high - td->o.start_delay;
+
+ if (td->o.use_os_rand) {
+ r = os_random_long(&td->delay_state);
+ delayrange = (unsigned long long) ((double) delayrange * (r / (OS_RAND_MAX + 1.0)));
+ } else {
+ r = __rand(&td->__delay_state);
+ delayrange = (unsigned long long) ((double) delayrange * (r / (FRAND_MAX + 1.0)));
+ }
+
+ delayrange += td->o.start_delay;
+ return delayrange;
+}
+
/*
* Lazy way of fixing up options that depend on each other. We could also
* define option callback handlers, but this is easier.
@@ -496,6 +522,9 @@ static int fixup_options(struct thread_data *td)
if (!o->file_size_high)
o->file_size_high = o->file_size_low;
+ if (o->start_delay_high)
+ o->start_delay = get_rand_start_delay(td);
+
if (o->norandommap && o->verify != VERIFY_NONE
&& !fixed_block_size(o)) {
log_err("fio: norandommap given for variable block sizes, "
@@ -711,6 +740,7 @@ static void td_fill_rand_seeds_os(struct thread_data *td)
os_random_seed(td->rand_seeds[FIO_RAND_FILE_SIZE_OFF], &td->file_size_state);
os_random_seed(td->rand_seeds[FIO_RAND_TRIM_OFF], &td->trim_state);
+ os_random_seed(td->rand_seeds[FIO_RAND_START_DELAY], &td->delay_state);
if (!td_random(td))
return;
@@ -736,6 +766,7 @@ static void td_fill_rand_seeds_internal(struct thread_data *td)
init_rand_seed(&td->__file_size_state, td->rand_seeds[FIO_RAND_FILE_SIZE_OFF]);
init_rand_seed(&td->__trim_state, td->rand_seeds[FIO_RAND_TRIM_OFF]);
+ init_rand_seed(&td->__delay_state, td->rand_seeds[FIO_RAND_START_DELAY]);
if (!td_random(td))
return;
@@ -751,6 +782,12 @@ static void td_fill_rand_seeds_internal(struct thread_data *td)
void td_fill_rand_seeds(struct thread_data *td)
{
+ if (td->o.allrand_repeatable) {
+ for (int i = 0; i < FIO_RAND_NR_OFFS; i++)
+ td->rand_seeds[i] = FIO_RANDSEED * td->thread_number
+ + i;
+ }
+
if (td->o.use_os_rand)
td_fill_rand_seeds_os(td);
else
@@ -989,10 +1026,10 @@ static int add_job(struct thread_data *td, const char *jobname, int job_add_num,
file_alloced = 1;
if (o->nr_files == 1 && exists_and_not_file(jobname))
- add_file(td, jobname);
+ add_file(td, jobname, job_add_num);
else {
for (i = 0; i < o->nr_files; i++)
- add_file(td, make_filename(fname, o, jobname, td->thread_number, i));
+ add_file(td, make_filename(fname, o, jobname, job_add_num, i), job_add_num);
}
}
@@ -1118,15 +1155,24 @@ static int add_job(struct thread_data *td, const char *jobname, int job_add_num,
td_new->o.new_group = 0;
if (file_alloced) {
- td_new->o.filename = NULL;
td_new->files_index = 0;
td_new->files_size = 0;
- td_new->files = NULL;
+ if (td_new->files) {
+ struct fio_file *f;
+ for_each_file(td_new, f, i) {
+ if (f->file_name)
+ free(f->file_name);
+ free(f);
+ }
+ td_new->files = NULL;
+ }
+ if (td_new->o.filename) {
+ free(td_new->o.filename);
+ td_new->o.filename = NULL;
+ }
}
- job_add_num = numjobs - 1;
-
- if (add_job(td_new, jobname, job_add_num, 1, client_type))
+ if (add_job(td_new, jobname, numjobs, 1, client_type))
goto err;
}
@@ -1676,6 +1722,9 @@ int parse_cmd_line(int argc, char *argv[], int client_type)
else
output_format = FIO_OUTPUT_NORMAL;
break;
+ case 'f':
+ append_terse_output = 1;
+ break;
case 'h':
if (!cur_client) {
usage(argv[0]);
@@ -1985,6 +2034,7 @@ int parse_options(int argc, char *argv[])
free(ini_file);
fio_options_free(&def_thread);
+ filesetup_mem_free();
if (!thread_number) {
if (parse_dryrun())
diff --git a/iolog.c b/iolog.c
index 5fd9416..eeaca29 100644
--- a/iolog.c
+++ b/iolog.c
@@ -324,7 +324,7 @@ static int read_iolog2(struct thread_data *td, FILE *f)
rw = DDIR_INVAL;
if (!strcmp(act, "add")) {
td->o.nr_files++;
- fileno = add_file(td, fname);
+ fileno = add_file(td, fname, 0);
file_action = FIO_LOG_ADD_FILE;
continue;
} else if (!strcmp(act, "open")) {
diff --git a/iolog.h b/iolog.h
index 3ec48f2..0716391 100644
--- a/iolog.h
+++ b/iolog.h
@@ -117,6 +117,7 @@ extern void write_iolog_close(struct thread_data *);
/*
* Logging
*/
+extern void finalize_logs(struct thread_data *td);
extern void add_lat_sample(struct thread_data *, enum fio_ddir, unsigned long,
unsigned int);
extern void add_clat_sample(struct thread_data *, enum fio_ddir, unsigned long,
diff --git a/options.c b/options.c
index ea51664..87a4432 100644
--- a/options.c
+++ b/options.c
@@ -729,11 +729,11 @@ static int str_random_distribution_cb(void *data, const char *str)
}
/*
- * Return next file in the string. Files are separated with ':'. If the ':'
+ * Return next name in the string. Files are separated with ':'. If the ':'
* is escaped with a '\', then that ':' is part of the filename and does not
* indicate a new file.
*/
-static char *get_next_file_name(char **ptr)
+static char *get_next_name(char **ptr)
{
char *str = *ptr;
char *p, *start;
@@ -774,6 +774,43 @@ static char *get_next_file_name(char **ptr)
return start;
}
+
+static int get_max_name_idx(char *input)
+{
+ unsigned int cur_idx;
+ char *str, *p;
+
+ p = str = strdup(input);
+ for (cur_idx = 0; ; cur_idx++)
+ if (get_next_name(&str) == NULL)
+ break;
+
+ free(p);
+ return cur_idx;
+}
+
+/*
+ * Returns the directory at the index, indexes > entires will be
+ * assigned via modulo division of the index
+ */
+int set_name_idx(char *target, char *input, int index)
+{
+ unsigned int cur_idx;
+ int len;
+ char *fname, *str, *p;
+
+ p = str = strdup(input);
+
+ index %= get_max_name_idx(input);
+ for (cur_idx = 0; cur_idx <= index; cur_idx++)
+ fname = get_next_name(&str);
+
+ len = sprintf(target, "%s/", fname);
+ free(p);
+
+ return len;
+}
+
static int str_filename_cb(void *data, const char *input)
{
struct thread_data *td = data;
@@ -787,10 +824,10 @@ static int str_filename_cb(void *data, const char *input)
if (!td->files_index)
td->o.nr_files = 0;
- while ((fname = get_next_file_name(&str)) != NULL) {
+ while ((fname = get_next_name(&str)) != NULL) {
if (!strlen(fname))
break;
- add_file(td, fname);
+ add_file(td, fname, 0);
td->o.nr_files++;
}
@@ -798,27 +835,35 @@ static int str_filename_cb(void *data, const char *input)
return 0;
}
-static int str_directory_cb(void *data, const char fio_unused *str)
+static int str_directory_cb(void *data, const char fio_unused *unused)
{
struct thread_data *td = data;
struct stat sb;
+ char *dirname, *str, *p;
+ int ret = 0;
if (parse_dryrun())
return 0;
- if (lstat(td->o.directory, &sb) < 0) {
- int ret = errno;
+ p = str = strdup(td->o.directory);
+ while ((dirname = get_next_name(&str)) != NULL) {
+ if (lstat(dirname, &sb) < 0) {
+ ret = errno;
- log_err("fio: %s is not a directory\n", td->o.directory);
- td_verror(td, ret, "lstat");
- return 1;
- }
- if (!S_ISDIR(sb.st_mode)) {
- log_err("fio: %s is not a directory\n", td->o.directory);
- return 1;
+ log_err("fio: %s is not a directory\n", dirname);
+ td_verror(td, ret, "lstat");
+ goto out;
+ }
+ if (!S_ISDIR(sb.st_mode)) {
+ log_err("fio: %s is not a directory\n", dirname);
+ ret = 1;
+ goto out;
+ }
}
- return 0;
+out:
+ free(p);
+ return ret;
}
static int str_lockfile_cb(void *data, const char fio_unused *str)
@@ -1784,6 +1829,15 @@ struct fio_option fio_options[FIO_MAX_OPTS] = {
.group = FIO_OPT_G_RANDOM,
},
{
+ .name = "allrandrepeat",
+ .type = FIO_OPT_BOOL,
+ .off1 = td_var_offset(allrand_repeatable),
+ .help = "Use repeatable random numbers for everything",
+ .def = "0",
+ .category = FIO_OPT_C_IO,
+ .group = FIO_OPT_G_RANDOM,
+ },
+ {
.name = "nrfiles",
.lname = "Number of files",
.alias = "nr_files",
@@ -2011,6 +2065,7 @@ struct fio_option fio_options[FIO_MAX_OPTS] = {
.lname = "Start delay",
.type = FIO_OPT_STR_VAL_TIME,
.off1 = td_var_offset(start_delay),
+ .off2 = td_var_offset(start_delay_high),
.help = "Only start job when this period has passed",
.def = "0",
.category = FIO_OPT_C_GENERAL,
@@ -2190,6 +2245,10 @@ struct fio_option fio_options[FIO_MAX_OPTS] = {
.oval = VERIFY_SHA512,
.help = "Use sha512 checksums for verification",
},
+ { .ival = "xxhash",
+ .oval = VERIFY_XXHASH,
+ .help = "Use xxhash checksums for verification",
+ },
{ .ival = "meta",
.oval = VERIFY_META,
.help = "Use io information",
@@ -3016,7 +3075,7 @@ struct fio_option fio_options[FIO_MAX_OPTS] = {
.type = FIO_OPT_INT,
.off1 = td_var_offset(compress_percentage),
.maxval = 100,
- .minval = 1,
+ .minval = 0,
.help = "How compressible the buffer is (approximately)",
.interval = 5,
.category = FIO_OPT_C_IO,
diff --git a/options.h b/options.h
index 3dc48a9..de9f610 100644
--- a/options.h
+++ b/options.h
@@ -17,6 +17,8 @@ void add_opt_posval(const char *, const char *, const char *);
void del_opt_posval(const char *, const char *);
struct thread_data;
void fio_options_free(struct thread_data *);
+char *get_name_idx(char *, int);
+int set_name_idx(char *, char *, int);
extern struct fio_option fio_options[FIO_MAX_OPTS];
diff --git a/parse.c b/parse.c
index e46fc14..c8bae03 100644
--- a/parse.c
+++ b/parse.c
@@ -122,21 +122,41 @@ static void show_option_help(struct fio_option *o, int is_err)
show_option_values(o);
}
-static unsigned long get_mult_time(char c)
+static unsigned long long get_mult_time(const char *str, int len)
{
- switch (c) {
- case 'm':
- case 'M':
- return 60;
- case 'h':
- case 'H':
- return 60 * 60;
- case 'd':
- case 'D':
- return 24 * 60 * 60;
- default:
- return 1;
+ const char *p = str;
+ char *c;
+ unsigned long long mult = 1000;
+
+ /*
+ * Go forward until we hit a non-digit, or +/- sign
+ */
+ while ((p - str) <= len) {
+ if (!isdigit((int) *p) && (*p != '+') && (*p != '-'))
+ break;
+ p++;
}
+
+ if (!isalpha((int) *p))
+ return 1000;
+
+ c = strdup(p);
+ for (int i = 0; i < strlen(c); i++)
+ c[i] = tolower(c[i]);
+
+ if (!strncmp("ms", c, 2))
+ mult = 1;
+ else if (!strcmp("s", c))
+ mult = 1000;
+ else if (!strcmp("m", c))
+ mult = 60 * 1000;
+ else if (!strcmp("h", c))
+ mult = 60 * 60 * 1000;
+ else if (!strcmp("d", c))
+ mult = 24 * 60 * 60 * 1000;
+
+ free(c);
+ return mult;
}
static int is_separator(char c)
@@ -275,7 +295,7 @@ int str_to_decimal(const char *str, long long *val, int kilo, void *data)
else
*val *= mult;
} else
- *val *= get_mult_time(str[len - 1]);
+ *val *= get_mult_time(str, len);
return 0;
}
diff --git a/server.h b/server.h
index adb190a..85d9d7e 100644
--- a/server.h
+++ b/server.h
@@ -38,7 +38,7 @@ struct fio_net_cmd_reply {
};
enum {
- FIO_SERVER_VER = 30,
+ FIO_SERVER_VER = 31,
FIO_SERVER_MAX_FRAGMENT_PDU = 1024,
diff --git a/stat.c b/stat.c
index 5d50ae8..bc01b51 100644
--- a/stat.c
+++ b/stat.c
@@ -1386,6 +1386,16 @@ static void __show_run_stats(void)
show_idle_prof_stats(FIO_OUTPUT_NORMAL, NULL);
}
+ if ( !(output_format == FIO_OUTPUT_TERSE) && append_terse_output) {
+ log_info("\nAdditional Terse Output:\n");
+
+ for (i = 0; i < nr_ts; i++) {
+ ts = &threadstats[i];
+ rs = &runstats[ts->groupid];
+ show_thread_status_terse(ts, rs);
+ }
+ }
+
log_info_flush();
free(runstats);
free(threadstats);
@@ -1569,6 +1579,37 @@ static inline void reset_io_stat(struct io_stat *ios)
ios->mean.u.f = ios->S.u.f = 0;
}
+static void _add_stat_to_log(struct io_log *iolog, unsigned long elapsed)
+{
+ /*
+ * Note an entry in the log. Use the mean from the logged samples,
+ * making sure to properly round up. Only write a log entry if we
+ * had actual samples done.
+ */
+ if (iolog->avg_window[DDIR_READ].samples) {
+ unsigned long mr;
+
+ mr = iolog->avg_window[DDIR_READ].mean.u.f + 0.50;
+ __add_log_sample(iolog, mr, DDIR_READ, 0, elapsed);
+ }
+ if (iolog->avg_window[DDIR_WRITE].samples) {
+ unsigned long mw;
+
+ mw = iolog->avg_window[DDIR_WRITE].mean.u.f + 0.50;
+ __add_log_sample(iolog, mw, DDIR_WRITE, 0, elapsed);
+ }
+ if (iolog->avg_window[DDIR_TRIM].samples) {
+ unsigned long mw;
+
+ mw = iolog->avg_window[DDIR_TRIM].mean.u.f + 0.50;
+ __add_log_sample(iolog, mw, DDIR_TRIM, 0, elapsed);
+ }
+
+ reset_io_stat(&iolog->avg_window[DDIR_READ]);
+ reset_io_stat(&iolog->avg_window[DDIR_WRITE]);
+ reset_io_stat(&iolog->avg_window[DDIR_TRIM]);
+}
+
static void add_log_sample(struct thread_data *td, struct io_log *iolog,
unsigned long val, enum fio_ddir ddir,
unsigned int bs)
@@ -1602,35 +1643,27 @@ static void add_log_sample(struct thread_data *td, struct io_log *iolog,
if (this_window < iolog->avg_msec)
return;
- /*
- * Note an entry in the log. Use the mean from the logged samples,
- * making sure to properly round up. Only write a log entry if we
- * had actual samples done.
- */
- if (iolog->avg_window[DDIR_READ].samples) {
- unsigned long mr;
+ _add_stat_to_log(iolog, elapsed);
- mr = iolog->avg_window[DDIR_READ].mean.u.f + 0.50;
- __add_log_sample(iolog, mr, DDIR_READ, 0, elapsed);
- }
- if (iolog->avg_window[DDIR_WRITE].samples) {
- unsigned long mw;
-
- mw = iolog->avg_window[DDIR_WRITE].mean.u.f + 0.50;
- __add_log_sample(iolog, mw, DDIR_WRITE, 0, elapsed);
- }
- if (iolog->avg_window[DDIR_TRIM].samples) {
- unsigned long mw;
+ iolog->avg_last = elapsed;
+}
- mw = iolog->avg_window[DDIR_TRIM].mean.u.f + 0.50;
- __add_log_sample(iolog, mw, DDIR_TRIM, 0, elapsed);
- }
+void finalize_logs(struct thread_data *td)
+{
+ unsigned long elapsed;
+ elapsed = mtime_since_now(&td->epoch);
- reset_io_stat(&iolog->avg_window[DDIR_READ]);
- reset_io_stat(&iolog->avg_window[DDIR_WRITE]);
- reset_io_stat(&iolog->avg_window[DDIR_TRIM]);
- iolog->avg_last = elapsed;
+ if (td->clat_log)
+ _add_stat_to_log(td->clat_log, elapsed);
+ if (td->slat_log)
+ _add_stat_to_log(td->slat_log, elapsed);
+ if (td->lat_log)
+ _add_stat_to_log(td->lat_log, elapsed);
+ if (td->bw_log)
+ _add_stat_to_log(td->bw_log, elapsed);
+ if (td->iops_log)
+ _add_stat_to_log(td->iops_log, elapsed);
}
void add_agg_sample(unsigned long val, enum fio_ddir ddir, unsigned int bs)
diff --git a/thread_options.h b/thread_options.h
index 79c3a89..b7a88ed 100644
--- a/thread_options.h
+++ b/thread_options.h
@@ -100,6 +100,7 @@ struct thread_options {
unsigned int do_disk_util;
unsigned int override_sync;
unsigned int rand_repeatable;
+ unsigned int allrand_repeatable;
unsigned long long rand_seed;
unsigned int use_os_rand;
unsigned int log_avg_msec;
@@ -129,6 +130,7 @@ struct thread_options {
unsigned int fdatasync_blocks;
unsigned int barrier_blocks;
unsigned long long start_delay;
+ unsigned long long start_delay_high;
unsigned long long timeout;
unsigned long long ramp_time;
unsigned int overwrite;
@@ -324,6 +326,7 @@ struct thread_options_pack {
uint32_t do_disk_util;
uint32_t override_sync;
uint32_t rand_repeatable;
+ uint32_t allrand_repeatable;
uint64_t rand_seed;
uint32_t use_os_rand;
uint32_t log_avg_msec;
@@ -350,6 +353,7 @@ struct thread_options_pack {
uint32_t fdatasync_blocks;
uint32_t barrier_blocks;
uint64_t start_delay;
+ uint64_t start_delay_high;
uint64_t timeout;
uint64_t ramp_time;
uint32_t overwrite;
diff --git a/time.c b/time.c
index 17f9f6f..b374d3e 100644
--- a/time.c
+++ b/time.c
@@ -71,7 +71,7 @@ int ramp_time_over(struct thread_data *td)
return 1;
fio_gettime(&tv, NULL);
- if (mtime_since(&td->epoch, &tv) >= td->o.ramp_time * 1000) {
+ if (mtime_since(&td->epoch, &tv) >= td->o.ramp_time ) {
td->ramp_time_over = 1;
reset_all_stats(td);
td_set_runstate(td, TD_RAMP);
diff --git a/verify.c b/verify.c
index 9373122..9eb532a 100644
--- a/verify.c
+++ b/verify.c
@@ -23,6 +23,7 @@
#include "crc/sha256.h"
#include "crc/sha512.h"
#include "crc/sha1.h"
+#include "crc/xxhash.h"
static void populate_hdr(struct thread_data *td, struct io_u *io_u,
struct verify_header *hdr, unsigned int header_num,
@@ -172,6 +173,9 @@ static inline unsigned int __hdr_size(int verify_type)
case VERIFY_SHA512:
len = sizeof(struct vhdr_sha512);
break;
+ case VERIFY_XXHASH:
+ len = sizeof(struct vhdr_xxhash);
+ break;
case VERIFY_META:
len = sizeof(struct vhdr_meta);
break;
@@ -404,6 +408,30 @@ static int verify_io_u_meta(struct verify_header *hdr, struct vcont *vc)
return ret;
}
+static int verify_io_u_xxhash(struct verify_header *hdr, struct vcont *vc)
+{
+ void *p = io_u_verify_off(hdr, vc);
+ struct vhdr_xxhash *vh = hdr_priv(hdr);
+ uint32_t hash;
+ void *state;
+
+ dprint(FD_VERIFY, "xxhash verify io_u %p, len %u\n", vc->io_u, hdr->len);
+
+ state = XXH32_init(1);
+ XXH32_update(state, p, hdr->len - hdr_size(hdr));
+ hash = XXH32_digest(state);
+
+ if (vh->hash == hash)
+ return 0;
+
+ vc->name = "xxhash";
+ vc->good_crc = &vh->hash;
+ vc->bad_crc = &hash;
+ vc->crc_len = sizeof(hash);
+ log_verify_failure(hdr, vc);
+ return EILSEQ;
+}
+
static int verify_io_u_sha512(struct verify_header *hdr, struct vcont *vc)
{
void *p = io_u_verify_off(hdr, vc);
@@ -793,6 +821,9 @@ int verify_io_u(struct thread_data *td, struct io_u *io_u)
case VERIFY_SHA512:
ret = verify_io_u_sha512(hdr, &vc);
break;
+ case VERIFY_XXHASH:
+ ret = verify_io_u_xxhash(hdr, &vc);
+ break;
case VERIFY_META:
ret = verify_io_u_meta(hdr, &vc);
break;
@@ -834,6 +865,16 @@ static void fill_meta(struct verify_header *hdr, struct thread_data *td,
vh->offset = io_u->offset + header_num * td->o.verify_interval;
}
+static void fill_xxhash(struct verify_header *hdr, void *p, unsigned int len)
+{
+ struct vhdr_xxhash *vh = hdr_priv(hdr);
+ void *state;
+
+ state = XXH32_init(1);
+ XXH32_update(state, p, len);
+ vh->hash = XXH32_digest(state);
+}
+
static void fill_sha512(struct verify_header *hdr, void *p, unsigned int len)
{
struct vhdr_sha512 *vh = hdr_priv(hdr);
@@ -973,6 +1014,11 @@ static void populate_hdr(struct thread_data *td, struct io_u *io_u,
io_u, hdr->len);
fill_sha512(hdr, data, data_len);
break;
+ case VERIFY_XXHASH:
+ dprint(FD_VERIFY, "fill xxhash io_u %p, len %u\n",
+ io_u, hdr->len);
+ fill_xxhash(hdr, data, data_len);
+ break;
case VERIFY_META:
dprint(FD_VERIFY, "fill meta io_u %p, len %u\n",
io_u, hdr->len);
diff --git a/verify.h b/verify.h
index 05d7b81..dba7743 100644
--- a/verify.h
+++ b/verify.h
@@ -16,6 +16,7 @@ enum {
VERIFY_CRC7, /* crc7 sum data blocks */
VERIFY_SHA256, /* sha256 sum data blocks */
VERIFY_SHA512, /* sha512 sum data blocks */
+ VERIFY_XXHASH, /* xxhash sum data blocks */
VERIFY_META, /* block_num, timestamp etc. */
VERIFY_SHA1, /* sha1 sum data blocks */
VERIFY_PATTERN, /* verify specific patterns */
@@ -66,6 +67,9 @@ struct vhdr_meta {
unsigned long time_sec;
unsigned long time_usec;
};
+struct vhdr_xxhash {
+ uint32_t hash;
+};
/*
* Verify helpers
--
To unsubscribe from this list: send the line "unsubscribe fio" in
the body of a message to majordomo@xxxxxxxxxxxxxxx
More majordomo info at http://vger.kernel.org/majordomo-info.html
