Hello, This is a RFC patch set that adds LZO compression support to makedumpfile and crash utility. LZO is as good as in size but by far better in speed than ZLIB, leading to reducing down time during generation of crash dump and refiltering. How to build: 1. Get LZO library, which is provided as lzo-devel package on recent linux distributions, and is also available on author's website: http://www.oberhumer.com/opensource/lzo/. 2. Apply the patch set to makedumpfile v1.4.0 and crash v6.0.0. 3. Build both using make. But for crash, do the following now: $ make CFLAGS="-llzo2" How to use: I've newly used -l option for lzo compression in this patch. So for example, do as follows: $ makedumpfile -l vmcore dumpfile $ crash vmlinux dumpfile Request of configure-like feature for crash utility: I would like configure-like feature on crash utility for users to select wheather to add LZO feature actually or not in build-time, that is: ./configure --enable-lzo or ./configure --disable-lzo. The reason is that support staff often downloads and installs the latest version of crash utility on machines where lzo library is not provided. Looking at the source code, it looks to me that crash does some kind of configuration processing in a local manner, around configure.c, and I guess it's difficult to use autoconf tools directly. Or is there another better way? Performance Comparison: Sample Data Ideally, I must have measured the performance for many enough vmcores generated from machines that was actually running, but now I don't have enough sample vmcores, I couldn't do so. So this comparison doesn't answer question on I/O time improvement. This is TODO for now. Instead, I choosed worst and best cases regarding compression ratio and speed only. Specifically, the former is /dev/urandom and the latter is /dev/zero. I get the sample data of 10MB, 100MB and 1GB by doing like this: $ dd bs=4096 count=$((1024*1024*1024/4096)) if=/dev/urandom of=urandom.1GB How to measure Then I performed compression for each block, 4096 bytes, and measured total compression time and output size. See attached mycompress.c. Result See attached file result.txt. Discussion For both kinds of data, lzo's compression was considerably quicker than zlib's. Compression ratio is about 37% for urandom data, and about 8.5% for zero data. Actual situation of physical memory would be in between the two cases, and so I guess average compression time ratio is between 37% and 8.5%. Although beyond the topic of this patch set, we can estimate worst compression time on more data size since compression is performed block size wise and the compression time increases linearly. Estimated worst time on 2TB memory is about 15 hours for lzo and about 40 hours for zlib. In this case, compressed data size is larger than the original, so they are really not used, compression time is fully meaningless. I think compression must be done in parallel, and I'll post such patch later. Diffstat * makedumpfile diskdump_mod.h | 3 +- makedumpfile.c | 98 +++++++++++++++++++++++++++++++++++++++++++++++++------ makedumpfile.h | 12 +++++++ 3 files changed, 101 insertions(+), 12 deletions(-) * crash defs.h | 1 + diskdump.c | 20 +++++++++++++++++++- diskdump.h | 3 ++- 3 files changed, 22 insertions(+), 2 deletions(-) TODO * evaluation including I/O time using actual vmcores Thanks. HATAYAMA, Daisuke
/*
* How to build:
*
* $ gcc -lz -llzo2 ./mycompress.c -o mycompress
*
*/
#include <stdio.h>
#include <stdlib.h>
#include <sys/time.h>
#include <zlib.h>
#include <lzo/lzo1x.h>
enum
{
BLOCK_SIZE = 4096
};
enum compression
{
GLIB = 0,
LZO
};
static inline size_t max(size_t a, size_t b)
{
return a >= b ? a : b;
}
static inline size_t compressBoundLZO(size_t size)
{
return size + size / 16 + 64 + 3;
}
static inline double getdtime(void)
{
struct timeval tv;
gettimeofday(&tv, NULL);
return (double)tv.tv_sec + (double)tv.tv_usec * 0.001 * 0.001;
}
static void print_usage(void)
{
printf("usage: mycompress [glib|lzo] <input file> <output file>\n");
}
int main(int argc, char **argv)
{
enum compression flag_compress;
FILE *fi = NULL, *fo = NULL;
char *bufout = NULL;
lzo_bytep wrkmem = NULL;
unsigned long size_out, len_bufout;
double t_compress_start, t_compress_end, t_compress_total;
double t_io_start, t_io_end, t_io_total;
size_t total_input_bytes, total_output_bytes;
if (argc != 4) {
print_usage();
return EXIT_FAILURE;
}
if (strncmp(argv[1], "glib", 4) == 0) {
flag_compress = GLIB;
}
else if (strncmp(argv[1], "lzo", 3) == 0) {
if (lzo_init() != LZO_E_OK) {
fprintf(stderr, "lzo_init() failed\n");
return EXIT_FAILURE;
}
flag_compress = LZO;
}
else {
return EXIT_FAILURE;
}
fi = fopen(argv[2], "r");
if (!fi) {
perror("fopen");
goto error;
}
fo = fopen(argv[3], "w");
if (!fo) {
perror("fopen");
goto error;
}
switch (flag_compress) {
case GLIB:
len_bufout = compressBound(BLOCK_SIZE);
break;
case LZO:
len_bufout = compressBoundLZO(BLOCK_SIZE);
break;
}
bufout = malloc(len_bufout);
if (!bufout) {
perror("malloc");
goto error;
}
wrkmem = malloc(LZO1X_1_MEM_COMPRESS);
if (!wrkmem) {
perror("malloc");
goto error;
}
t_compress_start = t_compress_end = t_compress_total = 0;
t_io_start = t_io_end = t_io_total = 0;
total_input_bytes = total_output_bytes = 0;
while (!ferror(fi)) {
char buf[BLOCK_SIZE];
unsigned long size_out;
total_input_bytes += BLOCK_SIZE;
if (fread(buf, sizeof(buf), 1, fi) != 1) {
if (feof(fi))
break;
perror("fread");
goto error;
}
/*
* For size_out, GLIB needs output buffer size, while
* LZO original data size.
*/
switch (flag_compress) {
case GLIB: size_out = len_bufout; break;
case LZO: size_out = BLOCK_SIZE; break;
}
t_compress_start = getdtime();
switch (flag_compress) {
case GLIB:
if (compress2(bufout, &size_out, buf, BLOCK_SIZE,
Z_BEST_SPEED) != Z_OK) {
fprintf(stderr, "glib compression failed\n");
goto error;
}
break;
case LZO:
if (lzo1x_1_compress(buf, BLOCK_SIZE, bufout,
&size_out, wrkmem) != LZO_E_OK) {
fprintf(stderr, "lzo compression failed\n");
goto error;
}
break;
}
t_compress_end = getdtime();
t_compress_total += t_compress_end - t_compress_start;
t_io_start = getdtime();
if (fwrite(bufout, size_out, 1, fo) != 1) {
perror("fwrite");
goto error;
}
t_io_end = getdtime();
t_io_total += t_io_end - t_io_start;
total_output_bytes += size_out;
}
printf("Input size: %d bytes\n", total_input_bytes);
printf("Compression Time: %lf seconds\n", t_compress_total);
printf("Output size: %d bytes\n", total_output_bytes);
printf("IO Time: %lf seconds\n", t_io_total);
error:
if (fi)
fclose(fi);
if (fo)
fclose(fo);
free(wrkmem);
return 0;
}
* /dev/urandom
lzo zlib compression time ratio(%)
10MBytes
compression time(sec) 0.271553 0.721236 37.65106012
output size(bytes) 10537086 10513920
compression size ratio(%) 100.4502445 100.2294026
100MBytes
compression time(sec) 2.671278 7.229928 36.94750487
output size(bytes) 105370774 105139200
compression size ratio(%) 100.4854756 100.2646381
1GBytes
compression time(sec) 27.627126 74.94164 36.86485377
output size(bytes) 1078997064 1076625408
compression size ratio(%) 100.489049 100.2681722
* /dev/zero
lzo zlib compression time ratio(%)
10MBytes
compression time(sec) 0.020834 0.238794 8.724674824
output size(bytes) 71680 104960
compression size ratio(%) 0.683326825 1.000585709
100MBytes
compression time(sec) 0.208346 2.441701 8.532821996
output size(bytes) 716800 1049600
compression size ratio(%) 0.683567048 1.000937463
1GBytes
compression time(sec) 2.123407 26.047801 8.151962617
output size(bytes) 7340032 10747904
compression size ratio(%) 0.683591142 1.000972744
Attachment:
0001-Add-LZO-Support.patch
Description: Binary data
Attachment:
0001-Support-LZO-compression-format.patch
Description: Binary data
Attachment:
0002-Add-Compression-and-IO-time-report.patch
Description: Binary data
-- Crash-utility mailing list Crash-utility@xxxxxxxxxx https://www.redhat.com/mailman/listinfo/crash-utility
