On 2024/03/28 17:15, Yulong TANG 汤玉龙 wrote:
> In Linux 5.1, the ZRAM block driver has changed its default compressor
> from "lzo" to "lzo-rle" to enhance LZO compression support. However,
> crash does not support the improved LZO algorithm, resulting in failure
> when reading memory.
>
> change default compressor : ce82f19fd5809f0cf87ea9f753c5cc65ca0673d6
>
> The issue was discovered when using the extension 'gcore' to generate a
> process coredump, which was found to be incomplete and unable to be
> opened properly with gdb.
> This patch is for Crash-utility tool, it enables the Crash-utility to
> support decompression of the "lzo-rle" compression algorithm used in
> zram. The patch has been tested with vmcore files from kernel version
> 5.4, and successfully allows reading of memory compressed with the zram
> compression algorithm.
>
> Testing:
> ========
>
> before apply this patch :
> crash> gcore -v 0 1
> gcore: WARNING: only the lzo compressor is supported
> gcore: WARNING: only the lzo compressor is supported
> gcore: WARNING: only the lzo compressor is supported
> after:
> crash> gcore -v 0
> 1 Saved core.1.init
>
> Changelog:
> ==========
> v2: keep the "if defined(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS)"
> related code of the copied kernel code, but change the "if defined"
> macro into a runtime check .
> v3: set a default value of HAVE_EFFICIENT_UNALIGNED_ACCESS depending on
> architecture, for no ikconfig kernels.
> v4: avoid checking CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS every
> call；move "include lzorle_decompress.h" to diskdump.c from def.h
thank you for the update.
The patch tested OK with lzo-rle zram, but I found a small issue and
redundant codes, and fixed them. Could you check the attached patch again?
* "make ctags" didn't work for lzorle_decompress.h, so added it to
SOURCE_FILES.
* There was no need to include stdint.h and string.h in
lzorle_decompress.c, so removed them.
* There are the same code blocks in "if (efficient_unligned_access)"
block and else block. Isn't this ok?
if (efficient_unaligned_access &&
(HAVE_IP(t + 15) && HAVE_OP(t + 15))) {
* Why are NEED_IP() and etc. new-lined? Replaced them with one lines
like the original kernel code.
#define NEED_IP(x) if (!HAVE_IP(x)) goto input_overrun
#define NEED_OP(x) if (!HAVE_OP(x)) goto output_overrun
#define TEST_LB(m_pos) if ((m_pos) < out) goto lookbehind_overrun
Thanks,
Kazu
>
> Patch:
> ==========
>
> See attachment.
>
>
>
> Thanks and regards,
> Yulong
>
>
> --
> Crash-utility mailing list -- devel@xxxxxxxxxxxxxxxxxxxxxxxxxxx
> To unsubscribe send an email to devel-leave@xxxxxxxxxxxxxxxxxxxxxxxxxxx
> https://${domain_name}/admin/lists/devel.lists.crash-utility.osci.io/
> Contribution Guidelines: https://github.com/crash-utility/crash/wikiFrom 27863d74a0b79f96ff776fac5c925b7eefd3f63c Mon Sep 17 00:00:00 2001
From: "yulong.tang" <yulong.tang@xxxxxxx>
Date: Tue, 20 Feb 2024 15:09:49 +0800
Subject: [PATCH] Adding the zram decompression algorithm "lzo-rle"
Port the improved decompression method for "lzo" in the kernel to
support decompression of "lzorle".
Since Linux 5.1, the default compression algorithm for zram was changed
from "lzo" to "lzo-rle". The crash-utility only supports decompression
for "lzo", when parsing vmcore files that utilize zram compression, such
as when using the gcore command to detach process core dump files,
parsing cannot be completed successfully.
before:
crash> gcore -v 0 1
gcore: WARNING: only the lzo compressor is supported
gcore: WARNING: only the lzo compressor is supported
gcore: WARNING: only the lzo compressor is supported
gcore: WARNING: only the lzo compressor is supported
after:
crash> gcore -v 0 1
Saved core.1.init
Signed-off-by: yulong.tang <yulong.tang@xxxxxxx>
Reviewed-by: Tao Liu <ltao@xxxxxxxxxx>
Signed-off-by: Kazuhito Hagio <k-hagio-ab@xxxxxxx>
---
Makefile | 13 +-
diskdump.c | 3 +
lzorle_decompress.c | 297 ++++++++++++++++++++++++++++++++++++++++++++
lzorle_decompress.h | 75 +++++++++++
4 files changed, 385 insertions(+), 3 deletions(-)
create mode 100644 lzorle_decompress.c
create mode 100644 lzorle_decompress.h
diff --git a/Makefile b/Makefile
index 9e9731324e43..60dad1870a92 100644
--- a/Makefile
+++ b/Makefile
@@ -60,6 +60,7 @@ SADUMP_HFILES=sadump.h
UNWIND_HFILES=unwind.h unwind_i.h rse.h unwind_x86.h unwind_x86_64.h
VMWARE_HFILES=vmware_vmss.h
MAPLE_TREE_HFILES=maple_tree.h
+LZORLE_HFILES=lzorle_decompress.h
CFILES=main.c tools.c global_data.c memory.c filesys.c help.c task.c \
kernel.c test.c gdb_interface.c configure.c net.c dev.c bpf.c \
@@ -74,12 +75,14 @@ CFILES=main.c tools.c global_data.c memory.c filesys.c help.c task.c \
xen_hyper.c xen_hyper_command.c xen_hyper_global_data.c \
xen_hyper_dump_tables.c kvmdump.c qemu.c qemu-load.c sadump.c ipcs.c \
ramdump.c vmware_vmss.c vmware_guestdump.c \
- xen_dom0.c kaslr_helper.c sbitmap.c maple_tree.c
+ xen_dom0.c kaslr_helper.c sbitmap.c maple_tree.c \
+ lzorle_decompress.c
SOURCE_FILES=${CFILES} ${GENERIC_HFILES} ${MCORE_HFILES} \
${REDHAT_CFILES} ${REDHAT_HFILES} ${UNWIND_HFILES} \
${LKCD_DUMP_HFILES} ${LKCD_TRACE_HFILES} ${LKCD_OBSOLETE_HFILES}\
- ${IBM_HFILES} ${SADUMP_HFILES} ${VMWARE_HFILES} ${MAPLE_TREE_HFILES}
+ ${IBM_HFILES} ${SADUMP_HFILES} ${VMWARE_HFILES} ${MAPLE_TREE_HFILES} \
+ ${LZORLE_HFILES}
OBJECT_FILES=main.o tools.o global_data.o memory.o filesys.o help.o task.o \
build_data.o kernel.o test.o gdb_interface.o net.o dev.o bpf.o \
@@ -94,7 +97,8 @@ OBJECT_FILES=main.o tools.o global_data.o memory.o filesys.o help.o task.o \
xen_hyper.o xen_hyper_command.o xen_hyper_global_data.o \
xen_hyper_dump_tables.o kvmdump.o qemu.o qemu-load.o sadump.o ipcs.o \
ramdump.o vmware_vmss.o vmware_guestdump.o \
- xen_dom0.o kaslr_helper.o sbitmap.o maple_tree.o
+ xen_dom0.o kaslr_helper.o sbitmap.o maple_tree.o \
+ lzorle_decompress.o
MEMORY_DRIVER_FILES=memory_driver/Makefile memory_driver/crash.c memory_driver/README
@@ -546,6 +550,9 @@ bpf.o: ${GENERIC_HFILES} bpf.c
maple_tree.o: ${GENERIC_HFILES} ${MAPLE_TREE_HFILES} maple_tree.c
${CC} -c ${CRASH_CFLAGS} maple_tree.c ${WARNING_OPTIONS} ${WARNING_ERROR}
+lzorle_decompress.o: lzorle_decompress.c
+ ${CC} -c ${CRASH_CFLAGS} lzorle_decompress.c ${WARNING_OPTIONS} ${WARNING_ERROR}
+
${PROGRAM}: force
@$(MAKE) all
diff --git a/diskdump.c b/diskdump.c
index 3ae7bf2da541..4a473e19b9f5 100644
--- a/diskdump.c
+++ b/diskdump.c
@@ -28,6 +28,7 @@
#include "xen_dom0.h"
#include "vmcore.h"
#include "maple_tree.h"
+#include "lzorle_decompress.h"
#define BITMAP_SECT_LEN 4096
@@ -3069,6 +3070,8 @@ try_zram_decompress(ulonglong pte_val, unsigned char *buf, ulong len, ulonglong
" with lzo library\n");
return 0;
#endif
+ } else if (STREQ(name, "lzo-rle")) {
+ decompressor = (void *)&lzorle_decompress_safe;
} else { /* todo: support more compressor */
error(WARNING, "only the lzo compressor is supported\n");
return 0;
diff --git a/lzorle_decompress.c b/lzorle_decompress.c
new file mode 100644
index 000000000000..9e3612145905
--- /dev/null
+++ b/lzorle_decompress.c
@@ -0,0 +1,297 @@
+/* lzorle_decompress.h
+ *
+ * from kernel lib/lzo/lzo1x_decompress_safe.c
+ *
+ * Copyright (C) 1996-2012 Markus F.X.J. Oberhumer <markus@xxxxxxxxxxxxx>
+ * Copyright (C) 2024 NIO
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ */
+
+#include "defs.h"
+#include "lzorle_decompress.h"
+
+/* This MAX_255_COUNT is the maximum number of times we can add 255 to a base
+ * count without overflowing an integer. The multiply will overflow when
+ * multiplying 255 by more than MAXINT/255. The sum will overflow earlier
+ * depending on the base count. Since the base count is taken from a u8
+ * and a few bits, it is safe to assume that it will always be lower than
+ * or equal to 2*255, thus we can always prevent any overflow by accepting
+ * two less 255 steps. See Documentation/lzo.txt for more information.
+ */
+#define MAX_255_COUNT ((((ulong)~0) / 255) - 2)
+
+static uint16_t get_unaligned_le16(const void *p) {
+ uint16_t value;
+ memcpy(&value, p, sizeof(uint16_t));
+ return value;
+}
+
+int lzorle_decompress_safe(const unsigned char *in, ulong in_len,
+ unsigned char *out, ulong *out_len, void *other/* NOT USED */) {
+ unsigned char *op;
+ const unsigned char *ip;
+ ulong t, next;
+ ulong state = 0;
+ const unsigned char *m_pos;
+ const unsigned char * const ip_end = in + in_len;
+ unsigned char * const op_end = out + *out_len;
+
+ unsigned char bitstream_version;
+
+ static int efficient_unaligned_access = -1;
+
+ if (efficient_unaligned_access == -1) {
+#if defined(ARM) || defined(ARM64) || defined(X86) || defined(X86_64) || defined(PPC) || defined(PPC64) || defined(S390)|| defined(S390X)
+ efficient_unaligned_access = TRUE;
+#else
+ efficient_unaligned_access = FALSE;
+#endif
+
+ if ((kt->ikconfig_flags & IKCONFIG_AVAIL) &&
+ (get_kernel_config("CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS", NULL) == IKCONFIG_Y))
+ efficient_unaligned_access = TRUE;
+ }
+
+ op = out;
+ ip = in;
+
+ if (in_len < 3)
+ goto input_overrun;
+
+ if (in_len >= 5 && *ip == 17) {
+ bitstream_version = ip[1];
+ ip += 2;
+ } else {
+ bitstream_version = 0;
+ }
+
+ if (*ip > 17) {
+ t = *ip++ - 17;
+ if (t < 4) {
+ next = t;
+ goto match_next;
+ }
+ goto copy_literal_run;
+ }
+
+ for (;;) {
+ t = *ip++;
+ if (t < 16) {
+ if (state == 0) {
+ if (t == 0) {
+ ulong offset;
+ const unsigned char *ip_last = ip;
+
+ while (*ip == 0) {
+ ip++;
+ NEED_IP(1);
+ }
+ offset = ip - ip_last;
+ if (offset > MAX_255_COUNT)
+ return LZO_E_ERROR;
+
+ offset = (offset << 8) - offset;
+ t += offset + 15 + *ip++;
+ }
+ t += 3;
+copy_literal_run:
+ if (efficient_unaligned_access &&
+ (HAVE_IP(t + 15) && HAVE_OP(t + 15))) {
+ const unsigned char *ie = ip + t;
+ unsigned char *oe = op + t;
+ do {
+ COPY8(op, ip);
+ op += 8;
+ ip += 8;
+ COPY8(op, ip);
+ op += 8;
+ ip += 8;
+ } while (ip < ie);
+ ip = ie;
+ op = oe;
+ } else {
+ NEED_OP(t);
+ NEED_IP(t + 3);
+ do {
+ *op++ = *ip++;
+ } while (--t > 0);
+ }
+ state = 4;
+ continue;
+ } else if (state != 4) {
+ next = t & 3;
+ m_pos = op - 1;
+ m_pos -= t >> 2;
+ m_pos -= *ip++ << 2;
+ TEST_LB(m_pos);
+ NEED_OP(2);
+ op[0] = m_pos[0];
+ op[1] = m_pos[1];
+ op += 2;
+ goto match_next;
+ } else {
+ next = t & 3;
+ m_pos = op - (1 + M2_MAX_OFFSET);
+ m_pos -= t >> 2;
+ m_pos -= *ip++ << 2;
+ t = 3;
+ }
+ } else if (t >= 64) {
+ next = t & 3;
+ m_pos = op - 1;
+ m_pos -= (t >> 2) & 7;
+ m_pos -= *ip++ << 3;
+ t = (t >> 5) - 1 + (3 - 1);
+ } else if (t >= 32) {
+ t = (t & 31) + (3 - 1);
+ if (t == 2) {
+ ulong offset;
+ const unsigned char *ip_last = ip;
+
+ while (*ip == 0) {
+ ip++;
+ NEED_IP(1);
+ }
+ offset = ip - ip_last;
+ if (offset > MAX_255_COUNT)
+ return LZO_E_ERROR;
+
+ offset = (offset << 8) - offset;
+ t += offset + 31 + *ip++;
+ NEED_IP(2);
+ }
+ m_pos = op - 1;
+
+ next = get_unaligned_le16(ip);
+ ip += 2;
+ m_pos -= next >> 2;
+ next &= 3;
+ } else {
+ NEED_IP(2);
+ next = get_unaligned_le16(ip);
+ if (((next & 0xfffc) == 0xfffc) &&
+ ((t & 0xf8) == 0x18) &&
+ bitstream_version) {
+ NEED_IP(3);
+ t &= 7;
+ t |= ip[2] << 3;
+ t += MIN_ZERO_RUN_LENGTH;
+ NEED_OP(t);
+ memset(op, 0, t);
+ op += t;
+ next &= 3;
+ ip += 3;
+ goto match_next;
+ } else {
+ m_pos = op;
+ m_pos -= (t & 8) << 11;
+ t = (t & 7) + (3 - 1);
+ if (t == 2) {
+ ulong offset;
+ const unsigned char *ip_last = ip;
+
+ while (*ip == 0) {
+ ip++;
+ NEED_IP(1);
+ }
+ offset = ip - ip_last;
+ if (offset > MAX_255_COUNT)
+ return LZO_E_ERROR;
+
+ offset = (offset << 8) - offset;
+ t += offset + 7 + *ip++;
+ NEED_IP(2);
+ next = get_unaligned_le16(ip);
+ }
+ ip += 2;
+ m_pos -= next >> 2;
+ next &= 3;
+ if (m_pos == op)
+ goto eof_found;
+ m_pos -= 0x4000;
+ }
+ }
+ TEST_LB(m_pos);
+
+ if (efficient_unaligned_access &&
+ (op - m_pos >= 8)) {
+ unsigned char *oe = op + t;
+ if (HAVE_OP(t + 15)) {
+ do {
+ COPY8(op, m_pos);
+ op += 8;
+ m_pos += 8;
+ COPY8(op, m_pos);
+ op += 8;
+ m_pos += 8;
+ } while (op < oe);
+ op = oe;
+ if (HAVE_IP(6)) {
+ state = next;
+ COPY4(op, ip);
+ op += next;
+ ip += next;
+ continue;
+ }
+ } else {
+ NEED_OP(t);
+ do {
+ *op++ = *m_pos++;
+ } while (op < oe);
+ }
+ } else {
+ unsigned char *oe = op + t;
+ NEED_OP(t);
+ op[0] = m_pos[0];
+ op[1] = m_pos[1];
+ op += 2;
+ m_pos += 2;
+ do {
+ *op++ = *m_pos++;
+ } while (op < oe);
+ }
+match_next:
+ state = next;
+ t = next;
+ if (efficient_unaligned_access &&
+ (HAVE_IP(6) && HAVE_OP(4))) {
+ COPY4(op, ip);
+ op += t;
+ ip += t;
+ } else {
+ NEED_IP(t + 3);
+ NEED_OP(t);
+ while (t > 0) {
+ *op++ = *ip++;
+ t--;
+ }
+ }
+ }
+
+eof_found:
+ *out_len = op - out;
+ return (t != 3 ? LZO_E_ERROR :
+ ip == ip_end ? LZO_E_OK :
+ ip < ip_end ? LZO_E_INPUT_NOT_CONSUMED : LZO_E_INPUT_OVERRUN);
+
+input_overrun:
+ *out_len = op - out;
+ return LZO_E_INPUT_OVERRUN;
+
+output_overrun:
+ *out_len = op - out;
+ return LZO_E_OUTPUT_OVERRUN;
+
+lookbehind_overrun:
+ *out_len = op - out;
+ return LZO_E_LOOKBEHIND_OVERRUN;
+}
diff --git a/lzorle_decompress.h b/lzorle_decompress.h
new file mode 100644
index 000000000000..8cdb3b108c70
--- /dev/null
+++ b/lzorle_decompress.h
@@ -0,0 +1,75 @@
+/* lzorle_decompress.h
+ *
+ * from kernel lib/lzo/lzodefs.h
+ *
+ * Copyright (C) 1996-2012 Markus F.X.J. Oberhumer <markus@xxxxxxxxxxxxx>
+ * Copyright (C) 2024 NIO
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ */
+
+#ifndef LZODEFS_H
+#define LZODEFS_H
+
+#define COPY4(dst, src) memcpy((dst), (src), sizeof(uint32_t))
+#define COPY8(dst, src) memcpy((dst), (src), sizeof(uint64_t))
+
+#define M1_MAX_OFFSET 0x0400
+#define M2_MAX_OFFSET 0x0800
+#define M3_MAX_OFFSET 0x4000
+#define M4_MAX_OFFSET_V0 0xbfff
+#define M4_MAX_OFFSET_V1 0xbffe
+
+#define M1_MIN_LEN 2
+#define M1_MAX_LEN 2
+#define M2_MIN_LEN 3
+#define M2_MAX_LEN 8
+#define M3_MIN_LEN 3
+#define M3_MAX_LEN 33
+#define M4_MIN_LEN 3
+#define M4_MAX_LEN 9
+
+#define M1_MARKER 0
+#define M2_MARKER 64
+#define M3_MARKER 32
+#define M4_MARKER 16
+
+#define MIN_ZERO_RUN_LENGTH 4
+#define MAX_ZERO_RUN_LENGTH (2047 + MIN_ZERO_RUN_LENGTH)
+
+#define lzo_dict_t unsigned short
+#define D_BITS 13
+#define D_SIZE (1u << D_BITS)
+#define D_MASK (D_SIZE - 1)
+#define D_HIGH ((D_MASK >> 1) + 1)
+
+#define LZO_E_OK 0
+#define LZO_E_ERROR (-1)
+#define LZO_E_OUT_OF_MEMORY (-2)
+#define LZO_E_NOT_COMPRESSIBLE (-3)
+#define LZO_E_INPUT_OVERRUN (-4)
+#define LZO_E_OUTPUT_OVERRUN (-5)
+#define LZO_E_LOOKBEHIND_OVERRUN (-6)
+#define LZO_E_EOF_NOT_FOUND (-7)
+#define LZO_E_INPUT_NOT_CONSUMED (-8)
+#define LZO_E_NOT_YET_IMPLEMELZO_HFILESNTED (-9)
+#define LZO_E_INVALID_ARGUMENT (-10)
+
+#define HAVE_IP(x) ((unsigned long)(ip_end - ip) >= (unsigned long)(x))
+#define HAVE_OP(x) ((unsigned long)(op_end - op) >= (unsigned long)(x))
+#define NEED_IP(x) if (!HAVE_IP(x)) goto input_overrun
+#define NEED_OP(x) if (!HAVE_OP(x)) goto output_overrun
+#define TEST_LB(m_pos) if ((m_pos) < out) goto lookbehind_overrun
+
+int lzorle_decompress_safe(const unsigned char *in, unsigned long in_len,
+ unsigned char *out, unsigned long *out_len, void *other/* NOT USED */);
+
+#endif
--
2.31.1
--
Crash-utility mailing list -- devel@xxxxxxxxxxxxxxxxxxxxxxxxxxx
To unsubscribe send an email to devel-leave@xxxxxxxxxxxxxxxxxxxxxxxxxxx
https://${domain_name}/admin/lists/devel.lists.crash-utility.osci.io/
Contribution Guidelines: https://github.com/crash-utility/crash/wiki
