Hi, Alexey
在 2020年10月10日 05:13, crash-utility-request@xxxxxxxxxx 写道:
> Date: Fri, 9 Oct 2020 14:13:49 -0700
> From: Alexey Makhalov <amakhalov@xxxxxxxxxx>
> To: <crash-utility@xxxxxxxxxx>, <amakhalov@xxxxxxxxxx>,
> <k-hagio-ab@xxxxxxx>, <d.hatayama@xxxxxxxxxxx>
> Subject: [PATCH v6] vmware_guestdump: new input format
> Message-ID: <20201009211349.23570-1-amakhalov@xxxxxxxxxx>
> Content-Type: text/plain
>
> vmware_guestdump is extension to vmware_vmss with ability to debug
> debug.guest and debug.vmem files.
>
> debug.guest.gz and debug.vmem.gz can be obtained using following
> .vmx options from VM running in debug mode:
> monitor.mini-suspend_on_panic = TRUE
> monitor.suspend_on_triplefault = TRUE
>
> guestdump (debug.guest) is simplified version of *.vmss which does
> not contain full VM state, but minimal guest state, such as memory
> layout and CPUs state, needed for debugger. is_vmware_guestdump()
> and vmware_guestdump_init() functions parse guestdump header and
> populate vmss data structure (from vmware_vmss.c). As result, all
> handlers (except mempry_dump) from vmware_vmss.c can be reused.
>
> How to use: $ crash /path/to/debug_file.guest vmlinux
> Companion debug_file.vmem must be present in the same folder as
> debug_file.guest. Otherwise crash will shot a message:
> vmw: Open the companion vmem file: /path/to/debug_file.vmem
> crash: vmw: /path/to/debug_file.vmem: No such file or directory
>
> Signed-off-by: Alexey Makhalov <amakhalov@xxxxxxxxxx>
> Acked-by: Kazuhito Hagio <k-hagio-ab@xxxxxxx>
> ---
> Makefile | 7 +-
> crash.8 | 12 +-
> defs.h | 8 ++
> filesys.c | 12 +-
> main.c | 14 +++
> memory.c | 8 +-
> vmware_guestdump.c | 315 +++++++++++++++++++++++++++++++++++++++++++++++++++++
Thank you for the patch.
I would recommend splitting it into two patches. For the vmware_vmss.h and and vmware_vmss.c
changes, it may be a separate patch, this changes are different from the new feature.
And the remaining changes(the above seven files) are the second patch, which represents the
new feature changes.
Thanks.
Lianbo
> vmware_vmss.c | 8 +-
> vmware_vmss.h | 8 ++
> 9 files changed, 376 insertions(+), 16 deletions(-)
> create mode 100644 vmware_guestdump.c
>
> diff --git a/Makefile b/Makefile
> index 7455410..d185719 100644
> --- a/Makefile
> +++ b/Makefile
> @@ -70,7 +70,7 @@ CFILES=main.c tools.c global_data.c memory.c filesys.c help.c task.c \
> unwind_x86_32_64.c unwind_arm.c \
> xen_hyper.c xen_hyper_command.c xen_hyper_global_data.c \
> xen_hyper_dump_tables.c kvmdump.c qemu.c qemu-load.c sadump.c ipcs.c \
> - ramdump.c vmware_vmss.c \
> + ramdump.c vmware_vmss.c vmware_guestdump.c \
> xen_dom0.c kaslr_helper.c
>
> SOURCE_FILES=${CFILES} ${GENERIC_HFILES} ${MCORE_HFILES} \
> @@ -89,7 +89,7 @@ OBJECT_FILES=main.o tools.o global_data.o memory.o filesys.o help.o task.o \
> unwind_x86_32_64.o unwind_arm.o \
> xen_hyper.o xen_hyper_command.o xen_hyper_global_data.o \
> xen_hyper_dump_tables.o kvmdump.o qemu.o qemu-load.o sadump.o ipcs.o \
> - ramdump.o vmware_vmss.o \
> + ramdump.o vmware_vmss.o vmware_guestdump.o \
> xen_dom0.o kaslr_helper.o
>
> MEMORY_DRIVER_FILES=memory_driver/Makefile memory_driver/crash.c memory_driver/README
> @@ -518,6 +518,9 @@ ramdump.o: ${GENERIC_HFILES} ${REDHAT_HFILES} ramdump.c
> vmware_vmss.o: ${GENERIC_HFILES} ${VMWARE_HFILES} vmware_vmss.c
> ${CC} -c ${CRASH_CFLAGS} vmware_vmss.c ${WARNING_OPTIONS} ${WARNING_ERROR}
>
> +vmware_guestdump.o: ${GENERIC_HFILES} ${VMWARE_HFILES} vmware_guestdump.c
> + ${CC} -c ${CRASH_CFLAGS} vmware_guestdump.c ${WARNING_OPTIONS} ${WARNING_ERROR}
> +
> kaslr_helper.o: ${GENERIC_HFILES} kaslr_helper.c
> ${CC} -c ${CRASH_CFLAGS} kaslr_helper.c ${WARNING_OPTIONS} ${WARNING_ERROR}
>
> diff --git a/crash.8 b/crash.8
> index 136ae78..5020ce1 100644
> --- a/crash.8
> +++ b/crash.8
> @@ -21,8 +21,9 @@ core dump has been created by the
> .I LKCD,
> .I kdump,
> .I xendump
> -or
> .I kvmdump
> +or
> +.I VMware
> facilities. It is loosely based on the SVR4 UNIX crash
> command, but has been significantly enhanced
> by completely merging it with the
> @@ -112,8 +113,9 @@ A kernel core dump file created by the
> .I LKCD
> .I kdump,
> .I xendump
> -or
> .I kvmdump
> +or
> +.I VMware
> facilities.
>
> If a MEMORY-IMAGE argument is not entered, the session will be invoked on
> @@ -144,6 +146,12 @@ in /var/tmp, which will only exist during the crash session. If a raw RAM
> dumpile represents a live memory source, such as that specified by the QEMU
> mem-path argument of a memory-backend-file object, then "live:" must be
> prepended to the MEMORY-IMAGE name.
> +
> +As VMware facility, the
> +.B crash
> +utility is able to process VMware VM memory dump generated by VM suspend
> +or guest core dump. In that case, .vmss or .guest file should be used as
> +a MEMORY-IMAGE and .vmem file must be located in the same folder.
> .TP
> .BI mapfile
> If the NAMELIST file is not the same kernel that is
> diff --git a/defs.h b/defs.h
> index c899fe2..3386709 100644
> --- a/defs.h
> +++ b/defs.h
> @@ -544,6 +544,7 @@ struct program_context {
> #define is_excluded_vmemmap() (pc->flags2 & EXCLUDED_VMEMMAP)
> #define MEMSRC_LOCAL (0x80000ULL)
> #define REDZONE (0x100000ULL)
> +#define VMWARE_VMSS_GUESTDUMP (0x200000ULL)
> char *cleanup;
> char *namelist_orig;
> char *namelist_debug_orig;
> @@ -6679,6 +6680,13 @@ int vmware_vmss_phys_base(ulong *phys_base);
> int vmware_vmss_set_phys_base(ulong);
>
> /*
> + * vmware_guestdump.c
> + */
> +int is_vmware_guestdump(char *filename);
> +int vmware_guestdump_init(char *filename, FILE *ofp);
> +int vmware_guestdump_memory_dump(FILE *);
> +
> +/*
> * kaslr_helper.c
> */
> int calc_kaslr_offset(ulong *, ulong *);
> diff --git a/filesys.c b/filesys.c
> index 2ec2b31..3361b6c 100644
> --- a/filesys.c
> +++ b/filesys.c
> @@ -253,9 +253,15 @@ memory_source_init(void)
> error(FATAL, "%s: initialization failed\n",
> pc->dumpfile);
> } else if (pc->flags & VMWARE_VMSS) {
> - if (!vmware_vmss_init(pc->dumpfile, fp))
> - error(FATAL, "%s: initialization failed\n",
> - pc->dumpfile);
> + if (pc->flags2 & VMWARE_VMSS_GUESTDUMP) {
> + if (!vmware_guestdump_init(pc->dumpfile, fp))
> + error(FATAL, "%s: initialization failed\n",
> + pc->dumpfile);
> + } else {
> + if (!vmware_vmss_init(pc->dumpfile, fp))
> + error(FATAL, "%s: initialization failed\n",
> + pc->dumpfile);
> + }
> }
> }
> }
> diff --git a/main.c b/main.c
> index 7f562e6..388ac46 100644
> --- a/main.c
> +++ b/main.c
> @@ -671,6 +671,18 @@ main(int argc, char **argv)
> pc->readmem = read_vmware_vmss;
> pc->writemem = write_vmware_vmss;
>
> + } else if (is_vmware_guestdump(argv[optind])) {
> + if (pc->flags & MEMORY_SOURCES) {
> + error(INFO,
> + "too many dumpfile arguments\n");
> + program_usage(SHORT_FORM);
> + }
> + pc->flags |= VMWARE_VMSS;
> + pc->flags2 |= VMWARE_VMSS_GUESTDUMP;
> + pc->dumpfile = argv[optind];
> + pc->readmem = read_vmware_vmss;
> + pc->writemem = write_vmware_vmss;
> +
> } else {
> error(INFO,
> "%s: not a supported file format\n",
> @@ -1486,6 +1498,8 @@ dump_program_context(void)
> fprintf(fp, "%sMEMSRC_LOCAL", others++ ? "|" : "");
> if (pc->flags2 & REDZONE)
> fprintf(fp, "%sREDZONE", others++ ? "|" : "");
> + if (pc->flags2 & VMWARE_VMSS_GUESTDUMP)
> + fprintf(fp, "%sVMWARE_VMSS_GUESTDUMP", others++ ? "|" : "");
> fprintf(fp, ")\n");
>
> fprintf(fp, " namelist: %s\n", pc->namelist);
> diff --git a/memory.c b/memory.c
> index c951827..0848097 100644
> --- a/memory.c
> +++ b/memory.c
> @@ -17115,8 +17115,12 @@ dumpfile_memory(int cmd)
> retval = kcore_memory_dump(fp);
> else if (pc->flags & SADUMP)
> retval = sadump_memory_dump(fp);
> - else if (pc->flags & VMWARE_VMSS)
> - retval = vmware_vmss_memory_dump(fp);
> + else if (pc->flags & VMWARE_VMSS) {
> + if (pc->flags2 & VMWARE_VMSS_GUESTDUMP)
> + retval = vmware_guestdump_memory_dump(fp);
> + else
> + retval = vmware_vmss_memory_dump(fp);
> + }
> break;
>
> case DUMPFILE_ENVIRONMENT:
> diff --git a/vmware_guestdump.c b/vmware_guestdump.c
> new file mode 100644
> index 0000000..d3fac59
> --- /dev/null
> +++ b/vmware_guestdump.c
> @@ -0,0 +1,315 @@
> +/*
> + * vmware_guestdump.c
> + *
> + * Copyright (c) 2020 VMware, Inc.
> + *
> + * This program is free software; you can redistribute it and/or modify
> + * it under the terms of the GNU General Public License as published by
> + * the Free Software Foundation; either version 2 of the License, or
> + * (at your option) any later version.
> + *
> + * This program is distributed in the hope that it will be useful,
> + * but WITHOUT ANY WARRANTY; without even the implied warranty of
> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
> + * GNU General Public License for more details.
> + *
> + * Author: Alexey Makhalov <amakhalov@xxxxxxxxxx>
> + */
> +
> +#include "defs.h"
> +#include "vmware_vmss.h"
> +
> +#define LOGPRX "vmw: "
> +
> +#define GUESTDUMP_VERSION 4
> +#define GUESTDUMP_MAGIC1 1
> +#define GUESTDUMP_MAGIC2 0
> +
> +struct guestdumpheader {
> + uint32_t version;
> + uint32_t num_vcpus;
> + uint8_t magic1;
> + uint8_t reserved1;
> + uint32_t cpu_vendor;
> + uint64_t magic2;
> + uint64_t last_addr;
> + uint64_t memsize_in_pages;
> + uint32_t reserved2;
> + uint32_t mem_holes;
> + struct memhole {
> + uint64_t ppn;
> + uint64_t pages;
> + } holes[2];
> +} __attribute__((packed));
> +
> +struct vcpu_state {
> + uint32_t cr0;
> + uint64_t cr2;
> + uint64_t cr3;
> + uint64_t cr4;
> + uint64_t reserved1[10];
> + uint64_t idt_base;
> + uint16_t reserved2[21];
> + struct x86_64_pt_regs {
> + uint64_t r15;
> + uint64_t r14;
> + uint64_t r13;
> + uint64_t r12;
> + uint64_t rbp;
> + uint64_t rbx;
> + uint64_t r11;
> + uint64_t r10;
> + uint64_t r9;
> + uint64_t r8;
> + uint64_t rax;
> + uint64_t rcx;
> + uint64_t rdx;
> + uint64_t rsi;
> + uint64_t rdi;
> + uint64_t orig_rax;
> + uint64_t rip;
> + uint64_t cs;
> + uint64_t eflags;
> + uint64_t rsp;
> + uint64_t ss;
> + } regs64;
> + uint8_t reserved3[65];
> +} __attribute__((packed));
> +
> +
> +/*
> + * vmware_guestdump is extension to vmware_vmss with ability to debug
> + * debug.guest and debug.vmem files.
> + *
> + * debug.guest.gz and debug.vmem.gz can be obtained using following
> + * .vmx options from VM running in debug mode:
> + * monitor.mini-suspend_on_panic = TRUE
> + * monitor.suspend_on_triplefault = TRUE
> + *
> + * guestdump (debug.guest) is simplified version of *.vmss which does
> + * not contain full VM state, but minimal guest state, such as memory
> + * layout and CPUs state, needed for debugger. is_vmware_guestdump()
> + * and vmware_guestdump_init() functions parse guestdump header and
> + * populate vmss data structure (from vmware_vmss.c). As result, all
> + * handlers (except mempry_dump) from vmware_vmss.c can be reused.
> + *
> + * debug.guest does not have dedicated header magic or signature for
> + * its format. To probe debug.guest we need to perform header fields
> + * and file size validity. In addition, check for the filename
> + * extension, which must be ".guest".
> + */
> +
> +int
> +is_vmware_guestdump(char *filename)
> +{
> + struct guestdumpheader hdr;
> + FILE *fp;
> + uint64_t filesize, holes_sum = 0;
> + int i;
> +
> + if (strcmp(filename + strlen(filename) - 6, ".guest"))
> + return FALSE;
> +
> + if ((fp = fopen(filename, "r")) == NULL) {
> + error(INFO, LOGPRX"Failed to open '%s': [Error %d] %s\n",
> + filename, errno, strerror(errno));
> + return FALSE;
> + }
> +
> + if (fread(&hdr, sizeof(struct guestdumpheader), 1, fp) != 1) {
> + error(INFO, LOGPRX"Failed to read '%s': [Error %d] %s\n",
> + filename, errno, strerror(errno));
> + fclose(fp);
> + return FALSE;
> + }
> +
> + if (fseek(fp, 0L, SEEK_END) == -1) {
> + error(INFO, LOGPRX"Failed to fseek '%s': [Error %d] %s\n",
> + filename, errno, strerror(errno));
> + fclose(fp);
> + return FALSE;
> + }
> + filesize = ftell(fp);
> + fclose(fp);
> +
> + if (hdr.mem_holes > 2)
> + goto unrecognized;
> +
> + for (i = 0; i < hdr.mem_holes; i++) {
> + /* hole start page */
> + vmss.regions[i].startpagenum = hdr.holes[i].ppn;
> + /* hole end page */
> + vmss.regions[i].startppn = hdr.holes[i].ppn + hdr.holes[i].pages;
> + holes_sum += hdr.holes[i].pages;
> + }
> +
> + if (hdr.version != GUESTDUMP_VERSION ||
> + hdr.magic1 != GUESTDUMP_MAGIC1 ||
> + hdr.magic2 != GUESTDUMP_MAGIC2 ||
> + (hdr.last_addr + 1) != ((hdr.memsize_in_pages + holes_sum) << VMW_PAGE_SHIFT) ||
> + filesize != sizeof(struct guestdumpheader) +
> + hdr.num_vcpus * (sizeof (struct vcpu_state) + VMW_PAGE_SIZE))
> + goto unrecognized;
> +
> + vmss.memsize = hdr.memsize_in_pages << VMW_PAGE_SHIFT;
> + vmss.regionscount = hdr.mem_holes + 1;
> + vmss.memoffset = 0;
> + vmss.num_vcpus = hdr.num_vcpus;
> + return TRUE;
> +
> +unrecognized:
> + if (CRASHDEBUG(1))
> + error(INFO, LOGPRX"Unrecognized debug.guest file.\n");
> + return FALSE;
> +}
> +
> +int
> +vmware_guestdump_init(char *filename, FILE *ofp)
> +{
> + FILE *fp = NULL;
> + int i, result = TRUE;
> + char *vmem_filename = NULL;
> + struct vcpu_state vs;
> + char *p;
> +
> + if (!machine_type("X86") && !machine_type("X86_64")) {
> + error(INFO,
> + LOGPRX"Invalid or unsupported host architecture for .vmss file: %s\n",
> + MACHINE_TYPE);
> + result = FALSE;
> + goto exit;
> + }
> +
> + if ((fp = fopen(filename, "r")) == NULL) {
> + error(INFO, LOGPRX"Failed to open '%s': [Error %d] %s\n",
> + filename, errno, strerror(errno));
> + result = FALSE;
> + goto exit;
> + }
> +
> + if (fseek(fp, sizeof(struct guestdumpheader), SEEK_SET) == -1) {
> + error(INFO, LOGPRX"Failed to fseek '%s': [Error %d] %s\n",
> + filename, errno, strerror(errno));
> + result = FALSE;
> + goto exit;
> + }
> +
> + vmss.vcpu_regs = malloc(vmss.num_vcpus * sizeof(uint32_t));
> + vmss.regs64 = calloc(vmss.num_vcpus, sizeof(void *));
> + if (!vmss.vcpu_regs || !vmss.regs64) {
> + error(INFO, LOGPRX"Failed to allocate memory\n");
> + result = FALSE;
> + goto exit;
> + }
> +
> + for (i = 0; i < vmss.num_vcpus; i++) {
> + if (fread(&vs, sizeof(struct vcpu_state), 1, fp) != 1) {
> + error(INFO, LOGPRX"Failed to read '%s': [Error %d] %s\n",
> + filename, errno, strerror(errno));
> + result = FALSE;
> + goto exit;
> + }
> + vmss.regs64[i] = calloc(1, sizeof(vmssregs64));
> + if (!vmss.regs64[i]) {
> + error(INFO, LOGPRX"Failed to allocate memory\n");
> + result = FALSE;
> + goto exit;
> + }
> + vmss.vcpu_regs[i] = 0;
> +
> + vmss.regs64[i]->rax = vs.regs64.rax;
> + vmss.regs64[i]->rcx = vs.regs64.rcx;
> + vmss.regs64[i]->rdx = vs.regs64.rdx;
> + vmss.regs64[i]->rbx = vs.regs64.rbx;
> + vmss.regs64[i]->rbp = vs.regs64.rbp;
> + vmss.regs64[i]->rsp = vs.regs64.rsp;
> + vmss.regs64[i]->rsi = vs.regs64.rsi;
> + vmss.regs64[i]->rdi = vs.regs64.rdi;
> + vmss.regs64[i]->r8 = vs.regs64.r8;
> + vmss.regs64[i]->r9 = vs.regs64.r9;
> + vmss.regs64[i]->r10 = vs.regs64.r10;
> + vmss.regs64[i]->r11 = vs.regs64.r11;
> + vmss.regs64[i]->r12 = vs.regs64.r12;
> + vmss.regs64[i]->r13 = vs.regs64.r13;
> + vmss.regs64[i]->r14 = vs.regs64.r14;
> + vmss.regs64[i]->r15 = vs.regs64.r15;
> + vmss.regs64[i]->idtr = vs.idt_base;
> + vmss.regs64[i]->cr[0] = vs.cr0;
> + vmss.regs64[i]->cr[2] = vs.cr2;
> + vmss.regs64[i]->cr[3] = vs.cr3;
> + vmss.regs64[i]->cr[4] = vs.cr4;
> + vmss.regs64[i]->rip = vs.regs64.rip;
> + vmss.regs64[i]->rflags = vs.regs64.eflags;
> +
> + vmss.vcpu_regs[i] = REGS_PRESENT_ALL;
> + }
> +
> + vmem_filename = strdup(filename);
> + p = vmem_filename + strlen(vmem_filename) - 5;
> + if (strcmp(p, "guest") != 0) {
> + result = FALSE;
> + goto exit;
> + }
> + strcpy(p, "vmem");
> +
> + fprintf(ofp, LOGPRX"Open the companion vmem file: %s\n", vmem_filename);
> + if ((vmss.dfp = fopen(vmem_filename, "r")) == NULL) {
> + error(INFO, LOGPRX"%s: %s\n", vmem_filename, strerror(errno));
> + result = FALSE;
> + goto exit;
> + }
> + fseek(vmss.dfp, 0L, SEEK_END);
> + if (vmss.memsize != ftell(vmss.dfp)) {
> + error(INFO, LOGPRX"%s: unexpected size\n", vmem_filename);
> + result = FALSE;
> + goto exit;
> + }
> + fseek(vmss.dfp, 0L, SEEK_SET);
> + fprintf(ofp, LOGPRX"vmem file: %s\n\n", vmem_filename);
> +
> +exit:
> + if (fp)
> + fclose(fp);
> + if (vmem_filename)
> + free(vmem_filename);
> + if (result == FALSE) {
> + if (vmss.dfp)
> + fclose(vmss.dfp);
> + if (vmss.regs64) {
> + for (i = 0; i < vmss.num_vcpus; i++) {
> + if (vmss.regs64[i])
> + free(vmss.regs64[i]);
> + }
> + free(vmss.regs64);
> + }
> + if (vmss.vcpu_regs)
> + free(vmss.vcpu_regs);
> + }
> + return result;
> +}
> +
> +int
> +vmware_guestdump_memory_dump(FILE *ofp)
> +{
> + fprintf(ofp, "vmware_guestdump:\n");
> + fprintf(ofp, " Header: version=%d num_vcpus=%ld\n",
> + GUESTDUMP_VERSION, vmss.num_vcpus);
> + fprintf(ofp, "Total memory: %ld\n", vmss.memsize);
> +
> + if (vmss.regionscount > 1) {
> + uint64_t holes_sum = 0;
> + unsigned i;
> +
> + fprintf(ofp, "Memory regions[%d]:\n", vmss.regionscount);
> + fprintf(ofp, " [0x%016x-", 0);
> + for (i = 0; i < vmss.regionscount - 1; i++) {
> + fprintf(ofp, "0x%016lx]\n", (uint64_t)vmss.regions[i].startpagenum << VMW_PAGE_SHIFT);
> + fprintf(ofp, " [0x%016lx-", (uint64_t)vmss.regions[i].startppn << VMW_PAGE_SHIFT);
> + holes_sum += vmss.regions[i].startppn - vmss.regions[i].startpagenum;
> + }
> + fprintf(ofp, "0x%016lx]\n", vmss.memsize + (holes_sum << VMW_PAGE_SHIFT));
> + }
> +
> + return TRUE;
> +}
> +
> diff --git a/vmware_vmss.c b/vmware_vmss.c
> index 252bfa2..b168f29 100644
> --- a/vmware_vmss.c
> +++ b/vmware_vmss.c
> @@ -23,13 +23,7 @@
>
> #define LOGPRX "vmw: "
>
> -/* VMware only supports X86/X86_64 virtual machines. */
> -#define VMW_PAGE_SIZE (4096)
> -#define VMW_PAGE_SHIFT (12)
> -
> -#define MAX_BLOCK_DUMP (128)
> -
> -static vmssdata vmss = { 0 };
> +vmssdata vmss = { 0 };
>
> int
> is_vmware_vmss(char *filename)
> diff --git a/vmware_vmss.h b/vmware_vmss.h
> index a5828a0..01d9446 100644
> --- a/vmware_vmss.h
> +++ b/vmware_vmss.h
> @@ -165,6 +165,14 @@ struct vmssdata {
> };
> typedef struct vmssdata vmssdata;
>
> +/* VMware only supports X86/X86_64 virtual machines. */
> +#define VMW_PAGE_SIZE (4096)
> +#define VMW_PAGE_SHIFT (12)
> +
> +#define MAX_BLOCK_DUMP (128)
> +
> +extern vmssdata vmss;
> +
> #define DEBUG_PARSE_PRINT(x) \
> do { \
> if (CRASHDEBUG(1)) { \
> -- 2.11.0
--
Crash-utility mailing list
Crash-utility@xxxxxxxxxx
https://www.redhat.com/mailman/listinfo/crash-utility
