Hi Michael, I made a few other changes to your cpu map/mask patch: - cpu_map_size() has been changed to use the symbol-name-creation scheme as cpu_map_addr() in order to absolutely maintain the backwards-compatibility issue brought up by Robin. So if the old symbol names still exist, the code does exactly the same thing that it used to. - Your changes to get_cpus_present() and get_cpus_possible() were incomplete because they were still using the hardwired "cpus_present_map" and "cpus_possible_map" symbols. - I didn't change the s390.c and s390x.c s390[x]_get_smp_cpus() functions beyond what you did, but I think they should be changed to use get_cpus_online() instead of the way they are doing it now. As it is now, they are restricting themselves to the number of cpu bits in a per-arch long. If the s390 CONFIG_NR_CPUS can exceed 32, or the s390x CONFIG_NR_CPUS can exceed 64, then those two functions are incorrect. If the architecture limits them to 32/64, then I suppose the way it's done now is OK. That's your call... Also, the attached patch contains the changes for searching for kernel module debuginfo files in the same directory tree specified with "mod -S <directory>". Alternatively there is a new "crash --mod <directory>" command line option, such that "mod -S" with no directory argument will default to searching the command-line specified directory. Anyway, I've only tested this on older kernels that still have the "cpu_xxxx_map" symbols. Can you test the patch on new kernels? And also let me know whether the s390[x] functions should be updated. Thanks, Dave
--- crash-4.0-8.9/main.c 2009-04-29 11:43:32.000000000 -0400 +++ crash-4.0-8.9p2/main.c 2009-04-28 13:11:52.000000000 -0400 @@ -1,8 +1,8 @@ /* main.c - core analysis suite * * Copyright (C) 1999, 2000, 2001, 2002 Mission Critical Linux, Inc. - * Copyright (C) 2002, 2003, 2004, 2005, 2006, 2007, 2008 David Anderson - * Copyright (C) 2002, 2003, 2004, 2005, 2006, 2007, 2008 Red Hat, Inc. All rights reserved. + * Copyright (C) 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009 David Anderson + * Copyright (C) 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009 Red Hat, Inc. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -59,6 +59,7 @@ {"reloc", required_argument, 0, 0}, {"active", 0, 0, 0}, {"minimal", 0, 0, 0}, + {"mod", required_argument, 0, 0}, {0, 0, 0, 0} }; @@ -203,6 +204,9 @@ else if (STREQ(long_options[option_index].name, "active")) tt->flags |= ACTIVE_ONLY; + else if (STREQ(long_options[option_index].name, "mod")) + kt->module_tree = optarg; + else if (STREQ(long_options[option_index].name, "reloc")) { if (!calculate(optarg, &kt->relocate, NULL, 0)) { error(INFO, "invalid --reloc argument: %s\n", @@ -1309,6 +1313,8 @@ fprintf(fp, "%sUD2A_INSTRUCTION", others ? "|" : ""); if (pc->curcmd_flags & IRQ_IN_USE) fprintf(fp, "%sIRQ_IN_USE", others ? "|" : ""); + if (pc->curcmd_flags & MODULE_TREE) + fprintf(fp, "%sMODULE_TREE", others ? "|" : ""); fprintf(fp, ")\n"); fprintf(fp, " curcmd_private: %llx\n", pc->curcmd_private); fprintf(fp, " sigint_cnt: %d\n", pc->sigint_cnt); --- crash-4.0-8.9/filesys.c 2009-04-29 11:43:32.000000000 -0400 +++ crash-4.0-8.9p2/filesys.c 2009-04-28 11:07:55.000000000 -0400 @@ -933,7 +933,7 @@ * When time permits, rewrite this doing the search by hand. */ char * -search_directory_tree(char *directory, char *file) +search_directory_tree(char *directory, char *file, int follow_links) { char command[BUFSIZE]; char buf[BUFSIZE]; @@ -949,8 +949,8 @@ return NULL; sprintf(command, - "/usr/bin/find %s -name %s -print; /bin/echo search done", - directory, file); + "/usr/bin/find %s %s -name %s -print; /bin/echo search done", + follow_links ? "-L" : "", directory, file); if ((pipe = popen(command, "r")) == NULL) { error(INFO, "%s: %s\n", command, strerror(errno)); --- crash-4.0-8.9/kernel.c 2009-04-29 11:43:32.000000000 -0400 +++ crash-4.0-8.9p2/kernel.c 2009-04-29 11:39:38.000000000 -0400 @@ -533,6 +533,62 @@ } /* + * Get cpu map address. Types are: possible, online, present and active. + * They exist as either: + * + * (1) cpu_<type>_map symbols, or + * (2) what is pointed to by cpu_<type>_mask + */ +ulong +cpu_map_addr(const char *type) +{ + char map_symbol[32]; + ulong addr; + + sprintf(map_symbol, "cpu_%s_map", type); + if (kernel_symbol_exists(map_symbol)) + return symbol_value(map_symbol); + + sprintf(map_symbol, "cpu_%s_mask", type); + if (kernel_symbol_exists(map_symbol)) { + get_symbol_data(map_symbol, sizeof(ulong), &addr); + return addr; + } + + return 0; +} + +/* + * Get cpu map (possible, online, etc.) size + */ +static int +cpu_map_size(const char *type) +{ + int len; + char map_symbol[32]; + struct gnu_request req; + + if (LKCD_KERNTYPES()) { + if ((len = STRUCT_SIZE("cpumask_t")) < 0) + error(FATAL, "cannot determine type cpumask_t\n"); + return len; + } + + sprintf(map_symbol, "cpu_%s_map", type); + if (kernel_symbol_exists(map_symbol)) { + len = get_symbol_type(map_symbol, NULL, &req) == + TYPE_CODE_UNDEF ? sizeof(ulong) : req.length; + return len; + } + + len = STRUCT_SIZE("cpumask_t"); + if (len < 0) + return sizeof(ulong); + else + return len; +} + +/* * If the cpu_present_map, cpu_online_map and cpu_possible_maps exist, * set up the kt->cpu_flags[NR_CPUS] with their settings. */ @@ -541,14 +597,14 @@ { int i, c, m, cpu, len; char *buf; - ulong *maskptr; + ulong *maskptr, addr; struct mapinfo { ulong cpu_flag; char *name; } mapinfo[] = { - { POSSIBLE, "cpu_possible_map" }, - { PRESENT, "cpu_present_map" }, - { ONLINE, "cpu_online_map" }, + { POSSIBLE, "possible" }, + { PRESENT, "present" }, + { ONLINE, "online" }, }; if ((len = STRUCT_SIZE("cpumask_t")) < 0) @@ -557,12 +613,13 @@ buf = GETBUF(len); for (m = 0; m < sizeof(mapinfo)/sizeof(struct mapinfo); m++) { - if (!kernel_symbol_exists(mapinfo[m].name)) + if (!(addr = cpu_map_addr(mapinfo[m].name))) continue; - if (!readmem(symbol_value(mapinfo[m].name), KVADDR, buf, len, + if (!readmem(addr, KVADDR, buf, len, mapinfo[m].name, RETURN_ON_ERROR)) { - error(WARNING, "cannot read %s\n", mapinfo[m].name); + error(WARNING, "cannot read cpu_%s_map\n", + mapinfo[m].name); continue; } @@ -578,7 +635,7 @@ } if (CRASHDEBUG(1)) { - fprintf(fp, "%s: ", mapinfo[m].name); + fprintf(fp, "cpu_%s_map: ", mapinfo[m].name); for (i = 0; i < NR_CPUS; i++) { if (kt->cpu_flags[i] & mapinfo[m].cpu_flag) fprintf(fp, "%d ", i); @@ -605,21 +662,21 @@ switch (map) { case POSSIBLE: - if (!kernel_symbol_exists("cpu_possible_map")) { + if (!cpu_map_addr("possible")) { error(INFO, "cpu_possible_map does not exist\n"); return FALSE; } return (kt->cpu_flags[cpu] & POSSIBLE); case PRESENT: - if (!kernel_symbol_exists("cpu_present_map")) { + if (!cpu_map_addr("present")) { error(INFO, "cpu_present_map does not exist\n"); return FALSE; } return (kt->cpu_flags[cpu] & PRESENT); case ONLINE: - if (!kernel_symbol_exists("cpu_online_map")) { + if (!cpu_map_addr("online")) { error(INFO, "cpu_online_map does not exist\n"); return FALSE; } @@ -2799,7 +2856,7 @@ cmd_mod(void) { int c; - char *objfile, *modref, *tree; + char *objfile, *modref, *tree, *symlink; ulong flag, address; char buf[BUFSIZE]; @@ -2817,11 +2874,11 @@ return; } - modref = objfile = tree = NULL; + modref = objfile = tree = symlink = NULL; address = 0; flag = LIST_MODULE_HDR; - while ((c = getopt(argcnt, args, "rd:Ds:St:o")) != EOF) { + while ((c = getopt(argcnt, args, "rd:Ds:So")) != EOF) { switch(c) { case 'r': @@ -2867,15 +2924,6 @@ st->flags |= USE_OLD_ADD_SYM; return; - case 't': - if (is_directory(optarg)) - tree = optarg; - else { - error(INFO, "%s is not a directory\n", args[2]); - cmd_usage(pc->curcmd, SYNOPSIS); - } - break; - case 'S': if (flag) cmd_usage(pc->curcmd, SYNOPSIS); @@ -2979,8 +3027,50 @@ break; } + if ((flag == LOAD_ALL_MODULE_SYMBOLS) && + (tree || kt->module_tree)) { + if (!tree) + tree = kt->module_tree; + + pc->curcmd_flags |= MODULE_TREE; + } + do_module_cmd(flag, modref, address, objfile, tree); + if (symlink) + FREEBUF(symlink); +} + +int +check_specified_module_tree(char *module, char *gdb_buffer) +{ + char *p1, *treebuf; + int retval; + + retval = FALSE; + + if (!(pc->curcmd_flags & MODULE_TREE)) + return retval; + /* + * Search for "/lib/modules" in the module name string + * and insert "/usr/lib/debug" there. + */ + if (strstr(module, "/lib/modules")) { + treebuf = GETBUF(strlen(module) + strlen("/usr/lib/debug") + + strlen(".debug") + 1); + strcpy(treebuf, module); + p1 = strstr(treebuf, "/lib/modules"); + shift_string_right(p1, strlen("/usr/lib/debug")); + BCOPY("/usr/lib/debug", p1, strlen("/usr/lib/debug")); + strcat(treebuf, ".debug"); + if (file_exists(treebuf, NULL)) { + strcpy(gdb_buffer, treebuf); + retval = TRUE; + } + FREEBUF(treebuf); + } + + return retval; } @@ -3290,12 +3380,12 @@ } if (tree) { - if (!(retbuf = search_directory_tree(tree, file))) { + if (!(retbuf = search_directory_tree(tree, file, 1))) { switch (kt->flags & (KMOD_V1|KMOD_V2)) { case KMOD_V2: sprintf(file, "%s.ko", modref); - retbuf = search_directory_tree(tree, file); + retbuf = search_directory_tree(tree, file, 1); } } return retbuf; @@ -3303,28 +3393,28 @@ sprintf(dir, "%s/%s", DEFAULT_REDHAT_DEBUG_LOCATION, kt->utsname.release); - retbuf = search_directory_tree(dir, file); + retbuf = search_directory_tree(dir, file, 0); if (!retbuf) { sprintf(dir, "/lib/modules/%s/updates", kt->utsname.release); - if (!(retbuf = search_directory_tree(dir, file))) { + if (!(retbuf = search_directory_tree(dir, file, 0))) { switch (kt->flags & (KMOD_V1|KMOD_V2)) { case KMOD_V2: sprintf(file, "%s.ko", modref); - retbuf = search_directory_tree(dir, file); + retbuf = search_directory_tree(dir, file, 0); } } } if (!retbuf) { sprintf(dir, "/lib/modules/%s", kt->utsname.release); - if (!(retbuf = search_directory_tree(dir, file))) { + if (!(retbuf = search_directory_tree(dir, file, 0))) { switch (kt->flags & (KMOD_V1|KMOD_V2)) { case KMOD_V2: sprintf(file, "%s.ko", modref); - retbuf = search_directory_tree(dir, file); + retbuf = search_directory_tree(dir, file, 0); } } } @@ -4101,6 +4191,8 @@ fprintf(fp, " module_list: %lx\n", kt->module_list); fprintf(fp, " kernel_module: %lx\n", kt->kernel_module); fprintf(fp, "mods_installed: %d\n", kt->mods_installed); + fprintf(fp, " module_tree: %s\n", kt->module_tree ? + kt->module_tree : "(not used)"); if (!(pc->flags & KERNEL_DEBUG_QUERY) && ACTIVE()) get_symbol_data("xtime", sizeof(struct timespec), &kt->date); fprintf(fp, " date: %s\n", @@ -4187,7 +4279,7 @@ } fprintf(fp, "\n"); fprintf(fp, " cpu_possible_map: "); - if (kernel_symbol_exists("cpu_possible_map")) { + if (cpu_map_addr("possible")) { for (i = 0; i < nr_cpus; i++) { if (kt->cpu_flags[i] & POSSIBLE) fprintf(fp, "%d ", i); @@ -4196,7 +4288,7 @@ } else fprintf(fp, "(does not exist)\n"); fprintf(fp, " cpu_present_map: "); - if (kernel_symbol_exists("cpu_present_map")) { + if (cpu_map_addr("present")) { for (i = 0; i < nr_cpus; i++) { if (kt->cpu_flags[i] & PRESENT) fprintf(fp, "%d ", i); @@ -4205,7 +4297,7 @@ } else fprintf(fp, "(does not exist)\n"); fprintf(fp, " cpu_online_map: "); - if (kernel_symbol_exists("cpu_online_map")) { + if (cpu_map_addr("online")) { for (i = 0; i < nr_cpus; i++) { if (kt->cpu_flags[i] & ONLINE) fprintf(fp, "%d ", i); @@ -5923,35 +6015,30 @@ get_cpus_online() { int i, len, online; - struct gnu_request req; char *buf; - ulong *maskptr; + ulong *maskptr, addr; - if (!symbol_exists("cpu_online_map")) + if (!(addr = cpu_map_addr("online"))) return 0; - if (LKCD_KERNTYPES()) { - if ((len = STRUCT_SIZE("cpumask_t")) < 0) - error(FATAL, "cannot determine type cpumask_t\n"); - } else - len = get_symbol_type("cpu_online_map", NULL, &req) == - TYPE_CODE_UNDEF ? sizeof(ulong) : req.length; + len = cpu_map_size("online"); buf = GETBUF(len); online = 0; - if (readmem(symbol_value("cpu_online_map"), KVADDR, buf, len, - "cpu_online_map", RETURN_ON_ERROR)) { + if (readmem(addr, KVADDR, buf, len, + "cpu_online_map", RETURN_ON_ERROR)) { maskptr = (ulong *)buf; for (i = 0; i < (len/sizeof(ulong)); i++, maskptr++) online += count_bits_long(*maskptr); - FREEBUF(buf); if (CRASHDEBUG(1)) error(INFO, "get_cpus_online: online: %d\n", online); } + FREEBUF(buf); + return online; } @@ -5962,35 +6049,30 @@ get_cpus_present() { int i, len, present; - struct gnu_request req; char *buf; - ulong *maskptr; + ulong *maskptr, addr; - if (!symbol_exists("cpu_present_map")) + if (!(addr = cpu_map_addr("present"))) return 0; - if (LKCD_KERNTYPES()) { - if ((len = STRUCT_SIZE("cpumask_t")) < 0) - error(FATAL, "cannot determine type cpumask_t\n"); - } else - len = get_symbol_type("cpu_present_map", NULL, &req) == - TYPE_CODE_UNDEF ? sizeof(ulong) : req.length; + len = cpu_map_size("present"); buf = GETBUF(len); present = 0; - if (readmem(symbol_value("cpu_present_map"), KVADDR, buf, len, + if (readmem(addr, KVADDR, buf, len, "cpu_present_map", RETURN_ON_ERROR)) { maskptr = (ulong *)buf; for (i = 0; i < (len/sizeof(ulong)); i++, maskptr++) present += count_bits_long(*maskptr); - FREEBUF(buf); if (CRASHDEBUG(1)) error(INFO, "get_cpus_present: present: %d\n", present); } + FREEBUF(buf); + return present; } @@ -6001,36 +6083,31 @@ get_cpus_possible() { int i, len, possible; - struct gnu_request req; char *buf; - ulong *maskptr; + ulong *maskptr, addr; - if (!symbol_exists("cpu_possible_map")) + if (!(addr = cpu_map_addr("possible"))) return 0; - if (LKCD_KERNTYPES()) { - if ((len = STRUCT_SIZE("cpumask_t")) < 0) - error(FATAL, "cannot determine type cpumask_t\n"); - } else - len = get_symbol_type("cpu_possible_map", NULL, &req) == - TYPE_CODE_UNDEF ? sizeof(ulong) : req.length; + len = cpu_map_size("possible"); buf = GETBUF(len); possible = 0; - if (readmem(symbol_value("cpu_possible_map"), KVADDR, buf, len, + if (readmem(addr, KVADDR, buf, len, "cpu_possible_map", RETURN_ON_ERROR)) { maskptr = (ulong *)buf; for (i = 0; i < (len/sizeof(ulong)); i++, maskptr++) possible += count_bits_long(*maskptr); - FREEBUF(buf); if (CRASHDEBUG(1)) error(INFO, "get_cpus_possible: possible: %d\n", possible); } + FREEBUF(buf); + return possible; } --- crash-4.0-8.9/s390.c 2009-04-29 11:43:32.000000000 -0400 +++ crash-4.0-8.9p2/s390.c 2009-04-29 11:39:47.000000000 -0400 @@ -1001,7 +1001,8 @@ { unsigned long map = 0, addr; int i, cpu_num = 0; - addr=symbol_value("cpu_online_map"); + + addr = cpu_map_addr("online"); readmem(addr, KVADDR, &map,sizeof(long), "cpu_online_map",FAULT_ON_ERROR); for(i = 0; i < sizeof(map)*8;i++){ if(map & 0x1UL) --- crash-4.0-8.9/s390x.c 2009-04-29 11:43:32.000000000 -0400 +++ crash-4.0-8.9p2/s390x.c 2009-04-29 11:39:50.000000000 -0400 @@ -1031,7 +1031,8 @@ { unsigned long map = 0, addr; int i, cpu_num = 0; - addr=symbol_value("cpu_online_map"); + + addr = cpu_map_addr("online"); readmem(addr, KVADDR, &map,sizeof(long), "cpu_online_map",FAULT_ON_ERROR); for(i = 0; i < sizeof(map)*8;i++){ if(map & 0x1UL) --- crash-4.0-8.9/ppc64.c 2009-04-29 11:43:32.000000000 -0400 +++ crash-4.0-8.9p2/ppc64.c 2009-04-29 11:39:55.000000000 -0400 @@ -2407,9 +2407,9 @@ if (!symbol_exists("paca")) error(FATAL, "PPC64: Could not find 'paca' symbol\n"); - if (symbol_exists("cpu_present_map")) + if (cpu_map_addr("present")) map = PRESENT; - else if (symbol_exists("cpu_online_map")) + else if (cpu_map_addr("online")) map = ONLINE; else error(FATAL, --- crash-4.0-8.9/symbols.c 2009-04-29 11:43:32.000000000 -0400 +++ crash-4.0-8.9p2/symbols.c 2009-04-28 11:08:16.000000000 -0400 @@ -428,11 +428,14 @@ } /* - * Callback for gdb to use a specified debug file. + * Callback for gdb to use a specified vmlinux.debug file. */ char * -check_specified_debug_file() +check_specified_kernel_debug_file() { + if (pc->flags & GDB_INIT) + return NULL; + return (pc->namelist_debug ? pc->namelist_debug : NULL); } --- crash-4.0-8.9/defs.h 2009-04-29 11:43:32.000000000 -0400 +++ crash-4.0-8.9p2/defs.h 2009-04-29 11:40:29.000000000 -0400 @@ -381,6 +381,7 @@ #define BAD_INSTRUCTION (0x80) #define UD2A_INSTRUCTION (0x100) #define IRQ_IN_USE (0x200) +#define MODULE_TREE (0x400) ulonglong curcmd_private; /* general purpose per-command info */ int cur_gdb_cmd; /* current gdb command */ int last_gdb_cmd; /* previously-executed gdb command */ @@ -547,6 +548,7 @@ ulong p2m_mfn_cache_hits; ulong p2m_page_cache_hits; ulong relocate; + char *module_tree; }; /* @@ -3281,7 +3283,7 @@ * symbols.c */ void symtab_init(void); -char *check_specified_debug_file(void); +char *check_specified_kernel_debug_file(void); void no_debugging_data(int); void get_text_init_space(void); int is_kernel_text(ulong); @@ -3427,7 +3429,7 @@ int file_exists(char *, struct stat *); int file_readable(char *); int is_directory(char *); -char *search_directory_tree(char *, char *); +char *search_directory_tree(char *, char *, int); void open_tmpfile(void); void close_tmpfile(void); void open_tmpfile2(void); @@ -3627,6 +3629,7 @@ void non_matching_kernel(void); struct load_module *modref_to_load_module(char *); void unlink_module(struct load_module *); +int check_specified_module_tree(char *, char *); int is_system_call(char *, ulong); void generic_dump_irq(int); int generic_dis_filter(ulong, char *); @@ -3647,6 +3650,7 @@ void paravirt_init(void); void print_stack_text_syms(struct bt_info *, ulong, ulong); void back_trace(struct bt_info *); +ulong cpu_map_addr(const char *type); #define BT_RAW (0x1ULL) #define BT_SYMBOLIC_ARGS (0x2ULL) #define BT_FULL (0x4ULL) --- crash-4.0-8.9/gdb-6.1/gdb/symtab.c 2009-04-29 11:43:32.000000000 -0400 +++ crash-4.0-8.9p2/gdb-6.1/gdb/symtab.c 2009-04-28 13:09:59.000000000 -0400 @@ -4,7 +4,7 @@ 1995, 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004 Free Software Foundation, Inc. Portions Copyright (C) 2001, 2002 Mission Critical Linux, Inc. - Copyright (c) 2002, 2003, 2004, 2005, 2007 Red Hat, Inc. All rights reserved. + Copyright (c) 2002, 2003, 2004, 2005, 2007, 2009 Red Hat, Inc. All rights reserved. This file is part of GDB. @@ -4696,7 +4696,8 @@ register struct objfile *objfile; ALL_OBJFILES(objfile) { - if (STREQ(objfile->name, req->name)) { + if (STREQ(objfile->name, req->name) || + same_file(objfile->name, req->name)) { free_objfile(objfile); break; } --- crash-4.0-8.9/gdb-6.1/gdb/symfile.c 2009-04-29 11:43:32.000000000 -0400 +++ crash-4.0-8.9p2/gdb-6.1/gdb/symfile.c 2009-04-28 13:10:32.000000000 -0400 @@ -3,7 +3,7 @@ Copyright 1990, 1991, 1992, 1993, 1994, 1995, 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004 Free Software Foundation, Inc. Portions Copyright (C) 2001, 2002 Mission Critical Linux, Inc. - Copyright (c) 2002, 2003, 2004, 2005, 2006, 2007 Red Hat, Inc. All rights reserved. + Copyright (c) 2002, 2003, 2004, 2005, 2006, 2007, 2009 Red Hat, Inc. All rights reserved. Contributed by Cygnus Support, using pieces from other GDB modules. @@ -1051,7 +1051,11 @@ char *basename; char *dir; char *debugfile; +#ifdef CRASH_MERGE char *name_copy; + extern int check_specified_module_tree(char *, char *); + extern char *check_specified_kernel_debug_file(); +#endif bfd_size_type debuglink_size; unsigned long crc32; int i; @@ -1105,6 +1109,17 @@ xfree (dir); return xstrdup (debugfile); } + +#ifdef CRASH_MERGE +{ + if (check_specified_module_tree(objfile->name, debugfile) && + separate_debug_file_exists(debugfile, crc32)) { + xfree(basename); + xfree(dir); + return xstrdup(debugfile); + } +} +#endif /* Then try in the global debugfile directory. */ strcpy (debugfile, debug_file_directory); @@ -1123,8 +1138,7 @@ xfree (dir); #ifdef CRASH_MERGE { - extern char *check_specified_debug_file(); - name_copy = check_specified_debug_file(); + name_copy = check_specified_kernel_debug_file(); return (name_copy ? xstrdup (name_copy) : NULL); } #else
-- Crash-utility mailing list Crash-utility@xxxxxxxxxx https://www.redhat.com/mailman/listinfo/crash-utility