Hi Kazu,
Thank you for supporting me with my first patch to crash.
04.02.2022 4:52, HAGIO KAZUHITO(萩尾 一仁) пишет:
> Hi Sergey,
>
> -----Original Message-----
>> Patch adds new 'sbitmapq' command. This command dumps
>> the contents of the sbitmap_queue structure and the used
>> bits in the bitmap. Also, it shows the dump of a structure
>> array associated with the sbitmap_queue.
>>
>> v1 -> v2:
>> - Update the help page (Lianbo)
>> - Use crash interfaces (offset_table, size_table, GETBUF()
>> and etc.) for reduce the number of readmem() (Kazu)
> Thank you for the update, the patch looks nice and clean!
>
> I have two slight questions inline with "Question:".
>
> As for the others, I've changed the code at my end and been testing,
> so I can post it when the questions are solved.
>
>> Signed-off-by: Sergey Samoylenko <s.samoylenko@xxxxxxxxx>
>> ---
>> Makefile | 7 +-
>> defs.h | 59 +++++
>> global_data.c | 1 +
>> help.c | 108 ++++++++
>> sbitmap.c | 664 ++++++++++++++++++++++++++++++++++++++++++++++++++
>> 5 files changed, 837 insertions(+), 2 deletions(-)
>> create mode 100644 sbitmap.c
>>
>> diff --git a/Makefile b/Makefile
>> index 4fd8b78..a381c5f 100644
>> --- a/Makefile
>> +++ b/Makefile
>> @@ -72,7 +72,7 @@ CFILES=main.c tools.c global_data.c memory.c filesys.c help.c task.c \
>> xen_hyper.c xen_hyper_command.c xen_hyper_global_data.c \
>> xen_hyper_dump_tables.c kvmdump.c qemu.c qemu-load.c sadump.c ipcs.c \
>> ramdump.c vmware_vmss.c vmware_guestdump.c \
>> - xen_dom0.c kaslr_helper.c
>> + xen_dom0.c kaslr_helper.c sbitmap.c
>>
>> SOURCE_FILES=${CFILES} ${GENERIC_HFILES} ${MCORE_HFILES} \
>> ${REDHAT_CFILES} ${REDHAT_HFILES} ${UNWIND_HFILES} \
>> @@ -92,7 +92,7 @@ OBJECT_FILES=main.o tools.o global_data.o memory.o filesys.o help.o task.o \
>> xen_hyper.o xen_hyper_command.o xen_hyper_global_data.o \
>> xen_hyper_dump_tables.o kvmdump.o qemu.o qemu-load.o sadump.o ipcs.o \
>> ramdump.o vmware_vmss.o vmware_guestdump.o \
>> - xen_dom0.o kaslr_helper.o
>> + xen_dom0.o kaslr_helper.o sbitmap.o
>>
>> MEMORY_DRIVER_FILES=memory_driver/Makefile memory_driver/crash.c memory_driver/README
>>
>> @@ -341,6 +341,9 @@ cmdline.o: ${GENERIC_HFILES} cmdline.c
>> tools.o: ${GENERIC_HFILES} tools.c
>> ${CC} -c ${CRASH_CFLAGS} tools.c ${WARNING_OPTIONS} ${WARNING_ERROR}
>>
>> +sbitmap.o: ${GENERIC_HFILES} sbitmap.c
>> + ${CC} -c ${CRASH_CFLAGS} sbitmap.c ${WARNING_OPTIONS} ${WARNING_ERROR}
>> +
>> global_data.o: ${GENERIC_HFILES} global_data.c
>> ${CC} -c ${CRASH_CFLAGS} global_data.c ${WARNING_OPTIONS} ${WARNING_ERROR}
>>
>> diff --git a/defs.h b/defs.h
>> index b63741c..d407025 100644
>> --- a/defs.h
>> +++ b/defs.h
>> @@ -18,6 +18,7 @@
>>
>> #ifndef GDB_COMMON
>>
>> +#include <stdbool.h>
>> #include <stdio.h>
>> #include <stdarg.h>
>> #include <stdint.h>
>> @@ -2146,6 +2147,23 @@ struct offset_table { /* stash of commonly-used offsets */
>> long wait_queue_entry_private;
>> long wait_queue_head_head;
>> long wait_queue_entry_entry;
>> + long sbitmap_word_depth;
>> + long sbitmap_word_word;
>> + long sbitmap_word_cleared;
>> + long sbitmap_depth;
>> + long sbitmap_shift;
>> + long sbitmap_map_nr;
>> + long sbitmap_map;
>> + long sbitmap_queue_sb;
>> + long sbitmap_queue_alloc_hint;
>> + long sbitmap_queue_wake_batch;
>> + long sbitmap_queue_wake_index;
>> + long sbitmap_queue_ws;
>> + long sbitmap_queue_ws_active;
>> + long sbitmap_queue_round_robin;
>> + long sbitmap_queue_min_shallow_depth;
>> + long sbq_wait_state_wait_cnt;
>> + long sbq_wait_state_wait;
>> };
>>
>> struct size_table { /* stash of commonly-used sizes */
>> @@ -2310,6 +2328,10 @@ struct size_table { /* stash of commonly-used sizes */
>> long prb_desc;
>> long wait_queue_entry;
>> long task_struct_state;
>> + long sbitmap_word;
>> + long sbitmap;
>> + long sbitmap_queue;
>> + long sbq_wait_state;
>> };
> The corresponding entries are needed in "help -o" output.
>
> --- a/symbols.c
> +++ b/symbols.c
> @@ -10690,6 +10690,24 @@ dump_offset_table(char *spec, ulong makestruct)
> fprintf(fp, " uts_namespace_name: %ld\n",
> OFFSET(uts_namespace_name));
>
> + fprintf(fp, " sbitmap_word_depth: %ld\n", OFFSET(sbitmap_word_depth));
> + fprintf(fp, " sbitmap_word_word: %ld\n", OFFSET(sbitmap_word_word));
> + fprintf(fp, " sbitmap_word_cleared: %ld\n", OFFSET(sbitmap_word_cleared));
> + fprintf(fp, " sbitmap_depth: %ld\n", OFFSET(sbitmap_depth));
> + fprintf(fp, " sbitmap_shift: %ld\n", OFFSET(sbitmap_shift));
> + fprintf(fp, " sbitmap_map_nr: %ld\n", OFFSET(sbitmap_map_nr));
> + fprintf(fp, " sbitmap_map: %ld\n", OFFSET(sbitmap_map));
> + fprintf(fp, " sbitmap_queue_sb: %ld\n", OFFSET(sbitmap_queue_sb));
> + fprintf(fp, " sbitmap_queue_alloc_hint: %ld\n", OFFSET(sbitmap_queue_alloc_hint));
> + fprintf(fp, " sbitmap_queue_wake_batch: %ld\n", OFFSET(sbitmap_queue_wake_batch));
> + fprintf(fp, " sbitmap_queue_wake_index: %ld\n", OFFSET(sbitmap_queue_wake_index));
> + fprintf(fp, " sbitmap_queue_ws: %ld\n", OFFSET(sbitmap_queue_ws));
> + fprintf(fp, " sbitmap_queue_ws_active: %ld\n", OFFSET(sbitmap_queue_ws_active));
> + fprintf(fp, " sbitmap_queue_round_robin: %ld\n", OFFSET(sbitmap_queue_round_robin));
> + fprintf(fp, "sbitmap_queue_min_shallow_depth: %ld\n", OFFSET(sbitmap_queue_min_shallow_depth));
> + fprintf(fp, " sbq_wait_state_wait_cnt: %ld\n", OFFSET(sbq_wait_state_wait_cnt));
> + fprintf(fp, " sbq_wait_state_wait: %ld\n", OFFSET(sbq_wait_state_wait));
> +
> fprintf(fp, "\n size_table:\n");
> fprintf(fp, " page: %ld\n", SIZE(page));
> fprintf(fp, " page_flags: %ld\n", SIZE(page_flags));
> @@ -10955,6 +10973,10 @@ dump_offset_table(char *spec, ulong makestruct)
> fprintf(fp, " printk_ringbuffer: %ld\n", SIZE(printk_ringbuffer));
> fprintf(fp, " prb_desc: %ld\n", SIZE(prb_desc));
>
> + fprintf(fp, " sbitmap_word: %ld\n", SIZE(sbitmap_word));
> + fprintf(fp, " sbitmap: %ld\n", SIZE(sbitmap));
> + fprintf(fp, " sbitmap_queue: %ld\n", SIZE(sbitmap_queue));
> + fprintf(fp, " sbq_wait_state: %ld\n", SIZE(sbq_wait_state));
>
> fprintf(fp, "\n array_table:\n");
> /*
Didn't notice this output. Thanks, added.
>> struct array_table {
>> @@ -2436,6 +2458,7 @@ DEF_LOADER(ushort);
>> DEF_LOADER(short);
>> typedef void *pointer_t;
>> DEF_LOADER(pointer_t);
>> +DEF_LOADER(bool);
>>
>> #define LOADER(TYPE) load_##TYPE
>>
>> @@ -2449,6 +2472,7 @@ DEF_LOADER(pointer_t);
>> #define SHORT(ADDR) LOADER(short) ((char *)(ADDR))
>> #define UCHAR(ADDR) *((unsigned char *)((char *)(ADDR)))
>> #define VOID_PTR(ADDR) ((void *) (LOADER(pointer_t) ((char *)(ADDR))))
>> +#define BOOL(ADDR) LOADER(bool) ((char *)(ADDR)))
>>
>> #else
>>
>> @@ -2462,6 +2486,7 @@ DEF_LOADER(pointer_t);
>> #define SHORT(ADDR) *((short *)((char *)(ADDR)))
>> #define UCHAR(ADDR) *((unsigned char *)((char *)(ADDR)))
>> #define VOID_PTR(ADDR) *((void **)((char *)(ADDR)))
>> +#define BOOL(ADDR) *((bool *)((char *)(ADDR)))
>>
>> #endif /* NEED_ALIGNED_MEM_ACCESS */
>>
>> @@ -4962,6 +4987,7 @@ void cmd_mach(void); /* main.c */
>> void cmd_help(void); /* help.c */
>> void cmd_test(void); /* test.c */
>> void cmd_ascii(void); /* tools.c */
>> +void cmd_sbitmapq(void); /* sbitmap.c */
>> void cmd_bpf(void); /* bfp.c */
>> void cmd_set(void); /* tools.c */
>> void cmd_eval(void); /* tools.c */
>> @@ -5575,6 +5601,7 @@ extern char *help_rd[];
>> extern char *help_repeat[];
>> extern char *help_runq[];
>> extern char *help_ipcs[];
>> +extern char *help_sbitmapq[];
>> extern char *help_search[];
>> extern char *help_set[];
>> extern char *help_sig[];
>> @@ -5844,6 +5871,38 @@ void devdump_info(void *, ulonglong, FILE *);
>> void ipcs_init(void);
>> ulong idr_find(ulong, int);
>>
>> +/*
>> + * sbitmap.c
>> + */
>> +/* sbitmap helpers */
>> +struct sbitmap_context {
>> + unsigned depth;
>> + unsigned shift;
>> + unsigned map_nr;
>> + ulong map_addr;
>> +};
>> +
>> +typedef bool (*sbitmap_for_each_fn)(unsigned int idx, void *p);
>> +
>> +void sbitmap_for_each_set(const struct sbitmap_context *sc,
>> + sbitmap_for_each_fn fn, void *data);
>> +void sbitmap_context_load(ulong addr, struct sbitmap_context *sc);
>> +
>> +/* sbitmap_queue helpers */
>> +typedef bool (*sbitmapq_for_each_fn)(unsigned int idx, ulong addr, void *p);
>> +
>> +struct sbitmapq_ops {
>> + /* array params associated with the bitmap */
>> + ulong addr;
>> + ulong size;
>> + /* callback params */
>> + sbitmapq_for_each_fn fn;
>> + void *p;
>> +};
>> +
>> +void sbitmapq_init(void);
>> +void sbitmapq_for_each_set(ulong addr, struct sbitmapq_ops *ops);
>> +
>> #ifdef ARM
>> void arm_init(int);
>> void arm_dump_machdep_table(ulong);
>> diff --git a/global_data.c b/global_data.c
>> index a316d1c..f9bb7d0 100644
>> --- a/global_data.c
>> +++ b/global_data.c
>> @@ -105,6 +105,7 @@ struct command_table_entry linux_command_table[] = {
>> {"rd", cmd_rd, help_rd, MINIMAL},
>> {"repeat", cmd_repeat, help_repeat, 0},
>> {"runq", cmd_runq, help_runq, REFRESH_TASK_TABLE},
>> + {"sbitmapq", cmd_sbitmapq, help_sbitmapq, 0},
>> {"search", cmd_search, help_search, 0},
>> {"set", cmd_set, help_set, REFRESH_TASK_TABLE | MINIMAL},
>> {"sig", cmd_sig, help_sig, REFRESH_TASK_TABLE},
>> diff --git a/help.c b/help.c
>> index 04a7eff..d151f2e 100644
>> --- a/help.c
>> +++ b/help.c
>> @@ -962,6 +962,114 @@ char *help_ascii[] = {
>> NULL
>> };
>>
>> +char *help_sbitmapq[] = {
>> +"sbitmapq",
>> +"sbitmap_queue struct contents",
>> +"[-s struct[.member[,member]] -p address [-v]] address",
>> +" The command dumps the contents of the sbitmap_queue structure and",
>> +" the used bits in the bitmap. Also, it shows the dump of a structure",
>> +" array associated with the sbitmap_queue.",
>> +"",
>> +" The arguments are as follows:",
>> +"",
>> +" -s struct - name of a C-code structure, that is stored in an array",
>> +" sssociated with sbitmap_queue structure. Use the",
> s/sssociated/associated/
>
>> +" \"struct.member\" format in order to display a particular",
>> +" member of the structure. -s option requires -p option",
>> +"",
>> +" -p address - address of a structure array associated with sbitmap_queue",
>> +" structure. The set bits in sbitmap are used for the index",
>> +" in an associated array.",
>> +"",
>> +" -x - override default output format with hexadecimal format.",
>> +"",
>> +" -d - override default output format with decimal format.",
>> +"",
>> +" -v - By default, the sbitmap command shows only a used sbitmap",
>> +" index and a structure address in the associated array.",
>> +" This flag says to print a formatted display of the",
>> +" contents of a structure in an associated array. -v option",
>> +" requires of -s.",
> I'd like to change these to the crash style like this:
>
> " -s struct name of a C-code structure, that is stored in an array",
> " associated with sbitmap_queue structure. Use the",
> " \"struct.member\" format in order to display a particular",
> " member of the structure. -s option requires -p option",
> " -p address address of a structure array associated with sbitmap_queue",
> " structure. The set bits in sbitmap are used for the index",
> " in an associated array.",
> " -x override default output format with hexadecimal format.",
> " -d override default output format with decimal format.",
> " -v By default, the sbitmap command shows only a used sbitmap",
> " index and a structure address in the associated array.",
> " This flag says to print a formatted display of the",
> " contents of a structure in an associated array. -v option",
> " requires of -s.",
>
>
>> +"",
>> +"EXAMPLES",
>> +"",
>> +" All examples are shown on the base of Linux Target system whit iSCSI",
> s/whit/with/
Fixed help and typos.
>
>> +" transport.",
>> +"",
>> +" Display the common sbitmap information for target session:",
>> +"",
>> +" %s> struct -oh se_session 0xc0000000e118c760 | grep sbitmap_queue",
>> +" [c0000000e118c808] struct sbitmap_queue sess_tag_pool;",
>> +" %s>",
>> +" %s> sbitmapq c0000000e118c808",
>> +" depth = 136",
>> +" busy = 4",
>> +" cleared = 26",
>> +" bits_per_word = 32",
>> +" map_nr = 5",
>> +" alloc_hint = {74, 36, 123, 101}",
>> +" wake_batch = 8",
>> +" wake_index = 0",
>> +" ws_active = 0",
>> +" ws = {",
>> +" { .wait_cnt = 8, .wait = inactive },",
>> +" { .wait_cnt = 8, .wait = inactive },",
>> +" { .wait_cnt = 8, .wait = inactive },",
>> +" { .wait_cnt = 8, .wait = inactive },",
>> +" { .wait_cnt = 8, .wait = inactive },",
>> +" { .wait_cnt = 8, .wait = inactive },",
>> +" { .wait_cnt = 8, .wait = inactive },",
>> +" { .wait_cnt = 8, .wait = inactive },",
>> +" }",
>> +" round_robin = 0",
>> +" min_shallow_depth = 4294967295",
>> +"",
>> +" 00000000: 0000 0000 0000 0000 0030 0000 0000 0000",
>> +" 00000010: 00",
>> +"",
>> +" Display the addresses of structure are associated with",
>> +" sbitmap_queue (for iscsi it is 'iscsi_cmd' structure):",
>> +"",
>> +" %s> struct se_session 0xc0000000e118c760 | grep sess_cmd_map",
>> +" sess_cmd_map = 0xc0000000671c0000,",
>> +" %s>",
>> +" %s> sbitmapq -s iscsi_cmd -p 0xc0000000671c0000 c0000000e118c808",
>> +" 76: 0xc0000000671d5600",
>> +" 77: 0xc0000000671d5a80",
>> +"",
>> +
>> +" Dump of formatted content of structures:",
>> +"",
>> +" %s> sbitmapq -s iscsi_cmd -p 0xc0000000671c0000 -v c0000000e118c808",
>> +" 76 (0xc0000000671d5600):",
>> +" struct iscsi_cmd {",
>> +" dataout_timer_flags = 0,",
>> +" dataout_timeout_retries = 0 '\\000',",
>> +" error_recovery_count = 0 '\\000',",
>> +" deferred_i_state = ISTATE_NO_STATE,",
>> +" i_state = ISTATE_SENT_STATUS,",
>> +" ...",
>> +" first_data_sg = 0xc0000000e306b080,",
>> +" first_data_sg_off = 0,",
>> +" kmapped_nents = 1,",
>> +" sense_reason = 0",
>> +" }",
>> +" 77 (0xc0000000671d5a80):",
>> +" struct iscsi_cmd {",
>> +" dataout_timer_flags = 0,",
>> +" dataout_timeout_retries = 0 '\\000',",
>> +" error_recovery_count = 0 '\\000',",
>> +" deferred_i_state = ISTATE_NO_STATE,",
>> +" i_state = ISTATE_NEW_CMD,",
>> +" ...",
>> +" first_data_sg = 0x0,",
>> +" first_data_sg_off = 0,",
>> +" kmapped_nents = 0,",
>> +" sense_reason = 0",
>> +" }",
>> +NULL
>> +};
>> +
>> char *help_quit[] = {
>> "quit",
>> "exit this session",
>> diff --git a/sbitmap.c b/sbitmap.c
>> new file mode 100644
>> index 0000000..5343a88
>> --- /dev/null
>> +++ b/sbitmap.c
>> @@ -0,0 +1,664 @@
>> +/* sbitmap.c - core analysis suite
>> + *
>> + * Copyright (C) 1999, 2000, 2001, 2002 Mission Critical Linux, Inc.
>> + * Copyright (C) 2002-2020 David Anderson
>> + * Copyright (C) 2002-2020 Red Hat, Inc. All rights reserved.
> Question:
> Would you like to change these copyright notices to yours?
> If you have a proper one, please let me know.
>
> e.g. in crash_target.c:
> * Copyright (c) 2021 VMware, Inc.
Fixed copyright.
>
>> + *
>> + * This program is free software; you can redistribute it and/or modify
>> + * it under the terms of the GNU General Public License as published by
>> + * the Free Software Foundation; either version 2 of the License, or
>> + * (at your option) any later version.
>> + *
>> + * This program is distributed in the hope that it will be useful,
>> + * but WITHOUT ANY WARRANTY; without even the implied warranty of
>> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
>> + * GNU General Public License for more details.
>> + */
>> +
>> +#include "defs.h"
>> +
>> +#define SBQ_WAIT_QUEUES 8
>> +
>> +/* sbitmap_queue struct context */
>> +struct sbitmap_queue_context {
>> + ulong sb_addr;
>> + ulong alloc_hint;
>> + unsigned int wake_batch;
>> + int wake_index;
>> + ulong ws_addr;
>> + int ws_active;
>> + bool round_robin;
>> + unsigned int min_shallow_depth;
>> +
>> +};
>> +
>> +struct sbitmapq_data {
>> +#define SBITMAPQ_DATA_FLAG_STRUCT_NAME (VERBOSE << 1)
>> +#define SBITMAPQ_DATA_FLAG_STRUCT_ADDR (VERBOSE << 2)
>> +#define SBITMAPQ_DATA_FLAG_STRUCT_MEMBER (VERBOSE << 3)
>> + ulong flags;
>> + int radix;
>> + /* sbitmap_queue info */
>> + ulong addr;
>> + /* data array info */
>> + ulong data_addr;
>> + char *data_name;
>> + int data_size;
>> +};
>> +
>> +#define SB_FLAG_INIT 0x01
>> +
>> +static uint sb_flags = 0;
>> +
>> +static inline unsigned int __const_hweight8(unsigned long w)
>> +{
>> + return
>> + (!!((w) & (1ULL << 0))) +
>> + (!!((w) & (1ULL << 1))) +
>> + (!!((w) & (1ULL << 2))) +
>> + (!!((w) & (1ULL << 3))) +
>> + (!!((w) & (1ULL << 4))) +
>> + (!!((w) & (1ULL << 5))) +
>> + (!!((w) & (1ULL << 6))) +
>> + (!!((w) & (1ULL << 7)));
>> +}
>> +
>> +#define __const_hweight16(w) (__const_hweight8(w) + __const_hweight8((w) >> 8))
>> +#define __const_hweight32(w) (__const_hweight16(w) + __const_hweight16((w) >> 16))
>> +#define __const_hweight64(w) (__const_hweight32(w) + __const_hweight32((w) >> 32))
>> +
>> +#define hweight32(w) __const_hweight32(w)
>> +#define hweight64(w) __const_hweight64(w)
>> +
>> +#define BIT(nr) (1UL << (nr))
>> +
>> +static inline unsigned long min(unsigned long a, unsigned long b)
>> +{
>> + return (a < b) ? a : b;
>> +}
>> +
>> +static unsigned long __last_word_mask(unsigned long nbits)
>> +{
>> + return ~0UL >> (-(nbits) & (BITS_PER_LONG - 1));
>> +}
>> +
>> +static unsigned long bitmap_hweight_long(unsigned long w)
>> +{
>> + return sizeof(w) == 4 ? hweight32(w) : hweight64(w);
>> +}
>> +
>> +static unsigned long bitmap_weight(unsigned long bitmap, unsigned int bits)
>> +{
>> + unsigned long w = 0;
>> +
>> + w += bitmap_hweight_long(bitmap);
>> + if (bits % BITS_PER_LONG)
>> + w += bitmap_hweight_long(bitmap & __last_word_mask(bits));
>> +
>> + return w;
>> +}
>> +
>> +static unsigned int __sbitmap_weight(const struct sbitmap_context *sc, bool set)
>> +{
>> + const ulong sbitmap_word_size = SIZE(sbitmap_word);
>> + const ulong w_depth_off = OFFSET(sbitmap_word_depth);
>> + const ulong w_word_off = OFFSET(sbitmap_word_word);
>> + const ulong w_cleared_off = OFFSET(sbitmap_word_cleared);
>> +
>> + unsigned int weight = 0;
>> + ulong addr = sc->map_addr;
>> + ulong depth, word, cleared;
>> + char *sbitmap_word_buf;
>> + int i;
>> +
>> + sbitmap_word_buf = GETBUF(sbitmap_word_size);
>> +
>> + for (i = 0; i < sc->map_nr; i++) {
>> + readmem(addr, KVADDR, sbitmap_word_buf, sbitmap_word_size, "sbitmap_word",
>> FAULT_ON_ERROR);
>> +
>> + depth = ULONG(sbitmap_word_buf + w_depth_off);
>> +
>> + if (set) {
>> + word = ULONG(sbitmap_word_buf + w_word_off);
>> + weight += bitmap_weight(word, depth);
>> + } else {
>> + cleared = ULONG(sbitmap_word_buf + w_cleared_off);
>> + weight += bitmap_weight(cleared, depth);
>> + }
>> +
>> + addr += sbitmap_word_size;
>> + }
>> +
>> + FREEBUF(sbitmap_word_buf);
>> +
>> + return weight;
>> +}
>> +
>> +static unsigned int sbitmap_weight(const struct sbitmap_context *sc)
>> +{
>> + return __sbitmap_weight(sc, true);
>> +}
>> +
>> +static unsigned int sbitmap_cleared(const struct sbitmap_context *sc)
>> +{
>> + return __sbitmap_weight(sc, false);
>> +}
>> +
>> +static void sbitmap_emit_byte(unsigned int offset, uint8_t byte)
>> +{
>> + if ((offset &0xf) == 0) {
>> + if (offset != 0)
>> + fputc('\n', fp);
>> + fprintf(fp, "%08x:", offset);
>> + }
>> + if ((offset & 0x1) == 0)
>> + fputc(' ', fp);
>> + fprintf(fp, "%02x", byte);
>> +}
>> +
>> +static void sbitmap_bitmap_show(const struct sbitmap_context *sc)
>> +{
>> + const ulong sbitmap_word_size = SIZE(sbitmap_word);
>> + const ulong w_depth_off = OFFSET(sbitmap_word_depth);
>> + const ulong w_word_off = OFFSET(sbitmap_word_word);
>> + const ulong w_cleared_off = OFFSET(sbitmap_word_cleared);
>> +
>> + uint8_t byte = 0;
>> + unsigned int byte_bits = 0;
>> + unsigned int offset = 0;
>> + ulong addr = sc->map_addr;
>> + char *sbitmap_word_buf;
>> + int i;
>> +
>> + sbitmap_word_buf = GETBUF(sbitmap_word_size);
>> +
>> + for (i = 0; i < sc->map_nr; i++) {
>> + unsigned long word, cleared, word_bits;
>> +
>> + readmem(addr, KVADDR, sbitmap_word_buf, sbitmap_word_size, "sbitmap_word",
>> FAULT_ON_ERROR);
>> +
>> + word = ULONG(sbitmap_word_buf + w_word_off);
>> + cleared = ULONG(sbitmap_word_buf + w_cleared_off);
>> + word_bits = ULONG(sbitmap_word_buf + w_depth_off);
>> +
>> + word &= ~cleared;
>> +
>> + while (word_bits > 0) {
>> + unsigned int bits = min(8 - byte_bits, word_bits);
>> +
>> + byte |= (word & (BIT(bits) - 1)) << byte_bits;
>> + byte_bits += bits;
>> + if (byte_bits == 8) {
>> + sbitmap_emit_byte(offset, byte);
>> + byte = 0;
>> + byte_bits = 0;
>> + offset++;
>> + }
>> + word >>= bits;
>> + word_bits -= bits;
>> + }
>> +
>> + addr += sbitmap_word_size;
>> + }
>> + if (byte_bits) {
>> + sbitmap_emit_byte(offset, byte);
>> + offset++;
>> + }
>> + if (offset)
>> + fputc('\n', fp);
>> +
>> + FREEBUF(sbitmap_word_buf);
>> +}
>> +
>> +static unsigned long sbitmap_find_next_bit(unsigned long word,
>> + unsigned long size, unsigned long offset)
>> +{
>> + if (size > BITS_PER_LONG)
>> + error(FATAL, "%s: word size isn't correct\n", __func__);
>> +
>> + for (; offset < size; offset++)
>> + if (word & (1UL << offset))
>> + return offset;
>> +
>> + return size;
>> +}
>> +
>> +static void __sbitmap_for_each_set(const struct sbitmap_context *sc,
>> + unsigned int start, sbitmap_for_each_fn fn, void *data)
>> +{
>> + const ulong sbitmap_word_size = SIZE(sbitmap_word);
>> + const ulong w_depth_off = OFFSET(sbitmap_word_depth);
>> + const ulong w_word_off = OFFSET(sbitmap_word_word);
>> + const ulong w_cleared_off = OFFSET(sbitmap_word_cleared);
>> +
>> + unsigned int index;
>> + unsigned int nr;
>> + unsigned int scanned = 0;
>> + char *sbitmap_word_buf;
>> +
>> + sbitmap_word_buf = GETBUF(sbitmap_word_size);
>> +
>> + if (start >= sc->map_nr)
>> + start = 0;
> Question: Is "(start >= sc->depth)" correct here? Currently start is
> always zero, though.
I have a plan to add range options to dump data attached to the
sbitmap_queue.
It will allow me to go through the bitmap from any index, not only first.
>
>> +
>> + index = start >> sc->shift;
>> + nr = start & ((1U << sc->shift) - 1U);
>> +
>> + while (scanned < sc->depth) {
>> + unsigned long w_addr = sc->map_addr + (sbitmap_word_size * index);
>> + unsigned long w_depth, w_word, w_cleared;
>> + unsigned long word, depth;
>> +
>> + readmem(w_addr, KVADDR, sbitmap_word_buf, sbitmap_word_size, "sbitmap_word",
>> FAULT_ON_ERROR);
>> +
>> + w_depth = ULONG(sbitmap_word_buf + w_depth_off);
>> + w_word = ULONG(sbitmap_word_buf + w_word_off);
>> + w_cleared = ULONG(sbitmap_word_buf + w_cleared_off);
>> +
>> + depth = min(w_depth - nr, sc->depth - scanned);
>> +
>> + scanned += depth;
>> + word = w_word & ~w_cleared;
>> + if (!word)
>> + goto next;
>> +
>> + /*
>> + * On the first iteration of the outer loop, we need to add the
>> + * bit offset back to the size of the word for find_next_bit().
>> + * On all other iterations, nr is zero, so this is a noop.
>> + */
>> + depth += nr;
>> + while (1) {
>> + nr = sbitmap_find_next_bit(word, depth, nr);
>> + if (nr >= depth)
>> + break;
>> + if (!fn((index << sc->shift) + nr, data))
>> + return;
>> +
>> + nr++;
>> + }
>> +next:
>> + nr = 0;
>> + if (++index >= sc->map_nr)
>> + index = 0;
>> + }
>> +
>> + FREEBUF(sbitmap_word_buf);
>> +}
>> +
>> +void sbitmap_for_each_set(const struct sbitmap_context *sc,
>> + sbitmap_for_each_fn fn, void *data)
>> +{
>> + __sbitmap_for_each_set(sc, 0, fn, data);
>> +}
>> +
>> +static void sbitmap_queue_show(const struct sbitmap_queue_context *sqc,
>> + const struct sbitmap_context *sc)
>> +{
>> + int cpus = get_cpus_possible();
>> + int sbq_wait_state_size, wait_cnt_off, wait_off, list_head_off;
>> + char *sbq_wait_state_buf;
>> + bool first;
>> + int i;
>> +
>> + fprintf(fp, "depth = %u\n", sc->depth);
>> + fprintf(fp, "busy = %u\n", sbitmap_weight(sc) - sbitmap_cleared(sc));
>> + fprintf(fp, "cleared = %u\n", sbitmap_cleared(sc));
>> + fprintf(fp, "bits_per_word = %u\n", 1U << sc->shift);
>> + fprintf(fp, "map_nr = %u\n", sc->map_nr);
>> +
>> + fputs("alloc_hint = {", fp);
>> + first = true;
>> + for (i = 0; i < cpus; i++) {
>> + ulong ptr;
>> + int val;
>> +
>> + if (!first)
>> + fprintf(fp, ", ");
>> + first = false;
>> +
>> + ptr = kt->__per_cpu_offset[i] + sqc->alloc_hint;
>> + readmem(ptr, KVADDR, &val, sizeof(val), "alloc_hint", FAULT_ON_ERROR);
>> +
>> + fprintf(fp, "%u", val);
>> + }
>> + fputs("}\n", fp);
>> +
>> + fprintf(fp, "wake_batch = %u\n", sqc->wake_batch);
>> + fprintf(fp, "wake_index = %d\n", sqc->wake_index);
>> + fprintf(fp, "ws_active = %d\n", sqc->ws_active);
>> +
>> + sbq_wait_state_size = SIZE(sbq_wait_state);
>> + wait_cnt_off = OFFSET(sbq_wait_state_wait_cnt);
>> + wait_off = OFFSET(sbq_wait_state_wait);
>> + list_head_off = OFFSET(wait_queue_head_head);
>> +
>> + sbq_wait_state_buf = GETBUF(sbq_wait_state_size);
>> +
>> + fputs("ws = {\n", fp);
>> + for (i = 0; i < SBQ_WAIT_QUEUES; i++) {
>> + ulong ws_addr = sqc->ws_addr + (sbq_wait_state_size * i);
>> + struct kernel_list_head *lh;
>> + ulong wait_cnt_addr, list_head_addr;
> will remove these unused variables:
>
> $ make clean ; make warn
> ...
> sbitmap.c: In function ‘sbitmap_queue_show’:
> sbitmap.c:342:24: warning: unused variable ‘list_head_addr’ [-Wunused-variable]
> ulong wait_cnt_addr, list_head_addr;
> ^~~~~~~~~~~~~~
> sbitmap.c:342:9: warning: unused variable ‘wait_cnt_addr’ [-Wunused-variable]
> ulong wait_cnt_addr, list_head_addr;
> ^~~~~~~~~~~~~
fixed
>
>> + ulong wait_cnt;
>> +
>> + readmem(ws_addr, KVADDR, sbq_wait_state_buf, sbq_wait_state_size, "sbq_wait_state",
>> FAULT_ON_ERROR);
>> +
>> + wait_cnt = INT(sbq_wait_state_buf + wait_cnt_off);
>> + lh = (struct kernel_list_head *)(sbq_wait_state_buf + wait_off + list_head_off);
>> +
>> + fprintf(fp, "\t{ .wait_cnt = %lu, .wait = %s },\n",
>> + wait_cnt, (lh->next == lh->prev) ? "inactive" : "active");
>> + }
>> + fputs("}\n", fp);
>> +
>> + FREEBUF(sbq_wait_state_buf);
>> +
>> + fprintf(fp, "round_robin = %d\n", sqc->round_robin);
>> + fprintf(fp, "min_shallow_depth = %u\n", sqc->min_shallow_depth);
>> +}
>> +
>> +static void sbitmap_queue_context_load(ulong addr, struct sbitmap_queue_context *sqc)
>> +{
>> + char *sbitmap_queue_buf;
>> +
>> + sqc->sb_addr = addr + OFFSET(sbitmap_queue_sb);
>> +
>> + sbitmap_queue_buf = GETBUF(SIZE(sbitmap_queue));
>> + readmem(addr, KVADDR, sbitmap_queue_buf, SIZE(sbitmap_queue), "sbitmap_queue", FAULT_ON_ERROR);
>> +
>> + sqc->alloc_hint = ULONG(sbitmap_queue_buf + OFFSET(sbitmap_queue_alloc_hint));
>> + sqc->wake_batch = UINT(sbitmap_queue_buf + OFFSET(sbitmap_queue_wake_batch));
>> + sqc->wake_index = INT(sbitmap_queue_buf + OFFSET(sbitmap_queue_wake_index));
>> + sqc->ws_addr = ULONG(sbitmap_queue_buf + OFFSET(sbitmap_queue_ws));
>> + sqc->ws_active = INT(sbitmap_queue_buf + OFFSET(sbitmap_queue_ws_active));
>> + sqc->round_robin = BOOL(sbitmap_queue_buf + OFFSET(sbitmap_queue_round_robin));
>> + sqc->min_shallow_depth = UINT(sbitmap_queue_buf + OFFSET(sbitmap_queue_min_shallow_depth));
>> +
>> + FREEBUF(sbitmap_queue_buf);
>> +}
>> +
>> +void sbitmap_context_load(ulong addr, struct sbitmap_context *sc)
>> +{
>> + char *sbitmap_buf;
>> +
>> + sbitmap_buf = GETBUF(SIZE(sbitmap));
>> + readmem(addr, KVADDR, sbitmap_buf, SIZE(sbitmap), "sbitmap", FAULT_ON_ERROR);
>> +
>> + sc->depth = UINT(sbitmap_buf + OFFSET(sbitmap_depth));
>> + sc->shift = UINT(sbitmap_buf + OFFSET(sbitmap_shift));
>> + sc->map_nr = UINT(sbitmap_buf + OFFSET(sbitmap_map_nr));
>> + sc->map_addr = ULONG(sbitmap_buf + OFFSET(sbitmap_map));
>> +
>> + FREEBUF(sbitmap_buf);
>> +}
>> +
>> +static bool for_each_func(unsigned int idx, void *p)
>> +{
>> + struct sbitmapq_ops *ops = p;
>> + ulong addr = ops->addr + (ops->size * idx);
>> +
>> + return ops->fn(idx, addr, ops->p);
>> +}
>> +
>> +void sbitmapq_for_each_set(ulong addr, struct sbitmapq_ops *ops)
>> +{
>> + struct sbitmap_queue_context sqc = {0};
>> + struct sbitmap_context sc = {0};
>> +
>> + sbitmap_queue_context_load(addr, &sqc);
>> + sbitmap_context_load(sqc.sb_addr, &sc);
>> +
>> + sbitmap_for_each_set(&sc, for_each_func, ops);
>> +}
>> +
>> +static void dump_struct_members(const char *s, ulong addr, unsigned radix)
>> +{
>> + int i, argc;
>> + char *p1, *p2;
>> + char *structname, *members;
>> + char *arglist[MAXARGS];
>> +
>> + structname = GETBUF(strlen(s) + 1);
>> + members = GETBUF(strlen(s) + 1);
>> +
>> + strcpy(structname, s);
>> + p1 = strstr(structname, ".") + 1;
>> +
>> + p2 = strstr(s, ".") + 1;
>> + strcpy(members, p2);
>> + replace_string(members, ",", ' ');
>> + argc = parse_line(members, arglist);
>> +
>> + for (i = 0; i < argc; i++) {
>> + *p1 = NULLCHAR;
>> + strcat(structname, arglist[i]);
>> + dump_struct_member(structname, addr, radix);
>> + }
>> +
>> + FREEBUF(structname);
>> + FREEBUF(members);
>> +}
>> +
>> +static bool sbitmap_data_print(unsigned int idx, ulong addr, void *p)
>> +{
>> + const struct sbitmapq_data *sd = p;
>> + bool verbose = !!(sd->flags & VERBOSE);
>> + bool members = !!(sd->flags & SBITMAPQ_DATA_FLAG_STRUCT_MEMBER);
>> +
>> + if (verbose) {
>> + fprintf(fp, "%d (0x%08lx):\n", idx, addr);
>> + if (members)
>> + dump_struct_members(sd->data_name, addr, sd->radix);
>> + else
>> + dump_struct(sd->data_name, addr, sd->radix);
>> + } else
>> + fprintf(fp, "%d: 0x%08lx\n", idx, addr);
>> +
>> + return true;
>> +}
>> +
>> +static void sbitmap_queue_data_dump(struct sbitmapq_data *sd)
>> +{
>> + struct sbitmapq_ops ops = {
>> + .addr = sd->data_addr,
>> + .size = sd->data_size,
>> + .fn = sbitmap_data_print,
>> + .p = sd
>> + };
>> +
>> + sbitmapq_for_each_set(sd->addr, &ops);
>> +}
>> +
>> +static void sbitmap_queue_dump(const struct sbitmapq_data *sd)
>> +{
>> + struct sbitmap_queue_context sqc ={0};
>> + struct sbitmap_context sc = {0};
>> +
>> + sbitmap_queue_context_load(sd->addr, &sqc);
>> + sbitmap_context_load(sqc.sb_addr, &sc);
>> +
>> + sbitmap_queue_show(&sqc, &sc);
>> + fputc('\n', fp);
>> + sbitmap_bitmap_show(&sc);
>> +}
>> +
>> +void sbitmapq_init(void)
>> +{
>> + if (sb_flags & SB_FLAG_INIT)
>> + return;
>> +
>> + STRUCT_SIZE_INIT(sbitmap_word, "sbitmap_word");
>> + STRUCT_SIZE_INIT(sbitmap, "sbitmap");
>> + STRUCT_SIZE_INIT(sbitmap_queue, "sbitmap_queue");
>> + STRUCT_SIZE_INIT(sbq_wait_state, "sbq_wait_state");
>> +
>> + MEMBER_OFFSET_INIT(sbitmap_word_depth, "sbitmap_word", "depth");
>> + MEMBER_OFFSET_INIT(sbitmap_word_word, "sbitmap_word", "word");
>> + MEMBER_OFFSET_INIT(sbitmap_word_cleared, "sbitmap_word", "cleared");
>> +
>> + MEMBER_OFFSET_INIT(sbitmap_depth, "sbitmap", "depth");
>> + MEMBER_OFFSET_INIT(sbitmap_shift, "sbitmap", "shift");
>> + MEMBER_OFFSET_INIT(sbitmap_map_nr, "sbitmap", "map_nr");
>> + MEMBER_OFFSET_INIT(sbitmap_map, "sbitmap", "map");
>> +
>> + MEMBER_OFFSET_INIT(sbitmap_queue_sb, "sbitmap_queue", "sb");
>> + MEMBER_OFFSET_INIT(sbitmap_queue_alloc_hint, "sbitmap_queue", "alloc_hint");
>> + MEMBER_OFFSET_INIT(sbitmap_queue_wake_batch, "sbitmap_queue", "wake_batch");
>> + MEMBER_OFFSET_INIT(sbitmap_queue_wake_index, "sbitmap_queue", "wake_index");
>> + MEMBER_OFFSET_INIT(sbitmap_queue_ws, "sbitmap_queue", "ws");
>> + MEMBER_OFFSET_INIT(sbitmap_queue_ws_active, "sbitmap_queue", "ws_active");
>> + MEMBER_OFFSET_INIT(sbitmap_queue_round_robin, "sbitmap_queue", "round_robin");
>> + MEMBER_OFFSET_INIT(sbitmap_queue_min_shallow_depth, "sbitmap_queue", "min_shallow_depth");
>> +
>> + MEMBER_OFFSET_INIT(sbq_wait_state_wait_cnt, "sbq_wait_state", "wait_cnt");
>> + MEMBER_OFFSET_INIT(sbq_wait_state_wait, "sbq_wait_state", "wait");
>> +
>> + if (!VALID_SIZE(sbitmap_word) ||
>> + !VALID_SIZE(sbitmap) ||
>> + !VALID_SIZE(sbitmap_queue) ||
>> + !VALID_SIZE(sbq_wait_state) ||
>> + INVALID_MEMBER(sbitmap_word_depth) ||
>> + INVALID_MEMBER(sbitmap_word_word) ||
>> + INVALID_MEMBER(sbitmap_word_cleared) ||
>> + INVALID_MEMBER(sbitmap_depth) ||
>> + INVALID_MEMBER(sbitmap_shift) ||
>> + INVALID_MEMBER(sbitmap_map_nr) ||
>> + INVALID_MEMBER(sbitmap_map) ||
>> + INVALID_MEMBER(sbitmap_queue_sb) ||
>> + INVALID_MEMBER(sbitmap_queue_alloc_hint) ||
>> + INVALID_MEMBER(sbitmap_queue_wake_batch) ||
>> + INVALID_MEMBER(sbitmap_queue_wake_index) ||
>> + INVALID_MEMBER(sbitmap_queue_ws) ||
>> + INVALID_MEMBER(sbitmap_queue_ws_active) ||
>> + INVALID_MEMBER(sbitmap_queue_round_robin) ||
>> + INVALID_MEMBER(sbitmap_queue_min_shallow_depth) ||
>> + INVALID_MEMBER(sbq_wait_state_wait_cnt) ||
>> + INVALID_MEMBER(sbq_wait_state_wait)) {
>> + command_not_supported();
>> + }
>> +
>> + sb_flags |= SB_FLAG_INIT;
>> +}
>> +
>> +static char *__get_struct_name(const char *s)
>> +{
>> + char *name, *p;
>> +
>> + name = GETBUF(strlen(s) + 1);
>> + strcpy(name, s);
>> +
>> + p = strstr(name, ".");
>> + *p = NULLCHAR;
>> +
>> + return name;
>> +}
>> +
>> +void cmd_sbitmapq(void)
>> +{
>> + struct sbitmapq_data sd = {0};
>> + int c;
>> +
>> + while ((c = getopt(argcnt, args, "s:p:xdv")) != EOF) {
>> + switch (c) {
>> + case 's':
>> + if (sd.flags & SBITMAPQ_DATA_FLAG_STRUCT_NAME)
>> + error(FATAL, "-s option (%s) already entered\n", sd.data_name);
>> +
>> + sd.data_name = optarg;
>> + sd.flags |= SBITMAPQ_DATA_FLAG_STRUCT_NAME;
>> +
>> + break;
>> +
>> + case 'p':
>> + if (sd.flags & SBITMAPQ_DATA_FLAG_STRUCT_ADDR)
>> + error(FATAL, "-p option (0x%lx) already entered\n", sd.data_addr);
>> + else if (!IS_A_NUMBER(optarg))
>> + error(FATAL, "invalid -p option: %s\n", optarg);
>> +
>> + sd.data_addr = htol(optarg, FAULT_ON_ERROR, NULL);
>> + if (!IS_KVADDR(sd.data_addr))
>> + error(FATAL, "invalid kernel virtual address: %s\n", optarg);
>> + sd.flags |= SBITMAPQ_DATA_FLAG_STRUCT_ADDR;
>> +
>> + break;
>> +
>> + case 'v':
>> + sd.flags |= VERBOSE;
>> + break;
>> +
>> + case 'x':
>> + if (sd.radix == 10)
>> + error(FATAL, "-d and -x are mutually exclusive\n");
>> + sd.radix = 16;
>> + break;
>> +
>> + case 'd':
>> + if (sd.radix == 16)
>> + error(FATAL, "-d and -x are mutually exclusive\n");
>> + sd.radix = 10;
>> + break;
>> +
>> + default:
>> + argerrs++;
>> + break;
>> + }
>> + }
>> +
>> + if (argerrs)
>> + cmd_usage(pc->curcmd, SYNOPSIS);
>> +
>> + if (!args[optind]) {
>> + error(INFO, "command argument is required\n");
>> + cmd_usage(pc->curcmd, SYNOPSIS);
>> + } else if (args[optind] && args[optind + 1]) {
>> + error(INFO, "too many arguments\n");
>> + cmd_usage(pc->curcmd, SYNOPSIS);
>> + } else if (!IS_A_NUMBER(args[optind])) {
>> + error(FATAL, "invalid command argument: %s\n", args[optind]);
>> + }
>> +
>> + sd.addr = htol(args[optind], FAULT_ON_ERROR, NULL);
>> + if (!IS_KVADDR(sd.addr))
>> + error(FATAL, "invalid kernel virtual address: %s\n", args[optind]);
>> +
>> + if ((sd.flags & SBITMAPQ_DATA_FLAG_STRUCT_NAME) &&
>> + !(sd.flags & SBITMAPQ_DATA_FLAG_STRUCT_ADDR)) {
>> + error(INFO, "-s option requires -p option");
>> + cmd_usage(pc->curcmd, SYNOPSIS);
>> + } else if ((sd.flags & SBITMAPQ_DATA_FLAG_STRUCT_ADDR) &&
>> + !(sd.flags & SBITMAPQ_DATA_FLAG_STRUCT_NAME)) {
>> + error(FATAL, "-p option is used with -s option only\n");
>> + cmd_usage(pc->curcmd, SYNOPSIS);
>> + }
>> +
>> + if (sd.flags & SBITMAPQ_DATA_FLAG_STRUCT_NAME) {
>> + bool error_flag = false;
>> +
>> + if (count_chars(sd.data_name, '.') > 0)
>> + sd.flags |= SBITMAPQ_DATA_FLAG_STRUCT_MEMBER;
>> +
>> + if (sd.flags & SBITMAPQ_DATA_FLAG_STRUCT_MEMBER) {
>> + char *data_name = __get_struct_name(sd.data_name);
>> +
>> + sd.data_size = STRUCT_SIZE(data_name);
>> + if (sd.data_size <= 0)
>> + error_flag = true;
>> +
>> + FREEBUF(data_name);
>> + } else {
>> + sd.data_size = STRUCT_SIZE(sd.data_name);
>> + if (sd.data_size <= 0)
>> + error_flag = true;
>> + }
>> + if (error_flag)
>> + error(FATAL, "invalid data structure reference: %s\n", sd.data_name);
>> + }
> The error_flag can be removed and this block can be shortened,
> will change like this:
>
> if (sd.flags & SBITMAPQ_DATA_FLAG_STRUCT_NAME) {
> if (count_chars(sd.data_name, '.') > 0)
> sd.flags |= SBITMAPQ_DATA_FLAG_STRUCT_MEMBER;
>
> if (sd.flags & SBITMAPQ_DATA_FLAG_STRUCT_MEMBER) {
> char *data_name = __get_struct_name(sd.data_name);
> sd.data_size = STRUCT_SIZE(data_name);
> FREEBUF(data_name);
> } else
> sd.data_size = STRUCT_SIZE(sd.data_name);
>
> if (sd.data_size <= 0)
> error(FATAL, "invalid data structure reference: %s\n", sd.data_name);
> }
>
> Thanks,
> Kazu
Agree, It looks better. Fixed.
>> +
>> + sbitmapq_init();
>> +
>> + if (sd.flags & SBITMAPQ_DATA_FLAG_STRUCT_NAME)
>> + sbitmap_queue_data_dump(&sd);
>> + else
>> + sbitmap_queue_dump(&sd);
>> +}
>> --
>> 2.25.1
>>
>>
>> --
>> Crash-utility mailing list
>> Crash-utility@xxxxxxxxxx
>> https://listman.redhat.com/mailman/listinfo/crash-utility
Thanks,
Sergey
--
Crash-utility mailing list
Crash-utility@xxxxxxxxxx
https://listman.redhat.com/mailman/listinfo/crash-utility
