Hi Dave, Thank you so much for your review and crash-4.0-4.2. And I am sorry for delaying my reply for a long time. I updated my extension for crash-4.0-4.2. Dave Anderson wrote: > The systemtaplog_init() function must be renamed "_init()" in order > for it to be called when the dlopen() call is made. Otherwise, > the extend command fails because no commands get registered: OK, I changed systemtaplog_init() to _init(). > The only other thing that makes me a little nervous is the temporary > relocation of the setjmp buffer. That was never meant to be used > by individual commands -- although nothing prevents it. I see that it > is used to force your cleanup() routine to be called. And your > cleanup() routine exists to free() the subbuf buffer and fclose() > the outfp. A couple suggestions: > > (1) if you use GETBUF() instead of malloc(), then the buffer will get freed > automatically by restore_sanity() prior to the next command prompt > -- even > if you don't do an accompanying FREEBUF(). > (2) with respect to the outfp file pointer, you could recognize whether it > was left open when the command runs again, and fclose() it then. Worse > case, a single file pointer would be left open for the remainder of the > crash session. > That's right, and I agreed with you. I'll stop relocation and implement this part based on your suggestions. > [...]So perhaps I could simply allow modules to register an additional > command with a new "CLEANUP" flag, which would not show up on the help > menu, but would be called during restore_sanity() prior to each > command prompt. Thanks to the new function of crash-4.0-4.2, I can register an cleanup routine of my extension. And I changed setup_global_data() to use symbol_value_module() which was introduced in crash-4.0-4.2 instead of my original function. Additionally, I changed command option. I removed -m option and added -o option. I can specify the output directory using -o option. You can use this extension like this: Preparation ============== (A) Build the shared-object(stplog.so). 1. Install crash-devel package. $ rpm -ivh crash-devel 2. $ cd libcrash_for_systemtap 3. Build $ make (B) Make the crash dump which includes SystemTap trace data. (*)If you analyze the live system memory, ignore this section. 1. Install kdump If you use FC6, see following URL. http://fedoraproject.org/wiki/FC6KdumpKexecHowTo?highlight=%28kdump%29 2. Use SystemTap on bulkmode $ stap -b foo.stp 3. Panic $ echo c > /proc/sysrq-trigger How to use ============== 1. start crash $ crash vmlinux vmcore (*) If you analyze the live system memory, you don't have to enter vmcore. 2. load the shared-object crash> extend $(WHERE_OBJ_PLACED)/stplog.so 3. retrieve the data crash> stplog -o output_dir mod_name 4. You can get output files under output_dir directory. Output ============== stplog command makes a file per cpu buffer of relayfs. Of cause, it also removes the padding bytes in the subbufs of relayfs automatically. Best Regards, -- --- Satoru MORIYA Linux Technology Center Hitachi, Ltd., Systems Development Laboratory E-mail: satoru.moriya.br@xxxxxxxxxxx
/* crash shared object for retrieving systemtap buffer Copyright (c) 2007 Hitachi,Ltd., Created by Satoru Moriya <satoru.moriya.br@xxxxxxxxxxx> This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ #include <crash/defs.h> #define STPLOG_NO_MOD -1 #define STPLOG_NO_SYM -2 struct rchan_offsets { long subbuf_size; long n_subbufs; long buf; long buf_start; long buf_offset; long buf_subbufs_produced; long buf_padding; }; struct fake_rchan_buf { void *start; size_t offset; size_t subbufs_produced; size_t *padding; }; struct fake_rchan { size_t subbuf_size; size_t n_subbufs; }; struct per_cpu_data { struct fake_rchan_buf buf; }; static struct rchan_offsets rchan_offsets; static struct fake_rchan chan; static struct per_cpu_data per_cpu[NR_CPUS]; static FILE *outfp; static char *subbuf; static int is_global; static int old_format; void cmd_stplog(void); void cmd_stplog_cleanup(void); char *help_stplog[]; char *help_stplog_cleanup[]; static struct command_table_entry command_table[] = { {"stplog", cmd_stplog, help_stplog, 0}, {"stplog_cleanup", cmd_stplog_cleanup, help_stplog_cleanup, CLEANUP}, {NULL, NULL, NULL, 0}, }; static void get_rchan_offsets(void) { rchan_offsets.subbuf_size = MEMBER_OFFSET("rchan", "subbuf_size"); if (rchan_offsets.subbuf_size < 0) goto ERR; rchan_offsets.n_subbufs = MEMBER_OFFSET("rchan", "n_subbufs"); if (rchan_offsets.n_subbufs < 0) goto ERR; rchan_offsets.buf = MEMBER_OFFSET("rchan", "buf"); if (rchan_offsets.buf < 0) goto ERR; rchan_offsets.buf_start = MEMBER_OFFSET("rchan_buf", "start"); if (rchan_offsets.buf_start < 0) goto ERR; rchan_offsets.buf_offset = MEMBER_OFFSET("rchan_buf", "offset"); if (rchan_offsets.buf_offset < 0) goto ERR; rchan_offsets.buf_subbufs_produced = MEMBER_OFFSET("rchan_buf", "subbufs_produced"); if (rchan_offsets.buf_subbufs_produced < 0) goto ERR; rchan_offsets.buf_padding = MEMBER_OFFSET("rchan_buf", "padding"); if (rchan_offsets.buf_padding < 0) goto ERR; return; ERR: error(FATAL, "cannot get rchan offset\n"); } static ulong get_rchan(ulong chan_addr) { ulong rchan; readmem(chan_addr, KVADDR, &rchan, sizeof(void*), "stp_channel", FAULT_ON_ERROR); readmem(rchan + rchan_offsets.subbuf_size, KVADDR, &chan.subbuf_size, sizeof(size_t), "stp_channel.subbuf_size", FAULT_ON_ERROR); readmem(rchan + rchan_offsets.n_subbufs, KVADDR, &chan.n_subbufs, sizeof(size_t), "stp_channel.n_subbufs", FAULT_ON_ERROR); return rchan; } static void get_rchan_buf(int cpu, ulong rchan) { ulong rchan_buf; struct per_cpu_data *pcd; pcd = &per_cpu[cpu]; readmem(rchan + rchan_offsets.buf + sizeof(void*) * cpu, KVADDR, &rchan_buf, sizeof(void*), "stp_channel.buf", FAULT_ON_ERROR); readmem(rchan_buf + rchan_offsets.buf_start, KVADDR, &pcd->buf.start, sizeof(void*), "stp_channel.buf.start", FAULT_ON_ERROR); readmem(rchan_buf + rchan_offsets.buf_offset, KVADDR, &pcd->buf.offset, sizeof(size_t), "stp_channel.buf.offset", FAULT_ON_ERROR); readmem(rchan_buf + rchan_offsets.buf_subbufs_produced, KVADDR, &pcd->buf.subbufs_produced, sizeof(size_t), "stp_channel.buf.subbufs_produced", FAULT_ON_ERROR); readmem(rchan_buf + rchan_offsets.buf_padding, KVADDR, &pcd->buf.padding, sizeof(size_t*), "stp_channel.buf.padding", FAULT_ON_ERROR); return; } static ulong get_rchan_addr(ulong stp_utt_addr) { ulong stp_utt; readmem(stp_utt_addr, KVADDR, &stp_utt, sizeof(void*), "stp_utt", FAULT_ON_ERROR); return (stp_utt + sizeof(int)); } static int check_global_buffer(ulong rchan) { int cpu; ulong rchan_buf[2]; for (cpu = 0; cpu < 2; cpu++) { readmem(rchan + rchan_offsets.buf + sizeof(void*) * cpu, KVADDR, &rchan_buf[cpu], sizeof(void*), "stp_channel.buf", FAULT_ON_ERROR); } if (rchan_buf[0] == rchan_buf[1]) return 1; return 0; } static void setup_global_data(char *module) { int i; ulong stp_utt_addr = 0; ulong stp_rchan_addr = 0; ulong rchan; stp_utt_addr = symbol_value_module("_stp_utt", module); if (stp_utt_addr == 0) { stp_rchan_addr = symbol_value_module("_stp_chan", module); if (stp_rchan_addr == 0) { error(FATAL, "Failed to find _stp_utt/_stp_chan.\n", module); } old_format = 1; } else { stp_rchan_addr = get_rchan_addr(stp_utt_addr); if (stp_rchan_addr == 0) { error(FATAL, "Failed to find _stp_utt/_stp_chan.\n", module); } } rchan = get_rchan(stp_rchan_addr); for (i = 0; i < kt->cpus; i++) get_rchan_buf(i, rchan); if (kt->cpus > 1) { is_global = check_global_buffer(rchan); } return; } static void output_cpu_logs(char *filename) { int i, max = 256; struct per_cpu_data *pcd; size_t n, idx, start, end, ready, len; unsigned padding; char fname[max + 1], *source; DIR *dir; /* check and create log directory */ dir = opendir(filename); if (dir) { closedir(dir); } else { if (mkdir(filename, S_IRWXU) < 0) { error(FATAL, "cannot create log directory '%s\n'", filename); } } /* allocate subbuf memory */ subbuf = GETBUF(chan.subbuf_size); if (!subbuf) { error(FATAL, "cannot allocate memory\n"); } fname[max] = '\0'; for (i = 0; i < kt->cpus; i++) { int adjust = 0; pcd = &per_cpu[i]; if (pcd->buf.offset == 0 || pcd->buf.offset == chan.subbuf_size + 1) { adjust = 0; } else { adjust = 1; } ready = pcd->buf.subbufs_produced + adjust; if (ready > chan.n_subbufs) { start = ready; end = start + chan.n_subbufs; } else { start = 0; end = ready; } /* print information */ fprintf(fp, "--- generating 'cpu%d' ---\n", i); fprintf(fp, " subbufs ready on relayfs:%ld\n", (long)ready); fprintf(fp, " n_subbufs:%ld, read from:%ld to:%ld (offset:%ld)\n\n", (long)chan.n_subbufs, (long)start, (long)end, (long)pcd->buf.offset); /* create log file */ snprintf(fname, max, "%s/cpu%d", filename, i); outfp = fopen(fname, "w"); if (!outfp) { error(FATAL, "cannot create log file '%s'\n", fname); } for (n = start; n < end; n++) { /* read relayfs subbufs and write to log file */ idx = n % chan.n_subbufs; source = pcd->buf.start + idx * chan.subbuf_size; readmem((ulong)pcd->buf.padding + sizeof(padding) * idx, KVADDR, &padding, sizeof(padding), "padding", FAULT_ON_ERROR); if (n == end - 1 && 0 < pcd->buf.offset && pcd->buf.offset < chan.subbuf_size) { len = pcd->buf.offset; } else { len = chan.subbuf_size; } if (old_format == 1) { source += sizeof(padding); len -= sizeof(padding) + padding; } else { len -= padding; } if (len) { readmem((ulong)source, KVADDR, subbuf, len, "subbuf", FAULT_ON_ERROR); if (fwrite(subbuf, len, 1, outfp) != 1) { error(FATAL, "cannot write log data\n"); } } } fclose(outfp); outfp = NULL; if (is_global == 1) break; } if (subbuf) { FREEBUF(subbuf); subbuf = NULL; } return; } static void do_stplog(char *module, char *filename) { setup_global_data(module); output_cpu_logs(filename); return; } void cmd_stplog(void) { int c; char *module = NULL; char *filename = NULL; while ((c = getopt(argcnt, args, "o:")) != EOF) { switch (c) { case 'o': filename = optarg; break; default: argerrs++; break; } } module = args[optind]; if (!module || argerrs) cmd_usage(pc->curcmd, SYNOPSIS); if (filename == NULL && module != NULL) filename = module; do_stplog(module, filename); return; } void cmd_stplog_cleanup(void) { if (outfp) { fclose(outfp); outfp = NULL; } return; } char *help_stplog[] = { "systemtaplog", "Retrieve SystemTap log data", "[-o dir_name] module_name", " Retrieve SystemTap's log data and write them to files.\n", " module_name All valid SystemTap log data made by the trace", " module which name is 'module_name' are written", " into log files. If you don't use -o option, the", " log files are created in `module_name` directory.", " The name of each log file is cpu0, cpu1...cpuN. ", " They have same format data as channel buffer", " except padding(This command removes padding). ", "", " -o file_name Specify the output directory.", NULL, }; char *help_stplog_cleanup[] = { "systemtaplog cleanup (hidden)", "Cleanup command for stplog", "", " This command is called during restore_sanity() prior to each ", " command prompt to close the files which was opened and failed to", " close by stplog command.", NULL, }; static void __attribute__ ((constructor)) _init(void) { get_rchan_offsets(); register_extension(command_table); return; } static void __attribute__ ((destructor)) _fini(void) { return; }
TARGET=stplog.so CFILE=stplog.c CFLAGS= -shared -rdynamic CFLAGS+= -I./crash -Wall PRJNAME=libcrash_for_systemtap VERSION=`date +%Y%m%d` ARCH=$(shell uname -i) ifeq ($(ARCH), i386) CFLAGS += -DX86 else ifeq ($(ARCH), ia64) CFLAGS += -DIA64 else ifeq ($(ARCH), x86_64) CFLAGS += -DX86_64 endif endif endif $(TARGET):$(CFILE) echo $(CFLAGS) gcc $(CFLAGS) -o $@ $(CFILE) clean: rm -f -r $(TARGET) *~ dist:distclean mkdir $(PRJNAME)-$(VERSION) cp $(CFILE) Makefile README $(PRJNAME)-$(VERSION) tar cvjf $(PRJNAME)-$(VERSION).tar.bz2 $(PRJNAME)-$(VERSION) rm -f -r $(PRJNAME)-$(VERSION) distclean: rm -f -r $(TARGET) *~ crash
-- Crash-utility mailing list Crash-utility@xxxxxxxxxx https://www.redhat.com/mailman/listinfo/crash-utility