Hi Dave,
Thank you so much for your review and crash-4.0-4.2.
And I am sorry for delaying my reply for a long time.
I updated my extension for crash-4.0-4.2.
Dave Anderson wrote:
> The systemtaplog_init() function must be renamed "_init()" in order
> for it to be called when the dlopen() call is made. Otherwise,
> the extend command fails because no commands get registered:
OK, I changed systemtaplog_init() to _init().
> The only other thing that makes me a little nervous is the temporary
> relocation of the setjmp buffer. That was never meant to be used
> by individual commands -- although nothing prevents it. I see that it
> is used to force your cleanup() routine to be called. And your
> cleanup() routine exists to free() the subbuf buffer and fclose()
> the outfp. A couple suggestions:
>
> (1) if you use GETBUF() instead of malloc(), then the buffer will get freed
> automatically by restore_sanity() prior to the next command prompt
> -- even
> if you don't do an accompanying FREEBUF().
> (2) with respect to the outfp file pointer, you could recognize whether it
> was left open when the command runs again, and fclose() it then. Worse
> case, a single file pointer would be left open for the remainder of the
> crash session.
>
That's right, and I agreed with you. I'll stop relocation and implement this
part based on your suggestions.
> [...]So perhaps I could simply allow modules to register an additional
> command with a new "CLEANUP" flag, which would not show up on the help
> menu, but would be called during restore_sanity() prior to each
> command prompt.
Thanks to the new function of crash-4.0-4.2, I can register an cleanup
routine of my extension. And I changed setup_global_data() to use
symbol_value_module() which was introduced in crash-4.0-4.2 instead of
my original function.
Additionally, I changed command option. I removed -m option and added
-o option. I can specify the output directory using -o option.
You can use this extension like this:
Preparation
==============
(A) Build the shared-object(stplog.so).
1. Install crash-devel package.
$ rpm -ivh crash-devel
2. $ cd libcrash_for_systemtap
3. Build
$ make
(B) Make the crash dump which includes SystemTap trace data.
(*)If you analyze the live system memory, ignore this section.
1. Install kdump
If you use FC6, see following URL.
http://fedoraproject.org/wiki/FC6KdumpKexecHowTo?highlight=%28kdump%29
2. Use SystemTap on bulkmode
$ stap -b foo.stp
3. Panic
$ echo c > /proc/sysrq-trigger
How to use
==============
1. start crash
$ crash vmlinux vmcore
(*) If you analyze the live system memory, you don't have to enter vmcore.
2. load the shared-object
crash> extend $(WHERE_OBJ_PLACED)/stplog.so
3. retrieve the data
crash> stplog -o output_dir mod_name
4. You can get output files under output_dir directory.
Output
==============
stplog command makes a file per cpu buffer of relayfs. Of cause, it also
removes the padding bytes in the subbufs of relayfs automatically.
Best Regards,
--
---
Satoru MORIYA
Linux Technology Center
Hitachi, Ltd., Systems Development Laboratory
E-mail: satoru.moriya.br@xxxxxxxxxxx
/*
crash shared object for retrieving systemtap buffer
Copyright (c) 2007 Hitachi,Ltd.,
Created by Satoru Moriya <satoru.moriya.br@xxxxxxxxxxx>
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 2 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
*/
#include <crash/defs.h>
#define STPLOG_NO_MOD -1
#define STPLOG_NO_SYM -2
struct rchan_offsets {
long subbuf_size;
long n_subbufs;
long buf;
long buf_start;
long buf_offset;
long buf_subbufs_produced;
long buf_padding;
};
struct fake_rchan_buf {
void *start;
size_t offset;
size_t subbufs_produced;
size_t *padding;
};
struct fake_rchan {
size_t subbuf_size;
size_t n_subbufs;
};
struct per_cpu_data {
struct fake_rchan_buf buf;
};
static struct rchan_offsets rchan_offsets;
static struct fake_rchan chan;
static struct per_cpu_data per_cpu[NR_CPUS];
static FILE *outfp;
static char *subbuf;
static int is_global;
static int old_format;
void cmd_stplog(void);
void cmd_stplog_cleanup(void);
char *help_stplog[];
char *help_stplog_cleanup[];
static struct command_table_entry command_table[] = {
{"stplog", cmd_stplog, help_stplog, 0},
{"stplog_cleanup", cmd_stplog_cleanup, help_stplog_cleanup, CLEANUP},
{NULL, NULL, NULL, 0},
};
static void get_rchan_offsets(void)
{
rchan_offsets.subbuf_size = MEMBER_OFFSET("rchan", "subbuf_size");
if (rchan_offsets.subbuf_size < 0)
goto ERR;
rchan_offsets.n_subbufs = MEMBER_OFFSET("rchan", "n_subbufs");
if (rchan_offsets.n_subbufs < 0)
goto ERR;
rchan_offsets.buf = MEMBER_OFFSET("rchan", "buf");
if (rchan_offsets.buf < 0)
goto ERR;
rchan_offsets.buf_start = MEMBER_OFFSET("rchan_buf", "start");
if (rchan_offsets.buf_start < 0)
goto ERR;
rchan_offsets.buf_offset = MEMBER_OFFSET("rchan_buf", "offset");
if (rchan_offsets.buf_offset < 0)
goto ERR;
rchan_offsets.buf_subbufs_produced
= MEMBER_OFFSET("rchan_buf", "subbufs_produced");
if (rchan_offsets.buf_subbufs_produced < 0)
goto ERR;
rchan_offsets.buf_padding = MEMBER_OFFSET("rchan_buf", "padding");
if (rchan_offsets.buf_padding < 0)
goto ERR;
return;
ERR:
error(FATAL, "cannot get rchan offset\n");
}
static ulong get_rchan(ulong chan_addr)
{
ulong rchan;
readmem(chan_addr, KVADDR, &rchan, sizeof(void*),
"stp_channel", FAULT_ON_ERROR);
readmem(rchan + rchan_offsets.subbuf_size,
KVADDR, &chan.subbuf_size, sizeof(size_t),
"stp_channel.subbuf_size", FAULT_ON_ERROR);
readmem(rchan + rchan_offsets.n_subbufs,
KVADDR, &chan.n_subbufs, sizeof(size_t),
"stp_channel.n_subbufs", FAULT_ON_ERROR);
return rchan;
}
static void get_rchan_buf(int cpu, ulong rchan)
{
ulong rchan_buf;
struct per_cpu_data *pcd;
pcd = &per_cpu[cpu];
readmem(rchan + rchan_offsets.buf + sizeof(void*) * cpu,
KVADDR, &rchan_buf, sizeof(void*),
"stp_channel.buf", FAULT_ON_ERROR);
readmem(rchan_buf + rchan_offsets.buf_start,
KVADDR, &pcd->buf.start, sizeof(void*),
"stp_channel.buf.start", FAULT_ON_ERROR);
readmem(rchan_buf + rchan_offsets.buf_offset,
KVADDR, &pcd->buf.offset, sizeof(size_t),
"stp_channel.buf.offset", FAULT_ON_ERROR);
readmem(rchan_buf + rchan_offsets.buf_subbufs_produced,
KVADDR, &pcd->buf.subbufs_produced, sizeof(size_t),
"stp_channel.buf.subbufs_produced", FAULT_ON_ERROR);
readmem(rchan_buf + rchan_offsets.buf_padding,
KVADDR, &pcd->buf.padding, sizeof(size_t*),
"stp_channel.buf.padding", FAULT_ON_ERROR);
return;
}
static ulong get_rchan_addr(ulong stp_utt_addr)
{
ulong stp_utt;
readmem(stp_utt_addr, KVADDR, &stp_utt, sizeof(void*),
"stp_utt", FAULT_ON_ERROR);
return (stp_utt + sizeof(int));
}
static int check_global_buffer(ulong rchan)
{
int cpu;
ulong rchan_buf[2];
for (cpu = 0; cpu < 2; cpu++) {
readmem(rchan + rchan_offsets.buf + sizeof(void*) * cpu,
KVADDR, &rchan_buf[cpu], sizeof(void*),
"stp_channel.buf", FAULT_ON_ERROR);
}
if (rchan_buf[0] == rchan_buf[1])
return 1;
return 0;
}
static void setup_global_data(char *module)
{
int i;
ulong stp_utt_addr = 0;
ulong stp_rchan_addr = 0;
ulong rchan;
stp_utt_addr = symbol_value_module("_stp_utt", module);
if (stp_utt_addr == 0) {
stp_rchan_addr = symbol_value_module("_stp_chan", module);
if (stp_rchan_addr == 0) {
error(FATAL, "Failed to find _stp_utt/_stp_chan.\n",
module);
}
old_format = 1;
} else {
stp_rchan_addr = get_rchan_addr(stp_utt_addr);
if (stp_rchan_addr == 0) {
error(FATAL, "Failed to find _stp_utt/_stp_chan.\n",
module);
}
}
rchan = get_rchan(stp_rchan_addr);
for (i = 0; i < kt->cpus; i++)
get_rchan_buf(i, rchan);
if (kt->cpus > 1) {
is_global = check_global_buffer(rchan);
}
return;
}
static void output_cpu_logs(char *filename)
{
int i, max = 256;
struct per_cpu_data *pcd;
size_t n, idx, start, end, ready, len;
unsigned padding;
char fname[max + 1], *source;
DIR *dir;
/* check and create log directory */
dir = opendir(filename);
if (dir) {
closedir(dir);
} else {
if (mkdir(filename, S_IRWXU) < 0) {
error(FATAL, "cannot create log directory '%s\n'", filename);
}
}
/* allocate subbuf memory */
subbuf = GETBUF(chan.subbuf_size);
if (!subbuf) {
error(FATAL, "cannot allocate memory\n");
}
fname[max] = '\0';
for (i = 0; i < kt->cpus; i++) {
int adjust = 0;
pcd = &per_cpu[i];
if (pcd->buf.offset == 0 ||
pcd->buf.offset == chan.subbuf_size + 1) {
adjust = 0;
} else {
adjust = 1;
}
ready = pcd->buf.subbufs_produced + adjust;
if (ready > chan.n_subbufs) {
start = ready;
end = start + chan.n_subbufs;
} else {
start = 0;
end = ready;
}
/* print information */
fprintf(fp, "--- generating 'cpu%d' ---\n", i);
fprintf(fp, " subbufs ready on relayfs:%ld\n", (long)ready);
fprintf(fp, " n_subbufs:%ld, read from:%ld to:%ld (offset:%ld)\n\n",
(long)chan.n_subbufs, (long)start, (long)end, (long)pcd->buf.offset);
/* create log file */
snprintf(fname, max, "%s/cpu%d", filename, i);
outfp = fopen(fname, "w");
if (!outfp) {
error(FATAL, "cannot create log file '%s'\n", fname);
}
for (n = start; n < end; n++) {
/* read relayfs subbufs and write to log file */
idx = n % chan.n_subbufs;
source = pcd->buf.start + idx * chan.subbuf_size;
readmem((ulong)pcd->buf.padding + sizeof(padding) * idx,
KVADDR, &padding, sizeof(padding),
"padding", FAULT_ON_ERROR);
if (n == end - 1 && 0 < pcd->buf.offset &&
pcd->buf.offset < chan.subbuf_size) {
len = pcd->buf.offset;
} else {
len = chan.subbuf_size;
}
if (old_format == 1) {
source += sizeof(padding);
len -= sizeof(padding) + padding;
} else {
len -= padding;
}
if (len) {
readmem((ulong)source, KVADDR, subbuf, len,
"subbuf", FAULT_ON_ERROR);
if (fwrite(subbuf, len, 1, outfp) != 1) {
error(FATAL, "cannot write log data\n");
}
}
}
fclose(outfp);
outfp = NULL;
if (is_global == 1)
break;
}
if (subbuf) {
FREEBUF(subbuf);
subbuf = NULL;
}
return;
}
static void do_stplog(char *module, char *filename)
{
setup_global_data(module);
output_cpu_logs(filename);
return;
}
void cmd_stplog(void)
{
int c;
char *module = NULL;
char *filename = NULL;
while ((c = getopt(argcnt, args, "o:")) != EOF) {
switch (c) {
case 'o':
filename = optarg;
break;
default:
argerrs++;
break;
}
}
module = args[optind];
if (!module || argerrs)
cmd_usage(pc->curcmd, SYNOPSIS);
if (filename == NULL && module != NULL)
filename = module;
do_stplog(module, filename);
return;
}
void cmd_stplog_cleanup(void)
{
if (outfp) {
fclose(outfp);
outfp = NULL;
}
return;
}
char *help_stplog[] = {
"systemtaplog",
"Retrieve SystemTap log data",
"[-o dir_name] module_name",
" Retrieve SystemTap's log data and write them to files.\n",
" module_name All valid SystemTap log data made by the trace",
" module which name is 'module_name' are written",
" into log files. If you don't use -o option, the",
" log files are created in `module_name` directory.",
" The name of each log file is cpu0, cpu1...cpuN. ",
" They have same format data as channel buffer",
" except padding(This command removes padding). ",
"",
" -o file_name Specify the output directory.",
NULL,
};
char *help_stplog_cleanup[] = {
"systemtaplog cleanup (hidden)",
"Cleanup command for stplog",
"",
" This command is called during restore_sanity() prior to each ",
" command prompt to close the files which was opened and failed to",
" close by stplog command.",
NULL,
};
static void __attribute__ ((constructor)) _init(void)
{
get_rchan_offsets();
register_extension(command_table);
return;
}
static void __attribute__ ((destructor)) _fini(void)
{
return;
}
TARGET=stplog.so
CFILE=stplog.c
CFLAGS= -shared -rdynamic
CFLAGS+= -I./crash -Wall
PRJNAME=libcrash_for_systemtap
VERSION=`date +%Y%m%d`
ARCH=$(shell uname -i)
ifeq ($(ARCH), i386)
CFLAGS += -DX86
else
ifeq ($(ARCH), ia64)
CFLAGS += -DIA64
else
ifeq ($(ARCH), x86_64)
CFLAGS += -DX86_64
endif
endif
endif
$(TARGET):$(CFILE)
echo $(CFLAGS)
gcc $(CFLAGS) -o $@ $(CFILE)
clean:
rm -f -r $(TARGET) *~
dist:distclean
mkdir $(PRJNAME)-$(VERSION)
cp $(CFILE) Makefile README $(PRJNAME)-$(VERSION)
tar cvjf $(PRJNAME)-$(VERSION).tar.bz2 $(PRJNAME)-$(VERSION)
rm -f -r $(PRJNAME)-$(VERSION)
distclean:
rm -f -r $(TARGET) *~ crash
--
Crash-utility mailing list
Crash-utility@xxxxxxxxxx
https://www.redhat.com/mailman/listinfo/crash-utility
