On Wed, Feb 14, 2018 at 12:03:31PM +0100, Jan Kara wrote:
> On Mon 12-02-18 13:46:48, Xiong Zhou wrote:
> > Stress test for fanotify and inotify. Exercise fanotify and
> > inotify user interfaces in loop while other stress tests going
> > on in the watched test directory.
> >
> > Watching slab object inotify_inode_mark size, report fail
> > it increases too fast. This may lead to a crash if OOM killer
> > invoked.
> >
> > kernel commit related to the fixes in v4.15-rc1:
> > 0d6ec07 fsnotify: pin both inode and vfsmount mark
> >
> > Signed-off-by: Xiong Zhou <xzhou@xxxxxxxxxx>
>
> I'm sorry for chiming in so late but I was on vacation. Just one question:
> Currently, all inotify and fanotify tests are part of LTP. Is there any
> good reason for putting this particular test to fstests and not LTP?
It's handy to test with bash and c. fstests is more convenient to do that.
Thanks,
Xiong
> Specifically I've refrained from putting notification framework tests to
> fstests because there's practically no relation of it to implementation of
> any particular filesystem. Also I'd prefer not to have fanotify / inotify
> tests in two different frameworks...
>
> Honza
>
> > ---
> > Thanks for reviewing!
> >
> > v4:
> > only list one commit in the series
> >
> > v3:
> > add wait in cleanup
> > add kernel commits fixed this issue in comments and commit msg
> > fix seq numbers
> > fix wording
> > v2:
> > add to dangerous group
> > new fsnotify group
> > watch inotify_inode_mark slab instead of free memory
> > watch inotify_inode_mark slab increasing speed instead
> > reduce running time to 2m since we are watching the speed
> > other than numbers
> > kill stress processes in cleanup
> >
> > .gitignore | 1 +
> > src/Makefile | 3 +-
> > src/fsnotify_stress.c | 339 ++++++++++++++++++++++++++++++++++++++++++++++++++
> > tests/generic/478 | 185 +++++++++++++++++++++++++++
> > tests/generic/478.out | 2 +
> > tests/generic/group | 1 +
> > 6 files changed, 530 insertions(+), 1 deletion(-)
> > create mode 100644 src/fsnotify_stress.c
> > create mode 100755 tests/generic/478
> > create mode 100644 tests/generic/478.out
> >
> > diff --git a/.gitignore b/.gitignore
> > index ee7eaed..ae01ed3 100644
> > --- a/.gitignore
> > +++ b/.gitignore
> > @@ -72,6 +72,7 @@
> > /src/fill
> > /src/fill2
> > /src/fs_perms
> > +/src/fsnotify_stress
> > /src/fssum
> > /src/fstest
> > /src/fsync-err
> > diff --git a/src/Makefile b/src/Makefile
> > index b96b8cf..6b9f296 100644
> > --- a/src/Makefile
> > +++ b/src/Makefile
> > @@ -14,7 +14,8 @@ TARGETS = dirstress fill fill2 getpagesize holes lstat64 \
> > t_mmap_writev t_truncate_cmtime dirhash_collide t_rename_overwrite \
> > holetest t_truncate_self t_mmap_dio af_unix t_mmap_stale_pmd \
> > t_mmap_cow_race t_mmap_fallocate fsync-err t_mmap_write_ro \
> > - t_ext4_dax_journal_corruption t_ext4_dax_inline_corruption
> > + t_ext4_dax_journal_corruption t_ext4_dax_inline_corruption \
> > + fsnotify_stress
> >
> > LINUX_TARGETS = xfsctl bstat t_mtab getdevicesize preallo_rw_pattern_reader \
> > preallo_rw_pattern_writer ftrunc trunc fs_perms testx looptest \
> > diff --git a/src/fsnotify_stress.c b/src/fsnotify_stress.c
> > new file mode 100644
> > index 0000000..f113918
> > --- /dev/null
> > +++ b/src/fsnotify_stress.c
> > @@ -0,0 +1,339 @@
> > +#ifndef _GNU_SOURCE
> > +#define _GNU_SOURCE /* Needed to get O_LARGEFILE definition */
> > +#endif
> > +#include <errno.h>
> > +#include <fcntl.h>
> > +#include <limits.h>
> > +#include <poll.h>
> > +#include <stdio.h>
> > +#include <stdlib.h>
> > +#include <sys/fanotify.h>
> > +#include <unistd.h>
> > +#include <string.h>
> > +#include <sys/types.h>
> > +#include <sys/stat.h>
> > +#include <dirent.h>
> > +#include <sys/inotify.h>
> > +
> > +static void handle_events(int fd)
> > +{
> > + const struct fanotify_event_metadata *metadata;
> > + struct fanotify_event_metadata buf[200];
> > + ssize_t len;
> > + struct fanotify_response response;
> > +
> > + /* Loop while events can be read from fanotify file descriptor */
> > + for(;;) {
> > + /* Read some events */
> > + len = read(fd, (void *) &buf, sizeof(buf));
> > + if (len == -1 && errno != EAGAIN) {
> > + perror("read");
> > + exit(EXIT_FAILURE);
> > + }
> > + /* Check if end of available data reached */
> > + if (len <= 0)
> > + break;
> > + /* Point to the first event in the buffer */
> > + metadata = buf;
> > + /* Loop over all events in the buffer */
> > + while (FAN_EVENT_OK(metadata, len)) {
> > + /* Check that run-time and compile-time structures match */
> > + if (metadata->vers != FANOTIFY_METADATA_VERSION) {
> > + fprintf(stderr,
> > + "Mismatch of fanotify metadata version.\n");
> > + exit(EXIT_FAILURE);
> > + }
> > + if (metadata->fd >= 0) {
> > + /* Handle open permission event */
> > + if (metadata->mask & FAN_OPEN_PERM) {
> > + /* Allow file to be opened */
> > + response.fd = metadata->fd;
> > + response.response = FAN_ALLOW;
> > + write(fd, &response,
> > + sizeof(struct fanotify_response));
> > + }
> > + /* Handle access permission event */
> > + if (metadata->mask & FAN_ACCESS_PERM) {
> > + /* Allow file to be accessed */
> > + response.fd = metadata->fd;
> > + response.response = FAN_ALLOW;
> > + write(fd, &response,
> > + sizeof(struct fanotify_response));
> > + }
> > + /* Close the file descriptor of the event */
> > + close(metadata->fd);
> > + }
> > + /* Advance to next event */
> > + metadata = FAN_EVENT_NEXT(metadata, len);
> > + }
> > + }
> > +}
> > +
> > +static int fanotify_watch(char *arg)
> > +{
> > + int fd, poll_num;
> > + nfds_t nfds;
> > + struct pollfd fds[2];
> > +
> > + /* Create the file descriptor for accessing the fanotify API */
> > + fd = fanotify_init(FAN_CLOEXEC | FAN_CLASS_CONTENT | FAN_NONBLOCK,
> > + O_RDONLY | O_LARGEFILE);
> > + if (fd == -1) {
> > + perror("fanotify_init");
> > + exit(EXIT_FAILURE);
> > + }
> > + /*
> > + * Mark the mount for:
> > + * - permission events before opening files
> > + * - notification events after closing a write-enabled
> > + file descriptor
> > + */
> > + if (fanotify_mark(fd, FAN_MARK_ADD | FAN_MARK_MOUNT,
> > + FAN_ACCESS | FAN_MODIFY | FAN_OPEN_PERM |
> > + FAN_CLOSE | FAN_OPEN | FAN_ACCESS_PERM |
> > + FAN_ONDIR | FAN_EVENT_ON_CHILD,
> > + -1, arg) == -1) {
> > + perror("fanotify_mark");
> > + exit(EXIT_FAILURE);
> > + }
> > + /* Prepare for polling */
> > + nfds = 1;
> > + /* Fanotify input */
> > + fds[0].fd = fd;
> > + fds[0].events = POLLIN;
> > + /* This is the loop to wait for incoming events */
> > + while (1) {
> > + poll_num = poll(fds, nfds, -1);
> > + if (poll_num == -1) {
> > + if (errno == EINTR) /* Interrupted by a signal */
> > + continue; /* Restart poll() */
> > + perror("poll"); /* Unexpected error */
> > + exit(EXIT_FAILURE);
> > + }
> > + if (poll_num > 0) {
> > + if (fds[0].revents & POLLIN) {
> > + /* Fanotify events are available */
> > + handle_events(fd);
> > + }
> > + }
> > + }
> > + exit(EXIT_SUCCESS);
> > +}
> > +
> > +static int fanotify_flush_stress(char *arg)
> > +{
> > + int fd;
> > +
> > + /* Create the file descriptor for accessing the fanotify API */
> > + fd = fanotify_init(FAN_CLOEXEC | FAN_CLASS_CONTENT | FAN_NONBLOCK,
> > + O_RDONLY | O_LARGEFILE);
> > + if (fd == -1) {
> > + perror("fanotify_init");
> > + exit(EXIT_FAILURE);
> > + }
> > +
> > + /* Loop marking all kinds of events and flush */
> > + while (1) {
> > + if (fanotify_mark(fd, FAN_MARK_ADD | FAN_MARK_MOUNT,
> > + FAN_ACCESS | FAN_MODIFY | FAN_OPEN_PERM | FAN_CLOSE |
> > + FAN_OPEN | FAN_ACCESS_PERM | FAN_ONDIR |
> > + FAN_EVENT_ON_CHILD, -1, arg) == -1)
> > + perror("fanotify_mark add");
> > +
> > + if (fanotify_mark(fd, FAN_MARK_FLUSH | FAN_MARK_MOUNT,
> > + 0, -1, arg) == -1)
> > + perror("fanotify_mark flush mount");
> > +
> > + if (fanotify_mark(fd, FAN_MARK_ADD | FAN_MARK_MOUNT,
> > + FAN_ACCESS | FAN_MODIFY | FAN_OPEN_PERM | FAN_CLOSE |
> > + FAN_OPEN | FAN_ACCESS_PERM | FAN_ONDIR |
> > + FAN_EVENT_ON_CHILD, -1, arg) == -1)
> > + perror("fanotify_mark add");
> > +
> > + if (fanotify_mark(fd, FAN_MARK_FLUSH, 0, -1, arg) == -1)
> > + perror("fanotify_mark flush");
> > + }
> > + close(fd);
> > + exit(EXIT_SUCCESS);
> > +}
> > +
> > +static int fanotify_init_stress(char *arg)
> > +{
> > + int fd;
> > +
> > + while (1) {
> > + /* Create the file descriptor for accessing the fanotify API */
> > + fd = fanotify_init(FAN_CLOEXEC | FAN_CLASS_CONTENT |
> > + FAN_NONBLOCK, O_RDONLY | O_LARGEFILE);
> > + if (fd == -1)
> > + perror("fanotify_init");
> > + if (fanotify_mark(fd, FAN_MARK_ADD | FAN_MARK_MOUNT,
> > + FAN_ACCESS | FAN_MODIFY | FAN_OPEN_PERM |
> > + FAN_CLOSE | FAN_OPEN | FAN_ACCESS_PERM |
> > + FAN_ONDIR | FAN_EVENT_ON_CHILD, -1,
> > + arg) == -1)
> > + perror("fanotify_mark");
> > + close(fd);
> > + }
> > + exit(EXIT_SUCCESS);
> > +}
> > +
> > +static void add_mark(int fd, uint64_t mask, char *path)
> > +{
> > + if (fanotify_mark(fd, FAN_MARK_ADD, mask, -1, path) == -1)
> > + perror("fanotify_mark add");
> > +}
> > +
> > +static void remove_mark(int fd, uint64_t mask, char *path)
> > +{
> > + if (fanotify_mark(fd, FAN_MARK_REMOVE, mask, -1, path) == -1)
> > + perror("fanotify_mark remove");
> > +}
> > +
> > +static int fanotify_mark_stress(char *arg)
> > +{
> > + int fd;
> > +
> > + /* Create the file descriptor for accessing the fanotify API */
> > + fd = fanotify_init(FAN_CLOEXEC | FAN_CLASS_CONTENT | FAN_NONBLOCK,
> > + O_RDONLY | O_LARGEFILE);
> > + if (fd == -1) {
> > + perror("fanotify_init");
> > + exit(EXIT_FAILURE);
> > + }
> > + /* Loop marking all kinds of events */
> > + while (1) {
> > + add_mark(fd, FAN_ACCESS, arg);
> > + remove_mark(fd, FAN_ACCESS, arg);
> > + add_mark(fd, FAN_MODIFY, arg);
> > + remove_mark(fd, FAN_MODIFY, arg);
> > + add_mark(fd, FAN_OPEN_PERM, arg);
> > + remove_mark(fd, FAN_OPEN_PERM, arg);
> > + add_mark(fd, FAN_CLOSE, arg);
> > + remove_mark(fd, FAN_CLOSE, arg);
> > + add_mark(fd, FAN_OPEN, arg);
> > + remove_mark(fd, FAN_OPEN, arg);
> > + add_mark(fd, FAN_ACCESS_PERM, arg);
> > + remove_mark(fd, FAN_ACCESS_PERM, arg);
> > + add_mark(fd, FAN_ONDIR, arg);
> > + remove_mark(fd, FAN_ONDIR, arg);
> > + add_mark(fd, FAN_EVENT_ON_CHILD, arg);
> > + remove_mark(fd, FAN_EVENT_ON_CHILD, arg);
> > + }
> > +
> > + close(fd);
> > + exit(EXIT_SUCCESS);
> > +}
> > +
> > +static int inotify_watch(char *arg)
> > +{
> > + int notify_fd;
> > + int wd, ret;
> > + char *buf;
> > +
> > + buf = malloc(sizeof(struct inotify_event) + NAME_MAX + 1);
> > + if (buf == NULL) {
> > + perror("malloc");
> > + return -1;
> > + }
> > +
> > + notify_fd = inotify_init1(IN_CLOEXEC);
> > + if (notify_fd == -1) {
> > + perror("inotify_init1");
> > + return -1;
> > + }
> > +
> > + wd = inotify_add_watch(notify_fd, arg,
> > + IN_ACCESS | IN_ATTRIB | IN_CLOSE_WRITE | IN_CLOSE_NOWRITE |
> > + IN_CREATE | IN_DELETE | IN_DELETE_SELF | IN_MODIFY |
> > + IN_MOVE_SELF | IN_MOVED_FROM | IN_MOVED_TO | IN_OPEN);
> > + if (wd < 0) {
> > + perror("inotify_add_watch");
> > + return -1;
> > + }
> > +
> > + while ((ret = read(notify_fd, buf, NAME_MAX)) != -1) {
> > + ;
> > + }
> > +
> > + ret = inotify_rm_watch(notify_fd, wd);
> > + if (ret < 0)
> > + perror("inotify_rm_watch");
> > +
> > + close(notify_fd);
> > + free(buf);
> > + return 0;
> > +}
> > +
> > +static int inotify_add_rm_watch_stress(char *arg)
> > +{
> > + int notify_fd;
> > + int wd, ret;
> > +
> > + notify_fd = inotify_init1(IN_CLOEXEC);
> > + if (notify_fd == -1) {
> > + perror("inotify_init1");
> > + return -1;
> > + }
> > +
> > + while (1) {
> > + wd = inotify_add_watch(notify_fd, arg,
> > + IN_ACCESS | IN_ATTRIB | IN_CLOSE_WRITE |
> > + IN_CLOSE_NOWRITE | IN_CREATE | IN_DELETE |
> > + IN_DELETE_SELF | IN_MODIFY | IN_MOVE_SELF |
> > + IN_MOVED_FROM | IN_MOVED_TO | IN_OPEN);
> > + if (wd < 0)
> > + perror("inotify_add_watch");
> > + ret = inotify_rm_watch(notify_fd, wd);
> > + if (ret < 0)
> > + perror("inotify_rm_watch");
> > + }
> > + close(notify_fd);
> > + return 0;
> > +}
> > +
> > +static int inotify_init_stress(void)
> > +{
> > + int notify_fd;
> > +
> > + while (1) {
> > + notify_fd = inotify_init1(IN_CLOEXEC);
> > + if (notify_fd == -1)
> > + perror("inotify_init1");
> > + close(notify_fd);
> > + }
> > + return 0;
> > +}
> > +
> > +int main(int argc, char *argv[])
> > +{
> > + pid_t pid;
> > +
> > + if (argc != 2) {
> > + fprintf(stderr, "Usage: %s testdir\n", argv[0]);
> > + exit(EXIT_FAILURE);
> > + }
> > +
> > + if ((pid = fork()) == 0)
> > + fanotify_watch(argv[1]);
> > +
> > + if ((pid = fork()) == 0)
> > + fanotify_flush_stress(argv[1]);
> > +
> > + if ((pid = fork()) == 0)
> > + fanotify_init_stress(argv[1]);
> > +
> > + if ((pid = fork()) == 0)
> > + fanotify_mark_stress(argv[1]);
> > +
> > + if ((pid = fork()) == 0)
> > + inotify_watch(argv[1]);
> > +
> > + if ((pid = fork()) == 0)
> > + inotify_add_rm_watch_stress(argv[1]);
> > +
> > + if ((pid = fork()) == 0)
> > + inotify_init_stress();
> > +
> > + return 0;
> > +}
> > diff --git a/tests/generic/478 b/tests/generic/478
> > new file mode 100755
> > index 0000000..7139837
> > --- /dev/null
> > +++ b/tests/generic/478
> > @@ -0,0 +1,185 @@
> > +#! /bin/bash
> > +# FS QA Test 478
> > +#
> > +# Stress test for fanotify and inotify.
> > +#
> > +# Exercise fanotify and inotify interfaces in loop while
> > +# other stress tests going on in the watched test directory.
> > +#
> > +# Watching slab object inotify_inode_mark size, report fail
> > +# it increases too fast. This may lead to a crash if OOM killer
> > +# invoked.
> > +#
> > +# kernel commit related to the fixes in v4.15-rc1:
> > +# 0d6ec07 fsnotify: pin both inode and vfsmount mark
> > +#
> > +#-----------------------------------------------------------------------
> > +# Copyright (c) 2018 Red Hat Inc. All Rights Reserved.
> > +#
> > +# This program is free software; you can redistribute it and/or
> > +# modify it under the terms of the GNU General Public License as
> > +# published by the Free Software Foundation.
> > +#
> > +# This program is distributed in the hope that it would be useful,
> > +# but WITHOUT ANY WARRANTY; without even the implied warranty of
> > +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
> > +# GNU General Public License for more details.
> > +#
> > +# You should have received a copy of the GNU General Public License
> > +# along with this program; if not, write the Free Software Foundation,
> > +# Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
> > +#-----------------------------------------------------------------------
> > +#
> > +
> > +seq=`basename $0`
> > +seqres=$RESULT_DIR/$seq
> > +echo "QA output created by $seq"
> > +
> > +here=`pwd`
> > +tmp=/tmp/$$
> > +status=1 # failure is the default!
> > +trap "_cleanup; exit \$status" 0 1 2 3 15
> > +
> > +_cleanup()
> > +{
> > + touch $stopfile
> > + while killall fsnotify_stress fsstress > /dev/null 2>&1 ; do
> > + sleep 1
> > + done
> > + wait
> > + cd /
> > + rm -f $tmp.*
> > +}
> > +
> > +# get standard environment, filters and checks
> > +. ./common/rc
> > +. ./common/filter
> > +
> > +# remove previous $seqres.full before test
> > +rm -f $seqres.full
> > +
> > +# real QA test starts here
> > +stopfile=$tmp.stop
> > +TIMEOUT=2m
> > +
> > +# Modify as appropriate.
> > +_supported_fs generic
> > +_supported_os Linux
> > +_require_scratch
> > +
> > +function add_files ()
> > +{
> > + local i=$((RANDOM))
> > +
> > + touch $SCRATCH_MNT/f-$i
> > + ln -s $SCRATCH_MNT/f-$i $SCRATCH_MNT/f-$i-sym
> > + ln $SCRATCH_MNT/f-$i $SCRATCH_MNT/f-$i-hdl
> > + mkdir $SCRATCH_MNT/d-$i
> > + mknod $SCRATCH_MNT/c-$i c 1 2
> > + mknod $SCRATCH_MNT/b-$i b 1 2
> > +}
> > +
> > +function mv_files ()
> > +{
> > + local i
> > + for i in $SCRATCH_MNT/* ; do
> > + mv -f f-$i f-$i-rename
> > + done
> > +}
> > +
> > +function read_files ()
> > +{
> > + find $SCRATCH_MNT/
> > + cat $SCRATCH_MNT/f-*
> > + ls -R $SCRATCH_MNT/d-*
> > +}
> > +
> > +function write_files ()
> > +{
> > + local i
> > + for i in $SCRATCH_MNT/* ; do
> > + echo 1 > $i
> > + echo 2 >> $i
> > + done
> > +}
> > +
> > +function rm_files ()
> > +{
> > + local i
> > + for i in $SCRATCH_MNT/* ; do
> > + rm -rf $i
> > + done
> > +}
> > +
> > +slab_cal_inotify_inode_mark()
> > +{
> > + echo 3 > /proc/sys/vm/drop_caches
> > + local num_obj=$(cat /proc/slabinfo | grep inotify_inode_mark | awk '{print $3}')
> > + local objsize=$(cat /proc/slabinfo | grep inotify_inode_mark | awk '{print $4}')
> > + echo $(($num_obj * $objsize))
> > +}
> > +
> > +_scratch_mkfs >>$seqres.full 2>&1
> > +_scratch_mount
> > +
> > +# inotify_inode_mark slab size before test
> > +iim_0=`slab_cal_inotify_inode_mark`
> > +
> > +NR_CPUS=$(grep -c processor /proc/cpuinfo)
> > +[ $NR_CPUS -lt 4 ] && NR_CPUS=4
> > +opts="-d $SCRATCH_MNT/ -p $NR_CPUS -n 50 -v -l 0 -c $FSSTRESS_AVOID"
> > +$FSSTRESS_PROG $opts >> $seqres.full 2>&1 &
> > +
> > +rm -f $stopfile
> > +for j in 1 2 ; do
> > +
> > +for i in `seq 1 $(($NR_CPUS/7 + 1))` ; do
> > + $here/src/fsnotify_stress $SCRATCH_MNT &
> > +done
> > +
> > +# run read/write files operations
> > +while [ ! -e $stopfile ]; do
> > + add_files > /dev/null 2>&1
> > +done &
> > +while [ ! -e $stopfile ]; do
> > + mv_files > /dev/null 2>&1
> > +done &
> > +while [ ! -e $stopfile ]; do
> > + read_files > /dev/null 2>&1
> > +done &
> > +while [ ! -e $stopfile ]; do
> > + write_files > /dev/null 2>&1
> > +done &
> > +while [ ! -e $stopfile ]; do
> > + rm_files > /dev/null 2>&1
> > +done &
> > +
> > +done
> > +
> > +# stressing for $TIMEOUT
> > +sleep $TIMEOUT
> > +
> > +# inotify_inode_mark slab size after test
> > +iim_1=`slab_cal_inotify_inode_mark`
> > +
> > +# cleanup stress processes
> > +touch $stopfile
> > +while killall fsnotify_stress fsstress > /dev/null 2>&1 ; do
> > + sleep 1
> > +done
> > +
> > +# wait _files functions done
> > +wait
> > +
> > +echo $iim_0 $iim_1 $(($iim_1 - $iim_0)) >> $seqres.full
> > +# If inotify_inode_mark slab size increases 1024 times of
> > +# itself in 2m, something bad happens because it could end up
> > +# invoking OOM killer, test fails.
> > +if [ $iim_1 -gt $iim_0 ] &&
> > + [ $((iim_1-iim_0)) -gt $((1024*$iim_0)) ] ; then
> > + echo inotify_inode_mark slab memory leaked
> > +fi
> > +# success, all done
> > +echo "Silence is golden"
> > +status=0
> > +exit
> > diff --git a/tests/generic/478.out b/tests/generic/478.out
> > new file mode 100644
> > index 0000000..4e2107a
> > --- /dev/null
> > +++ b/tests/generic/478.out
> > @@ -0,0 +1,2 @@
> > +QA output created by 478
> > +Silence is golden
> > diff --git a/tests/generic/group b/tests/generic/group
> > index cce03e9..8416957 100644
> > --- a/tests/generic/group
> > +++ b/tests/generic/group
> > @@ -480,3 +480,4 @@
> > 475 shutdown auto log metadata
> > 476 auto rw
> > 477 auto quick exportfs
> > +478 auto stress dangerous fsnotify
> > --
> > 1.8.3.1
> >
> --
> Jan Kara <jack@xxxxxxxx>
> SUSE Labs, CR
--
To unsubscribe from this list: send the line "unsubscribe fstests" in
the body of a message to majordomo@xxxxxxxxxxxxxxx
More majordomo info at http://vger.kernel.org/majordomo-info.html
