Re: [PATCH] ext4: add ratelimiting to ext4 messages

[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]

 



On 10/17/13 8:28 PM, Theodore Ts'o wrote:
> In the case of a storage device that suddenly disappears, or in the
> case of significant file system corruption, this can result in a huge
> flood of messages being sent to the console.  This can overflow the
> file system containing /var/log/messages, or if a serial console is
> configured, this can slow down the system so much that a hardware
> watchdog can end up triggering forcing a system reboot.

Just out of curiosity, after the fs shuts down, is there still a flood
of messages?  Shouldn't that clamp down on the errors?

If not, shouldn't it do so?  xfs has a lot of short-circuiting if
the filesystem is shut down, so it (I think) won't get into paths that
will generate more errors.

(If not, would just shutting up ext_* error message post-shutdown accomplish
the same thing w/ fewer lines & fewer /proc configuration knobs)?

Just a thought,

-Eric

> Google-Bug-Id: 7258357
> 
> Signed-off-by: "Theodore Ts'o" <tytso@xxxxxxx>
> ---
>  fs/ext4/ext4.h  |   6 +++
>  fs/ext4/super.c | 152 +++++++++++++++++++++++++++++++++++---------------------
>  2 files changed, 100 insertions(+), 58 deletions(-)
> 
> diff --git a/fs/ext4/ext4.h b/fs/ext4/ext4.h
> index af815ea..65485ab 100644
> --- a/fs/ext4/ext4.h
> +++ b/fs/ext4/ext4.h
> @@ -29,6 +29,7 @@
>  #include <linux/wait.h>
>  #include <linux/blockgroup_lock.h>
>  #include <linux/percpu_counter.h>
> +#include <linux/ratelimit.h>
>  #include <crypto/hash.h>
>  #ifdef __KERNEL__
>  #include <linux/compat.h>
> @@ -1314,6 +1315,11 @@ struct ext4_sb_info {
>  	unsigned long s_es_last_sorted;
>  	struct percpu_counter s_extent_cache_cnt;
>  	spinlock_t s_es_lru_lock ____cacheline_aligned_in_smp;
> +
> +	/* Ratelimit ext4 messages. */
> +	struct ratelimit_state s_err_ratelimit_state;
> +	struct ratelimit_state s_warning_ratelimit_state;
> +	struct ratelimit_state s_msg_ratelimit_state;
>  };
>  
>  static inline struct ext4_sb_info *EXT4_SB(struct super_block *sb)
> diff --git a/fs/ext4/super.c b/fs/ext4/super.c
> index 2c2e6cb..d3a857b 100644
> --- a/fs/ext4/super.c
> +++ b/fs/ext4/super.c
> @@ -411,20 +411,26 @@ static void ext4_handle_error(struct super_block *sb)
>  			sb->s_id);
>  }
>  
> +#define ext4_error_ratelimit(sb)					\
> +		___ratelimit(&(EXT4_SB(sb)->s_err_ratelimit_state),	\
> +			     "EXT4-fs error")
> +
>  void __ext4_error(struct super_block *sb, const char *function,
>  		  unsigned int line, const char *fmt, ...)
>  {
>  	struct va_format vaf;
>  	va_list args;
>  
> -	va_start(args, fmt);
> -	vaf.fmt = fmt;
> -	vaf.va = &args;
> -	printk(KERN_CRIT "EXT4-fs error (device %s): %s:%d: comm %s: %pV\n",
> -	       sb->s_id, function, line, current->comm, &vaf);
> -	va_end(args);
> +	if (ext4_error_ratelimit(sb)) {
> +		va_start(args, fmt);
> +		vaf.fmt = fmt;
> +		vaf.va = &args;
> +		printk(KERN_CRIT
> +		       "EXT4-fs error (device %s): %s:%d: comm %s: %pV\n",
> +		       sb->s_id, function, line, current->comm, &vaf);
> +		va_end(args);
> +	}
>  	save_error_info(sb, function, line);
> -
>  	ext4_handle_error(sb);
>  }
>  
> @@ -438,22 +444,23 @@ void __ext4_error_inode(struct inode *inode, const char *function,
>  
>  	es->s_last_error_ino = cpu_to_le32(inode->i_ino);
>  	es->s_last_error_block = cpu_to_le64(block);
> +	if (ext4_error_ratelimit(inode->i_sb)) {
> +		va_start(args, fmt);
> +		vaf.fmt = fmt;
> +		vaf.va = &args;
> +		if (block)
> +			printk(KERN_CRIT "EXT4-fs error (device %s): %s:%d: "
> +			       "inode #%lu: block %llu: comm %s: %pV\n",
> +			       inode->i_sb->s_id, function, line, inode->i_ino,
> +			       block, current->comm, &vaf);
> +		else
> +			printk(KERN_CRIT "EXT4-fs error (device %s): %s:%d: "
> +			       "inode #%lu: comm %s: %pV\n",
> +			       inode->i_sb->s_id, function, line, inode->i_ino,
> +			       current->comm, &vaf);
> +		va_end(args);
> +	}
>  	save_error_info(inode->i_sb, function, line);
> -	va_start(args, fmt);
> -	vaf.fmt = fmt;
> -	vaf.va = &args;
> -	if (block)
> -		printk(KERN_CRIT "EXT4-fs error (device %s): %s:%d: "
> -		       "inode #%lu: block %llu: comm %s: %pV\n",
> -		       inode->i_sb->s_id, function, line, inode->i_ino,
> -		       block, current->comm, &vaf);
> -	else
> -		printk(KERN_CRIT "EXT4-fs error (device %s): %s:%d: "
> -		       "inode #%lu: comm %s: %pV\n",
> -		       inode->i_sb->s_id, function, line, inode->i_ino,
> -		       current->comm, &vaf);
> -	va_end(args);
> -
>  	ext4_handle_error(inode->i_sb);
>  }
>  
> @@ -469,27 +476,28 @@ void __ext4_error_file(struct file *file, const char *function,
>  
>  	es = EXT4_SB(inode->i_sb)->s_es;
>  	es->s_last_error_ino = cpu_to_le32(inode->i_ino);
> +	if (ext4_error_ratelimit(inode->i_sb)) {
> +		path = d_path(&(file->f_path), pathname, sizeof(pathname));
> +		if (IS_ERR(path))
> +			path = "(unknown)";
> +		va_start(args, fmt);
> +		vaf.fmt = fmt;
> +		vaf.va = &args;
> +		if (block)
> +			printk(KERN_CRIT
> +			       "EXT4-fs error (device %s): %s:%d: inode #%lu: "
> +			       "block %llu: comm %s: path %s: %pV\n",
> +			       inode->i_sb->s_id, function, line, inode->i_ino,
> +			       block, current->comm, path, &vaf);
> +		else
> +			printk(KERN_CRIT
> +			       "EXT4-fs error (device %s): %s:%d: inode #%lu: "
> +			       "comm %s: path %s: %pV\n",
> +			       inode->i_sb->s_id, function, line, inode->i_ino,
> +			       current->comm, path, &vaf);
> +		va_end(args);
> +	}
>  	save_error_info(inode->i_sb, function, line);
> -	path = d_path(&(file->f_path), pathname, sizeof(pathname));
> -	if (IS_ERR(path))
> -		path = "(unknown)";
> -	va_start(args, fmt);
> -	vaf.fmt = fmt;
> -	vaf.va = &args;
> -	if (block)
> -		printk(KERN_CRIT
> -		       "EXT4-fs error (device %s): %s:%d: inode #%lu: "
> -		       "block %llu: comm %s: path %s: %pV\n",
> -		       inode->i_sb->s_id, function, line, inode->i_ino,
> -		       block, current->comm, path, &vaf);
> -	else
> -		printk(KERN_CRIT
> -		       "EXT4-fs error (device %s): %s:%d: inode #%lu: "
> -		       "comm %s: path %s: %pV\n",
> -		       inode->i_sb->s_id, function, line, inode->i_ino,
> -		       current->comm, path, &vaf);
> -	va_end(args);
> -
>  	ext4_handle_error(inode->i_sb);
>  }
>  
> @@ -543,11 +551,13 @@ void __ext4_std_error(struct super_block *sb, const char *function,
>  	    (sb->s_flags & MS_RDONLY))
>  		return;
>  
> -	errstr = ext4_decode_error(sb, errno, nbuf);
> -	printk(KERN_CRIT "EXT4-fs error (device %s) in %s:%d: %s\n",
> -	       sb->s_id, function, line, errstr);
> -	save_error_info(sb, function, line);
> +	if (ext4_error_ratelimit(sb)) {
> +		errstr = ext4_decode_error(sb, errno, nbuf);
> +		printk(KERN_CRIT "EXT4-fs error (device %s) in %s:%d: %s\n",
> +		       sb->s_id, function, line, errstr);
> +	}
>  
> +	save_error_info(sb, function, line);
>  	ext4_handle_error(sb);
>  }
>  
> @@ -597,6 +607,9 @@ void __ext4_msg(struct super_block *sb,
>  	struct va_format vaf;
>  	va_list args;
>  
> +	if (!___ratelimit(&(EXT4_SB(sb)->s_msg_ratelimit_state), "EXT4-fs"))
> +		return;
> +
>  	va_start(args, fmt);
>  	vaf.fmt = fmt;
>  	vaf.va = &args;
> @@ -610,6 +623,10 @@ void __ext4_warning(struct super_block *sb, const char *function,
>  	struct va_format vaf;
>  	va_list args;
>  
> +	if (!___ratelimit(&(EXT4_SB(sb)->s_warning_ratelimit_state),
> +			  "EXT4-fs warning"))
> +		return;
> +
>  	va_start(args, fmt);
>  	vaf.fmt = fmt;
>  	vaf.va = &args;
> @@ -633,18 +650,20 @@ __acquires(bitlock)
>  	es->s_last_error_block = cpu_to_le64(block);
>  	__save_error_info(sb, function, line);
>  
> -	va_start(args, fmt);
> -
> -	vaf.fmt = fmt;
> -	vaf.va = &args;
> -	printk(KERN_CRIT "EXT4-fs error (device %s): %s:%d: group %u, ",
> -	       sb->s_id, function, line, grp);
> -	if (ino)
> -		printk(KERN_CONT "inode %lu: ", ino);
> -	if (block)
> -		printk(KERN_CONT "block %llu:", (unsigned long long) block);
> -	printk(KERN_CONT "%pV\n", &vaf);
> -	va_end(args);
> +	if (ext4_error_ratelimit(sb)) {
> +		va_start(args, fmt);
> +		vaf.fmt = fmt;
> +		vaf.va = &args;
> +		printk(KERN_CRIT "EXT4-fs error (device %s): %s:%d: group %u, ",
> +		       sb->s_id, function, line, grp);
> +		if (ino)
> +			printk(KERN_CONT "inode %lu: ", ino);
> +		if (block)
> +			printk(KERN_CONT "block %llu:",
> +			       (unsigned long long) block);
> +		printk(KERN_CONT "%pV\n", &vaf);
> +		va_end(args);
> +	}
>  
>  	if (test_opt(sb, ERRORS_CONT)) {
>  		ext4_commit_super(sb, 0);
> @@ -2606,6 +2625,12 @@ EXT4_RW_ATTR_SBI_UI(mb_group_prealloc, s_mb_group_prealloc);
>  EXT4_DEPRECATED_ATTR(max_writeback_mb_bump, 128);
>  EXT4_RW_ATTR_SBI_UI(extent_max_zeroout_kb, s_extent_max_zeroout_kb);
>  EXT4_ATTR(trigger_fs_error, 0200, NULL, trigger_test_error);
> +EXT4_RW_ATTR_SBI_UI(err_ratelimit_interval_ms, s_err_ratelimit_state.interval);
> +EXT4_RW_ATTR_SBI_UI(err_ratelimit_burst, s_err_ratelimit_state.burst);
> +EXT4_RW_ATTR_SBI_UI(warning_ratelimit_interval_ms, s_warning_ratelimit_state.interval);
> +EXT4_RW_ATTR_SBI_UI(warning_ratelimit_burst, s_warning_ratelimit_state.burst);
> +EXT4_RW_ATTR_SBI_UI(msg_ratelimit_interval_ms, s_msg_ratelimit_state.interval);
> +EXT4_RW_ATTR_SBI_UI(msg_ratelimit_burst, s_msg_ratelimit_state.burst);
>  
>  static struct attribute *ext4_attrs[] = {
>  	ATTR_LIST(delayed_allocation_blocks),
> @@ -2623,6 +2648,12 @@ static struct attribute *ext4_attrs[] = {
>  	ATTR_LIST(max_writeback_mb_bump),
>  	ATTR_LIST(extent_max_zeroout_kb),
>  	ATTR_LIST(trigger_fs_error),
> +	ATTR_LIST(err_ratelimit_interval_ms),
> +	ATTR_LIST(err_ratelimit_burst),
> +	ATTR_LIST(warning_ratelimit_interval_ms),
> +	ATTR_LIST(warning_ratelimit_burst),
> +	ATTR_LIST(msg_ratelimit_interval_ms),
> +	ATTR_LIST(msg_ratelimit_burst),
>  	NULL,
>  };
>  
> @@ -4118,6 +4149,11 @@ no_journal:
>  	if (es->s_error_count)
>  		mod_timer(&sbi->s_err_report, jiffies + 300*HZ); /* 5 minutes */
>  
> +	/* Enable message ratelimiting. Default is 10 messages per 5 secs. */
> +	ratelimit_state_init(&sbi->s_err_ratelimit_state, 5 * HZ, 10);
> +	ratelimit_state_init(&sbi->s_warning_ratelimit_state, 5 * HZ, 10);
> +	ratelimit_state_init(&sbi->s_msg_ratelimit_state, 5 * HZ, 10);
> +
>  	kfree(orig_data);
>  	return 0;
>  
> 

--
To unsubscribe from this list: send the line "unsubscribe linux-ext4" in
the body of a message to majordomo@xxxxxxxxxxxxxxx
More majordomo info at  http://vger.kernel.org/majordomo-info.html




[Index of Archives]     [Reiser Filesystem Development]     [Ceph FS]     [Kernel Newbies]     [Security]     [Netfilter]     [Bugtraq]     [Linux FS]     [Yosemite National Park]     [MIPS Linux]     [ARM Linux]     [Linux Security]     [Linux RAID]     [Samba]     [Device Mapper]     [Linux Media]

  Powered by Linux