On Jan 20, 2014, at 11:17 PM, Theodore Ts'o <tytso@xxxxxxx> wrote:
> This feature is enabled via settings in /etc/mke2fs.conf. For
> example:
>
> hugefile = {
> features = extent,huge_file,flex_bg,uninit_bg,dir_nlink,extra_isize,^resize_inode,sparse_super2
> inode_size = 128
> packed_meta_blocks = 1
> make_hugefiles = 1
> inode_ratio = 4194304
> hugefiles_dir = /database
> hugefiles_uid = 120
> hugefiles_gid = 50
> hugefiles_name = storage
> hugefiles_digits = 4
> hugefile_size = 1G
> num_hugefiles = 0
> }
>
> Then "mke2fs -T hugefile /dev/sdXX" will create as many 1G files
> needed to fill the file system.
How is this different from using fallocate to allocate the files?
Is this just to create a test image for e2fsck or similar? It
might make sense to include f_hugefiles/script and expect.1 for it?
Cheers, Andreas
> Signed-off-by: "Theodore Ts'o" <tytso@xxxxxxx>
> ---
>
> This is an enhancement to mke2fs which I developed for a use case at
> work, where it is convenient to create a set of pre-allocated files
> which use all or most of the space in the file system. I've tried to
> make it to be as general as possible, but I'm still considering whether
> it's appropriate to include this in the e2fprogs sources, perhaps under
> a configure --enable-make-hugefiles option or some such. Nearly all of
> the new code is in a separate file for ease of maintenance.
>
>
> misc/Makefile.in | 3 +-
> misc/mk_hugefiles.c | 385 ++++++++++++++++++++++++++++++++++++++++++++++++++
> misc/mke2fs.c | 21 +--
> misc/mke2fs.conf.5.in | 45 ++++++
> misc/mke2fs.h | 29 ++++
> 5 files changed, 474 insertions(+), 9 deletions(-)
> create mode 100644 misc/mk_hugefiles.c
> create mode 100644 misc/mke2fs.h
>
> diff --git a/misc/Makefile.in b/misc/Makefile.in
> index 8342940..c5f332b 100644
> --- a/misc/Makefile.in
> +++ b/misc/Makefile.in
> @@ -42,7 +42,8 @@ LPROGS= @E2INITRD_PROG@
>
> TUNE2FS_OBJS= tune2fs.o util.o
> MKLPF_OBJS= mklost+found.o
> -MKE2FS_OBJS= mke2fs.o util.o profile.o prof_err.o default_profile.o
> +MKE2FS_OBJS= mke2fs.o util.o profile.o prof_err.o default_profile.o \
> + mk_hugefiles.o
> CHATTR_OBJS= chattr.o
> LSATTR_OBJS= lsattr.o
> UUIDGEN_OBJS= uuidgen.o
> diff --git a/misc/mk_hugefiles.c b/misc/mk_hugefiles.c
> new file mode 100644
> index 0000000..c43a2b0
> --- /dev/null
> +++ b/misc/mk_hugefiles.c
> @@ -0,0 +1,385 @@
> +/*
> + * mk_hugefiles.c -- create huge files
> + */
> +
> +#define _XOPEN_SOURCE 600 /* for inclusion of PATH_MAX in Solaris */
> +
> +#include "config.h"
> +#include <stdio.h>
> +#include <string.h>
> +#include <strings.h>
> +#include <fcntl.h>
> +#include <ctype.h>
> +#include <time.h>
> +#ifdef __linux__
> +#include <sys/utsname.h>
> +#endif
> +#ifdef HAVE_GETOPT_H
> +#include <getopt.h>
> +#else
> +extern char *optarg;
> +extern int optind;
> +#endif
> +#ifdef HAVE_UNISTD_H
> +#include <unistd.h>
> +#endif
> +#ifdef HAVE_STDLIB_H
> +#include <stdlib.h>
> +#endif
> +#ifdef HAVE_ERRNO_H
> +#include <errno.h>
> +#endif
> +#include <sys/ioctl.h>
> +#include <sys/types.h>
> +#include <sys/stat.h>
> +#include <libgen.h>
> +#include <limits.h>
> +#include <blkid/blkid.h>
> +
> +#include "ext2fs/ext2_fs.h"
> +#include "ext2fs/ext2fsP.h"
> +#include "et/com_err.h"
> +#include "uuid/uuid.h"
> +#include "e2p/e2p.h"
> +#include "ext2fs/ext2fs.h"
> +#include "util.h"
> +#include "profile.h"
> +#include "prof_err.h"
> +#include "nls-enable.h"
> +#include "mke2fs.h"
> +
> +static int uid;
> +static int gid;
> +static blk64_t num_blocks;
> +static blk64_t num_slack;
> +static unsigned long num_files;
> +static blk64_t goal;
> +static char *fn_prefix;
> +static int idx_digits;
> +static char *fn_buf;
> +static char *fn_numbuf;
> +
> +static errcode_t create_directory(ext2_filsys fs, char *dir,
> + ext2_ino_t *ret_ino)
> +
> +{
> + struct ext2_inode inode;
> + ext2_ino_t ino = EXT2_ROOT_INO;
> + ext2_ino_t newdir;
> + errcode_t retval;
> + char *fn, *cp, *next;
> +
> + fn = malloc(strlen(dir) + 1);
> + if (fn == NULL)
> + return ENOMEM;
> +
> + strcpy(fn, dir);
> + cp = fn;
> + while(1) {
> + next = strchr(cp, '/');
> + if (next)
> + *next++ = 0;
> + if (*cp) {
> + retval = ext2fs_new_inode(fs, ino, LINUX_S_IFDIR,
> + NULL, &newdir);
> + if (retval)
> + goto errout;
> +
> + retval = ext2fs_mkdir(fs, ino, newdir, cp);
> + if (retval)
> + goto errout;
> +
> + ino = newdir;
> + retval = ext2fs_read_inode(fs, ino, &inode);
> + if (retval)
> + goto errout;
> +
> + inode.i_uid = uid & 0xFFFF;
> + ext2fs_set_i_uid_high(inode, (uid >> 16) & 0xffff);
> + inode.i_gid = gid & 0xFFFF;
> + ext2fs_set_i_gid_high(inode, (gid >> 16) & 0xffff);
> + retval = ext2fs_write_inode(fs, ino, &inode);
> + if (retval)
> + goto errout;
> + }
> + if (next == NULL || *next == '\0')
> + break;
> + cp = next;
> + }
> +errout:
> + free(fn);
> + if (retval == 0)
> + *ret_ino = ino;
> + return retval;
> +}
> +
> +static errcode_t mk_hugefile(ext2_filsys fs, blk64_t num,
> + ext2_ino_t dir, int idx, ext2_ino_t *ino)
> +
> +{
> + errcode_t retval;
> + blk64_t lblk, blk, bend;
> + __u64 size;
> + blk64_t left;
> + blk64_t count = 0;
> + struct ext2_inode inode;
> + ext2_extent_handle_t handle;
> +
> + retval = ext2fs_new_inode(fs, 0, LINUX_S_IFREG, NULL, ino);
> + if (retval)
> + return retval;
> +
> + memset(&inode, 0, sizeof(struct ext2_inode));
> + inode.i_mode = LINUX_S_IFREG | (0666 & ~fs->umask);
> + inode.i_links_count = 1;
> + inode.i_uid = uid & 0xFFFF;
> + ext2fs_set_i_uid_high(inode, (uid >> 16) & 0xffff);
> + inode.i_gid = gid & 0xFFFF;
> + ext2fs_set_i_gid_high(inode, (gid >> 16) & 0xffff);
> +
> + retval = ext2fs_write_new_inode(fs, *ino, &inode);
> + if (retval)
> + return retval;
> +
> + ext2fs_inode_alloc_stats2(fs, *ino, +1, 0);
> +
> + retval = ext2fs_extent_open2(fs, *ino, &inode, &handle);
> + if (retval)
> + return retval;
> +
> + lblk = 0;
> + left = num ? num : 1;
> + while (left) {
> + blk64_t pblk, end;
> + blk64_t n = left;
> +
> + retval = ext2fs_find_first_zero_block_bitmap2(fs->block_map,
> + goal, ext2fs_blocks_count(fs->super) - 1, &end);
> + if (retval)
> + return ENOSPC;
> + goal = end;
> +
> + retval = ext2fs_find_first_set_block_bitmap2(fs->block_map, goal,
> + ext2fs_blocks_count(fs->super) - 1, &bend);
> + if (retval == ENOENT) {
> + bend = ext2fs_blocks_count(fs->super);
> + if (num == 0)
> + left = 0;
> + }
> + if (!num || bend - goal < left)
> + n = bend - goal;
> + pblk = goal;
> + if (num)
> + left -= n;
> + goal += n;
> + count += n;
> + ext2fs_block_alloc_stats_range(fs, pblk, n, +1);
> +
> + while (n) {
> + blk64_t l = n;
> + struct ext2fs_extent newextent;
> +
> + if (l > EXT_INIT_MAX_LEN)
> + l = EXT_INIT_MAX_LEN;
> +
> + newextent.e_len = l;
> + newextent.e_pblk = pblk;
> + newextent.e_lblk = lblk;
> + newextent.e_flags = 0;
> +
> + retval = ext2fs_extent_insert(handle,
> + EXT2_EXTENT_INSERT_AFTER, &newextent);
> + if (retval)
> + return retval;
> + pblk += l;
> + lblk += l;
> + n -= l;
> + }
> + }
> +
> + retval = ext2fs_read_inode(fs, *ino, &inode);
> + if (retval)
> + goto errout;
> +
> + retval = ext2fs_iblk_add_blocks(fs, &inode,
> + count / EXT2FS_CLUSTER_RATIO(fs));
> + if (retval)
> + goto errout;
> + size = (__u64) count * fs->blocksize;
> + inode.i_size = size & 0xffffffff;
> + inode.i_size_high = (size >> 32);
> +
> + retval = ext2fs_write_new_inode(fs, *ino, &inode);
> + if (retval)
> + goto errout;
> +
> + if (idx_digits)
> + sprintf(fn_numbuf, "%0*d", idx_digits, idx);
> + else if (num_files > 1)
> + sprintf(fn_numbuf, "%d", idx);
> +
> +retry:
> + retval = ext2fs_link(fs, dir, fn_buf, *ino, EXT2_FT_REG_FILE);
> + if (retval == EXT2_ET_DIR_NO_SPACE) {
> + retval = ext2fs_expand_dir(fs, dir);
> + if (retval)
> + goto errout;
> + goto retry;
> + }
> +
> + if (retval)
> + goto errout;
> +
> +errout:
> + if (handle)
> + ext2fs_extent_free(handle);
> +
> + return retval;
> +}
> +
> +static blk64_t calc_overhead(ext2_filsys fs, blk64_t num_blocks)
> +{
> + blk64_t e_blocks, e_blocks2, e_blocks3, e_blocks4;
> + int extents_per_block;
> + int extents = (num_blocks + EXT_INIT_MAX_LEN - 1) / EXT_INIT_MAX_LEN;
> +
> + if (extents <= 4)
> + return 0;
> +
> + /*
> + * This calculation is due to the fact that we are inefficient
> + * in how handle extent splits when appending to the end of
> + * the extent tree. Sigh. We should fix this so that we can
> + * actually store 340 extents per 4k block, instead of only 170.
> + */
> + extents_per_block = ((fs->blocksize -
> + sizeof(struct ext3_extent_header)) /
> + sizeof(struct ext3_extent));
> + extents_per_block = (extents_per_block/ 2) - 1;
> +
> + e_blocks = (extents + extents_per_block - 1) / extents_per_block;
> + e_blocks2 = (e_blocks + extents_per_block - 1) / extents_per_block;
> + e_blocks3 = (e_blocks2 + extents_per_block - 1) / extents_per_block;
> + e_blocks4 = (e_blocks3 + extents_per_block - 1) / extents_per_block;
> + return e_blocks + e_blocks2 + e_blocks3 + e_blocks4;
> +}
> +
> +/*
> + * Find the place where we should start allocating blocks for the huge
> + * files. Leave <slack> free blocks at the beginning of the file
> + * system for things like metadata blocks.
> + */
> +static blk64_t get_start_block(ext2_filsys fs, blk64_t slack)
> +{
> + errcode_t retval;
> + blk64_t goal = fs->super->s_first_data_block, next;
> + blk64_t last_blk = ext2fs_blocks_count(fs->super) - 1;
> +
> + while (slack) {
> + retval = ext2fs_find_first_zero_block_bitmap2(fs->block_map,
> + goal, last_blk, &goal);
> + if (retval)
> + break;
> +
> + retval = ext2fs_find_first_set_block_bitmap2(fs->block_map,
> + goal, last_blk, &next);
> + if (retval)
> + next = last_blk;
> + next--;
> +
> + if (next - goal > slack) {
> + goal += slack;
> + break;
> + }
> +
> + slack -= (next - goal);
> + goal = next;
> + }
> + return goal;
> +}
> +
> +errcode_t mk_hugefiles(ext2_filsys fs)
> +{
> + errcode_t retval;
> + ext2_ino_t dir;
> + int i;
> + char *t;
> +
> + if (!get_bool_from_profile(fs_types, "make_hugefiles", 0))
> + return 0;
> +
> + uid = get_int_from_profile(fs_types, "hugefiles_uid", 0);
> + gid = get_int_from_profile(fs_types, "hugefiles_gid", 0);
> + fs->umask = get_int_from_profile(fs_types, "hugefiles_umask", 077);
> + num_files = get_int_from_profile(fs_types, "num_hugefiles", 0);
> + t = get_string_from_profile(fs_types, "hugefiles_slack", "1M");
> + num_slack = parse_num_blocks2(t, fs->super->s_log_block_size);
> + t = get_string_from_profile(fs_types, "hugefiles_size", "0");
> + num_blocks = parse_num_blocks2(t, fs->super->s_log_block_size);
> +
> + retval = create_directory(fs, get_string_from_profile(fs_types,
> + "hugefiles_dir", "/"), &dir);
> + if (retval)
> + return dir;
> +
> + if (num_blocks == 0 && num_files == 0)
> + num_files = 1;
> +
> + if (num_files == 0 && num_blocks) {
> + blk64_t fs_blocks = ext2fs_free_blocks_count(fs->super);
> +
> + fs_blocks -= num_slack;
> + num_files = fs_blocks / num_blocks;
> + fs_blocks -= (num_files / 16) + 1;
> + fs_blocks -= calc_overhead(fs, num_blocks) * num_files;
> + num_files = fs_blocks / num_blocks;
> + }
> +
> + if (num_blocks == 0 && num_files > 1) {
> + blk64_t fs_blocks = ext2fs_free_blocks_count(fs->super);
> +
> + fs_blocks -= num_slack;
> + num_blocks = fs_blocks / num_files;
> +
> + fs_blocks -= calc_overhead(fs, num_blocks) * num_files;
> + fs_blocks -= num_slack;
> + num_blocks = fs_blocks / num_files;
> + printf("Using num_blocks %llu\n", num_blocks);
> + }
> +
> + num_slack += calc_overhead(fs, num_blocks) * num_files;
> + num_slack += (num_files / 16) + 1; /* space for dir entries */
> + goal = get_start_block(fs, num_slack);
> +
> + fn_prefix = get_string_from_profile(fs_types, "hugefiles_name",
> + "hugefile");
> + idx_digits = get_int_from_profile(fs_types, "hugefiles_digits", 5);
> + i = int_log10(num_files) + 1;
> + if (idx_digits > i)
> + i = idx_digits;
> + fn_buf = malloc(strlen(fn_prefix) + i + 1);
> + if (!fn_buf)
> + return ENOMEM;
> + strcpy(fn_buf, fn_prefix);
> + fn_numbuf = fn_buf + strlen(fn_prefix);
> +
> + if (!quiet) {
> + printf(_("Creating %d huge file(s) "), num_files);
> + if (num_blocks)
> + printf(_("with %llu blocks each"), num_blocks);
> + fputc('\n', stdout);
> + }
> + for (i=0; i < num_files; i++) {
> + ext2_ino_t ino;
> +
> + retval = mk_hugefile(fs, num_blocks, dir, i, &ino);
> + if (retval) {
> + com_err(program_name, retval,
> + _("while creating huge file %d"), i);
> + goto errout;
> + }
> + }
> +errout:
> + free(fn_buf);
> + return retval;
> +}
> diff --git a/misc/mke2fs.c b/misc/mke2fs.c
> index e798648..9d7673f 100644
> --- a/misc/mke2fs.c
> +++ b/misc/mke2fs.c
> @@ -62,6 +62,7 @@ extern int optind;
> #include "../version.h"
> #include "nls-enable.h"
> #include "quota/mkquota.h"
> +#include "mke2fs.h"
>
> #define STRIDE_LENGTH 8
>
> @@ -76,13 +77,13 @@ extern int optind;
> extern int isatty(int);
> extern FILE *fpopen(const char *cmd, const char *mode);
>
> -static const char * program_name = "mke2fs";
> +const char * program_name = "mke2fs";
> static const char * device_name /* = NULL */;
>
> /* Command line options */
> static int cflag;
> -static int verbose;
> -static int quiet;
> +int verbose;
> +int quiet;
> static int super_only;
> static int discard = 1; /* attempt to discard device before fs creation */
> static int direct_io;
> @@ -107,7 +108,7 @@ static char *volume_label;
> static char *mount_dir;
> char *journal_device;
> static int sync_kludge; /* Set using the MKE2FS_SYNC env. option */
> -static char **fs_types;
> +char **fs_types;
>
> static profile_t profile;
>
> @@ -142,7 +143,7 @@ static int int_log2(unsigned long long arg)
> return l;
> }
>
> -static int int_log10(unsigned long long arg)
> +int int_log10(unsigned long long arg)
> {
> int l;
>
> @@ -1253,7 +1254,7 @@ static char **parse_fs_type(const char *fs_type,
> return (list.list);
> }
>
> -static char *get_string_from_profile(char **types, const char *opt,
> +char *get_string_from_profile(char **types, const char *opt,
> const char *def_val)
> {
> char *ret = 0;
> @@ -1270,7 +1271,7 @@ static char *get_string_from_profile(char **types, const char *opt,
> return (ret);
> }
>
> -static int get_int_from_profile(char **types, const char *opt, int def_val)
> +int get_int_from_profile(char **types, const char *opt, int def_val)
> {
> int ret;
> char **cpp;
> @@ -1293,7 +1294,7 @@ static double get_double_from_profile(char **types, const char *opt,
> return ret;
> }
>
> -static int get_bool_from_profile(char **types, const char *opt, int def_val)
> +int get_bool_from_profile(char **types, const char *opt, int def_val)
> {
> int ret;
> char **cpp;
> @@ -2847,6 +2848,10 @@ no_journal:
> EXT4_FEATURE_RO_COMPAT_QUOTA))
> create_quota_inodes(fs);
>
> + retval = mk_hugefiles(fs);
> + if (retval)
> + com_err(program_name, retval, "while creating huge files");
> +
> if (!quiet)
> printf("%s", _("Writing superblocks and "
> "filesystem accounting information: "));
> diff --git a/misc/mke2fs.conf.5.in b/misc/mke2fs.conf.5.in
> index 1aba87b..8f628a7 100644
> --- a/misc/mke2fs.conf.5.in
> +++ b/misc/mke2fs.conf.5.in
> @@ -417,6 +417,51 @@ system feature is enabled. It can be overridden via the
> .B \-C
> command line option to
> .BR mke2fs (8)
> +.TP
> +.I make_hugefiles
> +This boolean relation enables the creation of pre-allocated files as
> +part of formatting the file system.
> +.TP
> +.I hugefiles_uid
> +This relation controls the user ownership for all of the files and
> +directories created by the
> +.I make_hugefiles
> +feature.
> +.TP
> +.I hugefiles_gid
> +This relation controls the group ownership for all of the files and
> +directories created by the
> +.I make_hugefiles
> +feature.
> +.TP
> +.I hugefiles_umask
> +This relation specifies the umask used when creating the files and
> +directories by the
> +.I make_hugefiles
> +feature.
> +.TP
> +.I num_hugefiles
> +This relation specifies the number of huge files to be created. If this
> +relation is not specified, or is set to zero, and the
> +.I hugefiles_size
> +relation is non-zero, then
> +.I make_hugefiles
> +will create as many huge files as can fit to fill the entire file system.
> +.TP
> +.I hugefiles_slack
> +This relation specifies how much space should be reserved for other
> +files.
> +.TP
> +.I hugefiles_size
> +This relation specifies the size of the huge files. If this relation is
> +not specified, the default is to fill th efile system.
> +.TP
> +.I hugefiles_name
> +This relation specifies the base file name for the huge files.
> +.TP
> +.I hugefiles_digits
> +This relation specifies the (zero-padded) width of the field for the
> +huge file number.
> .SH THE [devices] STANZA
> Each tag in the
> .I [devices]
> diff --git a/misc/mke2fs.h b/misc/mke2fs.h
> new file mode 100644
> index 0000000..73d8c71
> --- /dev/null
> +++ b/misc/mke2fs.h
> @@ -0,0 +1,29 @@
> +/*
> + * mke2fs.h
> + *
> + * Copyright (C) 1994, 1995, 1996, 1997, 1998, 1999, 2000, 2001, 2002,
> + * 2003, 2004, 2005 by Theodore Ts'o.
> + *
> + * %Begin-Header%
> + * This file may be redistributed under the terms of the GNU Public
> + * License.
> + * %End-Header%
> + */
> +
> +/* mke2fs.c */
> +extern const char * program_name;
> +extern int quiet;
> +extern int verbose;
> +extern char **fs_types;
> +
> +extern char *get_string_from_profile(char **types, const char *opt,
> + const char *def_val);
> +extern int get_int_from_profile(char **types, const char *opt, int def_val);
> +extern int get_bool_from_profile(char **types, const char *opt, int def_val);
> +extern int int_log10(unsigned long long arg);
> +
> +/* mk_hugefiles.c */
> +extern errcode_t mk_hugefiles(ext2_filsys fs);
> +
> +
> +
> --
> 1.8.5.rc3.362.gdf10213
>
> --
> To unsubscribe from this list: send the line "unsubscribe linux-ext4" in
> the body of a message to majordomo@xxxxxxxxxxxxxxx
> More majordomo info at http://vger.kernel.org/majordomo-info.html
Cheers, Andreas
Attachment:
signature.asc
Description: Message signed with OpenPGP using GPGMail
