From: Darrick J. Wong <djwong@xxxxxxxxxx>
Replace the file creation loops with a perl script that does everything
we want from a single process. This reduces the runtime of
_scratch_xfs_populate substantially by avoiding thousands of execve
overhead. On my system, this reduces the runtime of xfs/349 (with scrub
enabled) from ~140s to ~45s.
Signed-off-by: Darrick J. Wong <djwong@xxxxxxxxxx>
---
common/populate | 61 ++++++++++++++++++-----------------------------
src/popdir.pl | 72 +++++++++++++++++++++++++++++++++++++++++++++++++++++++
2 files changed, 96 insertions(+), 37 deletions(-)
create mode 100755 src/popdir.pl
diff --git a/common/populate b/common/populate
index 84f4b8e374..180540aedd 100644
--- a/common/populate
+++ b/common/populate
@@ -11,6 +11,7 @@ _require_populate_commands() {
_require_xfs_io_command "falloc"
_require_xfs_io_command "fpunch"
_require_test_program "punch-alternating"
+ _require_test_program "popdir.pl"
case "${FSTYP}" in
"xfs")
_require_command "$XFS_DB_PROG" "xfs_db"
@@ -54,55 +55,50 @@ __populate_fragment_file() {
# Create a large directory
__populate_create_dir() {
- name="$1"
- nr="$2"
- missing="$3"
+ local name="$1"
+ local nr="$2"
+ local missing="$3"
+ shift; shift; shift
mkdir -p "${name}"
- seq 0 "${nr}" | while read d; do
- creat=mkdir
- test "$((d % 20))" -eq 0 && creat=touch
- $creat "${name}/$(printf "%.08d" "$d")"
- done
+ $here/src/popdir.pl --dir "${name}" --end "${nr}" "$@"
test -z "${missing}" && return
- seq 1 2 "${nr}" | while read d; do
- rm -rf "${name}/$(printf "%.08d" "$d")"
- done
+ $here/src/popdir.pl --dir "${name}" --start 1 --incr 2 --end "${nr}" --remove "$@"
}
# Create a large directory and ensure that it's a btree format
__populate_xfs_create_btree_dir() {
local name="$1"
local isize="$2"
- local missing="$3"
+ local dblksz="$3"
+ local missing="$4"
local icore_size="$(_xfs_get_inode_core_bytes $SCRATCH_MNT)"
# We need enough extents to guarantee that the data fork is in
# btree format. Cycling the mount to use xfs_db is too slow, so
# watch for when the extent count exceeds the space after the
# inode core.
local max_nextents="$(((isize - icore_size) / 16))"
- local nr=0
+ local nr
+ local incr
+
+ # Add about one block's worth of dirents before we check the data fork
+ # format.
+ incr=$(( (dblksz / 8) / 100 * 100 ))
mkdir -p "${name}"
- while true; do
- local creat=mkdir
- test "$((nr % 20))" -eq 0 && creat=touch
- $creat "${name}/$(printf "%.08d" "$nr")"
+ for ((nr = 0; ; nr += incr)); do
+ $here/src/popdir.pl --dir "${name}" --start "${nr}" --end "$((nr + incr - 1))"
+
# Extent count checks use data blocks only to avoid the removal
# step from removing dabtree index blocks and reducing the
# number of extents below the required threshold.
- if [ "$((nr % 40))" -eq 0 ]; then
- local nextents="$(xfs_bmap ${name} | grep -v hole | wc -l)"
- [ "$((nextents - 1))" -gt $max_nextents ] && break
- fi
- nr=$((nr+1))
+ local nextents="$(xfs_bmap ${name} | grep -v hole | wc -l)"
+ [ "$((nextents - 1))" -gt $max_nextents ] && break
done
test -z "${missing}" && return
- seq 1 2 "${nr}" | while read d; do
- rm -rf "${name}/$(printf "%.08d" "$d")"
- done
+ $here/src/popdir.pl --dir "${name}" --start 1 --incr 2 --end "${nr}" --remove
}
# Add a bunch of attrs to a file
@@ -224,9 +220,7 @@ _scratch_xfs_populate() {
# Fill up the root inode chunk
echo "+ fill root ino chunk"
- seq 1 64 | while read f; do
- $XFS_IO_PROG -f -c "truncate 0" "${SCRATCH_MNT}/dummy${f}"
- done
+ $here/src/popdir.pl --dir "${SCRATCH_MNT}" --start 1 --end 64 --format "dummy%u" --file-mult 1
# Regular files
# - FMT_EXTENTS
@@ -261,7 +255,7 @@ _scratch_xfs_populate() {
# - BTREE
echo "+ btree dir"
- __populate_xfs_create_btree_dir "${SCRATCH_MNT}/S_IFDIR.FMT_BTREE" "$isize" true
+ __populate_xfs_create_btree_dir "${SCRATCH_MNT}/S_IFDIR.FMT_BTREE" "$isize" "$dblksz" true
# Symlinks
# - FMT_LOCAL
@@ -340,14 +334,7 @@ _scratch_xfs_populate() {
local rec_per_btblock=16
local nr="$(( 2 * (blksz / rec_per_btblock) * ino_per_rec ))"
local dir="${SCRATCH_MNT}/INOBT"
- mkdir -p "${dir}"
- seq 0 "${nr}" | while read f; do
- touch "${dir}/${f}"
- done
-
- seq 0 2 "${nr}" | while read f; do
- rm -f "${dir}/${f}"
- done
+ __populate_create_dir "${dir}" "${nr}" true --file-mult 1
# Reverse-mapping btree
is_rmapbt="$(_xfs_has_feature "$SCRATCH_MNT" rmapbt -v)"
diff --git a/src/popdir.pl b/src/popdir.pl
new file mode 100755
index 0000000000..dc0c046b7d
--- /dev/null
+++ b/src/popdir.pl
@@ -0,0 +1,72 @@
+#!/usr/bin/perl -w
+
+# Copyright (c) 2023 Oracle. All rights reserved.
+# SPDX-License-Identifier: GPL-2.0
+#
+# Create a bunch of files and subdirs in a directory.
+
+use Getopt::Long;
+use File::Basename;
+
+$progname=$0;
+GetOptions("start=i" => \$start,
+ "end=i" => \$end,
+ "file-mult=i" => \$file_mult,
+ "incr=i" => \$incr,
+ "format=s" => \$format,
+ "dir=s" => \$dir,
+ "remove!" => \$remove,
+ "help!" => \$help,
+ "verbose!" => \$verbose);
+
+
+# check/remove output directory, get filesystem info
+if (defined $help) {
+ # newline at end of die message suppresses line number
+ print STDERR <<"EOF";
+Usage: $progname [options]
+Options:
+ --dir chdir here before starting
+ --start=num create names starting with this number (0)
+ --incr=num increment file number by this much (1)
+ --end=num stop at this file number (100)
+ --file-mult create a regular file when file number is a multiple
+ of this quantity (20)
+ --remove remove instead of creating
+ --format=str printf formatting string for file name ("%08d")
+ --verbose verbose output
+ --help this help screen
+EOF
+ exit(1) unless defined $help;
+ # otherwise...
+ exit(0);
+}
+
+if (defined $dir) {
+ chdir($dir) or die("chdir $dir");
+}
+$start = 0 if (!defined $start);
+$end = 100 if (!defined $end);
+$file_mult = 20 if (!defined $file_mult);
+$format = "%08d" if (!defined $format);
+$incr = 1 if (!defined $incr);
+
+for ($i = $start; $i <= $end; $i += $incr) {
+ $fname = sprintf($format, $i);
+
+ if ($remove) {
+ $verbose && print "rm $fname\n";
+ unlink($fname) or rmdir($fname) or die("unlink $fname");
+ } elsif ($file_mult == 0 or ($i % $file_mult) == 0) {
+ # create a file
+ $verbose && print "touch $fname\n";
+ open(DONTCARE, ">$fname") or die("touch $fname");
+ close(DONTCARE);
+ } else {
+ # create a subdir
+ $verbose && print "mkdir $fname\n";
+ mkdir($fname, 0755) or die("mkdir $fname");
+ }
+}
+
+exit(0);
