populate: remove file creation loops that take forever

Replace the file creation loops with a perl script that does everything
we want from a single process.  This reduces the runtime of
_scratch_xfs_populate substantially by avoiding thousands of execve
overhead.  On my system, this reduces the runtime of xfs/349 (with scrub
enabled) from ~140s to ~45s.

[zlang: add popdir.pl into src/Makefile install list]

Signed-off-by: Darrick J. Wong <djwong@kernel.org>
Reviewed-by: Zorro Lang <zlang@redhat.com>
Signed-off-by: Zorro Lang <zlang@kernel.org>

diff --git a/common/populate b/common/populate
index 84f4b8e374a344f0b25be7a0549e5193417252c1..180540aedd44de37a8e5da9a20b6003d6f33f63b 100644 (file)
--- a/common/populate
+++ b/common/populate
@@ -11,6 +11,7 @@ _require_populate_commands() {
        _require_xfs_io_command "falloc"
        _require_xfs_io_command "fpunch"
        _require_test_program "punch-alternating"
+       _require_test_program "popdir.pl"
        case "${FSTYP}" in
        "xfs")
                _require_command "$XFS_DB_PROG" "xfs_db"
@@ -54,55 +55,50 @@ __populate_fragment_file() {
 
 # Create a large directory
 __populate_create_dir() {
-       name="$1"
-       nr="$2"
-       missing="$3"
+       local name="$1"
+       local nr="$2"
+       local missing="$3"
+       shift; shift; shift
 
        mkdir -p "${name}"
-       seq 0 "${nr}" | while read d; do
-               creat=mkdir
-               test "$((d % 20))" -eq 0 && creat=touch
-               $creat "${name}/$(printf "%.08d" "$d")"
-       done
+       $here/src/popdir.pl --dir "${name}" --end "${nr}" "$@"
 
        test -z "${missing}" && return
-       seq 1 2 "${nr}" | while read d; do
-               rm -rf "${name}/$(printf "%.08d" "$d")"
-       done
+       $here/src/popdir.pl --dir "${name}" --start 1 --incr 2 --end "${nr}" --remove "$@"
 }
 
 # Create a large directory and ensure that it's a btree format
 __populate_xfs_create_btree_dir() {
        local name="$1"
        local isize="$2"
-       local missing="$3"
+       local dblksz="$3"
+       local missing="$4"
        local icore_size="$(_xfs_get_inode_core_bytes $SCRATCH_MNT)"
        # We need enough extents to guarantee that the data fork is in
        # btree format.  Cycling the mount to use xfs_db is too slow, so
        # watch for when the extent count exceeds the space after the
        # inode core.
        local max_nextents="$(((isize - icore_size) / 16))"
-       local nr=0
+       local nr
+       local incr
+
+       # Add about one block's worth of dirents before we check the data fork
+       # format.
+       incr=$(( (dblksz / 8) / 100 * 100 ))
 
        mkdir -p "${name}"
-       while true; do
-               local creat=mkdir
-               test "$((nr % 20))" -eq 0 && creat=touch
-               $creat "${name}/$(printf "%.08d" "$nr")"
+       for ((nr = 0; ; nr += incr)); do
+               $here/src/popdir.pl --dir "${name}" --start "${nr}" --end "$((nr + incr - 1))"
+
                # Extent count checks use data blocks only to avoid the removal
                # step from removing dabtree index blocks and reducing the
                # number of extents below the required threshold.
-               if [ "$((nr % 40))" -eq 0 ]; then
-                       local nextents="$(xfs_bmap ${name} | grep -v hole | wc -l)"
-                       [ "$((nextents - 1))" -gt $max_nextents ] && break
-               fi
-               nr=$((nr+1))
+               local nextents="$(xfs_bmap ${name} | grep -v hole | wc -l)"
+               [ "$((nextents - 1))" -gt $max_nextents ] && break
        done
 
        test -z "${missing}" && return
-       seq 1 2 "${nr}" | while read d; do
-               rm -rf "${name}/$(printf "%.08d" "$d")"
-       done
+       $here/src/popdir.pl --dir "${name}" --start 1 --incr 2 --end "${nr}" --remove
 }
 
 # Add a bunch of attrs to a file
@@ -224,9 +220,7 @@ _scratch_xfs_populate() {
 
        # Fill up the root inode chunk
        echo "+ fill root ino chunk"
-       seq 1 64 | while read f; do
-               $XFS_IO_PROG -f -c "truncate 0" "${SCRATCH_MNT}/dummy${f}"
-       done
+       $here/src/popdir.pl --dir "${SCRATCH_MNT}" --start 1 --end 64 --format "dummy%u" --file-mult 1
 
        # Regular files
        # - FMT_EXTENTS
@@ -261,7 +255,7 @@ _scratch_xfs_populate() {
 
        # - BTREE
        echo "+ btree dir"
-       __populate_xfs_create_btree_dir "${SCRATCH_MNT}/S_IFDIR.FMT_BTREE" "$isize" true
+       __populate_xfs_create_btree_dir "${SCRATCH_MNT}/S_IFDIR.FMT_BTREE" "$isize" "$dblksz" true
 
        # Symlinks
        # - FMT_LOCAL
@@ -340,14 +334,7 @@ _scratch_xfs_populate() {
        local rec_per_btblock=16
        local nr="$(( 2 * (blksz / rec_per_btblock) * ino_per_rec ))"
        local dir="${SCRATCH_MNT}/INOBT"
-       mkdir -p "${dir}"
-       seq 0 "${nr}" | while read f; do
-               touch "${dir}/${f}"
-       done
-
-       seq 0 2 "${nr}" | while read f; do
-               rm -f "${dir}/${f}"
-       done
+       __populate_create_dir "${dir}" "${nr}" true --file-mult 1
 
        # Reverse-mapping btree
        is_rmapbt="$(_xfs_has_feature "$SCRATCH_MNT" rmapbt -v)"

diff --git a/src/Makefile b/src/Makefile
index afdf6b30c5486e8ce1b454da179969d5902a62c5..fb7298f808b165102483980753d2bfe7cca565ad 100644 (file)
--- a/src/Makefile
+++ b/src/Makefile
@@ -36,7 +36,7 @@ LINUX_TARGETS = xfsctl bstat t_mtab getdevicesize preallo_rw_pattern_reader \
        uuid_ioctl
 
 EXTRA_EXECS = dmerror fill2attr fill2fs fill2fs_check scaleread.sh \
-             btrfs_crc32c_forged_name.py
+             btrfs_crc32c_forged_name.py popdir.pl
 
 SUBDIRS = log-writes perf
 

diff --git a/src/popdir.pl b/src/popdir.pl
new file mode 100755 (executable)
index 0000000..dc0c046
--- /dev/null
+++ b/src/popdir.pl
@@ -0,0 +1,72 @@
+#!/usr/bin/perl -w
+
+# Copyright (c) 2023 Oracle.  All rights reserved.
+# SPDX-License-Identifier: GPL-2.0
+#
+# Create a bunch of files and subdirs in a directory.
+
+use Getopt::Long;
+use File::Basename;
+
+$progname=$0;
+GetOptions("start=i" => \$start,
+          "end=i" => \$end,
+          "file-mult=i" => \$file_mult,
+          "incr=i" => \$incr,
+          "format=s" => \$format,
+          "dir=s" => \$dir,
+          "remove!" => \$remove,
+          "help!" => \$help,
+          "verbose!" => \$verbose);
+
+
+# check/remove output directory, get filesystem info
+if (defined $help) {
+  # newline at end of die message suppresses line number
+  print STDERR <<"EOF";
+Usage: $progname [options]
+Options:
+  --dir             chdir here before starting
+  --start=num       create names starting with this number (0)
+  --incr=num        increment file number by this much (1)
+  --end=num         stop at this file number (100)
+  --file-mult       create a regular file when file number is a multiple
+                    of this quantity (20)
+  --remove          remove instead of creating
+  --format=str      printf formatting string for file name ("%08d")
+  --verbose         verbose output
+  --help            this help screen
+EOF
+  exit(1) unless defined $help;
+  # otherwise...
+  exit(0);
+}
+
+if (defined $dir) {
+       chdir($dir) or die("chdir $dir");
+}
+$start = 0 if (!defined $start);
+$end = 100 if (!defined $end);
+$file_mult = 20 if (!defined $file_mult);
+$format = "%08d" if (!defined $format);
+$incr = 1 if (!defined $incr);
+
+for ($i = $start; $i <= $end; $i += $incr) {
+       $fname = sprintf($format, $i);
+
+       if ($remove) {
+               $verbose && print "rm $fname\n";
+               unlink($fname) or rmdir($fname) or die("unlink $fname");
+       } elsif ($file_mult == 0 or ($i % $file_mult) == 0) {
+               # create a file
+               $verbose && print "touch $fname\n";
+               open(DONTCARE, ">$fname") or die("touch $fname");
+               close(DONTCARE);
+       } else {
+               # create a subdir
+               $verbose && print "mkdir $fname\n";
+               mkdir($fname, 0755) or die("mkdir $fname");
+       }
+}
+
+exit(0);