generic: check the behavior of programs opening a lot of O_TMPFILE files
authorDarrick J. Wong <darrick.wong@oracle.com>
Wed, 13 Feb 2019 20:49:05 +0000 (12:49 -0800)
committerEryu Guan <guaneryu@gmail.com>
Sat, 16 Feb 2019 12:46:30 +0000 (20:46 +0800)
Create a test (+ helper program) that opens as many unlinked files as it
possibly can on the scratch filesystem, then closes all the files at
once to stress-test unlinked file cleanup.  Add an xfs-specific test to
make sure that the fallback code doesn't bitrot.

Signed-off-by: Darrick J. Wong <darrick.wong@oracle.com>
Reviewed-by: Eryu Guan <guaneryu@gmail.com>
Signed-off-by: Eryu Guan <guaneryu@gmail.com>
13 files changed:
.gitignore
src/Makefile
src/t_open_tmpfiles.c [new file with mode: 0644]
tests/generic/530 [new file with mode: 0755]
tests/generic/530.out [new file with mode: 0644]
tests/generic/531 [new file with mode: 0755]
tests/generic/531.out [new file with mode: 0644]
tests/generic/group
tests/xfs/501 [new file with mode: 0755]
tests/xfs/501.out [new file with mode: 0644]
tests/xfs/502 [new file with mode: 0755]
tests/xfs/502.out [new file with mode: 0644]
tests/xfs/group

index 0933dc7..6454e4d 100644 (file)
 /src/t_mmap_writev
 /src/t_mtab
 /src/t_ofd_locks
+/src/t_open_tmpfiles
 /src/t_readdir_1
 /src/t_readdir_2
 /src/t_rename_overwrite
index ae09eb0..dbba7c7 100644 (file)
@@ -27,7 +27,7 @@ LINUX_TARGETS = xfsctl bstat t_mtab getdevicesize preallo_rw_pattern_reader \
        renameat2 t_getcwd e4compact test-nextquota punch-alternating \
        attr-list-by-handle-cursor-test listxattr dio-interleaved t_dir_type \
        dio-invalidate-cache stat_test t_encrypted_d_revalidate \
-       attr_replace_test swapon mkswap t_attr_corruption
+       attr_replace_test swapon mkswap t_attr_corruption t_open_tmpfiles
 
 SUBDIRS = log-writes perf
 
diff --git a/src/t_open_tmpfiles.c b/src/t_open_tmpfiles.c
new file mode 100644 (file)
index 0000000..9e70fba
--- /dev/null
@@ -0,0 +1,166 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * Copyright (C) 2019 Oracle.  All Rights Reserved.
+ * Author: Darrick J. Wong <darrick.wong@oracle.com>
+ *
+ * Test program to open unlinked files and leak them.
+ */
+#ifndef _GNU_SOURCE
+# define _GNU_SOURCE
+#endif
+#include <time.h>
+#include <unistd.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <fcntl.h>
+#include <errno.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <sys/ioctl.h>
+#include "global.h"
+
+static int min_fd = -1;
+static int max_fd = -1;
+static unsigned int nr_opened = 0;
+static float start_time;
+static int shutdown_fs = 0;
+
+void clock_time(float *time)
+{
+       static clockid_t clkid = CLOCK_MONOTONIC;
+       struct timespec ts;
+       int ret;
+
+retry:
+       ret = clock_gettime(clkid, &ts);
+       if (ret) {
+               if (clkid == CLOCK_MONOTONIC) {
+                       clkid = CLOCK_REALTIME;
+                       goto retry;
+               }
+               perror("clock_gettime");
+               exit(2);
+       }
+       *time = ts.tv_sec + ((float)ts.tv_nsec / 1000000000);
+}
+
+/*
+ * Exit the program due to an error.
+ *
+ * If we've exhausted all the file descriptors, make sure we close all the
+ * open fds in the order we received them in order to exploit a quirk of ext4
+ * and xfs where the oldest unlinked inodes are at the /end/ of the unlinked
+ * lists, which will make removing the unlinked files maximally painful.
+ *
+ * If it's some other error, just die and let the kernel sort it out.
+ */
+void die(void)
+{
+       float end_time;
+       int fd;
+
+       switch (errno) {
+       case EMFILE:
+       case ENFILE:
+       case ENOSPC:
+               clock_time(&end_time);
+               printf("Opened %u files in %.2fs.\n", nr_opened,
+                               end_time - start_time);
+               fflush(stdout);
+
+               if (shutdown_fs) {
+                       int flag = XFS_FSOP_GOING_FLAGS_NOLOGFLUSH;
+                       int ret;
+
+                       ret = ioctl(min_fd, XFS_IOC_GOINGDOWN, &flag);
+                       if (ret) {
+                               perror("shutdown");
+                               exit(2);
+                       }
+                       exit(0);
+               }
+
+               clock_time(&start_time);
+               for (fd = min_fd; fd <= max_fd; fd++)
+                       close(fd);
+               clock_time(&end_time);
+               printf("Closed %u files in %.2fs.\n", nr_opened,
+                               end_time - start_time);
+               exit(0);
+               break;
+       default:
+               perror("open?");
+               exit(2);
+               break;
+       }
+}
+
+/* Remember how many file we open and all that. */
+void remember_fd(int fd)
+{
+       if (min_fd == -1 || min_fd > fd)
+               min_fd = fd;
+       if (max_fd == -1 || max_fd < fd)
+               max_fd = fd;
+       nr_opened++;
+}
+
+/* Put an opened file on the unlinked list and leak the fd. */
+void leak_tmpfile(void)
+{
+       int fd = -1;
+       int ret;
+#ifdef O_TMPFILE
+       static int try_o_tmpfile = 1;
+#endif
+
+       /* Try to create an O_TMPFILE and leak the fd. */
+#ifdef O_TMPFILE
+       if (try_o_tmpfile) {
+               fd = open(".", O_TMPFILE | O_RDWR, 0644);
+               if (fd >= 0) {
+                       remember_fd(fd);
+                       return;
+               }
+               if (fd < 0) {
+                       if (errno == EOPNOTSUPP)
+                               try_o_tmpfile = 0;
+                       else
+                               die();
+               }
+       }
+#endif
+
+       /* Oh well, create a new file, unlink it, and leak the fd. */
+       fd = open("./moo", O_CREAT | O_RDWR, 0644);
+       if (fd < 0)
+               die();
+       ret = unlink("./moo");
+       if (ret)
+               die();
+       remember_fd(fd);
+}
+
+/*
+ * Try to put as many files on the unlinked list and then kill them.
+ * The first argument is a directory to chdir into; passing any second arg
+ * will shut down the fs instead of closing files.
+ */
+int main(int argc, char *argv[])
+{
+       int ret;
+
+       if (argc > 1) {
+               ret = chdir(argv[1]);
+               if (ret)
+                       perror(argv[1]);
+       }
+       if (argc > 2 && !strcmp(argv[2], "shutdown"))
+               shutdown_fs = 1;
+
+       clock_time(&start_time);
+       while (1)
+               leak_tmpfile();
+       return 0;
+}
diff --git a/tests/generic/530 b/tests/generic/530
new file mode 100755 (executable)
index 0000000..a2968d2
--- /dev/null
@@ -0,0 +1,65 @@
+#! /bin/bash
+# SPDX-License-Identifier: GPL-2.0+
+# Copyright (c) 2019 Oracle, Inc.  All Rights Reserved.
+#
+# FS QA Test No. 530
+#
+# Stress test creating a lot of unlinked O_TMPFILE files and recovering them
+# after a crash, checking that we don't blow up the filesystem.  This is sort
+# of a performance test for the xfs unlinked inode backref patchset, but it
+# applies to most other filesystems.
+#
+# Use only a single CPU to test the single threaded situation.
+#
+seq=`basename $0`
+seqres=$RESULT_DIR/$seq
+echo "QA output created by $seq"
+tmp=/tmp/$$
+status=1       # failure is the default!
+testfile=$TEST_DIR/$seq.txt
+trap "_cleanup; exit \$status" 0 1 2 3 15
+
+_cleanup()
+{
+       cd /
+       rm -f $tmp.*
+}
+
+# get standard environment, filters and checks
+. ./common/rc
+
+# real QA test starts here
+_supported_fs generic
+_supported_os Linux
+_require_scratch
+_require_scratch_shutdown
+_require_test_program "t_open_tmpfiles"
+
+rm -f $seqres.full
+_scratch_mkfs >> $seqres.full 2>&1
+_scratch_mount
+
+# Set ULIMIT_NOFILE to min(file-max, 50000 files per LOAD_FACTOR)
+# so that this test doesn't take forever or OOM the box
+max_files=$((50000 * LOAD_FACTOR))
+max_allowable_files=$(( $(cat /proc/sys/fs/file-max) ))
+test $max_allowable_files -gt 0 && test $max_files -gt $max_allowable_files && \
+       max_files=$max_allowable_files
+ulimit -n $max_files
+
+# Open a lot of unlinked files
+echo create >> $seqres.full
+$here/src/t_open_tmpfiles $SCRATCH_MNT shutdown >> $seqres.full
+
+# Unmount to prove that we can clean it all
+echo umount >> $seqres.full
+before=$(date +%s)
+_scratch_unmount
+after=$(date +%s)
+echo "Unmount took $((after - before))s." >> $seqres.full
+
+# Mount so that we can run the usual checks
+echo silence is golden
+_scratch_mount
+status=0
+exit
diff --git a/tests/generic/530.out b/tests/generic/530.out
new file mode 100644 (file)
index 0000000..baee78c
--- /dev/null
@@ -0,0 +1,2 @@
+QA output created by 530
+silence is golden
diff --git a/tests/generic/531 b/tests/generic/531
new file mode 100755 (executable)
index 0000000..f3eb5cd
--- /dev/null
@@ -0,0 +1,71 @@
+#! /bin/bash
+# SPDX-License-Identifier: GPL-2.0+
+# Copyright (c) 2019 Oracle, Inc.  All Rights Reserved.
+#
+# FS QA Test No. 531
+#
+# Stress test creating a lot of unlinked O_TMPFILE files and closing them
+# all at once, checking that we don't blow up the filesystem.  This is sort
+# of a performance test for the xfs unlinked inode backref patchset, but it
+# applies to most other filesystems.
+#
+# Use every CPU possible to stress the filesystem.
+#
+seq=`basename $0`
+seqres=$RESULT_DIR/$seq
+echo "QA output created by $seq"
+tmp=/tmp/$$
+status=1       # failure is the default!
+testfile=$TEST_DIR/$seq.txt
+trap "_cleanup; exit \$status" 0 1 2 3 15
+
+_cleanup()
+{
+       cd /
+       rm -f $tmp.*
+}
+
+# get standard environment, filters and checks
+. ./common/rc
+
+# real QA test starts here
+_supported_fs generic
+_supported_os Linux
+_require_scratch
+_require_test_program "t_open_tmpfiles"
+
+rm -f $seqres.full
+_scratch_mkfs >> $seqres.full 2>&1
+_scratch_mount
+
+# Try to load up all the CPUs, two threads per CPU.
+nr_cpus=$(( $(getconf _NPROCESSORS_ONLN) * 2 ))
+
+# Set ULIMIT_NOFILE to min(file-max, 50000 files per LOAD_FACTOR)
+# so that this test doesn't take forever or OOM the box
+max_files=$((50000 * LOAD_FACTOR))
+max_allowable_files=$(( $(cat /proc/sys/fs/file-max) ))
+test $max_allowable_files -gt 0 && test $max_files -gt $max_allowable_files && \
+       max_files=$max_allowable_files
+ulimit -n $max_files
+
+# Open a lot of unlinked files
+echo create >> $seqres.full
+for i in $(seq 1 $nr_cpus); do
+       mkdir $SCRATCH_MNT/$i
+       $here/src/t_open_tmpfiles $SCRATCH_MNT/$i >> $seqres.full &
+done
+wait
+
+# Unmount to prove that we can clean it all
+echo umount >> $seqres.full
+before=$(date +%s)
+_scratch_unmount
+after=$(date +%s)
+echo "Unmount took $((after - before))s." >> $seqres.full
+
+# Mount so that we can run the usual checks
+echo silence is golden
+_scratch_mount
+status=0
+exit
diff --git a/tests/generic/531.out b/tests/generic/531.out
new file mode 100644 (file)
index 0000000..7a3a89c
--- /dev/null
@@ -0,0 +1,2 @@
+QA output created by 531
+silence is golden
index ce5622e..008e89a 100644 (file)
 527 auto quick log
 528 auto quick
 529 auto quick attr
+530 auto quick unlink
+531 auto quick unlink
diff --git a/tests/xfs/501 b/tests/xfs/501
new file mode 100755 (executable)
index 0000000..51cdb02
--- /dev/null
@@ -0,0 +1,91 @@
+#! /bin/bash
+# SPDX-License-Identifier: GPL-2.0+
+# Copyright (c) 2019 Oracle, Inc.  All Rights Reserved.
+#
+# FS QA Test No. 501
+#
+# Stress test creating a lot of unlinked O_TMPFILE files and recovering them
+# after a crash, checking that we don't blow up the filesystem.  This is sort
+# of a performance test for the xfs unlinked inode backref patchset.
+#
+# Here we force the use of the slow iunlink bucket walk code in a single
+# threaded situation.
+#
+seq=`basename $0`
+seqres=$RESULT_DIR/$seq
+echo "QA output created by $seq"
+tmp=/tmp/$$
+status=1       # failure is the default!
+testfile=$TEST_DIR/$seq.txt
+trap "_cleanup; exit \$status" 0 1 2 3 15
+
+delay_knob="/sys/fs/xfs/debug/log_recovery_delay"
+
+_cleanup()
+{
+       cd /
+       test -e "$delay_knob" && echo 0 > "$delay_knob"
+       rm -f $tmp.*
+}
+
+# get standard environment, filters and checks
+. ./common/rc
+. ./common/inject
+
+# real QA test starts here
+_supported_fs xfs
+_supported_os Linux
+_require_xfs_io_error_injection "iunlink_fallback"
+_require_xfs_sysfs debug/log_recovery_delay
+_require_scratch
+_require_test_program "t_open_tmpfiles"
+
+rm -f $seqres.full
+_scratch_mkfs >> $seqres.full 2>&1
+_scratch_mount
+
+# Set ULIMIT_NOFILE to min(file-max, 30000 files per LOAD_FACTOR)
+# so that this test doesn't take forever or OOM the box
+max_files=$((30000 * LOAD_FACTOR))
+max_allowable_files=$(( $(cat /proc/sys/fs/file-max) ))
+test $max_allowable_files -gt 0 && test $max_files -gt $max_allowable_files && \
+       max_files=$max_allowable_files
+ulimit -n $max_files
+
+# Open a lot of unlinked files
+echo create >> $seqres.full
+$here/src/t_open_tmpfiles $SCRATCH_MNT shutdown >> $seqres.full
+
+# Unmount to prove that we can clean it all
+echo umount >> $seqres.full
+before=$(date +%s)
+_scratch_unmount
+after=$(date +%s)
+echo "Unmount took $((after - before))s." >> $seqres.full
+
+# Force xfs to use the iunlinked fallback 50% of the time
+injector() {
+       # Slow down log recovery by 5s to give us enough time to set up
+       # error injection.
+       echo 5 > "$delay_knob"
+
+       # Try for 10s to set our knob.
+       knob="$(_find_xfs_mountdev_errortag_knob "${SCRATCH_DEV}" iunlink_fallback)"
+       nr=0
+       while [ ! -e "$knob" ] && [ "$nr" -lt 20 ]; do
+               sleep 0.5
+               nr=$((nr+1))
+       done
+       if [ -e "$knob" ]; then
+               echo 2 > "$knob"
+       else
+               echo "unable to set iunlink_fallback?"
+       fi
+}
+
+# Mount so that we can run the usual checks
+echo silence is golden
+injector &
+_scratch_mount
+status=0
+exit
diff --git a/tests/xfs/501.out b/tests/xfs/501.out
new file mode 100644 (file)
index 0000000..fc95f46
--- /dev/null
@@ -0,0 +1,2 @@
+QA output created by 501
+silence is golden
diff --git a/tests/xfs/502 b/tests/xfs/502
new file mode 100755 (executable)
index 0000000..bfb063f
--- /dev/null
@@ -0,0 +1,76 @@
+#! /bin/bash
+# SPDX-License-Identifier: GPL-2.0+
+# Copyright (c) 2019 Oracle, Inc.  All Rights Reserved.
+#
+# FS QA Test No. 502
+#
+# Stress test creating a lot of unlinked O_TMPFILE files and closing them
+# all at once, checking that we don't blow up the filesystem.  This is sort
+# of a performance test for the xfs unlinked inode backref patchset.
+#
+# Here we force the use of the slow iunlink bucket walk code, using every
+# CPU possible.
+#
+seq=`basename $0`
+seqres=$RESULT_DIR/$seq
+echo "QA output created by $seq"
+tmp=/tmp/$$
+status=1       # failure is the default!
+testfile=$TEST_DIR/$seq.txt
+trap "_cleanup; exit \$status" 0 1 2 3 15
+
+_cleanup()
+{
+       cd /
+       rm -f $tmp.*
+}
+
+# get standard environment, filters and checks
+. ./common/rc
+. ./common/inject
+
+# real QA test starts here
+_supported_fs xfs
+_supported_os Linux
+_require_xfs_io_error_injection "iunlink_fallback"
+_require_scratch
+_require_test_program "t_open_tmpfiles"
+
+rm -f $seqres.full
+_scratch_mkfs >> $seqres.full 2>&1
+_scratch_mount
+
+# Load up all the CPUs, two threads per CPU.
+nr_cpus=$(( $(getconf _NPROCESSORS_ONLN) * 2 ))
+
+# Set ULIMIT_NOFILE to min(file-max, 30000 files per cpu per LOAD_FACTOR)
+# so that this test doesn't take forever or OOM the box
+max_files=$((30000 * LOAD_FACTOR))
+max_allowable_files=$(( $(cat /proc/sys/fs/file-max) ))
+test $max_allowable_files -gt 0 && test $max_files -gt $max_allowable_files && \
+       max_files=$max_allowable_files
+ulimit -n $max_files
+
+# Force xfs to use the iunlinked fallback 50% of the time
+_scratch_inject_error "iunlink_fallback" "2"
+
+# Open a lot of unlinked files
+echo create >> $seqres.full
+for i in $(seq 1 $nr_cpus); do
+       mkdir $SCRATCH_MNT/$i
+       $here/src/t_open_tmpfiles $SCRATCH_MNT/$i >> $seqres.full &
+done
+wait
+
+# Unmount to prove that we can clean it all
+echo umount >> $seqres.full
+before=$(date +%s)
+_scratch_unmount
+after=$(date +%s)
+echo "Unmount took $((after - before))s." >> $seqres.full
+
+# Mount so that we can run the usual checks
+echo silence is golden
+_scratch_mount
+status=0
+exit
diff --git a/tests/xfs/502.out b/tests/xfs/502.out
new file mode 100644 (file)
index 0000000..c1d4260
--- /dev/null
@@ -0,0 +1,2 @@
+QA output created by 502
+silence is golden
index b458f78..76b8a41 100644 (file)
 498 dangerous_fuzzers dangerous_norepair
 499 auto quick
 500 auto quick mkfs
+501 auto quick unlink
+502 auto quick unlink