From 15b4c8d2adc2d09b487d91fac27731496215606f Mon Sep 17 00:00:00 2001 From: Dave Chinner Date: Thu, 28 Jun 2007 15:57:11 +0000 Subject: [PATCH] QA tests for filestreams Merge of master-melb:xfs-cmds:29011a by kenmcd. QA tests for filestreams --- 170 | 50 ++++++++++++++ 170.out | 21 ++++++ 171 | 54 +++++++++++++++ 171.out | 21 ++++++ 172 | 48 +++++++++++++ 172.out | 11 +++ 173 | 51 ++++++++++++++ 173.out | 21 ++++++ 174 | 47 +++++++++++++ 174.out | 11 +++ common.filestreams | 168 +++++++++++++++++++++++++++++++++++++++++++++ group | 8 +++ 12 files changed, 511 insertions(+) create mode 100644 170 create mode 100644 170.out create mode 100644 171 create mode 100644 171.out create mode 100644 172 create mode 100644 172.out create mode 100644 173 create mode 100644 173.out create mode 100644 174 create mode 100644 174.out create mode 100644 common.filestreams diff --git a/170 b/170 new file mode 100644 index 00000000..c6bc4e15 --- /dev/null +++ b/170 @@ -0,0 +1,50 @@ +#! /bin/sh +# FSQA Test No. 170 +# +# Check the filestreams allocator is doing its job. +# Multi-file data streams should always write into seperate AGs. +# +#----------------------------------------------------------------------- +# Copyright (c) 2007 Silicon Graphics, Inc. All Rights Reserved. +#----------------------------------------------------------------------- +# +# creator - based on work from ddiss@sgi.co +owner=dgc@sgi.com + +seq=`basename $0` +echo "QA output created by $seq" + +here=`pwd` +tmp=/tmp/$$ +status=1 # failure is the default! +trap "_cleanup; exit \$status" 0 1 2 3 15 + +_cleanup() +{ + cd / + rm -rf ${SCRATCH_MNT}/stream* +} + +# get standard environment, filters and checks +. ./common.rc +. ./common.filter + +# real QA test starts here +_supported_fs xfs +_supported_os IRIX Linux + +_require_scratch + +. ./common.filestreams + +# test small stream, multiple I/O per file, 30s timeout +# XXX: irix timer mechanism? +echo 3000 > /proc/sys/fs/xfs/filestream_centisecs +_test_streams 8 16 4 8 3 0 0 +_test_streams 8 16 4 8 3 1 0 +_test_streams 8 16 4 8 3 0 1 +_test_streams 8 16 4 8 3 1 1 + +status=0 +exit + diff --git a/170.out b/170.out new file mode 100644 index 00000000..a1339cec --- /dev/null +++ b/170.out @@ -0,0 +1,21 @@ +QA output created by 170 +# testing 8 16 4 8 3 0 0 .... +# streaming +# sync AGs... +# checking stream AGs... ++ passed, streams are in seperate AGs +# testing 8 16 4 8 3 1 0 .... +# streaming +# sync AGs... +# checking stream AGs... ++ passed, streams are in seperate AGs +# testing 8 16 4 8 3 0 1 .... +# streaming +# sync AGs... +# checking stream AGs... ++ passed, streams are in seperate AGs +# testing 8 16 4 8 3 1 1 .... +# streaming +# sync AGs... +# checking stream AGs... ++ passed, streams are in seperate AGs diff --git a/171 b/171 new file mode 100644 index 00000000..e5cae7b6 --- /dev/null +++ b/171 @@ -0,0 +1,54 @@ +#! /bin/sh +# FSQA Test No. 171 +# +# Check the filestreams allocator is doing its job. +# Multi-file data streams should always write into seperate AGs. +# +#----------------------------------------------------------------------- +# Copyright (c) 2007 Silicon Graphics, Inc. All Rights Reserved. +#----------------------------------------------------------------------- +# +# creator +owner=dgc@sgi.com + +seq=`basename $0` +echo "QA output created by $seq" + +here=`pwd` +tmp=/tmp/$$ +status=1 # failure is the default! +trap "_cleanup; exit \$status" 0 1 2 3 15 + +_cleanup() +{ + cd / + rm -rf ${SCRATCH_MNT}/stream* +} + +# get standard environment, filters and checks +. ./common.rc +. ./common.filter + +# real QA test starts here +_supported_fs xfs +_supported_os IRIX Linux + +_require_scratch + +. ./common.filestreams + +# test large numbers of files, single I/O per file, 120s timeout +# Get close to filesystem full. +# 128 = ENOSPC +# 120 = 93.75% full, gets repeatable failures +# 112 = 87.5% full, should reliably succeed but doesn't *FIXME* +# 100 = 78.1% full, should reliably succeed +echo 12000 > /proc/sys/fs/xfs/filestream_centisecs +_test_streams 64 16 8 100 1 1 0 +_test_streams 64 16 8 100 1 1 1 +_test_streams 64 16 8 100 1 0 0 +_test_streams 64 16 8 100 1 0 1 + +status=0 +exit + diff --git a/171.out b/171.out new file mode 100644 index 00000000..89407cb2 --- /dev/null +++ b/171.out @@ -0,0 +1,21 @@ +QA output created by 171 +# testing 64 16 8 100 1 1 0 .... +# streaming +# sync AGs... +# checking stream AGs... ++ passed, streams are in seperate AGs +# testing 64 16 8 100 1 1 1 .... +# streaming +# sync AGs... +# checking stream AGs... ++ passed, streams are in seperate AGs +# testing 64 16 8 100 1 0 0 .... +# streaming +# sync AGs... +# checking stream AGs... ++ passed, streams are in seperate AGs +# testing 64 16 8 100 1 0 1 .... +# streaming +# sync AGs... +# checking stream AGs... ++ passed, streams are in seperate AGs diff --git a/172 b/172 new file mode 100644 index 00000000..f80ef517 --- /dev/null +++ b/172 @@ -0,0 +1,48 @@ +#! /bin/sh +# FSQA Test No. 172 +# +# Check the filestreams allocator is doing its job. +# Multi-file data streams should always write into seperate AGs. +# +#----------------------------------------------------------------------- +# Copyright (c) 2007 Silicon Graphics, Inc. All Rights Reserved. +#----------------------------------------------------------------------- +# +# creator +owner=dgc@sgi.com + +seq=`basename $0` +echo "QA output created by $seq" + +here=`pwd` +tmp=/tmp/$$ +status=1 # failure is the default! +trap "_cleanup; exit \$status" 0 1 2 3 15 + +_cleanup() +{ + cd / + rm -rf ${SCRATCH_MNT}/stream* +} + +# get standard environment, filters and checks +. ./common.rc +. ./common.filter + +# real QA test starts here +_supported_fs xfs +_supported_os IRIX Linux + +_require_scratch + +. ./common.filestreams + +# test reaper works by setting timeout low. Expected to fail +# for buffered, succeed for direct I/O. +echo 50 > /proc/sys/fs/xfs/filestream_centisecs +_test_streams 8 16 4 8 3 1 0 fail +_test_streams 64 16 20 10 1 0 1 + +status=0 +exit + diff --git a/172.out b/172.out new file mode 100644 index 00000000..ec2bcf67 --- /dev/null +++ b/172.out @@ -0,0 +1,11 @@ +QA output created by 172 +# testing 8 16 4 8 3 1 0 fail .... +# streaming +# sync AGs... +# checking stream AGs... ++ expected failure, matching AGs +# testing 64 16 20 10 1 0 1 .... +# streaming +# sync AGs... +# checking stream AGs... ++ passed, streams are in seperate AGs diff --git a/173 b/173 new file mode 100644 index 00000000..bfb8feab --- /dev/null +++ b/173 @@ -0,0 +1,51 @@ +#! /bin/sh +# FSQA Test No. 173 +# +# Check the filestreams allocator is doing its job. +# Multi-file data streams should always write into seperate AGs. +# +#----------------------------------------------------------------------- +# Copyright (c) 2007 Silicon Graphics, Inc. All Rights Reserved. +#----------------------------------------------------------------------- +# +# creator +owner=dgc@sgi.com + +seq=`basename $0` +echo "QA output created by $seq" + +here=`pwd` +tmp=/tmp/$$ +status=1 # failure is the default! +trap "_cleanup; exit \$status" 0 1 2 3 15 + +_cleanup() +{ + cd / + rm -rf ${SCRATCH_MNT}/stream* +} + +# get standard environment, filters and checks +. ./common.rc +. ./common.filter + +# real QA test starts here +_supported_fs xfs +_supported_os IRIX Linux + +_require_scratch + +. ./common.filestreams + +# test large number of streams, multiple I/O per file, 120s timeout +# Because each stream spills over an AG, the stream count needs to +# be less than or equal to half the AG count so we don't run out of AGs. +echo 12000 > /proc/sys/fs/xfs/filestream_centisecs +_test_streams 64 16 33 8 2 1 1 fail +_test_streams 64 16 32 8 2 0 1 +_test_streams 64 16 33 8 2 0 0 fail +_test_streams 64 16 32 8 2 1 0 + +status=0 +exit + diff --git a/173.out b/173.out new file mode 100644 index 00000000..21493057 --- /dev/null +++ b/173.out @@ -0,0 +1,21 @@ +QA output created by 173 +# testing 64 16 33 8 2 1 1 fail .... +# streaming +# sync AGs... +# checking stream AGs... ++ expected failure, matching AGs +# testing 64 16 32 8 2 0 1 .... +# streaming +# sync AGs... +# checking stream AGs... ++ passed, streams are in seperate AGs +# testing 64 16 33 8 2 0 0 fail .... +# streaming +# sync AGs... +# checking stream AGs... ++ expected failure, matching AGs +# testing 64 16 32 8 2 1 0 .... +# streaming +# sync AGs... +# checking stream AGs... ++ passed, streams are in seperate AGs diff --git a/174 b/174 new file mode 100644 index 00000000..21924dc9 --- /dev/null +++ b/174 @@ -0,0 +1,47 @@ +#! /bin/sh +# FSQA Test No. 174 +# +# Check the filestreams allocator is doing its job. +# Multi-file data streams should always write into seperate AGs. +# +#----------------------------------------------------------------------- +# Copyright (c) 2007 Silicon Graphics, Inc. All Rights Reserved. +#----------------------------------------------------------------------- +# +# creator +owner=dgc@sgi.com + +seq=`basename $0` +echo "QA output created by $seq" + +here=`pwd` +tmp=/tmp/$$ +status=1 # failure is the default! +trap "_cleanup; exit \$status" 0 1 2 3 15 + +_cleanup() +{ + cd / + rm -rf ${SCRATCH_MNT}/stream* +} + +# get standard environment, filters and checks +. ./common.rc +. ./common.filter + +# real QA test starts here +_supported_fs xfs +_supported_os IRIX Linux + +_require_scratch + +. ./common.filestreams + +# test number of streams greater than AGs. Expected to fail. +echo 6000 > /proc/sys/fs/xfs/filestream_centisecs +_test_streams 8 32 65 3 1 1 0 fail +_test_streams 8 32 65 3 1 0 1 fail + +status=0 +exit + diff --git a/174.out b/174.out new file mode 100644 index 00000000..5df581fe --- /dev/null +++ b/174.out @@ -0,0 +1,11 @@ +QA output created by 174 +# testing 8 32 65 3 1 1 0 fail .... +# streaming +# sync AGs... +# checking stream AGs... ++ expected failure, matching AGs +# testing 8 32 65 3 1 0 1 fail .... +# streaming +# sync AGs... +# checking stream AGs... ++ expected failure, matching AGs diff --git a/common.filestreams b/common.filestreams new file mode 100644 index 00000000..adabffd6 --- /dev/null +++ b/common.filestreams @@ -0,0 +1,168 @@ +##/bin/sh +# +# Copyright (c) 2007 Silicon Graphics, Inc. All Rights Reserved. +# +# Core of filestreams tests. +# + +_do_stream() +{ + local directory_name=$1 + local files=$2 + local file_size=$3 + local bsize=$4 + local iflag=$5 + local dio=$6 + + local count=`expr $file_size / $bsize` + mkdir $directory_name + if [ "$iflag" = "1" ]; then + $XFS_IO_PROG -x -c "chattr +S" $directory_name \ + || _fail "chattr of filestream flag" + fi + cd $directory_name + local i=1 + local oflags="" + if [ "$dio" = "1" ]; then + oflags="oflag=direct" + fi + while [ $i -le $files ]; do + dd if=/dev/zero of=frame-${i} $oflags \ + bs=$bsize count=$count >/dev/null 2>&1 + i=`expr $i + 1` + done +} + +_filter_agno() +{ + # the ag number is in column 4 of xfs_bmap output + perl -ne ' + $ag = (split /\s+/)[4] ; + if ($ag =~ /\d+/) {print "$ag "} ; + ' +} + +_get_stream_ags() +{ + local directory_name=$1 + local stream_ags=`xfs_bmap -vp ${directory_name}/* | _filter_agno` + echo $stream_ags +} + +_check_for_dupes() +{ + # check for duplicate numbers between two space seperated vars + local num_str_one=$1 + local num_str_two=$2 + + local this_num_one + local this_num_two + for this_num_one in $num_str_one; do + for this_num_two in $num_str_two; do + if [ "$this_num_one" == "$this_num_two" ]; then + echo "duplicate AG $this_num_one found" \ + >> $here/$seq.full + return 1 + fi + done + done + return 0 +} + +rm -f $here/$seq.full + +# test that filestreams are available +umount $SCRATCH_MNT > /dev/null 2>&1 + +_scratch_mkfs_xfs > /dev/null 2>&1 \ + || _fail "mkfs failed" + +_scratch_mount "-o filestreams" \ + || _notrun "filestreams mount not available" + +sync +umount $SCRATCH_MNT > /dev/null 2>&1 + +_test_streams() { + + echo "# testing $* ...." + local agcount="$1" + local agsize="$2" # in MB + local stream_count="$3" + local stream_files="$4" + local stream_file_size=`expr $5 \* 1024 \* 1024` + local use_iflag="$6" + local use_directio="$7" + local expected_result="$8" + + local size=`expr $agsize \* 1024 \* 1024 \* $agcount` + _scratch_mkfs_xfs -dsize=$size,agcount=$agcount >/dev/null 2>&1 \ + || _fail "mkfs failed" + + if [ "$use_iflag" = "0" ]; then + # mount using filestreams mount option + _scratch_mount "-o filestreams" \ + || _notrun "filestreams mount not available" + else + # test will set inode flag + _scratch_mount + fi + + + cd $SCRATCH_MNT + + # start four streams, each writing 24m (8 x 3m files) + echo "# streaming" + stream_pids="" + stream_index=1 + while [ $stream_index -le $stream_count ]; do + _do_stream stream${stream_index}-dir $stream_files \ + $stream_file_size 1048576 $use_iflag $use_directio & + stream_pids="$stream_pids $!" + stream_index=`expr $stream_index + 1` + done + + # wait for streams to finish + # XXX wait here not needed? -dgc + wait $stream_pids + + # sync the buffered streams out in parallel + echo "# sync AGs..." + stream_pids="" + stream_index=1 + while [ $stream_index -le $stream_count ]; do + _get_stream_ags stream${stream_index}-dir > /dev/null 2>&1 & + stream_pids="$stream_pids $!" + stream_index=`expr $stream_index + 1` + done + + # wait for streams to finish + wait $stream_pids + + # confirm streams are in seperate AGs + echo "# checking stream AGs..." + ags_seen="" + stream_index=1 + while [ $stream_index -le $stream_count ]; do + _get_stream_ags stream${stream_index}-dir > $tmp.${stream_index} & + this_stream_ags=`_get_stream_ags stream${stream_index}-dir` + echo "stream $stream_index AGs: $this_stream_ags" >> $here/$seq.full + _check_for_dupes "$ags_seen" "$this_stream_ags" + if [ $? -ne 0 ]; then + if [ "$expected_result" = "fail" ]; then + echo "+ expected failure, matching AGs" + expected_result="failed" + break + else + _fail "- failed, streams with matching AGs" + fi + fi + ags_seen="$ags_seen $this_stream_ags" + stream_index=`expr $stream_index + 1` + done + if [ "$expected_result" != "failed" ]; then + echo "+ passed, streams are in seperate AGs" + fi + cd $here + umount $SCRATCH_MNT +} diff --git a/group b/group index 3e64988b..ad59e0b3 100644 --- a/group +++ b/group @@ -83,6 +83,9 @@ pattern ajones@sgi.com # dmapi based tests dmapi +# filestreams based tests +filestreams dgc@sgi.com + # test-group association ... one line per test # 001 rw dir udf auto @@ -253,3 +256,8 @@ dmapi 166 rw metadata auto 167 rw metadata auto 168 dmapi auto +170 rw filestreams auto +171 rw filestreams auto +172 rw filestreams auto +173 rw filestreams auto +174 rw filestreams auto -- 2.47.3