common/populate

   1 ##/bin/bash
   2 # Routines for populating a scratch fs, and helpers to exercise an FS
   3 # once it's been fuzzed.
   4 #-----------------------------------------------------------------------
   5 #  Copyright (c) 2015 Oracle.  All Rights Reserved.
   6 #  This program is free software; you can redistribute it and/or modify
   7 #  it under the terms of the GNU General Public License as published by
   8 #  the Free Software Foundation; either version 2 of the License, or
   9 #  (at your option) any later version.
  10 #
  11 #  This program is distributed in the hope that it will be useful,
  12 #  but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 #  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14 #  GNU General Public License for more details.
  15 #
  16 #  You should have received a copy of the GNU General Public License
  17 #  along with this program; if not, write to the Free Software
  18 #  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307
  19 #  USA
  20 #
  21 #  Contact information: Silicon Graphics, Inc., 1500 Crittenden Lane,
  22 #  Mountain View, CA 94043, USA, or: http://www.sgi.com
  23 #-----------------------------------------------------------------------
  24 . ./common/quota
  25
  26 _require_populate_commands() {
  27         _require_xfs_io_command "falloc"
  28         _require_xfs_io_command "fpunch"
  29         _require_test_program "punch-alternating"
  30         _require_command "$XFS_DB_PROG" "xfs_db"
  31 }
  32
  33 _require_xfs_db_blocktrash_z_command() {
  34         test "${FSTYP}" = "xfs" || _notrun "cannot run xfs_db on ${FSTYP}"
  35         $XFS_DB_PROG -x -f -c 'blocktrash -z' "${TEST_DEV}" | grep -q 'nothing on stack' || _notrun "blocktrash -z not supported"
  36 }
  37
  38 # Attempt to make files of "every" format for data, dirs, attrs etc.
  39 # (with apologies to Eric Sandeen for mutating xfser.sh)
  40
  41 # Create a large directory
  42 __populate_create_dir() {
  43         name="$1"
  44         nr="$2"
  45         missing="$3"
  46
  47         mkdir -p "${name}"
  48         seq 0 "${nr}" | while read d; do
  49                 creat=mkdir
  50                 test "$((d % 20))" -eq 0 && creat=touch
  51                 $creat "${name}/$(printf "%.08d" "$d")"
  52         done
  53
  54         test -z "${missing}" && return
  55         seq 1 2 "${nr}" | while read d; do
  56                 rm -rf "${name}/$(printf "%.08d" "$d")"
  57         done
  58 }
  59
  60 # Add a bunch of attrs to a file
  61 __populate_create_attr() {
  62         name="$1"
  63         nr="$2"
  64         missing="$3"
  65
  66         touch "${name}"
  67         seq 0 "${nr}" | while read d; do
  68                 setfattr -n "user.$(printf "%.08d" "$d")" -v "$(printf "%.08d" "$d")" "${name}"
  69         done
  70
  71         test -z "${missing}" && return
  72         seq 1 2 "${nr}" | while read d; do
  73                 setfattr -x "user.$(printf "%.08d" "$d")" "${name}"
  74         done
  75 }
  76
  77 # Fill up some percentage of the remaining free space
  78 __populate_fill_fs() {
  79         dir="$1"
  80         pct="$2"
  81         test -z "${pct}" && pct=60
  82
  83         mkdir -p "${dir}/test/1"
  84         cp -pRdu "${dir}"/S_IFREG* "${dir}/test/1/"
  85
  86         SRC_SZ="$(du -ks "${dir}/test/1" | cut -f 1)"
  87         FS_SZ="$(( $(stat -f "${dir}" -c '%a * %S') / 1024 ))"
  88
  89         NR="$(( (FS_SZ * ${pct} / 100) / SRC_SZ ))"
  90
  91         echo "FILL FS"
  92         echo "src_sz $SRC_SZ fs_sz $FS_SZ nr $NR"
  93         seq 2 "${NR}" | while read nr; do
  94                 cp -pRdu "${dir}/test/1" "${dir}/test/${nr}"
  95         done
  96 }
  97
  98 # For XFS, force on all the quota options if quota is enabled
  99 # and the user didn't feed us noquota.
 100 _populate_xfs_qmount_option()
 101 {
 102         # User explicitly told us not to quota
 103         if echo "${MOUNT_OPTIONS}" | grep -q 'noquota'; then
 104                 return
 105         fi
 106
 107         # Don't bother if we can't turn on quotas
 108         if [ ! -f /proc/fs/xfs/xqmstat ]; then
 109                 # No quota support
 110                 return
 111         elif [ "${USE_EXTERNAL}" = "yes" ] && [ ! -z "${SCRATCH_RTDEV}" ]; then
 112                 # Quotas not supported on rt filesystems
 113                 return
 114         elif [ -z "${XFS_QUOTA_PROG}" ]; then
 115                 # xfs quota tools not installed
 116                 return
 117         fi
 118
 119         # Turn on all the quotas
 120         if xfs_info "${TEST_DIR}" | grep -q 'crc=1'; then
 121                 # v5 filesystems can have group & project quotas
 122                 quota="usrquota,grpquota,prjquota"
 123         else
 124                 # v4 filesystems cannot mix group & project quotas
 125                 quota="usrquota,grpquota"
 126         fi
 127
 128         # Inject our quota mount options
 129         if echo "${MOUNT_OPTIONS}" | grep -q "${quota}"; then
 130                 return
 131         elif echo "${MOUNT_OPTIONS}" | egrep -q '(quota|noenforce)'; then
 132                 _qmount_option "${quota}"
 133         else
 134                 export MOUNT_OPTIONS="$MOUNT_OPTIONS -o ${quota}"
 135                 echo "MOUNT_OPTIONS = $MOUNT_OPTIONS" >>$seqres.full
 136         fi
 137 }
 138
 139 # Populate an XFS on the scratch device with (we hope) all known
 140 # types of metadata block
 141 _scratch_xfs_populate() {
 142         fill=1
 143
 144         for arg in $@; do
 145                 case "${arg}" in
 146                 "nofill")
 147                         fill=0;;
 148                 esac
 149         done
 150
 151         _populate_xfs_qmount_option
 152         _scratch_mount
 153         blksz="$(stat -f -c '%s' "${SCRATCH_MNT}")"
 154         dblksz="$(xfs_info "${SCRATCH_MNT}" | grep naming.*bsize | sed -e 's/^.*bsize=//g' -e 's/\([0-9]*\).*$/\1/g')"
 155         leaf_lblk="$((32 * 1073741824 / blksz))"
 156         node_lblk="$((64 * 1073741824 / blksz))"
 157
 158         # Data:
 159
 160         # Fill up the root inode chunk
 161         echo "+ fill root ino chunk"
 162         seq 1 64 | while read f; do
 163                 $XFS_IO_PROG -f -c "truncate 0" "${SCRATCH_MNT}/dummy${f}"
 164         done
 165
 166         # Regular files
 167         # - FMT_EXTENTS
 168         echo "+ extents file"
 169         $XFS_IO_PROG -f -c "pwrite -S 0x61 0 ${blksz}" "${SCRATCH_MNT}/S_IFREG.FMT_EXTENTS"
 170
 171         # - FMT_BTREE
 172         echo "+ btree extents file"
 173         nr="$((blksz * 2 / 16))"
 174         $XFS_IO_PROG -f -c "pwrite -S 0x62 0 $((blksz * nr))" "${SCRATCH_MNT}/S_IFREG.FMT_BTREE"
 175         ./src/punch-alternating "${SCRATCH_MNT}/S_IFREG.FMT_BTREE"
 176
 177         # Directories
 178         # - INLINE
 179         echo "+ inline dir"
 180         __populate_create_dir "${SCRATCH_MNT}/S_IFDIR.FMT_INLINE" 1
 181
 182         # - BLOCK
 183         echo "+ block dir"
 184         __populate_create_dir "${SCRATCH_MNT}/S_IFDIR.FMT_BLOCK" "$((dblksz / 40))"
 185
 186         # - LEAF
 187         echo "+ leaf dir"
 188         __populate_create_dir "${SCRATCH_MNT}/S_IFDIR.FMT_LEAF" "$((dblksz / 12))"
 189
 190         # - NODE
 191         echo "+ node dir"
 192         __populate_create_dir "${SCRATCH_MNT}/S_IFDIR.FMT_NODE" "$((16 * dblksz / 40))" true
 193
 194         # - BTREE
 195         echo "+ btree dir"
 196         __populate_create_dir "${SCRATCH_MNT}/S_IFDIR.FMT_BTREE" "$((128 * dblksz / 40))" true
 197
 198         # Symlinks
 199         # - FMT_LOCAL
 200         echo "+ inline symlink"
 201         ln -s target "${SCRATCH_MNT}/S_IFLNK.FMT_LOCAL"
 202
 203         # - FMT_EXTENTS
 204         echo "+ extents symlink"
 205         ln -s "$(perl -e 'print "x" x 1023;')" "${SCRATCH_MNT}/S_IFLNK.FMT_EXTENTS"
 206
 207         # Char & block
 208         echo "+ special"
 209         mknod "${SCRATCH_MNT}/S_IFCHR" c 1 1
 210         mknod "${SCRATCH_MNT}/S_IFBLK" c 1 1
 211
 212         # special file with an xattr
 213         setfacl -P -m u:nobody:r ${SCRATCH_MNT}/S_IFCHR
 214
 215         # Attribute formats
 216         # LOCAL
 217         echo "+ local attr"
 218         __populate_create_attr "${SCRATCH_MNT}/ATTR.FMT_LOCAL" 1
 219
 220         # LEAF
 221         echo "+ leaf attr"
 222         __populate_create_attr "${SCRATCH_MNT}/ATTR.FMT_LEAF" "$((blksz / 40))"
 223
 224         # NODE
 225         echo "+ node attr"
 226         __populate_create_attr "${SCRATCH_MNT}/ATTR.FMT_NODE" "$((8 * blksz / 40))"
 227
 228         # BTREE
 229         echo "+ btree attr"
 230         __populate_create_attr "${SCRATCH_MNT}/ATTR.FMT_BTREE" "$((64 * blksz / 40))" true
 231
 232         # trusted namespace
 233         touch ${SCRATCH_MNT}/ATTR.TRUSTED
 234         setfattr -n trusted.moo -v urk ${SCRATCH_MNT}/ATTR.TRUSTED
 235
 236         # security namespace
 237         touch ${SCRATCH_MNT}/ATTR.SECURITY
 238         setfattr -n security.foo -v bar ${SCRATCH_MNT}/ATTR.SECURITY
 239
 240         # system namespace
 241         touch ${SCRATCH_MNT}/ATTR.SYSTEM
 242         setfacl -m u:root:r ${SCRATCH_MNT}/ATTR.SYSTEM
 243
 244         # FMT_EXTENTS with a remote less-than-a-block value
 245         echo "+ attr extents with a remote less-than-a-block value"
 246         touch "${SCRATCH_MNT}/ATTR.FMT_EXTENTS_REMOTE3K"
 247         $XFS_IO_PROG -f -c "pwrite -S 0x43 0 $((blksz - 300))" "${SCRATCH_MNT}/attrvalfile" > /dev/null
 248         attr -q -s user.remotebtreeattrname "${SCRATCH_MNT}/ATTR.FMT_EXTENTS_REMOTE3K" < "${SCRATCH_MNT}/attrvalfile"
 249
 250         # FMT_EXTENTS with a remote block-size value
 251         echo "+ attr extents with a remote one-block value"
 252         touch "${SCRATCH_MNT}/ATTR.FMT_EXTENTS_REMOTE4K"
 253         $XFS_IO_PROG -f -c "pwrite -S 0x44 0 ${blksz}" "${SCRATCH_MNT}/attrvalfile" > /dev/null
 254         attr -q -s user.remotebtreeattrname "${SCRATCH_MNT}/ATTR.FMT_EXTENTS_REMOTE4K" < "${SCRATCH_MNT}/attrvalfile"
 255         rm -rf "${SCRATCH_MNT}/attrvalfile"
 256
 257         # Make an unused inode
 258         echo "+ empty file"
 259         touch "${SCRATCH_MNT}/unused"
 260         $XFS_IO_PROG -f -c 'fsync' "${SCRATCH_MNT}/unused"
 261         rm -rf "${SCRATCH_MNT}/unused"
 262
 263         # Free space btree
 264         echo "+ freesp btree"
 265         nr="$((blksz * 2 / 8))"
 266         $XFS_IO_PROG -f -c "pwrite -S 0x62 0 $((blksz * nr))" "${SCRATCH_MNT}/BNOBT"
 267         ./src/punch-alternating "${SCRATCH_MNT}/BNOBT"
 268
 269         # Reverse-mapping btree
 270         is_rmapbt="$(xfs_info "${SCRATCH_MNT}" | grep -c 'rmapbt=1')"
 271         if [ $is_rmapbt -gt 0 ]; then
 272                 echo "+ rmapbt btree"
 273                 nr="$((blksz * 2 / 24))"
 274                 $XFS_IO_PROG -f -c "pwrite -S 0x62 0 $((blksz * nr))" "${SCRATCH_MNT}/RMAPBT"
 275                 ./src/punch-alternating "${SCRATCH_MNT}/RMAPBT"
 276         fi
 277
 278         # Realtime Reverse-mapping btree
 279         is_rt="$(xfs_info "${SCRATCH_MNT}" | grep -c 'rtextents=[1-9]')"
 280         if [ $is_rmapbt -gt 0 ] && [ $is_rt -gt 0 ]; then
 281                 echo "+ rtrmapbt btree"
 282                 nr="$((blksz * 2 / 32))"
 283                 $XFS_IO_PROG -f -R -c "pwrite -S 0x62 0 $((blksz * nr))" "${SCRATCH_MNT}/RTRMAPBT"
 284                 ./src/punch-alternating "${SCRATCH_MNT}/RTRMAPBT"
 285         fi
 286
 287         # Reference-count btree
 288         is_reflink="$(xfs_info "${SCRATCH_MNT}" | grep -c 'reflink=1')"
 289         if [ $is_reflink -gt 0 ]; then
 290                 echo "+ reflink btree"
 291                 nr="$((blksz * 2 / 12))"
 292                 $XFS_IO_PROG -f -c "pwrite -S 0x62 0 $((blksz * nr))" "${SCRATCH_MNT}/REFCOUNTBT"
 293                 cp --reflink=always "${SCRATCH_MNT}/REFCOUNTBT" "${SCRATCH_MNT}/REFCOUNTBT2"
 294                 ./src/punch-alternating "${SCRATCH_MNT}/REFCOUNTBT"
 295         fi
 296
 297         # Copy some real files (xfs tests, I guess...)
 298         echo "+ real files"
 299         test $fill -ne 0 && __populate_fill_fs "${SCRATCH_MNT}" 5
 300
 301         umount "${SCRATCH_MNT}"
 302 }
 303
 304 # Populate an ext4 on the scratch device with (we hope) all known
 305 # types of metadata block
 306 _scratch_ext4_populate() {
 307         fill=1
 308
 309         for arg in $@; do
 310                 case "${arg}" in
 311                 "nofill")
 312                         fill=0;;
 313                 esac
 314         done
 315
 316         _scratch_mount
 317         blksz="$(stat -f -c '%s' "${SCRATCH_MNT}")"
 318         dblksz="${blksz}"
 319         leaf_lblk="$((32 * 1073741824 / blksz))"
 320         node_lblk="$((64 * 1073741824 / blksz))"
 321
 322         # Data:
 323
 324         # Regular files
 325         # - FMT_INLINE
 326         echo "+ inline file"
 327         $XFS_IO_PROG -f -c "pwrite -S 0x61 0 1" "${SCRATCH_MNT}/S_IFREG.FMT_INLINE"
 328
 329         # - FMT_EXTENTS
 330         echo "+ extents file"
 331         $XFS_IO_PROG -f -c "pwrite -S 0x61 0 ${blksz}" "${SCRATCH_MNT}/S_IFREG.FMT_EXTENTS"
 332
 333         # - FMT_ETREE
 334         echo "+ extent tree file"
 335         nr="$((blksz * 2 / 12))"
 336         $XFS_IO_PROG -f -c "pwrite -S 0x62 0 $((blksz * nr))" "${SCRATCH_MNT}/S_IFREG.FMT_ETREE"
 337         ./src/punch-alternating "${SCRATCH_MNT}/S_IFREG.FMT_ETREE"
 338
 339         # Directories
 340         # - INLINE
 341         echo "+ inline dir"
 342         __populate_create_dir "${SCRATCH_MNT}/S_IFDIR.FMT_INLINE" 1
 343
 344         # - BLOCK
 345         echo "+ block dir"
 346         __populate_create_dir "${SCRATCH_MNT}/S_IFDIR.FMT_BLOCK" "$((dblksz / 24))"
 347
 348         # - HTREE
 349         echo "+ htree dir"
 350         __populate_create_dir "${SCRATCH_MNT}/S_IFDIR.FMT_HTREE" "$((4 * dblksz / 24))"
 351
 352         # Symlinks
 353         # - FMT_LOCAL
 354         echo "+ inline symlink"
 355         ln -s target "${SCRATCH_MNT}/S_IFLNK.FMT_LOCAL"
 356
 357         # - FMT_EXTENTS
 358         echo "+ extents symlink"
 359         ln -s "$(perl -e 'print "x" x 1023;')" "${SCRATCH_MNT}/S_IFLNK.FMT_EXTENTS"
 360
 361         # Char & block
 362         echo "+ special"
 363         mknod "${SCRATCH_MNT}/S_IFCHR" c 1 1
 364         mknod "${SCRATCH_MNT}/S_IFBLK" c 1 1
 365
 366         # special file with an xattr
 367         setfacl -P -m u:nobody:r ${SCRATCH_MNT}/S_IFCHR
 368
 369         # Attribute formats
 370         # LOCAL
 371         echo "+ local attr"
 372         __populate_create_attr "${SCRATCH_MNT}/ATTR.FMT_LOCAL" 1
 373
 374         # BLOCK
 375         echo "+ block attr"
 376         __populate_create_attr "${SCRATCH_MNT}/ATTR.FMT_BLOCK" "$((blksz / 40))"
 377
 378         # trusted namespace
 379         touch ${SCRATCH_MNT}/ATTR.TRUSTED
 380         setfattr -n trusted.moo -v urk ${SCRATCH_MNT}/ATTR.TRUSTED
 381
 382         # security namespace
 383         touch ${SCRATCH_MNT}/ATTR.SECURITY
 384         setfattr -n security.foo -v bar ${SCRATCH_MNT}/ATTR.SECURITY
 385
 386         # system namespace
 387         touch ${SCRATCH_MNT}/ATTR.SYSTEM
 388         setfacl -m u:root:r ${SCRATCH_MNT}/ATTR.SYSTEM
 389
 390         # Make an unused inode
 391         echo "+ empty file"
 392         touch "${SCRATCH_MNT}/unused"
 393         $XFS_IO_PROG -f -c 'fsync' "${SCRATCH_MNT}/unused"
 394         rm -rf "${SCRATCH_MNT}/unused"
 395
 396         # Copy some real files (xfs tests, I guess...)
 397         echo "+ real files"
 398         test $fill -ne 0 && __populate_fill_fs "${SCRATCH_MNT}" 5
 399
 400         umount "${SCRATCH_MNT}"
 401 }
 402
 403 # Find the inode number of a file
 404 __populate_find_inode() {
 405         name="$1"
 406         inode="$(stat -c '%i' "${name}")"
 407         echo "${inode}"
 408 }
 409
 410 # Check data fork format of XFS file
 411 __populate_check_xfs_dformat() {
 412         inode="$1"
 413         format="$2"
 414
 415         fmt="$(_scratch_xfs_db -c "inode ${inode}" -c 'p core.format' | sed -e 's/^.*(\([a-z]*\)).*$/\1/g')"
 416         test "${format}" = "${fmt}" || _fail "failed to create ino ${inode} dformat expected ${format} saw ${fmt}"
 417 }
 418
 419 # Check attr fork format of XFS file
 420 __populate_check_xfs_aformat() {
 421         inode="$1"
 422         format="$2"
 423
 424         fmt="$(_scratch_xfs_db -c "inode ${inode}" -c 'p core.aformat' | sed -e 's/^.*(\([a-z]*\)).*$/\1/g')"
 425         test "${format}" = "${fmt}" || _fail "failed to create ino ${inode} aformat expected ${format} saw ${fmt}"
 426 }
 427
 428 # Check structure of XFS directory
 429 __populate_check_xfs_dir() {
 430         inode="$1"
 431         dtype="$2"
 432
 433         (test -n "${leaf_lblk}" && test -n "${node_lblk}") || _fail "must define leaf_lblk and node_lblk before calling __populate_check_xfs_dir"
 434         datab=0
 435         leafb=0
 436         freeb=0
 437         #echo "== check dir ${inode} type ${dtype}" ; _scratch_xfs_db -x -c "inode ${inode}" -c "bmap"
 438         _scratch_xfs_db -x -c "inode ${inode}" -c "dblock 0" -c "stack" | grep -q 'file data block is unmapped' || datab=1
 439         _scratch_xfs_db -x -c "inode ${inode}" -c "dblock ${leaf_lblk}" -c "stack" | grep -q 'file data block is unmapped' || leafb=1
 440         _scratch_xfs_db -x -c "inode ${inode}" -c "dblock ${node_lblk}" -c "stack" | grep -q 'file data block is unmapped' || freeb=1
 441
 442         case "${dtype}" in
 443         "shortform"|"inline"|"local")
 444                 (test "${datab}" -eq 0 && test "${leafb}" -eq 0 && test "${freeb}" -eq 0) || _fail "failed to create ${dtype} dir ino ${inode} datab ${datab} leafb ${leafb} freeb ${freeb}"
 445                 ;;
 446         "block")
 447                 (test "${datab}" -eq 1 && test "${leafb}" -eq 0 && test "${freeb}" -eq 0) || _fail "failed to create ${dtype} dir ino ${inode} datab ${datab} leafb ${leafb} freeb ${freeb}"
 448                 ;;
 449         "leaf")
 450                 (test "${datab}" -eq 1 && test "${leafb}" -eq 1 && test "${freeb}" -eq 0) || _fail "failed to create ${dtype} dir ino ${inode} datab ${datab} leafb ${leafb} freeb ${freeb}"
 451                 ;;
 452         "node"|"btree")
 453                 (test "${datab}" -eq 1 && test "${leafb}" -eq 1 && test "${freeb}" -eq 1) || _fail "failed to create ${dtype} dir ino ${inode} datab ${datab} leafb ${leafb} freeb ${freeb}"
 454                 ;;
 455         *)
 456                 _fail "Unknown directory type ${dtype}"
 457         esac
 458 }
 459
 460 # Check structure of XFS attr
 461 __populate_check_xfs_attr() {
 462         inode="$1"
 463         atype="$2"
 464
 465         datab=0
 466         leafb=0
 467         #echo "== check attr ${inode} type ${dtype}" ; _scratch_xfs_db -x -c "inode ${inode}" -c "bmap -a"
 468         _scratch_xfs_db -x -c "inode ${inode}" -c "ablock 0" -c "stack" | grep -q 'file attr block is unmapped' || datab=1
 469         _scratch_xfs_db -x -c "inode ${inode}" -c "ablock 1" -c "stack" | grep -q 'file attr block is unmapped' || leafb=1
 470
 471         case "${atype}" in
 472         "shortform"|"inline"|"local")
 473                 (test "${datab}" -eq 0 && test "${leafb}" -eq 0) || _fail "failed to create ${atype} attr ino ${inode} datab ${datab} leafb ${leafb}"
 474                 ;;
 475         "leaf")
 476                 (test "${datab}" -eq 1 && test "${leafb}" -eq 0) || _fail "failed to create ${atype} attr ino ${inode} datab ${datab} leafb ${leafb}"
 477                 ;;
 478         "node"|"btree")
 479                 (test "${datab}" -eq 1 && test "${leafb}" -eq 1) || _fail "failed to create ${atype} attr ino ${inode} datab ${datab} leafb ${leafb}"
 480                 ;;
 481         *)
 482                 _fail "Unknown attribute type ${atype}"
 483         esac
 484 }
 485
 486 # Check that there's at least one per-AG btree with multiple levels
 487 __populate_check_xfs_agbtree_height() {
 488         bt_type="$1"
 489         nr_ags=$(_scratch_xfs_db -c 'sb 0' -c 'p agcount' | awk '{print $3}')
 490
 491         case "${bt_type}" in
 492         "bno"|"cnt"|"rmap"|"refcnt")
 493                 hdr="agf"
 494                 bt_prefix="${bt_type}"
 495                 ;;
 496         "ino")
 497                 hdr="agi"
 498                 bt_prefix=""
 499                 ;;
 500         "fino")
 501                 hdr="agi"
 502                 bt_prefix="free_"
 503                 ;;
 504         *)
 505                 _fail "Don't know about AG btree ${bt_type}"
 506                 ;;
 507         esac
 508
 509         seq 0 $((nr_ags - 1)) | while read ag; do
 510                 bt_level=$(_scratch_xfs_db -c "${hdr} ${ag}" -c "p ${bt_prefix}level" | awk '{print $3}')
 511                 if [ "${bt_level}" -gt 1 ]; then
 512                         return 100
 513                 fi
 514         done
 515         test $? -eq 100 || _fail "Failed to create ${bt_type} of sufficient height!"
 516         return 1
 517 }
 518
 519 # Check that populate created all the types of files we wanted
 520 _scratch_xfs_populate_check() {
 521         _scratch_mount
 522         extents_file="$(__populate_find_inode "${SCRATCH_MNT}/S_IFREG.FMT_EXTENTS")"
 523         btree_file="$(__populate_find_inode "${SCRATCH_MNT}/S_IFREG.FMT_BTREE")"
 524         inline_dir="$(__populate_find_inode "${SCRATCH_MNT}/S_IFDIR.FMT_INLINE")"
 525         block_dir="$(__populate_find_inode "${SCRATCH_MNT}/S_IFDIR.FMT_BLOCK")"
 526         leaf_dir="$(__populate_find_inode "${SCRATCH_MNT}/S_IFDIR.FMT_LEAF")"
 527         node_dir="$(__populate_find_inode "${SCRATCH_MNT}/S_IFDIR.FMT_NODE")"
 528         btree_dir="$(__populate_find_inode "${SCRATCH_MNT}/S_IFDIR.FMT_BTREE")"
 529         local_slink="$(__populate_find_inode "${SCRATCH_MNT}/S_IFLNK.FMT_LOCAL")"
 530         extents_slink="$(__populate_find_inode "${SCRATCH_MNT}/S_IFLNK.FMT_EXTENTS")"
 531         bdev="$(__populate_find_inode "${SCRATCH_MNT}/S_IFBLK")"
 532         cdev="$(__populate_find_inode "${SCRATCH_MNT}/S_IFCHR")"
 533         local_attr="$(__populate_find_inode "${SCRATCH_MNT}/ATTR.FMT_LOCAL")"
 534         leaf_attr="$(__populate_find_inode "${SCRATCH_MNT}/ATTR.FMT_LEAF")"
 535         node_attr="$(__populate_find_inode "${SCRATCH_MNT}/ATTR.FMT_NODE")"
 536         btree_attr="$(__populate_find_inode "${SCRATCH_MNT}/ATTR.FMT_BTREE")"
 537         is_finobt=$(xfs_info "${SCRATCH_MNT}" | grep -c 'finobt=1')
 538         is_rmapbt=$(xfs_info "${SCRATCH_MNT}" | grep -c 'rmapbt=1')
 539         is_reflink=$(xfs_info "${SCRATCH_MNT}" | grep -c 'reflink=1')
 540
 541         blksz="$(stat -f -c '%s' "${SCRATCH_MNT}")"
 542         dblksz="$(xfs_info "${SCRATCH_MNT}" | grep naming.*bsize | sed -e 's/^.*bsize=//g' -e 's/\([0-9]*\).*$/\1/g')"
 543         leaf_lblk="$((32 * 1073741824 / blksz))"
 544         node_lblk="$((64 * 1073741824 / blksz))"
 545         umount "${SCRATCH_MNT}"
 546
 547         __populate_check_xfs_dformat "${extents_file}" "extents"
 548         __populate_check_xfs_dformat "${btree_file}" "btree"
 549         __populate_check_xfs_dir "${inline_dir}" "inline"
 550         __populate_check_xfs_dir "${block_dir}" "block"
 551         __populate_check_xfs_dir "${leaf_dir}" "leaf"
 552         __populate_check_xfs_dir "${node_dir}" "node"
 553         __populate_check_xfs_dir "${btree_dir}" "btree"
 554         __populate_check_xfs_dformat "${btree_dir}" "btree"
 555         __populate_check_xfs_dformat "${bdev}" "dev"
 556         __populate_check_xfs_dformat "${cdev}" "dev"
 557         __populate_check_xfs_attr "${local_attr}" "local"
 558         __populate_check_xfs_attr "${leaf_attr}" "leaf"
 559         __populate_check_xfs_attr "${node_attr}" "node"
 560         __populate_check_xfs_attr "${btree_attr}" "btree"
 561         __populate_check_xfs_aformat "${btree_attr}" "btree"
 562         __populate_check_xfs_agbtree_height "bno"
 563         __populate_check_xfs_agbtree_height "cnt"
 564         test $is_rmapbt -ne 0 && __populate_check_xfs_agbtree_height "rmap"
 565         test $is_reflink -ne 0 && __populate_check_xfs_agbtree_height "refcnt"
 566 }
 567
 568 # Check data fork format of ext4 file
 569 __populate_check_ext4_dformat() {
 570         dev="${SCRATCH_DEV}"
 571         inode="$1"
 572         format="$2"
 573
 574         extents=0
 575         etree=0
 576         debugfs -R "stat <${inode}>" "${dev}" 2> /dev/null | grep 'ETB[0-9]' -q && etree=1
 577         iflags="$(debugfs -R "stat <${inode}>" "${dev}" 2> /dev/null | grep 'Flags:' | sed -e 's/^.*Flags: \([0-9a-fx]*\).*$/\1/g')"
 578         test "$(echo "${iflags}" | awk '{print and(strtonum($1), 0x80000);}')" -gt 0 && extents=1
 579
 580         case "${format}" in
 581         "blockmap")
 582                 test "${extents}" -eq 0 || _fail "failed to create ino ${inode} with blockmap"
 583                 ;;
 584         "extent"|"extents")
 585                 test "${extents}" -eq 1 || _fail "failed to create ino ${inode} with extents"
 586                 ;;
 587         "etree")
 588                 (test "${extents}" -eq 1 && test "${etree}" -eq 1) || _fail "failed to create ino ${inode} with extent tree"
 589                 ;;
 590         *)
 591                 _fail "Unknown dformat ${format}"
 592         esac
 593 }
 594
 595 # Check attr fork format of ext4 file
 596 __populate_check_ext4_aformat() {
 597         dev="${SCRATCH_DEV}"
 598         inode="$1"
 599         format="$2"
 600
 601         ablock=1
 602         debugfs -R "stat <${inode}>" "${dev}" 2> /dev/null | grep 'File ACL: 0' -q && ablock=0
 603
 604         case "${format}" in
 605         "local"|"inline")
 606                 test "${ablock}" -eq 0 || _fail "failed to create inode ${inode} with ${format} xattr"
 607                 ;;
 608         "block")
 609                 test "${extents}" -eq 1 || _fail "failed to create inode ${inode} with ${format} xattr"
 610                 ;;
 611         *)
 612                 _fail "Unknown aformat ${format}"
 613         esac
 614 }
 615
 616 # Check structure of ext4 dir
 617 __populate_check_ext4_dir() {
 618         dev="${SCRATCH_DEV}"
 619         inode="$1"
 620         dtype="$2"
 621
 622         htree=0
 623         inline=0
 624         iflags="$(debugfs -R "stat <${inode}>" "${dev}" 2> /dev/null | grep 'Flags:' | sed -e 's/^.*Flags: \([0-9a-fx]*\).*$/\1/g')"
 625         test "$(echo "${iflags}" | awk '{print and(strtonum($1), 0x1000);}')" -gt 0 && htree=1
 626         test "$(echo "${iflags}" | awk '{print and(strtonum($1), 0x10000000);}')" -gt 0 && inline=1
 627
 628         case "${dtype}" in
 629         "inline")
 630                 (test "${inline}" -eq 1 && test "${htree}" -eq 0) || _fail "failed to create ${dtype} dir ino ${inode} htree ${htree} inline ${inline}"
 631                 ;;
 632         "block")
 633                 (test "${inline}" -eq 0 && test "${htree}" -eq 0) || _fail "failed to create ${dtype} dir ino ${inode} htree ${htree} inline ${inline}"
 634                 ;;
 635         "htree")
 636                 (test "${inline}" -eq 0 && test "${htree}" -eq 1) || _fail "failed to create ${dtype} dir ino ${inode} htree ${htree} inline ${inline}"
 637                 ;;
 638         *)
 639                 _fail "Unknown directory type ${dtype}"
 640                 ;;
 641         esac
 642 }
 643
 644 # Check that populate created all the types of files we wanted
 645 _scratch_ext4_populate_check() {
 646         _scratch_mount
 647         extents_file="$(__populate_find_inode "${SCRATCH_MNT}/S_IFREG.FMT_EXTENTS")"
 648         etree_file="$(__populate_find_inode "${SCRATCH_MNT}/S_IFREG.FMT_ETREE")"
 649         block_dir="$(__populate_find_inode "${SCRATCH_MNT}/S_IFDIR.FMT_BLOCK")"
 650         htree_dir="$(__populate_find_inode "${SCRATCH_MNT}/S_IFDIR.FMT_HTREE")"
 651         extents_slink="$(__populate_find_inode "${SCRATCH_MNT}/S_IFLNK.FMT_EXTENTS")"
 652         local_attr="$(__populate_find_inode "${SCRATCH_MNT}/ATTR.FMT_LOCAL")"
 653         block_attr="$(__populate_find_inode "${SCRATCH_MNT}/ATTR.FMT_BLOCK")"
 654         umount "${SCRATCH_MNT}"
 655
 656         __populate_check_ext4_dformat "${extents_file}" "extents"
 657         __populate_check_ext4_dformat "${etree_file}" "etree"
 658         __populate_check_ext4_dir "${block_dir}" "block"
 659         __populate_check_ext4_dir "${htree_dir}" "htree"
 660         __populate_check_ext4_dformat "${extents_slink}" "extents"
 661         __populate_check_ext4_aformat "${local_attr}" "local"
 662         __populate_check_ext4_aformat "${block_attr}" "block"
 663 }
 664
 665 # Populate a scratch FS and check the contents to make sure we got that
 666 _scratch_populate() {
 667         case "${FSTYP}" in
 668         "xfs")
 669                 _scratch_xfs_populate
 670                 _scratch_xfs_populate_check
 671                 ;;
 672         "ext2"|"ext3"|"ext4")
 673                 _scratch_ext4_populate
 674                 _scratch_ext4_populate_check
 675                 ;;
 676         *)
 677                 _fail "Don't know how to populate a ${FSTYP} filesystem."
 678                 ;;
 679         esac
 680 }
 681
 682 # Fill a file system by repeatedly creating files in the given folder
 683 # starting with the given file size.  Files are reduced in size when
 684 # they can no longer fit until no more files can be created.
 685 _fill_fs()
 686 {
 687         local file_size=$1
 688         local dir=$2
 689         local block_size=$3
 690         local switch_user=$4
 691         local file_count=1
 692         local bytes_written=0
 693         local use_falloc=1;
 694
 695         if [ $# -ne 4 ]; then
 696                 echo "Usage: _fill_fs filesize dir blocksize switch_user"
 697                 exit 1
 698         fi
 699
 700         if [ $switch_user -eq 0 ]; then
 701                 mkdir -p $dir
 702         else
 703                 _user_do "mkdir -p $dir"
 704         fi
 705         if [ ! -d $dir ]; then
 706                 return 0;
 707         fi
 708
 709         testio=`$XFS_IO_PROG -F -fc "falloc 0 $block_size" $dir/$$.xfs_io 2>&1`
 710         echo $testio | grep -q "not found" && use_falloc=0
 711         echo $testio | grep -q "Operation not supported" && use_falloc=0
 712
 713         if [ $file_size -lt $block_size ]; then
 714                 $file_size = $block_size
 715         fi
 716
 717         while [ $file_size -ge $block_size ]; do
 718                 bytes_written=0
 719                 if [ $switch_user -eq 0 ]; then
 720                         if [ $use_falloc -eq 0 ]; then
 721                                 $XFS_IO_PROG -fc "pwrite -b 8388608 0 $file_size" \
 722                                         $dir/$file_count
 723                         else
 724                                 $XFS_IO_PROG -fc "falloc 0 $file_size" \
 725                                         $dir/$file_count
 726                         fi
 727                 else
 728                         if [ $use_falloc -eq 0 ]; then
 729                                 _user_do "$XFS_IO_PROG -f -c \"pwrite -b 8388608 0 \
 730                                         $file_size\" $dir/$file_count"
 731                         else
 732                                 _user_do "$XFS_IO_PROG -f -c \"falloc 0 \
 733                                         $file_size\" $dir/$file_count"
 734                         fi
 735                 fi
 736
 737                 if [ -f $dir/$file_count ]; then
 738                         bytes_written=$(stat -c '%s' $dir/$file_count)
 739                 fi
 740
 741                 # If there was no room to make the file, then divide it in
 742                 # half, and keep going
 743                 if [ $bytes_written -lt $file_size ]; then
 744                         file_size=$((file_size / 2))
 745                 fi
 746                 file_count=$((file_count + 1))
 747         done
 748 }
 749
 750 # Populate a scratch FS from scratch or from a cached image.
 751 _scratch_populate_cached() {
 752         POPULATE_METADUMP="${TEST_DIR}/__populate.${FSTYP}"
 753         POPULATE_METADUMP_DESCR="${TEST_DIR}/__populate.${FSTYP}.txt"
 754
 755         # Don't keep metadata images cached for more 48 hours...
 756         rm -rf "$(find "${POPULATE_METADUMP}" -mtime +2 2>/dev/null)"
 757
 758         # Throw away cached image if it doesn't match our spec.
 759         case "${FSTYP}" in
 760         "ext4")
 761                 extra_descr="LOGDEV ${SCRATCH_LOGDEV} USE_EXTERNAL ${USE_EXTERNAL}"
 762                 # ext4 cannot e2image external logs, so we cannot restore
 763                 test -n "${SCRATCH_LOGDEV}" && rm -f "${POPULATE_METADUMP}"
 764                 ;;
 765         "xfs")
 766                 extra_descr="LOGDEV ${SCRATCH_LOGDEV} USE_EXTERNAL ${USE_EXTERNAL} RTDEV ${SCRATCH_RTDEV}"
 767                 _populate_xfs_qmount_option
 768                 if echo "${MOUNT_OPTIONS}" | grep -q 'usrquota'; then
 769                         extra_descr="${extra_descr} QUOTAS"
 770                 fi
 771                 ;;
 772         *)
 773                 extra_descr="";;
 774         esac
 775         meta_descr="FSTYP ${FSTYP} MKFS_OPTIONS ${MKFS_OPTIONS} SIZE $(blockdev --getsz "${SCRATCH_DEV}") ${extra_descr} ARGS $@"
 776         cmp -s "${POPULATE_METADUMP_DESCR}" <(echo "${meta_descr}") || rm -rf "${POPULATE_METADUMP}"
 777
 778         # Do we have a cached image?
 779         if [ -r "${POPULATE_METADUMP}" ]; then
 780                 case "${FSTYP}" in
 781                 "xfs")
 782                         xfs_mdrestore "${POPULATE_METADUMP}" "${SCRATCH_DEV}" && return
 783                         ;;
 784                 "ext2"|"ext3"|"ext4")
 785                         e2image -r "${POPULATE_METADUMP}" "${SCRATCH_DEV}" && return
 786                         ;;
 787                 esac
 788         fi
 789
 790         # Oh well, just create one from scratch
 791         _scratch_mkfs
 792         echo "${meta_descr}" > "${POPULATE_METADUMP_DESCR}"
 793         case "${FSTYP}" in
 794         "xfs")
 795                 _scratch_xfs_populate $@
 796                 _scratch_xfs_populate_check
 797                 _scratch_metadump "${POPULATE_METADUMP}" -a -o
 798                 ;;
 799         "ext2"|"ext3"|"ext4")
 800                 _scratch_ext4_populate $@
 801                 _scratch_ext4_populate_check
 802                 e2image -Q "${SCRATCH_DEV}" "${POPULATE_METADUMP}"
 803                 ;;
 804         *)
 805                 _fail "Don't know how to populate a ${FSTYP} filesystem."
 806                 ;;
 807         esac
 808 }