common/populate

   1 ##/bin/bash
   2 # SPDX-License-Identifier: GPL-2.0+
   3 # Copyright (c) 2015 Oracle.  All Rights Reserved.
   4 #
   5 # Routines for populating a scratch fs, and helpers to exercise an FS
   6 # once it's been fuzzed.
   7
   8 . ./common/quota
   9
  10 _require_populate_commands() {
  11         _require_xfs_io_command "falloc"
  12         _require_xfs_io_command "fpunch"
  13         _require_test_program "punch-alternating"
  14         _require_command "$XFS_DB_PROG" "xfs_db"
  15 }
  16
  17 _require_xfs_db_blocktrash_z_command() {
  18         test "${FSTYP}" = "xfs" || _notrun "cannot run xfs_db on ${FSTYP}"
  19         $XFS_DB_PROG -x -f -c 'blocktrash -z' "${TEST_DEV}" | grep -q 'nothing on stack' || _notrun "blocktrash -z not supported"
  20 }
  21
  22 # Attempt to make files of "every" format for data, dirs, attrs etc.
  23 # (with apologies to Eric Sandeen for mutating xfser.sh)
  24
  25 # Create a file of a given size.
  26 __populate_create_file() {
  27         local sz="$1"
  28         local fname="$2"
  29
  30         $XFS_IO_PROG -f -c "pwrite -S 0x62 -W -b 1m 0 $sz" "${fname}"
  31 }
  32
  33 # Punch out every other hole in this file, if it exists.
  34 #
  35 # The goal here is to force the creation of a large number of metadata records
  36 # by creating a lot of tiny extent mappings in a file.  Callers should ensure
  37 # that fragmenting the file actually causes record creation.  Call this
  38 # function /after/ creating all other metadata structures.
  39 __populate_fragment_file() {
  40         local fname="$1"
  41
  42         test -f "${fname}" && $here/src/punch-alternating "${fname}"
  43 }
  44
  45 # Create a large directory
  46 __populate_create_dir() {
  47         name="$1"
  48         nr="$2"
  49         missing="$3"
  50
  51         mkdir -p "${name}"
  52         seq 0 "${nr}" | while read d; do
  53                 creat=mkdir
  54                 test "$((d % 20))" -eq 0 && creat=touch
  55                 $creat "${name}/$(printf "%.08d" "$d")"
  56         done
  57
  58         test -z "${missing}" && return
  59         seq 1 2 "${nr}" | while read d; do
  60                 rm -rf "${name}/$(printf "%.08d" "$d")"
  61         done
  62 }
  63
  64 # Add a bunch of attrs to a file
  65 __populate_create_attr() {
  66         name="$1"
  67         nr="$2"
  68         missing="$3"
  69
  70         touch "${name}"
  71         seq 0 "${nr}" | while read d; do
  72                 setfattr -n "user.$(printf "%.08d" "$d")" -v "$(printf "%.08d" "$d")" "${name}"
  73         done
  74
  75         test -z "${missing}" && return
  76         seq 1 2 "${nr}" | while read d; do
  77                 setfattr -x "user.$(printf "%.08d" "$d")" "${name}"
  78         done
  79 }
  80
  81 # Fill up some percentage of the remaining free space
  82 __populate_fill_fs() {
  83         dir="$1"
  84         pct="$2"
  85         test -z "${pct}" && pct=60
  86
  87         mkdir -p "${dir}/test/1"
  88         cp -pRdu "${dir}"/S_IFREG* "${dir}/test/1/"
  89
  90         SRC_SZ="$(du -ks "${dir}/test/1" | cut -f 1)"
  91         FS_SZ="$(( $(stat -f "${dir}" -c '%a * %S') / 1024 ))"
  92
  93         NR="$(( (FS_SZ * ${pct} / 100) / SRC_SZ ))"
  94
  95         echo "FILL FS"
  96         echo "src_sz $SRC_SZ fs_sz $FS_SZ nr $NR"
  97         seq 2 "${NR}" | while read nr; do
  98                 cp -pRdu "${dir}/test/1" "${dir}/test/${nr}"
  99         done
 100 }
 101
 102 # For XFS, force on all the quota options if quota is enabled
 103 # and the user didn't feed us noquota.
 104 _populate_xfs_qmount_option()
 105 {
 106         # User explicitly told us not to quota
 107         if echo "${MOUNT_OPTIONS}" | grep -q 'noquota'; then
 108                 return
 109         fi
 110
 111         # Don't bother if we can't turn on quotas
 112         if [ ! -f /proc/fs/xfs/xqmstat ]; then
 113                 # No quota support
 114                 return
 115         elif [ "${USE_EXTERNAL}" = "yes" ] && [ ! -z "${SCRATCH_RTDEV}" ]; then
 116                 # Quotas not supported on rt filesystems
 117                 return
 118         elif [ -z "${XFS_QUOTA_PROG}" ]; then
 119                 # xfs quota tools not installed
 120                 return
 121         fi
 122
 123         # Turn on all the quotas
 124         if $XFS_INFO_PROG "${TEST_DIR}" | grep -q 'crc=1'; then
 125                 # v5 filesystems can have group & project quotas
 126                 quota="usrquota,grpquota,prjquota"
 127         else
 128                 # v4 filesystems cannot mix group & project quotas
 129                 quota="usrquota,grpquota"
 130         fi
 131
 132         # Inject our quota mount options
 133         if echo "${MOUNT_OPTIONS}" | grep -q "${quota}"; then
 134                 return
 135         elif echo "${MOUNT_OPTIONS}" | egrep -q '(quota|noenforce)'; then
 136                 _qmount_option "${quota}"
 137         else
 138                 export MOUNT_OPTIONS="$MOUNT_OPTIONS -o ${quota}"
 139                 echo "MOUNT_OPTIONS = $MOUNT_OPTIONS" >>$seqres.full
 140         fi
 141 }
 142
 143 # Populate an XFS on the scratch device with (we hope) all known
 144 # types of metadata block
 145 _scratch_xfs_populate() {
 146         fill=1
 147
 148         for arg in $@; do
 149                 case "${arg}" in
 150                 "nofill")
 151                         fill=0;;
 152                 esac
 153         done
 154
 155         _populate_xfs_qmount_option
 156         _scratch_mount
 157
 158         # We cannot directly force the filesystem to create the metadata
 159         # structures we want; we can only achieve this indirectly by carefully
 160         # crafting files and a directory tree.  Therefore, we must have exact
 161         # control over the layout and device selection of all files created.
 162         # Clear the rtinherit flag on the root directory so that files are
 163         # always created on the data volume regardless of MKFS_OPTIONS.  We can
 164         # set the realtime flag when needed.
 165         $XFS_IO_PROG -c 'chattr -t' $SCRATCH_MNT
 166
 167         blksz="$(stat -f -c '%s' "${SCRATCH_MNT}")"
 168         dblksz="$($XFS_INFO_PROG "${SCRATCH_MNT}" | grep naming.*bsize | sed -e 's/^.*bsize=//g' -e 's/\([0-9]*\).*$/\1/g')"
 169         crc="$($XFS_INFO_PROG "${SCRATCH_MNT}" | grep crc= | sed -e 's/^.*crc=//g' -e 's/\([0-9]*\).*$/\1/g')"
 170         if [ $crc -eq 1 ]; then
 171                 leaf_hdr_size=64
 172         else
 173                 leaf_hdr_size=16
 174         fi
 175         leaf_lblk="$((32 * 1073741824 / blksz))"
 176         node_lblk="$((64 * 1073741824 / blksz))"
 177
 178         # Data:
 179
 180         # Fill up the root inode chunk
 181         echo "+ fill root ino chunk"
 182         seq 1 64 | while read f; do
 183                 $XFS_IO_PROG -f -c "truncate 0" "${SCRATCH_MNT}/dummy${f}"
 184         done
 185
 186         # Regular files
 187         # - FMT_EXTENTS
 188         echo "+ extents file"
 189         __populate_create_file $blksz "${SCRATCH_MNT}/S_IFREG.FMT_EXTENTS"
 190
 191         # - FMT_BTREE
 192         echo "+ btree extents file"
 193         nr="$((blksz * 2 / 16))"
 194         __populate_create_file $((blksz * nr)) "${SCRATCH_MNT}/S_IFREG.FMT_BTREE"
 195
 196         # Directories
 197         # - INLINE
 198         echo "+ inline dir"
 199         __populate_create_dir "${SCRATCH_MNT}/S_IFDIR.FMT_INLINE" 1
 200
 201         # - BLOCK
 202         echo "+ block dir"
 203         __populate_create_dir "${SCRATCH_MNT}/S_IFDIR.FMT_BLOCK" "$((dblksz / 40))"
 204
 205         # - LEAF
 206         echo "+ leaf dir"
 207         __populate_create_dir "${SCRATCH_MNT}/S_IFDIR.FMT_LEAF" "$((dblksz / 12))"
 208
 209         # - LEAFN
 210         echo "+ leafn dir"
 211         __populate_create_dir "${SCRATCH_MNT}/S_IFDIR.FMT_LEAFN" "$(( ((dblksz - leaf_hdr_size) / 8) - 3 ))"
 212
 213         # - NODE
 214         echo "+ node dir"
 215         __populate_create_dir "${SCRATCH_MNT}/S_IFDIR.FMT_NODE" "$((16 * dblksz / 40))" true
 216
 217         # - BTREE
 218         echo "+ btree dir"
 219         __populate_create_dir "${SCRATCH_MNT}/S_IFDIR.FMT_BTREE" "$((128 * dblksz / 40))" true
 220
 221         # Symlinks
 222         # - FMT_LOCAL
 223         echo "+ inline symlink"
 224         ln -s target "${SCRATCH_MNT}/S_IFLNK.FMT_LOCAL"
 225
 226         # - FMT_EXTENTS
 227         echo "+ extents symlink"
 228         ln -s "$(perl -e 'print "x" x 1023;')" "${SCRATCH_MNT}/S_IFLNK.FMT_EXTENTS"
 229
 230         # Char & block
 231         echo "+ special"
 232         mknod "${SCRATCH_MNT}/S_IFCHR" c 1 1
 233         mknod "${SCRATCH_MNT}/S_IFBLK" c 1 1
 234
 235         # special file with an xattr
 236         setfacl -P -m u:nobody:r ${SCRATCH_MNT}/S_IFCHR
 237
 238         # Attribute formats
 239         # LOCAL
 240         echo "+ local attr"
 241         __populate_create_attr "${SCRATCH_MNT}/ATTR.FMT_LOCAL" 1
 242
 243         # LEAF
 244         echo "+ leaf attr"
 245         __populate_create_attr "${SCRATCH_MNT}/ATTR.FMT_LEAF" "$((blksz / 40))"
 246
 247         # NODE
 248         echo "+ node attr"
 249         __populate_create_attr "${SCRATCH_MNT}/ATTR.FMT_NODE" "$((8 * blksz / 40))"
 250
 251         # BTREE
 252         echo "+ btree attr"
 253         __populate_create_attr "${SCRATCH_MNT}/ATTR.FMT_BTREE" "$((64 * blksz / 40))" true
 254
 255         # trusted namespace
 256         touch ${SCRATCH_MNT}/ATTR.TRUSTED
 257         setfattr -n trusted.moo -v urk ${SCRATCH_MNT}/ATTR.TRUSTED
 258
 259         # security namespace
 260         touch ${SCRATCH_MNT}/ATTR.SECURITY
 261         setfattr -n security.foo -v bar ${SCRATCH_MNT}/ATTR.SECURITY
 262
 263         # system namespace
 264         touch ${SCRATCH_MNT}/ATTR.SYSTEM
 265         setfacl -m u:root:r ${SCRATCH_MNT}/ATTR.SYSTEM
 266
 267         # FMT_EXTENTS with a remote less-than-a-block value
 268         echo "+ attr extents with a remote less-than-a-block value"
 269         touch "${SCRATCH_MNT}/ATTR.FMT_EXTENTS_REMOTE3K"
 270         $XFS_IO_PROG -f -c "pwrite -S 0x43 0 $((blksz - 300))" "${SCRATCH_MNT}/attrvalfile" > /dev/null
 271         attr -q -s user.remotebtreeattrname "${SCRATCH_MNT}/ATTR.FMT_EXTENTS_REMOTE3K" < "${SCRATCH_MNT}/attrvalfile"
 272
 273         # FMT_EXTENTS with a remote block-size value
 274         echo "+ attr extents with a remote one-block value"
 275         touch "${SCRATCH_MNT}/ATTR.FMT_EXTENTS_REMOTE4K"
 276         $XFS_IO_PROG -f -c "pwrite -S 0x44 0 ${blksz}" "${SCRATCH_MNT}/attrvalfile" > /dev/null
 277         attr -q -s user.remotebtreeattrname "${SCRATCH_MNT}/ATTR.FMT_EXTENTS_REMOTE4K" < "${SCRATCH_MNT}/attrvalfile"
 278         rm -rf "${SCRATCH_MNT}/attrvalfile"
 279
 280         # Make an unused inode
 281         echo "+ empty file"
 282         touch "${SCRATCH_MNT}/unused"
 283         $XFS_IO_PROG -f -c 'fsync' "${SCRATCH_MNT}/unused"
 284         rm -rf "${SCRATCH_MNT}/unused"
 285
 286         # Free space btree
 287         echo "+ freesp btree"
 288         nr="$((blksz * 2 / 8))"
 289         __populate_create_file $((blksz * nr)) "${SCRATCH_MNT}/BNOBT"
 290
 291         # Inode btree
 292         echo "+ inobt btree"
 293         local ino_per_rec=64
 294         local rec_per_btblock=16
 295         local nr="$(( 2 * (blksz / rec_per_btblock) * ino_per_rec ))"
 296         local dir="${SCRATCH_MNT}/INOBT"
 297         mkdir -p "${dir}"
 298         seq 0 "${nr}" | while read f; do
 299                 touch "${dir}/${f}"
 300         done
 301
 302         seq 0 2 "${nr}" | while read f; do
 303                 rm -f "${dir}/${f}"
 304         done
 305
 306         # Reverse-mapping btree
 307         is_rmapbt="$($XFS_INFO_PROG "${SCRATCH_MNT}" | grep -c 'rmapbt=1')"
 308         if [ $is_rmapbt -gt 0 ]; then
 309                 echo "+ rmapbt btree"
 310                 nr="$((blksz * 2 / 24))"
 311                 __populate_create_file $((blksz * nr)) "${SCRATCH_MNT}/RMAPBT"
 312         fi
 313
 314         # Realtime Reverse-mapping btree
 315         is_rt="$($XFS_INFO_PROG "${SCRATCH_MNT}" | grep -c 'rtextents=[1-9]')"
 316         if [ $is_rmapbt -gt 0 ] && [ $is_rt -gt 0 ]; then
 317                 echo "+ rtrmapbt btree"
 318                 nr="$((blksz * 2 / 32))"
 319                 $XFS_IO_PROG -R -f -c 'truncate 0' "${SCRATCH_MNT}/RTRMAPBT"
 320                 __populate_create_file $((blksz * nr)) "${SCRATCH_MNT}/RTRMAPBT"
 321         fi
 322
 323         # Reference-count btree
 324         is_reflink="$($XFS_INFO_PROG "${SCRATCH_MNT}" | grep -c 'reflink=1')"
 325         if [ $is_reflink -gt 0 ]; then
 326                 echo "+ reflink btree"
 327                 nr="$((blksz * 2 / 12))"
 328                 __populate_create_file $((blksz * nr)) "${SCRATCH_MNT}/REFCOUNTBT"
 329                 cp --reflink=always "${SCRATCH_MNT}/REFCOUNTBT" "${SCRATCH_MNT}/REFCOUNTBT2"
 330         fi
 331
 332         # Copy some real files (xfs tests, I guess...)
 333         echo "+ real files"
 334         test $fill -ne 0 && __populate_fill_fs "${SCRATCH_MNT}" 5
 335
 336         # Make sure we get all the fragmentation we asked for
 337         __populate_fragment_file "${SCRATCH_MNT}/S_IFREG.FMT_BTREE"
 338         __populate_fragment_file "${SCRATCH_MNT}/BNOBT"
 339         __populate_fragment_file "${SCRATCH_MNT}/RMAPBT"
 340         __populate_fragment_file "${SCRATCH_MNT}/RTRMAPBT"
 341         __populate_fragment_file "${SCRATCH_MNT}/REFCOUNTBT"
 342
 343         umount "${SCRATCH_MNT}"
 344 }
 345
 346 # Populate an ext4 on the scratch device with (we hope) all known
 347 # types of metadata block
 348 _scratch_ext4_populate() {
 349         fill=1
 350
 351         for arg in $@; do
 352                 case "${arg}" in
 353                 "nofill")
 354                         fill=0;;
 355                 esac
 356         done
 357
 358         _scratch_mount
 359         blksz="$(stat -f -c '%s' "${SCRATCH_MNT}")"
 360         dblksz="${blksz}"
 361         leaf_lblk="$((32 * 1073741824 / blksz))"
 362         node_lblk="$((64 * 1073741824 / blksz))"
 363
 364         # Data:
 365
 366         # Regular files
 367         # - FMT_INLINE
 368         echo "+ inline file"
 369         __populate_create_file 1 "${SCRATCH_MNT}/S_IFREG.FMT_INLINE"
 370
 371         # - FMT_EXTENTS
 372         echo "+ extents file"
 373         __populate_create_file $blksz "${SCRATCH_MNT}/S_IFREG.FMT_EXTENTS"
 374
 375         # - FMT_ETREE
 376         echo "+ extent tree file"
 377         nr="$((blksz * 2 / 12))"
 378         __populate_create_file $((blksz * nr)) "${SCRATCH_MNT}/S_IFREG.FMT_ETREE"
 379
 380         # Directories
 381         # - INLINE
 382         echo "+ inline dir"
 383         __populate_create_dir "${SCRATCH_MNT}/S_IFDIR.FMT_INLINE" 1
 384
 385         # - BLOCK
 386         echo "+ block dir"
 387         __populate_create_dir "${SCRATCH_MNT}/S_IFDIR.FMT_BLOCK" "$((dblksz / 32))"
 388
 389         # - HTREE
 390         echo "+ htree dir"
 391         __populate_create_dir "${SCRATCH_MNT}/S_IFDIR.FMT_HTREE" "$((4 * dblksz / 24))"
 392
 393         # Symlinks
 394         # - FMT_LOCAL
 395         echo "+ inline symlink"
 396         ln -s target "${SCRATCH_MNT}/S_IFLNK.FMT_LOCAL"
 397
 398         # - FMT_EXTENTS
 399         echo "+ extents symlink"
 400         ln -s "$(perl -e 'print "x" x 1023;')" "${SCRATCH_MNT}/S_IFLNK.FMT_EXTENTS"
 401
 402         # Char & block
 403         echo "+ special"
 404         mknod "${SCRATCH_MNT}/S_IFCHR" c 1 1
 405         mknod "${SCRATCH_MNT}/S_IFBLK" c 1 1
 406
 407         # special file with an xattr
 408         setfacl -P -m u:nobody:r ${SCRATCH_MNT}/S_IFCHR
 409
 410         # Attribute formats
 411         # LOCAL
 412         echo "+ local attr"
 413         __populate_create_attr "${SCRATCH_MNT}/ATTR.FMT_LOCAL" 0
 414
 415         # BLOCK
 416         echo "+ block attr"
 417         __populate_create_attr "${SCRATCH_MNT}/ATTR.FMT_BLOCK" "$((blksz / 40))"
 418
 419         # trusted namespace
 420         touch ${SCRATCH_MNT}/ATTR.TRUSTED
 421         setfattr -n trusted.moo -v urk ${SCRATCH_MNT}/ATTR.TRUSTED
 422
 423         # security namespace
 424         touch ${SCRATCH_MNT}/ATTR.SECURITY
 425         setfattr -n security.foo -v bar ${SCRATCH_MNT}/ATTR.SECURITY
 426
 427         # system namespace
 428         touch ${SCRATCH_MNT}/ATTR.SYSTEM
 429         setfacl -m u:root:r ${SCRATCH_MNT}/ATTR.SYSTEM
 430
 431         # Make an unused inode
 432         echo "+ empty file"
 433         touch "${SCRATCH_MNT}/unused"
 434         $XFS_IO_PROG -f -c 'fsync' "${SCRATCH_MNT}/unused"
 435         rm -rf "${SCRATCH_MNT}/unused"
 436
 437         # Copy some real files (xfs tests, I guess...)
 438         echo "+ real files"
 439         test $fill -ne 0 && __populate_fill_fs "${SCRATCH_MNT}" 5
 440
 441         # Make sure we get all the fragmentation we asked for
 442         __populate_fragment_file "${SCRATCH_MNT}/S_IFREG.FMT_ETREE"
 443
 444         umount "${SCRATCH_MNT}"
 445 }
 446
 447 # Find the inode number of a file
 448 __populate_find_inode() {
 449         name="$1"
 450         inode="$(stat -c '%i' "${name}")"
 451         echo "${inode}"
 452 }
 453
 454 # Check data fork format of XFS file
 455 __populate_check_xfs_dformat() {
 456         inode="$1"
 457         format="$2"
 458
 459         fmt="$(_scratch_xfs_db -c "inode ${inode}" -c 'p core.format' | sed -e 's/^.*(\([a-z]*\)).*$/\1/g')"
 460         test "${format}" = "${fmt}" || _fail "failed to create ino ${inode} dformat expected ${format} saw ${fmt}"
 461 }
 462
 463 # Check attr fork format of XFS file
 464 __populate_check_xfs_aformat() {
 465         inode="$1"
 466         format="$2"
 467
 468         fmt="$(_scratch_xfs_db -c "inode ${inode}" -c 'p core.aformat' | sed -e 's/^.*(\([a-z]*\)).*$/\1/g')"
 469         test "${format}" = "${fmt}" || _fail "failed to create ino ${inode} aformat expected ${format} saw ${fmt}"
 470 }
 471
 472 # Check structure of XFS directory
 473 __populate_check_xfs_dir() {
 474         inode="$1"
 475         dtype="$2"
 476
 477         (test -n "${leaf_lblk}" && test -n "${node_lblk}") || _fail "must define leaf_lblk and node_lblk before calling __populate_check_xfs_dir"
 478         datab=0
 479         leafb=0
 480         freeb=0
 481         #echo "== check dir ${inode} type ${dtype}" ; _scratch_xfs_db -x -c "inode ${inode}" -c "bmap"
 482         _scratch_xfs_db -x -c "inode ${inode}" -c "dblock 0" -c "stack" | grep -q 'file data block is unmapped' || datab=1
 483         _scratch_xfs_db -x -c "inode ${inode}" -c "dblock ${leaf_lblk}" -c "stack" | grep -q 'file data block is unmapped' || leafb=1
 484         _scratch_xfs_db -x -c "inode ${inode}" -c "dblock ${node_lblk}" -c "stack" | grep -q 'file data block is unmapped' || freeb=1
 485
 486         case "${dtype}" in
 487         "shortform"|"inline"|"local")
 488                 (test "${datab}" -eq 0 && test "${leafb}" -eq 0 && test "${freeb}" -eq 0) || _fail "failed to create ${dtype} dir ino ${inode} datab ${datab} leafb ${leafb} freeb ${freeb}"
 489                 ;;
 490         "block")
 491                 (test "${datab}" -eq 1 && test "${leafb}" -eq 0 && test "${freeb}" -eq 0) || _fail "failed to create ${dtype} dir ino ${inode} datab ${datab} leafb ${leafb} freeb ${freeb}"
 492                 ;;
 493         "leaf")
 494                 (test "${datab}" -eq 1 && test "${leafb}" -eq 1 && test "${freeb}" -eq 0) || _fail "failed to create ${dtype} dir ino ${inode} datab ${datab} leafb ${leafb} freeb ${freeb}"
 495                 ;;
 496         "leafn")
 497                 _scratch_xfs_db -x -c "inode ${inode}" -c "dblock ${leaf_lblk}" -c "p lhdr.info.hdr.magic" | grep -q '0x3dff' && return
 498                 _scratch_xfs_db -x -c "inode ${inode}" -c "dblock ${leaf_lblk}" -c "p lhdr.info.magic" | grep -q '0xd2ff' && return
 499                 _fail "failed to create ${dtype} dir ino ${inode} datab ${datab} leafb ${leafb} freeb ${freeb}"
 500                 ;;
 501         "node"|"btree")
 502                 (test "${datab}" -eq 1 && test "${leafb}" -eq 1 && test "${freeb}" -eq 1) || _fail "failed to create ${dtype} dir ino ${inode} datab ${datab} leafb ${leafb} freeb ${freeb}"
 503                 ;;
 504         *)
 505                 _fail "Unknown directory type ${dtype}"
 506         esac
 507 }
 508
 509 # Check structure of XFS attr
 510 __populate_check_xfs_attr() {
 511         inode="$1"
 512         atype="$2"
 513
 514         datab=0
 515         leafb=0
 516         #echo "== check attr ${inode} type ${dtype}" ; _scratch_xfs_db -x -c "inode ${inode}" -c "bmap -a"
 517         _scratch_xfs_db -x -c "inode ${inode}" -c "ablock 0" -c "stack" | grep -q 'file attr block is unmapped' || datab=1
 518         _scratch_xfs_db -x -c "inode ${inode}" -c "ablock 1" -c "stack" | grep -q 'file attr block is unmapped' || leafb=1
 519
 520         case "${atype}" in
 521         "shortform"|"inline"|"local")
 522                 (test "${datab}" -eq 0 && test "${leafb}" -eq 0) || _fail "failed to create ${atype} attr ino ${inode} datab ${datab} leafb ${leafb}"
 523                 ;;
 524         "leaf")
 525                 (test "${datab}" -eq 1 && test "${leafb}" -eq 0) || _fail "failed to create ${atype} attr ino ${inode} datab ${datab} leafb ${leafb}"
 526                 ;;
 527         "node"|"btree")
 528                 (test "${datab}" -eq 1 && test "${leafb}" -eq 1) || _fail "failed to create ${atype} attr ino ${inode} datab ${datab} leafb ${leafb}"
 529                 ;;
 530         *)
 531                 _fail "Unknown attribute type ${atype}"
 532         esac
 533 }
 534
 535 # Check that there's at least one per-AG btree with multiple levels
 536 __populate_check_xfs_agbtree_height() {
 537         bt_type="$1"
 538         nr_ags=$(_scratch_xfs_db -c 'sb 0' -c 'p agcount' | awk '{print $3}')
 539
 540         case "${bt_type}" in
 541         "bno"|"cnt"|"rmap"|"refcnt")
 542                 hdr="agf"
 543                 bt_prefix="${bt_type}"
 544                 ;;
 545         "ino")
 546                 hdr="agi"
 547                 bt_prefix=""
 548                 ;;
 549         "fino")
 550                 hdr="agi"
 551                 bt_prefix="free_"
 552                 ;;
 553         *)
 554                 _fail "Don't know about AG btree ${bt_type}"
 555                 ;;
 556         esac
 557
 558         seq 0 $((nr_ags - 1)) | while read ag; do
 559                 bt_level=$(_scratch_xfs_db -c "${hdr} ${ag}" -c "p ${bt_prefix}level" | awk '{print $3}')
 560                 if [ "${bt_level}" -gt 1 ]; then
 561                         return 100
 562                 fi
 563         done
 564         test $? -eq 100 || _fail "Failed to create ${bt_type} of sufficient height!"
 565         return 1
 566 }
 567
 568 # Check that populate created all the types of files we wanted
 569 _scratch_xfs_populate_check() {
 570         _scratch_mount
 571         extents_file="$(__populate_find_inode "${SCRATCH_MNT}/S_IFREG.FMT_EXTENTS")"
 572         btree_file="$(__populate_find_inode "${SCRATCH_MNT}/S_IFREG.FMT_BTREE")"
 573         inline_dir="$(__populate_find_inode "${SCRATCH_MNT}/S_IFDIR.FMT_INLINE")"
 574         block_dir="$(__populate_find_inode "${SCRATCH_MNT}/S_IFDIR.FMT_BLOCK")"
 575         leaf_dir="$(__populate_find_inode "${SCRATCH_MNT}/S_IFDIR.FMT_LEAF")"
 576         leafn_dir="$(__populate_find_inode "${SCRATCH_MNT}/S_IFDIR.FMT_LEAFN")"
 577         node_dir="$(__populate_find_inode "${SCRATCH_MNT}/S_IFDIR.FMT_NODE")"
 578         btree_dir="$(__populate_find_inode "${SCRATCH_MNT}/S_IFDIR.FMT_BTREE")"
 579         local_slink="$(__populate_find_inode "${SCRATCH_MNT}/S_IFLNK.FMT_LOCAL")"
 580         extents_slink="$(__populate_find_inode "${SCRATCH_MNT}/S_IFLNK.FMT_EXTENTS")"
 581         bdev="$(__populate_find_inode "${SCRATCH_MNT}/S_IFBLK")"
 582         cdev="$(__populate_find_inode "${SCRATCH_MNT}/S_IFCHR")"
 583         local_attr="$(__populate_find_inode "${SCRATCH_MNT}/ATTR.FMT_LOCAL")"
 584         leaf_attr="$(__populate_find_inode "${SCRATCH_MNT}/ATTR.FMT_LEAF")"
 585         node_attr="$(__populate_find_inode "${SCRATCH_MNT}/ATTR.FMT_NODE")"
 586         btree_attr="$(__populate_find_inode "${SCRATCH_MNT}/ATTR.FMT_BTREE")"
 587         is_finobt=$($XFS_INFO_PROG "${SCRATCH_MNT}" | grep -c 'finobt=1')
 588         is_rmapbt=$($XFS_INFO_PROG "${SCRATCH_MNT}" | grep -c 'rmapbt=1')
 589         is_reflink=$($XFS_INFO_PROG "${SCRATCH_MNT}" | grep -c 'reflink=1')
 590
 591         blksz="$(stat -f -c '%s' "${SCRATCH_MNT}")"
 592         dblksz="$($XFS_INFO_PROG "${SCRATCH_MNT}" | grep naming.*bsize | sed -e 's/^.*bsize=//g' -e 's/\([0-9]*\).*$/\1/g')"
 593         leaf_lblk="$((32 * 1073741824 / blksz))"
 594         node_lblk="$((64 * 1073741824 / blksz))"
 595         umount "${SCRATCH_MNT}"
 596
 597         __populate_check_xfs_dformat "${extents_file}" "extents"
 598         __populate_check_xfs_dformat "${btree_file}" "btree"
 599         __populate_check_xfs_dir "${inline_dir}" "inline"
 600         __populate_check_xfs_dir "${block_dir}" "block"
 601         __populate_check_xfs_dir "${leaf_dir}" "leaf"
 602         __populate_check_xfs_dir "${leafn_dir}" "leafn"
 603         __populate_check_xfs_dir "${node_dir}" "node"
 604         __populate_check_xfs_dir "${btree_dir}" "btree"
 605         __populate_check_xfs_dformat "${btree_dir}" "btree"
 606         __populate_check_xfs_dformat "${bdev}" "dev"
 607         __populate_check_xfs_dformat "${cdev}" "dev"
 608         __populate_check_xfs_attr "${local_attr}" "local"
 609         __populate_check_xfs_attr "${leaf_attr}" "leaf"
 610         __populate_check_xfs_attr "${node_attr}" "node"
 611         __populate_check_xfs_attr "${btree_attr}" "btree"
 612         __populate_check_xfs_aformat "${btree_attr}" "btree"
 613         __populate_check_xfs_agbtree_height "bno"
 614         __populate_check_xfs_agbtree_height "cnt"
 615         __populate_check_xfs_agbtree_height "ino"
 616         test $is_finobt -ne 0 && __populate_check_xfs_agbtree_height "fino"
 617         test $is_rmapbt -ne 0 && __populate_check_xfs_agbtree_height "rmap"
 618         test $is_reflink -ne 0 && __populate_check_xfs_agbtree_height "refcnt"
 619 }
 620
 621 # Check data fork format of ext4 file
 622 __populate_check_ext4_dformat() {
 623         dev="${SCRATCH_DEV}"
 624         inode="$1"
 625         format="$2"
 626
 627         extents=0
 628         etree=0
 629         debugfs -R "stat <${inode}>" "${dev}" 2> /dev/null | grep 'ETB[0-9]' -q && etree=1
 630         iflags="$(debugfs -R "stat <${inode}>" "${dev}" 2> /dev/null | grep 'Flags:' | sed -e 's/^.*Flags: \([0-9a-fx]*\).*$/\1/g')"
 631         test "$(echo "${iflags}" | awk '{print and(strtonum($1), 0x80000);}')" -gt 0 && extents=1
 632
 633         case "${format}" in
 634         "blockmap")
 635                 test "${extents}" -eq 0 || _fail "failed to create ino ${inode} with blockmap"
 636                 ;;
 637         "extent"|"extents")
 638                 test "${extents}" -eq 1 || _fail "failed to create ino ${inode} with extents"
 639                 ;;
 640         "etree")
 641                 (test "${extents}" -eq 1 && test "${etree}" -eq 1) || _fail "failed to create ino ${inode} with extent tree"
 642                 ;;
 643         *)
 644                 _fail "Unknown dformat ${format}"
 645         esac
 646 }
 647
 648 # Check attr fork format of ext4 file
 649 __populate_check_ext4_aformat() {
 650         dev="${SCRATCH_DEV}"
 651         inode="$1"
 652         format="$2"
 653
 654         ablock=1
 655         debugfs -R "stat <${inode}>" "${dev}" 2> /dev/null | grep 'File ACL: 0' -q && ablock=0
 656
 657         case "${format}" in
 658         "local"|"inline")
 659                 test "${ablock}" -eq 0 || _fail "failed to create inode ${inode} with ${format} xattr"
 660                 ;;
 661         "block")
 662                 test "${extents}" -eq 1 || _fail "failed to create inode ${inode} with ${format} xattr"
 663                 ;;
 664         *)
 665                 _fail "Unknown aformat ${format}"
 666         esac
 667 }
 668
 669 # Check structure of ext4 dir
 670 __populate_check_ext4_dir() {
 671         dev="${SCRATCH_DEV}"
 672         inode="$1"
 673         dtype="$2"
 674
 675         htree=0
 676         inline=0
 677         iflags="$(debugfs -R "stat <${inode}>" "${dev}" 2> /dev/null | grep 'Flags:' | sed -e 's/^.*Flags: \([0-9a-fx]*\).*$/\1/g')"
 678         test "$(echo "${iflags}" | awk '{print and(strtonum($1), 0x1000);}')" -gt 0 && htree=1
 679         test "$(echo "${iflags}" | awk '{print and(strtonum($1), 0x10000000);}')" -gt 0 && inline=1
 680
 681         case "${dtype}" in
 682         "inline")
 683                 (test "${inline}" -eq 1 && test "${htree}" -eq 0) || _fail "failed to create ${dtype} dir ino ${inode} htree ${htree} inline ${inline}"
 684                 ;;
 685         "block")
 686                 (test "${inline}" -eq 0 && test "${htree}" -eq 0) || _fail "failed to create ${dtype} dir ino ${inode} htree ${htree} inline ${inline}"
 687                 ;;
 688         "htree")
 689                 (test "${inline}" -eq 0 && test "${htree}" -eq 1) || _fail "failed to create ${dtype} dir ino ${inode} htree ${htree} inline ${inline}"
 690                 ;;
 691         *)
 692                 _fail "Unknown directory type ${dtype}"
 693                 ;;
 694         esac
 695 }
 696
 697 # Check that populate created all the types of files we wanted
 698 _scratch_ext4_populate_check() {
 699         _scratch_mount
 700         extents_file="$(__populate_find_inode "${SCRATCH_MNT}/S_IFREG.FMT_EXTENTS")"
 701         etree_file="$(__populate_find_inode "${SCRATCH_MNT}/S_IFREG.FMT_ETREE")"
 702         block_dir="$(__populate_find_inode "${SCRATCH_MNT}/S_IFDIR.FMT_BLOCK")"
 703         htree_dir="$(__populate_find_inode "${SCRATCH_MNT}/S_IFDIR.FMT_HTREE")"
 704         extents_slink="$(__populate_find_inode "${SCRATCH_MNT}/S_IFLNK.FMT_EXTENTS")"
 705         local_attr="$(__populate_find_inode "${SCRATCH_MNT}/ATTR.FMT_LOCAL")"
 706         block_attr="$(__populate_find_inode "${SCRATCH_MNT}/ATTR.FMT_BLOCK")"
 707         umount "${SCRATCH_MNT}"
 708
 709         __populate_check_ext4_dformat "${extents_file}" "extents"
 710         __populate_check_ext4_dformat "${etree_file}" "etree"
 711         __populate_check_ext4_dir "${block_dir}" "block"
 712         __populate_check_ext4_dir "${htree_dir}" "htree"
 713         __populate_check_ext4_dformat "${extents_slink}" "extents"
 714         __populate_check_ext4_aformat "${local_attr}" "local"
 715         __populate_check_ext4_aformat "${block_attr}" "block"
 716 }
 717
 718 # Populate a scratch FS and check the contents to make sure we got that
 719 _scratch_populate() {
 720         case "${FSTYP}" in
 721         "xfs")
 722                 _scratch_xfs_populate
 723                 _scratch_xfs_populate_check
 724                 ;;
 725         "ext2"|"ext3"|"ext4")
 726                 _scratch_ext4_populate
 727                 _scratch_ext4_populate_check
 728                 ;;
 729         *)
 730                 _fail "Don't know how to populate a ${FSTYP} filesystem."
 731                 ;;
 732         esac
 733 }
 734
 735 # Fill a file system by repeatedly creating files in the given folder
 736 # starting with the given file size.  Files are reduced in size when
 737 # they can no longer fit until no more files can be created.
 738 _fill_fs()
 739 {
 740         local file_size=$1
 741         local dir=$2
 742         local block_size=$3
 743         local switch_user=$4
 744         local file_count=1
 745         local bytes_written=0
 746         local use_falloc=1;
 747
 748         if [ $# -ne 4 ]; then
 749                 echo "Usage: _fill_fs filesize dir blocksize switch_user"
 750                 exit 1
 751         fi
 752
 753         if [ $switch_user -eq 0 ]; then
 754                 mkdir -p $dir
 755         else
 756                 _user_do "mkdir -p $dir"
 757         fi
 758         if [ ! -d $dir ]; then
 759                 return 0;
 760         fi
 761
 762         testio=`$XFS_IO_PROG -F -fc "falloc 0 $block_size" $dir/$$.xfs_io 2>&1`
 763         echo $testio | grep -q "not found" && use_falloc=0
 764         echo $testio | grep -q "Operation not supported" && use_falloc=0
 765
 766         if [ $file_size -lt $block_size ]; then
 767                 $file_size = $block_size
 768         fi
 769
 770         while [ $file_size -ge $block_size ]; do
 771                 bytes_written=0
 772                 if [ $switch_user -eq 0 ]; then
 773                         if [ $use_falloc -eq 0 ]; then
 774                                 $XFS_IO_PROG -fc "pwrite -b 8388608 0 $file_size" \
 775                                         $dir/$file_count
 776                         else
 777                                 $XFS_IO_PROG -fc "falloc 0 $file_size" \
 778                                         $dir/$file_count
 779                         fi
 780                 else
 781                         if [ $use_falloc -eq 0 ]; then
 782                                 _user_do "$XFS_IO_PROG -f -c \"pwrite -b 8388608 0 \
 783                                         $file_size\" $dir/$file_count"
 784                         else
 785                                 _user_do "$XFS_IO_PROG -f -c \"falloc 0 \
 786                                         $file_size\" $dir/$file_count"
 787                         fi
 788                 fi
 789
 790                 if [ -f $dir/$file_count ]; then
 791                         bytes_written=$(_get_filesize $dir/$file_count)
 792                 fi
 793
 794                 # If there was no room to make the file, then divide it in
 795                 # half, and keep going
 796                 if [ $bytes_written -lt $file_size ]; then
 797                         file_size=$((file_size / 2))
 798                 fi
 799                 file_count=$((file_count + 1))
 800         done
 801 }
 802
 803 # Compute the fs geometry description of a populated filesystem
 804 _scratch_populate_cache_tag() {
 805         local extra_descr=""
 806         local size="$(blockdev --getsz "${SCRATCH_DEV}")"
 807
 808         case "${FSTYP}" in
 809         "ext4")
 810                 extra_descr="LOGDEV ${SCRATCH_LOGDEV} USE_EXTERNAL ${USE_EXTERNAL}"
 811                 ;;
 812         "xfs")
 813                 extra_descr="LOGDEV ${SCRATCH_LOGDEV} USE_EXTERNAL ${USE_EXTERNAL} RTDEV ${SCRATCH_RTDEV}"
 814                 _populate_xfs_qmount_option
 815                 if echo "${MOUNT_OPTIONS}" | grep -q 'usrquota'; then
 816                         extra_descr="${extra_descr} QUOTAS"
 817                 fi
 818                 ;;
 819         esac
 820         echo "FSTYP ${FSTYP} MKFS_OPTIONS ${MKFS_OPTIONS} SIZE ${size} ${extra_descr} ARGS $@"
 821 }
 822
 823 # Restore a cached populated fs from a metadata dump
 824 _scratch_populate_restore_cached() {
 825         local metadump="$1"
 826
 827         case "${FSTYP}" in
 828         "xfs")
 829                 xfs_mdrestore "${metadump}" "${SCRATCH_DEV}" && return 0
 830                 ;;
 831         "ext2"|"ext3"|"ext4")
 832                 # ext4 cannot e2image external logs, so we cannot restore
 833                 test -n "${SCRATCH_LOGDEV}" && return 1
 834                 e2image -r "${metadump}" "${SCRATCH_DEV}" && return 0
 835                 ;;
 836         esac
 837         return 1
 838 }
 839
 840 # Populate a scratch FS from scratch or from a cached image.
 841 _scratch_populate_cached() {
 842         local meta_descr="$(_scratch_populate_cache_tag "$@")"
 843         local meta_tag="$(echo "${meta_descr}" | md5sum - | cut -d ' ' -f 1)"
 844         local metadump_stem="${TEST_DIR}/__populate.${FSTYP}.${meta_tag}"
 845
 846         # These variables are shared outside this function
 847         POPULATE_METADUMP="${metadump_stem}.metadump"
 848         POPULATE_METADUMP_DESCR="${metadump_stem}.txt"
 849
 850         # Don't keep metadata images cached for more 48 hours...
 851         rm -rf "$(find "${POPULATE_METADUMP}" -mtime +2 2>/dev/null)"
 852
 853         # Throw away cached image if it doesn't match our spec.
 854         cmp -s "${POPULATE_METADUMP_DESCR}" <(echo "${meta_descr}") || \
 855                 rm -rf "${POPULATE_METADUMP}"
 856
 857         # Try to restore from the metadump
 858         test -r "${POPULATE_METADUMP}" && \
 859                 _scratch_populate_restore_cached "${POPULATE_METADUMP}" && \
 860                 return
 861
 862         # Oh well, just create one from scratch
 863         _scratch_mkfs
 864         echo "${meta_descr}" > "${POPULATE_METADUMP_DESCR}"
 865         case "${FSTYP}" in
 866         "xfs")
 867                 _scratch_xfs_populate $@
 868                 _scratch_xfs_populate_check
 869                 _scratch_metadump "${POPULATE_METADUMP}" -a -o
 870                 ;;
 871         "ext2"|"ext3"|"ext4")
 872                 _scratch_ext4_populate $@
 873                 _scratch_ext4_populate_check
 874                 e2image -Q "${SCRATCH_DEV}" "${POPULATE_METADUMP}"
 875                 ;;
 876         *)
 877                 _fail "Don't know how to populate a ${FSTYP} filesystem."
 878                 ;;
 879         esac
 880 }