common/populate

   1 ##/bin/bash
   2 # SPDX-License-Identifier: GPL-2.0+
   3 # Copyright (c) 2015 Oracle.  All Rights Reserved.
   4 #
   5 # Routines for populating a scratch fs, and helpers to exercise an FS
   6 # once it's been fuzzed.
   7
   8 . ./common/quota
   9
  10 _require_populate_commands() {
  11         _require_xfs_io_command "falloc"
  12         _require_xfs_io_command "fpunch"
  13         _require_test_program "punch-alternating"
  14         _require_command "$XFS_DB_PROG" "xfs_db"
  15 }
  16
  17 _require_xfs_db_blocktrash_z_command() {
  18         test "${FSTYP}" = "xfs" || _notrun "cannot run xfs_db on ${FSTYP}"
  19         $XFS_DB_PROG -x -f -c 'blocktrash -z' "${TEST_DEV}" | grep -q 'nothing on stack' || _notrun "blocktrash -z not supported"
  20 }
  21
  22 # Attempt to make files of "every" format for data, dirs, attrs etc.
  23 # (with apologies to Eric Sandeen for mutating xfser.sh)
  24
  25 # Create a file of a given size.
  26 __populate_create_file() {
  27         local sz="$1"
  28         local fname="$2"
  29
  30         $XFS_IO_PROG -f -c "pwrite -S 0x62 -W -b 1m 0 $sz" "${fname}"
  31 }
  32
  33 # Punch out every other hole in this file, if it exists.
  34 #
  35 # The goal here is to force the creation of a large number of metadata records
  36 # by creating a lot of tiny extent mappings in a file.  Callers should ensure
  37 # that fragmenting the file actually causes record creation.  Call this
  38 # function /after/ creating all other metadata structures.
  39 __populate_fragment_file() {
  40         local fname="$1"
  41
  42         test -f "${fname}" && $here/src/punch-alternating "${fname}"
  43 }
  44
  45 # Create a large directory
  46 __populate_create_dir() {
  47         name="$1"
  48         nr="$2"
  49         missing="$3"
  50
  51         mkdir -p "${name}"
  52         seq 0 "${nr}" | while read d; do
  53                 creat=mkdir
  54                 test "$((d % 20))" -eq 0 && creat=touch
  55                 $creat "${name}/$(printf "%.08d" "$d")"
  56         done
  57
  58         test -z "${missing}" && return
  59         seq 1 2 "${nr}" | while read d; do
  60                 rm -rf "${name}/$(printf "%.08d" "$d")"
  61         done
  62 }
  63
  64 # Add a bunch of attrs to a file
  65 __populate_create_attr() {
  66         name="$1"
  67         nr="$2"
  68         missing="$3"
  69
  70         touch "${name}"
  71         seq 0 "${nr}" | while read d; do
  72                 setfattr -n "user.$(printf "%.08d" "$d")" -v "$(printf "%.08d" "$d")" "${name}"
  73         done
  74
  75         test -z "${missing}" && return
  76         seq 1 2 "${nr}" | while read d; do
  77                 setfattr -x "user.$(printf "%.08d" "$d")" "${name}"
  78         done
  79 }
  80
  81 # Fill up some percentage of the remaining free space
  82 __populate_fill_fs() {
  83         dir="$1"
  84         pct="$2"
  85         test -z "${pct}" && pct=60
  86
  87         mkdir -p "${dir}/test/1"
  88         cp -pRdu "${dir}"/S_IFREG* "${dir}/test/1/"
  89
  90         SRC_SZ="$(du -ks "${dir}/test/1" | cut -f 1)"
  91         FS_SZ="$(( $(stat -f "${dir}" -c '%a * %S') / 1024 ))"
  92
  93         NR="$(( (FS_SZ * ${pct} / 100) / SRC_SZ ))"
  94
  95         echo "FILL FS"
  96         echo "src_sz $SRC_SZ fs_sz $FS_SZ nr $NR"
  97         seq 2 "${NR}" | while read nr; do
  98                 cp -pRdu "${dir}/test/1" "${dir}/test/${nr}"
  99         done
 100 }
 101
 102 # For XFS, force on all the quota options if quota is enabled
 103 # and the user didn't feed us noquota.
 104 _populate_xfs_qmount_option()
 105 {
 106         # User explicitly told us not to quota
 107         if echo "${MOUNT_OPTIONS}" | grep -q 'noquota'; then
 108                 return
 109         fi
 110
 111         # Don't bother if we can't turn on quotas
 112         if [ ! -f /proc/fs/xfs/xqmstat ]; then
 113                 # No quota support
 114                 return
 115         elif [ "${USE_EXTERNAL}" = "yes" ] && [ ! -z "${SCRATCH_RTDEV}" ]; then
 116                 # Quotas not supported on rt filesystems
 117                 return
 118         elif [ -z "${XFS_QUOTA_PROG}" ]; then
 119                 # xfs quota tools not installed
 120                 return
 121         fi
 122
 123         # Turn on all the quotas
 124         if $XFS_INFO_PROG "${TEST_DIR}" | grep -q 'crc=1'; then
 125                 # v5 filesystems can have group & project quotas
 126                 quota="usrquota,grpquota,prjquota"
 127         else
 128                 # v4 filesystems cannot mix group & project quotas
 129                 quota="usrquota,grpquota"
 130         fi
 131
 132         # Inject our quota mount options
 133         if echo "${MOUNT_OPTIONS}" | grep -q "${quota}"; then
 134                 return
 135         elif echo "${MOUNT_OPTIONS}" | egrep -q '(quota|noenforce)'; then
 136                 _qmount_option "${quota}"
 137         else
 138                 export MOUNT_OPTIONS="$MOUNT_OPTIONS -o ${quota}"
 139                 echo "MOUNT_OPTIONS = $MOUNT_OPTIONS" >>$seqres.full
 140         fi
 141 }
 142
 143 # Populate an XFS on the scratch device with (we hope) all known
 144 # types of metadata block
 145 _scratch_xfs_populate() {
 146         fill=1
 147
 148         for arg in $@; do
 149                 case "${arg}" in
 150                 "nofill")
 151                         fill=0;;
 152                 esac
 153         done
 154
 155         _populate_xfs_qmount_option
 156         _scratch_mount
 157
 158         # We cannot directly force the filesystem to create the metadata
 159         # structures we want; we can only achieve this indirectly by carefully
 160         # crafting files and a directory tree.  Therefore, we must have exact
 161         # control over the layout and device selection of all files created.
 162         # Clear the rtinherit flag on the root directory so that files are
 163         # always created on the data volume regardless of MKFS_OPTIONS.  We can
 164         # set the realtime flag when needed.
 165         $XFS_IO_PROG -c 'chattr -t' $SCRATCH_MNT
 166
 167         blksz="$(stat -f -c '%s' "${SCRATCH_MNT}")"
 168         dblksz="$($XFS_INFO_PROG "${SCRATCH_MNT}" | grep naming.*bsize | sed -e 's/^.*bsize=//g' -e 's/\([0-9]*\).*$/\1/g')"
 169         crc="$($XFS_INFO_PROG "${SCRATCH_MNT}" | grep crc= | sed -e 's/^.*crc=//g' -e 's/\([0-9]*\).*$/\1/g')"
 170         if [ $crc -eq 1 ]; then
 171                 leaf_hdr_size=64
 172         else
 173                 leaf_hdr_size=16
 174         fi
 175         leaf_lblk="$((32 * 1073741824 / blksz))"
 176         node_lblk="$((64 * 1073741824 / blksz))"
 177
 178         # Data:
 179
 180         # Fill up the root inode chunk
 181         echo "+ fill root ino chunk"
 182         seq 1 64 | while read f; do
 183                 $XFS_IO_PROG -f -c "truncate 0" "${SCRATCH_MNT}/dummy${f}"
 184         done
 185
 186         # Regular files
 187         # - FMT_EXTENTS
 188         echo "+ extents file"
 189         __populate_create_file $blksz "${SCRATCH_MNT}/S_IFREG.FMT_EXTENTS"
 190
 191         # - FMT_BTREE
 192         echo "+ btree extents file"
 193         nr="$((blksz * 2 / 16))"
 194         __populate_create_file $((blksz * nr)) "${SCRATCH_MNT}/S_IFREG.FMT_BTREE"
 195
 196         # Directories
 197         # - INLINE
 198         echo "+ inline dir"
 199         __populate_create_dir "${SCRATCH_MNT}/S_IFDIR.FMT_INLINE" 1
 200
 201         # - BLOCK
 202         echo "+ block dir"
 203         __populate_create_dir "${SCRATCH_MNT}/S_IFDIR.FMT_BLOCK" "$((dblksz / 40))"
 204
 205         # - LEAF
 206         echo "+ leaf dir"
 207         __populate_create_dir "${SCRATCH_MNT}/S_IFDIR.FMT_LEAF" "$((dblksz / 12))"
 208
 209         # - LEAFN
 210         echo "+ leafn dir"
 211         __populate_create_dir "${SCRATCH_MNT}/S_IFDIR.FMT_LEAFN" "$(( ((dblksz - leaf_hdr_size) / 8) - 3 ))"
 212
 213         # - NODE
 214         echo "+ node dir"
 215         __populate_create_dir "${SCRATCH_MNT}/S_IFDIR.FMT_NODE" "$((16 * dblksz / 40))" true
 216
 217         # - BTREE
 218         echo "+ btree dir"
 219         __populate_create_dir "${SCRATCH_MNT}/S_IFDIR.FMT_BTREE" "$((128 * dblksz / 40))" true
 220
 221         # Symlinks
 222         # - FMT_LOCAL
 223         echo "+ inline symlink"
 224         ln -s target "${SCRATCH_MNT}/S_IFLNK.FMT_LOCAL"
 225
 226         # - FMT_EXTENTS
 227         echo "+ extents symlink"
 228         ln -s "$(perl -e 'print "x" x 1023;')" "${SCRATCH_MNT}/S_IFLNK.FMT_EXTENTS"
 229
 230         # Char & block
 231         echo "+ special"
 232         mknod "${SCRATCH_MNT}/S_IFCHR" c 1 1
 233         mknod "${SCRATCH_MNT}/S_IFBLK" b 1 1
 234         mknod "${SCRATCH_MNT}/S_IFIFO" p
 235
 236         # special file with an xattr
 237         setfacl -P -m u:nobody:r ${SCRATCH_MNT}/S_IFCHR
 238
 239         # Attribute formats
 240         # LOCAL
 241         echo "+ local attr"
 242         __populate_create_attr "${SCRATCH_MNT}/ATTR.FMT_LOCAL" 1
 243
 244         # LEAF
 245         echo "+ leaf attr"
 246         __populate_create_attr "${SCRATCH_MNT}/ATTR.FMT_LEAF" "$((blksz / 40))"
 247
 248         # NODE
 249         echo "+ node attr"
 250         __populate_create_attr "${SCRATCH_MNT}/ATTR.FMT_NODE" "$((8 * blksz / 40))"
 251
 252         # BTREE
 253         echo "+ btree attr"
 254         __populate_create_attr "${SCRATCH_MNT}/ATTR.FMT_BTREE" "$((64 * blksz / 40))" true
 255
 256         # trusted namespace
 257         touch ${SCRATCH_MNT}/ATTR.TRUSTED
 258         setfattr -n trusted.moo -v urk ${SCRATCH_MNT}/ATTR.TRUSTED
 259
 260         # security namespace
 261         touch ${SCRATCH_MNT}/ATTR.SECURITY
 262         setfattr -n security.foo -v bar ${SCRATCH_MNT}/ATTR.SECURITY
 263
 264         # system namespace
 265         touch ${SCRATCH_MNT}/ATTR.SYSTEM
 266         setfacl -m u:root:r ${SCRATCH_MNT}/ATTR.SYSTEM
 267
 268         # FMT_EXTENTS with a remote less-than-a-block value
 269         echo "+ attr extents with a remote less-than-a-block value"
 270         touch "${SCRATCH_MNT}/ATTR.FMT_EXTENTS_REMOTE3K"
 271         $XFS_IO_PROG -f -c "pwrite -S 0x43 0 $((blksz - 300))" "${SCRATCH_MNT}/attrvalfile" > /dev/null
 272         attr -q -s user.remotebtreeattrname "${SCRATCH_MNT}/ATTR.FMT_EXTENTS_REMOTE3K" < "${SCRATCH_MNT}/attrvalfile"
 273
 274         # FMT_EXTENTS with a remote block-size value
 275         echo "+ attr extents with a remote one-block value"
 276         touch "${SCRATCH_MNT}/ATTR.FMT_EXTENTS_REMOTE4K"
 277         $XFS_IO_PROG -f -c "pwrite -S 0x44 0 ${blksz}" "${SCRATCH_MNT}/attrvalfile" > /dev/null
 278         attr -q -s user.remotebtreeattrname "${SCRATCH_MNT}/ATTR.FMT_EXTENTS_REMOTE4K" < "${SCRATCH_MNT}/attrvalfile"
 279         rm -rf "${SCRATCH_MNT}/attrvalfile"
 280
 281         # Make an unused inode
 282         echo "+ empty file"
 283         touch "${SCRATCH_MNT}/unused"
 284         $XFS_IO_PROG -f -c 'fsync' "${SCRATCH_MNT}/unused"
 285         rm -rf "${SCRATCH_MNT}/unused"
 286
 287         # Free space btree
 288         echo "+ freesp btree"
 289         nr="$((blksz * 2 / 8))"
 290         __populate_create_file $((blksz * nr)) "${SCRATCH_MNT}/BNOBT"
 291
 292         # Inode btree
 293         echo "+ inobt btree"
 294         local ino_per_rec=64
 295         local rec_per_btblock=16
 296         local nr="$(( 2 * (blksz / rec_per_btblock) * ino_per_rec ))"
 297         local dir="${SCRATCH_MNT}/INOBT"
 298         mkdir -p "${dir}"
 299         seq 0 "${nr}" | while read f; do
 300                 touch "${dir}/${f}"
 301         done
 302
 303         seq 0 2 "${nr}" | while read f; do
 304                 rm -f "${dir}/${f}"
 305         done
 306
 307         # Reverse-mapping btree
 308         is_rmapbt="$($XFS_INFO_PROG "${SCRATCH_MNT}" | grep -c 'rmapbt=1')"
 309         if [ $is_rmapbt -gt 0 ]; then
 310                 echo "+ rmapbt btree"
 311                 nr="$((blksz * 2 / 24))"
 312                 __populate_create_file $((blksz * nr)) "${SCRATCH_MNT}/RMAPBT"
 313         fi
 314
 315         # Realtime Reverse-mapping btree
 316         is_rt="$($XFS_INFO_PROG "${SCRATCH_MNT}" | grep -c 'rtextents=[1-9]')"
 317         if [ $is_rmapbt -gt 0 ] && [ $is_rt -gt 0 ]; then
 318                 echo "+ rtrmapbt btree"
 319                 nr="$((blksz * 2 / 32))"
 320                 $XFS_IO_PROG -R -f -c 'truncate 0' "${SCRATCH_MNT}/RTRMAPBT"
 321                 __populate_create_file $((blksz * nr)) "${SCRATCH_MNT}/RTRMAPBT"
 322         fi
 323
 324         # Reference-count btree
 325         is_reflink="$($XFS_INFO_PROG "${SCRATCH_MNT}" | grep -c 'reflink=1')"
 326         if [ $is_reflink -gt 0 ]; then
 327                 echo "+ reflink btree"
 328                 nr="$((blksz * 2 / 12))"
 329                 __populate_create_file $((blksz * nr)) "${SCRATCH_MNT}/REFCOUNTBT"
 330                 cp --reflink=always "${SCRATCH_MNT}/REFCOUNTBT" "${SCRATCH_MNT}/REFCOUNTBT2"
 331         fi
 332
 333         # Copy some real files (xfs tests, I guess...)
 334         echo "+ real files"
 335         test $fill -ne 0 && __populate_fill_fs "${SCRATCH_MNT}" 5
 336
 337         # Make sure we get all the fragmentation we asked for
 338         __populate_fragment_file "${SCRATCH_MNT}/S_IFREG.FMT_BTREE"
 339         __populate_fragment_file "${SCRATCH_MNT}/BNOBT"
 340         __populate_fragment_file "${SCRATCH_MNT}/RMAPBT"
 341         __populate_fragment_file "${SCRATCH_MNT}/RTRMAPBT"
 342         __populate_fragment_file "${SCRATCH_MNT}/REFCOUNTBT"
 343
 344         umount "${SCRATCH_MNT}"
 345 }
 346
 347 # Populate an ext4 on the scratch device with (we hope) all known
 348 # types of metadata block
 349 _scratch_ext4_populate() {
 350         fill=1
 351
 352         for arg in $@; do
 353                 case "${arg}" in
 354                 "nofill")
 355                         fill=0;;
 356                 esac
 357         done
 358
 359         _scratch_mount
 360         blksz="$(stat -f -c '%s' "${SCRATCH_MNT}")"
 361         dblksz="${blksz}"
 362         leaf_lblk="$((32 * 1073741824 / blksz))"
 363         node_lblk="$((64 * 1073741824 / blksz))"
 364
 365         # Data:
 366
 367         # Regular files
 368         # - FMT_INLINE
 369         echo "+ inline file"
 370         __populate_create_file 1 "${SCRATCH_MNT}/S_IFREG.FMT_INLINE"
 371
 372         # - FMT_EXTENTS
 373         echo "+ extents file"
 374         __populate_create_file $blksz "${SCRATCH_MNT}/S_IFREG.FMT_EXTENTS"
 375
 376         # - FMT_ETREE
 377         echo "+ extent tree file"
 378         nr="$((blksz * 2 / 12))"
 379         __populate_create_file $((blksz * nr)) "${SCRATCH_MNT}/S_IFREG.FMT_ETREE"
 380
 381         # Directories
 382         # - INLINE
 383         echo "+ inline dir"
 384         __populate_create_dir "${SCRATCH_MNT}/S_IFDIR.FMT_INLINE" 1
 385
 386         # - BLOCK
 387         echo "+ block dir"
 388         __populate_create_dir "${SCRATCH_MNT}/S_IFDIR.FMT_BLOCK" "$((dblksz / 32))"
 389
 390         # - HTREE
 391         echo "+ htree dir"
 392         __populate_create_dir "${SCRATCH_MNT}/S_IFDIR.FMT_HTREE" "$((4 * dblksz / 24))"
 393
 394         # Symlinks
 395         # - FMT_LOCAL
 396         echo "+ inline symlink"
 397         ln -s target "${SCRATCH_MNT}/S_IFLNK.FMT_LOCAL"
 398
 399         # - FMT_EXTENTS
 400         echo "+ extents symlink"
 401         ln -s "$(perl -e 'print "x" x 1023;')" "${SCRATCH_MNT}/S_IFLNK.FMT_EXTENTS"
 402
 403         # Char & block
 404         echo "+ special"
 405         mknod "${SCRATCH_MNT}/S_IFCHR" c 1 1
 406         mknod "${SCRATCH_MNT}/S_IFBLK" b 1 1
 407         mknod "${SCRATCH_MNT}/S_IFIFO" p
 408
 409         # special file with an xattr
 410         setfacl -P -m u:nobody:r ${SCRATCH_MNT}/S_IFCHR
 411
 412         # Attribute formats
 413         # LOCAL
 414         echo "+ local attr"
 415         __populate_create_attr "${SCRATCH_MNT}/ATTR.FMT_LOCAL" 0
 416
 417         # BLOCK
 418         echo "+ block attr"
 419         __populate_create_attr "${SCRATCH_MNT}/ATTR.FMT_BLOCK" "$((blksz / 40))"
 420
 421         # trusted namespace
 422         touch ${SCRATCH_MNT}/ATTR.TRUSTED
 423         setfattr -n trusted.moo -v urk ${SCRATCH_MNT}/ATTR.TRUSTED
 424
 425         # security namespace
 426         touch ${SCRATCH_MNT}/ATTR.SECURITY
 427         setfattr -n security.foo -v bar ${SCRATCH_MNT}/ATTR.SECURITY
 428
 429         # system namespace
 430         touch ${SCRATCH_MNT}/ATTR.SYSTEM
 431         setfacl -m u:root:r ${SCRATCH_MNT}/ATTR.SYSTEM
 432
 433         # Make an unused inode
 434         echo "+ empty file"
 435         touch "${SCRATCH_MNT}/unused"
 436         $XFS_IO_PROG -f -c 'fsync' "${SCRATCH_MNT}/unused"
 437         rm -rf "${SCRATCH_MNT}/unused"
 438
 439         # Copy some real files (xfs tests, I guess...)
 440         echo "+ real files"
 441         test $fill -ne 0 && __populate_fill_fs "${SCRATCH_MNT}" 5
 442
 443         # Make sure we get all the fragmentation we asked for
 444         __populate_fragment_file "${SCRATCH_MNT}/S_IFREG.FMT_ETREE"
 445
 446         umount "${SCRATCH_MNT}"
 447 }
 448
 449 # Find the inode number of a file
 450 __populate_find_inode() {
 451         name="$1"
 452         inode="$(stat -c '%i' "${name}")"
 453         echo "${inode}"
 454 }
 455
 456 # Check data fork format of XFS file
 457 __populate_check_xfs_dformat() {
 458         inode="$1"
 459         format="$2"
 460
 461         fmt="$(_scratch_xfs_db -c "inode ${inode}" -c 'p core.format' | sed -e 's/^.*(\([a-z]*\)).*$/\1/g')"
 462         test "${format}" = "${fmt}" || _fail "failed to create ino ${inode} dformat expected ${format} saw ${fmt}"
 463 }
 464
 465 # Check attr fork format of XFS file
 466 __populate_check_xfs_aformat() {
 467         inode="$1"
 468         format="$2"
 469
 470         fmt="$(_scratch_xfs_db -c "inode ${inode}" -c 'p core.aformat' | sed -e 's/^.*(\([a-z]*\)).*$/\1/g')"
 471         test "${format}" = "${fmt}" || _fail "failed to create ino ${inode} aformat expected ${format} saw ${fmt}"
 472 }
 473
 474 # Check structure of XFS directory
 475 __populate_check_xfs_dir() {
 476         inode="$1"
 477         dtype="$2"
 478
 479         (test -n "${leaf_lblk}" && test -n "${node_lblk}") || _fail "must define leaf_lblk and node_lblk before calling __populate_check_xfs_dir"
 480         datab=0
 481         leafb=0
 482         freeb=0
 483         #echo "== check dir ${inode} type ${dtype}" ; _scratch_xfs_db -x -c "inode ${inode}" -c "bmap"
 484         _scratch_xfs_db -x -c "inode ${inode}" -c "dblock 0" -c "stack" | grep -q 'file data block is unmapped' || datab=1
 485         _scratch_xfs_db -x -c "inode ${inode}" -c "dblock ${leaf_lblk}" -c "stack" | grep -q 'file data block is unmapped' || leafb=1
 486         _scratch_xfs_db -x -c "inode ${inode}" -c "dblock ${node_lblk}" -c "stack" | grep -q 'file data block is unmapped' || freeb=1
 487
 488         case "${dtype}" in
 489         "shortform"|"inline"|"local")
 490                 (test "${datab}" -eq 0 && test "${leafb}" -eq 0 && test "${freeb}" -eq 0) || _fail "failed to create ${dtype} dir ino ${inode} datab ${datab} leafb ${leafb} freeb ${freeb}"
 491                 ;;
 492         "block")
 493                 (test "${datab}" -eq 1 && test "${leafb}" -eq 0 && test "${freeb}" -eq 0) || _fail "failed to create ${dtype} dir ino ${inode} datab ${datab} leafb ${leafb} freeb ${freeb}"
 494                 ;;
 495         "leaf")
 496                 (test "${datab}" -eq 1 && test "${leafb}" -eq 1 && test "${freeb}" -eq 0) || _fail "failed to create ${dtype} dir ino ${inode} datab ${datab} leafb ${leafb} freeb ${freeb}"
 497                 ;;
 498         "leafn")
 499                 _scratch_xfs_db -x -c "inode ${inode}" -c "dblock ${leaf_lblk}" -c "p lhdr.info.hdr.magic" | grep -q '0x3dff' && return
 500                 _scratch_xfs_db -x -c "inode ${inode}" -c "dblock ${leaf_lblk}" -c "p lhdr.info.magic" | grep -q '0xd2ff' && return
 501                 _fail "failed to create ${dtype} dir ino ${inode} datab ${datab} leafb ${leafb} freeb ${freeb}"
 502                 ;;
 503         "node"|"btree")
 504                 (test "${datab}" -eq 1 && test "${leafb}" -eq 1 && test "${freeb}" -eq 1) || _fail "failed to create ${dtype} dir ino ${inode} datab ${datab} leafb ${leafb} freeb ${freeb}"
 505                 ;;
 506         *)
 507                 _fail "Unknown directory type ${dtype}"
 508         esac
 509 }
 510
 511 # Check structure of XFS attr
 512 __populate_check_xfs_attr() {
 513         inode="$1"
 514         atype="$2"
 515
 516         datab=0
 517         leafb=0
 518         #echo "== check attr ${inode} type ${dtype}" ; _scratch_xfs_db -x -c "inode ${inode}" -c "bmap -a"
 519         _scratch_xfs_db -x -c "inode ${inode}" -c "ablock 0" -c "stack" | grep -q 'file attr block is unmapped' || datab=1
 520         _scratch_xfs_db -x -c "inode ${inode}" -c "ablock 1" -c "stack" | grep -q 'file attr block is unmapped' || leafb=1
 521
 522         case "${atype}" in
 523         "shortform"|"inline"|"local")
 524                 (test "${datab}" -eq 0 && test "${leafb}" -eq 0) || _fail "failed to create ${atype} attr ino ${inode} datab ${datab} leafb ${leafb}"
 525                 ;;
 526         "leaf")
 527                 (test "${datab}" -eq 1 && test "${leafb}" -eq 0) || _fail "failed to create ${atype} attr ino ${inode} datab ${datab} leafb ${leafb}"
 528                 ;;
 529         "node"|"btree")
 530                 (test "${datab}" -eq 1 && test "${leafb}" -eq 1) || _fail "failed to create ${atype} attr ino ${inode} datab ${datab} leafb ${leafb}"
 531                 ;;
 532         *)
 533                 _fail "Unknown attribute type ${atype}"
 534         esac
 535 }
 536
 537 # Check that there's at least one per-AG btree with multiple levels
 538 __populate_check_xfs_agbtree_height() {
 539         bt_type="$1"
 540         nr_ags=$(_scratch_xfs_db -c 'sb 0' -c 'p agcount' | awk '{print $3}')
 541
 542         case "${bt_type}" in
 543         "bno"|"cnt"|"rmap"|"refcnt")
 544                 hdr="agf"
 545                 bt_prefix="${bt_type}"
 546                 ;;
 547         "ino")
 548                 hdr="agi"
 549                 bt_prefix=""
 550                 ;;
 551         "fino")
 552                 hdr="agi"
 553                 bt_prefix="free_"
 554                 ;;
 555         *)
 556                 _fail "Don't know about AG btree ${bt_type}"
 557                 ;;
 558         esac
 559
 560         seq 0 $((nr_ags - 1)) | while read ag; do
 561                 bt_level=$(_scratch_xfs_db -c "${hdr} ${ag}" -c "p ${bt_prefix}level" | awk '{print $3}')
 562                 if [ "${bt_level}" -gt 1 ]; then
 563                         return 100
 564                 fi
 565         done
 566         test $? -eq 100 || _fail "Failed to create ${bt_type} of sufficient height!"
 567         return 1
 568 }
 569
 570 # Check that populate created all the types of files we wanted
 571 _scratch_xfs_populate_check() {
 572         _scratch_mount
 573         extents_file="$(__populate_find_inode "${SCRATCH_MNT}/S_IFREG.FMT_EXTENTS")"
 574         btree_file="$(__populate_find_inode "${SCRATCH_MNT}/S_IFREG.FMT_BTREE")"
 575         inline_dir="$(__populate_find_inode "${SCRATCH_MNT}/S_IFDIR.FMT_INLINE")"
 576         block_dir="$(__populate_find_inode "${SCRATCH_MNT}/S_IFDIR.FMT_BLOCK")"
 577         leaf_dir="$(__populate_find_inode "${SCRATCH_MNT}/S_IFDIR.FMT_LEAF")"
 578         leafn_dir="$(__populate_find_inode "${SCRATCH_MNT}/S_IFDIR.FMT_LEAFN")"
 579         node_dir="$(__populate_find_inode "${SCRATCH_MNT}/S_IFDIR.FMT_NODE")"
 580         btree_dir="$(__populate_find_inode "${SCRATCH_MNT}/S_IFDIR.FMT_BTREE")"
 581         local_slink="$(__populate_find_inode "${SCRATCH_MNT}/S_IFLNK.FMT_LOCAL")"
 582         extents_slink="$(__populate_find_inode "${SCRATCH_MNT}/S_IFLNK.FMT_EXTENTS")"
 583         bdev="$(__populate_find_inode "${SCRATCH_MNT}/S_IFBLK")"
 584         cdev="$(__populate_find_inode "${SCRATCH_MNT}/S_IFCHR")"
 585         fifo="$(__populate_find_inode "${SCRATCH_MNT}/S_IFIFO")"
 586         local_attr="$(__populate_find_inode "${SCRATCH_MNT}/ATTR.FMT_LOCAL")"
 587         leaf_attr="$(__populate_find_inode "${SCRATCH_MNT}/ATTR.FMT_LEAF")"
 588         node_attr="$(__populate_find_inode "${SCRATCH_MNT}/ATTR.FMT_NODE")"
 589         btree_attr="$(__populate_find_inode "${SCRATCH_MNT}/ATTR.FMT_BTREE")"
 590         is_finobt=$($XFS_INFO_PROG "${SCRATCH_MNT}" | grep -c 'finobt=1')
 591         is_rmapbt=$($XFS_INFO_PROG "${SCRATCH_MNT}" | grep -c 'rmapbt=1')
 592         is_reflink=$($XFS_INFO_PROG "${SCRATCH_MNT}" | grep -c 'reflink=1')
 593
 594         blksz="$(stat -f -c '%s' "${SCRATCH_MNT}")"
 595         dblksz="$($XFS_INFO_PROG "${SCRATCH_MNT}" | grep naming.*bsize | sed -e 's/^.*bsize=//g' -e 's/\([0-9]*\).*$/\1/g')"
 596         leaf_lblk="$((32 * 1073741824 / blksz))"
 597         node_lblk="$((64 * 1073741824 / blksz))"
 598         umount "${SCRATCH_MNT}"
 599
 600         __populate_check_xfs_dformat "${extents_file}" "extents"
 601         __populate_check_xfs_dformat "${btree_file}" "btree"
 602         __populate_check_xfs_dir "${inline_dir}" "inline"
 603         __populate_check_xfs_dir "${block_dir}" "block"
 604         __populate_check_xfs_dir "${leaf_dir}" "leaf"
 605         __populate_check_xfs_dir "${leafn_dir}" "leafn"
 606         __populate_check_xfs_dir "${node_dir}" "node"
 607         __populate_check_xfs_dir "${btree_dir}" "btree"
 608         __populate_check_xfs_dformat "${btree_dir}" "btree"
 609         __populate_check_xfs_dformat "${bdev}" "dev"
 610         __populate_check_xfs_dformat "${cdev}" "dev"
 611         __populate_check_xfs_dformat "${fifo}" "dev"
 612         __populate_check_xfs_attr "${local_attr}" "local"
 613         __populate_check_xfs_attr "${leaf_attr}" "leaf"
 614         __populate_check_xfs_attr "${node_attr}" "node"
 615         __populate_check_xfs_attr "${btree_attr}" "btree"
 616         __populate_check_xfs_aformat "${btree_attr}" "btree"
 617         __populate_check_xfs_agbtree_height "bno"
 618         __populate_check_xfs_agbtree_height "cnt"
 619         __populate_check_xfs_agbtree_height "ino"
 620         test $is_finobt -ne 0 && __populate_check_xfs_agbtree_height "fino"
 621         test $is_rmapbt -ne 0 && __populate_check_xfs_agbtree_height "rmap"
 622         test $is_reflink -ne 0 && __populate_check_xfs_agbtree_height "refcnt"
 623 }
 624
 625 # Check data fork format of ext4 file
 626 __populate_check_ext4_dformat() {
 627         dev="${SCRATCH_DEV}"
 628         inode="$1"
 629         format="$2"
 630
 631         extents=0
 632         etree=0
 633         debugfs -R "stat <${inode}>" "${dev}" 2> /dev/null | grep 'ETB[0-9]' -q && etree=1
 634         iflags="$(debugfs -R "stat <${inode}>" "${dev}" 2> /dev/null | grep 'Flags:' | sed -e 's/^.*Flags: \([0-9a-fx]*\).*$/\1/g')"
 635         test "$(echo "${iflags}" | awk '{print and(strtonum($1), 0x80000);}')" -gt 0 && extents=1
 636
 637         case "${format}" in
 638         "blockmap")
 639                 test "${extents}" -eq 0 || _fail "failed to create ino ${inode} with blockmap"
 640                 ;;
 641         "extent"|"extents")
 642                 test "${extents}" -eq 1 || _fail "failed to create ino ${inode} with extents"
 643                 ;;
 644         "etree")
 645                 (test "${extents}" -eq 1 && test "${etree}" -eq 1) || _fail "failed to create ino ${inode} with extent tree"
 646                 ;;
 647         *)
 648                 _fail "Unknown dformat ${format}"
 649         esac
 650 }
 651
 652 # Check attr fork format of ext4 file
 653 __populate_check_ext4_aformat() {
 654         dev="${SCRATCH_DEV}"
 655         inode="$1"
 656         format="$2"
 657
 658         ablock=1
 659         debugfs -R "stat <${inode}>" "${dev}" 2> /dev/null | grep 'File ACL: 0' -q && ablock=0
 660
 661         case "${format}" in
 662         "local"|"inline")
 663                 test "${ablock}" -eq 0 || _fail "failed to create inode ${inode} with ${format} xattr"
 664                 ;;
 665         "block")
 666                 test "${extents}" -eq 1 || _fail "failed to create inode ${inode} with ${format} xattr"
 667                 ;;
 668         *)
 669                 _fail "Unknown aformat ${format}"
 670         esac
 671 }
 672
 673 # Check structure of ext4 dir
 674 __populate_check_ext4_dir() {
 675         dev="${SCRATCH_DEV}"
 676         inode="$1"
 677         dtype="$2"
 678
 679         htree=0
 680         inline=0
 681         iflags="$(debugfs -R "stat <${inode}>" "${dev}" 2> /dev/null | grep 'Flags:' | sed -e 's/^.*Flags: \([0-9a-fx]*\).*$/\1/g')"
 682         test "$(echo "${iflags}" | awk '{print and(strtonum($1), 0x1000);}')" -gt 0 && htree=1
 683         test "$(echo "${iflags}" | awk '{print and(strtonum($1), 0x10000000);}')" -gt 0 && inline=1
 684
 685         case "${dtype}" in
 686         "inline")
 687                 (test "${inline}" -eq 1 && test "${htree}" -eq 0) || _fail "failed to create ${dtype} dir ino ${inode} htree ${htree} inline ${inline}"
 688                 ;;
 689         "block")
 690                 (test "${inline}" -eq 0 && test "${htree}" -eq 0) || _fail "failed to create ${dtype} dir ino ${inode} htree ${htree} inline ${inline}"
 691                 ;;
 692         "htree")
 693                 (test "${inline}" -eq 0 && test "${htree}" -eq 1) || _fail "failed to create ${dtype} dir ino ${inode} htree ${htree} inline ${inline}"
 694                 ;;
 695         *)
 696                 _fail "Unknown directory type ${dtype}"
 697                 ;;
 698         esac
 699 }
 700
 701 # Check that populate created all the types of files we wanted
 702 _scratch_ext4_populate_check() {
 703         _scratch_mount
 704         extents_file="$(__populate_find_inode "${SCRATCH_MNT}/S_IFREG.FMT_EXTENTS")"
 705         etree_file="$(__populate_find_inode "${SCRATCH_MNT}/S_IFREG.FMT_ETREE")"
 706         block_dir="$(__populate_find_inode "${SCRATCH_MNT}/S_IFDIR.FMT_BLOCK")"
 707         htree_dir="$(__populate_find_inode "${SCRATCH_MNT}/S_IFDIR.FMT_HTREE")"
 708         extents_slink="$(__populate_find_inode "${SCRATCH_MNT}/S_IFLNK.FMT_EXTENTS")"
 709         local_attr="$(__populate_find_inode "${SCRATCH_MNT}/ATTR.FMT_LOCAL")"
 710         block_attr="$(__populate_find_inode "${SCRATCH_MNT}/ATTR.FMT_BLOCK")"
 711         umount "${SCRATCH_MNT}"
 712
 713         __populate_check_ext4_dformat "${extents_file}" "extents"
 714         __populate_check_ext4_dformat "${etree_file}" "etree"
 715         __populate_check_ext4_dir "${block_dir}" "block"
 716         __populate_check_ext4_dir "${htree_dir}" "htree"
 717         __populate_check_ext4_dformat "${extents_slink}" "extents"
 718         __populate_check_ext4_aformat "${local_attr}" "local"
 719         __populate_check_ext4_aformat "${block_attr}" "block"
 720 }
 721
 722 # Populate a scratch FS and check the contents to make sure we got that
 723 _scratch_populate() {
 724         case "${FSTYP}" in
 725         "xfs")
 726                 _scratch_xfs_populate
 727                 _scratch_xfs_populate_check
 728                 ;;
 729         "ext2"|"ext3"|"ext4")
 730                 _scratch_ext4_populate
 731                 _scratch_ext4_populate_check
 732                 ;;
 733         *)
 734                 _fail "Don't know how to populate a ${FSTYP} filesystem."
 735                 ;;
 736         esac
 737 }
 738
 739 # Fill a file system by repeatedly creating files in the given folder
 740 # starting with the given file size.  Files are reduced in size when
 741 # they can no longer fit until no more files can be created.
 742 _fill_fs()
 743 {
 744         local file_size=$1
 745         local dir=$2
 746         local block_size=$3
 747         local switch_user=$4
 748         local file_count=1
 749         local bytes_written=0
 750         local use_falloc=1;
 751
 752         if [ $# -ne 4 ]; then
 753                 echo "Usage: _fill_fs filesize dir blocksize switch_user"
 754                 exit 1
 755         fi
 756
 757         if [ $switch_user -eq 0 ]; then
 758                 mkdir -p $dir
 759         else
 760                 _user_do "mkdir -p $dir"
 761         fi
 762         if [ ! -d $dir ]; then
 763                 return 0;
 764         fi
 765
 766         testio=`$XFS_IO_PROG -F -fc "falloc 0 $block_size" $dir/$$.xfs_io 2>&1`
 767         echo $testio | grep -q "not found" && use_falloc=0
 768         echo $testio | grep -q "Operation not supported" && use_falloc=0
 769
 770         if [ $file_size -lt $block_size ]; then
 771                 $file_size = $block_size
 772         fi
 773
 774         while [ $file_size -ge $block_size ]; do
 775                 bytes_written=0
 776                 if [ $switch_user -eq 0 ]; then
 777                         if [ $use_falloc -eq 0 ]; then
 778                                 $XFS_IO_PROG -fc "pwrite -b 8388608 0 $file_size" \
 779                                         $dir/$file_count
 780                         else
 781                                 $XFS_IO_PROG -fc "falloc 0 $file_size" \
 782                                         $dir/$file_count
 783                         fi
 784                 else
 785                         if [ $use_falloc -eq 0 ]; then
 786                                 _user_do "$XFS_IO_PROG -f -c \"pwrite -b 8388608 0 \
 787                                         $file_size\" $dir/$file_count"
 788                         else
 789                                 _user_do "$XFS_IO_PROG -f -c \"falloc 0 \
 790                                         $file_size\" $dir/$file_count"
 791                         fi
 792                 fi
 793
 794                 if [ -f $dir/$file_count ]; then
 795                         bytes_written=$(_get_filesize $dir/$file_count)
 796                 fi
 797
 798                 # If there was no room to make the file, then divide it in
 799                 # half, and keep going
 800                 if [ $bytes_written -lt $file_size ]; then
 801                         file_size=$((file_size / 2))
 802                 fi
 803                 file_count=$((file_count + 1))
 804         done
 805 }
 806
 807 # Compute the fs geometry description of a populated filesystem
 808 _scratch_populate_cache_tag() {
 809         local extra_descr=""
 810         local size="$(blockdev --getsz "${SCRATCH_DEV}")"
 811         local logdev_sz="none"
 812         local rtdev_sz="none"
 813
 814         if [ "${USE_EXTERNAL}" = "yes" ] && [ -n "${SCRATCH_LOGDEV}" ]; then
 815                 logdev_sz="$(blockdev --getsz "${SCRATCH_LOGDEV}")"
 816         fi
 817
 818         if [ "${USE_EXTERNAL}" = "yes" ] && [ -n "${SCRATCH_RTDEV}" ]; then
 819                 rtdev_sz="$(blockdev --getsz "${SCRATCH_RTDEV}")"
 820         fi
 821
 822         case "${FSTYP}" in
 823         "ext4")
 824                 extra_descr="LOGDEV_SIZE ${logdev_sz}"
 825                 ;;
 826         "xfs")
 827                 extra_descr="LOGDEV_SIZE ${logdev_sz} RTDEV_SIZE ${rtdev_sz}"
 828                 _populate_xfs_qmount_option
 829                 if echo "${MOUNT_OPTIONS}" | grep -q 'usrquota'; then
 830                         extra_descr="${extra_descr} QUOTAS"
 831                 fi
 832                 ;;
 833         esac
 834         echo "FSTYP ${FSTYP} MKFS_OPTIONS ${MKFS_OPTIONS} SIZE ${size} ${extra_descr} ARGS $@"
 835 }
 836
 837 # Restore a cached populated fs from a metadata dump
 838 _scratch_populate_restore_cached() {
 839         local metadump="$1"
 840
 841         # If we're configured for compressed dumps and there isn't already an
 842         # uncompressed dump, see if we can use DUMP_COMPRESSOR to decompress
 843         # something.
 844         if [ -n "$DUMP_COMPRESSOR" ]; then
 845                 for compr in "$metadump".*; do
 846                         [ -e "$compr" ] && $DUMP_COMPRESSOR -d -f -k "$compr" && break
 847                 done
 848         fi
 849
 850         test -r "$metadump" || return 1
 851
 852         case "${FSTYP}" in
 853         "xfs")
 854                 xfs_mdrestore "${metadump}" "${SCRATCH_DEV}" && return 0
 855                 ;;
 856         "ext2"|"ext3"|"ext4")
 857                 # ext4 cannot e2image external logs, so we cannot restore
 858                 test -n "${SCRATCH_LOGDEV}" && return 1
 859                 e2image -r "${metadump}" "${SCRATCH_DEV}" && return 0
 860                 ;;
 861         esac
 862         return 1
 863 }
 864
 865 # Populate a scratch FS from scratch or from a cached image.
 866 _scratch_populate_cached() {
 867         local meta_descr="$(_scratch_populate_cache_tag "$@")"
 868         local meta_tag="$(echo "${meta_descr}" | md5sum - | cut -d ' ' -f 1)"
 869         local metadump_stem="${TEST_DIR}/__populate.${FSTYP}.${meta_tag}"
 870
 871         # These variables are shared outside this function
 872         POPULATE_METADUMP="${metadump_stem}.metadump"
 873         POPULATE_METADUMP_DESCR="${metadump_stem}.txt"
 874
 875         # Don't keep metadata images cached for more 48 hours...
 876         rm -rf "$(find "${POPULATE_METADUMP}" -mtime +2 2>/dev/null)"
 877
 878         # Throw away cached image if it doesn't match our spec.
 879         cmp -s "${POPULATE_METADUMP_DESCR}" <(echo "${meta_descr}") || \
 880                 rm -rf "${POPULATE_METADUMP}"
 881
 882         # Try to restore from the metadump
 883         _scratch_populate_restore_cached "${POPULATE_METADUMP}" && \
 884                 return
 885
 886         # Oh well, just create one from scratch
 887         _scratch_mkfs
 888         echo "${meta_descr}" > "${POPULATE_METADUMP_DESCR}"
 889         case "${FSTYP}" in
 890         "xfs")
 891                 _scratch_xfs_populate $@
 892                 _scratch_xfs_populate_check
 893                 _scratch_xfs_metadump "${POPULATE_METADUMP}"
 894
 895                 local logdev=
 896                 [ "$USE_EXTERNAL" = yes -a ! -z "$SCRATCH_LOGDEV" ] && \
 897                         logdev=$SCRATCH_LOGDEV
 898
 899                 _xfs_metadump "$POPULATE_METADUMP" "$SCRATCH_DEV" "$logdev" \
 900                         compress
 901                 ;;
 902         "ext2"|"ext3"|"ext4")
 903                 _scratch_ext4_populate $@
 904                 _scratch_ext4_populate_check
 905                 _ext4_metadump "${SCRATCH_DEV}" "${POPULATE_METADUMP}" compress
 906                 ;;
 907         *)
 908                 _fail "Don't know how to populate a ${FSTYP} filesystem."
 909                 ;;
 910         esac
 911 }