common/populate

   1 ##/bin/bash
   2 # SPDX-License-Identifier: GPL-2.0+
   3 # Copyright (c) 2015 Oracle.  All Rights Reserved.
   4 #
   5 # Routines for populating a scratch fs, and helpers to exercise an FS
   6 # once it's been fuzzed.
   7
   8 . ./common/quota
   9
  10 _require_populate_commands() {
  11         _require_xfs_io_command "falloc"
  12         _require_xfs_io_command "fpunch"
  13         _require_test_program "punch-alternating"
  14         _require_command "$XFS_DB_PROG" "xfs_db"
  15 }
  16
  17 _require_xfs_db_blocktrash_z_command() {
  18         test "${FSTYP}" = "xfs" || _notrun "cannot run xfs_db on ${FSTYP}"
  19         $XFS_DB_PROG -x -f -c 'blocktrash -z' "${TEST_DEV}" | grep -q 'nothing on stack' || _notrun "blocktrash -z not supported"
  20 }
  21
  22 # Attempt to make files of "every" format for data, dirs, attrs etc.
  23 # (with apologies to Eric Sandeen for mutating xfser.sh)
  24
  25 # Create a large directory
  26 __populate_create_dir() {
  27         name="$1"
  28         nr="$2"
  29         missing="$3"
  30
  31         mkdir -p "${name}"
  32         seq 0 "${nr}" | while read d; do
  33                 creat=mkdir
  34                 test "$((d % 20))" -eq 0 && creat=touch
  35                 $creat "${name}/$(printf "%.08d" "$d")"
  36         done
  37
  38         test -z "${missing}" && return
  39         seq 1 2 "${nr}" | while read d; do
  40                 rm -rf "${name}/$(printf "%.08d" "$d")"
  41         done
  42 }
  43
  44 # Add a bunch of attrs to a file
  45 __populate_create_attr() {
  46         name="$1"
  47         nr="$2"
  48         missing="$3"
  49
  50         touch "${name}"
  51         seq 0 "${nr}" | while read d; do
  52                 setfattr -n "user.$(printf "%.08d" "$d")" -v "$(printf "%.08d" "$d")" "${name}"
  53         done
  54
  55         test -z "${missing}" && return
  56         seq 1 2 "${nr}" | while read d; do
  57                 setfattr -x "user.$(printf "%.08d" "$d")" "${name}"
  58         done
  59 }
  60
  61 # Fill up some percentage of the remaining free space
  62 __populate_fill_fs() {
  63         dir="$1"
  64         pct="$2"
  65         test -z "${pct}" && pct=60
  66
  67         mkdir -p "${dir}/test/1"
  68         cp -pRdu "${dir}"/S_IFREG* "${dir}/test/1/"
  69
  70         SRC_SZ="$(du -ks "${dir}/test/1" | cut -f 1)"
  71         FS_SZ="$(( $(stat -f "${dir}" -c '%a * %S') / 1024 ))"
  72
  73         NR="$(( (FS_SZ * ${pct} / 100) / SRC_SZ ))"
  74
  75         echo "FILL FS"
  76         echo "src_sz $SRC_SZ fs_sz $FS_SZ nr $NR"
  77         seq 2 "${NR}" | while read nr; do
  78                 cp -pRdu "${dir}/test/1" "${dir}/test/${nr}"
  79         done
  80 }
  81
  82 # For XFS, force on all the quota options if quota is enabled
  83 # and the user didn't feed us noquota.
  84 _populate_xfs_qmount_option()
  85 {
  86         # User explicitly told us not to quota
  87         if echo "${MOUNT_OPTIONS}" | grep -q 'noquota'; then
  88                 return
  89         fi
  90
  91         # Don't bother if we can't turn on quotas
  92         if [ ! -f /proc/fs/xfs/xqmstat ]; then
  93                 # No quota support
  94                 return
  95         elif [ "${USE_EXTERNAL}" = "yes" ] && [ ! -z "${SCRATCH_RTDEV}" ]; then
  96                 # Quotas not supported on rt filesystems
  97                 return
  98         elif [ -z "${XFS_QUOTA_PROG}" ]; then
  99                 # xfs quota tools not installed
 100                 return
 101         fi
 102
 103         # Turn on all the quotas
 104         if $XFS_INFO_PROG "${TEST_DIR}" | grep -q 'crc=1'; then
 105                 # v5 filesystems can have group & project quotas
 106                 quota="usrquota,grpquota,prjquota"
 107         else
 108                 # v4 filesystems cannot mix group & project quotas
 109                 quota="usrquota,grpquota"
 110         fi
 111
 112         # Inject our quota mount options
 113         if echo "${MOUNT_OPTIONS}" | grep -q "${quota}"; then
 114                 return
 115         elif echo "${MOUNT_OPTIONS}" | egrep -q '(quota|noenforce)'; then
 116                 _qmount_option "${quota}"
 117         else
 118                 export MOUNT_OPTIONS="$MOUNT_OPTIONS -o ${quota}"
 119                 echo "MOUNT_OPTIONS = $MOUNT_OPTIONS" >>$seqres.full
 120         fi
 121 }
 122
 123 # Populate an XFS on the scratch device with (we hope) all known
 124 # types of metadata block
 125 _scratch_xfs_populate() {
 126         fill=1
 127
 128         for arg in $@; do
 129                 case "${arg}" in
 130                 "nofill")
 131                         fill=0;;
 132                 esac
 133         done
 134
 135         _populate_xfs_qmount_option
 136         _scratch_mount
 137         blksz="$(stat -f -c '%s' "${SCRATCH_MNT}")"
 138         dblksz="$($XFS_INFO_PROG "${SCRATCH_MNT}" | grep naming.*bsize | sed -e 's/^.*bsize=//g' -e 's/\([0-9]*\).*$/\1/g')"
 139         crc="$($XFS_INFO_PROG "${SCRATCH_MNT}" | grep crc= | sed -e 's/^.*crc=//g' -e 's/\([0-9]*\).*$/\1/g')"
 140         if [ $crc -eq 1 ]; then
 141                 leaf_hdr_size=64
 142         else
 143                 leaf_hdr_size=16
 144         fi
 145         leaf_lblk="$((32 * 1073741824 / blksz))"
 146         node_lblk="$((64 * 1073741824 / blksz))"
 147
 148         # Data:
 149
 150         # Fill up the root inode chunk
 151         echo "+ fill root ino chunk"
 152         seq 1 64 | while read f; do
 153                 $XFS_IO_PROG -f -c "truncate 0" "${SCRATCH_MNT}/dummy${f}"
 154         done
 155
 156         # Regular files
 157         # - FMT_EXTENTS
 158         echo "+ extents file"
 159         $XFS_IO_PROG -f -c "pwrite -S 0x61 0 ${blksz}" "${SCRATCH_MNT}/S_IFREG.FMT_EXTENTS"
 160
 161         # - FMT_BTREE
 162         echo "+ btree extents file"
 163         nr="$((blksz * 2 / 16))"
 164         $XFS_IO_PROG -f -c "pwrite -S 0x62 0 $((blksz * nr))" "${SCRATCH_MNT}/S_IFREG.FMT_BTREE"
 165         ./src/punch-alternating "${SCRATCH_MNT}/S_IFREG.FMT_BTREE"
 166
 167         # Directories
 168         # - INLINE
 169         echo "+ inline dir"
 170         __populate_create_dir "${SCRATCH_MNT}/S_IFDIR.FMT_INLINE" 1
 171
 172         # - BLOCK
 173         echo "+ block dir"
 174         __populate_create_dir "${SCRATCH_MNT}/S_IFDIR.FMT_BLOCK" "$((dblksz / 40))"
 175
 176         # - LEAF
 177         echo "+ leaf dir"
 178         __populate_create_dir "${SCRATCH_MNT}/S_IFDIR.FMT_LEAF" "$((dblksz / 12))"
 179
 180         # - LEAFN
 181         echo "+ leafn dir"
 182         __populate_create_dir "${SCRATCH_MNT}/S_IFDIR.FMT_LEAFN" "$(( ((dblksz - leaf_hdr_size) / 8) - 3 ))"
 183
 184         # - NODE
 185         echo "+ node dir"
 186         __populate_create_dir "${SCRATCH_MNT}/S_IFDIR.FMT_NODE" "$((16 * dblksz / 40))" true
 187
 188         # - BTREE
 189         echo "+ btree dir"
 190         __populate_create_dir "${SCRATCH_MNT}/S_IFDIR.FMT_BTREE" "$((128 * dblksz / 40))" true
 191
 192         # Symlinks
 193         # - FMT_LOCAL
 194         echo "+ inline symlink"
 195         ln -s target "${SCRATCH_MNT}/S_IFLNK.FMT_LOCAL"
 196
 197         # - FMT_EXTENTS
 198         echo "+ extents symlink"
 199         ln -s "$(perl -e 'print "x" x 1023;')" "${SCRATCH_MNT}/S_IFLNK.FMT_EXTENTS"
 200
 201         # Char & block
 202         echo "+ special"
 203         mknod "${SCRATCH_MNT}/S_IFCHR" c 1 1
 204         mknod "${SCRATCH_MNT}/S_IFBLK" c 1 1
 205
 206         # special file with an xattr
 207         setfacl -P -m u:nobody:r ${SCRATCH_MNT}/S_IFCHR
 208
 209         # Attribute formats
 210         # LOCAL
 211         echo "+ local attr"
 212         __populate_create_attr "${SCRATCH_MNT}/ATTR.FMT_LOCAL" 1
 213
 214         # LEAF
 215         echo "+ leaf attr"
 216         __populate_create_attr "${SCRATCH_MNT}/ATTR.FMT_LEAF" "$((blksz / 40))"
 217
 218         # NODE
 219         echo "+ node attr"
 220         __populate_create_attr "${SCRATCH_MNT}/ATTR.FMT_NODE" "$((8 * blksz / 40))"
 221
 222         # BTREE
 223         echo "+ btree attr"
 224         __populate_create_attr "${SCRATCH_MNT}/ATTR.FMT_BTREE" "$((64 * blksz / 40))" true
 225
 226         # trusted namespace
 227         touch ${SCRATCH_MNT}/ATTR.TRUSTED
 228         setfattr -n trusted.moo -v urk ${SCRATCH_MNT}/ATTR.TRUSTED
 229
 230         # security namespace
 231         touch ${SCRATCH_MNT}/ATTR.SECURITY
 232         setfattr -n security.foo -v bar ${SCRATCH_MNT}/ATTR.SECURITY
 233
 234         # system namespace
 235         touch ${SCRATCH_MNT}/ATTR.SYSTEM
 236         setfacl -m u:root:r ${SCRATCH_MNT}/ATTR.SYSTEM
 237
 238         # FMT_EXTENTS with a remote less-than-a-block value
 239         echo "+ attr extents with a remote less-than-a-block value"
 240         touch "${SCRATCH_MNT}/ATTR.FMT_EXTENTS_REMOTE3K"
 241         $XFS_IO_PROG -f -c "pwrite -S 0x43 0 $((blksz - 300))" "${SCRATCH_MNT}/attrvalfile" > /dev/null
 242         attr -q -s user.remotebtreeattrname "${SCRATCH_MNT}/ATTR.FMT_EXTENTS_REMOTE3K" < "${SCRATCH_MNT}/attrvalfile"
 243
 244         # FMT_EXTENTS with a remote block-size value
 245         echo "+ attr extents with a remote one-block value"
 246         touch "${SCRATCH_MNT}/ATTR.FMT_EXTENTS_REMOTE4K"
 247         $XFS_IO_PROG -f -c "pwrite -S 0x44 0 ${blksz}" "${SCRATCH_MNT}/attrvalfile" > /dev/null
 248         attr -q -s user.remotebtreeattrname "${SCRATCH_MNT}/ATTR.FMT_EXTENTS_REMOTE4K" < "${SCRATCH_MNT}/attrvalfile"
 249         rm -rf "${SCRATCH_MNT}/attrvalfile"
 250
 251         # Make an unused inode
 252         echo "+ empty file"
 253         touch "${SCRATCH_MNT}/unused"
 254         $XFS_IO_PROG -f -c 'fsync' "${SCRATCH_MNT}/unused"
 255         rm -rf "${SCRATCH_MNT}/unused"
 256
 257         # Free space btree
 258         echo "+ freesp btree"
 259         nr="$((blksz * 2 / 8))"
 260         $XFS_IO_PROG -f -c "pwrite -S 0x62 0 $((blksz * nr))" "${SCRATCH_MNT}/BNOBT"
 261         ./src/punch-alternating "${SCRATCH_MNT}/BNOBT"
 262
 263         # Inode btree
 264         echo "+ inobt btree"
 265         local ino_per_rec=64
 266         local rec_per_btblock=16
 267         local nr="$(( 2 * (blksz / rec_per_btblock) * ino_per_rec ))"
 268         local dir="${SCRATCH_MNT}/INOBT"
 269         mkdir -p "${dir}"
 270         seq 0 "${nr}" | while read f; do
 271                 touch "${dir}/${f}"
 272         done
 273
 274         seq 0 "$((ino_per_rec + 1))" "${nr}" | while read f; do
 275                 rm -f "${dir}/${f}"
 276         done
 277
 278         # Reverse-mapping btree
 279         is_rmapbt="$($XFS_INFO_PROG "${SCRATCH_MNT}" | grep -c 'rmapbt=1')"
 280         if [ $is_rmapbt -gt 0 ]; then
 281                 echo "+ rmapbt btree"
 282                 nr="$((blksz * 2 / 24))"
 283                 $XFS_IO_PROG -f -c "pwrite -S 0x62 0 $((blksz * nr))" "${SCRATCH_MNT}/RMAPBT"
 284                 ./src/punch-alternating "${SCRATCH_MNT}/RMAPBT"
 285         fi
 286
 287         # Realtime Reverse-mapping btree
 288         is_rt="$($XFS_INFO_PROG "${SCRATCH_MNT}" | grep -c 'rtextents=[1-9]')"
 289         if [ $is_rmapbt -gt 0 ] && [ $is_rt -gt 0 ]; then
 290                 echo "+ rtrmapbt btree"
 291                 nr="$((blksz * 2 / 32))"
 292                 $XFS_IO_PROG -f -R -c "pwrite -S 0x62 0 $((blksz * nr))" "${SCRATCH_MNT}/RTRMAPBT"
 293                 ./src/punch-alternating "${SCRATCH_MNT}/RTRMAPBT"
 294         fi
 295
 296         # Reference-count btree
 297         is_reflink="$($XFS_INFO_PROG "${SCRATCH_MNT}" | grep -c 'reflink=1')"
 298         if [ $is_reflink -gt 0 ]; then
 299                 echo "+ reflink btree"
 300                 nr="$((blksz * 2 / 12))"
 301                 $XFS_IO_PROG -f -c "pwrite -S 0x62 0 $((blksz * nr))" "${SCRATCH_MNT}/REFCOUNTBT"
 302                 cp --reflink=always "${SCRATCH_MNT}/REFCOUNTBT" "${SCRATCH_MNT}/REFCOUNTBT2"
 303                 ./src/punch-alternating "${SCRATCH_MNT}/REFCOUNTBT"
 304         fi
 305
 306         # Copy some real files (xfs tests, I guess...)
 307         echo "+ real files"
 308         test $fill -ne 0 && __populate_fill_fs "${SCRATCH_MNT}" 5
 309
 310         umount "${SCRATCH_MNT}"
 311 }
 312
 313 # Populate an ext4 on the scratch device with (we hope) all known
 314 # types of metadata block
 315 _scratch_ext4_populate() {
 316         fill=1
 317
 318         for arg in $@; do
 319                 case "${arg}" in
 320                 "nofill")
 321                         fill=0;;
 322                 esac
 323         done
 324
 325         _scratch_mount
 326         blksz="$(stat -f -c '%s' "${SCRATCH_MNT}")"
 327         dblksz="${blksz}"
 328         leaf_lblk="$((32 * 1073741824 / blksz))"
 329         node_lblk="$((64 * 1073741824 / blksz))"
 330
 331         # Data:
 332
 333         # Regular files
 334         # - FMT_INLINE
 335         echo "+ inline file"
 336         $XFS_IO_PROG -f -c "pwrite -S 0x61 0 1" "${SCRATCH_MNT}/S_IFREG.FMT_INLINE"
 337
 338         # - FMT_EXTENTS
 339         echo "+ extents file"
 340         $XFS_IO_PROG -f -c "pwrite -S 0x61 0 ${blksz}" "${SCRATCH_MNT}/S_IFREG.FMT_EXTENTS"
 341
 342         # - FMT_ETREE
 343         echo "+ extent tree file"
 344         nr="$((blksz * 2 / 12))"
 345         $XFS_IO_PROG -f -c "pwrite -S 0x62 0 $((blksz * nr))" "${SCRATCH_MNT}/S_IFREG.FMT_ETREE"
 346         ./src/punch-alternating "${SCRATCH_MNT}/S_IFREG.FMT_ETREE"
 347
 348         # Directories
 349         # - INLINE
 350         echo "+ inline dir"
 351         __populate_create_dir "${SCRATCH_MNT}/S_IFDIR.FMT_INLINE" 1
 352
 353         # - BLOCK
 354         echo "+ block dir"
 355         __populate_create_dir "${SCRATCH_MNT}/S_IFDIR.FMT_BLOCK" "$((dblksz / 32))"
 356
 357         # - HTREE
 358         echo "+ htree dir"
 359         __populate_create_dir "${SCRATCH_MNT}/S_IFDIR.FMT_HTREE" "$((4 * dblksz / 24))"
 360
 361         # Symlinks
 362         # - FMT_LOCAL
 363         echo "+ inline symlink"
 364         ln -s target "${SCRATCH_MNT}/S_IFLNK.FMT_LOCAL"
 365
 366         # - FMT_EXTENTS
 367         echo "+ extents symlink"
 368         ln -s "$(perl -e 'print "x" x 1023;')" "${SCRATCH_MNT}/S_IFLNK.FMT_EXTENTS"
 369
 370         # Char & block
 371         echo "+ special"
 372         mknod "${SCRATCH_MNT}/S_IFCHR" c 1 1
 373         mknod "${SCRATCH_MNT}/S_IFBLK" c 1 1
 374
 375         # special file with an xattr
 376         setfacl -P -m u:nobody:r ${SCRATCH_MNT}/S_IFCHR
 377
 378         # Attribute formats
 379         # LOCAL
 380         echo "+ local attr"
 381         __populate_create_attr "${SCRATCH_MNT}/ATTR.FMT_LOCAL" 0
 382
 383         # BLOCK
 384         echo "+ block attr"
 385         __populate_create_attr "${SCRATCH_MNT}/ATTR.FMT_BLOCK" "$((blksz / 40))"
 386
 387         # trusted namespace
 388         touch ${SCRATCH_MNT}/ATTR.TRUSTED
 389         setfattr -n trusted.moo -v urk ${SCRATCH_MNT}/ATTR.TRUSTED
 390
 391         # security namespace
 392         touch ${SCRATCH_MNT}/ATTR.SECURITY
 393         setfattr -n security.foo -v bar ${SCRATCH_MNT}/ATTR.SECURITY
 394
 395         # system namespace
 396         touch ${SCRATCH_MNT}/ATTR.SYSTEM
 397         setfacl -m u:root:r ${SCRATCH_MNT}/ATTR.SYSTEM
 398
 399         # Make an unused inode
 400         echo "+ empty file"
 401         touch "${SCRATCH_MNT}/unused"
 402         $XFS_IO_PROG -f -c 'fsync' "${SCRATCH_MNT}/unused"
 403         rm -rf "${SCRATCH_MNT}/unused"
 404
 405         # Copy some real files (xfs tests, I guess...)
 406         echo "+ real files"
 407         test $fill -ne 0 && __populate_fill_fs "${SCRATCH_MNT}" 5
 408
 409         umount "${SCRATCH_MNT}"
 410 }
 411
 412 # Find the inode number of a file
 413 __populate_find_inode() {
 414         name="$1"
 415         inode="$(stat -c '%i' "${name}")"
 416         echo "${inode}"
 417 }
 418
 419 # Check data fork format of XFS file
 420 __populate_check_xfs_dformat() {
 421         inode="$1"
 422         format="$2"
 423
 424         fmt="$(_scratch_xfs_db -c "inode ${inode}" -c 'p core.format' | sed -e 's/^.*(\([a-z]*\)).*$/\1/g')"
 425         test "${format}" = "${fmt}" || _fail "failed to create ino ${inode} dformat expected ${format} saw ${fmt}"
 426 }
 427
 428 # Check attr fork format of XFS file
 429 __populate_check_xfs_aformat() {
 430         inode="$1"
 431         format="$2"
 432
 433         fmt="$(_scratch_xfs_db -c "inode ${inode}" -c 'p core.aformat' | sed -e 's/^.*(\([a-z]*\)).*$/\1/g')"
 434         test "${format}" = "${fmt}" || _fail "failed to create ino ${inode} aformat expected ${format} saw ${fmt}"
 435 }
 436
 437 # Check structure of XFS directory
 438 __populate_check_xfs_dir() {
 439         inode="$1"
 440         dtype="$2"
 441
 442         (test -n "${leaf_lblk}" && test -n "${node_lblk}") || _fail "must define leaf_lblk and node_lblk before calling __populate_check_xfs_dir"
 443         datab=0
 444         leafb=0
 445         freeb=0
 446         #echo "== check dir ${inode} type ${dtype}" ; _scratch_xfs_db -x -c "inode ${inode}" -c "bmap"
 447         _scratch_xfs_db -x -c "inode ${inode}" -c "dblock 0" -c "stack" | grep -q 'file data block is unmapped' || datab=1
 448         _scratch_xfs_db -x -c "inode ${inode}" -c "dblock ${leaf_lblk}" -c "stack" | grep -q 'file data block is unmapped' || leafb=1
 449         _scratch_xfs_db -x -c "inode ${inode}" -c "dblock ${node_lblk}" -c "stack" | grep -q 'file data block is unmapped' || freeb=1
 450
 451         case "${dtype}" in
 452         "shortform"|"inline"|"local")
 453                 (test "${datab}" -eq 0 && test "${leafb}" -eq 0 && test "${freeb}" -eq 0) || _fail "failed to create ${dtype} dir ino ${inode} datab ${datab} leafb ${leafb} freeb ${freeb}"
 454                 ;;
 455         "block")
 456                 (test "${datab}" -eq 1 && test "${leafb}" -eq 0 && test "${freeb}" -eq 0) || _fail "failed to create ${dtype} dir ino ${inode} datab ${datab} leafb ${leafb} freeb ${freeb}"
 457                 ;;
 458         "leaf")
 459                 (test "${datab}" -eq 1 && test "${leafb}" -eq 1 && test "${freeb}" -eq 0) || _fail "failed to create ${dtype} dir ino ${inode} datab ${datab} leafb ${leafb} freeb ${freeb}"
 460                 ;;
 461         "leafn")
 462                 _scratch_xfs_db -x -c "inode ${inode}" -c "dblock ${leaf_lblk}" -c "p lhdr.info.hdr.magic" | grep -q '0x3dff' && return
 463                 _scratch_xfs_db -x -c "inode ${inode}" -c "dblock ${leaf_lblk}" -c "p lhdr.info.magic" | grep -q '0xd2ff' && return
 464                 _fail "failed to create ${dtype} dir ino ${inode} datab ${datab} leafb ${leafb} freeb ${freeb}"
 465                 ;;
 466         "node"|"btree")
 467                 (test "${datab}" -eq 1 && test "${leafb}" -eq 1 && test "${freeb}" -eq 1) || _fail "failed to create ${dtype} dir ino ${inode} datab ${datab} leafb ${leafb} freeb ${freeb}"
 468                 ;;
 469         *)
 470                 _fail "Unknown directory type ${dtype}"
 471         esac
 472 }
 473
 474 # Check structure of XFS attr
 475 __populate_check_xfs_attr() {
 476         inode="$1"
 477         atype="$2"
 478
 479         datab=0
 480         leafb=0
 481         #echo "== check attr ${inode} type ${dtype}" ; _scratch_xfs_db -x -c "inode ${inode}" -c "bmap -a"
 482         _scratch_xfs_db -x -c "inode ${inode}" -c "ablock 0" -c "stack" | grep -q 'file attr block is unmapped' || datab=1
 483         _scratch_xfs_db -x -c "inode ${inode}" -c "ablock 1" -c "stack" | grep -q 'file attr block is unmapped' || leafb=1
 484
 485         case "${atype}" in
 486         "shortform"|"inline"|"local")
 487                 (test "${datab}" -eq 0 && test "${leafb}" -eq 0) || _fail "failed to create ${atype} attr ino ${inode} datab ${datab} leafb ${leafb}"
 488                 ;;
 489         "leaf")
 490                 (test "${datab}" -eq 1 && test "${leafb}" -eq 0) || _fail "failed to create ${atype} attr ino ${inode} datab ${datab} leafb ${leafb}"
 491                 ;;
 492         "node"|"btree")
 493                 (test "${datab}" -eq 1 && test "${leafb}" -eq 1) || _fail "failed to create ${atype} attr ino ${inode} datab ${datab} leafb ${leafb}"
 494                 ;;
 495         *)
 496                 _fail "Unknown attribute type ${atype}"
 497         esac
 498 }
 499
 500 # Check that there's at least one per-AG btree with multiple levels
 501 __populate_check_xfs_agbtree_height() {
 502         bt_type="$1"
 503         nr_ags=$(_scratch_xfs_db -c 'sb 0' -c 'p agcount' | awk '{print $3}')
 504
 505         case "${bt_type}" in
 506         "bno"|"cnt"|"rmap"|"refcnt")
 507                 hdr="agf"
 508                 bt_prefix="${bt_type}"
 509                 ;;
 510         "ino")
 511                 hdr="agi"
 512                 bt_prefix=""
 513                 ;;
 514         "fino")
 515                 hdr="agi"
 516                 bt_prefix="free_"
 517                 ;;
 518         *)
 519                 _fail "Don't know about AG btree ${bt_type}"
 520                 ;;
 521         esac
 522
 523         seq 0 $((nr_ags - 1)) | while read ag; do
 524                 bt_level=$(_scratch_xfs_db -c "${hdr} ${ag}" -c "p ${bt_prefix}level" | awk '{print $3}')
 525                 if [ "${bt_level}" -gt 1 ]; then
 526                         return 100
 527                 fi
 528         done
 529         test $? -eq 100 || _fail "Failed to create ${bt_type} of sufficient height!"
 530         return 1
 531 }
 532
 533 # Check that populate created all the types of files we wanted
 534 _scratch_xfs_populate_check() {
 535         _scratch_mount
 536         extents_file="$(__populate_find_inode "${SCRATCH_MNT}/S_IFREG.FMT_EXTENTS")"
 537         btree_file="$(__populate_find_inode "${SCRATCH_MNT}/S_IFREG.FMT_BTREE")"
 538         inline_dir="$(__populate_find_inode "${SCRATCH_MNT}/S_IFDIR.FMT_INLINE")"
 539         block_dir="$(__populate_find_inode "${SCRATCH_MNT}/S_IFDIR.FMT_BLOCK")"
 540         leaf_dir="$(__populate_find_inode "${SCRATCH_MNT}/S_IFDIR.FMT_LEAF")"
 541         leafn_dir="$(__populate_find_inode "${SCRATCH_MNT}/S_IFDIR.FMT_LEAFN")"
 542         node_dir="$(__populate_find_inode "${SCRATCH_MNT}/S_IFDIR.FMT_NODE")"
 543         btree_dir="$(__populate_find_inode "${SCRATCH_MNT}/S_IFDIR.FMT_BTREE")"
 544         local_slink="$(__populate_find_inode "${SCRATCH_MNT}/S_IFLNK.FMT_LOCAL")"
 545         extents_slink="$(__populate_find_inode "${SCRATCH_MNT}/S_IFLNK.FMT_EXTENTS")"
 546         bdev="$(__populate_find_inode "${SCRATCH_MNT}/S_IFBLK")"
 547         cdev="$(__populate_find_inode "${SCRATCH_MNT}/S_IFCHR")"
 548         local_attr="$(__populate_find_inode "${SCRATCH_MNT}/ATTR.FMT_LOCAL")"
 549         leaf_attr="$(__populate_find_inode "${SCRATCH_MNT}/ATTR.FMT_LEAF")"
 550         node_attr="$(__populate_find_inode "${SCRATCH_MNT}/ATTR.FMT_NODE")"
 551         btree_attr="$(__populate_find_inode "${SCRATCH_MNT}/ATTR.FMT_BTREE")"
 552         is_finobt=$($XFS_INFO_PROG "${SCRATCH_MNT}" | grep -c 'finobt=1')
 553         is_rmapbt=$($XFS_INFO_PROG "${SCRATCH_MNT}" | grep -c 'rmapbt=1')
 554         is_reflink=$($XFS_INFO_PROG "${SCRATCH_MNT}" | grep -c 'reflink=1')
 555
 556         blksz="$(stat -f -c '%s' "${SCRATCH_MNT}")"
 557         dblksz="$($XFS_INFO_PROG "${SCRATCH_MNT}" | grep naming.*bsize | sed -e 's/^.*bsize=//g' -e 's/\([0-9]*\).*$/\1/g')"
 558         leaf_lblk="$((32 * 1073741824 / blksz))"
 559         node_lblk="$((64 * 1073741824 / blksz))"
 560         umount "${SCRATCH_MNT}"
 561
 562         __populate_check_xfs_dformat "${extents_file}" "extents"
 563         __populate_check_xfs_dformat "${btree_file}" "btree"
 564         __populate_check_xfs_dir "${inline_dir}" "inline"
 565         __populate_check_xfs_dir "${block_dir}" "block"
 566         __populate_check_xfs_dir "${leaf_dir}" "leaf"
 567         __populate_check_xfs_dir "${leafn_dir}" "leafn"
 568         __populate_check_xfs_dir "${node_dir}" "node"
 569         __populate_check_xfs_dir "${btree_dir}" "btree"
 570         __populate_check_xfs_dformat "${btree_dir}" "btree"
 571         __populate_check_xfs_dformat "${bdev}" "dev"
 572         __populate_check_xfs_dformat "${cdev}" "dev"
 573         __populate_check_xfs_attr "${local_attr}" "local"
 574         __populate_check_xfs_attr "${leaf_attr}" "leaf"
 575         __populate_check_xfs_attr "${node_attr}" "node"
 576         __populate_check_xfs_attr "${btree_attr}" "btree"
 577         __populate_check_xfs_aformat "${btree_attr}" "btree"
 578         __populate_check_xfs_agbtree_height "bno"
 579         __populate_check_xfs_agbtree_height "cnt"
 580         __populate_check_xfs_agbtree_height "ino"
 581         test $is_finobt -ne 0 && __populate_check_xfs_agbtree_height "fino"
 582         test $is_rmapbt -ne 0 && __populate_check_xfs_agbtree_height "rmap"
 583         test $is_reflink -ne 0 && __populate_check_xfs_agbtree_height "refcnt"
 584 }
 585
 586 # Check data fork format of ext4 file
 587 __populate_check_ext4_dformat() {
 588         dev="${SCRATCH_DEV}"
 589         inode="$1"
 590         format="$2"
 591
 592         extents=0
 593         etree=0
 594         debugfs -R "stat <${inode}>" "${dev}" 2> /dev/null | grep 'ETB[0-9]' -q && etree=1
 595         iflags="$(debugfs -R "stat <${inode}>" "${dev}" 2> /dev/null | grep 'Flags:' | sed -e 's/^.*Flags: \([0-9a-fx]*\).*$/\1/g')"
 596         test "$(echo "${iflags}" | awk '{print and(strtonum($1), 0x80000);}')" -gt 0 && extents=1
 597
 598         case "${format}" in
 599         "blockmap")
 600                 test "${extents}" -eq 0 || _fail "failed to create ino ${inode} with blockmap"
 601                 ;;
 602         "extent"|"extents")
 603                 test "${extents}" -eq 1 || _fail "failed to create ino ${inode} with extents"
 604                 ;;
 605         "etree")
 606                 (test "${extents}" -eq 1 && test "${etree}" -eq 1) || _fail "failed to create ino ${inode} with extent tree"
 607                 ;;
 608         *)
 609                 _fail "Unknown dformat ${format}"
 610         esac
 611 }
 612
 613 # Check attr fork format of ext4 file
 614 __populate_check_ext4_aformat() {
 615         dev="${SCRATCH_DEV}"
 616         inode="$1"
 617         format="$2"
 618
 619         ablock=1
 620         debugfs -R "stat <${inode}>" "${dev}" 2> /dev/null | grep 'File ACL: 0' -q && ablock=0
 621
 622         case "${format}" in
 623         "local"|"inline")
 624                 test "${ablock}" -eq 0 || _fail "failed to create inode ${inode} with ${format} xattr"
 625                 ;;
 626         "block")
 627                 test "${extents}" -eq 1 || _fail "failed to create inode ${inode} with ${format} xattr"
 628                 ;;
 629         *)
 630                 _fail "Unknown aformat ${format}"
 631         esac
 632 }
 633
 634 # Check structure of ext4 dir
 635 __populate_check_ext4_dir() {
 636         dev="${SCRATCH_DEV}"
 637         inode="$1"
 638         dtype="$2"
 639
 640         htree=0
 641         inline=0
 642         iflags="$(debugfs -R "stat <${inode}>" "${dev}" 2> /dev/null | grep 'Flags:' | sed -e 's/^.*Flags: \([0-9a-fx]*\).*$/\1/g')"
 643         test "$(echo "${iflags}" | awk '{print and(strtonum($1), 0x1000);}')" -gt 0 && htree=1
 644         test "$(echo "${iflags}" | awk '{print and(strtonum($1), 0x10000000);}')" -gt 0 && inline=1
 645
 646         case "${dtype}" in
 647         "inline")
 648                 (test "${inline}" -eq 1 && test "${htree}" -eq 0) || _fail "failed to create ${dtype} dir ino ${inode} htree ${htree} inline ${inline}"
 649                 ;;
 650         "block")
 651                 (test "${inline}" -eq 0 && test "${htree}" -eq 0) || _fail "failed to create ${dtype} dir ino ${inode} htree ${htree} inline ${inline}"
 652                 ;;
 653         "htree")
 654                 (test "${inline}" -eq 0 && test "${htree}" -eq 1) || _fail "failed to create ${dtype} dir ino ${inode} htree ${htree} inline ${inline}"
 655                 ;;
 656         *)
 657                 _fail "Unknown directory type ${dtype}"
 658                 ;;
 659         esac
 660 }
 661
 662 # Check that populate created all the types of files we wanted
 663 _scratch_ext4_populate_check() {
 664         _scratch_mount
 665         extents_file="$(__populate_find_inode "${SCRATCH_MNT}/S_IFREG.FMT_EXTENTS")"
 666         etree_file="$(__populate_find_inode "${SCRATCH_MNT}/S_IFREG.FMT_ETREE")"
 667         block_dir="$(__populate_find_inode "${SCRATCH_MNT}/S_IFDIR.FMT_BLOCK")"
 668         htree_dir="$(__populate_find_inode "${SCRATCH_MNT}/S_IFDIR.FMT_HTREE")"
 669         extents_slink="$(__populate_find_inode "${SCRATCH_MNT}/S_IFLNK.FMT_EXTENTS")"
 670         local_attr="$(__populate_find_inode "${SCRATCH_MNT}/ATTR.FMT_LOCAL")"
 671         block_attr="$(__populate_find_inode "${SCRATCH_MNT}/ATTR.FMT_BLOCK")"
 672         umount "${SCRATCH_MNT}"
 673
 674         __populate_check_ext4_dformat "${extents_file}" "extents"
 675         __populate_check_ext4_dformat "${etree_file}" "etree"
 676         __populate_check_ext4_dir "${block_dir}" "block"
 677         __populate_check_ext4_dir "${htree_dir}" "htree"
 678         __populate_check_ext4_dformat "${extents_slink}" "extents"
 679         __populate_check_ext4_aformat "${local_attr}" "local"
 680         __populate_check_ext4_aformat "${block_attr}" "block"
 681 }
 682
 683 # Populate a scratch FS and check the contents to make sure we got that
 684 _scratch_populate() {
 685         case "${FSTYP}" in
 686         "xfs")
 687                 _scratch_xfs_populate
 688                 _scratch_xfs_populate_check
 689                 ;;
 690         "ext2"|"ext3"|"ext4")
 691                 _scratch_ext4_populate
 692                 _scratch_ext4_populate_check
 693                 ;;
 694         *)
 695                 _fail "Don't know how to populate a ${FSTYP} filesystem."
 696                 ;;
 697         esac
 698 }
 699
 700 # Fill a file system by repeatedly creating files in the given folder
 701 # starting with the given file size.  Files are reduced in size when
 702 # they can no longer fit until no more files can be created.
 703 _fill_fs()
 704 {
 705         local file_size=$1
 706         local dir=$2
 707         local block_size=$3
 708         local switch_user=$4
 709         local file_count=1
 710         local bytes_written=0
 711         local use_falloc=1;
 712
 713         if [ $# -ne 4 ]; then
 714                 echo "Usage: _fill_fs filesize dir blocksize switch_user"
 715                 exit 1
 716         fi
 717
 718         if [ $switch_user -eq 0 ]; then
 719                 mkdir -p $dir
 720         else
 721                 _user_do "mkdir -p $dir"
 722         fi
 723         if [ ! -d $dir ]; then
 724                 return 0;
 725         fi
 726
 727         testio=`$XFS_IO_PROG -F -fc "falloc 0 $block_size" $dir/$$.xfs_io 2>&1`
 728         echo $testio | grep -q "not found" && use_falloc=0
 729         echo $testio | grep -q "Operation not supported" && use_falloc=0
 730
 731         if [ $file_size -lt $block_size ]; then
 732                 $file_size = $block_size
 733         fi
 734
 735         while [ $file_size -ge $block_size ]; do
 736                 bytes_written=0
 737                 if [ $switch_user -eq 0 ]; then
 738                         if [ $use_falloc -eq 0 ]; then
 739                                 $XFS_IO_PROG -fc "pwrite -b 8388608 0 $file_size" \
 740                                         $dir/$file_count
 741                         else
 742                                 $XFS_IO_PROG -fc "falloc 0 $file_size" \
 743                                         $dir/$file_count
 744                         fi
 745                 else
 746                         if [ $use_falloc -eq 0 ]; then
 747                                 _user_do "$XFS_IO_PROG -f -c \"pwrite -b 8388608 0 \
 748                                         $file_size\" $dir/$file_count"
 749                         else
 750                                 _user_do "$XFS_IO_PROG -f -c \"falloc 0 \
 751                                         $file_size\" $dir/$file_count"
 752                         fi
 753                 fi
 754
 755                 if [ -f $dir/$file_count ]; then
 756                         bytes_written=$(stat -c '%s' $dir/$file_count)
 757                 fi
 758
 759                 # If there was no room to make the file, then divide it in
 760                 # half, and keep going
 761                 if [ $bytes_written -lt $file_size ]; then
 762                         file_size=$((file_size / 2))
 763                 fi
 764                 file_count=$((file_count + 1))
 765         done
 766 }
 767
 768 # Compute the fs geometry description of a populated filesystem
 769 _scratch_populate_cache_tag() {
 770         local extra_descr=""
 771         local size="$(blockdev --getsz "${SCRATCH_DEV}")"
 772
 773         case "${FSTYP}" in
 774         "ext4")
 775                 extra_descr="LOGDEV ${SCRATCH_LOGDEV} USE_EXTERNAL ${USE_EXTERNAL}"
 776                 ;;
 777         "xfs")
 778                 extra_descr="LOGDEV ${SCRATCH_LOGDEV} USE_EXTERNAL ${USE_EXTERNAL} RTDEV ${SCRATCH_RTDEV}"
 779                 _populate_xfs_qmount_option
 780                 if echo "${MOUNT_OPTIONS}" | grep -q 'usrquota'; then
 781                         extra_descr="${extra_descr} QUOTAS"
 782                 fi
 783                 ;;
 784         esac
 785         echo "FSTYP ${FSTYP} MKFS_OPTIONS ${MKFS_OPTIONS} SIZE ${size} ${extra_descr} ARGS $@"
 786 }
 787
 788 # Restore a cached populated fs from a metadata dump
 789 _scratch_populate_restore_cached() {
 790         local metadump="$1"
 791
 792         case "${FSTYP}" in
 793         "xfs")
 794                 xfs_mdrestore "${metadump}" "${SCRATCH_DEV}" && return 0
 795                 ;;
 796         "ext2"|"ext3"|"ext4")
 797                 # ext4 cannot e2image external logs, so we cannot restore
 798                 test -n "${SCRATCH_LOGDEV}" && return 1
 799                 e2image -r "${metadump}" "${SCRATCH_DEV}" && return 0
 800                 ;;
 801         esac
 802         return 1
 803 }
 804
 805 # Populate a scratch FS from scratch or from a cached image.
 806 _scratch_populate_cached() {
 807         local meta_descr="$(_scratch_populate_cache_tag "$@")"
 808
 809         # These variables are shared outside this function
 810         POPULATE_METADUMP="${TEST_DIR}/__populate.${FSTYP}"
 811         POPULATE_METADUMP_DESCR="${TEST_DIR}/__populate.${FSTYP}.txt"
 812
 813         # Don't keep metadata images cached for more 48 hours...
 814         rm -rf "$(find "${POPULATE_METADUMP}" -mtime +2 2>/dev/null)"
 815
 816         # Throw away cached image if it doesn't match our spec.
 817         cmp -s "${POPULATE_METADUMP_DESCR}" <(echo "${meta_descr}") || \
 818                 rm -rf "${POPULATE_METADUMP}"
 819
 820         # Try to restore from the metadump
 821         test -r "${POPULATE_METADUMP}" && \
 822                 _scratch_populate_restore_cached "${POPULATE_METADUMP}" && \
 823                 return
 824
 825         # Oh well, just create one from scratch
 826         _scratch_mkfs
 827         echo "${meta_descr}" > "${POPULATE_METADUMP_DESCR}"
 828         case "${FSTYP}" in
 829         "xfs")
 830                 _scratch_xfs_populate $@
 831                 _scratch_xfs_populate_check
 832                 _scratch_metadump "${POPULATE_METADUMP}" -a -o
 833                 ;;
 834         "ext2"|"ext3"|"ext4")
 835                 _scratch_ext4_populate $@
 836                 _scratch_ext4_populate_check
 837                 e2image -Q "${SCRATCH_DEV}" "${POPULATE_METADUMP}"
 838                 ;;
 839         *)
 840                 _fail "Don't know how to populate a ${FSTYP} filesystem."
 841                 ;;
 842         esac
 843 }