common/populate

   1 ##/bin/bash
   2 # Routines for populating a scratch fs, and helpers to exercise an FS
   3 # once it's been fuzzed.
   4 #-----------------------------------------------------------------------
   5 #  Copyright (c) 2015 Oracle.  All Rights Reserved.
   6 #  This program is free software; you can redistribute it and/or modify
   7 #  it under the terms of the GNU General Public License as published by
   8 #  the Free Software Foundation; either version 2 of the License, or
   9 #  (at your option) any later version.
  10 #
  11 #  This program is distributed in the hope that it will be useful,
  12 #  but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 #  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14 #  GNU General Public License for more details.
  15 #
  16 #  You should have received a copy of the GNU General Public License
  17 #  along with this program; if not, write to the Free Software
  18 #  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307
  19 #  USA
  20 #
  21 #  Contact information: Silicon Graphics, Inc., 1500 Crittenden Lane,
  22 #  Mountain View, CA 94043, USA, or: http://www.sgi.com
  23 #-----------------------------------------------------------------------
  24
  25 _require_populate_commands() {
  26         _require_xfs_io_command "falloc"
  27         _require_xfs_io_command "fpunch"
  28         _require_test_program "punch-alternating"
  29         _require_command "$XFS_DB_PROG" "xfs_db"
  30 }
  31
  32 _require_xfs_db_blocktrash_z_command() {
  33         test "${FSTYP}" = "xfs" || _notrun "cannot run xfs_db on ${FSTYP}"
  34         $XFS_DB_PROG -x -f -c 'blocktrash -z' "${TEST_DEV}" | grep -q 'nothing on stack' || _notrun "blocktrash -z not supported"
  35 }
  36
  37 # Attempt to make files of "every" format for data, dirs, attrs etc.
  38 # (with apologies to Eric Sandeen for mutating xfser.sh)
  39
  40 # Create a large directory
  41 __populate_create_dir() {
  42         name="$1"
  43         nr="$2"
  44         missing="$3"
  45
  46         mkdir -p "${name}"
  47         seq 0 "${nr}" | while read d; do
  48                 creat=mkdir
  49                 test "$((d % 20))" -eq 0 && creat=touch
  50                 $creat "${name}/$(printf "%.08d" "$d")"
  51         done
  52
  53         test -z "${missing}" && return
  54         seq 1 2 "${nr}" | while read d; do
  55                 rm -rf "${name}/$(printf "%.08d" "$d")"
  56         done
  57 }
  58
  59 # Add a bunch of attrs to a file
  60 __populate_create_attr() {
  61         name="$1"
  62         nr="$2"
  63         missing="$3"
  64
  65         touch "${name}"
  66         seq 0 "${nr}" | while read d; do
  67                 setfattr -n "user.$(printf "%.08d" "$d")" -v "$(printf "%.08d" "$d")" "${name}"
  68         done
  69
  70         test -z "${missing}" && return
  71         seq 1 2 "${nr}" | while read d; do
  72                 setfattr -x "user.$(printf "%.08d" "$d")" "${name}"
  73         done
  74 }
  75
  76 # Fill up some percentage of the remaining free space
  77 __populate_fill_fs() {
  78         dir="$1"
  79         pct="$2"
  80         test -z "${pct}" && pct=60
  81
  82         mkdir -p "${dir}/test/1"
  83         cp -pRdu "${dir}"/S_IFREG* "${dir}/test/1/"
  84
  85         SRC_SZ="$(du -ks "${dir}/test/1" | cut -f 1)"
  86         FS_SZ="$(( $(stat -f "${dir}" -c '%a * %S') / 1024 ))"
  87
  88         NR="$(( (FS_SZ * ${pct} / 100) / SRC_SZ ))"
  89
  90         echo "FILL FS"
  91         echo "src_sz $SRC_SZ fs_sz $FS_SZ nr $NR"
  92         seq 2 "${NR}" | while read nr; do
  93                 cp -pRdu "${dir}/test/1" "${dir}/test/${nr}"
  94         done
  95 }
  96
  97 # Populate an XFS on the scratch device with (we hope) all known
  98 # types of metadata block
  99 _scratch_xfs_populate() {
 100         fill=1
 101
 102         for arg in $@; do
 103                 case "${arg}" in
 104                 "nofill")
 105                         fill=0;;
 106                 esac
 107         done
 108
 109         _scratch_mount
 110         blksz="$(stat -f -c '%s' "${SCRATCH_MNT}")"
 111         dblksz="$(xfs_info "${SCRATCH_MNT}" | grep naming.*bsize | sed -e 's/^.*bsize=//g' -e 's/\([0-9]*\).*$/\1/g')"
 112         leaf_lblk="$((32 * 1073741824 / blksz))"
 113         node_lblk="$((64 * 1073741824 / blksz))"
 114
 115         # Data:
 116
 117         # Fill up the root inode chunk
 118         echo "+ fill root ino chunk"
 119         seq 1 64 | while read f; do
 120                 $XFS_IO_PROG -f -c "truncate 0" "${SCRATCH_MNT}/dummy${f}"
 121         done
 122
 123         # Regular files
 124         # - FMT_EXTENTS
 125         echo "+ extents file"
 126         $XFS_IO_PROG -f -c "pwrite -S 0x61 0 ${blksz}" "${SCRATCH_MNT}/S_IFREG.FMT_EXTENTS"
 127
 128         # - FMT_BTREE
 129         echo "+ btree extents file"
 130         nr="$((blksz * 2 / 16))"
 131         $XFS_IO_PROG -f -c "pwrite -S 0x62 0 $((blksz * nr))" "${SCRATCH_MNT}/S_IFREG.FMT_BTREE"
 132         ./src/punch-alternating "${SCRATCH_MNT}/S_IFREG.FMT_BTREE"
 133
 134         # Directories
 135         # - INLINE
 136         echo "+ inline dir"
 137         __populate_create_dir "${SCRATCH_MNT}/S_IFDIR.FMT_INLINE" 1
 138
 139         # - BLOCK
 140         echo "+ block dir"
 141         __populate_create_dir "${SCRATCH_MNT}/S_IFDIR.FMT_BLOCK" "$((dblksz / 40))"
 142
 143         # - LEAF
 144         echo "+ leaf dir"
 145         __populate_create_dir "${SCRATCH_MNT}/S_IFDIR.FMT_LEAF" "$((dblksz / 12))"
 146
 147         # - NODE
 148         echo "+ node dir"
 149         __populate_create_dir "${SCRATCH_MNT}/S_IFDIR.FMT_NODE" "$((16 * dblksz / 40))" true
 150
 151         # - BTREE
 152         echo "+ btree dir"
 153         __populate_create_dir "${SCRATCH_MNT}/S_IFDIR.FMT_BTREE" "$((128 * dblksz / 40))" true
 154
 155         # Symlinks
 156         # - FMT_LOCAL
 157         echo "+ inline symlink"
 158         ln -s target "${SCRATCH_MNT}/S_IFLNK.FMT_LOCAL"
 159
 160         # - FMT_EXTENTS
 161         echo "+ extents symlink"
 162         ln -s "$(perl -e 'print "x" x 1023;')" "${SCRATCH_MNT}/S_IFLNK.FMT_EXTENTS"
 163
 164         # Char & block
 165         echo "+ special"
 166         mknod "${SCRATCH_MNT}/S_IFCHR" c 1 1
 167         mknod "${SCRATCH_MNT}/S_IFBLK" c 1 1
 168
 169         # special file with an xattr
 170         setfacl -P -m u:nobody:r ${SCRATCH_MNT}/S_IFCHR
 171
 172         # Attribute formats
 173         # LOCAL
 174         echo "+ local attr"
 175         __populate_create_attr "${SCRATCH_MNT}/ATTR.FMT_LOCAL" 1
 176
 177         # LEAF
 178         echo "+ leaf attr"
 179         __populate_create_attr "${SCRATCH_MNT}/ATTR.FMT_LEAF" "$((blksz / 40))"
 180
 181         # NODE
 182         echo "+ node attr"
 183         __populate_create_attr "${SCRATCH_MNT}/ATTR.FMT_NODE" "$((8 * blksz / 40))"
 184
 185         # BTREE
 186         echo "+ btree attr"
 187         __populate_create_attr "${SCRATCH_MNT}/ATTR.FMT_BTREE" "$((64 * blksz / 40))" true
 188
 189         # trusted namespace
 190         touch ${SCRATCH_MNT}/ATTR.TRUSTED
 191         setfattr -n trusted.moo -v urk ${SCRATCH_MNT}/ATTR.TRUSTED
 192
 193         # security namespace
 194         touch ${SCRATCH_MNT}/ATTR.SECURITY
 195         setfattr -n security.foo -v bar ${SCRATCH_MNT}/ATTR.SECURITY
 196
 197         # system namespace
 198         touch ${SCRATCH_MNT}/ATTR.SYSTEM
 199         setfacl -m u:root:r ${SCRATCH_MNT}/ATTR.SYSTEM
 200
 201         # FMT_EXTENTS with a remote less-than-a-block value
 202         echo "+ attr extents with a remote less-than-a-block value"
 203         touch "${SCRATCH_MNT}/ATTR.FMT_EXTENTS_REMOTE3K"
 204         $XFS_IO_PROG -f -c "pwrite -S 0x43 0 $((blksz - 300))" "${SCRATCH_MNT}/attrvalfile" > /dev/null
 205         attr -q -s user.remotebtreeattrname "${SCRATCH_MNT}/ATTR.FMT_EXTENTS_REMOTE3K" < "${SCRATCH_MNT}/attrvalfile"
 206
 207         # FMT_EXTENTS with a remote block-size value
 208         echo "+ attr extents with a remote one-block value"
 209         touch "${SCRATCH_MNT}/ATTR.FMT_EXTENTS_REMOTE4K"
 210         $XFS_IO_PROG -f -c "pwrite -S 0x44 0 ${blksz}" "${SCRATCH_MNT}/attrvalfile" > /dev/null
 211         attr -q -s user.remotebtreeattrname "${SCRATCH_MNT}/ATTR.FMT_EXTENTS_REMOTE4K" < "${SCRATCH_MNT}/attrvalfile"
 212         rm -rf "${SCRATCH_MNT}/attrvalfile"
 213
 214         # Make an unused inode
 215         echo "+ empty file"
 216         touch "${SCRATCH_MNT}/unused"
 217         $XFS_IO_PROG -f -c 'fsync' "${SCRATCH_MNT}/unused"
 218         rm -rf "${SCRATCH_MNT}/unused"
 219
 220         # Free space btree
 221         echo "+ freesp btree"
 222         nr="$((blksz * 2 / 8))"
 223         $XFS_IO_PROG -f -c "pwrite -S 0x62 0 $((blksz * nr))" "${SCRATCH_MNT}/BNOBT"
 224         ./src/punch-alternating "${SCRATCH_MNT}/BNOBT"
 225
 226         # Reverse-mapping btree
 227         is_rmapbt="$(xfs_info "${SCRATCH_MNT}" | grep -c 'rmapbt=1')"
 228         if [ $is_rmapbt -gt 0 ]; then
 229                 echo "+ rmapbt btree"
 230                 nr="$((blksz * 2 / 24))"
 231                 $XFS_IO_PROG -f -c "pwrite -S 0x62 0 $((blksz * nr))" "${SCRATCH_MNT}/RMAPBT"
 232                 ./src/punch-alternating "${SCRATCH_MNT}/RMAPBT"
 233         fi
 234
 235         # Realtime Reverse-mapping btree
 236         is_rt="$(xfs_info "${SCRATCH_MNT}" | grep -c 'rtextents=[1-9]')"
 237         if [ $is_rmapbt -gt 0 ] && [ $is_rt -gt 0 ]; then
 238                 echo "+ rtrmapbt btree"
 239                 nr="$((blksz * 2 / 32))"
 240                 $XFS_IO_PROG -f -R -c "pwrite -S 0x62 0 $((blksz * nr))" "${SCRATCH_MNT}/RTRMAPBT"
 241                 ./src/punch-alternating "${SCRATCH_MNT}/RTRMAPBT"
 242         fi
 243
 244         # Reference-count btree
 245         is_reflink="$(xfs_info "${SCRATCH_MNT}" | grep -c 'reflink=1')"
 246         if [ $is_reflink -gt 0 ]; then
 247                 echo "+ reflink btree"
 248                 nr="$((blksz * 2 / 12))"
 249                 $XFS_IO_PROG -f -c "pwrite -S 0x62 0 $((blksz * nr))" "${SCRATCH_MNT}/REFCOUNTBT"
 250                 cp --reflink=always "${SCRATCH_MNT}/REFCOUNTBT" "${SCRATCH_MNT}/REFCOUNTBT2"
 251                 ./src/punch-alternating "${SCRATCH_MNT}/REFCOUNTBT"
 252         fi
 253
 254         # Copy some real files (xfs tests, I guess...)
 255         echo "+ real files"
 256         test $fill -ne 0 && __populate_fill_fs "${SCRATCH_MNT}" 5
 257
 258         umount "${SCRATCH_MNT}"
 259 }
 260
 261 # Populate an ext4 on the scratch device with (we hope) all known
 262 # types of metadata block
 263 _scratch_ext4_populate() {
 264         fill=1
 265
 266         for arg in $@; do
 267                 case "${arg}" in
 268                 "nofill")
 269                         fill=0;;
 270                 esac
 271         done
 272
 273         _scratch_mount
 274         blksz="$(stat -f -c '%s' "${SCRATCH_MNT}")"
 275         dblksz="${blksz}"
 276         leaf_lblk="$((32 * 1073741824 / blksz))"
 277         node_lblk="$((64 * 1073741824 / blksz))"
 278
 279         # Data:
 280
 281         # Regular files
 282         # - FMT_INLINE
 283         echo "+ inline file"
 284         $XFS_IO_PROG -f -c "pwrite -S 0x61 0 1" "${SCRATCH_MNT}/S_IFREG.FMT_INLINE"
 285
 286         # - FMT_EXTENTS
 287         echo "+ extents file"
 288         $XFS_IO_PROG -f -c "pwrite -S 0x61 0 ${blksz}" "${SCRATCH_MNT}/S_IFREG.FMT_EXTENTS"
 289
 290         # - FMT_ETREE
 291         echo "+ extent tree file"
 292         nr="$((blksz * 2 / 12))"
 293         $XFS_IO_PROG -f -c "pwrite -S 0x62 0 $((blksz * nr))" "${SCRATCH_MNT}/S_IFREG.FMT_ETREE"
 294         ./src/punch-alternating "${SCRATCH_MNT}/S_IFREG.FMT_ETREE"
 295
 296         # Directories
 297         # - INLINE
 298         echo "+ inline dir"
 299         __populate_create_dir "${SCRATCH_MNT}/S_IFDIR.FMT_INLINE" 1
 300
 301         # - BLOCK
 302         echo "+ block dir"
 303         __populate_create_dir "${SCRATCH_MNT}/S_IFDIR.FMT_BLOCK" "$((dblksz / 24))"
 304
 305         # - HTREE
 306         echo "+ htree dir"
 307         __populate_create_dir "${SCRATCH_MNT}/S_IFDIR.FMT_HTREE" "$((4 * dblksz / 24))"
 308
 309         # Symlinks
 310         # - FMT_LOCAL
 311         echo "+ inline symlink"
 312         ln -s target "${SCRATCH_MNT}/S_IFLNK.FMT_LOCAL"
 313
 314         # - FMT_EXTENTS
 315         echo "+ extents symlink"
 316         ln -s "$(perl -e 'print "x" x 1023;')" "${SCRATCH_MNT}/S_IFLNK.FMT_EXTENTS"
 317
 318         # Char & block
 319         echo "+ special"
 320         mknod "${SCRATCH_MNT}/S_IFCHR" c 1 1
 321         mknod "${SCRATCH_MNT}/S_IFBLK" c 1 1
 322
 323         # special file with an xattr
 324         setfacl -P -m u:nobody:r ${SCRATCH_MNT}/S_IFCHR
 325
 326         # Attribute formats
 327         # LOCAL
 328         echo "+ local attr"
 329         __populate_create_attr "${SCRATCH_MNT}/ATTR.FMT_LOCAL" 1
 330
 331         # BLOCK
 332         echo "+ block attr"
 333         __populate_create_attr "${SCRATCH_MNT}/ATTR.FMT_BLOCK" "$((blksz / 40))"
 334
 335         # trusted namespace
 336         touch ${SCRATCH_MNT}/ATTR.TRUSTED
 337         setfattr -n trusted.moo -v urk ${SCRATCH_MNT}/ATTR.TRUSTED
 338
 339         # security namespace
 340         touch ${SCRATCH_MNT}/ATTR.SECURITY
 341         setfattr -n security.foo -v bar ${SCRATCH_MNT}/ATTR.SECURITY
 342
 343         # system namespace
 344         touch ${SCRATCH_MNT}/ATTR.SYSTEM
 345         setfacl -m u:root:r ${SCRATCH_MNT}/ATTR.SYSTEM
 346
 347         # Make an unused inode
 348         echo "+ empty file"
 349         touch "${SCRATCH_MNT}/unused"
 350         $XFS_IO_PROG -f -c 'fsync' "${SCRATCH_MNT}/unused"
 351         rm -rf "${SCRATCH_MNT}/unused"
 352
 353         # Copy some real files (xfs tests, I guess...)
 354         echo "+ real files"
 355         test $fill -ne 0 && __populate_fill_fs "${SCRATCH_MNT}" 5
 356
 357         umount "${SCRATCH_MNT}"
 358 }
 359
 360 # Find the inode number of a file
 361 __populate_find_inode() {
 362         name="$1"
 363         inode="$(stat -c '%i' "${name}")"
 364         echo "${inode}"
 365 }
 366
 367 # Check data fork format of XFS file
 368 __populate_check_xfs_dformat() {
 369         inode="$1"
 370         format="$2"
 371
 372         fmt="$(_scratch_xfs_db -c "inode ${inode}" -c 'p core.format' | sed -e 's/^.*(\([a-z]*\)).*$/\1/g')"
 373         test "${format}" = "${fmt}" || _fail "failed to create ino ${inode} dformat expected ${format} saw ${fmt}"
 374 }
 375
 376 # Check attr fork format of XFS file
 377 __populate_check_xfs_aformat() {
 378         inode="$1"
 379         format="$2"
 380
 381         fmt="$(_scratch_xfs_db -c "inode ${inode}" -c 'p core.aformat' | sed -e 's/^.*(\([a-z]*\)).*$/\1/g')"
 382         test "${format}" = "${fmt}" || _fail "failed to create ino ${inode} aformat expected ${format} saw ${fmt}"
 383 }
 384
 385 # Check structure of XFS directory
 386 __populate_check_xfs_dir() {
 387         inode="$1"
 388         dtype="$2"
 389
 390         (test -n "${leaf_lblk}" && test -n "${node_lblk}") || _fail "must define leaf_lblk and node_lblk before calling __populate_check_xfs_dir"
 391         datab=0
 392         leafb=0
 393         freeb=0
 394         #echo "== check dir ${inode} type ${dtype}" ; _scratch_xfs_db -x -c "inode ${inode}" -c "bmap"
 395         _scratch_xfs_db -x -c "inode ${inode}" -c "dblock 0" -c "stack" | grep -q 'file data block is unmapped' || datab=1
 396         _scratch_xfs_db -x -c "inode ${inode}" -c "dblock ${leaf_lblk}" -c "stack" | grep -q 'file data block is unmapped' || leafb=1
 397         _scratch_xfs_db -x -c "inode ${inode}" -c "dblock ${node_lblk}" -c "stack" | grep -q 'file data block is unmapped' || freeb=1
 398
 399         case "${dtype}" in
 400         "shortform"|"inline"|"local")
 401                 (test "${datab}" -eq 0 && test "${leafb}" -eq 0 && test "${freeb}" -eq 0) || _fail "failed to create ${dtype} dir ino ${inode} datab ${datab} leafb ${leafb} freeb ${freeb}"
 402                 ;;
 403         "block")
 404                 (test "${datab}" -eq 1 && test "${leafb}" -eq 0 && test "${freeb}" -eq 0) || _fail "failed to create ${dtype} dir ino ${inode} datab ${datab} leafb ${leafb} freeb ${freeb}"
 405                 ;;
 406         "leaf")
 407                 (test "${datab}" -eq 1 && test "${leafb}" -eq 1 && test "${freeb}" -eq 0) || _fail "failed to create ${dtype} dir ino ${inode} datab ${datab} leafb ${leafb} freeb ${freeb}"
 408                 ;;
 409         "node"|"btree")
 410                 (test "${datab}" -eq 1 && test "${leafb}" -eq 1 && test "${freeb}" -eq 1) || _fail "failed to create ${dtype} dir ino ${inode} datab ${datab} leafb ${leafb} freeb ${freeb}"
 411                 ;;
 412         *)
 413                 _fail "Unknown directory type ${dtype}"
 414         esac
 415 }
 416
 417 # Check structure of XFS attr
 418 __populate_check_xfs_attr() {
 419         inode="$1"
 420         atype="$2"
 421
 422         datab=0
 423         leafb=0
 424         #echo "== check attr ${inode} type ${dtype}" ; _scratch_xfs_db -x -c "inode ${inode}" -c "bmap -a"
 425         _scratch_xfs_db -x -c "inode ${inode}" -c "ablock 0" -c "stack" | grep -q 'file attr block is unmapped' || datab=1
 426         _scratch_xfs_db -x -c "inode ${inode}" -c "ablock 1" -c "stack" | grep -q 'file attr block is unmapped' || leafb=1
 427
 428         case "${atype}" in
 429         "shortform"|"inline"|"local")
 430                 (test "${datab}" -eq 0 && test "${leafb}" -eq 0) || _fail "failed to create ${atype} attr ino ${inode} datab ${datab} leafb ${leafb}"
 431                 ;;
 432         "leaf")
 433                 (test "${datab}" -eq 1 && test "${leafb}" -eq 0) || _fail "failed to create ${atype} attr ino ${inode} datab ${datab} leafb ${leafb}"
 434                 ;;
 435         "node"|"btree")
 436                 (test "${datab}" -eq 1 && test "${leafb}" -eq 1) || _fail "failed to create ${atype} attr ino ${inode} datab ${datab} leafb ${leafb}"
 437                 ;;
 438         *)
 439                 _fail "Unknown attribute type ${atype}"
 440         esac
 441 }
 442
 443 # Check that there's at least one per-AG btree with multiple levels
 444 __populate_check_xfs_agbtree_height() {
 445         bt_type="$1"
 446         nr_ags=$(_scratch_xfs_db -c 'sb 0' -c 'p agcount' | awk '{print $3}')
 447
 448         case "${bt_type}" in
 449         "bno"|"cnt"|"rmap"|"refcnt")
 450                 hdr="agf"
 451                 bt_prefix="${bt_type}"
 452                 ;;
 453         "ino")
 454                 hdr="agi"
 455                 bt_prefix=""
 456                 ;;
 457         "fino")
 458                 hdr="agi"
 459                 bt_prefix="free_"
 460                 ;;
 461         *)
 462                 _fail "Don't know about AG btree ${bt_type}"
 463                 ;;
 464         esac
 465
 466         seq 0 $((nr_ags - 1)) | while read ag; do
 467                 bt_level=$(_scratch_xfs_db -c "${hdr} ${ag}" -c "p ${bt_prefix}level" | awk '{print $3}')
 468                 if [ "${bt_level}" -gt 1 ]; then
 469                         return 100
 470                 fi
 471         done
 472         test $? -eq 100 || _fail "Failed to create ${bt_type} of sufficient height!"
 473         return 1
 474 }
 475
 476 # Check that populate created all the types of files we wanted
 477 _scratch_xfs_populate_check() {
 478         _scratch_mount
 479         extents_file="$(__populate_find_inode "${SCRATCH_MNT}/S_IFREG.FMT_EXTENTS")"
 480         btree_file="$(__populate_find_inode "${SCRATCH_MNT}/S_IFREG.FMT_BTREE")"
 481         inline_dir="$(__populate_find_inode "${SCRATCH_MNT}/S_IFDIR.FMT_INLINE")"
 482         block_dir="$(__populate_find_inode "${SCRATCH_MNT}/S_IFDIR.FMT_BLOCK")"
 483         leaf_dir="$(__populate_find_inode "${SCRATCH_MNT}/S_IFDIR.FMT_LEAF")"
 484         node_dir="$(__populate_find_inode "${SCRATCH_MNT}/S_IFDIR.FMT_NODE")"
 485         btree_dir="$(__populate_find_inode "${SCRATCH_MNT}/S_IFDIR.FMT_BTREE")"
 486         local_slink="$(__populate_find_inode "${SCRATCH_MNT}/S_IFLNK.FMT_LOCAL")"
 487         extents_slink="$(__populate_find_inode "${SCRATCH_MNT}/S_IFLNK.FMT_EXTENTS")"
 488         bdev="$(__populate_find_inode "${SCRATCH_MNT}/S_IFBLK")"
 489         cdev="$(__populate_find_inode "${SCRATCH_MNT}/S_IFCHR")"
 490         local_attr="$(__populate_find_inode "${SCRATCH_MNT}/ATTR.FMT_LOCAL")"
 491         leaf_attr="$(__populate_find_inode "${SCRATCH_MNT}/ATTR.FMT_LEAF")"
 492         node_attr="$(__populate_find_inode "${SCRATCH_MNT}/ATTR.FMT_NODE")"
 493         btree_attr="$(__populate_find_inode "${SCRATCH_MNT}/ATTR.FMT_BTREE")"
 494         is_finobt=$(xfs_info "${SCRATCH_MNT}" | grep -c 'finobt=1')
 495         is_rmapbt=$(xfs_info "${SCRATCH_MNT}" | grep -c 'rmapbt=1')
 496         is_reflink=$(xfs_info "${SCRATCH_MNT}" | grep -c 'reflink=1')
 497
 498         blksz="$(stat -f -c '%s' "${SCRATCH_MNT}")"
 499         dblksz="$(xfs_info "${SCRATCH_MNT}" | grep naming.*bsize | sed -e 's/^.*bsize=//g' -e 's/\([0-9]*\).*$/\1/g')"
 500         leaf_lblk="$((32 * 1073741824 / blksz))"
 501         node_lblk="$((64 * 1073741824 / blksz))"
 502         umount "${SCRATCH_MNT}"
 503
 504         __populate_check_xfs_dformat "${extents_file}" "extents"
 505         __populate_check_xfs_dformat "${btree_file}" "btree"
 506         __populate_check_xfs_dir "${inline_dir}" "inline"
 507         __populate_check_xfs_dir "${block_dir}" "block"
 508         __populate_check_xfs_dir "${leaf_dir}" "leaf"
 509         __populate_check_xfs_dir "${node_dir}" "node"
 510         __populate_check_xfs_dir "${btree_dir}" "btree"
 511         __populate_check_xfs_dformat "${btree_dir}" "btree"
 512         __populate_check_xfs_dformat "${bdev}" "dev"
 513         __populate_check_xfs_dformat "${cdev}" "dev"
 514         __populate_check_xfs_attr "${local_attr}" "local"
 515         __populate_check_xfs_attr "${leaf_attr}" "leaf"
 516         __populate_check_xfs_attr "${node_attr}" "node"
 517         __populate_check_xfs_attr "${btree_attr}" "btree"
 518         __populate_check_xfs_aformat "${btree_attr}" "btree"
 519         __populate_check_xfs_agbtree_height "bno"
 520         __populate_check_xfs_agbtree_height "cnt"
 521         test $is_rmapbt -ne 0 && __populate_check_xfs_agbtree_height "rmap"
 522         test $is_reflink -ne 0 && __populate_check_xfs_agbtree_height "refcnt"
 523 }
 524
 525 # Check data fork format of ext4 file
 526 __populate_check_ext4_dformat() {
 527         dev="${SCRATCH_DEV}"
 528         inode="$1"
 529         format="$2"
 530
 531         extents=0
 532         etree=0
 533         debugfs -R "stat <${inode}>" "${dev}" 2> /dev/null | grep 'ETB[0-9]' -q && etree=1
 534         iflags="$(debugfs -R "stat <${inode}>" "${dev}" 2> /dev/null | grep 'Flags:' | sed -e 's/^.*Flags: \([0-9a-fx]*\).*$/\1/g')"
 535         test "$(echo "${iflags}" | awk '{print and(strtonum($1), 0x80000);}')" -gt 0 && extents=1
 536
 537         case "${format}" in
 538         "blockmap")
 539                 test "${extents}" -eq 0 || _fail "failed to create ino ${inode} with blockmap"
 540                 ;;
 541         "extent"|"extents")
 542                 test "${extents}" -eq 1 || _fail "failed to create ino ${inode} with extents"
 543                 ;;
 544         "etree")
 545                 (test "${extents}" -eq 1 && test "${etree}" -eq 1) || _fail "failed to create ino ${inode} with extent tree"
 546                 ;;
 547         *)
 548                 _fail "Unknown dformat ${format}"
 549         esac
 550 }
 551
 552 # Check attr fork format of ext4 file
 553 __populate_check_ext4_aformat() {
 554         dev="${SCRATCH_DEV}"
 555         inode="$1"
 556         format="$2"
 557
 558         ablock=1
 559         debugfs -R "stat <${inode}>" "${dev}" 2> /dev/null | grep 'File ACL: 0' -q && ablock=0
 560
 561         case "${format}" in
 562         "local"|"inline")
 563                 test "${ablock}" -eq 0 || _fail "failed to create inode ${inode} with ${format} xattr"
 564                 ;;
 565         "block")
 566                 test "${extents}" -eq 1 || _fail "failed to create inode ${inode} with ${format} xattr"
 567                 ;;
 568         *)
 569                 _fail "Unknown aformat ${format}"
 570         esac
 571 }
 572
 573 # Check structure of ext4 dir
 574 __populate_check_ext4_dir() {
 575         dev="${SCRATCH_DEV}"
 576         inode="$1"
 577         dtype="$2"
 578
 579         htree=0
 580         inline=0
 581         iflags="$(debugfs -R "stat <${inode}>" "${dev}" 2> /dev/null | grep 'Flags:' | sed -e 's/^.*Flags: \([0-9a-fx]*\).*$/\1/g')"
 582         test "$(echo "${iflags}" | awk '{print and(strtonum($1), 0x1000);}')" -gt 0 && htree=1
 583         test "$(echo "${iflags}" | awk '{print and(strtonum($1), 0x10000000);}')" -gt 0 && inline=1
 584
 585         case "${dtype}" in
 586         "inline")
 587                 (test "${inline}" -eq 1 && test "${htree}" -eq 0) || _fail "failed to create ${dtype} dir ino ${inode} htree ${htree} inline ${inline}"
 588                 ;;
 589         "block")
 590                 (test "${inline}" -eq 0 && test "${htree}" -eq 0) || _fail "failed to create ${dtype} dir ino ${inode} htree ${htree} inline ${inline}"
 591                 ;;
 592         "htree")
 593                 (test "${inline}" -eq 0 && test "${htree}" -eq 1) || _fail "failed to create ${dtype} dir ino ${inode} htree ${htree} inline ${inline}"
 594                 ;;
 595         *)
 596                 _fail "Unknown directory type ${dtype}"
 597                 ;;
 598         esac
 599 }
 600
 601 # Check that populate created all the types of files we wanted
 602 _scratch_ext4_populate_check() {
 603         _scratch_mount
 604         extents_file="$(__populate_find_inode "${SCRATCH_MNT}/S_IFREG.FMT_EXTENTS")"
 605         etree_file="$(__populate_find_inode "${SCRATCH_MNT}/S_IFREG.FMT_ETREE")"
 606         block_dir="$(__populate_find_inode "${SCRATCH_MNT}/S_IFDIR.FMT_BLOCK")"
 607         htree_dir="$(__populate_find_inode "${SCRATCH_MNT}/S_IFDIR.FMT_HTREE")"
 608         extents_slink="$(__populate_find_inode "${SCRATCH_MNT}/S_IFLNK.FMT_EXTENTS")"
 609         local_attr="$(__populate_find_inode "${SCRATCH_MNT}/ATTR.FMT_LOCAL")"
 610         block_attr="$(__populate_find_inode "${SCRATCH_MNT}/ATTR.FMT_BLOCK")"
 611         umount "${SCRATCH_MNT}"
 612
 613         __populate_check_ext4_dformat "${extents_file}" "extents"
 614         __populate_check_ext4_dformat "${etree_file}" "etree"
 615         __populate_check_ext4_dir "${block_dir}" "block"
 616         __populate_check_ext4_dir "${htree_dir}" "htree"
 617         __populate_check_ext4_dformat "${extents_slink}" "extents"
 618         __populate_check_ext4_aformat "${local_attr}" "local"
 619         __populate_check_ext4_aformat "${block_attr}" "block"
 620 }
 621
 622 # Populate a scratch FS and check the contents to make sure we got that
 623 _scratch_populate() {
 624         case "${FSTYP}" in
 625         "xfs")
 626                 _scratch_xfs_populate
 627                 _scratch_xfs_populate_check
 628                 ;;
 629         "ext2"|"ext3"|"ext4")
 630                 _scratch_ext4_populate
 631                 _scratch_ext4_populate_check
 632                 ;;
 633         *)
 634                 _fail "Don't know how to populate a ${FSTYP} filesystem."
 635                 ;;
 636         esac
 637 }
 638
 639 # Fill a file system by repeatedly creating files in the given folder
 640 # starting with the given file size.  Files are reduced in size when
 641 # they can no longer fit until no more files can be created.
 642 _fill_fs()
 643 {
 644         local file_size=$1
 645         local dir=$2
 646         local block_size=$3
 647         local switch_user=$4
 648         local file_count=1
 649         local bytes_written=0
 650         local use_falloc=1;
 651
 652         if [ $# -ne 4 ]; then
 653                 echo "Usage: _fill_fs filesize dir blocksize switch_user"
 654                 exit 1
 655         fi
 656
 657         if [ $switch_user -eq 0 ]; then
 658                 mkdir -p $dir
 659         else
 660                 _user_do "mkdir -p $dir"
 661         fi
 662         if [ ! -d $dir ]; then
 663                 return 0;
 664         fi
 665
 666         testio=`$XFS_IO_PROG -F -fc "falloc 0 $block_size" $dir/$$.xfs_io 2>&1`
 667         echo $testio | grep -q "not found" && use_falloc=0
 668         echo $testio | grep -q "Operation not supported" && use_falloc=0
 669
 670         if [ $file_size -lt $block_size ]; then
 671                 $file_size = $block_size
 672         fi
 673
 674         while [ $file_size -ge $block_size ]; do
 675                 bytes_written=0
 676                 if [ $switch_user -eq 0 ]; then
 677                         if [ $use_falloc -eq 0 ]; then
 678                                 $XFS_IO_PROG -fc "pwrite -b 8388608 0 $file_size" \
 679                                         $dir/$file_count
 680                         else
 681                                 $XFS_IO_PROG -fc "falloc 0 $file_size" \
 682                                         $dir/$file_count
 683                         fi
 684                 else
 685                         if [ $use_falloc -eq 0 ]; then
 686                                 _user_do "$XFS_IO_PROG -f -c \"pwrite -b 8388608 0 \
 687                                         $file_size\" $dir/$file_count"
 688                         else
 689                                 _user_do "$XFS_IO_PROG -f -c \"falloc 0 \
 690                                         $file_size\" $dir/$file_count"
 691                         fi
 692                 fi
 693
 694                 if [ -f $dir/$file_count ]; then
 695                         bytes_written=$(stat -c '%s' $dir/$file_count)
 696                 fi
 697
 698                 # If there was no room to make the file, then divide it in
 699                 # half, and keep going
 700                 if [ $bytes_written -lt $file_size ]; then
 701                         file_size=$((file_size / 2))
 702                 fi
 703                 file_count=$((file_count + 1))
 704         done
 705 }
 706
 707 # Populate a scratch FS from scratch or from a cached image.
 708 _scratch_populate_cached() {
 709         POPULATE_METADUMP="${TEST_DIR}/__populate.${FSTYP}"
 710         POPULATE_METADUMP_DESCR="${TEST_DIR}/__populate.${FSTYP}.txt"
 711
 712         # Don't keep metadata images cached for more 48 hours...
 713         rm -rf "$(find "${POPULATE_METADUMP}" -mtime +2 2>/dev/null)"
 714
 715         # Throw away cached image if it doesn't match our spec.
 716         case "${FSTYP}" in
 717         "ext4")
 718                 extra_descr="LOGDEV ${SCRATCH_LOGDEV} USE_EXTERNAL ${USE_EXTERNAL}"
 719                 # ext4 cannot e2image external logs, so we cannot restore
 720                 test -n "${SCRATCH_LOGDEV}" && rm -f "${POPULATE_METADUMP}"
 721                 ;;
 722         "xfs")
 723                 extra_descr="LOGDEV ${SCRATCH_LOGDEV} USE_EXTERNAL ${USE_EXTERNAL} RTDEV ${SCRATCH_RTDEV}";;
 724         *)
 725                 extra_descr="";;
 726         esac
 727         meta_descr="FSTYP ${FSTYP} MKFS_OPTIONS ${MKFS_OPTIONS} SIZE $(blockdev --getsz "${SCRATCH_DEV}") ${extra_descr} ARGS $@"
 728         cmp -s "${POPULATE_METADUMP_DESCR}" <(echo "${meta_descr}") || rm -rf "${POPULATE_METADUMP}"
 729
 730         # Do we have a cached image?
 731         if [ -r "${POPULATE_METADUMP}" ]; then
 732                 case "${FSTYP}" in
 733                 "xfs")
 734                         xfs_mdrestore "${POPULATE_METADUMP}" "${SCRATCH_DEV}" && return
 735                         ;;
 736                 "ext2"|"ext3"|"ext4")
 737                         e2image -r "${POPULATE_METADUMP}" "${SCRATCH_DEV}" && return
 738                         ;;
 739                 esac
 740         fi
 741
 742         # Oh well, just create one from scratch
 743         _scratch_mkfs
 744         echo "${meta_descr}" > "${POPULATE_METADUMP_DESCR}"
 745         case "${FSTYP}" in
 746         "xfs")
 747                 _scratch_xfs_populate $@
 748                 _scratch_xfs_populate_check
 749                 _scratch_metadump "${POPULATE_METADUMP}" -a -o
 750                 ;;
 751         "ext2"|"ext3"|"ext4")
 752                 _scratch_ext4_populate $@
 753                 _scratch_ext4_populate_check
 754                 e2image -Q "${SCRATCH_DEV}" "${POPULATE_METADUMP}"
 755                 ;;
 756         *)
 757                 _fail "Don't know how to populate a ${FSTYP} filesystem."
 758                 ;;
 759         esac
 760 }