generic: test MADV_POPULATE_READ with IO errors
[xfstests-dev.git] / common / populate
1 ##/bin/bash
2 # SPDX-License-Identifier: GPL-2.0+
3 # Copyright (c) 2015 Oracle.  All Rights Reserved.
4 #
5 # Routines for populating a scratch fs, and helpers to exercise an FS
6 # once it's been fuzzed.
7
8 . ./common/quota
9
10 _require_populate_commands() {
11         _require_xfs_io_command "falloc"
12         _require_xfs_io_command "fpunch"
13         _require_test_program "punch-alternating"
14         _require_test_program "popdir.pl"
15         if [ -n "${PYTHON3_PROG}" ]; then
16                 _require_command $PYTHON3_PROG python3
17                 _require_test_program "popattr.py"
18         fi
19         case "${FSTYP}" in
20         "xfs")
21                 _require_command "$XFS_DB_PROG" "xfs_db"
22                 _require_command "$WIPEFS_PROG" "wipefs"
23                 _require_command "$XFS_MDRESTORE_PROG" "xfs_mdrestore"
24                 ;;
25         ext*)
26                 _require_command "$DUMPE2FS_PROG" "dumpe2fs"
27                 _require_command "$E2IMAGE_PROG" "e2image"
28                 ;;
29         esac
30 }
31
32 _require_xfs_db_blocktrash_z_command() {
33         test "${FSTYP}" = "xfs" || _notrun "cannot run xfs_db on ${FSTYP}"
34         $XFS_DB_PROG -x -f -c 'blocktrash -z' "${TEST_DEV}" | grep -q 'nothing on stack' || _notrun "blocktrash -z not supported"
35 }
36
37 # Attempt to make files of "every" format for data, dirs, attrs etc.
38 # (with apologies to Eric Sandeen for mutating xfser.sh)
39
40 # Create a file of a given size.
41 __populate_create_file() {
42         local sz="$1"
43         local fname="$2"
44
45         $XFS_IO_PROG -f -c "pwrite -S 0x62 -W -b 1m 0 $sz" "${fname}"
46 }
47
48 # Fail the test if we failed to create some kind of filesystem metadata.
49 # Create a metadata dump of the failed filesystem so that we can analyze
50 # how things went rong.
51 __populate_fail() {
52         local flatdev="$(basename "$SCRATCH_DEV")"
53         local metadump="$seqres.$flatdev.populate.md"
54
55         case "$FSTYP" in
56         xfs)
57                 _scratch_unmount
58                 _scratch_xfs_metadump "$metadump" -a -o
59                 ;;
60         ext4)
61                 _scratch_unmount
62                 _ext4_metadump "${SCRATCH_DEV}" "$metadump"
63                 ;;
64         esac
65
66         _fail "$@"
67 }
68
69 # Punch out every other hole in this file, if it exists.
70 #
71 # The goal here is to force the creation of a large number of metadata records
72 # by creating a lot of tiny extent mappings in a file.  Callers should ensure
73 # that fragmenting the file actually causes record creation.  Call this
74 # function /after/ creating all other metadata structures.
75 __populate_fragment_file() {
76         local fname="$1"
77
78         test -f "${fname}" && $here/src/punch-alternating "${fname}"
79 }
80
81 # Create a large directory
82 __populate_create_dir() {
83         local name="$1"
84         local nr="$2"
85         local missing="$3"
86         shift; shift; shift
87
88         mkdir -p "${name}"
89         $here/src/popdir.pl --dir "${name}" --end "${nr}" "$@"
90
91         test -z "${missing}" && return
92         $here/src/popdir.pl --dir "${name}" --start 1 --incr 19 --end "${nr}" --remove "$@"
93 }
94
95 # Create a large directory and ensure that it's a btree format
96 __populate_xfs_create_btree_dir() {
97         local name="$1"
98         local isize="$2"
99         local dblksz="$3"
100         local missing="$4"
101         local icore_size="$(_xfs_get_inode_core_bytes $SCRATCH_MNT)"
102         # We need enough extents to guarantee that the data fork is in
103         # btree format.  Cycling the mount to use xfs_db is too slow, so
104         # watch for when the extent count exceeds the space after the
105         # inode core.
106         local max_nextents="$(((isize - icore_size) / 16))"
107         local nr
108         local incr
109
110         # Add about one block's worth of dirents before we check the data fork
111         # format.
112         incr=$(( (dblksz / 8) / 100 * 100 ))
113
114         mkdir -p "${name}"
115         for ((nr = 0; ; nr += incr)); do
116                 $here/src/popdir.pl --dir "${name}" --start "${nr}" --end "$((nr + incr - 1))"
117
118                 # Extent count checks use data blocks only to avoid the removal
119                 # step from removing dabtree index blocks and reducing the
120                 # number of extents below the required threshold.
121                 local nextents="$(xfs_bmap ${name} | grep -v hole | wc -l)"
122                 [ "$((nextents - 1))" -gt $max_nextents ] && break
123         done
124
125         test -z "${missing}" && return
126         $here/src/popdir.pl --dir "${name}" --start 1 --incr 19 --end "${nr}" --remove
127 }
128
129 # Add a bunch of attrs to a file
130 __populate_create_attr() {
131         name="$1"
132         nr="$2"
133         missing="$3"
134
135         touch "${name}"
136
137         if [ -n "${PYTHON3_PROG}" ]; then
138                 ${PYTHON3_PROG} $here/src/popattr.py --file "${name}" --end "${nr}"
139
140                 test -z "${missing}" && return
141                 ${PYTHON3_PROG} $here/src/popattr.py --file "${name}" --start 1 --incr 19 --end "${nr}" --remove
142                 return
143         fi
144
145         # Simulate a getfattr dump file so we can bulk-add attrs.
146         (
147                 echo "# file: ${name}";
148                 seq --format "user.%08g=\"abcdefgh\"" 0 "${nr}"
149                 echo
150         ) | setfattr --restore -
151
152         test -z "${missing}" && return
153         seq 1 2 "${nr}" | while read d; do
154                 setfattr -x "user.$(printf "%.08d" "$d")" "${name}"
155         done
156 }
157
158 # Create an extended attr structure and ensure that the fork is btree format
159 __populate_xfs_create_btree_attr() {
160         local name="$1"
161         local isize="$2"
162         local dblksz="$3"
163         local icore_size="$(_xfs_get_inode_core_bytes $SCRATCH_MNT)"
164         # We need enough extents to guarantee that the attr fork is in btree
165         # format.  Cycling the mount to use xfs_db is too slow, so watch for
166         # when the number of holes that we can punch in the attr fork by
167         # deleting remote xattrs exceeds the number of extent mappings that can
168         # fit in the inode core.
169         local max_nextents="$(((isize - icore_size) / 16))"
170         local nr
171         local i
172         local incr
173         local bigval
174
175         # Add about one block's worth of attrs in betweeen creating punchable
176         # remote value blocks.
177         incr=$(( (dblksz / 16) / 100 * 100 ))
178         bigval="$(perl -e "print \"@\" x $dblksz;")"
179
180         touch "${name}"
181
182         # We cannot control the mapping behaviors of the attr fork leaf and
183         # dabtree blocks, but we do know that remote values are stored in a
184         # single extent, and that those mappings are removed if the xattr is
185         # deleted.
186         #
187         # The extended attribute structure tends to grow from offset zero
188         # upwards, so we try to set up a sparse attr fork mapping by
189         # iteratively creating at least one leaf block's worth of local attrs,
190         # and then one remote attr, until the number of remote xattrs exceeds
191         # the number of mappings that fit in the inode core...
192         for ((nr = 0; nr < (incr * max_nextents); nr += incr)); do
193                 # Simulate a getfattr dump file so we can bulk-add attrs.
194                 (
195                         echo "# file: ${name}";
196                         seq --format "user.%08g=\"abcdefgh\"" "${nr}" "$((nr + incr + 1))"
197                         echo "user.v$(printf "%.08d" "$nr")=\"${bigval}\""
198                         echo
199                 ) | setfattr --restore -
200         done
201
202         # ... and in the second loop we delete all the remote attrs to
203         # fragment the attr fork mappings.
204         for ((i = 0; i < nr; i += incr)); do
205                 setfattr -x "user.v$(printf "%.08d" "$i")" "${name}"
206         done
207 }
208
209 # Fill up some percentage of the remaining free space
210 __populate_fill_fs() {
211         dir="$1"
212         pct="$2"
213         test -z "${pct}" && pct=60
214
215         mkdir -p "${dir}/test/1"
216         cp -pRdu "${dir}"/S_IFREG* "${dir}/test/1/"
217
218         SRC_SZ="$(du -ks "${dir}/test/1" | cut -f 1)"
219         FS_SZ="$(( $(stat -f "${dir}" -c '%a * %S') / 1024 ))"
220
221         NR="$(( (FS_SZ * ${pct} / 100) / SRC_SZ ))"
222
223         echo "FILL FS"
224         echo "src_sz $SRC_SZ fs_sz $FS_SZ nr $NR"
225         seq 2 "${NR}" | while read nr; do
226                 cp -pRdu "${dir}/test/1" "${dir}/test/${nr}"
227         done
228 }
229
230 # For XFS, force on all the quota options if quota is enabled
231 # and the user didn't feed us noquota.
232 _populate_xfs_qmount_option()
233 {
234         # User explicitly told us not to quota
235         if echo "${MOUNT_OPTIONS}" | grep -q 'noquota'; then
236                 return
237         fi
238
239         # Don't bother if we can't turn on quotas
240         if [ ! -f /proc/fs/xfs/xqmstat ]; then
241                 # No quota support
242                 return
243         elif [ "${USE_EXTERNAL}" = "yes" ] && [ ! -z "${SCRATCH_RTDEV}" ]; then
244                 # Quotas not supported on rt filesystems
245                 return
246         elif [ -z "${XFS_QUOTA_PROG}" ]; then
247                 # xfs quota tools not installed
248                 return
249         fi
250
251         # Turn on all the quotas
252         if _xfs_has_feature "$TEST_DIR" crc; then
253                 # v5 filesystems can have group & project quotas
254                 quota="usrquota,grpquota,prjquota"
255         else
256                 # v4 filesystems cannot mix group & project quotas
257                 quota="usrquota,grpquota"
258         fi
259
260         # Inject our quota mount options
261         if echo "${MOUNT_OPTIONS}" | grep -q "${quota}"; then
262                 return
263         elif echo "${MOUNT_OPTIONS}" | grep -Eq '(quota|noenforce)'; then
264                 _qmount_option "${quota}"
265         else
266                 export MOUNT_OPTIONS="$MOUNT_OPTIONS -o ${quota}"
267                 echo "MOUNT_OPTIONS = $MOUNT_OPTIONS" >>$seqres.full
268         fi
269 }
270
271 # Populate an XFS on the scratch device with (we hope) all known
272 # types of metadata block
273 _scratch_xfs_populate() {
274         fill=1
275
276         for arg in $@; do
277                 case "${arg}" in
278                 "nofill")
279                         fill=0;;
280                 esac
281         done
282
283         _populate_xfs_qmount_option
284         _scratch_mount
285
286         # We cannot directly force the filesystem to create the metadata
287         # structures we want; we can only achieve this indirectly by carefully
288         # crafting files and a directory tree.  Therefore, we must have exact
289         # control over the layout and device selection of all files created.
290         # Clear the rtinherit flag on the root directory so that files are
291         # always created on the data volume regardless of MKFS_OPTIONS.  We can
292         # set the realtime flag when needed.
293         _xfs_force_bdev data $SCRATCH_MNT
294
295         blksz="$(stat -f -c '%s' "${SCRATCH_MNT}")"
296         dblksz="$(_xfs_get_dir_blocksize "$SCRATCH_MNT")"
297         isize="$(_xfs_get_inode_size "$SCRATCH_MNT")"
298         crc="$(_xfs_has_feature "$SCRATCH_MNT" crc -v)"
299         if [ $crc -eq 1 ]; then
300                 leaf_hdr_size=64
301         else
302                 leaf_hdr_size=16
303         fi
304         leaf_lblk="$((32 * 1073741824 / blksz))"
305         node_lblk="$((64 * 1073741824 / blksz))"
306
307         # Data:
308
309         # Fill up the root inode chunk
310         echo "+ fill root ino chunk"
311         $here/src/popdir.pl --dir "${SCRATCH_MNT}" --start 1 --end 64 --format "dummy%u" --file-pct 100
312
313         # Regular files
314         # - FMT_EXTENTS
315         echo "+ extents file"
316         __populate_create_file $blksz "${SCRATCH_MNT}/S_IFREG.FMT_EXTENTS"
317
318         # - FMT_BTREE
319         echo "+ btree extents file"
320         nr="$((blksz * 2 / 16))"
321         __populate_create_file $((blksz * nr)) "${SCRATCH_MNT}/S_IFREG.FMT_BTREE"
322
323         # Directories
324         # - INLINE
325         echo "+ inline dir"
326         __populate_create_dir "${SCRATCH_MNT}/S_IFDIR.FMT_INLINE" 1
327
328         # - BLOCK
329         echo "+ block dir"
330         __populate_create_dir "${SCRATCH_MNT}/S_IFDIR.FMT_BLOCK" "$((dblksz / 40))"
331
332         # - LEAF
333         echo "+ leaf dir"
334         __populate_create_dir "${SCRATCH_MNT}/S_IFDIR.FMT_LEAF" "$((dblksz / 12))"
335
336         # - LEAFN
337         echo "+ leafn dir"
338         __populate_create_dir "${SCRATCH_MNT}/S_IFDIR.FMT_LEAFN" "$(( ((dblksz - leaf_hdr_size) / 8) - 3 ))"
339
340         # - NODE
341         echo "+ node dir"
342         __populate_create_dir "${SCRATCH_MNT}/S_IFDIR.FMT_NODE" "$((16 * dblksz / 40))" true
343
344         # - BTREE
345         echo "+ btree dir"
346         __populate_xfs_create_btree_dir "${SCRATCH_MNT}/S_IFDIR.FMT_BTREE" "$isize" "$dblksz" true
347
348         # Symlinks
349         # - FMT_LOCAL
350         echo "+ inline symlink"
351         ln -s target "${SCRATCH_MNT}/S_IFLNK.FMT_LOCAL"
352
353         # - FMT_EXTENTS
354         echo "+ extents symlink"
355         ln -s "$(perl -e 'print "x" x 1023;')" "${SCRATCH_MNT}/S_IFLNK.FMT_EXTENTS"
356
357         # Char & block
358         echo "+ special"
359         mknod "${SCRATCH_MNT}/S_IFCHR" c 1 1
360         mknod "${SCRATCH_MNT}/S_IFBLK" b 1 1
361         mknod "${SCRATCH_MNT}/S_IFIFO" p
362
363         # special file with an xattr
364         setfacl -P -m u:nobody:r ${SCRATCH_MNT}/S_IFCHR
365
366         # Attribute formats
367         # LOCAL
368         echo "+ local attr"
369         __populate_create_attr "${SCRATCH_MNT}/ATTR.FMT_LOCAL" 1
370
371         # LEAF
372         echo "+ leaf attr"
373         __populate_create_attr "${SCRATCH_MNT}/ATTR.FMT_LEAF" "$((blksz / 40))"
374
375         # NODE
376         echo "+ node attr"
377         __populate_create_attr "${SCRATCH_MNT}/ATTR.FMT_NODE" "$((8 * blksz / 40))"
378
379         # BTREE
380         echo "+ btree attr"
381         __populate_xfs_create_btree_attr "${SCRATCH_MNT}/ATTR.FMT_BTREE" "$isize" "$dblksz"
382
383         # trusted namespace
384         touch ${SCRATCH_MNT}/ATTR.TRUSTED
385         setfattr -n trusted.moo -v urk ${SCRATCH_MNT}/ATTR.TRUSTED
386
387         # security namespace
388         touch ${SCRATCH_MNT}/ATTR.SECURITY
389         setfattr -n security.foo -v bar ${SCRATCH_MNT}/ATTR.SECURITY
390
391         # system namespace
392         touch ${SCRATCH_MNT}/ATTR.SYSTEM
393         setfacl -m u:root:r ${SCRATCH_MNT}/ATTR.SYSTEM
394
395         # FMT_EXTENTS with a remote less-than-a-block value
396         echo "+ attr extents with a remote less-than-a-block value"
397         touch "${SCRATCH_MNT}/ATTR.FMT_EXTENTS_REMOTE3K"
398         $XFS_IO_PROG -f -c "pwrite -S 0x43 0 $((blksz - 300))" "${SCRATCH_MNT}/attrvalfile" > /dev/null
399         attr -q -s user.remotebtreeattrname "${SCRATCH_MNT}/ATTR.FMT_EXTENTS_REMOTE3K" < "${SCRATCH_MNT}/attrvalfile"
400
401         # FMT_EXTENTS with a remote block-size value
402         echo "+ attr extents with a remote one-block value"
403         touch "${SCRATCH_MNT}/ATTR.FMT_EXTENTS_REMOTE4K"
404         $XFS_IO_PROG -f -c "pwrite -S 0x44 0 ${blksz}" "${SCRATCH_MNT}/attrvalfile" > /dev/null
405         attr -q -s user.remotebtreeattrname "${SCRATCH_MNT}/ATTR.FMT_EXTENTS_REMOTE4K" < "${SCRATCH_MNT}/attrvalfile"
406         rm -rf "${SCRATCH_MNT}/attrvalfile"
407
408         # Make an unused inode
409         echo "+ empty file"
410         touch "${SCRATCH_MNT}/unused"
411         $XFS_IO_PROG -f -c 'fsync' "${SCRATCH_MNT}/unused"
412         rm -rf "${SCRATCH_MNT}/unused"
413
414         # Free space btree
415         echo "+ freesp btree"
416         nr="$((blksz * 2 / 8))"
417         __populate_create_file $((blksz * nr)) "${SCRATCH_MNT}/BNOBT"
418
419         # Inode btree
420         echo "+ inobt btree"
421         local ino_per_rec=64
422         local rec_per_btblock=16
423         local nr="$(( 2 * (blksz / rec_per_btblock) * ino_per_rec ))"
424         local dir="${SCRATCH_MNT}/INOBT"
425         __populate_create_dir "${dir}" "${nr}" true --file-pct 100
426
427         # Reverse-mapping btree
428         is_rmapbt="$(_xfs_has_feature "$SCRATCH_MNT" rmapbt -v)"
429         if [ $is_rmapbt -gt 0 ]; then
430                 echo "+ rmapbt btree"
431                 nr="$((blksz * 2 / 24))"
432                 __populate_create_file $((blksz * nr)) "${SCRATCH_MNT}/RMAPBT"
433         fi
434
435         # Realtime Reverse-mapping btree
436         is_rt="$(_xfs_get_rtextents "$SCRATCH_MNT")"
437         if [ $is_rmapbt -gt 0 ] && [ $is_rt -gt 0 ]; then
438                 echo "+ rtrmapbt btree"
439                 nr="$((blksz * 2 / 32))"
440                 $XFS_IO_PROG -R -f -c 'truncate 0' "${SCRATCH_MNT}/RTRMAPBT"
441                 __populate_create_file $((blksz * nr)) "${SCRATCH_MNT}/RTRMAPBT"
442         fi
443
444         # Reference-count btree
445         is_reflink="$(_xfs_has_feature "$SCRATCH_MNT" reflink -v)"
446         if [ $is_reflink -gt 0 ]; then
447                 echo "+ reflink btree"
448                 nr="$((blksz * 2 / 12))"
449                 __populate_create_file $((blksz * nr)) "${SCRATCH_MNT}/REFCOUNTBT"
450                 cp --reflink=always "${SCRATCH_MNT}/REFCOUNTBT" "${SCRATCH_MNT}/REFCOUNTBT2"
451         fi
452
453         # Copy some real files (xfs tests, I guess...)
454         echo "+ real files"
455         test $fill -ne 0 && __populate_fill_fs "${SCRATCH_MNT}" 5
456
457         # Make sure we get all the fragmentation we asked for
458         __populate_fragment_file "${SCRATCH_MNT}/S_IFREG.FMT_BTREE"
459         __populate_fragment_file "${SCRATCH_MNT}/BNOBT"
460         __populate_fragment_file "${SCRATCH_MNT}/RMAPBT"
461         __populate_fragment_file "${SCRATCH_MNT}/RTRMAPBT"
462         __populate_fragment_file "${SCRATCH_MNT}/REFCOUNTBT"
463
464         umount "${SCRATCH_MNT}"
465 }
466
467 # Populate an ext4 on the scratch device with (we hope) all known
468 # types of metadata block
469 _scratch_ext4_populate() {
470         fill=1
471
472         for arg in $@; do
473                 case "${arg}" in
474                 "nofill")
475                         fill=0;;
476                 esac
477         done
478
479         _scratch_mount
480         blksz="$(stat -f -c '%s' "${SCRATCH_MNT}")"
481         dblksz="${blksz}"
482         leaf_lblk="$((32 * 1073741824 / blksz))"
483         node_lblk="$((64 * 1073741824 / blksz))"
484
485         # Data:
486
487         # Regular files
488         # - FMT_INLINE
489         echo "+ inline file"
490         __populate_create_file 1 "${SCRATCH_MNT}/S_IFREG.FMT_INLINE"
491
492         # - FMT_EXTENTS
493         echo "+ extents file"
494         __populate_create_file $blksz "${SCRATCH_MNT}/S_IFREG.FMT_EXTENTS"
495
496         # - FMT_ETREE
497         echo "+ extent tree file"
498         nr="$((blksz * 2 / 12))"
499         __populate_create_file $((blksz * nr)) "${SCRATCH_MNT}/S_IFREG.FMT_ETREE"
500
501         # Directories
502         # - INLINE
503         echo "+ inline dir"
504         __populate_create_dir "${SCRATCH_MNT}/S_IFDIR.FMT_INLINE" 1
505
506         # - BLOCK
507         echo "+ block dir"
508         __populate_create_dir "${SCRATCH_MNT}/S_IFDIR.FMT_BLOCK" "$((dblksz / 32))"
509
510         # - HTREE
511         echo "+ htree dir"
512         __populate_create_dir "${SCRATCH_MNT}/S_IFDIR.FMT_HTREE" "$((4 * dblksz / 24))"
513
514         # Symlinks
515         # - FMT_LOCAL
516         echo "+ inline symlink"
517         ln -s target "${SCRATCH_MNT}/S_IFLNK.FMT_LOCAL"
518
519         # - FMT_EXTENTS
520         echo "+ extents symlink"
521         ln -s "$(perl -e 'print "x" x 1023;')" "${SCRATCH_MNT}/S_IFLNK.FMT_EXTENTS"
522
523         # Char & block
524         echo "+ special"
525         mknod "${SCRATCH_MNT}/S_IFCHR" c 1 1
526         mknod "${SCRATCH_MNT}/S_IFBLK" b 1 1
527         mknod "${SCRATCH_MNT}/S_IFIFO" p
528
529         # special file with an xattr
530         setfacl -P -m u:nobody:r ${SCRATCH_MNT}/S_IFCHR
531
532         # Attribute formats
533         # LOCAL
534         echo "+ local attr"
535         __populate_create_attr "${SCRATCH_MNT}/ATTR.FMT_LOCAL" 0
536
537         # BLOCK
538         echo "+ block attr"
539         __populate_create_attr "${SCRATCH_MNT}/ATTR.FMT_BLOCK" "$((blksz / 40))"
540
541         # trusted namespace
542         touch ${SCRATCH_MNT}/ATTR.TRUSTED
543         setfattr -n trusted.moo -v urk ${SCRATCH_MNT}/ATTR.TRUSTED
544
545         # security namespace
546         touch ${SCRATCH_MNT}/ATTR.SECURITY
547         setfattr -n security.foo -v bar ${SCRATCH_MNT}/ATTR.SECURITY
548
549         # system namespace
550         touch ${SCRATCH_MNT}/ATTR.SYSTEM
551         setfacl -m u:root:r ${SCRATCH_MNT}/ATTR.SYSTEM
552
553         # Make an unused inode
554         echo "+ empty file"
555         touch "${SCRATCH_MNT}/unused"
556         $XFS_IO_PROG -f -c 'fsync' "${SCRATCH_MNT}/unused"
557         rm -rf "${SCRATCH_MNT}/unused"
558
559         # Copy some real files (xfs tests, I guess...)
560         echo "+ real files"
561         test $fill -ne 0 && __populate_fill_fs "${SCRATCH_MNT}" 5
562
563         # Make sure we get all the fragmentation we asked for
564         __populate_fragment_file "${SCRATCH_MNT}/S_IFREG.FMT_ETREE"
565
566         umount "${SCRATCH_MNT}"
567 }
568
569 # Find the inode number of a file
570 __populate_find_inode() {
571         name="$1"
572         inode="$(stat -c '%i' "${name}")"
573         echo "${inode}"
574 }
575
576 # Check data fork format of XFS file
577 __populate_check_xfs_dformat() {
578         inode="$1"
579         format="$2"
580
581         fmt="$(_scratch_xfs_db -c "inode ${inode}" -c 'p core.format' | sed -e 's/^.*(\([a-z]*\)).*$/\1/g')"
582         test "${format}" = "${fmt}" || __populate_fail "failed to create ino ${inode} dformat expected ${format} saw ${fmt}"
583 }
584
585 # Check attr fork format of XFS file
586 __populate_check_xfs_aformat() {
587         inode="$1"
588         format="$2"
589
590         fmt="$(_scratch_xfs_db -c "inode ${inode}" -c 'p core.aformat' | sed -e 's/^.*(\([a-z]*\)).*$/\1/g')"
591         test "${format}" = "${fmt}" || __populate_fail "failed to create ino ${inode} aformat expected ${format} saw ${fmt}"
592 }
593
594 # Check structure of XFS directory
595 __populate_check_xfs_dir() {
596         inode="$1"
597         dtype="$2"
598
599         (test -n "${leaf_lblk}" && test -n "${node_lblk}") || _fail "must define leaf_lblk and node_lblk before calling __populate_check_xfs_dir"
600         datab=0
601         leafb=0
602         freeb=0
603         #echo "== check dir ${inode} type ${dtype}" ; _scratch_xfs_db -x -c "inode ${inode}" -c "bmap"
604         _scratch_xfs_db -x -c "inode ${inode}" -c "dblock 0" -c "stack" | grep -q 'file data block is unmapped' || datab=1
605         _scratch_xfs_db -x -c "inode ${inode}" -c "dblock ${leaf_lblk}" -c "stack" | grep -q 'file data block is unmapped' || leafb=1
606         _scratch_xfs_db -x -c "inode ${inode}" -c "dblock ${node_lblk}" -c "stack" | grep -q 'file data block is unmapped' || freeb=1
607
608         case "${dtype}" in
609         "shortform"|"inline"|"local")
610                 (test "${datab}" -eq 0 && test "${leafb}" -eq 0 && test "${freeb}" -eq 0) || __populate_fail "failed to create ${dtype} dir ino ${inode} datab ${datab} leafb ${leafb} freeb ${freeb}"
611                 ;;
612         "block")
613                 (test "${datab}" -eq 1 && test "${leafb}" -eq 0 && test "${freeb}" -eq 0) || __populate_fail "failed to create ${dtype} dir ino ${inode} datab ${datab} leafb ${leafb} freeb ${freeb}"
614                 ;;
615         "leaf")
616                 (test "${datab}" -eq 1 && test "${leafb}" -eq 1 && test "${freeb}" -eq 0) || __populate_fail "failed to create ${dtype} dir ino ${inode} datab ${datab} leafb ${leafb} freeb ${freeb}"
617                 ;;
618         "leafn")
619                 _scratch_xfs_db -x -c "inode ${inode}" -c "dblock ${leaf_lblk}" -c "p lhdr.info.hdr.magic" | grep -q '0x3dff' && return
620                 _scratch_xfs_db -x -c "inode ${inode}" -c "dblock ${leaf_lblk}" -c "p lhdr.info.magic" | grep -q '0xd2ff' && return
621                 __populate_fail "failed to create ${dtype} dir ino ${inode} datab ${datab} leafb ${leafb} freeb ${freeb}"
622                 ;;
623         "node"|"btree")
624                 (test "${datab}" -eq 1 && test "${leafb}" -eq 1 && test "${freeb}" -eq 1) || __populate_fail "failed to create ${dtype} dir ino ${inode} datab ${datab} leafb ${leafb} freeb ${freeb}"
625                 ;;
626         *)
627                 _fail "Unknown directory type ${dtype}"
628         esac
629 }
630
631 # Check structure of XFS attr
632 __populate_check_xfs_attr() {
633         inode="$1"
634         atype="$2"
635
636         datab=0
637         leafb=0
638         #echo "== check attr ${inode} type ${dtype}" ; _scratch_xfs_db -x -c "inode ${inode}" -c "bmap -a"
639         _scratch_xfs_db -x -c "inode ${inode}" -c "ablock 0" -c "stack" | grep -q 'file attr block is unmapped' || datab=1
640         _scratch_xfs_db -x -c "inode ${inode}" -c "ablock 1" -c "stack" | grep -q 'file attr block is unmapped' || leafb=1
641
642         case "${atype}" in
643         "shortform"|"inline"|"local")
644                 (test "${datab}" -eq 0 && test "${leafb}" -eq 0) || __populate_fail "failed to create ${atype} attr ino ${inode} datab ${datab} leafb ${leafb}"
645                 ;;
646         "leaf")
647                 (test "${datab}" -eq 1 && test "${leafb}" -eq 0) || __populate_fail "failed to create ${atype} attr ino ${inode} datab ${datab} leafb ${leafb}"
648                 ;;
649         "node"|"btree")
650                 (test "${datab}" -eq 1 && test "${leafb}" -eq 1) || __populate_fail "failed to create ${atype} attr ino ${inode} datab ${datab} leafb ${leafb}"
651                 ;;
652         *)
653                 _fail "Unknown attribute type ${atype}"
654         esac
655 }
656
657 # Check that there's at least one per-AG btree with multiple levels
658 __populate_check_xfs_agbtree_height() {
659         local bt_type="$1"
660         local agcount=$(_scratch_xfs_db -c 'sb 0' -c 'p agcount' | awk '{print $3}')
661
662         case "${bt_type}" in
663         "bno"|"cnt"|"rmap"|"refcnt")
664                 hdr="agf"
665                 bt_prefix="${bt_type}"
666                 ;;
667         "ino")
668                 hdr="agi"
669                 bt_prefix=""
670                 ;;
671         "fino")
672                 hdr="agi"
673                 bt_prefix="free_"
674                 ;;
675         *)
676                 _fail "Don't know about AG btree ${bt_type}"
677                 ;;
678         esac
679
680         for ((agno = 0; agno < agcount; agno++)); do
681                 bt_level=$(_scratch_xfs_db -c "${hdr} ${agno}" -c "p ${bt_prefix}level" | awk '{print $3}')
682                 # "level" is really the btree height
683                 if [ "${bt_level}" -gt 1 ]; then
684                         return 0
685                 fi
686         done
687         __populate_fail "Failed to create ${bt_type} of sufficient height!"
688         return 1
689 }
690
691 # Check that populate created all the types of files we wanted
692 _scratch_xfs_populate_check() {
693         _scratch_mount
694         extents_file="$(__populate_find_inode "${SCRATCH_MNT}/S_IFREG.FMT_EXTENTS")"
695         btree_file="$(__populate_find_inode "${SCRATCH_MNT}/S_IFREG.FMT_BTREE")"
696         inline_dir="$(__populate_find_inode "${SCRATCH_MNT}/S_IFDIR.FMT_INLINE")"
697         block_dir="$(__populate_find_inode "${SCRATCH_MNT}/S_IFDIR.FMT_BLOCK")"
698         leaf_dir="$(__populate_find_inode "${SCRATCH_MNT}/S_IFDIR.FMT_LEAF")"
699         leafn_dir="$(__populate_find_inode "${SCRATCH_MNT}/S_IFDIR.FMT_LEAFN")"
700         node_dir="$(__populate_find_inode "${SCRATCH_MNT}/S_IFDIR.FMT_NODE")"
701         btree_dir="$(__populate_find_inode "${SCRATCH_MNT}/S_IFDIR.FMT_BTREE")"
702         local_slink="$(__populate_find_inode "${SCRATCH_MNT}/S_IFLNK.FMT_LOCAL")"
703         extents_slink="$(__populate_find_inode "${SCRATCH_MNT}/S_IFLNK.FMT_EXTENTS")"
704         bdev="$(__populate_find_inode "${SCRATCH_MNT}/S_IFBLK")"
705         cdev="$(__populate_find_inode "${SCRATCH_MNT}/S_IFCHR")"
706         fifo="$(__populate_find_inode "${SCRATCH_MNT}/S_IFIFO")"
707         local_attr="$(__populate_find_inode "${SCRATCH_MNT}/ATTR.FMT_LOCAL")"
708         leaf_attr="$(__populate_find_inode "${SCRATCH_MNT}/ATTR.FMT_LEAF")"
709         node_attr="$(__populate_find_inode "${SCRATCH_MNT}/ATTR.FMT_NODE")"
710         btree_attr="$(__populate_find_inode "${SCRATCH_MNT}/ATTR.FMT_BTREE")"
711         is_finobt=$(_xfs_has_feature "$SCRATCH_MNT" finobt -v)
712         is_rmapbt=$(_xfs_has_feature "$SCRATCH_MNT" rmapbt -v)
713         is_reflink=$(_xfs_has_feature "$SCRATCH_MNT" reflink -v)
714
715         blksz="$(stat -f -c '%s' "${SCRATCH_MNT}")"
716         dblksz="$(_xfs_get_dir_blocksize "$SCRATCH_MNT")"
717         leaf_lblk="$((32 * 1073741824 / blksz))"
718         node_lblk="$((64 * 1073741824 / blksz))"
719         umount "${SCRATCH_MNT}"
720
721         __populate_check_xfs_dformat "${extents_file}" "extents"
722         __populate_check_xfs_dformat "${btree_file}" "btree"
723         __populate_check_xfs_dir "${inline_dir}" "inline"
724         __populate_check_xfs_dir "${block_dir}" "block"
725         __populate_check_xfs_dir "${leaf_dir}" "leaf"
726         __populate_check_xfs_dir "${leafn_dir}" "leafn"
727         __populate_check_xfs_dir "${node_dir}" "node"
728         __populate_check_xfs_dir "${btree_dir}" "btree"
729         __populate_check_xfs_dformat "${btree_dir}" "btree"
730         __populate_check_xfs_dformat "${bdev}" "dev"
731         __populate_check_xfs_dformat "${cdev}" "dev"
732         __populate_check_xfs_dformat "${fifo}" "dev"
733         __populate_check_xfs_attr "${local_attr}" "local"
734         __populate_check_xfs_attr "${leaf_attr}" "leaf"
735         __populate_check_xfs_attr "${node_attr}" "node"
736         __populate_check_xfs_attr "${btree_attr}" "btree"
737         __populate_check_xfs_aformat "${btree_attr}" "btree"
738         __populate_check_xfs_agbtree_height "bno"
739         __populate_check_xfs_agbtree_height "cnt"
740         __populate_check_xfs_agbtree_height "ino"
741         test $is_finobt -ne 0 && __populate_check_xfs_agbtree_height "fino"
742         test $is_rmapbt -ne 0 && __populate_check_xfs_agbtree_height "rmap"
743         test $is_reflink -ne 0 && __populate_check_xfs_agbtree_height "refcnt"
744 }
745
746 # Check data fork format of ext4 file
747 __populate_check_ext4_dformat() {
748         dev="${SCRATCH_DEV}"
749         inode="$1"
750         format="$2"
751
752         extents=0
753         etree=0
754         debugfs -R "stat <${inode}>" "${dev}" 2> /dev/null | grep 'ETB[0-9]' -q && etree=1
755         iflags="$(_ext4_get_inum_iflags "${dev}" "${inode}")"
756         test "$(echo "${iflags}" | awk '{print and(strtonum($1), 0x80000);}')" -gt 0 && extents=1
757
758         case "${format}" in
759         "blockmap")
760                 test "${extents}" -eq 0 || __populate_fail "failed to create ino ${inode} with blockmap"
761                 ;;
762         "extent"|"extents")
763                 test "${extents}" -eq 1 || __populate_fail "failed to create ino ${inode} with extents"
764                 ;;
765         "etree")
766                 (test "${extents}" -eq 1 && test "${etree}" -eq 1) || __populate_fail "failed to create ino ${inode} with extent tree"
767                 ;;
768         *)
769                 _fail "Unknown dformat ${format}"
770         esac
771 }
772
773 # Check attr fork format of ext4 file
774 __populate_check_ext4_aformat() {
775         dev="${SCRATCH_DEV}"
776         inode="$1"
777         format="$2"
778
779         ablock=1
780         debugfs -R "stat <${inode}>" "${dev}" 2> /dev/null | grep 'File ACL: 0' -q && ablock=0
781
782         case "${format}" in
783         "local"|"inline")
784                 test "${ablock}" -eq 0 || __populate_fail "failed to create inode ${inode} with ${format} xattr"
785                 ;;
786         "block")
787                 test "${extents}" -eq 1 || __populate_fail "failed to create inode ${inode} with ${format} xattr"
788                 ;;
789         *)
790                 _fail "Unknown aformat ${format}"
791         esac
792 }
793
794 # Check structure of ext4 dir
795 __populate_check_ext4_dir() {
796         dev="${SCRATCH_DEV}"
797         inode="$1"
798         dtype="$2"
799
800         htree=0
801         inline=0
802         iflags="$(_ext4_get_inum_iflags "${dev}" "${inode}")"
803         test "$(echo "${iflags}" | awk '{print and(strtonum($1), 0x1000);}')" -gt 0 && htree=1
804         test "$(echo "${iflags}" | awk '{print and(strtonum($1), 0x10000000);}')" -gt 0 && inline=1
805
806         case "${dtype}" in
807         "inline")
808                 (test "${inline}" -eq 1 && test "${htree}" -eq 0) || __populate_fail "failed to create ${dtype} dir ino ${inode} htree ${htree} inline ${inline}"
809                 ;;
810         "block")
811                 (test "${inline}" -eq 0 && test "${htree}" -eq 0) || __populate_fail "failed to create ${dtype} dir ino ${inode} htree ${htree} inline ${inline}"
812                 ;;
813         "htree")
814                 (test "${inline}" -eq 0 && test "${htree}" -eq 1) || __populate_fail "failed to create ${dtype} dir ino ${inode} htree ${htree} inline ${inline}"
815                 ;;
816         *)
817                 _fail "Unknown directory type ${dtype}"
818                 ;;
819         esac
820 }
821
822 # Check that populate created all the types of files we wanted
823 _scratch_ext4_populate_check() {
824         _scratch_mount
825         extents_file="$(__populate_find_inode "${SCRATCH_MNT}/S_IFREG.FMT_EXTENTS")"
826         etree_file="$(__populate_find_inode "${SCRATCH_MNT}/S_IFREG.FMT_ETREE")"
827         block_dir="$(__populate_find_inode "${SCRATCH_MNT}/S_IFDIR.FMT_BLOCK")"
828         htree_dir="$(__populate_find_inode "${SCRATCH_MNT}/S_IFDIR.FMT_HTREE")"
829         extents_slink="$(__populate_find_inode "${SCRATCH_MNT}/S_IFLNK.FMT_EXTENTS")"
830         local_attr="$(__populate_find_inode "${SCRATCH_MNT}/ATTR.FMT_LOCAL")"
831         block_attr="$(__populate_find_inode "${SCRATCH_MNT}/ATTR.FMT_BLOCK")"
832         umount "${SCRATCH_MNT}"
833
834         __populate_check_ext4_dformat "${extents_file}" "extents"
835         __populate_check_ext4_dformat "${etree_file}" "etree"
836         __populate_check_ext4_dir "${block_dir}" "block"
837         __populate_check_ext4_dir "${htree_dir}" "htree"
838         __populate_check_ext4_dformat "${extents_slink}" "extents"
839         __populate_check_ext4_aformat "${local_attr}" "local"
840         __populate_check_ext4_aformat "${block_attr}" "block"
841 }
842
843 # Populate a scratch FS and check the contents to make sure we got that
844 _scratch_populate() {
845         case "${FSTYP}" in
846         "xfs")
847                 _scratch_xfs_populate
848                 _scratch_xfs_populate_check
849                 ;;
850         "ext2"|"ext3"|"ext4")
851                 _scratch_ext4_populate
852                 _scratch_ext4_populate_check
853                 ;;
854         *)
855                 _fail "Don't know how to populate a ${FSTYP} filesystem."
856                 ;;
857         esac
858 }
859
860 # Fill a file system by repeatedly creating files in the given folder
861 # starting with the given file size.  Files are reduced in size when
862 # they can no longer fit until no more files can be created.
863 _fill_fs()
864 {
865         local file_size=$1
866         local dir=$2
867         local block_size=$3
868         local switch_user=$4
869         local file_count=1
870         local bytes_written=0
871         local use_falloc=1;
872
873         if [ $# -ne 4 ]; then
874                 echo "Usage: _fill_fs filesize dir blocksize switch_user"
875                 exit 1
876         fi
877
878         if [ $switch_user -eq 0 ]; then
879                 mkdir -p $dir
880         else
881                 _user_do "mkdir -p $dir"
882         fi
883         if [ ! -d $dir ]; then
884                 return 0;
885         fi
886
887         testio=`$XFS_IO_PROG -F -fc "falloc 0 $block_size" $dir/$$.xfs_io 2>&1`
888         echo $testio | grep -q "not found" && use_falloc=0
889         echo $testio | grep -q "Operation not supported" && use_falloc=0
890
891         if [ $file_size -lt $block_size ]; then
892                 $file_size = $block_size
893         fi
894
895         while [ $file_size -ge $block_size ]; do
896                 bytes_written=0
897                 if [ $switch_user -eq 0 ]; then
898                         if [ $use_falloc -eq 0 ]; then
899                                 $XFS_IO_PROG -fc "pwrite -b 8388608 0 $file_size" \
900                                         $dir/$file_count
901                         else
902                                 $XFS_IO_PROG -fc "falloc 0 $file_size" \
903                                         $dir/$file_count
904                         fi
905                 else
906                         if [ $use_falloc -eq 0 ]; then
907                                 _user_do "$XFS_IO_PROG -f -c \"pwrite -b 8388608 0 \
908                                         $file_size\" $dir/$file_count"
909                         else
910                                 _user_do "$XFS_IO_PROG -f -c \"falloc 0 \
911                                         $file_size\" $dir/$file_count"
912                         fi
913                 fi
914
915                 if [ -f $dir/$file_count ]; then
916                         bytes_written=$(_get_filesize $dir/$file_count)
917                 fi
918
919                 # If there was no room to make the file, then divide it in
920                 # half, and keep going
921                 if [ $bytes_written -lt $file_size ]; then
922                         file_size=$((file_size / 2))
923                 fi
924                 file_count=$((file_count + 1))
925         done
926 }
927
928 # Compute the fs geometry description of a populated filesystem
929 _scratch_populate_cache_tag() {
930         local extra_descr=""
931         local size="$(blockdev --getsz "${SCRATCH_DEV}")"
932         local logdev_sz="none"
933         local rtdev_sz="none"
934
935         if [ "${USE_EXTERNAL}" = "yes" ] && [ -n "${SCRATCH_LOGDEV}" ]; then
936                 logdev_sz="$(blockdev --getsz "${SCRATCH_LOGDEV}")"
937         fi
938
939         if [ "${USE_EXTERNAL}" = "yes" ] && [ -n "${SCRATCH_RTDEV}" ]; then
940                 rtdev_sz="$(blockdev --getsz "${SCRATCH_RTDEV}")"
941         fi
942
943         case "${FSTYP}" in
944         "ext4")
945                 extra_descr="LOGDEV_SIZE ${logdev_sz}"
946                 ;;
947         "xfs")
948                 extra_descr="LOGDEV_SIZE ${logdev_sz} RTDEV_SIZE ${rtdev_sz}"
949                 _populate_xfs_qmount_option
950                 if echo "${MOUNT_OPTIONS}" | grep -q 'usrquota'; then
951                         extra_descr="${extra_descr} QUOTAS"
952                 fi
953                 ;;
954         esac
955         echo "FSTYP ${FSTYP} MKFS_OPTIONS ${MKFS_OPTIONS} SIZE ${size} ${extra_descr} ARGS $@"
956 }
957
958 # Restore a cached populated fs from a metadata dump
959 _scratch_populate_restore_cached() {
960         local metadump="$1"
961
962         case "${FSTYP}" in
963         "xfs")
964                 _xfs_mdrestore "${metadump}" "${SCRATCH_DEV}"
965                 res=$?
966                 test $res -ne 0 && return $res
967
968                 # Cached images should have been unmounted cleanly, so if
969                 # there's an external log we need to wipe it and run repair to
970                 # format it to match this filesystem.
971                 if [ -n "${SCRATCH_LOGDEV}" ]; then
972                         $WIPEFS_PROG -a "${SCRATCH_LOGDEV}"
973                         _scratch_xfs_repair
974                         res=$?
975                 fi
976                 return $res
977                 ;;
978         "ext2"|"ext3"|"ext4")
979                 _ext4_mdrestore "${metadump}" "${SCRATCH_DEV}"
980                 ret=$?
981                 test $ret -ne 0 && return $ret
982
983                 # ext4 cannot e2image external logs, so we have to reformat
984                 # the scratch device to match the restored fs
985                 if [ -n "${SCRATCH_LOGDEV}" ]; then
986                         local fsuuid="$($DUMPE2FS_PROG -h "${SCRATCH_DEV}" 2>/dev/null | \
987                                         grep 'Journal UUID:' | \
988                                         sed -e 's/Journal UUID:[[:space:]]*//g')"
989                         $MKFS_EXT4_PROG -O journal_dev "${SCRATCH_LOGDEV}" \
990                                         -F -U "${fsuuid}"
991                 fi
992                 return 0
993                 ;;
994         esac
995         return 1
996 }
997
998 # Populate a scratch FS from scratch or from a cached image.
999 _scratch_populate_cached() {
1000         local meta_descr="$(_scratch_populate_cache_tag "$@")"
1001         local meta_tag="$(echo "${meta_descr}" | md5sum - | cut -d ' ' -f 1)"
1002         local metadump_stem="${TEST_DIR}/__populate.${FSTYP}.${meta_tag}"
1003
1004         # This variable is shared outside this function
1005         POPULATE_METADUMP="${metadump_stem}.metadump"
1006         local populate_metadump_descr="${metadump_stem}.txt"
1007
1008         # Don't keep metadata images cached for more 48 hours...
1009         rm -rf "$(find "${POPULATE_METADUMP}" -mtime +2 2>/dev/null)"
1010
1011         # Throw away cached image if it doesn't match our spec.
1012         cmp -s "${populate_metadump_descr}" <(echo "${meta_descr}") || \
1013                 rm -rf "${POPULATE_METADUMP}"
1014
1015         # Try to restore from the metadump
1016         _scratch_populate_restore_cached "${POPULATE_METADUMP}" && \
1017                 return
1018
1019         # Oh well, just create one from scratch
1020         _scratch_mkfs
1021         echo "${meta_descr}" > "${populate_metadump_descr}"
1022         case "${FSTYP}" in
1023         "xfs")
1024                 _scratch_xfs_populate $@
1025                 _scratch_xfs_populate_check
1026
1027                 local logdev=none
1028                 [ "$USE_EXTERNAL" = yes -a ! -z "$SCRATCH_LOGDEV" ] && \
1029                         logdev=$SCRATCH_LOGDEV
1030
1031                 _xfs_metadump "$POPULATE_METADUMP" "$SCRATCH_DEV" "$logdev" \
1032                         compress -a -o
1033                 ;;
1034         "ext2"|"ext3"|"ext4")
1035                 _scratch_ext4_populate $@
1036                 _scratch_ext4_populate_check
1037                 _ext4_metadump "${SCRATCH_DEV}" "${POPULATE_METADUMP}" compress
1038                 ;;
1039         *)
1040                 _fail "Don't know how to populate a ${FSTYP} filesystem."
1041                 ;;
1042         esac
1043 }