d484866abbfe4f5b2f3eb7e189d605c11f11268e
[xfstests-dev.git] / common / populate
1 ##/bin/bash
2 # SPDX-License-Identifier: GPL-2.0+
3 # Copyright (c) 2015 Oracle.  All Rights Reserved.
4 #
5 # Routines for populating a scratch fs, and helpers to exercise an FS
6 # once it's been fuzzed.
7
8 . ./common/quota
9
10 _require_populate_commands() {
11         _require_xfs_io_command "falloc"
12         _require_xfs_io_command "fpunch"
13         _require_test_program "punch-alternating"
14         _require_command "$XFS_DB_PROG" "xfs_db"
15 }
16
17 _require_xfs_db_blocktrash_z_command() {
18         test "${FSTYP}" = "xfs" || _notrun "cannot run xfs_db on ${FSTYP}"
19         $XFS_DB_PROG -x -f -c 'blocktrash -z' "${TEST_DEV}" | grep -q 'nothing on stack' || _notrun "blocktrash -z not supported"
20 }
21
22 # Attempt to make files of "every" format for data, dirs, attrs etc.
23 # (with apologies to Eric Sandeen for mutating xfser.sh)
24
25 # Create a file of a given size.
26 __populate_create_file() {
27         local sz="$1"
28         local fname="$2"
29
30         $XFS_IO_PROG -f -c "pwrite -S 0x62 -W -b 1m 0 $sz" "${fname}"
31 }
32
33 # Punch out every other hole in this file, if it exists.
34 #
35 # The goal here is to force the creation of a large number of metadata records
36 # by creating a lot of tiny extent mappings in a file.  Callers should ensure
37 # that fragmenting the file actually causes record creation.  Call this
38 # function /after/ creating all other metadata structures.
39 __populate_fragment_file() {
40         local fname="$1"
41
42         test -f "${fname}" && $here/src/punch-alternating "${fname}"
43 }
44
45 # Create a large directory
46 __populate_create_dir() {
47         name="$1"
48         nr="$2"
49         missing="$3"
50
51         mkdir -p "${name}"
52         seq 0 "${nr}" | while read d; do
53                 creat=mkdir
54                 test "$((d % 20))" -eq 0 && creat=touch
55                 $creat "${name}/$(printf "%.08d" "$d")"
56         done
57
58         test -z "${missing}" && return
59         seq 1 2 "${nr}" | while read d; do
60                 rm -rf "${name}/$(printf "%.08d" "$d")"
61         done
62 }
63
64 # Add a bunch of attrs to a file
65 __populate_create_attr() {
66         name="$1"
67         nr="$2"
68         missing="$3"
69
70         touch "${name}"
71         seq 0 "${nr}" | while read d; do
72                 setfattr -n "user.$(printf "%.08d" "$d")" -v "$(printf "%.08d" "$d")" "${name}"
73         done
74
75         test -z "${missing}" && return
76         seq 1 2 "${nr}" | while read d; do
77                 setfattr -x "user.$(printf "%.08d" "$d")" "${name}"
78         done
79 }
80
81 # Fill up some percentage of the remaining free space
82 __populate_fill_fs() {
83         dir="$1"
84         pct="$2"
85         test -z "${pct}" && pct=60
86
87         mkdir -p "${dir}/test/1"
88         cp -pRdu "${dir}"/S_IFREG* "${dir}/test/1/"
89
90         SRC_SZ="$(du -ks "${dir}/test/1" | cut -f 1)"
91         FS_SZ="$(( $(stat -f "${dir}" -c '%a * %S') / 1024 ))"
92
93         NR="$(( (FS_SZ * ${pct} / 100) / SRC_SZ ))"
94
95         echo "FILL FS"
96         echo "src_sz $SRC_SZ fs_sz $FS_SZ nr $NR"
97         seq 2 "${NR}" | while read nr; do
98                 cp -pRdu "${dir}/test/1" "${dir}/test/${nr}"
99         done
100 }
101
102 # For XFS, force on all the quota options if quota is enabled
103 # and the user didn't feed us noquota.
104 _populate_xfs_qmount_option()
105 {
106         # User explicitly told us not to quota
107         if echo "${MOUNT_OPTIONS}" | grep -q 'noquota'; then
108                 return
109         fi
110
111         # Don't bother if we can't turn on quotas
112         if [ ! -f /proc/fs/xfs/xqmstat ]; then
113                 # No quota support
114                 return
115         elif [ "${USE_EXTERNAL}" = "yes" ] && [ ! -z "${SCRATCH_RTDEV}" ]; then
116                 # Quotas not supported on rt filesystems
117                 return
118         elif [ -z "${XFS_QUOTA_PROG}" ]; then
119                 # xfs quota tools not installed
120                 return
121         fi
122
123         # Turn on all the quotas
124         if $XFS_INFO_PROG "${TEST_DIR}" | grep -q 'crc=1'; then
125                 # v5 filesystems can have group & project quotas
126                 quota="usrquota,grpquota,prjquota"
127         else
128                 # v4 filesystems cannot mix group & project quotas
129                 quota="usrquota,grpquota"
130         fi
131
132         # Inject our quota mount options
133         if echo "${MOUNT_OPTIONS}" | grep -q "${quota}"; then
134                 return
135         elif echo "${MOUNT_OPTIONS}" | egrep -q '(quota|noenforce)'; then
136                 _qmount_option "${quota}"
137         else
138                 export MOUNT_OPTIONS="$MOUNT_OPTIONS -o ${quota}"
139                 echo "MOUNT_OPTIONS = $MOUNT_OPTIONS" >>$seqres.full
140         fi
141 }
142
143 # Populate an XFS on the scratch device with (we hope) all known
144 # types of metadata block
145 _scratch_xfs_populate() {
146         fill=1
147
148         for arg in $@; do
149                 case "${arg}" in
150                 "nofill")
151                         fill=0;;
152                 esac
153         done
154
155         _populate_xfs_qmount_option
156         _scratch_mount
157
158         # We cannot directly force the filesystem to create the metadata
159         # structures we want; we can only achieve this indirectly by carefully
160         # crafting files and a directory tree.  Therefore, we must have exact
161         # control over the layout and device selection of all files created.
162         # Clear the rtinherit flag on the root directory so that files are
163         # always created on the data volume regardless of MKFS_OPTIONS.  We can
164         # set the realtime flag when needed.
165         $XFS_IO_PROG -c 'chattr -t' $SCRATCH_MNT
166
167         blksz="$(stat -f -c '%s' "${SCRATCH_MNT}")"
168         dblksz="$($XFS_INFO_PROG "${SCRATCH_MNT}" | grep naming.*bsize | sed -e 's/^.*bsize=//g' -e 's/\([0-9]*\).*$/\1/g')"
169         crc="$($XFS_INFO_PROG "${SCRATCH_MNT}" | grep crc= | sed -e 's/^.*crc=//g' -e 's/\([0-9]*\).*$/\1/g')"
170         if [ $crc -eq 1 ]; then
171                 leaf_hdr_size=64
172         else
173                 leaf_hdr_size=16
174         fi
175         leaf_lblk="$((32 * 1073741824 / blksz))"
176         node_lblk="$((64 * 1073741824 / blksz))"
177
178         # Data:
179
180         # Fill up the root inode chunk
181         echo "+ fill root ino chunk"
182         seq 1 64 | while read f; do
183                 $XFS_IO_PROG -f -c "truncate 0" "${SCRATCH_MNT}/dummy${f}"
184         done
185
186         # Regular files
187         # - FMT_EXTENTS
188         echo "+ extents file"
189         __populate_create_file $blksz "${SCRATCH_MNT}/S_IFREG.FMT_EXTENTS"
190
191         # - FMT_BTREE
192         echo "+ btree extents file"
193         nr="$((blksz * 2 / 16))"
194         __populate_create_file $((blksz * nr)) "${SCRATCH_MNT}/S_IFREG.FMT_BTREE"
195
196         # Directories
197         # - INLINE
198         echo "+ inline dir"
199         __populate_create_dir "${SCRATCH_MNT}/S_IFDIR.FMT_INLINE" 1
200
201         # - BLOCK
202         echo "+ block dir"
203         __populate_create_dir "${SCRATCH_MNT}/S_IFDIR.FMT_BLOCK" "$((dblksz / 40))"
204
205         # - LEAF
206         echo "+ leaf dir"
207         __populate_create_dir "${SCRATCH_MNT}/S_IFDIR.FMT_LEAF" "$((dblksz / 12))"
208
209         # - LEAFN
210         echo "+ leafn dir"
211         __populate_create_dir "${SCRATCH_MNT}/S_IFDIR.FMT_LEAFN" "$(( ((dblksz - leaf_hdr_size) / 8) - 3 ))"
212
213         # - NODE
214         echo "+ node dir"
215         __populate_create_dir "${SCRATCH_MNT}/S_IFDIR.FMT_NODE" "$((16 * dblksz / 40))" true
216
217         # - BTREE
218         echo "+ btree dir"
219         __populate_create_dir "${SCRATCH_MNT}/S_IFDIR.FMT_BTREE" "$((128 * dblksz / 40))" true
220
221         # Symlinks
222         # - FMT_LOCAL
223         echo "+ inline symlink"
224         ln -s target "${SCRATCH_MNT}/S_IFLNK.FMT_LOCAL"
225
226         # - FMT_EXTENTS
227         echo "+ extents symlink"
228         ln -s "$(perl -e 'print "x" x 1023;')" "${SCRATCH_MNT}/S_IFLNK.FMT_EXTENTS"
229
230         # Char & block
231         echo "+ special"
232         mknod "${SCRATCH_MNT}/S_IFCHR" c 1 1
233         mknod "${SCRATCH_MNT}/S_IFBLK" b 1 1
234         mknod "${SCRATCH_MNT}/S_IFIFO" p
235
236         # special file with an xattr
237         setfacl -P -m u:nobody:r ${SCRATCH_MNT}/S_IFCHR
238
239         # Attribute formats
240         # LOCAL
241         echo "+ local attr"
242         __populate_create_attr "${SCRATCH_MNT}/ATTR.FMT_LOCAL" 1
243
244         # LEAF
245         echo "+ leaf attr"
246         __populate_create_attr "${SCRATCH_MNT}/ATTR.FMT_LEAF" "$((blksz / 40))"
247
248         # NODE
249         echo "+ node attr"
250         __populate_create_attr "${SCRATCH_MNT}/ATTR.FMT_NODE" "$((8 * blksz / 40))"
251
252         # BTREE
253         echo "+ btree attr"
254         __populate_create_attr "${SCRATCH_MNT}/ATTR.FMT_BTREE" "$((64 * blksz / 40))" true
255
256         # trusted namespace
257         touch ${SCRATCH_MNT}/ATTR.TRUSTED
258         setfattr -n trusted.moo -v urk ${SCRATCH_MNT}/ATTR.TRUSTED
259
260         # security namespace
261         touch ${SCRATCH_MNT}/ATTR.SECURITY
262         setfattr -n security.foo -v bar ${SCRATCH_MNT}/ATTR.SECURITY
263
264         # system namespace
265         touch ${SCRATCH_MNT}/ATTR.SYSTEM
266         setfacl -m u:root:r ${SCRATCH_MNT}/ATTR.SYSTEM
267
268         # FMT_EXTENTS with a remote less-than-a-block value
269         echo "+ attr extents with a remote less-than-a-block value"
270         touch "${SCRATCH_MNT}/ATTR.FMT_EXTENTS_REMOTE3K"
271         $XFS_IO_PROG -f -c "pwrite -S 0x43 0 $((blksz - 300))" "${SCRATCH_MNT}/attrvalfile" > /dev/null
272         attr -q -s user.remotebtreeattrname "${SCRATCH_MNT}/ATTR.FMT_EXTENTS_REMOTE3K" < "${SCRATCH_MNT}/attrvalfile"
273
274         # FMT_EXTENTS with a remote block-size value
275         echo "+ attr extents with a remote one-block value"
276         touch "${SCRATCH_MNT}/ATTR.FMT_EXTENTS_REMOTE4K"
277         $XFS_IO_PROG -f -c "pwrite -S 0x44 0 ${blksz}" "${SCRATCH_MNT}/attrvalfile" > /dev/null
278         attr -q -s user.remotebtreeattrname "${SCRATCH_MNT}/ATTR.FMT_EXTENTS_REMOTE4K" < "${SCRATCH_MNT}/attrvalfile"
279         rm -rf "${SCRATCH_MNT}/attrvalfile"
280
281         # Make an unused inode
282         echo "+ empty file"
283         touch "${SCRATCH_MNT}/unused"
284         $XFS_IO_PROG -f -c 'fsync' "${SCRATCH_MNT}/unused"
285         rm -rf "${SCRATCH_MNT}/unused"
286
287         # Free space btree
288         echo "+ freesp btree"
289         nr="$((blksz * 2 / 8))"
290         __populate_create_file $((blksz * nr)) "${SCRATCH_MNT}/BNOBT"
291
292         # Inode btree
293         echo "+ inobt btree"
294         local ino_per_rec=64
295         local rec_per_btblock=16
296         local nr="$(( 2 * (blksz / rec_per_btblock) * ino_per_rec ))"
297         local dir="${SCRATCH_MNT}/INOBT"
298         mkdir -p "${dir}"
299         seq 0 "${nr}" | while read f; do
300                 touch "${dir}/${f}"
301         done
302
303         seq 0 2 "${nr}" | while read f; do
304                 rm -f "${dir}/${f}"
305         done
306
307         # Reverse-mapping btree
308         is_rmapbt="$($XFS_INFO_PROG "${SCRATCH_MNT}" | grep -c 'rmapbt=1')"
309         if [ $is_rmapbt -gt 0 ]; then
310                 echo "+ rmapbt btree"
311                 nr="$((blksz * 2 / 24))"
312                 __populate_create_file $((blksz * nr)) "${SCRATCH_MNT}/RMAPBT"
313         fi
314
315         # Realtime Reverse-mapping btree
316         is_rt="$($XFS_INFO_PROG "${SCRATCH_MNT}" | grep -c 'rtextents=[1-9]')"
317         if [ $is_rmapbt -gt 0 ] && [ $is_rt -gt 0 ]; then
318                 echo "+ rtrmapbt btree"
319                 nr="$((blksz * 2 / 32))"
320                 $XFS_IO_PROG -R -f -c 'truncate 0' "${SCRATCH_MNT}/RTRMAPBT"
321                 __populate_create_file $((blksz * nr)) "${SCRATCH_MNT}/RTRMAPBT"
322         fi
323
324         # Reference-count btree
325         is_reflink="$($XFS_INFO_PROG "${SCRATCH_MNT}" | grep -c 'reflink=1')"
326         if [ $is_reflink -gt 0 ]; then
327                 echo "+ reflink btree"
328                 nr="$((blksz * 2 / 12))"
329                 __populate_create_file $((blksz * nr)) "${SCRATCH_MNT}/REFCOUNTBT"
330                 cp --reflink=always "${SCRATCH_MNT}/REFCOUNTBT" "${SCRATCH_MNT}/REFCOUNTBT2"
331         fi
332
333         # Copy some real files (xfs tests, I guess...)
334         echo "+ real files"
335         test $fill -ne 0 && __populate_fill_fs "${SCRATCH_MNT}" 5
336
337         # Make sure we get all the fragmentation we asked for
338         __populate_fragment_file "${SCRATCH_MNT}/S_IFREG.FMT_BTREE"
339         __populate_fragment_file "${SCRATCH_MNT}/BNOBT"
340         __populate_fragment_file "${SCRATCH_MNT}/RMAPBT"
341         __populate_fragment_file "${SCRATCH_MNT}/RTRMAPBT"
342         __populate_fragment_file "${SCRATCH_MNT}/REFCOUNTBT"
343
344         umount "${SCRATCH_MNT}"
345 }
346
347 # Populate an ext4 on the scratch device with (we hope) all known
348 # types of metadata block
349 _scratch_ext4_populate() {
350         fill=1
351
352         for arg in $@; do
353                 case "${arg}" in
354                 "nofill")
355                         fill=0;;
356                 esac
357         done
358
359         _scratch_mount
360         blksz="$(stat -f -c '%s' "${SCRATCH_MNT}")"
361         dblksz="${blksz}"
362         leaf_lblk="$((32 * 1073741824 / blksz))"
363         node_lblk="$((64 * 1073741824 / blksz))"
364
365         # Data:
366
367         # Regular files
368         # - FMT_INLINE
369         echo "+ inline file"
370         __populate_create_file 1 "${SCRATCH_MNT}/S_IFREG.FMT_INLINE"
371
372         # - FMT_EXTENTS
373         echo "+ extents file"
374         __populate_create_file $blksz "${SCRATCH_MNT}/S_IFREG.FMT_EXTENTS"
375
376         # - FMT_ETREE
377         echo "+ extent tree file"
378         nr="$((blksz * 2 / 12))"
379         __populate_create_file $((blksz * nr)) "${SCRATCH_MNT}/S_IFREG.FMT_ETREE"
380
381         # Directories
382         # - INLINE
383         echo "+ inline dir"
384         __populate_create_dir "${SCRATCH_MNT}/S_IFDIR.FMT_INLINE" 1
385
386         # - BLOCK
387         echo "+ block dir"
388         __populate_create_dir "${SCRATCH_MNT}/S_IFDIR.FMT_BLOCK" "$((dblksz / 32))"
389
390         # - HTREE
391         echo "+ htree dir"
392         __populate_create_dir "${SCRATCH_MNT}/S_IFDIR.FMT_HTREE" "$((4 * dblksz / 24))"
393
394         # Symlinks
395         # - FMT_LOCAL
396         echo "+ inline symlink"
397         ln -s target "${SCRATCH_MNT}/S_IFLNK.FMT_LOCAL"
398
399         # - FMT_EXTENTS
400         echo "+ extents symlink"
401         ln -s "$(perl -e 'print "x" x 1023;')" "${SCRATCH_MNT}/S_IFLNK.FMT_EXTENTS"
402
403         # Char & block
404         echo "+ special"
405         mknod "${SCRATCH_MNT}/S_IFCHR" c 1 1
406         mknod "${SCRATCH_MNT}/S_IFBLK" b 1 1
407         mknod "${SCRATCH_MNT}/S_IFIFO" p
408
409         # special file with an xattr
410         setfacl -P -m u:nobody:r ${SCRATCH_MNT}/S_IFCHR
411
412         # Attribute formats
413         # LOCAL
414         echo "+ local attr"
415         __populate_create_attr "${SCRATCH_MNT}/ATTR.FMT_LOCAL" 0
416
417         # BLOCK
418         echo "+ block attr"
419         __populate_create_attr "${SCRATCH_MNT}/ATTR.FMT_BLOCK" "$((blksz / 40))"
420
421         # trusted namespace
422         touch ${SCRATCH_MNT}/ATTR.TRUSTED
423         setfattr -n trusted.moo -v urk ${SCRATCH_MNT}/ATTR.TRUSTED
424
425         # security namespace
426         touch ${SCRATCH_MNT}/ATTR.SECURITY
427         setfattr -n security.foo -v bar ${SCRATCH_MNT}/ATTR.SECURITY
428
429         # system namespace
430         touch ${SCRATCH_MNT}/ATTR.SYSTEM
431         setfacl -m u:root:r ${SCRATCH_MNT}/ATTR.SYSTEM
432
433         # Make an unused inode
434         echo "+ empty file"
435         touch "${SCRATCH_MNT}/unused"
436         $XFS_IO_PROG -f -c 'fsync' "${SCRATCH_MNT}/unused"
437         rm -rf "${SCRATCH_MNT}/unused"
438
439         # Copy some real files (xfs tests, I guess...)
440         echo "+ real files"
441         test $fill -ne 0 && __populate_fill_fs "${SCRATCH_MNT}" 5
442
443         # Make sure we get all the fragmentation we asked for
444         __populate_fragment_file "${SCRATCH_MNT}/S_IFREG.FMT_ETREE"
445
446         umount "${SCRATCH_MNT}"
447 }
448
449 # Find the inode number of a file
450 __populate_find_inode() {
451         name="$1"
452         inode="$(stat -c '%i' "${name}")"
453         echo "${inode}"
454 }
455
456 # Check data fork format of XFS file
457 __populate_check_xfs_dformat() {
458         inode="$1"
459         format="$2"
460
461         fmt="$(_scratch_xfs_db -c "inode ${inode}" -c 'p core.format' | sed -e 's/^.*(\([a-z]*\)).*$/\1/g')"
462         test "${format}" = "${fmt}" || _fail "failed to create ino ${inode} dformat expected ${format} saw ${fmt}"
463 }
464
465 # Check attr fork format of XFS file
466 __populate_check_xfs_aformat() {
467         inode="$1"
468         format="$2"
469
470         fmt="$(_scratch_xfs_db -c "inode ${inode}" -c 'p core.aformat' | sed -e 's/^.*(\([a-z]*\)).*$/\1/g')"
471         test "${format}" = "${fmt}" || _fail "failed to create ino ${inode} aformat expected ${format} saw ${fmt}"
472 }
473
474 # Check structure of XFS directory
475 __populate_check_xfs_dir() {
476         inode="$1"
477         dtype="$2"
478
479         (test -n "${leaf_lblk}" && test -n "${node_lblk}") || _fail "must define leaf_lblk and node_lblk before calling __populate_check_xfs_dir"
480         datab=0
481         leafb=0
482         freeb=0
483         #echo "== check dir ${inode} type ${dtype}" ; _scratch_xfs_db -x -c "inode ${inode}" -c "bmap"
484         _scratch_xfs_db -x -c "inode ${inode}" -c "dblock 0" -c "stack" | grep -q 'file data block is unmapped' || datab=1
485         _scratch_xfs_db -x -c "inode ${inode}" -c "dblock ${leaf_lblk}" -c "stack" | grep -q 'file data block is unmapped' || leafb=1
486         _scratch_xfs_db -x -c "inode ${inode}" -c "dblock ${node_lblk}" -c "stack" | grep -q 'file data block is unmapped' || freeb=1
487
488         case "${dtype}" in
489         "shortform"|"inline"|"local")
490                 (test "${datab}" -eq 0 && test "${leafb}" -eq 0 && test "${freeb}" -eq 0) || _fail "failed to create ${dtype} dir ino ${inode} datab ${datab} leafb ${leafb} freeb ${freeb}"
491                 ;;
492         "block")
493                 (test "${datab}" -eq 1 && test "${leafb}" -eq 0 && test "${freeb}" -eq 0) || _fail "failed to create ${dtype} dir ino ${inode} datab ${datab} leafb ${leafb} freeb ${freeb}"
494                 ;;
495         "leaf")
496                 (test "${datab}" -eq 1 && test "${leafb}" -eq 1 && test "${freeb}" -eq 0) || _fail "failed to create ${dtype} dir ino ${inode} datab ${datab} leafb ${leafb} freeb ${freeb}"
497                 ;;
498         "leafn")
499                 _scratch_xfs_db -x -c "inode ${inode}" -c "dblock ${leaf_lblk}" -c "p lhdr.info.hdr.magic" | grep -q '0x3dff' && return
500                 _scratch_xfs_db -x -c "inode ${inode}" -c "dblock ${leaf_lblk}" -c "p lhdr.info.magic" | grep -q '0xd2ff' && return
501                 _fail "failed to create ${dtype} dir ino ${inode} datab ${datab} leafb ${leafb} freeb ${freeb}"
502                 ;;
503         "node"|"btree")
504                 (test "${datab}" -eq 1 && test "${leafb}" -eq 1 && test "${freeb}" -eq 1) || _fail "failed to create ${dtype} dir ino ${inode} datab ${datab} leafb ${leafb} freeb ${freeb}"
505                 ;;
506         *)
507                 _fail "Unknown directory type ${dtype}"
508         esac
509 }
510
511 # Check structure of XFS attr
512 __populate_check_xfs_attr() {
513         inode="$1"
514         atype="$2"
515
516         datab=0
517         leafb=0
518         #echo "== check attr ${inode} type ${dtype}" ; _scratch_xfs_db -x -c "inode ${inode}" -c "bmap -a"
519         _scratch_xfs_db -x -c "inode ${inode}" -c "ablock 0" -c "stack" | grep -q 'file attr block is unmapped' || datab=1
520         _scratch_xfs_db -x -c "inode ${inode}" -c "ablock 1" -c "stack" | grep -q 'file attr block is unmapped' || leafb=1
521
522         case "${atype}" in
523         "shortform"|"inline"|"local")
524                 (test "${datab}" -eq 0 && test "${leafb}" -eq 0) || _fail "failed to create ${atype} attr ino ${inode} datab ${datab} leafb ${leafb}"
525                 ;;
526         "leaf")
527                 (test "${datab}" -eq 1 && test "${leafb}" -eq 0) || _fail "failed to create ${atype} attr ino ${inode} datab ${datab} leafb ${leafb}"
528                 ;;
529         "node"|"btree")
530                 (test "${datab}" -eq 1 && test "${leafb}" -eq 1) || _fail "failed to create ${atype} attr ino ${inode} datab ${datab} leafb ${leafb}"
531                 ;;
532         *)
533                 _fail "Unknown attribute type ${atype}"
534         esac
535 }
536
537 # Check that there's at least one per-AG btree with multiple levels
538 __populate_check_xfs_agbtree_height() {
539         bt_type="$1"
540         nr_ags=$(_scratch_xfs_db -c 'sb 0' -c 'p agcount' | awk '{print $3}')
541
542         case "${bt_type}" in
543         "bno"|"cnt"|"rmap"|"refcnt")
544                 hdr="agf"
545                 bt_prefix="${bt_type}"
546                 ;;
547         "ino")
548                 hdr="agi"
549                 bt_prefix=""
550                 ;;
551         "fino")
552                 hdr="agi"
553                 bt_prefix="free_"
554                 ;;
555         *)
556                 _fail "Don't know about AG btree ${bt_type}"
557                 ;;
558         esac
559
560         seq 0 $((nr_ags - 1)) | while read ag; do
561                 bt_level=$(_scratch_xfs_db -c "${hdr} ${ag}" -c "p ${bt_prefix}level" | awk '{print $3}')
562                 if [ "${bt_level}" -gt 1 ]; then
563                         return 100
564                 fi
565         done
566         test $? -eq 100 || _fail "Failed to create ${bt_type} of sufficient height!"
567         return 1
568 }
569
570 # Check that populate created all the types of files we wanted
571 _scratch_xfs_populate_check() {
572         _scratch_mount
573         extents_file="$(__populate_find_inode "${SCRATCH_MNT}/S_IFREG.FMT_EXTENTS")"
574         btree_file="$(__populate_find_inode "${SCRATCH_MNT}/S_IFREG.FMT_BTREE")"
575         inline_dir="$(__populate_find_inode "${SCRATCH_MNT}/S_IFDIR.FMT_INLINE")"
576         block_dir="$(__populate_find_inode "${SCRATCH_MNT}/S_IFDIR.FMT_BLOCK")"
577         leaf_dir="$(__populate_find_inode "${SCRATCH_MNT}/S_IFDIR.FMT_LEAF")"
578         leafn_dir="$(__populate_find_inode "${SCRATCH_MNT}/S_IFDIR.FMT_LEAFN")"
579         node_dir="$(__populate_find_inode "${SCRATCH_MNT}/S_IFDIR.FMT_NODE")"
580         btree_dir="$(__populate_find_inode "${SCRATCH_MNT}/S_IFDIR.FMT_BTREE")"
581         local_slink="$(__populate_find_inode "${SCRATCH_MNT}/S_IFLNK.FMT_LOCAL")"
582         extents_slink="$(__populate_find_inode "${SCRATCH_MNT}/S_IFLNK.FMT_EXTENTS")"
583         bdev="$(__populate_find_inode "${SCRATCH_MNT}/S_IFBLK")"
584         cdev="$(__populate_find_inode "${SCRATCH_MNT}/S_IFCHR")"
585         fifo="$(__populate_find_inode "${SCRATCH_MNT}/S_IFIFO")"
586         local_attr="$(__populate_find_inode "${SCRATCH_MNT}/ATTR.FMT_LOCAL")"
587         leaf_attr="$(__populate_find_inode "${SCRATCH_MNT}/ATTR.FMT_LEAF")"
588         node_attr="$(__populate_find_inode "${SCRATCH_MNT}/ATTR.FMT_NODE")"
589         btree_attr="$(__populate_find_inode "${SCRATCH_MNT}/ATTR.FMT_BTREE")"
590         is_finobt=$($XFS_INFO_PROG "${SCRATCH_MNT}" | grep -c 'finobt=1')
591         is_rmapbt=$($XFS_INFO_PROG "${SCRATCH_MNT}" | grep -c 'rmapbt=1')
592         is_reflink=$($XFS_INFO_PROG "${SCRATCH_MNT}" | grep -c 'reflink=1')
593
594         blksz="$(stat -f -c '%s' "${SCRATCH_MNT}")"
595         dblksz="$($XFS_INFO_PROG "${SCRATCH_MNT}" | grep naming.*bsize | sed -e 's/^.*bsize=//g' -e 's/\([0-9]*\).*$/\1/g')"
596         leaf_lblk="$((32 * 1073741824 / blksz))"
597         node_lblk="$((64 * 1073741824 / blksz))"
598         umount "${SCRATCH_MNT}"
599
600         __populate_check_xfs_dformat "${extents_file}" "extents"
601         __populate_check_xfs_dformat "${btree_file}" "btree"
602         __populate_check_xfs_dir "${inline_dir}" "inline"
603         __populate_check_xfs_dir "${block_dir}" "block"
604         __populate_check_xfs_dir "${leaf_dir}" "leaf"
605         __populate_check_xfs_dir "${leafn_dir}" "leafn"
606         __populate_check_xfs_dir "${node_dir}" "node"
607         __populate_check_xfs_dir "${btree_dir}" "btree"
608         __populate_check_xfs_dformat "${btree_dir}" "btree"
609         __populate_check_xfs_dformat "${bdev}" "dev"
610         __populate_check_xfs_dformat "${cdev}" "dev"
611         __populate_check_xfs_dformat "${fifo}" "dev"
612         __populate_check_xfs_attr "${local_attr}" "local"
613         __populate_check_xfs_attr "${leaf_attr}" "leaf"
614         __populate_check_xfs_attr "${node_attr}" "node"
615         __populate_check_xfs_attr "${btree_attr}" "btree"
616         __populate_check_xfs_aformat "${btree_attr}" "btree"
617         __populate_check_xfs_agbtree_height "bno"
618         __populate_check_xfs_agbtree_height "cnt"
619         __populate_check_xfs_agbtree_height "ino"
620         test $is_finobt -ne 0 && __populate_check_xfs_agbtree_height "fino"
621         test $is_rmapbt -ne 0 && __populate_check_xfs_agbtree_height "rmap"
622         test $is_reflink -ne 0 && __populate_check_xfs_agbtree_height "refcnt"
623 }
624
625 # Check data fork format of ext4 file
626 __populate_check_ext4_dformat() {
627         dev="${SCRATCH_DEV}"
628         inode="$1"
629         format="$2"
630
631         extents=0
632         etree=0
633         debugfs -R "stat <${inode}>" "${dev}" 2> /dev/null | grep 'ETB[0-9]' -q && etree=1
634         iflags="$(debugfs -R "stat <${inode}>" "${dev}" 2> /dev/null | grep 'Flags:' | sed -e 's/^.*Flags: \([0-9a-fx]*\).*$/\1/g')"
635         test "$(echo "${iflags}" | awk '{print and(strtonum($1), 0x80000);}')" -gt 0 && extents=1
636
637         case "${format}" in
638         "blockmap")
639                 test "${extents}" -eq 0 || _fail "failed to create ino ${inode} with blockmap"
640                 ;;
641         "extent"|"extents")
642                 test "${extents}" -eq 1 || _fail "failed to create ino ${inode} with extents"
643                 ;;
644         "etree")
645                 (test "${extents}" -eq 1 && test "${etree}" -eq 1) || _fail "failed to create ino ${inode} with extent tree"
646                 ;;
647         *)
648                 _fail "Unknown dformat ${format}"
649         esac
650 }
651
652 # Check attr fork format of ext4 file
653 __populate_check_ext4_aformat() {
654         dev="${SCRATCH_DEV}"
655         inode="$1"
656         format="$2"
657
658         ablock=1
659         debugfs -R "stat <${inode}>" "${dev}" 2> /dev/null | grep 'File ACL: 0' -q && ablock=0
660
661         case "${format}" in
662         "local"|"inline")
663                 test "${ablock}" -eq 0 || _fail "failed to create inode ${inode} with ${format} xattr"
664                 ;;
665         "block")
666                 test "${extents}" -eq 1 || _fail "failed to create inode ${inode} with ${format} xattr"
667                 ;;
668         *)
669                 _fail "Unknown aformat ${format}"
670         esac
671 }
672
673 # Check structure of ext4 dir
674 __populate_check_ext4_dir() {
675         dev="${SCRATCH_DEV}"
676         inode="$1"
677         dtype="$2"
678
679         htree=0
680         inline=0
681         iflags="$(debugfs -R "stat <${inode}>" "${dev}" 2> /dev/null | grep 'Flags:' | sed -e 's/^.*Flags: \([0-9a-fx]*\).*$/\1/g')"
682         test "$(echo "${iflags}" | awk '{print and(strtonum($1), 0x1000);}')" -gt 0 && htree=1
683         test "$(echo "${iflags}" | awk '{print and(strtonum($1), 0x10000000);}')" -gt 0 && inline=1
684
685         case "${dtype}" in
686         "inline")
687                 (test "${inline}" -eq 1 && test "${htree}" -eq 0) || _fail "failed to create ${dtype} dir ino ${inode} htree ${htree} inline ${inline}"
688                 ;;
689         "block")
690                 (test "${inline}" -eq 0 && test "${htree}" -eq 0) || _fail "failed to create ${dtype} dir ino ${inode} htree ${htree} inline ${inline}"
691                 ;;
692         "htree")
693                 (test "${inline}" -eq 0 && test "${htree}" -eq 1) || _fail "failed to create ${dtype} dir ino ${inode} htree ${htree} inline ${inline}"
694                 ;;
695         *)
696                 _fail "Unknown directory type ${dtype}"
697                 ;;
698         esac
699 }
700
701 # Check that populate created all the types of files we wanted
702 _scratch_ext4_populate_check() {
703         _scratch_mount
704         extents_file="$(__populate_find_inode "${SCRATCH_MNT}/S_IFREG.FMT_EXTENTS")"
705         etree_file="$(__populate_find_inode "${SCRATCH_MNT}/S_IFREG.FMT_ETREE")"
706         block_dir="$(__populate_find_inode "${SCRATCH_MNT}/S_IFDIR.FMT_BLOCK")"
707         htree_dir="$(__populate_find_inode "${SCRATCH_MNT}/S_IFDIR.FMT_HTREE")"
708         extents_slink="$(__populate_find_inode "${SCRATCH_MNT}/S_IFLNK.FMT_EXTENTS")"
709         local_attr="$(__populate_find_inode "${SCRATCH_MNT}/ATTR.FMT_LOCAL")"
710         block_attr="$(__populate_find_inode "${SCRATCH_MNT}/ATTR.FMT_BLOCK")"
711         umount "${SCRATCH_MNT}"
712
713         __populate_check_ext4_dformat "${extents_file}" "extents"
714         __populate_check_ext4_dformat "${etree_file}" "etree"
715         __populate_check_ext4_dir "${block_dir}" "block"
716         __populate_check_ext4_dir "${htree_dir}" "htree"
717         __populate_check_ext4_dformat "${extents_slink}" "extents"
718         __populate_check_ext4_aformat "${local_attr}" "local"
719         __populate_check_ext4_aformat "${block_attr}" "block"
720 }
721
722 # Populate a scratch FS and check the contents to make sure we got that
723 _scratch_populate() {
724         case "${FSTYP}" in
725         "xfs")
726                 _scratch_xfs_populate
727                 _scratch_xfs_populate_check
728                 ;;
729         "ext2"|"ext3"|"ext4")
730                 _scratch_ext4_populate
731                 _scratch_ext4_populate_check
732                 ;;
733         *)
734                 _fail "Don't know how to populate a ${FSTYP} filesystem."
735                 ;;
736         esac
737 }
738
739 # Fill a file system by repeatedly creating files in the given folder
740 # starting with the given file size.  Files are reduced in size when
741 # they can no longer fit until no more files can be created.
742 _fill_fs()
743 {
744         local file_size=$1
745         local dir=$2
746         local block_size=$3
747         local switch_user=$4
748         local file_count=1
749         local bytes_written=0
750         local use_falloc=1;
751
752         if [ $# -ne 4 ]; then
753                 echo "Usage: _fill_fs filesize dir blocksize switch_user"
754                 exit 1
755         fi
756
757         if [ $switch_user -eq 0 ]; then
758                 mkdir -p $dir
759         else
760                 _user_do "mkdir -p $dir"
761         fi
762         if [ ! -d $dir ]; then
763                 return 0;
764         fi
765
766         testio=`$XFS_IO_PROG -F -fc "falloc 0 $block_size" $dir/$$.xfs_io 2>&1`
767         echo $testio | grep -q "not found" && use_falloc=0
768         echo $testio | grep -q "Operation not supported" && use_falloc=0
769
770         if [ $file_size -lt $block_size ]; then
771                 $file_size = $block_size
772         fi
773
774         while [ $file_size -ge $block_size ]; do
775                 bytes_written=0
776                 if [ $switch_user -eq 0 ]; then
777                         if [ $use_falloc -eq 0 ]; then
778                                 $XFS_IO_PROG -fc "pwrite -b 8388608 0 $file_size" \
779                                         $dir/$file_count
780                         else
781                                 $XFS_IO_PROG -fc "falloc 0 $file_size" \
782                                         $dir/$file_count
783                         fi
784                 else
785                         if [ $use_falloc -eq 0 ]; then
786                                 _user_do "$XFS_IO_PROG -f -c \"pwrite -b 8388608 0 \
787                                         $file_size\" $dir/$file_count"
788                         else
789                                 _user_do "$XFS_IO_PROG -f -c \"falloc 0 \
790                                         $file_size\" $dir/$file_count"
791                         fi
792                 fi
793
794                 if [ -f $dir/$file_count ]; then
795                         bytes_written=$(_get_filesize $dir/$file_count)
796                 fi
797
798                 # If there was no room to make the file, then divide it in
799                 # half, and keep going
800                 if [ $bytes_written -lt $file_size ]; then
801                         file_size=$((file_size / 2))
802                 fi
803                 file_count=$((file_count + 1))
804         done
805 }
806
807 # Compute the fs geometry description of a populated filesystem
808 _scratch_populate_cache_tag() {
809         local extra_descr=""
810         local size="$(blockdev --getsz "${SCRATCH_DEV}")"
811         local logdev_sz="none"
812         local rtdev_sz="none"
813
814         if [ "${USE_EXTERNAL}" = "yes" ] && [ -n "${SCRATCH_LOGDEV}" ]; then
815                 logdev_sz="$(blockdev --getsz "${SCRATCH_LOGDEV}")"
816         fi
817
818         if [ "${USE_EXTERNAL}" = "yes" ] && [ -n "${SCRATCH_RTDEV}" ]; then
819                 rtdev_sz="$(blockdev --getsz "${SCRATCH_RTDEV}")"
820         fi
821
822         case "${FSTYP}" in
823         "ext4")
824                 extra_descr="LOGDEV_SIZE ${logdev_sz}"
825                 ;;
826         "xfs")
827                 extra_descr="LOGDEV_SIZE ${logdev_sz} RTDEV_SIZE ${rtdev_sz}"
828                 _populate_xfs_qmount_option
829                 if echo "${MOUNT_OPTIONS}" | grep -q 'usrquota'; then
830                         extra_descr="${extra_descr} QUOTAS"
831                 fi
832                 ;;
833         esac
834         echo "FSTYP ${FSTYP} MKFS_OPTIONS ${MKFS_OPTIONS} SIZE ${size} ${extra_descr} ARGS $@"
835 }
836
837 # Restore a cached populated fs from a metadata dump
838 _scratch_populate_restore_cached() {
839         local metadump="$1"
840
841         # If we're configured for compressed dumps and there isn't already an
842         # uncompressed dump, see if we can use DUMP_COMPRESSOR to decompress
843         # something.
844         if [ -n "$DUMP_COMPRESSOR" ]; then
845                 for compr in "$metadump".*; do
846                         [ -e "$compr" ] && $DUMP_COMPRESSOR -d -f -k "$compr" && break
847                 done
848         fi
849
850         test -r "$metadump" || return 1
851
852         case "${FSTYP}" in
853         "xfs")
854                 xfs_mdrestore "${metadump}" "${SCRATCH_DEV}" && return 0
855                 ;;
856         "ext2"|"ext3"|"ext4")
857                 # ext4 cannot e2image external logs, so we cannot restore
858                 test -n "${SCRATCH_LOGDEV}" && return 1
859                 e2image -r "${metadump}" "${SCRATCH_DEV}" && return 0
860                 ;;
861         esac
862         return 1
863 }
864
865 # Populate a scratch FS from scratch or from a cached image.
866 _scratch_populate_cached() {
867         local meta_descr="$(_scratch_populate_cache_tag "$@")"
868         local meta_tag="$(echo "${meta_descr}" | md5sum - | cut -d ' ' -f 1)"
869         local metadump_stem="${TEST_DIR}/__populate.${FSTYP}.${meta_tag}"
870
871         # These variables are shared outside this function
872         POPULATE_METADUMP="${metadump_stem}.metadump"
873         POPULATE_METADUMP_DESCR="${metadump_stem}.txt"
874
875         # Don't keep metadata images cached for more 48 hours...
876         rm -rf "$(find "${POPULATE_METADUMP}" -mtime +2 2>/dev/null)"
877
878         # Throw away cached image if it doesn't match our spec.
879         cmp -s "${POPULATE_METADUMP_DESCR}" <(echo "${meta_descr}") || \
880                 rm -rf "${POPULATE_METADUMP}"
881
882         # Try to restore from the metadump
883         _scratch_populate_restore_cached "${POPULATE_METADUMP}" && \
884                 return
885
886         # Oh well, just create one from scratch
887         _scratch_mkfs
888         echo "${meta_descr}" > "${POPULATE_METADUMP_DESCR}"
889         case "${FSTYP}" in
890         "xfs")
891                 _scratch_xfs_populate $@
892                 _scratch_xfs_populate_check
893                 _scratch_xfs_metadump "${POPULATE_METADUMP}"
894
895                 local logdev=
896                 [ "$USE_EXTERNAL" = yes -a ! -z "$SCRATCH_LOGDEV" ] && \
897                         logdev=$SCRATCH_LOGDEV
898
899                 _xfs_metadump "$POPULATE_METADUMP" "$SCRATCH_DEV" "$logdev" \
900                         compress
901                 ;;
902         "ext2"|"ext3"|"ext4")
903                 _scratch_ext4_populate $@
904                 _scratch_ext4_populate_check
905                 _ext4_metadump "${SCRATCH_DEV}" "${POPULATE_METADUMP}" compress
906                 ;;
907         *)
908                 _fail "Don't know how to populate a ${FSTYP} filesystem."
909                 ;;
910         esac
911 }