4fa118f0fbdb4a2697c0478a6e48b022aee9a672
[xfstests-dev.git] / common / populate
1 ##/bin/bash
2 # SPDX-License-Identifier: GPL-2.0+
3 # Copyright (c) 2015 Oracle.  All Rights Reserved.
4 #
5 # Routines for populating a scratch fs, and helpers to exercise an FS
6 # once it's been fuzzed.
7
8 . ./common/quota
9
10 _require_populate_commands() {
11         _require_xfs_io_command "falloc"
12         _require_xfs_io_command "fpunch"
13         _require_test_program "punch-alternating"
14         _require_command "$XFS_DB_PROG" "xfs_db"
15 }
16
17 _require_xfs_db_blocktrash_z_command() {
18         test "${FSTYP}" = "xfs" || _notrun "cannot run xfs_db on ${FSTYP}"
19         $XFS_DB_PROG -x -f -c 'blocktrash -z' "${TEST_DEV}" | grep -q 'nothing on stack' || _notrun "blocktrash -z not supported"
20 }
21
22 # Attempt to make files of "every" format for data, dirs, attrs etc.
23 # (with apologies to Eric Sandeen for mutating xfser.sh)
24
25 # Create a large directory
26 __populate_create_dir() {
27         name="$1"
28         nr="$2"
29         missing="$3"
30
31         mkdir -p "${name}"
32         seq 0 "${nr}" | while read d; do
33                 creat=mkdir
34                 test "$((d % 20))" -eq 0 && creat=touch
35                 $creat "${name}/$(printf "%.08d" "$d")"
36         done
37
38         test -z "${missing}" && return
39         seq 1 2 "${nr}" | while read d; do
40                 rm -rf "${name}/$(printf "%.08d" "$d")"
41         done
42 }
43
44 # Add a bunch of attrs to a file
45 __populate_create_attr() {
46         name="$1"
47         nr="$2"
48         missing="$3"
49
50         touch "${name}"
51         seq 0 "${nr}" | while read d; do
52                 setfattr -n "user.$(printf "%.08d" "$d")" -v "$(printf "%.08d" "$d")" "${name}"
53         done
54
55         test -z "${missing}" && return
56         seq 1 2 "${nr}" | while read d; do
57                 setfattr -x "user.$(printf "%.08d" "$d")" "${name}"
58         done
59 }
60
61 # Fill up some percentage of the remaining free space
62 __populate_fill_fs() {
63         dir="$1"
64         pct="$2"
65         test -z "${pct}" && pct=60
66
67         mkdir -p "${dir}/test/1"
68         cp -pRdu "${dir}"/S_IFREG* "${dir}/test/1/"
69
70         SRC_SZ="$(du -ks "${dir}/test/1" | cut -f 1)"
71         FS_SZ="$(( $(stat -f "${dir}" -c '%a * %S') / 1024 ))"
72
73         NR="$(( (FS_SZ * ${pct} / 100) / SRC_SZ ))"
74
75         echo "FILL FS"
76         echo "src_sz $SRC_SZ fs_sz $FS_SZ nr $NR"
77         seq 2 "${NR}" | while read nr; do
78                 cp -pRdu "${dir}/test/1" "${dir}/test/${nr}"
79         done
80 }
81
82 # For XFS, force on all the quota options if quota is enabled
83 # and the user didn't feed us noquota.
84 _populate_xfs_qmount_option()
85 {
86         # User explicitly told us not to quota
87         if echo "${MOUNT_OPTIONS}" | grep -q 'noquota'; then
88                 return
89         fi
90
91         # Don't bother if we can't turn on quotas
92         if [ ! -f /proc/fs/xfs/xqmstat ]; then
93                 # No quota support
94                 return
95         elif [ "${USE_EXTERNAL}" = "yes" ] && [ ! -z "${SCRATCH_RTDEV}" ]; then
96                 # Quotas not supported on rt filesystems
97                 return
98         elif [ -z "${XFS_QUOTA_PROG}" ]; then
99                 # xfs quota tools not installed
100                 return
101         fi
102
103         # Turn on all the quotas
104         if $XFS_INFO_PROG "${TEST_DIR}" | grep -q 'crc=1'; then
105                 # v5 filesystems can have group & project quotas
106                 quota="usrquota,grpquota,prjquota"
107         else
108                 # v4 filesystems cannot mix group & project quotas
109                 quota="usrquota,grpquota"
110         fi
111
112         # Inject our quota mount options
113         if echo "${MOUNT_OPTIONS}" | grep -q "${quota}"; then
114                 return
115         elif echo "${MOUNT_OPTIONS}" | egrep -q '(quota|noenforce)'; then
116                 _qmount_option "${quota}"
117         else
118                 export MOUNT_OPTIONS="$MOUNT_OPTIONS -o ${quota}"
119                 echo "MOUNT_OPTIONS = $MOUNT_OPTIONS" >>$seqres.full
120         fi
121 }
122
123 # Populate an XFS on the scratch device with (we hope) all known
124 # types of metadata block
125 _scratch_xfs_populate() {
126         fill=1
127
128         for arg in $@; do
129                 case "${arg}" in
130                 "nofill")
131                         fill=0;;
132                 esac
133         done
134
135         _populate_xfs_qmount_option
136         _scratch_mount
137         blksz="$(stat -f -c '%s' "${SCRATCH_MNT}")"
138         dblksz="$($XFS_INFO_PROG "${SCRATCH_MNT}" | grep naming.*bsize | sed -e 's/^.*bsize=//g' -e 's/\([0-9]*\).*$/\1/g')"
139         crc="$($XFS_INFO_PROG "${SCRATCH_MNT}" | grep crc= | sed -e 's/^.*crc=//g' -e 's/\([0-9]*\).*$/\1/g')"
140         if [ $crc -eq 1 ]; then
141                 leaf_hdr_size=64
142         else
143                 leaf_hdr_size=16
144         fi
145         leaf_lblk="$((32 * 1073741824 / blksz))"
146         node_lblk="$((64 * 1073741824 / blksz))"
147
148         # Data:
149
150         # Fill up the root inode chunk
151         echo "+ fill root ino chunk"
152         seq 1 64 | while read f; do
153                 $XFS_IO_PROG -f -c "truncate 0" "${SCRATCH_MNT}/dummy${f}"
154         done
155
156         # Regular files
157         # - FMT_EXTENTS
158         echo "+ extents file"
159         $XFS_IO_PROG -f -c "pwrite -S 0x61 0 ${blksz}" "${SCRATCH_MNT}/S_IFREG.FMT_EXTENTS"
160
161         # - FMT_BTREE
162         echo "+ btree extents file"
163         nr="$((blksz * 2 / 16))"
164         $XFS_IO_PROG -f -c "pwrite -S 0x62 0 $((blksz * nr))" "${SCRATCH_MNT}/S_IFREG.FMT_BTREE"
165         ./src/punch-alternating "${SCRATCH_MNT}/S_IFREG.FMT_BTREE"
166
167         # Directories
168         # - INLINE
169         echo "+ inline dir"
170         __populate_create_dir "${SCRATCH_MNT}/S_IFDIR.FMT_INLINE" 1
171
172         # - BLOCK
173         echo "+ block dir"
174         __populate_create_dir "${SCRATCH_MNT}/S_IFDIR.FMT_BLOCK" "$((dblksz / 40))"
175
176         # - LEAF
177         echo "+ leaf dir"
178         __populate_create_dir "${SCRATCH_MNT}/S_IFDIR.FMT_LEAF" "$((dblksz / 12))"
179
180         # - LEAFN
181         echo "+ leafn dir"
182         __populate_create_dir "${SCRATCH_MNT}/S_IFDIR.FMT_LEAFN" "$(( ((dblksz - leaf_hdr_size) / 8) - 3 ))"
183
184         # - NODE
185         echo "+ node dir"
186         __populate_create_dir "${SCRATCH_MNT}/S_IFDIR.FMT_NODE" "$((16 * dblksz / 40))" true
187
188         # - BTREE
189         echo "+ btree dir"
190         __populate_create_dir "${SCRATCH_MNT}/S_IFDIR.FMT_BTREE" "$((128 * dblksz / 40))" true
191
192         # Symlinks
193         # - FMT_LOCAL
194         echo "+ inline symlink"
195         ln -s target "${SCRATCH_MNT}/S_IFLNK.FMT_LOCAL"
196
197         # - FMT_EXTENTS
198         echo "+ extents symlink"
199         ln -s "$(perl -e 'print "x" x 1023;')" "${SCRATCH_MNT}/S_IFLNK.FMT_EXTENTS"
200
201         # Char & block
202         echo "+ special"
203         mknod "${SCRATCH_MNT}/S_IFCHR" c 1 1
204         mknod "${SCRATCH_MNT}/S_IFBLK" c 1 1
205
206         # special file with an xattr
207         setfacl -P -m u:nobody:r ${SCRATCH_MNT}/S_IFCHR
208
209         # Attribute formats
210         # LOCAL
211         echo "+ local attr"
212         __populate_create_attr "${SCRATCH_MNT}/ATTR.FMT_LOCAL" 1
213
214         # LEAF
215         echo "+ leaf attr"
216         __populate_create_attr "${SCRATCH_MNT}/ATTR.FMT_LEAF" "$((blksz / 40))"
217
218         # NODE
219         echo "+ node attr"
220         __populate_create_attr "${SCRATCH_MNT}/ATTR.FMT_NODE" "$((8 * blksz / 40))"
221
222         # BTREE
223         echo "+ btree attr"
224         __populate_create_attr "${SCRATCH_MNT}/ATTR.FMT_BTREE" "$((64 * blksz / 40))" true
225
226         # trusted namespace
227         touch ${SCRATCH_MNT}/ATTR.TRUSTED
228         setfattr -n trusted.moo -v urk ${SCRATCH_MNT}/ATTR.TRUSTED
229
230         # security namespace
231         touch ${SCRATCH_MNT}/ATTR.SECURITY
232         setfattr -n security.foo -v bar ${SCRATCH_MNT}/ATTR.SECURITY
233
234         # system namespace
235         touch ${SCRATCH_MNT}/ATTR.SYSTEM
236         setfacl -m u:root:r ${SCRATCH_MNT}/ATTR.SYSTEM
237
238         # FMT_EXTENTS with a remote less-than-a-block value
239         echo "+ attr extents with a remote less-than-a-block value"
240         touch "${SCRATCH_MNT}/ATTR.FMT_EXTENTS_REMOTE3K"
241         $XFS_IO_PROG -f -c "pwrite -S 0x43 0 $((blksz - 300))" "${SCRATCH_MNT}/attrvalfile" > /dev/null
242         attr -q -s user.remotebtreeattrname "${SCRATCH_MNT}/ATTR.FMT_EXTENTS_REMOTE3K" < "${SCRATCH_MNT}/attrvalfile"
243
244         # FMT_EXTENTS with a remote block-size value
245         echo "+ attr extents with a remote one-block value"
246         touch "${SCRATCH_MNT}/ATTR.FMT_EXTENTS_REMOTE4K"
247         $XFS_IO_PROG -f -c "pwrite -S 0x44 0 ${blksz}" "${SCRATCH_MNT}/attrvalfile" > /dev/null
248         attr -q -s user.remotebtreeattrname "${SCRATCH_MNT}/ATTR.FMT_EXTENTS_REMOTE4K" < "${SCRATCH_MNT}/attrvalfile"
249         rm -rf "${SCRATCH_MNT}/attrvalfile"
250
251         # Make an unused inode
252         echo "+ empty file"
253         touch "${SCRATCH_MNT}/unused"
254         $XFS_IO_PROG -f -c 'fsync' "${SCRATCH_MNT}/unused"
255         rm -rf "${SCRATCH_MNT}/unused"
256
257         # Free space btree
258         echo "+ freesp btree"
259         nr="$((blksz * 2 / 8))"
260         $XFS_IO_PROG -f -c "pwrite -S 0x62 0 $((blksz * nr))" "${SCRATCH_MNT}/BNOBT"
261         ./src/punch-alternating "${SCRATCH_MNT}/BNOBT"
262
263         # Inode btree
264         echo "+ inobt btree"
265         local ino_per_rec=64
266         local rec_per_btblock=16
267         local nr="$(( 2 * (blksz / rec_per_btblock) * ino_per_rec ))"
268         local dir="${SCRATCH_MNT}/INOBT"
269         mkdir -p "${dir}"
270         seq 0 "${nr}" | while read f; do
271                 touch "${dir}/${f}"
272         done
273
274         seq 0 "$((ino_per_rec + 1))" "${nr}" | while read f; do
275                 rm -f "${dir}/${f}"
276         done
277
278         # Reverse-mapping btree
279         is_rmapbt="$($XFS_INFO_PROG "${SCRATCH_MNT}" | grep -c 'rmapbt=1')"
280         if [ $is_rmapbt -gt 0 ]; then
281                 echo "+ rmapbt btree"
282                 nr="$((blksz * 2 / 24))"
283                 $XFS_IO_PROG -f -c "pwrite -S 0x62 0 $((blksz * nr))" "${SCRATCH_MNT}/RMAPBT"
284                 ./src/punch-alternating "${SCRATCH_MNT}/RMAPBT"
285         fi
286
287         # Realtime Reverse-mapping btree
288         is_rt="$($XFS_INFO_PROG "${SCRATCH_MNT}" | grep -c 'rtextents=[1-9]')"
289         if [ $is_rmapbt -gt 0 ] && [ $is_rt -gt 0 ]; then
290                 echo "+ rtrmapbt btree"
291                 nr="$((blksz * 2 / 32))"
292                 $XFS_IO_PROG -f -R -c "pwrite -S 0x62 0 $((blksz * nr))" "${SCRATCH_MNT}/RTRMAPBT"
293                 ./src/punch-alternating "${SCRATCH_MNT}/RTRMAPBT"
294         fi
295
296         # Reference-count btree
297         is_reflink="$($XFS_INFO_PROG "${SCRATCH_MNT}" | grep -c 'reflink=1')"
298         if [ $is_reflink -gt 0 ]; then
299                 echo "+ reflink btree"
300                 nr="$((blksz * 2 / 12))"
301                 $XFS_IO_PROG -f -c "pwrite -S 0x62 0 $((blksz * nr))" "${SCRATCH_MNT}/REFCOUNTBT"
302                 cp --reflink=always "${SCRATCH_MNT}/REFCOUNTBT" "${SCRATCH_MNT}/REFCOUNTBT2"
303                 ./src/punch-alternating "${SCRATCH_MNT}/REFCOUNTBT"
304         fi
305
306         # Copy some real files (xfs tests, I guess...)
307         echo "+ real files"
308         test $fill -ne 0 && __populate_fill_fs "${SCRATCH_MNT}" 5
309
310         umount "${SCRATCH_MNT}"
311 }
312
313 # Populate an ext4 on the scratch device with (we hope) all known
314 # types of metadata block
315 _scratch_ext4_populate() {
316         fill=1
317
318         for arg in $@; do
319                 case "${arg}" in
320                 "nofill")
321                         fill=0;;
322                 esac
323         done
324
325         _scratch_mount
326         blksz="$(stat -f -c '%s' "${SCRATCH_MNT}")"
327         dblksz="${blksz}"
328         leaf_lblk="$((32 * 1073741824 / blksz))"
329         node_lblk="$((64 * 1073741824 / blksz))"
330
331         # Data:
332
333         # Regular files
334         # - FMT_INLINE
335         echo "+ inline file"
336         $XFS_IO_PROG -f -c "pwrite -S 0x61 0 1" "${SCRATCH_MNT}/S_IFREG.FMT_INLINE"
337
338         # - FMT_EXTENTS
339         echo "+ extents file"
340         $XFS_IO_PROG -f -c "pwrite -S 0x61 0 ${blksz}" "${SCRATCH_MNT}/S_IFREG.FMT_EXTENTS"
341
342         # - FMT_ETREE
343         echo "+ extent tree file"
344         nr="$((blksz * 2 / 12))"
345         $XFS_IO_PROG -f -c "pwrite -S 0x62 0 $((blksz * nr))" "${SCRATCH_MNT}/S_IFREG.FMT_ETREE"
346         ./src/punch-alternating "${SCRATCH_MNT}/S_IFREG.FMT_ETREE"
347
348         # Directories
349         # - INLINE
350         echo "+ inline dir"
351         __populate_create_dir "${SCRATCH_MNT}/S_IFDIR.FMT_INLINE" 1
352
353         # - BLOCK
354         echo "+ block dir"
355         __populate_create_dir "${SCRATCH_MNT}/S_IFDIR.FMT_BLOCK" "$((dblksz / 32))"
356
357         # - HTREE
358         echo "+ htree dir"
359         __populate_create_dir "${SCRATCH_MNT}/S_IFDIR.FMT_HTREE" "$((4 * dblksz / 24))"
360
361         # Symlinks
362         # - FMT_LOCAL
363         echo "+ inline symlink"
364         ln -s target "${SCRATCH_MNT}/S_IFLNK.FMT_LOCAL"
365
366         # - FMT_EXTENTS
367         echo "+ extents symlink"
368         ln -s "$(perl -e 'print "x" x 1023;')" "${SCRATCH_MNT}/S_IFLNK.FMT_EXTENTS"
369
370         # Char & block
371         echo "+ special"
372         mknod "${SCRATCH_MNT}/S_IFCHR" c 1 1
373         mknod "${SCRATCH_MNT}/S_IFBLK" c 1 1
374
375         # special file with an xattr
376         setfacl -P -m u:nobody:r ${SCRATCH_MNT}/S_IFCHR
377
378         # Attribute formats
379         # LOCAL
380         echo "+ local attr"
381         __populate_create_attr "${SCRATCH_MNT}/ATTR.FMT_LOCAL" 0
382
383         # BLOCK
384         echo "+ block attr"
385         __populate_create_attr "${SCRATCH_MNT}/ATTR.FMT_BLOCK" "$((blksz / 40))"
386
387         # trusted namespace
388         touch ${SCRATCH_MNT}/ATTR.TRUSTED
389         setfattr -n trusted.moo -v urk ${SCRATCH_MNT}/ATTR.TRUSTED
390
391         # security namespace
392         touch ${SCRATCH_MNT}/ATTR.SECURITY
393         setfattr -n security.foo -v bar ${SCRATCH_MNT}/ATTR.SECURITY
394
395         # system namespace
396         touch ${SCRATCH_MNT}/ATTR.SYSTEM
397         setfacl -m u:root:r ${SCRATCH_MNT}/ATTR.SYSTEM
398
399         # Make an unused inode
400         echo "+ empty file"
401         touch "${SCRATCH_MNT}/unused"
402         $XFS_IO_PROG -f -c 'fsync' "${SCRATCH_MNT}/unused"
403         rm -rf "${SCRATCH_MNT}/unused"
404
405         # Copy some real files (xfs tests, I guess...)
406         echo "+ real files"
407         test $fill -ne 0 && __populate_fill_fs "${SCRATCH_MNT}" 5
408
409         umount "${SCRATCH_MNT}"
410 }
411
412 # Find the inode number of a file
413 __populate_find_inode() {
414         name="$1"
415         inode="$(stat -c '%i' "${name}")"
416         echo "${inode}"
417 }
418
419 # Check data fork format of XFS file
420 __populate_check_xfs_dformat() {
421         inode="$1"
422         format="$2"
423
424         fmt="$(_scratch_xfs_db -c "inode ${inode}" -c 'p core.format' | sed -e 's/^.*(\([a-z]*\)).*$/\1/g')"
425         test "${format}" = "${fmt}" || _fail "failed to create ino ${inode} dformat expected ${format} saw ${fmt}"
426 }
427
428 # Check attr fork format of XFS file
429 __populate_check_xfs_aformat() {
430         inode="$1"
431         format="$2"
432
433         fmt="$(_scratch_xfs_db -c "inode ${inode}" -c 'p core.aformat' | sed -e 's/^.*(\([a-z]*\)).*$/\1/g')"
434         test "${format}" = "${fmt}" || _fail "failed to create ino ${inode} aformat expected ${format} saw ${fmt}"
435 }
436
437 # Check structure of XFS directory
438 __populate_check_xfs_dir() {
439         inode="$1"
440         dtype="$2"
441
442         (test -n "${leaf_lblk}" && test -n "${node_lblk}") || _fail "must define leaf_lblk and node_lblk before calling __populate_check_xfs_dir"
443         datab=0
444         leafb=0
445         freeb=0
446         #echo "== check dir ${inode} type ${dtype}" ; _scratch_xfs_db -x -c "inode ${inode}" -c "bmap"
447         _scratch_xfs_db -x -c "inode ${inode}" -c "dblock 0" -c "stack" | grep -q 'file data block is unmapped' || datab=1
448         _scratch_xfs_db -x -c "inode ${inode}" -c "dblock ${leaf_lblk}" -c "stack" | grep -q 'file data block is unmapped' || leafb=1
449         _scratch_xfs_db -x -c "inode ${inode}" -c "dblock ${node_lblk}" -c "stack" | grep -q 'file data block is unmapped' || freeb=1
450
451         case "${dtype}" in
452         "shortform"|"inline"|"local")
453                 (test "${datab}" -eq 0 && test "${leafb}" -eq 0 && test "${freeb}" -eq 0) || _fail "failed to create ${dtype} dir ino ${inode} datab ${datab} leafb ${leafb} freeb ${freeb}"
454                 ;;
455         "block")
456                 (test "${datab}" -eq 1 && test "${leafb}" -eq 0 && test "${freeb}" -eq 0) || _fail "failed to create ${dtype} dir ino ${inode} datab ${datab} leafb ${leafb} freeb ${freeb}"
457                 ;;
458         "leaf")
459                 (test "${datab}" -eq 1 && test "${leafb}" -eq 1 && test "${freeb}" -eq 0) || _fail "failed to create ${dtype} dir ino ${inode} datab ${datab} leafb ${leafb} freeb ${freeb}"
460                 ;;
461         "leafn")
462                 _scratch_xfs_db -x -c "inode ${inode}" -c "dblock ${leaf_lblk}" -c "p lhdr.info.hdr.magic" | grep -q '0x3dff' && return
463                 _scratch_xfs_db -x -c "inode ${inode}" -c "dblock ${leaf_lblk}" -c "p lhdr.info.magic" | grep -q '0xd2ff' && return
464                 _fail "failed to create ${dtype} dir ino ${inode} datab ${datab} leafb ${leafb} freeb ${freeb}"
465                 ;;
466         "node"|"btree")
467                 (test "${datab}" -eq 1 && test "${leafb}" -eq 1 && test "${freeb}" -eq 1) || _fail "failed to create ${dtype} dir ino ${inode} datab ${datab} leafb ${leafb} freeb ${freeb}"
468                 ;;
469         *)
470                 _fail "Unknown directory type ${dtype}"
471         esac
472 }
473
474 # Check structure of XFS attr
475 __populate_check_xfs_attr() {
476         inode="$1"
477         atype="$2"
478
479         datab=0
480         leafb=0
481         #echo "== check attr ${inode} type ${dtype}" ; _scratch_xfs_db -x -c "inode ${inode}" -c "bmap -a"
482         _scratch_xfs_db -x -c "inode ${inode}" -c "ablock 0" -c "stack" | grep -q 'file attr block is unmapped' || datab=1
483         _scratch_xfs_db -x -c "inode ${inode}" -c "ablock 1" -c "stack" | grep -q 'file attr block is unmapped' || leafb=1
484
485         case "${atype}" in
486         "shortform"|"inline"|"local")
487                 (test "${datab}" -eq 0 && test "${leafb}" -eq 0) || _fail "failed to create ${atype} attr ino ${inode} datab ${datab} leafb ${leafb}"
488                 ;;
489         "leaf")
490                 (test "${datab}" -eq 1 && test "${leafb}" -eq 0) || _fail "failed to create ${atype} attr ino ${inode} datab ${datab} leafb ${leafb}"
491                 ;;
492         "node"|"btree")
493                 (test "${datab}" -eq 1 && test "${leafb}" -eq 1) || _fail "failed to create ${atype} attr ino ${inode} datab ${datab} leafb ${leafb}"
494                 ;;
495         *)
496                 _fail "Unknown attribute type ${atype}"
497         esac
498 }
499
500 # Check that there's at least one per-AG btree with multiple levels
501 __populate_check_xfs_agbtree_height() {
502         bt_type="$1"
503         nr_ags=$(_scratch_xfs_db -c 'sb 0' -c 'p agcount' | awk '{print $3}')
504
505         case "${bt_type}" in
506         "bno"|"cnt"|"rmap"|"refcnt")
507                 hdr="agf"
508                 bt_prefix="${bt_type}"
509                 ;;
510         "ino")
511                 hdr="agi"
512                 bt_prefix=""
513                 ;;
514         "fino")
515                 hdr="agi"
516                 bt_prefix="free_"
517                 ;;
518         *)
519                 _fail "Don't know about AG btree ${bt_type}"
520                 ;;
521         esac
522
523         seq 0 $((nr_ags - 1)) | while read ag; do
524                 bt_level=$(_scratch_xfs_db -c "${hdr} ${ag}" -c "p ${bt_prefix}level" | awk '{print $3}')
525                 if [ "${bt_level}" -gt 1 ]; then
526                         return 100
527                 fi
528         done
529         test $? -eq 100 || _fail "Failed to create ${bt_type} of sufficient height!"
530         return 1
531 }
532
533 # Check that populate created all the types of files we wanted
534 _scratch_xfs_populate_check() {
535         _scratch_mount
536         extents_file="$(__populate_find_inode "${SCRATCH_MNT}/S_IFREG.FMT_EXTENTS")"
537         btree_file="$(__populate_find_inode "${SCRATCH_MNT}/S_IFREG.FMT_BTREE")"
538         inline_dir="$(__populate_find_inode "${SCRATCH_MNT}/S_IFDIR.FMT_INLINE")"
539         block_dir="$(__populate_find_inode "${SCRATCH_MNT}/S_IFDIR.FMT_BLOCK")"
540         leaf_dir="$(__populate_find_inode "${SCRATCH_MNT}/S_IFDIR.FMT_LEAF")"
541         leafn_dir="$(__populate_find_inode "${SCRATCH_MNT}/S_IFDIR.FMT_LEAFN")"
542         node_dir="$(__populate_find_inode "${SCRATCH_MNT}/S_IFDIR.FMT_NODE")"
543         btree_dir="$(__populate_find_inode "${SCRATCH_MNT}/S_IFDIR.FMT_BTREE")"
544         local_slink="$(__populate_find_inode "${SCRATCH_MNT}/S_IFLNK.FMT_LOCAL")"
545         extents_slink="$(__populate_find_inode "${SCRATCH_MNT}/S_IFLNK.FMT_EXTENTS")"
546         bdev="$(__populate_find_inode "${SCRATCH_MNT}/S_IFBLK")"
547         cdev="$(__populate_find_inode "${SCRATCH_MNT}/S_IFCHR")"
548         local_attr="$(__populate_find_inode "${SCRATCH_MNT}/ATTR.FMT_LOCAL")"
549         leaf_attr="$(__populate_find_inode "${SCRATCH_MNT}/ATTR.FMT_LEAF")"
550         node_attr="$(__populate_find_inode "${SCRATCH_MNT}/ATTR.FMT_NODE")"
551         btree_attr="$(__populate_find_inode "${SCRATCH_MNT}/ATTR.FMT_BTREE")"
552         is_finobt=$($XFS_INFO_PROG "${SCRATCH_MNT}" | grep -c 'finobt=1')
553         is_rmapbt=$($XFS_INFO_PROG "${SCRATCH_MNT}" | grep -c 'rmapbt=1')
554         is_reflink=$($XFS_INFO_PROG "${SCRATCH_MNT}" | grep -c 'reflink=1')
555
556         blksz="$(stat -f -c '%s' "${SCRATCH_MNT}")"
557         dblksz="$($XFS_INFO_PROG "${SCRATCH_MNT}" | grep naming.*bsize | sed -e 's/^.*bsize=//g' -e 's/\([0-9]*\).*$/\1/g')"
558         leaf_lblk="$((32 * 1073741824 / blksz))"
559         node_lblk="$((64 * 1073741824 / blksz))"
560         umount "${SCRATCH_MNT}"
561
562         __populate_check_xfs_dformat "${extents_file}" "extents"
563         __populate_check_xfs_dformat "${btree_file}" "btree"
564         __populate_check_xfs_dir "${inline_dir}" "inline"
565         __populate_check_xfs_dir "${block_dir}" "block"
566         __populate_check_xfs_dir "${leaf_dir}" "leaf"
567         __populate_check_xfs_dir "${leafn_dir}" "leafn"
568         __populate_check_xfs_dir "${node_dir}" "node"
569         __populate_check_xfs_dir "${btree_dir}" "btree"
570         __populate_check_xfs_dformat "${btree_dir}" "btree"
571         __populate_check_xfs_dformat "${bdev}" "dev"
572         __populate_check_xfs_dformat "${cdev}" "dev"
573         __populate_check_xfs_attr "${local_attr}" "local"
574         __populate_check_xfs_attr "${leaf_attr}" "leaf"
575         __populate_check_xfs_attr "${node_attr}" "node"
576         __populate_check_xfs_attr "${btree_attr}" "btree"
577         __populate_check_xfs_aformat "${btree_attr}" "btree"
578         __populate_check_xfs_agbtree_height "bno"
579         __populate_check_xfs_agbtree_height "cnt"
580         __populate_check_xfs_agbtree_height "ino"
581         test $is_finobt -ne 0 && __populate_check_xfs_agbtree_height "fino"
582         test $is_rmapbt -ne 0 && __populate_check_xfs_agbtree_height "rmap"
583         test $is_reflink -ne 0 && __populate_check_xfs_agbtree_height "refcnt"
584 }
585
586 # Check data fork format of ext4 file
587 __populate_check_ext4_dformat() {
588         dev="${SCRATCH_DEV}"
589         inode="$1"
590         format="$2"
591
592         extents=0
593         etree=0
594         debugfs -R "stat <${inode}>" "${dev}" 2> /dev/null | grep 'ETB[0-9]' -q && etree=1
595         iflags="$(debugfs -R "stat <${inode}>" "${dev}" 2> /dev/null | grep 'Flags:' | sed -e 's/^.*Flags: \([0-9a-fx]*\).*$/\1/g')"
596         test "$(echo "${iflags}" | awk '{print and(strtonum($1), 0x80000);}')" -gt 0 && extents=1
597
598         case "${format}" in
599         "blockmap")
600                 test "${extents}" -eq 0 || _fail "failed to create ino ${inode} with blockmap"
601                 ;;
602         "extent"|"extents")
603                 test "${extents}" -eq 1 || _fail "failed to create ino ${inode} with extents"
604                 ;;
605         "etree")
606                 (test "${extents}" -eq 1 && test "${etree}" -eq 1) || _fail "failed to create ino ${inode} with extent tree"
607                 ;;
608         *)
609                 _fail "Unknown dformat ${format}"
610         esac
611 }
612
613 # Check attr fork format of ext4 file
614 __populate_check_ext4_aformat() {
615         dev="${SCRATCH_DEV}"
616         inode="$1"
617         format="$2"
618
619         ablock=1
620         debugfs -R "stat <${inode}>" "${dev}" 2> /dev/null | grep 'File ACL: 0' -q && ablock=0
621
622         case "${format}" in
623         "local"|"inline")
624                 test "${ablock}" -eq 0 || _fail "failed to create inode ${inode} with ${format} xattr"
625                 ;;
626         "block")
627                 test "${extents}" -eq 1 || _fail "failed to create inode ${inode} with ${format} xattr"
628                 ;;
629         *)
630                 _fail "Unknown aformat ${format}"
631         esac
632 }
633
634 # Check structure of ext4 dir
635 __populate_check_ext4_dir() {
636         dev="${SCRATCH_DEV}"
637         inode="$1"
638         dtype="$2"
639
640         htree=0
641         inline=0
642         iflags="$(debugfs -R "stat <${inode}>" "${dev}" 2> /dev/null | grep 'Flags:' | sed -e 's/^.*Flags: \([0-9a-fx]*\).*$/\1/g')"
643         test "$(echo "${iflags}" | awk '{print and(strtonum($1), 0x1000);}')" -gt 0 && htree=1
644         test "$(echo "${iflags}" | awk '{print and(strtonum($1), 0x10000000);}')" -gt 0 && inline=1
645
646         case "${dtype}" in
647         "inline")
648                 (test "${inline}" -eq 1 && test "${htree}" -eq 0) || _fail "failed to create ${dtype} dir ino ${inode} htree ${htree} inline ${inline}"
649                 ;;
650         "block")
651                 (test "${inline}" -eq 0 && test "${htree}" -eq 0) || _fail "failed to create ${dtype} dir ino ${inode} htree ${htree} inline ${inline}"
652                 ;;
653         "htree")
654                 (test "${inline}" -eq 0 && test "${htree}" -eq 1) || _fail "failed to create ${dtype} dir ino ${inode} htree ${htree} inline ${inline}"
655                 ;;
656         *)
657                 _fail "Unknown directory type ${dtype}"
658                 ;;
659         esac
660 }
661
662 # Check that populate created all the types of files we wanted
663 _scratch_ext4_populate_check() {
664         _scratch_mount
665         extents_file="$(__populate_find_inode "${SCRATCH_MNT}/S_IFREG.FMT_EXTENTS")"
666         etree_file="$(__populate_find_inode "${SCRATCH_MNT}/S_IFREG.FMT_ETREE")"
667         block_dir="$(__populate_find_inode "${SCRATCH_MNT}/S_IFDIR.FMT_BLOCK")"
668         htree_dir="$(__populate_find_inode "${SCRATCH_MNT}/S_IFDIR.FMT_HTREE")"
669         extents_slink="$(__populate_find_inode "${SCRATCH_MNT}/S_IFLNK.FMT_EXTENTS")"
670         local_attr="$(__populate_find_inode "${SCRATCH_MNT}/ATTR.FMT_LOCAL")"
671         block_attr="$(__populate_find_inode "${SCRATCH_MNT}/ATTR.FMT_BLOCK")"
672         umount "${SCRATCH_MNT}"
673
674         __populate_check_ext4_dformat "${extents_file}" "extents"
675         __populate_check_ext4_dformat "${etree_file}" "etree"
676         __populate_check_ext4_dir "${block_dir}" "block"
677         __populate_check_ext4_dir "${htree_dir}" "htree"
678         __populate_check_ext4_dformat "${extents_slink}" "extents"
679         __populate_check_ext4_aformat "${local_attr}" "local"
680         __populate_check_ext4_aformat "${block_attr}" "block"
681 }
682
683 # Populate a scratch FS and check the contents to make sure we got that
684 _scratch_populate() {
685         case "${FSTYP}" in
686         "xfs")
687                 _scratch_xfs_populate
688                 _scratch_xfs_populate_check
689                 ;;
690         "ext2"|"ext3"|"ext4")
691                 _scratch_ext4_populate
692                 _scratch_ext4_populate_check
693                 ;;
694         *)
695                 _fail "Don't know how to populate a ${FSTYP} filesystem."
696                 ;;
697         esac
698 }
699
700 # Fill a file system by repeatedly creating files in the given folder
701 # starting with the given file size.  Files are reduced in size when
702 # they can no longer fit until no more files can be created.
703 _fill_fs()
704 {
705         local file_size=$1
706         local dir=$2
707         local block_size=$3
708         local switch_user=$4
709         local file_count=1
710         local bytes_written=0
711         local use_falloc=1;
712
713         if [ $# -ne 4 ]; then
714                 echo "Usage: _fill_fs filesize dir blocksize switch_user"
715                 exit 1
716         fi
717
718         if [ $switch_user -eq 0 ]; then
719                 mkdir -p $dir
720         else
721                 _user_do "mkdir -p $dir"
722         fi
723         if [ ! -d $dir ]; then
724                 return 0;
725         fi
726
727         testio=`$XFS_IO_PROG -F -fc "falloc 0 $block_size" $dir/$$.xfs_io 2>&1`
728         echo $testio | grep -q "not found" && use_falloc=0
729         echo $testio | grep -q "Operation not supported" && use_falloc=0
730
731         if [ $file_size -lt $block_size ]; then
732                 $file_size = $block_size
733         fi
734
735         while [ $file_size -ge $block_size ]; do
736                 bytes_written=0
737                 if [ $switch_user -eq 0 ]; then
738                         if [ $use_falloc -eq 0 ]; then
739                                 $XFS_IO_PROG -fc "pwrite -b 8388608 0 $file_size" \
740                                         $dir/$file_count
741                         else
742                                 $XFS_IO_PROG -fc "falloc 0 $file_size" \
743                                         $dir/$file_count
744                         fi
745                 else
746                         if [ $use_falloc -eq 0 ]; then
747                                 _user_do "$XFS_IO_PROG -f -c \"pwrite -b 8388608 0 \
748                                         $file_size\" $dir/$file_count"
749                         else
750                                 _user_do "$XFS_IO_PROG -f -c \"falloc 0 \
751                                         $file_size\" $dir/$file_count"
752                         fi
753                 fi
754
755                 if [ -f $dir/$file_count ]; then
756                         bytes_written=$(stat -c '%s' $dir/$file_count)
757                 fi
758
759                 # If there was no room to make the file, then divide it in
760                 # half, and keep going
761                 if [ $bytes_written -lt $file_size ]; then
762                         file_size=$((file_size / 2))
763                 fi
764                 file_count=$((file_count + 1))
765         done
766 }
767
768 # Compute the fs geometry description of a populated filesystem
769 _scratch_populate_cache_tag() {
770         local extra_descr=""
771         local size="$(blockdev --getsz "${SCRATCH_DEV}")"
772
773         case "${FSTYP}" in
774         "ext4")
775                 extra_descr="LOGDEV ${SCRATCH_LOGDEV} USE_EXTERNAL ${USE_EXTERNAL}"
776                 ;;
777         "xfs")
778                 extra_descr="LOGDEV ${SCRATCH_LOGDEV} USE_EXTERNAL ${USE_EXTERNAL} RTDEV ${SCRATCH_RTDEV}"
779                 _populate_xfs_qmount_option
780                 if echo "${MOUNT_OPTIONS}" | grep -q 'usrquota'; then
781                         extra_descr="${extra_descr} QUOTAS"
782                 fi
783                 ;;
784         esac
785         echo "FSTYP ${FSTYP} MKFS_OPTIONS ${MKFS_OPTIONS} SIZE ${size} ${extra_descr} ARGS $@"
786 }
787
788 # Restore a cached populated fs from a metadata dump
789 _scratch_populate_restore_cached() {
790         local metadump="$1"
791
792         case "${FSTYP}" in
793         "xfs")
794                 xfs_mdrestore "${metadump}" "${SCRATCH_DEV}" && return 0
795                 ;;
796         "ext2"|"ext3"|"ext4")
797                 # ext4 cannot e2image external logs, so we cannot restore
798                 test -n "${SCRATCH_LOGDEV}" && return 1
799                 e2image -r "${metadump}" "${SCRATCH_DEV}" && return 0
800                 ;;
801         esac
802         return 1
803 }
804
805 # Populate a scratch FS from scratch or from a cached image.
806 _scratch_populate_cached() {
807         local meta_descr="$(_scratch_populate_cache_tag "$@")"
808         local meta_tag="$(echo "${meta_descr}" | md5sum - | cut -d ' ' -f 1)"
809         local metadump_stem="${TEST_DIR}/__populate.${FSTYP}.${meta_tag}"
810
811         # These variables are shared outside this function
812         POPULATE_METADUMP="${metadump_stem}.metadump"
813         POPULATE_METADUMP_DESCR="${metadump_stem}.txt"
814
815         # Don't keep metadata images cached for more 48 hours...
816         rm -rf "$(find "${POPULATE_METADUMP}" -mtime +2 2>/dev/null)"
817
818         # Throw away cached image if it doesn't match our spec.
819         cmp -s "${POPULATE_METADUMP_DESCR}" <(echo "${meta_descr}") || \
820                 rm -rf "${POPULATE_METADUMP}"
821
822         # Try to restore from the metadump
823         test -r "${POPULATE_METADUMP}" && \
824                 _scratch_populate_restore_cached "${POPULATE_METADUMP}" && \
825                 return
826
827         # Oh well, just create one from scratch
828         _scratch_mkfs
829         echo "${meta_descr}" > "${POPULATE_METADUMP_DESCR}"
830         case "${FSTYP}" in
831         "xfs")
832                 _scratch_xfs_populate $@
833                 _scratch_xfs_populate_check
834                 _scratch_metadump "${POPULATE_METADUMP}" -a -o
835                 ;;
836         "ext2"|"ext3"|"ext4")
837                 _scratch_ext4_populate $@
838                 _scratch_ext4_populate_check
839                 e2image -Q "${SCRATCH_DEV}" "${POPULATE_METADUMP}"
840                 ;;
841         *)
842                 _fail "Don't know how to populate a ${FSTYP} filesystem."
843                 ;;
844         esac
845 }