common/populate: make sure _scratch_xfs_populate puts its files on the data device
[xfstests-dev.git] / common / populate
1 ##/bin/bash
2 # SPDX-License-Identifier: GPL-2.0+
3 # Copyright (c) 2015 Oracle.  All Rights Reserved.
4 #
5 # Routines for populating a scratch fs, and helpers to exercise an FS
6 # once it's been fuzzed.
7
8 . ./common/quota
9
10 _require_populate_commands() {
11         _require_xfs_io_command "falloc"
12         _require_xfs_io_command "fpunch"
13         _require_test_program "punch-alternating"
14         _require_command "$XFS_DB_PROG" "xfs_db"
15 }
16
17 _require_xfs_db_blocktrash_z_command() {
18         test "${FSTYP}" = "xfs" || _notrun "cannot run xfs_db on ${FSTYP}"
19         $XFS_DB_PROG -x -f -c 'blocktrash -z' "${TEST_DEV}" | grep -q 'nothing on stack' || _notrun "blocktrash -z not supported"
20 }
21
22 # Attempt to make files of "every" format for data, dirs, attrs etc.
23 # (with apologies to Eric Sandeen for mutating xfser.sh)
24
25 # Create a file of a given size.
26 __populate_create_file() {
27         local sz="$1"
28         local fname="$2"
29
30         $XFS_IO_PROG -f -c "pwrite -S 0x62 -W -b 1m 0 $sz" "${fname}"
31 }
32
33 # Punch out every other hole in this file, if it exists.
34 #
35 # The goal here is to force the creation of a large number of metadata records
36 # by creating a lot of tiny extent mappings in a file.  Callers should ensure
37 # that fragmenting the file actually causes record creation.  Call this
38 # function /after/ creating all other metadata structures.
39 __populate_fragment_file() {
40         local fname="$1"
41
42         test -f "${fname}" && $here/src/punch-alternating "${fname}"
43 }
44
45 # Create a large directory
46 __populate_create_dir() {
47         name="$1"
48         nr="$2"
49         missing="$3"
50
51         mkdir -p "${name}"
52         seq 0 "${nr}" | while read d; do
53                 creat=mkdir
54                 test "$((d % 20))" -eq 0 && creat=touch
55                 $creat "${name}/$(printf "%.08d" "$d")"
56         done
57
58         test -z "${missing}" && return
59         seq 1 2 "${nr}" | while read d; do
60                 rm -rf "${name}/$(printf "%.08d" "$d")"
61         done
62 }
63
64 # Add a bunch of attrs to a file
65 __populate_create_attr() {
66         name="$1"
67         nr="$2"
68         missing="$3"
69
70         touch "${name}"
71         seq 0 "${nr}" | while read d; do
72                 setfattr -n "user.$(printf "%.08d" "$d")" -v "$(printf "%.08d" "$d")" "${name}"
73         done
74
75         test -z "${missing}" && return
76         seq 1 2 "${nr}" | while read d; do
77                 setfattr -x "user.$(printf "%.08d" "$d")" "${name}"
78         done
79 }
80
81 # Fill up some percentage of the remaining free space
82 __populate_fill_fs() {
83         dir="$1"
84         pct="$2"
85         test -z "${pct}" && pct=60
86
87         mkdir -p "${dir}/test/1"
88         cp -pRdu "${dir}"/S_IFREG* "${dir}/test/1/"
89
90         SRC_SZ="$(du -ks "${dir}/test/1" | cut -f 1)"
91         FS_SZ="$(( $(stat -f "${dir}" -c '%a * %S') / 1024 ))"
92
93         NR="$(( (FS_SZ * ${pct} / 100) / SRC_SZ ))"
94
95         echo "FILL FS"
96         echo "src_sz $SRC_SZ fs_sz $FS_SZ nr $NR"
97         seq 2 "${NR}" | while read nr; do
98                 cp -pRdu "${dir}/test/1" "${dir}/test/${nr}"
99         done
100 }
101
102 # For XFS, force on all the quota options if quota is enabled
103 # and the user didn't feed us noquota.
104 _populate_xfs_qmount_option()
105 {
106         # User explicitly told us not to quota
107         if echo "${MOUNT_OPTIONS}" | grep -q 'noquota'; then
108                 return
109         fi
110
111         # Don't bother if we can't turn on quotas
112         if [ ! -f /proc/fs/xfs/xqmstat ]; then
113                 # No quota support
114                 return
115         elif [ "${USE_EXTERNAL}" = "yes" ] && [ ! -z "${SCRATCH_RTDEV}" ]; then
116                 # Quotas not supported on rt filesystems
117                 return
118         elif [ -z "${XFS_QUOTA_PROG}" ]; then
119                 # xfs quota tools not installed
120                 return
121         fi
122
123         # Turn on all the quotas
124         if $XFS_INFO_PROG "${TEST_DIR}" | grep -q 'crc=1'; then
125                 # v5 filesystems can have group & project quotas
126                 quota="usrquota,grpquota,prjquota"
127         else
128                 # v4 filesystems cannot mix group & project quotas
129                 quota="usrquota,grpquota"
130         fi
131
132         # Inject our quota mount options
133         if echo "${MOUNT_OPTIONS}" | grep -q "${quota}"; then
134                 return
135         elif echo "${MOUNT_OPTIONS}" | egrep -q '(quota|noenforce)'; then
136                 _qmount_option "${quota}"
137         else
138                 export MOUNT_OPTIONS="$MOUNT_OPTIONS -o ${quota}"
139                 echo "MOUNT_OPTIONS = $MOUNT_OPTIONS" >>$seqres.full
140         fi
141 }
142
143 # Populate an XFS on the scratch device with (we hope) all known
144 # types of metadata block
145 _scratch_xfs_populate() {
146         fill=1
147
148         for arg in $@; do
149                 case "${arg}" in
150                 "nofill")
151                         fill=0;;
152                 esac
153         done
154
155         _populate_xfs_qmount_option
156         _scratch_mount
157
158         # We cannot directly force the filesystem to create the metadata
159         # structures we want; we can only achieve this indirectly by carefully
160         # crafting files and a directory tree.  Therefore, we must have exact
161         # control over the layout and device selection of all files created.
162         # Clear the rtinherit flag on the root directory so that files are
163         # always created on the data volume regardless of MKFS_OPTIONS.  We can
164         # set the realtime flag when needed.
165         $XFS_IO_PROG -c 'chattr -t' $SCRATCH_MNT
166
167         blksz="$(stat -f -c '%s' "${SCRATCH_MNT}")"
168         dblksz="$($XFS_INFO_PROG "${SCRATCH_MNT}" | grep naming.*bsize | sed -e 's/^.*bsize=//g' -e 's/\([0-9]*\).*$/\1/g')"
169         crc="$($XFS_INFO_PROG "${SCRATCH_MNT}" | grep crc= | sed -e 's/^.*crc=//g' -e 's/\([0-9]*\).*$/\1/g')"
170         if [ $crc -eq 1 ]; then
171                 leaf_hdr_size=64
172         else
173                 leaf_hdr_size=16
174         fi
175         leaf_lblk="$((32 * 1073741824 / blksz))"
176         node_lblk="$((64 * 1073741824 / blksz))"
177
178         # Data:
179
180         # Fill up the root inode chunk
181         echo "+ fill root ino chunk"
182         seq 1 64 | while read f; do
183                 $XFS_IO_PROG -f -c "truncate 0" "${SCRATCH_MNT}/dummy${f}"
184         done
185
186         # Regular files
187         # - FMT_EXTENTS
188         echo "+ extents file"
189         __populate_create_file $blksz "${SCRATCH_MNT}/S_IFREG.FMT_EXTENTS"
190
191         # - FMT_BTREE
192         echo "+ btree extents file"
193         nr="$((blksz * 2 / 16))"
194         __populate_create_file $((blksz * nr)) "${SCRATCH_MNT}/S_IFREG.FMT_BTREE"
195
196         # Directories
197         # - INLINE
198         echo "+ inline dir"
199         __populate_create_dir "${SCRATCH_MNT}/S_IFDIR.FMT_INLINE" 1
200
201         # - BLOCK
202         echo "+ block dir"
203         __populate_create_dir "${SCRATCH_MNT}/S_IFDIR.FMT_BLOCK" "$((dblksz / 40))"
204
205         # - LEAF
206         echo "+ leaf dir"
207         __populate_create_dir "${SCRATCH_MNT}/S_IFDIR.FMT_LEAF" "$((dblksz / 12))"
208
209         # - LEAFN
210         echo "+ leafn dir"
211         __populate_create_dir "${SCRATCH_MNT}/S_IFDIR.FMT_LEAFN" "$(( ((dblksz - leaf_hdr_size) / 8) - 3 ))"
212
213         # - NODE
214         echo "+ node dir"
215         __populate_create_dir "${SCRATCH_MNT}/S_IFDIR.FMT_NODE" "$((16 * dblksz / 40))" true
216
217         # - BTREE
218         echo "+ btree dir"
219         __populate_create_dir "${SCRATCH_MNT}/S_IFDIR.FMT_BTREE" "$((128 * dblksz / 40))" true
220
221         # Symlinks
222         # - FMT_LOCAL
223         echo "+ inline symlink"
224         ln -s target "${SCRATCH_MNT}/S_IFLNK.FMT_LOCAL"
225
226         # - FMT_EXTENTS
227         echo "+ extents symlink"
228         ln -s "$(perl -e 'print "x" x 1023;')" "${SCRATCH_MNT}/S_IFLNK.FMT_EXTENTS"
229
230         # Char & block
231         echo "+ special"
232         mknod "${SCRATCH_MNT}/S_IFCHR" c 1 1
233         mknod "${SCRATCH_MNT}/S_IFBLK" c 1 1
234
235         # special file with an xattr
236         setfacl -P -m u:nobody:r ${SCRATCH_MNT}/S_IFCHR
237
238         # Attribute formats
239         # LOCAL
240         echo "+ local attr"
241         __populate_create_attr "${SCRATCH_MNT}/ATTR.FMT_LOCAL" 1
242
243         # LEAF
244         echo "+ leaf attr"
245         __populate_create_attr "${SCRATCH_MNT}/ATTR.FMT_LEAF" "$((blksz / 40))"
246
247         # NODE
248         echo "+ node attr"
249         __populate_create_attr "${SCRATCH_MNT}/ATTR.FMT_NODE" "$((8 * blksz / 40))"
250
251         # BTREE
252         echo "+ btree attr"
253         __populate_create_attr "${SCRATCH_MNT}/ATTR.FMT_BTREE" "$((64 * blksz / 40))" true
254
255         # trusted namespace
256         touch ${SCRATCH_MNT}/ATTR.TRUSTED
257         setfattr -n trusted.moo -v urk ${SCRATCH_MNT}/ATTR.TRUSTED
258
259         # security namespace
260         touch ${SCRATCH_MNT}/ATTR.SECURITY
261         setfattr -n security.foo -v bar ${SCRATCH_MNT}/ATTR.SECURITY
262
263         # system namespace
264         touch ${SCRATCH_MNT}/ATTR.SYSTEM
265         setfacl -m u:root:r ${SCRATCH_MNT}/ATTR.SYSTEM
266
267         # FMT_EXTENTS with a remote less-than-a-block value
268         echo "+ attr extents with a remote less-than-a-block value"
269         touch "${SCRATCH_MNT}/ATTR.FMT_EXTENTS_REMOTE3K"
270         $XFS_IO_PROG -f -c "pwrite -S 0x43 0 $((blksz - 300))" "${SCRATCH_MNT}/attrvalfile" > /dev/null
271         attr -q -s user.remotebtreeattrname "${SCRATCH_MNT}/ATTR.FMT_EXTENTS_REMOTE3K" < "${SCRATCH_MNT}/attrvalfile"
272
273         # FMT_EXTENTS with a remote block-size value
274         echo "+ attr extents with a remote one-block value"
275         touch "${SCRATCH_MNT}/ATTR.FMT_EXTENTS_REMOTE4K"
276         $XFS_IO_PROG -f -c "pwrite -S 0x44 0 ${blksz}" "${SCRATCH_MNT}/attrvalfile" > /dev/null
277         attr -q -s user.remotebtreeattrname "${SCRATCH_MNT}/ATTR.FMT_EXTENTS_REMOTE4K" < "${SCRATCH_MNT}/attrvalfile"
278         rm -rf "${SCRATCH_MNT}/attrvalfile"
279
280         # Make an unused inode
281         echo "+ empty file"
282         touch "${SCRATCH_MNT}/unused"
283         $XFS_IO_PROG -f -c 'fsync' "${SCRATCH_MNT}/unused"
284         rm -rf "${SCRATCH_MNT}/unused"
285
286         # Free space btree
287         echo "+ freesp btree"
288         nr="$((blksz * 2 / 8))"
289         __populate_create_file $((blksz * nr)) "${SCRATCH_MNT}/BNOBT"
290
291         # Inode btree
292         echo "+ inobt btree"
293         local ino_per_rec=64
294         local rec_per_btblock=16
295         local nr="$(( 2 * (blksz / rec_per_btblock) * ino_per_rec ))"
296         local dir="${SCRATCH_MNT}/INOBT"
297         mkdir -p "${dir}"
298         seq 0 "${nr}" | while read f; do
299                 touch "${dir}/${f}"
300         done
301
302         seq 0 2 "${nr}" | while read f; do
303                 rm -f "${dir}/${f}"
304         done
305
306         # Reverse-mapping btree
307         is_rmapbt="$($XFS_INFO_PROG "${SCRATCH_MNT}" | grep -c 'rmapbt=1')"
308         if [ $is_rmapbt -gt 0 ]; then
309                 echo "+ rmapbt btree"
310                 nr="$((blksz * 2 / 24))"
311                 __populate_create_file $((blksz * nr)) "${SCRATCH_MNT}/RMAPBT"
312         fi
313
314         # Realtime Reverse-mapping btree
315         is_rt="$($XFS_INFO_PROG "${SCRATCH_MNT}" | grep -c 'rtextents=[1-9]')"
316         if [ $is_rmapbt -gt 0 ] && [ $is_rt -gt 0 ]; then
317                 echo "+ rtrmapbt btree"
318                 nr="$((blksz * 2 / 32))"
319                 $XFS_IO_PROG -R -f -c 'truncate 0' "${SCRATCH_MNT}/RTRMAPBT"
320                 __populate_create_file $((blksz * nr)) "${SCRATCH_MNT}/RTRMAPBT"
321         fi
322
323         # Reference-count btree
324         is_reflink="$($XFS_INFO_PROG "${SCRATCH_MNT}" | grep -c 'reflink=1')"
325         if [ $is_reflink -gt 0 ]; then
326                 echo "+ reflink btree"
327                 nr="$((blksz * 2 / 12))"
328                 __populate_create_file $((blksz * nr)) "${SCRATCH_MNT}/REFCOUNTBT"
329                 cp --reflink=always "${SCRATCH_MNT}/REFCOUNTBT" "${SCRATCH_MNT}/REFCOUNTBT2"
330         fi
331
332         # Copy some real files (xfs tests, I guess...)
333         echo "+ real files"
334         test $fill -ne 0 && __populate_fill_fs "${SCRATCH_MNT}" 5
335
336         # Make sure we get all the fragmentation we asked for
337         __populate_fragment_file "${SCRATCH_MNT}/S_IFREG.FMT_BTREE"
338         __populate_fragment_file "${SCRATCH_MNT}/BNOBT"
339         __populate_fragment_file "${SCRATCH_MNT}/RMAPBT"
340         __populate_fragment_file "${SCRATCH_MNT}/RTRMAPBT"
341         __populate_fragment_file "${SCRATCH_MNT}/REFCOUNTBT"
342
343         umount "${SCRATCH_MNT}"
344 }
345
346 # Populate an ext4 on the scratch device with (we hope) all known
347 # types of metadata block
348 _scratch_ext4_populate() {
349         fill=1
350
351         for arg in $@; do
352                 case "${arg}" in
353                 "nofill")
354                         fill=0;;
355                 esac
356         done
357
358         _scratch_mount
359         blksz="$(stat -f -c '%s' "${SCRATCH_MNT}")"
360         dblksz="${blksz}"
361         leaf_lblk="$((32 * 1073741824 / blksz))"
362         node_lblk="$((64 * 1073741824 / blksz))"
363
364         # Data:
365
366         # Regular files
367         # - FMT_INLINE
368         echo "+ inline file"
369         __populate_create_file 1 "${SCRATCH_MNT}/S_IFREG.FMT_INLINE"
370
371         # - FMT_EXTENTS
372         echo "+ extents file"
373         __populate_create_file $blksz "${SCRATCH_MNT}/S_IFREG.FMT_EXTENTS"
374
375         # - FMT_ETREE
376         echo "+ extent tree file"
377         nr="$((blksz * 2 / 12))"
378         __populate_create_file $((blksz * nr)) "${SCRATCH_MNT}/S_IFREG.FMT_ETREE"
379
380         # Directories
381         # - INLINE
382         echo "+ inline dir"
383         __populate_create_dir "${SCRATCH_MNT}/S_IFDIR.FMT_INLINE" 1
384
385         # - BLOCK
386         echo "+ block dir"
387         __populate_create_dir "${SCRATCH_MNT}/S_IFDIR.FMT_BLOCK" "$((dblksz / 32))"
388
389         # - HTREE
390         echo "+ htree dir"
391         __populate_create_dir "${SCRATCH_MNT}/S_IFDIR.FMT_HTREE" "$((4 * dblksz / 24))"
392
393         # Symlinks
394         # - FMT_LOCAL
395         echo "+ inline symlink"
396         ln -s target "${SCRATCH_MNT}/S_IFLNK.FMT_LOCAL"
397
398         # - FMT_EXTENTS
399         echo "+ extents symlink"
400         ln -s "$(perl -e 'print "x" x 1023;')" "${SCRATCH_MNT}/S_IFLNK.FMT_EXTENTS"
401
402         # Char & block
403         echo "+ special"
404         mknod "${SCRATCH_MNT}/S_IFCHR" c 1 1
405         mknod "${SCRATCH_MNT}/S_IFBLK" c 1 1
406
407         # special file with an xattr
408         setfacl -P -m u:nobody:r ${SCRATCH_MNT}/S_IFCHR
409
410         # Attribute formats
411         # LOCAL
412         echo "+ local attr"
413         __populate_create_attr "${SCRATCH_MNT}/ATTR.FMT_LOCAL" 0
414
415         # BLOCK
416         echo "+ block attr"
417         __populate_create_attr "${SCRATCH_MNT}/ATTR.FMT_BLOCK" "$((blksz / 40))"
418
419         # trusted namespace
420         touch ${SCRATCH_MNT}/ATTR.TRUSTED
421         setfattr -n trusted.moo -v urk ${SCRATCH_MNT}/ATTR.TRUSTED
422
423         # security namespace
424         touch ${SCRATCH_MNT}/ATTR.SECURITY
425         setfattr -n security.foo -v bar ${SCRATCH_MNT}/ATTR.SECURITY
426
427         # system namespace
428         touch ${SCRATCH_MNT}/ATTR.SYSTEM
429         setfacl -m u:root:r ${SCRATCH_MNT}/ATTR.SYSTEM
430
431         # Make an unused inode
432         echo "+ empty file"
433         touch "${SCRATCH_MNT}/unused"
434         $XFS_IO_PROG -f -c 'fsync' "${SCRATCH_MNT}/unused"
435         rm -rf "${SCRATCH_MNT}/unused"
436
437         # Copy some real files (xfs tests, I guess...)
438         echo "+ real files"
439         test $fill -ne 0 && __populate_fill_fs "${SCRATCH_MNT}" 5
440
441         # Make sure we get all the fragmentation we asked for
442         __populate_fragment_file "${SCRATCH_MNT}/S_IFREG.FMT_ETREE"
443
444         umount "${SCRATCH_MNT}"
445 }
446
447 # Find the inode number of a file
448 __populate_find_inode() {
449         name="$1"
450         inode="$(stat -c '%i' "${name}")"
451         echo "${inode}"
452 }
453
454 # Check data fork format of XFS file
455 __populate_check_xfs_dformat() {
456         inode="$1"
457         format="$2"
458
459         fmt="$(_scratch_xfs_db -c "inode ${inode}" -c 'p core.format' | sed -e 's/^.*(\([a-z]*\)).*$/\1/g')"
460         test "${format}" = "${fmt}" || _fail "failed to create ino ${inode} dformat expected ${format} saw ${fmt}"
461 }
462
463 # Check attr fork format of XFS file
464 __populate_check_xfs_aformat() {
465         inode="$1"
466         format="$2"
467
468         fmt="$(_scratch_xfs_db -c "inode ${inode}" -c 'p core.aformat' | sed -e 's/^.*(\([a-z]*\)).*$/\1/g')"
469         test "${format}" = "${fmt}" || _fail "failed to create ino ${inode} aformat expected ${format} saw ${fmt}"
470 }
471
472 # Check structure of XFS directory
473 __populate_check_xfs_dir() {
474         inode="$1"
475         dtype="$2"
476
477         (test -n "${leaf_lblk}" && test -n "${node_lblk}") || _fail "must define leaf_lblk and node_lblk before calling __populate_check_xfs_dir"
478         datab=0
479         leafb=0
480         freeb=0
481         #echo "== check dir ${inode} type ${dtype}" ; _scratch_xfs_db -x -c "inode ${inode}" -c "bmap"
482         _scratch_xfs_db -x -c "inode ${inode}" -c "dblock 0" -c "stack" | grep -q 'file data block is unmapped' || datab=1
483         _scratch_xfs_db -x -c "inode ${inode}" -c "dblock ${leaf_lblk}" -c "stack" | grep -q 'file data block is unmapped' || leafb=1
484         _scratch_xfs_db -x -c "inode ${inode}" -c "dblock ${node_lblk}" -c "stack" | grep -q 'file data block is unmapped' || freeb=1
485
486         case "${dtype}" in
487         "shortform"|"inline"|"local")
488                 (test "${datab}" -eq 0 && test "${leafb}" -eq 0 && test "${freeb}" -eq 0) || _fail "failed to create ${dtype} dir ino ${inode} datab ${datab} leafb ${leafb} freeb ${freeb}"
489                 ;;
490         "block")
491                 (test "${datab}" -eq 1 && test "${leafb}" -eq 0 && test "${freeb}" -eq 0) || _fail "failed to create ${dtype} dir ino ${inode} datab ${datab} leafb ${leafb} freeb ${freeb}"
492                 ;;
493         "leaf")
494                 (test "${datab}" -eq 1 && test "${leafb}" -eq 1 && test "${freeb}" -eq 0) || _fail "failed to create ${dtype} dir ino ${inode} datab ${datab} leafb ${leafb} freeb ${freeb}"
495                 ;;
496         "leafn")
497                 _scratch_xfs_db -x -c "inode ${inode}" -c "dblock ${leaf_lblk}" -c "p lhdr.info.hdr.magic" | grep -q '0x3dff' && return
498                 _scratch_xfs_db -x -c "inode ${inode}" -c "dblock ${leaf_lblk}" -c "p lhdr.info.magic" | grep -q '0xd2ff' && return
499                 _fail "failed to create ${dtype} dir ino ${inode} datab ${datab} leafb ${leafb} freeb ${freeb}"
500                 ;;
501         "node"|"btree")
502                 (test "${datab}" -eq 1 && test "${leafb}" -eq 1 && test "${freeb}" -eq 1) || _fail "failed to create ${dtype} dir ino ${inode} datab ${datab} leafb ${leafb} freeb ${freeb}"
503                 ;;
504         *)
505                 _fail "Unknown directory type ${dtype}"
506         esac
507 }
508
509 # Check structure of XFS attr
510 __populate_check_xfs_attr() {
511         inode="$1"
512         atype="$2"
513
514         datab=0
515         leafb=0
516         #echo "== check attr ${inode} type ${dtype}" ; _scratch_xfs_db -x -c "inode ${inode}" -c "bmap -a"
517         _scratch_xfs_db -x -c "inode ${inode}" -c "ablock 0" -c "stack" | grep -q 'file attr block is unmapped' || datab=1
518         _scratch_xfs_db -x -c "inode ${inode}" -c "ablock 1" -c "stack" | grep -q 'file attr block is unmapped' || leafb=1
519
520         case "${atype}" in
521         "shortform"|"inline"|"local")
522                 (test "${datab}" -eq 0 && test "${leafb}" -eq 0) || _fail "failed to create ${atype} attr ino ${inode} datab ${datab} leafb ${leafb}"
523                 ;;
524         "leaf")
525                 (test "${datab}" -eq 1 && test "${leafb}" -eq 0) || _fail "failed to create ${atype} attr ino ${inode} datab ${datab} leafb ${leafb}"
526                 ;;
527         "node"|"btree")
528                 (test "${datab}" -eq 1 && test "${leafb}" -eq 1) || _fail "failed to create ${atype} attr ino ${inode} datab ${datab} leafb ${leafb}"
529                 ;;
530         *)
531                 _fail "Unknown attribute type ${atype}"
532         esac
533 }
534
535 # Check that there's at least one per-AG btree with multiple levels
536 __populate_check_xfs_agbtree_height() {
537         bt_type="$1"
538         nr_ags=$(_scratch_xfs_db -c 'sb 0' -c 'p agcount' | awk '{print $3}')
539
540         case "${bt_type}" in
541         "bno"|"cnt"|"rmap"|"refcnt")
542                 hdr="agf"
543                 bt_prefix="${bt_type}"
544                 ;;
545         "ino")
546                 hdr="agi"
547                 bt_prefix=""
548                 ;;
549         "fino")
550                 hdr="agi"
551                 bt_prefix="free_"
552                 ;;
553         *)
554                 _fail "Don't know about AG btree ${bt_type}"
555                 ;;
556         esac
557
558         seq 0 $((nr_ags - 1)) | while read ag; do
559                 bt_level=$(_scratch_xfs_db -c "${hdr} ${ag}" -c "p ${bt_prefix}level" | awk '{print $3}')
560                 if [ "${bt_level}" -gt 1 ]; then
561                         return 100
562                 fi
563         done
564         test $? -eq 100 || _fail "Failed to create ${bt_type} of sufficient height!"
565         return 1
566 }
567
568 # Check that populate created all the types of files we wanted
569 _scratch_xfs_populate_check() {
570         _scratch_mount
571         extents_file="$(__populate_find_inode "${SCRATCH_MNT}/S_IFREG.FMT_EXTENTS")"
572         btree_file="$(__populate_find_inode "${SCRATCH_MNT}/S_IFREG.FMT_BTREE")"
573         inline_dir="$(__populate_find_inode "${SCRATCH_MNT}/S_IFDIR.FMT_INLINE")"
574         block_dir="$(__populate_find_inode "${SCRATCH_MNT}/S_IFDIR.FMT_BLOCK")"
575         leaf_dir="$(__populate_find_inode "${SCRATCH_MNT}/S_IFDIR.FMT_LEAF")"
576         leafn_dir="$(__populate_find_inode "${SCRATCH_MNT}/S_IFDIR.FMT_LEAFN")"
577         node_dir="$(__populate_find_inode "${SCRATCH_MNT}/S_IFDIR.FMT_NODE")"
578         btree_dir="$(__populate_find_inode "${SCRATCH_MNT}/S_IFDIR.FMT_BTREE")"
579         local_slink="$(__populate_find_inode "${SCRATCH_MNT}/S_IFLNK.FMT_LOCAL")"
580         extents_slink="$(__populate_find_inode "${SCRATCH_MNT}/S_IFLNK.FMT_EXTENTS")"
581         bdev="$(__populate_find_inode "${SCRATCH_MNT}/S_IFBLK")"
582         cdev="$(__populate_find_inode "${SCRATCH_MNT}/S_IFCHR")"
583         local_attr="$(__populate_find_inode "${SCRATCH_MNT}/ATTR.FMT_LOCAL")"
584         leaf_attr="$(__populate_find_inode "${SCRATCH_MNT}/ATTR.FMT_LEAF")"
585         node_attr="$(__populate_find_inode "${SCRATCH_MNT}/ATTR.FMT_NODE")"
586         btree_attr="$(__populate_find_inode "${SCRATCH_MNT}/ATTR.FMT_BTREE")"
587         is_finobt=$($XFS_INFO_PROG "${SCRATCH_MNT}" | grep -c 'finobt=1')
588         is_rmapbt=$($XFS_INFO_PROG "${SCRATCH_MNT}" | grep -c 'rmapbt=1')
589         is_reflink=$($XFS_INFO_PROG "${SCRATCH_MNT}" | grep -c 'reflink=1')
590
591         blksz="$(stat -f -c '%s' "${SCRATCH_MNT}")"
592         dblksz="$($XFS_INFO_PROG "${SCRATCH_MNT}" | grep naming.*bsize | sed -e 's/^.*bsize=//g' -e 's/\([0-9]*\).*$/\1/g')"
593         leaf_lblk="$((32 * 1073741824 / blksz))"
594         node_lblk="$((64 * 1073741824 / blksz))"
595         umount "${SCRATCH_MNT}"
596
597         __populate_check_xfs_dformat "${extents_file}" "extents"
598         __populate_check_xfs_dformat "${btree_file}" "btree"
599         __populate_check_xfs_dir "${inline_dir}" "inline"
600         __populate_check_xfs_dir "${block_dir}" "block"
601         __populate_check_xfs_dir "${leaf_dir}" "leaf"
602         __populate_check_xfs_dir "${leafn_dir}" "leafn"
603         __populate_check_xfs_dir "${node_dir}" "node"
604         __populate_check_xfs_dir "${btree_dir}" "btree"
605         __populate_check_xfs_dformat "${btree_dir}" "btree"
606         __populate_check_xfs_dformat "${bdev}" "dev"
607         __populate_check_xfs_dformat "${cdev}" "dev"
608         __populate_check_xfs_attr "${local_attr}" "local"
609         __populate_check_xfs_attr "${leaf_attr}" "leaf"
610         __populate_check_xfs_attr "${node_attr}" "node"
611         __populate_check_xfs_attr "${btree_attr}" "btree"
612         __populate_check_xfs_aformat "${btree_attr}" "btree"
613         __populate_check_xfs_agbtree_height "bno"
614         __populate_check_xfs_agbtree_height "cnt"
615         __populate_check_xfs_agbtree_height "ino"
616         test $is_finobt -ne 0 && __populate_check_xfs_agbtree_height "fino"
617         test $is_rmapbt -ne 0 && __populate_check_xfs_agbtree_height "rmap"
618         test $is_reflink -ne 0 && __populate_check_xfs_agbtree_height "refcnt"
619 }
620
621 # Check data fork format of ext4 file
622 __populate_check_ext4_dformat() {
623         dev="${SCRATCH_DEV}"
624         inode="$1"
625         format="$2"
626
627         extents=0
628         etree=0
629         debugfs -R "stat <${inode}>" "${dev}" 2> /dev/null | grep 'ETB[0-9]' -q && etree=1
630         iflags="$(debugfs -R "stat <${inode}>" "${dev}" 2> /dev/null | grep 'Flags:' | sed -e 's/^.*Flags: \([0-9a-fx]*\).*$/\1/g')"
631         test "$(echo "${iflags}" | awk '{print and(strtonum($1), 0x80000);}')" -gt 0 && extents=1
632
633         case "${format}" in
634         "blockmap")
635                 test "${extents}" -eq 0 || _fail "failed to create ino ${inode} with blockmap"
636                 ;;
637         "extent"|"extents")
638                 test "${extents}" -eq 1 || _fail "failed to create ino ${inode} with extents"
639                 ;;
640         "etree")
641                 (test "${extents}" -eq 1 && test "${etree}" -eq 1) || _fail "failed to create ino ${inode} with extent tree"
642                 ;;
643         *)
644                 _fail "Unknown dformat ${format}"
645         esac
646 }
647
648 # Check attr fork format of ext4 file
649 __populate_check_ext4_aformat() {
650         dev="${SCRATCH_DEV}"
651         inode="$1"
652         format="$2"
653
654         ablock=1
655         debugfs -R "stat <${inode}>" "${dev}" 2> /dev/null | grep 'File ACL: 0' -q && ablock=0
656
657         case "${format}" in
658         "local"|"inline")
659                 test "${ablock}" -eq 0 || _fail "failed to create inode ${inode} with ${format} xattr"
660                 ;;
661         "block")
662                 test "${extents}" -eq 1 || _fail "failed to create inode ${inode} with ${format} xattr"
663                 ;;
664         *)
665                 _fail "Unknown aformat ${format}"
666         esac
667 }
668
669 # Check structure of ext4 dir
670 __populate_check_ext4_dir() {
671         dev="${SCRATCH_DEV}"
672         inode="$1"
673         dtype="$2"
674
675         htree=0
676         inline=0
677         iflags="$(debugfs -R "stat <${inode}>" "${dev}" 2> /dev/null | grep 'Flags:' | sed -e 's/^.*Flags: \([0-9a-fx]*\).*$/\1/g')"
678         test "$(echo "${iflags}" | awk '{print and(strtonum($1), 0x1000);}')" -gt 0 && htree=1
679         test "$(echo "${iflags}" | awk '{print and(strtonum($1), 0x10000000);}')" -gt 0 && inline=1
680
681         case "${dtype}" in
682         "inline")
683                 (test "${inline}" -eq 1 && test "${htree}" -eq 0) || _fail "failed to create ${dtype} dir ino ${inode} htree ${htree} inline ${inline}"
684                 ;;
685         "block")
686                 (test "${inline}" -eq 0 && test "${htree}" -eq 0) || _fail "failed to create ${dtype} dir ino ${inode} htree ${htree} inline ${inline}"
687                 ;;
688         "htree")
689                 (test "${inline}" -eq 0 && test "${htree}" -eq 1) || _fail "failed to create ${dtype} dir ino ${inode} htree ${htree} inline ${inline}"
690                 ;;
691         *)
692                 _fail "Unknown directory type ${dtype}"
693                 ;;
694         esac
695 }
696
697 # Check that populate created all the types of files we wanted
698 _scratch_ext4_populate_check() {
699         _scratch_mount
700         extents_file="$(__populate_find_inode "${SCRATCH_MNT}/S_IFREG.FMT_EXTENTS")"
701         etree_file="$(__populate_find_inode "${SCRATCH_MNT}/S_IFREG.FMT_ETREE")"
702         block_dir="$(__populate_find_inode "${SCRATCH_MNT}/S_IFDIR.FMT_BLOCK")"
703         htree_dir="$(__populate_find_inode "${SCRATCH_MNT}/S_IFDIR.FMT_HTREE")"
704         extents_slink="$(__populate_find_inode "${SCRATCH_MNT}/S_IFLNK.FMT_EXTENTS")"
705         local_attr="$(__populate_find_inode "${SCRATCH_MNT}/ATTR.FMT_LOCAL")"
706         block_attr="$(__populate_find_inode "${SCRATCH_MNT}/ATTR.FMT_BLOCK")"
707         umount "${SCRATCH_MNT}"
708
709         __populate_check_ext4_dformat "${extents_file}" "extents"
710         __populate_check_ext4_dformat "${etree_file}" "etree"
711         __populate_check_ext4_dir "${block_dir}" "block"
712         __populate_check_ext4_dir "${htree_dir}" "htree"
713         __populate_check_ext4_dformat "${extents_slink}" "extents"
714         __populate_check_ext4_aformat "${local_attr}" "local"
715         __populate_check_ext4_aformat "${block_attr}" "block"
716 }
717
718 # Populate a scratch FS and check the contents to make sure we got that
719 _scratch_populate() {
720         case "${FSTYP}" in
721         "xfs")
722                 _scratch_xfs_populate
723                 _scratch_xfs_populate_check
724                 ;;
725         "ext2"|"ext3"|"ext4")
726                 _scratch_ext4_populate
727                 _scratch_ext4_populate_check
728                 ;;
729         *)
730                 _fail "Don't know how to populate a ${FSTYP} filesystem."
731                 ;;
732         esac
733 }
734
735 # Fill a file system by repeatedly creating files in the given folder
736 # starting with the given file size.  Files are reduced in size when
737 # they can no longer fit until no more files can be created.
738 _fill_fs()
739 {
740         local file_size=$1
741         local dir=$2
742         local block_size=$3
743         local switch_user=$4
744         local file_count=1
745         local bytes_written=0
746         local use_falloc=1;
747
748         if [ $# -ne 4 ]; then
749                 echo "Usage: _fill_fs filesize dir blocksize switch_user"
750                 exit 1
751         fi
752
753         if [ $switch_user -eq 0 ]; then
754                 mkdir -p $dir
755         else
756                 _user_do "mkdir -p $dir"
757         fi
758         if [ ! -d $dir ]; then
759                 return 0;
760         fi
761
762         testio=`$XFS_IO_PROG -F -fc "falloc 0 $block_size" $dir/$$.xfs_io 2>&1`
763         echo $testio | grep -q "not found" && use_falloc=0
764         echo $testio | grep -q "Operation not supported" && use_falloc=0
765
766         if [ $file_size -lt $block_size ]; then
767                 $file_size = $block_size
768         fi
769
770         while [ $file_size -ge $block_size ]; do
771                 bytes_written=0
772                 if [ $switch_user -eq 0 ]; then
773                         if [ $use_falloc -eq 0 ]; then
774                                 $XFS_IO_PROG -fc "pwrite -b 8388608 0 $file_size" \
775                                         $dir/$file_count
776                         else
777                                 $XFS_IO_PROG -fc "falloc 0 $file_size" \
778                                         $dir/$file_count
779                         fi
780                 else
781                         if [ $use_falloc -eq 0 ]; then
782                                 _user_do "$XFS_IO_PROG -f -c \"pwrite -b 8388608 0 \
783                                         $file_size\" $dir/$file_count"
784                         else
785                                 _user_do "$XFS_IO_PROG -f -c \"falloc 0 \
786                                         $file_size\" $dir/$file_count"
787                         fi
788                 fi
789
790                 if [ -f $dir/$file_count ]; then
791                         bytes_written=$(_get_filesize $dir/$file_count)
792                 fi
793
794                 # If there was no room to make the file, then divide it in
795                 # half, and keep going
796                 if [ $bytes_written -lt $file_size ]; then
797                         file_size=$((file_size / 2))
798                 fi
799                 file_count=$((file_count + 1))
800         done
801 }
802
803 # Compute the fs geometry description of a populated filesystem
804 _scratch_populate_cache_tag() {
805         local extra_descr=""
806         local size="$(blockdev --getsz "${SCRATCH_DEV}")"
807
808         case "${FSTYP}" in
809         "ext4")
810                 extra_descr="LOGDEV ${SCRATCH_LOGDEV} USE_EXTERNAL ${USE_EXTERNAL}"
811                 ;;
812         "xfs")
813                 extra_descr="LOGDEV ${SCRATCH_LOGDEV} USE_EXTERNAL ${USE_EXTERNAL} RTDEV ${SCRATCH_RTDEV}"
814                 _populate_xfs_qmount_option
815                 if echo "${MOUNT_OPTIONS}" | grep -q 'usrquota'; then
816                         extra_descr="${extra_descr} QUOTAS"
817                 fi
818                 ;;
819         esac
820         echo "FSTYP ${FSTYP} MKFS_OPTIONS ${MKFS_OPTIONS} SIZE ${size} ${extra_descr} ARGS $@"
821 }
822
823 # Restore a cached populated fs from a metadata dump
824 _scratch_populate_restore_cached() {
825         local metadump="$1"
826
827         case "${FSTYP}" in
828         "xfs")
829                 xfs_mdrestore "${metadump}" "${SCRATCH_DEV}" && return 0
830                 ;;
831         "ext2"|"ext3"|"ext4")
832                 # ext4 cannot e2image external logs, so we cannot restore
833                 test -n "${SCRATCH_LOGDEV}" && return 1
834                 e2image -r "${metadump}" "${SCRATCH_DEV}" && return 0
835                 ;;
836         esac
837         return 1
838 }
839
840 # Populate a scratch FS from scratch or from a cached image.
841 _scratch_populate_cached() {
842         local meta_descr="$(_scratch_populate_cache_tag "$@")"
843         local meta_tag="$(echo "${meta_descr}" | md5sum - | cut -d ' ' -f 1)"
844         local metadump_stem="${TEST_DIR}/__populate.${FSTYP}.${meta_tag}"
845
846         # These variables are shared outside this function
847         POPULATE_METADUMP="${metadump_stem}.metadump"
848         POPULATE_METADUMP_DESCR="${metadump_stem}.txt"
849
850         # Don't keep metadata images cached for more 48 hours...
851         rm -rf "$(find "${POPULATE_METADUMP}" -mtime +2 2>/dev/null)"
852
853         # Throw away cached image if it doesn't match our spec.
854         cmp -s "${POPULATE_METADUMP_DESCR}" <(echo "${meta_descr}") || \
855                 rm -rf "${POPULATE_METADUMP}"
856
857         # Try to restore from the metadump
858         test -r "${POPULATE_METADUMP}" && \
859                 _scratch_populate_restore_cached "${POPULATE_METADUMP}" && \
860                 return
861
862         # Oh well, just create one from scratch
863         _scratch_mkfs
864         echo "${meta_descr}" > "${POPULATE_METADUMP_DESCR}"
865         case "${FSTYP}" in
866         "xfs")
867                 _scratch_xfs_populate $@
868                 _scratch_xfs_populate_check
869                 _scratch_metadump "${POPULATE_METADUMP}" -a -o
870                 ;;
871         "ext2"|"ext3"|"ext4")
872                 _scratch_ext4_populate $@
873                 _scratch_ext4_populate_check
874                 e2image -Q "${SCRATCH_DEV}" "${POPULATE_METADUMP}"
875                 ;;
876         *)
877                 _fail "Don't know how to populate a ${FSTYP} filesystem."
878                 ;;
879         esac
880 }