common/rc: generalize _get_filesize()
[xfstests-dev.git] / common / populate
1 ##/bin/bash
2 # SPDX-License-Identifier: GPL-2.0+
3 # Copyright (c) 2015 Oracle.  All Rights Reserved.
4 #
5 # Routines for populating a scratch fs, and helpers to exercise an FS
6 # once it's been fuzzed.
7
8 . ./common/quota
9
10 _require_populate_commands() {
11         _require_xfs_io_command "falloc"
12         _require_xfs_io_command "fpunch"
13         _require_test_program "punch-alternating"
14         _require_command "$XFS_DB_PROG" "xfs_db"
15 }
16
17 _require_xfs_db_blocktrash_z_command() {
18         test "${FSTYP}" = "xfs" || _notrun "cannot run xfs_db on ${FSTYP}"
19         $XFS_DB_PROG -x -f -c 'blocktrash -z' "${TEST_DEV}" | grep -q 'nothing on stack' || _notrun "blocktrash -z not supported"
20 }
21
22 # Attempt to make files of "every" format for data, dirs, attrs etc.
23 # (with apologies to Eric Sandeen for mutating xfser.sh)
24
25 # Create a file of a given size.
26 __populate_create_file() {
27         local sz="$1"
28         local fname="$2"
29
30         $XFS_IO_PROG -f -c "pwrite -S 0x62 -W -b 1m 0 $sz" "${fname}"
31 }
32
33 # Punch out every other hole in this file, if it exists.
34 #
35 # The goal here is to force the creation of a large number of metadata records
36 # by creating a lot of tiny extent mappings in a file.  Callers should ensure
37 # that fragmenting the file actually causes record creation.  Call this
38 # function /after/ creating all other metadata structures.
39 __populate_fragment_file() {
40         local fname="$1"
41
42         test -f "${fname}" && $here/src/punch-alternating "${fname}"
43 }
44
45 # Create a large directory
46 __populate_create_dir() {
47         name="$1"
48         nr="$2"
49         missing="$3"
50
51         mkdir -p "${name}"
52         seq 0 "${nr}" | while read d; do
53                 creat=mkdir
54                 test "$((d % 20))" -eq 0 && creat=touch
55                 $creat "${name}/$(printf "%.08d" "$d")"
56         done
57
58         test -z "${missing}" && return
59         seq 1 2 "${nr}" | while read d; do
60                 rm -rf "${name}/$(printf "%.08d" "$d")"
61         done
62 }
63
64 # Add a bunch of attrs to a file
65 __populate_create_attr() {
66         name="$1"
67         nr="$2"
68         missing="$3"
69
70         touch "${name}"
71         seq 0 "${nr}" | while read d; do
72                 setfattr -n "user.$(printf "%.08d" "$d")" -v "$(printf "%.08d" "$d")" "${name}"
73         done
74
75         test -z "${missing}" && return
76         seq 1 2 "${nr}" | while read d; do
77                 setfattr -x "user.$(printf "%.08d" "$d")" "${name}"
78         done
79 }
80
81 # Fill up some percentage of the remaining free space
82 __populate_fill_fs() {
83         dir="$1"
84         pct="$2"
85         test -z "${pct}" && pct=60
86
87         mkdir -p "${dir}/test/1"
88         cp -pRdu "${dir}"/S_IFREG* "${dir}/test/1/"
89
90         SRC_SZ="$(du -ks "${dir}/test/1" | cut -f 1)"
91         FS_SZ="$(( $(stat -f "${dir}" -c '%a * %S') / 1024 ))"
92
93         NR="$(( (FS_SZ * ${pct} / 100) / SRC_SZ ))"
94
95         echo "FILL FS"
96         echo "src_sz $SRC_SZ fs_sz $FS_SZ nr $NR"
97         seq 2 "${NR}" | while read nr; do
98                 cp -pRdu "${dir}/test/1" "${dir}/test/${nr}"
99         done
100 }
101
102 # For XFS, force on all the quota options if quota is enabled
103 # and the user didn't feed us noquota.
104 _populate_xfs_qmount_option()
105 {
106         # User explicitly told us not to quota
107         if echo "${MOUNT_OPTIONS}" | grep -q 'noquota'; then
108                 return
109         fi
110
111         # Don't bother if we can't turn on quotas
112         if [ ! -f /proc/fs/xfs/xqmstat ]; then
113                 # No quota support
114                 return
115         elif [ "${USE_EXTERNAL}" = "yes" ] && [ ! -z "${SCRATCH_RTDEV}" ]; then
116                 # Quotas not supported on rt filesystems
117                 return
118         elif [ -z "${XFS_QUOTA_PROG}" ]; then
119                 # xfs quota tools not installed
120                 return
121         fi
122
123         # Turn on all the quotas
124         if $XFS_INFO_PROG "${TEST_DIR}" | grep -q 'crc=1'; then
125                 # v5 filesystems can have group & project quotas
126                 quota="usrquota,grpquota,prjquota"
127         else
128                 # v4 filesystems cannot mix group & project quotas
129                 quota="usrquota,grpquota"
130         fi
131
132         # Inject our quota mount options
133         if echo "${MOUNT_OPTIONS}" | grep -q "${quota}"; then
134                 return
135         elif echo "${MOUNT_OPTIONS}" | egrep -q '(quota|noenforce)'; then
136                 _qmount_option "${quota}"
137         else
138                 export MOUNT_OPTIONS="$MOUNT_OPTIONS -o ${quota}"
139                 echo "MOUNT_OPTIONS = $MOUNT_OPTIONS" >>$seqres.full
140         fi
141 }
142
143 # Populate an XFS on the scratch device with (we hope) all known
144 # types of metadata block
145 _scratch_xfs_populate() {
146         fill=1
147
148         for arg in $@; do
149                 case "${arg}" in
150                 "nofill")
151                         fill=0;;
152                 esac
153         done
154
155         _populate_xfs_qmount_option
156         _scratch_mount
157         blksz="$(stat -f -c '%s' "${SCRATCH_MNT}")"
158         dblksz="$($XFS_INFO_PROG "${SCRATCH_MNT}" | grep naming.*bsize | sed -e 's/^.*bsize=//g' -e 's/\([0-9]*\).*$/\1/g')"
159         crc="$($XFS_INFO_PROG "${SCRATCH_MNT}" | grep crc= | sed -e 's/^.*crc=//g' -e 's/\([0-9]*\).*$/\1/g')"
160         if [ $crc -eq 1 ]; then
161                 leaf_hdr_size=64
162         else
163                 leaf_hdr_size=16
164         fi
165         leaf_lblk="$((32 * 1073741824 / blksz))"
166         node_lblk="$((64 * 1073741824 / blksz))"
167
168         # Data:
169
170         # Fill up the root inode chunk
171         echo "+ fill root ino chunk"
172         seq 1 64 | while read f; do
173                 $XFS_IO_PROG -f -c "truncate 0" "${SCRATCH_MNT}/dummy${f}"
174         done
175
176         # Regular files
177         # - FMT_EXTENTS
178         echo "+ extents file"
179         __populate_create_file $blksz "${SCRATCH_MNT}/S_IFREG.FMT_EXTENTS"
180
181         # - FMT_BTREE
182         echo "+ btree extents file"
183         nr="$((blksz * 2 / 16))"
184         __populate_create_file $((blksz * nr)) "${SCRATCH_MNT}/S_IFREG.FMT_BTREE"
185
186         # Directories
187         # - INLINE
188         echo "+ inline dir"
189         __populate_create_dir "${SCRATCH_MNT}/S_IFDIR.FMT_INLINE" 1
190
191         # - BLOCK
192         echo "+ block dir"
193         __populate_create_dir "${SCRATCH_MNT}/S_IFDIR.FMT_BLOCK" "$((dblksz / 40))"
194
195         # - LEAF
196         echo "+ leaf dir"
197         __populate_create_dir "${SCRATCH_MNT}/S_IFDIR.FMT_LEAF" "$((dblksz / 12))"
198
199         # - LEAFN
200         echo "+ leafn dir"
201         __populate_create_dir "${SCRATCH_MNT}/S_IFDIR.FMT_LEAFN" "$(( ((dblksz - leaf_hdr_size) / 8) - 3 ))"
202
203         # - NODE
204         echo "+ node dir"
205         __populate_create_dir "${SCRATCH_MNT}/S_IFDIR.FMT_NODE" "$((16 * dblksz / 40))" true
206
207         # - BTREE
208         echo "+ btree dir"
209         __populate_create_dir "${SCRATCH_MNT}/S_IFDIR.FMT_BTREE" "$((128 * dblksz / 40))" true
210
211         # Symlinks
212         # - FMT_LOCAL
213         echo "+ inline symlink"
214         ln -s target "${SCRATCH_MNT}/S_IFLNK.FMT_LOCAL"
215
216         # - FMT_EXTENTS
217         echo "+ extents symlink"
218         ln -s "$(perl -e 'print "x" x 1023;')" "${SCRATCH_MNT}/S_IFLNK.FMT_EXTENTS"
219
220         # Char & block
221         echo "+ special"
222         mknod "${SCRATCH_MNT}/S_IFCHR" c 1 1
223         mknod "${SCRATCH_MNT}/S_IFBLK" c 1 1
224
225         # special file with an xattr
226         setfacl -P -m u:nobody:r ${SCRATCH_MNT}/S_IFCHR
227
228         # Attribute formats
229         # LOCAL
230         echo "+ local attr"
231         __populate_create_attr "${SCRATCH_MNT}/ATTR.FMT_LOCAL" 1
232
233         # LEAF
234         echo "+ leaf attr"
235         __populate_create_attr "${SCRATCH_MNT}/ATTR.FMT_LEAF" "$((blksz / 40))"
236
237         # NODE
238         echo "+ node attr"
239         __populate_create_attr "${SCRATCH_MNT}/ATTR.FMT_NODE" "$((8 * blksz / 40))"
240
241         # BTREE
242         echo "+ btree attr"
243         __populate_create_attr "${SCRATCH_MNT}/ATTR.FMT_BTREE" "$((64 * blksz / 40))" true
244
245         # trusted namespace
246         touch ${SCRATCH_MNT}/ATTR.TRUSTED
247         setfattr -n trusted.moo -v urk ${SCRATCH_MNT}/ATTR.TRUSTED
248
249         # security namespace
250         touch ${SCRATCH_MNT}/ATTR.SECURITY
251         setfattr -n security.foo -v bar ${SCRATCH_MNT}/ATTR.SECURITY
252
253         # system namespace
254         touch ${SCRATCH_MNT}/ATTR.SYSTEM
255         setfacl -m u:root:r ${SCRATCH_MNT}/ATTR.SYSTEM
256
257         # FMT_EXTENTS with a remote less-than-a-block value
258         echo "+ attr extents with a remote less-than-a-block value"
259         touch "${SCRATCH_MNT}/ATTR.FMT_EXTENTS_REMOTE3K"
260         $XFS_IO_PROG -f -c "pwrite -S 0x43 0 $((blksz - 300))" "${SCRATCH_MNT}/attrvalfile" > /dev/null
261         attr -q -s user.remotebtreeattrname "${SCRATCH_MNT}/ATTR.FMT_EXTENTS_REMOTE3K" < "${SCRATCH_MNT}/attrvalfile"
262
263         # FMT_EXTENTS with a remote block-size value
264         echo "+ attr extents with a remote one-block value"
265         touch "${SCRATCH_MNT}/ATTR.FMT_EXTENTS_REMOTE4K"
266         $XFS_IO_PROG -f -c "pwrite -S 0x44 0 ${blksz}" "${SCRATCH_MNT}/attrvalfile" > /dev/null
267         attr -q -s user.remotebtreeattrname "${SCRATCH_MNT}/ATTR.FMT_EXTENTS_REMOTE4K" < "${SCRATCH_MNT}/attrvalfile"
268         rm -rf "${SCRATCH_MNT}/attrvalfile"
269
270         # Make an unused inode
271         echo "+ empty file"
272         touch "${SCRATCH_MNT}/unused"
273         $XFS_IO_PROG -f -c 'fsync' "${SCRATCH_MNT}/unused"
274         rm -rf "${SCRATCH_MNT}/unused"
275
276         # Free space btree
277         echo "+ freesp btree"
278         nr="$((blksz * 2 / 8))"
279         __populate_create_file $((blksz * nr)) "${SCRATCH_MNT}/BNOBT"
280
281         # Inode btree
282         echo "+ inobt btree"
283         local ino_per_rec=64
284         local rec_per_btblock=16
285         local nr="$(( 2 * (blksz / rec_per_btblock) * ino_per_rec ))"
286         local dir="${SCRATCH_MNT}/INOBT"
287         mkdir -p "${dir}"
288         seq 0 "${nr}" | while read f; do
289                 touch "${dir}/${f}"
290         done
291
292         seq 0 2 "${nr}" | while read f; do
293                 rm -f "${dir}/${f}"
294         done
295
296         # Reverse-mapping btree
297         is_rmapbt="$($XFS_INFO_PROG "${SCRATCH_MNT}" | grep -c 'rmapbt=1')"
298         if [ $is_rmapbt -gt 0 ]; then
299                 echo "+ rmapbt btree"
300                 nr="$((blksz * 2 / 24))"
301                 __populate_create_file $((blksz * nr)) "${SCRATCH_MNT}/RMAPBT"
302         fi
303
304         # Realtime Reverse-mapping btree
305         is_rt="$($XFS_INFO_PROG "${SCRATCH_MNT}" | grep -c 'rtextents=[1-9]')"
306         if [ $is_rmapbt -gt 0 ] && [ $is_rt -gt 0 ]; then
307                 echo "+ rtrmapbt btree"
308                 nr="$((blksz * 2 / 32))"
309                 __populate_create_file $((blksz * nr)) "${SCRATCH_MNT}/RTRMAPBT"
310         fi
311
312         # Reference-count btree
313         is_reflink="$($XFS_INFO_PROG "${SCRATCH_MNT}" | grep -c 'reflink=1')"
314         if [ $is_reflink -gt 0 ]; then
315                 echo "+ reflink btree"
316                 nr="$((blksz * 2 / 12))"
317                 __populate_create_file $((blksz * nr)) "${SCRATCH_MNT}/REFCOUNTBT"
318                 cp --reflink=always "${SCRATCH_MNT}/REFCOUNTBT" "${SCRATCH_MNT}/REFCOUNTBT2"
319         fi
320
321         # Copy some real files (xfs tests, I guess...)
322         echo "+ real files"
323         test $fill -ne 0 && __populate_fill_fs "${SCRATCH_MNT}" 5
324
325         # Make sure we get all the fragmentation we asked for
326         __populate_fragment_file "${SCRATCH_MNT}/S_IFREG.FMT_BTREE"
327         __populate_fragment_file "${SCRATCH_MNT}/BNOBT"
328         __populate_fragment_file "${SCRATCH_MNT}/RMAPBT"
329         __populate_fragment_file "${SCRATCH_MNT}/RTRMAPBT"
330         __populate_fragment_file "${SCRATCH_MNT}/REFCOUNTBT"
331
332         umount "${SCRATCH_MNT}"
333 }
334
335 # Populate an ext4 on the scratch device with (we hope) all known
336 # types of metadata block
337 _scratch_ext4_populate() {
338         fill=1
339
340         for arg in $@; do
341                 case "${arg}" in
342                 "nofill")
343                         fill=0;;
344                 esac
345         done
346
347         _scratch_mount
348         blksz="$(stat -f -c '%s' "${SCRATCH_MNT}")"
349         dblksz="${blksz}"
350         leaf_lblk="$((32 * 1073741824 / blksz))"
351         node_lblk="$((64 * 1073741824 / blksz))"
352
353         # Data:
354
355         # Regular files
356         # - FMT_INLINE
357         echo "+ inline file"
358         __populate_create_file 1 "${SCRATCH_MNT}/S_IFREG.FMT_INLINE"
359
360         # - FMT_EXTENTS
361         echo "+ extents file"
362         __populate_create_file $blksz "${SCRATCH_MNT}/S_IFREG.FMT_EXTENTS"
363
364         # - FMT_ETREE
365         echo "+ extent tree file"
366         nr="$((blksz * 2 / 12))"
367         __populate_create_file $((blksz * nr)) "${SCRATCH_MNT}/S_IFREG.FMT_ETREE"
368
369         # Directories
370         # - INLINE
371         echo "+ inline dir"
372         __populate_create_dir "${SCRATCH_MNT}/S_IFDIR.FMT_INLINE" 1
373
374         # - BLOCK
375         echo "+ block dir"
376         __populate_create_dir "${SCRATCH_MNT}/S_IFDIR.FMT_BLOCK" "$((dblksz / 32))"
377
378         # - HTREE
379         echo "+ htree dir"
380         __populate_create_dir "${SCRATCH_MNT}/S_IFDIR.FMT_HTREE" "$((4 * dblksz / 24))"
381
382         # Symlinks
383         # - FMT_LOCAL
384         echo "+ inline symlink"
385         ln -s target "${SCRATCH_MNT}/S_IFLNK.FMT_LOCAL"
386
387         # - FMT_EXTENTS
388         echo "+ extents symlink"
389         ln -s "$(perl -e 'print "x" x 1023;')" "${SCRATCH_MNT}/S_IFLNK.FMT_EXTENTS"
390
391         # Char & block
392         echo "+ special"
393         mknod "${SCRATCH_MNT}/S_IFCHR" c 1 1
394         mknod "${SCRATCH_MNT}/S_IFBLK" c 1 1
395
396         # special file with an xattr
397         setfacl -P -m u:nobody:r ${SCRATCH_MNT}/S_IFCHR
398
399         # Attribute formats
400         # LOCAL
401         echo "+ local attr"
402         __populate_create_attr "${SCRATCH_MNT}/ATTR.FMT_LOCAL" 0
403
404         # BLOCK
405         echo "+ block attr"
406         __populate_create_attr "${SCRATCH_MNT}/ATTR.FMT_BLOCK" "$((blksz / 40))"
407
408         # trusted namespace
409         touch ${SCRATCH_MNT}/ATTR.TRUSTED
410         setfattr -n trusted.moo -v urk ${SCRATCH_MNT}/ATTR.TRUSTED
411
412         # security namespace
413         touch ${SCRATCH_MNT}/ATTR.SECURITY
414         setfattr -n security.foo -v bar ${SCRATCH_MNT}/ATTR.SECURITY
415
416         # system namespace
417         touch ${SCRATCH_MNT}/ATTR.SYSTEM
418         setfacl -m u:root:r ${SCRATCH_MNT}/ATTR.SYSTEM
419
420         # Make an unused inode
421         echo "+ empty file"
422         touch "${SCRATCH_MNT}/unused"
423         $XFS_IO_PROG -f -c 'fsync' "${SCRATCH_MNT}/unused"
424         rm -rf "${SCRATCH_MNT}/unused"
425
426         # Copy some real files (xfs tests, I guess...)
427         echo "+ real files"
428         test $fill -ne 0 && __populate_fill_fs "${SCRATCH_MNT}" 5
429
430         # Make sure we get all the fragmentation we asked for
431         __populate_fragment_file "${SCRATCH_MNT}/S_IFREG.FMT_ETREE"
432
433         umount "${SCRATCH_MNT}"
434 }
435
436 # Find the inode number of a file
437 __populate_find_inode() {
438         name="$1"
439         inode="$(stat -c '%i' "${name}")"
440         echo "${inode}"
441 }
442
443 # Check data fork format of XFS file
444 __populate_check_xfs_dformat() {
445         inode="$1"
446         format="$2"
447
448         fmt="$(_scratch_xfs_db -c "inode ${inode}" -c 'p core.format' | sed -e 's/^.*(\([a-z]*\)).*$/\1/g')"
449         test "${format}" = "${fmt}" || _fail "failed to create ino ${inode} dformat expected ${format} saw ${fmt}"
450 }
451
452 # Check attr fork format of XFS file
453 __populate_check_xfs_aformat() {
454         inode="$1"
455         format="$2"
456
457         fmt="$(_scratch_xfs_db -c "inode ${inode}" -c 'p core.aformat' | sed -e 's/^.*(\([a-z]*\)).*$/\1/g')"
458         test "${format}" = "${fmt}" || _fail "failed to create ino ${inode} aformat expected ${format} saw ${fmt}"
459 }
460
461 # Check structure of XFS directory
462 __populate_check_xfs_dir() {
463         inode="$1"
464         dtype="$2"
465
466         (test -n "${leaf_lblk}" && test -n "${node_lblk}") || _fail "must define leaf_lblk and node_lblk before calling __populate_check_xfs_dir"
467         datab=0
468         leafb=0
469         freeb=0
470         #echo "== check dir ${inode} type ${dtype}" ; _scratch_xfs_db -x -c "inode ${inode}" -c "bmap"
471         _scratch_xfs_db -x -c "inode ${inode}" -c "dblock 0" -c "stack" | grep -q 'file data block is unmapped' || datab=1
472         _scratch_xfs_db -x -c "inode ${inode}" -c "dblock ${leaf_lblk}" -c "stack" | grep -q 'file data block is unmapped' || leafb=1
473         _scratch_xfs_db -x -c "inode ${inode}" -c "dblock ${node_lblk}" -c "stack" | grep -q 'file data block is unmapped' || freeb=1
474
475         case "${dtype}" in
476         "shortform"|"inline"|"local")
477                 (test "${datab}" -eq 0 && test "${leafb}" -eq 0 && test "${freeb}" -eq 0) || _fail "failed to create ${dtype} dir ino ${inode} datab ${datab} leafb ${leafb} freeb ${freeb}"
478                 ;;
479         "block")
480                 (test "${datab}" -eq 1 && test "${leafb}" -eq 0 && test "${freeb}" -eq 0) || _fail "failed to create ${dtype} dir ino ${inode} datab ${datab} leafb ${leafb} freeb ${freeb}"
481                 ;;
482         "leaf")
483                 (test "${datab}" -eq 1 && test "${leafb}" -eq 1 && test "${freeb}" -eq 0) || _fail "failed to create ${dtype} dir ino ${inode} datab ${datab} leafb ${leafb} freeb ${freeb}"
484                 ;;
485         "leafn")
486                 _scratch_xfs_db -x -c "inode ${inode}" -c "dblock ${leaf_lblk}" -c "p lhdr.info.hdr.magic" | grep -q '0x3dff' && return
487                 _scratch_xfs_db -x -c "inode ${inode}" -c "dblock ${leaf_lblk}" -c "p lhdr.info.magic" | grep -q '0xd2ff' && return
488                 _fail "failed to create ${dtype} dir ino ${inode} datab ${datab} leafb ${leafb} freeb ${freeb}"
489                 ;;
490         "node"|"btree")
491                 (test "${datab}" -eq 1 && test "${leafb}" -eq 1 && test "${freeb}" -eq 1) || _fail "failed to create ${dtype} dir ino ${inode} datab ${datab} leafb ${leafb} freeb ${freeb}"
492                 ;;
493         *)
494                 _fail "Unknown directory type ${dtype}"
495         esac
496 }
497
498 # Check structure of XFS attr
499 __populate_check_xfs_attr() {
500         inode="$1"
501         atype="$2"
502
503         datab=0
504         leafb=0
505         #echo "== check attr ${inode} type ${dtype}" ; _scratch_xfs_db -x -c "inode ${inode}" -c "bmap -a"
506         _scratch_xfs_db -x -c "inode ${inode}" -c "ablock 0" -c "stack" | grep -q 'file attr block is unmapped' || datab=1
507         _scratch_xfs_db -x -c "inode ${inode}" -c "ablock 1" -c "stack" | grep -q 'file attr block is unmapped' || leafb=1
508
509         case "${atype}" in
510         "shortform"|"inline"|"local")
511                 (test "${datab}" -eq 0 && test "${leafb}" -eq 0) || _fail "failed to create ${atype} attr ino ${inode} datab ${datab} leafb ${leafb}"
512                 ;;
513         "leaf")
514                 (test "${datab}" -eq 1 && test "${leafb}" -eq 0) || _fail "failed to create ${atype} attr ino ${inode} datab ${datab} leafb ${leafb}"
515                 ;;
516         "node"|"btree")
517                 (test "${datab}" -eq 1 && test "${leafb}" -eq 1) || _fail "failed to create ${atype} attr ino ${inode} datab ${datab} leafb ${leafb}"
518                 ;;
519         *)
520                 _fail "Unknown attribute type ${atype}"
521         esac
522 }
523
524 # Check that there's at least one per-AG btree with multiple levels
525 __populate_check_xfs_agbtree_height() {
526         bt_type="$1"
527         nr_ags=$(_scratch_xfs_db -c 'sb 0' -c 'p agcount' | awk '{print $3}')
528
529         case "${bt_type}" in
530         "bno"|"cnt"|"rmap"|"refcnt")
531                 hdr="agf"
532                 bt_prefix="${bt_type}"
533                 ;;
534         "ino")
535                 hdr="agi"
536                 bt_prefix=""
537                 ;;
538         "fino")
539                 hdr="agi"
540                 bt_prefix="free_"
541                 ;;
542         *)
543                 _fail "Don't know about AG btree ${bt_type}"
544                 ;;
545         esac
546
547         seq 0 $((nr_ags - 1)) | while read ag; do
548                 bt_level=$(_scratch_xfs_db -c "${hdr} ${ag}" -c "p ${bt_prefix}level" | awk '{print $3}')
549                 if [ "${bt_level}" -gt 1 ]; then
550                         return 100
551                 fi
552         done
553         test $? -eq 100 || _fail "Failed to create ${bt_type} of sufficient height!"
554         return 1
555 }
556
557 # Check that populate created all the types of files we wanted
558 _scratch_xfs_populate_check() {
559         _scratch_mount
560         extents_file="$(__populate_find_inode "${SCRATCH_MNT}/S_IFREG.FMT_EXTENTS")"
561         btree_file="$(__populate_find_inode "${SCRATCH_MNT}/S_IFREG.FMT_BTREE")"
562         inline_dir="$(__populate_find_inode "${SCRATCH_MNT}/S_IFDIR.FMT_INLINE")"
563         block_dir="$(__populate_find_inode "${SCRATCH_MNT}/S_IFDIR.FMT_BLOCK")"
564         leaf_dir="$(__populate_find_inode "${SCRATCH_MNT}/S_IFDIR.FMT_LEAF")"
565         leafn_dir="$(__populate_find_inode "${SCRATCH_MNT}/S_IFDIR.FMT_LEAFN")"
566         node_dir="$(__populate_find_inode "${SCRATCH_MNT}/S_IFDIR.FMT_NODE")"
567         btree_dir="$(__populate_find_inode "${SCRATCH_MNT}/S_IFDIR.FMT_BTREE")"
568         local_slink="$(__populate_find_inode "${SCRATCH_MNT}/S_IFLNK.FMT_LOCAL")"
569         extents_slink="$(__populate_find_inode "${SCRATCH_MNT}/S_IFLNK.FMT_EXTENTS")"
570         bdev="$(__populate_find_inode "${SCRATCH_MNT}/S_IFBLK")"
571         cdev="$(__populate_find_inode "${SCRATCH_MNT}/S_IFCHR")"
572         local_attr="$(__populate_find_inode "${SCRATCH_MNT}/ATTR.FMT_LOCAL")"
573         leaf_attr="$(__populate_find_inode "${SCRATCH_MNT}/ATTR.FMT_LEAF")"
574         node_attr="$(__populate_find_inode "${SCRATCH_MNT}/ATTR.FMT_NODE")"
575         btree_attr="$(__populate_find_inode "${SCRATCH_MNT}/ATTR.FMT_BTREE")"
576         is_finobt=$($XFS_INFO_PROG "${SCRATCH_MNT}" | grep -c 'finobt=1')
577         is_rmapbt=$($XFS_INFO_PROG "${SCRATCH_MNT}" | grep -c 'rmapbt=1')
578         is_reflink=$($XFS_INFO_PROG "${SCRATCH_MNT}" | grep -c 'reflink=1')
579
580         blksz="$(stat -f -c '%s' "${SCRATCH_MNT}")"
581         dblksz="$($XFS_INFO_PROG "${SCRATCH_MNT}" | grep naming.*bsize | sed -e 's/^.*bsize=//g' -e 's/\([0-9]*\).*$/\1/g')"
582         leaf_lblk="$((32 * 1073741824 / blksz))"
583         node_lblk="$((64 * 1073741824 / blksz))"
584         umount "${SCRATCH_MNT}"
585
586         __populate_check_xfs_dformat "${extents_file}" "extents"
587         __populate_check_xfs_dformat "${btree_file}" "btree"
588         __populate_check_xfs_dir "${inline_dir}" "inline"
589         __populate_check_xfs_dir "${block_dir}" "block"
590         __populate_check_xfs_dir "${leaf_dir}" "leaf"
591         __populate_check_xfs_dir "${leafn_dir}" "leafn"
592         __populate_check_xfs_dir "${node_dir}" "node"
593         __populate_check_xfs_dir "${btree_dir}" "btree"
594         __populate_check_xfs_dformat "${btree_dir}" "btree"
595         __populate_check_xfs_dformat "${bdev}" "dev"
596         __populate_check_xfs_dformat "${cdev}" "dev"
597         __populate_check_xfs_attr "${local_attr}" "local"
598         __populate_check_xfs_attr "${leaf_attr}" "leaf"
599         __populate_check_xfs_attr "${node_attr}" "node"
600         __populate_check_xfs_attr "${btree_attr}" "btree"
601         __populate_check_xfs_aformat "${btree_attr}" "btree"
602         __populate_check_xfs_agbtree_height "bno"
603         __populate_check_xfs_agbtree_height "cnt"
604         __populate_check_xfs_agbtree_height "ino"
605         test $is_finobt -ne 0 && __populate_check_xfs_agbtree_height "fino"
606         test $is_rmapbt -ne 0 && __populate_check_xfs_agbtree_height "rmap"
607         test $is_reflink -ne 0 && __populate_check_xfs_agbtree_height "refcnt"
608 }
609
610 # Check data fork format of ext4 file
611 __populate_check_ext4_dformat() {
612         dev="${SCRATCH_DEV}"
613         inode="$1"
614         format="$2"
615
616         extents=0
617         etree=0
618         debugfs -R "stat <${inode}>" "${dev}" 2> /dev/null | grep 'ETB[0-9]' -q && etree=1
619         iflags="$(debugfs -R "stat <${inode}>" "${dev}" 2> /dev/null | grep 'Flags:' | sed -e 's/^.*Flags: \([0-9a-fx]*\).*$/\1/g')"
620         test "$(echo "${iflags}" | awk '{print and(strtonum($1), 0x80000);}')" -gt 0 && extents=1
621
622         case "${format}" in
623         "blockmap")
624                 test "${extents}" -eq 0 || _fail "failed to create ino ${inode} with blockmap"
625                 ;;
626         "extent"|"extents")
627                 test "${extents}" -eq 1 || _fail "failed to create ino ${inode} with extents"
628                 ;;
629         "etree")
630                 (test "${extents}" -eq 1 && test "${etree}" -eq 1) || _fail "failed to create ino ${inode} with extent tree"
631                 ;;
632         *)
633                 _fail "Unknown dformat ${format}"
634         esac
635 }
636
637 # Check attr fork format of ext4 file
638 __populate_check_ext4_aformat() {
639         dev="${SCRATCH_DEV}"
640         inode="$1"
641         format="$2"
642
643         ablock=1
644         debugfs -R "stat <${inode}>" "${dev}" 2> /dev/null | grep 'File ACL: 0' -q && ablock=0
645
646         case "${format}" in
647         "local"|"inline")
648                 test "${ablock}" -eq 0 || _fail "failed to create inode ${inode} with ${format} xattr"
649                 ;;
650         "block")
651                 test "${extents}" -eq 1 || _fail "failed to create inode ${inode} with ${format} xattr"
652                 ;;
653         *)
654                 _fail "Unknown aformat ${format}"
655         esac
656 }
657
658 # Check structure of ext4 dir
659 __populate_check_ext4_dir() {
660         dev="${SCRATCH_DEV}"
661         inode="$1"
662         dtype="$2"
663
664         htree=0
665         inline=0
666         iflags="$(debugfs -R "stat <${inode}>" "${dev}" 2> /dev/null | grep 'Flags:' | sed -e 's/^.*Flags: \([0-9a-fx]*\).*$/\1/g')"
667         test "$(echo "${iflags}" | awk '{print and(strtonum($1), 0x1000);}')" -gt 0 && htree=1
668         test "$(echo "${iflags}" | awk '{print and(strtonum($1), 0x10000000);}')" -gt 0 && inline=1
669
670         case "${dtype}" in
671         "inline")
672                 (test "${inline}" -eq 1 && test "${htree}" -eq 0) || _fail "failed to create ${dtype} dir ino ${inode} htree ${htree} inline ${inline}"
673                 ;;
674         "block")
675                 (test "${inline}" -eq 0 && test "${htree}" -eq 0) || _fail "failed to create ${dtype} dir ino ${inode} htree ${htree} inline ${inline}"
676                 ;;
677         "htree")
678                 (test "${inline}" -eq 0 && test "${htree}" -eq 1) || _fail "failed to create ${dtype} dir ino ${inode} htree ${htree} inline ${inline}"
679                 ;;
680         *)
681                 _fail "Unknown directory type ${dtype}"
682                 ;;
683         esac
684 }
685
686 # Check that populate created all the types of files we wanted
687 _scratch_ext4_populate_check() {
688         _scratch_mount
689         extents_file="$(__populate_find_inode "${SCRATCH_MNT}/S_IFREG.FMT_EXTENTS")"
690         etree_file="$(__populate_find_inode "${SCRATCH_MNT}/S_IFREG.FMT_ETREE")"
691         block_dir="$(__populate_find_inode "${SCRATCH_MNT}/S_IFDIR.FMT_BLOCK")"
692         htree_dir="$(__populate_find_inode "${SCRATCH_MNT}/S_IFDIR.FMT_HTREE")"
693         extents_slink="$(__populate_find_inode "${SCRATCH_MNT}/S_IFLNK.FMT_EXTENTS")"
694         local_attr="$(__populate_find_inode "${SCRATCH_MNT}/ATTR.FMT_LOCAL")"
695         block_attr="$(__populate_find_inode "${SCRATCH_MNT}/ATTR.FMT_BLOCK")"
696         umount "${SCRATCH_MNT}"
697
698         __populate_check_ext4_dformat "${extents_file}" "extents"
699         __populate_check_ext4_dformat "${etree_file}" "etree"
700         __populate_check_ext4_dir "${block_dir}" "block"
701         __populate_check_ext4_dir "${htree_dir}" "htree"
702         __populate_check_ext4_dformat "${extents_slink}" "extents"
703         __populate_check_ext4_aformat "${local_attr}" "local"
704         __populate_check_ext4_aformat "${block_attr}" "block"
705 }
706
707 # Populate a scratch FS and check the contents to make sure we got that
708 _scratch_populate() {
709         case "${FSTYP}" in
710         "xfs")
711                 _scratch_xfs_populate
712                 _scratch_xfs_populate_check
713                 ;;
714         "ext2"|"ext3"|"ext4")
715                 _scratch_ext4_populate
716                 _scratch_ext4_populate_check
717                 ;;
718         *)
719                 _fail "Don't know how to populate a ${FSTYP} filesystem."
720                 ;;
721         esac
722 }
723
724 # Fill a file system by repeatedly creating files in the given folder
725 # starting with the given file size.  Files are reduced in size when
726 # they can no longer fit until no more files can be created.
727 _fill_fs()
728 {
729         local file_size=$1
730         local dir=$2
731         local block_size=$3
732         local switch_user=$4
733         local file_count=1
734         local bytes_written=0
735         local use_falloc=1;
736
737         if [ $# -ne 4 ]; then
738                 echo "Usage: _fill_fs filesize dir blocksize switch_user"
739                 exit 1
740         fi
741
742         if [ $switch_user -eq 0 ]; then
743                 mkdir -p $dir
744         else
745                 _user_do "mkdir -p $dir"
746         fi
747         if [ ! -d $dir ]; then
748                 return 0;
749         fi
750
751         testio=`$XFS_IO_PROG -F -fc "falloc 0 $block_size" $dir/$$.xfs_io 2>&1`
752         echo $testio | grep -q "not found" && use_falloc=0
753         echo $testio | grep -q "Operation not supported" && use_falloc=0
754
755         if [ $file_size -lt $block_size ]; then
756                 $file_size = $block_size
757         fi
758
759         while [ $file_size -ge $block_size ]; do
760                 bytes_written=0
761                 if [ $switch_user -eq 0 ]; then
762                         if [ $use_falloc -eq 0 ]; then
763                                 $XFS_IO_PROG -fc "pwrite -b 8388608 0 $file_size" \
764                                         $dir/$file_count
765                         else
766                                 $XFS_IO_PROG -fc "falloc 0 $file_size" \
767                                         $dir/$file_count
768                         fi
769                 else
770                         if [ $use_falloc -eq 0 ]; then
771                                 _user_do "$XFS_IO_PROG -f -c \"pwrite -b 8388608 0 \
772                                         $file_size\" $dir/$file_count"
773                         else
774                                 _user_do "$XFS_IO_PROG -f -c \"falloc 0 \
775                                         $file_size\" $dir/$file_count"
776                         fi
777                 fi
778
779                 if [ -f $dir/$file_count ]; then
780                         bytes_written=$(_get_filesize $dir/$file_count)
781                 fi
782
783                 # If there was no room to make the file, then divide it in
784                 # half, and keep going
785                 if [ $bytes_written -lt $file_size ]; then
786                         file_size=$((file_size / 2))
787                 fi
788                 file_count=$((file_count + 1))
789         done
790 }
791
792 # Compute the fs geometry description of a populated filesystem
793 _scratch_populate_cache_tag() {
794         local extra_descr=""
795         local size="$(blockdev --getsz "${SCRATCH_DEV}")"
796
797         case "${FSTYP}" in
798         "ext4")
799                 extra_descr="LOGDEV ${SCRATCH_LOGDEV} USE_EXTERNAL ${USE_EXTERNAL}"
800                 ;;
801         "xfs")
802                 extra_descr="LOGDEV ${SCRATCH_LOGDEV} USE_EXTERNAL ${USE_EXTERNAL} RTDEV ${SCRATCH_RTDEV}"
803                 _populate_xfs_qmount_option
804                 if echo "${MOUNT_OPTIONS}" | grep -q 'usrquota'; then
805                         extra_descr="${extra_descr} QUOTAS"
806                 fi
807                 ;;
808         esac
809         echo "FSTYP ${FSTYP} MKFS_OPTIONS ${MKFS_OPTIONS} SIZE ${size} ${extra_descr} ARGS $@"
810 }
811
812 # Restore a cached populated fs from a metadata dump
813 _scratch_populate_restore_cached() {
814         local metadump="$1"
815
816         case "${FSTYP}" in
817         "xfs")
818                 xfs_mdrestore "${metadump}" "${SCRATCH_DEV}" && return 0
819                 ;;
820         "ext2"|"ext3"|"ext4")
821                 # ext4 cannot e2image external logs, so we cannot restore
822                 test -n "${SCRATCH_LOGDEV}" && return 1
823                 e2image -r "${metadump}" "${SCRATCH_DEV}" && return 0
824                 ;;
825         esac
826         return 1
827 }
828
829 # Populate a scratch FS from scratch or from a cached image.
830 _scratch_populate_cached() {
831         local meta_descr="$(_scratch_populate_cache_tag "$@")"
832         local meta_tag="$(echo "${meta_descr}" | md5sum - | cut -d ' ' -f 1)"
833         local metadump_stem="${TEST_DIR}/__populate.${FSTYP}.${meta_tag}"
834
835         # These variables are shared outside this function
836         POPULATE_METADUMP="${metadump_stem}.metadump"
837         POPULATE_METADUMP_DESCR="${metadump_stem}.txt"
838
839         # Don't keep metadata images cached for more 48 hours...
840         rm -rf "$(find "${POPULATE_METADUMP}" -mtime +2 2>/dev/null)"
841
842         # Throw away cached image if it doesn't match our spec.
843         cmp -s "${POPULATE_METADUMP_DESCR}" <(echo "${meta_descr}") || \
844                 rm -rf "${POPULATE_METADUMP}"
845
846         # Try to restore from the metadump
847         test -r "${POPULATE_METADUMP}" && \
848                 _scratch_populate_restore_cached "${POPULATE_METADUMP}" && \
849                 return
850
851         # Oh well, just create one from scratch
852         _scratch_mkfs
853         echo "${meta_descr}" > "${POPULATE_METADUMP_DESCR}"
854         case "${FSTYP}" in
855         "xfs")
856                 _scratch_xfs_populate $@
857                 _scratch_xfs_populate_check
858                 _scratch_metadump "${POPULATE_METADUMP}" -a -o
859                 ;;
860         "ext2"|"ext3"|"ext4")
861                 _scratch_ext4_populate $@
862                 _scratch_ext4_populate_check
863                 e2image -Q "${SCRATCH_DEV}" "${POPULATE_METADUMP}"
864                 ;;
865         *)
866                 _fail "Don't know how to populate a ${FSTYP} filesystem."
867                 ;;
868         esac
869 }