610425f9a186ad47a8ed5e5a1fcf56b72e552088
[xfstests-dev.git] / common / populate
1 ##/bin/bash
2 # SPDX-License-Identifier: GPL-2.0+
3 # Copyright (c) 2015 Oracle.  All Rights Reserved.
4 #
5 # Routines for populating a scratch fs, and helpers to exercise an FS
6 # once it's been fuzzed.
7
8 . ./common/quota
9
10 _require_populate_commands() {
11         _require_xfs_io_command "falloc"
12         _require_xfs_io_command "fpunch"
13         _require_test_program "punch-alternating"
14         _require_command "$XFS_DB_PROG" "xfs_db"
15 }
16
17 _require_xfs_db_blocktrash_z_command() {
18         test "${FSTYP}" = "xfs" || _notrun "cannot run xfs_db on ${FSTYP}"
19         $XFS_DB_PROG -x -f -c 'blocktrash -z' "${TEST_DEV}" | grep -q 'nothing on stack' || _notrun "blocktrash -z not supported"
20 }
21
22 # Attempt to make files of "every" format for data, dirs, attrs etc.
23 # (with apologies to Eric Sandeen for mutating xfser.sh)
24
25 # Create a large directory
26 __populate_create_dir() {
27         name="$1"
28         nr="$2"
29         missing="$3"
30
31         mkdir -p "${name}"
32         seq 0 "${nr}" | while read d; do
33                 creat=mkdir
34                 test "$((d % 20))" -eq 0 && creat=touch
35                 $creat "${name}/$(printf "%.08d" "$d")"
36         done
37
38         test -z "${missing}" && return
39         seq 1 2 "${nr}" | while read d; do
40                 rm -rf "${name}/$(printf "%.08d" "$d")"
41         done
42 }
43
44 # Add a bunch of attrs to a file
45 __populate_create_attr() {
46         name="$1"
47         nr="$2"
48         missing="$3"
49
50         touch "${name}"
51         seq 0 "${nr}" | while read d; do
52                 setfattr -n "user.$(printf "%.08d" "$d")" -v "$(printf "%.08d" "$d")" "${name}"
53         done
54
55         test -z "${missing}" && return
56         seq 1 2 "${nr}" | while read d; do
57                 setfattr -x "user.$(printf "%.08d" "$d")" "${name}"
58         done
59 }
60
61 # Fill up some percentage of the remaining free space
62 __populate_fill_fs() {
63         dir="$1"
64         pct="$2"
65         test -z "${pct}" && pct=60
66
67         mkdir -p "${dir}/test/1"
68         cp -pRdu "${dir}"/S_IFREG* "${dir}/test/1/"
69
70         SRC_SZ="$(du -ks "${dir}/test/1" | cut -f 1)"
71         FS_SZ="$(( $(stat -f "${dir}" -c '%a * %S') / 1024 ))"
72
73         NR="$(( (FS_SZ * ${pct} / 100) / SRC_SZ ))"
74
75         echo "FILL FS"
76         echo "src_sz $SRC_SZ fs_sz $FS_SZ nr $NR"
77         seq 2 "${NR}" | while read nr; do
78                 cp -pRdu "${dir}/test/1" "${dir}/test/${nr}"
79         done
80 }
81
82 # For XFS, force on all the quota options if quota is enabled
83 # and the user didn't feed us noquota.
84 _populate_xfs_qmount_option()
85 {
86         # User explicitly told us not to quota
87         if echo "${MOUNT_OPTIONS}" | grep -q 'noquota'; then
88                 return
89         fi
90
91         # Don't bother if we can't turn on quotas
92         if [ ! -f /proc/fs/xfs/xqmstat ]; then
93                 # No quota support
94                 return
95         elif [ "${USE_EXTERNAL}" = "yes" ] && [ ! -z "${SCRATCH_RTDEV}" ]; then
96                 # Quotas not supported on rt filesystems
97                 return
98         elif [ -z "${XFS_QUOTA_PROG}" ]; then
99                 # xfs quota tools not installed
100                 return
101         fi
102
103         # Turn on all the quotas
104         if $XFS_INFO_PROG "${TEST_DIR}" | grep -q 'crc=1'; then
105                 # v5 filesystems can have group & project quotas
106                 quota="usrquota,grpquota,prjquota"
107         else
108                 # v4 filesystems cannot mix group & project quotas
109                 quota="usrquota,grpquota"
110         fi
111
112         # Inject our quota mount options
113         if echo "${MOUNT_OPTIONS}" | grep -q "${quota}"; then
114                 return
115         elif echo "${MOUNT_OPTIONS}" | egrep -q '(quota|noenforce)'; then
116                 _qmount_option "${quota}"
117         else
118                 export MOUNT_OPTIONS="$MOUNT_OPTIONS -o ${quota}"
119                 echo "MOUNT_OPTIONS = $MOUNT_OPTIONS" >>$seqres.full
120         fi
121 }
122
123 # Populate an XFS on the scratch device with (we hope) all known
124 # types of metadata block
125 _scratch_xfs_populate() {
126         fill=1
127
128         for arg in $@; do
129                 case "${arg}" in
130                 "nofill")
131                         fill=0;;
132                 esac
133         done
134
135         _populate_xfs_qmount_option
136         _scratch_mount
137         blksz="$(stat -f -c '%s' "${SCRATCH_MNT}")"
138         dblksz="$($XFS_INFO_PROG "${SCRATCH_MNT}" | grep naming.*bsize | sed -e 's/^.*bsize=//g' -e 's/\([0-9]*\).*$/\1/g')"
139         crc="$($XFS_INFO_PROG "${SCRATCH_MNT}" | grep crc= | sed -e 's/^.*crc=//g' -e 's/\([0-9]*\).*$/\1/g')"
140         if [ $crc -eq 1 ]; then
141                 leaf_hdr_size=64
142         else
143                 leaf_hdr_size=16
144         fi
145         leaf_lblk="$((32 * 1073741824 / blksz))"
146         node_lblk="$((64 * 1073741824 / blksz))"
147
148         # Data:
149
150         # Fill up the root inode chunk
151         echo "+ fill root ino chunk"
152         seq 1 64 | while read f; do
153                 $XFS_IO_PROG -f -c "truncate 0" "${SCRATCH_MNT}/dummy${f}"
154         done
155
156         # Regular files
157         # - FMT_EXTENTS
158         echo "+ extents file"
159         $XFS_IO_PROG -f -c "pwrite -S 0x61 0 ${blksz}" "${SCRATCH_MNT}/S_IFREG.FMT_EXTENTS"
160
161         # - FMT_BTREE
162         echo "+ btree extents file"
163         nr="$((blksz * 2 / 16))"
164         $XFS_IO_PROG -f -c "pwrite -S 0x62 0 $((blksz * nr))" "${SCRATCH_MNT}/S_IFREG.FMT_BTREE"
165         ./src/punch-alternating "${SCRATCH_MNT}/S_IFREG.FMT_BTREE"
166
167         # Directories
168         # - INLINE
169         echo "+ inline dir"
170         __populate_create_dir "${SCRATCH_MNT}/S_IFDIR.FMT_INLINE" 1
171
172         # - BLOCK
173         echo "+ block dir"
174         __populate_create_dir "${SCRATCH_MNT}/S_IFDIR.FMT_BLOCK" "$((dblksz / 40))"
175
176         # - LEAF
177         echo "+ leaf dir"
178         __populate_create_dir "${SCRATCH_MNT}/S_IFDIR.FMT_LEAF" "$((dblksz / 12))"
179
180         # - LEAFN
181         echo "+ leafn dir"
182         __populate_create_dir "${SCRATCH_MNT}/S_IFDIR.FMT_LEAFN" "$(( ((dblksz - leaf_hdr_size) / 8) - 3 ))"
183
184         # - NODE
185         echo "+ node dir"
186         __populate_create_dir "${SCRATCH_MNT}/S_IFDIR.FMT_NODE" "$((16 * dblksz / 40))" true
187
188         # - BTREE
189         echo "+ btree dir"
190         __populate_create_dir "${SCRATCH_MNT}/S_IFDIR.FMT_BTREE" "$((128 * dblksz / 40))" true
191
192         # Symlinks
193         # - FMT_LOCAL
194         echo "+ inline symlink"
195         ln -s target "${SCRATCH_MNT}/S_IFLNK.FMT_LOCAL"
196
197         # - FMT_EXTENTS
198         echo "+ extents symlink"
199         ln -s "$(perl -e 'print "x" x 1023;')" "${SCRATCH_MNT}/S_IFLNK.FMT_EXTENTS"
200
201         # Char & block
202         echo "+ special"
203         mknod "${SCRATCH_MNT}/S_IFCHR" c 1 1
204         mknod "${SCRATCH_MNT}/S_IFBLK" c 1 1
205
206         # special file with an xattr
207         setfacl -P -m u:nobody:r ${SCRATCH_MNT}/S_IFCHR
208
209         # Attribute formats
210         # LOCAL
211         echo "+ local attr"
212         __populate_create_attr "${SCRATCH_MNT}/ATTR.FMT_LOCAL" 1
213
214         # LEAF
215         echo "+ leaf attr"
216         __populate_create_attr "${SCRATCH_MNT}/ATTR.FMT_LEAF" "$((blksz / 40))"
217
218         # NODE
219         echo "+ node attr"
220         __populate_create_attr "${SCRATCH_MNT}/ATTR.FMT_NODE" "$((8 * blksz / 40))"
221
222         # BTREE
223         echo "+ btree attr"
224         __populate_create_attr "${SCRATCH_MNT}/ATTR.FMT_BTREE" "$((64 * blksz / 40))" true
225
226         # trusted namespace
227         touch ${SCRATCH_MNT}/ATTR.TRUSTED
228         setfattr -n trusted.moo -v urk ${SCRATCH_MNT}/ATTR.TRUSTED
229
230         # security namespace
231         touch ${SCRATCH_MNT}/ATTR.SECURITY
232         setfattr -n security.foo -v bar ${SCRATCH_MNT}/ATTR.SECURITY
233
234         # system namespace
235         touch ${SCRATCH_MNT}/ATTR.SYSTEM
236         setfacl -m u:root:r ${SCRATCH_MNT}/ATTR.SYSTEM
237
238         # FMT_EXTENTS with a remote less-than-a-block value
239         echo "+ attr extents with a remote less-than-a-block value"
240         touch "${SCRATCH_MNT}/ATTR.FMT_EXTENTS_REMOTE3K"
241         $XFS_IO_PROG -f -c "pwrite -S 0x43 0 $((blksz - 300))" "${SCRATCH_MNT}/attrvalfile" > /dev/null
242         attr -q -s user.remotebtreeattrname "${SCRATCH_MNT}/ATTR.FMT_EXTENTS_REMOTE3K" < "${SCRATCH_MNT}/attrvalfile"
243
244         # FMT_EXTENTS with a remote block-size value
245         echo "+ attr extents with a remote one-block value"
246         touch "${SCRATCH_MNT}/ATTR.FMT_EXTENTS_REMOTE4K"
247         $XFS_IO_PROG -f -c "pwrite -S 0x44 0 ${blksz}" "${SCRATCH_MNT}/attrvalfile" > /dev/null
248         attr -q -s user.remotebtreeattrname "${SCRATCH_MNT}/ATTR.FMT_EXTENTS_REMOTE4K" < "${SCRATCH_MNT}/attrvalfile"
249         rm -rf "${SCRATCH_MNT}/attrvalfile"
250
251         # Make an unused inode
252         echo "+ empty file"
253         touch "${SCRATCH_MNT}/unused"
254         $XFS_IO_PROG -f -c 'fsync' "${SCRATCH_MNT}/unused"
255         rm -rf "${SCRATCH_MNT}/unused"
256
257         # Free space btree
258         echo "+ freesp btree"
259         nr="$((blksz * 2 / 8))"
260         $XFS_IO_PROG -f -c "pwrite -S 0x62 0 $((blksz * nr))" "${SCRATCH_MNT}/BNOBT"
261         ./src/punch-alternating "${SCRATCH_MNT}/BNOBT"
262
263         # Reverse-mapping btree
264         is_rmapbt="$($XFS_INFO_PROG "${SCRATCH_MNT}" | grep -c 'rmapbt=1')"
265         if [ $is_rmapbt -gt 0 ]; then
266                 echo "+ rmapbt btree"
267                 nr="$((blksz * 2 / 24))"
268                 $XFS_IO_PROG -f -c "pwrite -S 0x62 0 $((blksz * nr))" "${SCRATCH_MNT}/RMAPBT"
269                 ./src/punch-alternating "${SCRATCH_MNT}/RMAPBT"
270         fi
271
272         # Realtime Reverse-mapping btree
273         is_rt="$($XFS_INFO_PROG "${SCRATCH_MNT}" | grep -c 'rtextents=[1-9]')"
274         if [ $is_rmapbt -gt 0 ] && [ $is_rt -gt 0 ]; then
275                 echo "+ rtrmapbt btree"
276                 nr="$((blksz * 2 / 32))"
277                 $XFS_IO_PROG -f -R -c "pwrite -S 0x62 0 $((blksz * nr))" "${SCRATCH_MNT}/RTRMAPBT"
278                 ./src/punch-alternating "${SCRATCH_MNT}/RTRMAPBT"
279         fi
280
281         # Reference-count btree
282         is_reflink="$($XFS_INFO_PROG "${SCRATCH_MNT}" | grep -c 'reflink=1')"
283         if [ $is_reflink -gt 0 ]; then
284                 echo "+ reflink btree"
285                 nr="$((blksz * 2 / 12))"
286                 $XFS_IO_PROG -f -c "pwrite -S 0x62 0 $((blksz * nr))" "${SCRATCH_MNT}/REFCOUNTBT"
287                 cp --reflink=always "${SCRATCH_MNT}/REFCOUNTBT" "${SCRATCH_MNT}/REFCOUNTBT2"
288                 ./src/punch-alternating "${SCRATCH_MNT}/REFCOUNTBT"
289         fi
290
291         # Copy some real files (xfs tests, I guess...)
292         echo "+ real files"
293         test $fill -ne 0 && __populate_fill_fs "${SCRATCH_MNT}" 5
294
295         umount "${SCRATCH_MNT}"
296 }
297
298 # Populate an ext4 on the scratch device with (we hope) all known
299 # types of metadata block
300 _scratch_ext4_populate() {
301         fill=1
302
303         for arg in $@; do
304                 case "${arg}" in
305                 "nofill")
306                         fill=0;;
307                 esac
308         done
309
310         _scratch_mount
311         blksz="$(stat -f -c '%s' "${SCRATCH_MNT}")"
312         dblksz="${blksz}"
313         leaf_lblk="$((32 * 1073741824 / blksz))"
314         node_lblk="$((64 * 1073741824 / blksz))"
315
316         # Data:
317
318         # Regular files
319         # - FMT_INLINE
320         echo "+ inline file"
321         $XFS_IO_PROG -f -c "pwrite -S 0x61 0 1" "${SCRATCH_MNT}/S_IFREG.FMT_INLINE"
322
323         # - FMT_EXTENTS
324         echo "+ extents file"
325         $XFS_IO_PROG -f -c "pwrite -S 0x61 0 ${blksz}" "${SCRATCH_MNT}/S_IFREG.FMT_EXTENTS"
326
327         # - FMT_ETREE
328         echo "+ extent tree file"
329         nr="$((blksz * 2 / 12))"
330         $XFS_IO_PROG -f -c "pwrite -S 0x62 0 $((blksz * nr))" "${SCRATCH_MNT}/S_IFREG.FMT_ETREE"
331         ./src/punch-alternating "${SCRATCH_MNT}/S_IFREG.FMT_ETREE"
332
333         # Directories
334         # - INLINE
335         echo "+ inline dir"
336         __populate_create_dir "${SCRATCH_MNT}/S_IFDIR.FMT_INLINE" 1
337
338         # - BLOCK
339         echo "+ block dir"
340         __populate_create_dir "${SCRATCH_MNT}/S_IFDIR.FMT_BLOCK" "$((dblksz / 32))"
341
342         # - HTREE
343         echo "+ htree dir"
344         __populate_create_dir "${SCRATCH_MNT}/S_IFDIR.FMT_HTREE" "$((4 * dblksz / 24))"
345
346         # Symlinks
347         # - FMT_LOCAL
348         echo "+ inline symlink"
349         ln -s target "${SCRATCH_MNT}/S_IFLNK.FMT_LOCAL"
350
351         # - FMT_EXTENTS
352         echo "+ extents symlink"
353         ln -s "$(perl -e 'print "x" x 1023;')" "${SCRATCH_MNT}/S_IFLNK.FMT_EXTENTS"
354
355         # Char & block
356         echo "+ special"
357         mknod "${SCRATCH_MNT}/S_IFCHR" c 1 1
358         mknod "${SCRATCH_MNT}/S_IFBLK" c 1 1
359
360         # special file with an xattr
361         setfacl -P -m u:nobody:r ${SCRATCH_MNT}/S_IFCHR
362
363         # Attribute formats
364         # LOCAL
365         echo "+ local attr"
366         __populate_create_attr "${SCRATCH_MNT}/ATTR.FMT_LOCAL" 0
367
368         # BLOCK
369         echo "+ block attr"
370         __populate_create_attr "${SCRATCH_MNT}/ATTR.FMT_BLOCK" "$((blksz / 40))"
371
372         # trusted namespace
373         touch ${SCRATCH_MNT}/ATTR.TRUSTED
374         setfattr -n trusted.moo -v urk ${SCRATCH_MNT}/ATTR.TRUSTED
375
376         # security namespace
377         touch ${SCRATCH_MNT}/ATTR.SECURITY
378         setfattr -n security.foo -v bar ${SCRATCH_MNT}/ATTR.SECURITY
379
380         # system namespace
381         touch ${SCRATCH_MNT}/ATTR.SYSTEM
382         setfacl -m u:root:r ${SCRATCH_MNT}/ATTR.SYSTEM
383
384         # Make an unused inode
385         echo "+ empty file"
386         touch "${SCRATCH_MNT}/unused"
387         $XFS_IO_PROG -f -c 'fsync' "${SCRATCH_MNT}/unused"
388         rm -rf "${SCRATCH_MNT}/unused"
389
390         # Copy some real files (xfs tests, I guess...)
391         echo "+ real files"
392         test $fill -ne 0 && __populate_fill_fs "${SCRATCH_MNT}" 5
393
394         umount "${SCRATCH_MNT}"
395 }
396
397 # Find the inode number of a file
398 __populate_find_inode() {
399         name="$1"
400         inode="$(stat -c '%i' "${name}")"
401         echo "${inode}"
402 }
403
404 # Check data fork format of XFS file
405 __populate_check_xfs_dformat() {
406         inode="$1"
407         format="$2"
408
409         fmt="$(_scratch_xfs_db -c "inode ${inode}" -c 'p core.format' | sed -e 's/^.*(\([a-z]*\)).*$/\1/g')"
410         test "${format}" = "${fmt}" || _fail "failed to create ino ${inode} dformat expected ${format} saw ${fmt}"
411 }
412
413 # Check attr fork format of XFS file
414 __populate_check_xfs_aformat() {
415         inode="$1"
416         format="$2"
417
418         fmt="$(_scratch_xfs_db -c "inode ${inode}" -c 'p core.aformat' | sed -e 's/^.*(\([a-z]*\)).*$/\1/g')"
419         test "${format}" = "${fmt}" || _fail "failed to create ino ${inode} aformat expected ${format} saw ${fmt}"
420 }
421
422 # Check structure of XFS directory
423 __populate_check_xfs_dir() {
424         inode="$1"
425         dtype="$2"
426
427         (test -n "${leaf_lblk}" && test -n "${node_lblk}") || _fail "must define leaf_lblk and node_lblk before calling __populate_check_xfs_dir"
428         datab=0
429         leafb=0
430         freeb=0
431         #echo "== check dir ${inode} type ${dtype}" ; _scratch_xfs_db -x -c "inode ${inode}" -c "bmap"
432         _scratch_xfs_db -x -c "inode ${inode}" -c "dblock 0" -c "stack" | grep -q 'file data block is unmapped' || datab=1
433         _scratch_xfs_db -x -c "inode ${inode}" -c "dblock ${leaf_lblk}" -c "stack" | grep -q 'file data block is unmapped' || leafb=1
434         _scratch_xfs_db -x -c "inode ${inode}" -c "dblock ${node_lblk}" -c "stack" | grep -q 'file data block is unmapped' || freeb=1
435
436         case "${dtype}" in
437         "shortform"|"inline"|"local")
438                 (test "${datab}" -eq 0 && test "${leafb}" -eq 0 && test "${freeb}" -eq 0) || _fail "failed to create ${dtype} dir ino ${inode} datab ${datab} leafb ${leafb} freeb ${freeb}"
439                 ;;
440         "block")
441                 (test "${datab}" -eq 1 && test "${leafb}" -eq 0 && test "${freeb}" -eq 0) || _fail "failed to create ${dtype} dir ino ${inode} datab ${datab} leafb ${leafb} freeb ${freeb}"
442                 ;;
443         "leaf")
444                 (test "${datab}" -eq 1 && test "${leafb}" -eq 1 && test "${freeb}" -eq 0) || _fail "failed to create ${dtype} dir ino ${inode} datab ${datab} leafb ${leafb} freeb ${freeb}"
445                 ;;
446         "leafn")
447                 _scratch_xfs_db -x -c "inode ${inode}" -c "dblock ${leaf_lblk}" -c "p lhdr.info.hdr.magic" | egrep -q '(0x3dff|0xd2ff)' || _fail "failed to create ${dtype} dir ino ${inode} datab ${datab} leafb ${leafb} freeb ${freeb}"
448                 ;;
449         "node"|"btree")
450                 (test "${datab}" -eq 1 && test "${leafb}" -eq 1 && test "${freeb}" -eq 1) || _fail "failed to create ${dtype} dir ino ${inode} datab ${datab} leafb ${leafb} freeb ${freeb}"
451                 ;;
452         *)
453                 _fail "Unknown directory type ${dtype}"
454         esac
455 }
456
457 # Check structure of XFS attr
458 __populate_check_xfs_attr() {
459         inode="$1"
460         atype="$2"
461
462         datab=0
463         leafb=0
464         #echo "== check attr ${inode} type ${dtype}" ; _scratch_xfs_db -x -c "inode ${inode}" -c "bmap -a"
465         _scratch_xfs_db -x -c "inode ${inode}" -c "ablock 0" -c "stack" | grep -q 'file attr block is unmapped' || datab=1
466         _scratch_xfs_db -x -c "inode ${inode}" -c "ablock 1" -c "stack" | grep -q 'file attr block is unmapped' || leafb=1
467
468         case "${atype}" in
469         "shortform"|"inline"|"local")
470                 (test "${datab}" -eq 0 && test "${leafb}" -eq 0) || _fail "failed to create ${atype} attr ino ${inode} datab ${datab} leafb ${leafb}"
471                 ;;
472         "leaf")
473                 (test "${datab}" -eq 1 && test "${leafb}" -eq 0) || _fail "failed to create ${atype} attr ino ${inode} datab ${datab} leafb ${leafb}"
474                 ;;
475         "node"|"btree")
476                 (test "${datab}" -eq 1 && test "${leafb}" -eq 1) || _fail "failed to create ${atype} attr ino ${inode} datab ${datab} leafb ${leafb}"
477                 ;;
478         *)
479                 _fail "Unknown attribute type ${atype}"
480         esac
481 }
482
483 # Check that there's at least one per-AG btree with multiple levels
484 __populate_check_xfs_agbtree_height() {
485         bt_type="$1"
486         nr_ags=$(_scratch_xfs_db -c 'sb 0' -c 'p agcount' | awk '{print $3}')
487
488         case "${bt_type}" in
489         "bno"|"cnt"|"rmap"|"refcnt")
490                 hdr="agf"
491                 bt_prefix="${bt_type}"
492                 ;;
493         "ino")
494                 hdr="agi"
495                 bt_prefix=""
496                 ;;
497         "fino")
498                 hdr="agi"
499                 bt_prefix="free_"
500                 ;;
501         *)
502                 _fail "Don't know about AG btree ${bt_type}"
503                 ;;
504         esac
505
506         seq 0 $((nr_ags - 1)) | while read ag; do
507                 bt_level=$(_scratch_xfs_db -c "${hdr} ${ag}" -c "p ${bt_prefix}level" | awk '{print $3}')
508                 if [ "${bt_level}" -gt 1 ]; then
509                         return 100
510                 fi
511         done
512         test $? -eq 100 || _fail "Failed to create ${bt_type} of sufficient height!"
513         return 1
514 }
515
516 # Check that populate created all the types of files we wanted
517 _scratch_xfs_populate_check() {
518         _scratch_mount
519         extents_file="$(__populate_find_inode "${SCRATCH_MNT}/S_IFREG.FMT_EXTENTS")"
520         btree_file="$(__populate_find_inode "${SCRATCH_MNT}/S_IFREG.FMT_BTREE")"
521         inline_dir="$(__populate_find_inode "${SCRATCH_MNT}/S_IFDIR.FMT_INLINE")"
522         block_dir="$(__populate_find_inode "${SCRATCH_MNT}/S_IFDIR.FMT_BLOCK")"
523         leaf_dir="$(__populate_find_inode "${SCRATCH_MNT}/S_IFDIR.FMT_LEAF")"
524         leafn_dir="$(__populate_find_inode "${SCRATCH_MNT}/S_IFDIR.FMT_LEAFN")"
525         node_dir="$(__populate_find_inode "${SCRATCH_MNT}/S_IFDIR.FMT_NODE")"
526         btree_dir="$(__populate_find_inode "${SCRATCH_MNT}/S_IFDIR.FMT_BTREE")"
527         local_slink="$(__populate_find_inode "${SCRATCH_MNT}/S_IFLNK.FMT_LOCAL")"
528         extents_slink="$(__populate_find_inode "${SCRATCH_MNT}/S_IFLNK.FMT_EXTENTS")"
529         bdev="$(__populate_find_inode "${SCRATCH_MNT}/S_IFBLK")"
530         cdev="$(__populate_find_inode "${SCRATCH_MNT}/S_IFCHR")"
531         local_attr="$(__populate_find_inode "${SCRATCH_MNT}/ATTR.FMT_LOCAL")"
532         leaf_attr="$(__populate_find_inode "${SCRATCH_MNT}/ATTR.FMT_LEAF")"
533         node_attr="$(__populate_find_inode "${SCRATCH_MNT}/ATTR.FMT_NODE")"
534         btree_attr="$(__populate_find_inode "${SCRATCH_MNT}/ATTR.FMT_BTREE")"
535         is_finobt=$($XFS_INFO_PROG "${SCRATCH_MNT}" | grep -c 'finobt=1')
536         is_rmapbt=$($XFS_INFO_PROG "${SCRATCH_MNT}" | grep -c 'rmapbt=1')
537         is_reflink=$($XFS_INFO_PROG "${SCRATCH_MNT}" | grep -c 'reflink=1')
538
539         blksz="$(stat -f -c '%s' "${SCRATCH_MNT}")"
540         dblksz="$($XFS_INFO_PROG "${SCRATCH_MNT}" | grep naming.*bsize | sed -e 's/^.*bsize=//g' -e 's/\([0-9]*\).*$/\1/g')"
541         leaf_lblk="$((32 * 1073741824 / blksz))"
542         node_lblk="$((64 * 1073741824 / blksz))"
543         umount "${SCRATCH_MNT}"
544
545         __populate_check_xfs_dformat "${extents_file}" "extents"
546         __populate_check_xfs_dformat "${btree_file}" "btree"
547         __populate_check_xfs_dir "${inline_dir}" "inline"
548         __populate_check_xfs_dir "${block_dir}" "block"
549         __populate_check_xfs_dir "${leaf_dir}" "leaf"
550         __populate_check_xfs_dir "${leafn_dir}" "leafn"
551         __populate_check_xfs_dir "${node_dir}" "node"
552         __populate_check_xfs_dir "${btree_dir}" "btree"
553         __populate_check_xfs_dformat "${btree_dir}" "btree"
554         __populate_check_xfs_dformat "${bdev}" "dev"
555         __populate_check_xfs_dformat "${cdev}" "dev"
556         __populate_check_xfs_attr "${local_attr}" "local"
557         __populate_check_xfs_attr "${leaf_attr}" "leaf"
558         __populate_check_xfs_attr "${node_attr}" "node"
559         __populate_check_xfs_attr "${btree_attr}" "btree"
560         __populate_check_xfs_aformat "${btree_attr}" "btree"
561         __populate_check_xfs_agbtree_height "bno"
562         __populate_check_xfs_agbtree_height "cnt"
563         test $is_rmapbt -ne 0 && __populate_check_xfs_agbtree_height "rmap"
564         test $is_reflink -ne 0 && __populate_check_xfs_agbtree_height "refcnt"
565 }
566
567 # Check data fork format of ext4 file
568 __populate_check_ext4_dformat() {
569         dev="${SCRATCH_DEV}"
570         inode="$1"
571         format="$2"
572
573         extents=0
574         etree=0
575         debugfs -R "stat <${inode}>" "${dev}" 2> /dev/null | grep 'ETB[0-9]' -q && etree=1
576         iflags="$(debugfs -R "stat <${inode}>" "${dev}" 2> /dev/null | grep 'Flags:' | sed -e 's/^.*Flags: \([0-9a-fx]*\).*$/\1/g')"
577         test "$(echo "${iflags}" | awk '{print and(strtonum($1), 0x80000);}')" -gt 0 && extents=1
578
579         case "${format}" in
580         "blockmap")
581                 test "${extents}" -eq 0 || _fail "failed to create ino ${inode} with blockmap"
582                 ;;
583         "extent"|"extents")
584                 test "${extents}" -eq 1 || _fail "failed to create ino ${inode} with extents"
585                 ;;
586         "etree")
587                 (test "${extents}" -eq 1 && test "${etree}" -eq 1) || _fail "failed to create ino ${inode} with extent tree"
588                 ;;
589         *)
590                 _fail "Unknown dformat ${format}"
591         esac
592 }
593
594 # Check attr fork format of ext4 file
595 __populate_check_ext4_aformat() {
596         dev="${SCRATCH_DEV}"
597         inode="$1"
598         format="$2"
599
600         ablock=1
601         debugfs -R "stat <${inode}>" "${dev}" 2> /dev/null | grep 'File ACL: 0' -q && ablock=0
602
603         case "${format}" in
604         "local"|"inline")
605                 test "${ablock}" -eq 0 || _fail "failed to create inode ${inode} with ${format} xattr"
606                 ;;
607         "block")
608                 test "${extents}" -eq 1 || _fail "failed to create inode ${inode} with ${format} xattr"
609                 ;;
610         *)
611                 _fail "Unknown aformat ${format}"
612         esac
613 }
614
615 # Check structure of ext4 dir
616 __populate_check_ext4_dir() {
617         dev="${SCRATCH_DEV}"
618         inode="$1"
619         dtype="$2"
620
621         htree=0
622         inline=0
623         iflags="$(debugfs -R "stat <${inode}>" "${dev}" 2> /dev/null | grep 'Flags:' | sed -e 's/^.*Flags: \([0-9a-fx]*\).*$/\1/g')"
624         test "$(echo "${iflags}" | awk '{print and(strtonum($1), 0x1000);}')" -gt 0 && htree=1
625         test "$(echo "${iflags}" | awk '{print and(strtonum($1), 0x10000000);}')" -gt 0 && inline=1
626
627         case "${dtype}" in
628         "inline")
629                 (test "${inline}" -eq 1 && test "${htree}" -eq 0) || _fail "failed to create ${dtype} dir ino ${inode} htree ${htree} inline ${inline}"
630                 ;;
631         "block")
632                 (test "${inline}" -eq 0 && test "${htree}" -eq 0) || _fail "failed to create ${dtype} dir ino ${inode} htree ${htree} inline ${inline}"
633                 ;;
634         "htree")
635                 (test "${inline}" -eq 0 && test "${htree}" -eq 1) || _fail "failed to create ${dtype} dir ino ${inode} htree ${htree} inline ${inline}"
636                 ;;
637         *)
638                 _fail "Unknown directory type ${dtype}"
639                 ;;
640         esac
641 }
642
643 # Check that populate created all the types of files we wanted
644 _scratch_ext4_populate_check() {
645         _scratch_mount
646         extents_file="$(__populate_find_inode "${SCRATCH_MNT}/S_IFREG.FMT_EXTENTS")"
647         etree_file="$(__populate_find_inode "${SCRATCH_MNT}/S_IFREG.FMT_ETREE")"
648         block_dir="$(__populate_find_inode "${SCRATCH_MNT}/S_IFDIR.FMT_BLOCK")"
649         htree_dir="$(__populate_find_inode "${SCRATCH_MNT}/S_IFDIR.FMT_HTREE")"
650         extents_slink="$(__populate_find_inode "${SCRATCH_MNT}/S_IFLNK.FMT_EXTENTS")"
651         local_attr="$(__populate_find_inode "${SCRATCH_MNT}/ATTR.FMT_LOCAL")"
652         block_attr="$(__populate_find_inode "${SCRATCH_MNT}/ATTR.FMT_BLOCK")"
653         umount "${SCRATCH_MNT}"
654
655         __populate_check_ext4_dformat "${extents_file}" "extents"
656         __populate_check_ext4_dformat "${etree_file}" "etree"
657         __populate_check_ext4_dir "${block_dir}" "block"
658         __populate_check_ext4_dir "${htree_dir}" "htree"
659         __populate_check_ext4_dformat "${extents_slink}" "extents"
660         __populate_check_ext4_aformat "${local_attr}" "local"
661         __populate_check_ext4_aformat "${block_attr}" "block"
662 }
663
664 # Populate a scratch FS and check the contents to make sure we got that
665 _scratch_populate() {
666         case "${FSTYP}" in
667         "xfs")
668                 _scratch_xfs_populate
669                 _scratch_xfs_populate_check
670                 ;;
671         "ext2"|"ext3"|"ext4")
672                 _scratch_ext4_populate
673                 _scratch_ext4_populate_check
674                 ;;
675         *)
676                 _fail "Don't know how to populate a ${FSTYP} filesystem."
677                 ;;
678         esac
679 }
680
681 # Fill a file system by repeatedly creating files in the given folder
682 # starting with the given file size.  Files are reduced in size when
683 # they can no longer fit until no more files can be created.
684 _fill_fs()
685 {
686         local file_size=$1
687         local dir=$2
688         local block_size=$3
689         local switch_user=$4
690         local file_count=1
691         local bytes_written=0
692         local use_falloc=1;
693
694         if [ $# -ne 4 ]; then
695                 echo "Usage: _fill_fs filesize dir blocksize switch_user"
696                 exit 1
697         fi
698
699         if [ $switch_user -eq 0 ]; then
700                 mkdir -p $dir
701         else
702                 _user_do "mkdir -p $dir"
703         fi
704         if [ ! -d $dir ]; then
705                 return 0;
706         fi
707
708         testio=`$XFS_IO_PROG -F -fc "falloc 0 $block_size" $dir/$$.xfs_io 2>&1`
709         echo $testio | grep -q "not found" && use_falloc=0
710         echo $testio | grep -q "Operation not supported" && use_falloc=0
711
712         if [ $file_size -lt $block_size ]; then
713                 $file_size = $block_size
714         fi
715
716         while [ $file_size -ge $block_size ]; do
717                 bytes_written=0
718                 if [ $switch_user -eq 0 ]; then
719                         if [ $use_falloc -eq 0 ]; then
720                                 $XFS_IO_PROG -fc "pwrite -b 8388608 0 $file_size" \
721                                         $dir/$file_count
722                         else
723                                 $XFS_IO_PROG -fc "falloc 0 $file_size" \
724                                         $dir/$file_count
725                         fi
726                 else
727                         if [ $use_falloc -eq 0 ]; then
728                                 _user_do "$XFS_IO_PROG -f -c \"pwrite -b 8388608 0 \
729                                         $file_size\" $dir/$file_count"
730                         else
731                                 _user_do "$XFS_IO_PROG -f -c \"falloc 0 \
732                                         $file_size\" $dir/$file_count"
733                         fi
734                 fi
735
736                 if [ -f $dir/$file_count ]; then
737                         bytes_written=$(stat -c '%s' $dir/$file_count)
738                 fi
739
740                 # If there was no room to make the file, then divide it in
741                 # half, and keep going
742                 if [ $bytes_written -lt $file_size ]; then
743                         file_size=$((file_size / 2))
744                 fi
745                 file_count=$((file_count + 1))
746         done
747 }
748
749 # Populate a scratch FS from scratch or from a cached image.
750 _scratch_populate_cached() {
751         POPULATE_METADUMP="${TEST_DIR}/__populate.${FSTYP}"
752         POPULATE_METADUMP_DESCR="${TEST_DIR}/__populate.${FSTYP}.txt"
753
754         # Don't keep metadata images cached for more 48 hours...
755         rm -rf "$(find "${POPULATE_METADUMP}" -mtime +2 2>/dev/null)"
756
757         # Throw away cached image if it doesn't match our spec.
758         case "${FSTYP}" in
759         "ext4")
760                 extra_descr="LOGDEV ${SCRATCH_LOGDEV} USE_EXTERNAL ${USE_EXTERNAL}"
761                 # ext4 cannot e2image external logs, so we cannot restore
762                 test -n "${SCRATCH_LOGDEV}" && rm -f "${POPULATE_METADUMP}"
763                 ;;
764         "xfs")
765                 extra_descr="LOGDEV ${SCRATCH_LOGDEV} USE_EXTERNAL ${USE_EXTERNAL} RTDEV ${SCRATCH_RTDEV}"
766                 _populate_xfs_qmount_option
767                 if echo "${MOUNT_OPTIONS}" | grep -q 'usrquota'; then
768                         extra_descr="${extra_descr} QUOTAS"
769                 fi
770                 ;;
771         *)
772                 extra_descr="";;
773         esac
774         meta_descr="FSTYP ${FSTYP} MKFS_OPTIONS ${MKFS_OPTIONS} SIZE $(blockdev --getsz "${SCRATCH_DEV}") ${extra_descr} ARGS $@"
775         cmp -s "${POPULATE_METADUMP_DESCR}" <(echo "${meta_descr}") || rm -rf "${POPULATE_METADUMP}"
776
777         # Do we have a cached image?
778         if [ -r "${POPULATE_METADUMP}" ]; then
779                 case "${FSTYP}" in
780                 "xfs")
781                         xfs_mdrestore "${POPULATE_METADUMP}" "${SCRATCH_DEV}" && return
782                         ;;
783                 "ext2"|"ext3"|"ext4")
784                         e2image -r "${POPULATE_METADUMP}" "${SCRATCH_DEV}" && return
785                         ;;
786                 esac
787         fi
788
789         # Oh well, just create one from scratch
790         _scratch_mkfs
791         echo "${meta_descr}" > "${POPULATE_METADUMP_DESCR}"
792         case "${FSTYP}" in
793         "xfs")
794                 _scratch_xfs_populate $@
795                 _scratch_xfs_populate_check
796                 _scratch_metadump "${POPULATE_METADUMP}" -a -o
797                 ;;
798         "ext2"|"ext3"|"ext4")
799                 _scratch_ext4_populate $@
800                 _scratch_ext4_populate_check
801                 e2image -Q "${SCRATCH_DEV}" "${POPULATE_METADUMP}"
802                 ;;
803         *)
804                 _fail "Don't know how to populate a ${FSTYP} filesystem."
805                 ;;
806         esac
807 }