From 06ccadda049d3c2553b4b2f71fc4d875def1109b Mon Sep 17 00:00:00 2001 From: Christopher Hoffman Date: Tue, 8 Oct 2024 12:10:59 +0000 Subject: [PATCH] qa: Convert and create tests for libcephfs fscrypt Convert existing tests to use teuthology framework. Create tests to test N>1 fscrypt clients Fixes: https://tracker.ceph.com/issues/66577 Signed-off-by: Christopher Hoffman --- .../tasks/1-tests/fscrypt-snippets.yaml | 7 - qa/tasks/cephfs/mount.py | 124 +++++++- qa/tasks/cephfs/test_fscrypt.py | 234 ++++++++++++++- qa/workunits/suites/fscrypt-snippets.py | 278 ------------------ qa/workunits/suites/fscrypt-snippets.sh | 7 - 5 files changed, 341 insertions(+), 309 deletions(-) delete mode 100644 qa/suites/fs/fscrypt/tasks/1-tests/fscrypt-snippets.yaml delete mode 100755 qa/workunits/suites/fscrypt-snippets.py delete mode 100755 qa/workunits/suites/fscrypt-snippets.sh diff --git a/qa/suites/fs/fscrypt/tasks/1-tests/fscrypt-snippets.yaml b/qa/suites/fs/fscrypt/tasks/1-tests/fscrypt-snippets.yaml deleted file mode 100644 index 0b2947fad50bb..0000000000000 --- a/qa/suites/fs/fscrypt/tasks/1-tests/fscrypt-snippets.yaml +++ /dev/null @@ -1,7 +0,0 @@ -tasks: -- workunit: - timeout: 6h - clients: - client.0: - - fs/fscrypt.sh none fscrypt-snippets - - fs/fscrypt.sh unlocked fscrypt-snippets diff --git a/qa/tasks/cephfs/mount.py b/qa/tasks/cephfs/mount.py index 35d7c630dff2a..8b599403bbdc5 100644 --- a/qa/tasks/cephfs/mount.py +++ b/qa/tasks/cephfs/mount.py @@ -717,7 +717,7 @@ class CephFSMountBase(object): if r.exitstatus != 0: raise RuntimeError("Expected file {0} not found".format(suffix)) - def write_file(self, path, data, perms=None): + def write_file_ex(self, path, data, **kwargs): """ Write the given data at the given path and set the given perms to the file on the path. @@ -725,12 +725,22 @@ class CephFSMountBase(object): if path.find(self.hostfs_mntpt) == -1: path = os.path.join(self.hostfs_mntpt, path) - write_file(self.client_remote, path, data) + self.client_remote.write_file(path, data, **kwargs) + + def write_file(self, path, data, perms=None, **kwargs): + """ + Write the given data at the given path and set the given perms to the + file on the path. + """ + if path.find(self.hostfs_mntpt) == -1: + path = os.path.join(self.hostfs_mntpt, path) + + write_file(self.client_remote, path, data, **kwargs) if perms: self.run_shell(args=f'chmod {perms} {path}') - def read_file(self, path, sudo=False): + def read_file(self, path, sudo=False, offset=None, length=None): """ Return the data from the file on given path. """ @@ -740,7 +750,13 @@ class CephFSMountBase(object): args = [] if sudo: args.append('sudo') - args += ['cat', path] + args.append('dd') + args.append(f'if={path}') + args.append('bs=1') + if offset: + args.append(f'skip={offset}') + if length: + args.append(f'count={length}') return self.run_shell(args=args, omit_sudo=False).stdout.getvalue().strip() @@ -1492,6 +1508,84 @@ class CephFSMountBase(object): else: return proc + def lchown(self, fs_path, uid, gid): + """ + Change ownership of a link with uid and gid provided. + """ + + abs_path = os.path.join(self.hostfs_mntpt, fs_path) + pyscript = dedent(f""" + import os + import sys + + try: + os.lchown("{abs_path}", {uid}, {gid}) + except OSError as e: + sys.exit(e.errno) + """) + proc = self._run_python(pyscript) + proc.wait() + + def symlink(self, fs_path, symlink_path): + """ + Change ownership of a link with uid and gid provided. + """ + + src_path = os.path.join(self.hostfs_mntpt, fs_path) + sym_path = os.path.join(self.hostfs_mntpt, symlink_path) + pyscript = dedent(f""" + import os + import sys + + try: + os.symlink("{src_path}", "{sym_path}") + except OSError as e: + sys.exit(e.errno) + """) + proc = self._run_python(pyscript) + proc.wait() + + def copy_file_range(self, src, dest, length): + """ + Truncate a file of certain size + """ + + src_path = os.path.join(self.hostfs_mntpt, src) + dest_path = os.path.join(self.hostfs_mntpt, dest) + pyscript = dedent(f""" + import os + import sys + + try: + src_fd = os.open("{src_path}", os.O_RDONLY) + dest_fd = os.open("{dest_path}", os.O_WRONLY|os.O_TRUNC) + os.copy_file_range(src_fd, dest_fd, {length}) + os.close(src_fd) + os.close(dest_fd) + except OSError as e: + sys.exit(e.errno) + """) + proc = self._run_python(pyscript) + proc.wait() + + def truncate(self, fs_path, size): + """ + Truncate a file of certain size + """ + + abs_path = os.path.join(self.hostfs_mntpt, fs_path) + pyscript = dedent(f""" + import os + import sys + + try: + os.truncate("{abs_path}", {size}) + except OSError as e: + sys.exit(e.errno) + """) + proc = self._run_python(pyscript) + proc.wait() + def touch(self, fs_path): """ Create a dentry if it doesn't already exist. This python @@ -1515,6 +1609,28 @@ class CephFSMountBase(object): proc = self._run_python(pyscript) proc.wait() + def touch_os(self, fs_path): + """ + Create a dentry if it doesn't already exist. Use open in os module. + + :param fs_path: + :return: + """ + abs_path = os.path.join(self.hostfs_mntpt, fs_path) + pyscript = dedent(""" + import os + import sys + import errno + + try: + fd = os.open("{path}", os.O_RDONLY | os.O_CREAT) + os.close(fd) + except IOError as e: + sys.exit(errno.EIO) + """).format(path=abs_path) + proc = self._run_python(pyscript) + proc.wait() + def path_to_ino(self, fs_path, follow_symlinks=True): abs_path = os.path.join(self.hostfs_mntpt, fs_path) diff --git a/qa/tasks/cephfs/test_fscrypt.py b/qa/tasks/cephfs/test_fscrypt.py index a9a4fb5ae75a0..66dba7780fee2 100644 --- a/qa/tasks/cephfs/test_fscrypt.py +++ b/qa/tasks/cephfs/test_fscrypt.py @@ -2,6 +2,7 @@ from io import StringIO from os.path import basename import random import string +import time from logging import getLogger @@ -91,42 +92,249 @@ class TestFSCryptRecovery(FSCryptTestCase): self.mount_a.run_shell_payload(f"cd {self.path} && stat {file}") class TestFSCryptRMW(FSCryptTestCase): - def test_fscrypt_overwrite_block_boundary(): + CLIENTS_REQUIRED = 2 + def setUp(self): + super().setUp() + self.mount_b.run_shell_payload(f"sudo fscrypt unlock --quiet --key=/tmp/key {self.path}") + + def test_fscrypt_overwrite_block_boundary(self): """Test writing data with small, half write on previous block and trailing on new block""" - file = 'file.log' + file = f'{self.path}/file.log' size = 5529 offset = 3379 contents = 's' * size - self.mount_a.write_file(file, contents, offset) + self.mount_a.write_file_ex(path=file, bs=1, data=contents, offset=offset) - #s = write_fill(fd, 's', 5529, 6144) - sleep(10) + time.sleep(10) size = 4033 offset = 4127 contents = 't' * size - self.mount_a.write_file(file, contents, offset) - #s = write_fill(fd, 't', 4033, 6144) + self.mount_a.write_file_ex(path=file, bs=1, data=contents, offset=offset) def test_fscrypt_huge_hole(self): """Test writing data with huge hole, half write on previous block and trailing on new block""" - file = 'file.log' + file = f'{self.path}/file.log' size = 4096 offset = 2147477504 contents = 's' * size - self.mount_a.write_file(file, contents, offset) - #s = write_fill(fd, 's', 4096, 107374182400) - sleep(10) + self.mount_a.write_file_ex(path=file, bs=1, data=contents, offset=offset) + time.sleep(10) size = 8 offset = 12 contents = 't' * size - self.mount_a.write_file(file, contents, offset) - #s = write_fill(fd, 't', 8, 16) + self.mount_a.write_file_ex(path=file, bs=1, data=contents, offset=offset) + + def test_fscrypt_med_hole_write_boundary(self): + """Test writing data past many holes on offset 0 of block""" + + file = f'{self.path}/file.log' + + #reproducing sys calls after ffsb bench has started + size = 3192 + offset = 60653568 + contents = 's' * size + self.mount_a.write_file_ex(path=file, bs=1, data=contents, offset=offset) + + def test_fscrypt_simple_rmw(self): + """ Test simple rmw""" + + file = f'{self.path}/file.log' + + size = 32 + offset = 0 + contents = 's' * size + self.mount_a.write_file_ex(path=file, bs=1, data=contents, offset=offset) + + size = 8 + offset = 8 + contents = 't' * size + self.mount_a.write_file_ex(path=file, bs=1, data=contents, offset=offset) + + src_hash = self.mount_a.dir_checksum(path=file) + dest_hash = self.mount_b.dir_checksum(path=file) + + if src_hash != dest_hash: + raise ValueError + + def test_fscrypt_truncate_overwrite(self): + """ Test copy smaller file -> larger file gets new file size""" + + file1 = f'{self.path}/file1.log' + file2 = f'{self.path}/file2.log' + expected_size = 1024 + + self.mount_a.touch(file1) + self.mount_a.touch(file2) + + self.mount_a.truncate(file1, 1048576) + self.mount_a.truncate(file2, 1024) + + #simulate copy file2 -> file1 + self.mount_a.copy_file_range(file2, file1, 9223372035781033984) + actual_size = self.mount_a.stat(file1)['st_size'] + + if actual_size != expected_size: + raise ValueError + + def test_fscrypt_truncate_path(self): + """ Test overwrite/cp displays effective_size and not real size""" + + file = f'{self.path}/file.log' + expected_size = 68686 + + #fstest create test1 0644; + self.mount_a.touch_os(file) + + #fstest truncate test1 68686; + self.mount_a.truncate(file, expected_size) + + #fstest stat test1 size + if self.mount_a.lstat(file)['st_size'] != expected_size: + raise ValueError + #stat above command returns 69632 instead of truncated value. + + def test_fscrypt_lchown_symlink(self): + """ Test lchown to ensure target is set""" + + file1 = f'{self.path}/file1.log' + + self.mount_a.touch(file1) + + #fstest symlink file1 symlink1 + file2 = f'{self.path}/symlink' + self.mount_a.symlink(file1, file2) + + #fstest lchown symlink1 135 579 + self.mount_a.lchown(file2, 1000, 1000) + + # ls -l + #-rw-r--r--. 1 root root 0 Apr 22 18:11 file1 + #lrwxrwxrwx. 1 135 579 46 Apr 22 18:11 symlink1 -> ''$'\266\310''%'$'\005''W'$'\335''.'$'\355\211''kblD'$'\300''gq'$'\002\236\367''3'$'\255\201\001''Z6;'$'\221''&'$'\216\331\177''Q' + ###if os.readlink(file2) != file1: + ### raise Exception + + def test_fscrypt_900mhole_100mwrite(self): + """ Test 900m hole 100m data write""" + + size = 100 + offset = 900 + + files=[f'{self.path}/kfile.log', f'{self.path}/fuse_file.log'] + KERNEL_INDEX = 0 + FUSE_INDEX = 1 + + self.mount_a.write_n_mb(files[KERNEL_INDEX], size, seek=offset) + src_hash = self.mount_a.dir_checksum(path=files[KERNEL_INDEX]) + dest_hash = self.mount_b.dir_checksum(path=files[KERNEL_INDEX]) + + if src_hash != dest_hash: + raise ValueError + + self.mount_b.write_n_mb(files[FUSE_INDEX], size, seek=offset) + src_hash = self.mount_b.dir_checksum(path=files[FUSE_INDEX]) + dest_hash = self.mount_a.dir_checksum(path=files[FUSE_INDEX]) + + if src_hash != dest_hash: + raise ValueError + + def test_fscrypt_1gwrite_400m600mwrite(self): + """ Test 200M overwrite of 1G file""" + + file=f'{self.path}/file.log' + + self.mount_a.write_n_mb(file, 1000) + self.mount_b.write_n_mb(file, 200, seek=400) + client1_hash = self.mount_a.dir_checksum(path=file) + client2_hash = self.mount_b.dir_checksum(path=file) + + if client1_hash != client2_hash: + raise ValueError + + def test_fscrypt_truncate_ladder(self): + """ Test truncate down from 1GB""" + + file = f'{self.path}/file.log' + expected_sizes = [1024, 900, 500, 1] + + # define the truncate side and the read side + tside = self.mount_a + rside = self.mount_b + + tside.touch(file) + + for expected_size in expected_sizes: + tside.truncate(file, expected_size) + tside_size = tside.stat(file)['st_size'] + rside_size = rside.stat(file)['st_size'] + if tside_size != rside_size: + raise ValueError + + #swap which client does the truncate + tside, rside = rside, tside + + def strided_tests(self, fscrypt_block_size, write_size, num_writes, shared_file, fill): + wside = self.mount_a + rside = self.mount_b + + contents = fill * write_size * num_writes + + for i in range(num_writes): + offset = i * write_size + end_offset = offset + write_size + strided_write = contents[offset:end_offset] + s_size = len(strided_write) + print(f"=============== {offset} to - {end_offset} size: {s_size} ==============") + wside.write_file_ex(path=shared_file, data=strided_write, bs=1, offset=offset, sync=True) + wside, rside = rside, wside + + shared_contents1 = wside.read_file(shared_file) + shared_contents2 = rside.read_file(shared_file) + + if shared_contents1 != shared_contents2: + raise ValueError + + if contents != shared_contents1: + print(f"================= {contents} \n vs \n {shared_contents1}") + raise ValueError + + def test_fscrypt_strided_small(self): + """ Test strided i/o within a single fscrypt block""" + + fscrypt_block_size = 4096 + write_size = 256 + num_writes = 16 + shared_file = f'{self.path}/file.log' + fill = 's' + + self.strided_tests(fscrypt_block_size, write_size, num_writes, shared_file, fill) + + def test_fscrypt_strided_regular_write(self): + """ Test aligned strided i/o on fscrypt block""" + + fscrypt_block_size = 4096 + write_size = fscrypt_block_size + num_writes = 16 + shared_file = f'{self.path}/file.log' + fill = 's' + + self.strided_tests(fscrypt_block_size, write_size, num_writes, shared_file, fill) + + def test_unaligned_strided_write(self): + """ Test unaligned strided i/o on fscrypt block""" + + fscrypt_block_size = 4096 + write_size = 4000 + num_writes = 16 + shared_file = f'{self.path}/file.log' + fill = 's' + + self.strided_tests(fscrypt_block_size, write_size, num_writes, shared_file, fill) class TestFSCryptXFS(XFSTestsDev): diff --git a/qa/workunits/suites/fscrypt-snippets.py b/qa/workunits/suites/fscrypt-snippets.py deleted file mode 100755 index 0d861135d30c0..0000000000000 --- a/qa/workunits/suites/fscrypt-snippets.py +++ /dev/null @@ -1,278 +0,0 @@ -#! /usr/bin/env python3 - -import hashlib -import os -import sys -from time import sleep - -client_type = "" - -def write_fill(fd, fill, size, offset): - s = '' - for i in range(0,size): - s += fill - - os.lseek(fd, offset - int(size / 2), 0) - os.write(fd, str.encode(s)) - -def run_strided_test(fuse_file, kernel_file, size, offset): - a = 1 - -def test_overwrite_block_boundary(): - """Test writing data with small, half write on previous block and trailing on new block""" - - file = 'file.log' - fd = os.open(file, os.O_RDWR|os.O_CREAT) - - s = write_fill(fd, 's', 5529, 6144) - sleep(10) - s = write_fill(fd, 't', 4033, 6144) - - os.close(fd) - os.remove(file) - -def test_huge_hole(): - """Test writing data with huge hole, half write on previous block and trailing on new block""" - - file = 'file.log' - fd = os.open(file, os.O_RDWR|os.O_CREAT) - - s = write_fill(fd, 's', 4096, 107374182400) - sleep(10) - s = write_fill(fd, 't', 8, 16) - - os.close(fd) - os.remove(file) - -def test_med_hole_write_boundary(): - """Test writing data past many holes on offset 0 of block""" - - file = 'file.log' - - fd = os.open(file, os.O_RDWR|os.O_CREAT) - - #reproducing sys calls after ffsb bench has started - fill = '\0' - size = 3192 - offset = 60653568 - - s = '' - for i in range(0,size): - s += fill - - os.lseek(fd, offset, 0) - os.write(fd, str.encode(s)) - - os.close(fd) - os.remove(file) - -def test_simple_rmw(): - """ Test simple rmw""" - - file = 'file.log' - match_hash='08723317846e79780c8c9521b0f4bc49' - - fd = os.open(file, os.O_RDWR|os.O_CREAT) - - s = write_fill(fd, 's', 32, 16) - s = write_fill(fd, 't', 8, 16) - - os.close(fd) - - fd = os.open(file, os.O_RDWR|os.O_CREAT) - m = hashlib.md5() - - m.update(os.read(fd, 32)) - os.close(fd) - - if match_hash != m.hexdigest(): - raise Exception - - os.remove(file) - -def test_truncate_overwrite(): - """ Test copy smaller file -> larger file gets new file size""" - - file1 = 'file1.log' - file2 = 'file2.log' - expected_size = 1024 - - fd = os.open(file1, os.O_WRONLY|os.O_CREAT) - os.close(fd) - fd = os.open(file2, os.O_WRONLY|os.O_CREAT) - os.close(fd) - - os.truncate(file1, 1048576) - os.truncate(file2, 1024) - - #simulate copy file2 -> file1 - fd = os.open(file1, os.O_WRONLY|os.O_TRUNC) - fd2 = os.open(file2, os.O_RDONLY) - os.copy_file_range(fd2, fd, 9223372035781033984) - os.close(fd) - os.close(fd2) - - if os.stat(file1).st_size != expected_size: - raise Exception - - os.remove(file1) - os.remove(file2) - -def test_truncate_path(): - """ Test overwrite/cp displays effective_size and not real size""" - - file = 'file1.log' - expected_size = 68686 - - #fstest create test1 0644; - fd = os.open(file, os.O_WRONLY|os.O_CREAT) - os.close(fd) - #fstest truncate test1 68686; - os.truncate(file, expected_size) - - #fstest stat test1 size - if os.lstat(file).st_size != expected_size: - raise Exception - #stat above command returns 69632 instead of truncated value. - - os.remove(file) - -def test_lchown_symlink(): - """ Test lchown to ensure target is set""" - - file1 = 'file1.log' - fd = os.open(file1, os.O_WRONLY|os.O_CREAT) - os.close(fd) - - #fstest symlink file1 symlink1 - file2 = 'symlink' - os.symlink(file1, file2) - - #fstest lchown symlink1 135 579 - os.lchown(file2, 135, 579) - - # ls -l - #-rw-r--r--. 1 root root 0 Apr 22 18:11 file1 - #lrwxrwxrwx. 1 135 579 46 Apr 22 18:11 symlink1 -> ''$'\266\310''%'$'\005''W'$'\335''.'$'\355\211''kblD'$'\300''gq'$'\002\236\367''3'$'\255\201\001''Z6;'$'\221''&'$'\216\331\177''Q' - if os.readlink(file2) != file1: - raise Exception - - os.remove(file1) - os.remove(file2) - -def test_900mhole_100mwrite(): - """ Test 900m hole 100m data write""" - - MB = 1024 * 1024 - offset = 900 * MB - data_size = 100 * MB - - contents = '' - fill = 'a' - - for i in range(0,data_size): - contents+= fill - - fuse_path = '/mnt/mycephfs/' - kernel_path='/mnt/kclient/' - - #file originated in kernel mount - kfile = 'kfile.log' - - #file originated in fuse mount - fuse_file = 'fuse_file.log' - - fd = os.open(kernel_path + kfile, os.O_WRONLY|os.O_CREAT) - os.lseek(fd, offset, 0) - os.write(fd, str.encode(contents)) - os.close(fd) - -def test_1gwrite_400m600mwrite(): - """ Test 200M overwrite of 1G file""" - - GB = 1024 * 1024 * 1024 - MB = 1024 * 1024 - - kfile = '/mnt/kclient/enc1/kfile.log' - fuse_file = '/mnt/mycephfs/enc1/kfile.log' - - contents = "a" * GB - - fd = os.open(kfile, os.O_WRONLY|os.O_CREAT) - os.write(fd, str.encode(contents)) - os.close(fd) - - overwrite_contents = "b" * 200 * MB - fd = os.open(fuse_file, os.O_WRONLY) - os.lseek(fd, 400 * MB, 0) - os.write(fd, str.encode(overwrite_contents)) - os.close(fd) - os.remove(kfile) - -def test_truncate_ladder(): - """ Test truncate down from 1GB""" - MB = 1024 * 1024 - - expected_sizes = [1024, 900, 500, 1] - - kfiles = ['/mnt/kclient/enc1/kfile.log', '/mnt/kclient/enc1/fuse_file.log'] - fuse_files = ['/mnt/mycephfs/enc1/kfile.log', '/mnt/mycephfs/enc1/fuse_file.log'] - KERNEL_INDEX = 0 - FUSE_INDEX = 1 - - #generate files from kernel - fd = os.open(kfiles[KERNEL_INDEX], os.O_WRONLY|os.O_CREAT) - os.close(fd) - for expected_size in expected_sizes: - os.truncate(kfiles[KERNEL_INDEX], expected_size) - stat_size = os.stat(fuse_files[KERNEL_INDEX]).st_size - if os.stat(fuse_files[KERNEL_INDEX]).st_size != expected_size: - print(f"{expected_size} vs {stat_size} path:=%s" % (kfiles[FUSE_INDEX])) - #raise Exception - #os.remove(kfiles[KERNEL_INDEX]) - - # generate files from fuse - fd = os.open(fuse_files[FUSE_INDEX], os.O_WRONLY|os.O_CREAT) - os.close(fd) - for expected_size in expected_sizes: - os.truncate(fuse_files[FUSE_INDEX], expected_size) - stat_size = os.stat(kfiles[FUSE_INDEX]).st_size - if stat_size != expected_size: - print(f"{expected_size} 1vs {stat_size} path:=%s" % (kfiles[FUSE_INDEX])) - #os.remove(fuse_files[FUSE_INDEX]) - -def test_strided_small_write(): - """ Test strided writes within a single fscrypt block""" - a = 1 - -def test_strided_regular_write(): - """ Test aligned strided writes on fscrypt block""" - b = 1 - -def test_unaligned_strided_write(): - """ Test unaligned strided write on fscrypt block""" - c = 1 - -def main(): - if (len(sys.argv) == 2): - client_type = sys.argv[1] - else: - print("Usage is: %s " % sys.argv[0]) - sys.exit(1) - - test_overwrite_block_boundary() - test_huge_hole() - test_med_hole_write_boundary() - test_simple_rmw() - test_truncate_overwrite() - test_truncate_path() - test_lchown_symlink() - test_900mhole_100mwrite - test_1gwrite_400m600mwrite() - test_truncate_ladder() - test_strided_small_write() - test_strided_regular_write() - test_unaligned_strided_write() - -if __name__ == '__main__': - main() diff --git a/qa/workunits/suites/fscrypt-snippets.sh b/qa/workunits/suites/fscrypt-snippets.sh deleted file mode 100755 index ffb9a54f809e3..0000000000000 --- a/qa/workunits/suites/fscrypt-snippets.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/usr/bin/env bash -#find which dir shell script is ran from -DIR=$(dirname -- "$( readlink -f -- "$0"; )") -PYTHON_SCRIPT="fscrypt-snippets.py" - -#run it -${DIR}/${PYTHON_SCRIPT} -- 2.39.5