From dce06e10ec2ede70e71aedef00e2785c1a6ac21a Mon Sep 17 00:00:00 2001 From: Christopher Hoffman Date: Mon, 19 May 2025 18:56:20 +0000 Subject: [PATCH] client: Implement cloning fscrypt subvolume snaps Signed-off-by: Christopher Hoffman --- src/client/Client.cc | 143 +++++++++++++++++++--- src/client/Client.h | 21 ++-- src/pybind/mgr/volumes/fs/async_cloner.py | 7 +- 3 files changed, 144 insertions(+), 27 deletions(-) diff --git a/src/client/Client.cc b/src/client/Client.cc index 494ba2c0c20..b81759690bc 100644 --- a/src/client/Client.cc +++ b/src/client/Client.cc @@ -7991,7 +7991,7 @@ int Client::do_rename(const char *relfrom, const char *relto, const UserPerm& pe // dirs int Client::do_mkdirat(int dirfd, const char *relpath, mode_t mode, const UserPerm& perm, - std::string alternate_name) + std::string alternate_name, FSCrypt_Options fscrypt_options) { RWRef_t mref_reader(mount_state, CLIENT_MOUNTING); if (!mref_reader.is_state_satisfied()) @@ -8015,7 +8015,7 @@ int Client::do_mkdirat(int dirfd, const char *relpath, mode_t mode, const UserPe return rc; } - return _mkdir(wdr, mode, perm, 0, {}, std::move(alternate_name)); + return _mkdir(wdr, mode, perm, 0, {}, std::move(alternate_name), fscrypt_options); } int Client::mkdirs(const char *relpath, mode_t mode, const UserPerm& perms) @@ -8078,7 +8078,7 @@ int Client::mknod(const char *relpath, mode_t mode, const UserPerm& perms, dev_t // symlinks int Client::do_symlinkat(const char *target, int dirfd, const char *relpath, const UserPerm& perms, - std::string alternate_name) + std::string alternate_name, FSCrypt_Options fscrypt_options) { RWRef_t mref_reader(mount_state, CLIENT_MOUNTING); if (!mref_reader.is_state_satisfied()) { @@ -8096,7 +8096,7 @@ int Client::do_symlinkat(const char *target, int dirfd, const char *relpath, con if (int rc = get_fd_inode(dirfd, &dirinode); rc < 0) { return rc; } - return _symlink(dirinode.get(), relpath, target, perms, std::move(alternate_name)); + return _symlink(dirinode.get(), relpath, target, perms, std::move(alternate_name), 0, fscrypt_options); } int Client::readlink(const char *relpath, char *buf, loff_t size, const UserPerm& perms) @@ -10497,7 +10497,7 @@ int Client::getdir(const char *relpath, list& contents, int Client::create_and_open(int dirfd, const char *relpath, int flags, const UserPerm& perms, mode_t mode, int stripe_unit, int stripe_count, int object_size, const char *data_pool, - std::string alternate_name) { + std::string alternate_name, FSCrypt_Options fscrypt_options) { ceph_assert(ceph_mutex_is_locked_by_me(client_lock)); int cflags = ceph_flags_sys2wire(flags); tout(cct) << cflags << std::endl; @@ -10553,7 +10553,7 @@ int Client::create_and_open(int dirfd, const char *relpath, int flags, } r = _create(wdr, flags, mode, &in, &fh, stripe_unit, stripe_count, object_size, data_pool, &created, perms, - std::move(alternate_name)); + std::move(alternate_name), fscrypt_options); if (r < 0) goto out; } @@ -10583,7 +10583,8 @@ int Client::create_and_open(int dirfd, const char *relpath, int flags, int Client::do_openat(int dirfd, const char *relpath, int flags, const UserPerm& perms, mode_t mode, int stripe_unit, int stripe_count, int object_size, - const char *data_pool, std::string alternate_name) { + const char *data_pool, std::string alternate_name, + FSCrypt_Options fscrypt_options) { RWRef_t mref_reader(mount_state, CLIENT_MOUNTING); if (!mref_reader.is_state_satisfied()) { return -ENOTCONN; @@ -10598,7 +10599,7 @@ int Client::do_openat(int dirfd, const char *relpath, int flags, const UserPerm& std::scoped_lock locker(client_lock); // NEXT int r = create_and_open(dirfd, relpath, flags, perms, mode, stripe_unit, stripe_count, - object_size, data_pool, std::move(alternate_name)); + object_size, data_pool, std::move(alternate_name), fscrypt_options); tout(cct) << r << std::endl; ldout(cct, 3) << "openat exit(" << relpath << ")" << dendl; @@ -15679,7 +15680,8 @@ int Client::ll_mknodx(Inode *parent, const char *name, mode_t mode, int Client::_create(const walk_dentry_result& wdr, int flags, mode_t mode, InodeRef *inp, Fh **fhp, int stripe_unit, int stripe_count, int object_size, const char *data_pool, bool *created, - const UserPerm& perms, std::string alternate_name) + const UserPerm& perms, std::string alternate_name, + FSCrypt_Options fscrypt_options) { ldout(cct, 8) << "_create(" << *wdr.diri << " " << wdr.dname << ", 0" << oct << mode << dec << " " << perms << ")" << dendl; @@ -15718,7 +15720,14 @@ int Client::_create(const walk_dentry_result& wdr, int flags, mode_t mode, req->set_filepath(wdr.getpath()); req->set_alternate_name(alternate_name.empty() ? wdr.alternate_name : alternate_name); req->set_inode(wdr.diri); - wdr.diri->gen_inherited_fscrypt_auth(&req->fscrypt_auth); + if (fscrypt_options.fscrypt_auth.size()) + req->fscrypt_auth = fscrypt_options.fscrypt_auth; + else + wdr.diri->gen_inherited_fscrypt_auth(&req->fscrypt_auth); + + if (fscrypt_options.fscrypt_file.size()) + req->fscrypt_file = fscrypt_options.fscrypt_file; + req->head.args.open.flags = cflags | CEPH_O_CREAT; req->head.args.open.stripe_unit = stripe_unit; @@ -15774,7 +15783,7 @@ int Client::_create(const walk_dentry_result& wdr, int flags, mode_t mode, int Client::_mkdir(const walk_dentry_result& wdr, mode_t mode, const UserPerm& perm, InodeRef *inp, const std::map &metadata, - std::string alternate_name) + std::string alternate_name, FSCrypt_Options fscrypt_options) { ldout(cct, 8) << "_mkdir(" << wdr << ", 0o" << std::oct << mode << std::dec << ", uid " << perm.uid() @@ -15808,7 +15817,13 @@ int Client::_mkdir(const walk_dentry_result& wdr, mode_t mode, const UserPerm& p req->dentry_drop = CEPH_CAP_FILE_SHARED; req->dentry_unless = CEPH_CAP_FILE_EXCL; req->set_alternate_name(alternate_name.empty() ? wdr.alternate_name : alternate_name); - wdr.diri->gen_inherited_fscrypt_auth(&req->fscrypt_auth); + if (fscrypt_options.fscrypt_auth.size()) + req->fscrypt_auth = fscrypt_options.fscrypt_auth; + else + wdr.diri->gen_inherited_fscrypt_auth(&req->fscrypt_auth); + + if (fscrypt_options.fscrypt_file.size()) + req->fscrypt_file = fscrypt_options.fscrypt_file; mode |= S_IFDIR; bufferlist bl; @@ -15918,7 +15933,8 @@ int Client::ll_mkdirx(Inode *parent, const char *name, mode_t mode, Inode **out, } int Client::_symlink(Inode *dir, const char *name, const char *target, - const UserPerm& perms, std::string alternate_name, InodeRef *inp) + const UserPerm& perms, std::string alternate_name, InodeRef *inp, + FSCrypt_Options fscrypt_options) { ldout(cct, 8) << "_symlink(" << dir->ino << " " << name << ", " << target << ", uid " << perms.uid() << ", gid " << perms.gid() << ")" @@ -15946,9 +15962,16 @@ int Client::_symlink(Inode *dir, const char *name, const char *target, MetaRequest *req = new MetaRequest(CEPH_MDS_OP_SYMLINK); - wdr.diri->gen_inherited_fscrypt_auth(&req->fscrypt_auth); + if (fscrypt_options.fscrypt_auth.size()) + req->fscrypt_auth = fscrypt_options.fscrypt_auth; + else + wdr.diri->gen_inherited_fscrypt_auth(&req->fscrypt_auth); + + if (fscrypt_options.fscrypt_file.size()) + req->fscrypt_file = fscrypt_options.fscrypt_file; + auto fscrypt_ctx = fscrypt->init_ctx(req->fscrypt_auth); - if (fscrypt_ctx) { + if (fscrypt_ctx && cct->_conf.get_val("client_fscrypt_as")) { auto fscrypt_denc = fscrypt->get_fname_denc(fscrypt_ctx, nullptr, true); string enc_target; @@ -18569,7 +18592,95 @@ int Client::get_inode_flags(int fd, int* file_attr_out) { } int Client::fcopyfile(const char *spath, const char *dpath, UserPerm& perms, mode_t mode) { - return 0; + ldout(cct, 10) << "fcopyfile spath=" << spath << " dpath=" << dpath << " mode=" << mode << dendl; + + walk_dentry_result wdrsrc; + { + std::scoped_lock lock(client_lock); + if (int rc = path_walk(cwd, spath, &wdrsrc, perms, {.followsym = false}); rc < 0) { + return rc; + } + } + + auto& srcin = wdrsrc.target; + std::string alt_name = wdrsrc.alternate_name; + + FSCrypt_Options foptions; + foptions.fscrypt_auth = srcin->fscrypt_auth; + foptions.fscrypt_file = srcin->fscrypt_file; + + if (srcin->is_symlink()){ + char linkpath[4096]; + + int link_size = readlink(spath, linkpath, 4096, perms); + linkpath[link_size] = '\0'; + + int r = do_symlinkat(linkpath, CEPHFS_AT_FDCWD, dpath, perms, alt_name, foptions); + if (r < 0) { + ldout(cct, 10) << "fcopyfile could not create symlink=" << dpath << " r=" << r << dendl; + return r; + } + } else if(srcin->is_dir()) { + int r = do_mkdirat(CEPHFS_AT_FDCWD, dpath, mode, perms, alt_name, foptions); + if (r < 0) { + ldout(cct, 10) << "fcopyfile could not create dest dir=" << dpath << " r=" << r << dendl; + return r; + } + } else if(srcin->is_file()) { + int r = 0; + size_t size = srcin->size; + + int dest = do_openat(CEPHFS_AT_FDCWD, dpath, O_CREAT | O_TRUNC | O_WRONLY, perms, mode, 0,0,0, NULL, alt_name, foptions); + if (dest < 0) { + ldout(cct, 10) << "fcopyfile could not open dest file=" << dpath << " ret=" << dest << dendl; + return dest; + } + + bool need_read = true; + if (size == 0) + need_read = false; + + int src; + if (need_read) { + src = open(spath, O_RDONLY, perms, mode); + if (src < 0) { + ldout(cct, 10) << "fcopyfile could not open source file=" << spath << " ret=" << src << dendl; + close(dest); + return src; + } + + char in_buf[1048576]; + size_t off = 0; + size_t len = sizeof(in_buf) / sizeof(in_buf[0]); + + len = std::min(size, len); + + while (true) { + // include fstat here to reverify size (statx) + r = read(src, in_buf, len, off); + if (r < 0) { + ldout(cct, 10) << "fcopyfile: error reading copy data, r=" << r << dendl; + goto out; + } + + r = write(dest, in_buf, len, off); + if (r < 0) { + ldout(cct, 10) << "fcopyfile: error writing copy data, r=" << r << dendl; + goto out; + } + off = off + len; + + if (off == size) + break; + } + } + out: + close(dest); + if (need_read) + close(src); + return r; + } + return 0; } StandaloneClient::StandaloneClient(Messenger *m, MonClient *mc, diff --git a/src/client/Client.h b/src/client/Client.h index 5d8b4d54f15..9da9c086aab 100644 --- a/src/client/Client.h +++ b/src/client/Client.h @@ -995,6 +995,11 @@ public: protected: + struct FSCrypt_Options { + std::vector fscrypt_auth; + std::vector fscrypt_file; + }; + struct walk_dentry_result { DentryRef dn; InodeRef target; @@ -1075,13 +1080,14 @@ protected: int create_and_open(int dirfd, const char *relpath, int flags, const UserPerm& perms, mode_t mode, int stripe_unit, int stripe_count, int object_size, - const char *data_pool, std::string alternate_name); + const char *data_pool, std::string alternate_name, + FSCrypt_Options fscrypt_options={}); - int do_mkdirat(int dirfd, const char *relpath, mode_t mode, const UserPerm& perm, std::string alternate_name=""); + int do_mkdirat(int dirfd, const char *relpath, mode_t mode, const UserPerm& perm, std::string alternate_name="", FSCrypt_Options fscrypt_options={}); int do_rename(const char *from, const char *to, const UserPerm& perm, std::string alternate_name=""); int do_link(const char *existing, const char *newname, const UserPerm& perm, std::string alternate_name=""); - int do_symlinkat(const char *target, int dirfd, const char *linkpath, const UserPerm& perms, std::string alternate_name=""); - int do_openat(int dirfd, const char *path, int flags, const UserPerm& perms, mode_t mode, int stripe_unit, int stripe_count, int object_size, const char *data_pool, std::string alternate_name=""); + int do_symlinkat(const char *target, int dirfd, const char *linkpath, const UserPerm& perms, std::string alternate_name="", FSCrypt_Options fscrypt_options={}); + int do_openat(int dirfd, const char *path, int flags, const UserPerm& perms, mode_t mode, int stripe_unit, int stripe_count, int object_size, const char *data_pool, std::string alternate_name="", FSCrypt_Options fscrypt_options={}); struct PathWalk_ExtraOptions { bool followsym = true; @@ -1089,6 +1095,7 @@ protected: bool is_rename = false; bool require_target = true; }; + int path_walk(InodeRef dirinode, const filepath& fp, struct walk_dentry_result* result, const UserPerm& perms, const PathWalk_ExtraOptions& extra_options); int path_walk(InodeRef dirinode, const filepath& fp, InodeRef *end, const UserPerm& perms, const PathWalk_ExtraOptions& extra_options); @@ -2014,10 +2021,10 @@ private: int _rename(Inode *olddir, const char *oname, Inode *ndir, const char *nname, const UserPerm& perm, std::string alternate_name); int _mkdir(const walk_dentry_result& wdr, mode_t mode, const UserPerm& perm, InodeRef *inp = 0, const std::map &metadata={}, - std::string alternate_name=""); + std::string alternate_name="", FSCrypt_Options={}); int _rmdir(Inode *dir, const char *name, const UserPerm& perms, bool check_perms=true); int _symlink(Inode *dir, const char *name, const char *target, - const UserPerm& perms, std::string alternate_name, InodeRef *inp = 0); + const UserPerm& perms, std::string alternate_name, InodeRef *inp = 0, FSCrypt_Options fscrypt_options = {}); int _mknod(Inode *dir, const char *name, mode_t mode, dev_t rdev, const UserPerm& perms, InodeRef *inp = 0); bool make_absolute_path_string(const InodeRef& in, std::string& path); @@ -2055,7 +2062,7 @@ private: int _create(const walk_dentry_result& wdr, int flags, mode_t mode, InodeRef *inp, Fh **fhp, int stripe_unit, int stripe_count, int object_size, const char *data_pool, bool *created, const UserPerm &perms, - std::string alternate_name); + std::string alternate_name, FSCrypt_Options fscrypt_options={}); loff_t _lseek(Fh *fh, loff_t offset, int whence); int64_t _read(Fh *fh, int64_t offset, uint64_t size, bufferlist *bl, diff --git a/src/pybind/mgr/volumes/fs/async_cloner.py b/src/pybind/mgr/volumes/fs/async_cloner.py index 6fe2523ad2d..c22c6ee4816 100644 --- a/src/pybind/mgr/volumes/fs/async_cloner.py +++ b/src/pybind/mgr/volumes/fs/async_cloner.py @@ -135,22 +135,21 @@ def bulk_copy(fs_handle, source_path, dst_path, should_cancel): if stat.S_ISDIR(stx["mode"]): log.debug("cptree: (DIR) {0}".format(d_full_src)) try: - fs_handle.mkdir(d_full_dst, mo) + fs_handle.fcopyfile(d_full_src, d_full_dst, mo) except cephfs.Error as e: if not e.args[0] == errno.EEXIST: raise cptree(d_full_src, d_full_dst) elif stat.S_ISLNK(stx["mode"]): log.debug("cptree: (SYMLINK) {0}".format(d_full_src)) - target = fs_handle.readlink(d_full_src, 4096) try: - fs_handle.symlink(target[:stx["size"]], d_full_dst) + fs_handle.fcopyfile(d_full_src, d_full_dst, mo) except cephfs.Error as e: if not e.args[0] == errno.EEXIST: raise elif stat.S_ISREG(stx["mode"]): log.debug("cptree: (REG) {0}".format(d_full_src)) - copy_file(fs_handle, d_full_src, d_full_dst, mo, cancel_check=should_cancel) + fs_handle.fcopyfile(d_full_src, d_full_dst, mo) else: handled = False log.warning("cptree: (IGNORE) {0}".format(d_full_src)) -- 2.39.5