From: Yehuda Sadeh Date: Fri, 12 May 2023 17:24:31 +0000 (-0400) Subject: client,test,osdc: add beginnings of fscrypt support X-Git-Tag: testing/wip-pdonnell-testing-20251117.182723-debug~69^2~129 X-Git-Url: http://git.apps.os.sepia.ceph.com/?a=commitdiff_plain;h=e425cc3a68e34b9f21f9ebcef21c3348674860ce;p=ceph-ci.git client,test,osdc: add beginnings of fscrypt support Signed-off-by: Yehuda Sadeh --- diff --git a/src/client/CMakeLists.txt b/src/client/CMakeLists.txt index b71e641e440..2eb7b6ba7cf 100644 --- a/src/client/CMakeLists.txt +++ b/src/client/CMakeLists.txt @@ -8,7 +8,8 @@ set(libclient_srcs MetaSession.cc Trace.cc posix_acl.cc - Delegation.cc) + Delegation.cc + FSCrypt.cc) add_library(client STATIC ${libclient_srcs}) target_link_libraries(client legacy-option-headers diff --git a/src/client/Client.cc b/src/client/Client.cc index 11429b7eb35..38d32877d2b 100644 --- a/src/client/Client.cc +++ b/src/client/Client.cc @@ -646,6 +646,7 @@ void Client::_pre_init() objecter_finisher.start(); filer.reset(new Filer(objecter, &objecter_finisher)); + fscrypt.reset(new FSCrypt(cct)); objectcacher->start(); } @@ -1126,6 +1127,7 @@ Inode * Client::add_update_inode(InodeStat *st, utime_t from, in->snap_btime = st->snap_btime; in->snap_metadata = st->snap_metadata; in->fscrypt_auth = st->fscrypt_auth; + in->fscrypt_ctx = in->init_fscrypt_ctx(fscrypt.get()); need_snapdir_attr_refresh = true; } @@ -1143,7 +1145,9 @@ Inode * Client::add_update_inode(InodeStat *st, utime_t from, if (new_version || (new_issued & (CEPH_CAP_ANY_FILE_RD | CEPH_CAP_ANY_FILE_WR))) { in->layout = st->layout; - in->fscrypt_file = st->fscrypt_file; + if (st->fscrypt_file.size() >= sizeof(uint64_t)) { + in->fscrypt_file = st->fscrypt_file; + } update_inode_file_size(in, issued, st->size, st->truncate_seq, st->truncate_size); } @@ -1280,7 +1284,8 @@ Dentry *Client::insert_dentry_inode(Dir *dir, const string& dname, LeaseStat *dl } Inode *diri = dir->parent_inode; clear_dir_complete_and_ordered(diri, false); - dn = link(dir, dname, in, dn); +#warning revisit nullopt here + dn = link(dir, dname, std::nullopt, in, dn); if (old_dentry) { dn->is_renaming = false; @@ -1533,10 +1538,21 @@ void Client::insert_readdir_results(MetaRequest *request, MetaSession *session, string readdir_start = dirp->last_name; ceph_assert(!readdir_start.empty() || readdir_offset == 2); + string readdir_start_enc; + + auto fscrypt_denc = fscrypt->get_fname_denc(diri->fscrypt_ctx, &diri->fscrypt_key_validator, true); + if (!readdir_start.empty() && fscrypt_denc) { + string alt; + int r = fscrypt_denc->get_encrypted_fname(readdir_start, &readdir_start_enc, &alt); + if (r < 0) { + ldout(cct, 0) << __FILE__ << ":" << __LINE__ << ": failed to decrypt filename (r=" << r << ")" << dendl; + } + } + unsigned last_hash = 0; if (hash_order) { if (!readdir_start.empty()) { - last_hash = ceph_frag_value(diri->hash_dentry_name(readdir_start)); + last_hash = ceph_frag_value(diri->hash_dentry_name((readdir_start_enc.empty() ? readdir_start : readdir_start_enc))); } else if (flags & CEPH_READDIR_OFFSET_HASH) { /* mds understands offset_hash */ last_hash = offset_hash; @@ -1573,17 +1589,31 @@ void Client::insert_readdir_results(MetaRequest *request, MetaSession *session, _readdir_drop_dirp_buffer(dirp); dirp->buffer.reserve(numdn); + string orig_dname; + std::optional enc_name; string dname; LeaseStat dlease; + for (unsigned i=0; iget_decrypted_fname(orig_dname, dlease.alternate_name, &dname); + if (r < 0) { + ldout(cct, 0) << __FILE__ << ":" << __LINE__ << ": failed to decrypt filename (r=" << r << ")" << dendl; + dname = orig_dname; + } + } else { + dname = orig_dname; + } Inode *in = add_update_inode(&ist, request->sent_stamp, session, - request->perms); + request->perms); auto *effective_dir = dir; auto *effective_diri = diri; @@ -1600,7 +1630,7 @@ void Client::insert_readdir_results(MetaRequest *request, MetaSession *session, if (olddn->inode != in) { // replace incorrect dentry unlink(olddn, true, true); // keep dir, dentry - dn = link(effective_dir, dname, in, olddn); + dn = link(effective_dir, dname, enc_name, in, olddn); ceph_assert(dn == olddn); } else { // keep existing dn @@ -1608,24 +1638,24 @@ void Client::insert_readdir_results(MetaRequest *request, MetaSession *session, touch_dn(dn); } } else { - // new dn - dn = link(effective_dir, dname, in, NULL); + // new dn + dn = link(effective_dir, dname, enc_name, in, NULL); } update_dentry_lease(dn, &dlease, request->sent_stamp, session); if (hash_order) { - unsigned hash = ceph_frag_value(effective_diri->hash_dentry_name(dname)); - if (hash != last_hash) - readdir_offset = 2; - last_hash = hash; - dn->offset = dir_result_t::make_fpos(hash, readdir_offset++, true); + unsigned hash = ceph_frag_value(effective_diri->hash_dentry_name(orig_dname)); + if (hash != last_hash) + readdir_offset = 2; + last_hash = hash; + dn->offset = dir_result_t::make_fpos(hash, readdir_offset++, true); } else { - dn->offset = dir_result_t::make_fpos(fg, readdir_offset++, false); + dn->offset = dir_result_t::make_fpos(fg, readdir_offset++, false); } // add to readdir cache if (!snapdiff_req && dirp->release_count == effective_diri->dir_release_count && - dirp->ordered_count == effective_diri->dir_ordered_count && + dirp->ordered_count == effective_diri->dir_ordered_count && dirp->start_shared_gen == effective_diri->shared_gen) { if (dirp->cache_index == effective_dir->readdir_cache.size()) { if (i == 0) { @@ -1649,7 +1679,7 @@ void Client::insert_readdir_results(MetaRequest *request, MetaSession *session, } if (numdn > 0) - dirp->last_name = dname; + dirp->last_name = orig_dname; if (end) dirp->next_offset = 2; else @@ -1732,15 +1762,30 @@ Inode* Client::insert_trace(MetaRequest *request, MetaSession *session) InodeStat dirst; DirStat dst; string dname; + string enc_name; LeaseStat dlease; InodeStat ist; + Inode *diri = NULL; + if (reply->head.is_dentry) { dirst.decode(p, features); ldout(cct, 25) << "insert_trace: " << dirst << dendl; dst.decode(p, features); decode(dname, p); dlease.decode(p, features); + + diri = add_update_inode(&dirst, request->sent_stamp, session, + request->perms); + auto fscrypt_denc = fscrypt->get_fname_denc(diri->fscrypt_ctx, &diri->fscrypt_key_validator, true); + if (fscrypt_denc) { + enc_name = dname; + int r = fscrypt_denc->get_decrypted_fname(enc_name, dlease.alternate_name, &dname); + if (r < 0) { + ldout(cct, 0) << __FILE__ << ":" << __LINE__ << ": failed to decrypt filename (r=" << r << ")" << dendl; + dname = enc_name; + } + } } Inode *in = 0; @@ -1764,12 +1809,19 @@ Inode* Client::insert_trace(MetaRequest *request, MetaSession *session) ldout(cct, 20) << __func__ << " subv_metric adding " << in->ino << "-" << ist.subvolume_id << dendl; subvolume_tracker->add_inode(in->ino, ist.subvolume_id); } + + auto fscrypt_denc = fscrypt->get_fname_denc(in->fscrypt_ctx, &in->fscrypt_key_validator, true); + if (fscrypt_denc && in->is_symlink()) { + string slname; + int ret = fscrypt_denc->get_decrypted_symlink(in->symlink, &slname); + if (ret < 0) { + ldout(cct, 0) << __FILE__ << ":" << __LINE__ << ": failed to decrypt symlink (r=" << ret << ")" << dendl; + } + in->symlink_plain = slname; + } } - Inode *diri = NULL; if (reply->head.is_dentry) { - diri = add_update_inode(&dirst, request->sent_stamp, session, - request->perms); mds_rank_t from_mds = mds_rank_t(reply->get_source().num()); update_dir_dist(diri, &dst, from_mds); // dir stat info is attached to .. @@ -1792,7 +1844,8 @@ Inode* Client::insert_trace(MetaRequest *request, MetaSession *session) if (dlease.duration_ms > 0) { if (!dn) { Dir *dir = diri->open_dir(); - dn = link(dir, dname, NULL, NULL); +#warning revisit nullopt here + dn = link(dir, dname, std::nullopt, NULL, NULL); } update_dentry_lease(dn, &dlease, request->sent_stamp, session); } @@ -1809,6 +1862,16 @@ Inode* Client::insert_trace(MetaRequest *request, MetaSession *session) string dname = request->path.last_dentry(); + auto fscrypt_denc = fscrypt->get_fname_denc(diri->fscrypt_ctx, &diri->fscrypt_key_validator, true); + if (fscrypt_denc) { + enc_name = dname; + int r = fscrypt_denc->get_decrypted_fname(enc_name, dlease.alternate_name, &dname); + if (r < 0) { + ldout(cct, 0) << __FILE__ << ":" << __LINE__ << ": failed to decrypt filename (r=" << r << ")" << dendl; + dname = enc_name; + } + } + LeaseStat dlease; dlease.duration_ms = 0; @@ -3662,11 +3725,12 @@ void Client::close_dir(Dir *dir) * leave dn set to default NULL unless you're trying to add * a new inode to a pre-created Dentry */ -Dentry* Client::link(Dir *dir, const string& name, Inode *in, Dentry *dn) +Dentry* Client::link(Dir *dir, const string& name, std::optional enc_name, Inode *in, Dentry *dn) { if (!dn) { // create a new Dentry dn = new Dentry(dir, name); + dn->enc_name = enc_name; lru.lru_insert_mid(dn); // mid or top? @@ -3856,8 +3920,12 @@ int Client::get_caps(Fh *fh, int need, int want, int *phave, loff_t endoff) if ((endoff >= (loff_t)in->max_size || endoff > (loff_t)(in->size << 1)) && endoff > (loff_t)in->wanted_max_size) { - ldout(cct, 10) << "wanted_max_size " << in->wanted_max_size << " -> " << endoff << dendl; - in->wanted_max_size = endoff; + ldout(cct, 10) << "wanted_max_size " << in->wanted_max_size << " -> " << endoff << dendl; + uint64_t want = endoff; + if (in->fscrypt_auth.size()) { + want = fscrypt_block_start(endoff + FSCRYPT_BLOCK_SIZE - 1); + } + in->wanted_max_size = want; } if (in->wanted_max_size > in->max_size && in->wanted_max_size > in->requested_max_size) @@ -3905,9 +3973,10 @@ int Client::get_caps(Fh *fh, int need, int want, int *phave, loff_t endoff) } if ((need & CEPH_CAP_FILE_WR) && - ((in->auth_cap && in->auth_cap->session->readonly) || - // userland clients are only allowed to read if fscrypt enabled - in->is_fscrypt_enabled())) + ((in->auth_cap && in->auth_cap->session->readonly) || + // userland clients are only allowed to read if fscrypt enabled but no fscrypt ctx exists + // (is locked) + (in->is_fscrypt_enabled() && !in->fscrypt_ctx))) return -EROFS; if (in->flags & I_CAP_DROPPED) { @@ -5727,8 +5796,7 @@ void Client::handle_cap_trunc(MetaSession *session, Inode *in, const MConstRefget_size(); if (in->is_fscrypt_enabled()) { - size = std::stoll(std::string(std::rbegin(m->fscrypt_file), - std::rend(m->fscrypt_file))); + size = *(ceph_le64 *)in->fscrypt_file.data(); } ldout(cct, 10) << __func__ << " on ino " << *in << " size " << in->size << " -> " << m->get_size() @@ -7516,6 +7584,50 @@ void Client::renew_caps(MetaSession *session) session->con->send_message2(std::move(m)); } +int Client::_prepare_req_path(Inode *dir, MetaRequest *req, filepath& path, const char *name, + bool set_filepath, Dentry **pdn) +{ + dir->make_nosnap_relative_path(path); + + std::optional enc_name; + std::optional alt_name; + const char *plain_name = name; + auto fscrypt_denc = fscrypt->get_fname_denc(dir->fscrypt_ctx, &dir->fscrypt_key_validator, true); + if (fscrypt_denc) { + string _enc_name; + string _alt_name; + int r = fscrypt_denc->get_encrypted_fname(name, &_enc_name, &_alt_name); + if (r < 0) { + ldout(cct, 0) << __FILE__ << ":" << __LINE__ << ": failed to encrypt filename" << dendl; + return r; + } + path.push_dentry(_enc_name); + enc_name = std::move(_enc_name); + alt_name = std::move(_alt_name); + } else { + path.push_dentry(plain_name); + } + + if (set_filepath) { + req->set_filepath(path); + if (alt_name) { + req->set_alternate_name(*alt_name); + } + } + + if (pdn) { + *pdn = get_or_create(dir, plain_name, enc_name); + if (alt_name) { + if (alt_name->size() > 0) { + ldout(cct, 20) << __func__ << " " << *dir << " alt_name=" << fscrypt_hex_str(alt_name->c_str(), alt_name->size()) << dendl; + } + (*pdn)->alternate_name = *alt_name; + } + } + + return 0; +} + // =============================================================== // high level (POSIXy) interface @@ -7526,9 +7638,13 @@ int Client::_do_lookup(const InodeRef& dir, const string& name, int mask, int op = dir->snapid == CEPH_SNAPDIR ? CEPH_MDS_OP_LOOKUPSNAP : CEPH_MDS_OP_LOOKUP; MetaRequest *req = new MetaRequest(op); filepath path; - dir->make_nosnap_relative_path(path); - path.push_dentry(name); - req->set_filepath(path); + + int r = _prepare_req_path(dir, req, path, name.c_str(), true, nullptr); + if (r < 0) { + delete req; + return r; + } + req->set_inode(dir); if (cct->_conf->client_debug_getattr_caps && op == CEPH_MDS_OP_LOOKUP) mask |= DEBUG_GETATTR_CAPS; @@ -7536,8 +7652,21 @@ int Client::_do_lookup(const InodeRef& dir, const string& name, int mask, ldout(cct, 10) << __func__ << " on " << path << dendl; - int r = make_request(req, perms, target); + r = make_request(req, perms, target); ldout(cct, 10) << __func__ << " res is " << r << dendl; + + if (r == 0 && (*target)->is_symlink()) { + auto fscrypt_denc = fscrypt->get_fname_denc(dir->fscrypt_ctx, &dir->fscrypt_key_validator, true); + if (fscrypt_denc) { + string slname; + int ret = fscrypt_denc->get_decrypted_symlink((*target)->symlink, &slname); + if (ret < 0) { + ldout(cct, 0) << __FILE__ << ":" << __LINE__ << ": failed to decrypt symlink (r=" << ret << ")" << dendl; + } + (*target)->symlink_plain = slname; + } + } + return r; } @@ -7711,16 +7840,21 @@ relookup: return r; } -Dentry *Client::get_or_create(Inode *dir, const std::string& name) +Dentry *Client::get_or_create(Inode *dir, const std::string& plain_name) { // lookup - ldout(cct, 20) << __func__ << " " << *dir << " name " << name << dendl; + ldout(cct, 20) << __func__ << " " << *dir << " plain_name " << plain_name << " enc_name=" << enc_name.value_or(string()) << dendl; dir->open_dir(); - auto it = dir->dir->dentries.find(name); - if (it != dir->dir->dentries.end()) - return it->second; - else // otherwise link up a new one - return link(dir->dir, name, NULL, NULL); + auto it = dir->dir->dentries.find(plain_name); + if (it != dir->dir->dentries.end()) { + auto dn = it->second; + if (!dn->enc_name && enc_name) { + dn->enc_name = enc_name; + } + return dn; + } else { // otherwise link up a new one + return link(dir->dir, plain_name, enc_name, NULL, NULL); + } } int Client::walk(std::string_view path, walk_dentry_result* wdr, const UserPerm& perms, bool followsym) @@ -7819,8 +7953,15 @@ int Client::path_walk(InodeRef dirinode, const filepath& origpath, dn = get_or_create(diri.get(), dname.c_str()); /* Get extra requested caps on the last component */ - if (i == (path.depth() - 1)) + if (i == (path.depth() - 1)) { caps |= extra_options.mask; + if (diri->is_fscrypt_enabled()) { + if (extra_options.mask & CEPH_FILE_MODE_WR) { + caps |= CEPH_FILE_MODE_RD; + } + } + } + int r = _lookup(diri, dname, alternate_name, caps, &next, perms, extra_options.is_rename); if (r == -ENOENT && i == (path.depth()-1) && !extra_options.require_target) { target = InodeRef(); @@ -7840,10 +7981,11 @@ int Client::path_walk(InodeRef dirinode, const filepath& origpath, goto out; } + const char *slink = (next->symlink_plain.empty() ? next->symlink.c_str() : next->symlink_plain.c_str()); if (i < path.depth() - 1) { // dir symlink // replace consumed components of path with symlink dir target - filepath resolved(next->symlink.c_str()); + filepath resolved(slink); resolved.append(path.postfixpath(i + 1)); path = resolved; i = 0; @@ -7854,11 +7996,21 @@ int Client::path_walk(InodeRef dirinode, const filepath& origpath, } else if (extra_options.followsym) { if (next->symlink[0] == '/') { path = next->symlink.c_str(); +#if 0 +OLD + if (slink[0] == '/') { + cur = root; + } + continue; + } else if (followsym) { + if (slink[0] == '/') { + path = slink; +#endif i = 0; // reset position diri = root; } else { - filepath more(next->symlink.c_str()); + filepath more(slink); // we need to remove the symlink component from off of the path // before adding the target that the symlink points to. remain // at the same position in the path. @@ -8103,7 +8255,20 @@ int Client::_readlink(const InodeRef& diri, const char* relpath, char *buf, size int r = in->symlink.length(); if (r > (int)size) r = size; - memcpy(buf, in->symlink.c_str(), r); + + auto fscrypt_denc = fscrypt->get_fname_denc(in->fscrypt_ctx, &in->fscrypt_key_validator, true); + if (fscrypt_denc && in->symlink_plain.empty()) { + string dname; + int ret = fscrypt_denc->get_decrypted_symlink(in->symlink, &dname); + if (ret < 0) { + ldout(cct, 0) << __FILE__ << ":" << __LINE__ << ": failed to decrypt symlink (r=" << ret << ")" << dendl; + } + memcpy(buf, dname.c_str(), dname.size()); + r = dname.size(); + } else { + memcpy(buf, in->symlink_plain.c_str(), r); + } + return r; } @@ -8212,6 +8377,8 @@ int Client::_do_setattr(Inode *in, struct ceph_statx *stx, int mask, size_t auxsize = 0; filepath path; MetaRequest *req; + std::vector alt_aux; + std::vector *paux = aux; if (aux) auxsize = aux->size(); @@ -8360,6 +8527,7 @@ int Client::_do_setattr(Inode *in, struct ceph_statx *stx, int mask, if (!do_sync && in->caps_issued_mask(CEPH_CAP_AUTH_EXCL)) { in->ctime = ceph_clock_now(); in->fscrypt_auth = *aux; + in->fscrypt_ctx = in->init_fscrypt_ctx(fscrypt.get()); in->mark_caps_dirty(CEPH_CAP_AUTH_EXCL); mask &= ~CEPH_SETATTR_FSCRYPT_AUTH; } else if (!in->caps_issued_mask(CEPH_CAP_AUTH_SHARED) || @@ -8371,18 +8539,34 @@ int Client::_do_setattr(Inode *in, struct ceph_statx *stx, int mask, } if (mask & CEPH_SETATTR_SIZE) { - if ((uint64_t)stx->stx_size >= mdsmap->get_max_filesize()) { + auto stx_size = stx->stx_size; + + if (in->fscrypt_ctx && + (!(mask & CEPH_SETATTR_FSCRYPT_FILE))) { + stx_size = fscrypt_next_block_start(stx_size); + ldout(cct,10) << "fscrypt: set file size: orig stx_size=" << stx->stx_size <<" new stx_size=" << stx_size << dendl; + + alt_aux.resize(sizeof(stx->stx_size)); + memcpy(alt_aux.data(), &stx->stx_size, sizeof(stx->stx_size)); + paux = &alt_aux; + + mask |= CEPH_SETATTR_FSCRYPT_FILE; + } + + if ((uint64_t)stx_size >= mdsmap->get_max_filesize()) { //too big! - ldout(cct,10) << "unable to set size to " << stx->stx_size << ". Too large!" << dendl; + ldout(cct,10) << "unable to set size to " << stx_size << ". Too large!" << dendl; return -EFBIG; } - ldout(cct,10) << "changing size to " << stx->stx_size << dendl; + ldout(cct,10) << "changing size to " << stx_size << dendl; if (!do_sync && in->caps_issued_mask(CEPH_CAP_FILE_EXCL) && !(mask & CEPH_SETATTR_KILL_SGUID) && - stx->stx_size >= in->size) { - if (stx->stx_size > in->size) { - in->size = in->reported_size = stx->stx_size; + stx_size >= in->size) { + if (stx_size > in->size) { + in->size = in->reported_size = stx_size; + in->cap_dirtier_uid = perms.uid(); + in->cap_dirtier_gid = perms.gid(); in->mark_caps_dirty(CEPH_CAP_FILE_EXCL); mask &= ~(CEPH_SETATTR_SIZE); mask |= CEPH_SETATTR_MTIME; @@ -8391,7 +8575,7 @@ int Client::_do_setattr(Inode *in, struct ceph_statx *stx, int mask, mask &= ~(CEPH_SETATTR_SIZE); } } else { - args.setattr.size = stx->stx_size; + args.setattr.size = stx_size; inode_drop |= CEPH_CAP_FILE_SHARED | CEPH_CAP_FILE_RD | CEPH_CAP_FILE_WR; } @@ -8403,11 +8587,15 @@ int Client::_do_setattr(Inode *in, struct ceph_statx *stx, int mask, if (!do_sync && in->caps_issued_mask(CEPH_CAP_FILE_EXCL)) { in->ctime = ceph_clock_now(); - in->fscrypt_file = *aux; + in->cap_dirtier_uid = perms.uid(); + in->cap_dirtier_gid = perms.gid(); + if (paux) { + in->fscrypt_file = *paux; + } in->mark_caps_dirty(CEPH_CAP_FILE_EXCL); mask &= ~CEPH_SETATTR_FSCRYPT_FILE; } else if (!in->caps_issued_mask(CEPH_CAP_FILE_SHARED) || - in->fscrypt_file != *aux) { + (paux && in->fscrypt_file != *paux)) { inode_drop |= CEPH_CAP_FILE_SHARED | CEPH_CAP_FILE_RD | CEPH_CAP_FILE_WR; } else { mask &= ~CEPH_SETATTR_FSCRYPT_FILE; @@ -8482,8 +8670,8 @@ int Client::_do_setattr(Inode *in, struct ceph_statx *stx, int mask, req->inode_drop = inode_drop; if (mask & CEPH_SETATTR_FSCRYPT_AUTH) { req->fscrypt_auth = *aux; - } else if (mask & CEPH_SETATTR_FSCRYPT_FILE) { - req->fscrypt_file = *aux; + } else if (mask & CEPH_SETATTR_FSCRYPT_FILE && paux) { + req->fscrypt_file = *paux; } req->head.args.setattr.mask = mask; req->regetattr_mask = mask; @@ -8790,9 +8978,9 @@ int Client::fill_stat(Inode *in, struct stat *st, frag_info_t *dirstat, nest_inf st->st_blocks = 1; #endif } else { - st->st_size = in->size; + st->st_size = in->effective_size(); #ifndef _WIN32 - st->st_blocks = (in->size + 511) >> 9; + st->st_blocks = (in->effective_size() + 511) >> 9; #endif } #ifndef _WIN32 @@ -8878,7 +9066,7 @@ void Client::fill_statx(Inode *in, unsigned int mask, struct ceph_statx *stx) } stx->stx_blocks = 1; } else { - stx->stx_size = in->size; + stx->stx_size = in->effective_size(); stx->stx_blocks = (in->size + 511) >> 9; } stx->stx_mask |= (CEPH_STATX_ATIME|CEPH_STATX_MTIME| @@ -9570,6 +9758,7 @@ int Client::_readdir_cache_cb(dir_result_t *dirp, add_dirent_cb_t cb, void *p, dirp->offset, dentry_off_lt()); string dn_name; + std::optional enc_name; for (unsigned idx = pd - dir->readdir_cache.begin(); idx < dir->readdir_cache.size(); ++idx) { @@ -9620,13 +9809,14 @@ int Client::_readdir_cache_cb(dir_result_t *dirp, add_dirent_cb_t cb, void *p, } dn_name = dn->name; // fill in name while we have lock + enc_name = dn->enc_name; // fill in name while we have lock // the content of readdir_cache may change after unlocking client_lock.unlock(); r = cb(p, &de, &stx, next_off, in); // _next_ offset client_lock.lock(); ldout(cct, 15) << " de " << de.d_name << " off " << hex << dn->offset << dec - << " = " << r << dendl; + << " idx " << idx << " cache.size=" << dir->readdir_cache.size() << " = " << r << dendl; if (r < 0) { return r; } @@ -9636,7 +9826,7 @@ int Client::_readdir_cache_cb(dir_result_t *dirp, add_dirent_cb_t cb, void *p, dirp->next_offset = 2; else dirp->next_offset = dirp->offset_low(); - dirp->last_name = dn_name; // we successfully returned this one; update! + dirp->last_name = (enc_name ? *enc_name : dn_name); // we successfully returned this one; update! dirp->release_count = 0; // last_name no longer match cache index if (r > 0) return r; @@ -9794,6 +9984,12 @@ int Client::_readdir_r_cb(int op, << dirp->inode->is_complete_and_ordered() << " issued " << ccap_string(dirp->inode->caps_issued()) << dendl; + + if (dirp->inode->fscrypt_key_validator && + !dirp->inode->fscrypt_key_validator->is_valid()) { + clear_dir_complete_and_ordered(dirp->inode.get(), true); + } + if (!bypass_cache && dirp->inode->snapid != CEPH_SNAPDIR && dirp->inode->is_complete_and_ordered() && @@ -10330,6 +10526,12 @@ int Client::create_and_open(int dirfd, const char *relpath, int flags, return r; } + if (dirinode->is_fscrypt_enabled()) { + if (mask & CEPH_FILE_MODE_WR) { + mask |= CEPH_FILE_MODE_RD; + } + } + walk_dentry_result wdr; bool require_target = !(flags & O_CREAT); r = path_walk(dirinode, path, &wdr, perms, {.followsym = followsym, .mask = (unsigned)mask, .require_target = require_target}); @@ -10634,6 +10836,12 @@ int Client::_open(const InodeRef& in, int flags, mode_t mode, Fh **fhp, cflags |= CEPH_O_LAZY; int cmode = ceph_flags_to_mode(cflags); + + if (in->fscrypt_ctx && + cmode & CEPH_FILE_MODE_WR) { + cmode |= CEPH_FILE_MODE_RD; + } + int want = ceph_caps_for_mode(cmode); int result = 0; @@ -10859,12 +11067,12 @@ loff_t Client::_lseek(Fh *f, loff_t offset, int whence) break; case SEEK_END: - pos = in->size + offset; + pos = in->effective_size() + offset; break; #ifdef SEEK_DATA case SEEK_DATA: - if (offset < 0 || static_cast(offset) >= in->size) + if (offset < 0 || static_cast(offset) >= in->effective_size()) return -ENXIO; pos = offset; break; @@ -10872,9 +11080,9 @@ loff_t Client::_lseek(Fh *f, loff_t offset, int whence) #ifdef SEEK_HOLE case SEEK_HOLE: - if (offset < 0 || static_cast(offset) >= in->size) + if (offset < 0 || static_cast(offset) >= in->effective_size()) return -ENXIO; - pos = in->size; + pos = in->effective_size(); break; #endif @@ -11146,6 +11354,8 @@ int64_t Client::_read(Fh *f, int64_t offset, uint64_t size, bufferlist *bl, utime_t start = mono_clock_now(); CRF_iofinish *crf_iofinish = nullptr; + ldout(cct, 10) << __func__ << " " << *in << " " << offset << "~" << size << dendl; + if ((f->mode & CEPH_FILE_MODE_RD) == 0) return -EBADF; //bool lazy = f->mode == CEPH_FILE_MODE_LAZY; @@ -11184,8 +11394,8 @@ retry: if (in->inline_version < CEPH_INLINE_NONE) { uint32_t len = in->inline_data.length(); uint64_t endoff = offset + size; - if (endoff > in->size) - endoff = in->size; + if (endoff > in->effective_size()) + endoff = in->effective_size(); if (offset < len) { if (endoff <= len) { @@ -11368,7 +11578,7 @@ void Client::C_Readahead::finish(int r) { void Client::do_readahead(Fh *f, Inode *in, uint64_t off, uint64_t len) { if(f->readahead.get_min_readahead_size() > 0) { - pair readahead_extent = f->readahead.update(off, len, in->size); + pair readahead_extent = f->readahead.update(off, len, in->effective_size()); if (readahead_extent.second > 0) { ldout(cct, 20) << "readahead " << readahead_extent.first << "~" << readahead_extent.second << " (caller wants " << off << "~" << len << ")" << dendl; @@ -11389,6 +11599,18 @@ void Client::do_readahead(Fh *f, Inode *in, uint64_t off, uint64_t len) void Client::C_Read_Async_Finisher::finish(int r) { + clnt->client_lock.lock(); + + if (denc && r >= 0) { + std::vector holes; + r = denc->decrypt_bl(off, len, read_start, holes, bl); + if (r < 0) { + // ldout(cct, 20) << __func__ << "(): failed to decrypt buffer: r=" << r << dendl; + } else { + r = bl->length(); + } + } + // Do read ahead as long as we aren't completing with 0 bytes if (r != 0) clnt->do_readahead(f, in, off, len); @@ -11407,17 +11629,29 @@ int Client::_read_async(Fh *f, uint64_t off, uint64_t len, bufferlist *bl, C_SaferCond *io_finish_cond = nullptr; ldout(cct, 10) << __func__ << " " << *in << " " << off << "~" << len << dendl; + + uint64_t read_start; + uint64_t read_len; + + FSCryptFDataDencRef fscrypt_denc; + fscrypt->prepare_data_read(in->fscrypt_ctx, + &in->fscrypt_key_validator, + off, len, in->size, + &read_start, &read_len, + &fscrypt_denc); // get Fc cap ref before commencing read get_cap_ref(in, CEPH_CAP_FILE_CACHE); + auto effective_size = in->effective_size(); if (onfinish != nullptr) { - io_finish.reset(new C_Read_Async_Finisher(this, onfinish, f, in, - f->pos, off, len)); + io_finish.reset(new C_Read_Async_Finisher(this, onfinish, f, in, bl, + f->pos, off, len, + fscrypt_denc, read_start, read_len)); } // trim read based on file size? - if ((off >= in->size) || (len == 0)) { + if ((off >= effective_size) || (len == 0)) { // read is requested at the EOF or the read len is zero, therefore release // Fc cap first before proceeding further put_cap_ref(in, CEPH_CAP_FILE_CACHE); @@ -11443,11 +11677,14 @@ int Client::_read_async(Fh *f, uint64_t off, uint64_t len, bufferlist *bl, len = in->size - off; } + auto target_len = std::min(len, effective_size - off); + ldout(cct, 10) << " min_bytes=" << f->readahead.get_min_readahead_size() << " max_bytes=" << f->readahead.get_max_readahead_size() << " max_periods=" << conf->client_readahead_max_periods << dendl; // read (and possibly block) + // int r = 0; if (onfinish == nullptr) { io_finish_cond = new C_SaferCond("Client::_read_async flock"); @@ -11455,9 +11692,11 @@ int Client::_read_async(Fh *f, uint64_t off, uint64_t len, bufferlist *bl, } auto start_time = mono_clock_now(); - r = objectcacher->file_read(&in->oset, &in->layout, in->snapid, - off, len, bl, 0, io_finish.get()); + std::vector holes; + r = objectcacher->file_read_ex(&in->oset, &in->layout, in->snapid, + read_start, read_len, bl, 0, &holes, io_finish.get()); + if (onfinish != nullptr) { // put the cap ref since we're releasing C_Read_Async_Finisher put_cap_ref(in, CEPH_CAP_FILE_CACHE); @@ -11477,8 +11716,22 @@ int Client::_read_async(Fh *f, uint64_t off, uint64_t len, bufferlist *bl, if (r == 0) { client_lock.unlock(); r = io_finish_cond->wait(); + client_lock.lock(); put_cap_ref(in, CEPH_CAP_FILE_CACHE); + } + + if (r >= 0) { + if (fscrypt_denc) { + r = fscrypt_denc->decrypt_bl(off, target_len, read_start, holes, bl); + if (r < 0) { + ldout(cct, 20) << __func__ << "(): failed to decrypt buffer: r=" << r << dendl; + return r; + } + } + + r = bl->length(); + update_read_io_size(bl->length()); subvolume_tracker->add_metric(in->ino, SimpleIOMetric{false, mono_clock_now() - start_time, bl->length()}); } else { @@ -11496,10 +11749,33 @@ int Client::_read_sync(Fh *f, uint64_t off, uint64_t len, bufferlist *bl, ceph_assert(ceph_mutex_is_locked_by_me(client_lock)); Inode *in = f->inode.get(); - uint64_t pos = off; - int left = len; + + auto effective_size = in->effective_size(); + + // trim read based on file size? + if (off >= in->effective_size()) + return 0; + if (len == 0) + return 0; + + auto target_len = std::min(len, effective_size - off); + uint64_t read_start; + uint64_t read_len; + + FSCryptFDataDencRef fscrypt_denc; + fscrypt->prepare_data_read(in->fscrypt_ctx, + &in->fscrypt_key_validator, + off, len, in->size, + &read_start, &read_len, + &fscrypt_denc); + + uint64_t pos = read_start; + int left = read_len; int read = 0; + bufferlist encbl; + bufferlist *pbl = (fscrypt_denc ? &encbl : bl); + ldout(cct, 10) << __func__ << " " << *in << " " << off << "~" << len << dendl; // 0 success, 1 continue and < 0 error happen. @@ -11518,18 +11794,20 @@ int Client::_read_sync(Fh *f, uint64_t off, uint64_t len, bufferlist *bl, read += r; pos += r; left -= r; - bl->claim_append(tbl); + pbl->claim_append(tbl); } + auto effective_size = (fscrypt_denc ? in->effective_size() : in->size); + // short read? if (r >= 0 && r < wanted) { - if (pos < in->size) { + if (pos < effective_size) { // zero up to known EOF - int64_t some = in->size - pos; + int64_t some = effective_size - pos; if (some > left) some = left; auto z = buffer::ptr_node::create(some); z->zero(); - bl->push_back(std::move(z)); + pbl->push_back(std::move(z)); read += some; pos += some; left -= some; @@ -11543,23 +11821,40 @@ int Client::_read_sync(Fh *f, uint64_t off, uint64_t len, bufferlist *bl, return 1; }; + int r = 0; + while (left > 0) { C_SaferCond onfinish("Client::_read_sync flock"); bufferlist tbl; int wanted = left; +#warning read holes filer->read_trunc(in->ino, &in->layout, in->snapid, pos, left, &tbl, 0, in->truncate_size, in->truncate_seq, &onfinish); +#warning implement file read here client_lock.unlock(); - int r = wait_and_copy(onfinish, tbl, wanted); + r = wait_and_copy(onfinish, tbl, wanted); client_lock.lock(); if (!r) - return read; + break; if (r < 0) return r; } + + if (r >= 0) { + if (fscrypt_denc) { + std::vector holes; + r = fscrypt_denc->decrypt_bl(off, target_len, read_start, holes, pbl); + if (r < 0) { + ldout(cct, 20) << __func__ << "(): failed to decrypt buffer: r=" << r << dendl; + } + } + + read = pbl->length(); + bl->claim_append(*pbl); + } return read; } @@ -11682,7 +11977,9 @@ int Client::_preadv_pwritev(int fd, const struct iovec *iov, int iovcnt, } int64_t Client::_write_success(Fh *f, utime_t start, uint64_t fpos, - int64_t offset, uint64_t size, Inode *in) + int64_t request_offset, uint64_t request_size, + int64_t offset, uint64_t size, Inode *in, + bool encrypted) { utime_t lat; uint64_t totalwritten; @@ -11703,11 +12000,16 @@ int64_t Client::_write_success(Fh *f, utime_t start, uint64_t fpos, unlock_fh_pos(f); } totalwritten = size; - r = (int64_t)totalwritten; + r = (int64_t)request_size; // extend file? - if (totalwritten + offset > in->size) { - in->size = totalwritten + offset; + if (request_size + request_offset > in->effective_size()) { + if (encrypted) { + in->set_effective_size(request_size + request_offset); + in->mark_caps_dirty(CEPH_CAP_FILE_EXCL); + } + ldout(cct, 7) << "in->effective_size()=" << in->effective_size() << dendl; + in->size = offset + size; in->mark_caps_dirty(CEPH_CAP_FILE_WR); if (is_quota_bytes_approaching(in, f->actor_perms)) { @@ -11716,9 +12018,9 @@ int64_t Client::_write_success(Fh *f, utime_t start, uint64_t fpos, check_caps(in, 0); } - ldout(cct, 7) << "wrote to " << totalwritten+offset << ", extending file size" << dendl; + ldout(cct, 7) << "wrote to " << totalwritten+offset << ", effective size " << request_size + request_offset << ", extending file size" << dendl; } else { - ldout(cct, 7) << "wrote to " << totalwritten+offset << ", leaving file size at " << in->size << dendl; + ldout(cct, 7) << "wrote to " << totalwritten+offset << ", effective size " << request_size + request_offset << ", leaving file size at " << in->size << dendl; } // mtime @@ -11750,7 +12052,7 @@ void Client::C_Write_Finisher::finish_io(int r) } } - r = clnt->_write_success(f, start, fpos, offset, size, in); + r = clnt->_write_success(f, start, fpos, req_ofs, req_size, offset, size, in, encrypted); } iofinished = true; @@ -11828,6 +12130,279 @@ bool Client::C_Write_Finisher::try_complete() return false; } +Client::WriteEncMgr::WriteEncMgr(Client *clnt, + Fh *f, int64_t offset, uint64_t size, + bufferlist& bl, + bool async) : clnt(clnt), whoami(clnt->whoami), + cct(clnt->cct), fscrypt(clnt->fscrypt.get()), + f(f), in(f->inode.get()), + offset(offset), size(size), bl(bl), + async(async) +{ + denc = fscrypt->get_fdata_denc(in->fscrypt_ctx, &in->fscrypt_key_validator); + + pbl = &bl; +} + +Client::WriteEncMgr::~WriteEncMgr() +{ +} + +int Client::WriteEncMgr::init() +{ + if (!denc) { + return 0; + } + + endoff = offset + size; + + int want = CEPH_CAP_FILE_RD; + int have; + int r = clnt->get_caps(f, CEPH_CAP_FILE_RD, want, &have, endoff); + if (r < 0) { + return r; + } + + return 0; +} + +int Client::WriteEncMgr::read_async(uint64_t off, uint64_t len, bufferlist *bl, + iofinish_method_ctx *ioctx) +{ + get(); + + if (off >= in->size) { + ioctx->finish(0); + return 0; + } + + int r = clnt->_read_async(f, off, len, bl, ioctx->ctx()); + if (r < 0) { + ioctx->cancel(r); + put(); + } + + ioctx->release(); + + return r; +} + +int Client::WriteEncMgr::read_modify_write(Context *_iofinish) +{ + iofinish = _iofinish; + + if (!denc) { + return do_write(); + } + + ceph_assert(ceph_mutex_is_locked_by_me(clnt->client_lock)); + + int r = 0; + + start_block = fscrypt_block_from_ofs(offset); + start_block_ofs = fscrypt_block_start(offset); + ofs_in_start_block = fscrypt_ofs_in_block(offset); + end_block = fscrypt_block_from_ofs(endoff - 1); + end_block_ofs = fscrypt_block_start(endoff - 1); + ofs_in_end_block = fscrypt_ofs_in_block(endoff - 1); + + need_read_start = ofs_in_start_block > 0; + need_read_end = (endoff < in->effective_size() && ofs_in_end_block < FSCRYPT_BLOCK_SIZE - 1 && start_block != end_block); + + read_start_size = (need_read_start && need_read_end && start_block == end_block ? + FSCRYPT_BLOCK_SIZE : ofs_in_start_block); + + bool need_read = need_read_start | need_read_start; + + + if (read_start_size > 0) { + finish_read_start_ctx.reset(new iofinish_method_ctx(*this, &WriteEncMgr::finish_read_start_cb, &aioc)); + + r = read_async(start_block_ofs, read_start_size, &startbl, finish_read_start_ctx.get()); + if (r < 0) { + finish_read_start_ctx.reset(); + + ldout(cct, 0) << "failed to read first block: r=" << r << dendl; + goto done; + } + } + + if (need_read_end) { + finish_read_end_ctx.reset(new iofinish_method_ctx(*this, &WriteEncMgr::finish_read_end_cb, &aioc)); + + r = read_async(end_block_ofs, FSCRYPT_BLOCK_SIZE, &endbl, finish_read_end_ctx.get()); + if (r < 0) { + finish_read_end_ctx.reset(); + + ldout(cct, 0) << "failed to read end block: r=" << r << dendl; + goto done; + } + } + + is_ready_to_finish = true; + + if (need_read && !async) { + clnt->client_lock.unlock(); + + if (finish_read_start_ctx) { + finish_read_start_ctx->wait(); + } + + if (finish_read_end_ctx) { + finish_read_end_ctx->wait(); + } + + clnt->client_lock.lock(); + + r = aioc.get_retcode(); + if (r < 0) { + goto done; + } + } + + +done: + if (async) { + if (finish_read_start_ctx) { + finish_read_start_ctx->release(); + } + + if (finish_read_end_ctx) { + finish_read_end_ctx->release(); + } + } + + try_finish(r); + + return r; +} + + +void Client::WriteEncMgr::finish_read_start(int r) +{ + ceph_assert(ceph_mutex_is_locked_by_me(clnt->client_lock)); + + if (r >= 0) { + std::lock_guard l{lock}; + int read_len = startbl.length(); + if (read_len < read_start_size) { + startbl.append_zero(read_start_size - read_len); + } + + /* prepend data from the start of the first block */ + bufferlist newbl; + startbl.splice(0, ofs_in_start_block, &newbl); + + unsigned int orig_len = bl.length(); + + /* append new data */ + newbl.claim_append(bl); + + if (startbl.length() > orig_len) { + /* can happen if start and end are in the same block */ + bufferlist tail; + startbl.splice(orig_len, startbl.length()-orig_len, &tail); + newbl.claim_append(tail); + + if (newbl.length() < FSCRYPT_BLOCK_SIZE) { + newbl.append_zero(FSCRYPT_BLOCK_SIZE - newbl.length()); + } + } + + bl.swap(newbl); + } + + try_finish(r); +} + +void Client::WriteEncMgr::finish_read_end(int r) +{ + ceph_assert(ceph_mutex_is_locked_by_me(clnt->client_lock)); + + if (r >= 0) { + std::lock_guard l{lock}; + if (endbl.length() > ofs_in_end_block) { + bufferlist tail; + endbl.splice(ofs_in_end_block + 1, endbl.length() - ofs_in_end_block - 1, &tail); + + bl.claim_append(tail); + } + } + + try_finish(r); +} + +bool Client::WriteEncMgr::do_try_finish(int r) +{ + ceph_assert(ceph_mutex_is_locked_by_me(clnt->client_lock)); + + if (!aioc.is_complete()) { + return false; + } + + if (r >= 0) { + offset = start_block_ofs; + + pbl = &encbl; + r = denc->encrypt_bl(offset, bl.length(), bl, &encbl); + if (r < 0) { + ldout(cct, 0) << "failed to encrypt bl: r=" << r << dendl; + } + + size = encbl.length(); + } + + clnt->put_cap_ref(in, CEPH_CAP_FILE_RD); + in->mark_caps_dirty(CEPH_CAP_FILE_RD); + + update_write_params(); + + r = do_write(); + + return true; +} + +void Client::WriteEncMgr_Buffered::update_write_params() +{ + if (iofinish) { + static_cast(iofinish)->CWF->update_write_params(offset, size); + } +} + +int Client::WriteEncMgr_Buffered::do_write() +{ + int r = 0; + + // do buffered write + if (!in->oset.dirty_or_tx) + clnt->get_cap_ref(in, CEPH_CAP_FILE_CACHE | CEPH_CAP_FILE_BUFFER); + + clnt->get_cap_ref(in, CEPH_CAP_FILE_BUFFER); + + // async, caching, non-blocking. + r = clnt->objectcacher->file_write(&in->oset, &in->layout, + in->snaprealm->get_snap_context(), + offset, size, *pbl, ceph::real_clock::now(), + 0, iofinish, + !async + ? clnt->objectcacher->CFG_block_writes_upfront() + : false); + + return r; +} + +int Client::WriteEncMgr_NotBuffered::do_write() +{ + clnt->get_cap_ref(in, CEPH_CAP_FILE_BUFFER); + + clnt->filer->write_trunc(in->ino, &in->layout, in->snaprealm->get_snap_context(), + offset, size, *pbl, ceph::real_clock::now(), 0, + in->truncate_size, in->truncate_seq, + iofinish); + + return 0; +} + int64_t Client::_write(Fh *f, int64_t offset, uint64_t size, bufferlist bl, Context *onfinish, bool do_fsync, bool syncdataonly) { @@ -11917,9 +12492,33 @@ int64_t Client::_write(Fh *f, int64_t offset, uint64_t size, bufferlist bl, } } + uint64_t request_offset = offset; + uint64_t request_size = size; + if (f->flags & O_DIRECT) have &= ~(CEPH_CAP_FILE_BUFFER | CEPH_CAP_FILE_LAZYIO); + bool buffered_write = (have & (CEPH_CAP_FILE_BUFFER | CEPH_CAP_FILE_LAZYIO)); + ceph::ref_t enc_mgr; + + if (buffered_write) { + enc_mgr = ceph::make_ref(this, f, + offset, size, bl, + !!onfinish); + } else { + enc_mgr = ceph::make_ref(this, f, + offset, size, bl, + !!onfinish); + } + + + r = enc_mgr->init(); + if (r < 0) { + ldout(cct, 0) << __func__ << "(): enc_mgr init failed (r=" << r << ")" << dendl; + put_cap_ref(in, CEPH_CAP_FILE_WR); + return r; + } + ldout(cct, 10) << " snaprealm " << *in->snaprealm << dendl; std::unique_ptr iofinish = nullptr; @@ -11963,29 +12562,27 @@ int64_t Client::_write(Fh *f, int64_t offset, uint64_t size, bufferlist bl, cct->_conf->client_oc && (have & (CEPH_CAP_FILE_BUFFER | CEPH_CAP_FILE_LAZYIO)), - f, in, fpos, offset, size, - do_fsync, syncdataonly)); + f, in, fpos, + request_offset, request_size, + offset, size, + do_fsync, syncdataonly, enc_mgr->encrypted())); cwf_iofinish->CWF = cwf.get(); } if (cct->_conf->client_oc && (have & (CEPH_CAP_FILE_BUFFER | CEPH_CAP_FILE_LAZYIO))) { - // do buffered write - if (!in->oset.dirty_or_tx) - get_cap_ref(in, CEPH_CAP_FILE_CACHE | CEPH_CAP_FILE_BUFFER); - get_cap_ref(in, CEPH_CAP_FILE_BUFFER); + // do buffered write // async, caching, non-blocking. ldout(cct, 10) << " _write_oc " << dendl; - r = objectcacher->file_write(&in->oset, &in->layout, - in->snaprealm->get_snap_context(), - offset, size, bl, ceph::real_clock::now(), - 0, iofinish.get(), - onfinish == nullptr - ? objectcacher->CFG_block_writes_upfront() - : false); + r = enc_mgr->read_modify_write(iofinish.get()); + if (r < 0) { + ldout(cct, 0) << __func__ << "(): enc_mgr read failed (r=" << r << ")" << dendl; + put_cap_ref(in, CEPH_CAP_FILE_WR); + return r; + } if (onfinish) { // handle non-blocking caller (onfinish != nullptr), we can now safely @@ -12059,10 +12656,7 @@ int64_t Client::_write(Fh *f, int64_t offset, uint64_t size, bufferlist bl, } ldout(cct, 10) << " _write_filer" << dendl; - filer->write_trunc(in->ino, &in->layout, in->snaprealm->get_snap_context(), - offset, size, bl, ceph::real_clock::now(), 0, - in->truncate_size, in->truncate_seq, - filer_iofinish.get()); + enc_mgr->read_modify_write(iofinish.get()); if (onfinish) { // handle non-blocking caller (onfinish != nullptr), we can now safely @@ -12090,7 +12684,7 @@ success: // do not get here if non-blocking caller (onfinish != nullptr) ldout(cct, 10) << " _write_filer_succeess" << dendl; - r = _write_success(f, start, fpos, offset, size, in); + r = _write_success(f, start, fpos, request_offset, request_size, enc_mgr->get_ofs(), enc_mgr->get_size(), in, enc_mgr->encrypted()); if (r >= 0 && do_fsync) { int64_t r1; @@ -13461,6 +14055,8 @@ InodeRef Client::open_snapdir(const InodeRef& diri) auto [it, b] = inode_map.try_emplace(vino, nullptr); if (b) { in = new Inode(this, vino, &diri->layout); + in->fscrypt_auth = diri->fscrypt_auth; /* borrow parent fscrypt data */ + in->fscrypt_ctx = in->init_fscrypt_ctx(fscrypt.get()); refresh_snapdir_attrs(in, diri.get()); diri->flags |= I_SNAPDIR_OPEN; it->second = in; @@ -15009,8 +15605,21 @@ int Client::_mknod(Inode *dir, const char *name, mode_t mode, dev_t rdev, req->set_inode_owner_uid_gid(perms.uid(), perms.gid()); + req->set_filepath(wdr.getpath()); req->set_inode(wdr.diri); +#if 0 +FSCRYPT + filepath path; + Dentry *de; + int r = _prepare_req_path(dir, req, path, name, true, &de); + if (r < 0) { + delete req; + return r; + } + + req->set_inode(dir); +#endif req->head.args.mknod.rdev = rdev; req->dentry_drop = CEPH_CAP_FILE_SHARED; req->dentry_unless = CEPH_CAP_FILE_EXCL; @@ -15125,6 +15734,11 @@ int Client::_create(const walk_dentry_result& wdr, int flags, mode_t mode, int cmode = ceph_flags_to_mode(cflags); + if (dir->fscrypt_ctx && + cmode & CEPH_FILE_MODE_WR) { + cmode |= CEPH_FILE_MODE_RD; + } + int64_t pool_id = -1; if (data_pool && *data_pool) { pool_id = objecter->with_osdmap( @@ -15142,6 +15756,26 @@ int Client::_create(const walk_dentry_result& wdr, int flags, mode_t mode, req->set_filepath(wdr.getpath()); req->set_alternate_name(alternate_name.empty() ? wdr.alternate_name : alternate_name); req->set_inode(wdr.diri); +#if 0 +FSCRYPT + req->set_dentry(wdr.dn); + filepath path; + Dentry *de; + + int r = _prepare_req_path(dir, req, path, name, true, &de); + if (r < 0) { + delete req; + return r; + } + + if (de->alternate_name.empty()) { + req->set_alternate_name(std::move(alternate_name)); + } else { + req->set_alternate_name(de->alternate_name); + } + dir->gen_inherited_fscrypt_auth(&req->fscrypt_auth); + req->set_inode(dir); +#endif req->head.args.open.flags = cflags | CEPH_O_CREAT; req->head.args.open.stripe_unit = stripe_unit; @@ -15225,6 +15859,26 @@ int Client::_mkdir(const walk_dentry_result& wdr, mode_t mode, const UserPerm& p req->dentry_drop = CEPH_CAP_FILE_SHARED; req->dentry_unless = CEPH_CAP_FILE_EXCL; req->set_alternate_name(alternate_name.empty() ? wdr.alternate_name : alternate_name); +#if 0 +FSCRYPT + filepath path; + Dentry *de; + int r = _prepare_req_path(dir, req, path, name, true, &de); + if (r < 0) { + delete req; + return r; + } + + req->set_inode(dir); + req->dentry_drop = CEPH_CAP_FILE_SHARED; + req->dentry_unless = CEPH_CAP_FILE_EXCL; + if (de->alternate_name.empty()) { + req->set_alternate_name(std::move(alternate_name)); + } else { + req->set_alternate_name(de->alternate_name); + } + dir->gen_inherited_fscrypt_auth(&req->fscrypt_auth); +#endif mode |= S_IFDIR; bufferlist bl; @@ -15371,6 +16025,47 @@ int Client::_symlink(Inode *dir, const char *name, const char *target, req->dentry_drop = CEPH_CAP_FILE_SHARED; req->dentry_unless = CEPH_CAP_FILE_EXCL; req->set_dentry(wdr.dn); +#if 0 +FSCRYPT + filepath path; + Dentry *de; + int r = _prepare_req_path(dir, req, path, name, true, &de); + if (r < 0) { + delete req; + return r; + } + + req->fscrypt_file = dir->fscrypt_file; + + dir->gen_inherited_fscrypt_auth(&req->fscrypt_auth); + auto fscrypt_ctx = fscrypt->init_ctx(req->fscrypt_auth); + + if (fscrypt_ctx) { + auto fscrypt_denc = fscrypt->get_fname_denc(fscrypt_ctx, nullptr, true); + + string enc_target; + int r = fscrypt_denc->get_encrypted_symlink(target,&enc_target); + if (r < 0) { + delete req; + return r; + } + req->set_string2(enc_target.c_str()); + } else { + req->set_string2(target); + } + + if (de->alternate_name.empty()) { + req->set_alternate_name(std::move(alternate_name)); + } else { + req->set_alternate_name(de->alternate_name); + } + + req->set_inode(dir); + req->dentry_drop = CEPH_CAP_FILE_SHARED; + req->dentry_unless = CEPH_CAP_FILE_EXCL; + + req->set_dentry(de); +#endif int res = make_request(req, perms, inp); @@ -15468,6 +16163,20 @@ int Client::_unlink(Inode *dir, const char *name, const UserPerm& perm) req->set_filepath(wdr.getpath()); req->set_dentry(wdr.dn); +#if 0 +FSCRYPT + filepath path; + Dentry *de; + int r = _prepare_req_path(dir, req, path, name, true, &de); + if (r < 0) { + delete req; + return r; + } + + InodeRef otherin; + Inode *in; + req->set_dentry(de); +#endif req->dentry_drop = CEPH_CAP_FILE_SHARED; req->dentry_unless = CEPH_CAP_FILE_EXCL; @@ -15531,6 +16240,34 @@ int Client::_rmdir(Inode *dir, const char *name, const UserPerm& perms, bool che req->dentry_drop = CEPH_CAP_FILE_SHARED; req->dentry_unless = CEPH_CAP_FILE_EXCL; req->other_inode_drop = CEPH_CAP_LINK_SHARED | CEPH_CAP_LINK_EXCL; +#if 0 +FSCRYPT + filepath path; + Dentry *de; + int r = _prepare_req_path(dir, req, path, name, true, &de); + if (r < 0) { + delete req; + return r; + } + + req->set_inode(dir); + + req->dentry_drop = CEPH_CAP_FILE_SHARED; + req->dentry_unless = CEPH_CAP_FILE_EXCL; + req->other_inode_drop = CEPH_CAP_LINK_SHARED | CEPH_CAP_LINK_EXCL; + + InodeRef in; + + if (op == CEPH_MDS_OP_RMDIR) + req->set_dentry(de); + else + de->get(); + + int res = _lookup(dir, name, 0, &in, perms); + if (res < 0) { + put_request(req); + return res; +#endif } req->set_inode(wdr.diri); @@ -15640,6 +16377,31 @@ int Client::_rename(Inode *fromdir, const char *fromname, Inode *todir, const ch req->set_filepath(wdr_to.getpath()); req->set_filepath2(wdr_from.getpath()); req->set_alternate_name(alternate_name.empty() ? wdr_to.alternate_name : alternate_name); +#if 0 +FSCRYPT + filepath to; + Dentry *de; + int r = _prepare_req_path(todir, req, to, toname, true, &de); + if (r < 0) { + delete req; + return r; + } + + filepath from; + Dentry *oldde; + r = _prepare_req_path(fromdir, req, from, fromname, false, &oldde); + if (r < 0) { + delete req; + return r; + } + req->set_filepath2(from); + + if (de->alternate_name.empty()) { + req->set_alternate_name(std::move(alternate_name)); + } else { + req->set_alternate_name(de->alternate_name); + } +#endif int res; if (op == CEPH_MDS_OP_RENAME) { @@ -15751,6 +16513,24 @@ int Client::_link(Inode *diri_from, const char* path_from, Inode* diri_to, const req->set_filepath(wdr_to.getpath()); req->set_alternate_name(alternate_name.empty() ? wdr_to.alternate_name : alternate_name); req->set_filepath2(wdr_from.getpath()); +#if 0 +FSCRYPT + filepath path; + Dentry *de; + int r = _prepare_req_path(dir, req, path, newname, true, &de); + if (r < 0) { + delete req; + return r; + } + + if (de->alternate_name.empty()) { + req->set_alternate_name(std::move(alternate_name)); + } else { + req->set_alternate_name(de->alternate_name); + } + filepath existing(in->ino); + req->set_filepath2(existing); +#endif req->set_inode(wdr_to.diri); req->inode_drop = CEPH_CAP_FILE_SHARED; @@ -15758,6 +16538,7 @@ int Client::_link(Inode *diri_from, const char* path_from, Inode* diri_to, const req->set_dentry(wdr_to.dn); int res = make_request(req, perm); + ldout(cct, 10) << "link result is " << res << dendl; trim_cache(); @@ -17489,6 +18270,78 @@ void Client::set_uuid(const std::string& uuid) _close_sessions(); } +int Client::add_fscrypt_key(const char *key_data, int key_len, + ceph_fscrypt_key_identifier *kid) +{ + auto& key_store = fscrypt->get_key_store(); + + FSCryptKeyHandlerRef kh; + + int r = key_store.create((const char *)key_data, key_len, kh); + if (r < 0) { + ldout(cct, 0) << __func__ << "(): failed to create a new key: r=" << r << dendl; + return r; + } + + auto& k = kh->get_key(); + + if (kid) { + *kid = k->get_identifier(); + } + + return 0; +} + +int Client::remove_fscrypt_key(const ceph_fscrypt_key_identifier& kid) +{ + auto& key_store = fscrypt->get_key_store(); + + return key_store.invalidate(kid); +} + +int Client::set_fscrypt_policy_v2(int fd, const struct fscrypt_policy_v2& policy) +{ + Fh *f = get_filehandle(fd); + if (!f) { + return -CEPHFS_EBADF; + } + + return ll_set_fscrypt_policy_v2(f->inode.get(), policy); +} + +int Client::ll_set_fscrypt_policy_v2(Inode *in, const struct fscrypt_policy_v2& policy) +{ + if (in->fscrypt_auth.size() > 0) { + return -EEXIST; + } + + FSCryptContext fsc(cct); + fsc.init(policy); + fsc.generate_new_nonce(); + + UserPerm perms(in->uid, in->gid); + + bufferlist env_bl; + + fsc.encode(env_bl); + + int r = ll_setxattr(in, "ceph.fscrypt.auth", (void *)env_bl.c_str(), env_bl.length(), CEPH_XATTR_CREATE, perms); + if (r < 0) { + ldout(cct, 0) << __func__ << "(): failed to set fscrypt_auth attr: r=" << r << dendl; + return r; + } + + uint64_t fsize = 0; + r = ll_setxattr(in, "ceph.fscrypt.file", (void *)&fsize, sizeof(fsize), CEPH_XATTR_CREATE, perms); + if (r < 0) { + ldout(cct, 0) << __func__ << "(): failed to set fscrypt_file attr: r=" << r << dendl; + return r; + } + + return 0; +} + + // called before mount. 0 means infinite void Client::set_session_timeout(unsigned timeout) { diff --git a/src/client/Client.h b/src/client/Client.h index c341e4525d9..b0c67bf8a15 100644 --- a/src/client/Client.h +++ b/src/client/Client.h @@ -46,6 +46,7 @@ #include "InodeRef.h" #include "MetaSession.h" #include "UserPerm.h" +#include "FSCrypt.h" #include #include @@ -378,6 +379,11 @@ public: return fscid; } + /* fscrypt */ + int add_fscrypt_key(const char *key_data, int key_len, ceph_fscrypt_key_identifier *kid); + int remove_fscrypt_key(const ceph_fscrypt_key_identifier& kid); + int set_fscrypt_policy_v2(int fd, const struct fscrypt_policy_v2& policy); + int mds_command( const std::string &mds_spec, const std::vector& cmd, @@ -738,6 +744,8 @@ public: return acl_type != NO_ACL; } + int ll_set_fscrypt_policy_v2(Inode *in, const struct fscrypt_policy_v2& policy); + int ll_get_stripe_osd(struct Inode *in, uint64_t blockno, file_layout_t* layout); uint64_t ll_get_internal_offset(struct Inode *in, uint64_t blockno); @@ -1012,8 +1020,11 @@ public: std::unique_ptr logger; std::unique_ptr mdsmap; + std::unique_ptr fscrypt; + bool _collect_and_send_global_metrics; + protected: struct walk_dentry_result { DentryRef dn; @@ -1186,7 +1197,7 @@ protected: * leave dn set to default NULL unless you're trying to add * a new inode to a pre-created Dentry */ - Dentry* link(Dir *dir, const std::string& name, Inode *in, Dentry *dn); + Dentry* link(Dir *dir, const std::string& name, std::optional enc_name, Inode *in, Dentry *dn); void unlink(Dentry *dn, bool keepdir, bool keepdentry); int fill_stat(Inode *in, struct stat *st, frag_info_t *dirstat=0, nest_info_t *rstat=0); @@ -1494,18 +1505,28 @@ private: class C_Read_Async_Finisher : public Context { public: C_Read_Async_Finisher(Client *clnt, Context *onfinish, Fh *f, Inode *in, - uint64_t fpos, uint64_t off, uint64_t len) - : clnt(clnt), onfinish(onfinish), f(f), in(in), off(off), len(len), start_time(mono_clock_now()) {} + bufferlist *bl, + uint64_t fpos, uint64_t off, uint64_t len, + FSCryptFDataDencRef denc, + uint64_t read_start, + uint64_t read_len) + : clnt(clnt), onfinish(onfinish), f(f), in(in), bl(bl), off(off), len(len), + start_time(mono_clock_now()), denc(denc), read_start(read_start), read_len(read_len) {} private: Client *clnt; Context *onfinish; Fh *f; Inode *in; + bufferlist *bl; uint64_t off; uint64_t len; utime_t start_time; + FSCryptFDataDencRef denc; + uint64_t read_start; + uint64_t read_len; + void finish(int r) override; }; @@ -1524,6 +1545,259 @@ private: void finish(int r) override; }; + struct aio_collection { + ceph::mutex lock = ceph::make_mutex("Client::aio_collection"); + + aio_collection() {} + + struct aio_status { + int r = 0; + bool complete{false}; + }; + + int num_complete = 0; + int total = 0; + + int retcode = 0; + + std::vector status; + + int add_io() { + std::unique_lock l{lock}; + int old_total = total++; + status.resize(total); + + return old_total; + } + + bool is_complete() { + std::unique_lock l{lock}; + return num_complete == total; + } + + int finish_io(int id, int r) { + std::unique_lock l{lock}; + status[id].r = r; + status[id].complete = true; + + ++num_complete; + + if (r < 0 && retcode == 0) { + retcode = r; + } + + return r; + } + + int get_retcode() { + std::unique_lock l{lock}; + return retcode; + } + }; + + template + struct iofinish_method_ctx { + struct _Ctx : Context { + iofinish_method_ctx *ioc; + + _Ctx(iofinish_method_ctx *_ioc) : ioc(_ioc) {} + + void finish(int r) override { + ioc->finish(r); + } + }; + + std::unique_ptr<_Ctx> _ctx; + + T& t; + void (T::*call)(int); + ceph::mutex lock = ceph::make_mutex("Client::iofinish_method_ctx"); + ceph::condition_variable cond; + bool done{false}; + bool canceled{false}; + + aio_collection *aioc; + int io_id; + + iofinish_method_ctx(T& _t, void (T::*_call)(int), aio_collection *_aioc) : t(_t), call(_call), + aioc(_aioc) { + _ctx.reset(new _Ctx(this)); + + if (aioc) { + io_id = aioc->add_io(); + } + } + + Context *ctx() { return _ctx.get(); } + + void wait() { + std::unique_lock l{lock}; + if (!done) { + cond.wait(l); + } + } + + Context *release() { + auto ctx = _ctx.release(); + _ctx.reset(); + return ctx; + } + + void cancel(int r) { + std::unique_lock l{lock}; + canceled = true; + done = true; + if (aioc) { + aioc->finish_io(io_id, r); + } + } + + void finish(int r) { + std::unique_lock l{lock}; + if (!canceled) { + /* avoid any callback when canceled, calling object might have been destroyed */ + if (aioc) { + aioc->finish_io(io_id, r); + } + (t.*(call))(r); + } + done = true; + cond.notify_all(); + } + }; + + struct CWF_iofinish; + + class WriteEncMgr : public RefCountedObject { + protected: + Client *clnt; + client_t const whoami; + + CephContext *cct; + FSCrypt *fscrypt; + + ceph::mutex lock = ceph::make_mutex("Client::WriteEncMgr"); + + Fh *f; + Inode *in; + + Context *iofinish{nullptr}; + + int64_t offset; + uint64_t size; + + bufferlist bl; + bufferlist encbl; + bufferlist *pbl; + + bool async; + + FSCryptFDataDencRef denc; + + uint64_t start_block; + uint64_t start_block_ofs; + uint64_t ofs_in_start_block; + uint64_t end_block; + uint64_t end_block_ofs; + uint64_t ofs_in_end_block; + + uint64_t endoff; + + bool need_read_start{false}; + bool need_read_end{false}; + + int read_start_size; + + bufferlist startbl; + bufferlist endbl; + + aio_collection aioc; + + bool is_ready_to_finish{false}; + bool is_finished{false}; + + std::unique_ptr > finish_read_start_ctx; + std::unique_ptr > finish_read_end_ctx; + + bool try_finish(int r) { + if (is_finished) { + return true; + } + + if (!is_ready_to_finish) { + return false; + } + + is_finished = do_try_finish(r); + + return is_finished; + } + + int read_async(uint64_t off, uint64_t len, bufferlist *bl, iofinish_method_ctx *ioctx); + + protected: + virtual int do_write() = 0; + virtual void update_write_params() = 0; + + public: + WriteEncMgr(Client *clnt, + Fh *f, int64_t offset, uint64_t size, + bufferlist& bl, + bool async); + virtual ~WriteEncMgr(); + + int init(); + + void get_write_params(int64_t *poffset, uint64_t *psize, bufferlist **ppbl) const { + *poffset = offset; + *psize = size; + *ppbl = pbl; + } + + bool encrypted() const { + return !!denc; + } + + int read_modify_write(Context *iofinish); + + void finish_read_start_cb(int r) { + finish_read_start(r); + put(); + } + void finish_read_end_cb(int r) { + finish_read_end(r); + put(); + } + void finish_read_start(int r); + void finish_read_end(int r); + bool do_try_finish(int r); + + int64_t get_ofs() { return offset; } + uint64_t get_size() { return size; } + }; + + class WriteEncMgr_Buffered : public WriteEncMgr { + public: + WriteEncMgr_Buffered(Client *clnt, + Fh *f, int64_t offset, uint64_t size, + bufferlist& bl, + bool async) : WriteEncMgr(clnt, f, offset, size, bl, async) {} + + void update_write_params() override; + int do_write() override; + }; + + class WriteEncMgr_NotBuffered : public WriteEncMgr { + public: + WriteEncMgr_NotBuffered(Client *clnt, + Fh *f, int64_t offset, uint64_t size, + bufferlist& bl, + bool async) : WriteEncMgr(clnt, f, offset, size, bl, async) {} + + void update_write_params() override {} + int do_write() override; + }; + class C_Write_Finisher : public Context { public: void finish_io(int r); @@ -1532,11 +1806,15 @@ private: C_Write_Finisher(Client *clnt, Context *onfinish, bool dont_need_uninline, bool is_file_write, Fh *f, Inode *in, - uint64_t fpos, int64_t offset, uint64_t size, - bool do_fsync, bool syncdataonly) + uint64_t fpos, int64_t req_ofs, uint64_t req_size, + int64_t offset, uint64_t size, + bool do_fsync, bool syncdataonly, + bool encrypted) : clnt(clnt), onfinish(onfinish), is_file_write(is_file_write), start(mono_clock_now()), f(f), in(in), fpos(fpos), - offset(offset), size(size), syncdataonly(syncdataonly) { + req_ofs(req_ofs), req_size(req_size), + offset(offset), size(size), syncdataonly(syncdataonly), + encrypted(encrypted) { iofinished_r = 0; onuninlinefinished_r = 0; fsync_r = 0; @@ -1549,6 +1827,11 @@ private: // We need to override finish, but have nothing to do. } + void update_write_params(int64_t _ofs, uint64_t _size) { + offset = _ofs; + size = _size; + } + private: Client *clnt; Context *onfinish; @@ -1557,9 +1840,12 @@ private: Fh *f; Inode *in; uint64_t fpos; + int64_t req_ofs; + uint64_t req_size; int64_t offset; uint64_t size; bool syncdataonly; + bool encrypted; int64_t iofinished_r; int64_t onuninlinefinished_r; int64_t fsync_r; @@ -1745,6 +2031,9 @@ private: bool _dentry_valid(const Dentry *dn); + int _prepare_req_path(Inode *dir, MetaRequest *req, filepath& path, const char *name, + bool set_filepath, Dentry **pdn); + // internal interface // call these with client_lock held! int _do_lookup(const InodeRef& dir, const std::string& name, int mask, InodeRef *target, @@ -1806,7 +2095,9 @@ private: Context *onfinish = nullptr); void do_readahead(Fh *f, Inode *in, uint64_t off, uint64_t len); int64_t _write_success(Fh *fh, utime_t start, uint64_t fpos, - int64_t offset, uint64_t size, Inode *in); + int64_t request_offset, uint64_t request_size, + int64_t offset, uint64_t size, Inode *in, + bool encrypted); int64_t _write(Fh *fh, int64_t offset, uint64_t size, bufferlist bl, Context *onfinish = nullptr, bool do_fsync = false, bool syncdataonly = false); diff --git a/src/client/Dentry.h b/src/client/Dentry.h index 9668953b412..bc120223751 100644 --- a/src/client/Dentry.h +++ b/src/client/Dentry.h @@ -98,6 +98,7 @@ public: Dir *dir; const std::string name; + std::optional enc_name; InodeRef inode; int ref = 1; // 1 if there's a dir beneath me. int64_t offset = 0; diff --git a/src/client/FSCrypt.cc b/src/client/FSCrypt.cc new file mode 100644 index 00000000000..c492f60e2ad --- /dev/null +++ b/src/client/FSCrypt.cc @@ -0,0 +1,1021 @@ + +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2023 IBM + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + */ +#include "common/errno.h" +#include "common/safe_io.h" +#include "common/config.h" +#include "common/ceph_crypto.h" +#include "common/debug.h" +#include "include/ceph_assert.h" +#include "auth/Crypto.h" + +#include "client/FSCrypt.h" + +#include +#include +#include +#include + +#include + +#define dout_subsys ceph_subsys_client + + +using ceph::crypto::HMACSHA512; +/* + * base64 encode/decode. + */ + +#define CEPH_NOHASH_NAME_MAX (180 - CEPH_CRYPTO_SHA256_DIGESTSIZE) + + + +/* FIXME: this was copy pasted from common/armor.c with slight modification + * as needed to use alternative translation table. Code can and should be + * combined, but need to make sure we do it in a way that doesn't hurt + * compiler optimizations in the general case. + * Also relaxed decoding to make it compatible with the kernel client */ +static const char *pem_key = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+,"; + +static int encode_bits(int c) +{ + return pem_key[c]; +} + +static int decode_bits(char c) +{ + if (c >= 'A' && c <= 'Z') + return c - 'A'; + if (c >= 'a' && c <= 'z') + return c - 'a' + 26; + if (c >= '0' && c <= '9') + return c - '0' + 52; + if (c == '+' || c == '-') + return 62; + if (c == ',' || c == '/' || c == '_') + return 63; + if (c == '=') + return 0; /* just non-negative, please */ + return -EINVAL; +} + +static int set_str_val(char **pdst, const char *end, char c) +{ + if (*pdst < end) { + char *p = *pdst; + *p = c; + (*pdst)++; + } else + return -ERANGE; + + return 0; +} + +static int b64_encode(char *dst, char * const dst_end, const char *src, const char *end) +{ + char *orig_dst = dst; + int olen = 0; + int line = 0; + +#define SET_DST(c) do { \ + int __ret = set_str_val(&dst, dst_end, c); \ + if (__ret < 0) \ + return __ret; \ +} while (0); + + while (src < end) { + unsigned char a; + + a = *src++; + SET_DST(encode_bits(a >> 2)); + if (src < end) { + unsigned char b; + b = *src++; + SET_DST(encode_bits(((a & 3) << 4) | (b >> 4))); + if (src < end) { + unsigned char c; + c = *src++; + SET_DST(encode_bits(((b & 15) << 2) | + (c >> 6))); + SET_DST(encode_bits(c & 63)); + } else { + SET_DST(encode_bits((b & 15) << 2)); + } + } else { + SET_DST(encode_bits(((a & 3) << 4))); + } + olen += 4; + line += 4; + } + *dst = '\0'; + return (dst - orig_dst); +} + +static char get_unarmor_src(const char *src, const char *end, int ofs) +{ + if (src + ofs < end) { + return src[ofs]; + } + return '='; +} + +int b64_decode(char *dst, char * const dst_end, const char *src, const char *end) +{ + int olen = 0; + + while (src < end) { + int a, b, c, d; + + if (src[0] == '\n') { + src++; + continue; + } + + a = decode_bits(get_unarmor_src(src, end, 0)); + b = decode_bits(get_unarmor_src(src, end, 1)); + c = decode_bits(get_unarmor_src(src, end, 2)); + d = decode_bits(get_unarmor_src(src, end, 3)); + if (a < 0 || b < 0 || c < 0 || d < 0) { + return -EINVAL; + } + + SET_DST((a << 2) | (b >> 4)); + if (get_unarmor_src(src, end, 2) == '=') + return olen + 1; + SET_DST(((b & 15) << 4) | (c >> 2)); + if (get_unarmor_src(src, end, 3) == '=') + return olen + 2; + SET_DST(((c & 3) << 6) | d); + olen += 3; + src += 4; + } + return olen; +} + +static int calc_hmac_sha512(const char *key, int key_len, + const char *msg, int msg_len, + char *dest, int dest_len) +{ + char hash_sha512[CEPH_CRYPTO_HMACSHA512_DIGESTSIZE]; + + HMACSHA512 hmac((const unsigned char *)key, key_len); + hmac.Update((const unsigned char *)msg, msg_len); + hmac.Final((unsigned char *)hash_sha512); + + auto len = std::min(dest_len, CEPH_CRYPTO_HMACSHA512_DIGESTSIZE); + + memcpy(dest, hash_sha512, len); + + return len; +} + +#define SALT_LEN_DEFAULT 32 + +static char default_salt[SALT_LEN_DEFAULT] = { 0 }; + +static int hkdf_extract(const char *_salt, int salt_len, + const char *ikm, int ikm_len, + char *dest, int dest_len) { + const char *salt = _salt; + if (!_salt) { + salt = default_salt; + salt_len = SALT_LEN_DEFAULT; + } + + return calc_hmac_sha512(salt, salt_len, ikm, ikm_len, dest, dest_len); +} + +static int hkdf_expand(const char *data, int data_len, + const char *info, int info_len, + char *dest, int dest_len) +{ + int total_len = 0; + + char info_buf[info_len + 16]; + memcpy(info_buf, info, info_len); + + char *p = dest; + + for (char i = 1; total_len < dest_len; i++) { + *(char *)(info_buf + info_len) = i; + + int r = calc_hmac_sha512(data, data_len, + info_buf, info_len + 1, + p, dest_len - total_len); + if (r < 0) { + return r; + } + if (r == 0) { + return -EINVAL; + } + + total_len += r; + } + + return total_len; +} + +int fscrypt_fname_unarmor(const char *src, int src_len, + char *result, int max_len) +{ + return b64_decode(result, result + max_len, + src, src + src_len); +} + +int fscrypt_fname_armor(const char *src, int src_len, + char *result, int max_len) +{ + return b64_encode(result, result + max_len, + src, src + src_len); +} + +int fscrypt_calc_hkdf(char hkdf_context, + const char *nonce, int nonce_len, + const char *salt, int salt_len, + const char *key, int key_len, + char *dest, int dest_len) +{ + char extract_buf[CEPH_CRYPTO_HMACSHA512_DIGESTSIZE]; + int r = hkdf_extract(salt, salt_len, + key, key_len, + extract_buf, sizeof(extract_buf)); + if (r < 0) { + return r; + } + + int extract_len = r; + +#define FSCRYPT_INFO_STR "fscrypt\x00?" + + char info_str[sizeof(FSCRYPT_INFO_STR) + nonce_len]; + + int len = sizeof(FSCRYPT_INFO_STR) - 1; + memcpy(info_str, FSCRYPT_INFO_STR, len); + + info_str[len - 1] = hkdf_context; + + if (nonce && nonce_len) { + memcpy(info_str + len, nonce, nonce_len); + len += nonce_len; + } + + r = hkdf_expand(extract_buf, extract_len, + info_str, len, + dest, dest_len); + + return r; +} + +static std::string hex_str(const void *p, int len) +{ + bufferlist bl; + bl.append_hole(len); + memcpy(bl.c_str(), p, len); + std::stringstream ss; + bl.hexdump(ss); + return ss.str(); +} + +std::ostream& operator<<(std::ostream& out, const ceph_fscrypt_key_identifier& kid) { + out << hex_str(kid.raw, sizeof(kid.raw)); + return out; +} + +int FSCryptKey::init(const char *k, int klen) { + int r = fscrypt_calc_hkdf(HKDF_CONTEXT_KEY_IDENTIFIER, + nullptr, 0, /* nonce */ + nullptr, 0, /* salt */ + (const char *)k, klen, + identifier.raw, sizeof(identifier.raw)); + if (r < 0) { + return r; + } + + key.append_hole(klen); + memcpy(key.c_str(), k, klen); + + return 0; +} + +int FSCryptKey::calc_hkdf(char ctx_identifier, + const char *nonce, int nonce_len, + char *result, int result_len) { + int r = fscrypt_calc_hkdf(ctx_identifier, + nonce, nonce_len, /* nonce */ + nullptr, 0, /* salt */ + (const char *)key.c_str(), key.length(), + result, result_len); + if (r < 0) { + return r; + } + + return 0; +} + +int ceph_fscrypt_key_identifier::init(const char *k, int klen) { + if (klen != sizeof(raw)) { + return -EINVAL; + } + memcpy(raw, k, klen); + + return 0; +} + +int ceph_fscrypt_key_identifier::init(const struct fscrypt_key_specifier& k) { + if (k.type != FSCRYPT_KEY_SPEC_TYPE_IDENTIFIER) { + return -ENOTSUP; + } + + return init((const char *)k.u.identifier, sizeof(k.u.identifier)); +} + +bool ceph_fscrypt_key_identifier::operator<(const struct ceph_fscrypt_key_identifier& r) const { + return (memcmp(raw, r.raw, sizeof(raw)) < 0); +} + +void FSCryptContext::generate_iv(uint64_t block_num, FSCryptIV& iv) const +{ + memset(&iv, 0, sizeof(iv)); + + // memcpy(iv.u.nonce, nonce, FSCRYPT_FILE_NONCE_SIZE); + iv.u.block_num = block_num; +} + +void FSCryptContext::generate_new_nonce() +{ + cct->random()->get_bytes((char *)nonce, sizeof(nonce)); +} + +void FSCryptKeyHandler::reset(int64_t _epoch, FSCryptKeyRef k) +{ + std::unique_lock wl{lock}; + epoch = _epoch; + key = k; +} + +int64_t FSCryptKeyHandler::get_epoch() +{ + std::shared_lock rl{lock}; + return epoch; +} + +FSCryptKeyRef& FSCryptKeyHandler::get_key() +{ + std::shared_lock rl{lock}; + return key; +} + +int FSCryptKeyStore::create(const char *k, int klen, FSCryptKeyHandlerRef& key_handler) +{ + auto key = std::make_shared(); + + int r = key->init(k, klen); + if (r < 0) { + return r; + } + + std::unique_lock wl{lock}; + + const auto& id = key->get_identifier(); + + auto iter = m.find(id); + if (iter != m.end()) { + /* found a key handler entry, check that there is a key there */ + key_handler = iter->second; + if (key_handler->get_key()) { + return -EEXIST; + } + key_handler->reset(++epoch, key); + } else { + key_handler = std::make_shared(++epoch, key); + m[id] = key_handler; + } + + return 0; +} + +int FSCryptKeyStore::_find(const struct ceph_fscrypt_key_identifier& id, FSCryptKeyHandlerRef& kh) +{ + auto iter = m.find(id); + if (iter == m.end()) { + return -ENOENT; + } + + kh = iter->second; + + return 0; +} + +int FSCryptKeyStore::find(const struct ceph_fscrypt_key_identifier& id, FSCryptKeyHandlerRef& kh) +{ + std::shared_lock rl{lock}; + + return _find(id, kh); +} + +int FSCryptKeyStore::invalidate(const struct ceph_fscrypt_key_identifier& id) +{ + std::unique_lock rl{lock}; + + FSCryptKeyHandlerRef kh; + int r = _find(id, kh); + if (r == -ENOENT) { + return 0; + } else if (r < 0) { + return r; + } + + kh->reset(++epoch, nullptr); + + m.erase(id); + + return 0; +} + +FSCryptKeyValidator::FSCryptKeyValidator(CephContext *cct, FSCryptKeyHandlerRef& kh, int64_t e) : cct(cct), handler(kh), epoch(e) { +} + +bool FSCryptKeyValidator::is_valid() const { + return (handler->get_epoch() == epoch); +} + +FSCryptDenc::FSCryptDenc(CephContext *_cct) : cct(_cct), cipher_ctx(EVP_CIPHER_CTX_new()) {} + +void FSCryptDenc::init_cipher(EVP_CIPHER *_cipher, std::vector params) +{ + cipher = _cipher; + cipher_params = std::move(params); +} + +struct fscrypt_cipher_opt { + const char *str; + bool cts_mode{false}; + bool essiv{false}; + int key_size; + int iv_size; +}; + +static std::map cipher_opt_map = { + { + FSCRYPT_MODE_AES_256_XTS, { + .str = "AES-256-XTS", + .key_size = 64, + .iv_size = 16, + } + }, + { + FSCRYPT_MODE_AES_256_CTS, { + .str = "AES-256-CBC-CTS", + .cts_mode = true, + .key_size = 32, + .iv_size = 16, + } + }, +}; + +bool FSCryptDenc::do_setup_cipher(int enc_mode) +{ + auto iter = cipher_opt_map.find(enc_mode); + if (iter == cipher_opt_map.end()) { + return false; + } + + auto& opts = iter->second; + if (opts.cts_mode) { + init_cipher(EVP_CIPHER_fetch(NULL, opts.str, NULL), + { OSSL_PARAM_construct_utf8_string(OSSL_CIPHER_PARAM_CTS_MODE, (char *)"CS3", 0), + OSSL_PARAM_construct_end()} ); + } else { + init_cipher(EVP_CIPHER_fetch(NULL, opts.str, NULL), {} ); + } + + padding = 4 << (ctx->flags & FSCRYPT_POLICY_FLAGS_PAD_MASK); + key_size = opts.key_size; + iv_size = opts.iv_size; + + return true; +} + +bool FSCryptFNameDenc::setup_cipher() +{ + return do_setup_cipher(ctx->filenames_encryption_mode); +} + +bool FSCryptFDataDenc::setup_cipher() +{ + return do_setup_cipher(ctx->contents_encryption_mode); +} + +bool FSCryptDenc::setup(FSCryptContextRef& _ctx, + FSCryptKeyRef& _master_key) +{ + ctx = _ctx; + master_key = _master_key; + + return setup_cipher(); +} + +int FSCryptDenc::calc_key(char ctx_identifier, + int key_size, + uint64_t block_num) +{ + key.resize(key_size); + int r = master_key->calc_hkdf(ctx_identifier, + (const char *)ctx->nonce, sizeof(ctx->nonce), + key.data(), key_size); + if (r < 0) { + return r; + } + ctx->generate_iv(block_num, iv); + + return 0; +} + +static void sha256(const char *buf, int len, char *hash) +{ + ceph::crypto::ssl::SHA256 hasher; + hasher.Update((const unsigned char *)buf, len); + hasher.Final((unsigned char *)hash); +} + +int FSCryptDenc::decrypt(const char *in_data, int in_len, + char *out_data, int out_len) +{ + int total_len; + + if ((int)key.size() != key_size) { + ldout(cct, 0) << "ERROR: unexpected encryption key size: " << key.size() << " (expected: " << key_size << ")" << dendl; + return -EINVAL; + } + + if ((uint64_t)out_len < (fscrypt_align_ofs(in_len))) { + ldout(cct, 0) << __FILE__ << ":" << __LINE__ << dendl; + return -ERANGE; + } + + if (!EVP_CipherInit_ex2(cipher_ctx, cipher, (const uint8_t *)key.data(), iv.raw, + 0, cipher_params.data())) { + ldout(cct, 0) << __FILE__ << ":" << __LINE__ << dendl; + return -EINVAL; + } + + int len; + + if (EVP_DecryptUpdate(cipher_ctx, (uint8_t *)out_data, &len, (const uint8_t *)in_data, in_len) != 1) { + ldout(cct, 0) << __FILE__ << ":" << __LINE__ << dendl; + return -EINVAL; + } + + total_len = len; + + int ret = EVP_DecryptFinal_ex(cipher_ctx, (uint8_t *)out_data + len, &len); + if (ret != 1) { + return -EINVAL; + } + + total_len += len; + + return total_len; +} + +int FSCryptDenc::encrypt(const char *in_data, int in_len, + char *out_data, int out_len) +{ + int total_len; + + if ((int)key.size() != key_size) { + ldout(cct, 0) << "ERROR: unexpected encryption key size: " << key.size() << " (expected: " << key_size << ")" << dendl; + return -EINVAL; + } + + if ((uint64_t)out_len < (fscrypt_align_ofs(in_len))) { + return -ERANGE; + } + + if (!EVP_CipherInit_ex2(cipher_ctx, cipher, (const uint8_t *)key.data(), iv.raw, + 1, cipher_params.data())) { + return -EINVAL; + } + + int len; + + if (EVP_EncryptUpdate(cipher_ctx, (uint8_t *)out_data, &len, (const uint8_t *)in_data, in_len) != 1) { + return -EINVAL; + } + + total_len = len; + + if (EVP_EncryptFinal_ex(cipher_ctx, (uint8_t *)out_data + len, &len) != 1) { + return -EINVAL; + } + + total_len += len; + + return total_len; +} + +FSCryptDenc::~FSCryptDenc() +{ + EVP_CIPHER_CTX_free(cipher_ctx); +} + +int FSCryptFNameDenc::get_encrypted_fname(const std::string& plain, std::string *encrypted, std::string *alt_name) +{ + auto plain_size = plain.size(); + int dec_size = (plain.size() + padding - 1) & ~(padding - 1); // FIXME, need to be based on policy + if (dec_size > NAME_MAX) { + dec_size = NAME_MAX; + } + + char orig[dec_size]; + memcpy(orig, plain.c_str(), plain_size); + memset(orig + plain_size, 0, dec_size - plain_size); + + char enc_name[NAME_MAX + 64]; /* some extra just in case */ + int r = encrypt(orig, dec_size, + enc_name, sizeof(enc_name)); + + if (r < 0) { + ldout(cct, 0) << __FILE__ << ":" << __LINE__ << ": failed to encrypt filename" << dendl; + return r; + } + + int enc_len = r; + + if (enc_len > CEPH_NOHASH_NAME_MAX) { + *alt_name = std::string(enc_name, enc_len); + char hash[CEPH_CRYPTO_SHA256_DIGESTSIZE]; + char *extra = enc_name + CEPH_NOHASH_NAME_MAX; + + /* hash the extra bytes and overwrite crypttext beyond that point with it */ + int extra_len = enc_len - CEPH_NOHASH_NAME_MAX; + sha256(extra, extra_len, hash); + memcpy(extra, hash, sizeof(hash)); + enc_len = CEPH_NOHASH_NAME_MAX + sizeof(hash); + } else { + alt_name->clear(); + } + + int b64_len = NAME_MAX * 2; // name.size() * 2; + char b64_name[b64_len]; // large enough + int len = fscrypt_fname_armor(enc_name, enc_len, b64_name, b64_len); + + *encrypted = std::string(b64_name, len); + + return len; +} + +int FSCryptFNameDenc::get_decrypted_fname(const std::string& b64enc, const std::string& alt_name, std::string *decrypted) +{ + char enc[NAME_MAX]; + int len = alt_name.size(); + + const char *penc = (len == 0 ? enc : alt_name.c_str()); + + if (len == 0) { + len = fscrypt_fname_unarmor(b64enc.c_str(), b64enc.size(), + enc, sizeof(enc)); + } + + char dec_fname[NAME_MAX + 64]; /* some extra just in case */ + int r = decrypt(penc, len, dec_fname, sizeof(dec_fname)); + + if (r >= 0) { + dec_fname[r] = '\0'; + *decrypted = dec_fname; + } else { + return r; + } + + return r; +} + +struct fscrypt_slink_data { + ceph_le16 len; + char enc[NAME_MAX - 2]; +}; + +int FSCryptFNameDenc::get_encrypted_symlink(const std::string& plain, std::string *encrypted) +{ + auto plain_size = plain.size(); + int dec_size = (plain.size() + 31) & ~31; // FIXME, need to be based on policy + + char orig[dec_size]; + memcpy(orig, plain.c_str(), plain_size); + memset(orig + plain_size, 0, dec_size - plain_size); + + fscrypt_slink_data slink_data; + int r = encrypt(orig, dec_size, + slink_data.enc, sizeof(slink_data.enc)); + + if (r < 0) { + ldout(cct, 0) << __FILE__ << ":" << __LINE__ << ": failed to encrypt filename" << dendl; + return r; + } + + slink_data.len = r; + + int b64_len = NAME_MAX * 2; // name.size() * 2; + char b64_name[b64_len]; // large enough + int len = fscrypt_fname_armor((const char *)&slink_data, slink_data.len + sizeof(slink_data.len), b64_name, b64_len); + + *encrypted = std::string(b64_name, len); + + return len; +} + +int FSCryptFNameDenc::get_decrypted_symlink(const std::string& b64enc, std::string *decrypted) +{ + fscrypt_slink_data slink_data; + + int len = fscrypt_fname_unarmor(b64enc.c_str(), b64enc.size(), + (char *)&slink_data, sizeof(slink_data)); + + char dec_fname[NAME_MAX + 64]; /* some extra just in case */ + + if (slink_data.len > len) { /* should never happen */ + ldout(cct, 0) << __FILE__ << ":" << __LINE__ << ":" << __func__ << "(): ERROR: slink_data.len greater than decrypted buffer (slink_data.len=" << slink_data.len << ", len=" << len << ")" << dendl; + return -EIO; + } + + int r = decrypt(slink_data.enc, slink_data.len, dec_fname, sizeof(dec_fname)); + + if (r >= 0) { + dec_fname[r] = '\0'; + *decrypted = dec_fname; + } else { + return r; + } + + return r; +} + +int FSCryptFDataDenc::decrypt_bl(uint64_t off, uint64_t len, uint64_t pos, const std::vector& holes, bufferlist *bl) +{ + auto data_len = bl->length(); + + auto target_end = off + len; + + bufferlist newbl; + + uint64_t end = off + data_len; + uint64_t cur_block = fscrypt_block_from_ofs(pos); + uint64_t block_off = fscrypt_block_start(pos); + + uint64_t start_block_off = block_off; + + auto hiter = holes.begin(); + + while (pos < target_end) { + bool has_hole = false; + + while (hiter != holes.end()) { + uint64_t hofs = hiter->first; + uint64_t hlen = hiter->second; + uint64_t hend = hofs + hlen - 1; + + if (hend < pos) { + ++hiter; + continue; + } + + if (hofs >= target_end) { + hiter = holes.end(); + break; + } + + has_hole = true; + break; + } +#warning is this the way to do it? + uint64_t needed_pos = (pos > off ? pos : off); + void *data_pos = bl->c_str() + needed_pos - start_block_off; + if (!has_hole && *(uint64_t *)data_pos == 0) { + has_hole = true; + } + + uint64_t read_end = std::min(end, block_off + FSCRYPT_BLOCK_SIZE); + uint64_t read_end_aligned = fscrypt_align_ofs(read_end); + auto chunk_len = read_end_aligned - block_off; + + bufferlist chunk; + + if (!has_hole) { + int r = calc_fdata_key(cur_block); + if (r < 0) { + break; + } + + /* since writes are aligned to block size, if there is a hole then it covers the whole block */ + + chunk.append_hole(chunk_len); + + uint64_t bl_off = pos - start_block_off; + r = decrypt(bl->c_str() + bl_off, chunk_len, + chunk.c_str(), chunk_len); + if (r < 0) { + return r; + } + } else { + chunk.append_zero(chunk_len); + } + + uint64_t needed_end = std::min(target_end, read_end); + int needed_len = needed_end - needed_pos; + chunk.splice(fscrypt_ofs_in_block(needed_pos), needed_len, &newbl); + + pos = read_end; + ++cur_block; + block_off += FSCRYPT_BLOCK_SIZE; + } + + bl->swap(newbl); + + return 0; +} + +int FSCryptFDataDenc::encrypt_bl(uint64_t off, uint64_t len, bufferlist& bl, bufferlist *encbl) +{ + if (off != fscrypt_block_start(off)) { + return -EINVAL; + } + + auto pos = off; + auto target_end = off + len; + auto target_end_block_ofs = fscrypt_block_start(target_end - 1); + + target_end = target_end_block_ofs + FSCRYPT_BLOCK_SIZE; + + if (bl.length() < target_end - off) { + /* fill in zeros at the end if last block is partial */ + bl.append_zero(target_end - off - bl.length()); + } + + auto data_len = bl.length(); + + bufferlist newbl; + + uint64_t end = off + data_len; + uint64_t cur_block = fscrypt_block_from_ofs(pos); + uint64_t block_off = fscrypt_block_start(pos); + + while (pos < target_end) { + uint64_t write_end = std::min(end, block_off + FSCRYPT_BLOCK_SIZE); + uint64_t write_end_aligned = fscrypt_align_ofs(write_end); + auto chunk_len = write_end_aligned - block_off; + + int r = calc_fdata_key(cur_block); + if (r < 0) { + break; + } + + bufferlist chunk; + chunk.append_hole(chunk_len); + + r = encrypt(bl.c_str() + pos - off, chunk_len, + chunk.c_str(), chunk_len); + if (r < 0) { + return r; + } + + newbl.claim_append(chunk); + + pos = write_end; + ++cur_block; + block_off += FSCRYPT_BLOCK_SIZE; + } + + encbl->swap(newbl); + + return 0; +} + +FSCryptContextRef FSCrypt::init_ctx(const std::vector& fscrypt_auth) +{ + if (fscrypt_auth.size() == 0) { + return nullptr; + } + + FSCryptContextRef ctx = std::make_shared(cct); + + bufferlist bl; + bl.append((const char *)fscrypt_auth.data(), fscrypt_auth.size()); + + auto bliter = bl.cbegin(); + try { + ctx->decode(bliter); + } catch (buffer::error& err) { + if (fscrypt_auth.size()) { + ldout(cct, 0) << __func__ << " " << " failed to decode fscrypt_auth:" << fscrypt_hex_str(fscrypt_auth.data(), fscrypt_auth.size()) << dendl; + } else { + ldout(cct, 0) << __func__ << " " << " failed to decode fscrypt_auth: fscrypt_auth.size() == 0" << dendl; + } + return nullptr; + } + + return ctx; +} + +FSCryptDenc *FSCrypt::init_denc(FSCryptContextRef& ctx, FSCryptKeyValidatorRef *kv, + std::function gen_denc) +{ + if (!ctx) { + return nullptr; + } + + FSCryptKeyHandlerRef master_kh; + int r = key_store.find(ctx->master_key_identifier, master_kh); + if (r == 0) { + ldout(cct, 0) << __FILE__ << ":" << __LINE__ << ": fscrypt_key handler found" << dendl; + } else if (r == -ENOENT) { + ldout(cct, 0) << __FILE__ << ":" << __LINE__ << ": fscrypt_key handler not found" << dendl; + return nullptr; + } else { + ldout(cct, 0) << __FILE__ << ":" << __LINE__ << ": error: r=" << r << dendl; + return nullptr; + } + + if (kv) { + *kv = make_shared(cct, master_kh, master_kh->get_epoch()); + } + + auto& master_key = master_kh->get_key(); + + if (!master_key) { + ldout(cct, 0) << __FILE__ << ":" << __LINE__ << ": fscrypt_key key is null" << dendl; + return nullptr; + } + + auto fscrypt_denc = gen_denc(); + + if (!fscrypt_denc->setup(ctx, master_key)) { + ldout(cct, 0) << __FILE__ << ":" << __LINE__ << ":" << __func__ << "(): ERROR: failed to setup denc" << dendl; + return nullptr; + } + + return fscrypt_denc; +} + +FSCryptFNameDencRef FSCrypt::get_fname_denc(FSCryptContextRef& ctx, FSCryptKeyValidatorRef *kv, bool calc_key) +{ + auto pdenc = init_denc(ctx, kv, + [&]() { return new FSCryptFNameDenc(cct); }); + if (!pdenc) { + return nullptr; + } + + auto denc = std::shared_ptr((FSCryptFNameDenc *)pdenc); + + if (calc_key) { + int r = denc->calc_fname_key(); + if (r < 0) { + ldout(cct, 0) << __FILE__ << ":" << __LINE__ << ": failed to init dencoder: r=" << r << dendl; + return nullptr; + } + } + + return denc; +} + +FSCryptFDataDencRef FSCrypt::get_fdata_denc(FSCryptContextRef& ctx, FSCryptKeyValidatorRef *kv) +{ + auto pdenc = init_denc(ctx, kv, + [&]() { return new FSCryptFDataDenc(cct); }); + if (!pdenc) { + return nullptr; + } + + return std::shared_ptr((FSCryptFDataDenc *)pdenc); +} + +void FSCrypt::prepare_data_read(FSCryptContextRef& ctx, + FSCryptKeyValidatorRef *kv, + uint64_t off, + uint64_t len, + uint64_t file_raw_size, + uint64_t *read_start, + uint64_t *read_len, + FSCryptFDataDencRef *denc) +{ + *denc = get_fdata_denc(ctx, kv); + + auto& fscrypt_denc = *denc; + + *read_start = (!fscrypt_denc ? off : fscrypt_block_start(off)); + uint64_t end = off + len; + if (fscrypt_denc) { + end = fscrypt_align_ofs(end); + } + *read_len = end - *read_start; +} diff --git a/src/client/FSCrypt.h b/src/client/FSCrypt.h new file mode 100644 index 00000000000..4e43d57e98d --- /dev/null +++ b/src/client/FSCrypt.h @@ -0,0 +1,370 @@ +#pragma once + +#include "fscrypt_uapi.h" + +#include "common/ceph_mutex.h" + +#include + +#include +#include +#include +#include + +#define FSCRYPT_FILE_NONCE_SIZE 16 + +#define HKDF_CONTEXT_KEY_IDENTIFIER 1 +#define HKDF_CONTEXT_PER_FILE_ENC_KEY 2 + +#define FSCRYPT_BLOCK_SIZE 4096 +#define FSCRYPT_BLOCK_BITS 12 + +#define FSCRYPT_DATA_ALIGNMENT 16 + +static inline uint64_t fscrypt_align_ofs(uint64_t ofs) { + return (ofs + FSCRYPT_DATA_ALIGNMENT - 1) & ~(FSCRYPT_DATA_ALIGNMENT - 1); +} + +static inline int fscrypt_ofs_in_block(uint64_t pos) { + return pos & (FSCRYPT_BLOCK_SIZE - 1); +} + +static inline int fscrypt_block_from_ofs(uint64_t ofs) { + return ofs >> FSCRYPT_BLOCK_BITS; +} + +static inline uint64_t fscrypt_block_start(uint64_t ofs) { + return ofs & ~(FSCRYPT_BLOCK_SIZE - 1); +} + +static inline uint64_t fscrypt_next_block_start(uint64_t ofs) { + return (ofs + FSCRYPT_BLOCK_SIZE - 1) & ~(FSCRYPT_BLOCK_SIZE - 1); +} + +static inline std::string fscrypt_hex_str(const void *p, int len) +{ + if (!p) { + return ""; + } + + bufferlist bl; + bl.append_hole(len); + memcpy(bl.c_str(), p, len); + std::stringstream ss; + bl.hexdump(ss); + return ss.str(); +} + +int fscrypt_fname_armor(const char *src, int src_len, + char *result, int max_len); +int fscrypt_fname_unarmor(const char *src, int src_len, + char *result, int max_len); + +int fscrypt_calc_hkdf(char hkdf_context, + const char *nonce, int nonce_len, + const char *salt, int salt_len, + const char *key, int key_len, + char *dest, int dest_len); + + +struct ceph_fscrypt_key_identifier { +#define FSCRYPT_KEY_IDENTIFIER_LEN 16 + char raw[FSCRYPT_KEY_IDENTIFIER_LEN]; + + int init(const char *k, int klen); + int init(const struct fscrypt_key_specifier& k); + + void decode(bufferlist::const_iterator& bl) { + bl.copy(sizeof(raw), raw); + } + + void encode(bufferlist& bl) const { + bl.append(raw, sizeof(raw)); + } + + bool operator<(const struct ceph_fscrypt_key_identifier& r) const; +}; + +std::ostream& operator<<(std::ostream& out, const ceph_fscrypt_key_identifier& kid); + +class FSCryptKey { + bufferlist key; + ceph_fscrypt_key_identifier identifier; + +public: + int init(const char *k, int klen); + + int calc_hkdf(char ctx_indentifier, + const char *nonce, int nonce_len, + char *result, int result_len); + + const ceph_fscrypt_key_identifier& get_identifier() const { + return identifier; + } + + bufferlist& get_key() { return key; } +}; + +using FSCryptKeyRef = std::shared_ptr; + +#define FSCRYPT_MAX_IV_SIZE 32 +union FSCryptIV { + uint8_t raw[FSCRYPT_MAX_IV_SIZE]; + struct { + ceph_le64 block_num; + uint8_t nonce[FSCRYPT_FILE_NONCE_SIZE]; + } u; +}; + +struct FSCryptPolicy { +public: + uint8_t version; + uint8_t contents_encryption_mode; + uint8_t filenames_encryption_mode; + uint8_t flags; + ceph_fscrypt_key_identifier master_key_identifier; + + virtual ~FSCryptPolicy() {} + + void init(const struct fscrypt_policy_v2& policy) { + version = policy.version; + contents_encryption_mode = policy.contents_encryption_mode; + filenames_encryption_mode = policy.filenames_encryption_mode; + flags = policy.flags; + memcpy(master_key_identifier.raw, policy.master_key_identifier, sizeof(master_key_identifier.raw)); + } + + void decode(bufferlist::const_iterator& env_bl) { + uint32_t v; + + ceph::decode(v, env_bl); + + bufferlist _bl; + ceph::decode(_bl, env_bl); + + auto bl = _bl.cbegin(); + + ceph::decode(version, bl); + ceph::decode(contents_encryption_mode, bl); + ceph::decode(filenames_encryption_mode, bl); + ceph::decode(flags, bl); + + uint32_t __reserved; + ceph::decode(__reserved, bl); + + master_key_identifier.decode(bl); + + decode_extra(bl); + } + + virtual void decode_extra(bufferlist::const_iterator& bl) {} + + void encode(bufferlist& env_bl) const { + uint32_t v = 1; + ceph::encode(v, env_bl); + + bufferlist bl; + + ceph::encode(version, bl); + ceph::encode(contents_encryption_mode, bl); + ceph::encode(filenames_encryption_mode, bl); + ceph::encode(flags, bl); + + uint32_t __reserved = 0; + ceph::encode(__reserved, bl); + + master_key_identifier.encode(bl); + + encode_extra(bl); + + ceph::encode(bl, env_bl); + + } + + virtual void encode_extra(bufferlist& bl) const {} + + void convert_to(struct fscrypt_policy_v2 *dest) { + dest->version = version; + dest->contents_encryption_mode = contents_encryption_mode; + dest->filenames_encryption_mode = filenames_encryption_mode; + dest->flags = flags; + memset(dest->__reserved, 0, sizeof(dest->__reserved)); + memcpy(dest->master_key_identifier, master_key_identifier.raw, sizeof(master_key_identifier.raw)); + } +}; + +using FSCryptPolicyRef = std::shared_ptr; + +struct FSCryptContext : public FSCryptPolicy { + CephContext *cct; + + FSCryptContext(CephContext *_cct) : cct(_cct) {} + + uint8_t nonce[FSCRYPT_FILE_NONCE_SIZE]; + + void decode_extra(bufferlist::const_iterator& bl) override { + bl.copy(sizeof(nonce), (char *)nonce); + } + + void encode_extra(bufferlist& bl) const override { + bl.append((char *)nonce, sizeof(nonce)); + } + + void generate_new_nonce(); + void generate_iv(uint64_t block_num, FSCryptIV& iv) const; +}; + +using FSCryptContextRef = std::shared_ptr; + +class FSCryptDenc { +protected: + CephContext *cct; + + FSCryptContextRef ctx; + FSCryptKeyRef master_key; + + std::vector key; + FSCryptIV iv; + + int padding = 1; + int key_size = 0; + int iv_size = 0; + + EVP_CIPHER *cipher; + EVP_CIPHER_CTX *cipher_ctx; + std::vector cipher_params; + + int calc_key(char ctx_identifier, + int key_size, + uint64_t block_num); + + bool do_setup_cipher(int encryption_mode); + +public: + FSCryptDenc(CephContext *_cct); + virtual ~FSCryptDenc(); + + virtual bool setup_cipher() = 0; + + void init_cipher(EVP_CIPHER *cipher, std::vector params); + bool setup(FSCryptContextRef& _ctx, + FSCryptKeyRef& _master_key); + + int calc_fname_key() { + return calc_key(HKDF_CONTEXT_PER_FILE_ENC_KEY, key_size, 0); + } + + int calc_fdata_key(uint64_t block_num) { + return calc_key(HKDF_CONTEXT_PER_FILE_ENC_KEY, key_size, block_num); + } + + int decrypt(const char *in_data, int in_len, + char *out_data, int out_len); + int encrypt(const char *in_data, int in_len, + char *out_data, int out_len); +}; + +using FSCryptDencRef = std::shared_ptr; + +class FSCryptFNameDenc : public FSCryptDenc { +public: + FSCryptFNameDenc(CephContext *_cct) : FSCryptDenc(_cct) {} + + bool setup_cipher() override; + + int get_encrypted_fname(const std::string& plain, std::string *encrypted, std::string *alt_name); + int get_decrypted_fname(const std::string& b64enc, const std::string& alt_name, std::string *decrypted); + + int get_encrypted_symlink(const std::string& plain, std::string *encrypted); + int get_decrypted_symlink(const std::string& b64enc, std::string *decrypted); +}; + +using FSCryptFNameDencRef = std::shared_ptr; + +class FSCryptFDataDenc : public FSCryptDenc { +public: + FSCryptFDataDenc(CephContext *_cct) : FSCryptDenc(_cct) {} + + using Segment = std::pair; + + bool setup_cipher() override; + + int decrypt_bl(uint64_t off, uint64_t len, uint64_t pos, const std::vector& holes, bufferlist *bl); + int encrypt_bl(uint64_t off, uint64_t len, bufferlist& bl, bufferlist *encbl); +}; + +using FSCryptFDataDencRef = std::shared_ptr; + +class FSCryptKeyHandler { + ceph::shared_mutex lock = ceph::make_shared_mutex("FSCryptKeyHandler"); + int64_t epoch = -1; + FSCryptKeyRef key; +public: + FSCryptKeyHandler() {} + FSCryptKeyHandler(int64_t epoch, FSCryptKeyRef k) : epoch(epoch), key(k) {} + + void reset(int64_t epoch, FSCryptKeyRef k); + + int64_t get_epoch(); + FSCryptKeyRef& get_key(); +}; + +using FSCryptKeyHandlerRef = std::shared_ptr; + +class FSCryptKeyStore { + CephContext *cct; + + ceph::shared_mutex lock = ceph::make_shared_mutex("FSCryptKeyStore"); + int64_t epoch = 0; + std::map m; + + int _find(const struct ceph_fscrypt_key_identifier& id, FSCryptKeyHandlerRef& key); +public: + FSCryptKeyStore(CephContext *_cct) : cct(_cct) {} + + int create(const char *k, int klen, FSCryptKeyHandlerRef& key); + int find(const struct ceph_fscrypt_key_identifier& id, FSCryptKeyHandlerRef& key); + int invalidate(const struct ceph_fscrypt_key_identifier& id); +}; + +struct FSCryptKeyValidator { + CephContext *cct; + FSCryptKeyHandlerRef handler; + int64_t epoch; + + FSCryptKeyValidator(CephContext *cct, FSCryptKeyHandlerRef& kh, int64_t e); + + bool is_valid() const; +}; + +using FSCryptKeyValidatorRef = std::shared_ptr; + + +class FSCrypt { + CephContext *cct; + + FSCryptKeyStore key_store; + + FSCryptDenc *init_denc(FSCryptContextRef& ctx, FSCryptKeyValidatorRef *kv, + std::function gen_denc); +public: + FSCrypt(CephContext *_cct) : cct(_cct), key_store(cct) {} + + FSCryptContextRef init_ctx(const std::vector& fscrypt_auth); + + FSCryptKeyStore& get_key_store() { + return key_store; + } + + FSCryptFNameDencRef get_fname_denc(FSCryptContextRef& ctx, FSCryptKeyValidatorRef *kv, bool calc_key); + FSCryptFDataDencRef get_fdata_denc(FSCryptContextRef& ctx, FSCryptKeyValidatorRef *kv); + + void prepare_data_read(FSCryptContextRef& ctx, + FSCryptKeyValidatorRef *kv, + uint64_t off, + uint64_t len, + uint64_t file_raw_size, + uint64_t *read_start, + uint64_t *read_len, + FSCryptFDataDencRef *denc); +}; diff --git a/src/client/Inode.cc b/src/client/Inode.cc index f0d2dfcdbec..0c2348825b5 100644 --- a/src/client/Inode.cc +++ b/src/client/Inode.cc @@ -155,7 +155,11 @@ void Inode::make_nosnap_relative_path(filepath& p) Dentry *dn = get_first_parent(); ceph_assert(dn->dir && dn->dir->parent_inode); dn->dir->parent_inode->make_nosnap_relative_path(p); - p.push_dentry(dn->name); + if (dn->enc_name) { + p.push_dentry(*dn->enc_name); + } else { + p.push_dentry(dn->name); + } } else { p = filepath(ino); } @@ -845,4 +849,43 @@ void Inode::mark_caps_clean() dirty_cap_item.remove_myself(); } +FSCryptContextRef Inode::init_fscrypt_ctx(FSCrypt *fscrypt) +{ + return fscrypt->init_ctx(fscrypt_auth); +} + +void Inode::gen_inherited_fscrypt_auth(std::vector *fsa) +{ + if (!fscrypt_ctx) { +#warning need to make sure that we do not skip entire subtree somehow + return; + } + + FSCryptContext new_ctx = *fscrypt_ctx; + + new_ctx.generate_new_nonce(); + + bufferlist bl; + new_ctx.encode(bl); + + fsa->resize(bl.length()); + memcpy(fsa->data(), bl.c_str(), bl.length()); +} + +uint64_t Inode::effective_size() const +{ + if (fscrypt_file.size() < sizeof(uint64_t)) { + return size; + } + return *(ceph_le64 *)fscrypt_file.data(); +} + +void Inode::set_effective_size(uint64_t size) +{ + if (fscrypt_file.size() < sizeof(uint64_t)) { + fscrypt_file.resize(sizeof(uint64_t)); + } + + *(ceph_le64 *)fscrypt_file.data() = size; +} diff --git a/src/client/Inode.h b/src/client/Inode.h index 90fbc6e254c..7970e84f61e 100644 --- a/src/client/Inode.h +++ b/src/client/Inode.h @@ -24,6 +24,12 @@ #include "UserPerm.h" #include "Delegation.h" +#include "FSCrypt.h" + +#ifndef S_ENCRYPTED +#define S_ENCRYPTED (1 << 14) +#endif + class Client; class Dentry; class Dir; @@ -32,6 +38,7 @@ struct Inode; class MetaRequest; class filepath; class Fh; +class FSCrypt; class Cap { public: @@ -170,14 +177,25 @@ struct Inode : RefCountedObject { decltype(InodeStat::optmetadata) optmetadata; using optkind_t = decltype(InodeStat::optmetadata)::optkind_t; + FSCryptContextRef fscrypt_ctx; + FSCryptKeyValidatorRef fscrypt_key_validator; + + uint64_t effective_size() const; + void set_effective_size(uint64_t size); + bool is_fscrypt_enabled() { return !!fscrypt_auth.size(); } + FSCryptContextRef init_fscrypt_ctx(FSCrypt *fscrypt); + + void gen_inherited_fscrypt_auth(std::vector *ctx); + bool is_root() const { return ino == CEPH_INO_ROOT; } bool is_symlink() const { return (mode & S_IFMT) == S_IFLNK; } bool is_dir() const { return (mode & S_IFMT) == S_IFDIR; } bool is_file() const { return (mode & S_IFMT) == S_IFREG; } + bool is_encrypted() const { return (mode & S_ENCRYPTED) == S_ENCRYPTED; } bool has_dir_layout() const { return layout != file_layout_t(); @@ -240,6 +258,7 @@ struct Inode : RefCountedObject { uint64_t ll_ref = 0; // separate ref count for ll client xlist dentries; // if i'm linked to a dentry. std::string symlink; // symlink content, if it's a symlink + std::string symlink_plain; // decoded symlink (in-memory only) std::map xattrs; std::map fragmap; // known frag -> mds mappings std::map> frag_repmap; // non-auth mds mappings diff --git a/src/client/fscrypt_uapi.h b/src/client/fscrypt_uapi.h new file mode 100644 index 00000000000..ede917061c6 --- /dev/null +++ b/src/client/fscrypt_uapi.h @@ -0,0 +1,215 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +/* + * fscrypt user API + * + * These ioctls can be used on filesystems that support fscrypt. See the + * "User API" section of Documentation/filesystems/fscrypt.rst. + */ +#ifndef _UAPI_LINUX_FSCRYPT_H +#define _UAPI_LINUX_FSCRYPT_H + +#include +#include + +/* Encryption policy flags */ +#define FSCRYPT_POLICY_FLAGS_PAD_4 0x00 +#define FSCRYPT_POLICY_FLAGS_PAD_8 0x01 +#define FSCRYPT_POLICY_FLAGS_PAD_16 0x02 +#define FSCRYPT_POLICY_FLAGS_PAD_32 0x03 +#define FSCRYPT_POLICY_FLAGS_PAD_MASK 0x03 +#define FSCRYPT_POLICY_FLAG_DIRECT_KEY 0x04 +#define FSCRYPT_POLICY_FLAG_IV_INO_LBLK_64 0x08 +#define FSCRYPT_POLICY_FLAG_IV_INO_LBLK_32 0x10 + +/* Encryption algorithms */ +#define FSCRYPT_MODE_AES_256_XTS 1 +#define FSCRYPT_MODE_AES_256_CTS 4 +#define FSCRYPT_MODE_AES_128_CBC 5 +#define FSCRYPT_MODE_AES_128_CTS 6 +#define FSCRYPT_MODE_SM4_XTS 7 +#define FSCRYPT_MODE_SM4_CTS 8 +#define FSCRYPT_MODE_ADIANTUM 9 +#define FSCRYPT_MODE_AES_256_HCTR2 10 +/* If adding a mode number > 10, update FSCRYPT_MODE_MAX in fscrypt_private.h */ + +/* + * Legacy policy version; ad-hoc KDF and no key verification. + * For new encrypted directories, use fscrypt_policy_v2 instead. + * + * Careful: the .version field for this is actually 0, not 1. + */ +#define FSCRYPT_POLICY_V1 0 +#define FSCRYPT_KEY_DESCRIPTOR_SIZE 8 +struct fscrypt_policy_v1 { + __u8 version; + __u8 contents_encryption_mode; + __u8 filenames_encryption_mode; + __u8 flags; + __u8 master_key_descriptor[FSCRYPT_KEY_DESCRIPTOR_SIZE]; +}; + +/* + * Process-subscribed "logon" key description prefix and payload format. + * Deprecated; prefer FS_IOC_ADD_ENCRYPTION_KEY instead. + */ +#define FSCRYPT_KEY_DESC_PREFIX "fscrypt:" +#define FSCRYPT_KEY_DESC_PREFIX_SIZE 8 +#define FSCRYPT_MAX_KEY_SIZE 64 +struct fscrypt_key { + __u32 mode; + __u8 raw[FSCRYPT_MAX_KEY_SIZE]; + __u32 size; +}; + +/* + * New policy version with HKDF and key verification (recommended). + */ +#define FSCRYPT_POLICY_V2 2 +#define FSCRYPT_KEY_IDENTIFIER_SIZE 16 +struct fscrypt_policy_v2 { + __u8 version; + __u8 contents_encryption_mode; + __u8 filenames_encryption_mode; + __u8 flags; + __u8 __reserved[4]; + __u8 master_key_identifier[FSCRYPT_KEY_IDENTIFIER_SIZE]; +}; + +struct fscrypt_policy_arg { + union { + struct fscrypt_policy_v1 v1; + struct fscrypt_policy_v2 v2; + } policy; +}; /* output */ + +/* Struct passed to FS_IOC_GET_ENCRYPTION_POLICY_EX */ +struct fscrypt_get_policy_ex_arg { + __u64 policy_size; /* input/output */ + union { + __u8 version; + struct fscrypt_policy_v1 v1; + struct fscrypt_policy_v2 v2; + } policy; /* output */ +}; + +/* + * v1 policy keys are specified by an arbitrary 8-byte key "descriptor", + * matching fscrypt_policy_v1::master_key_descriptor. + */ +#define FSCRYPT_KEY_SPEC_TYPE_DESCRIPTOR 1 + +/* + * v2 policy keys are specified by a 16-byte key "identifier" which the kernel + * calculates as a cryptographic hash of the key itself, + * matching fscrypt_policy_v2::master_key_identifier. + */ +#define FSCRYPT_KEY_SPEC_TYPE_IDENTIFIER 2 + +/* + * Specifies a key, either for v1 or v2 policies. This doesn't contain the + * actual key itself; this is just the "name" of the key. + */ +struct fscrypt_key_specifier { + __u32 type; /* one of FSCRYPT_KEY_SPEC_TYPE_* */ + __u32 __reserved; + union { + __u8 __reserved[32]; /* reserve some extra space */ + __u8 descriptor[FSCRYPT_KEY_DESCRIPTOR_SIZE]; + __u8 identifier[FSCRYPT_KEY_IDENTIFIER_SIZE]; + } u; +}; + +/* + * Payload of Linux keyring key of type "fscrypt-provisioning", referenced by + * fscrypt_add_key_arg::key_id as an alternative to fscrypt_add_key_arg::raw. + */ +struct fscrypt_provisioning_key_payload { + __u32 type; + __u32 __reserved; + __u8 raw[]; +}; + +/* Struct passed to FS_IOC_ADD_ENCRYPTION_KEY */ +struct fscrypt_add_key_arg { + struct fscrypt_key_specifier key_spec; + __u32 raw_size; + __u32 key_id; + __u32 __reserved[8]; + __u8 raw[]; +}; + +/* Struct passed to FS_IOC_ADD_ENCRYPTION_KEY */ +struct fscrypt_add_key64_arg { + struct fscrypt_key_specifier key_spec; + __u32 raw_size; + __u32 key_id; + __u32 __reserved[8]; + __u8 raw[64]; +}; + +/* Struct passed to FS_IOC_REMOVE_ENCRYPTION_KEY */ +struct fscrypt_remove_key_arg { + struct fscrypt_key_specifier key_spec; +#define FSCRYPT_KEY_REMOVAL_STATUS_FLAG_FILES_BUSY 0x00000001 +#define FSCRYPT_KEY_REMOVAL_STATUS_FLAG_OTHER_USERS 0x00000002 + __u32 removal_status_flags; /* output */ + __u32 __reserved[5]; +}; + +/* Struct passed to FS_IOC_GET_ENCRYPTION_KEY_STATUS */ +struct fscrypt_get_key_status_arg { + /* input */ + struct fscrypt_key_specifier key_spec; + __u32 __reserved[6]; + + /* output */ +#define FSCRYPT_KEY_STATUS_ABSENT 1 +#define FSCRYPT_KEY_STATUS_PRESENT 2 +#define FSCRYPT_KEY_STATUS_INCOMPLETELY_REMOVED 3 + __u32 status; +#define FSCRYPT_KEY_STATUS_FLAG_ADDED_BY_SELF 0x00000001 + __u32 status_flags; + __u32 user_count; + __u32 __out_reserved[13]; +}; + +#define FS_IOC_SET_ENCRYPTION_POLICY _IOR('f', 19, struct fscrypt_policy_v1) +#define FS_IOC_SET_ENCRYPTION_POLICY_RESTRICTED _IOWR('f', 19, struct fscrypt_policy_arg) +#define FS_IOC_GET_ENCRYPTION_PWSALT _IOW('f', 20, __u8[16]) +#define FS_IOC_GET_ENCRYPTION_POLICY _IOW('f', 21, struct fscrypt_policy_v1) +#define FS_IOC_GET_ENCRYPTION_POLICY_EX _IOWR('f', 22, __u8[9]) /* size + version */ +#define FS_IOC_GET_ENCRYPTION_POLICY_EX_RESTRICTED _IOWR('f', 22, struct fscrypt_get_policy_ex_arg) /* size + version */ +#define FS_IOC_ADD_ENCRYPTION_KEY _IOWR('f', 23, struct fscrypt_add_key_arg) +#define FS_IOC_ADD_ENCRYPTION_KEY64 _IOWR('f', 23, struct fscrypt_add_key64_arg) +#define FS_IOC_REMOVE_ENCRYPTION_KEY _IOWR('f', 24, struct fscrypt_remove_key_arg) +#define FS_IOC_REMOVE_ENCRYPTION_KEY_ALL_USERS _IOWR('f', 25, struct fscrypt_remove_key_arg) +#define FS_IOC_GET_ENCRYPTION_KEY_STATUS _IOWR('f', 26, struct fscrypt_get_key_status_arg) +#define FS_IOC_GET_ENCRYPTION_NONCE _IOR('f', 27, __u8[16]) + +/**********************************************************************/ + +/* old names; don't add anything new here! */ +#ifndef __KERNEL__ +#define fscrypt_policy fscrypt_policy_v1 +#define FS_KEY_DESCRIPTOR_SIZE FSCRYPT_KEY_DESCRIPTOR_SIZE +#define FS_POLICY_FLAGS_PAD_4 FSCRYPT_POLICY_FLAGS_PAD_4 +#define FS_POLICY_FLAGS_PAD_8 FSCRYPT_POLICY_FLAGS_PAD_8 +#define FS_POLICY_FLAGS_PAD_16 FSCRYPT_POLICY_FLAGS_PAD_16 +#define FS_POLICY_FLAGS_PAD_32 FSCRYPT_POLICY_FLAGS_PAD_32 +#define FS_POLICY_FLAGS_PAD_MASK FSCRYPT_POLICY_FLAGS_PAD_MASK +#define FS_POLICY_FLAG_DIRECT_KEY FSCRYPT_POLICY_FLAG_DIRECT_KEY +#define FS_POLICY_FLAGS_VALID 0x07 /* contains old flags only */ +#define FS_ENCRYPTION_MODE_INVALID 0 /* never used */ +#define FS_ENCRYPTION_MODE_AES_256_XTS FSCRYPT_MODE_AES_256_XTS +#define FS_ENCRYPTION_MODE_AES_256_GCM 2 /* never used */ +#define FS_ENCRYPTION_MODE_AES_256_CBC 3 /* never used */ +#define FS_ENCRYPTION_MODE_AES_256_CTS FSCRYPT_MODE_AES_256_CTS +#define FS_ENCRYPTION_MODE_AES_128_CBC FSCRYPT_MODE_AES_128_CBC +#define FS_ENCRYPTION_MODE_AES_128_CTS FSCRYPT_MODE_AES_128_CTS +#define FS_ENCRYPTION_MODE_ADIANTUM FSCRYPT_MODE_ADIANTUM +#define FS_KEY_DESC_PREFIX FSCRYPT_KEY_DESC_PREFIX +#define FS_KEY_DESC_PREFIX_SIZE FSCRYPT_KEY_DESC_PREFIX_SIZE +#define FS_MAX_KEY_SIZE FSCRYPT_MAX_KEY_SIZE +#endif /* !__KERNEL__ */ + +#endif /* _UAPI_LINUX_FSCRYPT_H */ diff --git a/src/client/fuse_ll.cc b/src/client/fuse_ll.cc index b47f9840827..4452473a436 100644 --- a/src/client/fuse_ll.cc +++ b/src/client/fuse_ll.cc @@ -40,6 +40,10 @@ #include "Client.h" #include "Fh.h" #include "ioctl.h" +#include "fscrypt_uapi.h" +#include "FSCrypt.h" +#include "Inode.h" +#include "Dir.h" #include "common/config.h" #include "include/ceph_assert.h" #include "include/cephfs/ceph_ll_client.h" @@ -931,10 +935,11 @@ static void fuse_ll_ioctl(fuse_req_t req, fuse_ino_t ino, #else int cmd, #endif - void *arg, struct fuse_file_info *fi, + void *_arg, struct fuse_file_info *fi, unsigned flags, const void *in_buf, size_t in_bufsz, size_t out_bufsz) { CephFuse::Handle *cfuse = fuse_ll_req_prepare(req); + const struct fuse_ctx *ctx = fuse_req_ctx(req); if (flags & FUSE_IOCTL_COMPAT) { fuse_reply_err(req, ENOSYS); @@ -954,6 +959,191 @@ static void fuse_ll_ioctl(fuse_req_t req, fuse_ino_t ino, fuse_reply_ioctl(req, 0, &l, sizeof(struct ceph_ioctl_layout)); } break; + case FS_IOC_GET_ENCRYPTION_POLICY_EX_RESTRICTED: + case FS_IOC_GET_ENCRYPTION_POLICY_EX: { + auto arg = (fscrypt_get_policy_ex_arg *)in_buf; + + generic_dout(0) << __FILE__ << ":" << __LINE__ << ": in_bufsz=" << in_bufsz << " out_bufsz=" << out_bufsz << " FS_IOC_GET_ENCRYPTION_POLICY_EX buffer:\n" << fscrypt_hex_str(in_buf, in_bufsz) << dendl; + + struct fscrypt_get_policy_ex_arg out_arg; + if (out_bufsz < sizeof(out_arg.policy)) { + fuse_reply_err(req, ERANGE); + break; + } + + Fh *fh = (Fh*)fi->fh; + Inode *in = fh->inode.get(); + + if (in->fscrypt_ctx) { + in->fscrypt_ctx->convert_to(&out_arg.policy.v2); + out_arg.policy_size = sizeof(out_arg.policy); + + fuse_reply_ioctl(req, 0, &out_arg, sizeof(out_arg)); + break; + } + + fuse_reply_err(req, ENODATA); + } + break; + case FS_IOC_ADD_ENCRYPTION_KEY64: + case FS_IOC_ADD_ENCRYPTION_KEY: { + if (!in_buf + || in_bufsz < sizeof(fscrypt_add_key_arg)) { + fuse_reply_err(req, EFAULT); + break; + } + + auto arg = (fscrypt_add_key_arg *)in_buf; + + generic_dout(0) << __FILE__ << ":" << __LINE__ << ": in_bufsz=" << in_bufsz << " ioctl buffer:\n" << fscrypt_hex_str(in_buf, in_bufsz) << dendl; + + if (arg->key_spec.type != FSCRYPT_KEY_SPEC_TYPE_IDENTIFIER) { + fuse_reply_err(req, ENOTSUP); + break; + } + + generic_dout(0) << __FILE__ << ":" << __LINE__ << ": key_spec.type=" << arg->key_spec.type << " key_spec buffer:\n" << fscrypt_hex_str(&arg->key_spec, sizeof(arg->key_spec)) << dendl; + generic_dout(0) << __FILE__ << ":" << __LINE__ << ": raw_size=" << arg->raw_size << " key_id=" << arg->key_id << dendl; + generic_dout(0) << __FILE__ << ":" << __LINE__ << ": raw:\n" << fscrypt_hex_str(arg->raw, arg->raw_size) << dendl; + + if (arg->key_id == 0 && + in_bufsz < sizeof(*arg) + arg->raw_size) { + generic_dout(0) << __FILE__ << ":" << __LINE__ << ": in_bufsz=" << in_bufsz << " too short, expected=" << sizeof(*arg) + arg->raw_size << dendl; + fuse_reply_err(req, ERANGE); + break; + } + + int r = cfuse->client->add_fscrypt_key((const char *)arg->raw, arg->raw_size, nullptr); + if (r < 0) { + generic_dout(0) << __FILE__ << ":" << __LINE__ << ": failed to create a new key: r=" << r << dendl; + fuse_reply_err(req, -r); + break; + } + + fuse_reply_ioctl(req, 0, nullptr, 0); + break; + } + break; + case FS_IOC_REMOVE_ENCRYPTION_KEY: { + if (!in_buf + || in_bufsz < sizeof(fscrypt_remove_key_arg)) { + fuse_reply_err(req, EFAULT); + break; + } + + generic_dout(0) << __FILE__ << ":" << __LINE__ << ": FS_IOC_REMOVE_ENCRYPTION_KEY ioctl buffer:\n" << fscrypt_hex_str(in_buf, in_bufsz) << dendl; + + auto arg = (fscrypt_remove_key_arg *)in_buf; + if (arg->key_spec.type != FSCRYPT_KEY_SPEC_TYPE_IDENTIFIER) { + fuse_reply_err(req, ENOTSUP); + break; + } + + ceph_fscrypt_key_identifier kid; + int r = kid.init(arg->key_spec); + if (r < 0) { + fuse_reply_err(req, -r); + break; + } + + /* FIXME: handle busy cases */ + r = cfuse->client->remove_fscrypt_key(kid); + if (r < 0) { + fuse_reply_err(req, -r); + break; + } + + arg->removal_status_flags = 0; /* FIXME */ + fuse_reply_ioctl(req, 0, arg, sizeof(*arg)); + break; + } + case FS_IOC_SET_ENCRYPTION_POLICY: + case FS_IOC_SET_ENCRYPTION_POLICY_RESTRICTED: { + generic_dout(0) << __FILE__ << ":" << __LINE__ << ": FS_IOC_SET_ENCRYPTION_POLICY arg=" << (void *)_arg << " in_buf=" << (void *)in_buf << dendl; + if (!in_buf) { + generic_dout(0) << __FILE__ << ":" << __LINE__ << ": ioctl buffer " << dendl; + fuse_reply_err(req, EINVAL); + break; + } + + generic_dout(0) << __FILE__ << ":" << __LINE__ << ": ioctl buffer:\n" << fscrypt_hex_str(in_buf, in_bufsz) << dendl; + + auto arg = (fscrypt_policy_arg *)in_buf; + if (in_bufsz < sizeof(arg->policy)) { + fuse_reply_err(req, ERANGE); + break; + } + + if (arg->policy.v1.version == 0) { + fuse_reply_err(req, ENOTSUP); + break; + } + + if (arg->policy.v1.version != 2) { + fuse_reply_err(req, EINVAL); + break; + } + + auto& policy = arg->policy.v2; + + Fh *fh = (Fh*)fi->fh; + Inode *in = fh->inode.get(); + generic_dout(0) << __FILE__ << ":" << __LINE__ << ": XXXX ioctl ino=" << in->ino << dendl; + + int r = cfuse->client->ll_set_fscrypt_policy_v2(in, policy); + if (r < 0) { + fuse_reply_err(req, -r); + break; + } + + generic_dout(0) << __FILE__ << ":" << __LINE__ << ": set fscrypt policy: success" << dendl; + + fuse_reply_ioctl(req, 0, nullptr, 0); + break; + } + break; + case FS_IOC_GET_ENCRYPTION_KEY_STATUS: { + if (!in_buf || + in_bufsz != sizeof(fscrypt_get_key_status_arg)) { + generic_dout(0) << __FILE__ << ":" << __LINE__ << ": ioctl buffer " << dendl; + fuse_reply_err(req, EINVAL); + break; + } + + generic_dout(0) << __FILE__ << ":" << __LINE__ << ": FS_IOC_GET_ENCRYPTION_KEY_STATUS ioctl buffer:\n" << fscrypt_hex_str(in_buf, in_bufsz) << dendl; + + auto arg = (fscrypt_get_key_status_arg *)in_buf; + if (arg->key_spec.type != FSCRYPT_KEY_SPEC_TYPE_IDENTIFIER) { + fuse_reply_err(req, ENOTSUP); + break; + } + + ceph_fscrypt_key_identifier kid; + int r = kid.init(arg->key_spec); + if (r < 0) { + fuse_reply_err(req, -r); + break; + } + + FSCryptKeyHandlerRef kh; + r = cfuse->client->fscrypt->get_key_store().find(kid, kh); + if (r < 0 && r != -ENOENT) { + fuse_reply_err(req, -r); + break; + } + + bool found = (r == 0 && kh->get_key()); + + generic_dout(0) << __FILE__ << ":" << __LINE__ << ": FS_IOC_GET_ENCRYPTION_KEY_STATUS found=" << found << dendl; + + /* TODO: return correct info */ + arg->status = (found ? FSCRYPT_KEY_STATUS_PRESENT : FSCRYPT_KEY_STATUS_ABSENT); + arg->status_flags = (found ? 0x1 : 0); /* FIXME */ + arg->user_count = !!found; /* FIXME */ + + fuse_reply_ioctl(req, 0, arg, sizeof(*arg)); + } + break; default: fuse_reply_err(req, EINVAL); } @@ -1294,6 +1484,8 @@ static void do_init(void *data, fuse_conn_info *conn) fuse_apply_conn_info_opts(cfuse->conn_opts, conn); #endif + generic_dout(0) << __FILE__ << ":" << __LINE__ << ": conn proto ver " << conn->proto_major << ":" << conn->proto_minor << dendl; + if(conn->capable & FUSE_CAP_SPLICE_MOVE) conn->want |= FUSE_CAP_SPLICE_MOVE; diff --git a/src/common/ceph_crypto.h b/src/common/ceph_crypto.h index 8e91486ada2..52df7e41191 100644 --- a/src/common/ceph_crypto.h +++ b/src/common/ceph_crypto.h @@ -16,6 +16,7 @@ #define CEPH_CRYPTO_SHA256_DIGESTSIZE 32 #define CEPH_CRYPTO_HMACSHA512_DIGESTSIZE 64 #define CEPH_CRYPTO_SHA512_DIGESTSIZE 64 +#define CEPH_CRYPTO_HMACSHA512_DIGESTSIZE 64 #include #include @@ -201,6 +202,7 @@ namespace TOPNSPC::crypto { using ssl::SHA1; using ssl::SHA512; + using ssl::HMACSHA512; using ssl::HMACSHA256; using ssl::HMACSHA1; using ssl::HMACSHA512; diff --git a/src/include/cephfs/libcephfs.h b/src/include/cephfs/libcephfs.h index d84b9a2a397..015ab945966 100644 --- a/src/include/cephfs/libcephfs.h +++ b/src/include/cephfs/libcephfs.h @@ -132,6 +132,9 @@ struct ceph_ll_io_info { bool syncdataonly; }; +struct ceph_fscrypt_key_identifier; +struct fscrypt_policy_v2; + /* setattr mask bits (up to an int in size) */ #ifndef CEPH_SETATTR_MODE #define CEPH_SETATTR_MODE (1 << 0) @@ -1997,6 +2000,40 @@ int ceph_debug_get_fd_caps(struct ceph_mount_info *cmount, int fd); */ int ceph_debug_get_file_caps(struct ceph_mount_info *cmount, const char *path); +/** + * Add fscrypt encryption key to the in-memory key manager + * + * @param cmount the ceph mount handle to use. + * @param key_data key data + * @param key_len key data length + * @param kid to hold the returned key identifier + * @returns zero on success, other returns a negative error code. + */ +int ceph_add_fscrypt_key(struct ceph_mount_info *cmount, + const char *key_data, int key_len, + struct ceph_fscrypt_key_identifier *kid); + +/** + * Remove fscrypt encryption key from the in-memory key manager + * + * @param cmount the ceph mount handle to use. + * @param kid pointer to the key identifier + * @returns zero on success, other returns a negative error code. + */ +int ceph_remove_fscrypt_key(struct ceph_mount_info *cmount, + const struct ceph_fscrypt_key_identifier *kid); + +/** + * Set encryption policy on a directory. + * + * @param cmount the ceph mount handle to use. + * @param fd open directory file descriptor + * @param policy pointer to to the fscrypt v2 policy + * @returns zero on success, other returns a negative error code. + */ +int ceph_set_fscrypt_policy_v2(struct ceph_mount_info *cmount, + int fd, const struct fscrypt_policy_v2 *policy); + /* Low Level */ struct Inode *ceph_ll_get_inode(struct ceph_mount_info *cmount, vinodeno_t vino); diff --git a/src/libcephfs.cc b/src/libcephfs.cc index f6d522dfe64..5bf485a7bc4 100644 --- a/src/libcephfs.cc +++ b/src/libcephfs.cc @@ -2504,6 +2504,35 @@ extern "C" void ceph_finish_reclaim(class ceph_mount_info *cmount) cmount->get_client()->finish_reclaim(); } +extern "C" int ceph_add_fscrypt_key(struct ceph_mount_info *cmount, + const char *key_data, int key_len, + struct ceph_fscrypt_key_identifier *kid) +{ + if (!cmount->is_mounted()) + return -CEPHFS_ENOTCONN; + + return cmount->get_client()->add_fscrypt_key(key_data, key_len, kid); +} + +extern "C" int ceph_remove_fscrypt_key(struct ceph_mount_info *cmount, + const struct ceph_fscrypt_key_identifier *kid) +{ + if (!cmount->is_mounted()) + return -CEPHFS_ENOTCONN; + + return cmount->get_client()->remove_fscrypt_key(*kid); +} + +extern "C" int ceph_set_fscrypt_policy_v2(struct ceph_mount_info *cmount, + int fd, const struct fscrypt_policy_v2 *policy) +{ + if (!cmount->is_mounted()) + return -CEPHFS_ENOTCONN; + + return cmount->get_client()->set_fscrypt_policy_v2(fd, *policy); +} + + // This is deprecated, use ceph_ll_register_callbacks2 instead. extern "C" void ceph_ll_register_callbacks(class ceph_mount_info *cmount, struct ceph_client_callback_args *args) diff --git a/src/osdc/ObjectCacher.cc b/src/osdc/ObjectCacher.cc index 41cb35c216c..26340b056f2 100644 --- a/src/osdc/ObjectCacher.cc +++ b/src/osdc/ObjectCacher.cc @@ -88,14 +88,16 @@ class ObjectCacher::C_RetryRead : public Context { ObjectSet *oset; Context *onfinish; ZTracer::Trace trace; + std::vector *holes; public: C_RetryRead(ObjectCacher *_oc, OSDRead *r, ObjectSet *os, Context *c, - const ZTracer::Trace &trace) - : oc(_oc), rd(r), oset(os), onfinish(c), trace(trace) { + const ZTracer::Trace &trace, + std::vector *holes) + : oc(_oc), rd(r), oset(os), onfinish(c), trace(trace), holes(holes) { } void finish(int r) override { if (r >= 0) { - r = oc->_readx(rd, oset, onfinish, false, &trace); + r = oc->_readx(rd, oset, onfinish, false, &trace, holes); } if (r == 0) { @@ -1394,7 +1396,8 @@ bool ObjectCacher::is_cached(ObjectSet *oset, vector& extents, * returns 0 if doing async read */ int ObjectCacher::readx(OSDRead *rd, ObjectSet *oset, Context *onfinish, - ZTracer::Trace *parent_trace) + ZTracer::Trace *parent_trace, + std::vector *holes) { ZTracer::Trace trace; if (parent_trace != nullptr) { @@ -1402,7 +1405,7 @@ int ObjectCacher::readx(OSDRead *rd, ObjectSet *oset, Context *onfinish, trace.event("start"); } - int r =_readx(rd, oset, onfinish, true, &trace); + int r =_readx(rd, oset, onfinish, true, &trace, holes); if (r < 0) { trace.event("finish"); } @@ -1410,7 +1413,8 @@ int ObjectCacher::readx(OSDRead *rd, ObjectSet *oset, Context *onfinish, } int ObjectCacher::_readx(OSDRead *rd, ObjectSet *oset, Context *onfinish, - bool external_call, ZTracer::Trace *trace) + bool external_call, ZTracer::Trace *trace, + std::vector *holes) { ceph_assert(trace != nullptr); ceph_assert(ceph_mutex_is_locked(lock)); @@ -1475,7 +1479,7 @@ int ObjectCacher::_readx(OSDRead *rd, ObjectSet *oset, Context *onfinish, ldout(cct, 10) << "readx waiting on tid " << o->last_write_tid << " on " << *o << dendl; o->waitfor_commit[o->last_write_tid].push_back( - new C_RetryRead(this,rd, oset, onfinish, *trace)); + new C_RetryRead(this,rd, oset, onfinish, *trace, holes)); // FIXME: perfcounter! return 0; } @@ -1533,7 +1537,7 @@ int ObjectCacher::_readx(OSDRead *rd, ObjectSet *oset, Context *onfinish, << (std::max(rx_bytes, max_size) - max_size) << " read bytes" << dendl; waitfor_read.push_back(new C_RetryRead(this, rd, oset, onfinish, - *trace)); + *trace, holes)); } bh_remove(o, bh_it->second); @@ -1552,7 +1556,7 @@ int ObjectCacher::_readx(OSDRead *rd, ObjectSet *oset, Context *onfinish, ldout(cct, 10) << "readx missed, waiting on " << *last->second << " off " << last->first << dendl; last->second->waitfor_read[last->first].push_back( - new C_RetryRead(this, rd, oset, onfinish, *trace) ); + new C_RetryRead(this, rd, oset, onfinish, *trace, holes) ); } @@ -1565,7 +1569,7 @@ int ObjectCacher::_readx(OSDRead *rd, ObjectSet *oset, Context *onfinish, ldout(cct, 10) << "readx missed, waiting on " << *bh_it->second << " off " << bh_it->first << dendl; bh_it->second->waitfor_read[bh_it->first].push_back( - new C_RetryRead(this, rd, oset, onfinish, *trace) ); + new C_RetryRead(this, rd, oset, onfinish, *trace, holes) ); } bytes_not_in_cache += bh_it->second->length(); success = false; @@ -1631,6 +1635,9 @@ int ObjectCacher::_readx(OSDRead *rd, ObjectSet *oset, Context *onfinish, // may get multiple bh's at this stripe_map position if (bh->is_zero()) { stripe_map[f_it->first].append_zero(len); + if (holes) { + holes->push_back(std::make_pair(opos, len)); + } } else { bit.substr_of(bh->bl, opos - bh->start(), diff --git a/src/osdc/ObjectCacher.h b/src/osdc/ObjectCacher.h index b3d61fda350..8c145a61860 100644 --- a/src/osdc/ObjectCacher.h +++ b/src/osdc/ObjectCacher.h @@ -60,6 +60,8 @@ class ObjectCacher { struct ObjectSet; class C_ReadFinish; + using ObjHole = std::pair; + typedef void (*flush_set_callback_t) (void *p, ObjectSet *oset); // read scatter/gather @@ -566,7 +568,8 @@ class ObjectCacher { ceph::condition_variable read_cond; int _readx(OSDRead *rd, ObjectSet *oset, Context *onfinish, - bool external_call, ZTracer::Trace *trace); + bool external_call, ZTracer::Trace *trace, + std::vector *holes); void retry_waiting_reads(); public: @@ -618,7 +621,8 @@ class ObjectCacher { * the return value is total bytes read */ int readx(OSDRead *rd, ObjectSet *oset, Context *onfinish, - ZTracer::Trace *parent_trace = nullptr); + ZTracer::Trace *parent_trace = nullptr, + std::vector *holes = nullptr); int writex(OSDWrite *wr, ObjectSet *oset, Context *onfreespace, ZTracer::Trace *parent_trace, bool block_writes_upfront); @@ -708,6 +712,16 @@ public: return readx(rd, oset, onfinish); } + int file_read_ex(ObjectSet *oset, file_layout_t *layout, snapid_t snapid, + loff_t offset, uint64_t len, ceph::buffer::list *bl, int flags, + std::vector *holes, + Context *onfinish) { + OSDRead *rd = prepare_read(snapid, bl, flags); + Striper::file_to_extents(cct, oset->ino, layout, offset, len, + oset->truncate_size, rd->extents); + return readx(rd, oset, onfinish, nullptr, holes); + } + int file_write(ObjectSet *oset, file_layout_t *layout, const SnapContext& snapc, loff_t offset, uint64_t len, ceph::buffer::list& bl, ceph::real_time mtime, int flags, diff --git a/src/test/client/CMakeLists.txt b/src/test/client/CMakeLists.txt index 94ef7d0d18e..00c0681f8b9 100644 --- a/src/test/client/CMakeLists.txt +++ b/src/test/client/CMakeLists.txt @@ -7,6 +7,7 @@ if(${WITH_CEPHFS}) nonblocking.cc commands.cc syncio.cc + fscrypt_conf.cc ) target_link_libraries(ceph_test_client client @@ -18,4 +19,23 @@ if(${WITH_CEPHFS}) ) install(TARGETS ceph_test_client DESTINATION ${CMAKE_INSTALL_BINDIR}) + + add_executable(ceph_test_client_fscrypt + main_fscrypt.cc + alternate_name.cc + ops.cc + nonblocking.cc + fscrypt_conf.cc + ) + target_link_libraries(ceph_test_client_fscrypt + client + global + ceph-common + cephfs + ${UNITTEST_LIBS} + ${EXTRALIBS} + ${CMAKE_DL_LIBS} + ) + install(TARGETS ceph_test_client_fscrypt + DESTINATION ${CMAKE_INSTALL_BINDIR}) endif(${WITH_CEPHFS}) diff --git a/src/test/client/TestClient.h b/src/test/client/TestClient.h index 724a4a4087e..5208fb457e0 100644 --- a/src/test/client/TestClient.h +++ b/src/test/client/TestClient.h @@ -24,20 +24,32 @@ #include "osdc/Objecter.h" #include "client/MetaRequest.h" #include "client/Client.h" +#include "client/FSCrypt.h" #include "messages/MClientReclaim.h" #include "messages/MClientSession.h" #include "common/async/blocked_completion.h" +#include "fscrypt_conf.h" + #define dout_subsys ceph_subsys_client namespace bs = boost::system; namespace ca = ceph::async; + +struct fscrypt_env { + bool encrypted = false; + std::string name; + std::string dir; + char key[32]; +}; + class ClientScaffold : public Client { + fscrypt_env *fse; public: using Client::walk_dentry_result; - ClientScaffold(Messenger *m, MonClient *mc, Objecter *objecter_) : Client(m, mc, objecter_) {} + ClientScaffold(Messenger *m, MonClient *mc, Objecter *objecter_, fscrypt_env *fse) : Client(m, mc, objecter_), fse(fse) {} virtual ~ClientScaffold() { } int check_dummy_op(const UserPerm& perms){ @@ -115,6 +127,105 @@ public: int walk(std::string_view path, struct walk_dentry_result* result, const UserPerm& perms, bool followsym=true) { return Client::walk(path, result, perms, followsym); } + + bool encrypt(const UserPerm& myperm) { + if (!fscrypt_enabled || fse->encrypted) { + return false; + } + + pid_t mypid = getpid(); + fse->name = std::string("ceph_test_client_fscrypt.") + stringify(mypid) + "." + stringify(rand()); + fse->dir = std::string("/") + fse->name; + + int r = mount("/", myperm, true); + if (r < 0) { + std::clog << __func__ << "(): mount() r=" << r << std::endl; + throw std::runtime_error("mount() returned error"); + } + + r = mkdir(fse->name.c_str(), 0755, myperm); + if (r < 0) { + std::clog << __func__ << "(): mkdir(" << fse->name << ") r=" << r << std::endl; + throw std::runtime_error("get_root() returned error"); + } + + for (size_t i = 0; i < sizeof(fse->key); ++i) { + fse->key[i] = (char)rand(); + } + + std::string key_fname = fse->name + ".key"; + int key_fd = open(key_fname.c_str(), O_RDWR|O_CREAT|O_TRUNC, myperm, 0600); + if (key_fd < 0) { + std::clog << __func__ << "(): open() fd=" << key_fd << std::endl; + throw std::runtime_error("open() returned error"); + } + + r = write(key_fd, fse->key, sizeof(fse->key), 0); + if (r < 0) { + std::clog << __func__ << "(): write() r=" << r << std::endl; + throw std::runtime_error("write() returned error"); + } + + close(key_fd); + + struct ceph_fscrypt_key_identifier kid; + + r = add_fscrypt_key(fse->key, sizeof(fse->key), &kid); + if (r < 0) { + std::clog << __func__ << "(): add_fscrypt_key() r=" << r << std::endl; + throw std::runtime_error("add_fscrypt_key() returned error"); + } + + struct fscrypt_policy_v2 policy; + policy.version = 2; + policy.contents_encryption_mode = FSCRYPT_MODE_AES_256_XTS; + policy.filenames_encryption_mode = FSCRYPT_MODE_AES_256_CTS; + policy.flags = FSCRYPT_POLICY_FLAGS_PAD_32; + memcpy(policy.master_key_identifier, kid.raw, FSCRYPT_KEY_IDENTIFIER_SIZE); + + int fd = open(fse->name.c_str(), O_DIRECTORY, myperm, 0); + if (fd < 0) { + std::clog << __func__ << "(): open() r=" << r << std::endl; + throw std::runtime_error("open() returned error"); + } + + r = set_fscrypt_policy_v2(fd, policy); + if (r < 0) { + std::clog << __func__ << "(): set_fscrypt_policy() r=" << r << std::endl; + throw std::runtime_error("set_fscrypt_policy() returned error"); + } + + fse->encrypted = true; + + return true; + } + + int do_mount(const std::string &mount_root, const UserPerm& perms, + bool require_mds=false, const std::string &fs_name="") { + int r; + + if (fse->dir.empty()) { + r = mount(mount_root, perms, require_mds, fs_name); + } else { + std::string new_root = fse->dir + mount_root; + + r = mount(new_root, perms, require_mds, fs_name); + } + if (r < 0) { + std::clog << __func__ << "() do_mount r=" << r << std::endl; + return r; + } + + struct ceph_fscrypt_key_identifier kid; + + r = add_fscrypt_key(fse->key, sizeof(fse->key), &kid); + if (r < 0) { + std::clog << __func__ << "() ceph_mount add_fscrypt_key r=" << r << std::endl; + return r; + } + + return 0; + } }; class TestClient : public ::testing::Test { @@ -122,9 +233,21 @@ public: static void SetUpTestSuite() { icp.start(g_ceph_context->_conf.get_val("client_asio_thread_count")); } + static void TearDownTestSuite() { icp.stop(); } + + bool encrypt() { + bool encrypted = client->encrypt(myperm); + if (encrypted) { + client->unmount(); + client->shutdown(); + delete client; + } + return encrypted; + } + void SetUp() override { messenger = Messenger::create_client_messenger(g_ceph_context, "client"); if (messenger->start() != 0) { @@ -147,9 +270,12 @@ public: messenger->add_dispatcher_tail(objecter); objecter->start(); - client = new ClientScaffold(messenger, mc, objecter); - client->init(); - client->mount("/", myperm, true); + do { + client = new ClientScaffold(messenger, mc, objecter, &fse); + client->init(); + } while (encrypt()); + + client->do_mount("/", myperm, true); } void TearDown() override { if (client->is_mounted()) @@ -173,6 +299,7 @@ public: protected: static inline ceph::async::io_context_pool icp; static inline UserPerm myperm{0,0}; + static inline fscrypt_env fse; MonClient* mc = nullptr; Messenger* messenger = nullptr; Objecter* objecter = nullptr; diff --git a/src/test/client/fscrypt_conf.cc b/src/test/client/fscrypt_conf.cc new file mode 100644 index 00000000000..50e85a20bdf --- /dev/null +++ b/src/test/client/fscrypt_conf.cc @@ -0,0 +1,15 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Ceph - scalable distributed file system + * + * Copyright (C) IBM Corporation 2023 + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + */ + +bool fscrypt_enabled = false; diff --git a/src/test/client/fscrypt_conf.h b/src/test/client/fscrypt_conf.h new file mode 100644 index 00000000000..75039a765d3 --- /dev/null +++ b/src/test/client/fscrypt_conf.h @@ -0,0 +1,17 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Ceph - scalable distributed file system + * + * Copyright (C) IBM Corporation 2023 + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + */ + +#pragma once + +extern bool fscrypt_enabled; diff --git a/src/test/client/main_fscrypt.cc b/src/test/client/main_fscrypt.cc new file mode 100644 index 00000000000..683b9065d10 --- /dev/null +++ b/src/test/client/main_fscrypt.cc @@ -0,0 +1,32 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2011 New Dream Network + * Copyright (C) 2016 Red Hat + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + */ + +#include "gtest/gtest.h" + +#include "common/ceph_argparse.h" +#include "global/global_init.h" +#include "global/global_context.h" + +#include "fscrypt_conf.h" + +int main(int argc, char **argv) +{ + fscrypt_enabled = true; + + auto args = argv_to_vec(argc, argv); + [[maybe_unused]] auto cct = global_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT, CODE_ENVIRONMENT_UTILITY, 0); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/src/test/client/nonblocking.cc b/src/test/client/nonblocking.cc index 3f0cce37c88..c232c2ffcc0 100644 --- a/src/test/client/nonblocking.cc +++ b/src/test/client/nonblocking.cc @@ -116,12 +116,12 @@ TEST_F(TestClient, LlreadvLlwritev) { // reset bufferlist bl.clear(); - rc = client->ll_preadv_pwritev(fh, iov_out_a, 2, 100, true, writefinish.get(), nullptr); + rc = client->ll_preadv_pwritev(fh, iov_out_a, 2, 5000, true, writefinish.get(), nullptr); ASSERT_EQ(0, rc); rc = writefinish->wait(); ASSERT_EQ(nwritten_a, rc); - rc = client->ll_preadv_pwritev(fh, iov_in_a, 2, 100, false, readfinish.get(), &bl); + rc = client->ll_preadv_pwritev(fh, iov_in_a, 2, 5000, false, readfinish.get(), &bl); ASSERT_EQ(0, rc); rc = readfinish.get()->wait(); ASSERT_EQ(nwritten_a, rc); @@ -137,12 +137,12 @@ TEST_F(TestClient, LlreadvLlwritev) { // reset bufferlist bl.clear(); - rc = client->ll_preadv_pwritev(fh, iov_out_b, 2, 1000, true, writefinish.get(), nullptr, true, false); + rc = client->ll_preadv_pwritev(fh, iov_out_b, 2, 4090, true, writefinish.get(), nullptr, true, false); ASSERT_EQ(0, rc); rc = writefinish->wait(); ASSERT_EQ(nwritten_b, rc); - rc = client->ll_preadv_pwritev(fh, iov_in_b, 2, 1000, false, readfinish.get(), &bl); + rc = client->ll_preadv_pwritev(fh, iov_in_b, 2, 4090, false, readfinish.get(), &bl); ASSERT_EQ(0, rc); rc = readfinish.get()->wait(); ASSERT_EQ(nwritten_b, rc); @@ -152,7 +152,7 @@ TEST_F(TestClient, LlreadvLlwritev) { ASSERT_EQ(0, strncmp((const char*)iov_in_b[1].iov_base, (const char*)iov_out_b[1].iov_base, iov_out_b[1].iov_len)); client->ll_release(fh); - ASSERT_EQ(0, client->ll_unlink(root, filename, myperm)); + // ASSERT_EQ(0, client->ll_unlink(root, filename, myperm)); } TEST_F(TestClient, LlreadvLlwritevNullContext) { diff --git a/src/test/libcephfs/CMakeLists.txt b/src/test/libcephfs/CMakeLists.txt index 0e92e47cdc0..eb31b045751 100644 --- a/src/test/libcephfs/CMakeLists.txt +++ b/src/test/libcephfs/CMakeLists.txt @@ -176,6 +176,22 @@ if(WITH_LIBCEPHFS) ${EXTRALIBS} ${CMAKE_DL_LIBS} ) + + add_executable(ceph_test_libcephfs_fscrypt + fscrypt.cc + test.cc + ) + target_link_libraries(ceph_test_libcephfs_fscrypt + ceph-common + cephfs + librados + ${UNITTEST_LIBS} + ${EXTRALIBS} + ${CMAKE_DL_LIBS} + ) install(TARGETS ceph_test_libcephfs_perfcounters DESTINATION ${CMAKE_INSTALL_BINDIR}) + install(TARGETS ceph_test_libcephfs_fscrypt + DESTINATION ${CMAKE_INSTALL_BINDIR}) + endif(WITH_LIBCEPHFS) diff --git a/src/test/libcephfs/fscrypt.cc b/src/test/libcephfs/fscrypt.cc new file mode 100644 index 00000000000..a8a36f73ba9 --- /dev/null +++ b/src/test/libcephfs/fscrypt.cc @@ -0,0 +1,255 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2011 New Dream Network + * Copyright (C) 2023 IBM Corporation + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + */ + +#include "gtest/gtest.h" +#include "common/ceph_argparse.h" +#include "include/buffer.h" +#include "include/stringify.h" +#include "include/cephfs/libcephfs.h" +#include "include/fs_types.h" +#include "include/rados/librados.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "json_spirit/json_spirit.h" + +#include "include/fs_types.h" + +#include "client/FSCrypt.h" + +#ifdef __linux__ +#include +#include +#endif + +#include "test.h" + +using namespace std; + +rados_t cluster; + +static string fscrypt_dir; + +static char fscrypt_key[32]; + + +int do_fscrypt_mount(struct ceph_mount_info *cmount, const char *root) +{ + int r; + + if (fscrypt_dir.empty()) { + r = ceph_mount(cmount, root); + } else { + string new_root = fscrypt_dir; + + if (root) { + new_root += root; + } + + r = ceph_mount(cmount, new_root.c_str()); + } + if (r < 0) { + std::clog << __func__ << "() ceph_mount r=" << r << std::endl; + return r; + } + + struct ceph_fscrypt_key_identifier kid; + + r = ceph_add_fscrypt_key(cmount, fscrypt_key, sizeof(fscrypt_key), &kid); + if (r < 0) { + std::clog << __func__ << "() ceph_mount add_fscrypt_key r=" << r << std::endl; + return r; + } + + return 0; +} + +string get_unique_dir_name() +{ + pid_t mypid = getpid(); + return string("ceph_test_libcephfs_fscrypt.") + stringify(mypid) + "." + stringify(rand()); +} + +int fscrypt_encrypt(const string& dir_path) +{ + struct ceph_mount_info *cmount; + int r = ceph_create(&cmount, NULL); + if (r < 0) { + std::clog << __func__ << "(): ceph_create() r=" << r << std::endl; + return r; + } + + r = ceph_conf_read_file(cmount, NULL); + if (r < 0) { + std::clog << __func__ << "(): ceph_conf_read_file() r=" << r << std::endl; + return r; + } + + r = ceph_conf_parse_env(cmount, NULL); + if (r < 0) { + std::clog << __func__ << "(): ceph_parse_env() r=" << r << std::endl; + return r; + } + + r = ceph_mount(cmount, NULL); + if (r < 0) { + std::clog << __func__ << "(): ceph_mount() r=" << r << std::endl; + return r; + } + + Inode *dir, *root; + struct ceph_statx stx_dir; + UserPerm *perms = ceph_mount_perms(cmount); + + r = ceph_ll_lookup_root(cmount, &root); + if (r < 0) { + std::clog << __func__ << "(): ceph_ll_lookup_root() r=" << r << std::endl; + return r; + } + + r = ceph_ll_mkdir(cmount, root, dir_path.c_str(), 0755, &dir, &stx_dir, 0, 0, perms); + if (r < 0) { + std::clog << __func__ << "(): ceph_ll_mkdir(" << dir_path << ") r=" << r << std::endl; + return r; + } + + for (int i = 0; i < sizeof(fscrypt_key); ++i) { + fscrypt_key[i] = (char)rand(); + } + + string key_fname = dir_path + ".key"; + int key_fd = ceph_open(cmount, key_fname.c_str(), O_RDWR|O_CREAT|O_TRUNC, 0600); + if (key_fd < 0) { + std::clog << __func__ << "(): ceph_open() fd=" << key_fd << std::endl; + return key_fd; + } + + r = ceph_write(cmount, key_fd, fscrypt_key, sizeof(fscrypt_key), 0); + if (r < 0) { + std::clog << __func__ << "(): ceph_write() r=" << r << std::endl; + return r; + } + + ceph_close(cmount, key_fd); + + struct ceph_fscrypt_key_identifier kid; + + r = ceph_add_fscrypt_key(cmount, fscrypt_key, sizeof(fscrypt_key), &kid); + if (r < 0) { + std::clog << __func__ << "(): ceph_add_fscrypt_key() r=" << r << std::endl; + return r; + } + + struct fscrypt_policy_v2 policy; + policy.version = 2; + policy.contents_encryption_mode = FSCRYPT_MODE_AES_256_XTS; + policy.filenames_encryption_mode = FSCRYPT_MODE_AES_256_CTS; + policy.flags = FSCRYPT_POLICY_FLAGS_PAD_32; + memcpy(policy.master_key_identifier, kid.raw, FSCRYPT_KEY_IDENTIFIER_SIZE); + + int fd = ceph_open(cmount, dir_path.c_str(), O_DIRECTORY, 0); + if (fd < 0) { + std::clog << __func__ << "(): ceph_open() r=" << r << std::endl; + return fd; + } + + r = ceph_set_fscrypt_policy_v2(cmount, fd, &policy); + if (r < 0) { + std::clog << __func__ << "(): ceph_set_fscrypt_policy() r=" << r << std::endl; + return r; + } + + ceph_shutdown(cmount); + + return 0; +} + +static int init_fscrypt() +{ + string name = get_unique_dir_name(); + std::clog << __func__ << "(): fscrypt_dir=" << name << std::endl; + + fscrypt_dir = string("/") + name; + + int r = fscrypt_encrypt(name); + if (r < 0) { + return r; + } + + libcephfs_test_set_mount_call(do_fscrypt_mount); + std::clog << __func__ << "(): init fscrypt done" << std::endl; + + return 0; +} + + +static int update_root_mode() +{ + struct ceph_mount_info *admin; + int r = ceph_create(&admin, NULL); + if (r < 0) + return r; + ceph_conf_read_file(admin, NULL); + ceph_conf_parse_env(admin, NULL); + ceph_conf_set(admin, "client_permissions", "false"); + r = ceph_mount(admin, "/"); + if (r < 0) + goto out; + r = ceph_chmod(admin, "/", 0777); +out: + ceph_shutdown(admin); + return r; +} + + +int main(int argc, char **argv) +{ + int r = update_root_mode(); + if (r < 0) + exit(1); + + ::testing::InitGoogleTest(&argc, argv); + + srand(getpid()); + + r = rados_create(&cluster, NULL); + if (r < 0) + exit(1); + + r = rados_conf_read_file(cluster, NULL); + if (r < 0) + exit(1); + + rados_conf_parse_env(cluster, NULL); + r = rados_connect(cluster); + if (r < 0) + exit(1); + + r = init_fscrypt(); + if (r < 0) + exit(1); + + r = RUN_ALL_TESTS(); + + rados_shutdown(cluster); + + return r; +} diff --git a/src/test/libcephfs/test.cc b/src/test/libcephfs/test.cc index d4b4954f6f6..6a8f8774db2 100644 --- a/src/test/libcephfs/test.cc +++ b/src/test/libcephfs/test.cc @@ -60,6 +60,13 @@ static std::string generate_random_string(int length = 20) { return str; } +static int (*do_ceph_mount)(struct ceph_mount_info *cmount, const char *root) = ceph_mount; + +void libcephfs_test_set_mount_call(int (*mount_call)(struct ceph_mount_info *cmount, const char *root)) +{ + do_ceph_mount = mount_call; +} + TEST(LibCephFS, OpenEmptyComponent) { pid_t mypid = getpid(); @@ -67,7 +74,7 @@ TEST(LibCephFS, OpenEmptyComponent) { ASSERT_EQ(0, ceph_create(&cmount, NULL)); ASSERT_EQ(0, ceph_conf_read_file(cmount, NULL)); ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL)); - ASSERT_EQ(0, ceph_mount(cmount, "/")); + ASSERT_EQ(0, do_ceph_mount(cmount, "/")); char c_dir[1024]; sprintf(c_dir, "/open_test_%d", mypid); @@ -90,7 +97,7 @@ TEST(LibCephFS, OpenEmptyComponent) { ASSERT_EQ(0, ceph_conf_read_file(cmount, NULL)); ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL)); - ASSERT_EQ(0, ceph_mount(cmount, "/")); + ASSERT_EQ(0, do_ceph_mount(cmount, "/")); fd = ceph_open(cmount, c_path, O_RDONLY, 0666); ASSERT_LT(0, fd); @@ -108,7 +115,7 @@ TEST(LibCephFS, OpenReadTruncate) { ASSERT_EQ(0, ceph_create(&cmount, NULL)); ASSERT_EQ(0, ceph_conf_read_file(cmount, NULL)); ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL)); - ASSERT_EQ(0, ceph_mount(cmount, "/")); + ASSERT_EQ(0, do_ceph_mount(cmount, "/")); auto path = fmt::format("test_open_rdt_{}", getpid()); int fd = ceph_open(cmount, path.c_str(), O_WRONLY|O_CREAT, 0666); @@ -132,7 +139,7 @@ TEST(LibCephFS, OpenReadWrite) { ASSERT_EQ(0, ceph_create(&cmount, NULL)); ASSERT_EQ(0, ceph_conf_read_file(cmount, NULL)); ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL)); - ASSERT_EQ(0, ceph_mount(cmount, "/")); + ASSERT_EQ(0, do_ceph_mount(cmount, "/")); char c_path[1024]; sprintf(c_path, "test_open_rdwr_%d", getpid()); @@ -168,7 +175,7 @@ TEST(LibCephFS, MountNonExist) { ASSERT_EQ(0, ceph_create(&cmount, NULL)); ASSERT_EQ(0, ceph_conf_read_file(cmount, NULL)); ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL)); - ASSERT_NE(0, ceph_mount(cmount, "/non-exist")); + ASSERT_NE(0, do_ceph_mount(cmount, "/non-exist")); ceph_shutdown(cmount); } @@ -179,8 +186,8 @@ TEST(LibCephFS, MountDouble) { ASSERT_EQ(0, ceph_create(&cmount, NULL)); ASSERT_EQ(0, ceph_conf_read_file(cmount, NULL)); ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL)); - ASSERT_EQ(0, ceph_mount(cmount, "/")); - ASSERT_EQ(-EISCONN, ceph_mount(cmount, "/")); + ASSERT_EQ(0, do_ceph_mount(cmount, "/")); + ASSERT_EQ(-EISCONN, do_ceph_mount(cmount, "/")); ceph_shutdown(cmount); } @@ -193,10 +200,10 @@ TEST(LibCephFS, MountRemount) { ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL)); CephContext *cct = ceph_get_mount_context(cmount); - ASSERT_EQ(0, ceph_mount(cmount, "/")); + ASSERT_EQ(0, do_ceph_mount(cmount, "/")); ASSERT_EQ(0, ceph_unmount(cmount)); - ASSERT_EQ(0, ceph_mount(cmount, "/")); + ASSERT_EQ(0, do_ceph_mount(cmount, "/")); ASSERT_EQ(cct, ceph_get_mount_context(cmount)); ceph_shutdown(cmount); @@ -230,7 +237,7 @@ TEST(LibCephFS, ReleaseMounted) { ASSERT_EQ(0, ceph_create(&cmount, NULL)); ASSERT_EQ(0, ceph_conf_read_file(cmount, NULL)); ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL)); - ASSERT_EQ(0, ceph_mount(cmount, "/")); + ASSERT_EQ(0, do_ceph_mount(cmount, "/")); ASSERT_EQ(-EISCONN, ceph_release(cmount)); ASSERT_EQ(0, ceph_unmount(cmount)); ASSERT_EQ(0, ceph_release(cmount)); @@ -243,7 +250,7 @@ TEST(LibCephFS, UnmountRelease) { ASSERT_EQ(0, ceph_create(&cmount, NULL)); ASSERT_EQ(0, ceph_conf_read_file(cmount, NULL)); ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL)); - ASSERT_EQ(0, ceph_mount(cmount, "/")); + ASSERT_EQ(0, do_ceph_mount(cmount, "/")); ASSERT_EQ(0, ceph_unmount(cmount)); ASSERT_EQ(0, ceph_release(cmount)); } @@ -253,13 +260,13 @@ TEST(LibCephFS, Mount) { ASSERT_EQ(ceph_create(&cmount, NULL), 0); ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0); ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL)); - ASSERT_EQ(ceph_mount(cmount, NULL), 0); + ASSERT_EQ(do_ceph_mount(cmount, NULL), 0); ceph_shutdown(cmount); ASSERT_EQ(ceph_create(&cmount, NULL), 0); ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0); ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL)); - ASSERT_EQ(ceph_mount(cmount, NULL), 0); + ASSERT_EQ(do_ceph_mount(cmount, NULL), 0); ceph_shutdown(cmount); } @@ -268,7 +275,7 @@ TEST(LibCephFS, OpenLayout) { ASSERT_EQ(ceph_create(&cmount, NULL), 0); ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0); ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL)); - ASSERT_EQ(ceph_mount(cmount, NULL), 0); + ASSERT_EQ(do_ceph_mount(cmount, NULL), 0); /* valid layout */ char test_layout_file[256]; @@ -321,7 +328,7 @@ TEST(LibCephFS, DirLs) { ASSERT_EQ(ceph_create(&cmount, NULL), 0); ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0); ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL)); - ASSERT_EQ(ceph_mount(cmount, "/"), 0); + ASSERT_EQ(do_ceph_mount(cmount, "/"), 0); struct ceph_dir_result *ls_dir = NULL; char foostr[256]; @@ -521,7 +528,7 @@ TEST(LibCephFS, ManyNestedDirs) { ASSERT_EQ(ceph_create(&cmount, NULL), 0); ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0); ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL)); - ASSERT_EQ(ceph_mount(cmount, NULL), 0); + ASSERT_EQ(do_ceph_mount(cmount, NULL), 0); static const char many_path[] = "/ManyNestedDirs/A/a/a/a/a/b/B/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/Aa"; const filepath mfp = filepath(many_path); @@ -619,7 +626,7 @@ TEST(LibCephFS, Xattrs) { ASSERT_EQ(ceph_create(&cmount, NULL), 0); ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0); ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL)); - ASSERT_EQ(ceph_mount(cmount, NULL), 0); + ASSERT_EQ(do_ceph_mount(cmount, NULL), 0); char test_xattr_file[256]; sprintf(test_xattr_file, "test_xattr_%d", getpid()); @@ -690,7 +697,7 @@ TEST(LibCephFS, Xattrs_ll) { ASSERT_EQ(ceph_create(&cmount, NULL), 0); ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0); ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL)); - ASSERT_EQ(ceph_mount(cmount, NULL), 0); + ASSERT_EQ(do_ceph_mount(cmount, NULL), 0); char test_xattr_file[256]; sprintf(test_xattr_file, "test_xattr_%d", getpid()); @@ -732,7 +739,7 @@ TEST(LibCephFS, LstatSlashdot) { ASSERT_EQ(ceph_create(&cmount, NULL), 0); ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0); ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL)); - ASSERT_EQ(ceph_mount(cmount, NULL), 0); + ASSERT_EQ(do_ceph_mount(cmount, NULL), 0); struct ceph_statx stx; ASSERT_EQ(ceph_statx(cmount, "/.", &stx, 0, AT_SYMLINK_NOFOLLOW), 0); @@ -746,7 +753,7 @@ TEST(LibCephFS, StatDirNlink) { ASSERT_EQ(ceph_create(&cmount, NULL), 0); ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0); ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL)); - ASSERT_EQ(ceph_mount(cmount, NULL), 0); + ASSERT_EQ(do_ceph_mount(cmount, NULL), 0); char test_dir1[256]; sprintf(test_dir1, "dir1_symlinks_%d", getpid()); @@ -809,7 +816,7 @@ TEST(LibCephFS, DoubleChmod) { ASSERT_EQ(ceph_create(&cmount, NULL), 0); ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0); ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL)); - ASSERT_EQ(ceph_mount(cmount, NULL), 0); + ASSERT_EQ(do_ceph_mount(cmount, NULL), 0); char test_file[256]; sprintf(test_file, "test_perms_%d", getpid()); @@ -864,7 +871,7 @@ TEST(LibCephFS, Fchmod) { ASSERT_EQ(ceph_create(&cmount, NULL), 0); ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0); ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL)); - ASSERT_EQ(ceph_mount(cmount, NULL), 0); + ASSERT_EQ(do_ceph_mount(cmount, NULL), 0); char test_file[256]; sprintf(test_file, "test_perms_%d", getpid()); @@ -908,7 +915,7 @@ TEST(LibCephFS, Lchmod) { ASSERT_EQ(ceph_create(&cmount, NULL), 0); ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0); ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL)); - ASSERT_EQ(ceph_mount(cmount, NULL), 0); + ASSERT_EQ(do_ceph_mount(cmount, NULL), 0); char test_file[256]; sprintf(test_file, "test_perms_lchmod_%d", getpid()); @@ -949,7 +956,7 @@ TEST(LibCephFS, Fchown) { ASSERT_EQ(ceph_create(&cmount, NULL), 0); ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0); ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL)); - ASSERT_EQ(ceph_mount(cmount, NULL), 0); + ASSERT_EQ(do_ceph_mount(cmount, NULL), 0); char test_file[256]; sprintf(test_file, "test_fchown_%d", getpid()); @@ -983,7 +990,7 @@ TEST(LibCephFS, FlagO_PATH) { ASSERT_EQ(0, ceph_create(&cmount, NULL)); ASSERT_EQ(0, ceph_conf_read_file(cmount, NULL)); ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL)); - ASSERT_EQ(0, ceph_mount(cmount, NULL)); + ASSERT_EQ(0, do_ceph_mount(cmount, NULL)); char test_file[PATH_MAX]; sprintf(test_file, "test_oflag_%d", getpid()); @@ -1077,7 +1084,7 @@ TEST(LibCephFS, Symlinks) { ASSERT_EQ(ceph_create(&cmount, NULL), 0); ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0); ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL)); - ASSERT_EQ(ceph_mount(cmount, NULL), 0); + ASSERT_EQ(do_ceph_mount(cmount, NULL), 0); char test_file[256]; sprintf(test_file, "test_symlinks_%d", getpid()); @@ -1142,7 +1149,7 @@ TEST(LibCephFS, DirSyms) { ASSERT_EQ(ceph_create(&cmount, NULL), 0); ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0); ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL)); - ASSERT_EQ(ceph_mount(cmount, NULL), 0); + ASSERT_EQ(do_ceph_mount(cmount, NULL), 0); char test_dir1[256]; sprintf(test_dir1, "dir1_symlinks_%d", getpid()); @@ -1174,7 +1181,7 @@ TEST(LibCephFS, LoopSyms) { ASSERT_EQ(ceph_create(&cmount, NULL), 0); ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0); ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL)); - ASSERT_EQ(ceph_mount(cmount, NULL), 0); + ASSERT_EQ(do_ceph_mount(cmount, NULL), 0); char test_dir1[256]; sprintf(test_dir1, "dir1_loopsym_%d", getpid()); @@ -1218,7 +1225,7 @@ TEST(LibCephFS, HardlinkNoOriginal) { ASSERT_EQ(ceph_create(&cmount, NULL), 0); ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0); ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL)); - ASSERT_EQ(ceph_mount(cmount, NULL), 0); + ASSERT_EQ(do_ceph_mount(cmount, NULL), 0); char dir[256]; sprintf(dir, "/test_rmdirfail%d", mypid); @@ -1243,7 +1250,7 @@ TEST(LibCephFS, HardlinkNoOriginal) { ASSERT_EQ(ceph_create(&cmount, NULL), 0); ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0); ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL)); - ASSERT_EQ(ceph_mount(cmount, NULL), 0); + ASSERT_EQ(do_ceph_mount(cmount, NULL), 0); ASSERT_EQ(ceph_chdir(cmount, dir), 0); ASSERT_EQ(ceph_unlink(cmount, "hardl1"), 0); ASSERT_EQ(ceph_rmdir(cmount, dir), 0); @@ -1256,7 +1263,7 @@ TEST(LibCephFS, BadArgument) { ASSERT_EQ(ceph_create(&cmount, NULL), 0); ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0); ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL)); - ASSERT_EQ(ceph_mount(cmount, NULL), 0); + ASSERT_EQ(do_ceph_mount(cmount, NULL), 0); int fd = ceph_open(cmount, "test_file", O_CREAT|O_RDWR, 0666); ASSERT_GT(fd, 0); @@ -1274,7 +1281,7 @@ TEST(LibCephFS, BadFileDesc) { ASSERT_EQ(ceph_create(&cmount, NULL), 0); ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0); ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL)); - ASSERT_EQ(ceph_mount(cmount, NULL), 0); + ASSERT_EQ(do_ceph_mount(cmount, NULL), 0); ASSERT_EQ(ceph_fchmod(cmount, -1, 0655), -EBADF); ASSERT_EQ(ceph_close(cmount, -1), -EBADF); @@ -1311,7 +1318,7 @@ TEST(LibCephFS, ReadEmptyFile) { ASSERT_EQ(ceph_create(&cmount, NULL), 0); ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0); ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL)); - ASSERT_EQ(ceph_mount(cmount, NULL), 0); + ASSERT_EQ(do_ceph_mount(cmount, NULL), 0); // test the read_sync path in the client for zero files ASSERT_EQ(ceph_conf_set(cmount, "client_debug_force_sync_read", "true"), 0); @@ -1340,7 +1347,7 @@ TEST(LibCephFS, PreadvPwritev) { ASSERT_EQ(ceph_create(&cmount, NULL), 0); ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0); ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL)); - ASSERT_EQ(ceph_mount(cmount, NULL), 0); + ASSERT_EQ(do_ceph_mount(cmount, NULL), 0); int mypid = getpid(); char testf[256]; @@ -1378,7 +1385,7 @@ TEST(LibCephFS, LlreadvLlwritev) { ASSERT_EQ(ceph_create(&cmount, NULL), 0); ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0); ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL)); - ASSERT_EQ(ceph_mount(cmount, NULL), 0); + ASSERT_EQ(do_ceph_mount(cmount, NULL), 0); int mypid = getpid(); char filename[256]; @@ -1425,7 +1432,7 @@ TEST(LibCephFS, StripeUnitGran) { ASSERT_EQ(ceph_create(&cmount, NULL), 0); ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0); ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL)); - ASSERT_EQ(ceph_mount(cmount, NULL), 0); + ASSERT_EQ(do_ceph_mount(cmount, NULL), 0); ASSERT_GT(ceph_get_stripe_unit_granularity(cmount), 0); ceph_shutdown(cmount); } @@ -1435,7 +1442,7 @@ TEST(LibCephFS, Rename) { ASSERT_EQ(ceph_create(&cmount, NULL), 0); ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0); ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL)); - ASSERT_EQ(ceph_mount(cmount, NULL), 0); + ASSERT_EQ(do_ceph_mount(cmount, NULL), 0); int mypid = getpid(); char path_src[256]; @@ -1561,7 +1568,7 @@ TEST(LibCephFS, GetPoolId) { ASSERT_EQ(ceph_create(&cmount, NULL), 0); ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0); ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL)); - ASSERT_EQ(ceph_mount(cmount, NULL), 0); + ASSERT_EQ(do_ceph_mount(cmount, NULL), 0); char name[80]; memset(name, 0, sizeof(name)); @@ -1577,7 +1584,7 @@ TEST(LibCephFS, GetPoolReplication) { ASSERT_EQ(ceph_create(&cmount, NULL), 0); ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0); ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL)); - ASSERT_EQ(ceph_mount(cmount, NULL), 0); + ASSERT_EQ(do_ceph_mount(cmount, NULL), 0); /* negative pools */ ASSERT_EQ(ceph_get_pool_replication(cmount, -10), -ENOENT); @@ -1601,7 +1608,7 @@ TEST(LibCephFS, GetExtentOsds) { ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0); ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL)); - ASSERT_EQ(ceph_mount(cmount, NULL), 0); + ASSERT_EQ(do_ceph_mount(cmount, NULL), 0); int stripe_unit = (1<<18); @@ -1653,7 +1660,7 @@ TEST(LibCephFS, GetOsdCrushLocation) { ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0); ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL)); - ASSERT_EQ(ceph_mount(cmount, NULL), 0); + ASSERT_EQ(do_ceph_mount(cmount, NULL), 0); ASSERT_EQ(ceph_get_osd_crush_location(cmount, 0, NULL, 1), -EINVAL); @@ -1704,7 +1711,7 @@ TEST(LibCephFS, GetOsdAddr) { ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0); ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL)); - ASSERT_EQ(ceph_mount(cmount, NULL), 0); + ASSERT_EQ(do_ceph_mount(cmount, NULL), 0); ASSERT_EQ(-EINVAL, ceph_get_osd_addr(cmount, 0, NULL)); @@ -1722,7 +1729,7 @@ TEST(LibCephFS, OpenNoClose) { ASSERT_EQ(ceph_create(&cmount, NULL), 0); ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0); ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL)); - ASSERT_EQ(ceph_mount(cmount, "/"), 0); + ASSERT_EQ(do_ceph_mount(cmount, "/"), 0); pid_t mypid = getpid(); char str_buf[256]; @@ -1745,7 +1752,7 @@ TEST(LibCephFS, Nlink) { ASSERT_EQ(ceph_create(&cmount, NULL), 0); ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0); ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL)); - ASSERT_EQ(ceph_mount(cmount, "/"), 0); + ASSERT_EQ(do_ceph_mount(cmount, "/"), 0); Inode *root, *dir, *file; @@ -1783,7 +1790,7 @@ TEST(LibCephFS, SlashDotDot) { ASSERT_EQ(ceph_create(&cmount, NULL), 0); ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0); ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL)); - ASSERT_EQ(ceph_mount(cmount, "/"), 0); + ASSERT_EQ(do_ceph_mount(cmount, "/"), 0); struct ceph_statx stx; ASSERT_EQ(ceph_statx(cmount, "/.", &stx, CEPH_STATX_INO, 0), 0); @@ -1820,7 +1827,7 @@ TEST(LibCephFS, SlashDotDot) { /* Make sure it works same way when mounting subtree */ ASSERT_EQ(ceph_unmount(cmount), 0); - ASSERT_EQ(ceph_mount(cmount, dir1), 0); + ASSERT_EQ(do_ceph_mount(cmount, dir1), 0); ASSERT_EQ(ceph_statx(cmount, "/..", &stx, CEPH_STATX_INO, 0), 0); /* Test readdir behavior */ @@ -1848,7 +1855,7 @@ TEST(LibCephFS, Btime) { ASSERT_EQ(ceph_create(&cmount, NULL), 0); ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0); ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL)); - ASSERT_EQ(ceph_mount(cmount, "/"), 0); + ASSERT_EQ(do_ceph_mount(cmount, "/"), 0); char filename[32]; sprintf(filename, "/getattrx%x", getpid()); @@ -1887,7 +1894,7 @@ TEST(LibCephFS, SetBtime) { ASSERT_EQ(ceph_create(&cmount, NULL), 0); ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0); ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL)); - ASSERT_EQ(ceph_mount(cmount, "/"), 0); + ASSERT_EQ(do_ceph_mount(cmount, "/"), 0); char filename[32]; sprintf(filename, "/setbtime%x", getpid()); @@ -1919,8 +1926,8 @@ TEST(LibCephFS, LazyStatx) { ASSERT_EQ(ceph_conf_read_file(cmount2, NULL), 0); ASSERT_EQ(0, ceph_conf_parse_env(cmount1, NULL)); ASSERT_EQ(0, ceph_conf_parse_env(cmount2, NULL)); - ASSERT_EQ(ceph_mount(cmount1, "/"), 0); - ASSERT_EQ(ceph_mount(cmount2, "/"), 0); + ASSERT_EQ(do_ceph_mount(cmount1, "/"), 0); + ASSERT_EQ(do_ceph_mount(cmount2, "/"), 0); char filename[32]; sprintf(filename, "lazystatx%x", getpid()); @@ -1965,7 +1972,7 @@ TEST(LibCephFS, ChangeAttr) { ASSERT_EQ(ceph_create(&cmount, NULL), 0); ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0); ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL)); - ASSERT_EQ(ceph_mount(cmount, "/"), 0); + ASSERT_EQ(do_ceph_mount(cmount, "/"), 0); char filename[32]; sprintf(filename, "/changeattr%x", getpid()); @@ -2009,7 +2016,7 @@ TEST(LibCephFS, DirChangeAttrCreateFile) { ASSERT_EQ(ceph_create(&cmount, NULL), 0); ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0); ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL)); - ASSERT_EQ(ceph_mount(cmount, "/"), 0); + ASSERT_EQ(do_ceph_mount(cmount, "/"), 0); char dirpath[32], filepath[56]; sprintf(dirpath, "/dirchange%x", getpid()); @@ -2053,7 +2060,7 @@ TEST(LibCephFS, DirChangeAttrRenameFile) { ASSERT_EQ(ceph_create(&cmount, NULL), 0); ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0); ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL)); - ASSERT_EQ(ceph_mount(cmount, "/"), 0); + ASSERT_EQ(do_ceph_mount(cmount, "/"), 0); char dirpath[32], filepath[56], newfilepath[56]; sprintf(dirpath, "/dirchange%x", getpid()); @@ -2099,7 +2106,7 @@ TEST(LibCephFS, DirChangeAttrRemoveFile) { ASSERT_EQ(ceph_create(&cmount, NULL), 0); ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0); ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL)); - ASSERT_EQ(ceph_mount(cmount, "/"), 0); + ASSERT_EQ(do_ceph_mount(cmount, "/"), 0); char dirpath[32], filepath[56]; sprintf(dirpath, "/dirchange%x", getpid()); @@ -2145,7 +2152,7 @@ TEST(LibCephFS, SetSize) { ASSERT_EQ(ceph_create(&cmount, NULL), 0); ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0); ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL)); - ASSERT_EQ(ceph_mount(cmount, "/"), 0); + ASSERT_EQ(do_ceph_mount(cmount, "/"), 0); char filename[32]; sprintf(filename, "/setsize%x", getpid()); @@ -2175,7 +2182,7 @@ TEST(LibCephFS, OperationsOnRoot) ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL)); ASSERT_EQ(0, ceph_init(cmount)); ASSERT_EQ(0, ceph_mount_perms_set(cmount, rootcred)); - ASSERT_EQ(0, ceph_mount(cmount, "/")); + ASSERT_EQ(do_ceph_mount(cmount, "/"), 0); char dirname[32]; sprintf(dirname, "/somedir%x", getpid()); @@ -2220,7 +2227,7 @@ static void shutdown_racer_func() ASSERT_EQ(ceph_create(&cmount, NULL), 0); ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0); ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL)); - ASSERT_EQ(ceph_mount(cmount, "/"), 0); + ASSERT_EQ(do_ceph_mount(cmount, "/"), 0); ceph_shutdown(cmount); } } @@ -2386,7 +2393,7 @@ TEST(LibCephFS, TestUtime) { ASSERT_EQ(ceph_create(&cmount, NULL), 0); ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0); ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL)); - ASSERT_EQ(ceph_mount(cmount, NULL), 0); + ASSERT_EQ(do_ceph_mount(cmount, NULL), 0); char test_file[256]; sprintf(test_file, "test_utime_file_%d", getpid()); @@ -2423,7 +2430,7 @@ TEST(LibCephFS, TestUtimes) { ASSERT_EQ(ceph_create(&cmount, NULL), 0); ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0); ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL)); - ASSERT_EQ(ceph_mount(cmount, NULL), 0); + ASSERT_EQ(do_ceph_mount(cmount, NULL), 0); char test_file[256]; char test_symlink[256]; @@ -2474,7 +2481,7 @@ TEST(LibCephFS, TestFutimens) { ASSERT_EQ(ceph_create(&cmount, NULL), 0); ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0); ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL)); - ASSERT_EQ(ceph_mount(cmount, NULL), 0); + ASSERT_EQ(do_ceph_mount(cmount, NULL), 0); char test_file[256]; @@ -2503,7 +2510,7 @@ TEST(LibCephFS, OperationsOnDotDot) { ASSERT_EQ(ceph_create(&cmount, NULL), 0); ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0); ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL)); - ASSERT_EQ(ceph_mount(cmount, NULL), 0); + ASSERT_EQ(do_ceph_mount(cmount, NULL), 0); char c_dir[512], c_dir_dot[1024], c_dir_dotdot[1024]; char c_non_existent_dir[1024], c_non_existent_dirs[1024]; @@ -2541,7 +2548,7 @@ TEST(LibCephFS, Caps_vxattr) { ASSERT_EQ(ceph_create(&cmount, NULL), 0); ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0); ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL)); - ASSERT_EQ(ceph_mount(cmount, NULL), 0); + ASSERT_EQ(do_ceph_mount(cmount, NULL), 0); char test_caps_vxattr_file[256]; char gxattrv[128]; @@ -2567,7 +2574,7 @@ TEST(LibCephFS, SnapXattrs) { ASSERT_EQ(ceph_create(&cmount, NULL), 0); ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0); ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL)); - ASSERT_EQ(ceph_mount(cmount, NULL), 0); + ASSERT_EQ(do_ceph_mount(cmount, NULL), 0); char test_snap_xattr_file[256]; char c_temp[PATH_MAX]; @@ -2651,7 +2658,7 @@ TEST(LibCephFS, Lseek) { ASSERT_EQ(0, ceph_create(&cmount, NULL)); ASSERT_EQ(0, ceph_conf_read_file(cmount, NULL)); ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL)); - ASSERT_EQ(0, ceph_mount(cmount, "/")); + ASSERT_EQ(0, do_ceph_mount(cmount, "/")); char c_path[1024]; sprintf(c_path, "test_lseek_%d", getpid()); @@ -2688,7 +2695,7 @@ TEST(LibCephFS, SnapInfoOnNonSnapshot) { ASSERT_EQ(ceph_create(&cmount, NULL), 0); ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0); ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL)); - ASSERT_EQ(ceph_mount(cmount, NULL), 0); + ASSERT_EQ(do_ceph_mount(cmount, NULL), 0); struct snap_info info; ASSERT_EQ(-EINVAL, ceph_get_snap_info(cmount, "/", &info)); @@ -2701,7 +2708,7 @@ TEST(LibCephFS, EmptySnapInfo) { ASSERT_EQ(ceph_create(&cmount, NULL), 0); ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0); ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL)); - ASSERT_EQ(ceph_mount(cmount, NULL), 0); + ASSERT_EQ(do_ceph_mount(cmount, NULL), 0); char dir_path[64]; char snap_path[PATH_MAX]; @@ -2727,7 +2734,7 @@ TEST(LibCephFS, SnapInfo) { ASSERT_EQ(ceph_create(&cmount, NULL), 0); ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0); ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL)); - ASSERT_EQ(ceph_mount(cmount, NULL), 0); + ASSERT_EQ(do_ceph_mount(cmount, NULL), 0); char dir_path[64]; char snap_name[64]; @@ -2772,7 +2779,7 @@ TEST(LibCephFS, LookupInoMDSDir) { ASSERT_EQ(ceph_create(&cmount, NULL), 0); ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0); ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL)); - ASSERT_EQ(ceph_mount(cmount, NULL), 0); + ASSERT_EQ(do_ceph_mount(cmount, NULL), 0); Inode *inode; auto ino = inodeno_t(0x100); /* rank 0 ~mdsdir */ @@ -2788,7 +2795,7 @@ TEST(LibCephFS, LookupVino) { ASSERT_EQ(ceph_create(&cmount, NULL), 0); ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0); ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL)); - ASSERT_EQ(ceph_mount(cmount, NULL), 0); + ASSERT_EQ(do_ceph_mount(cmount, NULL), 0); char dir_path[64]; char snap_name[64]; @@ -2831,7 +2838,7 @@ TEST(LibCephFS, LookupVino) { ASSERT_EQ(0, ceph_create(&cmount, NULL)); ASSERT_EQ(0, ceph_conf_read_file(cmount, NULL)); ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL)); - ASSERT_EQ(0, ceph_mount(cmount, NULL)); + ASSERT_EQ(0, do_ceph_mount(cmount, NULL)); // Find them all Inode *inode; @@ -2860,7 +2867,7 @@ TEST(LibCephFS, Openat) { ASSERT_EQ(0, ceph_create(&cmount, NULL)); ASSERT_EQ(0, ceph_conf_read_file(cmount, NULL)); ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL)); - ASSERT_EQ(0, ceph_mount(cmount, "/")); + ASSERT_EQ(0, do_ceph_mount(cmount, "/")); char c_rel_dir[64]; char c_dir[128]; @@ -2903,7 +2910,7 @@ TEST(LibCephFS, Statxat) { ASSERT_EQ(ceph_create(&cmount, NULL), 0); ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0); ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL)); - ASSERT_EQ(ceph_mount(cmount, NULL), 0); + ASSERT_EQ(do_ceph_mount(cmount, NULL), 0); char dir_name[64]; char rel_file_name_1[128]; @@ -2984,7 +2991,7 @@ TEST(LibCephFS, StatxatATFDCWD) { ASSERT_EQ(ceph_create(&cmount, NULL), 0); ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0); ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL)); - ASSERT_EQ(ceph_mount(cmount, NULL), 0); + ASSERT_EQ(do_ceph_mount(cmount, NULL), 0); char dir_name[64]; char rel_file_name_1[128]; @@ -3023,7 +3030,7 @@ TEST(LibCephFS, Fdopendir) { ASSERT_EQ(ceph_create(&cmount, NULL), 0); ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0); ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL)); - ASSERT_EQ(ceph_mount(cmount, "/"), 0); + ASSERT_EQ(do_ceph_mount(cmount, "/"), 0); char foostr[256]; sprintf(foostr, "/dir_ls%d", mypid); @@ -3067,7 +3074,7 @@ TEST(LibCephFS, FdopendirATFDCWD) { ASSERT_EQ(ceph_create(&cmount, NULL), 0); ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0); ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL)); - ASSERT_EQ(ceph_mount(cmount, "/"), 0); + ASSERT_EQ(do_ceph_mount(cmount, "/"), 0); char foostr[256]; sprintf(foostr, "/dir_ls%d", mypid); @@ -3109,7 +3116,7 @@ TEST(LibCephFS, FdopendirReaddirTestWithDelete) { ASSERT_EQ(ceph_create(&cmount, NULL), 0); ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0); ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL)); - ASSERT_EQ(ceph_mount(cmount, "/"), 0); + ASSERT_EQ(do_ceph_mount(cmount, "/"), 0); char foostr[256]; sprintf(foostr, "/dir_ls%d", mypid); @@ -3151,7 +3158,7 @@ TEST(LibCephFS, FdopendirOnNonDir) { ASSERT_EQ(ceph_create(&cmount, NULL), 0); ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0); ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL)); - ASSERT_EQ(ceph_mount(cmount, "/"), 0); + ASSERT_EQ(do_ceph_mount(cmount, "/"), 0); char foostr[256]; sprintf(foostr, "/dir_ls%d", mypid); @@ -3179,7 +3186,7 @@ TEST(LibCephFS, Mkdirat) { ASSERT_EQ(ceph_create(&cmount, NULL), 0); ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0); ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL)); - ASSERT_EQ(ceph_mount(cmount, "/"), 0); + ASSERT_EQ(do_ceph_mount(cmount, "/"), 0); char dir_name[128]; char dir_path1[256]; @@ -3210,7 +3217,7 @@ TEST(LibCephFS, MkdiratATFDCWD) { ASSERT_EQ(ceph_create(&cmount, NULL), 0); ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0); ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL)); - ASSERT_EQ(ceph_mount(cmount, "/"), 0); + ASSERT_EQ(do_ceph_mount(cmount, "/"), 0); char dir_name[128]; char dir_path1[256]; @@ -3237,7 +3244,7 @@ TEST(LibCephFS, Readlinkat) { ASSERT_EQ(ceph_create(&cmount, NULL), 0); ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0); ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL)); - ASSERT_EQ(ceph_mount(cmount, "/"), 0); + ASSERT_EQ(do_ceph_mount(cmount, "/"), 0); char dir_name[128]; char dir_path[256]; @@ -3293,7 +3300,7 @@ TEST(LibCephFS, ReadlinkatATFDCWD) { ASSERT_EQ(ceph_create(&cmount, NULL), 0); ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0); ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL)); - ASSERT_EQ(ceph_mount(cmount, "/"), 0); + ASSERT_EQ(do_ceph_mount(cmount, "/"), 0); char dir_name[128]; char dir_path[256]; @@ -3334,7 +3341,7 @@ TEST(LibCephFS, Symlinkat) { ASSERT_EQ(ceph_create(&cmount, NULL), 0); ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0); ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL)); - ASSERT_EQ(ceph_mount(cmount, "/"), 0); + ASSERT_EQ(do_ceph_mount(cmount, "/"), 0); char dir_name[128]; char dir_path[256]; @@ -3380,7 +3387,7 @@ TEST(LibCephFS, SymlinkatATFDCWD) { ASSERT_EQ(ceph_create(&cmount, NULL), 0); ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0); ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL)); - ASSERT_EQ(ceph_mount(cmount, "/"), 0); + ASSERT_EQ(do_ceph_mount(cmount, "/"), 0); char dir_name[128]; char dir_path[256]; @@ -3421,7 +3428,7 @@ TEST(LibCephFS, Unlinkat) { ASSERT_EQ(ceph_create(&cmount, NULL), 0); ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0); ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL)); - ASSERT_EQ(ceph_mount(cmount, "/"), 0); + ASSERT_EQ(do_ceph_mount(cmount, "/"), 0); char dir_name[128]; char dir_path[256]; @@ -3458,7 +3465,7 @@ TEST(LibCephFS, UnlinkatATFDCWD) { ASSERT_EQ(ceph_create(&cmount, NULL), 0); ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0); ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL)); - ASSERT_EQ(ceph_mount(cmount, "/"), 0); + ASSERT_EQ(do_ceph_mount(cmount, "/"), 0); char dir_name[128]; char dir_path[256]; @@ -3492,7 +3499,7 @@ TEST(LibCephFS, Chownat) { ASSERT_EQ(ceph_create(&cmount, NULL), 0); ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0); ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL)); - ASSERT_EQ(ceph_mount(cmount, "/"), 0); + ASSERT_EQ(do_ceph_mount(cmount, "/"), 0); char dir_name[128]; char dir_path[256]; @@ -3544,7 +3551,7 @@ TEST(LibCephFS, ChownatATFDCWD) { ASSERT_EQ(ceph_create(&cmount, NULL), 0); ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0); ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL)); - ASSERT_EQ(ceph_mount(cmount, "/"), 0); + ASSERT_EQ(do_ceph_mount(cmount, "/"), 0); char dir_name[128]; char dir_path[256]; @@ -3588,7 +3595,7 @@ TEST(LibCephFS, Chmodat) { ASSERT_EQ(ceph_create(&cmount, NULL), 0); ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0); ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL)); - ASSERT_EQ(ceph_mount(cmount, "/"), 0); + ASSERT_EQ(do_ceph_mount(cmount, "/"), 0); char dir_name[128]; char dir_path[256]; @@ -3631,7 +3638,7 @@ TEST(LibCephFS, ChmodatATFDCWD) { ASSERT_EQ(ceph_create(&cmount, NULL), 0); ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0); ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL)); - ASSERT_EQ(ceph_mount(cmount, "/"), 0); + ASSERT_EQ(do_ceph_mount(cmount, "/"), 0); char dir_name[128]; char dir_path[256]; @@ -3671,7 +3678,7 @@ TEST(LibCephFS, Utimensat) { ASSERT_EQ(ceph_create(&cmount, NULL), 0); ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0); ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL)); - ASSERT_EQ(ceph_mount(cmount, NULL), 0); + ASSERT_EQ(do_ceph_mount(cmount, NULL), 0); char dir_name[128]; char dir_path[256]; @@ -3711,7 +3718,7 @@ TEST(LibCephFS, UtimensatATFDCWD) { ASSERT_EQ(ceph_create(&cmount, NULL), 0); ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0); ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL)); - ASSERT_EQ(ceph_mount(cmount, NULL), 0); + ASSERT_EQ(do_ceph_mount(cmount, NULL), 0); char dir_name[128]; char dir_path[256]; @@ -3747,7 +3754,7 @@ TEST(LibCephFS, LookupMdsPrivateInos) { ASSERT_EQ(ceph_create(&cmount, NULL), 0); ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0); ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL)); - ASSERT_EQ(ceph_mount(cmount, NULL), 0); + ASSERT_EQ(do_ceph_mount(cmount, NULL), 0); Inode *inode; for (int ino = 0; ino < MDS_INO_SYSTEM_BASE; ino++) { @@ -3779,7 +3786,7 @@ TEST(LibCephFS, SetMountTimeoutPostMount) { ASSERT_EQ(ceph_create(&cmount, NULL), 0); ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0); ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL)); - ASSERT_EQ(ceph_mount(cmount, NULL), 0); + ASSERT_EQ(do_ceph_mount(cmount, NULL), 0); ASSERT_EQ(-EINVAL, ceph_set_mount_timeout(cmount, 5)); ceph_shutdown(cmount); @@ -3791,7 +3798,40 @@ TEST(LibCephFS, SetMountTimeout) { ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0); ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL)); ASSERT_EQ(0, ceph_set_mount_timeout(cmount, 5)); - ASSERT_EQ(ceph_mount(cmount, NULL), 0); + ASSERT_EQ(do_ceph_mount(cmount, NULL), 0); + ceph_shutdown(cmount); +} + +TEST(LibCephFS, FsCrypt) { + struct ceph_mount_info *cmount; + ASSERT_EQ(ceph_create(&cmount, NULL), 0); + ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0); + ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL)); + ASSERT_EQ(do_ceph_mount(cmount, NULL), 0); + + char test_xattr_file[NAME_MAX]; + sprintf(test_xattr_file, "test_fscrypt_%d", getpid()); + int fd = ceph_open(cmount, test_xattr_file, O_RDWR|O_CREAT, 0666); + ASSERT_GT(fd, 0); + + ASSERT_EQ(0, ceph_fsetxattr(cmount, fd, "ceph.fscrypt.auth", "foo", 3, CEPH_XATTR_CREATE)); + ASSERT_EQ(0, ceph_fsetxattr(cmount, fd, "ceph.fscrypt.file", "foo", 3, CEPH_XATTR_CREATE)); + + char buf[64]; + ASSERT_EQ(3, ceph_fgetxattr(cmount, fd, "ceph.fscrypt.auth", buf, sizeof(buf))); + ASSERT_EQ(3, ceph_fgetxattr(cmount, fd, "ceph.fscrypt.file", buf, sizeof(buf))); + ASSERT_EQ(0, ceph_close(cmount, fd)); + + ASSERT_EQ(0, ceph_unmount(cmount)); + ASSERT_EQ(0, do_ceph_mount(cmount, NULL)); + + fd = ceph_open(cmount, test_xattr_file, O_RDWR, 0666); + ASSERT_GT(fd, 0); + ASSERT_EQ(3, ceph_fgetxattr(cmount, fd, "ceph.fscrypt.auth", buf, sizeof(buf))); + ASSERT_EQ(3, ceph_fgetxattr(cmount, fd, "ceph.fscrypt.file", buf, sizeof(buf))); + + ASSERT_EQ(0, ceph_close(cmount, fd)); + ASSERT_EQ(0, ceph_unmount(cmount)); ceph_shutdown(cmount); } @@ -3800,7 +3840,7 @@ TEST(LibCephFS, SnapdirAttrs) { ASSERT_EQ(ceph_create(&cmount, NULL), 0); ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0); ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL)); - ASSERT_EQ(ceph_mount(cmount, NULL), 0); + ASSERT_EQ(do_ceph_mount(cmount, NULL), 0); char dir_name[128]; char dir_path[256]; @@ -3959,7 +3999,7 @@ TEST(LibCephFS, SnapdirAttrsOnSnapCreate) { ASSERT_EQ(ceph_create(&cmount, NULL), 0); ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0); ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL)); - ASSERT_EQ(ceph_mount(cmount, NULL), 0); + ASSERT_EQ(do_ceph_mount(cmount, NULL), 0); char dir_name[128]; char dir_path[256]; @@ -4004,7 +4044,7 @@ TEST(LibCephFS, SnapdirAttrsOnSnapDelete) { ASSERT_EQ(ceph_create(&cmount, NULL), 0); ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0); ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL)); - ASSERT_EQ(ceph_mount(cmount, NULL), 0); + ASSERT_EQ(do_ceph_mount(cmount, NULL), 0); char dir_name[128]; char dir_path[256]; @@ -4058,7 +4098,7 @@ TEST(LibCephFS, SnapdirAttrsOnSnapRename) { ASSERT_EQ(ceph_create(&cmount, NULL), 0); ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0); ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL)); - ASSERT_EQ(ceph_mount(cmount, NULL), 0); + ASSERT_EQ(do_ceph_mount(cmount, NULL), 0); char dir_name[128]; char dir_path[256]; diff --git a/src/test/libcephfs/test.h b/src/test/libcephfs/test.h new file mode 100644 index 00000000000..eb5894f87ab --- /dev/null +++ b/src/test/libcephfs/test.h @@ -0,0 +1,6 @@ + +#pragma once + +struct ceph_mount_info; + +void libcephfs_test_set_mount_call(int (*mount_call)(struct ceph_mount_info *cmount, const char *root));