#include "fmt/format.h"
+#define dout_subsys ceph_subsys_rgw
namespace file::listing {
namespace bi = boost::intrusive;
typedef std::tuple<BucketCacheEntry<D, B>*, uint32_t> GetBucketResult;
- GetBucketResult get_bucket(const std::string& name, uint32_t flags)
+ GetBucketResult get_bucket(const DoutPrefixProvider* dpp, const std::string& name, uint32_t flags)
{
/* this fn returns a bucket locked appropriately, having atomically
* found or inserted the required BucketCacheEntry in_avl*/
} else {
/* BucketCacheEntry not in cache */
if (! (flags & BucketCache<D, B>::FLAG_CREATE)) {
- /* the caller does not want to instantiate a new cache
+ /* the caller does not want to instantiate a new cache
* entry (i.e., only wants to notify on an existing one) */
- return result;
- }
+ lat.lock->unlock();
+ return result;
+ }
/* we need to create it */
b = static_cast<BucketCacheEntry<D, B>*>(
lru.insert(&fac, cohort::lru::Edge::MRU, iflags));
int rc __attribute__((unused)) = 0;
GetBucketResult gbr =
- get_bucket(sal_bucket->get_name(),
+ get_bucket(dpp, sal_bucket->get_name(),
BucketCache<D, B>::FLAG_LOCK | BucketCache<D, B>::FLAG_CREATE);
auto [b /* BucketCacheEntry */, flags] = gbr;
if (b /* XXX again, can this fail? */) {
} else {
/* position at start of index */
auto rc = cursor.get(key, data, MDB_FIRST);
+ if (rc == MDB_NOTFOUND) {
+ /* no initial key */
+ return 0;
+ }
if (rc == MDB_SUCCESS) {
proc_result();
}
using namespace LMDBSafe;
int rc{0};
- GetBucketResult gbr = get_bucket(bname, BucketCache<D, B>::FLAG_LOCK);
+ GetBucketResult gbr = get_bucket(nullptr, bname, BucketCache<D, B>::FLAG_LOCK);
auto [b /* BucketCacheEntry */, flags] = gbr;
if (b) {
unique_lock ulk{b->mtx, std::adopt_lock};
if ((b->name != bname) ||
- (b != opaque) ||
+ (opaque && (b != opaque)) ||
(! (b->flags & BucketCacheEntry<D, B>::FLAG_FILLED))) {
/* do nothing */
return 0;
return rc;
} /* notify */
+ int add_entry(const DoutPrefixProvider* dpp, std::string bname, rgw_bucket_dir_entry bde) {
+ using namespace LMDBSafe;
+
+ GetBucketResult gbr = get_bucket(dpp, bname, BucketCache<D, B>::FLAG_LOCK);
+ auto [b /* BucketCacheEntry */, flags] = gbr;
+ if (b) {
+ unique_lock ulk{b->mtx, std::adopt_lock};
+ ulk.unlock();
+
+ auto txn = b->env->getRWTransaction();
+ auto concat_k = concat_key(bde.key);
+ std::string ser_data;
+ zpp::bits::out out(ser_data);
+ struct timespec ts {
+ ceph::real_clock::to_timespec(bde.meta.mtime)
+ };
+ auto errc =
+ out(bde.key.name, bde.key.instance, /* XXX bde.key.ns, */
+ bde.ver.pool, bde.ver.epoch, bde.exists, bde.meta.category,
+ bde.meta.size, ts.tv_sec, ts.tv_nsec, bde.meta.owner,
+ bde.meta.owner_display_name, bde.meta.accounted_size,
+ bde.meta.storage_class, bde.meta.appendable, bde.meta.etag);
+ if (errc.code != std::errc{0}) {
+ abort();
+ }
+ txn->put(b->dbi, concat_k, ser_data);
+
+ txn->commit();
+ lru.unref(b, cohort::lru::FLAG_NONE);
+ } /* b */
+
+ return 0;
+ } /* add_entry */
+
+ int remove_entry(const DoutPrefixProvider* dpp, std::string bname, cls_rgw_obj_key key) {
+ using namespace LMDBSafe;
+
+ GetBucketResult gbr = get_bucket(dpp, bname, BucketCache<D, B>::FLAG_LOCK);
+ auto [b /* BucketCacheEntry */, flags] = gbr;
+ if (b) {
+ unique_lock ulk{b->mtx, std::adopt_lock};
+ ulk.unlock();
+
+ auto txn = b->env->getRWTransaction();
+ auto concat_k = concat_key(key);
+ txn->del(b->dbi, concat_k);
+ txn->commit();
+
+ lru.unref(b, cohort::lru::FLAG_NONE);
+ } /* b */
+
+ return 0;
+ } /* remove_entry */
+
+ int invalidate_bucket(const DoutPrefixProvider* dpp, std::string bname) {
+ using namespace LMDBSafe;
+
+ GetBucketResult gbr = get_bucket(dpp, bname, BucketCache<D, B>::FLAG_LOCK);
+ auto [b /* BucketCacheEntry */, flags] = gbr;
+ if (b) {
+ unique_lock ulk{b->mtx, std::adopt_lock};
+
+ auto txn = b->env->getRWTransaction();
+ mdb_drop(*txn, b->dbi, 0);
+ txn->commit();
+ b->flags &= ~BucketCacheEntry<D, B>::FLAG_FILLED;
+
+ ulk.unlock();
+ } /* b */
+
+ return 0;
+ } /* invalidate_bucket */
}; /* BucketCache */
} // namespace file::listing
#include <sys/xattr.h>
#include <unistd.h>
#include "rgw_multi.h"
-#include "rgw_acl_s3.h"
#include "include/scope_guard.h"
#define dout_subsys ceph_subsys_rgw
namespace rgw { namespace sal {
-const int64_t READ_SIZE = 8 * 1024;
+const int64_t READ_SIZE = 128 * 1024;
const std::string ATTR_PREFIX = "user.X-RGW-";
#define RGW_POSIX_ATTR_BUCKET_INFO "POSIX-Bucket-Info"
#define RGW_POSIX_ATTR_MPUPLOAD "POSIX-Multipart-Upload"
#define RGW_POSIX_ATTR_OWNER "POSIX-Owner"
+#define RGW_POSIX_ATTR_OBJECT_TYPE "POSIX-Object-Type"
const std::string mp_ns = "multipart";
const std::string MP_OBJ_PART_PFX = "part-";
const std::string MP_OBJ_HEAD_NAME = MP_OBJ_PART_PFX + "00000";
+struct POSIXOwner {
+ rgw_user user;
+ std::string display_name;
+
+ POSIXOwner() {}
+
+ POSIXOwner(const rgw_user& _u, const std::string& _n) :
+ user(_u),
+ display_name(_n)
+ {}
+
+ void encode(bufferlist &bl) const {
+ ENCODE_START(1, 1, bl);
+ encode(user, bl);
+ encode(display_name, bl);
+ ENCODE_FINISH(bl);
+ }
+
+ void decode(bufferlist::const_iterator &bl) {
+ DECODE_START(1, bl);
+ decode(user, bl);
+ decode(display_name, bl);
+ DECODE_FINISH(bl);
+ }
+ friend inline std::ostream &operator<<(std::ostream &out,
+ const POSIXOwner &o) {
+ out << o.user << ":" << o.display_name;
+ return out;
+ }
+};
+WRITE_CLASS_ENCODER(POSIXOwner);
+
+std::string get_key_fname(rgw_obj_key& key, bool use_version)
+{
+ std::string oid;
+ if (use_version) {
+ oid = key.get_oid();
+ } else {
+ oid = key.get_index_key_name();
+ }
+ std::string fname = url_encode(oid, true);
+
+ if (!key.get_ns().empty()) {
+ /* Namespaced objects are hidden */
+ fname.insert(0, 1, '.');
+ }
+
+ return fname;
+}
+
+static inline std::string gen_rand_instance_name()
+{
+ enum { OBJ_INSTANCE_LEN = 32 };
+ char buf[OBJ_INSTANCE_LEN + 1];
+
+#if 0
+ gen_rand_alphanumeric_no_underscore(driver->ctx(), buf, OBJ_INSTANCE_LEN);
+#else
+ static uint64_t last_id = UINT64_MAX;
+ snprintf(buf, OBJ_INSTANCE_LEN, "%lx", last_id);
+ last_id--;
+#endif
+
+ return buf;
+}
+
+static inline std::string bucket_fname(std::string name, std::optional<std::string>& ns)
+{
+ std::string bname;
+
+ if (ns)
+ bname = "." + *ns + "_" + url_encode(name, true);
+ else
+ bname = url_encode(name, true);
+
+ return bname;
+}
+
+static inline bool get_attr(Attrs& attrs, const char* name, bufferlist& bl)
+{
+ auto iter = attrs.find(name);
+ if (iter == attrs.end()) {
+ return false;
+ }
+
+ bl = iter->second;
+ return true;
+}
+
+template <typename F>
+static bool decode_attr(Attrs &attrs, const char *name, F &f) {
+ bufferlist bl;
+ if (!get_attr(attrs, name, bl)) {
+ return false;
+ }
+ try {
+ auto bufit = bl.cbegin();
+ decode(f, bufit);
+ } catch (buffer::error &err) {
+ return false;
+ }
+
+ return true;
+}
+
static inline rgw_obj_key decode_obj_key(const char* fname)
{
std::string dname, oname, ns;
dname = url_decode(fname);
- rgw_obj_key::parse_index_key(dname, &oname, &ns);
- rgw_obj_key key(oname, std::string(), ns);
+ rgw_obj_key key;
+ rgw_obj_key::parse_raw_oid(dname, &key);
return key;
}
-static inline ceph::real_time from_statx_timestamp(const struct statx_timestamp& xts)
-{
- struct timespec ts{xts.tv_sec, xts.tv_nsec};
- return ceph::real_clock::from_timespec(ts);
-}
+static inline rgw_obj_key decode_obj_key(const std::string& fname)
+{
+ return decode_obj_key(fname.c_str());
+}
+
+int decode_owner(Attrs& attrs, POSIXOwner& owner)
+{
+ bufferlist bl;
+ if (!decode_attr(attrs, RGW_POSIX_ATTR_OWNER, owner)) {
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
+static inline ceph::real_time from_statx_timestamp(const struct statx_timestamp& xts)
+{
+ struct timespec ts{xts.tv_sec, xts.tv_nsec};
+ return ceph::real_clock::from_timespec(ts);
+}
+
+static inline int copy_dir_fd(int old_fd)
+{
+ return openat(old_fd, ".", O_RDONLY | O_DIRECTORY | O_NOFOLLOW);
+}
+
+static int get_x_attrs(optional_yield y, const DoutPrefixProvider* dpp, int fd,
+ Attrs& attrs, const std::string& display)
+{
+ char namebuf[64 * 1024]; // Max list size supported on linux
+ ssize_t buflen;
+ int ret;
+
+ buflen = flistxattr(fd, namebuf, sizeof(namebuf));
+ if (buflen < 0) {
+ ret = errno;
+ ldpp_dout(dpp, 0) << "ERROR: could not list attributes for " << display << ": "
+ << cpp_strerror(ret) << dendl;
+ return -ret;
+ }
+
+ char *keyptr = namebuf;
+ while (buflen > 0) {
+ std::string value;
+ ssize_t vallen, keylen;
+ char* vp;
+
+ keylen = strlen(keyptr) + 1;
+ std::string key(keyptr);
+ std::string::size_type prefixloc = key.find(ATTR_PREFIX);
+
+ if (prefixloc == std::string::npos) {
+ /* Not one of our attributes */
+ buflen -= keylen;
+ keyptr += keylen;
+ continue;
+ }
+
+ /* Make a key that has just the attribute name */
+ key.erase(prefixloc, ATTR_PREFIX.length());
+
+ vallen = fgetxattr(fd, keyptr, nullptr, 0);
+ if (vallen < 0) {
+ ret = errno;
+ ldpp_dout(dpp, 0) << "ERROR: could not get attribute " << keyptr << " for " << display << ": " << cpp_strerror(ret) << dendl;
+ return -ret;
+ } else if (vallen == 0) {
+ /* No attribute value for this name */
+ buflen -= keylen;
+ keyptr += keylen;
+ continue;
+ }
+
+ value.reserve(vallen + 1);
+ vp = &value[0];
+
+ vallen = fgetxattr(fd, keyptr, vp, vallen);
+ if (vallen < 0) {
+ ret = errno;
+ ldpp_dout(dpp, 0) << "ERROR: could not get attribute " << keyptr << " for " << display << ": " << cpp_strerror(ret) << dendl;
+ return -ret;
+ }
+
+ bufferlist bl;
+ bl.append(vp, vallen);
+ attrs.emplace(std::move(key), std::move(bl)); /* key and bl are r-value refs */
+
+ buflen -= keylen;
+ keyptr += keylen;
+ }
+
+ return 0;
+}
+
+static int write_x_attr(const DoutPrefixProvider* dpp, optional_yield y, int fd,
+ const std::string& key, bufferlist& value,
+ const std::string& display)
+{
+ int ret;
+ std::string attrname;
+
+ attrname = ATTR_PREFIX + key;
+
+ ret = fsetxattr(fd, attrname.c_str(), value.c_str(), value.length(), 0);
+ if (ret < 0) {
+ ret = errno;
+ ldpp_dout(dpp, 0) << "ERROR: could not write attribute " << attrname << " for " << display << ": " << cpp_strerror(ret) << dendl;
+ return -ret;
+ }
+
+ return 0;
+}
+
+static int remove_x_attr(const DoutPrefixProvider *dpp, optional_yield y,
+ int fd, const std::string &key,
+ const std::string &display)
+{
+ int ret;
+ std::string attrname{ATTR_PREFIX + key};
+
+ ret = fremovexattr(fd, attrname.c_str());
+ if (ret < 0) {
+ ret = errno;
+ ldpp_dout(dpp, 0) << "ERROR: could not remove attribute " << attrname << " for " << display << ": " << cpp_strerror(ret) << dendl;
+ return -ret;
+ }
+
+ return 0;
+}
+
+static int delete_directory(int parent_fd, const char* dname, bool delete_children,
+ const DoutPrefixProvider* dpp)
+{
+ int ret;
+ int dir_fd = -1;
+ DIR *dir;
+ struct dirent *entry;
+
+ dir_fd = openat(parent_fd, dname, O_RDONLY | O_DIRECTORY | O_NOFOLLOW);
+ if (dir_fd < 0) {
+ dir_fd = errno;
+ ldpp_dout(dpp, 0) << "ERROR: could not open subdir " << dname << ": "
+ << cpp_strerror(dir_fd) << dendl;
+ return -dir_fd;
+ }
+
+ dir = fdopendir(dir_fd);
+ if (dir == NULL) {
+ ret = errno;
+ ldpp_dout(dpp, 0) << "ERROR: could not open bucket " << dname
+ << " for listing: " << cpp_strerror(ret) << dendl;
+ return -ret;
+ }
+
+ errno = 0;
+ while ((entry = readdir(dir)) != NULL) {
+ struct statx stx;
+
+ if ((entry->d_name[0] == '.' && entry->d_name[1] == '\0') ||
+ (entry->d_name[0] == '.' && entry->d_name[1] == '.' &&
+ entry->d_name[2] == '\0')) {
+ /* Skip . and .. */
+ errno = 0;
+ continue;
+ }
+
+ std::string_view d_name = entry->d_name;
+ bool is_mp = d_name.starts_with("." + mp_ns);
+ if (!is_mp && !delete_children) {
+ return -ENOTEMPTY;
+ }
+
+ ret = statx(dir_fd, entry->d_name, AT_SYMLINK_NOFOLLOW, STATX_ALL, &stx);
+ if (ret < 0) {
+ ret = errno;
+ ldpp_dout(dpp, 0) << "ERROR: could not stat object " << entry->d_name
+ << ": " << cpp_strerror(ret) << dendl;
+ return -ret;
+ }
+
+ if (S_ISDIR(stx.stx_mode)) {
+ /* Recurse */
+ ret = delete_directory(dir_fd, entry->d_name, true, dpp);
+ if (ret < 0) {
+ return ret;
+ }
+
+ continue;
+ }
+
+ /* Otherwise, unlink */
+ ret = unlinkat(dir_fd, entry->d_name, 0);
+ if (ret < 0) {
+ ret = errno;
+ ldpp_dout(dpp, 0) << "ERROR: could not remove file " << entry->d_name
+ << ": " << cpp_strerror(ret) << dendl;
+ return -ret;
+ }
+ }
+
+ ret = unlinkat(parent_fd, dname, AT_REMOVEDIR);
+ if (ret < 0) {
+ ret = errno;
+ if (errno != ENOENT) {
+ ldpp_dout(dpp, 0) << "ERROR: could not remove bucket " << dname << ": "
+ << cpp_strerror(ret) << dendl;
+ return -ret;
+ }
+ }
+
+ return 0;
+}
+
+int FSEnt::stat(const DoutPrefixProvider* dpp, bool force)
+{
+ if (force) {
+ stat_done = false;
+ }
+
+ if (stat_done) {
+ return 0;
+ }
+
+ int ret = statx(parent->get_fd(), fname.c_str(), AT_SYMLINK_NOFOLLOW,
+ STATX_ALL, &stx);
+ if (ret < 0) {
+ ret = errno;
+ ldpp_dout(dpp, 0) << "ERROR: could not stat " << get_name() << ": "
+ << cpp_strerror(ret) << dendl;
+ exist = false;
+ return -ret;
+ }
+
+ exist = true;
+ stat_done = true;
+ return 0;
+}
+
+int FSEnt::write_attrs(const DoutPrefixProvider* dpp, optional_yield y, Attrs& attrs, Attrs* extra_attrs)
+{
+ int ret = open(dpp);
+ if (ret < 0) {
+ return ret;
+ }
+
+ /* Set the type */
+ bufferlist type_bl;
+ ObjectType type{get_type()};
+ type.encode(type_bl);
+ attrs[RGW_POSIX_ATTR_OBJECT_TYPE] = type_bl;
+
+ if (extra_attrs) {
+ for (auto &it : *extra_attrs) {
+ ret = write_x_attr(dpp, y, fd, it.first, it.second, get_name());
+ if (ret < 0) {
+ return ret;
+ }
+ }
+ }
+
+ for (auto& it : attrs) {
+ ret = write_x_attr(dpp, y, fd, it.first, it.second, get_name());
+ if (ret < 0) {
+ return ret;
+ }
+ }
+
+ return 0;
+}
+
+int FSEnt::read_attrs(const DoutPrefixProvider* dpp, optional_yield y, Attrs& attrs)
+{
+ int ret = open(dpp);
+ if (ret < 0) {
+ return ret;
+ }
+
+ return get_x_attrs(y, dpp, get_fd(), attrs, get_name());
+}
+
+int FSEnt::fill_cache(const DoutPrefixProvider *dpp, optional_yield y, fill_cache_cb_t& cb)
+{
+ rgw_bucket_dir_entry bde{};
+
+ rgw_obj_key key = decode_obj_key(get_name());
+ if (parent->get_type() == ObjectType::MULTIPART) {
+ key.ns = mp_ns;
+ }
+ key.get_index_key(&bde.key);
+ bde.ver.pool = 1;
+ bde.ver.epoch = 1;
+
+ switch (parent->get_type().type) {
+ case ObjectType::VERSIONED:
+ bde.flags = rgw_bucket_dir_entry::FLAG_VER;
+ bde.exists = true;
+ if (!key.have_instance()) {
+ bde.flags |= rgw_bucket_dir_entry::FLAG_CURRENT;
+ }
+ break;
+ case ObjectType::MULTIPART:
+ case ObjectType::DIRECTORY:
+ bde.exists = true;
+ break;
+ case ObjectType::UNKNOWN:
+ case ObjectType::FILE:
+ case ObjectType::SYMLINK:
+ return -EINVAL;
+ }
+
+ Attrs attrs;
+ int ret = open(dpp);
+ if (ret < 0)
+ return ret;
+
+ ret = get_x_attrs(y, dpp, get_fd(), attrs, get_name());
+ if (ret < 0)
+ return ret;
+
+ POSIXOwner o;
+ ret = decode_owner(attrs, o);
+ if (ret < 0) {
+ bde.meta.owner = "unknown";
+ bde.meta.owner_display_name = "unknown";
+ } else {
+ bde.meta.owner = o.user.to_str();
+ bde.meta.owner_display_name = o.display_name;
+ }
+ bde.meta.category = RGWObjCategory::Main;
+ bde.meta.size = stx.stx_size;
+ bde.meta.accounted_size = stx.stx_size;
+ bde.meta.mtime = from_statx_timestamp(stx.stx_mtime);
+ bde.meta.storage_class = RGW_STORAGE_CLASS_STANDARD;
+ bde.meta.appendable = true;
+ bufferlist etag_bl;
+ if (rgw::sal::get_attr(attrs, RGW_ATTR_ETAG, etag_bl)) {
+ bde.meta.etag = etag_bl.to_str();
+ }
+
+ return cb(dpp, bde);
+}
+
+int File::create(const DoutPrefixProvider *dpp, bool* existed, bool temp_file)
+{
+ int flags, ret;
+ std::string path;
+ if(temp_file) {
+ flags = O_TMPFILE | O_RDWR;
+ path = ".";
+ } else {
+ flags = O_CREAT | O_RDWR;
+ path = get_name();
+ }
+
+ ret = openat(parent->get_fd(), path.c_str(), flags | O_NOFOLLOW, S_IRWXU);
+ if (ret < 0) {
+ ret = errno;
+ if (ret == EEXIST) {
+ return 0;
+ }
+ ldpp_dout(dpp, 0) << "ERROR: could not open object " << get_name() << ": "
+ << cpp_strerror(ret) << dendl;
+ return -ret;
+ }
+
+ fd = ret;
+
+ return 0;
+}
+
+int File::open(const DoutPrefixProvider* dpp)
+{
+ if (fd >= 0) {
+ return 0;
+ }
+
+ int ret = openat(parent->get_fd(), fname.c_str(), O_RDWR, S_IRWXU);
+ if (ret < 0) {
+ ret = errno;
+ ldpp_dout(dpp, 0) << "ERROR: could not open object " << get_name() << ": "
+ << cpp_strerror(ret) << dendl;
+ return -ret;
+ }
+
+ fd = ret;
+
+ return 0;
+}
+
+int File::close()
+{
+ if (fd < 0) {
+ return 0;
+ }
+
+ int ret = ::fsync(fd);
+ if(ret < 0) {
+ return ret;
+ }
+
+ ret = ::close(fd);
+ if(ret < 0) {
+ return ret;
+ }
+ fd = -1;
+
+ return 0;
+}
+
+
+int File::stat(const DoutPrefixProvider* dpp, bool force)
+{
+ int ret = FSEnt::stat(dpp, force);
+ if (ret < 0) {
+ return ret;
+ }
+
+ if (!S_ISREG(stx.stx_mode)) {
+ /* Not a file */
+ ldpp_dout(dpp, 0) << "ERROR: " << get_name() << " is not a file" << dendl;
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
+int File::write(int64_t ofs, bufferlist& bl, const DoutPrefixProvider* dpp,
+ optional_yield y)
+{
+ int64_t left = bl.length();
+ char* curp = bl.c_str();
+ ssize_t ret;
+
+ ret = fchmod(fd, S_IRUSR|S_IWUSR);
+ if(ret < 0) {
+ ldpp_dout(dpp, 0) << "ERROR: could not change permissions on object " << get_name() << ": "
+ << cpp_strerror(ret) << dendl;
+ return ret;
+ }
+
+
+ ret = lseek(fd, ofs, SEEK_SET);
+ if (ret < 0) {
+ ret = errno;
+ ldpp_dout(dpp, 0) << "ERROR: could not seek object " << get_name() << " to "
+ << ofs << " :" << cpp_strerror(ret) << dendl;
+ return -ret;
+ }
+
+ while (left > 0) {
+ ret = ::write(fd, curp, left);
+ if (ret < 0) {
+ ret = errno;
+ ldpp_dout(dpp, 0) << "ERROR: could not write object " << get_name() << ": "
+ << cpp_strerror(ret) << dendl;
+ return -ret;
+ }
+
+ curp += ret;
+ left -= ret;
+ }
+
+ return 0;
+}
+
+int File::read(int64_t ofs, int64_t left, bufferlist& bl,
+ const DoutPrefixProvider* dpp, optional_yield y)
+{
+ int64_t len = std::min(left, READ_SIZE);
+ ssize_t ret;
+
+ ret = lseek(fd, ofs, SEEK_SET);
+ if (ret < 0) {
+ ret = errno;
+ ldpp_dout(dpp, 0) << "ERROR: could not seek object " << get_name() << " to "
+ << ofs << " :" << cpp_strerror(ret) << dendl;
+ return -ret;
+ }
+
+ char read_buf[READ_SIZE];
+ ret = ::read(fd, read_buf, len);
+ if (ret < 0) {
+ ret = errno;
+ ldpp_dout(dpp, 0) << "ERROR: could not read object " << get_name() << ": "
+ << cpp_strerror(ret) << dendl;
+ return -ret;
+ }
+
+ bl.append(read_buf, ret);
+
+ return ret;
+}
+
+int File::copy(const DoutPrefixProvider *dpp, optional_yield y,
+ Directory* dst_dir, const std::string& dst_name)
+{
+ off64_t scount = 0, dcount = 0;
+
+ int ret = stat(dpp);
+ if (ret < 0) {
+ ldpp_dout(dpp, 0) << "ERROR: could not stat source file " << get_name()
+ << dendl;
+ return ret;
+ }
+
+ ret = open(dpp);
+ if (ret < 0) {
+ ldpp_dout(dpp, 0) << "ERROR: could not open source file " << get_name()
+ << dendl;
+ return ret;
+ }
+
+ // Delete the target
+ {
+ std::unique_ptr<FSEnt> del;
+ ret = dst_dir->get_ent(dpp, y, dst_name, std::string(), del);
+ if (ret >= 0) {
+ ret = del->remove(dpp, y, /*delete_children=*/true);
+ if (ret < 0) {
+ ldpp_dout(dpp, 0) << "ERROR: could not remove dest " << dst_name
+ << dendl;
+ return ret;
+ }
+ }
+ }
+
+ std::unique_ptr<File> dest = clone();
+ dest->parent = dst_dir;
+ dest->fname = dst_name;
+
+ ret = dest->create(dpp);
+ if (ret < 0) {
+ ldpp_dout(dpp, 0) << "ERROR: could not create dest file "
+ << dest->get_name() << dendl;
+ return ret;
+ }
+ ret = dest->open(dpp);
+ if (ret < 0) {
+ ldpp_dout(dpp, 0) << "ERROR: could not open dest file "
+ << dest->get_name() << dendl;
+ return ret;
+ }
+
+ ret = copy_file_range(fd, &scount, dest->get_fd(), &dcount, get_size(), 0);
+ if (ret < 0) {
+ ret = errno;
+ ldpp_dout(dpp, 0) << "ERROR: could not copy object " << dest->get_name()
+ << ": " << cpp_strerror(ret) << dendl;
+ return -ret;
+ }
+
+ return 0;
+}
+
+int File::remove(const DoutPrefixProvider* dpp, optional_yield y, bool delete_children)
+{
+ if (!exists()) {
+ return 0;
+ }
+
+ int ret = unlinkat(parent->get_fd(), fname.c_str(), 0);
+ if (ret < 0) {
+ ret = errno;
+ if (errno != ENOENT) {
+ ldpp_dout(dpp, 0) << "ERROR: could not remove object " << get_name()
+ << ": " << cpp_strerror(ret) << dendl;
+ return -ret;
+ }
+ }
+
+ return 0;
+}
+
+int File::link_temp_file(const DoutPrefixProvider *dpp, optional_yield y, std::string temp_fname)
+{
+ if (fd < 0) {
+ return 0;
+ }
+
+ char temp_file_path[PATH_MAX];
+ // Only works on Linux - Non-portable
+ snprintf(temp_file_path, PATH_MAX, "/proc/self/fd/%d", fd);
+
+ int ret = linkat(AT_FDCWD, temp_file_path, parent->get_fd(), temp_fname.c_str(), AT_SYMLINK_FOLLOW);
+ if(ret < 0) {
+ ret = errno;
+ ldpp_dout(dpp, 0) << "ERROR: linkat for temp file could not finish: "
+ << cpp_strerror(ret) << dendl;
+ return -ret;
+ }
+
+ ret = renameat(parent->get_fd(), temp_fname.c_str(), parent->get_fd(), get_name().c_str());
+ if(ret < 0) {
+ ret = errno;
+ ldpp_dout(dpp, 0) << "ERROR: renameat for object could not finish: "
+ << cpp_strerror(ret) << dendl;
+ return -ret;
+ }
+
+ return 0;
+}
+
+bool Directory::file_exists(std::string& name)
+{
+ struct statx nstx;
+ int ret = statx(fd, name.c_str(), AT_SYMLINK_NOFOLLOW, STATX_ALL, &nstx);
+
+ return (ret >= 0);
+}
+
+int Directory::create(const DoutPrefixProvider* dpp, bool* existed, bool temp_file)
+{
+ if (temp_file) {
+ ldpp_dout(dpp, 0) << "ERROR: cannot create directory with temp_file " << get_name() << dendl;
+ return -EINVAL;
+ }
+
+ int ret = mkdirat(parent->get_fd(), fname.c_str(), S_IRWXU);
+ if (ret < 0) {
+ ret = errno;
+ if (ret != EEXIST) {
+ if (dpp)
+ ldpp_dout(dpp, 0) << "ERROR: could not create bucket " << get_name() << ": "
+ << cpp_strerror(ret) << dendl;
+ return -ret;
+ } else if (existed != nullptr) {
+ *existed = true;
+ }
+ }
+
+ return 0;
+}
+
+int Directory::open(const DoutPrefixProvider* dpp)
+{
+ if (fd >= 0) {
+ return 0;
+ }
+
+ int pfd{AT_FDCWD};
+ if (parent)
+ pfd = parent->get_fd();
+
+ int ret = openat(pfd, fname.c_str(), O_RDONLY | O_DIRECTORY | O_NOFOLLOW);
+ if (ret < 0) {
+ ret = errno;
+ ldpp_dout(dpp, 0) << "ERROR: could not open dir " << get_name() << ": "
+ << cpp_strerror(ret) << dendl;
+ return -ret;
+ }
+
+ fd = ret;
+
+ return 0;
+}
+
+int Directory::close()
+{
+ if (fd < 0) {
+ return 0;
+ }
+
+ ::close(fd);
+ fd = -1;
+
+ return 0;
+}
+
+int Directory::stat(const DoutPrefixProvider* dpp, bool force)
+{
+ int ret = FSEnt::stat(dpp, force);
+ if (ret < 0) {
+ return ret;
+ }
+
+ if (!S_ISDIR(stx.stx_mode)) {
+ /* Not a directory */
+ ldpp_dout(dpp, 0) << "ERROR: " << get_name() << " is not a directory" << dendl;
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
+int Directory::remove(const DoutPrefixProvider* dpp, optional_yield y, bool delete_children)
+{
+ return delete_directory(parent->get_fd(), fname.c_str(), delete_children, dpp);
+}
+
+int Directory::write(int64_t ofs, bufferlist& bl, const DoutPrefixProvider* dpp,
+ optional_yield y)
+{
+ return -EINVAL;
+}
+
+int Directory::read(int64_t ofs, int64_t left, bufferlist &bl,
+ const DoutPrefixProvider *dpp, optional_yield y)
+{
+ return -EINVAL;
+}
+
+int Directory::link_temp_file(const DoutPrefixProvider *dpp, optional_yield y,
+ std::string temp_fname)
+{
+ return -EINVAL;
+}
+
+template <typename F>
+int Directory::for_each(const DoutPrefixProvider* dpp, const F& func)
+{
+ DIR* dir;
+ struct dirent* entry;
+ int ret;
+
+ ret = open(dpp);
+ if (ret < 0) {
+ return ret;
+ }
+
+ dir = fdopendir(fd);
+ if (dir == NULL) {
+ ret = errno;
+ ldpp_dout(dpp, 0) << "ERROR: could not open dir " << get_name() << " for listing: "
+ << cpp_strerror(ret) << dendl;
+ return -ret;
+ }
+
+ rewinddir(dir);
+
+ ret = 0;
+ while ((entry = readdir(dir)) != NULL) {
+ std::string_view vname(entry->d_name);
+
+ if (vname == "." || vname == "..")
+ continue;
+
+ int r = func(entry->d_name);
+ if (r < 0) {
+ ret = r;
+ break;
+ }
+ }
+
+ if (ret == -EAGAIN) {
+ /* Limit reached */
+ ret = 0;
+ }
+ return ret;
+}
+
+int Directory::rename(const DoutPrefixProvider* dpp, optional_yield y, Directory* dst_dir, std::string dst_name)
+{
+ int flags = 0;
+ int ret;
+ std::string src_name = fname;
+ int parent_fd = parent->get_fd();
+
+ if (dst_dir->file_exists(dst_name)) {
+ flags = RENAME_EXCHANGE;
+ }
+ // swap
+ ret = renameat2(parent_fd, src_name.c_str(), dst_dir->get_fd(), dst_name.c_str(), flags);
+ if(ret < 0) {
+ ret = errno;
+ ldpp_dout(dpp, 0) << "ERROR: renameat2 for shadow object could not finish: "
+ << cpp_strerror(ret) << dendl;
+ return -ret;
+ }
+
+ /* Parent of this dir is now dest dir */
+ parent = dst_dir;
+ /* Name has changed */
+ fname = dst_name;
+
+ // Delete old one (could be file or directory)
+ struct statx stx;
+ ret = statx(parent_fd, src_name.c_str(), AT_SYMLINK_NOFOLLOW,
+ STATX_ALL, &stx);
+ if (ret < 0) {
+ ret = errno;
+ if (ret == ENOENT) {
+ return 0;
+ }
+ ldpp_dout(dpp, 0) << "ERROR: could not stat object " << get_name() << ": "
+ << cpp_strerror(ret) << dendl;
+ return -ret;
+ }
+
+ if (S_ISREG(stx.stx_mode)) {
+ ret = unlinkat(parent_fd, src_name.c_str(), 0);
+ } else if (S_ISDIR(stx.stx_mode)) {
+ ret = delete_directory(parent_fd, src_name.c_str(), true, dpp);
+ }
+ if (ret < 0) {
+ ret = errno;
+ ldpp_dout(dpp, 0) << "ERROR: could not remove old file " << get_name()
+ << ": " << cpp_strerror(ret) << dendl;
+ return -ret;
+ }
+
+ return 0;
+}
+
+int Directory::copy(const DoutPrefixProvider *dpp, optional_yield y,
+ Directory* dst_dir, const std::string& dst_name)
+{
+ int ret;
+
+ // Delete the target
+ {
+ std::unique_ptr<FSEnt> del;
+ ret = dst_dir->get_ent(dpp, y, dst_name, std::string(), del);
+ if (ret >= 0) {
+ ret = del->remove(dpp, y, /*delete_children=*/true);
+ if (ret < 0) {
+ ldpp_dout(dpp, 0) << "ERROR: could not remove dest " << dst_name
+ << dendl;
+ return ret;
+ }
+ }
+ }
+
+ ret = dst_dir->open(dpp);
+ std::unique_ptr<Directory> dest = clone_dir();
+ dest->parent = dst_dir;
+ dest->fname = dst_name;
+
+ ret = dest->create(dpp);
+ if (ret < 0) {
+ ldpp_dout(dpp, 0) << "ERROR: could not create dest " << dest->get_name() << dendl;
+ return ret;
+ }
+
+ Attrs attrs;
+ ret = read_attrs(dpp, y, attrs);
+ if (ret < 0) {
+ ldpp_dout(dpp, 0) << "ERROR: could not read attrs from " << get_name() << dendl;
+ return ret;
+ }
+ ret = dest->write_attrs(dpp, y, attrs, nullptr);
+ if (ret < 0) {
+ ldpp_dout(dpp, 0) << "ERROR: could not write attrs to " << dest->get_name() << dendl;
+ return ret;
+ }
+
+ ret = for_each(dpp, [this, &dest, &dpp, &y](const char* name) {
+ std::unique_ptr<FSEnt> sobj;
+
+ if (name[0] == '.') {
+ /* Skip dotfiles */
+ return 0;
+ }
+
+ int r = this->get_ent(dpp, y, name, std::string(), sobj);
+ if (r < 0)
+ return r;
+ return sobj->copy(dpp, y, dest.get(), name);
+ });
+
+ return ret;
+}
+
+int Directory::get_ent(const DoutPrefixProvider *dpp, optional_yield y, const std::string &name, const std::string& instance, std::unique_ptr<FSEnt>& ent)
+{
+ struct statx nstx;
+ std::unique_ptr<FSEnt> nent;
+
+ int ret = open(dpp);
+ if (ret < 0) {
+ ldpp_dout(dpp, 0) << "ERROR: could not open directory " << name << dendl;
+ return ret;
+ }
+
+ ret = statx(get_fd(), name.c_str(),
+ AT_SYMLINK_NOFOLLOW, STATX_ALL, &nstx);
+ if (ret < 0) {
+ ret = errno;
+ ldpp_dout(dpp, 0) << "ERROR: could not stat object " << name << " in dir "
+ << get_name() << " : " << cpp_strerror(ret) << dendl;
+ return -ret;
+ }
+ if (S_ISREG(nstx.stx_mode)) {
+ nent = std::make_unique<File>(name, this, nstx, ctx);
+ } else if (S_ISDIR(nstx.stx_mode)) {
+ ObjectType type{ObjectType::MULTIPART};
+ int tmpfd;
+ Attrs attrs;
+
+ tmpfd = openat(get_fd(), name.c_str(), O_RDONLY | O_DIRECTORY | O_NOFOLLOW);
+ if (tmpfd > 0) {
+ ret = get_x_attrs(y, dpp, tmpfd, attrs, name);
+ if (ret >= 0) {
+ decode_attr(attrs, RGW_POSIX_ATTR_OBJECT_TYPE, type);
+ }
+ }
+ switch (type.type) {
+ case ObjectType::VERSIONED:
+ nent = std::make_unique<VersionedDirectory>(name, this, instance, nstx, ctx);
+ break;
+ case ObjectType::MULTIPART:
+ nent = std::make_unique<MPDirectory>(name, this, nstx, ctx);
+ break;
+ case ObjectType::DIRECTORY:
+ nent = std::make_unique<Directory>(name, this, nstx, ctx);
+ break;
+ default:
+ ldpp_dout(dpp, 0) << "ERROR: invalid type " << type << dendl;
+ return -EINVAL;
+ }
+ } else if (S_ISLNK(nstx.stx_mode)) {
+ nent = std::make_unique<Symlink>(name, this, nstx, ctx);
+ } else {
+ return -EINVAL;
+ }
+
+ ent.swap(nent);
+ return 0;
+}
+
+int Directory::fill_cache(const DoutPrefixProvider *dpp, optional_yield y,
+ fill_cache_cb_t &cb)
+{
+ int ret = for_each(dpp, [this, &cb, &dpp, &y](const char *name) {
+ std::unique_ptr<FSEnt> ent;
+
+ if (name[0] == '.') {
+ /* Skip dotfiles */
+ return 0;
+ }
+
+ int ret = get_ent(dpp, y, name, std::string(), ent);
+ if (ret < 0)
+ return ret;
+
+ ent->stat(dpp); // Stat the object to get the type
+
+ ret = ent->fill_cache(dpp, y, cb);
+ if (ret < 0)
+ return ret;
+ return 0;
+ });
+
+ if (ret < 0) {
+ ldpp_dout(dpp, 0) << "ERROR: could not list directory " << get_name() << ": "
+ << cpp_strerror(ret) << dendl;
+ return ret;
+ }
+
+ return 0;
+}
+
+int Symlink::create(const DoutPrefixProvider* dpp, bool* existed, bool temp_file)
+{
+ if (temp_file) {
+ ldpp_dout(dpp, 0) << "ERROR: cannot create symlink with temp_file " << get_name() << dendl;
+ return -EINVAL;
+ }
+
+ int ret = symlinkat(target->get_name().c_str(), parent->get_fd(), fname.c_str());
+ if (ret < 0) {
+ ret = errno;
+ if (ret == EEXIST && existed != nullptr) {
+ *existed = true;
+ }
+ ldpp_dout(dpp, 0) << "ERROR: could not create bucket " << get_name() << ": "
+ << cpp_strerror(ret) << dendl;
+ return -ret;
+ }
+
+ return 0;
+}
+
+int Symlink::fill_target(const DoutPrefixProvider *dpp, Directory* parent, std::string sname, std::string tname, std::unique_ptr<FSEnt>& ent, CephContext* _ctx)
+{
+ int ret;
+
+ if (!tname.empty()) {
+ ret = parent->get_ent(dpp, null_yield, tname, std::string(), ent);
+ if (ret < 0) {
+ ent = std::make_unique<File>(tname, parent, _ctx);
+ }
+ return 0;
+ }
+
+ char link[PATH_MAX];
+ memset(link, 0, sizeof(link));
+ ret = readlinkat(parent->get_fd(), sname.c_str(), link, sizeof(link));
+ if (ret < 0) {
+ ret = errno;
+ return -ret;
+ }
+ ret = parent->get_ent(dpp, null_yield, link, std::string(), ent);
+ if (ret < 0) {
+ ent = std::make_unique<File>(link, parent, _ctx);
+ }
+ return 0;
+}
+
+int Symlink::stat(const DoutPrefixProvider* dpp, bool force)
+{
+ int ret = FSEnt::stat(dpp, force);
+ if (ret < 0) {
+ return ret;
+ }
+
+ if (!S_ISLNK(stx.stx_mode)) {
+ /* Not a symlink */
+ ldpp_dout(dpp, 0) << "ERROR: " << get_name() << " is not a symlink" << dendl;
+ return -EINVAL;
+ }
+
+ struct statx sstx;
+ ret = statx(parent->get_fd(), fname.c_str(), 0, STATX_BASIC_STATS, &sstx);
+ if (ret >= 0) {
+ stx.stx_size = sstx.stx_size;
+ }
+
+ exist = true;
+ return fill_target(dpp, parent, get_name(), std::string(), target, ctx);
+}
+
+int Symlink::fill_cache(const DoutPrefixProvider *dpp, optional_yield y, fill_cache_cb_t& cb)
+{
+ rgw_bucket_dir_entry bde{};
+ int ret;
+
+ rgw_obj_key key = decode_obj_key(get_name());
+ key.get_index_key(&bde.key);
+ bde.ver.pool = 1;
+ bde.ver.epoch = 1;
+
+ bde.flags = rgw_bucket_dir_entry::FLAG_VER;
+ bde.exists = true;
+ bde.flags |= rgw_bucket_dir_entry::FLAG_CURRENT;
+
+ if (!target) {
+ ret = stat(dpp, /*force=*/false);
+ if (ret < 0)
+ return ret;
+ }
+
+ Attrs attrs;
+ ret = target->read_attrs(dpp, y, attrs);
+ if (ret < 0)
+ return ret;
+
+ POSIXOwner o;
+ ret = decode_owner(attrs, o);
+ if (ret < 0) {
+ bde.meta.owner = "unknown";
+ bde.meta.owner_display_name = "unknown";
+ } else {
+ bde.meta.owner = o.user.to_str();
+ bde.meta.owner_display_name = o.display_name;
+ }
+ bde.meta.category = RGWObjCategory::Main;
+ bde.meta.size = stx.stx_size;
+ bde.meta.accounted_size = stx.stx_size;
+ bde.meta.mtime = from_statx_timestamp(stx.stx_mtime);
+ bde.meta.storage_class = RGW_STORAGE_CLASS_STANDARD;
+ bde.meta.appendable = true;
+ bufferlist etag_bl;
+ if (rgw::sal::get_attr(attrs, RGW_ATTR_ETAG, etag_bl)) {
+ bde.meta.etag = etag_bl.to_str();
+ }
+
+ return cb(dpp, bde);
+}
+
+int Symlink::read_attrs(const DoutPrefixProvider* dpp, optional_yield y, Attrs& attrs)
+{
+ if (target)
+ return target->read_attrs(dpp, y, attrs);
+
+ return FSEnt::read_attrs(dpp, y, attrs);
+}
+
+int Symlink::copy(const DoutPrefixProvider *dpp, optional_yield y,
+ Directory* dst_dir, const std::string& dst_name)
+{
+ int ret = stat(dpp);
+ if (ret < 0) {
+ ldpp_dout(dpp, 0) << "ERROR: could not stat source file " << get_name()
+ << dendl;
+ return ret;
+ }
+ rgw_obj_key skey = decode_obj_key(target->get_name());
+ rgw_obj_key dkey = decode_obj_key(dst_name);
+ dkey.instance = skey.instance;
+ std::string tgtname = get_key_fname(dkey, /*use_version=*/true);
+
+ ret = symlinkat(tgtname.c_str(), dst_dir->get_fd(), dst_name.c_str());
+
+ return 0;
+}
+
+int MPDirectory::create(const DoutPrefixProvider* dpp, bool* existed, bool temp_file)
+{
+ std::string path;
+
+ if(temp_file) {
+ tmpname = path = "._tmpname_" +
+ std::to_string(ceph::util::generate_random_number<uint64_t>());
+ } else {
+ path = get_name();
+ }
+
+ int ret = mkdirat(parent->get_fd(), path.c_str(), S_IRWXU);
+ if (ret < 0) {
+ ret = errno;
+ if (ret != EEXIST) {
+ if (dpp)
+ ldpp_dout(dpp, 0) << "ERROR: could not create bucket " << get_name() << ": "
+ << cpp_strerror(ret) << dendl;
+ return -ret;
+ } else if (existed != nullptr) {
+ *existed = true;
+ }
+ }
+
+ return 0;
+}
+
+int MPDirectory::read(int64_t ofs, int64_t left, bufferlist &bl,
+ const DoutPrefixProvider *dpp, optional_yield y)
+{
+ std::string pname;
+ for (auto part : parts) {
+ if (ofs < part.second) {
+ pname = part.first;
+ break;
+ }
+
+ ofs -= part.second;
+ }
+
+ if (pname.empty()) {
+ // ofs is past the end
+ return 0;
+ }
+
+ if (!cur_read_part || cur_read_part->get_name() != pname) {
+ cur_read_part = std::make_unique<File>(pname, this, ctx);
+ }
+ int ret = cur_read_part->open(dpp);
+ if (ret < 0) {
+ return ret;
+ }
+
+ return cur_read_part->read(ofs, left, bl, dpp, y);
+}
+
+int MPDirectory::link_temp_file(const DoutPrefixProvider *dpp, optional_yield y,
+ std::string temp_fname)
+{
+ if (tmpname.empty()) {
+ return 0;
+ }
+
+ /* Temporarily change name to tmpname, so we can reuse rename() */
+ std::string savename = fname;
+ fname = tmpname;
+ tmpname.clear();
+
+ return rename(dpp, y, parent, savename);
+}
+
+int MPDirectory::remove(const DoutPrefixProvider* dpp, optional_yield y, bool delete_children)
+{
+ return Directory::remove(dpp, y, /*delete_children=*/true);
+}
+
+int MPDirectory::stat(const DoutPrefixProvider* dpp, bool force)
+{
+ int ret = Directory::stat(dpp, force);
+ if (ret < 0) {
+ return ret;
+ }
+
+ uint64_t total_size{0};
+ for_each(dpp, [this, &total_size, &dpp](const char *name) {
+ int ret;
+ struct statx stx;
+ std::string sname = name;
+
+ if (sname.rfind(MP_OBJ_PART_PFX, 0) != 0) {
+ /* Skip non-parts */
+ return 0;
+ }
+
+ ret = statx(fd, name, AT_SYMLINK_NOFOLLOW, STATX_ALL, &stx);
+ if (ret < 0) {
+ ret = errno;
+ ldpp_dout(dpp, 0) << "ERROR: could not stat object " << name << ": "
+ << cpp_strerror(ret) << dendl;
+ return -ret;
+ }
+
+ if (!S_ISREG(stx.stx_mode)) {
+ /* Skip non-files */
+ return 0;
+ }
+
+ parts[name] = stx.stx_size;
+ total_size += stx.stx_size;
+ return 0;
+ });
+
+ stx.stx_size = total_size;
+
+ return 0;
+}
+
+
+std::unique_ptr<File> MPDirectory::get_part_file(int partnum)
+{
+ std::string partname = MP_OBJ_PART_PFX + fmt::format("{:0>5}", partnum);
+ rgw_obj_key part_key(partname);
+
+ return std::make_unique<File>(partname, this, ctx);
+}
+
+int MPDirectory::fill_cache(const DoutPrefixProvider *dpp, optional_yield y,
+ fill_cache_cb_t &cb)
+{
+ int ret = FSEnt::fill_cache(dpp, y, cb);
+ if (ret < 0)
+ return ret;
+
+ return Directory::fill_cache(dpp, y, cb);
+}
+
+int VersionedDirectory::open(const DoutPrefixProvider* dpp)
+{
+ if (fd > 0) {
+ return 0;
+ }
+ int ret = Directory::open(dpp);
+ if (ret < 0) {
+ return 0;
+ }
+
+ if (!instance_id.empty()) {
+ rgw_obj_key key = decode_obj_key(get_name());
+ key.instance = instance_id;
+ get_ent(dpp, null_yield, get_key_fname(key, /*use_version=*/true), std::string(), cur_version);
+ }
+
+ if (!cur_version) {
+ /* Can't open File, probably doesn't exist yet */
+ return 0;
+ }
+
+ return cur_version->open(dpp);
+}
+
+int VersionedDirectory::create(const DoutPrefixProvider* dpp, bool* existed, bool temp_file)
+{
+ int ret = mkdirat(parent->get_fd(), fname.c_str(), S_IRWXU);
+ if (ret < 0) {
+ ret = errno;
+ if (ret != EEXIST) {
+ if (dpp)
+ ldpp_dout(dpp, 0) << "ERROR: could not create versioned directory " << get_name() << ": "
+ << cpp_strerror(ret) << dendl;
+ return -ret;
+ }
+ }
+
+ ret = open(dpp);
+ if (ret < 0) {
+ ldpp_dout(dpp, 0) << "ERROR: could not open versioned directory " << get_name()
+ << dendl;
+ return ret;
+ }
+
+ /* Need type attribute written */
+ Attrs attrs;
+ ret = write_attrs(dpp, null_yield, attrs, nullptr);
+ if (ret < 0) {
+ ldpp_dout(dpp, 0) << "ERROR: could not write attrs for versioned directory " << get_name()
+ << dendl;
+ return ret;
+ }
+
+ if (temp_file) {
+ /* Want to create an actual versioned object */
+ rgw_obj_key key = decode_obj_key(get_name());
+ key.instance = instance_id;
+ std::unique_ptr<FSEnt> file =
+ std::make_unique<File>(get_key_fname(key, /*use_version=*/true), this, ctx);
+ ret = add_file(dpp, std::move(file), existed, temp_file);
+ if (ret < 0) {
+ return ret;
+ }
+ }
+
+ return 0;
+}
+
+std::string VersionedDirectory::get_new_instance()
+{
+ return gen_rand_instance_name();
+}
+
+int VersionedDirectory::add_file(const DoutPrefixProvider* dpp, std::unique_ptr<FSEnt>&& file, bool* existed, bool temp_file)
+{
+ int ret = open(dpp);
+ if (ret < 0) {
+ ldpp_dout(dpp, 0) << "ERROR: could not open versioned directory " << get_name()
+ << dendl;
+ return ret;
+ }
+
+ ret = file->create(dpp, existed, temp_file);
+ if (ret < 0) {
+ return ret;
+ }
+
+ if (!temp_file) {
+ return set_cur_version_ent(dpp, file.get());
+ }
+
+ cur_version = std::move(file);
+ return 0;
+}
+
+int VersionedDirectory::set_cur_version_ent(const DoutPrefixProvider* dpp, FSEnt* file)
+{
+ /* Delete current version symlink */
+ std::unique_ptr<FSEnt> del;
+ int ret = get_ent(dpp, null_yield, get_name(), std::string(), del);
+ if (ret >= 0) {
+ ret = del->remove(dpp, null_yield, /*delete_children=*/true);
+ if (ret < 0) {
+ ldpp_dout(dpp, 0) << "ERROR: could not remove cur_version " << get_name()
+ << dendl;
+ return ret;
+ }
+ }
+
+ /* Create new current version symlink */
+ std::unique_ptr<Symlink> sl =
+ std::make_unique<Symlink>(get_name(), this, file->get_name(), ctx);
+ ret = sl->create(dpp, /*existed=*/nullptr, /*temp_file=*/false);
+ if (ret < 0) {
+ ldpp_dout(dpp, 0) << "ERROR: could not create cur_version symlink "
+ << get_name() << dendl;
+ return ret;
+ }
+
+ return 0;
+}
+
+int VersionedDirectory::stat(const DoutPrefixProvider* dpp, bool force)
+{
+ int ret = Directory::stat(dpp, force);
+ if (ret < 0) {
+ return ret;
+ }
+
+ ret = open(dpp);
+ if (ret < 0)
+ return ret;
+
+ if (cur_version) {
+ /* Already have a File for the current version, use it */
+ ret = cur_version->stat(dpp);
+ if (ret < 0)
+ return ret;
+ stx.stx_size = cur_version->get_stx().stx_size;
+
+ return 0;
+ }
+
+ /* Try to read the symlink */
+ std::unique_ptr<Symlink> sl = std::make_unique<Symlink>(get_name(), this, ctx);
+ ret = sl->stat(dpp);
+ if (ret < 0) {
+ if (ret == -ENOENT)
+ return 0;
+ return ret;
+ }
+
+ if (!sl->exists()) {
+ stx.stx_size = 0;
+ return 0;
+ }
+
+ cur_version = sl->get_target()->clone_base();
+ ret = cur_version->open(dpp);
+ if (ret < 0) {
+ /* If target doesn't exist, it's a delete marker */
+ cur_version.reset();
+ stx.stx_size = 0;
+ return 0;
+ }
+ ret = cur_version->stat(dpp);
+ if (ret < 0)
+ return ret;
+ stx.stx_size = cur_version->get_stx().stx_size;
+
+ return 0;
+}
+
+int VersionedDirectory::read_attrs(const DoutPrefixProvider* dpp, optional_yield y, Attrs& attrs)
+{
+ if (!cur_version)
+ return FSEnt::read_attrs(dpp, y, attrs);
+
+ int ret = cur_version->read_attrs(dpp, y, attrs);
+ if (ret < 0) {
+ return ret;
+ }
+
+ /* Override type, it should be VERSIONED */
+ bufferlist type_bl;
+ ObjectType type{get_type()};
+ type.encode(type_bl);
+ attrs[RGW_POSIX_ATTR_OBJECT_TYPE] = type_bl;
+
+ return 0;
+}
+
+int VersionedDirectory::write_attrs(const DoutPrefixProvider* dpp, optional_yield y, Attrs& attrs, Attrs* extra_attrs)
+{
+ if (cur_version) {
+ int ret = cur_version->write_attrs(dpp, y, attrs, extra_attrs);
+ if (ret < 0)
+ return ret;
+ }
+
+ return FSEnt::write_attrs(dpp, y, attrs, extra_attrs);
+}
+
+int VersionedDirectory::write(int64_t ofs, bufferlist &bl,
+ const DoutPrefixProvider *dpp, optional_yield y)
+{
+ if (!cur_version)
+ return 0;
+ return cur_version->write(ofs, bl, dpp, y);
+}
+
+int VersionedDirectory::read(int64_t ofs, int64_t left, bufferlist &bl,
+ const DoutPrefixProvider *dpp, optional_yield y)
+{
+ if (!cur_version)
+ return 0;
+ return cur_version->read(ofs, left, bl, dpp, y);
+}
+
+int VersionedDirectory::link_temp_file(const DoutPrefixProvider *dpp, optional_yield y,
+ std::string temp_fname)
+{
+ if (!cur_version)
+ return -EINVAL;
+ int ret = cur_version->link_temp_file(dpp, y, temp_fname);
+ if (ret < 0)
+ return ret;
+
+ return set_cur_version_ent(dpp, cur_version.get());
+}
+
+int VersionedDirectory::copy(const DoutPrefixProvider *dpp, optional_yield y,
+ Directory* dst_dir, const std::string& dst_name)
+{
+ int ret;
+ rgw_obj_key dest_key = decode_obj_key(dst_name);
+ std::string basename = get_key_fname(dest_key, /*use_version=*/false);
+
+ // Delete the target
+ {
+ std::unique_ptr<FSEnt> del;
+ ret = dst_dir->get_ent(dpp, y, basename, std::string(), del);
+ if (ret >= 0) {
+ ret = del->remove(dpp, y, /*delete_children=*/true);
+ if (ret < 0) {
+ ldpp_dout(dpp, 0) << "ERROR: could not remove dest " << basename
+ << dendl;
+ return ret;
+ }
+ }
+ }
+
+ ret = dst_dir->open(dpp);
+ std::unique_ptr<VersionedDirectory> dest = clone();
+ dest->parent = dst_dir;
+ dest->fname = basename;
+
+ ret = dest->create(dpp);
+ if (ret < 0) {
+ ldpp_dout(dpp, 0) << "ERROR: could not create dest " << dest->get_name() << dendl;
+ return ret;
+ }
+
+ Attrs attrs;
+ ret = read_attrs(dpp, y, attrs);
+ if (ret < 0) {
+ ldpp_dout(dpp, 0) << "ERROR: could not read attrs from " << get_name() << dendl;
+ return ret;
+ }
+ ret = dest->write_attrs(dpp, y, attrs, nullptr);
+ if (ret < 0) {
+ ldpp_dout(dpp, 0) << "ERROR: could not write attrs to " << dest->get_name() << dendl;
+ return ret;
+ }
+
+ std::string tgtname;
+ ret = for_each(dpp, [this, &dest, &dest_key, &tgtname, &dpp, &y](const char* name) {
+ std::unique_ptr<FSEnt> sobj;
-static inline void bucket_statx_save(struct statx& stx, RGWBucketEnt& ent, ceph::real_time& mtime)
-{
- mtime = ceph::real_clock::from_time_t(stx.stx_mtime.tv_sec);
- ent.creation_time = ceph::real_clock::from_time_t(stx.stx_btime.tv_sec);
- // TODO Calculate size of bucket (or save it somewhere?)
- //ent.size = stx.stx_size;
- //ent.size_rounded = stx.stx_blocks * 512;
-}
+ if (name[0] == '.') {
+ /* Skip dotfiles */
+ return 0;
+ }
+ rgw_obj_key key = decode_obj_key(name);
+ if (!dest_key.instance.empty() && dest_key.instance != key.instance) {
+ /* Were asked to copy a single version, and this is not it */
+ return 0;
+ }
-static inline int copy_dir_fd(int old_fd)
-{
- return openat(old_fd, ".", O_RDONLY | O_DIRECTORY | O_NOFOLLOW);
+ int r = this->get_ent(dpp, y, name, std::string(), sobj);
+ if (r < 0)
+ return r;
+ key.name = dest_key.name;
+ tgtname = get_key_fname(key, /*use_version=*/true);
+ return sobj->copy(dpp, y, dest.get(), tgtname);
+ });
+
+ if (!dest_key.instance.empty()) {
+ /* We didn't copy the symlink, make a new one */
+ std::unique_ptr<Symlink> sl = std::make_unique<Symlink>(basename, dest.get(), tgtname, ctx);
+ ret = sl->create(dpp, /*existed=*/nullptr, /*temp_file=*/false);
+ }
+
+ return ret;
}
-static int get_x_attrs(optional_yield y, const DoutPrefixProvider* dpp, int fd,
- Attrs& attrs, const std::string& display)
+int VersionedDirectory::remove(const DoutPrefixProvider* dpp, optional_yield y, bool delete_children)
{
- char namebuf[64 * 1024]; // Max list size supported on linux
- ssize_t buflen;
- int ret;
-
- buflen = flistxattr(fd, namebuf, sizeof(namebuf));
- if (buflen < 0) {
- ret = errno;
- ldpp_dout(dpp, 0) << "ERROR: could not list attributes for " << display << ": "
- << cpp_strerror(ret) << dendl;
- return -ret;
- }
+ std::string tgtname;
+ bool newlink = false;
- char *keyptr = namebuf;
- while (buflen > 0) {
- std::string value;
- ssize_t vallen, keylen;
- char* vp;
+ int ret = open(dpp);
+ if (ret < 0)
+ return ret;
- keylen = strlen(keyptr) + 1;
- std::string key(keyptr);
- std::string::size_type prefixloc = key.find(ATTR_PREFIX);
+ if (instance_id.empty()) {
+ /* Check if directory is empty */
+ ret = for_each(dpp, [](const char *n) {
+ return -ENOENT;
+ });
- if (prefixloc == std::string::npos) {
- /* Not one of our attributes */
- buflen -= keylen;
- keyptr += keylen;
- continue;
+ if (ret == 0) {
+ /* We're empty, nuke us */
+ return Directory::remove(dpp, y, /*delete_children=*/true);
}
- /* Make a key that has just the attribute name */
- key.erase(prefixloc, ATTR_PREFIX.length());
-
- vallen = fgetxattr(fd, keyptr, nullptr, 0);
- if (vallen < 0) {
- ret = errno;
- ldpp_dout(dpp, 0) << "ERROR: could not get attribute " << keyptr << " for " << display << ": " << cpp_strerror(ret) << dendl;
- return -ret;
- } else if (vallen == 0) {
- /* No attribute value for this name */
- buflen -= keylen;
- keyptr += keylen;
- continue;
+ /* Add a delete marker */
+ rgw_obj_key key = decode_obj_key(get_name());
+ key.instance = gen_rand_instance_name();
+ tgtname = get_key_fname(key, /*use_version=*/true);
+ newlink = true;
+ ret = remove_symlink(dpp, y);
+ if (ret < 0) {
+ return ret;
}
+ } else {
+ /* Delete specific version */
+ rgw_obj_key key = decode_obj_key(get_name());
+ key.instance = instance_id;
+ std::string name = get_key_fname(key, /*use_version=*/true);
- value.reserve(vallen + 1);
- vp = &value[0];
+ std::unique_ptr<FSEnt> f;
+ ret = get_ent(dpp, y, name, std::string(), f);
+ if (ret == 0) {
+ ret = f->stat(dpp);
+ if (ret < 0)
+ return ret;
+ ret = f->remove(dpp, y, /*delete_children=*/true);
+ if (ret < 0)
+ return ret;
+ } else if (ret == -ENOENT) {
+ /* See if we're removing a delete marker */
+ std::unique_ptr<Symlink> sl =
+ std::make_unique<Symlink>(get_name(), this, ctx);
+ ret = sl->stat(dpp);
+ if (ret == 0) {
+ if (name != sl->get_target()->get_name()) {
+ /* Symlink didn't match, don't change anything */
+ return 0;
+ }
+ }
+ /* FALLTHROUGH */
+ } else {
+ return ret;
+ }
- vallen = fgetxattr(fd, keyptr, vp, vallen);
- if (vallen < 0) {
- ret = errno;
- ldpp_dout(dpp, 0) << "ERROR: could not get attribute " << keyptr << " for " << display << ": " << cpp_strerror(ret) << dendl;
- return -ret;
+ /* Possibly move symlink */
+ ret = remove_symlink(dpp, y, name);
+ if (ret < 0) {
+ if (ret == -ENOKEY) {
+ return 0;
+ }
+ return ret;
}
+ newlink = true;
+ /* Create new current version symlink */
+ ret = for_each(dpp, [&tgtname](const char *n) {
+ if (n[0] == '.') {
+ /* Skip dotfiles */
+ return 0;
+ }
- bufferlist bl;
- bl.append(vp, vallen);
- attrs.emplace(std::move(key), std::move(bl)); /* key and bl are r-value refs */
+ tgtname = n;
+ return 0;
+ });
- buflen -= keylen;
- keyptr += keylen;
+ if (tgtname.empty()) {
+ /* We're empty, nuke us */
+ exist = false;
+ return Directory::remove(dpp, y, /*delete_children=*/true);
+ }
}
+ if (newlink) {
+ exist = true;
+ std::unique_ptr<Symlink> sl =
+ std::make_unique<Symlink>(get_name(), this, tgtname, ctx);
+ return sl->create(dpp, /*existed=*/nullptr, /*temp_file=*/false);
+ }
return 0;
}
-static int write_x_attr(const DoutPrefixProvider* dpp, optional_yield y, int fd,
- const std::string& key, bufferlist& value,
- const std::string& display)
+int VersionedDirectory::fill_cache(const DoutPrefixProvider *dpp, optional_yield y,
+ fill_cache_cb_t &cb)
{
- int ret;
- std::string attrname;
+ int ret = for_each(dpp, [this, &cb, &dpp, &y](const char *name) {
+ std::unique_ptr<FSEnt> ent;
- attrname = ATTR_PREFIX + key;
+ if (name[0] == '.') {
+ /* Skip dotfiles */
+ return 0;
+ }
+
+ int ret = get_ent(dpp, y, name, std::string(), ent);
+ if (ret < 0)
+ return ret;
+
+ ent->stat(dpp); // Stat the object to get the type
+
+ ret = ent->fill_cache(dpp, y, cb);
+ if (ret < 0)
+ return ret;
+ return 0;
+ });
- ret = fsetxattr(fd, attrname.c_str(), value.c_str(), value.length(), 0);
if (ret < 0) {
- ret = errno;
- ldpp_dout(dpp, 0) << "ERROR: could not write attribute " << attrname << " for " << display << ": " << cpp_strerror(ret) << dendl;
- return -ret;
+ ldpp_dout(dpp, 0) << "ERROR: could not list directory " << get_name() << ": "
+ << cpp_strerror(ret) << dendl;
+ return ret;
}
return 0;
}
-static int delete_directory(int parent_fd, const char* dname, bool delete_children,
- const DoutPrefixProvider* dpp)
+std::string VersionedDirectory::get_cur_version()
{
- int ret;
- int dir_fd = -1;
- DIR *dir;
- struct dirent *entry;
-
- dir_fd = openat(parent_fd, dname, O_RDONLY | O_DIRECTORY | O_NOFOLLOW);
- if (dir_fd < 0) {
- dir_fd = errno;
- ldpp_dout(dpp, 0) << "ERROR: could not open subdir " << dname << ": "
- << cpp_strerror(dir_fd) << dendl;
- return -dir_fd;
- }
-
- dir = fdopendir(dir_fd);
- if (dir == NULL) {
- ret = errno;
- ldpp_dout(dpp, 0) << "ERROR: could not open bucket " << dname
- << " for listing: " << cpp_strerror(ret) << dendl;
- return -ret;
- }
-
- errno = 0;
- while ((entry = readdir(dir)) != NULL) {
- struct statx stx;
-
- if ((entry->d_name[0] == '.' && entry->d_name[1] == '\0') ||
- (entry->d_name[0] == '.' && entry->d_name[1] == '.' &&
- entry->d_name[2] == '\0')) {
- /* Skip . and .. */
- errno = 0;
- continue;
- }
+ if (!cur_version)
+ return "";
- std::string_view d_name = entry->d_name;
- bool is_mp = d_name.starts_with("." + mp_ns);
- if (!is_mp && !delete_children) {
- return -ENOTEMPTY;
- }
+ rgw_obj_key key = decode_obj_key(cur_version->get_name());
- ret = statx(dir_fd, entry->d_name, AT_SYMLINK_NOFOLLOW, STATX_ALL, &stx);
- if (ret < 0) {
- ret = errno;
- ldpp_dout(dpp, 0) << "ERROR: could not stat object " << entry->d_name
- << ": " << cpp_strerror(ret) << dendl;
- return -ret;
- }
+ return key.instance;
+}
- if (S_ISDIR(stx.stx_mode)) {
- /* Recurse */
- ret = delete_directory(dir_fd, entry->d_name, true, dpp);
- if (ret < 0) {
- return ret;
- }
+int VersionedDirectory::remove_symlink(const DoutPrefixProvider *dpp, optional_yield y, std::string match)
+{
+ int ret;
- continue;
- }
+ std::unique_ptr<Symlink> sl =
+ std::make_unique<Symlink>(get_name(), this, ctx);
+ ret = sl->stat(dpp);
+ if (ret < 0) {
+ /* Doesn't exist, nothing to do */
+ if (ret == -ENOENT)
+ return 0;
+ return ret;
+ }
- /* Otherwise, unlink */
- ret = unlinkat(dir_fd, entry->d_name, 0);
- if (ret < 0) {
- ret = errno;
- ldpp_dout(dpp, 0) << "ERROR: could not remove file " << entry->d_name
- << ": " << cpp_strerror(ret) << dendl;
- return -ret;
- }
+ if (!match.empty()) {
+ if (match != sl->get_target()->get_name())
+ return -ENOKEY;
}
- ret = unlinkat(parent_fd, dname, AT_REMOVEDIR);
+ ret = sl->remove(dpp, y, /*delete_children=*/false);
if (ret < 0) {
- ret = errno;
- if (errno != ENOENT) {
- ldpp_dout(dpp, 0) << "ERROR: could not remove bucket " << dname << ": "
- << cpp_strerror(ret) << dendl;
- return -ret;
- }
+ return ret;
}
return 0;
g_conf().get_val<int64_t>("rgw_posix_cache_partitions"),
g_conf().get_val<int64_t>("rgw_posix_cache_lmdb_count")));
- root_fd = openat(-1, base_path.c_str(), O_RDONLY | O_DIRECTORY | O_NOFOLLOW);
- if (root_fd == -1) {
- int err = errno;
- if (err == ENOTDIR) {
+ root_dir = std::make_unique<Directory>(base_path, nullptr, ctx());
+ int ret = root_dir->open(dpp);
+ if (ret < 0) {
+ if (ret == -ENOTDIR) {
ldpp_dout(dpp, 0) << " ERROR: base path (" << base_path
<< "): was not a directory." << dendl;
- return -err;
- } else if (err == ENOENT) {
- err = mkdir(base_path.c_str(), S_IRWXU);
- if (err < 0) {
- err = errno;
+ return ret;
+ } else if (ret == -ENOENT) {
+ ret = root_dir->create(dpp);
+ if (ret < 0) {
ldpp_dout(dpp, 0) << " ERROR: could not create base path ("
- << base_path << "): " << cpp_strerror(err) << dendl;
- return -err;
+ << base_path << "): " << cpp_strerror(-ret) << dendl;
+ return ret;
}
- root_fd = ::open(base_path.c_str(), O_RDONLY | O_DIRECTORY | O_NOFOLLOW);
}
}
- if (root_fd == -1) {
- int err = errno;
- ldpp_dout(dpp, 0) << " ERROR: could not open base path ("
- << base_path << "): " << cpp_strerror(err) << dendl;
- return -err;
- }
+ ldpp_dout(dpp, 20) << "root_fd: " << root_dir->get_fd() << dendl;
ldpp_dout(dpp, 20) << "SUCCESS" << dendl;
return 0;
int POSIXDriver::load_bucket(const DoutPrefixProvider* dpp, const rgw_bucket& b, std::unique_ptr<Bucket>* bucket, optional_yield y)
{
- *bucket = std::make_unique<POSIXBucket>(this, root_fd, b);
+ *bucket = std::make_unique<POSIXBucket>(this, root_dir.get(), b);
return (*bucket)->load_bucket(dpp, y);
}
std::unique_ptr<Bucket> POSIXDriver::get_bucket(const RGWBucketInfo& i)
{
/* Don't need to fetch the bucket info, use the provided one */
- return std::make_unique<POSIXBucket>(this, root_fd, i);
+ return std::make_unique<POSIXBucket>(this, root_dir.get(), i);
}
std::string POSIXDriver::zone_unique_trans_id(const uint64_t unique_num)
_user_id, _user_tenant, _req_id, y);
}
-int POSIXDriver::close()
-{
- if (root_fd < 0) {
- return 0;
- }
-
- ::close(root_fd);
- root_fd = -1;
-
- return 0;
-}
-
// TODO: marker and other params
int POSIXDriver::list_buckets(const DoutPrefixProvider* dpp, const rgw_owner& owner,
const std::string& tenant, const std::string& marker,
info.quota = *params.quota;
}
- int ret = set_attrs(attrs);
+ int ret = set_attrs(params.attrs);
if (ret < 0) {
return ret;
}
return std::make_unique<POSIXObject>(driver, k, this);
}
-int POSIXObject::fill_bde(const DoutPrefixProvider *dpp, optional_yield y, rgw_bucket_dir_entry& bde)
+int POSIXObject::fill_cache(const DoutPrefixProvider *dpp, optional_yield y, fill_cache_cb_t& cb)
{
- std::unique_ptr<User> owner;
- (void)get_owner(dpp, y, &owner);
-
- get_key().get_index_key(&bde.key);
- bde.ver.pool = 1;
- bde.ver.epoch = 1;
- bde.exists = true;
- bde.meta.category = RGWObjCategory::Main;
- bde.meta.size = get_size();
- bde.meta.mtime = get_mtime();
- if (owner) {
- bde.meta.owner = owner->get_id().to_str();
- bde.meta.owner_display_name = owner->get_display_name();
- } else {
- bde.meta.owner = "unknown";
- bde.meta.owner_display_name = "unknown";
- }
- bde.meta.accounted_size = get_size();
- bde.meta.storage_class = RGW_STORAGE_CLASS_STANDARD;
- bde.meta.appendable = true;
- bufferlist etag_bl;
- if (get_attr(RGW_ATTR_ETAG, etag_bl)) {
- bde.meta.etag = etag_bl.to_str();
- }
-
- return 0;
+ return ent->fill_cache(dpp, y, cb);
}
int POSIXDriver::mint_listing_entry(const std::string &bname,
if (ret < 0)
return ret;
- obj = b->get_object(decode_obj_key(bde.key.name.c_str()));
+ obj = b->get_object(decode_obj_key(bde.key.name));
pobj = static_cast<POSIXObject *>(obj.get());
if (!pobj->check_exists(nullptr)) {
if (ret < 0)
return ret;
- ret = pobj->fill_bde(nullptr, null_yield, bde);
- if (ret < 0)
- return ret;
-
- return 0;
-}
-int POSIXBucket::fill_cache(const DoutPrefixProvider* dpp, optional_yield y,
- fill_cache_cb_t cb)
-{
- int ret = for_each(dpp, [this, &cb, &dpp, &y](const char* name) {
- int ret;
- std::unique_ptr<Object> obj;
- POSIXObject* pobj;
-
- if (name[0] == '.') {
- /* Skip dotfiles */
- return 0;
- }
-
- obj = get_object(decode_obj_key(name));
- pobj = static_cast<POSIXObject*>(obj.get());
-
- if (!pobj->check_exists(dpp)) {
- ret = errno;
- ldpp_dout(dpp, 0) << "ERROR: could not stat object " << name << ": "
- << cpp_strerror(ret) << dendl;
- return -ret;
- }
-
- ret = pobj->get_obj_attrs(y, dpp);
- if (ret < 0)
- return ret;
-
- rgw_bucket_dir_entry bde{};
- ret = pobj->fill_bde(dpp, y, bde);
- if (ret < 0)
- return ret;
-
- cb(dpp, bde);
-
- return 0;
- });
- if (ret < 0) {
- ldpp_dout(dpp, 0) << "ERROR: could not list bucket " << get_name() << ": "
- << cpp_strerror(ret) << dendl;
- return ret;
- }
-
- return 0;
+ ret = pobj->fill_cache(nullptr, null_yield,
+ [&bde](const DoutPrefixProvider *dpp, rgw_bucket_dir_entry &nbde) -> int {
+ bde = nbde;
+ return 0;
+ });
+
+ return ret;
+}
+int POSIXBucket::fill_cache(const DoutPrefixProvider* dpp, optional_yield y,
+ fill_cache_cb_t& cb)
+{
+ return dir->fill_cache(dpp, y, cb);
}
-// TODO marker and other params
int POSIXBucket::list(const DoutPrefixProvider* dpp, ListParams& params,
int max, ListResults& results, optional_yield y)
{
int count{0};
bool in_prefix{false};
// Names in the cache are in OID format
+ rgw_obj_key marker_key(params.marker);
+ params.marker = marker_key.get_oid();
{
- rgw_obj_key key(params.marker);
- params.marker = key.get_oid();
- key.set(params.prefix);
- params.prefix = key.get_oid();
- }
- // Names are url_encoded, so encode prefix and delimiter
- // Names seem to not be url_encoded in cache
- //params.prefix = url_encode(params.prefix);
- //params.delim = url_encode(params.delim);
+ rgw_obj_key key(params.prefix);
+ params.prefix = key.name;
+ }
if (max <= 0) {
return 0;
}
+ //params.list_versions
int ret = driver->get_bucket_cache()->list_bucket(
dpp, y, this, params.marker.name, [&](const rgw_bucket_dir_entry& bde) -> bool
{
std::string ns;
// bde.key can be encoded with the namespace. Decode it here
- if (!params.marker.empty() && params.marker == bde.key.name) {
+ rgw_obj_key bde_key{bde.key};
+ if (!params.list_versions && !bde.is_visible()) {
+ return true;
+ }
+ if (params.list_versions && versioned() && bde_key.instance.empty()) {
+ return true;
+ }
+ if (bde_key.ns != params.ns) {
+ // Namespace must match
+ return true;
+ }
+ if (!marker_key.empty() && marker_key == bde_key.name) {
// Skip marker
return true;
}
if (!params.prefix.empty()) {
// We have a prefix, only match
- if (!bde.key.name.starts_with(params.prefix)) {
+ if (!bde_key.name.starts_with(params.prefix)) {
// Prefix doesn't match; skip
if (in_prefix) {
return false;
}
return true;
}
- auto delim_pos = bde.key.name.find(params.delim, params.prefix.size());
+ auto delim_pos = bde_key.name.find(params.delim, params.prefix.size());
if (delim_pos == std::string_view::npos) {
// Straight prefix match
results.next_marker.set(bde.key);
}
return true;
}
- std::string prefix_key =
- bde.key.name.substr(0, delim_pos + params.delim.length());
- rgw_obj_key::parse_raw_oid(prefix_key, &results.next_marker);
- // Use results.next_marker.name for prefix_key, since it's been decoded
+ results.next_marker =
+ bde_key.name.substr(0, delim_pos + params.delim.length());
if (!results.common_prefixes.contains(results.next_marker.name)) {
results.common_prefixes[results.next_marker.name] = true;
count++; // Count will be checked when we exit prefix
}
if (!params.delim.empty()) {
// Delimiter, but no prefix
- auto delim_pos = bde.key.name.find(params.delim) ;
+ auto delim_pos = bde_key.name.find(params.delim) ;
if (delim_pos == std::string_view::npos) {
// Delimiter doesn't match, insert
results.next_marker.set(bde.key);
return true;
}
std::string prefix_key =
- bde.key.name.substr(0, delim_pos + params.delim.length());
- if (!params.marker.empty() && params.marker == prefix_key) {
+ bde_key.name.substr(0, delim_pos + params.delim.length());
+ if (!marker_key.empty() && marker_key == prefix_key) {
// Skip marker
return true;
}
bool delete_children,
optional_yield y)
{
- return delete_directory(parent_fd, get_fname().c_str(),
- delete_children, dpp);
+ int ret = dir->remove(dpp, y, delete_children);
+ if (ret < 0) {
+ return ret;
+ }
+
+ driver->get_bucket_cache()->invalidate_bucket(dpp, get_name());
+
+ return ret;
}
int POSIXBucket::remove_bypass_gc(int concurrent_max,
/* Skip dotfiles */
return -ERR_INVALID_OBJECT_NAME;
}
- ret = stat(dpp);
+ ret = dir->stat(dpp);
if (ret < 0) {
return ret;
}
- mtime = ceph::real_clock::from_time_t(stx.stx_mtime.tv_sec);
- info.creation_time = ceph::real_clock::from_time_t(stx.stx_btime.tv_sec);
+ mtime = ceph::real_clock::from_time_t(dir->get_stx().stx_mtime.tv_sec);
+ info.creation_time = ceph::real_clock::from_time_t(dir->get_stx().stx_btime.tv_sec);
- ret = open(dpp);
+ ret = dir->open(dpp);
if (ret < 0) {
return ret;
}
- get_x_attrs(y, dpp, dir_fd, attrs, get_name());
- auto iter = attrs.find(RGW_POSIX_ATTR_BUCKET_INFO);
- if (iter != attrs.end()) {
- // Proper bucket with saved info
- try {
- auto bufit = iter->second.cbegin();
- decode(info, bufit);
- } catch (buffer::error &err) {
- ldout(driver->ctx(), 0) << "ERROR: " << __func__ << ": failed to decode " RGW_POSIX_ATTR_BUCKET_INFO " attr" << dendl;
- return -EINVAL;
- }
- // info isn't stored in attrs
- attrs.erase(RGW_POSIX_ATTR_BUCKET_INFO);
- } else {
+ ret = dir->read_attrs(dpp, y, attrs);
+ if (ret < 0) {
+ return ret;
+ }
+
+ RGWBucketInfo bak_info = info;;
+ ret = decode_attr(attrs, RGW_POSIX_ATTR_BUCKET_INFO, info);
+ if (ret < 0) {
// TODO dang: fake info up (UID to owner conversion?)
+ info = bak_info;
+ } else {
+ // Don't leave info visible in attributes
+ attrs.erase(RGW_POSIX_ATTR_BUCKET_INFO);
}
return 0;
auto& main = stats[RGWObjCategory::Main];
// TODO: bucket stats shouldn't have to list all objects
- return for_each(dpp, [this, dpp, &main] (const char* name) {
+ return dir->for_each(dpp, [this, dpp, &main] (const char* name) {
if (name[0] == '.') {
/* Skip dotfiles */
return 0;
}
- struct statx lstx;
- int ret = statx(dir_fd, name, AT_SYMLINK_NOFOLLOW, STATX_ALL, &lstx);
+ std::unique_ptr<FSEnt> dent;
+ int ret = dir->get_ent(dpp, null_yield, name, std::string(), dent);
+ if (ret < 0) {
+ ret = errno;
+ ldpp_dout(dpp, 0) << "ERROR: could not get ent for object " << name << ": "
+ << cpp_strerror(ret) << dendl;
+ return -ret;
+ }
+
+ ret = dent->stat(dpp);
if (ret < 0) {
ret = errno;
ldpp_dout(dpp, 0) << "ERROR: could not stat object " << name << ": "
return -ret;
}
+ struct statx& lstx = dent->get_stx();
+
if (S_ISREG(lstx.stx_mode) || S_ISDIR(lstx.stx_mode)) {
main.num_objects++;
main.size += lstx.stx_size;
return 0;
});
+ return 0;
}
int POSIXBucket::read_stats_async(const DoutPrefixProvider *dpp,
struct timespec ts[2];
ts[0].tv_nsec = UTIME_OMIT;
ts[1] = ceph::real_clock::to_timespec(mtime);
- int ret = utimensat(parent_fd, get_fname().c_str(), ts, AT_SYMLINK_NOFOLLOW);
+ int ret = utimensat(dir->get_parent()->get_fd(), get_fname().c_str(), ts, AT_SYMLINK_NOFOLLOW);
if (ret < 0) {
ret = errno;
ldpp_dout(dpp, 0) << "ERROR: could not set mtime on bucket " << get_name() << ": "
int POSIXBucket::write_attrs(const DoutPrefixProvider* dpp, optional_yield y)
{
- int ret = open(dpp);
+ int ret = dir->open(dpp);
if (ret < 0) {
return ret;
}
- // Bucket info is stored as an attribute, but on in attrs[]
+ // Bucket info is stored as an attribute, but not in attrs[]
bufferlist bl;
encode(info, bl);
- ret = write_x_attr(dpp, y, dir_fd, RGW_POSIX_ATTR_BUCKET_INFO, bl, get_name());
- if (ret < 0) {
- return ret;
- }
+ Attrs extra_attrs;
+ extra_attrs[RGW_POSIX_ATTR_BUCKET_INFO] = bl;
- for (auto& it : attrs) {
- ret = write_x_attr(dpp, y, dir_fd, it.first, it.second, get_name());
- if (ret < 0) {
- return ret;
- }
- }
- return 0;
+ return dir->write_attrs(dpp, y, attrs, &extra_attrs);
}
int POSIXBucket::check_empty(const DoutPrefixProvider* dpp, optional_yield y)
{
- DIR* dir;
- struct dirent* entry;
- int ret;
-
- ret = open(dpp);
- if (ret < 0) {
- return ret;
- }
-
- dir = fdopendir(dir_fd);
- if (dir == NULL) {
- ret = errno;
- ldpp_dout(dpp, 0) << "ERROR: could not open bucket " << get_name() << " for listing: "
- << cpp_strerror(ret) << dendl;
- return -ret;
- }
-
- errno = 0;
- while ((entry = readdir(dir)) != NULL) {
- if (entry->d_name[0] != '.') {
- return -ENOTEMPTY;
- }
- if (entry->d_name[1] == '.' || entry->d_name[1] == '\0') {
- continue;
- }
- }
- return 0;
+ return dir->for_each(dpp, [](const char* name) {
+ /* for_each filters out "." and "..", so reaching here is not empty */
+ return -ENOTEMPTY;
+ });
}
int POSIXBucket::check_quota(const DoutPrefixProvider *dpp, RGWQuota& quota, uint64_t obj_size,
{
*pmtime = mtime;
- int ret = open(dpp);
+ int ret = dir->open(dpp);
if (ret < 0) {
return ret;
}
- get_x_attrs(y, dpp, dir_fd, attrs, get_name());
- return 0;
+ return dir->read_attrs(dpp, y, attrs);
}
int POSIXBucket::read_usage(const DoutPrefixProvider *dpp, uint64_t start_epoch,
std::map<std::string, bool> *common_prefixes,
bool *is_truncated, optional_yield y)
{
- //std::vector<std::unique_ptr<MultipartUpload>> nup;
- //int ret;
-//
- //ret = next->list_multiparts(dpp, prefix, marker, delim, max_uploads, nup,
- //common_prefixes, is_truncated);
- //if (ret < 0)
- //return ret;
-//
- //for (auto& ent : nup) {
- //uploads.emplace_back(std::make_unique<POSIXMultipartUpload>(std::move(ent), this, driver));
- //}
-
- return 0;
-}
-
-int POSIXBucket::abort_multiparts(const DoutPrefixProvider* dpp, CephContext* cct, optional_yield y)
-{
- return 0;
-}
-
-int POSIXBucket::create(const DoutPrefixProvider* dpp, optional_yield y, bool* existed)
-{
- int ret = mkdirat(parent_fd, get_fname().c_str(), S_IRWXU);
- if (ret < 0) {
- ret = errno;
- if (ret != EEXIST) {
- if (dpp)
- ldpp_dout(dpp, 0) << "ERROR: could not create bucket " << get_name() << ": "
- << cpp_strerror(ret) << dendl;
- return -ret;
- } else if (existed != nullptr) {
- *existed = true;
- }
- return -ret;
- }
-
- return write_attrs(dpp, y);
-}
-
-std::string POSIXBucket::get_fname()
-{
- std::string name;
-
- if (ns)
- name = "." + *ns + "_" + url_encode(get_name(), true);
- else
- name = url_encode(get_name(), true);
-
- return name;
-}
-
-int POSIXBucket::get_shadow_bucket(const DoutPrefixProvider* dpp, optional_yield y,
- const std::string& ns,
- const std::string& tenant, const std::string& name,
- bool create, std::unique_ptr<POSIXBucket>* shadow)
-{
- std::optional<std::string> ons{std::nullopt};
+ int count = 0;
int ret;
- POSIXBucket* bp;
- rgw_bucket b;
-
- b.tenant = tenant;
- b.name = name;
-
- if (!ns.empty()) {
- ons = ns;
- }
-
- open(dpp);
-
- bp = new POSIXBucket(driver, dir_fd, b, ons);
- ret = bp->load_bucket(dpp, y);
- if (ret == -ENOENT && create) {
- /* Create it if it doesn't exist */
- ret = bp->create(dpp, y, nullptr);
- }
- if (ret < 0) {
- delete bp;
- return ret;
- }
- shadow->reset(bp);
- return 0;
-}
+ ret = dir->for_each(dpp, [this, dpp, y, &count, &max_uploads, &is_truncated, &uploads] (const char* name) {
+ std::string_view d_name = name;
+ static std::string mp_pre{"." + mp_ns + "_"};
+ if (!d_name.starts_with(mp_pre)) {
+ /* Skip non-uploads */
+ return 0;
+ }
-template <typename F>
-int POSIXBucket::for_each(const DoutPrefixProvider* dpp, const F& func)
-{
- DIR* dir;
- struct dirent* entry;
- int ret;
+ if (count >= max_uploads) {
+ if (is_truncated) {
+ *is_truncated = true;
+ }
- ret = open(dpp);
- if (ret < 0) {
- return ret;
- }
+ return -EAGAIN;
+ }
- dir = fdopendir(dir_fd);
- if (dir == NULL) {
- ret = errno;
- ldpp_dout(dpp, 0) << "ERROR: could not open bucket " << get_name() << " for listing: "
- << cpp_strerror(ret) << dendl;
- return -ret;
- }
+ d_name.remove_prefix(mp_pre.size());
- rewinddir(dir);
+ ACLOwner owner;
+ std::unique_ptr<MultipartUpload> upload =
+ std::make_unique<POSIXMultipartUpload>(
+ driver, this, std::string(d_name), std::nullopt, owner,
+ real_clock::now());
+ rgw_placement_rule* rule{nullptr};
+ int ret = upload->get_info(dpp, y, &rule, nullptr);
+ if (ret < 0)
+ return 0;
+ uploads.emplace(uploads.end(), std::move(upload));
+ count++;
- while ((entry = readdir(dir)) != NULL) {
- int r = func(entry->d_name);
- if (r < 0) {
- ret = r;
- }
- }
+ return 0;
+ });
- if (ret == -EAGAIN) {
- /* Limit reached */
- ret = 0;
- }
return ret;
}
-int POSIXBucket::open(const DoutPrefixProvider* dpp)
+int POSIXBucket::abort_multiparts(const DoutPrefixProvider* dpp, CephContext* cct, optional_yield y)
{
- if (dir_fd >= 0) {
- return 0;
- }
-
- int ret = openat(parent_fd, get_fname().c_str(),
- O_RDONLY | O_DIRECTORY | O_NOFOLLOW);
- if (ret < 0) {
- ret = errno;
- ldpp_dout(dpp, 0) << "ERROR: could not open bucket " << get_name() << ": "
- << cpp_strerror(ret) << dendl;
- return -ret;
- }
-
- dir_fd = ret;
-
return 0;
}
-// This is for renaming a shadow bucket to a MP object. It won't work work for a normal bucket
-int POSIXBucket::rename(const DoutPrefixProvider* dpp, optional_yield y, Object* target_obj)
+int POSIXBucket::create(const DoutPrefixProvider* dpp, optional_yield y, bool* existed)
{
- POSIXObject *to = static_cast<POSIXObject*>(target_obj);
- POSIXBucket *tb = static_cast<POSIXBucket*>(target_obj->get_bucket());
- std::string src_fname = get_fname();
- std::string dst_fname = to->get_fname();
- int flags = 0;
-
- if (to->check_exists(dpp)) {
- flags = RENAME_EXCHANGE;
- }
- // swap
- int ret = renameat2(tb->get_dir_fd(dpp), src_fname.c_str(), tb->get_dir_fd(dpp), dst_fname.c_str(), flags);
- if(ret < 0) {
- ret = errno;
- ldpp_dout(dpp, 0) << "ERROR: renameat2 for shadow object could not finish: "
- << cpp_strerror(ret) << dendl;
- return -ret;
- }
-
- // Update saved bucket info
- info.bucket.name = to->get_name();
- bufferlist bl;
- encode(info, bl);
- ret = write_x_attr(dpp, y, dir_fd, RGW_POSIX_ATTR_BUCKET_INFO, bl, get_name());
+ int ret = dir->create(dpp, existed);
if (ret < 0) {
return ret;
}
- // Delete old one (could be file or directory)
- struct statx stx;
- ret = statx(parent_fd, src_fname.c_str(), AT_SYMLINK_NOFOLLOW,
- STATX_ALL, &stx);
- if (ret < 0) {
- ret = errno;
- if (ret == ENOENT) {
- return 0;
- }
- ldpp_dout(dpp, 0) << "ERROR: could not stat object " << get_name() << ": "
- << cpp_strerror(ret) << dendl;
- return -ret;
- }
-
- if (S_ISREG(stx.stx_mode)) {
- ret = unlinkat(parent_fd, src_fname.c_str(), 0);
- } else if (S_ISDIR(stx.stx_mode)) {
- ret = delete_directory(parent_fd, src_fname.c_str(), true, dpp);
- }
- if (ret < 0) {
- ret = errno;
- ldpp_dout(dpp, 0) << "ERROR: could not remove old file " << get_name()
- << ": " << cpp_strerror(ret) << dendl;
- return -ret;
- }
-
- return 0;
-}
-
-int POSIXBucket::close()
-{
- if (dir_fd < 0) {
- return 0;
- }
-
- ::close(dir_fd);
- dir_fd = -1;
-
- return 0;
+ return write_attrs(dpp, y);
}
-int POSIXBucket::stat(const DoutPrefixProvider* dpp)
+std::string POSIXBucket::get_fname()
{
- if (stat_done) {
- return 0;
- }
-
- int ret = statx(parent_fd, get_fname().c_str(), AT_SYMLINK_NOFOLLOW,
- STATX_ALL, &stx);
- if (ret < 0) {
- ret = errno;
- ldpp_dout(dpp, 0) << "ERROR: could not stat bucket " << get_name() << ": "
- << cpp_strerror(ret) << dendl;
- return -ret;
- }
- if (!S_ISDIR(stx.stx_mode)) {
- /* Not a bucket */
- return -EINVAL;
- }
-
- stat_done = true;
- return 0;
+ return bucket_fname(get_name(), ns);
}
-/* This is a shadow bucket. Copy it into a new shadow bucket in the destination
- * bucket */
-int POSIXBucket::copy(const DoutPrefixProvider *dpp, optional_yield y,
- POSIXBucket* db, POSIXObject* dest)
+int POSIXBucket::rename(const DoutPrefixProvider* dpp, optional_yield y, Object* target_obj)
{
- std::unique_ptr<POSIXBucket> dsb;
-
- // Delete the target, in case it's not a multipart
- int ret = dest->delete_object(dpp, y, rgw::sal::FLAG_LOG_OP);
- if (ret < 0) {
- ldpp_dout(dpp, 0) << "ERROR: could not remove dest object "
- << dest->get_name() << dendl;
- return ret;
- }
-
- ret = db->get_shadow_bucket(dpp, y, std::string(), std::string(), dest->get_fname(), true, &dsb);
- if (ret < 0) {
- ldpp_dout(dpp, 0) << "ERROR: could not create shadow bucket " << dest->get_name()
- << " in bucket " << db->get_name() << dendl;
- return ret;
- }
-
- ret = for_each(dpp, [this, &dsb, &dpp, &y](const char *name) {
- int ret;
- std::unique_ptr<Object> sobj;
- POSIXObject* sop;
- std::unique_ptr<Object> dobj;
- POSIXObject* dop;
-
- if (name[0] == '.') {
- /* Skip dotfiles */
- return 0;
- }
+ int ret;
+ Directory* dst_dir = dir->get_parent();
- sobj = this->get_object(decode_obj_key(name));
- sop = static_cast<POSIXObject*>(sobj.get());
- if (!sop->check_exists(dpp)) {
- ret = errno;
- ldpp_dout(dpp, 0) << "ERROR: could not stat object " << name << ": "
- << cpp_strerror(ret) << dendl;
- return -ret;
- }
- ret = sop->open(dpp, true);
+ info.bucket.name = target_obj->get_key().get_oid();
+ ns.reset();
+
+ if (!target_obj->get_instance().empty()) {
+ /* This is a versioned object. Need to handle versioneddirectory */
+ POSIXObject *to = static_cast<POSIXObject *>(target_obj);
+ ret = to->open(dpp, true, false);
if (ret < 0) {
- ldpp_dout(dpp, 0) << "ERROR: could not open source object " << get_name()
- << dendl;
+ ldpp_dout(dpp, 0) << "ERROR: could not open target obj " << to->get_name() << dendl;
return ret;
}
+ dst_dir = static_cast<Directory *>(to->get_fsent());
+ }
- dobj = dsb->get_object(decode_obj_key(name));
- dop = static_cast<POSIXObject*>(dobj.get());
-
- return sop->copy(dpp, y, this, dsb.get(), dop);
- });
-
- return ret;
+ return dir->rename(dpp, y, dst_dir, get_fname());
}
int POSIXObject::delete_object(const DoutPrefixProvider* dpp,
return ret;
}
- if (!b->versioned()) {
- if (shadow) {
- ret = shadow->remove(dpp, true, y);
- if (ret < 0) {
- return ret;
- }
- shadow.reset(nullptr);
- }
-
- int ret = unlinkat(b->get_dir_fd(dpp), get_fname().c_str(), 0);
- if (ret < 0) {
- ret = errno;
- if (errno != ENOENT) {
- ldpp_dout(dpp, 0) << "ERROR: could not remove object " << get_name()
- << ": " << cpp_strerror(ret) << dendl;
- return -ret;
- }
- }
- return 0;
- }
-
- // Versioned directory. Need to remove all objects matching
- b->for_each(dpp, [this, &dpp, &b](const char* name) {
- int ret;
- std::string_view vname(name);
+ ret = ent->remove(dpp, y, /*delete_children=*/false);
- if (vname.find(get_fname().c_str()) != std::string_view::npos) {
- ret = unlinkat(b->get_dir_fd(dpp), name, 0);
- if (ret < 0) {
- ret = errno;
- if (errno != ENOENT) {
- ldpp_dout(dpp, 0) << "ERROR: could not remove object " << name
- << ": " << cpp_strerror(ret) << dendl;
- return -ret;
- }
- }
- }
- return 0;
- });
+ cls_rgw_obj_key key;
+ get_key().get_index_key(&key);
+ driver->get_bucket_cache()->remove_entry(dpp, b->get_name(), key);
+ if (!key.instance.empty() && !ent->exists()) {
+ /* Remove the non-versiond key as well */
+ key.instance.clear();
+ driver->get_bucket_cache()->remove_entry(dpp, b->get_name(), key);
+ }
return 0;
}
<< dendl;
return -EINVAL;
}
+ bool has_instance = !get_key().instance.empty();
// Source must exist, and we need to know if it's a shadow obj
if (!check_exists(dpp)) {
return -ret;
}
- if (shadow) {
- return shadow->copy(dpp, y, db, dobj);
- } else {
- return copy(dpp, y, sb, db, dobj);
+ if (!get_key().instance.empty() && !has_instance) {
+ /* For copy, no instance meance copy all instances. Clear intance id if it
+ * was passed in clear. */
+ get_key().instance.clear();
+ }
+
+ if (state.obj != dobj->state.obj) {
+ /* An actual copy, copy the data */
+ ret = copy(dpp, y, sb, db, dobj);
+ if (ret < 0) {
+ ldpp_dout(dpp, 0) << "ERROR: failed to copy object " << get_key()
+ << dendl;
+ return ret;
+ }
+ }
+ dobj->make_ent(ent->get_type());
+
+ /* Set up attributes for destination */
+ Attrs src_attrs = state.attrset;
+ /* Come attrs are never copied */
+ src_attrs.erase(RGW_ATTR_DELETE_AT);
+ src_attrs.erase(RGW_ATTR_OBJECT_RETENTION);
+ src_attrs.erase(RGW_ATTR_OBJECT_LEGAL_HOLD);
+ /* Some attrs, if they exist, always come from the call */
+ src_attrs[RGW_ATTR_ACL] = attrs[RGW_ATTR_ACL];
+ bufferlist rt;
+ if (get_attr(RGW_ATTR_OBJECT_RETENTION, rt)) {
+ src_attrs[RGW_ATTR_OBJECT_RETENTION] = rt;
+ }
+ bufferlist lh;
+ if (get_attr(RGW_ATTR_OBJECT_LEGAL_HOLD, lh)) {
+ src_attrs[RGW_ATTR_OBJECT_LEGAL_HOLD] = lh;
+ }
+
+ bufferlist tt;
+ switch (attrs_mod) {
+ case ATTRSMOD_REPLACE:
+ /* Keep tags if not set */
+ if (!attrs[RGW_ATTR_ETAG].length()) {
+ attrs[RGW_ATTR_ETAG] = src_attrs[RGW_ATTR_ETAG];
+ }
+ if (!attrs[RGW_ATTR_TAIL_TAG].length() &&
+ rgw::sal::get_attr(src_attrs, RGW_ATTR_TAIL_TAG, tt)) {
+ attrs[RGW_ATTR_TAIL_TAG] = tt;
+ }
+ break;
+
+ case ATTRSMOD_MERGE:
+ for (auto it = src_attrs.begin(); it != src_attrs.end(); ++it) {
+ if (attrs.find(it->first) == attrs.end()) {
+ attrs[it->first] = it->second;
+ }
+ }
+ break;
+ case ATTRSMOD_NONE:
+ attrs = src_attrs;
+ ret = 0;
+ break;
+ }
+
+ /* Some attrs always come from the source */
+ bufferlist com;
+ if (rgw::sal::get_attr(src_attrs, RGW_ATTR_COMPRESSION, com)) {
+ attrs[RGW_ATTR_COMPRESSION] = com;
}
+ bufferlist mpu;
+ if (rgw::sal::get_attr(src_attrs, RGW_POSIX_ATTR_MPUPLOAD, mpu)) {
+ attrs[RGW_POSIX_ATTR_MPUPLOAD] = mpu;
+ }
+ bufferlist ownerbl;
+ if (rgw::sal::get_attr(src_attrs, RGW_POSIX_ATTR_OWNER, ownerbl)) {
+ attrs[RGW_POSIX_ATTR_OWNER] = ownerbl;
+ }
+ bufferlist pot;
+ if (rgw::sal::get_attr(src_attrs, RGW_POSIX_ATTR_OBJECT_TYPE, pot)) {
+ attrs[RGW_POSIX_ATTR_OBJECT_TYPE] = pot;
+ }
+ return dobj->set_obj_attrs(dpp, &attrs, nullptr, y, rgw::sal::FLAG_LOG_OP);
}
int POSIXObject::load_obj_state(const DoutPrefixProvider* dpp, optional_yield y, bool follow_olh)
{
if (delattrs) {
for (auto& it : *delattrs) {
+ if (it.first == RGW_POSIX_ATTR_OBJECT_TYPE) {
+ // Don't delete type
+ continue;
+ }
state.attrset.erase(it.first);
}
}
if (setattrs) {
for (auto& it : *setattrs) {
+ if (it.first == RGW_POSIX_ATTR_OBJECT_TYPE) {
+ // Don't overwrite type
+ continue;
+ }
state.attrset[it.first] = it.second;
}
}
- for (auto& it : state.attrset) {
- int ret = write_attr(dpp, y, it.first, it.second);
- if (ret < 0) {
- return ret;
- }
- }
+ write_attrs(dpp, y);
return 0;
}
int POSIXObject::get_obj_attrs(optional_yield y, const DoutPrefixProvider* dpp,
rgw_obj* target_obj)
{
+ //int fd;
+
int ret = open(dpp, false);
if (ret < 0) {
return ret;
}
- return get_x_attrs(y, dpp, obj_fd, state.attrset, get_name());
+ ret = ent->read_attrs(dpp, y, state.attrset);
+ if (ret == 0)
+ state.has_attrs = true;
+ else
+ state.has_attrs = false;
+
+ return ret;
}
int POSIXObject::modify_obj_attrs(const char* attr_name, bufferlist& attr_val,
optional_yield y, const DoutPrefixProvider* dpp)
{
state.attrset[attr_name] = attr_val;
- return write_attr(dpp, y, attr_name, attr_val);
+ return write_attrs(dpp, y);
}
int POSIXObject::delete_obj_attrs(const DoutPrefixProvider* dpp, const char* attr_name,
{
state.attrset.erase(attr_name);
- int ret = open(dpp, true);
+ int ret = open(dpp);
if (ret < 0) {
return ret;
}
- ret = fremovexattr(obj_fd, attr_name);
+ ret = remove_x_attr(dpp, y, ent->get_fd(), attr_name, get_name());
if (ret < 0) {
ret = errno;
ldpp_dout(dpp, 0) << "ERROR: could not remover attribute " << attr_name << " for " << get_name() << ": " << cpp_strerror(ret) << dendl;
bool POSIXObject::is_expired()
{
- bufferlist bl;
- if (get_attr(RGW_ATTR_DELETE_AT, bl)) {
- utime_t delete_at;
- try {
- auto bufit = bl.cbegin();
- decode(delete_at, bufit);
- } catch (buffer::error& err) {
- ldout(driver->ctx(), 0) << "ERROR: " << __func__ << ": failed to decode " RGW_ATTR_DELETE_AT " attr" << dendl;
- return false;
- }
+ utime_t delete_at;
+ if (!decode_attr(state.attrset, RGW_ATTR_DELETE_AT, delete_at)) {
+ ldout(driver->ctx(), 0)
+ << "ERROR: " << __func__
+ << ": failed to decode " RGW_ATTR_DELETE_AT " attr" << dendl;
+ return false;
+ }
- if (delete_at <= ceph_clock_now() && !delete_at.is_zero()) {
- return true;
- }
+ if (delete_at <= ceph_clock_now() && !delete_at.is_zero()) {
+ return true;
}
return false;
void POSIXObject::gen_rand_obj_instance_name()
{
- enum { OBJ_INSTANCE_LEN = 32 };
- char buf[OBJ_INSTANCE_LEN + 1];
-
- gen_rand_alphanumeric_no_underscore(driver->ctx(), buf, OBJ_INSTANCE_LEN);
- state.obj.key.set_instance(buf);
+ state.obj.key.set_instance(gen_rand_instance_name());
}
std::unique_ptr<MPSerializer> POSIXObject::get_serializer(const DoutPrefixProvider *dpp, const std::string& lock_name)
return -ENOENT;
}
- return 0;
+ POSIXBucket* b = static_cast<POSIXBucket*>(obj->get_bucket());
+ if (b->get_dir()->get_type() == ObjectType::MULTIPART && b->get_dir_fd(dpp) > 0) {
+ return 0;
+ }
+
+ return -ENOENT;
}
int POSIXObject::transition(Bucket* bucket,
int uid = 0;
int gid = 0;
- int ret = fchownat(b->get_dir_fd(dpp), get_fname().c_str(), uid, gid, AT_SYMLINK_NOFOLLOW);
+ int ret = fchownat(b->get_dir_fd(dpp), get_fname(/*use_version=*/true).c_str(), uid, gid, AT_SYMLINK_NOFOLLOW);
if (ret < 0) {
ret = errno;
ldpp_dout(dpp, 0) << "ERROR: could not remove object " << get_name() << ": "
return 0;
}
-int POSIXObject::stat(const DoutPrefixProvider* dpp)
-{
- if (stat_done) {
- return 0;
- }
-
- state.exists = false;
- POSIXBucket *b = static_cast<POSIXBucket*>(get_bucket());
- if (!b) {
- ldpp_dout(dpp, 0) << "ERROR: could not get bucket for " << get_name() << dendl;
- return -EINVAL;
- }
-
- int ret = statx(b->get_dir_fd(dpp), get_fname().c_str(), AT_SYMLINK_NOFOLLOW,
- STATX_ALL, &stx);
- if (ret < 0) {
- ret = errno;
- ldpp_dout(dpp, 0) << "ERROR: could not stat object " << get_name() << ": "
- << cpp_strerror(ret) << dendl;
- return -ret;
- }
- if (S_ISREG(stx.stx_mode)) {
- /* Normal object */
- state.accounted_size = state.size = stx.stx_size;
- state.mtime = from_statx_timestamp(stx.stx_mtime);
- } else if (S_ISDIR(stx.stx_mode)) {
- /* multipart object */
- /* Get the shadow bucket */
- POSIXBucket* pb = static_cast<POSIXBucket*>(bucket);
- ret = pb->get_shadow_bucket(dpp, null_yield, std::string(),
- std::string(), get_fname(), false, &shadow);
- if (ret < 0) {
- return ret;
- }
-
- state.mtime = from_statx_timestamp(stx.stx_mtime);
- /* Add up size of parts */
- uint64_t total_size{0};
- int fd = shadow->get_dir_fd(dpp);
- shadow->for_each(dpp, [this, &total_size, fd, &dpp](const char* name) {
- int ret;
- struct statx stx;
- std::string sname = name;
-
- if (sname.rfind(MP_OBJ_PART_PFX, 0) != 0) {
- /* Skip non-parts */
- return 0;
- }
-
- ret = statx(fd, name, AT_SYMLINK_NOFOLLOW, STATX_ALL, &stx);
- if (ret < 0) {
- ret = errno;
- ldpp_dout(dpp, 0) << "ERROR: could not stat object " << name << ": " << cpp_strerror(ret) << dendl;
- return -ret;
- }
-
- if (!S_ISREG(stx.stx_mode)) {
- /* Skip non-files */
- return 0;
- }
-
- parts[name] = stx.stx_size;
- total_size += stx.stx_size;
- return 0;
- });
- state.accounted_size = state.size = total_size;
- } else {
- /* Not an object */
- return -EINVAL;
- }
-
- stat_done = true;
- state.exists = true;
-
- return 0;
-}
-
-int POSIXObject::get_owner(const DoutPrefixProvider *dpp, optional_yield y, std::unique_ptr<User> *owner)
-{
- bufferlist bl;
- rgw_user u;
- if (!get_attr(RGW_POSIX_ATTR_OWNER, bl)) {
- ldpp_dout(dpp, 0) << "ERROR: " << __func__
- << ": No " RGW_POSIX_ATTR_OWNER " attr" << dendl;
- return -EINVAL;
- }
-
- try {
- auto bufit = bl.cbegin();
- decode(u, bufit);
- } catch (buffer::error &err) {
- ldpp_dout(dpp, 0) << "ERROR: " << __func__
- << ": failed to decode " RGW_POSIX_ATTR_OWNER " attr" << dendl;
- return -EINVAL;
- }
-
- *owner = driver->get_user(u);
- (*owner)->load_user(dpp, y);
- return 0;
-}
-
-std::unique_ptr<Object::ReadOp> POSIXObject::get_read_op()
-{
- return std::make_unique<POSIXReadOp>(this);
-}
-
-std::unique_ptr<Object::DeleteOp> POSIXObject::get_delete_op()
-{
- return std::make_unique<POSIXDeleteOp>(this);
-}
-
-int POSIXObject::open(const DoutPrefixProvider* dpp, bool create, bool temp_file)
-{
- if (obj_fd >= 0) {
- return 0;
- }
-
- stat(dpp);
-
- if (shadow) {
- obj_fd = shadow->get_dir_fd(dpp);
- return obj_fd;
- }
-
- POSIXBucket *b = static_cast<POSIXBucket*>(get_bucket());
- if (!b) {
- ldpp_dout(dpp, 0) << "ERROR: could not get bucket for " << get_name() << dendl;
- return -EINVAL;
- }
-
- int ret, flags;
- std::string path;
-
- if(temp_file) {
- flags = O_TMPFILE | O_RDWR;
- path = ".";
- } else {
- flags = O_RDWR | O_NOFOLLOW;
- if (create)
- flags |= O_CREAT;
- path = get_fname();
- }
- ret = openat(b->get_dir_fd(dpp), path.c_str(), flags, S_IRWXU);
- if (ret < 0) {
- ret = errno;
- ldpp_dout(dpp, 0) << "ERROR: could not open object " << get_name() << ": "
- << cpp_strerror(ret) << dendl;
- return -ret;
- }
-
- obj_fd = ret;
-
- return 0;
-}
-
-int POSIXObject::link_temp_file(const DoutPrefixProvider *dpp, optional_yield y, uint32_t flags)
+int POSIXObject::get_cur_version(const DoutPrefixProvider* dpp, rgw_obj_key& key)
{
- if (obj_fd < 0) {
- return 0;
- }
-
- char temp_file_path[PATH_MAX];
- // Only works on Linux - Non-portable
- snprintf(temp_file_path, PATH_MAX, "/proc/self/fd/%d", obj_fd);
-
- POSIXBucket *b = static_cast<POSIXBucket*>(get_bucket());
-
- if (!b) {
- ldpp_dout(dpp, 0) << "ERROR: could not get bucket for " << get_name() << dendl;
- return -EINVAL;
- }
-
- int ret = linkat(AT_FDCWD, temp_file_path, b->get_dir_fd(dpp), get_temp_fname().c_str(), AT_SYMLINK_FOLLOW);
- if(ret < 0) {
- ret = errno;
- ldpp_dout(dpp, 0) << "ERROR: linkat for temp file could not finish: "
- << cpp_strerror(ret) << dendl;
- return -ret;
- }
-
- // Delete the target, in case it's a multipart
- ret = delete_object(dpp, y, flags);
- if (ret < 0) {
- ldpp_dout(dpp, 0) << "ERROR: could not remove dest object "
- << get_name() << dendl;
- return ret;
- }
-
- ret = renameat(b->get_dir_fd(dpp), get_temp_fname().c_str(), b->get_dir_fd(dpp), get_fname().c_str());
- if(ret < 0) {
- ret = errno;
- ldpp_dout(dpp, 0) << "ERROR: renameat for object could not finish: "
- << cpp_strerror(ret) << dendl;
- return -ret;
- }
-
return 0;
}
-
-int POSIXObject::close()
+int POSIXObject::set_cur_version(const DoutPrefixProvider *dpp)
{
- if (obj_fd < 0) {
- return 0;
- }
-
- int ret = ::fsync(obj_fd);
- if(ret < 0) {
- return ret;
- }
-
- ret = ::close(obj_fd);
- if(ret < 0) {
+ VersionedDirectory* vdir = static_cast<VersionedDirectory*>(ent.get());
+ std::unique_ptr<FSEnt> child;
+ int ret = vdir->get_ent(dpp, null_yield, get_fname(true), std::string(), child);
+ if (ret < 0)
return ret;
- }
- obj_fd = -1;
- return 0;
+ ret = vdir->set_cur_version_ent(dpp, child.get());
+ return ret;
}
-int POSIXObject::read(int64_t ofs, int64_t left, bufferlist& bl,
- const DoutPrefixProvider* dpp, optional_yield y)
+int POSIXObject::stat(const DoutPrefixProvider* dpp)
{
- if (!shadow) {
- // Normal file, just read it
- int64_t len = std::min(left + 1, READ_SIZE);
- ssize_t ret;
-
- ret = lseek(obj_fd, ofs, SEEK_SET);
- if (ret < 0) {
- ret = errno;
- ldpp_dout(dpp, 0) << "ERROR: could not seek object " << get_name() << " to "
- << ofs << " :" << cpp_strerror(ret) << dendl;
- return -ret;
- }
+ int ret;
- char read_buf[READ_SIZE];
- ret = ::read(obj_fd, read_buf, len);
+ if (!ent) {
+ ret = static_cast<POSIXBucket *>(bucket)->get_dir()->get_ent(
+ dpp, null_yield, get_fname(/*use_version=*/false), state.obj.key.instance, ent);
if (ret < 0) {
- ret = errno;
- ldpp_dout(dpp, 0) << "ERROR: could not read object " << get_name() << ": "
- << cpp_strerror(ret) << dendl;
- return -ret;
- }
-
- bl.append(read_buf, ret);
-
- return ret;
- }
-
- // It's a multipart object, find the correct file, open it, and read it
- std::string pname;
- for (auto part : parts) {
- if (ofs < part.second) {
- pname = part.first;
- break;
+ state.exists = false;
+ return ret;
}
+ }
- ofs -= part.second;
+ ret = ent->stat(dpp);
+ if (ret < 0) {
+ state.exists = false;
+ return ret;
}
- if (pname.empty()) {
- // ofs is past the end
- return 0;
+ if (state.obj.key.instance.empty()) {
+ state.obj.key.instance = ent->get_cur_version();
}
- POSIXObject* shadow_obj;
- std::unique_ptr<rgw::sal::Object> obj = shadow->get_object(rgw_obj_key(pname));
- shadow_obj = static_cast<POSIXObject*>(obj.get());
- int ret = shadow_obj->open(dpp, false);
- if (ret < 0) {
- return ret;
+ state.exists = ent->exists();
+ if (!state.exists) {
+ return 0;
}
- return shadow_obj->read(ofs, left, bl, dpp, y);
+ state.accounted_size = state.size = ent->get_stx().stx_size;
+ state.mtime = from_statx_timestamp(ent->get_stx().stx_mtime);
+
+ return 0;
}
-int POSIXObject::write(int64_t ofs, bufferlist& bl, const DoutPrefixProvider* dpp,
- optional_yield y)
+int POSIXObject::make_ent(ObjectType type)
{
- if (shadow) {
- // Can't write to a MP file
- return -EINVAL;
+ if (ent)
+ return 0;
+
+ switch (type.type) {
+ case ObjectType::UNKNOWN:
+ return -EINVAL;
+ case ObjectType::FILE:
+ ent = std::make_unique<File>(
+ get_fname(/*use_version=*/true), static_cast<POSIXBucket *>(bucket)->get_dir(), driver->ctx());
+ break;
+ case ObjectType::DIRECTORY:
+ ent = std::make_unique<Directory>(
+ get_fname(/*use_version=*/true), static_cast<POSIXBucket *>(bucket)->get_dir(), driver->ctx());
+ break;
+ case ObjectType::SYMLINK:
+ ent = std::make_unique<Symlink>(
+ get_fname(/*use_version=*/true), static_cast<POSIXBucket *>(bucket)->get_dir(), driver->ctx());
+ break;
+ case ObjectType::MULTIPART:
+ ent = std::make_unique<MPDirectory>(
+ get_fname(/*use_version=*/true), static_cast<POSIXBucket *>(bucket)->get_dir(), driver->ctx());
+ break;
+ case ObjectType::VERSIONED:
+ ent = std::make_unique<VersionedDirectory>(
+ get_fname(/*use_version=*/false), static_cast<POSIXBucket *>(bucket)->get_dir(), get_instance(), driver->ctx());
+ break;
}
- int64_t left = bl.length();
- char* curp = bl.c_str();
- ssize_t ret;
+ return 0;
+}
- ret = fchmod(obj_fd, S_IRUSR|S_IWUSR);
- if(ret < 0) {
- ldpp_dout(dpp, 0) << "ERROR: could not change permissions on object " << get_name() << ": "
- << cpp_strerror(ret) << dendl;
+int POSIXObject::get_owner(const DoutPrefixProvider *dpp, optional_yield y, std::unique_ptr<User> *owner)
+{
+ POSIXOwner o;
+ int ret = decode_owner(get_attrs(), o);
+ if (ret < 0) {
+ ldpp_dout(dpp, 0) << "ERROR: " << __func__
+ << ": No " RGW_POSIX_ATTR_OWNER " attr" << dendl;
return ret;
}
+ *owner = driver->get_user(o.user);
+ (*owner)->load_user(dpp, y);
+ return 0;
+}
+
+std::unique_ptr<Object::ReadOp> POSIXObject::get_read_op()
+{
+ return std::make_unique<POSIXReadOp>(this);
+}
+
+std::unique_ptr<Object::DeleteOp> POSIXObject::get_delete_op()
+{
+ return std::make_unique<POSIXDeleteOp>(this);
+}
+
+int POSIXObject::open(const DoutPrefixProvider* dpp, bool create, bool temp_file)
+{
+ int ret{0};
- ret = lseek(obj_fd, ofs, SEEK_SET);
+ if (!ent) {
+ ret = stat(dpp);
+ if (ret < 0) {
+ if (!create) {
+ return ret;
+ }
+ if (versioned()) {
+ ret = make_ent(ObjectType::VERSIONED);
+ } else {
+ ret = make_ent(ObjectType::FILE);
+ }
+ }
+ }
if (ret < 0) {
- ret = errno;
- ldpp_dout(dpp, 0) << "ERROR: could not seek object " << get_name() << " to "
- << ofs << " :" << cpp_strerror(ret) << dendl;
- return -ret;
+ return ret;
}
- while (left > 0) {
- ret = ::write(obj_fd, curp, left);
+ if (create) {
+ ret = ent->create(dpp, nullptr, temp_file);
if (ret < 0) {
- ret = errno;
- ldpp_dout(dpp, 0) << "ERROR: could not write object " << get_name() << ": "
- << cpp_strerror(ret) << dendl;
- return -ret;
+ ldpp_dout(dpp, 0) << "ERROR: could not create " << ent->get_name() << dendl;
+ return ret;
}
+ }
- curp += ret;
- left -= ret;
+ return ent->open(dpp);
+}
+
+int POSIXObject::link_temp_file(const DoutPrefixProvider *dpp, optional_yield y)
+{
+ std::string temp_fname = gen_temp_fname();
+ int ret = ent->link_temp_file(dpp, y, temp_fname);
+ if (ret < 0)
+ return ret;
+
+ POSIXBucket *b = static_cast<POSIXBucket *>(get_bucket());
+ if (!b) {
+ ldpp_dout(dpp, 0) << "ERROR: could not get bucket for " << get_name()
+ << dendl;
+ return -EINVAL;
}
+ fill_cache( nullptr, null_yield,
+ [&](const DoutPrefixProvider *dpp, rgw_bucket_dir_entry &bde) -> int {
+ driver->get_bucket_cache()->add_entry(dpp, b->get_name(), bde);
+ return 0;
+ });
return 0;
}
-int POSIXObject::write_attr(const DoutPrefixProvider* dpp, optional_yield y, const std::string& key, bufferlist& value)
+
+int POSIXObject::close()
{
- int ret;
- std::string attrname;
+ if (ent)
+ return ent->close();
- ret = open(dpp, true);
- if (ret < 0) {
- return ret;
- }
+ return 0;
+}
+
+int POSIXObject::read(int64_t ofs, int64_t left, bufferlist& bl,
+ const DoutPrefixProvider* dpp, optional_yield y)
+{
+ if (!ent)
+ return -ENOENT;
+ return ent->read(ofs, left, bl, dpp, y);
+}
+
+int POSIXObject::write(int64_t ofs, bufferlist& bl, const DoutPrefixProvider* dpp,
+ optional_yield y)
+{
+ return ent->write(ofs, bl, dpp, y);
+}
- return write_x_attr(dpp, y, obj_fd, key, value, get_name());
+int POSIXObject::write_attrs(const DoutPrefixProvider* dpp, optional_yield y)
+{
+ return ent->write_attrs(dpp, y, state.attrset, nullptr);
}
int POSIXObject::POSIXReadOp::prepare(optional_yield y, const DoutPrefixProvider* dpp)
{
int ret;
- /* Generate an ETAG */
- if (shadow) {
- ret = generate_mp_etag(dpp, y);
- } else {
- ret = generate_etag(dpp, y);
- }
-
+ ret = generate_etag(dpp, y);
return ret;
}
int POSIXObject::generate_mp_etag(const DoutPrefixProvider* dpp, optional_yield y)
{
- int32_t count = 0;
- char etag_buf[CEPH_CRYPTO_MD5_DIGESTSIZE];
- char final_etag_str[CEPH_CRYPTO_MD5_DIGESTSIZE * 2 + 16];
- std::string etag;
- bufferlist etag_bl;
- MD5 hash;
- // Allow use of MD5 digest in FIPS mode for non-cryptographic purposes
- hash.SetFlags(EVP_MD_CTX_FLAG_NON_FIPS_ALLOW);
- int ret;
- rgw::sal::Bucket::ListParams params;
- rgw::sal::Bucket::ListResults results;
-
- do {
- static constexpr auto MAX_LIST_OBJS = 100u;
- ret = shadow->list(dpp, params, MAX_LIST_OBJS, results, y);
- if (ret < 0) {
- return ret;
- }
- for (rgw_bucket_dir_entry& ent : results.objs) {
- std::unique_ptr<rgw::sal::Object> obj;
- POSIXObject* shadow_obj;
-
- if (MP_OBJ_PART_PFX.compare(0, std::string::npos, ent.key.name,
- MP_OBJ_PART_PFX.size() != 0)) {
- // Skip non-parts
- continue;
- }
-
- obj = shadow->get_object(rgw_obj_key(ent.key));
- shadow_obj = static_cast<POSIXObject*>(obj.get());
- ret = shadow_obj->get_obj_attrs(y, dpp);
- if (ret < 0) {
- return ret;
- }
- bufferlist etag_bl;
- if (!shadow_obj->get_attr(RGW_ATTR_ETAG, etag_bl)) {
- // Generate part's etag
- ret = shadow_obj->generate_etag(dpp, y);
- if (ret < 0)
- return ret;
- }
- if (!shadow_obj->get_attr(RGW_ATTR_ETAG, etag_bl)) {
- // Can't get etag.
- return -EINVAL;
- }
- hex_to_buf(etag_bl.c_str(), etag_buf, CEPH_CRYPTO_MD5_DIGESTSIZE);
- hash.Update((const unsigned char *)etag_buf, sizeof(etag_buf));
- count++;
- }
- } while (results.is_truncated);
-
- hash.Final((unsigned char *)etag_buf);
-
- buf_to_hex((unsigned char *)etag_buf, sizeof(etag_buf), final_etag_str);
- snprintf(&final_etag_str[CEPH_CRYPTO_MD5_DIGESTSIZE * 2],
- sizeof(final_etag_str) - CEPH_CRYPTO_MD5_DIGESTSIZE * 2,
- "-%" PRId32, count);
- etag = final_etag_str;
- ldpp_dout(dpp, 10) << "calculated etag: " << etag << dendl;
-
- etag_bl.append(etag);
- (void)write_attr(dpp, y, RGW_ATTR_ETAG, etag_bl);
- get_attrs().emplace(std::move(RGW_ATTR_ETAG), std::move(etag_bl));
-
return 0;
}
hash.Final(m);
buf_to_hex(m, CEPH_CRYPTO_MD5_DIGESTSIZE, calc_md5);
etag_bl.append(calc_md5, sizeof(calc_md5));
- (void)write_attr(dpp, y, RGW_ATTR_ETAG, etag_bl);
get_attrs().emplace(std::move(RGW_ATTR_ETAG), std::move(etag_bl));
-
- return 0;
+ return write_attrs(dpp, y);
}
-const std::string POSIXObject::get_fname()
+const std::string POSIXObject::get_fname(bool use_version)
{
- std::string fname = url_encode(get_obj().get_oid(), true);
-
- if (!get_obj().key.get_ns().empty()) {
- /* Namespaced objects are hidden */
- fname.insert(0, 1, '.');
- }
-
- return fname;
+ return get_key_fname(state.obj.key, use_version);
}
-void POSIXObject::gen_temp_fname()
+std::string POSIXObject::gen_temp_fname()
{
+ std::string temp_fname;
enum { RAND_SUFFIX_SIZE = 8 };
char buf[RAND_SUFFIX_SIZE + 1];
gen_rand_alphanumeric_no_underscore(driver->ctx(), buf, RAND_SUFFIX_SIZE);
- temp_fname = "." + get_fname() + ".";
+ temp_fname = "." + get_fname(/*use_version=*/true) + ".";
temp_fname.append(buf);
-}
-const std::string POSIXObject::get_temp_fname()
-{
return temp_fname;
}
/* Read some */
int ret = cb->handle_data(bl, 0, len);
if (ret < 0) {
- ldpp_dout(dpp, 0) << " ERROR: callback failed on " << source->get_name() << dendl;
+ ldpp_dout(dpp, 0) << " ERROR: callback failed on " << source->get_name() << ": " << ret << dendl;
return ret;
}
int POSIXObject::copy(const DoutPrefixProvider *dpp, optional_yield y,
POSIXBucket *sb, POSIXBucket *db, POSIXObject *dobj)
{
- off64_t scount = 0, dcount = 0;
-
- int ret = open(dpp, false);
- if (ret < 0) {
- ldpp_dout(dpp, 0) << "ERROR: could not open source object " << get_name()
- << dendl;
- return ret;
- }
-
- // Delete the target, in case it's a multipart
- ret = dobj->delete_object(dpp, y, rgw::sal::FLAG_LOG_OP);
- if (ret < 0) {
- ldpp_dout(dpp, 0) << "ERROR: could not remove dest object "
- << dobj->get_name() << dendl;
- return ret;
- }
-
- ret = dobj->open(dpp, true);
- if (ret < 0) {
- ldpp_dout(dpp, 0) << "ERROR: could not open dest object "
- << dobj->get_name() << dendl;
- return ret;
- }
-
- ret = copy_file_range(obj_fd, &scount, dobj->get_fd(), &dcount, stx.stx_size, 0);
- if (ret < 0) {
- ret = errno;
- ldpp_dout(dpp, 0) << "ERROR: could not copy object " << dobj->get_name()
- << ": " << cpp_strerror(ret) << dendl;
- return -ret;
- }
-
- ret = get_obj_attrs(y, dpp);
- if (ret < 0) {
- ldpp_dout(dpp, 0) << "ERROR: could not get attrs for source object "
- << get_name() << dendl;
- return ret;
- }
+ rgw_obj_key dst_key = dobj->get_key();
+ if (!get_key().instance.empty())
+ dst_key.instance = get_key().instance;
- ret = dobj->set_obj_attrs(dpp, &get_attrs(), NULL, y, rgw::sal::FLAG_LOG_OP);
- if (ret < 0) {
- ldpp_dout(dpp, 0) << "ERROR: could not write attrs to dest object "
- << dobj->get_name() << dendl;
- return ret;
- }
-
- return 0;
+ return ent->copy(dpp, y, db->get_dir(), get_key_fname(dst_key, /*use_version=*/true));
}
void POSIXMPObj::init_gen(POSIXDriver* driver, const std::string& _oid, ACLOwner& _owner)
int POSIXMultipartPart::load(const DoutPrefixProvider* dpp, optional_yield y,
POSIXDriver* driver, rgw_obj_key& key)
{
- if (shadow) {
+ if (part_file) {
/* Already loaded */
return 0;
}
- shadow = std::make_unique<POSIXObject>(driver, key, upload->get_shadow());
+ part_file = std::make_unique<File>(get_key_fname(key, false), upload->get_shadow()->get_dir(), driver->ctx());
- // Stat the shadow object to get things like size
- int ret = shadow->load_obj_state(dpp, y);
+ // Stat the part_file object to get things like size
+ int ret = part_file->stat(dpp, y);
if (ret < 0) {
return ret;
}
- ret = shadow->get_obj_attrs(y, dpp);
+ Attrs attrs;
+ ret = part_file->read_attrs(dpp, y, attrs);
if (ret < 0) {
return ret;
}
- auto ait = shadow->get_attrs().find(RGW_POSIX_ATTR_MPUPLOAD);
- if (ait == shadow->get_attrs().end()) {
- ldout(driver->ctx(), 0) << "ERROR: " << __func__ << ": Not a part: " << key << dendl;
- return -EINVAL;
- }
-
- try {
- auto bit = ait->second.cbegin();
- decode(info, bit);
- } catch (buffer::error& err) {
- ldout(driver->ctx(), 0) << "ERROR: " << __func__ << ": failed to decode part info: " << key << dendl;
- return -EINVAL;
+ ret = decode_attr(attrs, RGW_POSIX_ATTR_MPUPLOAD, info);
+ if (ret < 0) {
+ ldpp_dout(dpp, 0) << "ERROR: " << __func__ << ": failed to decode part info: " << key << dendl;
+ return ret;
}
return 0;
}
-int POSIXMultipartUpload::load(bool create)
+int POSIXMultipartUpload::load(const DoutPrefixProvider *dpp, bool create)
{
+ int ret = 0;
if (!shadow) {
POSIXBucket* pb = static_cast<POSIXBucket*>(bucket);
- return pb->get_shadow_bucket(nullptr, null_yield, mp_ns,
- std::string(), get_meta(), create, &shadow);
+ std::optional<std::string> ns{mp_ns};
+
+ std::unique_ptr<Directory> mpdir = std::make_unique<MPDirectory>(bucket_fname(get_meta(), ns), pb->get_dir(), driver->ctx());
+
+ shadow = std::make_unique<POSIXBucket>(driver, std::move(mpdir), rgw_bucket(std::string(), get_meta()), mp_ns);
+
+ ret = shadow->load_bucket(dpp, null_yield);
+ if (ret == -ENOENT && create) {
+ ret = shadow->create(dpp, null_yield, nullptr);
+ }
}
- return 0;
+ return ret;
}
std::unique_ptr<rgw::sal::Object> POSIXMultipartUpload::get_meta_obj()
{
- load();
+ load(nullptr);
if (!shadow) {
// This upload doesn't exist, but the API doesn't check this until it calls
// on the *serializer*. So make a fake object in the parent bucket that
int ret;
/* Create the shadow bucket */
- ret = load(true);
+ ret = load(dpp, true);
if (ret < 0) {
- ldpp_dout(dpp, 0) << " ERROR: could not get shadow bucket for mp upload "
+ ldpp_dout(dpp, 0) << " ERROR: could not get shadow dir for mp upload "
<< get_key() << dendl;
return ret;
}
meta_obj = get_meta_obj();
+ ret = static_cast<POSIXObject*>(meta_obj.get())->open(dpp, true);
+ if (ret < 0) {
+ return ret;
+ }
+
mp_obj.upload_info.cksum_type = cksum_type;
mp_obj.upload_info.dest_placement = dest_placement;
+ mp_obj.owner = owner;
bufferlist bl;
encode(mp_obj, bl);
int ret;
int last_num = 0;
- ret = load();
+ ret = load(dpp);
if (ret < 0) {
return ret;
}
params.prefix = MP_OBJ_PART_PFX;
params.marker = MP_OBJ_PART_PFX + fmt::format("{:0>5}", marker);
+ params.marker.ns = mp_ns;
+ params.ns = mp_ns;
ret = shadow->list(dpp, params, num_parts + 1, results, y);
if (ret < 0) {
POSIXMultipartPart* ppart = static_cast<POSIXMultipartPart*>(part.get());
rgw_obj_key key(ent.key);
+ // Parts are namespaced in the bucket listing
+ key.ns.clear();
ret = ppart->load(dpp, y, driver, key);
if (ret == 0) {
/* Skip anything that's not a part */
{
int ret;
- ret = load();
+ ret = load(dpp);
if (ret < 0) {
+ if (ret == -ENOENT)
+ ret = ERR_NO_SUCH_UPLOAD;
return ret;
}
auto etags_iter = part_etags.begin();
rgw::sal::Attrs& attrs = target_obj->get_attrs();
+ ofs = accounted_size = 0;
+
do {
ret = list_parts(dpp, cct, max_parts, marker, &marker, &truncated, y);
if (ret == -ENOENT) {
sizeof(final_etag_str) - CEPH_CRYPTO_MD5_DIGESTSIZE * 2,
"-%lld", (long long)part_etags.size());
etag = final_etag_str;
- ldpp_dout(dpp, 10) << "calculated etag: " << etag << dendl;
etag_bl.append(etag);
}
// Rename to target_obj
- return shadow->rename(dpp, y, target_obj);
+ ret = shadow->rename(dpp, y, target_obj);
+ if (ret < 0) {
+ ldpp_dout(dpp, 0) << "ERROR: failed to rename to final name " << target_obj->get_name()
+ << ": " << cpp_strerror(ret) << dendl;
+ return ret;
+ }
+
+ POSIXObject *to = static_cast<POSIXObject*>(target_obj);
+ POSIXBucket *sb = static_cast<POSIXBucket*>(target_obj->get_bucket());
+ if (sb->versioned()) {
+ ret = to->set_cur_version(dpp);
+ if (ret < 0) {
+ return ret;
+ }
+ }
+ return 0;
}
int POSIXMultipartUpload::get_info(const DoutPrefixProvider *dpp, optional_yield y,
}
if (rule) {
- if (mp_obj.oid.empty()) {
+ if (mp_obj.upload_info.dest_placement.name.empty()) {
if (!meta_obj) {
meta_obj = get_meta_obj();
- ret = meta_obj->get_obj_attrs(y, dpp);
- if (ret < 0) {
- ldpp_dout(dpp, 0) << " ERROR: could not get meta object for mp upload "
- << get_key() << dendl;
- return ret;
- }
}
- bufferlist bl;
- if (!meta_obj->get_attr(RGW_POSIX_ATTR_MPUPLOAD, bl)) {
+ ret = meta_obj->get_obj_attrs(y, dpp);
+ if (ret < 0) {
+ ldpp_dout(dpp, 0) << " ERROR: could not get meta object for mp upload "
+ << get_key() << dendl;
+ return ret;
+ }
+ ret = decode_attr(meta_obj->get_attrs(), RGW_POSIX_ATTR_MPUPLOAD, mp_obj);
+ if (ret < 0) {
ldpp_dout(dpp, 0) << " ERROR: could not get meta object attrs for mp upload "
<< get_key() << dendl;
return ret;
}
- auto biter = bl.cbegin();
- decode(mp_obj, biter);
}
*rule = &mp_obj.upload_info.dest_placement;
}
return 0;
}
+std::string POSIXMultipartUpload::get_fname()
+{
+ std::string name;
+
+ name = "." + mp_ns + "_" + url_encode(get_meta(), true);
+
+ return name;
+}
+
std::unique_ptr<Writer> POSIXMultipartUpload::get_writer(
const DoutPrefixProvider *dpp,
optional_yield y,
std::string fname = MP_OBJ_PART_PFX + fmt::format("{:0>5}", part_num);
rgw_obj_key part_key(fname);
- load();
+ load(dpp);
- return std::make_unique<POSIXMultipartWriter>(dpp, y, shadow->clone(), part_key, driver,
- owner, ptail_placement_rule, part_num);
+ return std::make_unique<POSIXMultipartWriter>(dpp, y, shadow.get(), part_key,
+ driver, owner,
+ ptail_placement_rule, part_num);
}
int POSIXMultipartWriter::prepare(optional_yield y)
{
- return obj->open(dpp, true);
+ int ret = part_file->create(dpp, /*existed=*/nullptr, /*tempfile=*/false);
+ if (ret < 0) {
+ return ret;
+ }
+
+ return part_file->open(dpp);
}
int POSIXMultipartWriter::process(bufferlist&& data, uint64_t offset)
{
- return obj->write(offset, data, dpp, null_yield);
+ return part_file->write(offset, data, dpp, null_yield);
}
int POSIXMultipartWriter::complete(
if (if_match) {
if (strcmp(if_match, "*") == 0) {
// test the object is existing
- if (!obj->check_exists(dpp)) {
+ if (!part_file->exists()) {
return -ERR_PRECONDITION_FAILED;
}
} else {
+ Attrs attrs;
bufferlist bl;
- if (!obj->get_attr(RGW_ATTR_ETAG, bl)) {
+ ret = part_file->read_attrs(rctx.dpp, rctx.y, attrs);
+ if (ret < 0) {
+ return -ERR_PRECONDITION_FAILED;
+ }
+ if (!get_attr(attrs, RGW_ATTR_ETAG, bl)) {
return -ERR_PRECONDITION_FAILED;
}
if (strncmp(if_match, bl.c_str(), bl.length()) != 0) {
encode(info, bl);
attrs[RGW_POSIX_ATTR_MPUPLOAD] = bl;
- for (auto& attr : attrs) {
- ret = obj->write_attr(rctx.dpp, rctx.y, attr.first, attr.second);
- if (ret < 0) {
- ldpp_dout(rctx.dpp, 20) << "ERROR: failed writing attr " << attr.first << dendl;
- return ret;
- }
+ ret = part_file->write_attrs(rctx.dpp, rctx.y, attrs, /*extra_attrs=*/nullptr);
+ if (ret < 0) {
+ ldpp_dout(rctx.dpp, 20) << "ERROR: failed writing attrs for " << part_file->get_name() << dendl;
+ return ret;
}
- ret = obj->close();
+ ret = part_file->close();
if (ret < 0) {
ldpp_dout(rctx.dpp, 20) << "ERROR: failed closing file" << dendl;
return ret;
int POSIXAtomicWriter::prepare(optional_yield y)
{
- obj.get_obj_attrs(y, dpp);
- obj.close();
- obj.gen_temp_fname();
- return obj.open(dpp, true, true);
+ int ret;
+
+ if (obj->versioned()) {
+ ret = obj->make_ent(ObjectType::VERSIONED);
+ } else {
+ ret = obj->make_ent(ObjectType::FILE);
+ }
+ if (ret < 0) {
+ return ret;
+ }
+ obj->get_obj_attrs(y, dpp);
+ obj->close();
+ return obj->open(dpp, true, true);
}
int POSIXAtomicWriter::process(bufferlist&& data, uint64_t offset)
{
- return obj.write(offset, data, dpp, null_yield);
+ return obj->write(offset, data, dpp, null_yield);
}
int POSIXAtomicWriter::complete(size_t accounted_size, const std::string& etag,
if (if_match) {
if (strcmp(if_match, "*") == 0) {
// test the object is existing
- if (!obj.check_exists(dpp)) {
+ if (!obj->check_exists(dpp)) {
return -ERR_PRECONDITION_FAILED;
}
} else {
bufferlist bl;
- if (!obj.get_attr(RGW_ATTR_ETAG, bl)) {
+ if (!get_attr(obj->get_attrs(), RGW_ATTR_ETAG, bl)) {
return -ERR_PRECONDITION_FAILED;
}
if (strncmp(if_match, bl.c_str(), bl.length()) != 0) {
if (if_nomatch) {
if (strcmp(if_nomatch, "*") == 0) {
// test the object is not existing
- if (obj.check_exists(dpp)) {
+ if (obj->check_exists(dpp)) {
return -ERR_PRECONDITION_FAILED;
}
} else {
bufferlist bl;
- if (!obj.get_attr(RGW_ATTR_ETAG, bl)) {
+ if (!get_attr(obj->get_attrs(), RGW_ATTR_ETAG, bl)) {
return -ERR_PRECONDITION_FAILED;
}
if (strncmp(if_nomatch, bl.c_str(), bl.length()) == 0) {
}
}
- bufferlist bl;
- encode(owner, bl);
- attrs[RGW_POSIX_ATTR_OWNER] = bl;
+ bufferlist owner_bl;
+ std::unique_ptr<User> user;
+ user = driver->get_user(std::get<rgw_user>(owner.id));
+ user->load_user(rctx.dpp, rctx.y);
+ POSIXOwner po{std::get<rgw_user>(owner.id), user->get_display_name()};
+ encode(po, owner_bl);
+ attrs[RGW_POSIX_ATTR_OWNER] = owner_bl;
- for (auto attr : attrs) {
- ret = obj.write_attr(rctx.dpp, rctx.y, attr.first, attr.second);
- if (ret < 0) {
- ldpp_dout(rctx.dpp, 20) << "ERROR: POSIXAtomicWriter failed writing attr " << attr.first << dendl;
- return ret;
- }
+ bufferlist type_bl;
+
+ obj->set_attrs(attrs);
+ ret = obj->write_attrs(rctx.dpp, rctx.y);
+ if (ret < 0) {
+ ldpp_dout(rctx.dpp, 20) << "ERROR: POSIXAtomicWriter failed writing attrs for "
+ << obj->get_name() << dendl;
+ return ret;
}
- ret = obj.link_temp_file(rctx.dpp, rctx.y, flags);
+ ret = obj->link_temp_file(rctx.dpp, rctx.y);
if (ret < 0) {
ldpp_dout(dpp, 20) << "ERROR: POSIXAtomicWriter failed writing temp file" << dendl;
return ret;
}
- ret = obj.close();
+ ret = obj->open(dpp);
+ if (ret < 0) {
+ ldpp_dout(rctx.dpp, 20) << "ERROR: POSIXAtomicWriter failed opening file" << dendl;
+ return ret;
+ }
+
+ ret = obj->stat(dpp);
if (ret < 0) {
ldpp_dout(rctx.dpp, 20) << "ERROR: POSIXAtomicWriter failed closing file" << dendl;
return ret;
using BucketCache = file::listing::BucketCache<POSIXDriver, POSIXBucket>;
+/* integration w/bucket listing cache */
+using fill_cache_cb_t = file::listing::fill_cache_cb_t;
+
+struct ObjectType {
+ enum Type {
+ UNKNOWN = 0,
+ FILE = 1,
+ DIRECTORY = 2,
+ VERSIONED = 3,
+ MULTIPART = 4,
+ SYMLINK = 5,
+ };
+ uint32_t type{UNKNOWN};
+
+ ObjectType &operator=(ObjectType::Type &&_t) {
+ type = _t;
+ return *this;
+ };
+
+ ObjectType() {}
+ ObjectType(Type _t) : type(_t){}
+
+ bool operator==(const ObjectType &t) const { return (type == t.type); }
+ bool operator==(const ObjectType::Type &t) const { return (type == t); }
+
+ void encode(bufferlist &bl) const {
+ ENCODE_START(1, 1, bl);
+ encode(type, bl);
+ ENCODE_FINISH(bl);
+ }
+
+ void decode(bufferlist::const_iterator &bl) {
+ DECODE_START(1, bl);
+ ceph::decode(type, bl);
+ DECODE_FINISH(bl);
+ }
+ friend inline std::ostream &operator<<(std::ostream &out,
+ const ObjectType &t) {
+ switch (t.type) {
+ case UNKNOWN:
+ out << "UNKNOWN";
+ break;
+ case FILE:
+ out << "FILE";
+ break;
+ case DIRECTORY:
+ out << "DIRECTORY";
+ break;
+ case VERSIONED:
+ out << "VERSIONED";
+ break;
+ case MULTIPART:
+ out << "MULTIPART";
+ break;
+ case SYMLINK:
+ out << "SYMLINK";
+ break;
+ }
+ return out;
+ }
+};
+WRITE_CLASS_ENCODER(ObjectType);
+
+class Directory;
+
+class FSEnt {
+protected:
+ std::string fname;
+ Directory* parent;
+ int fd{-1};
+ bool exist{false};
+ struct statx stx;
+ bool stat_done{false};
+ CephContext* ctx;
+
+public:
+ FSEnt(std::string _name, Directory* _parent, CephContext* _ctx) : fname(_name), parent(_parent), ctx(_ctx) {}
+ FSEnt(std::string _name, Directory* _parent, struct statx& _stx, CephContext* _ctx) : fname(_name), parent(_parent), exist(true), stx(_stx), stat_done(true), ctx(_ctx) {}
+ FSEnt(const FSEnt& _e) :
+ fname(_e.fname),
+ parent(_e.parent),
+ exist(_e.exist),
+ stx(_e.stx),
+ stat_done(_e.stat_done),
+ ctx(_e.ctx)
+ { }
+
+ virtual ~FSEnt() { }
+
+ int get_fd() { return fd; };
+ std::string& get_name() { return fname; }
+ Directory* get_parent() { return parent; }
+ bool exists() { return exist; }
+ struct statx& get_stx() { return stx; }
+ virtual ObjectType get_type() { return ObjectType::UNKNOWN; };
+
+ virtual int create(const DoutPrefixProvider *dpp, bool* existed = nullptr, bool temp_file = false) = 0;
+ virtual int open(const DoutPrefixProvider *dpp) = 0;
+ virtual int close() = 0;
+ virtual int stat(const DoutPrefixProvider *dpp, bool force = false);
+ virtual int remove(const DoutPrefixProvider* dpp, optional_yield y, bool delete_children) = 0;
+ virtual int write(int64_t ofs, bufferlist& bl, const DoutPrefixProvider* dpp, optional_yield y) = 0;
+ virtual int read(int64_t ofs, int64_t end, bufferlist& bl, const DoutPrefixProvider* dpp, optional_yield y) = 0;
+ virtual int write_attrs(const DoutPrefixProvider* dpp, optional_yield y, Attrs& attrs, Attrs* extra_attrs);
+ virtual int read_attrs(const DoutPrefixProvider* dpp, optional_yield y, Attrs& attrs);
+ virtual int copy(const DoutPrefixProvider *dpp, optional_yield y, Directory* dst_dir, const std::string& name) = 0;
+ virtual int link_temp_file(const DoutPrefixProvider* dpp, optional_yield y, std::string target_fname) = 0;
+ virtual std::unique_ptr<FSEnt> clone_base() = 0;
+ virtual int fill_cache(const DoutPrefixProvider* dpp, optional_yield y, fill_cache_cb_t& cb);
+ virtual std::string get_cur_version() { return ""; };
+};
+
+class File : public FSEnt {
+protected:
+
+public:
+ File(std::string _name, Directory* _parent, CephContext* _ctx) : FSEnt(_name, _parent, _ctx)
+ {}
+ File(std::string _name, Directory* _parent, struct statx& _stx, CephContext* _ctx) : FSEnt(_name, _parent, _stx, _ctx)
+ {}
+ File(const File& _f) : FSEnt(_f) {}
+ virtual ~File() { close(); }
+
+ virtual uint64_t get_size() { return stx.stx_size; }
+ virtual ObjectType get_type() override { return ObjectType::FILE; };
+
+
+ virtual int create(const DoutPrefixProvider *dpp, bool* existed = nullptr, bool temp_file = false) override;
+ virtual int open(const DoutPrefixProvider *dpp) override;
+ virtual int close() override;
+ virtual int stat(const DoutPrefixProvider *dpp, bool force = false) override;
+ virtual int remove(const DoutPrefixProvider* dpp, optional_yield y, bool delete_children) override;
+ virtual int write(int64_t ofs, bufferlist& bl, const DoutPrefixProvider* dpp, optional_yield y) override;
+ virtual int read(int64_t ofs, int64_t end, bufferlist& bl, const DoutPrefixProvider* dpp, optional_yield y) override;
+ virtual int copy(const DoutPrefixProvider *dpp, optional_yield y, Directory* dst_dir, const std::string& name) override;
+ virtual int link_temp_file(const DoutPrefixProvider* dpp, optional_yield y, std::string target_fname) override;
+ virtual std::unique_ptr<FSEnt> clone_base() override {
+ return std::make_unique<File>(*this);
+ }
+ std::unique_ptr<File> clone() {
+ return std::make_unique<File>(*this);
+ }
+};
+
+class Directory : public FSEnt {
+protected:
+
+public:
+ Directory(std::string _name, Directory* _parent, CephContext* _ctx) : FSEnt(_name, _parent, _ctx)
+ {}
+ Directory(std::string _name, Directory* _parent, struct statx& _stx, CephContext* _ctx) : FSEnt(_name, _parent, _stx, _ctx)
+ {}
+ Directory(const Directory& _d) : FSEnt(_d) {}
+ virtual ~Directory() { close(); }
+
+ virtual ObjectType get_type() override { return ObjectType::DIRECTORY; };
+
+ virtual bool file_exists(std::string& name);
+
+ virtual int create(const DoutPrefixProvider *dpp, bool* existed = nullptr, bool temp_file = false) override;
+ virtual int open(const DoutPrefixProvider *dpp) override;
+ virtual int close() override;
+ virtual int stat(const DoutPrefixProvider *dpp, bool force = false) override;
+ virtual int remove(const DoutPrefixProvider* dpp, optional_yield y, bool delete_children) override;
+ template <typename F>
+ int for_each(const DoutPrefixProvider* dpp, const F& func);
+ virtual int rename(const DoutPrefixProvider* dpp, optional_yield y, Directory* dst_dir, std::string dst_name);
+ virtual int write(int64_t ofs, bufferlist& bl, const DoutPrefixProvider* dpp, optional_yield y) override;
+ virtual int read(int64_t ofs, int64_t end, bufferlist& bl, const DoutPrefixProvider* dpp, optional_yield y) override;
+ virtual std::unique_ptr<FSEnt> clone_base() override {
+ return std::make_unique<Directory>(*this);
+ }
+ virtual std::unique_ptr<Directory> clone_dir() {
+ return std::make_unique<Directory>(*this);
+ }
+ std::unique_ptr<Directory> clone() {
+ return std::make_unique<Directory>(*this);
+ }
+ virtual int copy(const DoutPrefixProvider *dpp, optional_yield y, Directory* dst_dir, const std::string& name) override;
+ virtual int link_temp_file(const DoutPrefixProvider* dpp, optional_yield y, std::string target_fname) override;
+ virtual int fill_cache(const DoutPrefixProvider* dpp, optional_yield y, fill_cache_cb_t& cb) override;
+
+ int get_ent(const DoutPrefixProvider *dpp, optional_yield y, const std::string& name, const std::string& version, std::unique_ptr<FSEnt>& ent);
+};
+
+class Symlink: public File {
+ std::unique_ptr<FSEnt> target;
+public:
+ Symlink(std::string _name, Directory* _parent, std::string _tgt, CephContext* _ctx) :
+ File(_name, _parent, _ctx)
+ { fill_target(nullptr, parent, fname,_tgt, target, _ctx); }
+ Symlink(std::string _name, Directory* _parent, CephContext* _ctx) :
+ File(_name, _parent, _ctx)
+ {}
+ Symlink(std::string _name, Directory* _parent, struct statx& _stx, std::string _tgt, CephContext* _ctx) :
+ File(_name, _parent, _stx, _ctx)
+ { fill_target(nullptr, parent, fname,_tgt, target, _ctx); }
+ Symlink(std::string _name, Directory* _parent, struct statx& _stx, CephContext* _ctx) :
+ File(_name, _parent, _stx, _ctx)
+ {}
+ Symlink(const Symlink& _s) : File(_s) {}
+ virtual ~Symlink() { close(); }
+
+ static int fill_target(const DoutPrefixProvider *dpp, Directory* parent, std::string sname, std::string tname, std::unique_ptr<FSEnt>& ent, CephContext* _ctx);
+
+ virtual ObjectType get_type() override { return ObjectType::SYMLINK; };
+ virtual int create(const DoutPrefixProvider *dpp, bool* existed = nullptr, bool temp_file = false) override;
+ virtual int stat(const DoutPrefixProvider *dpp, bool force = false) override;
+ virtual int read_attrs(const DoutPrefixProvider* dpp, optional_yield y, Attrs& attrs) override;
+ FSEnt* get_target() { return target.get(); }
+ virtual std::unique_ptr<FSEnt> clone_base() override {
+ return std::make_unique<Symlink>(*this);
+ }
+ std::unique_ptr<Symlink> clone() {
+ return std::make_unique<Symlink>(*this);
+ }
+ virtual int copy(const DoutPrefixProvider *dpp, optional_yield y, Directory* dst_dir, const std::string& name) override;
+ virtual int fill_cache(const DoutPrefixProvider* dpp, optional_yield y, fill_cache_cb_t& cb) override;
+};
+
+class MPDirectory : public Directory {
+ std::string tmpname;
+protected:
+ std::map<std::string, int64_t> parts;
+ std::unique_ptr<FSEnt> cur_read_part;
+
+public:
+ MPDirectory(std::string _name, Directory* _parent, CephContext* _ctx) : Directory(_name, _parent, _ctx)
+ {}
+ MPDirectory(std::string _name, Directory* _parent, struct statx& _stx, CephContext* _ctx) : Directory(_name, _parent, _stx, _ctx)
+ {}
+ MPDirectory(const MPDirectory& _d) :
+ Directory(_d),
+ parts(_d.parts)
+ { if (_d.cur_read_part) cur_read_part = _d.cur_read_part->clone_base(); }
+ virtual ~MPDirectory() { close(); }
+
+ virtual ObjectType get_type() override { return ObjectType::MULTIPART; };
+ virtual int create(const DoutPrefixProvider *dpp, bool* existed = nullptr, bool temp_file = false) override;
+ virtual int read(int64_t ofs, int64_t end, bufferlist& bl, const DoutPrefixProvider* dpp, optional_yield y) override;
+ virtual int link_temp_file(const DoutPrefixProvider* dpp, optional_yield y, std::string target_fname) override;
+ virtual int remove(const DoutPrefixProvider* dpp, optional_yield y, bool delete_children) override;
+ virtual int stat(const DoutPrefixProvider *dpp, bool force = false) override;
+ std::unique_ptr<File> get_part_file(int partnum);
+ virtual std::unique_ptr<FSEnt> clone_base() override {
+ return std::make_unique<MPDirectory>(*this);
+ }
+ virtual std::unique_ptr<Directory> clone_dir() override {
+ return std::make_unique<MPDirectory>(*this);
+ }
+ std::unique_ptr<MPDirectory> clone() {
+ return std::make_unique<MPDirectory>(*this);
+ }
+ virtual int fill_cache(const DoutPrefixProvider* dpp, optional_yield y, fill_cache_cb_t& cb) override;
+};
+
+class VersionedDirectory : public Directory {
+protected:
+ std::string instance_id;
+ std::unique_ptr<FSEnt> cur_version;
+
+public:
+ VersionedDirectory(std::string _name, Directory* _parent, CephContext* _ctx) : Directory(_name, _parent, _ctx)
+ {}
+ VersionedDirectory(std::string _name, Directory* _parent, std::string _instance_id, CephContext* _ctx) :
+ Directory(_name, _parent, _ctx),
+ instance_id(_instance_id)
+ {}
+ VersionedDirectory(std::string _name, Directory* _parent, std::unique_ptr<FSEnt>&& _cur, CephContext* _ctx) :
+ Directory(_name, _parent, _ctx),
+ cur_version(std::move(_cur))
+ {}
+ VersionedDirectory(std::string _name, Directory* _parent, struct statx& _stx, CephContext* _ctx) : Directory(_name, _parent, _stx, _ctx)
+ {}
+ VersionedDirectory(std::string _name, Directory* _parent, std::string _instance_id, struct statx& _stx, CephContext* _ctx) :
+ Directory(_name, _parent, _stx, _ctx),
+ instance_id(_instance_id)
+ {}
+ VersionedDirectory(const VersionedDirectory& _d) :
+ Directory(_d),
+ instance_id(_d.instance_id),
+ cur_version(_d.cur_version ? _d.cur_version->clone_base() : nullptr)
+ { }
+ VersionedDirectory(const Directory& _d) :
+ Directory(_d)
+ { }
+ virtual ~VersionedDirectory() { close(); }
+
+ virtual ObjectType get_type() override { return ObjectType::VERSIONED; };
+ virtual int create(const DoutPrefixProvider *dpp, bool* existed = nullptr, bool temp_file = false) override;
+ virtual int open(const DoutPrefixProvider *dpp) override;
+ virtual int stat(const DoutPrefixProvider *dpp, bool force = false) override;
+ virtual int read_attrs(const DoutPrefixProvider* dpp, optional_yield y, Attrs& attrs) override;
+ virtual int write_attrs(const DoutPrefixProvider* dpp, optional_yield y, Attrs& attrs, Attrs* extra_attrs) override;
+ virtual int write(int64_t ofs, bufferlist& bl, const DoutPrefixProvider* dpp, optional_yield y) override;
+ virtual int read(int64_t ofs, int64_t end, bufferlist& bl, const DoutPrefixProvider* dpp, optional_yield y) override;
+ virtual int link_temp_file(const DoutPrefixProvider* dpp, optional_yield y, std::string target_fname) override;
+ virtual int remove(const DoutPrefixProvider* dpp, optional_yield y, bool delete_children) override;
+ virtual std::string get_cur_version() override;
+ std::string get_new_instance();
+ int remove_symlink(const DoutPrefixProvider *dpp, optional_yield y, std::string match = "");
+ int add_file(const DoutPrefixProvider *dpp, std::unique_ptr<FSEnt>&& file, bool* existed = nullptr, bool temp_file = false);
+ FSEnt* get_cur_version_ent() { return cur_version.get(); };
+ int set_cur_version_ent(const DoutPrefixProvider *dpp, FSEnt* file);
+ virtual std::unique_ptr<FSEnt> clone_base() override {
+ return std::make_unique<VersionedDirectory>(*this);
+ }
+ virtual std::unique_ptr<Directory> clone_dir() override {
+ return std::make_unique<VersionedDirectory>(*this);
+ }
+ std::unique_ptr<VersionedDirectory> clone() {
+ return std::make_unique<VersionedDirectory>(*this);
+ }
+ virtual int copy(const DoutPrefixProvider *dpp, optional_yield y, Directory* dst_dir, const std::string& name) override;
+ virtual int fill_cache(const DoutPrefixProvider* dpp, optional_yield y, fill_cache_cb_t& cb) override;
+};
+
+std::string get_key_fname(rgw_obj_key& key, bool use_version);
+
class POSIXDriver : public FilterDriver {
-private:
+protected:
std::unique_ptr<BucketCache> bucket_cache;
std::string base_path;
+ std::unique_ptr<Directory> root_dir;
int root_fd;
public:
POSIXDriver(Driver* _next) : FilterDriver(_next)
{ }
- virtual ~POSIXDriver() { close(); }
+ virtual ~POSIXDriver() { }
virtual int initialize(CephContext *cct, const DoutPrefixProvider *dpp) override;
virtual std::unique_ptr<User> get_user(const rgw_user& u) override;
virtual int get_user_by_access_key(const DoutPrefixProvider* dpp, const
optional_yield y) override;
/* Internal APIs */
- int get_root_fd() { return root_fd; }
+ int get_root_fd() { return root_dir->get_fd(); }
+ Directory* get_root_dir() { return root_dir.get(); }
const std::string& get_base_path() const { return base_path; }
BucketCache* get_bucket_cache() { return bucket_cache.get(); }
- int close();
-
/* called by BucketCache layer when a new object is discovered
* by inotify or similar */
int mint_listing_entry(
class POSIXBucket : public StoreBucket {
private:
POSIXDriver* driver;
- int parent_fd{-1};
- int dir_fd{-1};
- struct statx stx;
- bool stat_done{false};
RGWAccessControlPolicy acls;
- std::optional<std::string> ns;
+ std::optional<std::string> ns{std::nullopt};
+ std::unique_ptr<Directory> dir;
public:
- POSIXBucket(POSIXDriver *_dr, int _p_fd, const rgw_bucket& _b, std::optional<std::string> _ns = std::nullopt)
+ POSIXBucket(POSIXDriver *_dr, Directory* _p_dir, const rgw_bucket& _b, std::optional<std::string> _ns = std::nullopt)
+ : StoreBucket(_b),
+ driver(_dr),
+ acls(),
+ ns(_ns),
+ dir(std::make_unique<Directory>(get_fname(), _p_dir, _dr->ctx()))
+ { }
+
+ POSIXBucket(POSIXDriver *_dr, std::unique_ptr<Directory> _this_dir, const rgw_bucket& _b, std::optional<std::string> _ns = std::nullopt)
: StoreBucket(_b),
driver(_dr),
- parent_fd(_p_fd),
acls(),
- ns(_ns)
+ ns(_ns),
+ dir(std::move(_this_dir))
{ }
- POSIXBucket(POSIXDriver *_dr, int _p_fd, const RGWBucketInfo& _i)
+ POSIXBucket(POSIXDriver *_dr, Directory* _p_dir, const RGWBucketInfo& _i)
: StoreBucket(_i),
driver(_dr),
- parent_fd(_p_fd),
- acls()
+ acls(),
+ ns(),
+ dir(std::make_unique<Directory>(get_fname(), _p_dir, _dr->ctx()))
{ }
POSIXBucket(const POSIXBucket& _b) :
StoreBucket(_b),
driver(_b.driver),
- parent_fd(_b.parent_fd),
- /* Don't want to copy dir_fd */
- stx(_b.stx),
- stat_done(_b.stat_done),
acls(_b.acls),
- ns(_b.ns) {}
+ ns(_b.ns)
+ {
+ dir.reset(static_cast<Directory*>(_b.dir->clone().get()));
+ }
- virtual ~POSIXBucket() { close(); }
+ virtual ~POSIXBucket() { }
virtual std::unique_ptr<Object> get_object(const rgw_obj_key& key) override;
virtual int list(const DoutPrefixProvider* dpp, ListParams&, int,
/* Internal APIs */
int create(const DoutPrefixProvider *dpp, optional_yield y, bool* existed);
- void set_stat(struct statx _stx) { stx = _stx; stat_done = true; }
- int get_dir_fd(const DoutPrefixProvider *dpp) { open(dpp); return dir_fd; }
+ Directory* get_dir() { return dir.get(); }
+ int get_dir_fd(const DoutPrefixProvider *dpp) { dir->open(dpp); return dir->get_fd(); }
/* TODO dang Escape the bucket name for file use */
std::string get_fname();
- int get_shadow_bucket(const DoutPrefixProvider* dpp, optional_yield y,
- const std::string& ns, const std::string& tenant,
- const std::string& name, bool create,
- std::unique_ptr<POSIXBucket>* shadow);
- template <typename F>
- int for_each(const DoutPrefixProvider* dpp, const F& func);
- int open(const DoutPrefixProvider *dpp);
- int close();
+ std::optional<std::string> get_ns() { return ns; }
int rename(const DoutPrefixProvider* dpp, optional_yield y, Object* target_obj);
- int copy(const DoutPrefixProvider *dpp, optional_yield y, POSIXBucket* db, POSIXObject* dobj);
-
- /* integration w/bucket listing cache */
- using fill_cache_cb_t = file::listing::fill_cache_cb_t;
/* enumerate all entries by callback, in any order */
- int fill_cache(const DoutPrefixProvider* dpp, optional_yield y, fill_cache_cb_t cb);
+ int fill_cache(const DoutPrefixProvider* dpp, optional_yield y, fill_cache_cb_t& cb);
private:
- int stat(const DoutPrefixProvider *dpp);
int write_attrs(const DoutPrefixProvider *dpp, optional_yield y);
}; /* POSIXBucket */
class POSIXObject : public StoreObject {
+public:
private:
POSIXDriver* driver;
RGWAccessControlPolicy acls;
- int obj_fd{-1};
- struct statx stx;
- bool stat_done{false};
- std::unique_ptr<rgw::sal::POSIXBucket> shadow;
- std::string temp_fname;
+ std::unique_ptr<FSEnt> ent;
std::map<std::string, int64_t> parts;
public:
POSIXObject(POSIXDriver *_dr, const rgw_obj_key& _k)
: StoreObject(_k),
driver(_dr),
- acls() {}
+ acls()
+ {}
POSIXObject(POSIXDriver* _driver, const rgw_obj_key& _k, Bucket* _b) :
StoreObject(_k, _b),
driver(_driver),
- acls() {}
+ acls()
+ {}
POSIXObject(const POSIXObject& _o) :
StoreObject(_o),
- driver(_o.driver) {}
+ driver(_o.driver)
+ {}
virtual ~POSIXObject() { close(); }
return std::unique_ptr<Object>(new POSIXObject(*this));
}
- int open(const DoutPrefixProvider *dpp, bool create, bool temp_file = false);
+ FSEnt* get_fsent() { return ent.get(); }
+ int open(const DoutPrefixProvider *dpp, bool create = false, bool temp_file = false);
int close();
int write(int64_t ofs, bufferlist& bl, const DoutPrefixProvider* dpp, optional_yield y);
- int write_attr(const DoutPrefixProvider* dpp, optional_yield y, const std::string& key, bufferlist& value);
- int link_temp_file(const DoutPrefixProvider* dpp, optional_yield y, uint32_t flags);
- void gen_temp_fname();
- /* TODO dang Escape the object name for file use */
- const std::string get_fname();
+ int write_attrs(const DoutPrefixProvider* dpp, optional_yield y);
+ int link_temp_file(const DoutPrefixProvider* dpp, optional_yield y);
+ std::string gen_temp_fname();
+ const std::string get_fname(bool use_version);
bool check_exists(const DoutPrefixProvider* dpp) { stat(dpp); return state.exists; }
int get_owner(const DoutPrefixProvider *dpp, optional_yield y, std::unique_ptr<User> *owner);
int copy(const DoutPrefixProvider *dpp, optional_yield y, POSIXBucket *sb,
POSIXBucket *db, POSIXObject *dobj);
- int fill_bde(const DoutPrefixProvider *dpp, optional_yield y, rgw_bucket_dir_entry &bde);
+ int fill_cache(const DoutPrefixProvider *dpp, optional_yield y, fill_cache_cb_t& cb);
+ int set_cur_version(const DoutPrefixProvider *dpp);
+ int stat(const DoutPrefixProvider *dpp);
+ int make_ent(ObjectType type);
+ bool versioned() { return bucket->versioned(); }
protected:
int read(int64_t ofs, int64_t end, bufferlist& bl, const DoutPrefixProvider* dpp, optional_yield y);
int generate_attrs(const DoutPrefixProvider* dpp, optional_yield y);
- int get_fd() { return obj_fd; };
private:
- const std::string get_temp_fname();
- int stat(const DoutPrefixProvider *dpp);
int generate_mp_etag(const DoutPrefixProvider* dpp, optional_yield y);
int generate_etag(const DoutPrefixProvider* dpp, optional_yield y);
+ int get_cur_version(const DoutPrefixProvider *dpp, rgw_obj_key &key);
};
struct POSIXMPObj {
protected:
POSIXUploadPartInfo info;
POSIXMultipartUpload* upload;
- std::unique_ptr<rgw::sal::POSIXObject> shadow;
+ std::unique_ptr<File> part_file;
public:
POSIXMultipartPart(POSIXMultipartUpload* _upload) :
virtual ~POSIXMultipartPart() = default;
virtual uint32_t get_num() { return info.num; }
- virtual uint64_t get_size() { return shadow->get_size(); }
+ virtual uint64_t get_size() { return part_file->get_size(); }
virtual const std::string& get_etag() { return info.etag; }
virtual ceph::real_time& get_mtime() { return info.mtime; }
virtual const std::optional<rgw::cksum::Cksum>& get_cksum() {
POSIXBucket* get_shadow() { return shadow.get(); }
private:
- int load(bool create=false);
+ std::string get_fname();
+ int load(const DoutPrefixProvider *dpp, bool create=false);
};
class POSIXAtomicWriter : public StoreWriter {
const rgw_placement_rule *ptail_placement_rule;
uint64_t olh_epoch;
const std::string& unique_tag;
- POSIXObject obj;
+ POSIXObject* obj;
public:
POSIXAtomicWriter(const DoutPrefixProvider *dpp,
ptail_placement_rule(_ptail_placement_rule),
olh_epoch(_olh_epoch),
unique_tag(_unique_tag),
- obj(_driver, _head_obj->get_key(), _head_obj->get_bucket()) {}
+ obj(static_cast<POSIXObject*>(_head_obj)) {}
virtual ~POSIXAtomicWriter() = default;
virtual int prepare(optional_yield y) override;
const ACLOwner& owner;
const rgw_placement_rule *ptail_placement_rule;
uint64_t part_num;
- std::unique_ptr<Bucket> shadow_bucket;
- std::unique_ptr<POSIXObject> obj;
+ std::unique_ptr<Directory> upload_dir;
+ std::unique_ptr<File> part_file;
public:
POSIXMultipartWriter(const DoutPrefixProvider *dpp,
optional_yield y,
- std::unique_ptr<Bucket> _shadow_bucket,
+ POSIXBucket* _shadow_bucket,
rgw_obj_key& _key,
POSIXDriver* _driver,
const ACLOwner& _owner,
owner(_owner),
ptail_placement_rule(_ptail_placement_rule),
part_num(_part_num),
- shadow_bucket(std::move(_shadow_bucket)),
- obj(std::make_unique<POSIXObject>(_driver, _key, shadow_bucket.get())) {}
+ upload_dir(_shadow_bucket->get_dir()->clone()),
+ part_file(std::make_unique<File>(get_key_fname(_key, false), upload_dir.get(), _driver->ctx()))
+ { upload_dir->open(dpp); }
virtual ~POSIXMultipartWriter() = default;
virtual int prepare(optional_yield y) override;
SYSTEM PRIVATE "${CMAKE_SOURCE_DIR}/src/rgw/driver/posix")
target_link_libraries(unittest_posix_bucket_cache ${UNITTEST_LIBS}
${rgw_libs} ${LMDB_LIBRARIES})
+
+# unittest_rgw_posix_driver
+add_executable(unittest_rgw_posix_driver
+ test_rgw_posix_driver.cc)
+add_ceph_unittest(unittest_rgw_posix_driver)
+target_compile_definitions(unittest_rgw_posix_driver PUBLIC LMDB_SAFE_NO_CPP_UTILITIES)
+target_include_directories(unittest_rgw_posix_driver
+ SYSTEM PRIVATE "${CMAKE_SOURCE_DIR}/src/rgw"
+ SYSTEM PRIVATE "${CMAKE_SOURCE_DIR}/src/rgw/driver/posix")
+target_link_libraries(unittest_rgw_posix_driver ${UNITTEST_LIBS}
+ ${rgw_libs} ${LMDB_LIBRARIES})
endif(WITH_RADOSGW_POSIX)
#include <fmt/format.h>
#include <gtest/gtest.h>
+#include "common/common_init.h"
+#include "global/global_init.h"
using namespace std::chrono_literals;
static constexpr std::string_view bucket1_marker = ""; // start at the beginning
DoutPrefixProvider* dpp{nullptr};
+ std::vector<std::string> bvec;
class MockSalDriver
{
+ std::vector<const char *> args;
+
public:
+ boost::intrusive_ptr<CephContext> cct;
+ MockSalDriver() {
+ /* Proceed with environment setup */
+ cct = global_init(nullptr, args, CEPH_ENTITY_TYPE_CLIENT,
+ CODE_ENVIRONMENT_UTILITY,
+ CINIT_FLAG_NO_DEFAULT_CONFIG_FILE);
+ }
/* called by BucketCache layer when a new object is discovered
* by inotify or similar */
int mint_listing_entry(
return 0;
}
+ CephContext *ctx(void) { return cct.get(); }
}; /* MockSalDriver */
class MockSalBucket
--- /dev/null
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab ft=cpp
+
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2020 Red Hat, Inc
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation. See file COPYING.
+ */
+
+#include "rgw_sal_posix.h"
+#include <gtest/gtest.h>
+#include <iostream>
+#include <filesystem>
+#include "common/common_init.h"
+#include "global/global_init.h"
+
+using namespace rgw::sal;
+
+const std::string ATTR1{"attr1"};
+const std::string ATTR2{"attr2"};
+const std::string ATTR3{"attr3"};
+const std::string ATTR_OBJECT_TYPE{"POSIX-Object-Type"};
+
+namespace sf = std::filesystem;
+class Environment* env;
+sf::path base_path{"posixtest"};
+std::unique_ptr<Directory> root;
+std::vector<const char*> args;
+
+class Environment : public ::testing::Environment {
+public:
+ boost::intrusive_ptr<CephContext> cct;
+ DoutPrefixProvider* dpp{nullptr};
+
+ Environment() {}
+
+ virtual ~Environment() {}
+
+ void SetUp() override {
+ sf::remove_all(base_path);
+ sf::create_directory(base_path);
+
+ args.push_back("--rgw_multipart_min_part_size=32");
+ args.push_back("--debug-rgw=20");
+ args.push_back("--debug-ms=1");
+
+ /* Proceed with environment setup */
+ cct = global_init(nullptr, args, CEPH_ENTITY_TYPE_CLIENT,
+ CODE_ENVIRONMENT_UTILITY,
+ CINIT_FLAG_NO_DEFAULT_CONFIG_FILE);
+
+ dpp = nullptr;
+ //dpp = new NoDoutPrefix(cct.get(), 1);
+
+ root = std::make_unique<Directory>(base_path, nullptr, cct.get());
+ ASSERT_EQ(root->open(dpp), 0);
+ }
+
+ void TearDown() override {
+ sf::remove_all(base_path);
+ }
+};
+
+
+static inline void add_attr(Attrs& attrs, const std::string& name, const std::string& value)
+{
+ bufferlist bl;
+ encode(value, bl);
+
+ attrs[name] = bl;
+}
+
+static inline bool get_attr(Attrs& attrs, const char* name, bufferlist& bl)
+{
+ auto iter = attrs.find(name);
+ if (iter == attrs.end()) {
+ return false;
+ }
+
+ bl = iter->second;
+ return true;
+}
+
+template <typename F>
+static bool decode_attr(Attrs &attrs, const char *name, F &f) {
+ bufferlist bl;
+ if (!get_attr(attrs, name, bl)) {
+ return false;
+ }
+ F tmpf;
+ try {
+ auto bufit = bl.cbegin();
+ decode(tmpf, bufit);
+ } catch (buffer::error &err) {
+ return false;
+ }
+
+ f = tmpf;
+ return true;
+}
+
+class TestDirectory : public Directory {
+public:
+ TestDirectory(std::string _name, Directory* _parent, CephContext* _ctx) : Directory(_name, _parent, _ctx)
+ {}
+ TestDirectory(std::string _name, Directory* _parent, struct statx& _stx, CephContext* _ctx) : Directory(_name, _parent, _stx, _ctx)
+ {}
+ virtual ~TestDirectory() { close(); }
+
+ bool get_stat_done() { return stat_done; }
+};
+
+class TestFile : public File {
+public:
+ TestFile(std::string _name, Directory* _parent, CephContext* _ctx) : File(_name, _parent, _ctx)
+ {}
+ TestFile(std::string _name, Directory* _parent, struct statx& _stx, CephContext* _ctx) : File(_name, _parent, _stx, _ctx)
+ {}
+ virtual ~TestFile() { close(); }
+
+ bool get_stat_done() { return stat_done; }
+};
+
+std::string get_test_name()
+{
+ std::string suitename =
+ testing::UnitTest::GetInstance()->current_test_info()->test_suite_name();
+ std::string testname =
+ testing::UnitTest::GetInstance()->current_test_info()->name();
+
+ return suitename + testname;
+}
+
+
+// Directory
+
+TEST(FSEnt, DirCreate)
+{
+ std::string dirname = get_test_name();
+ sf::path tp{base_path / dirname};
+ std::unique_ptr<Directory> testdir = std::make_unique<Directory>(dirname, root.get(), env->cct.get());
+
+ EXPECT_FALSE(sf::exists(tp));
+
+ bool existed;
+ int ret = testdir->create(env->dpp, &existed);
+
+ EXPECT_EQ(ret, 0);
+ EXPECT_FALSE(existed);
+ EXPECT_TRUE(sf::exists(tp));
+ EXPECT_TRUE(sf::is_directory(tp));
+}
+
+TEST(FSEnt, DirBase)
+{
+ std::string dirname = get_test_name();
+ sf::path tp{base_path / dirname};
+ std::unique_ptr<TestDirectory> testdir = std::make_unique<TestDirectory>(dirname, root.get(), env->cct.get());
+
+ EXPECT_FALSE(sf::exists(tp));
+
+ bool existed;
+ int ret = testdir->create(env->dpp, &existed);
+
+ EXPECT_EQ(ret, 0);
+ EXPECT_FALSE(existed);
+ EXPECT_TRUE(sf::exists(tp));
+ EXPECT_TRUE(sf::is_directory(tp));
+
+ EXPECT_EQ(testdir->get_fd(), -1);
+ EXPECT_EQ(testdir->get_name(), dirname);
+ EXPECT_EQ(testdir->get_parent(), root.get());
+ EXPECT_FALSE(testdir->exists());
+ EXPECT_EQ(testdir->get_type(), ObjectType::DIRECTORY);
+ EXPECT_FALSE(testdir->get_stat_done());
+
+ ret = testdir->open(env->dpp);
+ EXPECT_EQ(ret, 0);
+ EXPECT_GT(testdir->get_fd(), 0);
+
+ ret = testdir->stat(env->dpp, false);
+ EXPECT_EQ(ret, 0);
+ EXPECT_TRUE(testdir->get_stat_done());
+ EXPECT_TRUE(S_ISDIR(testdir->get_stx().stx_mode));
+
+ Attrs attrs;
+ add_attr(attrs, ATTR1, ATTR1);
+ add_attr(attrs, ATTR2, ATTR2);
+ Attrs extra_attrs;
+ add_attr(extra_attrs, ATTR3, ATTR3);
+
+ ret = testdir->write_attrs(env->dpp, null_yield, attrs, &extra_attrs);
+ EXPECT_EQ(ret, 0);
+
+ attrs.clear();
+ ret = testdir->read_attrs(env->dpp, null_yield, attrs);
+ EXPECT_EQ(ret, 0);
+ EXPECT_EQ(attrs.size(), 4);
+ std::string val;
+ bool success = decode_attr(attrs, ATTR1.c_str(), val);
+ EXPECT_TRUE(success);
+ EXPECT_EQ(val, ATTR1);
+ success = decode_attr(attrs, ATTR2.c_str(), val);
+ EXPECT_TRUE(success);
+ EXPECT_EQ(val, ATTR2);
+ success = decode_attr(attrs, ATTR3.c_str(), val);
+ EXPECT_TRUE(success);
+ EXPECT_EQ(val, ATTR3);
+ ObjectType type;
+ success = decode_attr(attrs, ATTR_OBJECT_TYPE.c_str(), type);
+ EXPECT_TRUE(success);
+ EXPECT_EQ(type.type, ObjectType::DIRECTORY);
+
+ ret = testdir->close();
+ EXPECT_EQ(ret, 0);
+ EXPECT_EQ(testdir->get_fd(), -1);
+
+ bufferlist bl;
+ ret = testdir->write(0, bl, env->dpp, null_yield);
+ EXPECT_EQ(ret, -EINVAL);
+
+ ret = testdir->read(0, 50, bl, env->dpp, null_yield);
+ EXPECT_EQ(ret, -EINVAL);
+
+ ret = testdir->link_temp_file(env->dpp, null_yield, dirname);
+ EXPECT_EQ(ret, -EINVAL);
+
+ std::string copyname{dirname + "-copy"};
+ sf::path cp{base_path / copyname};
+ sf::remove_all(cp);
+ EXPECT_FALSE(sf::exists(cp));
+ ret = testdir->copy(env->dpp, null_yield, root.get(), copyname);
+ EXPECT_EQ(ret, 0);
+ EXPECT_TRUE(sf::exists(cp));
+ EXPECT_TRUE(sf::is_directory(tp));
+
+ std::unique_ptr<TestDirectory> copydir = std::make_unique<TestDirectory>(copyname, root.get(), env->cct.get());
+ ret = copydir->open(env->dpp);
+ EXPECT_EQ(ret, 0);
+ EXPECT_GT(copydir->get_fd(), 0);
+
+ ret = copydir->stat(env->dpp, false);
+ EXPECT_EQ(ret, 0);
+ EXPECT_TRUE(copydir->get_stat_done());
+ EXPECT_TRUE(S_ISDIR(copydir->get_stx().stx_mode));
+
+ attrs.clear();
+ ret = copydir->read_attrs(env->dpp, null_yield, attrs);
+ EXPECT_EQ(ret, 0);
+ EXPECT_EQ(attrs.size(), 4);
+ success = decode_attr(attrs, ATTR1.c_str(), val);
+ EXPECT_TRUE(success);
+ EXPECT_EQ(val, ATTR1);
+ success = decode_attr(attrs, ATTR2.c_str(), val);
+ EXPECT_TRUE(success);
+ EXPECT_EQ(val, ATTR2);
+ success = decode_attr(attrs, ATTR3.c_str(), val);
+ EXPECT_TRUE(success);
+ EXPECT_EQ(val, ATTR3);
+
+ ret = copydir->close();
+ EXPECT_EQ(ret, 0);
+ EXPECT_EQ(copydir->get_fd(), -1);
+
+ std::unique_ptr<FSEnt> ent;
+ ret = root->get_ent(env->dpp, null_yield, dirname, std::string(), ent);
+ EXPECT_EQ(ret, 0);
+ EXPECT_EQ(ent->get_type(), ObjectType::DIRECTORY);
+
+ ret = testdir->remove(env->dpp, null_yield, false);
+ EXPECT_EQ(ret, 0);
+ EXPECT_FALSE(sf::exists(tp));
+}
+
+TEST(FSEnt, DirAddDir)
+{
+ bool existed;
+ std::string dirname = get_test_name();
+ sf::path tp{base_path / dirname};
+ std::unique_ptr<Directory> testdir = std::make_unique<Directory>(dirname, root.get(), env->cct.get());
+ int ret = testdir->create(env->dpp, &existed);
+ EXPECT_EQ(ret, 0);
+
+ ret = testdir->open(env->dpp);
+ EXPECT_EQ(ret, 0);
+
+ std::string subdirname{"SubDir"};
+ sf::path sp{base_path / dirname / subdirname};
+ std::unique_ptr<Directory> subdir = std::make_unique<Directory>(subdirname, testdir.get(), env->cct.get());
+ ret = subdir->create(env->dpp, &existed);
+ EXPECT_EQ(ret, 0);
+ EXPECT_FALSE(existed);
+ EXPECT_TRUE(sf::exists(sp));
+ EXPECT_TRUE(sf::is_directory(sp));
+
+ ret = subdir->open(env->dpp);
+ EXPECT_EQ(ret, 0);
+
+ std::string subsubdirname{"SubSubDir"};
+ sf::path ssp{base_path / dirname / subdirname / subsubdirname };
+ std::unique_ptr<Directory> subsubdir = std::make_unique<Directory>(subsubdirname, subdir.get(), env->cct.get());
+ ret = subsubdir->create(env->dpp, &existed);
+ EXPECT_EQ(ret, 0);
+ EXPECT_FALSE(existed);
+ EXPECT_TRUE(sf::exists(ssp));
+ EXPECT_TRUE(sf::is_directory(ssp));
+}
+
+TEST(FSEnt, DirRename)
+{
+ bool existed;
+ std::string dirname = get_test_name();
+ sf::path tp{base_path / dirname};
+ std::unique_ptr<Directory> testdir = std::make_unique<Directory>(dirname, root.get(), env->cct.get());
+ int ret = testdir->create(env->dpp, &existed);
+ EXPECT_EQ(ret, 0);
+
+ ret = testdir->open(env->dpp);
+ EXPECT_EQ(ret, 0);
+
+ std::string subdirname{"SubDir"};
+ sf::path sp{base_path / dirname / subdirname};
+ std::unique_ptr<Directory> subdir = std::make_unique<Directory>(subdirname, testdir.get(), env->cct.get());
+ ret = subdir->create(env->dpp, &existed);
+ EXPECT_EQ(ret, 0);
+ EXPECT_FALSE(existed);
+ EXPECT_TRUE(sf::exists(sp));
+ EXPECT_TRUE(sf::is_directory(sp));
+
+ ret = subdir->open(env->dpp);
+ EXPECT_EQ(ret, 0);
+
+ std::string newsubdirname{"SubDir2"};
+ sf::path nsp{base_path / dirname / newsubdirname};
+ ret = subdir->rename(env->dpp, null_yield, testdir.get(), newsubdirname);
+ EXPECT_EQ(ret, 0);
+ EXPECT_TRUE(sf::exists(nsp));
+ EXPECT_TRUE(sf::is_directory(nsp));
+ EXPECT_FALSE(sf::exists(sp));
+ EXPECT_FALSE(sf::is_directory(sp));
+}
+
+
+// File
+
+TEST(FSEnt, FileCreateReal)
+{
+ std::string fname = get_test_name();
+ sf::path tp{base_path / fname};
+ TestFile testfile{fname, root.get(), env->cct.get()};
+
+ EXPECT_FALSE(sf::exists(tp));
+
+ bool existed;
+ int ret = testfile.create(env->dpp, &existed);
+
+ EXPECT_EQ(ret, 0);
+ EXPECT_FALSE(existed);
+ EXPECT_TRUE(sf::exists(tp));
+ EXPECT_TRUE(sf::is_regular_file(tp));
+}
+
+TEST(FSEnt, FileCreateTemp)
+{
+ std::string fname = get_test_name();
+ sf::path tp{base_path / fname};
+ TestFile testfile{fname, root.get(), env->cct.get()};
+
+ EXPECT_FALSE(sf::exists(tp));
+
+ bool existed;
+ int ret = testfile.create(env->dpp, &existed, true);
+ EXPECT_EQ(ret, 0);
+ EXPECT_FALSE(existed);
+ EXPECT_FALSE(sf::exists(tp));
+
+ std::string temp_fname{fname + "-blargh"};
+ ret = testfile.link_temp_file(env->dpp, null_yield, temp_fname);
+ EXPECT_EQ(ret, 0);
+ EXPECT_TRUE(sf::exists(tp));
+ EXPECT_TRUE(sf::is_regular_file(tp));
+}
+
+TEST(FSEnt, FileBase)
+{
+ std::string fname = get_test_name();
+ sf::path tp{base_path / fname};
+ std::unique_ptr<TestFile> testfile = std::make_unique<TestFile>(fname, root.get(), env->cct.get());
+
+ EXPECT_FALSE(sf::exists(tp));
+ EXPECT_EQ(testfile->get_fd(), -1);
+
+ bool existed;
+ int ret = testfile->create(env->dpp, &existed);
+
+ EXPECT_EQ(ret, 0);
+ EXPECT_FALSE(existed);
+ EXPECT_TRUE(sf::exists(tp));
+ EXPECT_TRUE(sf::is_regular_file(tp));
+ // create() opens
+ EXPECT_GT(testfile->get_fd(), 0);
+
+ EXPECT_EQ(testfile->get_name(), fname);
+ EXPECT_EQ(testfile->get_parent(), root.get());
+ EXPECT_FALSE(testfile->exists());
+ EXPECT_EQ(testfile->get_type(), ObjectType::FILE);
+ EXPECT_FALSE(testfile->get_stat_done());
+
+ ret = testfile->open(env->dpp);
+ EXPECT_EQ(ret, 0);
+ EXPECT_GT(testfile->get_fd(), 0);
+
+ ret = testfile->stat(env->dpp, false);
+ EXPECT_EQ(ret, 0);
+ EXPECT_TRUE(testfile->get_stat_done());
+ EXPECT_TRUE(S_ISREG(testfile->get_stx().stx_mode));
+
+ Attrs attrs;
+ add_attr(attrs, ATTR1, ATTR1);
+ add_attr(attrs, ATTR2, ATTR2);
+ Attrs extra_attrs;
+ add_attr(extra_attrs, ATTR3, ATTR3);
+
+ ret = testfile->write_attrs(env->dpp, null_yield, attrs, &extra_attrs);
+ EXPECT_EQ(ret, 0);
+
+ attrs.clear();
+ ret = testfile->read_attrs(env->dpp, null_yield, attrs);
+ EXPECT_EQ(ret, 0);
+ EXPECT_EQ(attrs.size(), 4);
+ std::string val;
+ bool success = decode_attr(attrs, ATTR1.c_str(), val);
+ EXPECT_TRUE(success);
+ EXPECT_EQ(val, ATTR1);
+ success = decode_attr(attrs, ATTR2.c_str(), val);
+ EXPECT_TRUE(success);
+ EXPECT_EQ(val, ATTR2);
+ success = decode_attr(attrs, ATTR3.c_str(), val);
+ EXPECT_TRUE(success);
+ EXPECT_EQ(val, ATTR3);
+ ObjectType type;
+ success = decode_attr(attrs, ATTR_OBJECT_TYPE.c_str(), type);
+ EXPECT_TRUE(success);
+ EXPECT_EQ(type.type, ObjectType::FILE);
+
+ ret = testfile->close();
+ EXPECT_EQ(ret, 0);
+ EXPECT_EQ(testfile->get_fd(), -1);
+
+ std::string copyname{fname + "-copy"};
+ sf::path cp{base_path / copyname};
+ EXPECT_FALSE(sf::exists(cp));
+ ret = testfile->copy(env->dpp, null_yield, root.get(), copyname);
+ EXPECT_EQ(ret, 0);
+ EXPECT_TRUE(sf::exists(cp));
+ EXPECT_TRUE(sf::is_regular_file(tp));
+
+ std::unique_ptr<TestFile> copyfile = std::make_unique<TestFile>(copyname, root.get(), env->cct.get());
+ ret = copyfile->open(env->dpp);
+ EXPECT_EQ(ret, 0);
+ EXPECT_GT(copyfile->get_fd(), 0);
+
+ ret = copyfile->stat(env->dpp, false);
+ EXPECT_EQ(ret, 0);
+ EXPECT_TRUE(copyfile->get_stat_done());
+ EXPECT_TRUE(S_ISREG(copyfile->get_stx().stx_mode));
+
+ attrs.clear();
+ ret = copyfile->read_attrs(env->dpp, null_yield, attrs);
+ EXPECT_EQ(ret, 0);
+ EXPECT_EQ(attrs.size(), 0);
+
+ ret = copyfile->close();
+ EXPECT_EQ(ret, 0);
+ EXPECT_EQ(copyfile->get_fd(), -1);
+
+ std::unique_ptr<FSEnt> ent;
+ ret = root->get_ent(env->dpp, null_yield, fname, std::string(), ent);
+ EXPECT_EQ(ret, 0);
+ EXPECT_EQ(ent->get_type(), ObjectType::FILE);
+
+ ret = testfile->remove(env->dpp, null_yield, false);
+ EXPECT_EQ(ret, 0);
+ EXPECT_FALSE(sf::exists(tp));
+}
+
+TEST(FSEnt, FileReadWrite)
+{
+ std::string fname = get_test_name();
+ sf::path tp{base_path / fname};
+ std::unique_ptr<File> testfile{std::make_unique<File>(fname, root.get(), env->cct.get())};
+
+ int ret = testfile->create(env->dpp);
+ EXPECT_EQ(ret, 0);
+ EXPECT_TRUE(sf::exists(tp));
+ EXPECT_TRUE(sf::is_regular_file(tp));
+
+ bufferlist bl;
+ encode(fname, bl);
+ int len = bl.length();
+ ret = testfile->write(0, bl, env->dpp, null_yield);
+ EXPECT_EQ(ret, 0);
+ EXPECT_EQ(sf::file_size(tp), len);
+
+ bl.clear();
+ ret = testfile->read(0, 50, bl, env->dpp, null_yield);
+ EXPECT_EQ(ret, len);
+
+ std::string result;
+ EXPECT_NO_THROW({
+ auto bufit = bl.cbegin();
+ decode(result, bufit);
+ });
+
+ EXPECT_EQ(result, fname);
+}
+
+TEST(FSEnt, SymlinkBase)
+{
+ std::string fname = get_test_name();
+ sf::path tp{base_path / fname};
+ std::string target{"symlinktarget"};
+ std::unique_ptr<Symlink> testlink = std::make_unique<Symlink>(fname, root.get(), target, env->cct.get());
+
+ EXPECT_FALSE(sf::exists(tp));
+
+ bool existed;
+ int ret = testlink->create(env->dpp, &existed);
+
+ EXPECT_EQ(ret, 0);
+ EXPECT_FALSE(existed);
+ EXPECT_TRUE(sf::is_symlink(tp));
+ EXPECT_EQ(sf::read_symlink(tp), target);
+
+ EXPECT_EQ(testlink->get_name(), fname);
+ EXPECT_EQ(testlink->get_parent(), root.get());
+ EXPECT_FALSE(testlink->exists());
+ EXPECT_EQ(testlink->get_type(), ObjectType::SYMLINK);
+
+ std::unique_ptr<FSEnt> ent;
+ ret = root->get_ent(env->dpp, null_yield, fname, std::string(), ent);
+ EXPECT_EQ(ret, 0);
+ EXPECT_EQ(ent->get_type(), ObjectType::SYMLINK);
+
+
+ ret = testlink->remove(env->dpp, null_yield, false);
+ EXPECT_EQ(ret, 0);
+ EXPECT_FALSE(sf::exists(tp));
+}
+
+TEST(FSEnt, MPDirBase)
+{
+ std::string dirname = get_test_name();
+ sf::path tp{base_path / dirname};
+ std::unique_ptr<MPDirectory> testdir = std::make_unique<MPDirectory>(dirname, root.get(), env->cct.get());
+
+ EXPECT_FALSE(sf::exists(tp));
+
+ bool existed;
+ int ret = testdir->create(env->dpp, &existed);
+
+ EXPECT_EQ(ret, 0);
+ EXPECT_FALSE(existed);
+ EXPECT_TRUE(sf::exists(tp));
+ EXPECT_TRUE(sf::is_directory(tp));
+
+ EXPECT_EQ(testdir->get_fd(), -1);
+ EXPECT_EQ(testdir->get_name(), dirname);
+ EXPECT_EQ(testdir->get_parent(), root.get());
+ EXPECT_FALSE(testdir->exists());
+ EXPECT_EQ(testdir->get_type(), ObjectType::MULTIPART);
+
+ ret = testdir->open(env->dpp);
+ EXPECT_EQ(ret, 0);
+ EXPECT_GT(testdir->get_fd(), 0);
+
+ ret = testdir->stat(env->dpp, false);
+ EXPECT_EQ(ret, 0);
+ EXPECT_TRUE(S_ISDIR(testdir->get_stx().stx_mode));
+
+ Attrs attrs;
+ add_attr(attrs, ATTR1, ATTR1);
+ add_attr(attrs, ATTR2, ATTR2);
+ Attrs extra_attrs;
+ add_attr(extra_attrs, ATTR3, ATTR3);
+
+ ret = testdir->write_attrs(env->dpp, null_yield, attrs, &extra_attrs);
+ EXPECT_EQ(ret, 0);
+
+ attrs.clear();
+ ret = testdir->read_attrs(env->dpp, null_yield, attrs);
+ EXPECT_EQ(ret, 0);
+ EXPECT_EQ(attrs.size(), 4);
+ std::string val;
+ bool success = decode_attr(attrs, ATTR1.c_str(), val);
+ EXPECT_TRUE(success);
+ EXPECT_EQ(val, ATTR1);
+ success = decode_attr(attrs, ATTR2.c_str(), val);
+ EXPECT_TRUE(success);
+ EXPECT_EQ(val, ATTR2);
+ success = decode_attr(attrs, ATTR3.c_str(), val);
+ EXPECT_TRUE(success);
+ EXPECT_EQ(val, ATTR3);
+ ObjectType type;
+ success = decode_attr(attrs, ATTR_OBJECT_TYPE.c_str(), type);
+ EXPECT_TRUE(success);
+ EXPECT_EQ(type.type, ObjectType::MULTIPART);
+
+ ret = testdir->close();
+ EXPECT_EQ(ret, 0);
+ EXPECT_EQ(testdir->get_fd(), -1);
+
+ bufferlist bl;
+ ret = testdir->write(0, bl, env->dpp, null_yield);
+ EXPECT_EQ(ret, -EINVAL);
+
+ ret = testdir->read(0, 50, bl, env->dpp, null_yield);
+ EXPECT_EQ(ret, 0);
+
+ ret = testdir->link_temp_file(env->dpp, null_yield, dirname);
+ EXPECT_EQ(ret, 0);
+
+ std::string copyname{dirname + "-copy"};
+ sf::path cp{base_path / copyname};
+ sf::remove_all(cp);
+ EXPECT_FALSE(sf::exists(cp));
+ ret = testdir->copy(env->dpp, null_yield, root.get(), copyname);
+ EXPECT_EQ(ret, 0);
+ EXPECT_TRUE(sf::exists(cp));
+ EXPECT_TRUE(sf::is_directory(tp));
+
+ std::unique_ptr<MPDirectory> copydir = std::make_unique<MPDirectory>(copyname, root.get(), env->cct.get());
+ ret = copydir->open(env->dpp);
+ EXPECT_EQ(ret, 0);
+ EXPECT_GT(copydir->get_fd(), 0);
+
+ ret = copydir->stat(env->dpp, false);
+ EXPECT_EQ(ret, 0);
+ EXPECT_TRUE(S_ISDIR(copydir->get_stx().stx_mode));
+
+ attrs.clear();
+ ret = copydir->read_attrs(env->dpp, null_yield, attrs);
+ EXPECT_EQ(ret, 0);
+ EXPECT_EQ(attrs.size(), 4);
+ success = decode_attr(attrs, ATTR1.c_str(), val);
+ EXPECT_TRUE(success);
+ EXPECT_EQ(val, ATTR1);
+ success = decode_attr(attrs, ATTR2.c_str(), val);
+ EXPECT_TRUE(success);
+ EXPECT_EQ(val, ATTR2);
+ success = decode_attr(attrs, ATTR3.c_str(), val);
+ EXPECT_TRUE(success);
+ EXPECT_EQ(val, ATTR3);
+
+ ret = copydir->close();
+ EXPECT_EQ(ret, 0);
+ EXPECT_EQ(copydir->get_fd(), -1);
+
+ std::unique_ptr<FSEnt> ent;
+ ret = root->get_ent(env->dpp, null_yield, dirname, std::string(), ent);
+ EXPECT_EQ(ret, 0);
+ EXPECT_EQ(ent->get_type(), ObjectType::MULTIPART);
+
+ ret = testdir->remove(env->dpp, null_yield, false);
+ EXPECT_EQ(ret, 0);
+ EXPECT_FALSE(sf::exists(tp));
+}
+
+TEST(FSEnt, MPDirTemp)
+{
+ std::string dirname = get_test_name();
+ sf::path tp{base_path / dirname};
+ std::unique_ptr<MPDirectory> testdir = std::make_unique<MPDirectory>(dirname, root.get(), env->cct.get());
+
+ EXPECT_FALSE(sf::exists(tp));
+
+ bool existed;
+ int ret = testdir->create(env->dpp, &existed, true);
+
+ EXPECT_EQ(ret, 0);
+ EXPECT_FALSE(existed);
+ EXPECT_FALSE(sf::exists(tp));
+
+ std::string temp_fname; // unused
+ ret = testdir->link_temp_file(env->dpp, null_yield, temp_fname);
+ EXPECT_EQ(ret, 0);
+ EXPECT_TRUE(sf::exists(tp));
+ EXPECT_TRUE(sf::is_directory(tp));
+
+ EXPECT_EQ(testdir->get_name(), dirname);
+ EXPECT_EQ(testdir->get_parent(), root.get());
+ EXPECT_EQ(testdir->get_type(), ObjectType::MULTIPART);
+}
+
+TEST(FSEnt, MPDirReadWrite)
+{
+ std::string dirname = get_test_name();
+ sf::path tp{base_path / dirname};
+ std::unique_ptr<MPDirectory> testdir = std::make_unique<MPDirectory>(dirname, root.get(), env->cct.get());
+ int ret = testdir->create(env->dpp, nullptr);
+ EXPECT_EQ(ret, 0);
+ EXPECT_TRUE(sf::exists(tp));
+ EXPECT_TRUE(sf::is_directory(tp));
+
+ ret = testdir->open(env->dpp);
+ EXPECT_EQ(ret, 0);
+
+ bufferlist write_bl;
+ int total_len{0};
+ for (int part_num = 0; part_num < 4; ++part_num) {
+ std::unique_ptr<File> testfile = testdir->get_part_file(part_num);
+ sf::path pp{tp / testfile->get_name()};
+
+ ret = testfile->create(env->dpp, /*existed=*/nullptr, /*temp_file=*/false);
+ EXPECT_EQ(ret, 0);
+ EXPECT_TRUE(sf::exists(pp));
+ EXPECT_TRUE(sf::is_regular_file(pp));
+
+ bufferlist bl;
+ encode(dirname, bl);
+ int len = bl.length();
+ total_len += len;
+ ret = testfile->write(0, bl, env->dpp, null_yield);
+ EXPECT_EQ(ret, 0);
+ EXPECT_EQ(sf::file_size(pp), len);
+ write_bl.claim_append(bl);
+ }
+
+ ret = testdir->stat(env->dpp, true);
+ EXPECT_EQ(ret, 0);
+
+ bufferlist total_bl;
+ std::string read_data;
+ int left = total_len;
+ int ofs{0};
+ while (left > 0) {
+ bufferlist bl;
+ ret = testdir->read(ofs, left, bl, env->dpp, null_yield);
+ EXPECT_GE(ret, 0);
+ if (ret == 0)
+ break;
+
+ std::string result;
+ EXPECT_NO_THROW({
+ auto bufit = bl.cbegin();
+ decode(result, bufit);
+ });
+ read_data += result;
+ total_bl.claim_append(bl);
+ left -= ret;
+ ofs += ret;
+ }
+
+ EXPECT_EQ(total_bl.length(), total_len);
+ EXPECT_EQ(total_bl, write_bl);
+
+ EXPECT_EQ(read_data, dirname + dirname + dirname + dirname);
+}
+
+TEST(FSEnt, VerDirBase)
+{
+ std::string dirname = get_test_name();
+ sf::path tp{base_path / dirname};
+ std::unique_ptr<VersionedDirectory> testdir = std::make_unique<VersionedDirectory>(dirname, root.get(), env->cct.get());
+
+ EXPECT_FALSE(sf::exists(tp));
+
+ bool existed;
+ int ret = testdir->create(env->dpp, &existed);
+
+ EXPECT_EQ(ret, 0);
+ EXPECT_FALSE(existed);
+ EXPECT_TRUE(sf::exists(tp));
+ EXPECT_TRUE(sf::is_directory(tp));
+
+ /* Create opens */
+ EXPECT_NE(testdir->get_fd(), -1);
+ EXPECT_EQ(testdir->get_name(), dirname);
+ EXPECT_EQ(testdir->get_parent(), root.get());
+ EXPECT_FALSE(testdir->exists());
+ EXPECT_EQ(testdir->get_type(), ObjectType::VERSIONED);
+
+ ret = testdir->open(env->dpp);
+ EXPECT_EQ(ret, 0);
+ EXPECT_GT(testdir->get_fd(), 0);
+
+ ret = testdir->stat(env->dpp, false);
+ EXPECT_EQ(ret, 0);
+ EXPECT_TRUE(S_ISDIR(testdir->get_stx().stx_mode));
+
+ Attrs attrs;
+ add_attr(attrs, ATTR1, ATTR1);
+ add_attr(attrs, ATTR2, ATTR2);
+ Attrs extra_attrs;
+ add_attr(extra_attrs, ATTR3, ATTR3);
+
+ ret = testdir->write_attrs(env->dpp, null_yield, attrs, &extra_attrs);
+ EXPECT_EQ(ret, 0);
+
+ attrs.clear();
+ ret = testdir->read_attrs(env->dpp, null_yield, attrs);
+ EXPECT_EQ(ret, 0);
+ EXPECT_EQ(attrs.size(), 4);
+ std::string val;
+ bool success = decode_attr(attrs, ATTR1.c_str(), val);
+ EXPECT_TRUE(success);
+ EXPECT_EQ(val, ATTR1);
+ success = decode_attr(attrs, ATTR2.c_str(), val);
+ EXPECT_TRUE(success);
+ EXPECT_EQ(val, ATTR2);
+ success = decode_attr(attrs, ATTR3.c_str(), val);
+ EXPECT_TRUE(success);
+ EXPECT_EQ(val, ATTR3);
+ ObjectType type;
+ success = decode_attr(attrs, ATTR_OBJECT_TYPE.c_str(), type);
+ EXPECT_TRUE(success);
+ EXPECT_EQ(type.type, ObjectType::VERSIONED);
+
+ ret = testdir->close();
+ EXPECT_EQ(ret, 0);
+ EXPECT_EQ(testdir->get_fd(), -1);
+
+ bufferlist bl;
+ ret = testdir->write(0, bl, env->dpp, null_yield);
+ EXPECT_EQ(ret, 0);
+
+ ret = testdir->read(0, 50, bl, env->dpp, null_yield);
+ EXPECT_EQ(ret, 0);
+
+ ret = testdir->link_temp_file(env->dpp, null_yield, dirname);
+ EXPECT_EQ(ret, -EINVAL);
+
+ std::string copyname{dirname + "-copy"};
+ sf::path cp{base_path / copyname};
+ sf::remove_all(cp);
+ EXPECT_FALSE(sf::exists(cp));
+ ret = testdir->copy(env->dpp, null_yield, root.get(), copyname);
+ EXPECT_EQ(ret, 0);
+ EXPECT_TRUE(sf::exists(cp));
+ EXPECT_TRUE(sf::is_directory(tp));
+
+ std::unique_ptr<VersionedDirectory> copydir = std::make_unique<VersionedDirectory>(copyname, root.get(), env->cct.get());
+ ret = copydir->open(env->dpp);
+ EXPECT_EQ(ret, 0);
+ EXPECT_GT(copydir->get_fd(), 0);
+
+ ret = copydir->stat(env->dpp, false);
+ EXPECT_EQ(ret, 0);
+ EXPECT_TRUE(S_ISDIR(copydir->get_stx().stx_mode));
+
+ attrs.clear();
+ ret = copydir->read_attrs(env->dpp, null_yield, attrs);
+ EXPECT_EQ(ret, 0);
+ EXPECT_EQ(attrs.size(), 4);
+ success = decode_attr(attrs, ATTR1.c_str(), val);
+ EXPECT_TRUE(success);
+ EXPECT_EQ(val, ATTR1);
+ success = decode_attr(attrs, ATTR2.c_str(), val);
+ EXPECT_TRUE(success);
+ EXPECT_EQ(val, ATTR2);
+ success = decode_attr(attrs, ATTR3.c_str(), val);
+ EXPECT_TRUE(success);
+ EXPECT_EQ(val, ATTR3);
+
+ ret = copydir->close();
+ EXPECT_EQ(ret, 0);
+ EXPECT_EQ(copydir->get_fd(), -1);
+
+ std::unique_ptr<FSEnt> ent;
+ ret = root->get_ent(env->dpp, null_yield, dirname, std::string(), ent);
+ EXPECT_EQ(ret, 0);
+ EXPECT_EQ(ent->get_type(), ObjectType::VERSIONED);
+
+ ret = testdir->remove(env->dpp, null_yield, false);
+ EXPECT_EQ(ret, 0);
+ EXPECT_FALSE(sf::exists(tp));
+}
+
+TEST(FSEnt, VerDirReadWrite)
+{
+ std::string fname = get_test_name();
+ std::unique_ptr<VersionedDirectory> verdir{
+ std::make_unique<VersionedDirectory>(fname, root.get(), env->cct.get())};
+ std::string instance_id{verdir->get_new_instance()};
+ std::string vfname{"_%3A" + instance_id + "_" + fname};
+ sf::path tp{base_path / fname};
+ sf::path fp{tp / vfname};
+ sf::path lp{tp / fname};
+
+ int ret = verdir->create(env->dpp, /*existed=*/nullptr, /*temp_file=*/false);
+ EXPECT_EQ(ret, 0);
+
+ std::unique_ptr<File> testfile{std::make_unique<File>(vfname, verdir.get(), env->cct.get())};
+ ret = verdir->add_file(env->dpp, std::move(testfile), /*existed=*/nullptr, /*temp_file=*/true);
+ EXPECT_EQ(ret, 0);
+
+ std::string temp_fname{fname + "-blargh"};
+ ret = verdir->link_temp_file(env->dpp, null_yield, temp_fname);
+ EXPECT_TRUE(sf::exists(tp));
+ EXPECT_TRUE(sf::is_directory(tp));
+ EXPECT_TRUE(sf::exists(fp));
+ EXPECT_TRUE(sf::is_regular_file(fp));
+ EXPECT_TRUE(sf::exists(lp));
+ EXPECT_TRUE(sf::is_symlink(lp));
+ EXPECT_EQ(sf::read_symlink(lp), vfname);
+
+ bufferlist bl;
+ encode(fname, bl);
+ int len = bl.length();
+ ret = verdir->write(0, bl, env->dpp, null_yield);
+ EXPECT_EQ(ret, 0);
+ EXPECT_EQ(sf::file_size(fp), len);
+
+ bl.clear();
+ ret = verdir->read(0, 50, bl, env->dpp, null_yield);
+ EXPECT_EQ(ret, len);
+
+ std::string result;
+ EXPECT_NO_THROW({
+ auto bufit = bl.cbegin();
+ decode(result, bufit);
+ });
+
+ EXPECT_EQ(result, fname);
+
+ Attrs attrs;
+ add_attr(attrs, ATTR1, ATTR1);
+ add_attr(attrs, ATTR2, ATTR2);
+ Attrs extra_attrs;
+ add_attr(extra_attrs, ATTR3, ATTR3);
+
+ ret = verdir->write_attrs(env->dpp, null_yield, attrs, &extra_attrs);
+ EXPECT_EQ(ret, 0);
+
+ attrs.clear();
+ ret = verdir->read_attrs(env->dpp, null_yield, attrs);
+ EXPECT_EQ(ret, 0);
+ EXPECT_EQ(attrs.size(), 4);
+ std::string val;
+ bool success = decode_attr(attrs, ATTR1.c_str(), val);
+ EXPECT_TRUE(success);
+ EXPECT_EQ(val, ATTR1);
+ success = decode_attr(attrs, ATTR2.c_str(), val);
+ EXPECT_TRUE(success);
+ EXPECT_EQ(val, ATTR2);
+ success = decode_attr(attrs, ATTR3.c_str(), val);
+ EXPECT_TRUE(success);
+ EXPECT_EQ(val, ATTR3);
+ ObjectType type;
+ success = decode_attr(attrs, ATTR_OBJECT_TYPE.c_str(), type);
+ EXPECT_TRUE(success);
+ EXPECT_EQ(type.type, ObjectType::VERSIONED);
+
+ FSEnt* ent = verdir->get_cur_version_ent();
+ attrs.clear();
+ ret = ent->read_attrs(env->dpp, null_yield, attrs);
+ EXPECT_EQ(ret, 0);
+ EXPECT_EQ(attrs.size(), 4);
+ success = decode_attr(attrs, ATTR1.c_str(), val);
+ EXPECT_TRUE(success);
+ EXPECT_EQ(val, ATTR1);
+ success = decode_attr(attrs, ATTR2.c_str(), val);
+ EXPECT_TRUE(success);
+ EXPECT_EQ(val, ATTR2);
+ success = decode_attr(attrs, ATTR3.c_str(), val);
+ EXPECT_TRUE(success);
+ EXPECT_EQ(val, ATTR3);
+ success = decode_attr(attrs, ATTR_OBJECT_TYPE.c_str(), type);
+ EXPECT_TRUE(success);
+ EXPECT_EQ(type.type, ObjectType::FILE);
+}
+
+TEST(FSEnt, MPVerDirReadWrite)
+{
+ std::string testname = get_test_name();
+ std::unique_ptr<VersionedDirectory> verdir{
+ std::make_unique<VersionedDirectory>(testname, root.get(), env->cct.get())};
+ std::string instance_id{verdir->get_new_instance()};
+ std::string vfname{"_%3A" + instance_id + "_" + testname};
+ sf::path vp{base_path / testname};
+ sf::path mp{vp / vfname};
+ sf::path lp{vp / testname};
+
+ int ret = verdir->create(env->dpp, /*existed=*/nullptr, /*temp_file=*/false);
+ EXPECT_EQ(ret, 0);
+
+ std::unique_ptr<MPDirectory> mpdir{std::make_unique<MPDirectory>(vfname, verdir.get(), env->cct.get())};
+ ret = verdir->add_file(env->dpp, std::move(mpdir), /*existed=*/nullptr, /*temp_file=*/true);
+ EXPECT_EQ(ret, 0);
+
+ std::string temp_fname{testname + "-blargh"};
+ ret = verdir->link_temp_file(env->dpp, null_yield, temp_fname);
+ EXPECT_TRUE(sf::exists(vp));
+ EXPECT_TRUE(sf::is_directory(vp));
+ EXPECT_TRUE(sf::exists(mp));
+ EXPECT_TRUE(sf::is_directory(mp));
+ EXPECT_TRUE(sf::exists(lp));
+ EXPECT_TRUE(sf::is_symlink(lp));
+ EXPECT_EQ(sf::read_symlink(lp), vfname);
+
+ ret = verdir->open(env->dpp);
+ EXPECT_EQ(ret, 0);
+
+ MPDirectory* mpp = static_cast<MPDirectory*>(verdir->get_cur_version_ent());
+ EXPECT_NE(mpp, nullptr);
+
+ ret = mpp->open(env->dpp);
+ EXPECT_EQ(ret, 0);
+
+ bufferlist write_bl;
+ int total_len{0};
+ for (int part_num = 0; part_num < 4; ++part_num) {
+ std::unique_ptr<File> testfile = mpp->get_part_file(part_num);
+ sf::path pp{mp / testfile->get_name()};
+
+ ret = testfile->create(env->dpp, /*existed=*/nullptr, /*temp_file=*/false);
+ EXPECT_EQ(ret, 0);
+ EXPECT_TRUE(sf::exists(pp));
+ EXPECT_TRUE(sf::is_regular_file(pp));
+
+ bufferlist bl;
+ encode(testname, bl);
+ int len = bl.length();
+ total_len += len;
+ ret = testfile->write(0, bl, env->dpp, null_yield);
+ EXPECT_EQ(ret, 0);
+ EXPECT_EQ(sf::file_size(pp), len);
+ write_bl.claim_append(bl);
+ }
+
+ ret = verdir->stat(env->dpp, true);
+ EXPECT_EQ(ret, 0);
+
+ bufferlist total_bl;
+ std::string read_data;
+ int left = total_len;
+ int ofs{0};
+ while (left > 0) {
+ bufferlist bl;
+ ret = verdir->read(ofs, left, bl, env->dpp, null_yield);
+ EXPECT_GE(ret, 0);
+ if (ret == 0)
+ break;
+
+ std::string result;
+ EXPECT_NO_THROW({
+ auto bufit = bl.cbegin();
+ decode(result, bufit);
+ });
+ read_data += result;
+ total_bl.claim_append(bl);
+ left -= ret;
+ ofs += ret;
+ }
+
+ EXPECT_EQ(total_bl.length(), total_len);
+ EXPECT_EQ(total_bl, write_bl);
+
+ EXPECT_EQ(read_data, testname + testname + testname + testname);
+}
+
+class TestUser;
+class TestDriver : public POSIXDriver
+{
+public:
+ std::string driver_base;
+
+ TestDriver(std::string _base_path) : POSIXDriver(nullptr), driver_base(_base_path)
+ { }
+ virtual ~TestDriver() = default;
+
+ int init(const DoutPrefixProvider* dpp)
+ {
+ std::string cache_base = driver_base + "/cache";
+ base_path = driver_base + "/root";
+
+ root_dir = std::make_unique<Directory>(base_path, nullptr, env->cct.get());
+ int ret = root_dir->open(env->dpp);
+ if (ret < 0) {
+ if (ret == -ENOTDIR) {
+ ldpp_dout(env->dpp, 0) << " ERROR: base path (" << base_path
+ << "): was not a directory." << dendl;
+ return ret;
+ } else if (ret == -ENOENT) {
+ ret = root_dir->create(env->dpp);
+ if (ret < 0) {
+ ldpp_dout(env->dpp, 0)
+ << " ERROR: could not create base path (" << base_path
+ << "): " << cpp_strerror(-ret) << dendl;
+ return ret;
+ }
+ }
+ }
+
+ /* ordered listing cache */
+ bucket_cache.reset(new BucketCache(
+ this, base_path, cache_base, 100, 3, 3, 3));
+
+ ldpp_dout(env->dpp, 20) << "SUCCESS" << dendl;
+ return 0;
+ }
+ virtual CephContext* ctx(void) override {
+ return get_pointer(env->cct);
+ }
+
+ virtual std::unique_ptr<User> get_user(const rgw_user& u) override;
+};
+
+class TestUser : public StoreUser {
+ Attrs attrs;
+
+public:
+ TestUser(TestDriver *_dr, const rgw_user& _u) : StoreUser(_u) { }
+ TestUser(TestDriver *_dr, const RGWUserInfo& _i) : StoreUser(_i) { }
+ TestUser(TestDriver *_dr) { }
+ TestUser(TestUser& _o) = default;
+ virtual ~TestUser() = default;
+
+ virtual std::unique_ptr<User> clone() override {
+ return std::unique_ptr<User>(new TestUser(*this));
+ }
+ virtual Attrs& get_attrs() override { return attrs; }
+ virtual void set_attrs(Attrs &_attrs) override { attrs = _attrs; }
+ virtual int read_attrs(const DoutPrefixProvider* dpp, optional_yield y) override { return 0; }
+ virtual int merge_and_store_attrs(const DoutPrefixProvider* dpp, Attrs&
+ new_attrs, optional_yield y) override { return 0; }
+ virtual int read_usage(const DoutPrefixProvider* dpp, uint64_t start_epoch,
+ uint64_t end_epoch, uint32_t max_entries, bool* is_truncated,
+ RGWUsageIter &usage_iter,
+ std::map<rgw_user_bucket, rgw_usage_log_entry> &usage) override { return 0; }
+ virtual int trim_usage(const DoutPrefixProvider* dpp, uint64_t start_epoch,
+ uint64_t end_epoch, optional_yield y) override { return 0; }
+ virtual int load_user(const DoutPrefixProvider* dpp, optional_yield y) override { return 0; }
+ virtual int store_user(const DoutPrefixProvider* dpp, optional_yield y, bool
+ exclusive, RGWUserInfo* old_info = nullptr) override { return 0; }
+ virtual int remove_user(const DoutPrefixProvider* dpp, optional_yield y) override { return 0; }
+ virtual int verify_mfa(const std::string &mfa_str, bool *verified,
+ const DoutPrefixProvider* dpp,
+ optional_yield y) override { return 0; }
+ virtual int list_groups(const DoutPrefixProvider *dpp, optional_yield y,
+ std::string_view marker, uint32_t max_items,
+ GroupList &listing) override { return -ENOTSUP; }
+};
+
+std::unique_ptr<User> TestDriver::get_user(const rgw_user &u)
+{
+ return std::make_unique<TestUser>(this, u);
+}
+
+TEST(POSIXDriver, CreateDriver)
+{
+ std::string name = get_test_name();
+ sf::path bp{sf::absolute(sf::path{base_path / name})};
+ sf::create_directory(bp);
+ sf::create_directory(bp / "cache");
+ sf::create_directory(bp / "root");
+ TestDriver driver{bp};
+
+ sf::path tp{bp / "root"};
+
+ int ret = driver.init(env->dpp);
+ EXPECT_EQ(ret, 0);
+ EXPECT_TRUE(sf::exists(tp));
+ EXPECT_TRUE(sf::is_directory(tp));
+}
+
+class POSIXDriverTest : public ::testing::Test {
+ protected:
+ std::unique_ptr<TestDriver> driver;
+ rgw_owner owner;
+ ACLOwner acl_owner;
+ sf::path bp;
+ std::string testname;
+
+ public:
+ POSIXDriverTest() {}
+
+ void SetUp() {
+ testname = get_test_name();
+ bp = sf::path{sf::absolute(sf::path{base_path / testname})};
+ sf::create_directory(bp);
+ sf::create_directory(bp / "cache");
+ sf::create_directory(bp / "root");
+ driver = std::make_unique<TestDriver>(bp);
+ int ret = driver->init(env->dpp);
+ EXPECT_EQ(ret, 0);
+
+ rgw_user uid{"tenant", testname};
+ owner = uid;
+ acl_owner.id = owner;
+ EXPECT_EQ(ret, 0);
+ }
+
+ void TearDown() { sf::remove_all(bp); }
+};
+
+TEST_F(POSIXDriverTest, Bucket)
+{
+ RGWBucketInfo info;
+ info.bucket.name = testname;
+ info.owner = owner;
+ info.creation_time = ceph::real_clock::now();
+
+ std::unique_ptr<rgw::sal::Bucket> bucket = driver->get_bucket(info);
+ EXPECT_NE(bucket.get(), nullptr);
+ EXPECT_EQ(bucket->get_name(), testname);
+ EXPECT_EQ(bucket->get_key().name, testname);
+ EXPECT_EQ(bucket->get_key().tenant, "");
+ EXPECT_EQ(bucket->get_key().bucket_id, "");
+ EXPECT_FALSE(bucket->versioned());
+ EXPECT_FALSE(bucket->versioning_enabled());
+
+}
+
+TEST_F(POSIXDriverTest, BucketCreate)
+{
+ std::unique_ptr<rgw::sal::Bucket> bucket;
+ bool bucket_exists;
+ rgw::sal::Bucket::CreateParams createparams;
+
+ RGWBucketInfo info;
+ info.bucket.name = testname;
+ info.owner = owner;
+ info.creation_time = ceph::real_clock::now();
+ bucket = driver->get_bucket(info);
+ EXPECT_NE(bucket.get(), nullptr);
+
+ createparams.owner = owner;
+
+ int ret = bucket->create(env->dpp, createparams, null_yield);
+ EXPECT_EQ(ret, 0);
+ EXPECT_EQ(bucket->get_name(), testname);
+ EXPECT_EQ(bucket->get_key().name, testname);
+ EXPECT_EQ(bucket->get_key().tenant, "");
+ EXPECT_EQ(bucket->get_key().bucket_id, "");
+ EXPECT_FALSE(bucket_exists);
+
+ sf::path tp{bp / "root" / testname};
+ EXPECT_TRUE(sf::exists(tp));
+ EXPECT_TRUE(sf::is_directory(tp));
+}
+
+class POSIXBucketTest : public POSIXDriverTest {
+protected:
+ std::unique_ptr<rgw::sal::Bucket> bucket;
+
+public:
+ POSIXBucketTest() {}
+
+ void SetUp() {
+ POSIXDriverTest::SetUp();
+
+ RGWBucketInfo info;
+ info.bucket.name = testname;
+ info.owner = owner;
+ info.creation_time = ceph::real_clock::now();
+
+ bucket = driver->get_bucket(info);
+ EXPECT_NE(bucket.get(), nullptr);
+
+ rgw::sal::Bucket::CreateParams createparams;
+ createparams.owner = owner;
+ int ret = bucket->create(env->dpp, createparams, null_yield);
+ EXPECT_EQ(ret, 0);
+ }
+
+ void TearDown() {
+ POSIXDriverTest::TearDown();
+ }
+};
+
+TEST_F(POSIXBucketTest, Object)
+{
+ std::unique_ptr<rgw::sal::Object> object = bucket->get_object(rgw_obj_key(testname, "instance", "namespace"));
+ EXPECT_NE(object.get(), nullptr);
+ EXPECT_EQ(object->get_name(), testname);
+ EXPECT_EQ(object->get_key().name, testname);
+ EXPECT_EQ(object->get_bucket(), bucket.get());
+ EXPECT_EQ(object->get_oid(), "_namespace:instance_" + testname);
+ EXPECT_EQ(object->get_instance(), "instance");
+
+}
+
+TEST_F(POSIXBucketTest, ObjectWrite)
+{
+ sf::path tp{bp / "root" / testname / testname};
+ EXPECT_FALSE(sf::exists(tp));
+
+ std::unique_ptr<rgw::sal::Object> object = bucket->get_object(rgw_obj_key(testname));
+ EXPECT_NE(object.get(), nullptr);
+
+ std::unique_ptr<rgw::sal::Writer> writer = driver->get_atomic_writer(
+ env->dpp, null_yield, object.get(), acl_owner, nullptr, 0, testname);
+ EXPECT_NE(writer.get(), nullptr);
+
+ int ret = writer->prepare(null_yield);
+ EXPECT_EQ(ret, 0);
+
+ int ofs{0};
+ std::string etag;
+ for (int i = 0; i < 4; ++i) {
+ bufferlist bl;
+ encode(testname, bl);
+ int len = bl.length();
+
+ ret = writer->process(std::move(bl), ofs);
+ EXPECT_EQ(ret, 0);
+
+ ofs += len;
+ }
+
+ ret = writer->process({}, ofs);
+ EXPECT_EQ(ret, 0);
+
+ ceph::real_time mtime;
+ Attrs attrs;
+ bufferlist bl;
+ encode(ATTR1, bl);
+ attrs[ATTR1] = bl;
+ req_context rctx{env->dpp, null_yield, nullptr};
+
+ ret = writer->complete(ofs, etag, &mtime, real_time(), attrs, std::nullopt,
+ real_time(), nullptr, nullptr, nullptr, nullptr,
+ nullptr, rctx, 0);
+ EXPECT_EQ(ret, 0);
+ EXPECT_EQ(object->get_size(), ofs);
+
+ bufferlist getbl = object->get_attrs()[ATTR1];
+ EXPECT_EQ(bl, getbl);
+
+ EXPECT_TRUE(sf::exists(tp));
+ EXPECT_TRUE(sf::is_regular_file(tp));
+}
+
+class POSIXObjectTest : public POSIXBucketTest {
+protected:
+ std::unique_ptr<rgw::sal::Object> object;
+ uint64_t write_size{0};
+ bufferlist write_data;
+
+public:
+ POSIXObjectTest() {}
+
+ void SetUp() {
+ POSIXBucketTest::SetUp();
+ object = write_object(testname);
+ }
+
+ std::unique_ptr<rgw::sal::Object> write_object(std::string objname) {
+ std::unique_ptr<rgw::sal::Object> obj = bucket->get_object(rgw_obj_key(objname));
+ EXPECT_NE(obj.get(), nullptr);
+
+ std::unique_ptr<rgw::sal::Writer> writer = driver->get_atomic_writer(
+ env->dpp, null_yield, obj.get(), acl_owner, nullptr, 0, testname);
+ EXPECT_NE(writer.get(), nullptr);
+
+ int ret = writer->prepare(null_yield);
+ EXPECT_EQ(ret, 0);
+
+ std::string etag;
+ for (int i = 0; i < 4; ++i) {
+ bufferlist bl;
+ encode(objname, bl);
+ int len = bl.length();
+
+ write_data.append(bl);
+
+ ret = writer->process(std::move(bl), write_size);
+ EXPECT_EQ(ret, 0);
+
+ write_size += len;
+ }
+
+ ret = writer->process({}, write_size);
+ EXPECT_EQ(ret, 0);
+
+ ceph::real_time mtime;
+ Attrs attrs;
+ add_attr(attrs, ATTR1, ATTR1);
+ req_context rctx{env->dpp, null_yield, nullptr};
+ ret = writer->complete(write_size, etag, &mtime, real_time(), attrs,
+ std::nullopt, real_time(), nullptr, nullptr, nullptr,
+ nullptr, nullptr, rctx, 0);
+ EXPECT_EQ(ret, 0);
+
+ return obj;
+ }
+
+ void TearDown() { POSIXBucketTest::TearDown(); }
+};
+
+class Read_CB : public RGWGetDataCB
+{
+public:
+ bufferlist *save_bl;
+ explicit Read_CB(bufferlist *_bl) : save_bl(_bl) {}
+ ~Read_CB() override {}
+
+ int handle_data(bufferlist& bl, off_t bl_ofs, off_t bl_len) override {
+ save_bl->append(bl);
+ return 0;
+ }
+};
+
+TEST_F(POSIXObjectTest, BucketList)
+{
+ std::unique_ptr<rgw::sal::Object> obj1 = write_object(testname + "-1");
+ EXPECT_NE(obj1.get(), nullptr);
+ std::unique_ptr<rgw::sal::Object> obj2 = write_object(testname + "-2");
+ EXPECT_NE(obj2.get(), nullptr);
+ std::unique_ptr<rgw::sal::Object> obj3 = write_object(testname + "-3");
+ EXPECT_NE(obj3.get(), nullptr);
+
+ rgw::sal::Bucket::ListParams params;
+ rgw::sal::Bucket::ListResults results;
+
+ int ret = bucket->list(env->dpp, params, 128, results, null_yield);
+ EXPECT_EQ(ret, 0);
+
+ EXPECT_EQ(results.is_truncated, false);
+
+ EXPECT_EQ(results.objs.size(), 4);
+
+ rgw_obj_key key(results.objs[0].key);
+ EXPECT_EQ(key, object->get_key());
+ rgw_obj_key key1(results.objs[1].key);
+ EXPECT_EQ(key1, obj1->get_key());
+ rgw_obj_key key2(results.objs[2].key);
+ EXPECT_EQ(key2, obj2->get_key());
+ rgw_obj_key key3(results.objs[3].key);
+ EXPECT_EQ(key3, obj3->get_key());
+}
+
+TEST_F(POSIXObjectTest, BucketListV)
+{
+ std::unique_ptr<rgw::sal::Object> obj1 = write_object(testname + "-1");
+ EXPECT_NE(obj1.get(), nullptr);
+ std::unique_ptr<rgw::sal::Object> obj2 = write_object(testname + "-2");
+ EXPECT_NE(obj2.get(), nullptr);
+ std::unique_ptr<rgw::sal::Object> obj3 = write_object(testname + "-3");
+ EXPECT_NE(obj3.get(), nullptr);
+
+ rgw::sal::Bucket::ListParams params;
+ params.list_versions = true;
+ rgw::sal::Bucket::ListResults results;
+
+ int ret = bucket->list(env->dpp, params, 128, results, null_yield);
+ EXPECT_EQ(ret, 0);
+
+ EXPECT_EQ(results.is_truncated, false);
+
+ EXPECT_EQ(results.objs.size(), 4);
+
+ rgw_obj_key key(results.objs[0].key);
+ EXPECT_EQ(key, object->get_key());
+ rgw_obj_key key1(results.objs[1].key);
+ EXPECT_EQ(key1, obj1->get_key());
+ rgw_obj_key key2(results.objs[2].key);
+ EXPECT_EQ(key2, obj2->get_key());
+ rgw_obj_key key3(results.objs[3].key);
+ EXPECT_EQ(key3, obj3->get_key());
+}
+
+TEST_F(POSIXObjectTest, ObjectRead)
+{
+ std::unique_ptr<rgw::sal::Object::ReadOp> read_op(object->get_read_op());
+
+ int ret = read_op->prepare(null_yield, env->dpp);
+ EXPECT_EQ(ret, 0);
+
+ EXPECT_EQ(object->get_size(), write_size);
+
+ bufferlist bl;
+ Read_CB cb(&bl);
+ ret = read_op->iterate(env->dpp, 0, write_size, &cb, null_yield);
+ EXPECT_EQ(ret, 0);
+ EXPECT_EQ(write_data, bl);
+}
+
+TEST_F(POSIXObjectTest, ObjectDelete)
+{
+ sf::path tp{bp / "root" / testname / testname};
+ EXPECT_TRUE(sf::exists(tp));
+
+ std::unique_ptr<rgw::sal::Object::DeleteOp> del_op = object->get_delete_op();
+ int ret = del_op->delete_obj(env->dpp, null_yield, 0);
+ EXPECT_EQ(ret, 0);
+
+ EXPECT_FALSE(sf::exists(tp));
+}
+
+TEST_F(POSIXObjectTest, ObjectCopy)
+{
+ sf::path sp{bp / "root" / testname / testname};
+ EXPECT_TRUE(sf::exists(sp));
+
+ std::string dstname{testname + "-dst"};
+ sf::path dp{bp / "root" / testname / dstname};
+
+ std::unique_ptr<rgw::sal::Object> dstobj = bucket->get_object(rgw_obj_key(dstname));
+ EXPECT_NE(dstobj.get(), nullptr);
+ EXPECT_FALSE(sf::exists(dp));
+
+ RGWEnv rgw_env;
+ req_info info(env->cct.get(), &rgw_env);
+ rgw_zone_id zone;
+ rgw_placement_rule placement;
+ ceph::real_time mtime;
+ Attrs attrs;
+ std::string tag;
+
+ int ret = object->copy_object(acl_owner,
+ std::get<rgw_user>(owner),
+ &info,
+ zone,
+ dstobj.get(),
+ bucket.get(),
+ bucket.get(),
+ placement,
+ &mtime,
+ &mtime,
+ &mtime,
+ &mtime,
+ false,
+ nullptr,
+ nullptr,
+ ATTRSMOD_NONE,
+ false,
+ attrs,
+ RGWObjCategory::Main,
+ 0,
+ boost::none,
+ nullptr,
+ &tag, /* use req_id as tag */
+ &tag,
+ nullptr,
+ nullptr,
+ env->dpp,
+ null_yield);
+ EXPECT_EQ(ret, 0);
+
+ EXPECT_TRUE(sf::exists(sp));
+ EXPECT_TRUE(sf::exists(dp));
+}
+
+TEST_F(POSIXObjectTest, ObjectAttrs)
+{
+ int ret = object->get_obj_attrs(null_yield, env->dpp);
+ EXPECT_EQ(ret, 0);
+
+ bufferlist origbl;
+ encode(ATTR1, origbl);
+
+ // POSIXDriver adds attributes ("POSIX-Owner", and "POSIX-Object-Type")
+ EXPECT_EQ(object->get_attrs().size(), 3);
+ EXPECT_EQ(object->get_attrs()[ATTR1], origbl);
+ EXPECT_TRUE(object->get_attrs().contains("POSIX-Owner"));
+ EXPECT_TRUE(object->get_attrs().contains(ATTR_OBJECT_TYPE));
+
+ std::string addattr{"AddAttrO"};
+ bufferlist addbl;
+ encode(addattr, addbl);
+
+ ret = object->modify_obj_attrs(addattr.c_str(), addbl, null_yield, env->dpp);
+ EXPECT_EQ(ret, 0);
+
+ EXPECT_EQ(object->get_attrs().size(), 4);
+ EXPECT_EQ(object->get_attrs()[ATTR1], origbl);
+ EXPECT_EQ(object->get_attrs()[addattr], addbl);
+ EXPECT_TRUE(object->get_attrs().contains("POSIX-Owner"));
+ EXPECT_TRUE(object->get_attrs().contains(ATTR_OBJECT_TYPE));
+
+ ret = object->delete_obj_attrs(env->dpp, ATTR1.c_str(), null_yield);
+ EXPECT_EQ(ret, 0);
+ EXPECT_EQ(object->get_attrs().size(), 3);
+ EXPECT_EQ(object->get_attrs()[addattr], addbl);
+ EXPECT_TRUE(object->get_attrs().contains("POSIX-Owner"));
+ EXPECT_TRUE(object->get_attrs().contains(ATTR_OBJECT_TYPE));
+}
+
+TEST_F(POSIXBucketTest, MultipartUpload)
+{
+ std::string upload_id = "c0ffee";
+ std::unique_ptr<rgw::sal::MultipartUpload> upload = bucket->get_multipart_upload(testname, upload_id);
+
+ EXPECT_NE(upload.get(), nullptr);
+ EXPECT_EQ(upload->get_meta(), testname + "." + upload_id);
+ EXPECT_EQ(upload->get_key(), testname);
+ EXPECT_EQ(upload->get_upload_id(), upload_id);
+}
+
+TEST_F(POSIXBucketTest, MPUploadCreate)
+{
+ std::string upload_id = "c0ffee";
+ std::string mpname{".multipart_" + testname + "." + upload_id};
+ sf::path tp{bp / "root" / testname / mpname};
+ EXPECT_FALSE(sf::exists(tp));
+
+ std::unique_ptr<rgw::sal::MultipartUpload> upload = bucket->get_multipart_upload(testname, upload_id);
+ EXPECT_NE(upload.get(), nullptr);
+
+ rgw_placement_rule placement;
+ Attrs attrs;
+ add_attr(attrs, ATTR1, ATTR1);
+ int ret = upload->init(env->dpp, null_yield, acl_owner, placement, attrs);
+ EXPECT_EQ(ret, 0);
+
+ EXPECT_TRUE(sf::exists(tp));
+ EXPECT_TRUE(sf::is_directory(tp));
+}
+
+class POSIXMPObjectTest : public POSIXBucketTest {
+protected:
+ std::unique_ptr<rgw::sal::MultipartUpload> def_upload;
+ std::string mpname;
+ bufferlist write_data;
+
+public:
+ POSIXMPObjectTest() {}
+
+ void SetUp() {
+ def_upload = get_upload("c0ffee");
+ }
+
+ std::unique_ptr<rgw::sal::MultipartUpload> get_upload(std::string upload_id) {
+ POSIXBucketTest::SetUp();
+ mpname = ".multipart_" + testname + "." + upload_id;
+
+ std::unique_ptr<rgw::sal::MultipartUpload> upload =
+ bucket->get_multipart_upload(testname, upload_id);
+ EXPECT_NE(upload.get(), nullptr);
+
+ rgw_placement_rule placement;
+ Attrs attrs;
+ add_attr(attrs, ATTR1, ATTR1);
+ int ret = upload->init(env->dpp, null_yield, acl_owner, placement, attrs);
+ EXPECT_EQ(ret, 0);
+
+ return upload;
+ }
+
+ void TearDown() {
+ POSIXBucketTest::TearDown();
+ }
+
+ int write_part(rgw::sal::MultipartUpload* upload, int part_num) {
+ std::unique_ptr<rgw::sal::Writer> writer;
+ rgw_placement_rule placement;
+ std::string part_name = "part-" + fmt::format("{:0>5}", part_num);
+ ACLOwner owner{bucket->get_owner()};
+
+ writer = upload->get_writer(env->dpp, null_yield, nullptr, owner,
+ &placement, part_num, part_name);
+ EXPECT_NE(writer.get(), nullptr);
+
+ int ret = writer->prepare(null_yield);
+ EXPECT_EQ(ret, 0);
+
+ int ofs{0};
+ for (int i = 0; i < 4; ++i) {
+ bufferlist bl;
+ encode(testname + part_name, bl);
+ int len = bl.length();
+
+ write_data.append(bl);
+
+ ret = writer->process(std::move(bl), ofs);
+ EXPECT_EQ(ret, 0);
+
+ ofs += len;
+ }
+
+ ret = writer->process({}, ofs);
+ EXPECT_EQ(ret, 0);
+
+ ceph::real_time mtime;
+ Attrs attrs;
+ bufferlist bl;
+ encode(ATTR1, bl);
+ attrs[ATTR1] = bl;
+ req_context rctx{env->dpp, null_yield, nullptr};
+
+ ret = writer->complete(ofs, part_name, &mtime, real_time(), attrs,
+ std::nullopt, real_time(), nullptr, nullptr, nullptr,
+ nullptr, nullptr, rctx, 0);
+ EXPECT_EQ(ret, 0);
+
+ return ofs;
+ }
+ uint64_t create_MPObj(rgw::sal::MultipartUpload* upload, std::string name) {
+ std::map<int, std::string> parts;
+ int part_count{4};
+ uint64_t write_size{0};
+
+ for (int i = 1; i <= part_count; ++i) {
+ write_size += write_part(upload, i);
+ parts[i] = "part-" + fmt::format("{:0>5}", i);
+ }
+
+ std::list<rgw_obj_index_key> remove_objs;
+ bool compressed = false;
+ RGWCompressionInfo cs_info;
+ std::unique_ptr<Object> mp_obj = bucket->get_object(rgw_obj_key(name));
+ off_t ofs{0};
+ uint64_t accounted_size{0};
+ std::string tag;
+ ACLOwner owner;
+ owner.id = bucket->get_owner();
+
+ int ret = upload->complete(env->dpp, null_yield, get_pointer(env->cct), parts,
+ remove_objs, accounted_size, compressed, cs_info,
+ ofs, tag, owner, 0, mp_obj.get());
+ EXPECT_EQ(ret, 0);
+ EXPECT_EQ(write_size, ofs);
+ EXPECT_EQ(write_size, accounted_size);
+
+ for (int i = 1; i <= part_count; ++i) {
+ std::string part_name = "part-" + fmt::format("{:0>5}", i);
+ sf::path tp{bp / "root" / testname / name / part_name};
+ EXPECT_TRUE(sf::exists(tp));
+ EXPECT_TRUE(sf::is_regular_file(tp));
+ }
+
+ return write_size;
+ }
+};
+
+TEST_F(POSIXMPObjectTest, MPUploadWrite)
+{
+ std::unique_ptr<rgw::sal::Writer> writer;
+ rgw_placement_rule placement;
+
+ writer = def_upload->get_writer(env->dpp, null_yield, nullptr, acl_owner,
+ &placement, 1, "00001");
+ EXPECT_NE(writer.get(), nullptr);
+
+ int ret = writer->prepare(null_yield);
+ EXPECT_EQ(ret, 0);
+
+ int ofs{0};
+ std::string etag{"part-00001"};
+ for (int i = 0; i < 4; ++i) {
+ bufferlist bl;
+ encode(testname, bl);
+ int len = bl.length();
+
+ ret = writer->process(std::move(bl), ofs);
+ EXPECT_EQ(ret, 0);
+
+ ofs += len;
+ }
+
+ ret = writer->process({}, ofs);
+ EXPECT_EQ(ret, 0);
+
+ ceph::real_time mtime;
+ Attrs attrs;
+ bufferlist bl;
+ encode(ATTR1, bl);
+ attrs[ATTR1] = bl;
+ req_context rctx{env->dpp, null_yield, nullptr};
+
+ ret = writer->complete(ofs, etag, &mtime, real_time(), attrs, std::nullopt,
+ real_time(), nullptr, nullptr, nullptr, nullptr,
+ nullptr, rctx, 0);
+ EXPECT_EQ(ret, 0);
+
+ sf::path tp{bp / "root" / testname / mpname / "part-00001" };
+ EXPECT_TRUE(sf::exists(tp));
+ EXPECT_TRUE(sf::is_regular_file(tp));
+}
+
+TEST_F(POSIXMPObjectTest, MPUploadComplete)
+{
+ create_MPObj(def_upload.get(), testname);
+}
+
+TEST_F(POSIXMPObjectTest, MPUploadRead)
+{
+ uint64_t write_size = create_MPObj(def_upload.get(), testname);
+
+ std::unique_ptr<Object> mp_obj = bucket->get_object(rgw_obj_key(testname));
+ std::unique_ptr<rgw::sal::Object::ReadOp> read_op(mp_obj->get_read_op());
+
+ int ret = read_op->prepare(null_yield, env->dpp);
+ EXPECT_EQ(ret, 0);
+
+ EXPECT_EQ(mp_obj->get_size(), write_size);
+
+ bufferlist bl;
+ Read_CB cb(&bl);
+ ret = read_op->iterate(env->dpp, 0, write_size, &cb, null_yield);
+ EXPECT_EQ(ret, 0);
+ EXPECT_EQ(write_data, bl);
+}
+
+TEST_F(POSIXMPObjectTest, MPUploadCopy)
+{
+ create_MPObj(def_upload.get(), testname);
+
+ sf::path sp{bp / "root" / testname / testname};
+ std::string dstname{testname + "-dst"};
+ sf::path dp{bp / "root" / testname / dstname};
+
+ std::unique_ptr<Object> object = bucket->get_object(rgw_obj_key(testname));
+ EXPECT_NE(object.get(), nullptr);
+ std::unique_ptr<rgw::sal::Object> dstobj = bucket->get_object(rgw_obj_key(dstname));
+ EXPECT_NE(dstobj.get(), nullptr);
+ EXPECT_FALSE(sf::exists(dp));
+
+ RGWEnv rgw_env;
+ req_info info(env->cct.get(), &rgw_env);
+ rgw_zone_id zone;
+ rgw_placement_rule placement;
+ ceph::real_time mtime;
+ Attrs attrs;
+ std::string tag;
+
+ int ret = object->copy_object(acl_owner,
+ std::get<rgw_user>(owner),
+ &info,
+ zone,
+ dstobj.get(),
+ bucket.get(),
+ bucket.get(),
+ placement,
+ &mtime,
+ &mtime,
+ &mtime,
+ &mtime,
+ false,
+ nullptr,
+ nullptr,
+ ATTRSMOD_NONE,
+ false,
+ attrs,
+ RGWObjCategory::Main,
+ 0,
+ boost::none,
+ nullptr,
+ &tag, /* use req_id as tag */
+ &tag,
+ nullptr,
+ nullptr,
+ env->dpp,
+ null_yield);
+ EXPECT_EQ(ret, 0);
+
+ EXPECT_TRUE(sf::exists(sp));
+ EXPECT_TRUE(sf::exists(dp));
+}
+
+TEST_F(POSIXMPObjectTest, BucketList)
+{
+ std::unique_ptr<rgw::sal::MultipartUpload> up1 = get_upload("c0ffee-1");
+ create_MPObj(up1.get(), testname + "-1");
+ std::unique_ptr<Object> obj1 = bucket->get_object(rgw_obj_key(testname + "-1"));
+ EXPECT_NE(obj1.get(), nullptr);
+ std::unique_ptr<rgw::sal::MultipartUpload> up2 = get_upload("c0ffee-2");
+ create_MPObj(up2.get(), testname + "-2");
+ std::unique_ptr<Object> obj2 = bucket->get_object(rgw_obj_key(testname + "-2"));
+ EXPECT_NE(obj2.get(), nullptr);
+ std::unique_ptr<rgw::sal::MultipartUpload> up3 = get_upload("c0ffee-3");
+ create_MPObj(up3.get(), testname + "-3");
+ std::unique_ptr<Object> obj3 = bucket->get_object(rgw_obj_key(testname + "-3"));
+ EXPECT_NE(obj3.get(), nullptr);
+
+ rgw::sal::Bucket::ListParams params;
+ params.list_versions = true;
+ rgw::sal::Bucket::ListResults results;
+
+ int ret = bucket->list(env->dpp, params, 128, results, null_yield);
+ EXPECT_EQ(ret, 0);
+
+ EXPECT_EQ(results.is_truncated, false);
+
+ EXPECT_EQ(results.objs.size(), 3);
+
+ for (auto ent : results.objs) {
+ printf("%s\n", ent.key.to_string().c_str());
+ }
+
+ rgw_obj_key key1(results.objs[0].key);
+ EXPECT_EQ(key1, obj1->get_key());
+ rgw_obj_key key2(results.objs[1].key);
+ EXPECT_EQ(key2, obj2->get_key());
+ rgw_obj_key key3(results.objs[2].key);
+ EXPECT_EQ(key3, obj3->get_key());
+}
+
+TEST_F(POSIXBucketTest, VersionedObjectWrite)
+{
+ bucket->get_info().flags |= BUCKET_VERSIONED;
+ sf::path tp{bp / "root" / testname / testname};
+ EXPECT_FALSE(sf::exists(tp));
+
+ std::unique_ptr<rgw::sal::Object> object = bucket->get_object(rgw_obj_key(testname));
+ EXPECT_NE(object.get(), nullptr);
+
+ object->gen_rand_obj_instance_name();
+ std::string inst_id = object->get_instance();
+
+ std::unique_ptr<rgw::sal::Writer> writer = driver->get_atomic_writer(
+ env->dpp, null_yield, object.get(), acl_owner, nullptr, 0, testname);
+ EXPECT_NE(writer.get(), nullptr);
+
+ int ret = writer->prepare(null_yield);
+ EXPECT_EQ(ret, 0);
+
+ int ofs{0};
+ std::string etag;
+ for (int i = 0; i < 4; ++i) {
+ bufferlist bl;
+ encode(testname, bl);
+ int len = bl.length();
+
+ ret = writer->process(std::move(bl), ofs);
+ EXPECT_EQ(ret, 0);
+
+ ofs += len;
+ }
+
+ ret = writer->process({}, ofs);
+ EXPECT_EQ(ret, 0);
+
+ ceph::real_time mtime;
+ Attrs attrs;
+ bufferlist bl;
+ encode(ATTR1, bl);
+ attrs[ATTR1] = bl;
+ req_context rctx{env->dpp, null_yield, nullptr};
+
+ ret = writer->complete(ofs, etag, &mtime, real_time(), attrs, std::nullopt,
+ real_time(), nullptr, nullptr, nullptr, nullptr,
+ nullptr, rctx, 0);
+ EXPECT_EQ(ret, 0);
+ EXPECT_EQ(object->get_size(), ofs);
+
+ bufferlist getbl = object->get_attrs()[ATTR1];
+ EXPECT_EQ(bl, getbl);
+
+ EXPECT_TRUE(sf::exists(tp));
+ EXPECT_TRUE(sf::is_directory(tp));
+
+ std::string vfname{"_%3A" + inst_id + "_" + testname};
+ sf::path op{tp / vfname};
+ EXPECT_TRUE(sf::exists(op));
+ EXPECT_TRUE(sf::is_regular_file(op));
+
+ sf::path curver{tp / testname};
+ EXPECT_TRUE(sf::exists(curver));
+ EXPECT_TRUE(sf::is_symlink(curver));
+ EXPECT_EQ(sf::read_symlink(curver), vfname);
+
+ std::unique_ptr<rgw::sal::Object> obj2 = bucket->get_object(rgw_obj_key(testname));
+ EXPECT_NE(obj2.get(), nullptr);
+
+ obj2->gen_rand_obj_instance_name();
+ inst_id = obj2->get_instance();
+
+ writer.reset();
+ writer = driver->get_atomic_writer(env->dpp, null_yield, obj2.get(), acl_owner, nullptr,
+ 0, testname);
+ EXPECT_NE(writer.get(), nullptr);
+
+ ret = writer->prepare(null_yield);
+ EXPECT_EQ(ret, 0);
+
+ std::string obj2_content{testname + "ver2"};
+ ofs = 0;
+ for (int i = 0; i < 4; ++i) {
+ bufferlist bl;
+ encode(obj2_content, bl);
+ int len = bl.length();
+
+ ret = writer->process(std::move(bl), ofs);
+ EXPECT_EQ(ret, 0);
+
+ ofs += len;
+ }
+
+ ret = writer->process({}, ofs);
+ EXPECT_EQ(ret, 0);
+
+ bl.clear();
+ encode(ATTR1, bl);
+ attrs[ATTR1] = bl;
+
+ ret = writer->complete(ofs, etag, &mtime, real_time(), attrs, std::nullopt,
+ real_time(), nullptr, nullptr, nullptr, nullptr,
+ nullptr, rctx, 0);
+ EXPECT_EQ(ret, 0);
+ EXPECT_EQ(obj2->get_size(), ofs);
+
+ getbl = obj2->get_attrs()[ATTR1];
+ EXPECT_EQ(bl, getbl);
+
+ EXPECT_TRUE(sf::exists(tp));
+ EXPECT_TRUE(sf::is_directory(tp));
+
+ vfname = "_%3A" + inst_id + "_" + testname;
+ sf::path o2p{tp / vfname};
+ EXPECT_TRUE(sf::exists(o2p));
+ EXPECT_TRUE(sf::is_regular_file(o2p));
+
+ EXPECT_TRUE(sf::exists(curver));
+ EXPECT_TRUE(sf::is_symlink(curver));
+ EXPECT_EQ(sf::read_symlink(curver), vfname);
+
+ std::unique_ptr<rgw::sal::Object> robj = bucket->get_object(rgw_obj_key(testname));
+ EXPECT_NE(robj.get(), nullptr);
+
+ std::unique_ptr<rgw::sal::Object::ReadOp> read_op(robj->get_read_op());
+ ret = read_op->prepare(null_yield, env->dpp);
+ EXPECT_EQ(ret, 0);
+ EXPECT_EQ(robj->get_key().instance, inst_id);
+}
+
+class POSIXVerObjectTest : public POSIXBucketTest {
+protected:
+ uint64_t write_size{0};
+ bufferlist write_data;
+
+public:
+ POSIXVerObjectTest() {}
+
+ void SetUp() {
+ POSIXBucketTest::SetUp();
+
+ bucket->get_info().flags |= BUCKET_VERSIONED;
+ }
+
+ std::unique_ptr<rgw::sal::Object> write_version(std::string objname) {
+ std::unique_ptr<rgw::sal::Object> object = bucket->get_object(rgw_obj_key(objname));
+ EXPECT_NE(object.get(), nullptr);
+ object->gen_rand_obj_instance_name();
+
+ std::unique_ptr<rgw::sal::Writer> writer = driver->get_atomic_writer(
+ env->dpp, null_yield, object.get(), acl_owner, nullptr, 0, testname);
+ EXPECT_NE(writer.get(), nullptr);
+
+ int ret = writer->prepare(null_yield);
+ EXPECT_EQ(ret, 0);
+
+ std::string etag;
+ for (int i = 0; i < 4; ++i) {
+ bufferlist bl;
+ encode(objname, bl);
+ int len = bl.length();
+
+ write_data.claim_append(bl);
+
+ ret = writer->process(std::move(bl), write_size);
+ EXPECT_EQ(ret, 0);
+
+ write_size += len;
+ }
+
+ ret = writer->process({}, write_size);
+ EXPECT_EQ(ret, 0);
+
+ ceph::real_time mtime;
+ Attrs attrs;
+ add_attr(attrs, ATTR1, ATTR1);
+ req_context rctx{env->dpp, null_yield, nullptr};
+ ret = writer->complete(write_size, etag, &mtime, real_time(), attrs,
+ std::nullopt, real_time(), nullptr, nullptr, nullptr,
+ nullptr, nullptr, rctx, 0);
+ EXPECT_EQ(ret, 0);
+
+ return object;
+ }
+
+ void TearDown() { POSIXBucketTest::TearDown(); }
+};
+
+TEST_F(POSIXVerObjectTest, url_encode)
+{
+ std::string objname = testname + "&foo";
+ std::string encname = url_encode(objname, true);
+ sf::path sp{bp / "root" / testname / encname};
+ std::unique_ptr<rgw::sal::Object> obj1v1 = write_version(objname);
+ EXPECT_NE(obj1v1.get(), nullptr);
+ std::string obj1v1_inst = obj1v1->get_instance();
+
+ EXPECT_TRUE(sf::exists(sp));
+ EXPECT_TRUE(sf::is_directory(sp));
+
+ std::string vfname1{"_%3A" + obj1v1_inst + "_" + encname};
+ sf::path op1{sp / vfname1};
+ EXPECT_TRUE(sf::exists(op1));
+ EXPECT_TRUE(sf::is_regular_file(op1));
+}
+
+
+TEST_F(POSIXVerObjectTest, BucketListV)
+{
+ std::unique_ptr<rgw::sal::Object> obj1v1 = write_version(testname + "-1");
+ EXPECT_NE(obj1v1.get(), nullptr);
+ std::unique_ptr<rgw::sal::Object> obj1v2 = write_version(testname + "-1");
+ EXPECT_NE(obj1v2.get(), nullptr);
+ std::unique_ptr<rgw::sal::Object> obj2v1 = write_version(testname + "-2");
+ EXPECT_NE(obj2v1.get(), nullptr);
+ std::unique_ptr<rgw::sal::Object> obj2v2 = write_version(testname + "-2");
+ EXPECT_NE(obj2v2.get(), nullptr);
+
+ rgw::sal::Bucket::ListParams params;
+ params.list_versions = true;
+ rgw::sal::Bucket::ListResults results;
+
+ int ret = bucket->list(env->dpp, params, 128, results, null_yield);
+ EXPECT_EQ(ret, 0);
+
+ EXPECT_EQ(results.is_truncated, false);
+
+ EXPECT_EQ(results.objs.size(), 4);
+
+ for (auto ent : results.objs) {
+ printf("%s\n", ent.key.to_string().c_str());
+ }
+
+ rgw_obj_key key1(results.objs[0].key);
+ EXPECT_EQ(key1, obj1v2->get_key());
+ rgw_obj_key key2(results.objs[1].key);
+ EXPECT_EQ(key2, obj1v1->get_key());
+
+ rgw_obj_key key4(results.objs[2].key);
+ EXPECT_EQ(key4, obj2v2->get_key());
+ rgw_obj_key key5(results.objs[3].key);
+ EXPECT_EQ(key5, obj2v1->get_key());
+}
+
+
+TEST_F(POSIXVerObjectTest, DeleteCurVersion)
+{
+ std::string srcname{testname + "-1"};
+ std::unique_ptr<rgw::sal::Object> obj1v1 = write_version(srcname);
+ EXPECT_NE(obj1v1.get(), nullptr);
+ std::string obj1v1_inst = obj1v1->get_instance();
+ std::unique_ptr<rgw::sal::Object> obj1v2 = write_version(srcname);
+ EXPECT_NE(obj1v2.get(), nullptr);
+ std::string obj1v2_inst = obj1v2->get_instance();
+ std::unique_ptr<rgw::sal::Object> obj1v3 = write_version(srcname);
+ EXPECT_NE(obj1v3.get(), nullptr);
+ std::string obj1v3_inst = obj1v3->get_instance();
+ sf::path sp{bp / "root" / testname / srcname};
+ std::unique_ptr<rgw::sal::Object> delobj = bucket->get_object(rgw_obj_key(srcname));
+ EXPECT_NE(delobj.get(), nullptr);
+ EXPECT_TRUE(sf::exists(sp));
+ sf::path ops{sp / srcname};
+
+ std::string vfname1{"_%3A" + obj1v1_inst + "_" + srcname};
+ sf::path op1{sp / vfname1};
+ EXPECT_TRUE(sf::exists(op1));
+ EXPECT_TRUE(sf::is_regular_file(op1));
+ std::string vfname2{"_%3A" + obj1v2_inst + "_" + srcname};
+ sf::path op2{sp / vfname2};
+ EXPECT_TRUE(sf::exists(op2));
+ EXPECT_TRUE(sf::is_regular_file(op2));
+ std::string vfname3{"_%3A" + obj1v3_inst + "_" + srcname};
+ sf::path op3{sp / vfname3};
+ EXPECT_TRUE(sf::exists(op3));
+ EXPECT_TRUE(sf::is_regular_file(op3));
+ EXPECT_TRUE(sf::exists(ops));
+ EXPECT_TRUE(sf::is_symlink(ops));
+ EXPECT_EQ(sf::read_symlink(ops), vfname3);
+
+
+ std::unique_ptr<rgw::sal::Object::DeleteOp> del_op = delobj->get_delete_op();
+ int ret = del_op->delete_obj(env->dpp, null_yield, 0);
+ EXPECT_EQ(ret, 0);
+ std::string delobj_inst = delobj->get_instance();
+ std::string dfname{"_%3A" + delobj_inst + "_" + srcname};
+
+ EXPECT_TRUE(sf::exists(op1));
+ EXPECT_TRUE(sf::is_regular_file(op1));
+ EXPECT_TRUE(sf::exists(op2));
+ EXPECT_TRUE(sf::is_regular_file(op2));
+ EXPECT_TRUE(sf::exists(op3));
+ EXPECT_TRUE(sf::is_regular_file(op3));
+ EXPECT_FALSE(sf::exists(ops));
+ EXPECT_TRUE(sf::is_symlink(ops));
+ /* Need to find a way to get the correct version */
+ //EXPECT_EQ(sf::read_symlink(ops), dfname);
+
+}
+
+TEST_F(POSIXVerObjectTest, DeleteOldVersion)
+{
+ std::string srcname{testname + "-1"};
+ std::unique_ptr<rgw::sal::Object> obj1v1 = write_version(srcname);
+ EXPECT_NE(obj1v1.get(), nullptr);
+ std::string obj1v1_inst = obj1v1->get_instance();
+ std::unique_ptr<rgw::sal::Object> obj1v2 = write_version(srcname);
+ EXPECT_NE(obj1v2.get(), nullptr);
+ std::string obj1v2_inst = obj1v2->get_instance();
+ std::unique_ptr<rgw::sal::Object> obj1v3 = write_version(srcname);
+ EXPECT_NE(obj1v3.get(), nullptr);
+ std::string obj1v3_inst = obj1v3->get_instance();
+ sf::path sp{bp / "root" / testname / srcname};
+ std::unique_ptr<rgw::sal::Object> delobj = bucket->get_object(rgw_obj_key(srcname, obj1v2_inst));
+ EXPECT_NE(delobj.get(), nullptr);
+ EXPECT_TRUE(sf::exists(sp));
+ sf::path ops{sp / srcname};
+
+ std::string vfname1{"_%3A" + obj1v1_inst + "_" + srcname};
+ sf::path op1{sp / vfname1};
+ EXPECT_TRUE(sf::exists(op1));
+ EXPECT_TRUE(sf::is_regular_file(op1));
+ std::string vfname2{"_%3A" + obj1v2_inst + "_" + srcname};
+ sf::path op2{sp / vfname2};
+ EXPECT_TRUE(sf::exists(op2));
+ EXPECT_TRUE(sf::is_regular_file(op2));
+ std::string vfname3{"_%3A" + obj1v3_inst + "_" + srcname};
+ sf::path op3{sp / vfname3};
+ EXPECT_TRUE(sf::exists(op3));
+ EXPECT_TRUE(sf::is_regular_file(op3));
+ EXPECT_TRUE(sf::exists(ops));
+ EXPECT_TRUE(sf::is_symlink(ops));
+ EXPECT_EQ(sf::read_symlink(ops), vfname3);
+
+
+ std::unique_ptr<rgw::sal::Object::DeleteOp> del_op = delobj->get_delete_op();
+ int ret = del_op->delete_obj(env->dpp, null_yield, 0);
+ EXPECT_EQ(ret, 0);
+
+ EXPECT_TRUE(sf::exists(op1));
+ EXPECT_TRUE(sf::is_regular_file(op1));
+ EXPECT_FALSE(sf::exists(op2));
+ EXPECT_TRUE(sf::exists(op3));
+ EXPECT_TRUE(sf::is_regular_file(op3));
+ EXPECT_TRUE(sf::exists(ops));
+ EXPECT_TRUE(sf::is_symlink(ops));
+ EXPECT_EQ(sf::read_symlink(ops), vfname3);
+
+}
+
+TEST_F(POSIXVerObjectTest, ObjectCopy)
+{
+ std::string srcname{testname + "-1"};
+ std::unique_ptr<rgw::sal::Object> obj1v1 = write_version(srcname);
+ EXPECT_NE(obj1v1.get(), nullptr);
+ std::string obj1v1_inst = obj1v1->get_instance();
+ std::unique_ptr<rgw::sal::Object> obj1v2 = write_version(srcname);
+ EXPECT_NE(obj1v2.get(), nullptr);
+ std::string obj1v2_inst = obj1v2->get_instance();
+ sf::path sp{bp / "root" / testname / srcname};
+ std::unique_ptr<rgw::sal::Object> srcobj = bucket->get_object(rgw_obj_key(srcname));
+ EXPECT_NE(srcobj.get(), nullptr);
+ EXPECT_TRUE(sf::exists(sp));
+
+
+ std::string dstname{testname + "-dst"};
+ sf::path dp{bp / "root" / testname / dstname};
+
+ std::unique_ptr<rgw::sal::Object> dstobj = bucket->get_object(rgw_obj_key(dstname));
+ EXPECT_NE(dstobj.get(), nullptr);
+ EXPECT_FALSE(sf::exists(dp));
+
+ RGWEnv rgw_env;
+ req_info info(env->cct.get(), &rgw_env);
+ rgw_zone_id zone;
+ rgw_placement_rule placement;
+ ceph::real_time mtime;
+ Attrs attrs;
+ std::string tag;
+
+ int ret = srcobj->copy_object(acl_owner,
+ std::get<rgw_user>(owner),
+ &info,
+ zone,
+ dstobj.get(),
+ bucket.get(),
+ bucket.get(),
+ placement,
+ &mtime,
+ &mtime,
+ &mtime,
+ &mtime,
+ false,
+ nullptr,
+ nullptr,
+ ATTRSMOD_NONE,
+ false,
+ attrs,
+ RGWObjCategory::Main,
+ 0,
+ boost::none,
+ nullptr,
+ &tag, /* use req_id as tag */
+ &tag,
+ nullptr,
+ nullptr,
+ env->dpp,
+ null_yield);
+ EXPECT_EQ(ret, 0);
+ EXPECT_TRUE(sf::exists(sp));
+ EXPECT_TRUE(sf::exists(dp));
+
+ std::string vfname1{"_%3A" + obj1v1_inst + "_" + dstname};
+ sf::path op1{dp / vfname1};
+ EXPECT_TRUE(sf::exists(op1));
+ EXPECT_TRUE(sf::is_regular_file(op1));
+ std::string vfname2{"_%3A" + obj1v2_inst + "_" + dstname};
+ sf::path op2{dp / vfname2};
+ EXPECT_TRUE(sf::exists(op2));
+ EXPECT_TRUE(sf::is_regular_file(op2));
+ sf::path ops{dp / dstname};
+ EXPECT_TRUE(sf::exists(ops));
+ EXPECT_TRUE(sf::is_symlink(ops));
+ EXPECT_EQ(sf::read_symlink(ops), vfname2);
+}
+
+TEST_F(POSIXVerObjectTest, CopyVersion)
+{
+ std::string srcname{testname + "-1"};
+ std::unique_ptr<rgw::sal::Object> obj1v1 = write_version(srcname);
+ EXPECT_NE(obj1v1.get(), nullptr);
+ std::string obj1v1_inst = obj1v1->get_instance();
+ std::unique_ptr<rgw::sal::Object> obj1v2 = write_version(srcname);
+ EXPECT_NE(obj1v2.get(), nullptr);
+ std::string obj1v2_inst = obj1v2->get_instance();
+ std::string vsrcname{"_%3A" + obj1v1_inst + "_" + srcname};
+ sf::path sp{bp / "root" / testname / srcname / vsrcname};
+ std::unique_ptr<rgw::sal::Object> srcobj = bucket->get_object(rgw_obj_key(srcname, obj1v1_inst));
+ EXPECT_NE(srcobj.get(), nullptr);
+ EXPECT_TRUE(sf::exists(sp));
+
+
+ std::string dstname{testname + "-dst"};
+ sf::path dp{bp / "root" / testname / dstname};
+
+ std::unique_ptr<rgw::sal::Object> dstobj = bucket->get_object(rgw_obj_key(dstname));
+ EXPECT_NE(dstobj.get(), nullptr);
+ EXPECT_FALSE(sf::exists(dp));
+
+ RGWEnv rgw_env;
+ req_info info(env->cct.get(), &rgw_env);
+ rgw_zone_id zone;
+ rgw_placement_rule placement;
+ ceph::real_time mtime;
+ Attrs attrs;
+ std::string tag;
+
+ int ret = srcobj->copy_object(acl_owner,
+ std::get<rgw_user>(owner),
+ &info,
+ zone,
+ dstobj.get(),
+ bucket.get(),
+ bucket.get(),
+ placement,
+ &mtime,
+ &mtime,
+ &mtime,
+ &mtime,
+ false,
+ nullptr,
+ nullptr,
+ ATTRSMOD_NONE,
+ false,
+ attrs,
+ RGWObjCategory::Main,
+ 0,
+ boost::none,
+ nullptr,
+ &tag, /* use req_id as tag */
+ &tag,
+ nullptr,
+ nullptr,
+ env->dpp,
+ null_yield);
+ EXPECT_EQ(ret, 0);
+ EXPECT_TRUE(sf::exists(sp));
+ EXPECT_TRUE(sf::exists(dp));
+
+ std::string vfname1{"_%3A" + obj1v1_inst + "_" + dstname};
+ sf::path op1{dp / vfname1};
+ EXPECT_TRUE(sf::exists(op1));
+ EXPECT_TRUE(sf::is_regular_file(op1));
+ std::string vfname2{"_%3A" + obj1v2_inst + "_" + dstname};
+ sf::path op2{dp / vfname2};
+ EXPECT_FALSE(sf::exists(op2));
+ sf::path ops{dp / dstname};
+ EXPECT_TRUE(sf::exists(ops));
+ EXPECT_TRUE(sf::is_symlink(ops));
+ EXPECT_EQ(sf::read_symlink(ops), vfname1);
+}
+
+class POSIXVerMPObjectTest : public POSIXVerObjectTest {
+protected:
+ std::unique_ptr<rgw::sal::MultipartUpload> upload;
+ std::string upload_id = "c0ffee";
+ std::string mpname;
+ uint64_t write_size{0};
+ bufferlist write_data;
+
+public:
+ POSIXVerMPObjectTest() {}
+
+ void SetUp() {
+ POSIXVerObjectTest::SetUp();
+ mpname = ".multipart_" + testname + "." + upload_id;
+
+ upload = bucket->get_multipart_upload(testname, upload_id);
+ EXPECT_NE(upload.get(), nullptr);
+
+ rgw_placement_rule placement;
+ Attrs attrs;
+ add_attr(attrs, ATTR1, ATTR1);
+ int ret = upload->init(env->dpp, null_yield, acl_owner, placement, attrs);
+ EXPECT_EQ(ret, 0);
+ }
+
+ void TearDown() {
+ POSIXVerObjectTest::TearDown();
+ }
+
+ int write_part(int part_num) {
+ std::unique_ptr<rgw::sal::Writer> writer;
+ rgw_placement_rule placement;
+ std::string part_name = "part-" + fmt::format("{:0>5}", part_num);
+
+ writer = upload->get_writer(env->dpp, null_yield, nullptr, acl_owner,
+ &placement, part_num, part_name);
+ EXPECT_NE(writer.get(), nullptr);
+
+ int ret = writer->prepare(null_yield);
+ EXPECT_EQ(ret, 0);
+
+ int ofs{0};
+ for (int i = 0; i < 4; ++i) {
+ bufferlist bl;
+ encode(testname + part_name, bl);
+ int len = bl.length();
+
+ write_data.append(bl);
+
+ ret = writer->process(std::move(bl), ofs);
+ EXPECT_EQ(ret, 0);
+
+ ofs += len;
+ }
+
+ ret = writer->process({}, ofs);
+ EXPECT_EQ(ret, 0);
+
+ ceph::real_time mtime;
+ Attrs attrs;
+ bufferlist bl;
+ encode(ATTR1, bl);
+ attrs[ATTR1] = bl;
+ req_context rctx{env->dpp, null_yield, nullptr};
+
+ ret = writer->complete(ofs, part_name, &mtime, real_time(), attrs,
+ std::nullopt, real_time(), nullptr, nullptr, nullptr,
+ nullptr, nullptr, rctx, 0);
+ EXPECT_EQ(ret, 0);
+
+ return ofs;
+ }
+ void create_MPObj(std::string objname) {
+ std::map<int, std::string> parts;
+ int part_count{4};
+
+ for (int i = 1; i <= part_count; ++i) {
+ write_size += write_part(i);
+ parts[i] = "part-" + fmt::format("{:0>5}", i);
+ }
+
+ std::list<rgw_obj_index_key> remove_objs;
+ bool compressed = false;
+ RGWCompressionInfo cs_info;
+ std::unique_ptr<Object> mp_obj = bucket->get_object(rgw_obj_key(objname));
+ off_t ofs{0};
+ uint64_t accounted_size{0};
+ std::string tag;
+ ACLOwner owner;
+ owner.id = bucket->get_owner();
+ mp_obj->gen_rand_obj_instance_name();
+ std::string inst_id = mp_obj->get_instance();
+ std::string vfname{"_%3A" + inst_id + "_" + objname};
+ sf::path op{bp / "root" / testname / objname / vfname };
+
+ int ret = upload->complete(env->dpp, null_yield, get_pointer(env->cct), parts,
+ remove_objs, accounted_size, compressed, cs_info,
+ ofs, tag, owner, 0, mp_obj.get());
+ EXPECT_EQ(ret, 0);
+ EXPECT_EQ(write_size, ofs);
+ EXPECT_EQ(write_size, accounted_size);
+ EXPECT_TRUE(sf::exists(op));
+ EXPECT_TRUE(sf::is_directory(op));
+
+ for (int i = 1; i <= part_count; ++i) {
+ std::string part_name = "part-" + fmt::format("{:0>5}", i);
+ sf::path pp{bp / "root" / testname / objname / vfname / part_name};
+ EXPECT_TRUE(sf::exists(pp));
+ EXPECT_TRUE(sf::is_regular_file(pp));
+ }
+
+ std::unique_ptr<Object> object = bucket->get_object(rgw_obj_key(objname));
+ std::unique_ptr<rgw::sal::Object::ReadOp> read_op(object->get_read_op());
+
+ ret = read_op->prepare(null_yield, env->dpp);
+ EXPECT_EQ(ret, 0);
+
+ std::string getver = object->get_instance();
+ EXPECT_EQ(inst_id, getver);
+
+ ObjectType type{ObjectType::VERSIONED};
+ ret = decode_attr(object->get_attrs(), ATTR_OBJECT_TYPE.c_str(), type);
+ EXPECT_EQ(type.type, ObjectType::VERSIONED);
+
+ std::unique_ptr<Object> vobj = bucket->get_object(rgw_obj_key(objname, inst_id));
+ std::unique_ptr<rgw::sal::Object::ReadOp> vread_op(vobj->get_read_op());
+
+ ret = vread_op->prepare(null_yield, env->dpp);
+ EXPECT_EQ(ret, 0);
+
+ ret = decode_attr(vobj->get_attrs(), ATTR_OBJECT_TYPE.c_str(), type);
+ EXPECT_EQ(type.type, ObjectType::VERSIONED);
+
+ sf::path ops{bp / "root" / testname / objname / objname};
+ EXPECT_TRUE(sf::exists(ops));
+ EXPECT_TRUE(sf::is_symlink(ops));
+ EXPECT_EQ(sf::read_symlink(ops), vfname);
+ }
+};
+
+TEST_F(POSIXVerMPObjectTest, MPUploadComplete)
+{
+ create_MPObj(testname + "MPVER");
+}
+
+
+int main(int argc, char *argv[]) {
+ ::testing::InitGoogleTest(&argc, argv);
+
+ env = new Environment();
+ ::testing::AddGlobalTestEnvironment(env);
+
+ return RUN_ALL_TESTS();
+}
projects / ceph.git / commitdiff