From 5cf5ab470e6e9119ae4a6a6111974cb03610ba3e Mon Sep 17 00:00:00 2001 From: Yehuda Sadeh Date: Thu, 17 Nov 2016 15:26:35 -0800 Subject: [PATCH] rgw: add rgw_obj_index_key, kill RGWObjEnt Use rgw_obj_index_key to represent entries in bucket index (typedef of cls_rgw_obj_key). Get rid of RGWObjEnt, it was duplicate of rgw_bucket_dir_entry anyway. Signed-off-by: Yehuda Sadeh --- src/cls/rgw/cls_rgw_types.h | 10 ++ src/rgw/rgw_admin.cc | 38 +++---- src/rgw/rgw_bucket.cc | 49 ++++----- src/rgw/rgw_bucket.h | 2 +- src/rgw/rgw_cache.h | 2 +- src/rgw/rgw_common.h | 149 ++++++++++++++-------------- src/rgw/rgw_cr_rados.cc | 9 +- src/rgw/rgw_formats.cc | 6 +- src/rgw/rgw_formats.h | 2 +- src/rgw/rgw_gc.cc | 11 +- src/rgw/rgw_json_enc.cc | 16 --- src/rgw/rgw_lc.cc | 29 +++--- src/rgw/rgw_op.cc | 78 +++++++-------- src/rgw/rgw_op.h | 6 +- src/rgw/rgw_orphan.cc | 9 +- src/rgw/rgw_rados.cc | 193 +++++++++++++++--------------------- src/rgw/rgw_rados.h | 38 +++---- src/rgw/rgw_rest.cc | 2 +- src/rgw/rgw_rest.h | 2 +- src/rgw/rgw_rest_s3.cc | 36 +++---- src/rgw/rgw_rest_swift.cc | 18 ++-- 21 files changed, 332 insertions(+), 373 deletions(-) diff --git a/src/cls/rgw/cls_rgw_types.h b/src/cls/rgw/cls_rgw_types.h index 248a642b5a0..8e48e60affb 100644 --- a/src/cls/rgw/cls_rgw_types.h +++ b/src/cls/rgw/cls_rgw_types.h @@ -239,6 +239,10 @@ struct cls_rgw_obj_key { cls_rgw_obj_key(const string &_name) : name(_name) {} cls_rgw_obj_key(const string& n, const string& i) : name(n), instance(i) {} + void set(const string& _name) { + name = _name; + } + bool operator==(const cls_rgw_obj_key& k) const { return (name.compare(k.name) == 0) && (instance.compare(k.instance) == 0); @@ -250,6 +254,12 @@ struct cls_rgw_obj_key { } return (r < 0); } + bool operator<=(const cls_rgw_obj_key& k) const { + return !(k < *this); + } + bool empty() { + return name.empty(); + } void encode(bufferlist &bl) const { ENCODE_START(1, 1, bl); ::encode(name, bl); diff --git a/src/rgw/rgw_admin.cc b/src/rgw/rgw_admin.cc index 00eaf47ad3f..6cc3dbb97d4 100644 --- a/src/rgw/rgw_admin.cc +++ b/src/rgw/rgw_admin.cc @@ -1449,7 +1449,7 @@ int do_check_object_locator(const string& tenant_name, const string& bucket_name string prefix; string delim; - vector result; + vector result; map common_prefixes; string ns; @@ -1475,7 +1475,7 @@ int do_check_object_locator(const string& tenant_name, const string& bucket_name count += result.size(); - for (vector::iterator iter = result.begin(); iter != result.end(); ++iter) { + for (vector::iterator iter = result.begin(); iter != result.end(); ++iter) { rgw_obj_key key = iter->key; rgw_obj obj(bucket, key); @@ -4831,7 +4831,7 @@ int main(int argc, const char **argv) string prefix; string delim; - vector result; + vector result; map common_prefixes; string ns; @@ -4854,8 +4854,8 @@ int main(int argc, const char **argv) count += result.size(); - for (vector::iterator iter = result.begin(); iter != result.end(); ++iter) { - RGWObjEnt& entry = *iter; + for (vector::iterator iter = result.begin(); iter != result.end(); ++iter) { + rgw_bucket_dir_entry& entry = *iter; encode_json("entry", entry, formatter); } formatter->flush(cout); @@ -5224,8 +5224,7 @@ next: return 1; } - rgw_obj obj(bucket, key.name); - obj.set_instance(key.instance); + rgw_obj obj(bucket, key); ret = store->bi_put(bucket, obj, entry); if (ret < 0) { @@ -5428,14 +5427,14 @@ next: bool is_truncated = true; - rgw_obj_key marker; + rgw_obj_index_key marker; string prefix; formatter->open_object_section("result"); formatter->dump_string("bucket", bucket_name); formatter->open_array_section("objects"); while (is_truncated) { - map result; + map result; int r = store->cls_bucket_list(bucket_info, RGW_NO_SHARD, marker, prefix, 1000, true, result, &is_truncated, &marker, bucket_object_check_filter); @@ -5447,26 +5446,25 @@ next: if (r == -ENOENT) break; - map::iterator iter; + map::iterator iter; for (iter = result.begin(); iter != result.end(); ++iter) { rgw_obj_key key = iter->second.key; - RGWObjEnt& entry = iter->second; + rgw_bucket_dir_entry& entry = iter->second; formatter->open_object_section("object"); formatter->dump_string("name", key.name); formatter->dump_string("instance", key.instance); - formatter->dump_int("size", entry.size); - utime_t ut(entry.mtime); + formatter->dump_int("size", entry.meta.size); + utime_t ut(entry.meta.mtime); ut.gmtime(formatter->dump_stream("mtime")); - if ((entry.size < min_rewrite_size) || - (entry.size > max_rewrite_size) || + if ((entry.meta.size < min_rewrite_size) || + (entry.meta.size > max_rewrite_size) || (start_epoch > 0 && start_epoch > (uint64_t)ut.sec()) || (end_epoch > 0 && end_epoch < (uint64_t)ut.sec())) { formatter->dump_string("status", "Skipped"); } else { - rgw_obj obj(bucket, key.name); - obj.set_instance(key.instance); + rgw_obj obj(bucket, key); bool need_rewrite = true; if (min_rewrite_stripe_size > 0) { @@ -5656,9 +5654,11 @@ next: cerr << "ERROR: could not init bucket: " << cpp_strerror(-ret) << std::endl; return -ret; } - list oid_list; + list oid_list; rgw_obj_key key(object, object_version); - oid_list.push_back(key); + rgw_obj_index_key index_key; + key.get_index_key(&index_key); + oid_list.push_back(index_key); ret = store->remove_objs_from_index(bucket_info, oid_list); if (ret < 0) { cerr << "ERROR: remove_obj_from_index() returned error: " << cpp_strerror(-ret) << std::endl; diff --git a/src/rgw/rgw_bucket.cc b/src/rgw/rgw_bucket.cc index 966c8ea34ec..5f13be24bd1 100644 --- a/src/rgw/rgw_bucket.cc +++ b/src/rgw/rgw_bucket.cc @@ -421,11 +421,11 @@ int rgw_bucket_set_attrs(RGWRados *store, RGWBucketInfo& bucket_info, return rgw_bucket_instance_store_info(store, key, bl, false, &attrs, objv_tracker, real_time()); } -static void dump_mulipart_index_results(list& objs_to_unlink, +static void dump_mulipart_index_results(list& objs_to_unlink, Formatter *f) { // make sure that an appropiately titled header has been opened previously - list::iterator oiter = objs_to_unlink.begin(); + auto oiter = objs_to_unlink.begin(); f->open_array_section("invalid_multipart_entries"); @@ -521,7 +521,7 @@ int rgw_remove_bucket(RGWRados *store, rgw_bucket& bucket, bool delete_children) { int ret; map stats; - std::vector objs; + std::vector objs; map common_prefixes; RGWBucketInfo info; RGWObjectCtx obj_ctx(store); @@ -552,9 +552,10 @@ int rgw_remove_bucket(RGWRados *store, rgw_bucket& bucket, bool delete_children) if (ret < 0) return ret; - std::vector::iterator it = objs.begin(); + std::vector::iterator it = objs.begin(); for (; it != objs.end(); ++it) { - ret = rgw_remove_object(store, info, bucket, (*it).key); + rgw_obj_key key(it->key); + ret = rgw_remove_object(store, info, bucket, key); if (ret < 0) return ret; } @@ -612,7 +613,7 @@ int rgw_remove_bucket_bypass_gc(RGWRados *store, rgw_bucket& bucket, { int ret; map stats; - std::vector objs; + std::vector objs; map common_prefixes; RGWBucketInfo info; RGWObjectCtx obj_ctx(store); @@ -642,11 +643,10 @@ int rgw_remove_bucket_bypass_gc(RGWRados *store, rgw_bucket& bucket, return ret; while (!objs.empty()) { - std::vector::iterator it = objs.begin(); + std::vector::iterator it = objs.begin(); for (; it != objs.end(); ++it) { RGWObjState *astate = NULL; - rgw_obj obj(bucket, (*it).key.name); - obj.set_instance((*it).key.instance); + rgw_obj obj(bucket, (*it).key); ret = store->get_obj_state(&obj_ctx, obj, &astate, false); if (ret == -ENOENT) { @@ -953,9 +953,9 @@ int RGWBucket::remove_object(RGWBucketAdminOpState& op_state, std::string *err_m return 0; } -static void dump_bucket_index(map result, Formatter *f) +static void dump_bucket_index(map result, Formatter *f) { - map::iterator iter; + map::iterator iter; for (iter = result.begin(); iter != result.end(); ++iter) { f->dump_string("object", iter->first); } @@ -991,7 +991,7 @@ static void dump_index_check(map existing_stats } int RGWBucket::check_bad_index_multipart(RGWBucketAdminOpState& op_state, - list& objs_to_unlink, std::string *err_msg) + list& objs_to_unlink, std::string *err_msg) { bool fix_index = op_state.will_fix_index(); rgw_bucket bucket = op_state.get_bucket(); @@ -1003,7 +1003,7 @@ int RGWBucket::check_bad_index_multipart(RGWBucketAdminOpState& op_state, bool is_truncated; map meta_objs; - map all_objs; + map all_objs; RGWBucketInfo bucket_info; RGWObjectCtx obj_ctx(store); @@ -1019,7 +1019,7 @@ int RGWBucket::check_bad_index_multipart(RGWBucketAdminOpState& op_state, list_op.params.list_versions = true; do { - vector result; + vector result; int r = list_op.list_objects(max, &result, &common_prefixes, &is_truncated); if (r < 0) { set_err_msg(err_msg, "failed to list objects in bucket=" + bucket.name + @@ -1028,14 +1028,14 @@ int RGWBucket::check_bad_index_multipart(RGWBucketAdminOpState& op_state, return r; } - vector::iterator iter; + vector::iterator iter; for (iter = result.begin(); iter != result.end(); ++iter) { - RGWObjEnt& ent = *iter; + rgw_bucket_dir_entry& ent = *iter; rgw_obj obj(bucket, ent.key); obj.set_ns(ns); - rgw_obj_key key; + rgw_obj_index_key key; obj.get_index_key(&key); string oid = key.name; @@ -1059,10 +1059,11 @@ int RGWBucket::check_bad_index_multipart(RGWBucketAdminOpState& op_state, } while (is_truncated); - map::iterator aiter; - for (aiter = all_objs.begin(); aiter != all_objs.end(); ++aiter) { + for (auto aiter = all_objs.begin(); aiter != all_objs.end(); ++aiter) { string& name = aiter->second; + rgw_obj_index_key index_key(aiter->first); + if (meta_objs.find(name) == meta_objs.end()) { objs_to_unlink.push_back(aiter->first); } @@ -1101,13 +1102,13 @@ int RGWBucket::check_object_index(RGWBucketAdminOpState& op_state, store->cls_obj_set_bucket_tag_timeout(bucket_info, BUCKET_TAG_TIMEOUT); string prefix; - rgw_obj_key marker; + rgw_obj_index_key marker; bool is_truncated = true; Formatter *formatter = flusher.get_formatter(); formatter->open_object_section("objects"); while (is_truncated) { - map result; + map result; int r = store->cls_bucket_list(bucket_info, RGW_NO_SHARD, marker, prefix, 1000, true, result, &is_truncated, &marker, @@ -1302,10 +1303,10 @@ int RGWBucketAdminOp::check_index(RGWRados *store, RGWBucketAdminOpState& op_sta RGWFormatterFlusher& flusher) { int ret; - map result; + map result; map existing_stats; map calculated_stats; - list objs_to_unlink; + list objs_to_unlink; RGWBucket bucket; @@ -1463,7 +1464,7 @@ int RGWBucketAdminOp::info(RGWRados *store, RGWBucketAdminOpState& op_state, formatter->open_array_section("buckets"); if (store->list_buckets_init(&handle) >= 0) { - RGWObjEnt obj; + rgw_bucket_dir_entry obj; while (store->list_buckets_next(obj, &handle) >= 0) { if (show_stats) bucket_stats(store, user_id.tenant, obj.key.name, formatter); diff --git a/src/rgw/rgw_bucket.h b/src/rgw/rgw_bucket.h index 056bfd81099..5497a0ddff4 100644 --- a/src/rgw/rgw_bucket.h +++ b/src/rgw/rgw_bucket.h @@ -274,7 +274,7 @@ public: int init(RGWRados *storage, RGWBucketAdminOpState& op_state); int check_bad_index_multipart(RGWBucketAdminOpState& op_state, - list& objs_to_unlink, std::string *err_msg = NULL); + list& objs_to_unlink, std::string *err_msg = NULL); int check_object_index(RGWBucketAdminOpState& op_state, RGWFormatterFlusher& flusher, diff --git a/src/rgw/rgw_cache.h b/src/rgw/rgw_cache.h index af3933108fa..583bdc8c76c 100644 --- a/src/rgw/rgw_cache.h +++ b/src/rgw/rgw_cache.h @@ -176,7 +176,7 @@ class RGWCache : public T int list_objects_raw_init(rgw_pool& pool, RGWAccessHandle *handle) { return T::list_objects_raw_init(pool, handle); } - int list_objects_raw_next(RGWObjEnt& obj, RGWAccessHandle *handle) { + int list_objects_raw_next(rgw_bucket_dir_entry& obj, RGWAccessHandle *handle) { return T::list_objects_raw_next(obj, handle); } diff --git a/src/rgw/rgw_common.h b/src/rgw/rgw_common.h index bac7019f39a..0608b5e1f9a 100644 --- a/src/rgw/rgw_common.h +++ b/src/rgw/rgw_common.h @@ -1340,42 +1340,74 @@ struct req_info { void init_meta_info(bool *found_bad_meta); }; +typedef cls_rgw_obj_key rgw_obj_index_key; + struct rgw_obj_key { string name; string instance; + string ns; rgw_obj_key() {} // cppcheck-suppress noExplicitConstructor - rgw_obj_key(const string& n) { - set(n); - } - rgw_obj_key(const string& n, const string& i) { - set(n, i); - } + rgw_obj_key(const string& n) : name(n) {} + rgw_obj_key(const string& n, const string& i) : name(n), instance(i) {} - // cppcheck-suppress noExplicitConstructor - rgw_obj_key(const cls_rgw_obj_key& k) { - set(k); - } - - void set(const cls_rgw_obj_key& k) { - name = k.name; + rgw_obj_key(const rgw_obj_index_key& k) { + parse_index_key(k.name, &name, &ns); instance = k.instance; } - void transform(cls_rgw_obj_key *k) { - k->name = name; - k->instance = instance; + static void parse_index_key(const string& key, string *name, string *ns) { + if (key[0] != '_') { + *name = key; + ns->clear(); + return; + } + if (key[1] == '_') { + *name = key.substr(1); + ns->clear(); + return; + } + ssize_t pos = key.find('_', 1); + if (pos < 0) { + /* shouldn't happen, just use key */ + *name = key; + ns->clear(); + return; + } + + *name = key.substr(pos + 1); + *ns = key.substr(1, pos -1); } void set(const string& n) { name = n; instance.clear(); + ns.clear(); + } + + string get_index_key_name() const { + if (ns.empty()) { + if (name.size() < 1 || name[0] != '_') { + return name; + } + return string("_") + name; + }; + + char buf[ns.size() + 16]; + snprintf(buf, sizeof(buf), "_%s_", ns.c_str()); + return string(buf) + name; + }; + + void get_index_key(rgw_obj_index_key *key) const { + key->name = get_index_key_name(); + key->instance = instance; } void set(const string& n, const string& i) { name = n; instance = i; + ns.clear(); } bool empty() const { @@ -1396,15 +1428,19 @@ struct rgw_obj_key { return !(k < *this); } void encode(bufferlist& bl) const { - ENCODE_START(1, 1, bl); + ENCODE_START(2, 1, bl); ::encode(name, bl); ::encode(instance, bl); + ::encode(ns, bl); ENCODE_FINISH(bl); } void decode(bufferlist::iterator& bl) { - DECODE_START(1, bl); + DECODE_START(2, bl); ::decode(name, bl); ::decode(instance, bl); + if (struct_v >= 2) { + ::decode(ns, bl); + } DECODE_FINISH(bl); } void dump(Formatter *f) const; @@ -1419,6 +1455,14 @@ inline ostream& operator<<(ostream& out, const rgw_obj_key &o) { } } +inline ostream& operator<<(ostream& out, const rgw_obj_index_key &o) { + if (o.instance.empty()) { + return out << o.name; + } else { + return out << o.name << "[" << o.instance << "]"; + } +} + struct rgw_aws4_auth { string date; string expires; @@ -1551,36 +1595,6 @@ struct req_state { ~req_state(); }; -/** Store basic data on an object */ -struct RGWObjEnt { - rgw_obj_key key; - std::string ns; - rgw_user owner; - std::string owner_display_name; - uint64_t size{0}; - uint64_t accounted_size{0}; - ceph::real_time mtime; - string etag; - string content_type; - string tag; - uint32_t flags; - uint64_t versioned_epoch; - - RGWObjEnt() : flags(0), versioned_epoch(0) {} - - void dump(Formatter *f) const; - - bool is_current() { - uint32_t test_flags = RGW_BUCKET_DIRENT_FLAG_VER | RGW_BUCKET_DIRENT_FLAG_CURRENT; - return (flags & RGW_BUCKET_DIRENT_FLAG_VER) == 0 || - (flags & test_flags) == test_flags; - } - bool is_delete_marker() { return (flags & RGW_BUCKET_DIRENT_FLAG_DELETE_MARKER) != 0; } - bool is_visible() { - return is_current() && !is_delete_marker(); - } -}; - /** Store basic data on bucket */ struct RGWBucketEnt { rgw_bucket bucket; @@ -1669,8 +1683,13 @@ struct rgw_obj { rgw_obj(const rgw_bucket& b, const std::string& name) { init(b, name); } - rgw_obj(const rgw_bucket& b, const rgw_obj_key& k) : in_extra_data(false) { - from_index_key(b, k); + rgw_obj(const rgw_bucket& b, const rgw_obj_key& k) { + init(b, k.name); + instance = k.instance; + } + rgw_obj(const rgw_bucket& b, const rgw_obj_index_key& k) : bucket(b) { + rgw_obj_key::parse_index_key(k.name, &name, &ns); + set_instance(k.instance); } void init(const rgw_bucket& b, const std::string& name) { bucket = b; @@ -1726,6 +1745,11 @@ struct rgw_obj { name = n; } + void set_key(const rgw_obj_key& k) { + set_name(k.name); + set_instance(k.instance); + } + string get_oid() const { if (ns.empty() && !need_to_encode_instance()) { if (name.size() < 1 || name[0] != '_') { @@ -1760,30 +1784,7 @@ struct rgw_obj { return string(buf) + name; }; - void from_index_key(const rgw_bucket& b, const rgw_obj_key& key) { - if (key.name[0] != '_') { - init(b, key.name); - set_instance(key.instance); - return; - } - if (key.name[1] == '_') { - init(b, key.name.substr(1)); - set_instance(key.instance); - return; - } - ssize_t pos = key.name.find('_', 1); - if (pos < 0) { - /* shouldn't happen, just use key */ - init(b, key.name); - set_instance(key.instance); - return; - } - - init_ns(b, key.name.substr(pos + 1), key.name.substr(1, pos -1)); - set_instance(key.instance); - } - - void get_index_key(rgw_obj_key *key) const { + void get_index_key(rgw_obj_index_key *key) const { key->name = get_index_key_name(); key->instance = instance; } diff --git a/src/rgw/rgw_cr_rados.cc b/src/rgw/rgw_cr_rados.cc index 68d0c64ab27..2e3207777cb 100644 --- a/src/rgw/rgw_cr_rados.cc +++ b/src/rgw/rgw_cr_rados.cc @@ -479,8 +479,7 @@ int RGWAsyncFetchRemoteObj::_send_request() string op_id = store->unique_id(store->get_new_req_id()); map attrs; - rgw_obj src_obj(bucket_info.bucket, key.name); - src_obj.set_instance(key.instance); + rgw_obj src_obj(bucket_info.bucket, key); rgw_obj dest_obj(src_obj); @@ -531,8 +530,7 @@ int RGWAsyncStatRemoteObj::_send_request() string client_id = store->zone_id() + buf; string op_id = store->unique_id(store->get_new_req_id()); - rgw_obj src_obj(bucket_info.bucket, key.name); - src_obj.set_instance(key.instance); + rgw_obj src_obj(bucket_info.bucket, key); rgw_obj dest_obj(src_obj); @@ -566,8 +564,7 @@ int RGWAsyncRemoveObj::_send_request() { RGWObjectCtx obj_ctx(store); - rgw_obj obj(bucket_info.bucket, key.name); - obj.set_instance(key.instance); + rgw_obj obj(bucket_info.bucket, key); ldout(store->ctx(), 0) << __func__ << "(): deleting obj=" << obj << dendl; diff --git a/src/rgw/rgw_formats.cc b/src/rgw/rgw_formats.cc index 7a0a1bf7624..8a84083c757 100644 --- a/src/rgw/rgw_formats.cc +++ b/src/rgw/rgw_formats.cc @@ -343,7 +343,7 @@ std::string RGWSwiftWebsiteListingFormatter::format_name( return item_name.substr(prefix.length()); } -void RGWSwiftWebsiteListingFormatter::dump_object(const RGWObjEnt& objent) +void RGWSwiftWebsiteListingFormatter::dump_object(const rgw_bucket_dir_entry& objent) { const auto name = format_name(objent.key.name); ss << boost::format(R"()") @@ -351,9 +351,9 @@ void RGWSwiftWebsiteListingFormatter::dump_object(const RGWObjEnt& objent) << boost::format(R"(%s)") % url_encode(name) % HTMLHelper::escape(name) - << boost::format(R"(%lld)") % objent.size + << boost::format(R"(%lld)") % objent.meta.size << boost::format(R"(%s)") - % dump_time_to_str(objent.mtime) + % dump_time_to_str(objent.meta.mtime) << R"()"; } diff --git a/src/rgw/rgw_formats.h b/src/rgw/rgw_formats.h index acf3a0fb7de..13e4cd83a47 100644 --- a/src/rgw/rgw_formats.h +++ b/src/rgw/rgw_formats.h @@ -79,7 +79,7 @@ public: void generate_header(const std::string& dir_path, const std::string& css_path); void generate_footer(); - void dump_object(const RGWObjEnt& objent); + void dump_object(const rgw_bucket_dir_entry& objent); void dump_subdir(const std::string& name); }; diff --git a/src/rgw/rgw_gc.cc b/src/rgw/rgw_gc.cc index 6f1dd448028..6973a955ee2 100644 --- a/src/rgw/rgw_gc.cc +++ b/src/rgw/rgw_gc.cc @@ -196,19 +196,18 @@ int RGWGC::process(int index, int max_secs) } ctx->locator_set_key(obj.loc); - rgw_obj key_obj; - key_obj.set_name(obj.key.name); - key_obj.set_instance(obj.key.instance); - dout(0) << "gc::process: removing " << obj.pool << ":" << key_obj.get_oid() << dendl; + const string& oid = obj.key.name; /* just stored raw oid there */ + + dout(0) << "gc::process: removing " << obj.pool << ":" << obj.key.name << dendl; ObjectWriteOperation op; cls_refcount_put(op, info.tag, true); - ret = ctx->operate(key_obj.get_oid(), &op); + ret = ctx->operate(oid, &op); if (ret == -ENOENT) ret = 0; if (ret < 0) { remove_tag = false; - dout(0) << "failed to remove " << obj.pool << ":" << key_obj.get_oid() << "@" << obj.loc << dendl; + dout(0) << "failed to remove " << obj.pool << ":" << oid << "@" << obj.loc << dendl; } if (going_down()) // leave early, even if tag isn't removed, it's ok diff --git a/src/rgw/rgw_json_enc.cc b/src/rgw/rgw_json_enc.cc index bb274fd3aa1..ac5460b84c9 100644 --- a/src/rgw/rgw_json_enc.cc +++ b/src/rgw/rgw_json_enc.cc @@ -763,22 +763,6 @@ void rgw_obj_key::dump(Formatter *f) const encode_json("instance", instance, f); } -void RGWObjEnt::dump(Formatter *f) const -{ - encode_json("name", key.name, f); - encode_json("instance", key.instance, f); - encode_json("namespace", ns, f); - encode_json("owner", owner.to_str(), f); - encode_json("owner_display_name", owner_display_name, f); - encode_json("size", size, f); - utime_t ut(mtime); - encode_json("mtime", ut, f); - encode_json("etag", etag, f); - encode_json("content_type", content_type, f); - encode_json("tag", tag, f); - encode_json("flags", flags, f); -} - void RGWBucketEnt::dump(Formatter *f) const { encode_json("bucket", bucket, f); diff --git a/src/rgw/rgw_lc.cc b/src/rgw/rgw_lc.cc index b9770e332a0..6bf7441d7fa 100644 --- a/src/rgw/rgw_lc.cc +++ b/src/rgw/rgw_lc.cc @@ -272,7 +272,7 @@ int RGWLC::bucket_lc_process(string& shard_id) map bucket_attrs; string next_marker, no_ns, list_versions; bool is_truncated; - vector objs; + vector objs; RGWObjectCtx obj_ctx(store); vector result; result = split(shard_id, ':'); @@ -329,21 +329,27 @@ int RGWLC::bucket_lc_process(string& shard_id) utime_t now = ceph_clock_now(); for (auto obj_iter = objs.begin(); obj_iter != objs.end(); ++obj_iter) { - if (obj_has_expired(now - ceph::real_clock::to_time_t(obj_iter->mtime), prefix_iter->second.expiration)) { + rgw_obj_key key(obj_iter->key); + + if (!key.ns.empty()) { + continue; + } + + if (obj_has_expired(now - ceph::real_clock::to_time_t(obj_iter->meta.mtime), prefix_iter->second.expiration)) { RGWObjectCtx rctx(store); - rgw_obj obj(bucket_info.bucket, obj_iter->key.name); + rgw_obj obj(bucket_info.bucket, key); RGWObjState *state; int ret = store->get_obj_state(&rctx, obj, &state, false); if (ret < 0) { return ret; } - if (state->mtime != obj_iter->mtime)//Check mtime again to avoid delete a recently update object as much as possible + if (state->mtime != obj_iter->meta.mtime)//Check mtime again to avoid delete a recently update object as much as possible continue; ret = remove_expired_obj(bucket_info, obj_iter->key, true); if (ret < 0) { ldout(cct, 0) << "ERROR: remove_expired_obj " << dendl; } else { - ldout(cct, 10) << "DELETED:" << bucket_name << ":" << obj_iter->key.name << dendl; + ldout(cct, 10) << "DELETED:" << bucket_name << ":" << key << dendl; } } } @@ -363,7 +369,7 @@ int RGWLC::bucket_lc_process(string& shard_id) pre_marker = list_op.get_next_marker(); } list_op.params.prefix = prefix_iter->first; - RGWObjEnt pre_obj; + rgw_bucket_dir_entry pre_obj; do { if (!objs.empty()) { pre_obj = objs.back(); @@ -402,34 +408,33 @@ int RGWLC::bucket_lc_process(string& shard_id) } else { remove_indeed = false; } - mtime = obj_iter->mtime; + mtime = obj_iter->meta.mtime; expiration = prefix_iter->second.expiration; } else { if (prefix_iter->second.noncur_expiration <=0) { continue; } remove_indeed = true; - mtime = (obj_iter == objs.begin())?pre_obj.mtime:(obj_iter - 1)->mtime; + mtime = (obj_iter == objs.begin())?pre_obj.meta.mtime:(obj_iter - 1)->meta.mtime; expiration = prefix_iter->second.noncur_expiration; } if (obj_has_expired(now - ceph::real_clock::to_time_t(mtime), expiration)) { if (obj_iter->is_visible()) { RGWObjectCtx rctx(store); - rgw_obj obj(bucket_info.bucket, obj_iter->key.name); - obj.set_instance(obj_iter->key.instance); + rgw_obj obj(bucket_info.bucket, obj_iter->key); RGWObjState *state; int ret = store->get_obj_state(&rctx, obj, &state, false); if (ret < 0) { return ret; } - if (state->mtime != obj_iter->mtime)//Check mtime again to avoid delete a recently update object as much as possible + if (state->mtime != obj_iter->meta.mtime)//Check mtime again to avoid delete a recently update object as much as possible continue; } ret = remove_expired_obj(bucket_info, obj_iter->key, remove_indeed); if (ret < 0) { ldout(cct, 0) << "ERROR: remove_expired_obj " << dendl; } else { - ldout(cct, 10) << "DELETED:" << bucket_name << ":" << obj_iter->key.name << dendl; + ldout(cct, 10) << "DELETED:" << bucket_name << ":" << obj_iter->key << dendl; } } } diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc index 9d92bede6c7..f0bc7fdff27 100644 --- a/src/rgw/rgw_op.cc +++ b/src/rgw/rgw_op.cc @@ -346,8 +346,7 @@ static int read_obj_policy(RGWRados *store, obj.init_ns(bucket, oid, mp_ns); obj.set_in_extra_data(true); } else { - obj = rgw_obj(bucket, object.name); - obj.set_instance(object.instance); + obj = rgw_obj(bucket, object); } RGWObjectCtx *obj_ctx = static_cast(s->obj_ctx); int ret = get_obj_policy_from_attr(s->cct, store, *obj_ctx, @@ -537,8 +536,7 @@ int rgw_build_object_policies(RGWRados *store, struct req_state *s, } s->object_acl = new RGWAccessControlPolicy(s->cct); - rgw_obj obj(s->bucket, s->object.name); - obj.set_instance(s->object.instance); + rgw_obj obj(s->bucket, s->object); store->set_atomic(s->obj_ctx, obj); if (prefetch_data) { @@ -560,8 +558,7 @@ static void rgw_bucket_object_pre_exec(struct req_state *s) int RGWGetObj::verify_permission() { - obj = rgw_obj(s->bucket, s->object.name); - obj.set_instance(s->object.instance); + obj = rgw_obj(s->bucket, s->object); store->set_atomic(s->obj_ctx, obj); if (get_data) { store->set_prefetch_data(s->obj_ctx, obj); @@ -806,7 +803,7 @@ bool RGWOp::generate_cors_headers(string& origin, string& method, string& header } int RGWGetObj::read_user_manifest_part(rgw_bucket& bucket, - const RGWObjEnt& ent, + const rgw_bucket_dir_entry& ent, RGWAccessControlPolicy * const bucket_policy, const off_t start_ofs, const off_t end_ofs) @@ -835,7 +832,7 @@ int RGWGetObj::read_user_manifest_part(rgw_bucket& bucket, RGWRados::Object op_target(store, s->bucket_info, obj_ctx, part); RGWRados::Object::Read read_op(&op_target); - read_op.conds.if_match = ent.etag.c_str(); + read_op.conds.if_match = ent.meta.etag.c_str(); read_op.params.attrs = &attrs; read_op.params.obj_size = &obj_size; read_op.params.perr = &s->err; @@ -855,10 +852,10 @@ int RGWGetObj::read_user_manifest_part(rgw_bucket& bucket, if (need_decompress) { - if (cs_info.orig_size != ent.size) { + if (cs_info.orig_size != ent.meta.size) { // hmm.. something wrong, object not as expected, abort! ldout(s->cct, 0) << "ERROR: expected cs_info.orig_size=" << cs_info.orig_size << - ", actual read size=" << ent.size << dendl; + ", actual read size=" << ent.meta.size << dendl; return -EIO; } decompress.emplace(s->cct, &cs_info, partial_content, filter); @@ -866,9 +863,9 @@ int RGWGetObj::read_user_manifest_part(rgw_bucket& bucket, } else { - if (obj_size != ent.size) { + if (obj_size != ent.meta.size) { // hmm.. something wrong, object not as expected, abort! - ldout(s->cct, 0) << "ERROR: expected obj_size=" << obj_size << ", actual read size=" << ent.size << dendl; + ldout(s->cct, 0) << "ERROR: expected obj_size=" << obj_size << ", actual read size=" << ent.meta.size << dendl; return -EIO; } } @@ -884,7 +881,7 @@ int RGWGetObj::read_user_manifest_part(rgw_bucket& bucket, return -EPERM; } - if (ent.size == 0) { + if (ent.meta.size == 0) { return 0; } @@ -907,7 +904,7 @@ static int iterate_user_manifest_parts(CephContext * const cct, uint64_t * const pobj_size, string * const pobj_sum, int (*cb)(rgw_bucket& bucket, - const RGWObjEnt& ent, + const rgw_bucket_dir_entry& ent, RGWAccessControlPolicy * const bucket_policy, off_t start_ofs, off_t end_ofs, @@ -919,7 +916,7 @@ static int iterate_user_manifest_parts(CephContext * const cct, bool found_start = false, found_end = false, handled_end = false; string delim; bool is_truncated; - vector objs; + vector objs; utime_t start_time = ceph_clock_now(); @@ -937,19 +934,19 @@ static int iterate_user_manifest_parts(CephContext * const cct, return r; } - for (RGWObjEnt& ent : objs) { + for (rgw_bucket_dir_entry& ent : objs) { uint64_t cur_total_len = obj_ofs; - uint64_t start_ofs = 0, end_ofs = ent.size; + uint64_t start_ofs = 0, end_ofs = ent.meta.size; - if (!found_start && cur_total_len + ent.size > (uint64_t)ofs) { + if (!found_start && cur_total_len + ent.meta.size > (uint64_t)ofs) { start_ofs = ofs - obj_ofs; found_start = true; } - obj_ofs += ent.size; + obj_ofs += ent.meta.size; if (pobj_sum) { - etag_sum.Update((const byte *)ent.etag.c_str(), - ent.etag.length()); + etag_sum.Update((const byte *)ent.meta.etag.c_str(), + ent.meta.etag.length()); } if (!found_end && obj_ofs > (uint64_t)end) { @@ -1005,7 +1002,7 @@ static int iterate_slo_parts(CephContext *cct, off_t end, map& slo_parts, int (*cb)(rgw_bucket& bucket, - const RGWObjEnt& ent, + const rgw_bucket_dir_entry& ent, RGWAccessControlPolicy *bucket_policy, off_t start_ofs, off_t end_ofs, @@ -1029,21 +1026,21 @@ static int iterate_slo_parts(CephContext *cct, for (; iter != slo_parts.end() && !found_end; ++iter) { rgw_slo_part& part = iter->second; - RGWObjEnt ent; + rgw_bucket_dir_entry ent; ent.key.name = part.obj_name; - ent.size = part.size; - ent.etag = part.etag; + ent.meta.size = part.size; + ent.meta.etag = part.etag; uint64_t cur_total_len = obj_ofs; - uint64_t start_ofs = 0, end_ofs = ent.size; + uint64_t start_ofs = 0, end_ofs = ent.meta.size; - if (!found_start && cur_total_len + ent.size > (uint64_t)ofs) { + if (!found_start && cur_total_len + ent.meta.size > (uint64_t)ofs) { start_ofs = ofs - obj_ofs; found_start = true; } - obj_ofs += ent.size; + obj_ofs += ent.meta.size; if (!found_end && obj_ofs > (uint64_t)end) { end_ofs = end - cur_total_len + 1; @@ -1068,7 +1065,7 @@ static int iterate_slo_parts(CephContext *cct, } static int get_obj_user_manifest_iterate_cb(rgw_bucket& bucket, - const RGWObjEnt& ent, + const rgw_bucket_dir_entry& ent, RGWAccessControlPolicy * const bucket_policy, const off_t start_ofs, const off_t end_ofs, @@ -2571,8 +2568,7 @@ int RGWPutObj::verify_permission() rgw_bucket cs_bucket(copy_source_bucket_info.bucket); rgw_obj_key cs_object(copy_source_object_name, copy_source_version_id); - rgw_obj obj(cs_bucket, cs_object.name); - obj.set_instance(cs_object.instance); + rgw_obj obj(cs_bucket, cs_object); store->set_atomic(s->obj_ctx, obj); store->set_prefetch_data(s->obj_ctx, obj); @@ -2823,8 +2819,7 @@ int RGWPutObj::get_data(const off_t fst, const off_t lst, bufferlist& bl) new_end = lst; rgw_obj_key obj_key(copy_source_object_name, copy_source_version_id); - rgw_obj obj(copy_source_bucket_info.bucket, obj_key.name); - obj.set_instance(obj_key.instance); + rgw_obj obj(copy_source_bucket_info.bucket, obj_key); RGWRados::Object op_target(store, copy_source_bucket_info, *static_cast(s->obj_ctx), obj); RGWRados::Object::Read read_op(&op_target); @@ -3716,8 +3711,7 @@ void RGWDeleteObj::execute() return; } - rgw_obj obj(s->bucket, s->object.name); - obj.set_instance(s->object.instance); + rgw_obj obj(s->bucket, s->object); map attrs; @@ -4846,7 +4840,7 @@ void RGWCompleteMultipart::execute() uint64_t min_part_size = s->cct->_conf->rgw_multipart_min_part_size; - list remove_objs; /* objects to be removed from index listing */ + list remove_objs; /* objects to be removed from index listing */ bool versioned_object = s->bucket_info.versioning_enabled(); @@ -4950,7 +4944,7 @@ void RGWCompleteMultipart::execute() compressed = true; } - rgw_obj_key remove_key; + rgw_obj_index_key remove_key; src_obj.get_index_key(&remove_key); remove_objs.push_back(remove_key); @@ -5055,7 +5049,7 @@ void RGWAbortMultipart::execute() meta_obj.index_hash_source = s->object.name; cls_rgw_obj_chain chain; - list remove_objs; + list remove_objs; do { op_ret = list_multipart_parts(store, s, upload_id, meta_oid, max_parts, @@ -5083,7 +5077,7 @@ void RGWAbortMultipart::execute() rgw_raw_obj raw_head = oiter.get_location().get_raw_obj(store); rgw_raw_obj_to_obj(s->bucket, raw_head, &head); - rgw_obj_key key; + rgw_obj_index_key key; head.get_index_key(&key); remove_objs.push_back(key); } @@ -5163,7 +5157,7 @@ void RGWListBucketMultiparts::pre_exec() void RGWListBucketMultiparts::execute() { - vector objs; + vector objs; string marker_meta; op_ret = get_params(); @@ -5196,10 +5190,10 @@ void RGWListBucketMultiparts::execute() op_ret = list_op.list_objects(max_uploads, &objs, &common_prefixes, &is_truncated); if (!objs.empty()) { - vector::iterator iter; + vector::iterator iter; RGWMultipartUploadEntry entry; for (iter = objs.begin(); iter != objs.end(); ++iter) { - rgw_obj_key& key = iter->key; + rgw_obj_key key(iter->key); if (!entry.mp.from_meta(key.name)) continue; entry.obj = *iter; diff --git a/src/rgw/rgw_op.h b/src/rgw/rgw_op.h index 30be7c48c64..717c6ef318d 100644 --- a/src/rgw/rgw_op.h +++ b/src/rgw/rgw_op.h @@ -182,7 +182,7 @@ public: void pre_exec(); void execute(); int read_user_manifest_part(rgw_bucket& bucket, - const RGWObjEnt& ent, + const rgw_bucket_dir_entry& ent, RGWAccessControlPolicy *bucket_policy, off_t start_ofs, off_t end_ofs); @@ -429,7 +429,7 @@ protected: string encoding_type; bool list_versions; int max; - vector objs; + vector objs; map common_prefixes; int default_max; @@ -1429,7 +1429,7 @@ public: }; struct RGWMultipartUploadEntry { - RGWObjEnt obj; + rgw_bucket_dir_entry obj; RGWMPObj mp; }; diff --git a/src/rgw/rgw_orphan.cc b/src/rgw/rgw_orphan.cc index fa04badf152..73800731bc8 100644 --- a/src/rgw/rgw_orphan.cc +++ b/src/rgw/rgw_orphan.cc @@ -503,7 +503,7 @@ int RGWOrphanSearch::build_linked_oids_for_bucket(const string& bucket_instance_ int count = 0; do { - vector result; + vector result; #define MAX_LIST_OBJS_ENTRIES 100 ret = list_op.list_objects(MAX_LIST_OBJS_ENTRIES, &result, NULL, &truncated); @@ -512,17 +512,16 @@ int RGWOrphanSearch::build_linked_oids_for_bucket(const string& bucket_instance_ return -ret; } - for (vector::iterator iter = result.begin(); iter != result.end(); ++iter) { - RGWObjEnt& entry = *iter; + for (vector::iterator iter = result.begin(); iter != result.end(); ++iter) { + rgw_bucket_dir_entry& entry = *iter; if (entry.key.instance.empty()) { ldout(store->ctx(), 20) << "obj entry: " << entry.key.name << dendl; } else { ldout(store->ctx(), 20) << "obj entry: " << entry.key.name << " [" << entry.key.instance << "]" << dendl; } - ldout(store->ctx(), 20) << __func__ << ": entry.key.name=" << entry.key.name << " entry.key.instance=" << entry.key.instance << " entry.ns=" << entry.ns << dendl; + ldout(store->ctx(), 20) << __func__ << ": entry.key.name=" << entry.key.name << " entry.key.instance=" << entry.key.instance << dendl; rgw_obj obj(bucket_info.bucket, entry.key); - obj.set_ns(entry.ns); RGWRados::Object op_target(store, bucket_info, obj_ctx, obj); diff --git a/src/rgw/rgw_rados.cc b/src/rgw/rgw_rados.cc index 84da096835e..cf12b75a55f 100644 --- a/src/rgw/rgw_rados.cc +++ b/src/rgw/rgw_rados.cc @@ -4436,7 +4436,7 @@ int RGWRados::list_buckets_init(RGWAccessHandle *handle) * handle is updated. * returns 0 on success, -ERR# otherwise. */ -int RGWRados::list_buckets_next(RGWObjEnt& obj, RGWAccessHandle *handle) +int RGWRados::list_buckets_next(rgw_bucket_dir_entry& obj, RGWAccessHandle *handle) { librados::NObjectIterator *state = (librados::NObjectIterator *)*handle; @@ -4446,7 +4446,7 @@ int RGWRados::list_buckets_next(RGWObjEnt& obj, RGWAccessHandle *handle) return -ENOENT; } - obj.key.set((*state)->get_oid()); + obj.key.name = (*state)->get_oid(); if (obj.key.name[0] == '_') { obj.key.name = obj.key.name.substr(1); } @@ -4929,7 +4929,7 @@ string RGWRados::objexp_hint_get_shardname(int shard_num) #define MAX_OBJEXP_SHARDS_PRIME 7877 -int RGWRados::objexp_key_shard(const rgw_obj_key& key) +int RGWRados::objexp_key_shard(const rgw_obj_index_key& key) { string obj_key = key.name + key.instance; int num_shards = cct->_conf->rgw_objexp_hints_num_shards; @@ -4952,7 +4952,7 @@ int RGWRados::objexp_hint_add(const ceph::real_time& delete_at, const string& tenant_name, const string& bucket_name, const string& bucket_id, - const rgw_obj_key& obj_key) + const rgw_obj_index_key& obj_key) { const string keyext = objexp_hint_get_keyext(tenant_name, bucket_name, bucket_id, obj_key); @@ -5124,7 +5124,7 @@ int rgw_policy_from_attrset(CephContext *cct, map& attrset, * common_prefixes: if delim is filled in, any matching prefixes are placed here. * is_truncated: if number of objects in the bucket is bigger than max, then truncated. */ -int RGWRados::Bucket::List::list_objects(int max, vector *result, +int RGWRados::Bucket::List::list_objects(int max, vector *result, map *common_prefixes, bool *is_truncated) { @@ -5142,13 +5142,13 @@ int RGWRados::Bucket::List::list_objects(int max, vector *result, rgw_obj marker_obj(b, params.marker); rgw_obj end_marker_obj(b, params.end_marker); rgw_obj prefix_obj; - rgw_obj_key cur_end_marker; + rgw_obj_index_key cur_end_marker; if (!params.ns.empty()) { marker_obj.set_ns(params.ns); end_marker_obj.set_ns(params.ns); end_marker_obj.get_index_key(&cur_end_marker); } - rgw_obj_key cur_marker; + rgw_obj_index_key cur_marker; marker_obj.get_index_key(&cur_marker); const bool cur_end_marker_valid = !params.end_marker.empty(); @@ -5186,27 +5186,26 @@ int RGWRados::Bucket::List::list_objects(int max, vector *result, cur_marker.set(skip_after_delim); ldout(cct, 20) << "setting cur_marker=" << cur_marker.name << "[" << cur_marker.instance << "]" << dendl; } - std::map ent_map; + std::map ent_map; int r = store->cls_bucket_list(target->get_bucket_info(), shard_id, cur_marker, cur_prefix, read_ahead + 1 - count, params.list_versions, ent_map, &truncated, &cur_marker); if (r < 0) return r; - std::map::iterator eiter; + std::map::iterator eiter; for (eiter = ent_map.begin(); eiter != ent_map.end(); ++eiter) { - rgw_obj_key obj = eiter->second.key; - RGWObjEnt& entry = eiter->second; - rgw_obj_key key = obj; - string instance; - string ns; + rgw_bucket_dir_entry& entry = eiter->second; + rgw_obj_index_key index_key = entry.key; - bool valid = rgw_obj::parse_raw_oid(obj.name, &obj.name, &instance, &ns); + rgw_obj_key obj(index_key); + + bool valid = rgw_obj::parse_raw_oid(index_key.name, &obj.name, &obj.instance, &obj.ns); if (!valid) { ldout(cct, 0) << "ERROR: could not parse object name: " << obj.name << dendl; continue; } - bool check_ns = (ns == params.ns); + bool check_ns = (obj.ns == params.ns); if (!params.list_versions && !entry.is_visible()) { continue; } @@ -5222,17 +5221,17 @@ int RGWRados::Bucket::List::list_objects(int max, vector *result, continue; } - if (cur_end_marker_valid && cur_end_marker <= obj) { + if (cur_end_marker_valid && cur_end_marker <= index_key) { truncated = false; goto done; } if (count < max) { - params.marker = key; - next_marker = key; + params.marker = index_key; + next_marker = index_key; } - if (params.filter && !params.filter->filter(obj.name, key.name)) + if (params.filter && !params.filter->filter(obj.name, index_key.name)) continue; if (params.prefix.size() && (obj.name.compare(0, params.prefix.size(), params.prefix) != 0)) @@ -5270,8 +5269,6 @@ int RGWRados::Bucket::List::list_objects(int max, vector *result, goto done; } - entry.key = obj; - entry.ns = ns; result->emplace_back(std::move(entry)); count++; } @@ -6088,7 +6085,7 @@ int RGWRados::BucketShard::init(const rgw_bucket& _bucket, int sid) int RGWRados::on_last_entry_in_listing(RGWBucketInfo& bucket_info, const std::string& obj_prefix, const std::string& obj_delim, - std::function handler) + std::function handler) { RGWRados::Bucket target(this, bucket_info); RGWRados::Bucket::List list_op(&target); @@ -6103,12 +6100,12 @@ int RGWRados::on_last_entry_in_listing(RGWBucketInfo& bucket_info, bool is_truncated = false; - boost::optional last_entry; + boost::optional last_entry; /* We need to rewind to the last object in a listing. */ do { /* List bucket entries in chunks. */ static constexpr int MAX_LIST_OBJS = 100; - std::vector entries(MAX_LIST_OBJS); + std::vector entries(MAX_LIST_OBJS); int ret = list_op.list_objects(MAX_LIST_OBJS, &entries, nullptr, &is_truncated); @@ -6246,7 +6243,7 @@ int RGWRados::swift_versioning_restore(RGWObjectCtx& obj_ctx, } /* This code will be executed on latest version of the object. */ - const auto handler = [&](const RGWObjEnt& entry) -> int { + const auto handler = [&](const rgw_bucket_dir_entry& entry) -> int { std::string no_client_id; std::string no_op_id; std::string no_zone; @@ -6504,7 +6501,7 @@ int RGWRados::Object::Write::_do_write_meta(uint64_t size, uint64_t accounted_si } if (!real_clock::is_zero(meta.delete_at)) { - rgw_obj_key obj_key; + rgw_obj_index_key obj_key; obj.get_index_key(&obj_key); r = store->objexp_hint_add(meta.delete_at, @@ -7823,8 +7820,8 @@ int RGWRados::delete_bucket(RGWBucketInfo& bucket_info, RGWObjVersionTracker& ob if (r < 0) return r; - std::map ent_map; - rgw_obj_key marker; + std::map ent_map; + rgw_obj_index_key marker; string prefix; bool is_truncated; @@ -7836,8 +7833,8 @@ int RGWRados::delete_bucket(RGWBucketInfo& bucket_info, RGWObjVersionTracker& ob return r; string ns; - std::map::iterator eiter; - rgw_obj_key obj; + std::map::iterator eiter; + rgw_obj_index_key obj; string instance; for (eiter = ent_map.begin(); eiter != ent_map.end(); ++eiter) { obj = eiter->second.key; @@ -9112,7 +9109,7 @@ int RGWRados::set_attrs(void *ctx, rgw_obj& obj, try { ::decode(ts, bl); - rgw_obj_key obj_key; + rgw_obj_index_key obj_key; obj.get_index_key(&obj_key); objexp_hint_add(ts, bucket.tenant, bucket.name, bucket.bucket_id, obj_key); @@ -9410,7 +9407,7 @@ int RGWRados::Bucket::UpdateIndex::complete(int64_t poolid, uint64_t epoch, const string& content_type, bufferlist *acl_bl, RGWObjCategory category, - list *remove_objs) + list *remove_objs) { if (blind) { return 0; @@ -9423,12 +9420,12 @@ int RGWRados::Bucket::UpdateIndex::complete(int64_t poolid, uint64_t epoch, return ret; } - RGWObjEnt ent; + rgw_bucket_dir_entry ent; obj.get_index_key(&ent.key); - ent.size = size; - ent.accounted_size = accounted_size; - ent.mtime = ut; - ent.etag = etag; + ent.meta.size = size; + ent.meta.accounted_size = accounted_size; + ent.meta.mtime = ut; + ent.meta.etag = etag; ACLOwner owner; if (acl_bl && acl_bl->length()) { int ret = store->decode_policy(*acl_bl, &owner); @@ -9436,9 +9433,9 @@ int RGWRados::Bucket::UpdateIndex::complete(int64_t poolid, uint64_t epoch, ldout(store->ctx(), 0) << "WARNING: could not decode policy ret=" << ret << dendl; } } - ent.owner = owner.get_id(); - ent.owner_display_name = owner.get_display_name(); - ent.content_type = content_type; + ent.meta.owner = owner.get_id().to_str(); + ent.meta.owner_display_name = owner.get_display_name(); + ent.meta.content_type = content_type; ret = store->cls_obj_complete_add(*bs, optag, poolid, epoch, ent, category, remove_objs, bilog_flags); @@ -9452,7 +9449,7 @@ int RGWRados::Bucket::UpdateIndex::complete(int64_t poolid, uint64_t epoch, int RGWRados::Bucket::UpdateIndex::complete_del(int64_t poolid, uint64_t epoch, real_time& removed_mtime, - list *remove_objs) + list *remove_objs) { if (blind) { return 0; @@ -11558,7 +11555,7 @@ int RGWRados::pool_iterate_begin(const rgw_pool& pool, RGWPoolIterCtx& ctx) return 0; } -int RGWRados::pool_iterate(RGWPoolIterCtx& ctx, uint32_t num, vector& objs, +int RGWRados::pool_iterate(RGWPoolIterCtx& ctx, uint32_t num, vector& objs, bool *is_truncated, RGWAccessListFilter *filter) { librados::IoCtx& io_ctx = ctx.io_ctx; @@ -11570,7 +11567,7 @@ int RGWRados::pool_iterate(RGWPoolIterCtx& ctx, uint32_t num, vector& uint32_t i; for (i = 0; i < num && iter != io_ctx.nobjects_end(); ++i, ++iter) { - RGWObjEnt e; + rgw_bucket_dir_entry e; string oid = iter->get_oid(); ldout(cct, 20) << "RGWRados::pool_iterate: got " << oid << dendl; @@ -11612,7 +11609,7 @@ int RGWRados::list_raw_objects(const rgw_pool& pool, const string& prefix_filter ctx.initialized = true; } - vector objs; + vector objs; int r = pool_iterate(ctx.iter_ctx, max, objs, is_truncated, &filter); if (r < 0) { if(r != -ENOENT) @@ -11620,7 +11617,7 @@ int RGWRados::list_raw_objects(const rgw_pool& pool, const string& prefix_filter return r; } - vector::iterator iter; + vector::iterator iter; for (iter = objs.begin(); iter != objs.end(); ++iter) { oids.push_back(iter->key.name); } @@ -11933,30 +11930,22 @@ int RGWRados::cls_obj_prepare_op(BucketShard& bs, RGWModifyOp op, string& tag, int RGWRados::cls_obj_complete_op(BucketShard& bs, RGWModifyOp op, string& tag, int64_t pool, uint64_t epoch, - RGWObjEnt& ent, RGWObjCategory category, - list *remove_objs, uint16_t bilog_flags) + rgw_bucket_dir_entry& ent, RGWObjCategory category, + list *remove_objs, uint16_t bilog_flags) { list *pro = NULL; list ro; if (remove_objs) { - for (list::iterator iter = remove_objs->begin(); iter != remove_objs->end(); ++iter) { - cls_rgw_obj_key k; - iter->transform(&k); - ro.push_back(k); + for (auto iter = remove_objs->begin(); iter != remove_objs->end(); ++iter) { + ro.push_back(*iter); } pro = &ro; } ObjectWriteOperation o; rgw_bucket_dir_entry_meta dir_meta; - dir_meta.size = ent.size; - dir_meta.accounted_size = ent.accounted_size; - dir_meta.mtime = ent.mtime; - dir_meta.etag = ent.etag; - dir_meta.owner = ent.owner.to_str(); - dir_meta.owner_display_name = ent.owner_display_name; - dir_meta.content_type = ent.content_type; + dir_meta = ent.meta; dir_meta.category = category; rgw_bucket_entry_ver ver; @@ -11974,8 +11963,8 @@ int RGWRados::cls_obj_complete_op(BucketShard& bs, RGWModifyOp op, string& tag, int RGWRados::cls_obj_complete_add(BucketShard& bs, string& tag, int64_t pool, uint64_t epoch, - RGWObjEnt& ent, RGWObjCategory category, - list *remove_objs, uint16_t bilog_flags) + rgw_bucket_dir_entry& ent, RGWObjCategory category, + list *remove_objs, uint16_t bilog_flags) { return cls_obj_complete_op(bs, CLS_RGW_OP_ADD, tag, pool, epoch, ent, category, remove_objs, bilog_flags); } @@ -11984,18 +11973,18 @@ int RGWRados::cls_obj_complete_del(BucketShard& bs, string& tag, int64_t pool, uint64_t epoch, rgw_obj& obj, real_time& removed_mtime, - list *remove_objs, + list *remove_objs, uint16_t bilog_flags) { - RGWObjEnt ent; - ent.mtime = removed_mtime; + rgw_bucket_dir_entry ent; + ent.meta.mtime = removed_mtime; obj.get_index_key(&ent.key); return cls_obj_complete_op(bs, CLS_RGW_OP_DEL, tag, pool, epoch, ent, RGW_OBJ_CATEGORY_NONE, remove_objs, bilog_flags); } int RGWRados::cls_obj_complete_cancel(BucketShard& bs, string& tag, rgw_obj& obj, uint16_t bilog_flags) { - RGWObjEnt ent; + rgw_bucket_dir_entry ent; obj.get_index_key(&ent.key); return cls_obj_complete_op(bs, CLS_RGW_OP_CANCEL, tag, -1 /* pool id */, 0, ent, RGW_OBJ_CATEGORY_NONE, NULL, bilog_flags); } @@ -12011,9 +12000,9 @@ int RGWRados::cls_obj_set_bucket_tag_timeout(RGWBucketInfo& bucket_info, uint64_ return CLSRGWIssueSetTagTimeout(index_ctx, bucket_objs, cct->_conf->rgw_bucket_index_max_aio, timeout)(); } -int RGWRados::cls_bucket_list(RGWBucketInfo& bucket_info, int shard_id, rgw_obj_key& start, const string& prefix, - uint32_t num_entries, bool list_versions, map& m, - bool *is_truncated, rgw_obj_key *last_entry, +int RGWRados::cls_bucket_list(RGWBucketInfo& bucket_info, int shard_id, rgw_obj_index_key& start, const string& prefix, + uint32_t num_entries, bool list_versions, map& m, + bool *is_truncated, rgw_obj_index_key *last_entry, bool (*force_check_filter)(const string& name)) { ldout(cct, 10) << "cls_bucket_list " << bucket_info.bucket << " start " << start.name << "[" << start.instance << "] num_entries " << num_entries << dendl; @@ -12065,34 +12054,20 @@ int RGWRados::cls_bucket_list(RGWBucketInfo& bucket_info, int shard_id, rgw_obj_ const string& name = vcurrents[pos]->first; struct rgw_bucket_dir_entry& dirent = vcurrents[pos]->second; - // fill it in with initial values; we may correct later - RGWObjEnt e; - e.key.set(dirent.key.name, dirent.key.instance); - e.size = dirent.meta.size; - e.accounted_size = dirent.meta.accounted_size; - e.mtime = dirent.meta.mtime; - e.etag = dirent.meta.etag; - e.owner = dirent.meta.owner; - e.owner_display_name = dirent.meta.owner_display_name; - e.content_type = dirent.meta.content_type; - e.tag = dirent.tag; - e.flags = dirent.flags; - e.versioned_epoch = dirent.versioned_epoch; - bool force_check = force_check_filter && force_check_filter(dirent.key.name); if ((!dirent.exists && !dirent.is_delete_marker()) || !dirent.pending_map.empty() || force_check) { /* there are uncommitted ops. We need to check the current state, * and if the tags are old we need to do cleanup as well. */ librados::IoCtx sub_ctx; sub_ctx.dup(index_ctx); - r = check_disk_state(sub_ctx, bucket_info.bucket, dirent, e, updates[vnames[pos]]); + r = check_disk_state(sub_ctx, bucket_info.bucket, dirent, dirent, updates[vnames[pos]]); if (r < 0 && r != -ENOENT) { return r; } } if (r >= 0) { - ldout(cct, 10) << "RGWRados::cls_bucket_list: got " << e.key.name << "[" << e.key.instance << "]" << dendl; - m[name] = std::move(e); + ldout(cct, 10) << "RGWRados::cls_bucket_list: got " << dirent.key.name << "[" << dirent.key.instance << "]" << dendl; + m[name] = std::move(dirent); ++count; } @@ -12184,7 +12159,7 @@ int RGWRados::cls_obj_usage_log_trim(string& oid, string& user, uint64_t start_e return r; } -int RGWRados::remove_objs_from_index(RGWBucketInfo& bucket_info, list& oid_list) +int RGWRados::remove_objs_from_index(RGWBucketInfo& bucket_info, list& oid_list) { librados::IoCtx index_ctx; string dir_oid; @@ -12197,14 +12172,11 @@ int RGWRados::remove_objs_from_index(RGWBucketInfo& bucket_info, list::iterator iter; - - for (iter = oid_list.begin(); iter != oid_list.end(); ++iter) { - rgw_obj_key& key = *iter; - dout(2) << "RGWRados::remove_objs_from_index bucket=" << bucket_info.bucket << " obj=" << key.name << ":" << key.instance << dendl; + for (auto iter = oid_list.begin(); iter != oid_list.end(); ++iter) { rgw_bucket_dir_entry entry; + entry.key = *iter; + dout(2) << "RGWRados::remove_objs_from_index bucket=" << bucket_info.bucket << " obj=" << entry.key.name << ":" << entry.key.instance << dendl; entry.ver.epoch = (uint64_t)-1; // ULLONG_MAX, needed to that objclass doesn't skip out request - key.transform(&entry.key); updates.append(CEPH_RGW_REMOVE | suggest_flag); ::encode(entry, updates); } @@ -12219,23 +12191,18 @@ int RGWRados::remove_objs_from_index(RGWBucketInfo& bucket_info, list::iterator iter; + vector::iterator iter; for (iter = objs.begin(); iter != objs.end(); ++iter) { const char *section_name = (iter->is_delete_marker() ? "DeleteMarker" : "Version"); @@ -527,14 +527,15 @@ void RGWListBucket_ObjStore_S3::send_versioned_response() if (objs_container) { s->formatter->dump_bool("IsDeleteMarker", iter->is_delete_marker()); } + rgw_obj_key key(iter->key); if (encode_key) { string key_name; - url_encode(iter->key.name, key_name); + url_encode(key.name, key_name); s->formatter->dump_string("Key", key_name); } else { - s->formatter->dump_string("Key", iter->key.name); + s->formatter->dump_string("Key", key.name); } - string version_id = iter->key.instance; + string version_id = key.instance; if (version_id.empty()) { version_id = "null"; } @@ -543,18 +544,18 @@ void RGWListBucket_ObjStore_S3::send_versioned_response() s->formatter->dump_int("VersionedEpoch", iter->versioned_epoch); } s->formatter->dump_string("RgwxTag", iter->tag); - utime_t ut(iter->mtime); + utime_t ut(iter->meta.mtime); ut.gmtime_nsec(s->formatter->dump_stream("RgwxMtime")); } s->formatter->dump_string("VersionId", version_id); s->formatter->dump_bool("IsLatest", iter->is_current()); - dump_time(s, "LastModified", &iter->mtime); + dump_time(s, "LastModified", &iter->meta.mtime); if (!iter->is_delete_marker()) { - s->formatter->dump_format("ETag", "\"%s\"", iter->etag.c_str()); - s->formatter->dump_int("Size", iter->accounted_size); + s->formatter->dump_format("ETag", "\"%s\"", iter->meta.etag.c_str()); + s->formatter->dump_int("Size", iter->meta.accounted_size); s->formatter->dump_string("StorageClass", "STANDARD"); } - dump_owner(s, iter->owner, iter->owner_display_name); + dump_owner(s, iter->meta.owner, iter->meta.owner_display_name); s->formatter->close_section(); } if (objs_container) { @@ -613,21 +614,22 @@ void RGWListBucket_ObjStore_S3::send_response() } if (op_ret >= 0) { - vector::iterator iter; + vector::iterator iter; for (iter = objs.begin(); iter != objs.end(); ++iter) { + rgw_obj_key key(iter->key); s->formatter->open_array_section("Contents"); if (encode_key) { string key_name; - url_encode(iter->key.name, key_name); + url_encode(key.name, key_name); s->formatter->dump_string("Key", key_name); } else { - s->formatter->dump_string("Key", iter->key.name); + s->formatter->dump_string("Key", key.name); } - dump_time(s, "LastModified", &iter->mtime); - s->formatter->dump_format("ETag", "\"%s\"", iter->etag.c_str()); - s->formatter->dump_int("Size", iter->accounted_size); + dump_time(s, "LastModified", &iter->meta.mtime); + s->formatter->dump_format("ETag", "\"%s\"", iter->meta.etag.c_str()); + s->formatter->dump_int("Size", iter->meta.accounted_size); s->formatter->dump_string("StorageClass", "STANDARD"); - dump_owner(s, iter->owner, iter->owner_display_name); + dump_owner(s, iter->meta.owner, iter->meta.owner_display_name); if (s->system_request) { s->formatter->dump_string("RgwxTag", iter->tag); } @@ -2800,7 +2802,7 @@ void RGWListBucketMultiparts_ObjStore_S3::send_response() dump_owner(s, s->user->user_id, s->user->display_name, "Initiator"); dump_owner(s, s->user->user_id, s->user->display_name); s->formatter->dump_string("StorageClass", "STANDARD"); - dump_time(s, "Initiated", &iter->obj.mtime); + dump_time(s, "Initiated", &iter->obj.meta.mtime); s->formatter->close_section(); } if (!common_prefixes.empty()) { diff --git a/src/rgw/rgw_rest_swift.cc b/src/rgw/rgw_rest_swift.cc index 23353a28b8f..757c8e57284 100644 --- a/src/rgw/rgw_rest_swift.cc +++ b/src/rgw/rgw_rest_swift.cc @@ -271,7 +271,7 @@ static void dump_container_metadata(struct req_state *, void RGWListBucket_ObjStore_SWIFT::send_response() { - vector::iterator iter = objs.begin(); + vector::iterator iter = objs.begin(); map::iterator pref_iter = common_prefixes.begin(); dump_start(s); @@ -283,7 +283,7 @@ void RGWListBucket_ObjStore_SWIFT::send_response() while (iter != objs.end() || pref_iter != common_prefixes.end()) { bool do_pref = false; bool do_objs = false; - rgw_obj_key& key = iter->key; + rgw_obj_key key(iter->key); if (pref_iter == common_prefixes.end()) do_objs = true; else if (iter == objs.end()) @@ -302,12 +302,12 @@ void RGWListBucket_ObjStore_SWIFT::send_response() s->formatter->open_object_section("object"); s->formatter->dump_string("name", key.name); - s->formatter->dump_string("hash", iter->etag); - s->formatter->dump_int("bytes", iter->accounted_size); - string single_content_type = iter->content_type; - if (iter->content_type.size()) { + s->formatter->dump_string("hash", iter->meta.etag); + s->formatter->dump_int("bytes", iter->meta.accounted_size); + string single_content_type = iter->meta.content_type; + if (iter->meta.content_type.size()) { // content type might hold multiple values, just dump the last one - ssize_t pos = iter->content_type.rfind(','); + ssize_t pos = iter->meta.content_type.rfind(','); if (pos > 0) { ++pos; while (single_content_type[pos] == ' ') @@ -316,7 +316,7 @@ void RGWListBucket_ObjStore_SWIFT::send_response() } s->formatter->dump_string("content_type", single_content_type); } - dump_time(s, "last_modified", &iter->mtime); + dump_time(s, "last_modified", &iter->meta.mtime); s->formatter->close_section(); } @@ -1775,7 +1775,7 @@ RGWOp* RGWSwiftWebsiteHandler::get_ws_listing_op() htmler.dump_subdir(subdir_name); } - for (const RGWObjEnt& obj : objs) { + for (const rgw_bucket_dir_entry& obj : objs) { if (! common_prefixes.count(obj.key.name + '/')) { htmler.dump_object(obj); } -- 2.39.5