From dbf8cf0b5d46cc49eaf0a14bcc05a4fb215d98fb Mon Sep 17 00:00:00 2001 From: Radoslaw Zarzynski Date: Fri, 13 May 2016 18:23:12 +0200 Subject: [PATCH] rgw: improve support for Swift's object versioning. This patch allows RadosGW to pass the RefStack with an accuracy to the RFC7230 violation issue which is clearly a Tempest bug. Fixes: http://tracker.ceph.com/issues/15925 Signed-off-by: Radoslaw Zarzynski (cherry picked from commit 237ad1210f5a6e4f191293ce532ca15869612a93) Conflicts: src/rgw/rgw_op.cc In contrast to master, Jewel doesn't support container quota of Swift API. All tracks of this feature have been eradicated from the patch. src/rgw/rgw_rest_swift.cc Jewel doesn't include boost/optional.hpp but has boost/utility/in_place_factory.hpp. The conflict has been resolved to pull in both headers. --- doc/radosgw/config-ref.rst | 14 +++ doc/radosgw/swift.rst | 2 +- src/rgw/rgw_common.h | 7 +- src/rgw/rgw_op.cc | 100 ++++++++++++++----- src/rgw/rgw_op.h | 6 +- src/rgw/rgw_rados.cc | 191 +++++++++++++++++++++++++++++++++---- src/rgw/rgw_rados.h | 36 +++++-- src/rgw/rgw_rest_swift.cc | 39 ++++++-- 8 files changed, 331 insertions(+), 64 deletions(-) diff --git a/doc/radosgw/config-ref.rst b/doc/radosgw/config-ref.rst index 8bd24756eee2c..c41f686da334b 100644 --- a/doc/radosgw/config-ref.rst +++ b/doc/radosgw/config-ref.rst @@ -778,6 +778,20 @@ Swift Settings :Default: ``auth`` +``rgw swift versioning enabled`` + +:Description: Enables the Object Versioning of OpenStack Object Storage API. + This allows clients to put the ``X-Versions-Location`` attribute + on containers that should be versioned. The attribute specifies + the name of container storing archived versions. It must be owned + by the same user that the versioned container due to access + control verification - ACLs are NOT taken into consideration. + Those containers cannot be versioned by the S3 object versioning + mechanism. +:Type: Boolean +:Default: ``false`` + + Logging Settings ================ diff --git a/doc/radosgw/swift.rst b/doc/radosgw/swift.rst index dc445ddc2264a..6113b7216a497 100644 --- a/doc/radosgw/swift.rst +++ b/doc/radosgw/swift.rst @@ -67,7 +67,7 @@ The following table describes the support status for current Swift functional fe +---------------------------------+-----------------+----------------------------------------+ | **Expiring Objects** | Supported | | +---------------------------------+-----------------+----------------------------------------+ -| **Object Versioning** | Not Supported | | +| **Object Versioning** | Supported | | +---------------------------------+-----------------+----------------------------------------+ | **CORS** | Not Supported | | +---------------------------------+-----------------+----------------------------------------+ diff --git a/src/rgw/rgw_common.h b/src/rgw/rgw_common.h index a907f441bfcda..ccdb2d08ee04a 100644 --- a/src/rgw/rgw_common.h +++ b/src/rgw/rgw_common.h @@ -992,11 +992,14 @@ struct RGWBucketInfo void decode_json(JSONObj *obj); - bool versioned() { return (flags & BUCKET_VERSIONED) != 0; } + bool versioned() const { return (flags & BUCKET_VERSIONED) != 0; } int versioning_status() { return flags & (BUCKET_VERSIONED | BUCKET_VERSIONS_SUSPENDED); } bool versioning_enabled() { return versioning_status() == BUCKET_VERSIONED; } - bool has_swift_versioning() { return swift_versioning; } + bool has_swift_versioning() const { + /* A bucket may be versioned through one mechanism only. */ + return swift_versioning && !versioned(); + } RGWBucketInfo() : flags(0), has_instance_obj(false), num_shards(0), bucket_index_shard_hash_type(MOD), requester_pays(false), has_website(false), swift_versioning(false) {} diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc index 218526b93ae2f..9183fdf7c389c 100644 --- a/src/rgw/rgw_op.cc +++ b/src/rgw/rgw_op.cc @@ -2000,10 +2000,17 @@ void RGWCreateBucket::execute() s->bucket.tenant = s->bucket_tenant; /* ignored if bucket exists */ s->bucket.name = s->bucket_name; + + /* Handle updates of the metadata for Swift's object versioning. */ + if (swift_ver_location) { + s->bucket_info.swift_ver_location = *swift_ver_location; + s->bucket_info.swift_versioning = (! swift_ver_location->empty()); + } + op_ret = store->create_bucket(*(s->user), s->bucket, zonegroup_id, - placement_rule, swift_ver_location, attrs, - info, pobjv, &ep_objv, creation_time, - pmaster_bucket, true); + placement_rule, s->bucket_info.swift_ver_location, + attrs, info, pobjv, &ep_objv, creation_time, + pmaster_bucket, true); /* continue if EEXIST and create_bucket will fail below. this way we can * recover from a partial create by retrying it. */ ldout(s->cct, 20) << "rgw_create_bucket returned ret=" << op_ret << " bucket=" << s->bucket << dendl; @@ -2071,6 +2078,12 @@ void RGWCreateBucket::execute() prepare_add_del_attrs(s->bucket_attrs, rmattr_names, attrs); populate_with_generic_attrs(s, attrs); + /* Handle updates of the metadata for Swift's object versioning. */ + if (swift_ver_location) { + s->bucket_info.swift_ver_location = *swift_ver_location; + s->bucket_info.swift_versioning = (! swift_ver_location->empty()); + } + op_ret = rgw_bucket_set_attrs(store, s->bucket_info, attrs, &s->bucket_info.objv_tracker); } while (op_ret == -ECANCELED && tries++ < 20); @@ -2414,6 +2427,18 @@ void RGWPutObj::execute() processor = select_processor(*static_cast(s->obj_ctx), &multipart); + /* Handle object versioning of Swift API. */ + if (! multipart) { + rgw_obj obj(s->bucket, s->object); + op_ret = store->swift_versioning_copy(*static_cast(s->obj_ctx), + s->bucket_owner.get_id(), + s->bucket_info, + obj); + if (op_ret < 0) { + return; + } + } + op_ret = processor->prepare(store, NULL); if (op_ret < 0) { ldout(s->cct, 20) << "processor->prepare() returned ret=" << op_ret @@ -2883,8 +2908,10 @@ void RGWPutMetadataBucket::execute() prepare_add_del_attrs(s->bucket_attrs, rmattr_names, attrs); populate_with_generic_attrs(s, attrs); - s->bucket_info.swift_ver_location = swift_ver_location; - s->bucket_info.swift_versioning = (!swift_ver_location.empty()); + if (swift_ver_location) { + s->bucket_info.swift_ver_location = *swift_ver_location; + s->bucket_info.swift_versioning = (! swift_ver_location->empty()); + } op_ret = rgw_bucket_set_attrs(store, s->bucket_info, attrs, &s->bucket_info.objv_tracker); @@ -3056,35 +3083,46 @@ void RGWDeleteObj::execute() } RGWObjectCtx *obj_ctx = static_cast(s->obj_ctx); - obj_ctx->set_atomic(obj); - RGWRados::Object del_target(store, s->bucket_info, *obj_ctx, obj); - RGWRados::Object::Delete del_op(&del_target); - - op_ret = get_system_versioning_params(s, &del_op.params.olh_epoch, - &del_op.params.marker_version_id); + bool ver_restored = false; + op_ret = store->swift_versioning_restore(*obj_ctx, s->bucket_owner.get_id(), + s->bucket_info, obj, ver_restored); if (op_ret < 0) { return; } - del_op.params.bucket_owner = s->bucket_owner.get_id(); - del_op.params.versioning_status = s->bucket_info.versioning_status(); - del_op.params.obj_owner = s->owner; - del_op.params.unmod_since = unmod_since; - del_op.params.high_precision_time = s->system_request; /* system request uses high precision time */ + if (!ver_restored) { + /* Swift's versioning mechanism hasn't found any previous version of + * the object that could be restored. This means we should proceed + * with the regular delete path. */ + RGWRados::Object del_target(store, s->bucket_info, *obj_ctx, obj); + RGWRados::Object::Delete del_op(&del_target); - op_ret = del_op.delete_obj(); - if (op_ret >= 0) { - delete_marker = del_op.result.delete_marker; - version_id = del_op.result.version_id; - } + op_ret = get_system_versioning_params(s, &del_op.params.olh_epoch, + &del_op.params.marker_version_id); + if (op_ret < 0) { + return; + } - /* Check whether the object has expired. Swift API documentation - * stands that we should return 404 Not Found in such case. */ - if (need_object_expiration() && object_is_expired(attrs)) { - op_ret = -ENOENT; - return; + del_op.params.bucket_owner = s->bucket_owner.get_id(); + del_op.params.versioning_status = s->bucket_info.versioning_status(); + del_op.params.obj_owner = s->owner; + del_op.params.unmod_since = unmod_since; + del_op.params.high_precision_time = s->system_request; /* system request uses high precision time */ + + op_ret = del_op.delete_obj(); + if (op_ret >= 0) { + delete_marker = del_op.result.delete_marker; + version_id = del_op.result.version_id; + } + + /* Check whether the object has expired. Swift API documentation + * stands that we should return 404 Not Found in such case. */ + if (need_object_expiration() && object_is_expired(attrs)) { + op_ret = -ENOENT; + return; + } } if (op_ret == -ERR_PRECONDITION_FAILED && no_precondition_error) { @@ -3306,6 +3344,16 @@ void RGWCopyObj::execute() bool high_precision_time = (s->system_request); + /* Handle object versioning of Swift API. In case of copying to remote this + * should fail gently (op_ret == 0) as the dst_obj will not exist here. */ + op_ret = store->swift_versioning_copy(obj_ctx, + dest_bucket_info.owner, + dest_bucket_info, + dst_obj); + if (op_ret < 0) { + return; + } + op_ret = store->copy_obj(obj_ctx, s->user->user_id, client_id, diff --git a/src/rgw/rgw_op.h b/src/rgw/rgw_op.h index d0884ea5b3b09..f66d87700c910 100644 --- a/src/rgw/rgw_op.h +++ b/src/rgw/rgw_op.h @@ -18,6 +18,8 @@ #include #include +#include + #include "common/armor.h" #include "common/mime.h" #include "common/utf8.h" @@ -535,7 +537,7 @@ protected: obj_version ep_objv; bool has_cors; RGWCORSConfiguration cors_config; - string swift_ver_location; + boost::optional swift_ver_location; map attrs; set rmattr_names; @@ -781,7 +783,7 @@ protected: RGWAccessControlPolicy policy; RGWCORSConfiguration cors_config; string placement_rule; - string swift_ver_location; + boost::optional swift_ver_location; public: RGWPutMetadataBucket() diff --git a/src/rgw/rgw_rados.cc b/src/rgw/rgw_rados.cc index a76c276bb2cf6..303b0907cad01 100644 --- a/src/rgw/rgw_rados.cc +++ b/src/rgw/rgw_rados.cc @@ -6,6 +6,9 @@ #include #include +#include +#include + #include "common/ceph_json.h" #include "common/utf8.h" @@ -5693,13 +5696,69 @@ int RGWRados::BucketShard::init(rgw_bucket& _bucket, rgw_obj& obj) } -int RGWRados::swift_versioning_copy(RGWBucketInfo& bucket_info, RGWRados::Object *source, RGWObjState *state, - rgw_user& user) +/* Execute @handler on last item in bucket listing for bucket specified + * in @bucket_info. @obj_prefix and @obj_delim narrow down the listing + * to objects matching these criterias. */ +int RGWRados::on_last_entry_in_listing(RGWBucketInfo& bucket_info, + const std::string& obj_prefix, + const std::string& obj_delim, + std::function handler) +{ + RGWRados::Bucket target(this, bucket_info); + RGWRados::Bucket::List list_op(&target); + + list_op.params.prefix = obj_prefix; + list_op.params.delim = obj_delim; + + ldout(cct, 20) << "iterating listing for bucket=" << bucket_info.bucket.name + << ", obj_prefix=" << obj_prefix + << ", obj_delim=" << obj_delim + << dendl; + + bool is_truncated = false; + + boost::optional last_entry; + /* We need to rewind to the last object in a listing. */ + do { + /* List bucket entries in chunks. */ + static constexpr int MAX_LIST_OBJS = 100; + std::vector entries(MAX_LIST_OBJS); + + int ret = list_op.list_objects(MAX_LIST_OBJS, &entries, nullptr, + &is_truncated); + if (ret < 0) { + return ret; + } else if (!entries.empty()) { + last_entry = last_entry = entries.back(); + } + } while (is_truncated); + + if (last_entry) { + return handler(*last_entry); + } + + /* Empty listing - no items we can run handler on. */ + return 0; +} + + +int RGWRados::swift_versioning_copy(RGWObjectCtx& obj_ctx, + const rgw_user& user, + RGWBucketInfo& bucket_info, + rgw_obj& obj) { - if (!bucket_info.has_swift_versioning() || bucket_info.swift_ver_location.empty()) { + if (! swift_versioning_enabled(bucket_info)) { return 0; } + obj_ctx.set_atomic(obj); + + RGWObjState * state = nullptr; + int r = get_obj_state(&obj_ctx, obj, &state, false); + if (r < 0) { + return r; + } + if (!state->exists) { return 0; } @@ -5707,16 +5766,15 @@ int RGWRados::swift_versioning_copy(RGWBucketInfo& bucket_info, RGWRados::Object string client_id; string op_id; - rgw_obj& obj = source->get_obj(); const string& src_name = obj.get_object(); char buf[src_name.size() + 32]; struct timespec ts = ceph::real_clock::to_timespec(state->mtime); - snprintf(buf, sizeof(buf), "%03d%s/%lld.%06ld", (int)src_name.size(), + snprintf(buf, sizeof(buf), "%03x%s/%lld.%06ld", (int)src_name.size(), src_name.c_str(), (long long)ts.tv_sec, ts.tv_nsec / 1000); RGWBucketInfo dest_bucket_info; - int r = get_bucket_info(source->get_ctx(), bucket_info.bucket.tenant, bucket_info.swift_ver_location, dest_bucket_info, NULL, NULL); + r = get_bucket_info(obj_ctx, bucket_info.bucket.tenant, bucket_info.swift_ver_location, dest_bucket_info, NULL, NULL); if (r < 0) { ldout(cct, 10) << "failed to read dest bucket info: r=" << r << dendl; return r; @@ -5727,10 +5785,11 @@ int RGWRados::swift_versioning_copy(RGWBucketInfo& bucket_info, RGWRados::Object } rgw_obj dest_obj(dest_bucket_info.bucket, buf); + obj_ctx.set_atomic(dest_obj); string no_zone; - r = copy_obj(source->get_ctx(), + r = copy_obj(obj_ctx, user, client_id, op_id, @@ -5759,13 +5818,121 @@ int RGWRados::swift_versioning_copy(RGWBucketInfo& bucket_info, RGWRados::Object NULL, /* struct rgw_err *err */ NULL, /* void (*progress_cb)(off_t, void *) */ NULL); /* void *progress_data */ - if (r == -ECANCELED || r == -ENOENT) { /* has already been overwritten, meaning another rgw process already copied it out */ + if (r == -ECANCELED || r == -ENOENT) { + /* Has already been overwritten, meaning another rgw process already + * copied it out */ return 0; } return r; } +int RGWRados::swift_versioning_restore(RGWObjectCtx& obj_ctx, + const rgw_user& user, + RGWBucketInfo& bucket_info, + rgw_obj& obj, + bool& restored) /* out */ +{ + if (! swift_versioning_enabled(bucket_info)) { + return 0; + } + + /* Bucket info of the bucket that stores previous versions of our object. */ + RGWBucketInfo archive_binfo; + + int ret = get_bucket_info(obj_ctx, bucket_info.bucket.tenant, + bucket_info.swift_ver_location, archive_binfo, + nullptr, nullptr); + if (ret < 0) { + return ret; + } + + /* Abort the operation if the bucket storing our archive belongs to someone + * else. This is a limitation in comparison to Swift as we aren't taking ACLs + * into consideration. For we can live with that. + * + * TODO: delegate this check to un upper layer and compare with ACLs. */ + if (bucket_info.owner != archive_binfo.owner) { + return -EPERM; + } + + /* This code will be executed on latest version of the object. */ + const auto handler = [&](const RGWObjEnt& entry) -> int { + std::string no_client_id; + std::string no_op_id; + std::string no_zone; + + /* We don't support object versioning of Swift API on those buckets that + * are already versioned using the S3 mechanism. This affects also bucket + * storing archived objects. Otherwise the delete operation would create + * a deletion marker. */ + if (archive_binfo.versioned()) { + restored = false; + return -ERR_PRECONDITION_FAILED; + } + + /* We are requesting ATTRSMOD_NONE so the attr attribute is perfectly + * irrelevant and may be safely skipped. */ + std::map no_attrs; + + rgw_obj archive_obj(archive_binfo.bucket, entry.key); + obj_ctx.set_atomic(archive_obj); + obj_ctx.set_atomic(obj); + + int ret = copy_obj(obj_ctx, + user, + no_client_id, + no_op_id, + nullptr, /* req_info *info */ + no_zone, + obj, /* dest obj */ + archive_obj, /* src obj */ + bucket_info, /* dest bucket info */ + archive_binfo, /* src bucket info */ + nullptr, /* time_t *src_mtime */ + nullptr, /* time_t *mtime */ + nullptr, /* const time_t *mod_ptr */ + nullptr, /* const time_t *unmod_ptr */ + false, /* bool high_precision_time */ + nullptr, /* const char *if_match */ + nullptr, /* const char *if_nomatch */ + RGWRados::ATTRSMOD_NONE, + true, /* bool copy_if_newer */ + no_attrs, + RGW_OBJ_CATEGORY_MAIN, + 0, /* uint64_t olh_epoch */ + real_time(), /* time_t delete_at */ + nullptr, /* string *version_id */ + nullptr, /* string *ptag */ + nullptr, /* string *petag */ + nullptr, /* struct rgw_err *err */ + nullptr, /* void (*progress_cb)(off_t, void *) */ + nullptr); /* void *progress_data */ + if (ret == -ECANCELED || ret == -ENOENT) { + /* Has already been overwritten, meaning another rgw process already + * copied it out */ + return 0; + } else if (ret < 0) { + return ret; + } else { + restored = true; + } + + /* Need to remove the archived copy. */ + ret = delete_obj(obj_ctx, archive_binfo, archive_obj, + archive_binfo.versioning_status()); + + return ret; + }; + + const std::string& obj_name = obj.get_object(); + const auto prefix = boost::str(boost::format("%03x%s") % obj_name.size() + % obj_name); + + return on_last_entry_in_listing(archive_binfo, prefix, std::string(), + handler); +} + /** * Write/overwrite an object to the bucket storage. * bucket: the bucket to store the object in @@ -5901,10 +6068,6 @@ int RGWRados::Object::Write::write_meta(uint64_t size, index_op.set_bilog_flags(RGW_BILOG_FLAG_VERSIONED_OP); } - r = store->swift_versioning_copy(bucket_info, target, state, meta.owner); - if (r < 0) { - goto done_cancel; - } r = index_op.prepare(CLS_RGW_OP_ADD); if (r < 0) @@ -7546,10 +7709,6 @@ int RGWRados::Object::Delete::delete_obj() index_op.set_bilog_flags(params.bilog_flags); - r = store->swift_versioning_copy(bucket_info, target, state, params.bucket_owner); - if (r < 0) { - return r; - } r = index_op.prepare(CLS_RGW_OP_DEL); if (r < 0) diff --git a/src/rgw/rgw_rados.h b/src/rgw/rgw_rados.h index 9704fb6d1d61b..041979a806989 100644 --- a/src/rgw/rgw_rados.h +++ b/src/rgw/rgw_rados.h @@ -4,6 +4,8 @@ #ifndef CEPH_RGWRADOS_H #define CEPH_RGWRADOS_H +#include + #include "include/rados/librados.hpp" #include "include/Context.h" #include "common/RefCountedObj.h" @@ -2442,6 +2444,32 @@ public: virtual int aio_wait(void *handle); virtual bool aio_completed(void *handle); + int on_last_entry_in_listing(RGWBucketInfo& bucket_info, + const std::string& obj_prefix, + const std::string& obj_delim, + std::function handler); + + bool swift_versioning_enabled(const RGWBucketInfo& bucket_info) const { + return bucket_info.has_swift_versioning() && + bucket_info.swift_ver_location.size(); + } + + int swift_versioning_copy(RGWObjectCtx& obj_ctx, /* in/out */ + const rgw_user& user, /* in */ + RGWBucketInfo& bucket_info, /* in */ + rgw_obj& obj); /* in */ + int swift_versioning_restore(RGWObjectCtx& obj_ctx, /* in/out */ + const rgw_user& user, /* in */ + RGWBucketInfo& bucket_info, /* in */ + rgw_obj& obj, /* in */ + bool& restored); /* out */ + int copy_obj_to_remote_dest(RGWObjState *astate, + map& src_attrs, + RGWRados::Object::Read& read_op, + const rgw_user& user_id, + rgw_obj& dest_obj, + ceph::real_time *mtime); + enum AttrsMod { ATTRSMOD_NONE = 0, ATTRSMOD_REPLACE = 1, @@ -2479,14 +2507,6 @@ public: struct rgw_err *err, void (*progress_cb)(off_t, void *), void *progress_data); - int swift_versioning_copy(RGWBucketInfo& bucket_info, RGWRados::Object *source, RGWObjState *state, - rgw_user& user); - int copy_obj_to_remote_dest(RGWObjState *astate, - map& src_attrs, - RGWRados::Object::Read& read_op, - const rgw_user& user_id, - rgw_obj& dest_obj, - ceph::real_time *mtime); /** * Copy an object. * dest_obj: the object to copy into diff --git a/src/rgw/rgw_rest_swift.cc b/src/rgw/rgw_rest_swift.cc index 00acfc6891377..a6d362363f1ac 100644 --- a/src/rgw/rgw_rest_swift.cc +++ b/src/rgw/rgw_rest_swift.cc @@ -1,6 +1,7 @@ // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- // vim: ts=8 sw=2 smarttab +#include #include #include "include/assert.h" @@ -507,6 +508,33 @@ static void get_rmattrs_from_headers(const req_state * const s, } } +static int get_swift_versioning_settings( + req_state * const s, + boost::optional& swift_ver_location) +{ + /* Removing the Swift's versions location has lower priority than setting + * a new one. That's the reason why we're handling it first. */ + const std::string vlocdel = + s->info.env->get("HTTP_X_REMOVE_VERSIONS_LOCATION", ""); + if (vlocdel.size()) { + swift_ver_location = boost::in_place(std::string()); + } + + std::string vloc = s->info.env->get("HTTP_X_VERSIONS_LOCATION", ""); + if (vloc.size()) { + /* If the Swift's versioning is globally disabled but someone wants to + * enable it for a given container, new version of Swift will generate + * the precondition failed error. */ + if (! s->cct->_conf->rgw_swift_versioning_enabled) { + return -ERR_PRECONDITION_FAILED; + } + + swift_ver_location = std::move(vloc); + } + + return 0; +} + int RGWCreateBucket_ObjStore_SWIFT::get_params() { bool has_policy; @@ -525,11 +553,7 @@ int RGWCreateBucket_ObjStore_SWIFT::get_params() CONT_REMOVE_ATTR_PREFIX, rmattr_names); placement_rule = s->info.env->get("HTTP_X_STORAGE_POLICY", ""); - if (s->cct->_conf->rgw_swift_versioning_enabled) { - swift_ver_location = s->info.env->get("HTTP_X_VERSIONS_LOCATION", ""); - } - - return 0; + return get_swift_versioning_settings(s, swift_ver_location); } void RGWCreateBucket_ObjStore_SWIFT::send_response() @@ -743,10 +767,7 @@ int RGWPutMetadataBucket_ObjStore_SWIFT::get_params() rmattr_names); placement_rule = s->info.env->get("HTTP_X_STORAGE_POLICY", ""); - if (s->cct->_conf->rgw_swift_versioning_enabled) { - swift_ver_location = s->info.env->get("HTTP_X_VERSIONS_LOCATION", ""); - } - return 0; + return get_swift_versioning_settings(s, swift_ver_location); } void RGWPutMetadataBucket_ObjStore_SWIFT::send_response() -- 2.39.5