From 5ffd58107994f0c40730babc65341e852faa83b7 Mon Sep 17 00:00:00 2001 From: Yehuda Sadeh Date: Tue, 13 Nov 2018 18:29:54 -0800 Subject: [PATCH] rgw: archive: fixes and api adjustments following rebase Signed-off-by: Yehuda Sadeh --- src/cls/version/cls_version.cc | 2 +- src/rgw/rgw_bucket.cc | 9 ++++----- src/rgw/rgw_cr_rados.cc | 2 +- src/rgw/rgw_cr_rados.h | 8 +++++++- src/rgw/rgw_data_sync.cc | 24 ++++++++++++++++++------ src/rgw/rgw_data_sync.h | 1 + src/rgw/rgw_rados.cc | 9 +++++++-- src/rgw/rgw_rados.h | 1 + 8 files changed, 40 insertions(+), 16 deletions(-) diff --git a/src/cls/version/cls_version.cc b/src/cls/version/cls_version.cc index aa10e2a898f..61f9fec022b 100644 --- a/src/cls/version/cls_version.cc +++ b/src/cls/version/cls_version.cc @@ -70,6 +70,7 @@ static int read_version(cls_method_context_t hctx, obj_version *objv, bool impli CLS_LOG(0, "ERROR: read_version(): failed to decode version entry\n"); return -EIO; } + CLS_LOG(20, "cls_version: read_version %s:%d", objv->tag.c_str(), (int)objv->ver); return 0; } @@ -185,7 +186,6 @@ static int cls_version_check(cls_method_context_t hctx, bufferlist *in, bufferli int ret = read_version(hctx, &objv, false); if (ret < 0) return ret; - CLS_LOG(20, "cls_version: read_version %s:%d", objv.tag.c_str(), (int)objv.ver); if (!check_conds(op.conds, objv)) { CLS_LOG(20, "cls_version: failed condition check"); diff --git a/src/rgw/rgw_bucket.cc b/src/rgw/rgw_bucket.cc index 9a8b290f22c..55c4995174a 100644 --- a/src/rgw/rgw_bucket.cc +++ b/src/rgw/rgw_bucket.cc @@ -2513,7 +2513,7 @@ struct archive_meta_info { return false; } - auto bliter = iter->second.begin(); + auto bliter = iter->second.cbegin(); try { decode(bliter); } catch (buffer::error& err) { @@ -2534,7 +2534,7 @@ struct archive_meta_info { ENCODE_FINISH(bl); } - void decode(bufferlist::iterator& bl) { + void decode(bufferlist::const_iterator& bl) { DECODE_START(1, bl); decode(orig_bucket, bl); DECODE_FINISH(bl); @@ -2555,7 +2555,7 @@ public: /* read original entrypoint */ RGWBucketEntryPoint be; - RGWObjectCtx obj_ctx(store); + auto obj_ctx = store->svc.sysobj->init_obj_ctx(); map attrs; int ret = store->get_bucket_entrypoint_info(obj_ctx, tenant_name, bucket_name, be, &objv_tracker, &mtime, &attrs); if (ret < 0) { @@ -2598,7 +2598,6 @@ public: get_md5_digest(&new_be, md5_digest); new_bucket_name = ami.orig_bucket.name + "-deleted-" + md5_digest; -ldout(store->ctx(), 0) << __FILE__ << ":" << __LINE__ << ": new_bucket_name==" << new_bucket_name << dendl; new_bi.bucket.name = new_bucket_name; new_bi.objv_tracker.clear(); @@ -2647,7 +2646,7 @@ ldout(store->ctx(), 0) << __FILE__ << ":" << __LINE__ << ": new_bucket_name==" < lderr(store->ctx()) << "could not delete bucket=" << entry << dendl; } - ret = rgw_delete_system_obj(store, store->get_zone_params().domain_root, RGW_BUCKET_INSTANCE_MD_PREFIX + meta_name, NULL); + ret = rgw_delete_system_obj(store, store->svc.zone->get_zone_params().domain_root, RGW_BUCKET_INSTANCE_MD_PREFIX + meta_name, NULL); /* idempotent */ diff --git a/src/rgw/rgw_cr_rados.cc b/src/rgw/rgw_cr_rados.cc index b83ad4a6299..6f12ca85958 100644 --- a/src/rgw/rgw_cr_rados.cc +++ b/src/rgw/rgw_cr_rados.cc @@ -582,7 +582,7 @@ int RGWAsyncFetchRemoteObj::_send_request() rgw_obj src_obj(bucket_info.bucket, key); - rgw_obj dest_obj(src_obj); + rgw_obj dest_obj(bucket_info.bucket, dest_key.value_or(key)); int r = store->fetch_remote_obj(obj_ctx, user_id, diff --git a/src/rgw/rgw_cr_rados.h b/src/rgw/rgw_cr_rados.h index 5740b6846d4..4480c5e8e05 100644 --- a/src/rgw/rgw_cr_rados.h +++ b/src/rgw/rgw_cr_rados.h @@ -854,6 +854,7 @@ class RGWAsyncFetchRemoteObj : public RGWAsyncRadosRequest { std::optional dest_placement_rule; rgw_obj_key key; + std::optional dest_key; std::optional versioned_epoch; real_time src_mtime; @@ -869,12 +870,14 @@ public: RGWBucketInfo& _bucket_info, std::optional _dest_placement_rule, const rgw_obj_key& _key, + const std::optional& _dest_key, std::optional _versioned_epoch, bool _if_newer, rgw_zone_set *_zones_trace) : RGWAsyncRadosRequest(caller, cn), store(_store), source_zone(_source_zone), bucket_info(_bucket_info), dest_placement_rule(_dest_placement_rule), key(_key), + dest_key(_dest_key), versioned_epoch(_versioned_epoch), copy_if_newer(_if_newer) { @@ -894,6 +897,7 @@ class RGWFetchRemoteObjCR : public RGWSimpleCoroutine { std::optional dest_placement_rule; rgw_obj_key key; + std::optional dest_key; std::optional versioned_epoch; real_time src_mtime; @@ -909,6 +913,7 @@ public: RGWBucketInfo& _bucket_info, std::optional _dest_placement_rule, const rgw_obj_key& _key, + const std::optional& _dest_key, std::optional _versioned_epoch, bool _if_newer, rgw_zone_set *_zones_trace) : RGWSimpleCoroutine(_store->ctx()), cct(_store->ctx()), async_rados(_async_rados), store(_store), @@ -916,6 +921,7 @@ public: bucket_info(_bucket_info), dest_placement_rule(_dest_placement_rule), key(_key), + dest_key(_dest_key), versioned_epoch(_versioned_epoch), copy_if_newer(_if_newer), req(NULL), zones_trace(_zones_trace) {} @@ -934,7 +940,7 @@ public: int send_request() override { req = new RGWAsyncFetchRemoteObj(this, stack->create_completion_notifier(), store, source_zone, bucket_info, dest_placement_rule, - key, versioned_epoch, copy_if_newer, zones_trace); + key, dest_key, versioned_epoch, copy_if_newer, zones_trace); async_rados->queue(req); return 0; } diff --git a/src/rgw/rgw_data_sync.cc b/src/rgw/rgw_data_sync.cc index 5effbb61786..00a7b5c297e 100644 --- a/src/rgw/rgw_data_sync.cc +++ b/src/rgw/rgw_data_sync.cc @@ -1667,7 +1667,7 @@ RGWCoroutine *RGWDefaultDataSyncModule::sync_object(RGWDataSyncEnv *sync_env, RG { return new RGWFetchRemoteObjCR(sync_env->async_rados, sync_env->store, sync_env->source_zone, bucket_info, std::nullopt, - key, versioned_epoch, + key, std::nullopt, versioned_epoch, true, zones_trace); } @@ -1693,7 +1693,7 @@ public: void init(RGWDataSyncEnv *sync_env, uint64_t instance_id) override; - RGWCoroutine *sync_object(RGWDataSyncEnv *sync_env, RGWBucketInfo& bucket_info, rgw_obj_key& key, uint64_t versioned_epoch, rgw_zone_set *zones_trace) override; + RGWCoroutine *sync_object(RGWDataSyncEnv *sync_env, RGWBucketInfo& bucket_info, rgw_obj_key& key, std::optional versioned_epoch, rgw_zone_set *zones_trace) override; RGWCoroutine *remove_object(RGWDataSyncEnv *sync_env, RGWBucketInfo& bucket_info, rgw_obj_key& key, real_time& mtime, bool versioned, uint64_t versioned_epoch, rgw_zone_set *zones_trace) override; RGWCoroutine *create_delete_marker(RGWDataSyncEnv *sync_env, RGWBucketInfo& bucket_info, rgw_obj_key& key, real_time& mtime, rgw_bucket_entry_owner& owner, bool versioned, uint64_t versioned_epoch, rgw_zone_set *zones_trace) override; @@ -1734,9 +1734,9 @@ void RGWArchiveDataSyncModule::init(RGWDataSyncEnv *sync_env, uint64_t instance_ //current_meta_mgr = new_meta_mgr; } -RGWCoroutine *RGWArchiveDataSyncModule::sync_object(RGWDataSyncEnv *sync_env, RGWBucketInfo& bucket_info, rgw_obj_key& key, uint64_t versioned_epoch, rgw_zone_set *zones_trace) +RGWCoroutine *RGWArchiveDataSyncModule::sync_object(RGWDataSyncEnv *sync_env, RGWBucketInfo& bucket_info, rgw_obj_key& key, std::optional versioned_epoch, rgw_zone_set *zones_trace) { - ldout(sync_env->cct, 0) << "SYNC_ARCHIVE: sync_object: b=" << bucket_info.bucket << " k=" << key << " versioned_epoch=" << versioned_epoch << dendl; + ldout(sync_env->cct, 0) << "SYNC_ARCHIVE: sync_object: b=" << bucket_info.bucket << " k=" << key << " versioned_epoch=" << versioned_epoch.value_or(0) << dendl; if (!bucket_info.versioned() || (bucket_info.flags & BUCKET_VERSIONS_SUSPENDED)) { ldout(sync_env->cct, 0) << "SYNC_ARCHIVE: sync_object: enabling object versioning for archive bucket" << dendl; @@ -1747,8 +1747,20 @@ RGWCoroutine *RGWArchiveDataSyncModule::sync_object(RGWDataSyncEnv *sync_env, RG return NULL; } } - return new RGWFetchRemoteObjCR(sync_env->async_rados, sync_env->store, sync_env->source_zone, bucket_info, - key, versioned_epoch, + + std::optional dest_key; + + if (versioned_epoch.value_or(0) == 0) { /* force version if not set */ + versioned_epoch = 0; + dest_key = key; + if (key.instance.empty()) { + sync_env->store->gen_rand_obj_instance_name(&(*dest_key)); + } + } + + return new RGWFetchRemoteObjCR(sync_env->async_rados, sync_env->store, sync_env->source_zone, + bucket_info, std::nullopt, + key, dest_key, versioned_epoch, true, zones_trace); } diff --git a/src/rgw/rgw_data_sync.h b/src/rgw/rgw_data_sync.h index 20fc822eeab..4c8dcb36c19 100644 --- a/src/rgw/rgw_data_sync.h +++ b/src/rgw/rgw_data_sync.h @@ -600,6 +600,7 @@ public: class RGWArchiveSyncModule : public RGWDefaultSyncModule { public: RGWArchiveSyncModule() {} + bool supports_writes() override { return false; } bool supports_data_export() override { return false; } int create_instance(CephContext *cct, const JSONFormattable& config, RGWSyncModuleInstanceRef *instance) override; }; diff --git a/src/rgw/rgw_rados.cc b/src/rgw/rgw_rados.cc index b2627f41286..aad73d01fbf 100644 --- a/src/rgw/rgw_rados.cc +++ b/src/rgw/rgw_rados.cc @@ -7577,7 +7577,7 @@ int RGWRados::unlink_obj_instance(RGWObjectCtx& obj_ctx, RGWBucketInfo& bucket_i return 0; } -void RGWRados::gen_rand_obj_instance_name(rgw_obj *target_obj) +void RGWRados::gen_rand_obj_instance_name(rgw_obj_key *target_key) { #define OBJ_INSTANCE_LEN 32 char buf[OBJ_INSTANCE_LEN + 1]; @@ -7585,7 +7585,12 @@ void RGWRados::gen_rand_obj_instance_name(rgw_obj *target_obj) gen_rand_alphanumeric_no_underscore(cct, buf, OBJ_INSTANCE_LEN); /* don't want it to get url escaped, no underscore for instance name due to the way we encode the raw keys */ - target_obj->key.set_instance(buf); + target_key->set_instance(buf); +} + +void RGWRados::gen_rand_obj_instance_name(rgw_obj *target_obj) +{ + gen_rand_obj_instance_name(&target_obj->key); } int RGWRados::get_olh(const RGWBucketInfo& bucket_info, const rgw_obj& obj, RGWOLHInfo *olh) diff --git a/src/rgw/rgw_rados.h b/src/rgw/rgw_rados.h index 552e4f938d6..6e371f909a4 100644 --- a/src/rgw/rgw_rados.h +++ b/src/rgw/rgw_rados.h @@ -2097,6 +2097,7 @@ public: int follow_olh(const RGWBucketInfo& bucket_info, RGWObjectCtx& ctx, RGWObjState *state, const rgw_obj& olh_obj, rgw_obj *target); int get_olh(const RGWBucketInfo& bucket_info, const rgw_obj& obj, RGWOLHInfo *olh); + void gen_rand_obj_instance_name(rgw_obj_key *target_key); void gen_rand_obj_instance_name(rgw_obj *target); int update_containers_stats(map& m); -- 2.39.5