From f00ac7c96f0ac48e0ca487ecb5918db42e6cf234 Mon Sep 17 00:00:00 2001 From: Soumya Koduri Date: Sat, 24 May 2025 03:07:58 +0530 Subject: [PATCH] rgw/cloud-restore: Fixing issues with initializing and resetting FIFO In addition, added some more debug statements and done code cleanup Reviewed-by: Adam Emerson Reviewed-by: Jiffin Tony Thottan Signed-off-by: Soumya Koduri --- src/rgw/driver/rados/rgw_lc_tier.cc | 2 +- src/rgw/driver/rados/rgw_rados.cc | 7 +- src/rgw/driver/rados/rgw_sal_rados.cc | 69 +++++++++++++-- src/rgw/driver/rados/rgw_sal_rados.h | 31 ++++--- src/rgw/rgw_restore.cc | 119 ++++++++++++++++++-------- src/rgw/rgw_restore.h | 4 +- src/rgw/rgw_sal.h | 5 +- src/rgw/rgw_sal_dbstore.cc | 3 +- src/rgw/rgw_sal_dbstore.h | 3 +- src/rgw/rgw_sal_filter.cc | 10 ++- src/rgw/rgw_sal_filter.h | 5 +- src/rgw/rgw_sal_store.h | 6 ++ 12 files changed, 188 insertions(+), 76 deletions(-) diff --git a/src/rgw/driver/rados/rgw_lc_tier.cc b/src/rgw/driver/rados/rgw_lc_tier.cc index c536d6fc10cad..c1dcd2387a540 100644 --- a/src/rgw/driver/rados/rgw_lc_tier.cc +++ b/src/rgw/driver/rados/rgw_lc_tier.cc @@ -292,7 +292,7 @@ int rgw_cloud_tier_restore_object(RGWLCCloudTierCtx& tier_ctx, } // now send HEAD request and verify if restore is complete on glacier/tape endpoint - static constexpr int MAX_RETRIES = 10; + static constexpr int MAX_RETRIES = 2; uint32_t retries = 0; do { ret = rgw_cloud_tier_get_object(tier_ctx, true, headers, nullptr, etag, diff --git a/src/rgw/driver/rados/rgw_rados.cc b/src/rgw/driver/rados/rgw_rados.cc index 452957c3130ba..347a8b7350bdc 100644 --- a/src/rgw/driver/rados/rgw_rados.cc +++ b/src/rgw/driver/rados/rgw_rados.cc @@ -1379,7 +1379,12 @@ int RGWRados::init_complete(const DoutPrefixProvider *dpp, optional_yield y, rgw lc->start_processor(); restore = new RGWRestore(); - restore->initialize(cct, this->driver); + ret = restore->initialize(cct, this->driver); + + if (ret < 0) { + ldpp_dout(dpp, 0) << "ERROR: failed to initialize restore thread" << dendl; + return ret; + } if (use_restore_thread) restore->start_processor(); diff --git a/src/rgw/driver/rados/rgw_sal_rados.cc b/src/rgw/driver/rados/rgw_sal_rados.cc index 7568b84d30c79..c1061965fb5d2 100644 --- a/src/rgw/driver/rados/rgw_sal_rados.cc +++ b/src/rgw/driver/rados/rgw_sal_rados.cc @@ -2026,10 +2026,9 @@ std::unique_ptr RadosStore::get_lifecycle(void) return std::make_unique(this); } -std::unique_ptr RadosStore::get_restore(const int n_objs, - const std::vector& obj_names) +std::unique_ptr RadosStore::get_restore(void) { - return std::make_unique(this, n_objs, obj_names); + return std::make_unique(this); } bool RadosStore::process_expired_objects(const DoutPrefixProvider *dpp, @@ -4673,12 +4672,45 @@ std::unique_ptr RadosRestore::get_serializer( return std::make_unique(store, oid, lock_name, cookie); } +int RadosRestore::initialize(const DoutPrefixProvider* dpp, optional_yield y, + int n_objs, std::vector& o_names) +{ + int ret = 0; + num_objs = n_objs; + obj_names = o_names; + + for (auto i=0; i < num_objs; i++) { + std::unique_ptr fifo_tmp; + ret = rgw::cls::fifo::FIFO::create(dpp, ioctx, obj_names[i], &fifo_tmp, y); + + ldpp_dout(dpp, 20) << "creating fifo object for index=" << i + << ", objname=" << obj_names[i] << + " returned ret=" << ret << dendl; + + if (ret) { + return ret; + } + + fifos.push_back(std::move(fifo_tmp)); + } + + return ret; +} + +void RadosRestore::finalize() { + obj_names.clear(); + fifos.clear(); +} + int RadosRestore::add_entry(const DoutPrefixProvider* dpp, optional_yield y, int index, const RGWRestoreEntry& entry) { bufferlist bl; encode(entry, bl); + ldpp_dout(dpp, 20) << __PRETTY_FUNCTION__ + << "Adding entry:(" << entry.bucket << "," << entry.obj_key << ") to FIFO:" << obj_names[index] << dendl; + auto ret = push(dpp, y, index, std::move(bl)); if (ret < 0) { ldpp_dout(dpp, -1) << "ERROR: push() returned " << ret << dendl; @@ -4701,6 +4733,9 @@ int RadosRestore::add_entries(const DoutPrefixProvider* dpp, optional_yield y, } + ldpp_dout(dpp, 20) << __PRETTY_FUNCTION__ + << "Adding " << restore_entries.size() << " entries to FIFO:" << obj_names[index] << dendl; + int ret = push(dpp, y, index, std::move(ent_list)); if (ret < 0) { @@ -4713,7 +4748,10 @@ int RadosRestore::add_entries(const DoutPrefixProvider* dpp, optional_yield y, int RadosRestore::push(const DoutPrefixProvider *dpp, optional_yield y, int index, std::vector&& items) { - auto r = fifos[index].push(dpp, items, y); + ldpp_dout(dpp, 20) << __PRETTY_FUNCTION__ + << "Pushing entries to FIFO:" << obj_names[index] << dendl; + + auto r = fifos[index]->push(dpp, items, y); if (r < 0) { ldpp_dout(dpp, -1) << __PRETTY_FUNCTION__ << ": unable to push to FIFO: " << obj_names[index] @@ -4724,7 +4762,10 @@ int RadosRestore::push(const DoutPrefixProvider *dpp, optional_yield y, int RadosRestore::push(const DoutPrefixProvider *dpp, optional_yield y, int index, ceph::buffer::list&& bl) { - auto r = fifos[index].push(dpp, std::move(bl), y); + ldpp_dout(dpp, 20) << __PRETTY_FUNCTION__ + << "Pushing entry to FIFO:" << obj_names[index] << dendl; + + auto r = fifos[index]->push(dpp, std::move(bl), y); if (r < 0) { ldpp_dout(dpp, -1) << __PRETTY_FUNCTION__ << ": unable to push to FIFO: " << obj_names[index] @@ -4768,8 +4809,10 @@ int RadosRestore::list(const DoutPrefixProvider *dpp, optional_yield y, std::vector restore_entries; bool more = false; - auto r = fifos[index].list(dpp, max_entries, marker, &restore_entries, &more, y); + ldpp_dout(dpp, 20) << __PRETTY_FUNCTION__ + << "Listing entries from FIFO:" << obj_names[index] << dendl; + auto r = fifos[index]->list(dpp, max_entries, marker, &restore_entries, &more, y); if (r < 0) { ldpp_dout(dpp, -1) << __PRETTY_FUNCTION__ << ": unable to list FIFO: " << obj_names[index] @@ -4804,6 +4847,11 @@ int RadosRestore::list(const DoutPrefixProvider *dpp, optional_yield y, *out_marker = restore_entries.back().marker; } + ldpp_dout(dpp, 20) << __PRETTY_FUNCTION__ + << "Listing from FIFO:" << obj_names[index] << ", returned:" + << restore_entries.size() << " entries, truncated:" << more + << ", out_marker:" << (out_marker ? *out_marker : "") << dendl; + return 0; } @@ -4818,7 +4866,10 @@ int RadosRestore::trim_entries(const DoutPrefixProvider *dpp, optional_yield y, int RadosRestore::trim(const DoutPrefixProvider *dpp, optional_yield y, int index, const std::string_view& marker) { - auto r = fifos[index].trim(dpp, marker, false, y); + ldpp_dout(dpp, 20) << __PRETTY_FUNCTION__ + << "Trimming FIFO:" << obj_names[index] << " upto marker:" << marker << dendl; + + auto r = fifos[index]->trim(dpp, marker, false, y); if (r < 0) { ldpp_dout(dpp, -1) << __PRETTY_FUNCTION__ << ": unable to trim FIFO: " << obj_names[index] @@ -4838,7 +4889,7 @@ int RadosRestore::is_empty(const DoutPrefixProvider *dpp, optional_yield y) { bool more = false; for (auto shard = 0u; shard < fifos.size(); ++shard) { - auto r = fifos[shard].list(dpp, 1, {}, &restore_entries, &more, y); + auto r = fifos[shard]->list(dpp, 1, {}, &restore_entries, &more, y); if (r < 0) { ldpp_dout(dpp, -1) << __PRETTY_FUNCTION__ << ": unable to list FIFO: " << obj_names[shard] @@ -4846,6 +4897,8 @@ int RadosRestore::is_empty(const DoutPrefixProvider *dpp, optional_yield y) { return r; } if (!restore_entries.empty()) { + ldpp_dout(dpp, 20) << __PRETTY_FUNCTION__ + << "Entries found in FIFO:" << obj_names[shard] << dendl; return 0; } } diff --git a/src/rgw/driver/rados/rgw_sal_rados.h b/src/rgw/driver/rados/rgw_sal_rados.h index ff22de78acf62..e7520139e2cf1 100644 --- a/src/rgw/driver/rados/rgw_sal_rados.h +++ b/src/rgw/driver/rados/rgw_sal_rados.h @@ -27,7 +27,6 @@ #include "rgw_putobj_processor.h" #include "services/svc_tier_rados.h" #include "cls/lock/cls_lock_client.h" -#include "rgw_log_backing.h" namespace rgw { namespace sal { @@ -283,8 +282,7 @@ class RadosStore : public StoreDriver { virtual int list_all_zones(const DoutPrefixProvider* dpp, std::list& zone_ids) override; virtual int cluster_stat(RGWClusterStat& stats) override; virtual std::unique_ptr get_lifecycle(void) override; - virtual std::unique_ptr get_restore(const int n_objs, - const std::vector& obj_names) override; + virtual std::unique_ptr get_restore(void) override; virtual bool process_expired_objects(const DoutPrefixProvider *dpp, optional_yield y) override; virtual std::unique_ptr get_notification(rgw::sal::Object* obj, rgw::sal::Object* src_obj, req_state* s, rgw::notify::EventType event_type, optional_yield y, const std::string* object_name=nullptr) override; virtual std::unique_ptr get_notification( @@ -961,23 +959,24 @@ public: } }; -class RadosRestore : public Restore { +class RadosRestore : public StoreRestore { RadosStore* store; - const int num_objs; - const std::vector& obj_names; librados::IoCtx& ioctx; - ceph::containers::tiny_vector fifos; + int num_objs; + std::vector obj_names; + std::vector> fifos; public: - RadosRestore(RadosStore* _st, const int n_objs, const std::vector& o_names) : store(_st), - num_objs(n_objs), obj_names(o_names), - ioctx(*store->getRados()->get_restore_pool_ctx()), - fifos(num_objs, - [&](const size_t ix, auto emplacer) { - emplacer.emplace(ioctx, std::string(obj_names[ix])); - }) {} - - ~RadosRestore() override = default; + RadosRestore(RadosStore* _st) : store(_st), + ioctx(*store->getRados()->get_restore_pool_ctx()) {} + + ~RadosRestore() override { + finalize(); + } + + virtual int initialize(const DoutPrefixProvider* dpp, optional_yield y, + int n_objs, std::vector& obj_names) override; + void finalize(); virtual int add_entry(const DoutPrefixProvider* dpp, optional_yield y, int index, const RGWRestoreEntry& r_entry) override; diff --git a/src/rgw/rgw_restore.cc b/src/rgw/rgw_restore.cc index 842cbd178b3ad..b5fb5bc9081a7 100644 --- a/src/rgw/rgw_restore.cc +++ b/src/rgw/rgw_restore.cc @@ -105,25 +105,48 @@ void RGWRestoreEntry::generate_test_instances(std::list& l) l.push_back(new RGWRestoreEntry); } -void RGWRestore::initialize(CephContext *_cct, rgw::sal::Driver* _driver) { +int RGWRestore::initialize(CephContext *_cct, rgw::sal::Driver* _driver) { + int ret = 0; cct = _cct; driver = _driver; + ldpp_dout(this, 20) << __PRETTY_FUNCTION__ << ": initializing RGWRestore handle" << dendl; /* max_objs indicates the number of shards or objects * used to store Restore Entries */ max_objs = cct->_conf->rgw_restore_max_objs; if (max_objs > HASH_PRIME) max_objs = HASH_PRIME; + obj_names.clear(); for (int i = 0; i < max_objs; i++) { - obj_names.push_back(fmt::format("{}.{}", restore_oid_prefix, i)); + std::string s = fmt::format("{}.{}", restore_oid_prefix, i); + obj_names.push_back(s); + ldpp_dout(this, 30) << __PRETTY_FUNCTION__ << ": obj_name_i=" << obj_names[i] << dendl; + } + + sal_restore = driver->get_restore(); + + if (!sal_restore) { + ldpp_dout(this, -1) << __PRETTY_FUNCTION__ << ": failed to create sal_restore" << dendl; + return -EINVAL; } - sal_restore = driver->get_restore(max_objs, obj_names); + + ret = sal_restore->initialize(this, null_yield, max_objs, obj_names); + + if (ret < 0) { + ldpp_dout(this, -1) << __PRETTY_FUNCTION__ << ": failed to initialize sal_restore" << dendl; + } + + ldpp_dout(this, 20) << __PRETTY_FUNCTION__ << ": initializing RGWRestore handle completed" << dendl; + + return ret; } void RGWRestore::finalize() { + sal_restore.reset(nullptr); obj_names.clear(); + ldpp_dout(this, 20) << __PRETTY_FUNCTION__ << ": finalize RGWRestore handle" << dendl; } static inline std::ostream& operator<<(std::ostream &os, RGWRestoreEntry& ent) { @@ -193,7 +216,7 @@ void *RGWRestore::RestoreWorker::entry() { int r = 0; r = restore->process(this, null_yield); if (r < 0) { - ldpp_dout(dpp, -1) << "ERROR: restore process() returned error r=" << r << dendl; + ldpp_dout(dpp, -1) << __PRETTY_FUNCTION__ << ": ERROR: restore process() returned error r=" << r << dendl; } if (restore->going_down()) break; @@ -236,7 +259,7 @@ int RGWRestore::process(RestoreWorker* worker, optional_yield y) */ int RGWRestore::process(int index, int max_secs, optional_yield y) { - ldpp_dout(this, 20) << "RGWRestore::process entered with Restore index_shard=" + ldpp_dout(this, 20) << __PRETTY_FUNCTION__ << ": process entered index=" << index << ", max_secs=" << max_secs << dendl; /* list used to gather still IN_PROGRESS */ @@ -261,7 +284,7 @@ int RGWRestore::process(int index, int max_secs, optional_yield y) int ret = serializer->try_lock(this, time, null_yield); if (ret == -EBUSY || ret == -EEXIST) { /* already locked by another lc processor */ - ldpp_dout(this, 0) << "RGWRestore::process() failed to acquire lock on " + ldpp_dout(this, 0) << __PRETTY_FUNCTION__ << ": failed to acquire lock on " << obj_names[index] << dendl; return -EBUSY; } @@ -278,9 +301,10 @@ int RGWRestore::process(int index, int max_secs, optional_yield y) std::vector entries; ret = sal_restore->list(this, y, index, marker, &next_marker, max, entries, &truncated); - ldpp_dout(this, 20) << - "RGWRestore::process sal_restore->list returned with returned:" << ret << + ldpp_dout(this, 20) << __PRETTY_FUNCTION__ << + ": list on shard:" << obj_names[index] << " returned:" << ret << ", entries.size=" << entries.size() << ", truncated=" << truncated << + ", marker='" << marker << "'" << ", next_marker='" << next_marker << "'" << dendl; if (entries.size() == 0) { @@ -298,8 +322,11 @@ int RGWRestore::process(int index, int max_secs, optional_yield y) ret = process_restore_entry(entry, y); - if (entry.status == rgw::sal::RGWRestoreStatus::RestoreAlreadyInProgress) { + if (!ret && entry.status == rgw::sal::RGWRestoreStatus::RestoreAlreadyInProgress) { r_entries.push_back(entry); + ldpp_dout(this, 20) << __PRETTY_FUNCTION__ << ": re-pushing entry: '" << entry + << "' on shard:" + << obj_names[index] << dendl; } ///process all entries, trim and re-add @@ -314,16 +341,18 @@ int RGWRestore::process(int index, int max_secs, optional_yield y) goto done; } } + ldpp_dout(this, 20) << __PRETTY_FUNCTION__ << ": trimming till marker: '" << marker + << "' on shard:" + << obj_names[index] << dendl; ret = sal_restore->trim_entries(this, y, index, marker); if (ret < 0) { - ldpp_dout(this, -1) << "ERROR: RGWRestore::process() failed to trim entries on " - << obj_names[index] << dendl; + ldpp_dout(this, -1) << __PRETTY_FUNCTION__ << ": ERROR: failed to trim entries on " << obj_names[index] << dendl; } if (!r_entries.empty()) { ret = sal_restore->add_entries(this, y, index, r_entries); if (ret < 0) { - ldpp_dout(this, -1) << "ERROR: RGWRestore::process() failed to add entries on " + ldpp_dout(this, -1) << __PRETTY_FUNCTION__ << ": ERROR: failed to add entries on " << obj_names[index] << dendl; } } @@ -340,10 +369,15 @@ done: int RGWRestore::process_restore_entry(RGWRestoreEntry& entry, optional_yield y) { int ret = 0; + bool in_progress = true; std::unique_ptr bucket; std::unique_ptr obj; std::unique_ptr tier; std::optional days = entry.days; + rgw::sal::RGWRestoreStatus restore_status = rgw::sal::RGWRestoreStatus::None; + RGWObjState* obj_state{nullptr}; + rgw_placement_rule target_placement; + // Ensure its the same source zone processing temp entries as we do not // replicate temp restored copies if (days) { // temp copy @@ -358,7 +392,8 @@ int RGWRestore::process_restore_entry(RGWRestoreEntry& entry, optional_yield y) // bucket, obj, days, state=in_progress ret = driver->load_bucket(this, entry.bucket, &bucket, null_yield); if (ret < 0) { - ldpp_dout(this, -1) << "ERROR: Restore:get_bucket for " << bucket->get_name() + ldpp_dout(this, -1) << __PRETTY_FUNCTION__ << ": ERROR: get_bucket for " + << bucket->get_name() << " failed" << dendl; return ret; } @@ -367,26 +402,27 @@ int RGWRestore::process_restore_entry(RGWRestoreEntry& entry, optional_yield y) ret = obj->load_obj_state(this, null_yield, true); if (ret < 0) { - ldpp_dout(this, 0) << "ERROR: Restore:get_object for " << entry.obj_key + ldpp_dout(this, -1) << __PRETTY_FUNCTION__ << ": ERROR: get_object for " + << entry.obj_key << " failed" << dendl; return ret; } - rgw_placement_rule target_placement; target_placement.inherit_from(bucket->get_placement_rule()); auto& attrs = obj->get_attrs(); auto attr_iter = attrs.find(RGW_ATTR_RESTORE_STATUS); - rgw::sal::RGWRestoreStatus restore_status = rgw::sal::RGWRestoreStatus::None; if (attr_iter != attrs.end()) { bufferlist bl = attr_iter->second; auto iter = bl.cbegin(); decode(restore_status, iter); } - if (restore_status == rgw::sal::RGWRestoreStatus::CloudRestored) { + if (restore_status != rgw::sal::RGWRestoreStatus::RestoreAlreadyInProgress) { // XXX: Check if expiry-date needs to be update - ldpp_dout(this, 5) << "Restore of object " << obj->get_key() << " already done" << dendl; - entry.status = rgw::sal::RGWRestoreStatus::CloudRestored; + ldpp_dout(this, 5) << __PRETTY_FUNCTION__ << ": Restore of object " << obj->get_key() + << " not in progress state" << dendl; + + entry.status = restore_status; return 0; } @@ -397,17 +433,19 @@ int RGWRestore::process_restore_entry(RGWRestoreEntry& entry, optional_yield y) ret = driver->get_zone()->get_zonegroup().get_placement_tier(target_placement, &tier); if (ret < 0) { - ldpp_dout(this, -1) << "ERROR: failed to fetch tier placement handle, ret = " << ret << dendl; - return ret; + ldpp_dout(this, -1) << __PRETTY_FUNCTION__ << ": ERROR: failed to fetch tier placement handle, ret = " << ret << dendl; + goto done; } else { - ldpp_dout(this, 20) << "getting tier placement handle cloud tier for " << + ldpp_dout(this, 20) << __PRETTY_FUNCTION__ << ": getting tier placement handle" + << " cloud tier for " << " storage class " << target_placement.storage_class << dendl; } if (!tier->is_tier_type_s3()) { - ldpp_dout(this, -1) << "ERROR: not s3 tier type - " << tier->get_tier_type() << + ldpp_dout(this, -1) << __PRETTY_FUNCTION__ << ": ERROR: not s3 tier type - " + << tier->get_tier_type() << " for storage class " << target_placement.storage_class << dendl; - return -EINVAL; + goto done; } // now go ahead with restoring object @@ -416,20 +454,25 @@ int RGWRestore::process_restore_entry(RGWRestoreEntry& entry, optional_yield y) ret = obj->restore_obj_from_cloud(bucket.get(), tier.get(), cct, days, in_progress, this, y); if (ret < 0) { - ldpp_dout(this, -1) << "Restore of object(" << obj->get_key() << ") failed" << ret << dendl; + ldpp_dout(this, -1) << __PRETTY_FUNCTION__ << ": Restore of object(" << obj->get_key() << ") failed" << ret << dendl; auto reset_ret = set_cloud_restore_status(this, obj.get(), y, rgw::sal::RGWRestoreStatus::RestoreFailed); - entry.status = rgw::sal::RGWRestoreStatus::RestoreFailed; if (reset_ret < 0) { - ldpp_dout(this, -1) << "Setting restore status ad RestoreFailed failed for object(" << obj->get_key() << ") " << reset_ret << dendl; + ldpp_dout(this, -1) << __PRETTY_FUNCTION__ << ": Setting restore status ad RestoreFailed failed for object(" << obj->get_key() << ") " << reset_ret << dendl; } - return ret; + goto done; } if (in_progress) { - ldpp_dout(this, 15) << "Restore of object " << obj->get_key() << " is still in progress" << dendl; + ldpp_dout(this, 15) << __PRETTY_FUNCTION__ << ": Restore of object " << obj->get_key() << " is still in progress" << dendl; entry.status = rgw::sal::RGWRestoreStatus::RestoreAlreadyInProgress; } else { - ldpp_dout(this, 15) << "Restore of object " << obj->get_key() << " succeeded" << dendl; + ldpp_dout(this, 15) << __PRETTY_FUNCTION__ << ": Restore of object " << obj->get_key() << " succeeded" << dendl; + entry.status = rgw::sal::RGWRestoreStatus::CloudRestored; + } + +done: + if (ret < 0) { + ldpp_dout(this, -1) << __PRETTY_FUNCTION__ << ": Restore of entry:'" << entry << "' failed" << ret << dendl; entry.status = rgw::sal::RGWRestoreStatus::RestoreFailed; } return ret; @@ -473,14 +516,14 @@ int RGWRestore::restore_obj_from_cloud(rgw::sal::Bucket* pbucket, int ret = 0; if (!pbucket || !pobj) { - ldpp_dout(this, -1) << "ERROR: Invalid bucket/object. Restore failed" << dendl; + ldpp_dout(this, -1) << __PRETTY_FUNCTION__ << ": ERROR: Invalid bucket/object. Restore failed" << dendl; return -EINVAL; } // set restore_status as RESTORE_ALREADY_IN_PROGRESS ret = set_cloud_restore_status(this, pobj, y, rgw::sal::RGWRestoreStatus::RestoreAlreadyInProgress); if (ret < 0) { - ldpp_dout(this, 0) << " Setting cloud restore status to RESTORE_ALREADY_IN_PROGRESS for the object(" << pobj->get_key() << " failed, ret=" << ret << dendl; + ldpp_dout(this, 0) << __PRETTY_FUNCTION__ << ": Setting cloud restore status to RESTORE_ALREADY_IN_PROGRESS for the object(" << pobj->get_key() << " failed, ret=" << ret << dendl; return ret; } @@ -489,11 +532,11 @@ int RGWRestore::restore_obj_from_cloud(rgw::sal::Bucket* pbucket, ret = pobj->restore_obj_from_cloud(pbucket, tier, cct, days, in_progress, dpp, y); if (ret < 0) { - ldpp_dout(this, 0) << "ERROR: object " << pobj->get_key() << " fetching failed" << ret << dendl; + ldpp_dout(this, -1) << __PRETTY_FUNCTION__ << ": ERROR: object " << pobj->get_key() << " fetching failed" << ret << dendl; auto reset_ret = set_cloud_restore_status(this, pobj, y, rgw::sal::RGWRestoreStatus::RestoreFailed); if (reset_ret < 0) { - ldpp_dout(this, -1) << "Setting restore status to RestoreFailed failed for object(" << pobj->get_key() << ") " << reset_ret << dendl; + ldpp_dout(this, -1) << __PRETTY_FUNCTION__ << ": Setting restore status to RestoreFailed failed for object(" << pobj->get_key() << ") " << reset_ret << dendl; } return ret; @@ -511,20 +554,22 @@ int RGWRestore::restore_obj_from_cloud(rgw::sal::Bucket* pbucket, ldpp_dout(this, 10) << "RGWRestore:: Adding restore entry of object(" << pobj->get_key() << ") entry: " << entry << dendl; int index = choose_oid(entry); + ldpp_dout(this, 10) << __PRETTY_FUNCTION__ << ": Adding restore entry of object(" << pobj->get_key() << ") entry: " << entry << ", to shard:" << obj_names[index] << dendl; + ret = sal_restore->add_entry(this, y, index, entry); if (ret < 0) { - ldpp_dout(this, -1) << "ERROR: Adding restore entry of object(" << pobj->get_key() << ") failed" << ret << dendl; + ldpp_dout(this, -1) << __PRETTY_FUNCTION__ << ": ERROR: Adding restore entry of object(" << pobj->get_key() << ") failed" << ret << dendl; auto reset_ret = set_cloud_restore_status(this, pobj, y, rgw::sal::RGWRestoreStatus::RestoreFailed); if (reset_ret < 0) { - ldpp_dout(this, -1) << "Setting restore status as RestoreFailed failed for object(" << pobj->get_key() << ") " << reset_ret << dendl; + ldpp_dout(this, -1) << __PRETTY_FUNCTION__ << ": Setting restore status as RestoreFailed failed for object(" << pobj->get_key() << ") " << reset_ret << dendl; } return ret; } } - ldpp_dout(this, 10) << "Restore of object " << pobj->get_key() << (in_progress ? " is in progress" : " succeeded") << dendl; + ldpp_dout(this, 10) << __PRETTY_FUNCTION__ << ": Restore of object " << pobj->get_key() << (in_progress ? " is in progress" : " succeeded") << dendl; return ret; } diff --git a/src/rgw/rgw_restore.h b/src/rgw/rgw_restore.h index d18366cb78730..b37c80c25c0a5 100644 --- a/src/rgw/rgw_restore.h +++ b/src/rgw/rgw_restore.h @@ -68,7 +68,7 @@ class RGWRestore : public DoutPrefixProvider { rgw::sal::Driver* driver; std::unique_ptr sal_restore; int max_objs{0}; - std::vector obj_names; + std::vector obj_names; std::atomic down_flag = { false }; class RestoreWorker : public Thread @@ -107,7 +107,7 @@ public: RGWRestore() : cct(nullptr), driver(nullptr), max_objs(0) {} - void initialize(CephContext *_cct, rgw::sal::Driver* _driver); + int initialize(CephContext *_cct, rgw::sal::Driver* _driver); void finalize(); bool going_down(); diff --git a/src/rgw/rgw_sal.h b/src/rgw/rgw_sal.h index 9373b4fb8d4b3..36858228fe9a3 100644 --- a/src/rgw/rgw_sal.h +++ b/src/rgw/rgw_sal.h @@ -475,8 +475,7 @@ class Driver { /** Get a @a Lifecycle object. Used to manage/run lifecycle transitions */ virtual std::unique_ptr get_lifecycle(void) = 0; /** Get a @a Restore object. Used to manage/run restore objects */ - virtual std::unique_ptr get_restore(const int n_objs, - const std::vector& obj_names) = 0; + virtual std::unique_ptr get_restore(void) = 0; /** Reset the temporarily restored objects which are expired */ virtual bool process_expired_objects(const DoutPrefixProvider *dpp, optional_yield y) = 0; @@ -1691,6 +1690,8 @@ class Restore { public: Restore() = default; virtual ~Restore() = default; + virtual int initialize(const DoutPrefixProvider* dpp, optional_yield y, + int n_objs, std::vector& obj_names) = 0; /** Add a single restore entry state */ virtual int add_entry(const DoutPrefixProvider* dpp, optional_yield y, int index, const RGWRestoreEntry& r_entry) = 0; diff --git a/src/rgw/rgw_sal_dbstore.cc b/src/rgw/rgw_sal_dbstore.cc index 077f9583831cd..e1bbe2cdf891a 100644 --- a/src/rgw/rgw_sal_dbstore.cc +++ b/src/rgw/rgw_sal_dbstore.cc @@ -1920,8 +1920,7 @@ namespace rgw::sal { return std::make_unique(store, oid, lock_name, cookie); } - std::unique_ptr DBStore::get_restore(const int n_objs, - const std::vector& obj_names) + std::unique_ptr DBStore::get_restore() { return nullptr; } diff --git a/src/rgw/rgw_sal_dbstore.h b/src/rgw/rgw_sal_dbstore.h index 8a9451ae79478..3c7a3cc324f56 100644 --- a/src/rgw/rgw_sal_dbstore.h +++ b/src/rgw/rgw_sal_dbstore.h @@ -893,8 +893,7 @@ public: virtual int list_all_zones(const DoutPrefixProvider* dpp, std::list& zone_ids) override; virtual int cluster_stat(RGWClusterStat& stats) override; virtual std::unique_ptr get_lifecycle(void) override; - virtual std::unique_ptr get_restore(const int n_objs, - const std::vector& obj_names) override; + virtual std::unique_ptr get_restore(void) override; virtual bool process_expired_objects(const DoutPrefixProvider *dpp, optional_yield y) override; virtual std::unique_ptr get_notification( diff --git a/src/rgw/rgw_sal_filter.cc b/src/rgw/rgw_sal_filter.cc index 62132df10b819..09c38e45d9a72 100644 --- a/src/rgw/rgw_sal_filter.cc +++ b/src/rgw/rgw_sal_filter.cc @@ -440,10 +440,9 @@ bool FilterDriver::process_expired_objects(const DoutPrefixProvider *dpp, return next->process_expired_objects(dpp, y); } -std::unique_ptr FilterDriver::get_restore(const int n_objs, - const std::vector& obj_names) +std::unique_ptr FilterDriver::get_restore() { - std::unique_ptr restore = next->get_restore(n_objs, obj_names); + std::unique_ptr restore = next->get_restore(); return std::make_unique(std::move(restore)); } @@ -1471,6 +1470,11 @@ std::unique_ptr FilterRestore::get_serializer(const std::stri return std::make_unique(std::move(ns)); } +int FilterRestore::initialize(const DoutPrefixProvider* dpp, optional_yield y, + int n_objs, std::vector& obj_names) { + return next->initialize(dpp, y, n_objs, obj_names); +} + int FilterRestore::add_entry(const DoutPrefixProvider* dpp, optional_yield y, int index, const RGWRestoreEntry& r_entry) { return next->add_entry(dpp, y, index, r_entry); diff --git a/src/rgw/rgw_sal_filter.h b/src/rgw/rgw_sal_filter.h index 79b08465ca7af..55568fd6df84b 100644 --- a/src/rgw/rgw_sal_filter.h +++ b/src/rgw/rgw_sal_filter.h @@ -292,8 +292,7 @@ public: } virtual int cluster_stat(RGWClusterStat& stats) override; virtual std::unique_ptr get_lifecycle(void) override; - virtual std::unique_ptr get_restore(const int n_objs, - const std::vector& obj_names) override; + virtual std::unique_ptr get_restore(void) override; virtual bool process_expired_objects(const DoutPrefixProvider *dpp, optional_yield y) override; virtual std::unique_ptr get_notification(rgw::sal::Object* obj, @@ -1080,6 +1079,8 @@ public: FilterRestore(std::unique_ptr _next) : next(std::move(_next)) {} ~FilterRestore() override = default; + virtual int initialize(const DoutPrefixProvider* dpp, optional_yield y, + int n_objs, std::vector& obj_names) override; virtual int add_entry(const DoutPrefixProvider* dpp, optional_yield y, int index, const RGWRestoreEntry& r_entry) override; virtual int add_entries(const DoutPrefixProvider* dpp, optional_yield y, diff --git a/src/rgw/rgw_sal_store.h b/src/rgw/rgw_sal_store.h index 0364c84d1ac03..08499b0bb543c 100644 --- a/src/rgw/rgw_sal_store.h +++ b/src/rgw/rgw_sal_store.h @@ -468,6 +468,12 @@ public: virtual void print(std::ostream& out) const override { out << oid; } }; +class StoreRestore : public Restore { + +public: + StoreRestore() = default; + virtual ~StoreRestore() = default; +}; class StoreNotification : public Notification { protected: -- 2.39.5