From 504b024fa9f4cb054109c00e527eb0dc08b9b4ce Mon Sep 17 00:00:00 2001 From: "Adam C. Emerson" Date: Sun, 3 Jan 2021 18:32:50 -0500 Subject: [PATCH] rgw/datalog: make get_oid take generation Signed-off-by: Adam C. Emerson (cherry picked from commit f7b850f7aa84d9cf24b4eaebbe51c7ee221bbd44) Signed-off-by: Adam C. Emerson --- src/rgw/rgw_datalog.cc | 27 +++++++------ src/rgw/rgw_datalog.h | 78 ++++++++++++++++++++----------------- src/rgw/rgw_trim_datalog.cc | 2 +- 3 files changed, 57 insertions(+), 50 deletions(-) diff --git a/src/rgw/rgw_datalog.cc b/src/rgw/rgw_datalog.cc index 9fc2fff83c103..329657d463125 100644 --- a/src/rgw/rgw_datalog.cc +++ b/src/rgw/rgw_datalog.cc @@ -73,14 +73,13 @@ void rgw_data_change_log_entry::decode_json(JSONObj *obj) { class RGWDataChangesOmap final : public RGWDataChangesBE { using centries = std::list; std::vector oids; - std::string get_oid(int i) const { - return datalog.get_oid(i); - } + public: RGWDataChangesOmap(lr::IoCtx& ioctx, RGWDataChangesLog& datalog, + uint64_t gen_id, int num_shards) - : RGWDataChangesBE(ioctx, datalog) { + : RGWDataChangesBE(ioctx, datalog, gen_id) { oids.reserve(num_shards); for (auto i = 0; i < num_shards; ++i) { oids.push_back(get_oid(i)); @@ -208,14 +207,12 @@ public: class RGWDataChangesFIFO final : public RGWDataChangesBE { using centries = std::vector; std::vector> fifos; - std::string get_oid(int i) const { - return datalog.get_oid(i); - } + public: RGWDataChangesFIFO(lr::IoCtx& ioctx, RGWDataChangesLog& datalog, - int shards) - : RGWDataChangesBE(ioctx, datalog) { + uint64_t gen_id, int shards) + : RGWDataChangesBE(ioctx, datalog, gen_id) { fifos.resize(shards); for (auto i = 0; i < shards; ++i) { auto r = rgw::cls::fifo::FIFO::create(ioctx, get_oid(i), @@ -395,7 +392,7 @@ int RGWDataChangesLog::start(const RGWZone* _zone, } auto found = log_backing_type(ioctx, *defbacking, num_shards, - [this](int i) { return get_oid(i); }, + [this](int i) { return get_oid(0, i); }, null_yield); if (!found) { @@ -406,10 +403,10 @@ int RGWDataChangesLog::start(const RGWZone* _zone, try { switch (*found) { case log_type::omap: - be = std::make_unique(ioctx, *this, num_shards); + be = std::make_unique(ioctx, *this, 0, num_shards); break; case log_type::fifo: - be = std::make_unique(ioctx, *this, num_shards); + be = std::make_unique(ioctx, *this, 0, num_shards); break; } } catch (bs::system_error& e) { @@ -530,8 +527,10 @@ bool RGWDataChangesLog::filter_bucket(const rgw_bucket& bucket, return bucket_filter(bucket, y); } -std::string RGWDataChangesLog::get_oid(int i) const { - return fmt::format("{}.{}", prefix, i); +std::string RGWDataChangesLog::get_oid(uint64_t gen_id, int i) const { + return (gen_id > 0 ? + fmt::format("{}@G{}.{}", prefix, gen_id, i) : + fmt::format("{}.{}", prefix, i)); } int RGWDataChangesLog::add_entry(const RGWBucketInfo& bucket_info, int shard_id) { diff --git a/src/rgw/rgw_datalog.h b/src/rgw/rgw_datalog.h index 387d50a1d4964..2a73237b38d2d 100644 --- a/src/rgw/rgw_datalog.h +++ b/src/rgw/rgw_datalog.h @@ -117,40 +117,7 @@ struct RGWDataChangesLogMarker { class RGWDataChangesLog; -class RGWDataChangesBE { -protected: - librados::IoCtx& ioctx; - CephContext* const cct; - RGWDataChangesLog& datalog; -private: -public: - using entries = std::variant, - std::vector>; - - RGWDataChangesBE(librados::IoCtx& ioctx, - RGWDataChangesLog& datalog) - : ioctx(ioctx), cct(static_cast(ioctx.cct())), - datalog(datalog) {} - virtual ~RGWDataChangesBE() = default; - - virtual void prepare(ceph::real_time now, - const std::string& key, - ceph::buffer::list&& entry, - entries& out) = 0; - virtual int push(int index, entries&& items) = 0; - virtual int push(int index, ceph::real_time now, - const std::string& key, - ceph::buffer::list&& bl) = 0; - virtual int list(int shard, int max_entries, - std::vector& entries, - std::optional marker, - std::string* out_marker, bool* truncated) = 0; - virtual int get_info(int index, RGWDataChangesLogInfo *info) = 0; - virtual int trim(int index, std::string_view marker) = 0; - virtual int trim(int index, std::string_view marker, - librados::AioCompletion* c) = 0; - virtual std::string_view max_marker() const = 0; -}; +class RGWDataChangesBE; class RGWDataChangesLog { CephContext *cct; @@ -247,7 +214,48 @@ public: } // a marker that compares greater than any other std::string_view max_marker() const; - std::string get_oid(int shard_id) const; + std::string get_oid(uint64_t gen_id, int shard_id) const; +}; + +class RGWDataChangesBE { +protected: + librados::IoCtx& ioctx; + CephContext* const cct; + RGWDataChangesLog& datalog; + uint64_t gen_id; + + std::string get_oid(int shard_id) { + return datalog.get_oid(gen_id, shard_id); + } +public: + using entries = std::variant, + std::vector>; + + RGWDataChangesBE(librados::IoCtx& ioctx, + RGWDataChangesLog& datalog, + uint64_t gen_id) + : ioctx(ioctx), cct(static_cast(ioctx.cct())), + datalog(datalog), gen_id(gen_id) {} + virtual ~RGWDataChangesBE() = default; + + virtual void prepare(ceph::real_time now, + const std::string& key, + ceph::buffer::list&& entry, + entries& out) = 0; + virtual int push(int index, entries&& items) = 0; + virtual int push(int index, ceph::real_time now, + const std::string& key, + ceph::buffer::list&& bl) = 0; + virtual int list(int shard, int max_entries, + std::vector& entries, + std::optional marker, + std::string* out_marker, bool* truncated) = 0; + virtual int get_info(int index, RGWDataChangesLogInfo *info) = 0; + virtual int trim(int index, std::string_view marker) = 0; + virtual int trim(int index, std::string_view marker, + librados::AioCompletion* c) = 0; + virtual std::string_view max_marker() const = 0; }; + #endif diff --git a/src/rgw/rgw_trim_datalog.cc b/src/rgw/rgw_trim_datalog.cc index 62f6c07d17205..85c19a7c4437b 100644 --- a/src/rgw/rgw_trim_datalog.cc +++ b/src/rgw/rgw_trim_datalog.cc @@ -202,7 +202,7 @@ class DataLogTrimPollCR : public RGWCoroutine { int num_shards, utime_t interval) : RGWCoroutine(store->ctx()), store(store), http(http), num_shards(num_shards), interval(interval), - lock_oid(store->svc()->datalog_rados->get_oid(0)), + lock_oid(store->svc()->datalog_rados->get_oid(0, 0)), lock_cookie(RGWSimpleRadosLockCR::gen_random_cookie(cct)), last_trim(num_shards) {} -- 2.39.5