From 0ecc2b8fc07011c77ef99d09b9c3f12bf4c5d9b4 Mon Sep 17 00:00:00 2001 From: Matt Benjamin Date: Tue, 1 Mar 2022 15:28:00 -0500 Subject: [PATCH] rgwlc: prevent multiple runs per scheduling cycle Restore (and robustify) the assertion that, in general, each bucket shard should be processed once per scheduling cycle. If the prior cycle did not finish, processing in the current cyhcle will continue from the marker where the last cycle left off. Signed-off-by: Matt Benjamin --- src/cls/rgw/cls_rgw_types.h | 12 ++++++++++-- src/rgw/rgw_lc.cc | 32 +++++++++++++++++++++++++++++--- src/rgw/rgw_sal.h | 1 + src/rgw/rgw_sal_rados.cc | 2 ++ 4 files changed, 42 insertions(+), 5 deletions(-) diff --git a/src/cls/rgw/cls_rgw_types.h b/src/cls/rgw/cls_rgw_types.h index 9a9626af99a9..8e72f5081208 100644 --- a/src/cls/rgw/cls_rgw_types.h +++ b/src/cls/rgw/cls_rgw_types.h @@ -1239,23 +1239,31 @@ struct cls_rgw_lc_obj_head { time_t start_date = 0; std::string marker; + time_t shard_rollover_date = 0; cls_rgw_lc_obj_head() {} void encode(ceph::buffer::list& bl) const { - ENCODE_START(1, 1, bl); + ENCODE_START(2, 2, bl); uint64_t t = start_date; encode(t, bl); encode(marker, bl); + encode(shard_rollover_date, bl); ENCODE_FINISH(bl); } void decode(ceph::buffer::list::const_iterator& bl) { - DECODE_START(1, bl); + DECODE_START(2, bl); uint64_t t; decode(t, bl); start_date = static_cast(t); decode(marker, bl); + if (struct_v < 2) { + shard_rollover_date = 0; + } else { + decode(t, bl); + shard_rollover_date = static_cast(t); + } DECODE_FINISH(bl); } diff --git a/src/rgw/rgw_lc.cc b/src/rgw/rgw_lc.cc index 620576a9a496..b5c84a58195e 100644 --- a/src/rgw/rgw_lc.cc +++ b/src/rgw/rgw_lc.cc @@ -2097,6 +2097,21 @@ int RGWLC::process_bucket(int index, int max_lock_secs, LCWorker* worker, return ret; } /* RGWLC::process_bucket */ +static inline bool allow_shard_rollover(CephContext* cct, time_t now, time_t shard_rollover_date) +{ + /* return true iff: + * - non-debug scheduling is in effect, and + * - the current shard has not rolled over in the last 24 hours + */ + if (((shard_rollover_date < now) && + (now - shard_rollover_date > 24*60*60)) || + (! shard_rollover_date /* no rollover date stored */) || + (cct->_conf->rgw_lc_debug_interval > 0 /* defaults to -1 == disabled */)) { + return true; + } + return false; +} + int RGWLC::process(int index, int max_lock_secs, LCWorker* worker, bool once = false) { @@ -2147,7 +2162,15 @@ int RGWLC::process(int index, int max_lock_secs, LCWorker* worker, /* if there is nothing at head, try to reinitialize head.marker with the * first entry in the queue */ - if (head.marker.empty()) { + if (head.marker.empty() && + allow_shard_rollover(cct, now, head.shard_rollover_date) /* prevent multiple passes by diff. + * rgws,in same cycle */) { + + ldpp_dout(this, 5) << "RGWLC::process() process shard rollover lc_shard=" << lc_shard + << " head.marker=" << head.marker + << " head.shard_rollover_date=" << head.shard_rollover_date + << dendl; + vector entries; int ret = sal_lc->list_entries(lc_shard, head.marker, 1, entries); if (ret < 0) { @@ -2159,6 +2182,7 @@ int RGWLC::process(int index, int max_lock_secs, LCWorker* worker, entry = entries.front(); head.marker = entry.bucket; head.start_date = now; + head.shard_rollover_date = 0; } } else { ldpp_dout(this, 0) << "RGWLC::process() head.marker !empty() at START for shard==" @@ -2190,8 +2214,9 @@ int RGWLC::process(int index, int max_lock_secs, LCWorker* worker, } } } else { - ldpp_dout(this, 0) << "RGWLC::process() entry.bucket.empty() == true at START 1" - << " (this is impossible, but stop now)" + ldpp_dout(this, 5) << "RGWLC::process() entry.bucket.empty() == true at START 1" + << " (this is possible mainly before any lc policy has been stored" + << " or after removal of an lc_shard object)" << dendl; goto exit; } @@ -2281,6 +2306,7 @@ int RGWLC::process(int index, int max_lock_secs, LCWorker* worker, "RGWLC::process() cycle finished lc_shard=" << lc_shard << dendl; + head.shard_rollover_date = ceph_clock_now(); ret = sal_lc->put_head(lc_shard, head); if (ret < 0) { ldpp_dout(this, 0) << "RGWLC::process() failed to put head " diff --git a/src/rgw/rgw_sal.h b/src/rgw/rgw_sal.h index a04e200973af..4dde27c5162c 100644 --- a/src/rgw/rgw_sal.h +++ b/src/rgw/rgw_sal.h @@ -1286,6 +1286,7 @@ public: struct LCHead { time_t start_date{0}; std::string marker; + time_t shard_rollover_date{0}; LCHead() = default; LCHead(time_t _date, std::string& _marker) : start_date(_date), marker(_marker) {} diff --git a/src/rgw/rgw_sal_rados.cc b/src/rgw/rgw_sal_rados.cc index f84a5abec539..913e265306c3 100644 --- a/src/rgw/rgw_sal_rados.cc +++ b/src/rgw/rgw_sal_rados.cc @@ -2562,6 +2562,7 @@ int RadosLifecycle::get_head(const std::string& oid, LCHead& head) head.marker = cls_head.marker; head.start_date = cls_head.start_date; + head.shard_rollover_date = cls_head.shard_rollover_date; return ret; } @@ -2572,6 +2573,7 @@ int RadosLifecycle::put_head(const std::string& oid, const LCHead& head) cls_head.marker = head.marker; cls_head.start_date = head.start_date; + cls_head.shard_rollover_date = head.shard_rollover_date; return cls_rgw_lc_put_head(*store->getRados()->get_lc_pool_ctx(), oid, cls_head); } -- 2.47.3