rgwlc: prevent multiple runs per scheduling cycle

Restore (and robustify) the assertion that, in general, each bucket
shard should be processed once per scheduling cycle.

If the prior cycle did not finish, processing in the current cyhcle
will continue from the marker where the last cycle left off.

Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>

diff --git a/src/cls/rgw/cls_rgw_types.h b/src/cls/rgw/cls_rgw_types.h
index 9a9626af99a9b2647326b667670fe8e3afd5ee51..8e72f50812086d046dbc1fb8bec18d1c633aa024 100644 (file)
--- a/src/cls/rgw/cls_rgw_types.h
+++ b/src/cls/rgw/cls_rgw_types.h
@@ -1239,23 +1239,31 @@ struct cls_rgw_lc_obj_head
 {
   time_t start_date = 0;
   std::string marker;
+  time_t shard_rollover_date = 0;
 
   cls_rgw_lc_obj_head() {}
 
   void encode(ceph::buffer::list& bl) const {
-    ENCODE_START(1, 1, bl);
+    ENCODE_START(2, 2, bl);
     uint64_t t = start_date;
     encode(t, bl);
     encode(marker, bl);
+    encode(shard_rollover_date, bl);
     ENCODE_FINISH(bl);
   }
 
   void decode(ceph::buffer::list::const_iterator& bl) {
-    DECODE_START(1, bl);
+    DECODE_START(2, bl);
     uint64_t t;
     decode(t, bl);
     start_date = static_cast<time_t>(t);
     decode(marker, bl);
+    if (struct_v < 2) {
+      shard_rollover_date = 0;
+    } else {
+      decode(t, bl);
+      shard_rollover_date = static_cast<time_t>(t);
+    }
     DECODE_FINISH(bl);
   }
 

diff --git a/src/rgw/rgw_lc.cc b/src/rgw/rgw_lc.cc
index 620576a9a496c67d60919745703ba3409fb84561..b5c84a58195ee222c018d6b4932a15254cff4ded 100644 (file)
--- a/src/rgw/rgw_lc.cc
+++ b/src/rgw/rgw_lc.cc
@@ -2097,6 +2097,21 @@ int RGWLC::process_bucket(int index, int max_lock_secs, LCWorker* worker,
   return ret;
 } /* RGWLC::process_bucket */
 
+static inline bool allow_shard_rollover(CephContext* cct, time_t now, time_t shard_rollover_date)
+{
+  /* return true iff:
+   *    - non-debug scheduling is in effect, and
+   *    - the current shard has not rolled over in the last 24 hours
+   */
+  if (((shard_rollover_date < now) &&
+       (now - shard_rollover_date > 24*60*60)) ||
+      (! shard_rollover_date /* no rollover date stored */) ||
+      (cct->_conf->rgw_lc_debug_interval > 0 /* defaults to -1 == disabled */)) {
+    return true;
+  }
+  return false;
+}
+
 int RGWLC::process(int index, int max_lock_secs, LCWorker* worker,
                   bool once = false)
 {
@@ -2147,7 +2162,15 @@ int RGWLC::process(int index, int max_lock_secs, LCWorker* worker,
 
     /* if there is nothing at head, try to reinitialize head.marker with the
      * first entry in the queue */
-    if (head.marker.empty()) {
+    if (head.marker.empty() &&
+       allow_shard_rollover(cct, now, head.shard_rollover_date) /* prevent multiple passes by diff.
+                                                                 * rgws,in same cycle */) {
+
+      ldpp_dout(this, 5) << "RGWLC::process() process shard rollover lc_shard=" << lc_shard
+                        << " head.marker=" << head.marker
+                        << " head.shard_rollover_date=" << head.shard_rollover_date
+                        << dendl;
+
       vector<rgw::sal::Lifecycle::LCEntry> entries;
       int ret = sal_lc->list_entries(lc_shard, head.marker, 1, entries);
       if (ret < 0) {
@@ -2159,6 +2182,7 @@ int RGWLC::process(int index, int max_lock_secs, LCWorker* worker,
        entry = entries.front();
        head.marker = entry.bucket;
        head.start_date = now;
+       head.shard_rollover_date = 0;
       }
     } else {
       ldpp_dout(this, 0) << "RGWLC::process() head.marker !empty() at START for shard=="
@@ -2190,8 +2214,9 @@ int RGWLC::process(int index, int max_lock_secs, LCWorker* worker,
         }
       }
     } else {
-      ldpp_dout(this, 0) << "RGWLC::process() entry.bucket.empty() == true at START 1"
-                        << " (this is impossible, but stop now)"
+      ldpp_dout(this, 5) << "RGWLC::process() entry.bucket.empty() == true at START 1"
+                        << " (this is possible mainly before any lc policy has been stored"
+                        << " or after removal of an lc_shard object)"
                          << dendl;
       goto exit;
     }
@@ -2281,6 +2306,7 @@ int RGWLC::process(int index, int max_lock_secs, LCWorker* worker,
        "RGWLC::process() cycle finished lc_shard="
                         << lc_shard
                         << dendl;
+      head.shard_rollover_date = ceph_clock_now();
       ret = sal_lc->put_head(lc_shard,  head);
       if (ret < 0) {
        ldpp_dout(this, 0) << "RGWLC::process() failed to put head "

diff --git a/src/rgw/rgw_sal.h b/src/rgw/rgw_sal.h
index a04e200973af32c3c50248ed696831eb4499e341..4dde27c5162cca7a938eec8d2ecbadfb98d36b27 100644 (file)
--- a/src/rgw/rgw_sal.h
+++ b/src/rgw/rgw_sal.h
@@ -1286,6 +1286,7 @@ public:
   struct LCHead {
     time_t start_date{0};
     std::string marker;
+    time_t shard_rollover_date{0};
 
     LCHead() = default;
     LCHead(time_t _date, std::string& _marker) : start_date(_date), marker(_marker) {}

diff --git a/src/rgw/rgw_sal_rados.cc b/src/rgw/rgw_sal_rados.cc
index f84a5abec5398a601ba50f025f4479653c937185..913e265306c3f60b6f62b6e20fa403f1e394a2bd 100644 (file)
--- a/src/rgw/rgw_sal_rados.cc
+++ b/src/rgw/rgw_sal_rados.cc
@@ -2562,6 +2562,7 @@ int RadosLifecycle::get_head(const std::string& oid, LCHead& head)
 
   head.marker = cls_head.marker;
   head.start_date = cls_head.start_date;
+  head.shard_rollover_date = cls_head.shard_rollover_date;
 
   return ret;
 }
@@ -2572,6 +2573,7 @@ int RadosLifecycle::put_head(const std::string& oid, const LCHead& head)
 
   cls_head.marker = head.marker;
   cls_head.start_date = head.start_date;
+  cls_head.shard_rollover_date = head.shard_rollover_date;
 
   return cls_rgw_lc_put_head(*store->getRados()->get_lc_pool_ctx(), oid, cls_head);
 }