rgw: add realm epoch to RGWRealm and RGWPeriod

RGWRealm::epoch is incremented on each new period, and
RGWPeriod::realm_epoch tracks the realm epoch when it was created

this makes it easy to determine whether a given period comes before or
after another period, which is needed to build a view of the period
history.  it also makes it easy to detect forks in the period history,
where two different periods share the same realm_epoch

Signed-off-by: Casey Bodley <cbodley@redhat.com>

diff --git a/src/rgw/rgw_json_enc.cc b/src/rgw/rgw_json_enc.cc
index 7e0682ee6b20fe92472fef3de6cef1eb48cc8a5a..6bbee35ab1e06dea296322c72481a23eea78d5fc 100644 (file)
--- a/src/rgw/rgw_json_enc.cc
+++ b/src/rgw/rgw_json_enc.cc
@@ -758,6 +758,7 @@ void RGWPeriod::dump(Formatter *f) const
   encode_json("period_config", period_config, f);
   encode_json("realm_id", realm_id, f);
   encode_json("realm_name", realm_name, f);
+  encode_json("realm_epoch", realm_epoch, f);
 }
 
 void RGWPeriod::decode_json(JSONObj *obj)
@@ -772,6 +773,7 @@ void RGWPeriod::decode_json(JSONObj *obj)
   JSONDecoder::decode_json("period_config", period_config, obj);
   JSONDecoder::decode_json("realm_id", realm_id, obj);
   JSONDecoder::decode_json("realm_name", realm_name, obj);
+  JSONDecoder::decode_json("realm_epoch", realm_epoch, obj);
 }
 
 void RGWZoneParams::dump(Formatter *f) const
@@ -1034,6 +1036,7 @@ void RGWRealm::dump(Formatter *f) const
 {
   RGWSystemMetaObj::dump(f);
   encode_json("current_period", current_period, f);
+  encode_json("epoch", epoch, f);
 }
 
 
@@ -1041,6 +1044,7 @@ void RGWRealm::decode_json(JSONObj *obj)
 {
   RGWSystemMetaObj::decode_json(obj);
   JSONDecoder::decode_json("current_period", current_period, obj);
+  JSONDecoder::decode_json("epoch", epoch, obj);
 }
 
 void KeystoneToken::Metadata::decode_json(JSONObj *obj)

diff --git a/src/rgw/rgw_rados.cc b/src/rgw/rgw_rados.cc
index 3f1d0f5311f8365c4a8f72748f23814b8679a82c..3418418e6ff914afa836880ed23558c30369785c 100644 (file)
--- a/src/rgw/rgw_rados.cc
+++ b/src/rgw/rgw_rados.cc
@@ -785,7 +785,20 @@ const string& RGWRealm::get_info_oid_prefix(bool old_format)
 
 int RGWRealm::set_current_period(RGWPeriod& period)
 {
+  // update realm epoch to match the period's
+  if (epoch > period.get_realm_epoch()) {
+    lderr(cct) << "ERROR: set_current_period with old realm epoch "
+        << period.get_realm_epoch() << ", current epoch=" << epoch << dendl;
+    return -EINVAL;
+  }
+  if (epoch == period.get_realm_epoch() && current_period != period.get_id()) {
+    lderr(cct) << "ERROR: set_current_period with same realm epoch "
+        << period.get_realm_epoch() << ", but different period id "
+        << period.get_id() << " != " << current_period << dendl;
+    return -EINVAL;
+  }
 
+  epoch = period.get_realm_epoch();
   current_period = period.get_id();
 
   int ret = update();
@@ -1208,6 +1221,7 @@ void RGWPeriod::fork()
   predecessor_uuid = id;
   id = get_staging_id(realm_id);
   period_map.reset();
+  realm_epoch++;
 }
 
 void RGWPeriod::update(const RGWZoneGroupMap& map)
@@ -1273,6 +1287,13 @@ int RGWPeriod::commit(RGWRealm& realm, const RGWPeriod& current_period)
         << dendl;
     return -EINVAL;
   }
+  // realm epoch must be 1 greater than current period
+  if (realm_epoch != current_period.get_realm_epoch() + 1) {
+    lderr(cct) << "period's realm epoch " << realm_epoch
+        << " does not come directly after current realm epoch "
+        << current_period.get_realm_epoch() << dendl;
+    return -EINVAL;
+  }
   // did the master zone change?
   if (master_zone != current_period.get_master_zone()) {
     // store the current metadata sync status in the period
@@ -1310,6 +1331,7 @@ int RGWPeriod::commit(RGWRealm& realm, const RGWPeriod& current_period)
   set_id(current_period.get_id());
   set_epoch(current_period.get_epoch() + 1);
   set_predecessor(current_period.get_predecessor());
+  realm_epoch = current_period.get_realm_epoch();
   // write the period to rados
   int r = store_info(false);
   if (r < 0) {

diff --git a/src/rgw/rgw_rados.h b/src/rgw/rgw_rados.h
index 6413c7f825cdf2682e8a3d823acda10f5c513511..c35e01a5e23d93eb473779256d33c07d85a307c7 100644 (file)
--- a/src/rgw/rgw_rados.h
+++ b/src/rgw/rgw_rados.h
@@ -1287,6 +1287,7 @@ class RGWPeriod;
 class RGWRealm : public RGWSystemMetaObj
 {
   string current_period;
+  epoch_t epoch{0}; //< realm epoch, incremented for each new period
 
   int create_control();
   int delete_control();
@@ -1300,6 +1301,7 @@ public:
     ENCODE_START(1, 1, bl);
     RGWSystemMetaObj::encode(bl);
     ::encode(current_period, bl);
+    ::encode(epoch, bl);
     ENCODE_FINISH(bl);
   }
 
@@ -1307,6 +1309,7 @@ public:
     DECODE_START(1, bl);
     RGWSystemMetaObj::decode(bl);
     ::decode(current_period, bl);
+    ::decode(epoch, bl);
     DECODE_FINISH(bl);
   }
 
@@ -1326,6 +1329,8 @@ public:
   }
   int set_current_period(RGWPeriod& period);
 
+  epoch_t get_epoch() const { return epoch; }
+
   string get_control_oid();
   /// send a notify on the realm control object
   int notify_zone(bufferlist& bl);
@@ -1367,6 +1372,7 @@ class RGWPeriod
 
   string realm_id;
   string realm_name;
+  epoch_t realm_epoch{1}; //< realm epoch when period was made current
 
   CephContext *cct;
   RGWRados *store;
@@ -1390,6 +1396,7 @@ public:
 
   const string& get_id() const { return id; }
   epoch_t get_epoch() const { return epoch; }
+  epoch_t get_realm_epoch() const { return realm_epoch; }
   const string& get_predecessor() const { return predecessor_uuid; }
   const string& get_master_zone() const { return master_zone; }
   const string& get_master_zonegroup() const { return master_zonegroup; }
@@ -1444,9 +1451,10 @@ public:
   int commit(RGWRealm& realm, const RGWPeriod &current_period);
 
   void encode(bufferlist& bl) const {
-    ENCODE_START(1, 1, bl);    
+    ENCODE_START(1, 1, bl);
     ::encode(id, bl);
     ::encode(epoch, bl);
+    ::encode(realm_epoch, bl);
     ::encode(predecessor_uuid, bl);
     ::encode(sync_status, bl);
     ::encode(period_map, bl);
@@ -1462,6 +1470,7 @@ public:
     DECODE_START(1, bl);
     ::decode(id, bl);
     ::decode(epoch, bl);
+    ::decode(realm_epoch, bl);
     ::decode(predecessor_uuid, bl);
     ::decode(sync_status, bl);
     ::decode(period_map, bl);