osd/PeeringState: piggyback lease and ack on activation messages

The lease goes out with the MOSDPGLog or info, and the ack comes back with
the info.

We no longer need to renew the lease explicitly in
all_activated_and_committed() because we *just* piggybacked on activation.
We can just wait for the normal renew event to fire.

Signed-off-by: Sage Weil <sage@redhat.com>

diff --git a/src/messages/MOSDPGInfo2.h b/src/messages/MOSDPGInfo2.h
index 4cf1eff8a6469bef144e4345072b8a3740960be1..8163d402b7436c738453370a5e0a800b94bebc3b 100644 (file)
--- a/src/messages/MOSDPGInfo2.h
+++ b/src/messages/MOSDPGInfo2.h
@@ -16,6 +16,8 @@ public:
   epoch_t epoch_sent;
   epoch_t min_epoch;
   pg_info_t info;
+  std::optional<pg_lease_t> lease;
+  std::optional<pg_lease_ack_t> lease_ack;
 
   spg_t get_spg() const override {
     return spgid;
@@ -34,7 +36,9 @@ public:
       MInfoRec(
        pg_shard_t(get_source().num(), info.pgid.shard),
        info,
-       epoch_sent));
+       epoch_sent,
+       lease,
+       lease_ack));
   }
 
   MOSDPGInfo2() : MOSDPeeringOp{MSG_OSD_PG_INFO2,
@@ -45,12 +49,16 @@ public:
     spg_t s,
     pg_info_t q,
     epoch_t sent,
-    epoch_t min)
+    epoch_t min,
+    std::optional<pg_lease_t> l,
+    std::optional<pg_lease_ack_t> la)
     : MOSDPeeringOp{MSG_OSD_PG_INFO2, HEAD_VERSION, COMPAT_VERSION},
       spgid(s),
       epoch_sent(sent),
       min_epoch(min),
-      info(q) {
+      info(q),
+      lease(l),
+      lease_ack(la) {
     set_priority(CEPH_MSG_PRIO_HIGH);
   }
 
@@ -71,6 +79,8 @@ public:
     encode(epoch_sent, payload);
     encode(min_epoch, payload);
     encode(info, payload);
+    encode(lease, payload);
+    encode(lease_ack, payload);
   }
   void decode_payload() override {
     using ceph::decode;
@@ -79,6 +89,8 @@ public:
     decode(epoch_sent, p);
     decode(min_epoch, p);
     decode(info, p);
+    decode(lease, p);
+    decode(lease_ack, p);
   }
 private:
   template<class T, typename... Args>

diff --git a/src/osd/PGPeeringEvent.h b/src/osd/PGPeeringEvent.h
index 04290f83af724ed7f633d4dbc5528afb685237f8..efe191d35cb8ceff3096085a81936290309b6989 100644 (file)
--- a/src/osd/PGPeeringEvent.h
+++ b/src/osd/PGPeeringEvent.h
@@ -73,10 +73,21 @@ struct MInfoRec : boost::statechart::event< MInfoRec > {
   pg_shard_t from;
   pg_info_t info;
   epoch_t msg_epoch;
-  MInfoRec(pg_shard_t from, const pg_info_t &info, epoch_t msg_epoch) :
-    from(from), info(info), msg_epoch(msg_epoch) {}
+  std::optional<pg_lease_t> lease;
+  std::optional<pg_lease_ack_t> lease_ack;
+  MInfoRec(pg_shard_t from, const pg_info_t &info, epoch_t msg_epoch,
+          std::optional<pg_lease_t> l = {},
+          std::optional<pg_lease_ack_t> la = {})
+    : from(from), info(info), msg_epoch(msg_epoch),
+      lease(l), lease_ack(la) {}
   void print(std::ostream *out) const {
     *out << "MInfoRec from " << from << " info: " << info;
+    if (lease) {
+      *out << " " << *lease;
+    }
+    if (lease_ack) {
+      *out << " " << *lease_ack;
+    }
   }
 };
 

diff --git a/src/osd/PeeringState.cc b/src/osd/PeeringState.cc
index 1770e0f2fade9c63721aa44d14f714bacb0b03ee..a3293e8301567aceeea87ac3a41b1a3fd0b5a745 100644 (file)
--- a/src/osd/PeeringState.cc
+++ b/src/osd/PeeringState.cc
@@ -63,7 +63,9 @@ void BufferedRecoveryMessages::send_info(
   spg_t to_spgid,
   epoch_t min_epoch,
   epoch_t cur_epoch,
-  const pg_info_t &info)
+  const pg_info_t &info,
+  std::optional<pg_lease_t> lease,
+  std::optional<pg_lease_ack_t> lease_ack)
 {
   if (require_osd_release >= ceph_release_t::octopus) {
     send_osd_message(
@@ -72,7 +74,9 @@ void BufferedRecoveryMessages::send_info(
        to_spgid,
        info,
        cur_epoch,
-       min_epoch)
+       min_epoch,
+       lease,
+       lease_ack)
       );
   } else {
     send_osd_message(
@@ -1203,7 +1207,8 @@ void PeeringState::recalc_readable_until()
   }
   readable_until = min;
   readable_until_ub = min;
-  dout(20) << __func__ << " readable_until[_ub] " << readable_until << dendl;
+  dout(20) << __func__ << " readable_until[_ub] " << readable_until
+          << " (sent " << readable_until_ub_sent << ")" << dendl;
 }
 
 bool PeeringState::check_prior_readable_down_osds(const OSDMapRef& map)
@@ -2406,7 +2411,8 @@ void PeeringState::activate(
            spg_t(info.pgid.pgid, peer.shard),
            get_osdmap_epoch(), // fixme: use lower epoch?
            get_osdmap_epoch(),
-           info);
+           info,
+           get_lease());
        } else {
          psdout(10) << "activate peer osd." << peer
                     << " is up to date, but sending pg_log anyway" << dendl;
@@ -5631,13 +5637,16 @@ boost::statechart::result PeeringState::Active::react(const MInfoRec& infoevt)
   ceph_assert(ps->is_primary());
 
   ceph_assert(!ps->acting_recovery_backfill.empty());
+  if (infoevt.lease_ack) {
+    ps->proc_lease_ack(infoevt.from.osd, *infoevt.lease_ack);
+  }
   // don't update history (yet) if we are active and primary; the replica
   // may be telling us they have activated (and committed) but we can't
   // share that until _everyone_ does the same.
   if (ps->is_acting_recovery_backfill(infoevt.from) &&
       ps->peer_activated.count(infoevt.from) == 0) {
     psdout(10) << " peer osd." << infoevt.from
-                      << " activated and committed" << dendl;
+              << " activated and committed" << dendl;
     ps->peer_activated.insert(infoevt.from);
     ps->blocked_by.erase(infoevt.from.shard);
     pl->publish_stats_to_osd();
@@ -5831,8 +5840,6 @@ void PeeringState::Active::all_activated_and_committed()
   ceph_assert(!ps->acting_recovery_backfill.empty());
   ceph_assert(ps->blocked_by.empty());
 
-  ps->send_lease();
-
   // Degraded?
   ps->update_calc_stats();
   if (ps->info.stats.stats.sum.num_objects_degraded) {
@@ -5907,7 +5914,9 @@ boost::statechart::result PeeringState::ReplicaActive::react(
     spg_t(ps->info.pgid.pgid, ps->get_primary().shard),
     epoch,
     epoch,
-    i);
+    i,
+    {}, /* lease */
+    ps->get_lease_ack());
 
   if (ps->acting.size() >= ps->pool.info.min_size) {
     ps->state_set(PG_STATE_ACTIVE);
@@ -6071,6 +6080,10 @@ boost::statechart::result PeeringState::Stray::react(const MInfoRec& infoevt)
     ps->info.hit_set = infoevt.info.hit_set;
   }
 
+  if (infoevt.lease) {
+    ps->proc_lease(*infoevt.lease);
+  }
+
   ceph_assert(infoevt.info.last_update == ps->info.last_update);
   ceph_assert(ps->pg_log.get_head() == ps->info.last_update);
 

diff --git a/src/osd/PeeringState.h b/src/osd/PeeringState.h
index f5752c02b4fd8a3c9e5c46e3225bfc8a33271b85..a32be2a3bf5a948b34f03a8558e3a8e07dc120a3 100644 (file)
--- a/src/osd/PeeringState.h
+++ b/src/osd/PeeringState.h
@@ -88,7 +88,9 @@ struct BufferedRecoveryMessages {
   void send_query(int to, spg_t spgid, const pg_query_t &q);
   void send_info(int to, spg_t to_spgid,
                 epoch_t min_epoch, epoch_t cur_epoch,
-                const pg_info_t &info);
+                const pg_info_t &info,
+                std::optional<pg_lease_t> lease = {},
+                std::optional<pg_lease_ack_t> lease_ack = {});
 };
 
 struct HeartbeatStamps : public RefCountedObject {
@@ -236,8 +238,11 @@ struct PeeringCtxWrapper {
   }
   void send_info(int to, spg_t to_spgid,
                 epoch_t min_epoch, epoch_t cur_epoch,
-                const pg_info_t &info) {
-    msgs.send_info(to, to_spgid, min_epoch, cur_epoch, info);
+                const pg_info_t &info,
+                std::optional<pg_lease_t> lease = {},
+                std::optional<pg_lease_ack_t> lease_ack = {}) {
+    msgs.send_info(to, to_spgid, min_epoch, cur_epoch, info,
+                  lease, lease_ack);
   }
 };