osd: schedule regular lease renewals

Signed-off-by: Sage Weil <sage@redhat.com>

diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc
index 0b18c96c3c4c9f785a2f6dfeed89d7b3f2ed5345..78d01a920afdf7ebaeaba03c82a89e83c84c30aa 100644 (file)
--- a/src/osd/OSD.cc
+++ b/src/osd/OSD.cc
@@ -455,6 +455,16 @@ HeartbeatStampsRef OSDService::get_hb_stamps(unsigned peer)
   return hb_stamps[peer];
 }
 
+void OSDService::queue_renew_lease(epoch_t epoch, spg_t spgid)
+{
+  osd->enqueue_peering_evt(
+    spgid,
+    PGPeeringEventRef(
+      std::make_shared<PGPeeringEvent>(
+       epoch, epoch,
+       RenewLease())));
+}
+
 void OSDService::start_shutdown()
 {
   {

diff --git a/src/osd/OSD.h b/src/osd/OSD.h
index efa7678c871170b2026d949eb0c86d833087c7fa..57c8a87c3c61bb02f0f38bb9c6f67c7e6a153de1 100644 (file)
--- a/src/osd/OSD.h
+++ b/src/osd/OSD.h
@@ -862,6 +862,8 @@ public:
   // Timer for readable leases
   ceph::timer<ceph::mono_clock> mono_timer = ceph::timer<ceph::mono_clock>{ceph::construct_suspended};
 
+  void queue_renew_lease(epoch_t epoch, spg_t spgid);
+
   // -- stopping --
   ceph::mutex is_stopping_lock = ceph::make_mutex("OSDService::is_stopping_lock");
   ceph::condition_variable is_stopping_cond;

diff --git a/src/osd/PG.cc b/src/osd/PG.cc
index ae28ad704d19ee1a4a9fe3e4afe101d4e80bf385..2401d9defdd57f2ff86985b130d0b88363467272 100644 (file)
--- a/src/osd/PG.cc
+++ b/src/osd/PG.cc
@@ -1807,6 +1807,17 @@ HeartbeatStampsRef PG::get_hb_stamps(int peer)
   return osd->get_hb_stamps(peer);
 }
 
+void PG::schedule_renew_lease(epoch_t lpr, ceph::timespan delay)
+{
+  auto spgid = info.pgid;
+  auto o = osd;
+  osd->mono_timer.add_event(
+    delay,
+    [o, lpr, spgid]() {
+      o->queue_renew_lease(lpr, spgid);
+    });
+}
+
 void PG::rebuild_missing_set_with_deletes(PGLog &pglog)
 {
   pglog.rebuild_missing_set_with_deletes(

diff --git a/src/osd/PG.h b/src/osd/PG.h
index 1eb149d55454cca90a128b0ae3585bf9fa90c0c8..43a3a750237b24909760d7375f587eb5eff58982 100644 (file)
--- a/src/osd/PG.h
+++ b/src/osd/PG.h
@@ -480,6 +480,7 @@ public:
 
   ceph::signedspan get_mnow() override;
   HeartbeatStampsRef get_hb_stamps(int peer) override;
+  void schedule_renew_lease(epoch_t lpr, ceph::timespan delay) override;
 
   void rebuild_missing_set_with_deletes(PGLog &pglog) override;
 

diff --git a/src/osd/PGPeeringEvent.h b/src/osd/PGPeeringEvent.h
index aaa8b0e939a2f3faa1e54ae1bc3726febfb7f9fe..04290f83af724ed7f633d4dbc5528afb685237f8 100644 (file)
--- a/src/osd/PGPeeringEvent.h
+++ b/src/osd/PGPeeringEvent.h
@@ -207,3 +207,5 @@ struct DeferBackfill : boost::statechart::event<DeferBackfill> {
     *out << "DeferBackfill: delay " << delay;
   }
 };
+
+TrivialEvent(RenewLease)

diff --git a/src/osd/PeeringState.cc b/src/osd/PeeringState.cc
index 038eeb193825c5b5f3c435143933b7ee20fc82b0..a0f77b81eafcbc1fe478efce7dbccdb03aa1e4e2 100644 (file)
--- a/src/osd/PeeringState.cc
+++ b/src/osd/PeeringState.cc
@@ -747,9 +747,6 @@ void PeeringState::on_new_interval()
   acting_readable_until_ub.clear();
   if (is_primary()) {
     acting_readable_until_ub.resize(acting.size(), ceph::signedspan::zero());
-
-    // start lease here, so that we get acks during peering
-    renew_lease(pl->get_mnow());
   }
 
   pl->on_new_interval();
@@ -1082,8 +1079,34 @@ bool PeeringState::set_force_backfill(bool b)
   return did;
 }
 
+void PeeringState::schedule_renew_lease()
+{
+  pl->schedule_renew_lease(
+    last_peering_reset,
+    readable_interval / 2);
+}
+
+void PeeringState::send_lease()
+{
+  epoch_t epoch = pl->get_osdmap_epoch();
+  for (auto peer : actingset) {
+    if (peer == pg_whoami) {
+      continue;
+    }
+    pl->send_cluster_message(
+      peer.osd,
+      new MOSDPGLease(epoch,
+                     spg_t(spgid.pgid, peer.shard),
+                     get_lease()),
+      epoch);
+  }
+}
+
 void PeeringState::proc_lease(const pg_lease_t& l)
 {
+  if (get_role() < 0) {
+    return;
+  }
   psdout(10) << __func__ << " " << l << dendl;
   if (l.readable_until_ub > readable_until_ub_from_primary) {
     readable_until_ub_from_primary = l.readable_until_ub;
@@ -2432,6 +2455,10 @@ void PeeringState::activate(
       }
     }
 
+    renew_lease(pl->get_mnow());
+    send_lease();
+    schedule_renew_lease();
+
     // Set up missing_loc
     set<pg_shard_t> complete_shards;
     for (set<pg_shard_t>::iterator i = acting_recovery_backfill.begin();
@@ -5689,6 +5716,15 @@ boost::statechart::result PeeringState::Active::react(const AllReplicasActivated
   return discard_event();
 }
 
+boost::statechart::result PeeringState::Active::react(const RenewLease& rl)
+{
+  DECLARE_LOCALS;
+  ps->renew_lease(pl->get_mnow());
+  ps->send_lease();
+  ps->schedule_renew_lease();
+  return discard_event();
+}
+
 boost::statechart::result PeeringState::Active::react(const MLeaseAck& la)
 {
   DECLARE_LOCALS;
@@ -5805,7 +5841,9 @@ boost::statechart::result PeeringState::ReplicaActive::react(const MLease& l)
   ps->proc_lease(l.lease);
   pl->send_cluster_message(
     ps->get_primary().osd,
-    new MOSDPGLeaseAck(epoch, spgid, ps->get_lease_ack()),
+    new MOSDPGLeaseAck(epoch,
+                      spg_t(spgid.pgid, ps->get_primary().shard),
+                      ps->get_lease_ack()),
     epoch);
   return discard_event();
 }

diff --git a/src/osd/PeeringState.h b/src/osd/PeeringState.h
index e6a2408fbe850a4cdd746ea86beaf21f492b36dc..d486507c2aca97b70cc591929a98c9a38a52b271 100644 (file)
--- a/src/osd/PeeringState.h
+++ b/src/osd/PeeringState.h
@@ -272,6 +272,7 @@ public:
 
     virtual ceph::signedspan get_mnow() = 0;
     virtual HeartbeatStampsRef get_hb_stamps(int peer) = 0;
+    virtual void schedule_renew_lease(epoch_t plr, ceph::timespan delay) = 0;
 
     // ============ Flush state ==================
     /**
@@ -813,6 +814,7 @@ public:
       boost::statechart::custom_reaction< RemoteReservationRevokedTooFull>,
       boost::statechart::custom_reaction< RemoteReservationRevoked>,
       boost::statechart::custom_reaction< DoRecovery>,
+      boost::statechart::custom_reaction< RenewLease>,
       boost::statechart::custom_reaction< MLeaseAck>
       > reactions;
     boost::statechart::result react(const QueryState& q);
@@ -827,6 +829,7 @@ public:
     }
     boost::statechart::result react(const ActivateCommitted&);
     boost::statechart::result react(const AllReplicasActivated&);
+    boost::statechart::result react(const RenewLease&);
     boost::statechart::result react(const MLeaseAck&);
     boost::statechart::result react(const DeferRecovery& evt) {
       return discard_event();
@@ -1949,7 +1952,7 @@ public:
       recalc_readable_until();
     }
   }
-
+  void send_lease();
   void schedule_renew_lease();
 
   pg_lease_t get_lease() {