From: Sage Weil Date: Tue, 6 Aug 2019 22:04:44 +0000 (-0500) Subject: osd/PeeringState: piggyback lease and ack on activation messages X-Git-Tag: v15.1.0~1379^2~7 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=39529a7f92479edb96178ef0548a29bd56967cb2;p=ceph.git osd/PeeringState: piggyback lease and ack on activation messages The lease goes out with the MOSDPGLog or info, and the ack comes back with the info. We no longer need to renew the lease explicitly in all_activated_and_committed() because we *just* piggybacked on activation. We can just wait for the normal renew event to fire. Signed-off-by: Sage Weil --- diff --git a/src/messages/MOSDPGInfo2.h b/src/messages/MOSDPGInfo2.h index 4cf1eff8a64..8163d402b74 100644 --- a/src/messages/MOSDPGInfo2.h +++ b/src/messages/MOSDPGInfo2.h @@ -16,6 +16,8 @@ public: epoch_t epoch_sent; epoch_t min_epoch; pg_info_t info; + std::optional lease; + std::optional lease_ack; spg_t get_spg() const override { return spgid; @@ -34,7 +36,9 @@ public: MInfoRec( pg_shard_t(get_source().num(), info.pgid.shard), info, - epoch_sent)); + epoch_sent, + lease, + lease_ack)); } MOSDPGInfo2() : MOSDPeeringOp{MSG_OSD_PG_INFO2, @@ -45,12 +49,16 @@ public: spg_t s, pg_info_t q, epoch_t sent, - epoch_t min) + epoch_t min, + std::optional l, + std::optional la) : MOSDPeeringOp{MSG_OSD_PG_INFO2, HEAD_VERSION, COMPAT_VERSION}, spgid(s), epoch_sent(sent), min_epoch(min), - info(q) { + info(q), + lease(l), + lease_ack(la) { set_priority(CEPH_MSG_PRIO_HIGH); } @@ -71,6 +79,8 @@ public: encode(epoch_sent, payload); encode(min_epoch, payload); encode(info, payload); + encode(lease, payload); + encode(lease_ack, payload); } void decode_payload() override { using ceph::decode; @@ -79,6 +89,8 @@ public: decode(epoch_sent, p); decode(min_epoch, p); decode(info, p); + decode(lease, p); + decode(lease_ack, p); } private: template diff --git a/src/osd/PGPeeringEvent.h b/src/osd/PGPeeringEvent.h index 04290f83af7..efe191d35cb 100644 --- a/src/osd/PGPeeringEvent.h +++ b/src/osd/PGPeeringEvent.h @@ -73,10 +73,21 @@ struct MInfoRec : boost::statechart::event< MInfoRec > { pg_shard_t from; pg_info_t info; epoch_t msg_epoch; - MInfoRec(pg_shard_t from, const pg_info_t &info, epoch_t msg_epoch) : - from(from), info(info), msg_epoch(msg_epoch) {} + std::optional lease; + std::optional lease_ack; + MInfoRec(pg_shard_t from, const pg_info_t &info, epoch_t msg_epoch, + std::optional l = {}, + std::optional la = {}) + : from(from), info(info), msg_epoch(msg_epoch), + lease(l), lease_ack(la) {} void print(std::ostream *out) const { *out << "MInfoRec from " << from << " info: " << info; + if (lease) { + *out << " " << *lease; + } + if (lease_ack) { + *out << " " << *lease_ack; + } } }; diff --git a/src/osd/PeeringState.cc b/src/osd/PeeringState.cc index 1770e0f2fad..a3293e83015 100644 --- a/src/osd/PeeringState.cc +++ b/src/osd/PeeringState.cc @@ -63,7 +63,9 @@ void BufferedRecoveryMessages::send_info( spg_t to_spgid, epoch_t min_epoch, epoch_t cur_epoch, - const pg_info_t &info) + const pg_info_t &info, + std::optional lease, + std::optional lease_ack) { if (require_osd_release >= ceph_release_t::octopus) { send_osd_message( @@ -72,7 +74,9 @@ void BufferedRecoveryMessages::send_info( to_spgid, info, cur_epoch, - min_epoch) + min_epoch, + lease, + lease_ack) ); } else { send_osd_message( @@ -1203,7 +1207,8 @@ void PeeringState::recalc_readable_until() } readable_until = min; readable_until_ub = min; - dout(20) << __func__ << " readable_until[_ub] " << readable_until << dendl; + dout(20) << __func__ << " readable_until[_ub] " << readable_until + << " (sent " << readable_until_ub_sent << ")" << dendl; } bool PeeringState::check_prior_readable_down_osds(const OSDMapRef& map) @@ -2406,7 +2411,8 @@ void PeeringState::activate( spg_t(info.pgid.pgid, peer.shard), get_osdmap_epoch(), // fixme: use lower epoch? get_osdmap_epoch(), - info); + info, + get_lease()); } else { psdout(10) << "activate peer osd." << peer << " is up to date, but sending pg_log anyway" << dendl; @@ -5631,13 +5637,16 @@ boost::statechart::result PeeringState::Active::react(const MInfoRec& infoevt) ceph_assert(ps->is_primary()); ceph_assert(!ps->acting_recovery_backfill.empty()); + if (infoevt.lease_ack) { + ps->proc_lease_ack(infoevt.from.osd, *infoevt.lease_ack); + } // don't update history (yet) if we are active and primary; the replica // may be telling us they have activated (and committed) but we can't // share that until _everyone_ does the same. if (ps->is_acting_recovery_backfill(infoevt.from) && ps->peer_activated.count(infoevt.from) == 0) { psdout(10) << " peer osd." << infoevt.from - << " activated and committed" << dendl; + << " activated and committed" << dendl; ps->peer_activated.insert(infoevt.from); ps->blocked_by.erase(infoevt.from.shard); pl->publish_stats_to_osd(); @@ -5831,8 +5840,6 @@ void PeeringState::Active::all_activated_and_committed() ceph_assert(!ps->acting_recovery_backfill.empty()); ceph_assert(ps->blocked_by.empty()); - ps->send_lease(); - // Degraded? ps->update_calc_stats(); if (ps->info.stats.stats.sum.num_objects_degraded) { @@ -5907,7 +5914,9 @@ boost::statechart::result PeeringState::ReplicaActive::react( spg_t(ps->info.pgid.pgid, ps->get_primary().shard), epoch, epoch, - i); + i, + {}, /* lease */ + ps->get_lease_ack()); if (ps->acting.size() >= ps->pool.info.min_size) { ps->state_set(PG_STATE_ACTIVE); @@ -6071,6 +6080,10 @@ boost::statechart::result PeeringState::Stray::react(const MInfoRec& infoevt) ps->info.hit_set = infoevt.info.hit_set; } + if (infoevt.lease) { + ps->proc_lease(*infoevt.lease); + } + ceph_assert(infoevt.info.last_update == ps->info.last_update); ceph_assert(ps->pg_log.get_head() == ps->info.last_update); diff --git a/src/osd/PeeringState.h b/src/osd/PeeringState.h index f5752c02b4f..a32be2a3bf5 100644 --- a/src/osd/PeeringState.h +++ b/src/osd/PeeringState.h @@ -88,7 +88,9 @@ struct BufferedRecoveryMessages { void send_query(int to, spg_t spgid, const pg_query_t &q); void send_info(int to, spg_t to_spgid, epoch_t min_epoch, epoch_t cur_epoch, - const pg_info_t &info); + const pg_info_t &info, + std::optional lease = {}, + std::optional lease_ack = {}); }; struct HeartbeatStamps : public RefCountedObject { @@ -236,8 +238,11 @@ struct PeeringCtxWrapper { } void send_info(int to, spg_t to_spgid, epoch_t min_epoch, epoch_t cur_epoch, - const pg_info_t &info) { - msgs.send_info(to, to_spgid, min_epoch, cur_epoch, info); + const pg_info_t &info, + std::optional lease = {}, + std::optional lease_ack = {}) { + msgs.send_info(to, to_spgid, min_epoch, cur_epoch, info, + lease, lease_ack); } };