From 8eac58354f3de0a6ca6bc496438b23c8068c4e91 Mon Sep 17 00:00:00 2001 From: Greg Farnum Date: Wed, 19 Jun 2019 16:32:30 -0700 Subject: [PATCH] elector: split ElectionLogic into its own compilation unit Just copy the files, then delete the inappropriate bits. Switched to using ldout in ElectionLogic.cc, added a few functions I missed to the ElectionOwner interface, and moved Elector() into the .cc so I could grab cct out of it. Signed-off-by: Greg Farnum --- src/mon/CMakeLists.txt | 1 + src/mon/ElectionLogic.cc | 219 +++++++++++++++++++++++++++++++++++++++ src/mon/ElectionLogic.h | 69 ++++++++++++ src/mon/Elector.cc | 199 +---------------------------------- src/mon/Elector.h | 48 +-------- 5 files changed, 297 insertions(+), 239 deletions(-) create mode 100644 src/mon/ElectionLogic.cc create mode 100644 src/mon/ElectionLogic.h diff --git a/src/mon/CMakeLists.txt b/src/mon/CMakeLists.txt index 62b41dc2acd99..f1ac49bde5646 100644 --- a/src/mon/CMakeLists.txt +++ b/src/mon/CMakeLists.txt @@ -18,6 +18,7 @@ set(lib_mon_srcs ConfigMap.cc ConfigMonitor.cc Elector.cc + ElectionLogic.cc HealthMonitor.cc ConfigKeyService.cc ../mds/MDSAuthCaps.cc diff --git a/src/mon/ElectionLogic.cc b/src/mon/ElectionLogic.cc new file mode 100644 index 0000000000000..0d456ba713ea7 --- /dev/null +++ b/src/mon/ElectionLogic.cc @@ -0,0 +1,219 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2004-2006 Sage Weil + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + */ + +#include "ElectionLogic.h" + +#include "include/ceph_assert.h" +#include "common/dout.h" + +#define dout_subsys ceph_subsys_mon +#undef dout_prefix +#define dout_prefix _prefix(_dout, epoch, elector) +static ostream& _prefix(std::ostream *_dout, epoch_t epoch, ElectionOwner* elector) { + return *_dout << "paxos." << elector->get_my_rank() + << ").electionLogic(" << epoch << ") "; +} +void ElectionLogic::init() +{ + epoch = elector->read_persisted_epoch(); + if (!epoch) { + ldout(cct, 1) << "init, first boot, initializing epoch at 1 " << dendl; + epoch = 1; + } else if (epoch % 2) { + ldout(cct, 1) << "init, last seen epoch " << epoch + << ", mid-election, bumping" << dendl; + ++epoch; + elector->persist_epoch(epoch); + } else { + ldout(cct, 1) << "init, last seen epoch " << epoch << dendl; + } +} + +void ElectionLogic::bump_epoch(epoch_t e) +{ + ldout(cct, 10) << __func__ << epoch << " to " << e << dendl; + ceph_assert(epoch <= e); + epoch = e; + elector->validate_store(); + // clear up some state + electing_me = false; + acked_me.clear(); + elector->notify_bump_epoch(); +} + +void ElectionLogic::declare_standalone_victory() +{ + assert(elector->paxos_size() == 1 && elector->get_my_rank() == 0); + init(); + bump_epoch(epoch+1); +} + +void ElectionLogic::start() +{ + if (!participating) { + ldout(cct, 0) << "not starting new election -- not participating" << dendl; + return; + } + ldout(cct, 5) << "start -- can i be leader?" << dendl; + + acked_me.clear(); + init(); + + // start by trying to elect me + if (epoch % 2 == 0) { + bump_epoch(epoch+1); // odd == election cycle + } else { + elector->validate_store(); + } + electing_me = true; + acked_me.insert(elector->get_my_rank()); + leader_acked = -1; + + elector->propose_to_peers(epoch); + elector->_start(); +} + +void ElectionLogic::defer(int who) +{ + ldout(cct, 5) << "defer to " << who << dendl; + + if (electing_me) { + // drop out + acked_me.clear(); + electing_me = false; + } + + // ack them + leader_acked = who; + elector->_defer_to(who); +} + +void ElectionLogic::end_election_period() +{ + ldout(cct, 5) << "election period ended" << dendl; + + // did i win? + if (electing_me && + acked_me.size() > (elector->paxos_size() / 2)) { + // i win + declare_victory(); + } else { + // whoever i deferred to didn't declare victory quickly enough. + if (elector->ever_participated()) + start(); + else + elector->reset_election(); + } +} + + +void ElectionLogic::declare_victory() +{ + leader_acked = -1; + electing_me = false; + + set new_quorum; + new_quorum.swap(acked_me); + + ceph_assert(epoch % 2 == 1); // election + bump_epoch(epoch+1); // is over! + + elector->message_victory(new_quorum); +} + +void ElectionLogic::receive_propose(epoch_t mepoch, int from) +{ + if (mepoch > epoch) { + bump_epoch(mepoch); + } else if (mepoch < epoch) { + // got an "old" propose, + if (epoch % 2 == 0 && // in a non-election cycle + !elector->is_current_member(from)) { // from someone outside the quorum + // a mon just started up, call a new election so they can rejoin! + ldout(cct, 5) << " got propose from old epoch, " + << from << " must have just started" << dendl; + // we may be active; make sure we reset things in the monitor appropriately. + elector->trigger_new_election(); + } else { + ldout(cct, 5) << " ignoring old propose" << dendl; + return; + } + } + + if (elector->get_my_rank() < from) { + // i would win over them. + if (leader_acked >= 0) { // we already acked someone + ceph_assert(leader_acked < from); // and they still win, of course + ldout(cct, 5) << "no, we already acked " << leader_acked << dendl; + } else { + // wait, i should win! + if (!electing_me) { + elector->trigger_new_election(); + } + } + } else { + // they would win over me + if (leader_acked < 0 || // haven't acked anyone yet, or + leader_acked > from || // they would win over who you did ack, or + leader_acked == from) { // this is the guy we're already deferring to + defer(from); + } else { + // ignore them! + ldout(cct, 5) << "no, we already acked " << leader_acked << dendl; + } + } +} + +void ElectionLogic::receive_ack(int from, epoch_t from_epoch) +{ + ceph_assert(from_epoch % 2 == 1); // sender in an election epoch + if (from_epoch > epoch) { + ldout(cct, 5) << "woah, that's a newer epoch, i must have rebooted. bumping and re-starting!" << dendl; + bump_epoch(from_epoch); + start(); + return; + } + // is that _everyone_? + if (electing_me) { + acked_me.insert(from); + if (acked_me.size() == elector->paxos_size()) { + // if yes, shortcut to election finish + declare_victory(); + } + } else { + // ignore, i'm deferring already. + ceph_assert(leader_acked >= 0); + } +} + +bool ElectionLogic::receive_victory_claim(int from, epoch_t from_epoch) +{ + ceph_assert(from < elector->get_my_rank()); + ceph_assert(from_epoch % 2 == 0); + + leader_acked = -1; + + // i should have seen this election if i'm getting the victory. + if (from_epoch != epoch + 1) { + ldout(cct, 5) << "woah, that's a funny epoch, i must have rebooted. bumping and re-starting!" << dendl; + bump_epoch(from_epoch); + start(); + return false; + } + + bump_epoch(from_epoch); + + // they win + return true; +} diff --git a/src/mon/ElectionLogic.h b/src/mon/ElectionLogic.h new file mode 100644 index 0000000000000..974ee3fca39a7 --- /dev/null +++ b/src/mon/ElectionLogic.h @@ -0,0 +1,69 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2004-2006 Sage Weil + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + */ + + +#ifndef CEPH_ELECTIONLOGIC_H +#define CEPH_ELECTIONLOGIC_H + +#include +#include "include/types.h" + +class ElectionOwner { +public: + virtual void persist_epoch(epoch_t e) = 0; + virtual epoch_t read_persisted_epoch() = 0; + virtual void validate_store() = 0; + virtual void notify_bump_epoch() = 0; + virtual void trigger_new_election() = 0; + virtual int get_my_rank() = 0; + virtual void propose_to_peers(epoch_t e) = 0; + virtual void reset_election() = 0; + virtual bool ever_participated() = 0; + virtual unsigned paxos_size() = 0; + virtual void _start() = 0; + virtual void _defer_to(int who) = 0; + virtual void message_victory(const set& quorum) = 0; + virtual bool is_current_member(int rank) = 0; + virtual ~ElectionOwner() {} +}; + +class ElectionLogic { +public: + ElectionOwner *elector; + CephContext *cct; + epoch_t epoch = 0; + bool participating; + bool electing_me; + set acked_me; + int leader_acked; + + ElectionLogic(ElectionOwner *e, CephContext *c) : elector(e), cct(c), + participating(true), + electing_me(false), leader_acked(-1) {} + void declare_standalone_victory(); + void start(); + void defer(int who); + void end_election_period(); + void receive_propose(epoch_t mepoch, int from); + void receive_ack(int from, epoch_t from_epoch); + bool receive_victory_claim(int from, epoch_t from_epoch); + + +private: + void init(); + void bump_epoch(epoch_t e); + void declare_victory(); +}; + +#endif diff --git a/src/mon/Elector.cc b/src/mon/Elector.cc index fa5a5654223dd..beb2884b55a4f 100644 --- a/src/mon/Elector.cc +++ b/src/mon/Elector.cc @@ -31,6 +31,10 @@ static ostream& _prefix(std::ostream *_dout, Elector* elector) { << ").elector(" << elector->get_epoch() << ") "; } +Elector::Elector(Monitor *m) : logic(this, m->cct), + mon(m), elector(this) {} + + void Elector::persist_epoch(epoch_t e) { auto t(std::make_shared()); @@ -81,78 +85,17 @@ unsigned Elector::paxos_size() return (unsigned)mon->monmap->size(); } - -void ElectionLogic::init() -{ - epoch = elector->read_persisted_epoch(); - if (!epoch) { - dout(1) << "init, first boot, initializing epoch at 1 " << dendl; - epoch = 1; - } else if (epoch % 2) { - dout(1) << "init, last seen epoch " << epoch - << ", mid-election, bumping" << dendl; - ++epoch; - elector->persist_epoch(epoch); - } else { - dout(1) << "init, last seen epoch " << epoch << dendl; - } -} - void Elector::shutdown() { cancel_timer(); } -void ElectionLogic::bump_epoch(epoch_t e) -{ - dout(10) << __func__ << epoch << " to " << e << dendl; - ceph_assert(epoch <= e); - epoch = e; - elector->validate_store(); - // clear up some state - electing_me = false; - acked_me.clear(); - elector->notify_bump_epoch(); -} - void Elector::notify_bump_epoch() { peer_info.clear(); mon->join_election(); } -void ElectionLogic::declare_standalone_victory() -{ - assert(elector->paxos_size() == 1 && elector->get_my_rank() == 0); - init(); - bump_epoch(epoch+1); -} - -void ElectionLogic::start() -{ - if (!participating) { - dout(0) << "not starting new election -- not participating" << dendl; - return; - } - dout(5) << "start -- can i be leader?" << dendl; - - acked_me.clear(); - init(); - - // start by trying to elect me - if (epoch % 2 == 0) { - bump_epoch(epoch+1); // odd == election cycle - } else { - elector->validate_store(); - } - electing_me = true; - acked_me.insert(elector->get_my_rank()); - leader_acked = -1; - - elector->propose_to_peers(epoch); - elector->_start(); -} - void Elector::propose_to_peers(epoch_t e) { // bcast to everyone else @@ -176,21 +119,6 @@ void Elector::_start() reset_timer(); } -void ElectionLogic::defer(int who) -{ - dout(5) << "defer to " << who << dendl; - - if (electing_me) { - // drop out - acked_me.clear(); - electing_me = false; - } - - // ack them - leader_acked = who; - elector->_defer_to(who); -} - void Elector::_defer_to(int who) { MMonElection *m = new MMonElection(MMonElection::OP_ACK, logic.epoch, mon->monmap); @@ -238,39 +166,6 @@ void Elector::cancel_timer() } } -void ElectionLogic::end_election_period() -{ - dout(5) << "election period ended" << dendl; - - // did i win? - if (electing_me && - acked_me.size() > (elector->paxos_size() / 2)) { - // i win - declare_victory(); - } else { - // whoever i deferred to didn't declare victory quickly enough. - if (elector->ever_participated()) - start(); - else - elector->reset_election(); - } -} - - -void ElectionLogic::declare_victory() -{ - leader_acked = -1; - electing_me = false; - - set new_quorum; - new_quorum.swap(acked_me); - - ceph_assert(epoch % 2 == 1); // election - bump_epoch(epoch+1); // is over! - - elector->message_victory(new_quorum); -} - void Elector::message_victory(const set& quorum) { @@ -357,71 +252,6 @@ void Elector::handle_propose(MonOpRequestRef op) logic.receive_propose(m->epoch, from); } -void ElectionLogic::receive_propose(epoch_t mepoch, int from) -{ - if (mepoch > epoch) { - bump_epoch(mepoch); - } else if (mepoch < epoch) { - // got an "old" propose, - if (epoch % 2 == 0 && // in a non-election cycle - !elector->is_current_member(from)) { // from someone outside the quorum - // a mon just started up, call a new election so they can rejoin! - dout(5) << " got propose from old epoch, " - << from << " must have just started" << dendl; - // we may be active; make sure we reset things in the monitor appropriately. - elector->trigger_new_election(); - } else { - dout(5) << " ignoring old propose" << dendl; - return; - } - } - - if (elector->get_my_rank() < from) { - // i would win over them. - if (leader_acked >= 0) { // we already acked someone - ceph_assert(leader_acked < from); // and they still win, of course - dout(5) << "no, we already acked " << leader_acked << dendl; - } else { - // wait, i should win! - if (!electing_me) { - elector->trigger_new_election(); - } - } - } else { - // they would win over me - if (leader_acked < 0 || // haven't acked anyone yet, or - leader_acked > from || // they would win over who you did ack, or - leader_acked == from) { // this is the guy we're already deferring to - defer(from); - } else { - // ignore them! - dout(5) << "no, we already acked " << leader_acked << dendl; - } - } -} - -void ElectionLogic::receive_ack(int from, epoch_t from_epoch) -{ - ceph_assert(from_epoch % 2 == 1); // sender in an election epoch - if (from_epoch > epoch) { - dout(5) << "woah, that's a newer epoch, i must have rebooted. bumping and re-starting!" << dendl; - bump_epoch(from_epoch); - start(); - return; - } - // is that _everyone_? - if (electing_me) { - acked_me.insert(from); - if (acked_me.size() == elector->paxos_size()) { - // if yes, shortcut to election finish - declare_victory(); - } - } else { - // ignore, i'm deferring already. - ceph_assert(leader_acked >= 0); - } -} - void Elector::handle_ack(MonOpRequestRef op) { op->mark_event("elector:handle_ack"); @@ -471,27 +301,6 @@ void Elector::handle_ack(MonOpRequestRef op) logic.receive_ack(from, m->epoch); } -bool ElectionLogic::receive_victory_claim(int from, epoch_t from_epoch) -{ - ceph_assert(from < elector->get_my_rank()); - ceph_assert(from_epoch % 2 == 0); - - leader_acked = -1; - - // i should have seen this election if i'm getting the victory. - if (from_epoch != epoch + 1) { - dout(5) << "woah, that's a funny epoch, i must have rebooted. bumping and re-starting!" << dendl; - bump_epoch(from_epoch); - start(); - return false; - } - - bump_epoch(from_epoch); - - // they win - return true; -} - void Elector::handle_victory(MonOpRequestRef op) { op->mark_event("elector:handle_victory"); diff --git a/src/mon/Elector.h b/src/mon/Elector.h index 8db45ad8b439e..359ce94e9b6ba 100644 --- a/src/mon/Elector.h +++ b/src/mon/Elector.h @@ -22,56 +22,16 @@ #include "include/Context.h" #include "mon/MonOpRequest.h" #include "mon/mon_types.h" +#include "mon/ElectionLogic.h" class Monitor; -class Elector; -class ElectionOwner { -public: - virtual void persist_epoch(epoch_t e) = 0; - virtual epoch_t read_persisted_epoch() = 0; - virtual void validate_store() = 0; - virtual void notify_bump_epoch() = 0; - virtual void trigger_new_election() = 0; - virtual int get_my_rank() = 0; - virtual void propose_to_peers(epoch_t e) = 0; - virtual void reset_election() = 0; - virtual bool ever_participated() = 0; - virtual unsigned paxos_size() = 0; - virtual ~ElectionOwner() = 0; -}; - -class ElectionLogic { -public: - Elector *elector; - epoch_t epoch; - bool participating; - bool electing_me; - set acked_me; - int leader_acked; - - ElectionLogic(Elector *e) : elector(e), epoch(0), participating(true), - electing_me(false), leader_acked(-1) {} - void declare_standalone_victory(); - void start(); - void defer(int who); - void end_election_period(); - void receive_propose(epoch_t mepoch, int from); - void receive_ack(int from, epoch_t from_epoch); - bool receive_victory_claim(int from, epoch_t from_epoch); - - -private: - void init(); - void bump_epoch(epoch_t e); - void declare_victory(); -}; /** * This class is responsible for maintaining the local state when electing * a new Leader. We may win or we may lose. If we win, it means we became the * Leader; if we lose, it means we are a Peon. */ -class Elector { +class Elector : public ElectionOwner { /** * @defgroup Elector_h_class Elector * @{ @@ -385,8 +345,8 @@ private: * * @param m A Monitor instance */ - explicit Elector(Monitor *m) : logic(this), - mon(m), elector(this) {} + explicit Elector(Monitor *m); + virtual ~Elector() {} /** * Initiate the Elector class. -- 2.39.5