From: Sage Weil Date: Fri, 26 Jun 2009 21:26:53 +0000 (-0700) Subject: osd: switch to MonClient, fix cmds and ceph X-Git-Tag: v0.10~114 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=6e3412b17edeb146d6fc8ace41948a5959ddb18a;p=ceph.git osd: switch to MonClient, fix cmds and ceph --- diff --git a/src/ceph.cc b/src/ceph.cc index 975372efb163..c9dfc19f23bb 100644 --- a/src/ceph.cc +++ b/src/ceph.cc @@ -630,6 +630,8 @@ int main(int argc, const char **argv, const char *envp[]) rank.start(); rank.set_policy(entity_name_t::TYPE_MON, SimpleMessenger::Policy::lossy_fail_after(1.0)); + mc.set_messenger(messenger); + dispatcher.link_dispatcher(&mc); if (mc.get_monmap() < 0) return -1; diff --git a/src/cosd.cc b/src/cosd.cc index 6c49a468dc9f..68f5fe8f7387 100644 --- a/src/cosd.cc +++ b/src/cosd.cc @@ -155,7 +155,7 @@ int main(int argc, const char **argv) rank.start(); // start osd - OSD *osd = new OSD(whoami, m, hbm, &mc.monmap, g_conf.osd_data, g_conf.osd_journal); + OSD *osd = new OSD(whoami, m, hbm, &mc, g_conf.osd_data, g_conf.osd_journal); if (osd->init() < 0) { cout << "error initializing osd" << std::endl; return 1; diff --git a/src/mds/MDS.cc b/src/mds/MDS.cc index 63308bc7580d..576d44fdf9d7 100644 --- a/src/mds/MDS.cc +++ b/src/mds/MDS.cc @@ -372,6 +372,11 @@ void MDS::send_message_client(Message *m, entity_inst_t clientinst) int MDS::init() { + // get monmap + monc->set_messenger(messenger); + link_dispatcher(monc); + monc->get_monmap(); + mds_lock.Lock(); // starting beacon. this will induce an MDSMap from the monitor diff --git a/src/messages/MClass.h b/src/messages/MClass.h index 173564fb05c9..14c094cef048 100644 --- a/src/messages/MClass.h +++ b/src/messages/MClass.h @@ -37,12 +37,12 @@ public: MClass() : PaxosServiceMessage(MSG_CLASS, 0) {} #if 0 - MClass(ceph_fsid_t& f, deque& e) : + MClass(const ceph_fsid_t& f, deque& e) : PaxosServiceMessage(MSG_CLASS, 0), fsid(f), entries(e), last(0), action(0) {} #endif - MClass(ceph_fsid_t& f, version_t l) : PaxosServiceMessage(MSG_CLASS, 0), fsid(f), last(l) {} - MClass(ceph_fsid_t& f, version_t l, version_t paxos_version) : + MClass(const ceph_fsid_t& f, version_t l) : PaxosServiceMessage(MSG_CLASS, 0), fsid(f), last(l) {} + MClass(const ceph_fsid_t& f, version_t l, version_t paxos_version) : PaxosServiceMessage(MSG_CLASS, paxos_version), fsid(f), last(l) {} const char *get_type_name() { return "class"; } diff --git a/src/messages/MOSDFailure.h b/src/messages/MOSDFailure.h index 434613690f72..89c67c7a032c 100644 --- a/src/messages/MOSDFailure.h +++ b/src/messages/MOSDFailure.h @@ -26,7 +26,7 @@ class MOSDFailure : public PaxosServiceMessage { epoch_t epoch; MOSDFailure() : PaxosServiceMessage(MSG_OSD_FAILURE, 0) {} - MOSDFailure(ceph_fsid_t &fs, entity_inst_t f, epoch_t e) : + MOSDFailure(const ceph_fsid_t &fs, entity_inst_t f, epoch_t e) : PaxosServiceMessage(MSG_OSD_FAILURE, e), fsid(fs), failed(f), epoch(e) {} diff --git a/src/messages/MOSDMap.h b/src/messages/MOSDMap.h index c4c9874a6fa1..677f8032e9ba 100644 --- a/src/messages/MOSDMap.h +++ b/src/messages/MOSDMap.h @@ -47,8 +47,8 @@ class MOSDMap : public Message { MOSDMap() : Message(CEPH_MSG_OSD_MAP) { } - MOSDMap(ceph_fsid_t &f, OSDMap *oc=0) : Message(CEPH_MSG_OSD_MAP), - fsid(f) { + MOSDMap(const ceph_fsid_t &f, OSDMap *oc=0) : Message(CEPH_MSG_OSD_MAP), + fsid(f) { if (oc) oc->encode(maps[oc->get_epoch()]); } diff --git a/src/mon/MonClient.cc b/src/mon/MonClient.cc index ab453dea7e37..a6b1c6f4015c 100644 --- a/src/mon/MonClient.cc +++ b/src/mon/MonClient.cc @@ -287,4 +287,10 @@ void MonClient::send_mon_message(Message *m, bool newmon) messenger->send_message(m, monmap.mon_inst[mon]); } - +void MonClient::pick_new_mon() +{ + Mutex::Locker l(monc_lock); + int oldmon = monmap.pick_mon(); + messenger->mark_down(monmap.get_inst(oldmon).addr); + monmap.pick_mon(true); +} diff --git a/src/mon/MonClient.h b/src/mon/MonClient.h index 2f80ae9df7a9..08370a29d30f 100644 --- a/src/mon/MonClient.h +++ b/src/mon/MonClient.h @@ -83,6 +83,7 @@ private: void note_mon_leader(int m) { monmap.last_mon = m; } + void pick_new_mon(); const ceph_fsid_t& get_fsid() { return monmap.fsid; diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc index 538b77dc88a4..f979616d0db2 100644 --- a/src/osd/OSD.cc +++ b/src/osd/OSD.cc @@ -35,6 +35,8 @@ #include "msg/Messenger.h" #include "msg/Message.h" +#include "mon/MonClient.h" + #include "messages/MLog.h" #include "messages/MGenericMessage.h" @@ -219,14 +221,14 @@ int OSD::peek_super(const char *dev, nstring& magic, ceph_fsid_t& fsid, int& who // cons/des -OSD::OSD(int id, Messenger *m, Messenger *hbm, MonMap *mm, const char *dev, const char *jdev) : +OSD::OSD(int id, Messenger *m, Messenger *hbm, MonClient *mc, const char *dev, const char *jdev) : osd_lock("OSD::osd_lock"), timer(osd_lock), messenger(m), + monc(mc), logger(NULL), store(NULL), - monmap(mm), - logclient(messenger, monmap), + logclient(messenger, &mc->monmap), whoami(id), dev_path(dev), journal_path(jdev), state(STATE_BOOTING), boot_epoch(0), up_epoch(0), @@ -261,6 +263,8 @@ OSD::OSD(int id, Messenger *m, Messenger *hbm, MonMap *mm, const char *dev, cons snap_trim_wq(this, &disk_tp), scrub_wq(this, &disk_tp) { + monc->set_messenger(messenger); + osdmap = 0; memset(&my_stat, 0, sizeof(my_stat)); @@ -555,8 +559,8 @@ int OSD::read_superblock() dout(10) << "read_superblock " << superblock << dendl; - if (ceph_fsid_compare(&superblock.fsid, &monmap->fsid)) { - derr(0) << "read_superblock fsid " << superblock.fsid << " != monmap " << monmap->fsid << dendl; + if (ceph_fsid_compare(&superblock.fsid, &monc->get_fsid())) { + derr(0) << "read_superblock fsid " << superblock.fsid << " != monmap " << monc->get_fsid() << dendl; return -1; } @@ -1166,9 +1170,7 @@ void OSD::heartbeat() if (now - last_mon_heartbeat > g_conf.osd_mon_heartbeat_interval) { last_mon_heartbeat = now; dout(10) << "i have no heartbeat peers; checking mon for new map" << dendl; - int mon = monmap->pick_mon(); - messenger->send_message(new MOSDGetMap(monmap->fsid, osdmap->get_epoch()+1), - monmap->get_inst(mon)); + monc->send_mon_message(new MOSDGetMap(monc->get_fsid(), osdmap->get_epoch()+1)); } } @@ -1257,10 +1259,8 @@ void OSD::do_mon_report() pg_stat_queue_lock.Unlock(); if (retry) { - int oldmon = monmap->pick_mon(); - messenger->mark_down(monmap->get_inst(oldmon).addr); - int mon = monmap->pick_mon(true); - dout(10) << "marked down old mon" << oldmon << ", chose new mon" << mon << dendl; + monc->pick_new_mon(); + dout(10) << "picked a new mon" << dendl; } // do any pending reports @@ -1276,10 +1276,8 @@ void OSD::do_mon_report() void OSD::send_boot() { - int mon = monmap->pick_mon(true); - dout(10) << "send_boot to mon" << mon << dendl; - messenger->send_message(new MOSDBoot(superblock), - monmap->get_inst(mon)); + dout(10) << "send_boot" << dendl; + monc->send_mon_message(new MOSDBoot(superblock)); } void OSD::queue_want_up_thru(epoch_t want) @@ -1309,20 +1307,16 @@ void OSD::send_alive() dout(10) << "send_alive up_thru currently " << up_thru << " want " << up_thru_wanted << dendl; if (up_thru_wanted > up_thru) { up_thru_pending = up_thru_wanted; - int mon = monmap->pick_mon(); - dout(10) << "send_alive to mon" << mon << " (want " << up_thru_wanted << ")" << dendl; - messenger->send_message(new MOSDAlive(osdmap->get_epoch()), - monmap->get_inst(mon)); + dout(10) << "send_alive want " << up_thru_wanted << dendl; + monc->send_mon_message(new MOSDAlive(osdmap->get_epoch())); } } void OSD::send_failures() { - int mon = monmap->pick_mon(); while (!failure_queue.empty()) { int osd = *failure_queue.begin(); - messenger->send_message(new MOSDFailure(monmap->fsid, osdmap->get_inst(osd), osdmap->get_epoch()), - monmap->get_inst(mon)); + monc->send_mon_message(new MOSDFailure(monc->get_fsid(), osdmap->get_inst(osd), osdmap->get_epoch())); failure_queue.erase(osd); } } @@ -1375,8 +1369,7 @@ void OSD::send_pg_stats() m->osd_stat.hb_out.push_back(p->first); dout(20) << " osd_stat " << m->osd_stat << dendl; - int mon = monmap->pick_mon(); - messenger->send_message(m, monmap->get_inst(mon)); + monc->send_mon_message(m); } pg_stat_queue_lock.Unlock(); @@ -1676,8 +1669,8 @@ void OSD::handle_scrub(MOSDScrub *m) { dout(10) << "handle_scrub " << *m << dendl; - if (ceph_fsid_compare(&m->fsid, &monmap->fsid)) { - dout(0) << "handle_scrub fsid " << m->fsid << " != " << monmap->fsid << dendl; + if (ceph_fsid_compare(&m->fsid, &monc->get_fsid())) { + dout(0) << "handle_scrub fsid " << m->fsid << " != " << monc->get_fsid() << dendl; delete m; return; } @@ -1725,9 +1718,7 @@ void OSD::wait_for_new_map(Message *m) { // ask if (waiting_for_osdmap.empty()) { - int mon = monmap->pick_mon(); - messenger->send_message(new MOSDGetMap(monmap->fsid, osdmap->get_epoch()+1), - monmap->get_inst(mon)); + monc->send_mon_message(new MOSDGetMap(monc->get_fsid(), osdmap->get_epoch()+1)); } waiting_for_osdmap.push_back(m); @@ -1769,8 +1760,8 @@ void OSD::note_up_osd(int osd) void OSD::handle_osd_map(MOSDMap *m) { assert(osd_lock.is_locked()); - if (ceph_fsid_compare(&m->fsid, &monmap->fsid)) { - dout(0) << "handle_osd_map fsid " << m->fsid << " != " << monmap->fsid << dendl; + if (ceph_fsid_compare(&m->fsid, &monc->get_fsid())) { + dout(0) << "handle_osd_map fsid " << m->fsid << " != " << monc->get_fsid() << dendl; delete m; return; } @@ -1947,8 +1938,7 @@ void OSD::handle_osd_map(MOSDMap *m) } else { dout(10) << "handle_osd_map missing epoch " << cur+1 << dendl; - int mon = monmap->pick_mon(); - messenger->send_message(new MOSDGetMap(monmap->fsid, cur+1), monmap->get_inst(mon)); + monc->send_mon_message(new MOSDGetMap(monc->get_fsid(), cur+1)); break; } @@ -2297,7 +2287,7 @@ void OSD::send_incremental_map(epoch_t since, const entity_inst_t& inst, bool fu dout(10) << "send_incremental_map " << since << " -> " << osdmap->get_epoch() << " to " << inst << dendl; - MOSDMap *m = new MOSDMap(monmap->fsid); + MOSDMap *m = new MOSDMap(monc->get_fsid()); for (epoch_t e = osdmap->get_epoch(); e > since; @@ -3953,14 +3943,13 @@ void OSD::handle_class(MClass *m) void OSD::send_class_request(const char *cname, ClassVersion& version) { dout(10) << "send_class_request class=" << cname << " version=" << version << dendl; - MClass *m = new MClass(monmap->get_fsid(), 0); + MClass *m = new MClass(monc->get_fsid(), 0); ClassInfo info; info.name = cname; info.version = version; m->info.push_back(info); m->action = CLASS_GET; - int mon = monmap->pick_mon(); - messenger->send_message(m, monmap->get_inst(mon)); + monc->send_mon_message(m); } diff --git a/src/osd/OSD.h b/src/osd/OSD.h index 5ddca9c75fb7..f032cf3217c2 100644 --- a/src/osd/OSD.h +++ b/src/osd/OSD.h @@ -23,8 +23,6 @@ #include "common/WorkQueue.h" #include "common/LogClient.h" -#include "mon/MonMap.h" - #include "os/ObjectStore.h" #include "PG.h" @@ -82,6 +80,7 @@ enum { class Messenger; class Message; +class MonClient; class Logger; class ObjectStore; class OSDMap; @@ -96,9 +95,9 @@ protected: SafeTimer timer; // safe timer (osd_lock) Messenger *messenger; + MonClient *monc; Logger *logger; ObjectStore *store; - MonMap *monmap; LogClient logclient; @@ -770,7 +769,7 @@ protected: private: virtual bool dispatch_impl(Message *m); public: - OSD(int id, Messenger *m, Messenger *hbm, MonMap *mm, const char *dev = 0, const char *jdev = 0); + OSD(int id, Messenger *m, Messenger *hbm, MonClient *mc, const char *dev = 0, const char *jdev = 0); ~OSD(); // static bits