osd: switch to MonClient, fix cmds and ceph

diff --git a/src/ceph.cc b/src/ceph.cc
index 975372efb163595dbe7c3c58ae09caf52b186055..c9dfc19f23bbcbd0fbdead7a991656c7777b3416 100644 (file)
--- a/src/ceph.cc
+++ b/src/ceph.cc
@@ -630,6 +630,8 @@ int main(int argc, const char **argv, const char *envp[])
   rank.start();
   rank.set_policy(entity_name_t::TYPE_MON, SimpleMessenger::Policy::lossy_fail_after(1.0));
 
+  mc.set_messenger(messenger);
+  dispatcher.link_dispatcher(&mc);
   if (mc.get_monmap() < 0)
     return -1;
 

diff --git a/src/cosd.cc b/src/cosd.cc
index 6c49a468dc9fe014a723d136e4deb127019f3eec..68f5fe8f7387dfc8790f4e7093e5f9e49529e79a 100644 (file)
--- a/src/cosd.cc
+++ b/src/cosd.cc
@@ -155,7 +155,7 @@ int main(int argc, const char **argv)
   rank.start();
 
   // start osd
-  OSD *osd = new OSD(whoami, m, hbm, &mc.monmap, g_conf.osd_data, g_conf.osd_journal);
+  OSD *osd = new OSD(whoami, m, hbm, &mc, g_conf.osd_data, g_conf.osd_journal);
   if (osd->init() < 0) {
     cout << "error initializing osd" << std::endl;
     return 1;

diff --git a/src/mds/MDS.cc b/src/mds/MDS.cc
index 63308bc7580dd5ea33c0f29fbd9cc26ec49be287..576d44fdf9d704e28786d2354f27519c9537b519 100644 (file)
--- a/src/mds/MDS.cc
+++ b/src/mds/MDS.cc
@@ -372,6 +372,11 @@ void MDS::send_message_client(Message *m, entity_inst_t clientinst)
 
 int MDS::init()
 {
+  // get monmap
+  monc->set_messenger(messenger);
+  link_dispatcher(monc);
+  monc->get_monmap();
+
   mds_lock.Lock();
 
   // starting beacon.  this will induce an MDSMap from the monitor

diff --git a/src/messages/MClass.h b/src/messages/MClass.h
index 173564fb05c9b7f671710c396eedba8ffbe56a2a..14c094cef048435de2dd357d70eca7e66f5abed8 100644 (file)
--- a/src/messages/MClass.h
+++ b/src/messages/MClass.h
@@ -37,12 +37,12 @@ public:
 
   MClass() : PaxosServiceMessage(MSG_CLASS, 0) {}
 #if 0
-  MClass(ceph_fsid_t& f, deque<ClassLibraryIncremental>& e) :
+  MClass(const ceph_fsid_t& f, deque<ClassLibraryIncremental>& e) :
     PaxosServiceMessage(MSG_CLASS, 0),
     fsid(f), entries(e), last(0), action(0) {}
 #endif
-  MClass(ceph_fsid_t& f, version_t l) : PaxosServiceMessage(MSG_CLASS, 0), fsid(f), last(l) {}
-  MClass(ceph_fsid_t& f, version_t l, version_t paxos_version) :
+  MClass(const ceph_fsid_t& f, version_t l) : PaxosServiceMessage(MSG_CLASS, 0), fsid(f), last(l) {}
+  MClass(const ceph_fsid_t& f, version_t l, version_t paxos_version) :
     PaxosServiceMessage(MSG_CLASS, paxos_version), fsid(f), last(l) {}
 
   const char *get_type_name() { return "class"; }

diff --git a/src/messages/MOSDFailure.h b/src/messages/MOSDFailure.h
index 434613690f727cf52df669afbbe32cc9438204c0..89c67c7a032c13d350ddfbc78cd3f840eff2a381 100644 (file)
--- a/src/messages/MOSDFailure.h
+++ b/src/messages/MOSDFailure.h
@@ -26,7 +26,7 @@ class MOSDFailure : public PaxosServiceMessage {
   epoch_t       epoch;
 
   MOSDFailure() : PaxosServiceMessage(MSG_OSD_FAILURE, 0) {}
-  MOSDFailure(ceph_fsid_t &fs, entity_inst_t f, epoch_t e) : 
+  MOSDFailure(const ceph_fsid_t &fs, entity_inst_t f, epoch_t e) : 
     PaxosServiceMessage(MSG_OSD_FAILURE, e),
     fsid(fs), failed(f), epoch(e) {}
  

diff --git a/src/messages/MOSDMap.h b/src/messages/MOSDMap.h
index c4c9874a6fa18ff744b51070eddd88bdb3ccd71f..677f8032e9ba2d000a788237126b079aa1167631 100644 (file)
--- a/src/messages/MOSDMap.h
+++ b/src/messages/MOSDMap.h
@@ -47,8 +47,8 @@ class MOSDMap : public Message {
 
 
   MOSDMap() : Message(CEPH_MSG_OSD_MAP) { }
-  MOSDMap(ceph_fsid_t &f, OSDMap *oc=0) : Message(CEPH_MSG_OSD_MAP),
-                                     fsid(f) {
+  MOSDMap(const ceph_fsid_t &f, OSDMap *oc=0) : Message(CEPH_MSG_OSD_MAP),
+                                               fsid(f) {
     if (oc)
       oc->encode(maps[oc->get_epoch()]);
   }

diff --git a/src/mon/MonClient.cc b/src/mon/MonClient.cc
index ab453dea7e37267fb4f3a4ef29811907ad42e400..a6b1c6f4015cc536bfdb75a1f70b84469fd4a479 100644 (file)
--- a/src/mon/MonClient.cc
+++ b/src/mon/MonClient.cc
@@ -287,4 +287,10 @@ void MonClient::send_mon_message(Message *m, bool newmon)
   messenger->send_message(m, monmap.mon_inst[mon]);
 }
 
-
+void MonClient::pick_new_mon()
+{
+  Mutex::Locker l(monc_lock);
+  int oldmon = monmap.pick_mon();
+  messenger->mark_down(monmap.get_inst(oldmon).addr);
+  monmap.pick_mon(true);
+}

diff --git a/src/mon/MonClient.h b/src/mon/MonClient.h
index 2f80ae9df7a9d98a942765c665795400cb21a89b..08370a29d30f3f81189c512d17005bae12368e11 100644 (file)
--- a/src/mon/MonClient.h
+++ b/src/mon/MonClient.h
@@ -83,6 +83,7 @@ private:
   void note_mon_leader(int m) {
     monmap.last_mon = m;
   }
+  void pick_new_mon();
 
   const ceph_fsid_t& get_fsid() {
     return monmap.fsid;

diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc
index 538b77dc88a407664f93844cf54041740a197b1b..f979616d0db21156c098936b738ca30a18a21c79 100644 (file)
--- a/src/osd/OSD.cc
+++ b/src/osd/OSD.cc
@@ -35,6 +35,8 @@
 #include "msg/Messenger.h"
 #include "msg/Message.h"
 
+#include "mon/MonClient.h"
+
 #include "messages/MLog.h"
 
 #include "messages/MGenericMessage.h"
@@ -219,14 +221,14 @@ int OSD::peek_super(const char *dev, nstring& magic, ceph_fsid_t& fsid, int& who
 
 // cons/des
 
-OSD::OSD(int id, Messenger *m, Messenger *hbm, MonMap *mm, const char *dev, const char *jdev) : 
+OSD::OSD(int id, Messenger *m, Messenger *hbm, MonClient *mc, const char *dev, const char *jdev) : 
   osd_lock("OSD::osd_lock"),
   timer(osd_lock),
   messenger(m),
+  monc(mc),
   logger(NULL),
   store(NULL),
-  monmap(mm),
-  logclient(messenger, monmap),
+  logclient(messenger, &mc->monmap),
   whoami(id),
   dev_path(dev), journal_path(jdev),
   state(STATE_BOOTING), boot_epoch(0), up_epoch(0),
@@ -261,6 +263,8 @@ OSD::OSD(int id, Messenger *m, Messenger *hbm, MonMap *mm, const char *dev, cons
   snap_trim_wq(this, &disk_tp),
   scrub_wq(this, &disk_tp)
 {
+  monc->set_messenger(messenger);
+  
   osdmap = 0;
 
   memset(&my_stat, 0, sizeof(my_stat));
@@ -555,8 +559,8 @@ int OSD::read_superblock()
 
   dout(10) << "read_superblock " << superblock << dendl;
 
-  if (ceph_fsid_compare(&superblock.fsid, &monmap->fsid)) {
-    derr(0) << "read_superblock fsid " << superblock.fsid << " != monmap " << monmap->fsid << dendl;
+  if (ceph_fsid_compare(&superblock.fsid, &monc->get_fsid())) {
+    derr(0) << "read_superblock fsid " << superblock.fsid << " != monmap " << monc->get_fsid() << dendl;
     return -1;
   }
 
@@ -1166,9 +1170,7 @@ void OSD::heartbeat()
     if (now - last_mon_heartbeat > g_conf.osd_mon_heartbeat_interval) {
       last_mon_heartbeat = now;
       dout(10) << "i have no heartbeat peers; checking mon for new map" << dendl;
-      int mon = monmap->pick_mon();
-      messenger->send_message(new MOSDGetMap(monmap->fsid, osdmap->get_epoch()+1),
-                             monmap->get_inst(mon));
+      monc->send_mon_message(new MOSDGetMap(monc->get_fsid(), osdmap->get_epoch()+1));
     }
   }
 
@@ -1257,10 +1259,8 @@ void OSD::do_mon_report()
   pg_stat_queue_lock.Unlock();
 
   if (retry) {
-    int oldmon = monmap->pick_mon();
-    messenger->mark_down(monmap->get_inst(oldmon).addr);
-    int mon = monmap->pick_mon(true);
-    dout(10) << "marked down old mon" << oldmon << ", chose new mon" << mon << dendl;
+    monc->pick_new_mon();
+    dout(10) << "picked a new mon" << dendl;
   }
 
   // do any pending reports
@@ -1276,10 +1276,8 @@ void OSD::do_mon_report()
 
 void OSD::send_boot()
 {
-  int mon = monmap->pick_mon(true);
-  dout(10) << "send_boot to mon" << mon << dendl;
-  messenger->send_message(new MOSDBoot(superblock), 
-                         monmap->get_inst(mon));
+  dout(10) << "send_boot" << dendl;
+  monc->send_mon_message(new MOSDBoot(superblock));
 }
 
 void OSD::queue_want_up_thru(epoch_t want)
@@ -1309,20 +1307,16 @@ void OSD::send_alive()
   dout(10) << "send_alive up_thru currently " << up_thru << " want " << up_thru_wanted << dendl;
   if (up_thru_wanted > up_thru) {
     up_thru_pending = up_thru_wanted;
-    int mon = monmap->pick_mon();
-    dout(10) << "send_alive to mon" << mon << " (want " << up_thru_wanted << ")" << dendl;
-    messenger->send_message(new MOSDAlive(osdmap->get_epoch()),
-                           monmap->get_inst(mon));
+    dout(10) << "send_alive want " << up_thru_wanted << dendl;
+    monc->send_mon_message(new MOSDAlive(osdmap->get_epoch()));
   }
 }
 
 void OSD::send_failures()
 {
-  int mon = monmap->pick_mon();
   while (!failure_queue.empty()) {
     int osd = *failure_queue.begin();
-    messenger->send_message(new MOSDFailure(monmap->fsid, osdmap->get_inst(osd), osdmap->get_epoch()),
-                           monmap->get_inst(mon));
+    monc->send_mon_message(new MOSDFailure(monc->get_fsid(), osdmap->get_inst(osd), osdmap->get_epoch()));
     failure_queue.erase(osd);
   }
 }
@@ -1375,8 +1369,7 @@ void OSD::send_pg_stats()
       m->osd_stat.hb_out.push_back(p->first);
     dout(20) << " osd_stat " << m->osd_stat << dendl;
     
-    int mon = monmap->pick_mon();
-    messenger->send_message(m, monmap->get_inst(mon));  
+    monc->send_mon_message(m);
   }
 
   pg_stat_queue_lock.Unlock();
@@ -1676,8 +1669,8 @@ void OSD::handle_scrub(MOSDScrub *m)
 {
   dout(10) << "handle_scrub " << *m << dendl;
   
-  if (ceph_fsid_compare(&m->fsid, &monmap->fsid)) {
-    dout(0) << "handle_scrub fsid " << m->fsid << " != " << monmap->fsid << dendl;
+  if (ceph_fsid_compare(&m->fsid, &monc->get_fsid())) {
+    dout(0) << "handle_scrub fsid " << m->fsid << " != " << monc->get_fsid() << dendl;
     delete m;
     return;
   }
@@ -1725,9 +1718,7 @@ void OSD::wait_for_new_map(Message *m)
 {
   // ask 
   if (waiting_for_osdmap.empty()) {
-    int mon = monmap->pick_mon();
-    messenger->send_message(new MOSDGetMap(monmap->fsid, osdmap->get_epoch()+1),
-                            monmap->get_inst(mon));
+    monc->send_mon_message(new MOSDGetMap(monc->get_fsid(), osdmap->get_epoch()+1));
   }
   
   waiting_for_osdmap.push_back(m);
@@ -1769,8 +1760,8 @@ void OSD::note_up_osd(int osd)
 void OSD::handle_osd_map(MOSDMap *m)
 {
   assert(osd_lock.is_locked());
-  if (ceph_fsid_compare(&m->fsid, &monmap->fsid)) {
-    dout(0) << "handle_osd_map fsid " << m->fsid << " != " << monmap->fsid << dendl;
+  if (ceph_fsid_compare(&m->fsid, &monc->get_fsid())) {
+    dout(0) << "handle_osd_map fsid " << m->fsid << " != " << monc->get_fsid() << dendl;
     delete m;
     return;
   }
@@ -1947,8 +1938,7 @@ void OSD::handle_osd_map(MOSDMap *m)
     }
     else {
       dout(10) << "handle_osd_map missing epoch " << cur+1 << dendl;
-      int mon = monmap->pick_mon();
-      messenger->send_message(new MOSDGetMap(monmap->fsid, cur+1), monmap->get_inst(mon));
+      monc->send_mon_message(new MOSDGetMap(monc->get_fsid(), cur+1));
       break;
     }
 
@@ -2297,7 +2287,7 @@ void OSD::send_incremental_map(epoch_t since, const entity_inst_t& inst, bool fu
   dout(10) << "send_incremental_map " << since << " -> " << osdmap->get_epoch()
            << " to " << inst << dendl;
   
-  MOSDMap *m = new MOSDMap(monmap->fsid);
+  MOSDMap *m = new MOSDMap(monc->get_fsid());
   
   for (epoch_t e = osdmap->get_epoch();
        e > since;
@@ -3953,14 +3943,13 @@ void OSD::handle_class(MClass *m)
 void OSD::send_class_request(const char *cname, ClassVersion& version)
 {
   dout(10) << "send_class_request class=" << cname << " version=" << version << dendl;
-  MClass *m = new MClass(monmap->get_fsid(), 0);
+  MClass *m = new MClass(monc->get_fsid(), 0);
   ClassInfo info;
   info.name = cname;
   info.version = version;
   m->info.push_back(info);
   m->action = CLASS_GET;
-  int mon = monmap->pick_mon();
-  messenger->send_message(m, monmap->get_inst(mon));
+  monc->send_mon_message(m);
 }
 
 

diff --git a/src/osd/OSD.h b/src/osd/OSD.h
index 5ddca9c75fb7a50d113760f92374ef66a5cc2776..f032cf3217c218c24d758b89ff05bc99d9a78203 100644 (file)
--- a/src/osd/OSD.h
+++ b/src/osd/OSD.h
@@ -23,8 +23,6 @@
 #include "common/WorkQueue.h"
 #include "common/LogClient.h"
 
-#include "mon/MonMap.h"
-
 #include "os/ObjectStore.h"
 #include "PG.h"
 
@@ -82,6 +80,7 @@ enum {
 
 class Messenger;
 class Message;
+class MonClient;
 class Logger;
 class ObjectStore;
 class OSDMap;
@@ -96,9 +95,9 @@ protected:
   SafeTimer timer;    // safe timer (osd_lock)
 
   Messenger   *messenger; 
+  MonClient   *monc;
   Logger      *logger;
   ObjectStore *store;
-  MonMap      *monmap;
 
   LogClient   logclient;
 
@@ -770,7 +769,7 @@ protected:
  private:
   virtual bool dispatch_impl(Message *m);
  public:
-  OSD(int id, Messenger *m, Messenger *hbm, MonMap *mm, const char *dev = 0, const char *jdev = 0);
+  OSD(int id, Messenger *m, Messenger *hbm, MonClient *mc, const char *dev = 0, const char *jdev = 0);
   ~OSD();
 
   // static bits