From: Sage Weil <sage@newdream.net>
Date: Fri, 26 Jun 2009 21:12:56 +0000 (-0700)
Subject: mds, objecter, ceph: use MonClient
X-Git-Tag: v0.10~115
X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=34168a9048d294f92df74a7fc04c3156e3408a2a;p=ceph.git

mds, objecter, ceph: use MonClient
---

diff --git a/src/ceph.cc b/src/ceph.cc
index 8b9415333c6d..975372efb163 100644
--- a/src/ceph.cc
+++ b/src/ceph.cc
@@ -630,6 +630,9 @@ int main(int argc, const char **argv, const char *envp[])
   rank.start();
   rank.set_policy(entity_name_t::TYPE_MON, SimpleMessenger::Policy::lossy_fail_after(1.0));
 
+  if (mc.get_monmap() < 0)
+    return -1;
+
   if (watch) {
     lock.Lock();
     get_status();
diff --git a/src/client/Client.cc b/src/client/Client.cc
index ce1704afcba0..6e72c5948d6e 100644
--- a/src/client/Client.cc
+++ b/src/client/Client.cc
@@ -133,7 +133,7 @@ Client::Client(Messenger *m, MonClient *mc) : timer(client_lock), client_lock("C
   // osd interfaces
   osdmap = new OSDMap;     // initially blank.. see mount()
   mdsmap = new MDSMap;
-  objecter = new Objecter(messenger, &monclient->monmap, osdmap, client_lock);
+  objecter = new Objecter(messenger, monclient, osdmap, client_lock);
   objecter->set_client_incarnation(0);  // client always 0, for now.
   objectcacher = new ObjectCacher(objecter, client_lock, 
 				  0,                            // all ack callback
diff --git a/src/cmds.cc b/src/cmds.cc
index aaa453fafd4c..5f6a56099ada 100644
--- a/src/cmds.cc
+++ b/src/cmds.cc
@@ -86,7 +86,7 @@ int main(int argc, const char **argv)
   rank.start();
   
   // start mds
-  MDS *mds = new MDS(g_conf.id, m, &mc.monmap);
+  MDS *mds = new MDS(g_conf.id, m, &mc);
   mds->init();
   
   rank.wait();
diff --git a/src/dumpjournal.cc b/src/dumpjournal.cc
index 7c39b37ce192..5906565f1f55 100644
--- a/src/dumpjournal.cc
+++ b/src/dumpjournal.cc
@@ -97,7 +97,7 @@ int main(int argc, const char **argv, const char *envp[])
   inodeno_t ino = MDS_INO_LOG_OFFSET + mds;
   unsigned pg_pool = CEPH_METADATA_RULE;
 
-  objecter = new Objecter(messenger, &mc.monmap, &osdmap, lock);
+  objecter = new Objecter(messenger, &mc, &osdmap, lock);
   journaler = new Journaler(ino, pg_pool, CEPH_FS_ONDISK_MAGIC, objecter, 0, 0,  &lock);
 
   objecter->set_client_incarnation(0);
diff --git a/src/librados.cc b/src/librados.cc
index 78059c6a8c14..dd0f37ecd62a 100644
--- a/src/librados.cc
+++ b/src/librados.cc
@@ -287,7 +287,7 @@ bool RadosClient::init()
 
   monclient.link_dispatcher(this);
 
-  objecter = new Objecter(messenger, &monclient.monmap, &osdmap, lock);
+  objecter = new Objecter(messenger, &monclient, &osdmap, lock);
   if (!objecter)
     return false;
 
diff --git a/src/mds/MDS.cc b/src/mds/MDS.cc
index 212694aa1bce..63308bc7580d 100644
--- a/src/mds/MDS.cc
+++ b/src/mds/MDS.cc
@@ -18,6 +18,7 @@
 #include "common/Clock.h"
 
 #include "msg/Messenger.h"
+#include "mon/MonClient.h"
 
 #include "osd/OSDMap.h"
 #include "osdc/Objecter.h"
@@ -71,7 +72,7 @@
 
 
 // cons/des
-MDS::MDS(const char *n, Messenger *m, MonMap *mm) : 
+MDS::MDS(const char *n, Messenger *m, MonClient *mc) : 
   mds_lock("MDS::mds_lock"),
   timer(mds_lock),
   name(n),
@@ -79,16 +80,18 @@ MDS::MDS(const char *n, Messenger *m, MonMap *mm) :
   standby_for_rank(-1),
   standby_replay_for(-1),
   messenger(m),
-  monmap(mm),
-  logclient(messenger, monmap),
+  monc(mc),
+  logclient(messenger, &mc->monmap),
   sessionmap(this) {
 
   last_tid = 0;
 
+  monc->set_messenger(messenger);
+
   mdsmap = new MDSMap;
   osdmap = new OSDMap;
 
-  objecter = new Objecter(messenger, monmap, osdmap, mds_lock);
+  objecter = new Objecter(messenger, monc, osdmap, mds_lock);
   filer = new Filer(objecter);
 
   mdcache = new MDCache(this);
@@ -299,7 +302,7 @@ void MDS::send_message_mds(Message *m, int mds)
 {
   // send mdsmap first?
   if (peer_mdsmap_epoch[mds] < mdsmap->get_epoch()) {
-    messenger->send_message(new MMDSMap(monmap->fsid, mdsmap), 
+    messenger->send_message(new MMDSMap(monc->get_fsid(), mdsmap), 
 			    mdsmap->get_inst(mds));
     peer_mdsmap_epoch[mds] = mdsmap->get_epoch();
   }
@@ -337,7 +340,7 @@ void MDS::forward_message_mds(Message *m, int mds)
 
   // send mdsmap first?
   if (peer_mdsmap_epoch[mds] < mdsmap->get_epoch()) {
-    messenger->send_message(new MMDSMap(monmap->fsid, mdsmap), 
+    messenger->send_message(new MMDSMap(monc->get_fsid(), mdsmap), 
 			    mdsmap->get_inst(mds));
     peer_mdsmap_epoch[mds] = mdsmap->get_epoch();
   }
@@ -477,15 +480,16 @@ void MDS::beacon_send()
 	   << dendl;
 
   // pick new random mon if we have any outstanding beacons...
-  int mon = monmap->pick_mon(beacon_seq_stamp.size());
+  bool newmon = beacon_seq_stamp.size();
 
   beacon_seq_stamp[beacon_last_seq] = g_clock.now();
   
-  MMDSBeacon *beacon = new MMDSBeacon(monmap->fsid, name, mdsmap->get_epoch(), 
+  MMDSBeacon *beacon = new MMDSBeacon(monc->get_fsid(), name, mdsmap->get_epoch(), 
 				      want_state, beacon_last_seq);
   beacon->set_standby_for_rank(standby_for_rank);
   beacon->set_standby_for_name(standby_for_name);
-  messenger->send_message(beacon, monmap->get_inst(mon));
+
+  monc->send_mon_message(beacon, newmon);
 
   // schedule next sender
   if (beacon_sender) timer.cancel_event(beacon_sender);
@@ -500,7 +504,7 @@ void MDS::handle_mds_beacon(MMDSBeacon *m)
   version_t seq = m->get_seq();
 
   // make note of which mon 
-  monmap->last_mon = m->get_source().num();
+  monc->note_mon_leader(m->get_source().num());
 
   // update lab
   if (beacon_seq_stamp.count(seq)) {
@@ -631,9 +635,7 @@ void MDS::handle_mds_map(MMDSMap *m)
     // do i need an osdmap?
     if (oldwhoami < 0) {
       // we need an osdmap too.
-      int mon = monmap->pick_mon();
-      messenger->send_message(new MOSDGetMap(monmap->fsid, 0),
-			      monmap->get_inst(mon));
+      monc->send_mon_message(new MOSDGetMap(monc->get_fsid(), 0));
     }
   }
 
@@ -764,7 +766,7 @@ void MDS::bcast_mds_map()
   for (set<Session*>::const_iterator p = clients.begin();
        p != clients.end();
        ++p) 
-    messenger->send_message(new MMDSMap(monmap->fsid, mdsmap), (*p)->inst);
+    messenger->send_message(new MMDSMap(monc->get_fsid(), mdsmap), (*p)->inst);
   last_client_mdsmap_bcast = mdsmap->get_epoch();
 }
 
diff --git a/src/mds/MDS.h b/src/mds/MDS.h
index 1ff4f8c245c1..80a8bf66a042 100644
--- a/src/mds/MDS.h
+++ b/src/mds/MDS.h
@@ -29,7 +29,6 @@
 #include "common/Timer.h"
 #include "common/LogClient.h"
 
-#include "mon/MonMap.h"
 #include "MDSMap.h"
 
 #include "SessionMap.h"
@@ -104,6 +103,8 @@ enum {
 
 class filepath;
 
+class MonClient;
+
 class OSDMap;
 class Objecter;
 class Filer;
@@ -149,7 +150,7 @@ class MDS : public Dispatcher {
   int standby_replay_for;
 
   Messenger    *messenger;
-  MonMap       *monmap;
+  MonClient    *monc;
   MDSMap       *mdsmap;
   OSDMap       *osdmap;
   Objecter     *objecter;
@@ -292,7 +293,7 @@ class MDS : public Dispatcher {
  private:
   virtual bool dispatch_impl(Message *m);
  public:
-  MDS(const char *n, Messenger *m, MonMap *mm);
+  MDS(const char *n, Messenger *m, MonClient *mc);
   ~MDS();
 
   // who am i etc
diff --git a/src/mds/SnapServer.cc b/src/mds/SnapServer.cc
index e4d06cc9d50c..70a54d97b3d9 100644
--- a/src/mds/SnapServer.cc
+++ b/src/mds/SnapServer.cc
@@ -15,6 +15,7 @@
 #include "SnapServer.h"
 #include "MDS.h"
 #include "osd/OSDMap.h"
+#include "mon/MonClient.h"
 
 #include "include/types.h"
 #include "messages/MMDSTableRequest.h"
@@ -249,8 +250,7 @@ void SnapServer::check_osd_map(bool force)
   if (!all_purge.empty()) {
     dout(10) << "requesting removal of " << all_purge << dendl;
     MRemoveSnaps *m = new MRemoveSnaps(all_purge);
-    int mon = mds->monmap->pick_mon();
-    mds->messenger->send_message(m, mds->monmap->get_inst(mon));
+    mds->monc->send_mon_message(m);
   }
 
   last_checked_osdmap = version;
diff --git a/src/messages/MGetPoolStats.h b/src/messages/MGetPoolStats.h
index 47529458a611..0b2664205469 100644
--- a/src/messages/MGetPoolStats.h
+++ b/src/messages/MGetPoolStats.h
@@ -25,7 +25,7 @@ public:
   vector<string> pools;
 
   MGetPoolStats() : PaxosServiceMessage(MSG_GETPOOLSTATS, 0) {}
-  MGetPoolStats(ceph_fsid_t& f, tid_t t, vector<string>& ls, version_t l) :
+  MGetPoolStats(const ceph_fsid_t& f, tid_t t, vector<string>& ls, version_t l) :
     PaxosServiceMessage(MSG_GETPOOLSTATS, l),
     fsid(f), tid(t), pools(ls) { }
 
diff --git a/src/messages/MMDSBeacon.h b/src/messages/MMDSBeacon.h
index 1025e2b835c8..ec0d27ed2165 100644
--- a/src/messages/MMDSBeacon.h
+++ b/src/messages/MMDSBeacon.h
@@ -32,7 +32,7 @@ class MMDSBeacon : public PaxosServiceMessage {
 
  public:
   MMDSBeacon() : PaxosServiceMessage(MSG_MDS_BEACON, 0) {}
-  MMDSBeacon(ceph_fsid_t &f, string& n, epoch_t les, int st, version_t se) : 
+  MMDSBeacon(const ceph_fsid_t &f, string& n, epoch_t les, int st, version_t se) : 
     PaxosServiceMessage(MSG_MDS_BEACON, les), 
     fsid(f), name(n), state(st), seq(se),
     standby_for_rank(-1) { }
diff --git a/src/messages/MMDSMap.h b/src/messages/MMDSMap.h
index bcd0166b2e09..b307ef39f370 100644
--- a/src/messages/MMDSMap.h
+++ b/src/messages/MMDSMap.h
@@ -54,7 +54,7 @@ class MMDSMap : public Message {
 
   MMDSMap() : 
     Message(CEPH_MSG_MDS_MAP) {}
-  MMDSMap(ceph_fsid_t &f, MDSMap *mm) :
+  MMDSMap(const ceph_fsid_t &f, MDSMap *mm) :
     Message(CEPH_MSG_MDS_MAP),
     fsid(f) {
     epoch = mm->get_epoch();
diff --git a/src/messages/MOSDGetMap.h b/src/messages/MOSDGetMap.h
index 2506e5b5297f..e0ac5a1b7aba 100644
--- a/src/messages/MOSDGetMap.h
+++ b/src/messages/MOSDGetMap.h
@@ -25,7 +25,7 @@ class MOSDGetMap : public PaxosServiceMessage {
   epoch_t start;  // this is the first incremental the sender wants (he has start-1)
 
   MOSDGetMap() : PaxosServiceMessage(CEPH_MSG_OSD_GETMAP, 0) {}
-  MOSDGetMap(ceph_fsid_t& f, epoch_t s=0) : 
+  MOSDGetMap(const ceph_fsid_t& f, epoch_t s=0) : 
     PaxosServiceMessage(CEPH_MSG_OSD_GETMAP, s>0 ? s-1 : 0),
     fsid(f), start(s) { }
 
diff --git a/src/messages/MPoolSnap.h b/src/messages/MPoolSnap.h
index 4d1b8445c3c8..806707b30145 100644
--- a/src/messages/MPoolSnap.h
+++ b/src/messages/MPoolSnap.h
@@ -27,7 +27,7 @@ public:
   bool create;
 
   MPoolSnap() : PaxosServiceMessage(MSG_POOLSNAP, 0) {}
-  MPoolSnap( ceph_fsid_t& f, tid_t t, int p, string& n, bool c, version_t v) :
+  MPoolSnap(const ceph_fsid_t& f, tid_t t, int p, string& n, bool c, version_t v) :
     PaxosServiceMessage(MSG_POOLSNAP, v), fsid(f), tid(t), pool(p), name(n), create(c) {}
 
   const char *get_type_name() { return "poolsnap"; }
diff --git a/src/messages/MStatfs.h b/src/messages/MStatfs.h
index 3204cd872dfa..253520c69557 100644
--- a/src/messages/MStatfs.h
+++ b/src/messages/MStatfs.h
@@ -25,7 +25,7 @@ public:
   tid_t tid;
 
   MStatfs() : PaxosServiceMessage(CEPH_MSG_STATFS, 0) {}
-  MStatfs(ceph_fsid_t& f, tid_t t, version_t v) :
+  MStatfs(const ceph_fsid_t& f, tid_t t, version_t v) :
     PaxosServiceMessage(CEPH_MSG_STATFS, v), fsid(f), tid(t) {}
 
   const char *get_type_name() { return "statfs"; }
diff --git a/src/mon/MonClient.cc b/src/mon/MonClient.cc
index 9f7752051e97..ab453dea7e37 100644
--- a/src/mon/MonClient.cc
+++ b/src/mon/MonClient.cc
@@ -91,23 +91,27 @@ int MonClient::get_monmap()
   dout(10) << "get_monmap" << dendl;
   Mutex::Locker l(monc_lock);
   
-  SimpleMessenger rank; 
-
-  rank.bind();
-  
-  Messenger *msgr = rank.register_entity(entity_name_t::CLIENT(-1));
-  msgr->set_dispatcher(this);
-  
-  rank.start(true);  // do not daemonize!
+  SimpleMessenger *rank; 
+  bool temp_msgr = false;
+  if (!messenger) {
+    rank = new SimpleMessenger;
+    rank->bind();
+    messenger = rank->register_entity(entity_name_t::CLIENT(-1));
+    messenger->set_dispatcher(this);
+    rank->start(true);  // do not daemonize!
+    temp_msgr = true; 
+  }
   
   int attempt = 10;
   int i = 0;
 
   srand(getpid());
+  dout(10) << "have " << monmap.epoch << dendl;
+    
   while (monmap.epoch == 0) {
     i = rand() % monmap.mon_inst.size();
     dout(10) << "querying " << monmap.mon_inst[i] << dendl;
-    msgr->send_message(new MMonGetMap, monmap.mon_inst[i]);
+    messenger->send_message(new MMonGetMap, monmap.mon_inst[i]);
     
     if (--attempt == 0)
       break;
@@ -116,9 +120,12 @@ int MonClient::get_monmap()
     map_cond.WaitInterval(monc_lock, interval);
   }
   
-  msgr->shutdown();
-  rank.wait();
-  msgr->destroy();
+  if (temp_msgr) {
+    messenger->shutdown();
+    rank->wait();
+    messenger->destroy();
+    messenger = 0;
+  }
 
   if (monmap.epoch)
     return 0;
diff --git a/src/mon/MonClient.h b/src/mon/MonClient.h
index 4b34c4c2ba05..2f80ae9df7a9 100644
--- a/src/mon/MonClient.h
+++ b/src/mon/MonClient.h
@@ -80,6 +80,9 @@ private:
   int unmount();
 
   void send_mon_message(Message *m, bool new_mon=false);
+  void note_mon_leader(int m) {
+    monmap.last_mon = m;
+  }
 
   const ceph_fsid_t& get_fsid() {
     return monmap.fsid;
diff --git a/src/mon/MonMap.h b/src/mon/MonMap.h
index 126320345281..231ebeab4028 100644
--- a/src/mon/MonMap.h
+++ b/src/mon/MonMap.h
@@ -29,7 +29,7 @@ class MonMap {
 
   int       last_mon;    // last mon i talked to
 
-  MonMap(int s=0) : epoch(s?1:0), mon_inst(s), last_mon(-1) {
+  MonMap() : epoch(0), last_mon(-1) {
     memset(&fsid, 0, sizeof(fsid));
   }
 
@@ -40,7 +40,6 @@ class MonMap {
   }
 
   void add_mon(entity_inst_t inst) {
-    if (!epoch) epoch = 1;
     mon_inst.push_back(inst);
   }
 
diff --git a/src/osdc/Objecter.cc b/src/osdc/Objecter.cc
index 03926140bad0..81ba990298e3 100644
--- a/src/osdc/Objecter.cc
+++ b/src/osdc/Objecter.cc
@@ -15,7 +15,8 @@
 #include "Objecter.h"
 #include "osd/OSDMap.h"
 #include "osd/PGLS.h"
-#include "mon/MonMap.h"
+
+#include "mon/MonClient.h"
 
 #include "msg/Messenger.h"
 #include "msg/Message.h"
@@ -94,8 +95,8 @@ void Objecter::handle_osd_map(MOSDMap *m)
 {
   assert(osdmap); 
 
-  if (ceph_fsid_compare(&m->fsid, &monmap->fsid)) {
-    dout(0) << "handle_osd_map fsid " << m->fsid << " != " << monmap->fsid << dendl;
+  if (ceph_fsid_compare(&m->fsid, &monc->get_fsid())) {
+    dout(0) << "handle_osd_map fsid " << m->fsid << " != " << monc->get_fsid() << dendl;
     delete m;
     return;
   }
@@ -140,9 +141,7 @@ void Objecter::handle_osd_map(MOSDMap *m)
 	}
 	else {
 	  dout(3) << "handle_osd_map requesting missing epoch " << osdmap->get_epoch()+1 << dendl;
-	  int mon = monmap->pick_mon();
-	  messenger->send_message(new MOSDGetMap(monmap->fsid, osdmap->get_epoch()+1), 
-				  monmap->get_inst(mon));
+	  monc->send_mon_message(new MOSDGetMap(monc->get_fsid(), osdmap->get_epoch()+1));
 	  break;
 	}
 	
@@ -174,9 +173,7 @@ void Objecter::handle_osd_map(MOSDMap *m)
 	scan_pgs(changed_pgs);
       } else {
 	dout(3) << "handle_osd_map hmm, i want a full map, requesting" << dendl;
-	int mon = monmap->pick_mon();
-	messenger->send_message(new MOSDGetMap(monmap->fsid, 0),
-				monmap->get_inst(mon));
+	monc->send_mon_message(new MOSDGetMap(monc->get_fsid(), 0));
       }
     }
 
@@ -209,8 +206,7 @@ void Objecter::maybe_request_map()
   dout(10) << "maybe_request_map requesting next osd map" << dendl;
   last_epoch_requested_stamp = now;
   last_epoch_requested = osdmap->get_epoch()+1;
-  messenger->send_message(new MOSDGetMap(monmap->fsid, last_epoch_requested),
-			  monmap->get_inst(monmap->pick_mon()));
+  monc->send_mon_message(new MOSDGetMap(monc->get_fsid(), last_epoch_requested));
 }
 
 
@@ -689,10 +685,8 @@ void Objecter::delete_pool_snap(int *reply, int pool, string& snapName, Context
 
 void Objecter::pool_snap_submit(SnapOp *op) {
   dout(10) << "pool_snap_submit " << op->tid << dendl;
-  MPoolSnap *m = new MPoolSnap(monmap->fsid, op->tid, op->pool,
-			       op->name, op->create, last_seen_version);
-  int mon = monmap->pick_mon();
-  messenger->send_message(m, monmap->get_inst(mon));
+  monc->send_mon_message(new MPoolSnap(monc->get_fsid(), op->tid, op->pool,
+				       op->name, op->create, last_seen_version));
   op->last_submit = g_clock.now();
 }
 
@@ -744,9 +738,7 @@ void Objecter::get_pool_stats(vector<string>& pools, map<string,pool_stat_t> *re
 void Objecter::poolstat_submit(PoolStatOp *op)
 {
   dout(10) << "poolstat_submit " << op->tid << dendl;
-  MGetPoolStats *m = new MGetPoolStats(monmap->fsid, op->tid, op->pools, last_seen_version);
-  int mon = monmap->pick_mon();
-  messenger->send_message(m, monmap->get_inst(mon));
+  monc->send_mon_message(new MGetPoolStats(monc->get_fsid(), op->tid, op->pools, last_seen_version));
   op->last_submit = g_clock.now();
 }
 
@@ -785,11 +777,10 @@ void Objecter::get_fs_stats(ceph_statfs& result, Context *onfinish) {
   fs_stats_submit(op);
 }
 
-void Objecter::fs_stats_submit(StatfsOp *op) {
+void Objecter::fs_stats_submit(StatfsOp *op)
+{
   dout(10) << "fs_stats_submit" << op->tid << dendl;
-  MStatfs *m = new MStatfs(monmap->fsid, op->tid, last_seen_version);
-  int mon = monmap->pick_mon();
-  messenger->send_message(m, monmap->get_inst(mon));
+  monc->send_mon_message(new MStatfs(monc->get_fsid(), op->tid, last_seen_version));
   op->last_submit = g_clock.now();
 }
 
diff --git a/src/osdc/Objecter.h b/src/osdc/Objecter.h
index 6c9978e56789..7468e7ead0bc 100644
--- a/src/osdc/Objecter.h
+++ b/src/osdc/Objecter.h
@@ -32,7 +32,7 @@ using namespace __gnu_cxx;
 class Context;
 class Messenger;
 class OSDMap;
-class MonMap;
+class MonClient;
 class Message;
 
 class MPoolSnapReply;
@@ -174,7 +174,7 @@ struct ObjectOperation {
 class Objecter {
  public:  
   Messenger *messenger;
-  MonMap    *monmap;
+  MonClient *monc;
   OSDMap    *osdmap;
 
   bufferlist signed_ticket;
@@ -375,8 +375,8 @@ class Objecter {
 
 
  public:
-  Objecter(Messenger *m, MonMap *mm, OSDMap *om, Mutex& l) : 
-    messenger(m), monmap(mm), osdmap(om),
+  Objecter(Messenger *m, MonClient *mc, OSDMap *om, Mutex& l) : 
+    messenger(m), monc(mc), osdmap(om),
     last_tid(0), client_inc(-1),
     num_unacked(0), num_uncommitted(0),
     last_epoch_requested(0),