From: Yehuda Sadeh <yehuda@inktank.com>
Date: Wed, 4 Jun 2014 18:46:50 +0000 (-0700)
Subject: objecter: split objecter initialization
X-Git-Tag: v0.86~213^2~90
X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=9b811a33d51b6bdeaea4285411f5fafbaad72a22;p=ceph.git

objecter: split objecter initialization

Separate objecter initialization to non cluster related work (e.g.,
internal data structures, other registrations), and to operations that
can initiate cluster interaction. This is so that we don't hit a rare
race where we can get called indirectly from one of the dispatcher callbacks
e.g., into handle_osd_map() when not yet being initialized.
This requires that objecter->init() should be called before
messenger->add_dispatcher_head(), and objecter->start() after it.

Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
---

diff --git a/src/client/Client.cc b/src/client/Client.cc
index b57c44a14efd..cc42c5d05c0f 100644
--- a/src/client/Client.cc
+++ b/src/client/Client.cc
@@ -350,6 +350,7 @@ int Client::init()
 
   objectcacher->start();
 
+  objecter->init();
   // ok!
   messenger->add_dispatcher_head(this);
 
@@ -362,10 +363,7 @@ int Client::init()
     monclient->shutdown();
     return r;
   }
-
-  client_lock.Unlock();
-  objecter->init();
-  client_lock.Lock();
+  objecter->start();
 
   monclient->set_want_keys(CEPH_ENTITY_TYPE_MDS | CEPH_ENTITY_TYPE_OSD);
   monclient->sub_want("mdsmap", 0, 0);
diff --git a/src/librados/RadosClient.cc b/src/librados/RadosClient.cc
index 6c0c2b8406d1..c6c6ca307946 100644
--- a/src/librados/RadosClient.cc
+++ b/src/librados/RadosClient.cc
@@ -245,6 +245,7 @@ int librados::RadosClient::connect()
 
   monclient.set_messenger(messenger);
 
+  objecter->init();
   messenger->add_dispatcher_head(this);
 
   messenger->start();
@@ -268,7 +269,7 @@ int librados::RadosClient::connect()
   messenger->set_myname(entity_name_t::CLIENT(monclient.get_global_id()));
 
   objecter->set_client_incarnation(0);
-  objecter->init();
+  objecter->start();
   lock.Lock();
 
   timer.init();
diff --git a/src/mds/MDS.cc b/src/mds/MDS.cc
index 8ce9fe968f40..c40fdb96a9d4 100644
--- a/src/mds/MDS.cc
+++ b/src/mds/MDS.cc
@@ -594,6 +594,8 @@ int MDS::init(MDSMap::DaemonState wanted_state)
   dout(10) << sizeof(Capability) << "\tCapability " << dendl;
   dout(10) << sizeof(xlist<void*>::item) << "\t xlist<>::item   *2=" << 2*sizeof(xlist<void*>::item) << dendl;
 
+  objecter->init();
+
   messenger->add_dispatcher_tail(this);
 
   // get monmap
@@ -616,7 +618,7 @@ int MDS::init(MDSMap::DaemonState wanted_state)
   while (monc->wait_auth_rotating(30.0) < 0) {
     derr << "unable to obtain rotating service keys; retrying" << dendl;
   }
-  objecter->init();
+  objecter->start();
 
   mds_lock.Lock();
   if (want_state == CEPH_MDS_STATE_DNE) {
diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc
index 0892820b108b..33ab0e59f193 100644
--- a/src/osd/OSD.cc
+++ b/src/osd/OSD.cc
@@ -244,7 +244,9 @@ OSDService::OSDService(OSD *osd) :
 #ifdef PG_DEBUG_REFS
   , pgid_lock("OSDService::pgid_lock")
 #endif
-{}
+{
+  objecter->init();
+}
 
 OSDService::~OSDService()
 {
@@ -460,7 +462,7 @@ void OSDService::init()
   reserver_finisher.start();
   objecter_finisher.start();
   objecter->set_client_incarnation(0);
-  objecter->init();
+  objecter->start();
   watch_timer.init();
   agent_timer.init();
 
@@ -1989,7 +1991,7 @@ int OSD::init()
   dout(2) << "superblock: i am osd." << superblock.whoami << dendl;
 
   create_logger();
-    
+
   // i'm ready!
   client_messenger->add_dispatcher_head(this);
   cluster_messenger->add_dispatcher_head(this);
diff --git a/src/osdc/Objecter.cc b/src/osdc/Objecter.cc
index 5f53176c8963..fef6faf73f87 100644
--- a/src/osdc/Objecter.cc
+++ b/src/osdc/Objecter.cc
@@ -168,6 +168,9 @@ void Objecter::handle_conf_change(const struct md_config_t *conf,
 
 // messages ------------------------------
 
+/*
+ * initialize only internal data structures, don't initiate cluster interaction
+ */
 void Objecter::init()
 {
   assert(!initialized.read());
@@ -266,26 +269,31 @@ void Objecter::init()
 
   timer.init();
 
-  rwlock.get_read();
+  initialized.set(1);
+}
+
+/*
+ * ok, cluster interaction can happen
+ */
+void Objecter::start()
+{
+  RWLock::RLocker rl(rwlock);
 
   schedule_tick();
   if (osdmap->get_epoch() == 0) {
     int r = _maybe_request_map();
     assert (r == 0 || osdmap->get_epoch() > 0);
   }
-
-  rwlock.unlock();
-
-  initialized.set(1);
 }
 
 void Objecter::shutdown()
 {
   assert(initialized.read());
-  initialized.set(0);
 
   RWLock::WLocker wl(rwlock);
 
+  initialized.set(0);
+
   map<int,OSDSession*>::iterator p;
   while (!osd_sessions.empty()) {
     p = osd_sessions.begin();
diff --git a/src/osdc/Objecter.h b/src/osdc/Objecter.h
index 1a14aa73c370..2440dcec4810 100644
--- a/src/osdc/Objecter.h
+++ b/src/osdc/Objecter.h
@@ -1608,6 +1608,7 @@ public:
   }
 
   void init();
+  void start();
   void shutdown();
 
   /**
diff --git a/src/test/mon/test_mon_workloadgen.cc b/src/test/mon/test_mon_workloadgen.cc
index 55a5bbe8aa97..b52760141401 100644
--- a/src/test/mon/test_mon_workloadgen.cc
+++ b/src/test/mon/test_mon_workloadgen.cc
@@ -260,6 +260,7 @@ class ClientStub : public TestStub
     objecter->set_balanced_budget();
 
     monc.set_messenger(messenger.get());
+    objecter->init();
     messenger->add_dispatcher_head(this);
     messenger->start();
     monc.set_want_keys(CEPH_ENTITY_TYPE_MON|CEPH_ENTITY_TYPE_OSD);
@@ -281,7 +282,7 @@ class ClientStub : public TestStub
     monc.wait_auth_rotating(30.0);
 
     objecter->set_client_incarnation(0);
-    objecter->init();
+    objecter->start();
 
     lock.Lock();
     timer.init();