objecter: split objecter initialization

Separate objecter initialization to non cluster related work (e.g.,
internal data structures, other registrations), and to operations that
can initiate cluster interaction. This is so that we don't hit a rare
race where we can get called indirectly from one of the dispatcher callbacks
e.g., into handle_osd_map() when not yet being initialized.
This requires that objecter->init() should be called before
messenger->add_dispatcher_head(), and objecter->start() after it.

Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>

diff --git a/src/client/Client.cc b/src/client/Client.cc
index b57c44a14efd885c416c0838cf48efdbbb8385bc..cc42c5d05c0f1a2e031995c342fbf906e67f2598 100644 (file)
--- a/src/client/Client.cc
+++ b/src/client/Client.cc
@@ -350,6 +350,7 @@ int Client::init()
 
   objectcacher->start();
 
+  objecter->init();
   // ok!
   messenger->add_dispatcher_head(this);
 
@@ -362,10 +363,7 @@ int Client::init()
     monclient->shutdown();
     return r;
   }
-
-  client_lock.Unlock();
-  objecter->init();
-  client_lock.Lock();
+  objecter->start();
 
   monclient->set_want_keys(CEPH_ENTITY_TYPE_MDS | CEPH_ENTITY_TYPE_OSD);
   monclient->sub_want("mdsmap", 0, 0);

diff --git a/src/librados/RadosClient.cc b/src/librados/RadosClient.cc
index 6c0c2b8406d19c658f1cb0c01299232086abfdb5..c6c6ca30794688762e3aa5e9cb4e46feead1b547 100644 (file)
--- a/src/librados/RadosClient.cc
+++ b/src/librados/RadosClient.cc
@@ -245,6 +245,7 @@ int librados::RadosClient::connect()
 
   monclient.set_messenger(messenger);
 
+  objecter->init();
   messenger->add_dispatcher_head(this);
 
   messenger->start();
@@ -268,7 +269,7 @@ int librados::RadosClient::connect()
   messenger->set_myname(entity_name_t::CLIENT(monclient.get_global_id()));
 
   objecter->set_client_incarnation(0);
-  objecter->init();
+  objecter->start();
   lock.Lock();
 
   timer.init();

diff --git a/src/mds/MDS.cc b/src/mds/MDS.cc
index 8ce9fe968f4080fbb348cda44dc97f4ee3e099d3..c40fdb96a9d414015cac3804f7d07e6ad44c3edf 100644 (file)
--- a/src/mds/MDS.cc
+++ b/src/mds/MDS.cc
@@ -594,6 +594,8 @@ int MDS::init(MDSMap::DaemonState wanted_state)
   dout(10) << sizeof(Capability) << "\tCapability " << dendl;
   dout(10) << sizeof(xlist<void*>::item) << "\t xlist<>::item   *2=" << 2*sizeof(xlist<void*>::item) << dendl;
 
+  objecter->init();
+
   messenger->add_dispatcher_tail(this);
 
   // get monmap
@@ -616,7 +618,7 @@ int MDS::init(MDSMap::DaemonState wanted_state)
   while (monc->wait_auth_rotating(30.0) < 0) {
     derr << "unable to obtain rotating service keys; retrying" << dendl;
   }
-  objecter->init();
+  objecter->start();
 
   mds_lock.Lock();
   if (want_state == CEPH_MDS_STATE_DNE) {

diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc
index 0892820b108ba66aaded337a4f7d837460070ba7..33ab0e59f193b023bb1a76e9fd0c7c9e20e95f74 100644 (file)
--- a/src/osd/OSD.cc
+++ b/src/osd/OSD.cc
@@ -244,7 +244,9 @@ OSDService::OSDService(OSD *osd) :
 #ifdef PG_DEBUG_REFS
   , pgid_lock("OSDService::pgid_lock")
 #endif
-{}
+{
+  objecter->init();
+}
 
 OSDService::~OSDService()
 {
@@ -460,7 +462,7 @@ void OSDService::init()
   reserver_finisher.start();
   objecter_finisher.start();
   objecter->set_client_incarnation(0);
-  objecter->init();
+  objecter->start();
   watch_timer.init();
   agent_timer.init();
 
@@ -1989,7 +1991,7 @@ int OSD::init()
   dout(2) << "superblock: i am osd." << superblock.whoami << dendl;
 
   create_logger();
-    
+
   // i'm ready!
   client_messenger->add_dispatcher_head(this);
   cluster_messenger->add_dispatcher_head(this);

diff --git a/src/osdc/Objecter.cc b/src/osdc/Objecter.cc
index 5f53176c89633a2087a9c690437239dd5c3befc5..fef6faf73f87073d9ca2c2a2398993502eb1d9ae 100644 (file)
--- a/src/osdc/Objecter.cc
+++ b/src/osdc/Objecter.cc
@@ -168,6 +168,9 @@ void Objecter::handle_conf_change(const struct md_config_t *conf,
 
 // messages ------------------------------
 
+/*
+ * initialize only internal data structures, don't initiate cluster interaction
+ */
 void Objecter::init()
 {
   assert(!initialized.read());
@@ -266,26 +269,31 @@ void Objecter::init()
 
   timer.init();
 
-  rwlock.get_read();
+  initialized.set(1);
+}
+
+/*
+ * ok, cluster interaction can happen
+ */
+void Objecter::start()
+{
+  RWLock::RLocker rl(rwlock);
 
   schedule_tick();
   if (osdmap->get_epoch() == 0) {
     int r = _maybe_request_map();
     assert (r == 0 || osdmap->get_epoch() > 0);
   }
-
-  rwlock.unlock();
-
-  initialized.set(1);
 }
 
 void Objecter::shutdown()
 {
   assert(initialized.read());
-  initialized.set(0);
 
   RWLock::WLocker wl(rwlock);
 
+  initialized.set(0);
+
   map<int,OSDSession*>::iterator p;
   while (!osd_sessions.empty()) {
     p = osd_sessions.begin();

diff --git a/src/osdc/Objecter.h b/src/osdc/Objecter.h
index 1a14aa73c370951aae8ac653c3c8820e7e7d2f7f..2440dcec48105fc55671ecc0823e7a17a90160e8 100644 (file)
--- a/src/osdc/Objecter.h
+++ b/src/osdc/Objecter.h
@@ -1608,6 +1608,7 @@ public:
   }
 
   void init();
+  void start();
   void shutdown();
 
   /**

diff --git a/src/test/mon/test_mon_workloadgen.cc b/src/test/mon/test_mon_workloadgen.cc
index 55a5bbe8aa9725ab317049af30a3b6ada923dc22..b52760141401fff43ed0402c14fdd2f5f85a956a 100644 (file)
--- a/src/test/mon/test_mon_workloadgen.cc
+++ b/src/test/mon/test_mon_workloadgen.cc
@@ -260,6 +260,7 @@ class ClientStub : public TestStub
     objecter->set_balanced_budget();
 
     monc.set_messenger(messenger.get());
+    objecter->init();
     messenger->add_dispatcher_head(this);
     messenger->start();
     monc.set_want_keys(CEPH_ENTITY_TYPE_MON|CEPH_ENTITY_TYPE_OSD);
@@ -281,7 +282,7 @@ class ClientStub : public TestStub
     monc.wait_auth_rotating(30.0);
 
     objecter->set_client_incarnation(0);
-    objecter->init();
+    objecter->start();
 
     lock.Lock();
     timer.init();