objecter: separate locked and unlocked init/shutdown

We don't want to hold the lock while we register the admin socket commands
or else we create a lock cycle when we try to process them later.

Signed-off-by: Sage Weil <sage@inktank.com>

diff --git a/src/librados/RadosClient.cc b/src/librados/RadosClient.cc
index 56a31f81ea56c37d84315c67cf084328acb742a6..ae56c76dfc4b884c77453eeddee66396d1a9ea5d 100644 (file)
--- a/src/librados/RadosClient.cc
+++ b/src/librados/RadosClient.cc
@@ -181,12 +181,13 @@ int librados::RadosClient::connect()
   }
   messenger->set_myname(entity_name_t::CLIENT(monclient.get_global_id()));
 
+  objecter->init_unlocked();
   lock.Lock();
 
   timer.init();
 
   objecter->set_client_incarnation(0);
-  objecter->init();
+  objecter->init_locked();
   monclient.renew_subs();
 
   while (osdmap.get_epoch() == 0) {
@@ -220,11 +221,16 @@ void librados::RadosClient::shutdown()
     finisher.stop();
   }
   monclient.shutdown();
-  if (objecter && state == CONNECTED)
-    objecter->shutdown();
+  bool need_objecter = false;
+  if (objecter && state == CONNECTED) {
+    need_objecter = true;
+    objecter->shutdown_locked();
+  }
   state = DISCONNECTED;
   timer.shutdown();   // will drop+retake lock
   lock.Unlock();
+  if (need_objecter)
+    objecter->shutdown_unlocked();
   if (messenger) {
     messenger->shutdown();
     messenger->wait();

diff --git a/src/mds/Dumper.cc b/src/mds/Dumper.cc
index 793625581dc40bd2a8d9e5c2b9b04bb88837e8b4..917403ee2088502644b6e9e5dd5e6f408080dd08 100644 (file)
--- a/src/mds/Dumper.cc
+++ b/src/mds/Dumper.cc
@@ -66,8 +66,9 @@ void Dumper::init(int rank)
   client_t whoami = monc->get_global_id();
   messenger->set_myname(entity_name_t::CLIENT(whoami.v));
 
+  objecter->init_unlocked();
   lock.Lock();
-  objecter->init();
+  objecter->init_locked();
   objecter->wait_for_osd_map();
   timer.init();
   lock.Unlock();
@@ -77,7 +78,9 @@ void Dumper::shutdown()
 {
   lock.Lock();
   timer.shutdown();
+  objecter->shutdown_locked();
   lock.Unlock();
+  objecter->shutdown_unlocked();
 }
 
 void Dumper::dump(const char *dump_file)

diff --git a/src/mds/MDS.cc b/src/mds/MDS.cc
index 3df617ccb07f62545b03b08f11a0b45cac3b29d2..fde5ff11ff00cf10b9b22d1f625e188edbb1187d 100644 (file)
--- a/src/mds/MDS.cc
+++ b/src/mds/MDS.cc
@@ -462,6 +462,8 @@ int MDS::init(int wanted_state)
   monc->authenticate();
   monc->wait_auth_rotating(30.0);
 
+  objecter->init_unlocked();
+
   mds_lock.Lock();
   if (want_state == CEPH_MDS_STATE_DNE) {
     mds_lock.Unlock();
@@ -504,8 +506,8 @@ int MDS::init(int wanted_state)
   beacon_start();
   whoami = -1;
   messenger->set_myname(entity_name_t::MDS(whoami));
-
-  objecter->init();
+  
+  objecter->init_locked();
 
   monc->sub_want("mdsmap", 0, 0);
   monc->renew_subs();
@@ -1560,7 +1562,7 @@ void MDS::suicide()
   // shut down cache
   mdcache->shutdown();
 
-  objecter->shutdown();
+  objecter->shutdown_locked();
   
   // shut down messenger
   messenger->shutdown();

diff --git a/src/mds/Resetter.cc b/src/mds/Resetter.cc
index 63139f56588456ff8362beafc4abb63348aa12f9..e968cdcada6ee8bdcc8a792757b20adb70bb5fe8 100644 (file)
--- a/src/mds/Resetter.cc
+++ b/src/mds/Resetter.cc
@@ -76,8 +76,9 @@ void Resetter::init(int rank)
   client_t whoami = monc->get_global_id();
   messenger->set_myname(entity_name_t::CLIENT(whoami.v));
 
+  objecter->init_unlocked();
   lock.Lock();
-  objecter->init();
+  objecter->init_locked();
   objecter->wait_for_osd_map();
   timer.init();
   lock.Unlock();
@@ -87,7 +88,9 @@ void Resetter::shutdown()
 {
   lock.Lock();
   timer.shutdown();
+  objecter->shutdown_locked();
   lock.Unlock();
+  objecter->shutdown_unlocked();
   messenger->shutdown();
   messenger->wait();
 }

diff --git a/src/osdc/Objecter.cc b/src/osdc/Objecter.cc
index 6643cfd790bbbf438cc7eacf84e69943194ff2d6..5e36749898d79324794630e91327c6ea57e2df34 100644 (file)
--- a/src/osdc/Objecter.cc
+++ b/src/osdc/Objecter.cc
@@ -121,9 +121,8 @@ enum {
 
 // messages ------------------------------
 
-void Objecter::init()
+void Objecter::init_unlocked()
 {
-  assert(client_lock.is_locked());
   assert(!initialized);
 
   if (!logger) {
@@ -209,6 +208,12 @@ void Objecter::init()
     lderr(cct) << "error registering admin socket command: "
               << cpp_strerror(-ret) << dendl;
   }
+}
+
+void Objecter::init_locked()
+{
+  assert(client_lock.is_locked());
+  assert(!initialized);
 
   schedule_tick();
   maybe_request_map();
@@ -216,7 +221,7 @@ void Objecter::init()
   initialized = true;
 }
 
-void Objecter::shutdown() 
+void Objecter::shutdown_locked() 
 {
   assert(client_lock.is_locked());
   assert(initialized);
@@ -232,7 +237,10 @@ void Objecter::shutdown()
     timer.cancel_event(tick_event);
     tick_event = NULL;
   }
+}
 
+void Objecter::shutdown_unlocked()
+{
   if (m_request_state_hook) {
     AdminSocket* admin_socket = cct->get_admin_socket();
     admin_socket->unregister_command("objecter_requests");

diff --git a/src/osdc/Objecter.h b/src/osdc/Objecter.h
index 54fbaf3e46bd22547ff63ccc120a869281b9c575..e96cb48690b245123b1b8177852694ed85d64257 100644 (file)
--- a/src/osdc/Objecter.h
+++ b/src/osdc/Objecter.h
@@ -969,8 +969,10 @@ public:
     assert(!logger);
   }
 
-  void init();
-  void shutdown();
+  void init_unlocked();
+  void init_locked();
+  void shutdown_locked();
+  void shutdown_unlocked();
 
   /**
    * Tell the objecter to throttle outgoing ops according to its