osd: defer boot until we have rotating keys

Make sure we have our rotating keys before we start booting.  This
ensures we can open connections with peers *before* we add ourselves to
the osdmap.  This behaviors marks instances of #3292, although it is
not clear whether it is responsible for the actual crash.

Signed-off-by: Sage Weil <sage@inktank.com>
Reviewed-by: Sam Just <sam.just@inktank.com>

diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc
index 60b860ca1b4fe8ed8db303650f57d407910391d8..d2a2a8ab05cca8b606658960467fbb2cd4c8d989 100644 (file)
--- a/src/osd/OSD.cc
+++ b/src/osd/OSD.cc
@@ -712,7 +712,7 @@ OSD::OSD(int id, Messenger *internal_messenger, Messenger *external_messenger,
   dev_path(dev), journal_path(jdev),
   dispatch_running(false),
   osd_compat(get_osd_compat_set()),
-  state(STATE_BOOTING), boot_epoch(0), up_epoch(0), bind_epoch(0),
+  state(STATE_INITIALIZING), boot_epoch(0), up_epoch(0), bind_epoch(0),
   op_tp(external_messenger->cct, "OSD::op_tp", g_conf->osd_op_threads, "osd_op_threads"),
   recovery_tp(external_messenger->cct, "OSD::recovery_tp", g_conf->osd_recovery_threads, "osd_recovery_threads"),
   disk_tp(external_messenger->cct, "OSD::disk_tp", g_conf->osd_disk_threads, "osd_disk_threads"),
@@ -901,20 +901,6 @@ int OSD::init()
   // tell monc about log_client so it will know about mon session resets
   monc->set_log_client(&clog);
 
-  osd_lock.Unlock();
-
-  r = monc->authenticate();
-  if (r < 0) {
-    monc->shutdown();
-    store->umount();
-    osd_lock.Lock(); // locker is going to unlock this on function exit
-    return r;
-  }
-
-  monc->wait_auth_rotating(30.0);
-
-  osd_lock.Lock();
-
   op_tp.start();
   recovery_tp.start();
   disk_tp.start();
@@ -947,6 +933,24 @@ int OSD::init()
   service.init();
   service.publish_map(osdmap);
   service.publish_superblock(superblock);
+
+  osd_lock.Unlock();
+
+  r = monc->authenticate();
+  if (r < 0) {
+    monc->shutdown();
+    store->umount();
+    osd_lock.Lock(); // locker is going to unlock this on function exit
+    return r;
+  }
+
+  while (monc->wait_auth_rotating(30.0) < 0) {
+    derr << "unable to obtain rotating service keys; retrying" << dendl;
+  }
+
+  state = STATE_BOOTING;
+
+  osd_lock.Lock();
   return 0;
 }
 
@@ -2394,6 +2398,11 @@ void OSD::_maybe_boot(epoch_t oldest, epoch_t newest)
   Mutex::Locker l(osd_lock);
   dout(10) << "_maybe_boot mon has osdmaps " << oldest << ".." << newest << dendl;
 
+  if (is_initializing()) {
+    dout(10) << "still initializing" << dendl;
+    return;
+  }
+
   // if our map within recent history, try to add ourselves to the osdmap.
   if (osdmap->test_flag(CEPH_OSDMAP_NOUP)) {
     dout(5) << "osdmap NOUP flag is set, waiting for it to clear" << dendl;

diff --git a/src/osd/OSD.h b/src/osd/OSD.h
index 4b018d7f5d1f568231b3a62a9e1f62b24fdb6415..ff277e1e5b8bce48af094e62affa4abdcd6dbae9 100644 (file)
--- a/src/osd/OSD.h
+++ b/src/osd/OSD.h
@@ -416,9 +416,10 @@ private:
 
   // -- state --
 public:
-  static const int STATE_BOOTING = 1;
-  static const int STATE_ACTIVE = 2;
-  static const int STATE_STOPPING = 3;
+  static const int STATE_INITIALIZING = 1;
+  static const int STATE_BOOTING = 2;
+  static const int STATE_ACTIVE = 3;
+  static const int STATE_STOPPING = 4;
 
 private:
   int state;
@@ -427,6 +428,7 @@ private:
   epoch_t bind_epoch;  // epoch we last did a bind to new ip:ports
 
 public:
+  bool is_initializing() { return state == STATE_INITIALIZING; }
   bool is_booting() { return state == STATE_BOOTING; }
   bool is_active() { return state == STATE_ACTIVE; }
   bool is_stopping() { return state == STATE_STOPPING; }