osd: embed a MgrClient

Signed-off-by: John Spray <john.spray@redhat.com>

diff --git a/src/ceph_osd.cc b/src/ceph_osd.cc
index 33fafe1a3830f734be5042fe5621243d8321c3d9..61a0ddb5f71c74478fd0f1947e1345d3c392155b 100644 (file)
--- a/src/ceph_osd.cc
+++ b/src/ceph_osd.cc
@@ -510,6 +510,12 @@ int main(int argc, const char **argv)
                                                               CEPH_FEATURE_UID |
                                                               CEPH_FEATURE_PGID64 |
                                                               CEPH_FEATURE_OSDENC));
+  ms_public->set_policy(entity_name_t::TYPE_MGR,
+                               Messenger::Policy::lossy_client(supported,
+                                                              CEPH_FEATURE_UID |
+                                                              CEPH_FEATURE_PGID64 |
+                                                              CEPH_FEATURE_OSDENC));
+
   //try to poison pill any OSD connections on the wrong address
   ms_public->set_policy(entity_name_t::TYPE_OSD,
                        Messenger::Policy::stateless_server(0,0));

diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc
index 9e00c34ca57226db54a96e04cb02195a61e19817..1199a362a28059bc59620c8c377545e9a08be07a 100644 (file)
--- a/src/osd/OSD.cc
+++ b/src/osd/OSD.cc
@@ -1645,6 +1645,7 @@ OSD::OSD(CephContext *cct_, ObjectStore *store_,
   client_messenger(external_messenger),
   objecter_messenger(osdc_messenger),
   monc(mc),
+  mgrc(cct_, client_messenger),
   logger(NULL),
   recoverystate_perf(NULL),
   store(store_),
@@ -2174,11 +2175,15 @@ int OSD::init()
 
   objecter_messenger->add_dispatcher_head(service.objecter);
 
-  monc->set_want_keys(CEPH_ENTITY_TYPE_MON | CEPH_ENTITY_TYPE_OSD);
+  monc->set_want_keys(CEPH_ENTITY_TYPE_MON | CEPH_ENTITY_TYPE_OSD
+                      | CEPH_ENTITY_TYPE_MGR);
   r = monc->init();
   if (r < 0)
     goto out;
 
+  mgrc.init();
+  client_messenger->add_dispatcher_head(&mgrc);
+
   // tell monc about log_client so it will know about mon session resets
   monc->set_log_client(&log_client);
   update_log_config();
@@ -2252,6 +2257,9 @@ int OSD::init()
   // subscribe to any pg creations
   monc->sub_want("osd_pg_creates", last_pg_create_epoch, 0);
 
+  // MgrClient needs this (it doesn't have MonClient reference itself)
+  monc->sub_want("mgrmap", 0, 0);
+
   // we don't need to ask for an osdmap here; objecter will
   //monc->sub_want("osdmap", osdmap->get_epoch(), CEPH_SUBSCRIBE_ONETIME);
 
@@ -2261,6 +2269,7 @@ int OSD::init()
 
   return 0;
 monout:
+  mgrc.shutdown();
   monc->shutdown();
 
 out:
@@ -2757,6 +2766,7 @@ int OSD::shutdown()
   store = 0;
   dout(10) << "Store synced" << dendl;
 
+  mgrc.shutdown();
   monc->shutdown();
   osd_lock.Unlock();
 
@@ -4730,7 +4740,8 @@ void OSD::ms_handle_connect(Connection *con)
 
 void OSD::ms_handle_fast_connect(Connection *con)
 {
-  if (con->get_peer_type() != CEPH_ENTITY_TYPE_MON) {
+  if (con->get_peer_type() != CEPH_ENTITY_TYPE_MON &&
+      con->get_peer_type() != CEPH_ENTITY_TYPE_MGR) {
     Session *s = static_cast<Session*>(con->get_priv());
     if (!s) {
       s = new Session(cct);
@@ -4748,7 +4759,8 @@ void OSD::ms_handle_fast_connect(Connection *con)
 
 void OSD::ms_handle_fast_accept(Connection *con)
 {
-  if (con->get_peer_type() != CEPH_ENTITY_TYPE_MON) {
+  if (con->get_peer_type() != CEPH_ENTITY_TYPE_MON &&
+      con->get_peer_type() != CEPH_ENTITY_TYPE_MGR) {
     Session *s = static_cast<Session*>(con->get_priv());
     if (!s) {
       s = new Session(cct);
@@ -5835,6 +5847,7 @@ bool OSD::heartbeat_dispatch(Message *m)
 
 bool OSD::ms_dispatch(Message *m)
 {
+  dout(20) << "OSD::ms_dispatch: " << *m << dendl;
   if (m->get_type() == MSG_OSD_MARK_ME_DOWN) {
     service.got_stop_ack();
     m->put();
@@ -6008,11 +6021,14 @@ bool OSD::ms_get_authorizer(int dest_type, AuthAuthorizer **authorizer, bool for
   if (force_new) {
     /* the MonClient checks keys every tick(), so we should just wait for that cycle
        to get through */
-    if (monc->wait_auth_rotating(10) < 0)
+    if (monc->wait_auth_rotating(10) < 0) {
+      derr << "OSD::ms_get_authorizer wait_auth_rotating failed" << dendl;
       return false;
+    }
   }
 
   *authorizer = monc->auth->build_authorizer(dest_type);
+  derr << "OSD::ms_get_authorizer build_authorizer returned " << *authorizer << dendl;
   return *authorizer != NULL;
 }
 
@@ -6029,6 +6045,7 @@ bool OSD::ms_verify_authorizer(Connection *con, int peer_type,
      * this makes the 'cluster' consistent w/ monitor's usage.
      */
   case CEPH_ENTITY_TYPE_OSD:
+  case CEPH_ENTITY_TYPE_MGR:
     authorize_handler = authorize_handler_cluster_registry->get_handler(protocol);
     break;
   default:

diff --git a/src/osd/OSD.h b/src/osd/OSD.h
index dc6e84da6cd8de2b77d4d7c54a76cbd9d7a8a9b1..0b835c0ad0fe3974da3547961819fac12a4c7720 100644 (file)
--- a/src/osd/OSD.h
+++ b/src/osd/OSD.h
@@ -24,6 +24,10 @@
 #include "common/Timer.h"
 #include "common/WorkQueue.h"
 #include "common/AsyncReserver.h"
+#include "common/ceph_context.h"
+
+#include "mgr/MgrClient.h"
+
 #include "os/ObjectStore.h"
 #include "OSDCap.h" 
  
@@ -1207,6 +1211,7 @@ protected:
   Messenger   *client_messenger;
   Messenger   *objecter_messenger;
   MonClient   *monc; // check the "monc helpers" list before accessing directly
+  MgrClient   mgrc;
   PerfCounters      *logger;
   PerfCounters      *recoverystate_perf;
   ObjectStore *store;

diff --git a/src/vstart.sh b/src/vstart.sh
index 90626afc6c54ce023cf5e44e95df74845181fe11..847658a734e80f9dcda4c21ea3ec4a1a892fe8fa 100755 (executable)
--- a/src/vstart.sh
+++ b/src/vstart.sh
@@ -362,6 +362,7 @@ else
         debug osd = 25
         debug objecter = 20
         debug monc = 20
+        debug mgrc = 20
         debug journal = 20
         debug filestore = 20
         debug bluestore = 30