From 10da6d23cd447044ff38079807a7f36a5c347270 Mon Sep 17 00:00:00 2001 From: John Spray Date: Thu, 19 May 2016 12:59:56 +0100 Subject: [PATCH] osd: embed a MgrClient Signed-off-by: John Spray --- src/ceph_osd.cc | 6 ++++++ src/osd/OSD.cc | 25 +++++++++++++++++++++---- src/osd/OSD.h | 5 +++++ src/vstart.sh | 1 + 4 files changed, 33 insertions(+), 4 deletions(-) diff --git a/src/ceph_osd.cc b/src/ceph_osd.cc index 33fafe1a3830f..61a0ddb5f71c7 100644 --- a/src/ceph_osd.cc +++ b/src/ceph_osd.cc @@ -510,6 +510,12 @@ int main(int argc, const char **argv) CEPH_FEATURE_UID | CEPH_FEATURE_PGID64 | CEPH_FEATURE_OSDENC)); + ms_public->set_policy(entity_name_t::TYPE_MGR, + Messenger::Policy::lossy_client(supported, + CEPH_FEATURE_UID | + CEPH_FEATURE_PGID64 | + CEPH_FEATURE_OSDENC)); + //try to poison pill any OSD connections on the wrong address ms_public->set_policy(entity_name_t::TYPE_OSD, Messenger::Policy::stateless_server(0,0)); diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc index 9e00c34ca5722..1199a362a2805 100644 --- a/src/osd/OSD.cc +++ b/src/osd/OSD.cc @@ -1645,6 +1645,7 @@ OSD::OSD(CephContext *cct_, ObjectStore *store_, client_messenger(external_messenger), objecter_messenger(osdc_messenger), monc(mc), + mgrc(cct_, client_messenger), logger(NULL), recoverystate_perf(NULL), store(store_), @@ -2174,11 +2175,15 @@ int OSD::init() objecter_messenger->add_dispatcher_head(service.objecter); - monc->set_want_keys(CEPH_ENTITY_TYPE_MON | CEPH_ENTITY_TYPE_OSD); + monc->set_want_keys(CEPH_ENTITY_TYPE_MON | CEPH_ENTITY_TYPE_OSD + | CEPH_ENTITY_TYPE_MGR); r = monc->init(); if (r < 0) goto out; + mgrc.init(); + client_messenger->add_dispatcher_head(&mgrc); + // tell monc about log_client so it will know about mon session resets monc->set_log_client(&log_client); update_log_config(); @@ -2252,6 +2257,9 @@ int OSD::init() // subscribe to any pg creations monc->sub_want("osd_pg_creates", last_pg_create_epoch, 0); + // MgrClient needs this (it doesn't have MonClient reference itself) + monc->sub_want("mgrmap", 0, 0); + // we don't need to ask for an osdmap here; objecter will //monc->sub_want("osdmap", osdmap->get_epoch(), CEPH_SUBSCRIBE_ONETIME); @@ -2261,6 +2269,7 @@ int OSD::init() return 0; monout: + mgrc.shutdown(); monc->shutdown(); out: @@ -2757,6 +2766,7 @@ int OSD::shutdown() store = 0; dout(10) << "Store synced" << dendl; + mgrc.shutdown(); monc->shutdown(); osd_lock.Unlock(); @@ -4730,7 +4740,8 @@ void OSD::ms_handle_connect(Connection *con) void OSD::ms_handle_fast_connect(Connection *con) { - if (con->get_peer_type() != CEPH_ENTITY_TYPE_MON) { + if (con->get_peer_type() != CEPH_ENTITY_TYPE_MON && + con->get_peer_type() != CEPH_ENTITY_TYPE_MGR) { Session *s = static_cast(con->get_priv()); if (!s) { s = new Session(cct); @@ -4748,7 +4759,8 @@ void OSD::ms_handle_fast_connect(Connection *con) void OSD::ms_handle_fast_accept(Connection *con) { - if (con->get_peer_type() != CEPH_ENTITY_TYPE_MON) { + if (con->get_peer_type() != CEPH_ENTITY_TYPE_MON && + con->get_peer_type() != CEPH_ENTITY_TYPE_MGR) { Session *s = static_cast(con->get_priv()); if (!s) { s = new Session(cct); @@ -5835,6 +5847,7 @@ bool OSD::heartbeat_dispatch(Message *m) bool OSD::ms_dispatch(Message *m) { + dout(20) << "OSD::ms_dispatch: " << *m << dendl; if (m->get_type() == MSG_OSD_MARK_ME_DOWN) { service.got_stop_ack(); m->put(); @@ -6008,11 +6021,14 @@ bool OSD::ms_get_authorizer(int dest_type, AuthAuthorizer **authorizer, bool for if (force_new) { /* the MonClient checks keys every tick(), so we should just wait for that cycle to get through */ - if (monc->wait_auth_rotating(10) < 0) + if (monc->wait_auth_rotating(10) < 0) { + derr << "OSD::ms_get_authorizer wait_auth_rotating failed" << dendl; return false; + } } *authorizer = monc->auth->build_authorizer(dest_type); + derr << "OSD::ms_get_authorizer build_authorizer returned " << *authorizer << dendl; return *authorizer != NULL; } @@ -6029,6 +6045,7 @@ bool OSD::ms_verify_authorizer(Connection *con, int peer_type, * this makes the 'cluster' consistent w/ monitor's usage. */ case CEPH_ENTITY_TYPE_OSD: + case CEPH_ENTITY_TYPE_MGR: authorize_handler = authorize_handler_cluster_registry->get_handler(protocol); break; default: diff --git a/src/osd/OSD.h b/src/osd/OSD.h index dc6e84da6cd8d..0b835c0ad0fe3 100644 --- a/src/osd/OSD.h +++ b/src/osd/OSD.h @@ -24,6 +24,10 @@ #include "common/Timer.h" #include "common/WorkQueue.h" #include "common/AsyncReserver.h" +#include "common/ceph_context.h" + +#include "mgr/MgrClient.h" + #include "os/ObjectStore.h" #include "OSDCap.h" @@ -1207,6 +1211,7 @@ protected: Messenger *client_messenger; Messenger *objecter_messenger; MonClient *monc; // check the "monc helpers" list before accessing directly + MgrClient mgrc; PerfCounters *logger; PerfCounters *recoverystate_perf; ObjectStore *store; diff --git a/src/vstart.sh b/src/vstart.sh index 90626afc6c54c..847658a734e80 100755 --- a/src/vstart.sh +++ b/src/vstart.sh @@ -362,6 +362,7 @@ else debug osd = 25 debug objecter = 20 debug monc = 20 + debug mgrc = 20 debug journal = 20 debug filestore = 20 debug bluestore = 30 -- 2.39.5