From: Sage Weil Date: Mon, 21 Jan 2019 16:22:26 +0000 (-0600) Subject: switch monc, daemons to use new msgr2 auth frame exchange X-Git-Tag: v14.1.0~183^2~42 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=1d29722f801ccdfe9c4080df2f2e79ddd4379c20;p=ceph.git switch monc, daemons to use new msgr2 auth frame exchange - MonClient implements AuthClient to authenticate as a client - MonClient implements AuthServer to allow daemons to verify authorizers - Monitor implements AuthServer to allow clients to authenticate with an exchange of msgr2 frames - Monitor implements AuthClient to authenticate with other monitors After this change ProtocolV1 and SimpleMessenger still use all of the old Dispatcher-based callbacks, but ProtocolV2 doesn't need them at all (except for ms_handle_authentication when we finish). Signed-off-by: Sage Weil --- diff --git a/src/auth/Auth.h b/src/auth/Auth.h index 00f852d25f8..465d6d8471a 100644 --- a/src/auth/Auth.h +++ b/src/auth/Auth.h @@ -19,6 +19,7 @@ #include "common/entity_name.h" enum { + AUTH_MODE_NONE = 0, AUTH_MODE_AUTHORIZER = 1, AUTH_MODE_MON = 100, }; @@ -159,7 +160,24 @@ struct AuthConnectionMeta { /// client: initial empty, but populated if server said bad method std::vector allowed_methods; - int auth_mode = 0; ///< server: AUTH_MODE_* + int auth_mode = 0; ///< AUTH_MODE_* + + enum { + CON_MODE_INTEGRITY, // crc: protect against bit errors + CON_MODE_AUTHENTICITY, // secure hash: protect against MITM + CON_MODE_SECRECY, // encrypted + }; + int con_mode = CON_MODE_INTEGRITY; + + bool is_integrity_mode() { + return con_mode == CON_MODE_INTEGRITY; + } + bool is_authenticity_mode() { + return con_mode == CON_MODE_AUTHENTICITY; + } + bool is_secrecy_mode() { + return con_mode == CON_MODE_SECRECY; + } CryptoKey session_key; ///< per-ticket key CryptoKey connection_secret; ///< per-connection key diff --git a/src/mds/MDSDaemon.cc b/src/mds/MDSDaemon.cc index 9080e2949eb..2eaab22781d 100644 --- a/src/mds/MDSDaemon.cc +++ b/src/mds/MDSDaemon.cc @@ -492,6 +492,10 @@ int MDSDaemon::init() return r; } + messenger->set_auth_client(monc); + messenger->set_auth_server(monc); + monc->set_handle_authentication_dispatcher(this); + // tell monc about log_client so it will know about mon session resets monc->set_log_client(&log_client); diff --git a/src/mgr/DaemonServer.cc b/src/mgr/DaemonServer.cc index a05dd37c8be..1298f64f0ac 100644 --- a/src/mgr/DaemonServer.cc +++ b/src/mgr/DaemonServer.cc @@ -111,6 +111,8 @@ int DaemonServer::init(uint64_t gid, entity_addrvec_t client_addrs) getpid(), 0); msgr->set_default_policy(Messenger::Policy::stateless_server(0)); + msgr->set_auth_client(monc); + // throttle clients msgr->set_policy_throttlers(entity_name_t::TYPE_CLIENT, client_byte_throttler.get(), @@ -145,6 +147,9 @@ int DaemonServer::init(uint64_t gid, entity_addrvec_t client_addrs) msgr->start(); msgr->add_dispatcher_tail(this); + msgr->set_auth_server(monc); + monc->set_handle_authentication_dispatcher(this); + started_at = ceph_clock_now(); std::lock_guard l(lock); diff --git a/src/mon/AuthMonitor.h b/src/mon/AuthMonitor.h index ecde5b6b60f..3a34e80f794 100644 --- a/src/mon/AuthMonitor.h +++ b/src/mon/AuthMonitor.h @@ -144,7 +144,9 @@ private: bool prepare_global_id(MonOpRequestRef op); bool should_increase_max_global_id(); void increase_max_global_id(); +public: uint64_t assign_global_id(bool should_increase_max); +private: // propose pending update to peers void encode_pending(MonitorDBStore::TransactionRef t) override; void encode_full(MonitorDBStore::TransactionRef t) override; diff --git a/src/mon/MonClient.cc b/src/mon/MonClient.cc index 6f0dc40f278..d37b9b65719 100644 --- a/src/mon/MonClient.cc +++ b/src/mon/MonClient.cc @@ -50,6 +50,7 @@ MonClient::MonClient(CephContext *cct_) : Dispatcher(cct_), + AuthServer(cct_), messenger(NULL), monc_lock("MonClient::monc_lock"), timer(cct_, monc_lock), @@ -267,6 +268,9 @@ bool MonClient::ms_dispatch(Message *m) case MSG_LOGACK: case MSG_CONFIG: break; + case CEPH_MSG_PING: + m->put(); + return true; default: return false; } @@ -418,6 +422,7 @@ int MonClient::init() ldout(cct, 10) << __func__ << dendl; messenger->add_dispatcher_head(this); + messenger->set_auth_client(this); entity_name = cct->_conf->name; @@ -593,37 +598,7 @@ void MonClient::handle_auth(MAuthReply *m) pending_cons.clear(); } - _finish_hunting(); - - if (!auth_err) { - last_rotating_renew_sent = utime_t(); - while (!waiting_for_session.empty()) { - _send_mon_message(waiting_for_session.front()); - waiting_for_session.pop_front(); - } - _resend_mon_commands(); - send_log(true); - if (active_con) { - std::swap(auth, active_con->get_auth()); - if (global_id && global_id != active_con->get_global_id()) { - lderr(cct) << __func__ << " global_id changed from " << global_id - << " to " << active_con->get_global_id() << dendl; - } - global_id = active_con->get_global_id(); - } - } - _finish_auth(auth_err); - if (!auth_err) { - Context *cb = nullptr; - if (session_established_context) { - cb = session_established_context.release(); - } - if (cb) { - monc_lock.Unlock(); - cb->complete(0); - monc_lock.Lock(); - } - } + _finish_hunting(auth_err); } void MonClient::_finish_auth(int auth_err) @@ -696,7 +671,7 @@ MonConnection& MonClient::_add_conn(unsigned rank, uint64_t global_id) { auto peer = monmap.get_addrs(rank); auto conn = messenger->connect_to_mon(peer); - MonConnection mc(cct, conn, global_id); + MonConnection mc(cct, conn, global_id, auth_supported->get_supported_set()); auto inserted = pending_cons.insert(make_pair(peer, move(mc))); ldout(cct, 10) << "picked mon." << monmap.get_name(rank) << " con " << conn @@ -786,7 +761,7 @@ void MonClient::_start_hunting() } } -void MonClient::_finish_hunting() +void MonClient::_finish_hunting(int auth_err) { ceph_assert(monc_lock.is_locked()); // the pending conns have been cleaned. @@ -802,6 +777,36 @@ void MonClient::_finish_hunting() had_a_connection = true; _un_backoff(); + + if (!auth_err) { + last_rotating_renew_sent = utime_t(); + while (!waiting_for_session.empty()) { + _send_mon_message(waiting_for_session.front()); + waiting_for_session.pop_front(); + } + _resend_mon_commands(); + send_log(true); + if (active_con) { + std::swap(auth, active_con->get_auth()); + if (global_id && global_id != active_con->get_global_id()) { + lderr(cct) << __func__ << " global_id changed from " << global_id + << " to " << active_con->get_global_id() << dendl; + } + global_id = active_con->get_global_id(); + } + } + _finish_auth(auth_err); + if (!auth_err) { + Context *cb = nullptr; + if (session_established_context) { + cb = session_established_context.release(); + } + if (cb) { + monc_lock.Unlock(); + cb->complete(0); + monc_lock.Lock(); + } + } } void MonClient::tick() @@ -1222,6 +1227,192 @@ void MonClient::handle_get_version_reply(MMonGetVersionReply* m) m->put(); } +int MonClient::get_auth_request( + Connection *con, + uint32_t *auth_method, + bufferlist *bl) +{ + std::lock_guard l(monc_lock); + ldout(cct,10) << __func__ << " con " << con << " auth_method " << *auth_method + << dendl; + + // connection to mon? + if (con->get_peer_type() == CEPH_ENTITY_TYPE_MON) { + ceph_assert(!con->get_auth_meta()->authorizer); + for (auto& i : pending_cons) { + if (i.second.is_con(con)) { + return i.second.get_auth_request( + auth_method, bl, + entity_name, want_keys, rotating_secrets.get()); + } + } + return -ENOENT; + } + + // generate authorizer + auto auth_meta = con->get_auth_meta(); + if (!auth) { + lderr(cct) << __func__ << " but no auth handler is set up" << dendl; + return -1; + } + auth_meta->authorizer.reset(auth->build_authorizer(con->get_peer_type())); + auth_meta->auth_method = auth_meta->authorizer->protocol; + *bl = auth_meta->authorizer->bl; + return 0; +} + +int MonClient::handle_auth_reply_more( + Connection *con, + const bufferlist& bl, + bufferlist *reply) +{ + std::lock_guard l(monc_lock); + + if (con->get_peer_type() == CEPH_ENTITY_TYPE_MON) { + for (auto& i : pending_cons) { + if (i.second.is_con(con)) { + return i.second.handle_auth_reply_more(bl, reply); + } + } + return -ENOENT; + } + + // authorizer challenges + auto auth_meta = con->get_auth_meta(); + if (!auth || !auth_meta->authorizer) { + lderr(cct) << __func__ << " no authorizer?" << dendl; + return -1; + } + auth_meta->authorizer->add_challenge(cct, bl); + *reply = auth_meta->authorizer->bl; + return 0; +} + +int MonClient::handle_auth_done( + Connection *con, + uint64_t global_id, + const bufferlist& bl, + CryptoKey *session_key, + CryptoKey *connection_key) +{ + if (con->get_peer_type() == CEPH_ENTITY_TYPE_MON) { + std::lock_guard l(monc_lock); + for (auto& i : pending_cons) { + if (i.second.is_con(con)) { + int r = i.second.handle_auth_done( + global_id, bl, + session_key, connection_key); + if (r) { + pending_cons.erase(i.first); + if (!pending_cons.empty()) { + return r; + } + } else { + active_con.reset(new MonConnection(std::move(i.second))); + pending_cons.clear(); + ceph_assert(active_con->have_session()); + } + + _finish_hunting(r); + return r; + } + } + return -ENOENT; + } else { + // verify authorizer reply + auto auth_meta = con->get_auth_meta(); + auto p = bl.begin(); + if (!auth_meta->authorizer->verify_reply(p, &auth_meta->connection_secret)) { + ldout(cct, 0) << __func__ << " failed verifying authorizer reply" + << dendl; + return -EACCES; + } + auth_meta->session_key = auth_meta->authorizer->session_key; + return 0; + } +} + +int MonClient::handle_auth_bad_method( + Connection *con, + uint32_t old_auth_method, + int result, + const std::vector& allowed_methods) +{ + con->get_auth_meta()->allowed_methods = allowed_methods; + + std::lock_guard l(monc_lock); + if (con->get_peer_type() == CEPH_ENTITY_TYPE_MON) { + for (auto& i : pending_cons) { + if (i.second.is_con(con)) { + int r = i.second.handle_auth_bad_method(old_auth_method, + result, + allowed_methods); + if (r == 0) { + return r; // try another method on this con + } + pending_cons.erase(i.first); + if (!pending_cons.empty()) { + return r; // fail this con, maybe another con will succeed + } + // fail hunt + _finish_hunting(r); + return r; + } + } + return -ENOENT; + } else { + // huh... + ldout(cct,10) << __func__ << " hmm, they didn't like " << old_auth_method + << " result " << cpp_strerror(result) + << " and auth is " << (auth ? auth->get_protocol() : 0) + << dendl; + return -EACCES; + } +} + +int MonClient::handle_auth_request( + Connection *con, + bool more, + uint32_t auth_method, + const bufferlist& payload, + bufferlist *reply) +{ + auto auth_meta = con->get_auth_meta(); + auth_meta->auth_mode = payload[0]; + if (auth_meta->auth_mode != AUTH_MODE_AUTHORIZER) { + return -EACCES; + } + AuthAuthorizeHandler *ah = get_auth_authorize_handler(con->get_peer_type(), + auth_method); + if (!ah) { + lderr(cct) << __func__ << " no AuthAuthorizeHandler found for auth method " + << auth_method << dendl; + return -EOPNOTSUPP; + } + bool was_challenge = (bool)auth_meta->authorizer_challenge; + bool isvalid = ah->verify_authorizer( + cct, + rotating_secrets.get(), + payload, + reply, + &con->peer_name, + &con->peer_global_id, + &con->peer_caps_info, + &auth_meta->session_key, + &auth_meta->connection_secret, + &auth_meta->authorizer_challenge); + if (isvalid) { + handle_authentication_dispatcher->ms_handle_authentication(con); + return 1; + } + if (!more && !was_challenge && auth_meta->authorizer_challenge) { + ldout(cct,10) << __func__ << " added challenge on " << con << dendl; + return 0; + } + ldout(cct,10) << __func__ << " bad authorizer on " << con << dendl; + return -EACCES; +} + AuthAuthorizer* MonClient::build_authorizer(int service_id) const { std::lock_guard l(monc_lock); if (auth) { @@ -1237,8 +1428,10 @@ AuthAuthorizer* MonClient::build_authorizer(int service_id) const { #undef dout_prefix #define dout_prefix *_dout << "monclient" << (have_session() ? ": " : "(hunting): ") -MonConnection::MonConnection(CephContext *cct, ConnectionRef con, uint64_t global_id) - : cct(cct), con(con), global_id(global_id) +MonConnection::MonConnection( + CephContext *cct, ConnectionRef con, uint64_t global_id, + const list& auth_supported) + : cct(cct), con(con), global_id(global_id), auth_supported(auth_supported) {} MonConnection::~MonConnection() @@ -1258,6 +1451,13 @@ void MonConnection::start(epoch_t epoch, const EntityName& entity_name, const AuthMethodList& auth_supported) { + if (con->get_peer_addr().is_msgr2()) { + ldout(cct, 10) << __func__ << " opening mon connection" << dendl; + state = State::AUTHENTICATING; + con->send_message(new MPing()); + return; + } + // restart authentication handshake state = State::NEGOTIATING; @@ -1277,6 +1477,116 @@ void MonConnection::start(epoch_t epoch, con->send_message(m); } +int MonConnection::get_auth_request( + uint32_t *method, bufferlist *bl, + const EntityName& entity_name, + uint32_t want_keys, + RotatingKeyRing* keyring) +{ + // choose method + if (auth_method < 0) { + auth_method = auth_supported.front(); + } + *method = auth_method; + ldout(cct,10) << __func__ << " method " << *method << dendl; + + if (auth) { + auth.reset(); + } + int r = _init_auth(*method, entity_name, want_keys, keyring, true); + ceph_assert(r == 0); + + // initial requset includes some boilerplate... + encode((char)AUTH_MODE_MON, *bl); + encode(entity_name, *bl); + encode(global_id, *bl); + + // and (maybe) some method-specific initial payload + auth->build_initial_request(bl); + + return 0; +} + +int MonConnection::handle_auth_reply_more( + const bufferlist& bl, + bufferlist *reply) +{ + ldout(cct, 10) << __func__ << " payload " << bl.length() << dendl; + ldout(cct, 30) << __func__ << " got\n"; + bl.hexdump(*_dout); + *_dout << dendl; + + auto p = bl.cbegin(); + ldout(cct, 10) << __func__ << " payload_len " << bl.length() << dendl; + auto auth_meta = con->get_auth_meta(); + int r = auth->handle_response(0, p, &auth_meta->session_key, + &auth_meta->connection_secret); + if (r == -EAGAIN) { + auth->prepare_build_request(); + auth->build_request(*reply); + ldout(cct, 10) << __func__ << " responding with " << reply->length() + << " bytes" << dendl; + r = 0; + } else if (r < 0) { + lderr(cct) << __func__ << " handle_response returned " << r << dendl; + } else { + ldout(cct, 10) << __func__ << " authenticated!" << dendl; + // FIXME + ceph_abort(cct, "write me"); + } + return r; +} + +int MonConnection::handle_auth_done( + uint64_t new_global_id, + const bufferlist& bl, + CryptoKey *session_key, + CryptoKey *connection_secret) +{ + ldout(cct,10) << __func__ << " global_id " << new_global_id + << " payload " << bl.length() + << dendl; + global_id = new_global_id; + auth->set_global_id(global_id); + auto p = bl.begin(); + auto auth_meta = con->get_auth_meta(); + int auth_err = auth->handle_response(0, p, &auth_meta->session_key, + &auth_meta->connection_secret); + if (auth_err >= 0) { + state = State::HAVE_SESSION; + } + return auth_err; +} + +int MonConnection::handle_auth_bad_method( + uint32_t old_auth_method, + int result, + const std::vector& allowed_methods) +{ + ldout(cct,10) << __func__ << " old_auth_method " << old_auth_method + << " result " << cpp_strerror(result) + << " allowed_methods " << allowed_methods << dendl; + auto p = std::find(auth_supported.begin(), auth_supported.end(), + old_auth_method); + assert(p != auth_supported.end()); + + while (p != auth_supported.end()) { + ++p; + if (std::find(allowed_methods.begin(), allowed_methods.end(), *p) != + allowed_methods.end()) { + break; + } + } + if (p == auth_supported.end()) { + lderr(cct) << __func__ << " server allowed_methods " << allowed_methods + << " but i only support " << auth_supported << dendl; + return -EACCES; + } + auth_method = *p; + ldout(cct,10) << __func__ << " will try " << auth_method << " next" << dendl; + return 0; +} + int MonConnection::handle_auth(MAuthReply* m, const EntityName& entity_name, uint32_t want_keys, @@ -1307,23 +1617,39 @@ int MonConnection::_negotiate(MAuthReply *m, return 0; } - auth.reset( - AuthClientHandler::create(cct,m->protocol, keyring)); - if (!auth) { - ldout(cct, 10) << "no handler for protocol " << m->protocol << dendl; + int r = _init_auth(m->protocol, entity_name, want_keys, keyring, false); + if (r == -ENOTSUP) { if (m->result == -ENOTSUP) { ldout(cct, 10) << "none of our auth protocols are supported by the server" << dendl; } return m->result; } + return r; +} + +int MonConnection::_init_auth( + uint32_t method, + const EntityName& entity_name, + uint32_t want_keys, + RotatingKeyRing* keyring, + bool msgr2) +{ + ldout(cct,10) << __func__ << " method " << method << dendl; + auth.reset( + AuthClientHandler::create(cct, method, keyring)); + if (!auth) { + ldout(cct, 10) << " no handler for protocol " << method << dendl; + return -ENOTSUP; + } // do not request MGR key unless the mon has the SERVER_KRAKEN // feature. otherwise it will give us an auth error. note that // we have to use the FEATUREMASK because pre-jewel the kraken // feature bit was used for something else. - if ((want_keys & CEPH_ENTITY_TYPE_MGR) && - !(m->get_connection()->has_features(CEPH_FEATUREMASK_SERVER_KRAKEN))) { + if (!msgr2 && + (want_keys & CEPH_ENTITY_TYPE_MGR) && + !(con->has_features(CEPH_FEATUREMASK_SERVER_KRAKEN))) { ldout(cct, 1) << __func__ << " not requesting MGR keys from pre-kraken monitor" << dendl; diff --git a/src/mon/MonClient.h b/src/mon/MonClient.h index e66e5cdbf1b..0f4f21c2161 100644 --- a/src/mon/MonClient.h +++ b/src/mon/MonClient.h @@ -25,6 +25,9 @@ #include "common/Finisher.h" #include "common/config.h" +#include "auth/AuthClient.h" +#include "auth/AuthServer.h" + class MMonMap; class MConfig; class MMonGetVersionReply; @@ -98,7 +101,8 @@ class MonConnection { public: MonConnection(CephContext *cct, ConnectionRef conn, - uint64_t global_id); + uint64_t global_id, + const list& auto_supported); ~MonConnection(); MonConnection(MonConnection&& rhs) = default; MonConnection& operator=(MonConnection&&) = default; @@ -123,31 +127,63 @@ public: return auth; } + int get_auth_request( + uint32_t *method, bufferlist *out, + const EntityName& entity_name, + uint32_t want_keys, + RotatingKeyRing* keyring); + int handle_auth_reply_more( + const bufferlist& bl, + bufferlist *reply); + int handle_auth_done( + uint64_t global_id, + const bufferlist& bl, + CryptoKey *session_key, + CryptoKey *connection_secret); + int handle_auth_bad_method( + uint32_t old_auth_method, + int result, + const std::vector& allowed_methods); + + bool is_con(Connection *c) const { + return con.get() == c; + } + private: int _negotiate(MAuthReply *m, const EntityName& entity_name, uint32_t want_keys, RotatingKeyRing* keyring); + int _init_auth(uint32_t method, + const EntityName& entity_name, + uint32_t want_keys, + RotatingKeyRing* keyring, + bool msgr2); private: CephContext *cct; enum class State { NONE, - NEGOTIATING, - AUTHENTICATING, + NEGOTIATING, // v1 only + AUTHENTICATING, // v1 and v2 HAVE_SESSION, }; State state = State::NONE; ConnectionRef con; + int auth_method = -1; std::unique_ptr auth; uint64_t global_id; + const list& auth_supported; }; -class MonClient : public Dispatcher { +class MonClient : public Dispatcher, + public AuthClient, + public AuthServer /* for mgr, osd, mds */ { public: MonMap monmap; map config_mgr; + private: Messenger *messenger; @@ -207,11 +243,13 @@ private: std::unique_ptr session_established_context; bool had_a_connection; double reopen_interval_multiplier; + + Dispatcher *handle_authentication_dispatcher = nullptr; bool _opened() const; bool _hunting() const; void _start_hunting(); - void _finish_hunting(); + void _finish_hunting(int auth_err); void _finish_auth(int auth_err); void _reopen_session(int rank = -1); MonConnection& _add_conn(unsigned rank, uint64_t global_id); @@ -230,8 +268,38 @@ private: } public: - void set_entity_name(EntityName name) { entity_name = name; } + // AuthClient + int get_auth_request( + Connection *con, + uint32_t *method, + bufferlist *bl) override; + int handle_auth_reply_more( + Connection *con, + const bufferlist& bl, + bufferlist *reply) override; + int handle_auth_done( + Connection *con, + uint64_t global_id, + const bufferlist& bl, + CryptoKey *session_key, + CryptoKey *connection_key) override; + int handle_auth_bad_method( + Connection *con, + uint32_t old_auth_method, + int result, + const std::vector& allowed_methods) override; + // AuthServer + int handle_auth_request( + Connection *con, + bool more, + uint32_t auth_method, + const bufferlist& bl, + bufferlist *reply); + void set_entity_name(EntityName name) { entity_name = name; } + void set_handle_authentication_dispatcher(Dispatcher *d) { + handle_authentication_dispatcher = d; + } int _check_auth_tickets(); int _check_auth_rotating(); int wait_auth_rotating(double timeout); diff --git a/src/mon/Monitor.cc b/src/mon/Monitor.cc index 09acf7dc46e..7eec0cc45b1 100644 --- a/src/mon/Monitor.cc +++ b/src/mon/Monitor.cc @@ -121,6 +121,7 @@ MonCommand mon_commands[] = { #undef COMMAND_WITH_FLAG + void C_MonContext::finish(int r) { if (mon->is_shutdown()) return; @@ -130,6 +131,7 @@ void C_MonContext::finish(int r) { Monitor::Monitor(CephContext* cct_, string nm, MonitorDBStore *s, Messenger *m, Messenger *mgr_m, MonMap *map) : Dispatcher(cct_), + AuthServer(cct_), name(nm), rank(-1), messenger(m), @@ -873,10 +875,13 @@ int Monitor::init() // i'm ready! messenger->add_dispatcher_tail(this); + messenger->set_auth_client(this); + messenger->set_auth_server(this); mgr_client.init(); mgr_messenger->add_dispatcher_tail(&mgr_client); mgr_messenger->add_dispatcher_tail(this); // for auth ms_* calls + mgr_messenger->set_auth_client(this); bootstrap(); // add features of myself into feature_map @@ -5109,7 +5114,7 @@ bool Monitor::ms_handle_reset(Connection *con) std::lock_guard l(lock); dout(10) << "reset/close on session " << s->name << " " << s->addrs << dendl; - if (!s->closed) { + if (!s->closed && s->item.is_on_list()) { std::lock_guard l(session_map_lock); remove_session(s); } @@ -5857,6 +5862,66 @@ void Monitor::extract_save_mon_key(KeyRing& keyring) } } +// AuthClient methods -- for mon <-> mon communication +int Monitor::get_auth_request( + Connection *con, + uint32_t *method, bufferlist *out) +{ + AuthAuthorizer *auth; + if (!ms_get_authorizer(con->get_peer_type(), &auth)) { + return -EACCES; + } + auto auth_meta = con->get_auth_meta(); + auth_meta->authorizer.reset(auth); + *method = auth->protocol; + *out = auth->bl; + return 0; +} + +int Monitor::handle_auth_reply_more( + Connection *con, + const bufferlist& bl, + bufferlist *reply) +{ + auto auth_meta = con->get_auth_meta(); + if (!auth_meta->authorizer) { + derr << __func__ << " no authorizer?" << dendl; + return -EACCES; + } + auth_meta->authorizer->add_challenge(cct, bl); + *reply = auth_meta->authorizer->bl; + return 0; +} + +int Monitor::handle_auth_done( + Connection *con, + uint64_t global_id, + const bufferlist& bl, + CryptoKey *session_key, + CryptoKey *connection_key) +{ + // verify authorizer reply + auto auth_meta = con->get_auth_meta(); + auto p = bl.begin(); + if (!auth_meta->authorizer->verify_reply(p, &auth_meta->connection_secret)) { + dout(0) << __func__ << " failed verifying authorizer reply" << dendl; + return -EACCES; + } + auth_meta->session_key = auth_meta->authorizer->session_key; + return 0; +} + +int Monitor::handle_auth_bad_method( + Connection *con, + uint32_t old_auth_method, + int result, + const std::vector& allowed_methods) +{ + derr << __func__ << " hmm, they didn't like " << old_auth_method + << " result " << cpp_strerror(result) << dendl; + return -EACCES; +} + bool Monitor::ms_get_authorizer(int service_id, AuthAuthorizer **authorizer) { dout(10) << "ms_get_authorizer for " << ceph_entity_type_name(service_id) @@ -5950,6 +6015,172 @@ KeyStore *Monitor::ms_get_auth1_authorizer_keystore() return &keyring; } +int Monitor::handle_auth_request( + Connection *con, + bool more, + uint32_t auth_method, + const bufferlist &payload, + bufferlist *reply) +{ + // NOTE: be careful, the Connection hasn't fully negotiated yet, so + // e.g., peer_features, peer_addrs, and others are still unknown. + + dout(10) << __func__ << " con " << con << (more ? " (more)":" (start)") + << " method " << auth_method + << " payload " << payload.length() + << dendl; + auto auth_meta = con->get_auth_meta(); + if (!more) { + auth_meta->auth_mode = payload[0]; + } + + if (auth_meta->auth_mode == AUTH_MODE_AUTHORIZER) { + AuthAuthorizeHandler *ah = get_auth_authorize_handler(con->get_peer_type(), + auth_method); + if (!ah) { + lderr(cct) << __func__ << " no AuthAuthorizeHandler found for auth method " + << auth_method << dendl; + return -EOPNOTSUPP; + } + bool was_challenge = (bool)auth_meta->authorizer_challenge; + bool isvalid = ah->verify_authorizer( + cct, + &keyring, + payload, + reply, + &con->peer_name, + &con->peer_global_id, + &con->peer_caps_info, + &auth_meta->session_key, + &auth_meta->connection_secret, + &auth_meta->authorizer_challenge); + if (isvalid) { + ms_handle_authentication(con); + return 1; + } + if (!more && !was_challenge && auth_meta->authorizer_challenge) { + return 0; + } + dout(10) << __func__ << " bad authorizer on " << con << dendl; + return -EACCES; + } else if (auth_meta->auth_mode != AUTH_MODE_MON) { + derr << __func__ << " unrecognized auth mode " << auth_meta->auth_mode + << dendl; + return -EACCES; + } + + RefCountedPtr priv; + MonSession *s; + int32_t r = 0; + auto p = payload.begin(); + if (!more) { + if (con->get_priv()) { + return -EACCES; // wtf + } + + // handler? + AuthServiceHandler *auth_handler = get_auth_service_handler( + auth_method, g_ceph_context, &key_server); + if (!auth_handler) { + dout(1) << __func__ << " auth_method " << auth_method << " not supported" + << dendl; + return -EOPNOTSUPP; + } + + uint8_t mode; + EntityName entity_name; + + decode(mode, p); + assert(mode == AUTH_MODE_MON); + decode(entity_name, p); + decode(con->peer_global_id, p); + + // supported method? + if (entity_name.get_type() == CEPH_ENTITY_TYPE_MON || + entity_name.get_type() == CEPH_ENTITY_TYPE_OSD || + entity_name.get_type() == CEPH_ENTITY_TYPE_MDS || + entity_name.get_type() == CEPH_ENTITY_TYPE_MGR) { + if (!auth_cluster_required.is_supported_auth(auth_method)) { + dout(10) << __func__ << " entity " << entity_name << " method " + << auth_method << " not among supported " + << auth_cluster_required.get_supported_set() << dendl; + return -EOPNOTSUPP; + } + } else { + if (!auth_service_required.is_supported_auth(auth_method)) { + dout(10) << __func__ << " entity " << entity_name << " method " + << auth_method << " not among supported " + << auth_cluster_required.get_supported_set() << dendl; + return -EOPNOTSUPP; + } + } + + // for msgr1 we would do some weirdness here to ensure signatures + // are supported by the client if we require it. for msgr2 that + // is not necessary. + + if (!con->peer_global_id) { + con->peer_global_id = authmon()->assign_global_id(false); + if (!con->peer_global_id) { + dout(1) << __func__ << " failed to assign global_id" << dendl; + return -EBUSY; + } + dout(10) << __func__ << " assigned global_id " << con->peer_global_id + << dendl; + } + + // set up partial session + s = new MonSession(con); + s->auth_handler = auth_handler; + con->set_priv(RefCountedPtr{s, false}); + + r = s->auth_handler->start_session(entity_name, reply, + &con->peer_caps_info, + &auth_meta->session_key, + &auth_meta->connection_secret); + } else { + priv = con->get_priv(); + s = static_cast(priv.get()); + r = s->auth_handler->handle_request(p, reply, &con->peer_global_id, + &con->peer_caps_info, + &auth_meta->session_key, + &auth_meta->connection_secret); + } + if (r > 0 && + !s->authenticated) { + ms_handle_authentication(con); + } + + dout(30) << " r " << r << " reply:\n"; + reply->hexdump(*_dout); + *_dout << dendl; + return r; +} + +void Monitor::ms_handle_accept(Connection *con) +{ + auto priv = con->get_priv(); + MonSession *s = static_cast(priv.get()); + if (!s) { + // legacy protocol v1? + dout(10) << __func__ << " con " << con << " no session" << dendl; + return; + } + + if (s->item.is_on_list()) { + dout(10) << __func__ << " con " << con << " session " << s + << " already on list" << dendl; + } else { + dout(10) << __func__ << " con " << con << " session " << s + << " registering session for " + << con->get_peer_addrs() << dendl; + s->_ident(entity_name_t(con->get_peer_type(), con->get_peer_id()), + con->get_peer_addrs()); + std::lock_guard l(session_map_lock); + session_map.add_session(s); + } +} + int Monitor::ms_handle_authentication(Connection *con) { if (con->get_peer_type() == CEPH_ENTITY_TYPE_MON) { diff --git a/src/mon/Monitor.h b/src/mon/Monitor.h index 4528b651b44..7aa912c8daf 100644 --- a/src/mon/Monitor.h +++ b/src/mon/Monitor.h @@ -43,6 +43,8 @@ #include "common/config_obs.h" #include "common/LogClient.h" +#include "auth/AuthClient.h" +#include "auth/AuthServer.h" #include "auth/cephx/CephxKeyServer.h" #include "auth/AuthMethodList.h" #include "auth/KeyRing.h" @@ -123,6 +125,8 @@ public: }; class Monitor : public Dispatcher, + public AuthClient, + public AuthServer, public md_config_obs_t { public: int orig_argc = 0; @@ -897,10 +901,40 @@ public: public: // for AuthMonitor msgr1: int ms_handle_authentication(Connection *con) override; private: + void ms_handle_accept(Connection *con) override; bool ms_handle_reset(Connection *con) override; void ms_handle_remote_reset(Connection *con) override {} bool ms_handle_refused(Connection *con) override; + // AuthClient + int get_auth_request( + Connection *con, + uint32_t *method, bufferlist *out) override; + int handle_auth_reply_more( + Connection *con, + const bufferlist& bl, + bufferlist *reply) override; + int handle_auth_done( + Connection *con, + uint64_t global_id, + const bufferlist& bl, + CryptoKey *session_key, + CryptoKey *connection_key) override; + int handle_auth_bad_method( + Connection *con, + uint32_t old_auth_method, + int result, + const std::vector& allowed_methods) override; + // /AuthClient + // AuthServer + int handle_auth_request( + Connection *con, + bool more, + uint32_t auth_method, + const bufferlist& bl, + bufferlist *reply) override; + // /AuthServer + int write_default_keyring(bufferlist& bl); void extract_save_mon_key(KeyRing& keyring); diff --git a/src/msg/async/ProtocolV2.cc b/src/msg/async/ProtocolV2.cc index 1cfd1b5bf87..8e0c979fc09 100644 --- a/src/msg/async/ProtocolV2.cc +++ b/src/msg/async/ProtocolV2.cc @@ -8,6 +8,8 @@ #include "common/ceph_crypto.h" #include "common/errno.h" #include "include/random.h" +#include "auth/AuthClient.h" +#include "auth/AuthServer.h" #define dout_subsys ceph_subsys_ms #undef dout_prefix @@ -44,13 +46,6 @@ struct DecryptionError : public std::exception {}; void ProtocolV2::get_auth_allowed_methods( int peer_type, std::vector &allowed_methods) { - // FIXME: this is for legacy MAuth-based authentication - if (messenger->get_mytype() == CEPH_ENTITY_TYPE_MON && - peer_type != CEPH_ENTITY_TYPE_MON) { - allowed_methods.push_back(CEPH_AUTH_NONE); - return; - } - std::string method; if (!cct->_conf->auth_supported.empty()) { method = cct->_conf->auth_supported; @@ -260,58 +255,50 @@ struct HelloFrame : public PayloadFrame { + : public PayloadFrame { const ProtocolV2::Tag tag = ProtocolV2::Tag::AUTH_REQUEST; using PayloadFrame::PayloadFrame; - AuthRequestFrame(uint32_t method) - : AuthRequestFrame(method, 0, bufferlist()) {} + AuthRequestFrame(uint32_t method) : AuthRequestFrame(method, bufferlist()) {} inline uint32_t &method() { return get_val<0>(); } - inline bufferlist &auth_payload() { return get_val<2>(); } + inline bufferlist &auth_payload() { return get_val<1>(); } }; struct AuthBadMethodFrame - : public PayloadFrame> { + : public PayloadFrame> { const ProtocolV2::Tag tag = ProtocolV2::Tag::AUTH_BAD_METHOD; using PayloadFrame::PayloadFrame; inline uint32_t &method() { return get_val<0>(); } - inline std::vector &allowed_methods() { return get_val<1>(); } -}; - -struct AuthBadAuthFrame - : public PayloadFrame { - const ProtocolV2::Tag tag = ProtocolV2::Tag::AUTH_BAD_AUTH; - using PayloadFrame::PayloadFrame; - - inline uint32_t &error_code() { return get_val<0>(); } - inline std::string &error_msg() { return get_val<1>(); } + inline int32_t &result() { return get_val<1>(); } + inline std::vector &allowed_methods() { return get_val<2>(); } }; struct AuthReplyMoreFrame - : public PayloadFrame { + : public PayloadFrame { const ProtocolV2::Tag tag = ProtocolV2::Tag::AUTH_REPLY_MORE; using PayloadFrame::PayloadFrame; - inline bufferlist &auth_payload() { return get_val<1>(); } + inline bufferlist &auth_payload() { return get_val<0>(); } }; struct AuthRequestMoreFrame - : public PayloadFrame { + : public PayloadFrame { const ProtocolV2::Tag tag = ProtocolV2::Tag::AUTH_REQUEST_MORE; using PayloadFrame::PayloadFrame; - inline bufferlist &auth_payload() { return get_val<1>(); } + inline bufferlist &auth_payload() { return get_val<0>(); } }; struct AuthDoneFrame - : public PayloadFrame { + : public PayloadFrame { const ProtocolV2::Tag tag = ProtocolV2::Tag::AUTH_DONE; using PayloadFrame::PayloadFrame; - inline uint64_t &flags() { return get_val<0>(); } - inline bufferlist &auth_payload() { return get_val<2>(); } + inline uint64_t &global_id() { return get_val<0>(); } + inline bufferlist &auth_payload() { return get_val<1>(); } }; template @@ -463,9 +450,6 @@ ProtocolV2::ProtocolV2(AsyncConnection *connection) temp_buffer(nullptr), state(NONE), peer_required_features(0), - authorizer(nullptr), - got_bad_method(0), - auth_flags(0), cookie(0), global_seq(0), connect_seq(0), @@ -482,20 +466,11 @@ ProtocolV2::ProtocolV2(AsyncConnection *connection) ProtocolV2::~ProtocolV2() { delete[] temp_buffer; - if (authorizer) { - delete authorizer; - } } void ProtocolV2::connect() { ldout(cct, 1) << __func__ << dendl; state = START_CONNECT; - - got_bad_method = 0; - if (authorizer) { - delete authorizer; - authorizer = nullptr; - } } void ProtocolV2::accept() { @@ -615,11 +590,7 @@ uint64_t ProtocolV2::discard_requeued_up_to(uint64_t out_seq, uint64_t seq) { void ProtocolV2::reset_recv_state() { if (state == CONNECTING) { - if (authorizer) { - delete authorizer; - } - authorizer = nullptr; - got_bad_method = 0; + auth_meta.authorizer.reset(nullptr); } // clean read and write callbacks @@ -1089,7 +1060,7 @@ bool ProtocolV2::is_queued() { void ProtocolV2::sign_payload(bufferlist &payload) { ldout(cct, 21) << __func__ << " len=" << payload.length() << dendl; - if (sign_frames() && session_security) { + if (auth_meta.is_authenticity_mode() && session_security) { uint32_t pad_len; calculate_payload_size(payload.length(), nullptr, &pad_len); auto padding = bufferptr(buffer::create(pad_len)); @@ -1115,7 +1086,7 @@ void ProtocolV2::sign_payload(bufferlist &payload) { void ProtocolV2::verify_signature(char *payload, uint32_t length) { ldout(cct, 21) << __func__ << " len=" << length << dendl; - if (sign_frames() && session_security) { + if (auth_meta.is_authenticity_mode() && session_security) { uint32_t payload_len = length - CEPH_CRYPTO_HMACSHA256_DIGESTSIZE; const char *p = payload + payload_len; char signature[CEPH_CRYPTO_HMACSHA256_DIGESTSIZE]; @@ -1132,7 +1103,7 @@ void ProtocolV2::verify_signature(char *payload, uint32_t length) { void ProtocolV2::encrypt_payload(bufferlist &payload) { ldout(cct, 21) << __func__ << " len=" << payload.length() << dendl; - if (encrypt_frames() && session_security) { + if (auth_meta.is_secrecy_mode() && session_security) { uint32_t pad_len; calculate_payload_size(payload.length(), nullptr, nullptr, &pad_len); if (pad_len) { @@ -1149,7 +1120,7 @@ void ProtocolV2::encrypt_payload(bufferlist &payload) { void ProtocolV2::decrypt_payload(char *payload, uint32_t &length) { ldout(cct, 21) << __func__ << " len=" << length << dendl; - if (encrypt_frames() && session_security) { + if (auth_meta.is_secrecy_mode() && session_security) { bufferlist in; in.push_back(buffer::create_static(length, payload)); bufferlist out; @@ -1165,8 +1136,8 @@ void ProtocolV2::decrypt_payload(char *payload, uint32_t &length) { void ProtocolV2::calculate_payload_size(uint32_t length, uint32_t *total_len, uint32_t *sig_pad_len, uint32_t *enc_pad_len) { - bool is_signed = sign_frames(); - bool is_encrypted = encrypt_frames(); + bool is_signed = auth_meta.is_authenticity_mode(); + bool is_encrypted = auth_meta.is_secrecy_mode(); uint32_t sig_pad_l = 0; uint32_t enc_pad_l = 0; @@ -1448,7 +1419,6 @@ CtPtr ProtocolV2::handle_read_frame_length_and_tag(char *buffer, int r) { case Tag::HELLO: case Tag::AUTH_REQUEST: case Tag::AUTH_BAD_METHOD: - case Tag::AUTH_BAD_AUTH: case Tag::AUTH_REPLY_MORE: case Tag::AUTH_REQUEST_MORE: case Tag::AUTH_DONE: @@ -1472,7 +1442,7 @@ CtPtr ProtocolV2::handle_read_frame_length_and_tag(char *buffer, int r) { lderr(cct) << __func__ << " received unknown tag=" << static_cast(next_tag) << dendl; - ceph_abort(); + return _fault(); } } @@ -1488,6 +1458,12 @@ CtPtr ProtocolV2::handle_frame_payload(char *buffer, int r) { return _fault(); } + ldout(cct, 30) << __func__ << "\n"; + bufferlist bl; + bl.append(buffer, next_payload_len); + bl.hexdump(*_dout); + *_dout << dendl; + switch (next_tag) { case Tag::HELLO: return handle_hello(buffer, next_payload_len); @@ -1495,8 +1471,6 @@ CtPtr ProtocolV2::handle_frame_payload(char *buffer, int r) { return handle_auth_request(buffer, next_payload_len); case Tag::AUTH_BAD_METHOD: return handle_auth_bad_method(buffer, next_payload_len); - case Tag::AUTH_BAD_AUTH: - return handle_auth_bad_auth(buffer, next_payload_len); case Tag::AUTH_REPLY_MORE: return handle_auth_reply_more(buffer, next_payload_len); case Tag::AUTH_REQUEST_MORE: @@ -1827,7 +1801,8 @@ CtPtr ProtocolV2::read_message_data() { // the message payload ldout(cct, 1) << __func__ << " reading message payload extra bytes left=" << next_payload_len << dendl; - ceph_assert(session_security && (sign_frames() || encrypt_frames())); + ceph_assert(session_security && (auth_meta.is_authenticity_mode() || + auth_meta.is_secrecy_mode())); extra.push_back(buffer::create(next_payload_len)); return READB(next_payload_len, extra.c_str(), handle_message_extra_bytes); } @@ -1889,7 +1864,7 @@ CtPtr ProtocolV2::handle_message_complete() { ceph_msg_footer footer{current_header.front_crc, current_header.middle_crc, current_header.data_crc, 0, current_header.flags}; - if (sign_frames() || encrypt_frames()) { + if (auth_meta.is_authenticity_mode() || auth_meta.is_secrecy_mode()) { bufferlist msg_payload; msg_payload.claim_append(front); msg_payload.claim_append(middle); @@ -2109,138 +2084,101 @@ CtPtr ProtocolV2::post_client_banner_exchange() { } CtPtr ProtocolV2::send_auth_request(std::vector &allowed_methods) { - ldout(cct, 20) << __func__ << dendl; + ldout(cct, 20) << __func__ << " peer_type " << (int)connection->peer_type + << " auth_client " << messenger->auth_client << dendl; + ceph_assert(messenger->auth_client); - if (!authorizer) { - authorizer = - messenger->ms_deliver_get_authorizer(connection->peer_type); + bufferlist bl; + connection->lock.unlock(); + int r = messenger->auth_client->get_auth_request( + connection, &auth_meta.auth_method, &bl); + connection->lock.lock(); + if (state != State::CONNECTING) { + return _fault(); } - - auth_method = CEPH_AUTH_NONE; - if (!authorizer) { - if (!allowed_methods.empty() && - std::find(allowed_methods.begin(), allowed_methods.end(), - auth_method) == allowed_methods.end()) { - ldout(cct, 0) << __func__ - << " peer requires authentication, stopping connection" - << dendl; - stop(); - connection->dispatch_queue->queue_reset(connection); - return nullptr; - } - ldout(cct, 10) << __func__ << " authorizer not found for peer_type=" - << connection->peer_type << dendl; - AuthRequestFrame authFrame(auth_method); - bufferlist &bl = authFrame.get_buffer(); - return WRITE(bl, "auth request", read_frame); - } - - auth_method = authorizer->protocol; - if (!allowed_methods.empty() && - std::find(allowed_methods.begin(), allowed_methods.end(), auth_method) == - allowed_methods.end()) { - ldout(cct, 0) << __func__ << " peer does not allow authentication method=" - << auth_method << dendl; + if (r < 0) { + ldout(cct, 0) << __func__ << " get_initial_auth_request returned " << r + << dendl; stop(); connection->dispatch_queue->queue_reset(connection); return nullptr; } - - ldout(cct, 10) << __func__ << " sending auth request method=" << auth_method - << " len=" << authorizer->bl.length() << dendl; - - // we need to copy authorizer->bl because we might need it again in a - // reconnect - bufferlist auth_blob; - auth_blob.append(authorizer->bl); - AuthRequestFrame authFrame(auth_method, authorizer->bl.length(), - auth_blob); - bufferlist &bl = authFrame.get_buffer(); - return WRITE(bl, "auth request", read_frame); + AuthRequestFrame frame(auth_meta.auth_method, bl); + return WRITE(frame.get_buffer(), "auth request", read_frame); } CtPtr ProtocolV2::handle_auth_bad_method(char *payload, uint32_t length) { ldout(cct, 20) << __func__ << " payload_len=" << length << dendl; AuthBadMethodFrame bad_method(payload, length); - ldout(cct, 1) << __func__ << " auth method=" << bad_method.method() - << " rejected, allowed methods=" << bad_method.allowed_methods() + ldout(cct, 1) << __func__ << " method=" << bad_method.method() + << " result " << cpp_strerror(bad_method.result()) + << ", allowed methods=" << bad_method.allowed_methods() << dendl; - - if (got_bad_method == bad_method.allowed_methods().size()) { - ldout(cct, 1) << __func__ << " too many attempts, closing connection" - << dendl; + ceph_assert(messenger->auth_client); + connection->lock.unlock(); + int r = messenger->auth_client->handle_auth_bad_method( + connection, bad_method.method(), bad_method.result(), + bad_method.allowed_methods()); + connection->lock.lock(); + if (state != State::CONNECTING || r < 0) { return _fault(); } - got_bad_method++; - return send_auth_request(bad_method.allowed_methods()); } -CtPtr ProtocolV2::handle_auth_bad_auth(char *payload, uint32_t length) { - ldout(cct, 20) << __func__ << " payload_len=" << length << dendl; - - AuthBadAuthFrame bad_auth(payload, length); - ldout(cct, 1) << __func__ << " authentication failed" - << " error code=" << bad_auth.error_code() - << " error message=" << bad_auth.error_msg() << dendl; - - return _fault(); -} - CtPtr ProtocolV2::handle_auth_reply_more(char *payload, uint32_t length) { ldout(cct, 20) << __func__ << " payload_len=" << length << dendl; - AuthReplyMoreFrame auth_more(payload, length); ldout(cct, 5) << __func__ << " auth reply more len=" << auth_more.auth_payload().length() << dendl; - ldout(cct, 10) << __func__ << " connect got auth challenge" << dendl; - if (auth_method == CEPH_AUTH_CEPHX) { - ceph_assert(authorizer); - authorizer->add_challenge(cct, auth_more.auth_payload()); - AuthRequestMoreFrame more_reply(authorizer->bl.length(), authorizer->bl); - return WRITE(more_reply.get_buffer(), "auth request more", read_frame); - } else { - ceph_abort("Auth method %d not implemented", auth_method); + ceph_assert(messenger->auth_client); + bufferlist bl; + bl.append(payload, length); + bufferlist reply; + connection->lock.unlock(); + int r = messenger->auth_client->handle_auth_reply_more( + connection, auth_more.auth_payload(), &reply); + connection->lock.lock(); + if (state != State::CONNECTING) { + return _fault(); } - return nullptr; + if (r < 0) { + lderr(cct) << __func__ << " auth_client handle_auth_reply_more returned " + << r << dendl; + return _fault(); + } + AuthRequestMoreFrame more_reply(reply); + return WRITE(more_reply.get_buffer(), "auth request more", read_frame); } CtPtr ProtocolV2::handle_auth_done(char *payload, uint32_t length) { ldout(cct, 20) << __func__ << " payload_len=" << length << dendl; AuthDoneFrame auth_done(payload, length); - CryptoKey connection_secret; - if (authorizer) { - auto iter = auth_done.auth_payload().cbegin(); - if (!authorizer->verify_reply(iter, &connection_secret)) { - ldout(cct, 0) << __func__ << " failed verifying authorize reply" << dendl; - return _fault(); - } - } - ldout(cct, 1) << __func__ << " authentication done," - << " flags=" << std::hex << auth_done.flags() << std::dec - << dendl; - - if (authorizer) { - ldout(cct, 10) << __func__ << " setting up session_security with auth " - << authorizer << dendl; - session_security.reset(get_auth_session_handler( - cct, authorizer->protocol, authorizer->session_key, - connection_secret, - CEPH_FEATURE_MSG_AUTH | CEPH_FEATURE_CEPHX_V2)); - auth_flags = auth_done.flags(); - } else { - // We have no authorizer, so we shouldn't be applying security to messages - // in this connection. - ldout(cct, 10) << __func__ << " no authorizer, clearing session_security" - << dendl; - session_security.reset(); - auth_flags = 0; + ceph_assert(messenger->auth_client); + connection->lock.unlock(); + int r = messenger->auth_client->handle_auth_done( + connection, + auth_done.global_id(), + auth_done.auth_payload(), + &auth_meta.session_key, + &auth_meta.connection_secret); + connection->lock.lock(); + if (state != State::CONNECTING) { + return _fault(); + } + if (r < 0) { + return _fault(); } + session_security.reset( + get_auth_session_handler( + cct, auth_meta.auth_method, auth_meta.session_key, + auth_meta.connection_secret, + CEPH_FEATURE_MSG_AUTH | CEPH_FEATURE_CEPHX_V2)); if (!cookie) { ceph_assert(connect_seq == 0); @@ -2468,74 +2406,24 @@ CtPtr ProtocolV2::post_server_banner_exchange() { } CtPtr ProtocolV2::handle_auth_request(char *payload, uint32_t length) { - ldout(cct, 20) << __func__ << " payload_len=" << length << dendl; - AuthRequestFrame auth_request(payload, length); - ldout(cct, 10) << __func__ << " AuthRequest(method=" << auth_request.method() << ", auth_len=" << auth_request.auth_payload().length() << ")" << dendl; - - std::vector allowed_methods; - get_auth_allowed_methods(connection->peer_type, - allowed_methods); - - bool found = std::find(allowed_methods.begin(), allowed_methods.end(), - auth_request.method()) != allowed_methods.end(); - if (!found) { - ldout(cct, 1) << __func__ << " auth method=" << auth_request.method() - << " not allowed" << dendl; - AuthBadMethodFrame bad_method(auth_request.method(), allowed_methods); - bufferlist &bl = bad_method.get_buffer(); - return WRITE(bl, "bad auth method", read_frame); - } - - ldout(cct, 10) << __func__ << " auth method=" << auth_request.method() - << " accepted, authorizer payload len=" - << auth_request.auth_payload().length() << dendl; - - auth_method = auth_request.method(); - - return _handle_authorizer(auth_request.auth_payload(), false); + auth_meta.auth_method = auth_request.method(); + return _handle_auth_request(auth_request.auth_payload(), false); } -CtPtr ProtocolV2::_handle_authorizer(bufferlist& auth_payload, bool more) +CtPtr ProtocolV2::_handle_auth_request(bufferlist& auth_payload, bool more) { - bool authorizer_valid; - bufferlist authorizer_reply; - bool had_challenge = (bool)authorizer_challenge; - - connection->lock.unlock(); - if (!messenger->ms_deliver_verify_authorizer( - connection, connection->peer_type, auth_method, - auth_payload, - authorizer_reply, authorizer_valid, session_key, - &connection_secret, - &authorizer_challenge) || - !authorizer_valid) { - connection->lock.lock(); - if (state != ACCEPTING) { - ldout(cct, 1) << __func__ - << " state changed while accept, it must be mark_down" - << dendl; - ceph_assert(state == CLOSED); - return _fault(); - } - - if (!had_challenge && authorizer_challenge) { - ldout(cct, 10) << __func__ << " challenging authorizer" << dendl; - ceph_assert(authorizer_reply.length()); - AuthReplyMoreFrame more(authorizer_reply.length(), authorizer_reply); - return WRITE(more.get_buffer(), "auth reply more", read_frame); - } else { - ldout(cct, 0) << __func__ << " got bad authorizer, auth_reply_len=" - << authorizer_reply.length() << dendl; - session_security.reset(); - AuthBadAuthFrame bad_auth(EPERM, "Bad Authorizer"); - return WRITE(bad_auth.get_buffer(), "bad auth", read_frame); - } + if (!messenger->auth_server) { + return _fault(); } - + bufferlist reply; + connection->lock.unlock(); + int r = messenger->auth_server->handle_auth_request( + connection, more, auth_meta.auth_method, auth_payload, + &reply); connection->lock.lock(); if (state != ACCEPTING) { ldout(cct, 1) << __func__ @@ -2544,35 +2432,34 @@ CtPtr ProtocolV2::_handle_authorizer(bufferlist& auth_payload, bool more) ceph_assert(state == CLOSED); return _fault(); } - - session_security.reset( - get_auth_session_handler(cct, auth_method, session_key, - connection_secret, - CEPH_FEATURE_MSG_AUTH | CEPH_FEATURE_CEPHX_V2)); - - if (auth_method == CEPH_AUTH_CEPHX) { -#warning fix msgr2 sign/encrypt options - if (cct->_conf.get_val("ms_msgr2_sign_messages")) { - auth_flags |= static_cast(AuthFlag::SIGNED); - } - if (cct->_conf.get_val("ms_msgr2_encrypt_messages")) { - auth_flags |= static_cast(AuthFlag::ENCRYPTED); - } + if (r == 1) { + AuthDoneFrame auth_done(connection->peer_global_id, reply); + return WRITE(auth_done.get_buffer(), "auth done", read_frame); + } else if (r == 0) { + AuthReplyMoreFrame more(reply); + return WRITE(more.get_buffer(), "auth reply more", read_frame); + } else if (r == -EBUSY) { + // kick the client and maybe they'll come back later + return _fault(); + } else { + ceph_assert(r < 0); + std::vector allowed_methods; + messenger->auth_server->get_supported_auth_methods( + connection->get_peer_type(), &allowed_methods); + ldout(cct, 1) << __func__ << " auth_method " << auth_meta.auth_method + << " r " << cpp_strerror(r) + << ", allowed_methods " << allowed_methods + << dendl; + AuthBadMethodFrame bad_method(auth_meta.auth_method, r, allowed_methods); + return WRITE(bad_method.get_buffer(), "bad auth method", read_frame); } - AuthDoneFrame auth_done(auth_flags, authorizer_reply.length(), - authorizer_reply); - return WRITE(auth_done.get_buffer(), "auth done", read_frame); } CtPtr ProtocolV2::handle_auth_request_more(char *payload, uint32_t length) { ldout(cct, 20) << __func__ << " payload_len=" << length << dendl; - AuthRequestMoreFrame auth_more(payload, length); - ldout(cct, 5) << __func__ - << " auth request more len=" << auth_more.auth_payload().length() - << dendl; - return _handle_authorizer(auth_more.auth_payload(), true); + return _handle_auth_request(auth_more.auth_payload(), true); } CtPtr ProtocolV2::handle_client_ident(char *payload, uint32_t length) { @@ -2596,6 +2483,7 @@ CtPtr ProtocolV2::handle_client_ident(char *payload, uint32_t length) { connection->target_addr = connection->_infer_target_addr(client_ident.addrs()); peer_name = entity_name_t(connection->get_peer_type(), client_ident.gid()); + connection->set_peer_id(client_ident.gid()); uint64_t feat_missing = connection->policy.features_required & ~(uint64_t)client_ident.supported_features(); @@ -2911,10 +2799,11 @@ CtPtr ProtocolV2::reuse_connection(AsyncConnectionRef existing, exproto->reconnecting = reconnecting; exproto->replacing = true; exproto->session_security = session_security; - exproto->auth_method = auth_method; - exproto->auth_flags = auth_flags; - exproto->session_key = session_key; - exproto->authorizer_challenge = std::move(authorizer_challenge); + exproto->auth_meta.con_mode = auth_meta.con_mode; + exproto->auth_meta.auth_method = auth_meta.auth_method; + exproto->auth_meta.session_key = auth_meta.session_key; + exproto->auth_meta.connection_secret = auth_meta.connection_secret; + exproto->auth_meta.authorizer_challenge = std::move(auth_meta.authorizer_challenge); existing->state_offset = 0; // avoid previous thread modify event exproto->state = NONE; diff --git a/src/msg/async/ProtocolV2.h b/src/msg/async/ProtocolV2.h index 6006f59abfb..8e31aeb21dd 100644 --- a/src/msg/async/ProtocolV2.h +++ b/src/msg/async/ProtocolV2.h @@ -11,9 +11,9 @@ private: enum State { NONE, START_CONNECT, - CONNECTING, + CONNECTING, // banner + authentication + ident START_ACCEPT, - ACCEPTING, + ACCEPTING, // banner + authentication + ident ACCEPTING_SESSION, READY, THROTTLE_MESSAGE, @@ -50,7 +50,6 @@ public: HELLO = 1, AUTH_REQUEST, AUTH_BAD_METHOD, - AUTH_BAD_AUTH, AUTH_REPLY_MORE, AUTH_REQUEST_MORE, AUTH_DONE, @@ -75,14 +74,8 @@ private: char *temp_buffer; State state; uint64_t peer_required_features; - AuthAuthorizer *authorizer; - uint32_t auth_method; - uint32_t got_bad_method; - CryptoKey session_key; - CryptoKey connection_secret; std::shared_ptr session_security; - std::unique_ptr authorizer_challenge; - uint64_t auth_flags; + uint64_t cookie; uint64_t global_seq; uint64_t connect_seq; @@ -119,12 +112,6 @@ private: ostream &_conn_prefix(std::ostream *_dout); void run_continuation(Ct *continuation); void calc_signature(const char *in, uint32_t length, char *out); - inline bool sign_frames() { - return auth_flags & static_cast(AuthFlag::SIGNED); - } - inline bool encrypt_frames() { - return auth_flags & static_cast(AuthFlag::ENCRYPTED); - } Ct *read(CONTINUATION_PARAM(next, ProtocolV2, char *, int), int len, char *buffer = nullptr); @@ -234,7 +221,6 @@ private: } Ct *send_auth_request(std::vector &allowed_methods); Ct *handle_auth_bad_method(char *payload, uint32_t length); - Ct *handle_auth_bad_auth(char *payload, uint32_t length); Ct *handle_auth_reply_more(char *payload, uint32_t length); Ct *handle_auth_done(char *payload, uint32_t length); Ct *send_client_ident(); @@ -256,7 +242,7 @@ private: Ct *post_server_banner_exchange(); Ct *handle_auth_request(char *payload, uint32_t length); Ct *handle_auth_request_more(char *payload, uint32_t length); - Ct *_handle_authorizer(bufferlist& auth_payload, bool more); + Ct *_handle_auth_request(bufferlist& auth_payload, bool more); Ct *handle_client_ident(char *payload, uint32_t length); Ct *handle_ident_missing_features_write(int r); Ct *handle_reconnect(char *payload, uint32_t length); diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc index 9c85c2b396c..6f97f884c95 100644 --- a/src/osd/OSD.cc +++ b/src/osd/OSD.cc @@ -3034,6 +3034,22 @@ int OSD::init() if (r < 0) goto out; + // client_messenger auth_client is already set up by monc. + for (auto m : { cluster_messenger, + objecter_messenger, + hb_front_client_messenger, + hb_back_client_messenger, + hb_front_server_messenger, + hb_back_server_messenger } ) { + m->set_auth_client(monc); + } + for (auto m : { cluster_messenger, + hb_front_server_messenger, + hb_back_server_messenger }) { + m->set_auth_server(monc); + } + monc->set_handle_authentication_dispatcher(this); + mgrc.set_pgstats_cb([this](){ return collect_pg_stats(); }); mgrc.set_perf_metric_query_cb( [this](const std::map &queries) {