From: Sage Weil Date: Fri, 25 May 2018 15:59:19 +0000 (-0500) Subject: auth/cephx: add authorizer challenge X-Git-Tag: v10.2.11~6^2~3 X-Git-Url: http://git.apps.os.sepia.ceph.com/?a=commitdiff_plain;h=26816cd80ae245d351d5ce34d8af434fbc798602;p=ceph.git auth/cephx: add authorizer challenge Allow the accepting side of a connection to reject an initial authorizer with a random challenge. The connecting side then has to respond with an updated authorizer proving they are able to decrypt the service's challenge and that the new authorizer was produced for this specific connection instance. The accepting side requires this challenge and response unconditionally if the client side advertises they have the feature bit. Servers wishing to require this improved level of authentication simply have to require the appropriate feature. Signed-off-by: Sage Weil (cherry picked from commit f80b848d3f830eb6dba50123e04385173fa4540b) # Conflicts: # src/auth/Auth.h # src/auth/cephx/CephxProtocol.cc # src/auth/cephx/CephxProtocol.h # src/auth/none/AuthNoneProtocol.h # src/msg/Dispatcher.h # src/msg/async/AsyncConnection.cc - const_iterator - ::decode vs decode - AsyncConnection ctor arg noise - get_random_bytes(), not cct->random() (cherry picked from commit 5ead97120e07054d80623dada90a5cc764c28468) # Conflicts: # src/auth/cephx/CephxAuthorizeHandler.h # src/auth/cephx/CephxProtocol.h # src/auth/none/AuthNoneAuthorizeHandler.h # src/auth/none/AuthNoneProtocol.h # src/auth/unknown/AuthUnknownAuthorizeHandler.h # src/mds/MDSDaemon.cc # src/mds/MDSDaemon.h # src/mgr/DaemonServer.cc # src/mgr/DaemonServer.h # src/mon/Monitor.cc # src/mon/Monitor.h # src/msg/async/AsyncConnection.cc # src/osd/OSD.cc # src/osd/OSD.h # src/test/messenger/simple_dispatcher.h # src/test/msgr/perf_msgr_client.cc # src/test/msgr/perf_msgr_server.cc # src/test/msgr/test_msgr.cc - lots of override annotation conflicts - _refused() callbacks no present in jewel - lots of msg/async conflicts (code has changed a fair bit) - we inherited some upstream rotating keys checks, see f159a093ecab4aa53693ec106c00d5ecb383c467 --- diff --git a/src/auth/Auth.h b/src/auth/Auth.h index 12d49094b437b..cacf6d9025637 100644 --- a/src/auth/Auth.h +++ b/src/auth/Auth.h @@ -139,6 +139,11 @@ struct AuthAuthorizer { explicit AuthAuthorizer(__u32 p) : protocol(p) {} virtual ~AuthAuthorizer() {} virtual bool verify_reply(bufferlist::iterator& reply) = 0; + virtual bool add_challenge(CephContext *cct, bufferlist& challenge) = 0; +}; + +struct AuthAuthorizerChallenge { + virtual ~AuthAuthorizerChallenge() {} }; diff --git a/src/auth/AuthAuthorizeHandler.h b/src/auth/AuthAuthorizeHandler.h index 7b67876e44be3..394ba1a47c59b 100644 --- a/src/auth/AuthAuthorizeHandler.h +++ b/src/auth/AuthAuthorizeHandler.h @@ -34,7 +34,9 @@ struct AuthAuthorizeHandler { virtual bool verify_authorizer(CephContext *cct, KeyStore *keys, bufferlist& authorizer_data, bufferlist& authorizer_reply, EntityName& entity_name, uint64_t& global_id, - AuthCapsInfo& caps_info, CryptoKey& session_key, uint64_t *auid = NULL) = 0; + AuthCapsInfo& caps_info, CryptoKey& session_key, + uint64_t *auid, + std::unique_ptr *challenge) = 0; virtual int authorizer_session_crypto() = 0; }; diff --git a/src/auth/cephx/CephxAuthorizeHandler.cc b/src/auth/cephx/CephxAuthorizeHandler.cc index 5b28b9e5ade1d..4065c73330c49 100644 --- a/src/auth/cephx/CephxAuthorizeHandler.cc +++ b/src/auth/cephx/CephxAuthorizeHandler.cc @@ -8,9 +8,12 @@ -bool CephxAuthorizeHandler::verify_authorizer(CephContext *cct, KeyStore *keys, - bufferlist& authorizer_data, bufferlist& authorizer_reply, - EntityName& entity_name, uint64_t& global_id, AuthCapsInfo& caps_info, CryptoKey& session_key, uint64_t *auid) +bool CephxAuthorizeHandler::verify_authorizer( + CephContext *cct, KeyStore *keys, + bufferlist& authorizer_data, bufferlist& authorizer_reply, + EntityName& entity_name, uint64_t& global_id, AuthCapsInfo& caps_info, + CryptoKey& session_key, uint64_t *auid, + std::unique_ptr *challenge) { bufferlist::iterator iter = authorizer_data.begin(); @@ -21,7 +24,8 @@ bool CephxAuthorizeHandler::verify_authorizer(CephContext *cct, KeyStore *keys, CephXServiceTicketInfo auth_ticket_info; - bool isvalid = cephx_verify_authorizer(cct, keys, iter, auth_ticket_info, authorizer_reply); + bool isvalid = cephx_verify_authorizer(cct, keys, iter, auth_ticket_info, challenge, + authorizer_reply); if (isvalid) { caps_info = auth_ticket_info.ticket.caps; diff --git a/src/auth/cephx/CephxAuthorizeHandler.h b/src/auth/cephx/CephxAuthorizeHandler.h index d17a692f71c3f..d3753a635440a 100644 --- a/src/auth/cephx/CephxAuthorizeHandler.h +++ b/src/auth/cephx/CephxAuthorizeHandler.h @@ -23,8 +23,9 @@ struct CephxAuthorizeHandler : public AuthAuthorizeHandler { bool verify_authorizer(CephContext *cct, KeyStore *keys, bufferlist& authorizer_data, bufferlist& authorizer_reply, EntityName& entity_name, uint64_t& global_id, - AuthCapsInfo& caps_info, CryptoKey& session_key, uint64_t *auid = NULL); - int authorizer_session_crypto(); + AuthCapsInfo& caps_info, CryptoKey& session_key, uint64_t *auid, + std::unique_ptr *challenge) override; + int authorizer_session_crypto() override; }; diff --git a/src/auth/cephx/CephxProtocol.cc b/src/auth/cephx/CephxProtocol.cc index 0e668c67cf5e2..711a9a53da065 100644 --- a/src/auth/cephx/CephxProtocol.cc +++ b/src/auth/cephx/CephxProtocol.cc @@ -304,6 +304,7 @@ CephXAuthorizer *CephXTicketHandler::build_authorizer(uint64_t global_id) const ::encode(service_id, a->bl); ::encode(ticket, a->bl); + a->base_bl = a->bl; CephXAuthorize msg; msg.nonce = a->nonce; @@ -390,7 +391,9 @@ bool cephx_decode_ticket(CephContext *cct, KeyStore *keys, uint32_t service_id, */ bool cephx_verify_authorizer(CephContext *cct, KeyStore *keys, bufferlist::iterator& indata, - CephXServiceTicketInfo& ticket_info, bufferlist& reply_bl) + CephXServiceTicketInfo& ticket_info, + std::unique_ptr *challenge, + bufferlist& reply_bl) { __u8 authorizer_v; uint32_t service_id; @@ -457,6 +460,30 @@ bool cephx_verify_authorizer(CephContext *cct, KeyStore *keys, return false; } + if (challenge) { + auto *c = static_cast(challenge->get()); + if (!auth_msg.have_challenge || !c) { + c = new CephXAuthorizeChallenge; + challenge->reset(c); + get_random_bytes((char*)&c->server_challenge, sizeof(c->server_challenge)); + ldout(cct,10) << __func__ << " adding server_challenge " << c->server_challenge + << dendl; + + encode_encrypt_enc_bl(cct, *c, ticket_info.session_key, reply_bl, error); + if (!error.empty()) { + ldout(cct, 10) << "verify_authorizer: encode_encrypt error: " << error << dendl; + return false; + } + return false; + } + ldout(cct, 10) << __func__ << " got server_challenge+1 " + << auth_msg.server_challenge_plus_one + << " expecting " << c->server_challenge + 1 << dendl; + if (c->server_challenge + 1 != auth_msg.server_challenge_plus_one) { + return false; + } + } + /* * Reply authorizer: * {timestamp + 1}^session_key @@ -493,3 +520,31 @@ bool CephXAuthorizer::verify_reply(bufferlist::iterator& indata) return true; } +bool CephXAuthorizer::add_challenge(CephContext *cct, bufferlist& challenge) +{ + bl = base_bl; + + CephXAuthorize msg; + msg.nonce = nonce; + + auto p = challenge.begin(); + if (!p.end()) { + std::string error; + CephXAuthorizeChallenge ch; + decode_decrypt_enc_bl(cct, ch, session_key, challenge, error); + if (!error.empty()) { + ldout(cct, 0) << "failed to decrypt challenge (" << challenge.length() << " bytes): " + << error << dendl; + return false; + } + msg.have_challenge = true; + msg.server_challenge_plus_one = ch.server_challenge + 1; + } + + std::string error; + if (encode_encrypt(cct, msg, session_key, bl, error)) { + ldout(cct, 0) << __func__ << " failed to encrypt authorizer: " << error << dendl; + return false; + } + return true; +} diff --git a/src/auth/cephx/CephxProtocol.h b/src/auth/cephx/CephxProtocol.h index 4a5fb5d3dcbee..a239bbc90e46c 100644 --- a/src/auth/cephx/CephxProtocol.h +++ b/src/auth/cephx/CephxProtocol.h @@ -276,12 +276,14 @@ private: CephContext *cct; public: uint64_t nonce; + bufferlist base_bl; explicit CephXAuthorizer(CephContext *cct_) : AuthAuthorizer(CEPH_AUTH_CEPHX), cct(cct_), nonce(0) {} bool build_authorizer(); - bool verify_reply(bufferlist::iterator& reply); + bool verify_reply(bufferlist::iterator& reply) override; + bool add_challenge(CephContext *cct, bufferlist& challenge) override; }; @@ -387,17 +389,41 @@ struct CephXServiceTicketInfo { }; WRITE_CLASS_ENCODER(CephXServiceTicketInfo) +struct CephXAuthorizeChallenge : public AuthAuthorizerChallenge { + uint64_t server_challenge; + void encode(bufferlist& bl) const { + __u8 struct_v = 1; + ::encode(struct_v, bl); + ::encode(server_challenge, bl); + } + void decode(bufferlist::iterator& bl) { + __u8 struct_v; + ::decode(struct_v, bl); + ::decode(server_challenge, bl); + } +}; +WRITE_CLASS_ENCODER(CephXAuthorizeChallenge) + struct CephXAuthorize { uint64_t nonce; + bool have_challenge = false; + uint64_t server_challenge_plus_one = 0; void encode(bufferlist& bl) const { - __u8 struct_v = 1; + __u8 struct_v = 2; ::encode(struct_v, bl); ::encode(nonce, bl); + ::encode(have_challenge, bl); + ::encode(server_challenge_plus_one, bl); } void decode(bufferlist::iterator& bl) { __u8 struct_v; ::decode(struct_v, bl); ::decode(nonce, bl); + if (struct_v >= 2) { + ::decode(have_challenge, bl); + ::decode(server_challenge_plus_one, bl); + } + } }; WRITE_CLASS_ENCODER(CephXAuthorize) @@ -412,9 +438,12 @@ bool cephx_decode_ticket(CephContext *cct, KeyStore *keys, /* * Verify authorizer and generate reply authorizer */ -extern bool cephx_verify_authorizer(CephContext *cct, KeyStore *keys, - bufferlist::iterator& indata, - CephXServiceTicketInfo& ticket_info, bufferlist& reply_bl); +extern bool cephx_verify_authorizer( + CephContext *cct, KeyStore *keys, + bufferlist::iterator& indata, + CephXServiceTicketInfo& ticket_info, + std::unique_ptr *challenge, + bufferlist& reply_bl); diff --git a/src/auth/cephx/CephxServiceHandler.cc b/src/auth/cephx/CephxServiceHandler.cc index 7b6212ab5b40d..71a9b2f15d3fb 100644 --- a/src/auth/cephx/CephxServiceHandler.cc +++ b/src/auth/cephx/CephxServiceHandler.cc @@ -156,7 +156,9 @@ int CephxServiceHandler::handle_request(bufferlist::iterator& indata, bufferlist bufferlist tmp_bl; CephXServiceTicketInfo auth_ticket_info; - if (!cephx_verify_authorizer(cct, key_server, indata, auth_ticket_info, tmp_bl)) { + // note: no challenge here. + if (!cephx_verify_authorizer(cct, key_server, indata, auth_ticket_info, nullptr, + tmp_bl)) { ret = -EPERM; break; } diff --git a/src/auth/none/AuthNoneAuthorizeHandler.cc b/src/auth/none/AuthNoneAuthorizeHandler.cc index 8b55e9e9ba8b4..5767eacffd105 100644 --- a/src/auth/none/AuthNoneAuthorizeHandler.cc +++ b/src/auth/none/AuthNoneAuthorizeHandler.cc @@ -17,10 +17,13 @@ #define dout_subsys ceph_subsys_auth -bool AuthNoneAuthorizeHandler::verify_authorizer(CephContext *cct, KeyStore *keys, - bufferlist& authorizer_data, bufferlist& authorizer_reply, - EntityName& entity_name, uint64_t& global_id, AuthCapsInfo& caps_info, CryptoKey& session_key, -uint64_t *auid) +bool AuthNoneAuthorizeHandler::verify_authorizer( + CephContext *cct, KeyStore *keys, + bufferlist& authorizer_data, bufferlist& authorizer_reply, + EntityName& entity_name, uint64_t& global_id, AuthCapsInfo& caps_info, + CryptoKey& session_key, + uint64_t *auid, + std::unique_ptr *challenge) { bufferlist::iterator iter = authorizer_data.begin(); diff --git a/src/auth/none/AuthNoneAuthorizeHandler.h b/src/auth/none/AuthNoneAuthorizeHandler.h index cc00d3d6cdb2d..1179698a43ac9 100644 --- a/src/auth/none/AuthNoneAuthorizeHandler.h +++ b/src/auth/none/AuthNoneAuthorizeHandler.h @@ -23,8 +23,9 @@ struct AuthNoneAuthorizeHandler : public AuthAuthorizeHandler { bool verify_authorizer(CephContext *cct, KeyStore *keys, bufferlist& authorizer_data, bufferlist& authorizer_reply, EntityName& entity_name, uint64_t& global_id, - AuthCapsInfo& caps_info, CryptoKey& session_key, uint64_t *auid=NULL); - int authorizer_session_crypto(); + AuthCapsInfo& caps_info, CryptoKey& session_key, uint64_t *auid, + std::unique_ptr *challenge) override; + int authorizer_session_crypto() override; }; diff --git a/src/auth/none/AuthNoneProtocol.h b/src/auth/none/AuthNoneProtocol.h index a3c989128f65e..e3a889303596a 100644 --- a/src/auth/none/AuthNoneProtocol.h +++ b/src/auth/none/AuthNoneProtocol.h @@ -17,6 +17,8 @@ #include "../Auth.h" +class CephContext; + struct AuthNoneAuthorizer : public AuthAuthorizer { AuthNoneAuthorizer() : AuthAuthorizer(CEPH_AUTH_NONE) { } bool build_authorizer(const EntityName &ename, uint64_t global_id) { @@ -26,7 +28,8 @@ struct AuthNoneAuthorizer : public AuthAuthorizer { ::encode(global_id, bl); return 0; } - bool verify_reply(bufferlist::iterator& reply) { return true; } + bool verify_reply(bufferlist::iterator& reply) override { return true; } + bool add_challenge(CephContext *cct, bufferlist& ch) override { return true; } }; #endif diff --git a/src/auth/unknown/AuthUnknownAuthorizeHandler.cc b/src/auth/unknown/AuthUnknownAuthorizeHandler.cc index de66058c4896a..f974ee0a5114f 100644 --- a/src/auth/unknown/AuthUnknownAuthorizeHandler.cc +++ b/src/auth/unknown/AuthUnknownAuthorizeHandler.cc @@ -17,10 +17,13 @@ #define dout_subsys ceph_subsys_auth -bool AuthUnknownAuthorizeHandler::verify_authorizer(CephContext *cct, KeyStore *keys, - bufferlist& authorizer_data, bufferlist& authorizer_reply, - EntityName& entity_name, uint64_t& global_id, AuthCapsInfo& caps_info, CryptoKey& session_key, -uint64_t *auid) +bool AuthUnknownAuthorizeHandler::verify_authorizer( + CephContext *cct, KeyStore *keys, + bufferlist& authorizer_data, bufferlist& authorizer_reply, + EntityName& entity_name, uint64_t& global_id, AuthCapsInfo& caps_info, + CryptoKey& session_key, + uint64_t *auid, + std::unique_ptr *challenge) { // For unknown authorizers, there's nothing to verify. They're "OK" by definition. PLR diff --git a/src/auth/unknown/AuthUnknownAuthorizeHandler.h b/src/auth/unknown/AuthUnknownAuthorizeHandler.h index 0156fbd85d97b..2c7cdf89e0c8a 100644 --- a/src/auth/unknown/AuthUnknownAuthorizeHandler.h +++ b/src/auth/unknown/AuthUnknownAuthorizeHandler.h @@ -23,8 +23,9 @@ struct AuthUnknownAuthorizeHandler : public AuthAuthorizeHandler { bool verify_authorizer(CephContext *cct, KeyStore *keys, bufferlist& authorizer_data, bufferlist& authorizer_reply, EntityName& entity_name, uint64_t& global_id, - AuthCapsInfo& caps_info, CryptoKey& session_key, uint64_t *auid=NULL); - int authorizer_session_crypto(); + AuthCapsInfo& caps_info, CryptoKey& session_key, uint64_t *auid, + std::unique_ptr *challenge) override; + int authorizer_session_crypto() override; }; diff --git a/src/include/msgr.h b/src/include/msgr.h index ac98344a2cdd9..41baebcd0d122 100644 --- a/src/include/msgr.h +++ b/src/include/msgr.h @@ -92,7 +92,7 @@ struct ceph_entity_inst { #define CEPH_MSGR_TAG_SEQ 13 /* 64-bit int follows with seen seq number */ #define CEPH_MSGR_TAG_KEEPALIVE2 14 #define CEPH_MSGR_TAG_KEEPALIVE2_ACK 15 /* keepalive reply */ - +#define CEPH_MSGR_TAG_CHALLENGE_AUTHORIZER 16 /* ceph v2 doing server challenge */ /* * connection negotiation diff --git a/src/mds/MDSDaemon.cc b/src/mds/MDSDaemon.cc index a9b4ecbaf41ce..ddd179212daf4 100644 --- a/src/mds/MDSDaemon.cc +++ b/src/mds/MDSDaemon.cc @@ -1295,7 +1295,8 @@ void MDSDaemon::ms_handle_remote_reset(Connection *con) bool MDSDaemon::ms_verify_authorizer(Connection *con, int peer_type, int protocol, bufferlist& authorizer_data, bufferlist& authorizer_reply, - bool& is_valid, CryptoKey& session_key) + bool& is_valid, CryptoKey& session_key, + std::unique_ptr *challenge) { Mutex::Locker l(mds_lock); if (stopping) { @@ -1322,8 +1323,16 @@ bool MDSDaemon::ms_verify_authorizer(Connection *con, int peer_type, EntityName name; uint64_t global_id; - is_valid = authorize_handler->verify_authorizer(cct, monc->rotating_secrets, - authorizer_data, authorizer_reply, name, global_id, caps_info, session_key); + RotatingKeyRing *keys = monc->rotating_secrets; + if (keys) { + is_valid = authorize_handler->verify_authorizer( + cct, keys, + authorizer_data, authorizer_reply, name, global_id, caps_info, + session_key, nullptr, challenge); + } else { + dout(10) << __func__ << " no rotating_keys (yet), denied" << dendl; + is_valid = false; + } if (is_valid) { entity_name_t n(con->get_peer_type(), global_id); diff --git a/src/mds/MDSDaemon.h b/src/mds/MDSDaemon.h index d193dac66c216..6d59263f6af8d 100644 --- a/src/mds/MDSDaemon.h +++ b/src/mds/MDSDaemon.h @@ -146,11 +146,12 @@ class MDSDaemon : public Dispatcher, public md_config_obs_t { bool ms_get_authorizer(int dest_type, AuthAuthorizer **authorizer, bool force_new); bool ms_verify_authorizer(Connection *con, int peer_type, int protocol, bufferlist& authorizer_data, bufferlist& authorizer_reply, - bool& isvalid, CryptoKey& session_key); - void ms_handle_accept(Connection *con); - void ms_handle_connect(Connection *con); - bool ms_handle_reset(Connection *con); - void ms_handle_remote_reset(Connection *con); + bool& isvalid, CryptoKey& session_key, + std::unique_ptr *challenge) override; + void ms_handle_accept(Connection *con) override; + void ms_handle_connect(Connection *con) override; + bool ms_handle_reset(Connection *con) override; + void ms_handle_remote_reset(Connection *con) override; protected: // admin socket handling diff --git a/src/mon/Monitor.cc b/src/mon/Monitor.cc index 392e0af89b3a0..66fa93924a4f4 100644 --- a/src/mon/Monitor.cc +++ b/src/mon/Monitor.cc @@ -5115,8 +5115,10 @@ bool Monitor::ms_get_authorizer(int service_id, AuthAuthorizer **authorizer, boo } bool Monitor::ms_verify_authorizer(Connection *con, int peer_type, - int protocol, bufferlist& authorizer_data, bufferlist& authorizer_reply, - bool& isvalid, CryptoKey& session_key) + int protocol, bufferlist& authorizer_data, + bufferlist& authorizer_reply, + bool& isvalid, CryptoKey& session_key, + std::unique_ptr *challenge) { dout(10) << "ms_verify_authorizer " << con->get_peer_addr() << " " << ceph_entity_type_name(peer_type) @@ -5135,7 +5137,7 @@ bool Monitor::ms_verify_authorizer(Connection *con, int peer_type, if (authorizer_data.length()) { bool ret = cephx_verify_authorizer(g_ceph_context, &keyring, iter, - auth_ticket_info, authorizer_reply); + auth_ticket_info, challenge, authorizer_reply); if (ret) { session_key = auth_ticket_info.session_key; isvalid = true; diff --git a/src/mon/Monitor.h b/src/mon/Monitor.h index 75020ad13acd6..1ff8a8f49bcb8 100644 --- a/src/mon/Monitor.h +++ b/src/mon/Monitor.h @@ -857,9 +857,10 @@ public: bool ms_get_authorizer(int dest_type, AuthAuthorizer **authorizer, bool force_new); bool ms_verify_authorizer(Connection *con, int peer_type, int protocol, bufferlist& authorizer_data, bufferlist& authorizer_reply, - bool& isvalid, CryptoKey& session_key); - bool ms_handle_reset(Connection *con); - void ms_handle_remote_reset(Connection *con) {} + bool& isvalid, CryptoKey& session_key, + std::unique_ptr *challenge) override; + bool ms_handle_reset(Connection *con) override; + void ms_handle_remote_reset(Connection *con) override {} int write_default_keyring(bufferlist& bl); void extract_save_mon_key(KeyRing& keyring); diff --git a/src/msg/Dispatcher.h b/src/msg/Dispatcher.h index f7de0dedffb5d..29ea5f1b78d8d 100644 --- a/src/msg/Dispatcher.h +++ b/src/msg/Dispatcher.h @@ -17,6 +17,7 @@ #define CEPH_DISPATCHER_H #include "include/assert.h" +#include #include "include/buffer_fwd.h" #include "include/assert.h" @@ -26,6 +27,7 @@ class Connection; class AuthAuthorizer; class CryptoKey; class CephContext; +class AuthAuthorizerChallenge; class Dispatcher { public: @@ -194,7 +196,10 @@ public: ceph::bufferlist& authorizer, ceph::bufferlist& authorizer_reply, bool& isvalid, - CryptoKey& session_key) { return false; } + CryptoKey& session_key, + std::unique_ptr *challenge) { + return false; + } /** * @} //Authentication */ diff --git a/src/msg/Messenger.h b/src/msg/Messenger.h index 8acfd6816ddf2..32769ede5eb0a 100644 --- a/src/msg/Messenger.h +++ b/src/msg/Messenger.h @@ -704,11 +704,13 @@ public: */ bool ms_deliver_verify_authorizer(Connection *con, int peer_type, int protocol, bufferlist& authorizer, bufferlist& authorizer_reply, - bool& isvalid, CryptoKey& session_key) { + bool& isvalid, CryptoKey& session_key, + std::unique_ptr *challenge) { for (list::iterator p = dispatchers.begin(); p != dispatchers.end(); ++p) { - if ((*p)->ms_verify_authorizer(con, peer_type, protocol, authorizer, authorizer_reply, isvalid, session_key)) + if ((*p)->ms_verify_authorizer(con, peer_type, protocol, authorizer, authorizer_reply, + isvalid, session_key, challenge)) return true; } return false; diff --git a/src/msg/async/AsyncConnection.cc b/src/msg/async/AsyncConnection.cc index b67452964c187..70af3d602b6b4 100644 --- a/src/msg/async/AsyncConnection.cc +++ b/src/msg/async/AsyncConnection.cc @@ -1202,8 +1202,7 @@ ssize_t AsyncConnection::_process_connection() case STATE_CONNECTING_SEND_CONNECT_MSG: { - if (!got_bad_auth) { - delete authorizer; + if (!authorizer) { authorizer = async_msgr->get_authorizer(peer_type, false); } bufferlist bl; @@ -1284,7 +1283,15 @@ ssize_t AsyncConnection::_process_connection() } authorizer_reply.append(state_buffer, connect_reply.authorizer_len); - bufferlist::iterator iter = authorizer_reply.begin(); + + if (connect_reply.tag == CEPH_MSGR_TAG_CHALLENGE_AUTHORIZER) { + ldout(async_msgr->cct,10) << __func__ << " connect got auth challenge" << dendl; + authorizer->add_challenge(async_msgr->cct, authorizer_reply); + state = STATE_CONNECTING_SEND_CONNECT_MSG; + break; + } + + auto iter = authorizer_reply.begin(); if (authorizer && !authorizer->verify_reply(iter)) { ldout(async_msgr->cct, 0) << __func__ << " failed verifying authorize reply" << dendl; goto fail; @@ -1692,11 +1699,25 @@ ssize_t AsyncConnection::handle_connect_msg(ceph_msg_connect &connect, bufferlis lock.Unlock(); bool authorizer_valid; - if (!async_msgr->verify_authorizer(this, peer_type, connect.authorizer_protocol, authorizer_bl, - authorizer_reply, authorizer_valid, session_key) || !authorizer_valid) { - ldout(async_msgr->cct,0) << __func__ << ": got bad authorizer" << dendl; + bool need_challenge = connect.features & CEPH_FEATURE_CEPHX_V2; + bool had_challenge = (bool)authorizer_challenge; + if (!async_msgr->verify_authorizer( + this, peer_type, connect.authorizer_protocol, authorizer_bl, + authorizer_reply, authorizer_valid, session_key, + need_challenge ? &authorizer_challenge : nullptr) || + !authorizer_valid) { + char tag; + if (need_challenge && !had_challenge && authorizer_challenge) { + ldout(async_msgr->cct,0) << __func__ << ": challenging authorizer" + << dendl; + assert(authorizer_reply.length()); + tag = CEPH_MSGR_TAG_CHALLENGE_AUTHORIZER; + } else { + ldout(async_msgr->cct,0) << __func__ << ": got bad authorizer" << dendl; + tag = CEPH_MSGR_TAG_BADAUTHORIZER; + } session_security.reset(); - return _reply_accept(CEPH_MSGR_TAG_BADAUTHORIZER, connect, reply, authorizer_reply); + return _reply_accept(tag, connect, reply, authorizer_reply); } // We've verified the authorizer for this AsyncConnection, so set up the session security structure. PLR @@ -1900,6 +1921,7 @@ ssize_t AsyncConnection::handle_connect_msg(ceph_msg_connect &connect, bufferlis ldout(async_msgr->cct, 0) << __func__ << " reply fault for existing connection." << dendl; existing->fault(); } + existing->authorizer_challenge.reset(); ldout(async_msgr->cct, 1) << __func__ << " stop myself to swap existing" << dendl; _stop(); diff --git a/src/msg/async/AsyncConnection.h b/src/msg/async/AsyncConnection.h index 45ec461fc16ab..e92c668db2b54 100644 --- a/src/msg/async/AsyncConnection.h +++ b/src/msg/async/AsyncConnection.h @@ -350,6 +350,7 @@ class AsyncConnection : public Connection { NetHandler net; EventCenter *center; ceph::shared_ptr session_security; + std::unique_ptr authorizer_challenge; // accept side #if !defined(MSG_NOSIGNAL) && !defined(SO_NOSIGPIPE) sigset_t sigpipe_mask; diff --git a/src/msg/async/AsyncMessenger.h b/src/msg/async/AsyncMessenger.h index 3025518dddfca..e349dad054709 100644 --- a/src/msg/async/AsyncMessenger.h +++ b/src/msg/async/AsyncMessenger.h @@ -485,9 +485,10 @@ public: * This wraps ms_deliver_verify_authorizer; we use it for AsyncConnection. */ bool verify_authorizer(Connection *con, int peer_type, int protocol, bufferlist& auth, bufferlist& auth_reply, - bool& isvalid, CryptoKey& session_key) { + bool& isvalid, CryptoKey& session_key, + std::unique_ptr *challenge) { return ms_deliver_verify_authorizer(con, peer_type, protocol, auth, - auth_reply, isvalid, session_key); + auth_reply, isvalid, session_key, challenge); } /** * Increment the global sequence for this AsyncMessenger and return it. diff --git a/src/msg/simple/Pipe.cc b/src/msg/simple/Pipe.cc index e65918954bbdd..912632f98757a 100644 --- a/src/msg/simple/Pipe.cc +++ b/src/msg/simple/Pipe.cc @@ -288,6 +288,10 @@ int Pipe::accept() // used for reading in the remote acked seq on connect uint64_t newly_acked_seq = 0; + bool need_challenge = false; + bool had_challenge = false; + std::unique_ptr authorizer_challenge; + recv_reset(); set_socket_options(); @@ -448,14 +452,27 @@ int Pipe::accept() pipe_lock.Unlock(); - if (!msgr->verify_authorizer(connection_state.get(), peer_type, connect.authorizer_protocol, authorizer, - authorizer_reply, authorizer_valid, session_key) || + need_challenge = connect.features & CEPH_FEATURE_CEPHX_V2; + had_challenge = (bool)authorizer_challenge; + authorizer_reply.clear(); + if (!msgr->verify_authorizer( + connection_state.get(), peer_type, connect.authorizer_protocol, authorizer, + authorizer_reply, authorizer_valid, session_key, + need_challenge ? &authorizer_challenge : nullptr) || !authorizer_valid) { - ldout(msgr->cct,0) << "accept: got bad authorizer" << dendl; pipe_lock.Lock(); if (state != STATE_ACCEPTING) goto shutting_down_msgr_unlocked; - reply.tag = CEPH_MSGR_TAG_BADAUTHORIZER; + if (!had_challenge && need_challenge && authorizer_challenge) { + ldout(msgr->cct,0) << "accept: challenging authorizer " + << authorizer_reply.length() + << " bytes" << dendl; + assert(authorizer_reply.length()); + reply.tag = CEPH_MSGR_TAG_CHALLENGE_AUTHORIZER; + } else { + ldout(msgr->cct,0) << "accept: got bad authorizer" << dendl; + reply.tag = CEPH_MSGR_TAG_BADAUTHORIZER; + } session_security.reset(); goto reply; } @@ -1042,8 +1059,9 @@ int Pipe::connect() while (1) { - delete authorizer; - authorizer = msgr->get_authorizer(peer_type, false); + if (!authorizer) { + authorizer = msgr->get_authorizer(peer_type, false); + } bufferlist authorizer_reply; ceph_msg_connect connect; @@ -1113,6 +1131,13 @@ int Pipe::connect() authorizer_reply.push_back(bp); } + if (reply.tag == CEPH_MSGR_TAG_CHALLENGE_AUTHORIZER) { + authorizer->add_challenge(msgr->cct, authorizer_reply); + ldout(msgr->cct,10) << " got authorizer challenge, " << authorizer_reply.length() + << " bytes" << dendl; + continue; + } + if (authorizer) { bufferlist::iterator iter = authorizer_reply.begin(); if (!authorizer->verify_reply(iter)) { diff --git a/src/msg/simple/SimpleMessenger.cc b/src/msg/simple/SimpleMessenger.cc index f0ea282f1ac92..adf22166c9024 100644 --- a/src/msg/simple/SimpleMessenger.cc +++ b/src/msg/simple/SimpleMessenger.cc @@ -393,9 +393,12 @@ AuthAuthorizer *SimpleMessenger::get_authorizer(int peer_type, bool force_new) bool SimpleMessenger::verify_authorizer(Connection *con, int peer_type, int protocol, bufferlist& authorizer, bufferlist& authorizer_reply, - bool& isvalid,CryptoKey& session_key) + bool& isvalid,CryptoKey& session_key, + std::unique_ptr *challenge) { - return ms_deliver_verify_authorizer(con, peer_type, protocol, authorizer, authorizer_reply, isvalid,session_key); + return ms_deliver_verify_authorizer(con, peer_type, protocol, authorizer, authorizer_reply, + isvalid, session_key, + challenge); } ConnectionRef SimpleMessenger::get_connection(const entity_inst_t& dest) diff --git a/src/msg/simple/SimpleMessenger.h b/src/msg/simple/SimpleMessenger.h index 5eb441e86ef6d..e3fab1c8f50fa 100644 --- a/src/msg/simple/SimpleMessenger.h +++ b/src/msg/simple/SimpleMessenger.h @@ -346,8 +346,10 @@ public: /** * This wraps ms_deliver_verify_authorizer; we use it for Pipe. */ - bool verify_authorizer(Connection *con, int peer_type, int protocol, bufferlist& auth, bufferlist& auth_reply, - bool& isvalid,CryptoKey& session_key); + bool verify_authorizer(Connection *con, int peer_type, int protocol, bufferlist& auth, + bufferlist& auth_reply, + bool& isvalid,CryptoKey& session_key, + std::unique_ptr *challenge); /** * Increment the global sequence for this SimpleMessenger and return it. * This is for the connect protocol, although it doesn't hurt if somebody diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc index 72b36c0dcf752..e4579e4567bcd 100644 --- a/src/osd/OSD.cc +++ b/src/osd/OSD.cc @@ -6038,9 +6038,11 @@ bool OSD::ms_get_authorizer(int dest_type, AuthAuthorizer **authorizer, bool for } -bool OSD::ms_verify_authorizer(Connection *con, int peer_type, - int protocol, bufferlist& authorizer_data, bufferlist& authorizer_reply, - bool& isvalid, CryptoKey& session_key) +bool OSD::ms_verify_authorizer( + Connection *con, int peer_type, + int protocol, bufferlist& authorizer_data, bufferlist& authorizer_reply, + bool& isvalid, CryptoKey& session_key, + std::unique_ptr *challenge) { AuthAuthorizeHandler *authorize_handler = 0; switch (peer_type) { @@ -6066,8 +6068,16 @@ bool OSD::ms_verify_authorizer(Connection *con, int peer_type, uint64_t global_id; uint64_t auid = CEPH_AUTH_UID_DEFAULT; - isvalid = authorize_handler->verify_authorizer(cct, monc->rotating_secrets, - authorizer_data, authorizer_reply, name, global_id, caps_info, session_key, &auid); + RotatingKeyRing *keys = monc->rotating_secrets; + if (keys) { + isvalid = authorize_handler->verify_authorizer( + cct, keys, + authorizer_data, authorizer_reply, name, global_id, caps_info, session_key, + &auid, challenge); + } else { + dout(10) << __func__ << " no rotating_keys (yet), denied" << dendl; + isvalid = false; + } if (isvalid) { Session *s = static_cast(con->get_priv()); diff --git a/src/osd/OSD.h b/src/osd/OSD.h index 458a1ca94e75f..87ecff6a463e7 100644 --- a/src/osd/OSD.h +++ b/src/osd/OSD.h @@ -1631,7 +1631,8 @@ public: void ms_handle_remote_reset(Connection *con) {} bool ms_verify_authorizer(Connection *con, int peer_type, int protocol, bufferlist& authorizer_data, bufferlist& authorizer_reply, - bool& isvalid, CryptoKey& session_key) { + bool& isvalid, CryptoKey& session_key, + std::unique_ptr *challenge) override { isvalid = true; return true; } @@ -2379,12 +2380,13 @@ protected: bool ms_get_authorizer(int dest_type, AuthAuthorizer **authorizer, bool force_new); bool ms_verify_authorizer(Connection *con, int peer_type, int protocol, bufferlist& authorizer, bufferlist& authorizer_reply, - bool& isvalid, CryptoKey& session_key); - void ms_handle_connect(Connection *con); - void ms_handle_fast_connect(Connection *con); - void ms_handle_fast_accept(Connection *con); - bool ms_handle_reset(Connection *con); - void ms_handle_remote_reset(Connection *con) {} + bool& isvalid, CryptoKey& session_key, + std::unique_ptr *challenge) override; + void ms_handle_connect(Connection *con) override; + void ms_handle_fast_connect(Connection *con) override; + void ms_handle_fast_accept(Connection *con) override; + bool ms_handle_reset(Connection *con) override; + void ms_handle_remote_reset(Connection *con) override {} io_queue get_io_queue() const { if (cct->_conf->osd_op_queue == "debug_random") { diff --git a/src/test/messenger/simple_dispatcher.h b/src/test/messenger/simple_dispatcher.h index 0c08003b76101..6dea1a8dfe0d8 100644 --- a/src/test/messenger/simple_dispatcher.h +++ b/src/test/messenger/simple_dispatcher.h @@ -110,10 +110,11 @@ public: * @return True if we were able to prove or disprove correctness of * authorizer, false otherwise. */ - virtual bool ms_verify_authorizer(Connection *con, int peer_type, - int protocol, bufferlist& authorizer, - bufferlist& authorizer_reply, - bool& isvalid, CryptoKey& session_key) { + bool ms_verify_authorizer(Connection *con, int peer_type, + int protocol, bufferlist& authorizer, + bufferlist& authorizer_reply, + bool& isvalid, CryptoKey& session_key, + std::unique_ptr *challenge) override { /* always succeed */ isvalid = true; return true; diff --git a/src/test/messenger/xio_dispatcher.h b/src/test/messenger/xio_dispatcher.h index 29c71a0ba17ac..60253660b073e 100644 --- a/src/test/messenger/xio_dispatcher.h +++ b/src/test/messenger/xio_dispatcher.h @@ -113,7 +113,8 @@ public: virtual bool ms_verify_authorizer(Connection *con, int peer_type, int protocol, bufferlist& authorizer, bufferlist& authorizer_reply, - bool& isvalid, CryptoKey& session_key) { + bool& isvalid, CryptoKey& session_key, + std::unique_ptr *challenge) { /* always succeed */ isvalid = true; return true; diff --git a/src/test/msgr/perf_msgr_client.cc b/src/test/msgr/perf_msgr_client.cc index eb5434900be53..6a8b8cda0fbb6 100644 --- a/src/test/msgr/perf_msgr_client.cc +++ b/src/test/msgr/perf_msgr_client.cc @@ -56,7 +56,8 @@ class MessengerClient { void ms_handle_remote_reset(Connection *con) {} bool ms_verify_authorizer(Connection *con, int peer_type, int protocol, bufferlist& authorizer, bufferlist& authorizer_reply, - bool& isvalid, CryptoKey& session_key) { + bool& isvalid, CryptoKey& session_key, + std::unique_ptr *challenge) override { isvalid = true; return true; } diff --git a/src/test/msgr/perf_msgr_server.cc b/src/test/msgr/perf_msgr_server.cc index 7b31adc3bf5ad..324b9174b7640 100644 --- a/src/test/msgr/perf_msgr_server.cc +++ b/src/test/msgr/perf_msgr_server.cc @@ -100,7 +100,8 @@ class ServerDispatcher : public Dispatcher { } bool ms_verify_authorizer(Connection *con, int peer_type, int protocol, bufferlist& authorizer, bufferlist& authorizer_reply, - bool& isvalid, CryptoKey& session_key) { + bool& isvalid, CryptoKey& session_key, + std::unique_ptr *challenge) override { isvalid = true; return true; } diff --git a/src/test/msgr/test_msgr.cc b/src/test/msgr/test_msgr.cc index 37549227fc9e5..41ddeff468eea 100644 --- a/src/test/msgr/test_msgr.cc +++ b/src/test/msgr/test_msgr.cc @@ -197,7 +197,8 @@ class FakeDispatcher : public Dispatcher { bool ms_verify_authorizer(Connection *con, int peer_type, int protocol, bufferlist& authorizer, bufferlist& authorizer_reply, - bool& isvalid, CryptoKey& session_key) { + bool& isvalid, CryptoKey& session_key, + std::unique_ptr *challenge) override { isvalid = true; return true; } @@ -815,7 +816,8 @@ class SyntheticDispatcher : public Dispatcher { bool ms_verify_authorizer(Connection *con, int peer_type, int protocol, bufferlist& authorizer, bufferlist& authorizer_reply, - bool& isvalid, CryptoKey& session_key) { + bool& isvalid, CryptoKey& session_key, + std::unique_ptr *challenge) override { isvalid = true; return true; } @@ -1347,7 +1349,8 @@ class MarkdownDispatcher : public Dispatcher { } bool ms_verify_authorizer(Connection *con, int peer_type, int protocol, bufferlist& authorizer, bufferlist& authorizer_reply, - bool& isvalid, CryptoKey& session_key) { + bool& isvalid, CryptoKey& session_key, + std::unique_ptr *challenge) override { isvalid = true; return true; }