From 660977925dde39c1bd0e6bb2a1f4ac6ce8d02e37 Mon Sep 17 00:00:00 2001 From: Yehuda Sadeh Date: Tue, 22 Sep 2009 16:51:09 -0700 Subject: [PATCH] auth: monclient creates session asynchronously --- src/auth/AuthClientHandler.cc | 61 ++++++++++++++++++++++++++++++----- src/auth/AuthClientHandler.h | 3 ++ src/auth/KeyRing.cc | 16 +++++++++ src/auth/KeyRing.h | 2 ++ src/auth/KeysServer.cc | 24 +++++++++----- src/auth/KeysServer.h | 6 ++-- src/mon/MonClient.cc | 44 +++++++++++++++++++++---- src/mon/MonClient.h | 18 +++++++++-- 8 files changed, 146 insertions(+), 28 deletions(-) diff --git a/src/auth/AuthClientHandler.cc b/src/auth/AuthClientHandler.cc index 16af87f987653..879d5a16830c0 100644 --- a/src/auth/AuthClientHandler.cc +++ b/src/auth/AuthClientHandler.cc @@ -23,7 +23,7 @@ AuthClientProtocolHandler::AuthClientProtocolHandler(AuthClientHandler *client) : msg(NULL), got_response(false), got_timeout(false), - timeout_event(NULL) + timeout_event(NULL), lock("AuthClientProtocolHandler") { dout(0) << "AuthClientProtocolHandler::AuthClientProtocolHandler" << dendl; this->client = client; @@ -61,21 +61,47 @@ int AuthClientProtocolHandler::do_request(double timeout) timeout_event = new C_OpTimeout(this, timeout); client->timer.add_event_after(timeout, timeout_event); - cond.Wait(client->lock); + cond.Wait(lock); dout(0) << "got_response=" << got_response << " got_timeout=" << got_timeout << dendl; // finish. - client->timer.cancel_event(timeout_event); - timeout_event = NULL; + if (timeout_event) { + client->timer.cancel_event(timeout_event); + timeout_event = NULL; + } return status; } +int AuthClientProtocolHandler::do_async_request(double timeout) +{ + got_response = false; + client->client->send_message(msg); + +#if 0 + // schedule timeout? + assert(timeout_event == 0); + timeout_event = new C_OpTimeout(this, timeout); + client->timer.add_event_after(timeout, timeout_event); + + + dout(0) << "got_response=" << got_response << " got_timeout=" << got_timeout << dendl; + + // finish. + if (timeout_event) { + client->timer.cancel_event(timeout_event); + timeout_event = NULL; + } +#endif + + return 0; +} + void AuthClientProtocolHandler::_request_timeout(double timeout) { dout(10) << "_request_timeout" << dendl; - timeout_event = 0; + timeout_event = NULL; if (!got_response) { got_timeout = 1; cond.Signal(); @@ -85,12 +111,14 @@ void AuthClientProtocolHandler::_request_timeout(double timeout) int AuthClientProtocolHandler::handle_response(int ret, bufferlist::iterator& iter) { - Mutex::Locker l(client->lock); + if (!client) { + derr(0) << "AuthClientProtocolHandler::handle_response() but client is NULL" << dendl; + return -EINVAL; + } - got_response = true; + Mutex::Locker l(lock); status = _handle_response(ret, iter); - cond.Signal(); return status; } @@ -394,6 +422,23 @@ int AuthClientHandler::start_session(AuthClient *client, double timeout) return err; } +int AuthClientHandler::send_session_request(AuthClient *client, AuthClientProtocolHandler *handler, double timeout) +{ + Mutex::Locker l(lock); + this->client = client; + dout(10) << "start_session" << dendl; + + int err = handler->build_request(); + dout(0) << "handler.build_request returned " << err << dendl; + if (err < 0) + return err; + + err = handler->do_async_request(timeout); + dout(0) << "handler.do_async_request returned " << err << dendl; + + return err; +} + int AuthClientHandler::authorize(uint32_t service_id, double timeout) { Mutex::Locker l(lock); diff --git a/src/auth/AuthClientHandler.h b/src/auth/AuthClientHandler.h index 4acc4e7b99f1b..fcf12846b986b 100644 --- a/src/auth/AuthClientHandler.h +++ b/src/auth/AuthClientHandler.h @@ -52,6 +52,7 @@ protected: bool got_timeout; Context *timeout_event; uint32_t id; + Mutex lock; // session state int status; @@ -76,6 +77,7 @@ public: int handle_response(int ret, bufferlist::iterator& iter); int do_request(double timeout); + int do_async_request(double timeout); }; class AuthClientAuthenticateHandler : public AuthClientProtocolHandler { @@ -186,6 +188,7 @@ public: int handle_response(Message *response); int start_session(AuthClient *client, double timeout); + int send_session_request(AuthClient *client, AuthClientProtocolHandler *handler, double timeout); int authorize(uint32_t service_id, double timeout); void tick(); }; diff --git a/src/auth/KeyRing.cc b/src/auth/KeyRing.cc index 0064e12010119..aa22ba75bd6b2 100644 --- a/src/auth/KeyRing.cc +++ b/src/auth/KeyRing.cc @@ -108,3 +108,19 @@ void KeyRing::get_master(CryptoKey& dest) dest = master; } +bool KeyRing::need_rotating_secrets() +{ + Mutex::Locker l(lock); + + if (rotating_secrets.secrets.size() < KEY_ROTATE_NUM) + return true; + + map::iterator iter = rotating_secrets.secrets.lower_bound(0); + ExpiringCryptoKey& key = iter->second; + if (key.expiration < g_clock.now()) { + dout(0) << "key.expiration=" << key.expiration << " now=" << g_clock.now() << dendl; + return true; + } + + return false; +} diff --git a/src/auth/KeyRing.h b/src/auth/KeyRing.h index 69457153b4f1b..5ecd5b65aa246 100644 --- a/src/auth/KeyRing.h +++ b/src/auth/KeyRing.h @@ -36,6 +36,8 @@ public: void set_rotating(RotatingSecrets& secrets); void get_master(CryptoKey& dest); + + bool need_rotating_secrets(); }; diff --git a/src/auth/KeysServer.cc b/src/auth/KeysServer.cc index 143e2c73676f0..b7562dcced5af 100644 --- a/src/auth/KeysServer.cc +++ b/src/auth/KeysServer.cc @@ -77,7 +77,7 @@ int KeysServer::start_server(bool init) Mutex::Locker l(lock); if (init) { - _generate_all_rotating_secrets(); + _generate_all_rotating_secrets(init); } rotate_event = new C_RotateTimeout(this, KEY_ROTATE_TIME); if (!rotate_event) @@ -86,13 +86,21 @@ int KeysServer::start_server(bool init) return 0; } -void KeysServer::_generate_all_rotating_secrets() +void KeysServer::_generate_all_rotating_secrets(bool init) { data.rotating_ver++; dout(0) << "generate_all_rotating_secrets()" << dendl; - _rotate_secret(CEPHX_PRINCIPAL_MON); - _rotate_secret(CEPHX_PRINCIPAL_OSD); - _rotate_secret(CEPHX_PRINCIPAL_MDS); + + int i = KEY_ROTATE_NUM; + + if (init) + i = 1; + + for (; i <= KEY_ROTATE_NUM; i++) { + _rotate_secret(CEPHX_PRINCIPAL_MON, i); + _rotate_secret(CEPHX_PRINCIPAL_OSD, i); + _rotate_secret(CEPHX_PRINCIPAL_MDS, i); + } dout(0) << "generated: " << dendl; @@ -112,12 +120,12 @@ void KeysServer::_generate_all_rotating_secrets() } } -void KeysServer::_rotate_secret(uint32_t service_id) +void KeysServer::_rotate_secret(uint32_t service_id, int factor) { ExpiringCryptoKey ek; generate_secret(ek.key); ek.expiration = g_clock.now(); - ek.expiration += (KEY_ROTATE_TIME * 3); + ek.expiration += (KEY_ROTATE_TIME * factor); data.add_rotating_secret(service_id, ek); } @@ -125,7 +133,7 @@ void KeysServer::_rotate_secret(uint32_t service_id) void KeysServer::rotate_timeout(double timeout) { dout(0) << "KeysServer::rotate_timeout" << dendl; - _generate_all_rotating_secrets(); + _generate_all_rotating_secrets(false); rotate_event = new C_RotateTimeout(this, timeout); timer.add_event_after(timeout, rotate_event); diff --git a/src/auth/KeysServer.h b/src/auth/KeysServer.h index e100d2b96dab9..b233ea88faa6a 100644 --- a/src/auth/KeysServer.h +++ b/src/auth/KeysServer.h @@ -21,7 +21,7 @@ #include "common/Timer.h" #include "Auth.h" -#define KEY_ROTATE_TIME 5 +#define KEY_ROTATE_TIME 20 #define KEY_ROTATE_NUM 3 @@ -100,8 +100,8 @@ class KeysServer { Context *rotate_event; bool generate_secret(CryptoKey& secret); - void _rotate_secret(uint32_t service_id); - void _generate_all_rotating_secrets(); + void _rotate_secret(uint32_t service_id, int factor); + void _generate_all_rotating_secrets(bool init); public: KeysServer(); diff --git a/src/mon/MonClient.cc b/src/mon/MonClient.cc index 45e40128cace7..53030d39f3dc3 100644 --- a/src/mon/MonClient.cc +++ b/src/mon/MonClient.cc @@ -187,7 +187,9 @@ bool MonClient::ms_dispatch(Message *m) switch (m->get_type()) { case CEPH_MSG_MON_MAP: + dout(0) << "CEPH_MSG_MON_MAP begin" << dendl; handle_monmap((MMonMap*)m); + dout(0) << "CEPH_MSG_MON_MAP end" << dendl; return true; case CEPH_MSG_CLIENT_MOUNT_ACK: @@ -195,8 +197,7 @@ bool MonClient::ms_dispatch(Message *m) return true; case CEPH_MSG_AUTH_REPLY: - auth.handle_response((MAuthReply*)m); - delete m; + handle_auth((MAuthReply*)m); return true; case CEPH_MSG_MON_SUBSCRIBE_ACK: @@ -302,6 +303,20 @@ void MonClient::handle_mount_ack(MClientMountAck* m) delete m; } +void MonClient::handle_auth(MAuthReply *m) +{ + int ret = auth.handle_response((MAuthReply*)m); + delete m; + + if (ret == -EAGAIN) { + auth.send_session_request(this, &auth_handler, 30.0); + } else { + state = MC_STATE_AUTHENTICATED; + _reopen_session(); + } +} + + int MonClient::authenticate(double timeout) { Mutex::Locker lock(monc_lock); @@ -338,9 +353,17 @@ void MonClient::_reopen_session() _pick_new_mon(); dout(0) << "_reopen_session 0" << dendl; - auth.start_session(this, 30.0); - dout(0) << "_reopen_session 1" << dendl; - _start_auth_rotating(KEY_ROTATE_TIME); + if (state == MC_STATE_NONE) { + state = MC_STATE_AUTHENTICATING; + auth.send_session_request(this, &auth_handler, 30.0); + return; + } + + if (state == MC_STATE_AUTHENTICATING) + return; + + if (keyring && keyring->need_rotating_secrets()) + _start_auth_rotating(KEY_ROTATE_TIME); dout(0) << "_reopen_session 2" << dendl; if (mounting) @@ -376,6 +399,11 @@ void MonClient::tick() { dout(10) << "tick" << dendl; + if (keyring && keyring->need_rotating_secrets()) { + dout(0) << "MonClient::tick: need rotating secret" << dendl; + _start_auth_rotating(KEY_ROTATE_TIME); + } + if (hunting) { dout(0) << "continuing hunt" << dendl; auth.start_session(this, 30.0); @@ -466,8 +494,10 @@ int MonClient::_start_auth_rotating(double timeout) return -ETIMEDOUT; } - timer.cancel_event(auth_timeout_event); - auth_timeout_event = NULL; + if (auth_timeout_event) { + timer.cancel_event(auth_timeout_event); + auth_timeout_event = NULL; + } return 0; } diff --git a/src/mon/MonClient.h b/src/mon/MonClient.h index 87b52aec0e6d2..1cb9f3e631044 100644 --- a/src/mon/MonClient.h +++ b/src/mon/MonClient.h @@ -34,10 +34,19 @@ class MClientMountAck; class MMonSubscribeAck; class MAuthRotating; + +enum MonClientState { + MC_STATE_NONE, + MC_STATE_AUTHENTICATING, + MC_STATE_AUTHENTICATED, +}; + class MonClient : public Dispatcher, public AuthClient { public: MonMap monmap; private: + MonClientState state; + Messenger *messenger; int cur_mon; @@ -57,6 +66,8 @@ private: void handle_monmap(MMonMap *m); + void handle_auth(MAuthReply *m); + // monitor session bool hunting; @@ -150,6 +161,7 @@ private: // auth tickets public: AuthClientHandler auth; + AuthClientAuthenticateHandler auth_handler; double auth_timeout; public: void renew_subs() { @@ -170,14 +182,16 @@ public: } public: - MonClient() : messenger(NULL), cur_mon(-1), + MonClient() : state(MC_STATE_NONE), + messenger(NULL), cur_mon(-1), monc_lock("MonClient::monc_lock"), timer(monc_lock), hunting(false), auth_timeout_event(NULL), auth_got_timeout(false), keyring(NULL), - mounting(0), mount_err(0) { } + mounting(0), mount_err(0), + auth_handler(&auth, 0, 0) { } ~MonClient() { timer.cancel_all_events(); } -- 2.39.5