rgw: asio/beast add ssl hot-reload

Adds the `ssl_reload` config option to the beast frontend.
This sets an interval in seconds to periodically reload the ssl context to pick up changes without restarting. It can be disabled (default) be setting it to `0`.

Fixes: 65470
Signed-off-by: Henry Richter <henry.richter@cloudandheat.com>

diff --git a/doc/radosgw/frontends.rst b/doc/radosgw/frontends.rst
index cec1daf1de0658678530ad3ad8429cf2b5194f0c..6d3a361aa37149c8cfaf54d31fc8a634710fbe25 100644 (file)
--- a/doc/radosgw/frontends.rst
+++ b/doc/radosgw/frontends.rst
@@ -64,6 +64,17 @@ Options
 :Type: String
 :Default: None
 
+``ssl_reload``
+
+:Description: Optional interval in seconds to periodically recreate the SSL
+              context, which reloads the SSL certificate and private key from
+              their specified paths. A value of ``0`` disables this feature.
+              The reload is non-disruptive to existing connections. If the re-
+              load fails, the previous context continues to be used.
+
+:Type: Integer
+:Default: 0
+
 ``ssl_options``
 
 :Description: Optional colon separated list of SSL context options:

diff --git a/src/rgw/rgw_asio_frontend.cc b/src/rgw/rgw_asio_frontend.cc
index 7d8aa19adc3aa7c80d5dc2db9603ca3c632a8b96..2fbda94963d4cbe12821f08cd612667aad59d90a 100644 (file)
--- a/src/rgw/rgw_asio_frontend.cc
+++ b/src/rgw/rgw_asio_frontend.cc
@@ -2,6 +2,7 @@
 // vim: ts=8 sw=2 sts=2 expandtab ft=cpp
 
 #include <atomic>
+#include <chrono>
 #include <ctime>
 #include <iomanip>
 #include <list>
@@ -473,13 +474,20 @@ class AsioFrontend {
   ceph::timespan request_timeout = std::chrono::milliseconds(REQUEST_TIMEOUT);
   size_t header_limit = 16384;
 #ifdef WITH_RADOSGW_BEAST_OPENSSL
-  boost::optional<ssl::context> ssl_context;
+#ifdef __cpp_lib_atomic_shared_ptr
+  std::atomic<std::shared_ptr<ssl::context>> ssl_context;
+#else
+  std::shared_ptr<ssl::context> ssl_context;
+#endif
+  boost::asio::steady_timer ssl_reload_timer;
   int get_config_key_val(string name,
                          const string& type,
                          bufferlist *pbl);
-  int ssl_set_private_key(const string& name, bool is_ssl_cert);
-  int ssl_set_certificate_chain(const string& name);
-  int init_ssl();
+  int ssl_set_private_key(ssl::context& ctx, const string& name, bool is_ssl_cert);
+  int ssl_set_certificate_chain(ssl::context& ctx, const string& name);
+  int ssl_init();
+  int ssl_reload();
+  int ssl_reload_timer_start();
 #endif
   SharedMutex pause_mutex;
   std::unique_ptr<rgw::dmclock::Scheduler> scheduler;
@@ -514,6 +522,9 @@ class AsioFrontend {
               dmc::SchedulerCtx& sched_ctx,
               boost::asio::io_context& context)
     : env(env), conf(conf), context(context),
+#ifdef WITH_RADOSGW_BEAST_OPENSSL
+      ssl_reload_timer(context),
+#endif
       pause_mutex(context.get_executor()),
       backoff(context)
   {
@@ -669,10 +680,21 @@ int AsioFrontend::init()
   }
 
 #ifdef WITH_RADOSGW_BEAST_OPENSSL
-  int r = init_ssl();
+  int r = ssl_init();
   if (r < 0) {
     return r;
   }
+#ifdef __cpp_lib_atomic_shared_ptr
+  const auto ssl_ctx = ssl_context.load(std::memory_order_acquire);
+#else
+  const auto ssl_ctx = std::atomic_load_explicit(&ssl_context, std::memory_order_acquire);
+#endif
+  if (ssl_ctx != nullptr) {
+    r = ssl_reload_timer_start();
+    if (r < 0) {
+      return r;
+    }
+  }
 #endif
 
   // parse endpoints
@@ -887,12 +909,12 @@ int AsioFrontend::get_config_key_val(string name,
   return 0;
 }
 
-int AsioFrontend::ssl_set_private_key(const string& name, bool is_ssl_certificate)
+int AsioFrontend::ssl_set_private_key(ssl::context& ssl_ctx, const string& name, bool is_ssl_certificate)
 {
   boost::system::error_code ec;
 
   if (!boost::algorithm::starts_with(name, config_val_prefix)) {
-    ssl_context->use_private_key_file(name, ssl::context::pem, ec);
+    ssl_ctx.use_private_key_file(name, ssl::context::pem, ec);
   } else {
     bufferlist bl;
     int r = get_config_key_val(name.substr(config_val_prefix.size()),
@@ -901,7 +923,7 @@ int AsioFrontend::ssl_set_private_key(const string& name, bool is_ssl_certificat
     if (r < 0) {
       return r;
     }
-    ssl_context->use_private_key(boost::asio::buffer(bl.c_str(), bl.length()),
+    ssl_ctx.use_private_key(boost::asio::buffer(bl.c_str(), bl.length()),
                                  ssl::context::pem, ec);
   }
 
@@ -919,12 +941,12 @@ int AsioFrontend::ssl_set_private_key(const string& name, bool is_ssl_certificat
   return 0;
 }
 
-int AsioFrontend::ssl_set_certificate_chain(const string& name)
+int AsioFrontend::ssl_set_certificate_chain(ssl::context& ssl_ctx, const string& name)
 {
   boost::system::error_code ec;
 
   if (!boost::algorithm::starts_with(name, config_val_prefix)) {
-    ssl_context->use_certificate_chain_file(name, ec);
+    ssl_ctx.use_certificate_chain_file(name, ec);
   } else {
     bufferlist bl;
     int r = get_config_key_val(name.substr(config_val_prefix.size()),
@@ -933,7 +955,7 @@ int AsioFrontend::ssl_set_certificate_chain(const string& name)
     if (r < 0) {
       return r;
     }
-    ssl_context->use_certificate_chain(boost::asio::buffer(bl.c_str(), bl.length()),
+    ssl_ctx.use_certificate_chain(boost::asio::buffer(bl.c_str(), bl.length()),
                                  ec);
   }
 
@@ -946,21 +968,62 @@ int AsioFrontend::ssl_set_certificate_chain(const string& name)
   return 0;
 }
 
-int AsioFrontend::init_ssl()
+int AsioFrontend::ssl_init()
 {
   boost::system::error_code ec;
   auto& config = conf->get_config_map();
 
-  // ssl configuration
-  std::optional<string> cert = conf->get_val("ssl_certificate");
-  if (cert) {
-    // only initialize the ssl context if it's going to be used
-    ssl_context = boost::in_place(ssl::context::tls);
+  auto ports = config.equal_range("ssl_port");
+  auto endpoints = config.equal_range("ssl_endpoint");
+
+  /*
+   * don't try to config certificate if frontend isn't configured for ssl
+   */
+  if (ports.first == ports.second &&
+      endpoints.first == endpoints.second) {
+    return 0;
   }
 
-  std::optional<string> key = conf->get_val("ssl_private_key");
-  bool have_cert = false;
+  int r = ssl_reload();
+  if (r < 0) {
+    return r;
+  }
+
+  // parse ssl endpoints
+  for (auto i = ports.first; i != ports.second; ++i) {
+    auto port = parse_port(i->second.c_str(), ec);
+    if (ec) {
+      lderr(ctx()) << "failed to parse ssl_port=" << i->second << dendl;
+      return -ec.value();
+    }
+    listeners.emplace_back(context);
+    listeners.back().endpoint.port(port);
+    listeners.back().use_ssl = true;
+
+    listeners.emplace_back(context);
+    listeners.back().endpoint = tcp::endpoint(tcp::v6(), port);
+    listeners.back().use_ssl = true;
+  }
 
+  for (auto i = endpoints.first; i != endpoints.second; ++i) {
+    auto endpoint = parse_endpoint(i->second, 443, ec);
+    if (ec) {
+      lderr(ctx()) << "failed to parse ssl_endpoint=" << i->second << dendl;
+      return -ec.value();
+    }
+    listeners.emplace_back(context);
+    listeners.back().endpoint = endpoint;
+    listeners.back().use_ssl = true;
+  }
+
+  return 0;
+}
+
+int AsioFrontend::ssl_reload() {
+  const auto ssl_ctx = std::make_shared<ssl::context>(ssl::context(ssl::context::tls));
+
+  std::optional<string> cert = conf->get_val("ssl_certificate");
+  std::optional<string> key = conf->get_val("ssl_private_key");
   if (key && !cert) {
     lderr(ctx()) << "no ssl_certificate configured for ssl_private_key" << dendl;
     return -EINVAL;
@@ -979,21 +1042,21 @@ int AsioFrontend::init_ssl()
   if (options) {
     for (auto &option : ceph::split(*options, ":")) {
       if (option == "default_workarounds") {
-        ssl_context->set_options(ssl::context::default_workarounds);
+        ssl_ctx->set_options(ssl::context::default_workarounds);
       } else if (option == "no_compression") {
-        ssl_context->set_options(ssl::context::no_compression);
+        ssl_ctx->set_options(ssl::context::no_compression);
       } else if (option == "no_sslv2") {
-        ssl_context->set_options(ssl::context::no_sslv2);
+        ssl_ctx->set_options(ssl::context::no_sslv2);
       } else if (option == "no_sslv3") {
-        ssl_context->set_options(ssl::context::no_sslv3);
+        ssl_ctx->set_options(ssl::context::no_sslv3);
       } else if (option == "no_tlsv1") {
-        ssl_context->set_options(ssl::context::no_tlsv1);
+        ssl_ctx->set_options(ssl::context::no_tlsv1);
       } else if (option == "no_tlsv1_1") {
-        ssl_context->set_options(ssl::context::no_tlsv1_1);
+        ssl_ctx->set_options(ssl::context::no_tlsv1_1);
       } else if (option == "no_tlsv1_2") {
-        ssl_context->set_options(ssl::context::no_tlsv1_2);
+        ssl_ctx->set_options(ssl::context::no_tlsv1_2);
       } else if (option == "single_dh_use") {
-        ssl_context->set_options(ssl::context::single_dh_use);
+        ssl_ctx->set_options(ssl::context::single_dh_use);
       } else {
         lderr(ctx()) << "ignoring unknown ssl option '" << option << "'" << dendl;
       }
@@ -1007,8 +1070,7 @@ int AsioFrontend::init_ssl()
       return -EINVAL;
     }
 
-    int r = SSL_CTX_set_cipher_list(ssl_context->native_handle(),
-                                    ciphers->c_str());
+    int r = SSL_CTX_set_cipher_list(ssl_ctx->native_handle(), ciphers->c_str());
     if (r == 0) {
       lderr(ctx()) << "no cipher could be selected from ssl_ciphers: "
                    << *ciphers << dendl;
@@ -1016,19 +1078,8 @@ int AsioFrontend::init_ssl()
     }
   }
 
-  auto ports = config.equal_range("ssl_port");
-  auto endpoints = config.equal_range("ssl_endpoint");
-
-  /*
-   * don't try to config certificate if frontend isn't configured for ssl
-   */
-  if (ports.first == ports.second &&
-      endpoints.first == endpoints.second) {
-    return 0;
-  }
-
   bool key_is_cert = false;
-
+  bool have_cert = false;
   if (cert) {
     if (!key) {
       key = cert;
@@ -1036,59 +1087,71 @@ int AsioFrontend::init_ssl()
     }
 
     ExpandMetaVar emv(env.driver->get_zone());
-
     cert = emv.process_str(*cert);
     key = emv.process_str(*key);
 
-    int r = ssl_set_private_key(*key, key_is_cert);
+    int r = ssl_set_private_key(*ssl_ctx, *key, key_is_cert);
     bool have_private_key = (r >= 0);
     if (r < 0) {
       if (!key_is_cert) {
-        r = ssl_set_private_key(*cert, true);
+        r = ssl_set_private_key(*ssl_ctx, *cert, true);
         have_private_key = (r >= 0);
       }
     }
 
     if (have_private_key) {
-      int r = ssl_set_certificate_chain(*cert);
+      int r = ssl_set_certificate_chain(*ssl_ctx, *cert);
       have_cert = (r >= 0);
     }
   }
 
-  // parse ssl endpoints
-  for (auto i = ports.first; i != ports.second; ++i) {
-    if (!have_cert) {
-      lderr(ctx()) << "no ssl_certificate configured for ssl_port" << dendl;
-      return -EINVAL;
-    }
-    auto port = parse_port(i->second.c_str(), ec);
-    if (ec) {
-      lderr(ctx()) << "failed to parse ssl_port=" << i->second << dendl;
-      return -ec.value();
-    }
-    listeners.emplace_back(context);
-    listeners.back().endpoint.port(port);
-    listeners.back().use_ssl = true;
+  if (!have_cert) {
+    lderr(ctx()) << "no ssl_certificate configured" << dendl;
+    return -EINVAL;
+  }
 
-    listeners.emplace_back(context);
-    listeners.back().endpoint = tcp::endpoint(tcp::v6(), port);
-    listeners.back().use_ssl = true;
+#ifdef __cpp_lib_atomic_shared_ptr
+  ssl_context.store(ssl_ctx, std::memory_order_release);
+#else
+  std::atomic_store_explicit(&ssl_context, ssl_ctx, std::memory_order_release);
+#endif
+
+  return 0;
+}
+
+int AsioFrontend::ssl_reload_timer_start() {
+  const auto interval_str = conf->get_val("ssl_reload");
+  if (!interval_str) {
+    return 0;
   }
 
-  for (auto i = endpoints.first; i != endpoints.second; ++i) {
-    if (!have_cert) {
-      lderr(ctx()) << "no ssl_certificate configured for ssl_endpoint" << dendl;
-      return -EINVAL;
-    }
-    auto endpoint = parse_endpoint(i->second, 443, ec);
+  const auto interval = ceph::parse<uint64_t>(*interval_str);
+  if (!interval) {
+    lderr(ctx()) << "failed to parse ssl_reload=" << *interval_str << dendl;
+    return -EINVAL;
+  };
+
+  if (*interval == 0) {
+    return 0;
+  }
+
+  ssl_reload_timer.expires_after(std::chrono::seconds(*interval));
+  ssl_reload_timer.async_wait([this](const boost::system::error_code &ec) {
     if (ec) {
-      lderr(ctx()) << "failed to parse ssl_endpoint=" << i->second << dendl;
-      return -ec.value();
+      return;
     }
-    listeners.emplace_back(context);
-    listeners.back().endpoint = endpoint;
-    listeners.back().use_ssl = true;
-  }
+
+    ldout(ctx(), 4) << "ssl reload triggered" << dendl;
+    if (ssl_reload() < 0) {
+      lderr(ctx()) << "ssl reload failed, continuing with existing context"
+                   << dendl;
+    } else {
+      ldout(ctx(), 4) << "ssl reload successful" << dendl;
+    }
+
+    ssl_reload_timer_start();
+  });
+
   return 0;
 }
 #endif // WITH_RADOSGW_BEAST_OPENSSL
@@ -1126,16 +1189,21 @@ void AsioFrontend::on_accept(Listener& l, tcp::socket stream)
 {
   boost::system::error_code ec;
   stream.set_option(tcp::no_delay(l.use_nodelay), ec);
-  
+
   // spawn a coroutine to handle the connection
 #ifdef WITH_RADOSGW_BEAST_OPENSSL
   if (l.use_ssl) {
+#ifdef __cpp_lib_atomic_shared_ptr
+    const auto ssl_ctx = ssl_context.load(std::memory_order_acquire);
+#else
+    const auto ssl_ctx = std::atomic_load_excplicit(&ssl_context, std::memory_order_acquire);
+#endif
     boost::asio::spawn(make_strand(context), std::allocator_arg, make_stack_allocator(),
-      [this, s=std::move(stream)] (boost::asio::yield_context yield) mutable {
+      [this, s=std::move(stream), ssl_ctx] (boost::asio::yield_context yield) mutable {
         auto conn = boost::intrusive_ptr{new Connection(std::move(s))};
         auto c = connections.add(*conn);
         // wrap the tcp stream in an ssl stream
-        boost::asio::ssl::stream<tcp::socket&> stream{conn->socket, *ssl_context};
+        boost::asio::ssl::stream<tcp::socket&> stream{conn->socket, *ssl_ctx};
         auto timeout = timeout_timer{context.get_executor(), request_timeout, conn};
         // do ssl handshake
         boost::system::error_code ec;
@@ -1196,6 +1264,7 @@ void AsioFrontend::stop()
     // signal cancellation of accept()
     listener.signal.emit(boost::asio::cancellation_type::terminal);
   }
+  ssl_reload_timer.cancel();
 
   const bool graceful_stop{ g_ceph_context->_conf->rgw_graceful_stop };
   if (graceful_stop) {