rgw: Handle multiple listening addreses w/ optional ssl "correctly" with civetweb.

For civetweb: accept a range of port numbers joined with '+'.
Port numbers may include an ipaddress: prefix and 's' suffix.
Additionally, use "mg_get_local_addr" to correctly deduce host port per
incoming connection.

civetweb can accept connections on multiple ports, some of which
might have SSL turned on and some not.  Both s3 and swift have various
authorization protocols in which the port number matters.  In the generic
radosgw frontend process, each frontend only has one port number, but
we should want to have both ssl and non-ssl connections managed within
one rgw frontend, because the thread pool is also per front-end, and
that *is* a scarce resource.

So, this patch enables the use of multiple ports with a single civetweb
frontend.  To indicate https: append an 's' to portno.  To use multiple
ports, use +.  So 80+443s indicates use of the usual default http ports.
The parsed port is not stored in the frontend structure,

So instead, this patch adds logic to use the results of
mg_get_local_addr() on a per-connection basis insetad of the generic
front-end port number.  This will affect "v4" s3 authorization, and also
affect swift pre-signed URLs.

mg_get_local_addr() is a new customization to civetweb; that submodule
was updated (in a temporary repository) by the previous commit to this.

Signed-off-by: Marcus Watts <mwatts@redhat.com>

diff --git a/src/rgw/rgw_civetweb.cc b/src/rgw/rgw_civetweb.cc
index 2ed6fa7acfc5dfcec996d6cf5d50eb2f8141bc6b..65283c61b1fb099053c94a67d51d39de51781e08 100644 (file)
--- a/src/rgw/rgw_civetweb.cc
+++ b/src/rgw/rgw_civetweb.cc
@@ -29,13 +29,23 @@ size_t RGWCivetWeb::write_data(const char *buf, const size_t len)
   return len;
 }
 
-RGWCivetWeb::RGWCivetWeb(mg_connection* const conn, const int port)
+RGWCivetWeb::RGWCivetWeb(mg_connection* const conn)
   : conn(conn),
-    port(port),
     explicit_keepalive(false),
     explicit_conn_close(false),
     txbuf(*this)
 {
+    sockaddr *lsa = mg_get_local_addr(conn);
+    switch(lsa->sa_family) {
+    case AF_INET:
+       port = ntohs(((struct sockaddr_in*)lsa)->sin_port);
+       break;
+    case AF_INET6:
+       port = ntohs(((struct sockaddr_in6*)lsa)->sin6_port);
+       break;
+    default:
+       port = -1;
+    }
 }
 
 size_t RGWCivetWeb::read_data(char *buf, size_t len)
@@ -110,14 +120,12 @@ void RGWCivetWeb::init_env(CephContext *cct)
     env.set("REMOTE_USER", info->remote_user);
   }
 
+  if (port <= 0)
+    lderr(cct) << "init_env: bug: invalid port number" << dendl;
   char port_buf[16];
   snprintf(port_buf, sizeof(port_buf), "%d", port);
   env.set("SERVER_PORT", port_buf);
-
   if (info->is_ssl) {
-    if (port == 0) {
-      strcpy(port_buf,"443");
-    }
     env.set("SERVER_PORT_SECURE", port_buf);
   }
 }

diff --git a/src/rgw/rgw_civetweb.h b/src/rgw/rgw_civetweb.h
index a6c5ea2567c4da1e6a2d0513161e0f0c9d432959..2be7c2ed143286348b0e1635a4e724de8a1f453a 100644 (file)
--- a/src/rgw/rgw_civetweb.h
+++ b/src/rgw/rgw_civetweb.h
@@ -52,7 +52,7 @@ public:
     return env;
   }
 
-  RGWCivetWeb(mg_connection *_conn, int _port);
+  RGWCivetWeb(mg_connection *_conn);
 };
 
 #endif

diff --git a/src/rgw/rgw_civetweb_frontend.cc b/src/rgw/rgw_civetweb_frontend.cc
index 17686f6e641aa6748bdb96620d0ccf579c07ca82..977883dc74d273dc2ee59c6a64e10fc09c29e261 100644 (file)
--- a/src/rgw/rgw_civetweb_frontend.cc
+++ b/src/rgw/rgw_civetweb_frontend.cc
@@ -22,7 +22,7 @@ int RGWCivetWebFrontend::process(struct mg_connection*  const conn)
   /* Hold a read lock over access to env.store for reconfiguration. */
   RWLock::RLocker lock(env.mutex);
 
-  RGWCivetWeb cw_client(conn, env.port);
+  RGWCivetWeb cw_client(conn);
   auto real_client_io = rgw::io::add_reordering(
                           rgw::io::add_buffering(
                             rgw::io::add_chunking(
@@ -45,14 +45,17 @@ int RGWCivetWebFrontend::process(struct mg_connection*  const conn)
 int RGWCivetWebFrontend::run()
 {
   auto& conf_map = conf->get_config_map();
+  string port_str;
 
   set_conf_default(conf_map, "num_threads",
                    std::to_string(g_conf->rgw_thread_pool_size));
   set_conf_default(conf_map, "decode_url", "no");
   set_conf_default(conf_map, "enable_keep_alive", "yes");
-  conf_map["listening_ports"] = conf->get_val("port", "80");
   set_conf_default(conf_map, "validate_http_method", "no");
   set_conf_default(conf_map, "canonicalize_url_path", "no");
+  conf->get_val("port", "80", &port_str);
+  std::replace(port_str.begin(), port_str.end(), '+', ',');
+  conf_map["listening_ports"] = port_str;
 
   /* Set run_as_user. This will cause civetweb to invoke setuid() and setgid()
    * based on pw_uid and pw_gid obtained from pw_name. */

diff --git a/src/rgw/rgw_main.cc b/src/rgw/rgw_main.cc
index 8138b8798d8c05911e41292a2958985d0e259443..4c4685ec1434b14bb09abe742a785ab3b002ca1c 100644 (file)
--- a/src/rgw/rgw_main.cc
+++ b/src/rgw/rgw_main.cc
@@ -187,7 +187,6 @@ static void reloader_handler(int signum)
   sighup_handler(signum);
 }
 
-
 /*
  * start up the RADOS connection and then handle HTTP messages as they come in
  */
@@ -470,12 +469,10 @@ int main(int argc, const char **argv)
 
       fe = new RGWFCGXFrontend(fcgi_pe, config);
     } else if (framework == "civetweb" || framework == "mongoose") {
-      int port;
-      config->get_val("port", 80, &port);
       std::string uri_prefix;
       config->get_val("prefix", "", &uri_prefix);
 
-      RGWProcessEnv env = { store, &rest, olog, port, uri_prefix };
+      RGWProcessEnv env = { store, &rest, olog, 0, uri_prefix };
 
       fe = new RGWCivetWebFrontend(env, config);
     } else if (framework == "loadgen") {