From: Marcus Watts <mwatts@redhat.com>
Date: Wed, 3 Aug 2016 21:36:55 +0000 (-0400)
Subject: rgw: Handle multiple listening addreses w/ optional ssl "correctly" with civetweb.
X-Git-Tag: v12.0.0~269^2
X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=8bc6decc0774fae9ac881f91e73da55deebe3360;p=ceph.git

rgw: Handle multiple listening addreses w/ optional ssl "correctly" with civetweb.

For civetweb: accept a range of port numbers joined with '+'.
Port numbers may include an ipaddress: prefix and 's' suffix.
Additionally, use "mg_get_local_addr" to correctly deduce host port per
incoming connection.

civetweb can accept connections on multiple ports, some of which
might have SSL turned on and some not.  Both s3 and swift have various
authorization protocols in which the port number matters.  In the generic
radosgw frontend process, each frontend only has one port number, but
we should want to have both ssl and non-ssl connections managed within
one rgw frontend, because the thread pool is also per front-end, and
that *is* a scarce resource.

So, this patch enables the use of multiple ports with a single civetweb
frontend.  To indicate https: append an 's' to portno.  To use multiple
ports, use +.  So 80+443s indicates use of the usual default http ports.
The parsed port is not stored in the frontend structure,

So instead, this patch adds logic to use the results of
mg_get_local_addr() on a per-connection basis insetad of the generic
front-end port number.  This will affect "v4" s3 authorization, and also
affect swift pre-signed URLs.

mg_get_local_addr() is a new customization to civetweb; that submodule
was updated (in a temporary repository) by the previous commit to this.

Signed-off-by: Marcus Watts <mwatts@redhat.com>
---

diff --git a/src/rgw/rgw_civetweb.cc b/src/rgw/rgw_civetweb.cc
index 2ed6fa7acfc5..65283c61b1fb 100644
--- a/src/rgw/rgw_civetweb.cc
+++ b/src/rgw/rgw_civetweb.cc
@@ -29,13 +29,23 @@ size_t RGWCivetWeb::write_data(const char *buf, const size_t len)
   return len;
 }
 
-RGWCivetWeb::RGWCivetWeb(mg_connection* const conn, const int port)
+RGWCivetWeb::RGWCivetWeb(mg_connection* const conn)
   : conn(conn),
-    port(port),
     explicit_keepalive(false),
     explicit_conn_close(false),
     txbuf(*this)
 {
+    sockaddr *lsa = mg_get_local_addr(conn);
+    switch(lsa->sa_family) {
+    case AF_INET:
+	port = ntohs(((struct sockaddr_in*)lsa)->sin_port);
+	break;
+    case AF_INET6:
+	port = ntohs(((struct sockaddr_in6*)lsa)->sin6_port);
+	break;
+    default:
+	port = -1;
+    }
 }
 
 size_t RGWCivetWeb::read_data(char *buf, size_t len)
@@ -110,14 +120,12 @@ void RGWCivetWeb::init_env(CephContext *cct)
     env.set("REMOTE_USER", info->remote_user);
   }
 
+  if (port <= 0)
+    lderr(cct) << "init_env: bug: invalid port number" << dendl;
   char port_buf[16];
   snprintf(port_buf, sizeof(port_buf), "%d", port);
   env.set("SERVER_PORT", port_buf);
-
   if (info->is_ssl) {
-    if (port == 0) {
-      strcpy(port_buf,"443");
-    }
     env.set("SERVER_PORT_SECURE", port_buf);
   }
 }
diff --git a/src/rgw/rgw_civetweb.h b/src/rgw/rgw_civetweb.h
index a6c5ea2567c4..2be7c2ed1432 100644
--- a/src/rgw/rgw_civetweb.h
+++ b/src/rgw/rgw_civetweb.h
@@ -52,7 +52,7 @@ public:
     return env;
   }
 
-  RGWCivetWeb(mg_connection *_conn, int _port);
+  RGWCivetWeb(mg_connection *_conn);
 };
 
 #endif
diff --git a/src/rgw/rgw_civetweb_frontend.cc b/src/rgw/rgw_civetweb_frontend.cc
index 17686f6e641a..977883dc74d2 100644
--- a/src/rgw/rgw_civetweb_frontend.cc
+++ b/src/rgw/rgw_civetweb_frontend.cc
@@ -22,7 +22,7 @@ int RGWCivetWebFrontend::process(struct mg_connection*  const conn)
   /* Hold a read lock over access to env.store for reconfiguration. */
   RWLock::RLocker lock(env.mutex);
 
-  RGWCivetWeb cw_client(conn, env.port);
+  RGWCivetWeb cw_client(conn);
   auto real_client_io = rgw::io::add_reordering(
                           rgw::io::add_buffering(
                             rgw::io::add_chunking(
@@ -45,14 +45,17 @@ int RGWCivetWebFrontend::process(struct mg_connection*  const conn)
 int RGWCivetWebFrontend::run()
 {
   auto& conf_map = conf->get_config_map();
+  string port_str;
 
   set_conf_default(conf_map, "num_threads",
                    std::to_string(g_conf->rgw_thread_pool_size));
   set_conf_default(conf_map, "decode_url", "no");
   set_conf_default(conf_map, "enable_keep_alive", "yes");
-  conf_map["listening_ports"] = conf->get_val("port", "80");
   set_conf_default(conf_map, "validate_http_method", "no");
   set_conf_default(conf_map, "canonicalize_url_path", "no");
+  conf->get_val("port", "80", &port_str);
+  std::replace(port_str.begin(), port_str.end(), '+', ',');
+  conf_map["listening_ports"] = port_str;
 
   /* Set run_as_user. This will cause civetweb to invoke setuid() and setgid()
    * based on pw_uid and pw_gid obtained from pw_name. */
diff --git a/src/rgw/rgw_main.cc b/src/rgw/rgw_main.cc
index 8138b8798d8c..4c4685ec1434 100644
--- a/src/rgw/rgw_main.cc
+++ b/src/rgw/rgw_main.cc
@@ -187,7 +187,6 @@ static void reloader_handler(int signum)
   sighup_handler(signum);
 }
 
-
 /*
  * start up the RADOS connection and then handle HTTP messages as they come in
  */
@@ -470,12 +469,10 @@ int main(int argc, const char **argv)
 
       fe = new RGWFCGXFrontend(fcgi_pe, config);
     } else if (framework == "civetweb" || framework == "mongoose") {
-      int port;
-      config->get_val("port", 80, &port);
       std::string uri_prefix;
       config->get_val("prefix", "", &uri_prefix);
 
-      RGWProcessEnv env = { store, &rest, olog, port, uri_prefix };
+      RGWProcessEnv env = { store, &rest, olog, 0, uri_prefix };
 
       fe = new RGWCivetWebFrontend(env, config);
     } else if (framework == "loadgen") {