rgw: add control logic for chunked transfer encoding.

Needed for tempest.api.object_storage.test_object_slo.ObjectSloTest.test_delete_large_object.

Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>

diff --git a/src/rgw/rgw_client_io.h b/src/rgw/rgw_client_io.h
index 05879e4c47f3d42c7c4994dc0dfed704c15910d7..e85b47f9c9882b370a6e651d13e0a138540fc238 100644 (file)
--- a/src/rgw/rgw_client_io.h
+++ b/src/rgw/rgw_client_io.h
@@ -64,7 +64,24 @@ public:
   virtual std::size_t send_header(const boost::string_ref& name,
                                   const boost::string_ref& value) = 0;
 
+  /* Inform a client about a content length. Takes number of bytes supplied in
+   * @len XOR one of the alternative modes for dealing with it passed as @mode.
+   * On success returns number of bytes sent to the direct client of RadosGW.
+   * On failure throws int containing errno.
+   *
+   * CALL ORDER:
+   *  - The method must be called EXACTLY ONE time.
+   *  - The method must be preceeded with a call to send_status().
+   *  - The method must not be called after complete_header(). */
   virtual std::size_t send_content_length(uint64_t len) = 0;
+
+  virtual std::size_t send_chunked_transfer_encoding() {
+    /* This is a null implementation. We don't send anything here, even the HTTP
+     * header. The intended behaviour should be provided through a decorator or
+     * directly by a given front-end. */
+    return 0;
+  }
+
   virtual std::size_t complete_header() = 0;
 
   /* Receive body. On success Returns number of bytes sent to the direct
@@ -98,6 +115,7 @@ public:
   virtual std::size_t send_header(const boost::string_ref& name,
                                   const boost::string_ref& value) noexcept = 0;
   virtual int send_content_length(uint64_t len) = 0;
+  virtual int send_chunked_transfer_encoding() = 0;
   virtual int complete_header() = 0;
 
   virtual int recv_body(char* buf, std::size_t max) = 0;
@@ -186,6 +204,10 @@ public:
     EXCPT_TO_RC(get_decoratee().send_content_length(len));
   }
 
+  int send_chunked_transfer_encoding() override {
+    EXCPT_TO_RC(get_decoratee().send_chunked_transfer_encoding());
+  }
+
   int complete_header() override {
     EXCPT_TO_RC(get_decoratee().complete_header());
   }

diff --git a/src/rgw/rgw_client_io_decoimpl.h b/src/rgw/rgw_client_io_decoimpl.h
index ba50a7a60008fd63e30693d6942b4ed3e4c34dca..b492383accda2d627c7e5a19fc9362cac8887629 100644 (file)
--- a/src/rgw/rgw_client_io_decoimpl.h
+++ b/src/rgw/rgw_client_io_decoimpl.h
@@ -69,6 +69,10 @@ public:
     return get_decoratee().send_content_length(len);
   }
 
+  std::size_t send_chunked_transfer_encoding() override {
+    return get_decoratee().send_chunked_transfer_encoding();
+  }
+
   std::size_t complete_header() override {
     return get_decoratee().complete_header();
   }
@@ -146,6 +150,14 @@ public:
     return sent;
   }
 
+  std::size_t send_chunked_transfer_encoding() override {
+    const auto sent = RGWDecoratedStreamIO<T>::send_chunked_transfer_encoding();
+    if (enabled) {
+      total_sent += sent;
+    }
+    return sent;
+  }
+
   std::size_t complete_header() override {
     const auto sent = RGWDecoratedStreamIO<T>::complete_header();
     if (enabled) {
@@ -205,6 +217,7 @@ public:
   }
 
   std::size_t send_content_length(const uint64_t len) override;
+  std::size_t send_chunked_transfer_encoding() override;
   std::size_t complete_header() override;
   std::size_t send_body(const char* buf, std::size_t len) override;
   int complete_request() override;
@@ -229,6 +242,13 @@ std::size_t RGWStreamIOBufferingEngine<T>::send_content_length(const uint64_t le
   return RGWDecoratedStreamIO<T>::send_content_length(len);
 }
 
+template <typename T>
+std::size_t RGWStreamIOBufferingEngine<T>::send_chunked_transfer_encoding()
+{
+  has_content_length = true;
+  return RGWDecoratedStreamIO<T>::send_chunked_transfer_encoding();
+}
+
 template <typename T>
 std::size_t RGWStreamIOBufferingEngine<T>::complete_header()
 {
@@ -286,16 +306,10 @@ public:
     return RGWDecoratedStreamIO<T>::send_content_length(len);
   }
 
-  std::size_t complete_header() override {
-    size_t sent = 0;
-
-    if (! has_content_length) {
-      sent += RGWDecoratedStreamIO<T>::send_header("Transfer-Enconding",
-                                                   "chunked");
-      chunking_enabled = true;
-    }
-
-    return sent + RGWDecoratedStreamIO<T>::complete_header();
+  std::size_t send_chunked_transfer_encoding() override {
+    has_content_length = false;
+    chunking_enabled = true;
+    return RGWDecoratedStreamIO<T>::send_header("Transfer-Encoding", "chunked");
   }
 
   std::size_t send_body(const char* buf,
@@ -315,6 +329,18 @@ public:
       return sent;
     }
   }
+
+  int complete_request() override {
+    std::size_t sent = 0;
+
+    if (chunking_enabled) {
+      static constexpr char CHUNKED_RESP_END[] = "0\r\n\r\n";
+      sent += RGWDecoratedStreamIO<T>::send_body(CHUNKED_RESP_END,
+                                                 sizeof(CHUNKED_RESP_END) - 1);
+    }
+
+    return sent + RGWDecoratedStreamIO<T>::complete_request();
+  }
 };
 
 template <typename T>

diff --git a/src/rgw/rgw_rest.cc b/src/rgw/rgw_rest.cc
index 37555b9a052b831cf7a3da7145c7d0de501cc733..7434e71a7721484ec127967e9773ff7e712f179c 100644 (file)
--- a/src/rgw/rgw_rest.cc
+++ b/src/rgw/rgw_rest.cc
@@ -419,6 +419,16 @@ void dump_content_length(struct req_state* const s, const uint64_t len)
   dump_header(s, "Accept-Ranges", "bytes");
 }
 
+static void dump_chunked_encoding(struct req_state* const s)
+{
+  try {
+    STREAM_IO(s)->send_chunked_transfer_encoding();
+  } catch (RGWRestfulIOEngine::Exception& e) {
+    ldout(s->cct, 0) << "ERROR: STREAM_IO(s)->send_chunked_transfer_encoding()"
+                     << " returned err=" << e.what() << dendl;
+  }
+}
+
 void dump_etag(struct req_state* const s,
                const boost::string_ref& etag,
                const bool quoted)
@@ -698,7 +708,9 @@ void end_header(struct req_state* s, RGWOp* op, const char *content_type,
     s->formatter->output_footer();
     dump_content_length(s, s->formatter->get_len());
   } else {
-    if (proposed_content_length != NO_CONTENT_LENGTH) {
+    if (proposed_content_length == CHUNKED_TRANSFER_ENCODING) {
+      dump_chunked_encoding(s);
+    } else if (proposed_content_length != NO_CONTENT_LENGTH) {
       dump_content_length(s, proposed_content_length);
     }
   }

diff --git a/src/rgw/rgw_rest.h b/src/rgw/rgw_rest.h
index 01413ae72ab6312f480d63a940ab57c55f201746..63d5c50fd06b3e412e442cdb641341d3b3d227d4 100644 (file)
--- a/src/rgw/rgw_rest.h
+++ b/src/rgw/rgw_rest.h
@@ -513,6 +513,7 @@ public:
 };
 
 static constexpr int64_t NO_CONTENT_LENGTH = -1;
+static constexpr int64_t CHUNKED_TRANSFER_ENCODING = -2;
 
 extern void set_req_state_err(struct rgw_err &err, int err_no, int prot_flags);
 extern void set_req_state_err(struct req_state *s, int err_no);

diff --git a/src/rgw/rgw_rest_swift.cc b/src/rgw/rgw_rest_swift.cc
index 5f8c553002eb446b666f1d2f57f45c9756a82c40..ab0aa6ca07e25394af9d728915cb56e135262404 100644 (file)
--- a/src/rgw/rgw_rest_swift.cc
+++ b/src/rgw/rgw_rest_swift.cc
@@ -1031,7 +1031,7 @@ void RGWDeleteObj_ObjStore_SWIFT::send_response()
 
   if (multipart_delete) {
     end_header(s, this /* RGWOp */, nullptr /* contype */,
-               NO_CONTENT_LENGTH);
+               CHUNKED_TRANSFER_ENCODING);
 
     if (deleter) {
       bulkdelete_respond(deleter->get_num_deleted(),
@@ -1387,7 +1387,8 @@ void RGWBulkDelete_ObjStore_SWIFT::send_response()
 {
   set_req_state_err(s, op_ret);
   dump_errno(s);
-  end_header(s, NULL);
+  end_header(s, this /* RGWOp */, nullptr /* contype */,
+             CHUNKED_TRANSFER_ENCODING);
 
   bulkdelete_respond(deleter->get_num_deleted(),
                      deleter->get_num_unfound(),