From: liubingrun <liubr1@chinatelecom.cn>
Date: Sun, 3 Sep 2023 18:29:47 +0000 (-0400)
Subject: rgw: improve the efficiency of buffer list utilization of chunk upload
X-Git-Tag: v19.0.0~365^2
X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=e47bb22f69ee83479431994ce6514bb739f5dd98;p=ceph.git

rgw: improve the efficiency of buffer list utilization of chunk upload

Reduced waste of buffer::ptr by receiving multiple chunks and filling them into the buffer

AWSv4ComplMulti::recv_body() just receive one chunk and fill it into buffer.
Each 4MB buffer is actually only utilizing 64KB, leading to frequent buffer allocations.
~800GB virtual memory consumption has been observed.

Signed-off-by: liubingrun <liubr1@chinatelecom.cn>
---

diff --git a/src/rgw/rgw_auth_s3.cc b/src/rgw/rgw_auth_s3.cc
index a2def87040efa..0797f8184aad8 100644
--- a/src/rgw/rgw_auth_s3.cc
+++ b/src/rgw/rgw_auth_s3.cc
@@ -1141,7 +1141,7 @@ bool AWSv4ComplMulti::is_signature_mismatched()
   }
 }
 
-size_t AWSv4ComplMulti::recv_body(char* const buf, const size_t buf_max)
+size_t AWSv4ComplMulti::recv_chunk(char* const buf, const size_t buf_max, bool& eof)
 {
   /* Buffer stores only parsed stream. Raw values reflect the stream
    * we're getting from a client. */
@@ -1166,6 +1166,7 @@ size_t AWSv4ComplMulti::recv_body(char* const buf, const size_t buf_max)
                                                    to_extract);
       parsing_buf.resize(parsing_buf.size() - (to_extract - received));
       if (received == 0) {
+        eof = true;
         break;
       }
 
@@ -1215,6 +1216,7 @@ size_t AWSv4ComplMulti::recv_body(char* const buf, const size_t buf_max)
     dout(30) << "AWSv4ComplMulti: to_extract=" << to_extract << ", received=" << received << dendl;
 
     if (received == 0) {
+      eof = true;
       break;
     }
 
@@ -1229,6 +1231,19 @@ size_t AWSv4ComplMulti::recv_body(char* const buf, const size_t buf_max)
   return buf_pos;
 }
 
+size_t AWSv4ComplMulti::recv_body(char* const buf, const size_t buf_max)
+{
+  bool eof = false;
+  size_t total = 0;
+
+  while (total < buf_max && !eof) {
+    const size_t received = recv_chunk(buf + total, buf_max - total, eof);
+    total += received;
+  }
+  dout(20) << "AWSv4ComplMulti: received=" << total << dendl;
+  return total;
+}
+
 void AWSv4ComplMulti::modify_request_state(const DoutPrefixProvider* dpp, req_state* const s_rw)
 {
   const char* const decoded_length = \
diff --git a/src/rgw/rgw_auth_s3.h b/src/rgw/rgw_auth_s3.h
index 5dbd1c998a573..c03dfad825dd2 100644
--- a/src/rgw/rgw_auth_s3.h
+++ b/src/rgw/rgw_auth_s3.h
@@ -331,6 +331,7 @@ class AWSv4ComplMulti : public rgw::auth::Completer,
 
   bool is_signature_mismatched();
   std::string calc_chunk_signature(const std::string& payload_hash) const;
+  size_t recv_chunk(char* buf, size_t max, bool& eof);
 
 public:
   /* We need the constructor to be public because of the std::make_shared that