rgw_cksum: permit fallback to checksum-type from create-multipart, in upload-part

There appear to be workloads that provide a checksum algorithm in
create-multipart-upload, but do not provide (what must be) the
corresponding algorithm when uploading the parts.  (complete-multipart-upload
has no checksum argument, the value is implicit.)

This behavior is inconsistent with at least some SDKs, but it is
possibly accepted behavior in AWS S3, and is not logically necessary,
since the originally supplied checksum type is already known.

Therefore, change the behavior of upload-part to fall back to a
checksum type that was sent with the corresponding create-multipart-upload
request, if one is present, rather than failing with InvalidRequest.

Fixes: https://tracker.ceph.com/issues/68513
Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
(cherry picked from commit 6b487a4c6dbadf3f470c8b12ddd5f2521c6920c6)

diff --git a/src/rgw/rgw_cksum_pipe.cc b/src/rgw/rgw_cksum_pipe.cc
index e06957e2715d526f0786406d8f78ca2979bfa243..0bec8d341af078bbc3eca7f20c90c1126b35c599 100644 (file)
--- a/src/rgw/rgw_cksum_pipe.cc
+++ b/src/rgw/rgw_cksum_pipe.cc
@@ -18,6 +18,7 @@
 #include <string>
 #include <fmt/format.h>
 #include <boost/algorithm/string.hpp>
+#include "rgw_cksum.h"
 #include "rgw_common.h"
 #include "common/dout.h"
 #include "rgw_client_io.h"
@@ -34,7 +35,8 @@ namespace rgw::putobj {
   {}
 
   std::unique_ptr<RGWPutObj_Cksum> RGWPutObj_Cksum::Factory(
-    rgw::sal::DataProcessor* next, const RGWEnv& env)
+    rgw::sal::DataProcessor* next, const RGWEnv& env,
+    rgw::cksum::Type override_type)
   {
     /* look for matching headers */
     auto algo_header = cksum_algorithm_hdr(env);
@@ -49,6 +51,13 @@ namespace rgw::putobj {
       throw rgw::io::Exception(EINVAL, std::system_category());
     }
     /* no checksum header */
+    if (override_type != rgw::cksum::Type::none) {
+      /* XXXX safe? do we need to fixup env as well? */
+      auto algo_header = cksum_algorithm_hdr(override_type);
+      return
+       std::make_unique<RGWPutObj_Cksum>(
+                          next, override_type, std::move(algo_header));
+    }
     return std::unique_ptr<RGWPutObj_Cksum>();
   }
 

diff --git a/src/rgw/rgw_cksum_pipe.h b/src/rgw/rgw_cksum_pipe.h
index fddcd283c84bc2a1a7742f2440cb35be2c7c8459..c459d156335422585c9339142d5aa8a23e2ba407 100644 (file)
--- a/src/rgw/rgw_cksum_pipe.h
+++ b/src/rgw/rgw_cksum_pipe.h
@@ -20,6 +20,7 @@
 #include <tuple>
 #include <cstring>
 #include <boost/algorithm/string/case_conv.hpp>
+#include "rgw_cksum.h"
 #include "rgw_cksum_digest.h"
 #include "rgw_common.h"
 #include "rgw_putobj.h"
@@ -29,6 +30,38 @@ namespace rgw::putobj {
   namespace cksum = rgw::cksum;
   using cksum_hdr_t = std::pair<const char*, const char*>;
 
+  static inline const cksum_hdr_t cksum_algorithm_hdr(rgw::cksum::Type t) {
+    static constexpr std::string_view hdr =
+      "HTTP_X_AMZ_SDK_CHECKSUM_ALGORITHM";
+    using rgw::cksum::Type;
+    switch (t) {
+    case Type::sha256:
+      return cksum_hdr_t(hdr.data(), "SHA256");
+      break;
+    case Type::crc32:
+      return cksum_hdr_t(hdr.data(), "CRC32");
+      break;
+    case Type::crc32c:
+      return cksum_hdr_t(hdr.data(), "CRC32C");
+      break;
+    case Type::xxh3:
+      return cksum_hdr_t(hdr.data(), "XX3");
+      break;
+    case Type::sha1:
+      return cksum_hdr_t(hdr.data(), "SHA1");
+      break;
+    case Type::sha512:
+      return cksum_hdr_t(hdr.data(), "SHA512");
+      break;
+    case Type::blake3:
+      return cksum_hdr_t(hdr.data(), "BLAKE3");
+      break;
+    default:
+      break;
+    };
+    return cksum_hdr_t(nullptr, nullptr);;
+  }
+
   static inline const cksum_hdr_t cksum_algorithm_hdr(const RGWEnv& env) {
     /* If the individual checksum value you provide through
        x-amz-checksum-algorithm doesn't match the checksum algorithm
@@ -102,7 +135,8 @@ namespace rgw::putobj {
     using VerifyResult = std::tuple<bool, const cksum::Cksum&>;
 
     static std::unique_ptr<RGWPutObj_Cksum> Factory(
-      rgw::sal::DataProcessor* next, const RGWEnv&);
+      rgw::sal::DataProcessor* next, const RGWEnv&,
+      rgw::cksum::Type override_type);
 
     RGWPutObj_Cksum(rgw::sal::DataProcessor* next, rgw::cksum::Type _type,
                    cksum_hdr_t&& _hdr);

diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc
index 687480caf8867bfaff91579511df8ffd147810ca..866b959e3ac9cc27b31fc610eb72cab13c4f27cc 100644 (file)
--- a/src/rgw/rgw_op.cc
+++ b/src/rgw/rgw_op.cc
@@ -25,6 +25,7 @@
 #include "common/ceph_json.h"
 #include "common/static_ptr.h"
 #include "common/perf_counters_key.h"
+#include "rgw_cksum.h"
 #include "rgw_cksum_digest.h"
 #include "rgw_common.h"
 #include "common/split.h"
@@ -4333,6 +4334,9 @@ void RGWPutObj::execute(optional_yield y)
       }
       return;
     }
+
+    multipart_cksum_type = upload->cksum_type;
+
     /* upload will go out of scope, so copy the dest placement for later use */
     s->dest_placement = *pdest_placement;
     pdest_placement = &s->dest_placement;
@@ -4463,11 +4467,12 @@ void RGWPutObj::execute(optional_yield y)
     /* optional streaming checksum */
     try {
       cksum_filter =
-       rgw::putobj::RGWPutObj_Cksum::Factory(filter, *s->info.env);
+       rgw::putobj::RGWPutObj_Cksum::Factory(filter, *s->info.env, multipart_cksum_type);
     } catch (const rgw::io::Exception& e) {
       op_ret = -e.code().value();
       return;
     }
+
     if (cksum_filter) {
       filter = &*cksum_filter;
     }
@@ -4614,10 +4619,12 @@ void RGWPutObj::execute(optional_yield y)
 
   if (cksum_filter) {
     const auto& hdr = cksum_filter->header();
+    auto expected_ck = cksum_filter->expected(*s->info.env);
     auto cksum_verify =
       cksum_filter->verify(*s->info.env); // valid or no supplied cksum
     cksum = get<1>(cksum_verify);
-    if (std::get<0>(cksum_verify)) {
+    if ((!expected_ck) ||
+       std::get<0>(cksum_verify)) {
       buffer::list cksum_bl;
 
       ldpp_dout_fmt(this, 16,
@@ -4625,14 +4632,13 @@ void RGWPutObj::execute(optional_yield y)
                    "\n\tcomputed={} == \n\texpected={}",
                    hdr.second,
                    cksum->to_armor(),
-                   cksum_filter->expected(*s->info.env));
+                   (!!expected_ck) ? expected_ck : "(checksum unavailable)");
 
       cksum->encode(cksum_bl);
       emplace_attr(RGW_ATTR_CKSUM, std::move(cksum_bl));
     } else {
       /* content checksum mismatch */
       auto computed_ck = cksum->to_armor();
-      auto expected_ck = cksum_filter->expected(*s->info.env);
 
       ldpp_dout_fmt(this, 4,
                    "{} content checksum mismatch"
@@ -4835,7 +4841,8 @@ void RGWPostObj::execute(optional_yield y)
     /* optional streaming checksum */
     try {
       cksum_filter =
-       rgw::putobj::RGWPutObj_Cksum::Factory(filter, *s->info.env);
+       rgw::putobj::RGWPutObj_Cksum::Factory(
+         filter, *s->info.env, rgw::cksum::Type::none /* no override */);
     } catch (const rgw::io::Exception& e) {
       op_ret = -e.code().value();
       return;

diff --git a/src/rgw/rgw_op.h b/src/rgw/rgw_op.h
index d131fdf0b2bf6eb4b9e8fdd883af8214aaa53468..dfc2b914b5b56658b4ea043265a85753b86156c9 100644 (file)
--- a/src/rgw/rgw_op.h
+++ b/src/rgw/rgw_op.h
@@ -1238,6 +1238,7 @@ protected:
   std::string multipart_upload_id;
   std::string multipart_part_str;
   int multipart_part_num = 0;
+  rgw::cksum::Type multipart_cksum_type{rgw::cksum::Type::none};
   jspan_ptr multipart_trace;
 
   boost::optional<ceph::real_time> delete_at;