if (manifest /* params.parts_count */) {
RGWObjManifest::obj_iterator end = manifest->obj_end(dpp);
auto cur_part_id = end.get_cur_part_id();
- params.parts_count = (cur_part_id == 1) ? 1 : cur_part_id - 1;;
+ if (cur_part_id != 0 ) {
+ /* end.get_cur_part_id() returns 0 for non-multipart manifests */
+ params.parts_count = (cur_part_id == 1) ? 1 : cur_part_id - 1;
+ }
}
if (!astate->exists) {
#include <stdio.h>
#include "BLAKE3/c/blake3.h"
-#define XXH_INLINE_ALL 1 /* required for streaming variants */
-#include "xxhash.h"
-
namespace rgw { namespace digest {
class Blake3 {
#pragma once
+#include <boost/algorithm/string/case_conv.hpp>
#include <boost/algorithm/string/predicate.hpp>
#include <cstdint>
#include <cstring>
#include <string_view>
#include <array>
#include <iterator>
-#include <boost/variant.hpp>
-#include <boost/blank.hpp>
#include <boost/algorithm/string.hpp>
#include "fmt/format.h"
-#include "common/ceph_crypto.h"
#include "common/armor.h"
-#include "rgw_blake3_digest.h"
-#include "rgw_crc_digest.h"
-#include "rgw_xxh_digest.h"
#include <boost/algorithm/hex.hpp>
#include "rgw_hex.h"
#include "rgw_b64.h"
namespace ba = boost::algorithm;
- static inline std::string safe_upcase_str(std::string s) {
- std::transform(s.begin(), s.end(), s.begin(), ::toupper);
- return s;
- }
-
class Cksum {
public:
static constexpr std::array<Desc, 8> checksums =
Cksum(Type _type, const char* _armored_text)
: type(_type) {
const auto& ckd = checksums[uint16_t(type)];
- (void) ceph_unarmor((char*) digest.begin(), (char*) digest.begin() + ckd.digest_size,
- _armored_text, _armored_text + std::strlen(_armored_text));
+ (void) ceph_unarmor((char*) digest.begin(),
+ (char*) digest.begin() + ckd.digest_size,
+ _armored_text,
+ _armored_text + std::strlen(_armored_text));
}
const char* type_string() const {
std::string element_name() const {
std::string ts{type_string()};
- return fmt::format("Checksum{}", safe_upcase_str(ts));
+ return fmt::format("Checksum{}", boost::to_upper_copy(ts));
}
std::string_view raw() const {
std::string hs;
const auto& ckd = checksums[uint16_t(type)];
hs.resize(ckd.armored_size);
- memset(hs.data(), 0, hs.length());
ceph_armor((char*) hs.data(), (char*) hs.data() + ckd.armored_size,
(char*) digest.begin(), (char*) digest.begin() +
ckd.digest_size);
static inline const std::optional<rgw::cksum::Cksum> no_cksum{std::nullopt};
+ static inline std::string to_string(const Type type) {
+ std::string hs;
+ const auto& ckd = Cksum::checksums[uint16_t(type)];
+ return ckd.name;
+ }
+
static inline Type parse_cksum_type(const char* name)
{
for (const auto& ck : Cksum::checksums) {
parse_cksum_type_hdr(hdr_name) != Type::none;
} /* is_cksum_hdr */
- class Digest {
- public:
- virtual void Restart() = 0;
- virtual void Update (const unsigned char *input, size_t length) = 0;
- virtual void Update(const ceph::buffer::list& bl) = 0;
- virtual void Final (unsigned char *digest) = 0;
- virtual ~Digest() {}
- };
-
- template<class T>
- class TDigest : public Digest
- {
- T d;
- public:
- TDigest() {}
- TDigest(TDigest&& rhs) noexcept
- : d(std::move(rhs.d))
- {}
- void Restart() override { d.Restart(); }
- void Update(const unsigned char* data, uint64_t len) override {
- d.Update(data, len);
- }
- void Update(const ceph::buffer::list& bl) {
- for (auto& p : bl.buffers()) {
- d.Update((const unsigned char *)p.c_str(), p.length());
- }
- }
- void Final(unsigned char* digest) override {
- d.Final(digest);
- }
- };
-
- typedef TDigest<rgw::digest::Blake3> Blake3;
- typedef TDigest<rgw::digest::Crc32> Crc32;
- typedef TDigest<rgw::digest::Crc32c> Crc32c;
- typedef TDigest<rgw::digest::XXH3> XXH3;
- typedef TDigest<ceph::crypto::SHA1> SHA1;
- typedef TDigest<ceph::crypto::SHA256> SHA256;
- typedef TDigest<ceph::crypto::SHA512> SHA512;
-
- typedef boost::variant<boost::blank,
- Blake3,
- Crc32,
- Crc32c,
- XXH3,
- SHA1,
- SHA256,
- SHA512> DigestVariant;
-
- struct get_digest_ptr : public boost::static_visitor<Digest*>
- {
- get_digest_ptr() {};
- Digest* operator()(const boost::blank& b) const { return nullptr; }
- Digest* operator()(Blake3& digest) const { return &digest; }
- Digest* operator()(Crc32& digest) const { return &digest; }
- Digest* operator()(Crc32c& digest) const { return &digest; }
- Digest* operator()(XXH3& digest) const { return &digest; }
- Digest* operator()(SHA1& digest) const { return &digest; }
- Digest* operator()(SHA256& digest) const { return &digest; }
- Digest* operator()(SHA512& digest) const { return &digest; }
- };
-
- static inline Digest* get_digest(DigestVariant& ev)
- {
- return boost::apply_visitor(get_digest_ptr{}, ev);
- }
-
- static inline DigestVariant digest_factory(const Type cksum_type)
- {
- switch (cksum_type) {
- case Type::blake3:
- return Blake3();
- break;
- case Type::sha256:
- return SHA256();
- break;
- case Type::crc32:
- return Crc32();
- break;
- case Type::crc32c:
- return Crc32c();
- break;
- case Type::xxh3:
- return XXH3();
- break;
- case Type::sha512:
- return SHA512();
- break;
- case Type::sha1:
- return SHA1();
- break;
- case Type::none:
- break;
- };
- return boost::blank();
- } /* digest_factory */
-
- static inline Cksum finalize_digest(Digest* digest, Type type)
- {
- Cksum cksum(type);
- if (digest) {
- auto data = cksum.digest.data();
- digest->Final(data);
- }
- return cksum;
- }
-
- static inline std::string to_string(const Type type) {
- std::string hs;
- const auto& ckd = Cksum::checksums[uint16_t(type)];
- return ckd.name;
- }
-
}} /* namespace */
#include <boost/variant.hpp>
#include <boost/blank.hpp>
+#include "common/ceph_crypto.h"
#include "rgw_blake3_digest.h"
#include "rgw_crc_digest.h"
#include "rgw_xxh_digest.h"
: Pipe(next),
_type(_typ),
dv(rgw::cksum::digest_factory(_type)),
- _digest(cksum::get_digest(dv)), cksum_hdr(_hdr),
- _state(State::DIGEST)
+ _digest(cksum::get_digest(dv)), cksum_hdr(_hdr)
{}
std::unique_ptr<RGWPutObj_Cksum> RGWPutObj_Cksum::Factory(
#include <utility>
#include <tuple>
#include <cstring>
-#include "rgw_cksum.h"
+#include <boost/algorithm/string/case_conv.hpp>
+#include "rgw_cksum_digest.h"
#include "rgw_common.h"
#include "rgw_putobj.h"
ix <= uint16_t(cksum::Type::blake3); ++ix) {
cksum_type = cksum::Type(ix);
auto hk = fmt::format("HTTP_X_AMZ_CHECKSUM_{}",
- safe_upcase_str(to_string(cksum_type)));
+ boost::to_upper_copy(to_string(cksum_type)));
auto hv = env.get(hk.c_str());
if (hv) {
return
// PutObj filter for streaming checksums
class RGWPutObj_Cksum : public rgw::putobj::Pipe {
- enum class State : uint16_t {
- START,
- DIGEST,
- FINAL
- };
-
cksum::Type _type;
cksum::DigestVariant dv;
cksum::Digest* _digest;
cksum::Cksum _cksum;
cksum_hdr_t cksum_hdr;
- State _state;
public:
cksum::Type type() { return _type; }
cksum::Digest* digest() const { return _digest; }
const cksum::Cksum& cksum() { return _cksum; };
- State state() const { return _state; }
const cksum_hdr_t& header() const {
return cksum_hdr;
const cksum::Cksum& finalize() {
_cksum = finalize_digest(_digest, _type);
- _state = State::FINAL;
return _cksum;
}
}
VerifyResult verify(const RGWEnv& env) {
- if (_state == State::DIGEST) [[likely]] {
+ if (_cksum.type == cksum::Type::none) [[likely]] {
(void) finalize();
}
auto hv = expected(env);
#include <boost/container/flat_map.hpp>
#include <boost/container/flat_set.hpp>
+#include "common/dout_fmt.h"
#include "common/ceph_crypto.h"
#include "common/random_string.h"
#include "common/tracer.h"
#define RGW_ATTR_CORS RGW_ATTR_PREFIX "cors"
#define RGW_ATTR_ETAG RGW_ATTR_PREFIX "etag"
#define RGW_ATTR_CKSUM RGW_ATTR_PREFIX "cksum"
-#define RGW_ATTR_CKSUM_ALGORITHM RGW_ATTR_PREFIX "x-amz-checksum-algorithm"
#define RGW_ATTR_BUCKETS RGW_ATTR_PREFIX "buckets"
#define RGW_ATTR_META_PREFIX RGW_ATTR_PREFIX RGW_AMZ_META_PREFIX
#define RGW_ATTR_CONTENT_TYPE RGW_ATTR_PREFIX "content_type"
return out;
} /* ys_header_mangle */
-static inline std::string& upcase_str(std::string& s) {
- std::transform(s.begin(), s.end(), s.begin(), ::toupper);
- return s;
-}
-
-static inline std::string safe_upcase_str(std::string s) {
- std::transform(s.begin(), s.end(), s.begin(), ::toupper);
- return s;
-}
-
extern int rgw_bucket_parse_bucket_instance(const std::string& bucket_instance, std::string *bucket_name, std::string *bucket_id, int *shard_id);
boost::intrusive_ptr<CephContext>
#include "common/ceph_json.h"
#include "common/static_ptr.h"
#include "common/perf_counters_key.h"
-#include "rgw_cksum.h"
+#include "rgw_cksum_digest.h"
#include "rgw_common.h"
#include "rgw_tracer.h"
cksum_filter =
rgw::putobj::RGWPutObj_Cksum::Factory(filter, *s->info.env);
} catch (const rgw::io::Exception& e) {
- op_ret = e.code().value();
+ op_ret = -e.code().value();
return;
}
if (cksum_filter) {
cksum = get<1>(cksum_verify);
if (std::get<0>(cksum_verify)) {
buffer::list cksum_bl;
- ldpp_dout(this, 16)
- << fmt::format("{} checksum verified ", hdr.second)
- << fmt::format("\n\tcomputed={} == \n\texpected= {}",
- cksum->to_armor(),
- cksum_filter->expected(*s->info.env))
- << dendl;
+
+ ldpp_dout_fmt(this, 16,
+ "{} checksum verified "
+ "\n\tcomputed={} == \n\texpected={}",
+ hdr.second,
+ cksum->to_armor(),
+ cksum_filter->expected(*s->info.env));
+
cksum->encode(cksum_bl);
emplace_attr(RGW_ATTR_CKSUM, std::move(cksum_bl));
} else {
auto computed_ck = cksum->to_armor();
auto expected_ck = cksum_filter->expected(*s->info.env);
- ldpp_dout(this, 4)
- << fmt::format("{} content checksum mismatch", hdr.second)
- << fmt::format("\n\tcalculated={} != \n\texpected={}",
- computed_ck,
- (!!expected_ck) ? expected_ck : "(checksum unavailable)")
- << dendl;
+ ldpp_dout_fmt(this, 4,
+ "{} content checksum mismatch"
+ "\n\tcalculated={} != \n\texpected={}",
+ hdr.second,
+ computed_ck,
+ (!!expected_ck) ? expected_ck : "(checksum unavailable)");
+
op_ret = -ERR_INVALID_REQUEST;
return;
}
cksum_filter =
rgw::putobj::RGWPutObj_Cksum::Factory(filter, *s->info.env);
} catch (const rgw::io::Exception& e) {
- op_ret = e.code().value();
+ op_ret = -e.code().value();
return;
}
if (cksum_filter) {
} else {
/* content checksum mismatch */
const auto &hdr = cksum_filter->header();
- ldpp_dout(this, 4) << fmt::format("{} content checksum mismatch",
- hdr.second)
- << fmt::format(
- "\n\tcalculated={} != \n\texpected={}",
- cksum->to_armor(),
- cksum_filter->expected(*s->info.env))
- << dendl;
+
+ ldpp_dout_fmt(this, 4,
+ "{} content checksum mismatch"
+ "\n\tcalculated={} != \n\texpected={}",
+ hdr.second,
+ cksum->to_armor(),
+ cksum_filter->expected(*s->info.env));
+
op_ret = -ERR_INVALID_REQUEST;
return;
}
}
if (truncated) {
- ldpp_dout(dpp, 20)
- << fmt::format(
- "WARNING: {} upload->list_parts {} {} truncated, again_count={}!",
- __func__, num_parts, marker, again_count)
- << dendl;
+
+ ldpp_dout_fmt(dpp, 20,
+ "WARNING: {} upload->list_parts {} {} truncated, "
+ "again_count={}!",
+ __func__, num_parts, marker, again_count);
+
truncated = false;
++again_count;
goto again;
for (auto& part : parts_map) {
++parts_ix;
auto& part_cksum = part.second->get_cksum();
- ldpp_dout(dpp, 16)
- << fmt::format("INFO: {} iterate part: {} {} {}",
- __func__, parts_ix, part_cksum->type_string(),
- part_cksum->to_armor())
- << dendl;
+
+ ldpp_dout_fmt(dpp, 16,
+ "INFO: {} iterate part: {} {} {}",
+ __func__, parts_ix, part_cksum->type_string(),
+ part_cksum->to_armor());
+
if ((part_cksum->type != cksum_type)) {
/* if parts have inconsistent checksum, fail now */
- ldpp_dout(dpp, 4)
- << fmt::format(
- "ERROR: multipart part checksum type mismatch\n\tcomplete "
- "multipart header={} part={}",
- to_string(part_cksum->type), to_string(cksum_type))
- << dendl;
- op_ret = -ERR_INVALID_REQUEST;
+
+ ldpp_dout_fmt(dpp, 14,
+ "ERROR: multipart part checksum type mismatch\n\tcomplete "
+ "multipart header={} part={}",
+ to_string(part_cksum->type), to_string(cksum_type));
+
+ op_ret = -ERR_INVALID_REQUEST;
return op_ret;
}
/* we cannot verify this checksum, only compute it */
out_cksum = rgw::cksum::finalize_digest(digest, cksum_type);
- ldpp_dout(dpp, 16)
- << fmt::format("INFO: {} combined checksum {} {}-{}",
- __func__,
- out_cksum->type_string(),
- out_cksum->to_armor(), num_parts)
- << dendl;
+ ldpp_dout_fmt(dpp, 16,
+ "INFO: {} combined checksum {} {}-{}",
+ __func__,
+ out_cksum->type_string(),
+ out_cksum->to_armor(), num_parts);
return op_ret;
} /* try_sum_part_chksums */
}
}
- auto target_attrs = meta_obj->get_attrs();
+ auto& target_attrs = meta_obj->get_attrs();
if (cksum) {
armored_cksum =
/* validate computed checksum against supplied checksum, if present */
auto [hdr_cksum, supplied_cksum] =
rgw::putobj::find_hdr_cksum(*(s->info.env));
- ldpp_dout(this, 10) << fmt::format("INFO: client supplied checksum {}: {}",
- hdr_cksum.header_name(), supplied_cksum)
- << dendl;
+
+ ldpp_dout_fmt(this, 10,
+ "INFO: client supplied checksum {}: {}",
+ hdr_cksum.header_name(), supplied_cksum);
if (! (supplied_cksum.empty()) &&
(supplied_cksum != armored_cksum)) {
// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
// vim: ts=8 sw=2 smarttab ft=cpp
+#include <boost/algorithm/string/case_conv.hpp>
#include <cstdint>
#include <errno.h>
#include <array>
} catch (const buffer::error&) {}
}
- if (multipart_parts_count && multipart_parts_count > 0) {
+ if (multipart_parts_count && *multipart_parts_count > 0) {
dump_header(s, "x-amz-mp-parts-count", *multipart_parts_count);
}
} catch (buffer::error& err) {
ldpp_dout(this, 0) << "ERROR: failed to decode rgw::cksum::Cksum"
<< dendl;
- /* XXX return error here? the user might prefer data we have */
+ /* XXX agreed to handle this case as if there is no checksum
+ * to avoid data unavailable */
}
}
} /* checksum_mode */
auto cksum_type = rgw::cksum::parse_cksum_type_hdr(k);
if (cksum_type != rgw::cksum::Type::none) {
put_prop("HTTP_X_AMZ_CHECKSUM_ALGORITHM",
- safe_upcase_str(to_string(cksum_type)));
+ boost::to_upper_copy(to_string(cksum_type)));
bufferlist& d = p.second.data;
std::string v {
rgw_trim_whitespace(std::string_view(d.c_str(), d.length()))};
dump_header_if_nonempty(s, "x-amz-abort-rule-id", rule_id);
}
if (cksum_algo != rgw::cksum::Type::none) {
- dump_header(s, "x-amz-checksum-algorithm", safe_upcase_str(to_string(cksum_algo)));
+ dump_header(s, "x-amz-checksum-algorithm",
+ boost::to_upper_copy(to_string(cksum_algo)));
}
end_header(s, this, to_mime_type(s->format));
if (op_ret == 0) {
Container element that identifies who initiated the multipart upload. If the initiator is an AWS account, this element provides the same information as the Owner element. If the initiator is an IAM User, this element provides the user ARN and display name, see https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListParts.html */
if (cksum && cksum->aws()) {
- s->formatter->dump_string("ChecksumAlgorithm", safe_upcase_str(cksum->type_string()));
+ s->formatter->dump_string("ChecksumAlgorithm",
+ boost::to_upper_copy(std::string(cksum->type_string())));
}
for (; iter != upload->get_parts().end(); ++iter) {
auto& part_cksum = part->get_cksum();
if (part_cksum && part_cksum->aws()) {
s->formatter->dump_string(part_cksum->element_name(),
- fmt::format("{}", part_cksum->to_armor()));
+ part_cksum->to_armor());
}
s->formatter->close_section();
}
ASSERT_EQ(ck2.to_armor(), ck3.first.to_armor());
}
-TEST(RGWCksum, Output)
-{
- auto o_mode = std::ios::out|std::ios::trunc;
- std::ofstream of;
- of.open("/tmp/lorem", o_mode);
- of << lorem;
- of.close();
-
- of.open("/tmp/dolor", o_mode);
- of << dolor;
- of.close();
-}
-
TEST(RGWCksum, DigestCRC32)
{
auto t = cksum::Type::crc32;
projects / ceph.git / commitdiff