From: Adam C. Emerson Date: Thu, 26 Mar 2026 01:12:02 +0000 (-0400) Subject: rgw: `buf_to_hex` no longer uses VLAs X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=f722480d0d425d3d4a90b6fe76a55e5a1ec6ce53;p=ceph.git rgw: `buf_to_hex` no longer uses VLAs This was surprisingly more involved than you would think. Signed-off-by: Adam C. Emerson --- diff --git a/src/rgw/driver/daos/rgw_sal_daos.cc b/src/rgw/driver/daos/rgw_sal_daos.cc index 35ad0824d749..4b322e998902 100644 --- a/src/rgw/driver/daos/rgw_sal_daos.cc +++ b/src/rgw/driver/daos/rgw_sal_daos.cc @@ -1696,9 +1696,6 @@ int DaosMultipartUpload::complete( prefix_map_t& processed_prefixes) { ldpp_dout(dpp, 20) << "DEBUG: complete" << dendl; char final_etag[CEPH_CRYPTO_MD5_DIGESTSIZE]; - char final_etag_str[CEPH_CRYPTO_MD5_DIGESTSIZE * 2 + 16]; - std::string etag; - bufferlist etag_bl; MD5 hash; // Allow use of MD5 digest in FIPS mode for non-cryptographic purposes hash.SetFlags(EVP_MD_CTX_FLAG_NON_FIPS_ALLOW); @@ -1831,15 +1828,14 @@ int DaosMultipartUpload::complete( } while (truncated); hash.Final((unsigned char*)final_etag); - buf_to_hex((unsigned char*)final_etag, sizeof(final_etag), final_etag_str); - snprintf(&final_etag_str[CEPH_CRYPTO_MD5_DIGESTSIZE * 2], - sizeof(final_etag_str) - CEPH_CRYPTO_MD5_DIGESTSIZE * 2, "-%lld", - (long long)part_etags.size()); - etag = final_etag_str; - ldpp_dout(dpp, 10) << "calculated etag: " << etag << dendl; - - etag_bl.append(etag); + append_bl(etag_bl, CEPH_CRYPTO_MD5_DIGESTSIZE * 2 + 16, [&](auto iter) { + auto start = iter; + iter = buf_to_hex(final_etag, sizeof(final_etag), iter); + iter = fmt::format_to(iter, "-{}", part_etags.size()); + ldpp_dout(dpp, 10) << "calculated etag: " << std::string_view{start, iter} << dendl; + return iter; + } attrs[RGW_ATTR_ETAG] = etag_bl; if (compressed) { diff --git a/src/rgw/driver/motr/rgw_sal_motr.cc b/src/rgw/driver/motr/rgw_sal_motr.cc index 4cd90d0c7dcc..6167df46aedd 100644 --- a/src/rgw/driver/motr/rgw_sal_motr.cc +++ b/src/rgw/driver/motr/rgw_sal_motr.cc @@ -2681,9 +2681,6 @@ int MotrMultipartUpload::complete(const DoutPrefixProvider *dpp, const char *if_nomatch) { char final_etag[CEPH_CRYPTO_MD5_DIGESTSIZE]; - char final_etag_str[CEPH_CRYPTO_MD5_DIGESTSIZE * 2 + 16]; - std::string etag; - bufferlist etag_bl; MD5 hash; // Allow use of MD5 digest in FIPS mode for non-cryptographic purposes hash.SetFlags(EVP_MD_CTX_FLAG_NON_FIPS_ALLOW); @@ -2822,13 +2819,13 @@ int MotrMultipartUpload::complete(const DoutPrefixProvider *dpp, } while (truncated); hash.Final((unsigned char *)final_etag); - buf_to_hex((unsigned char *)final_etag, sizeof(final_etag), final_etag_str); - snprintf(&final_etag_str[CEPH_CRYPTO_MD5_DIGESTSIZE * 2], - sizeof(final_etag_str) - CEPH_CRYPTO_MD5_DIGESTSIZE * 2, - "-%lld", (long long)part_etags.size()); - etag = final_etag_str; - ldpp_dout(dpp, 20) << "calculated etag: " << etag << dendl; - etag_bl.append(etag); + append_bl(etag_bl, CEPH_CRYPTO_MD5_DIGESTSIZE * 2 + 16, [&](auto iter) { + auto start = iter; + iter = buf_to_hex(final_etag, sizeof(final_etag), iter); + iter = fmt::format_to(iter, "-{}", part_etags.size()); + ldpp_dout(dpp, 10) << "calculated etag: " << std::string_view{start, iter} << dendl; + return iter; + } attrs[RGW_ATTR_ETAG] = etag_bl; if (compressed) { diff --git a/src/rgw/driver/posix/rgw_sal_posix.cc b/src/rgw/driver/posix/rgw_sal_posix.cc index cbf99ef1a305..203ab37d4f14 100644 --- a/src/rgw/driver/posix/rgw_sal_posix.cc +++ b/src/rgw/driver/posix/rgw_sal_posix.cc @@ -3635,11 +3635,8 @@ int POSIXObject::generate_etag(const DoutPrefixProvider* dpp, optional_yield y) MD5 hash; // Allow use of MD5 digest in FIPS mode for non-cryptographic purposes hash.SetFlags(EVP_MD_CTX_FLAG_NON_FIPS_ALLOW); - char calc_md5[CEPH_CRYPTO_MD5_DIGESTSIZE * 2 + 1]; unsigned char m[CEPH_CRYPTO_MD5_DIGESTSIZE]; - bufferlist etag_bl; - while (left > 0) { bufferlist bl; int len = read(cur_ofs, left, bl, dpp, y); @@ -3658,8 +3655,12 @@ int POSIXObject::generate_etag(const DoutPrefixProvider* dpp, optional_yield y) } hash.Final(m); - buf_to_hex(m, CEPH_CRYPTO_MD5_DIGESTSIZE, calc_md5); - etag_bl.append(calc_md5, sizeof(calc_md5)); + bufferlist etag_bl; + append_bl(etag_bl, CEPH_CRYPTO_MD5_DIGESTSIZE * 2 + 1, [&](auto iter) { + iter = buf_to_hex(m, iter); + *iter++ = '\0'; + return iter; + }); get_attrs().emplace(std::move(RGW_ATTR_ETAG), std::move(etag_bl)); return write_attrs(dpp, y); } @@ -3942,9 +3943,6 @@ int POSIXMultipartUpload::complete(const DoutPrefixProvider *dpp, const char *if_nomatch) { char final_etag[CEPH_CRYPTO_MD5_DIGESTSIZE]; - char final_etag_str[CEPH_CRYPTO_MD5_DIGESTSIZE * 2 + 16]; - std::string etag; - bufferlist etag_bl; MD5 hash; // Allow use of MD5 digest in FIPS mode for non-cryptographic purposes hash.SetFlags(EVP_MD_CTX_FLAG_NON_FIPS_ALLOW); @@ -4047,15 +4045,14 @@ int POSIXMultipartUpload::complete(const DoutPrefixProvider *dpp, } while (truncated); hash.Final((unsigned char *)final_etag); - buf_to_hex((unsigned char *)final_etag, sizeof(final_etag), final_etag_str); - snprintf(&final_etag_str[CEPH_CRYPTO_MD5_DIGESTSIZE * 2], - sizeof(final_etag_str) - CEPH_CRYPTO_MD5_DIGESTSIZE * 2, - "-%lld", (long long)part_etags.size()); - etag = final_etag_str; - - etag_bl.append(etag); + bufferlist etag_bl; + append_bl(etag_bl, CEPH_CRYPTO_MD5_DIGESTSIZE * 2 + 16, [&](auto iter) { + iter = buf_to_hex(final_etag, iter); + iter = fmt::format_to(iter, "-{}", part_etags.size()); + return iter; + }); - attrs[RGW_ATTR_ETAG] = etag_bl; + attrs[RGW_ATTR_ETAG] = std::move(etag_bl); if (compressed) { // write compression attribute to full object diff --git a/src/rgw/driver/rados/rgw_bucket.cc b/src/rgw/driver/rados/rgw_bucket.cc index be0b3d7007ba..1fe74f3e8611 100644 --- a/src/rgw/driver/rados/rgw_bucket.cc +++ b/src/rgw/driver/rados/rgw_bucket.cc @@ -2613,7 +2613,6 @@ std::string RGWBucketMetadataHandler::get_marker(void *handle) static void get_md5_digest(const RGWBucketEntryPoint *be, string& md5_digest) { - char md5[CEPH_CRYPTO_MD5_DIGESTSIZE * 2 + 1]; unsigned char m[CEPH_CRYPTO_MD5_DIGESTSIZE]; bufferlist bl; @@ -2627,11 +2626,11 @@ static void get_md5_digest(const RGWBucketEntryPoint *be, string& md5_digest) { hash.Update((const unsigned char *)bl.c_str(), bl.length()); hash.Final(m); - buf_to_hex(m, CEPH_CRYPTO_MD5_DIGESTSIZE, md5); + md5_digest.clear(); + md5_digest.reserve(CEPH_CRYPTO_MD5_DIGESTSIZE * 2); + buf_to_hex(m, std::back_inserter(md5_digest)); delete f; - - md5_digest = md5; } #define ARCHIVE_META_ATTR RGW_ATTR_PREFIX "zone.archive.info" diff --git a/src/rgw/driver/rados/rgw_etag_verifier.cc b/src/rgw/driver/rados/rgw_etag_verifier.cc index 07c6c71b019d..c3006e52c6f6 100644 --- a/src/rgw/driver/rados/rgw_etag_verifier.cc +++ b/src/rgw/driver/rados/rgw_etag_verifier.cc @@ -91,23 +91,22 @@ int ETagVerifier_Atomic::process(bufferlist&& in, uint64_t logical_offset) void ETagVerifier_Atomic::calculate_etag() { unsigned char m[CEPH_CRYPTO_MD5_DIGESTSIZE]; - char calc_md5[CEPH_CRYPTO_MD5_DIGESTSIZE * 2 + 1]; /* Return early if ETag has already been calculated */ if (!calculated_etag.empty()) return; hash.Final(m); - buf_to_hex(m, CEPH_CRYPTO_MD5_DIGESTSIZE, calc_md5); - calculated_etag = calc_md5; + calculated_etag.clear(); + calculated_etag.reserve(CEPH_CRYPTO_MD5_DIGESTSIZE * 2); + buf_to_hex(m, std::back_inserter(calculated_etag)); ldout(cct, 20) << "Single part object: " << " etag:" << calculated_etag - << dendl; + << dendl; } void ETagVerifier_MPU::process_end_of_MPU_part() { unsigned char m[CEPH_CRYPTO_MD5_DIGESTSIZE]; - char calc_md5_part[CEPH_CRYPTO_MD5_DIGESTSIZE * 2 + 1]; std::string calculated_etag_part; hash.Final(m); @@ -115,8 +114,10 @@ void ETagVerifier_MPU::process_end_of_MPU_part() hash.Restart(); if (cct->_conf->subsys.should_gather(dout_subsys, 20)) { - buf_to_hex(m, CEPH_CRYPTO_MD5_DIGESTSIZE, calc_md5_part); - calculated_etag_part = calc_md5_part; + calculated_etag_part.clear(); + calculated_etag_part.reserve(CEPH_CRYPTO_MD5_DIGESTSIZE * 2); + + buf_to_hex(m, std::back_inserter(calculated_etag_part)); ldout(cct, 20) << "Part etag: " << calculated_etag_part << dendl; } @@ -168,23 +169,21 @@ void ETagVerifier_MPU::calculate_etag() constexpr auto extra = 2 + digits10; // add "-%u\0" at the end unsigned char m[CEPH_CRYPTO_MD5_DIGESTSIZE], mpu_m[CEPH_CRYPTO_MD5_DIGESTSIZE]; - char final_etag_str[CEPH_CRYPTO_MD5_DIGESTSIZE * 2 + extra]; /* Return early if ETag has already been calculated */ if (!calculated_etag.empty()) return; + calculated_etag.reserve(CEPH_CRYPTO_MD5_DIGESTSIZE * 2 + extra); + hash.Final(m); mpu_etag_hash.Update((const unsigned char *)m, sizeof(m)); /* Refer RGWCompleteMultipart::execute() for ETag calculation for MPU object */ mpu_etag_hash.Final(mpu_m); - buf_to_hex(mpu_m, CEPH_CRYPTO_MD5_DIGESTSIZE, final_etag_str); - snprintf(&final_etag_str[CEPH_CRYPTO_MD5_DIGESTSIZE * 2], - sizeof(final_etag_str) - CEPH_CRYPTO_MD5_DIGESTSIZE * 2, - "-%u", parts); + buf_to_hex(mpu_m, std::back_inserter(calculated_etag)); + fmt::format_to(std::back_inserter(calculated_etag), "-{}", parts); - calculated_etag = final_etag_str; ldout(cct, 20) << "MPU calculated ETag:" << calculated_etag << dendl; } diff --git a/src/rgw/driver/rados/rgw_putobj_processor.cc b/src/rgw/driver/rados/rgw_putobj_processor.cc index e2ba23a056f4..ee194eb7533b 100644 --- a/src/rgw/driver/rados/rgw_putobj_processor.cc +++ b/src/rgw/driver/rados/rgw_putobj_processor.cc @@ -807,18 +807,20 @@ int AppendObjectProcessor::complete( hash.SetFlags(EVP_MD_CTX_FLAG_NON_FIPS_ALLOW); char petag[CEPH_CRYPTO_MD5_DIGESTSIZE]; char final_etag[CEPH_CRYPTO_MD5_DIGESTSIZE]; - char final_etag_str[CEPH_CRYPTO_MD5_DIGESTSIZE * 2 + 16]; hex_to_buf(cur_etag.c_str(), petag, CEPH_CRYPTO_MD5_DIGESTSIZE); hash.Update((const unsigned char *)petag, sizeof(petag)); hex_to_buf(etag.c_str(), petag, CEPH_CRYPTO_MD5_DIGESTSIZE); hash.Update((const unsigned char *)petag, sizeof(petag)); hash.Final((unsigned char *)final_etag); - buf_to_hex((unsigned char *)final_etag, sizeof(final_etag), final_etag_str); - snprintf(&final_etag_str[CEPH_CRYPTO_MD5_DIGESTSIZE * 2], sizeof(final_etag_str) - CEPH_CRYPTO_MD5_DIGESTSIZE * 2, - "-%lld", (long long)cur_part_num); + bufferlist etag_bl; - etag_bl.append(final_etag_str, strlen(final_etag_str) + 1); - attrs[RGW_ATTR_ETAG] = etag_bl; + append_bl(etag_bl, CEPH_CRYPTO_MD5_DIGESTSIZE * 2 + 16, [&](auto iter) { + iter = buf_to_hex(final_etag, iter); + iter = fmt::format_to(iter, "-{}", cur_part_num); + *iter++ = '\0'; + return iter; + }); + attrs[RGW_ATTR_ETAG] = std::move(etag_bl); } r = obj_op.write_meta(actual_size + cur_size, accounted_size + *cur_accounted_size, diff --git a/src/rgw/driver/rados/rgw_rados.cc b/src/rgw/driver/rados/rgw_rados.cc index 1a1b07cc92e4..2483eb13f6ee 100644 --- a/src/rgw/driver/rados/rgw_rados.cc +++ b/src/rgw/driver/rados/rgw_rados.cc @@ -6882,7 +6882,6 @@ static void generate_fake_tag(const DoutPrefixProvider *dpp, RGWRados* store, ma } unsigned char md5[CEPH_CRYPTO_MD5_DIGESTSIZE]; - char md5_str[CEPH_CRYPTO_MD5_DIGESTSIZE * 2 + 1]; MD5 hash; // Allow use of MD5 digest in FIPS mode for non-cryptographic purposes hash.SetFlags(EVP_MD_CTX_FLAG_NON_FIPS_ALLOW); @@ -6895,8 +6894,7 @@ static void generate_fake_tag(const DoutPrefixProvider *dpp, RGWRados* store, ma } hash.Final(md5); - buf_to_hex(md5, CEPH_CRYPTO_MD5_DIGESTSIZE, md5_str); - tag.append(md5_str); + buf_to_hex(md5, std::back_inserter(tag)); ldpp_dout(dpp, 10) << "generate_fake_tag new tag=" << tag << dendl; diff --git a/src/rgw/driver/rados/rgw_sal_rados.cc b/src/rgw/driver/rados/rgw_sal_rados.cc index 31b8841f0d65..fc712266e253 100644 --- a/src/rgw/driver/rados/rgw_sal_rados.cc +++ b/src/rgw/driver/rados/rgw_sal_rados.cc @@ -4325,9 +4325,6 @@ int RadosMultipartUpload::complete(const DoutPrefixProvider *dpp, const char *if_nomatch) { char final_etag[CEPH_CRYPTO_MD5_DIGESTSIZE]; - char final_etag_str[CEPH_CRYPTO_MD5_DIGESTSIZE * 2 + 16]; - std::string etag; - bufferlist etag_bl; MD5 hash; // Allow use of MD5 digest in FIPS mode for non-cryptographic purposes hash.SetFlags(EVP_MD_CTX_FLAG_NON_FIPS_ALLOW); @@ -4461,16 +4458,16 @@ int RadosMultipartUpload::complete(const DoutPrefixProvider *dpp, } while (truncated); hash.Final((unsigned char *)final_etag); - buf_to_hex((unsigned char *)final_etag, sizeof(final_etag), final_etag_str); - snprintf(&final_etag_str[CEPH_CRYPTO_MD5_DIGESTSIZE * 2], - sizeof(final_etag_str) - CEPH_CRYPTO_MD5_DIGESTSIZE * 2, - "-%lld", (long long)part_etags.size()); - etag = final_etag_str; - ldpp_dout(dpp, 10) << "calculated etag: " << etag << dendl; - - etag_bl.append(etag); - - attrs[RGW_ATTR_ETAG] = etag_bl; + bufferlist etag_bl; + append_bl(etag_bl, CEPH_CRYPTO_MD5_DIGESTSIZE * 2 + 16, [&](auto iter) { + auto start = iter; + iter = buf_to_hex(final_etag, iter); + iter = fmt::format_to(iter, "-{}", part_etags.size()); + ldpp_dout(dpp, 10) << "calculated etag: " << std::string_view{start, iter} + << dendl; + return iter; + }); + attrs[RGW_ATTR_ETAG] = std::move(etag_bl); rgw_placement_rule* ru; ru = &placement; diff --git a/src/rgw/rgw_auth_s3.cc b/src/rgw/rgw_auth_s3.cc index c14a937685aa..dd0e1eeb88d1 100644 --- a/src/rgw/rgw_auth_s3.cc +++ b/src/rgw/rgw_auth_s3.cc @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include @@ -1006,7 +1007,7 @@ get_v4_signature(const std::string_view& credential_scope, using srv_signature_t = AWSEngine::VersionAbstractor::server_signature_t; srv_signature_t signature(srv_signature_t::initialized_later(), digest.SIZE * 2); - buf_to_hex(digest.v, digest.SIZE, signature.begin()); + buf_to_hex(std::span{digest.v, digest.SIZE}, signature.begin()); ldpp_dout(dpp, 10) << "generated signature = " << signature << dendl; diff --git a/src/rgw/rgw_common.cc b/src/rgw/rgw_common.cc index 5fe9f3647614..b712d0bda2d6 100644 --- a/src/rgw/rgw_common.cc +++ b/src/rgw/rgw_common.cc @@ -820,17 +820,18 @@ string calc_hash_sha256_close_stream(SHA256 **phash) if (!hash) { hash = calc_hash_sha256_open_stream(); } - char hash_sha256[CEPH_CRYPTO_HMACSHA256_DIGESTSIZE]; + char hash_sha256[CEPH_CRYPTO_SHA256_DIGESTSIZE]; hash->Final((unsigned char *)hash_sha256); - char hex_str[(CEPH_CRYPTO_SHA256_DIGESTSIZE * 2) + 1]; - buf_to_hex((unsigned char *)hash_sha256, CEPH_CRYPTO_SHA256_DIGESTSIZE, hex_str); + std::string hex_str; + hex_str.reserve(CEPH_CRYPTO_SHA256_DIGESTSIZE * 2); + buf_to_hex(hash_sha256, std::back_inserter(hex_str)); delete hash; *phash = NULL; - return std::string(hex_str); + return hex_str; } std::string calc_hash_sha256_restart_stream(SHA256 **phash) diff --git a/src/rgw/rgw_common.h b/src/rgw/rgw_common.h index 90e4d36c6afe..17ff15898ecf 100644 --- a/src/rgw/rgw_common.h +++ b/src/rgw/rgw_common.h @@ -2287,3 +2287,36 @@ static inline void get_obj_bucket_and_oid_loc(const rgw_obj& obj, std::string& o locator.clear(); } } + +/// Truncate a bufferlist to a given length +/// +/// \param[inout] bl The bufferlist to truncate +/// \param[in] len The new size +inline void +truncate_bl(buffer::list& bl, std::size_t len) +{ + if (len < bl.length()) { + bl.splice(len, bl.length() - len); + } +} + +/// Reserve a size and append to a bufferlist +/// +/// \param[inout] bl The list to which we append +/// \param[in] reserve The maximum that we will append +/// \param[in] appender A function taking an iterator that it increments +/// and returns. The function *must not* append more +/// than was reserved. +inline void +append_bl( + buffer::list& bl, + std::size_t reserve, + std::invocable auto&& appender) +requires std::is_same_v, char*> +{ + auto orig_size = bl.length(); + auto iter = bl.append_hole(reserve).c_str(); + const auto start = iter; + iter = std::forward(appender)(iter); + truncate_bl(bl, orig_size + (iter - start)); +} diff --git a/src/rgw/rgw_data_access.cc b/src/rgw/rgw_data_access.cc index d56155121d40..985d88143f1e 100644 --- a/src/rgw/rgw_data_access.cc +++ b/src/rgw/rgw_data_access.cc @@ -41,13 +41,13 @@ public: } void finish(std::string *etag) { char etag_buf[S]; - char etag_buf_str[S * 2 + 16]; hash.Final((unsigned char *)etag_buf); - buf_to_hex((const unsigned char *)etag_buf, S, - etag_buf_str); - - *etag = etag_buf_str; + if (etag) { + etag->clear(); + etag->reserve(S * 2); + buf_to_hex(etag_buf, std::back_inserter(*etag)); + } } }; diff --git a/src/rgw/rgw_file.cc b/src/rgw/rgw_file.cc index 6bfc06d84357..5032b5105eb4 100644 --- a/src/rgw/rgw_file.cc +++ b/src/rgw/rgw_file.cc @@ -1957,7 +1957,6 @@ namespace rgw { { buffer::list bl, aclbl, ux_key, ux_attrs; map::iterator iter; - char calc_md5[CEPH_CRYPTO_MD5_DIGESTSIZE * 2 + 1]; unsigned char m[CEPH_CRYPTO_MD5_DIGESTSIZE]; req_state* state = get_state(); const req_context rctx{this, state->yield, nullptr}; @@ -2000,8 +1999,9 @@ namespace rgw { << ", blocks=" << cs_info.blocks.size() << dendl; } - buf_to_hex(m, CEPH_CRYPTO_MD5_DIGESTSIZE, calc_md5); - etag = calc_md5; + etag.clear(); + etag.reserve(CEPH_CRYPTO_MD5_DIGESTSIZE * 2); + buf_to_hex(m, std::back_inserter(etag)); bl.append(etag.c_str(), etag.size() + 1); emplace_attr(RGW_ATTR_ETAG, std::move(bl)); diff --git a/src/rgw/rgw_hex.h b/src/rgw/rgw_hex.h index 1bd9f9034d0e..683760012513 100644 --- a/src/rgw/rgw_hex.h +++ b/src/rgw/rgw_hex.h @@ -16,29 +16,38 @@ #pragma once #include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include -static inline void buf_to_hex(const unsigned char* const buf, - const size_t len, - char* const str) +inline auto +buf_to_hex( + std::ranges::input_range auto&& in, + std::output_iterator auto out) { - str[0] = '\0'; - for (size_t i = 0; i < len; i++) { - ::sprintf(&str[i*2], "%02x", static_cast(buf[i])); + static constexpr char hexits[] = "0123456789abcdef"; + for (const auto c_ : std::forward(in)) { + const auto c = static_cast(c_); + for (const auto digit : {c >> 4, c & 0xf}) { + *out++ = hexits[digit]; + } } + return out; } -template static inline std::array +template +inline std::array buf_to_hex(const std::array& buf) { - static_assert(N > 0, "The input array must be at least one element long"); + if constexpr (N == 0) { + return std::array{'\0'}; + } std::array hex_dest; - buf_to_hex(buf.data(), N, hex_dest.data()); + buf_to_hex(buf, hex_dest.begin()); + hex_dest[hex_dest.size() - 1] = '\0'; return hex_dest; } diff --git a/src/rgw/rgw_keystone.cc b/src/rgw/rgw_keystone.cc index 7f2ec8d5efa2..9317c22df278 100644 --- a/src/rgw/rgw_keystone.cc +++ b/src/rgw/rgw_keystone.cc @@ -46,9 +46,9 @@ void rgw_get_token_id(const string& token, string& token_id) hash.Update((const unsigned char *)token.c_str(), token.size()); hash.Final(m); - char calc_md5[CEPH_CRYPTO_MD5_DIGESTSIZE * 2 + 1]; - buf_to_hex(m, CEPH_CRYPTO_MD5_DIGESTSIZE, calc_md5); - token_id = calc_md5; + token_id.clear(); + token_id.reserve(CEPH_CRYPTO_MD5_DIGESTSIZE * 2 + 1); + buf_to_hex(m, std::back_inserter(token_id)); } diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc index bd3cfe2a4e2d..b0cfc3662e26 100644 --- a/src/rgw/rgw_op.cc +++ b/src/rgw/rgw_op.cc @@ -1,15 +1,16 @@ // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:nil -*- // vim: ts=8 sw=2 sts=2 expandtab ft=cpp -#include +#include #include -#include +#include #include -#include - +#include #include #include +#include + #include #include #include @@ -4563,8 +4564,10 @@ int RGWPutObj::get_lua_filter(std::unique_ptr* filter, void RGWPutObj::execute(optional_yield y) { char supplied_md5_bin[CEPH_CRYPTO_MD5_DIGESTSIZE + 1]; - char supplied_md5[CEPH_CRYPTO_MD5_DIGESTSIZE * 2 + 1]; - char calc_md5[CEPH_CRYPTO_MD5_DIGESTSIZE * 2 + 1]; + std::string supplied_md5; + supplied_md5.reserve(CEPH_CRYPTO_MD5_DIGESTSIZE * 2); + std::string calc_md5; + calc_md5.reserve(CEPH_CRYPTO_MD5_DIGESTSIZE * 2); unsigned char m[CEPH_CRYPTO_MD5_DIGESTSIZE]; MD5 hash; // Allow use of MD5 digest in FIPS mode for non-cryptographic purposes @@ -4614,7 +4617,7 @@ void RGWPutObj::execute(optional_yield y) return; } - buf_to_hex((const unsigned char *)supplied_md5_bin, CEPH_CRYPTO_MD5_DIGESTSIZE, supplied_md5); + buf_to_hex(std::span{supplied_md5_bin, CEPH_CRYPTO_MD5_DIGESTSIZE}, std::back_inserter(supplied_md5)); ldpp_dout(this, 15) << "supplied_md5=" << supplied_md5 << dendl; } @@ -4628,8 +4631,7 @@ void RGWPutObj::execute(optional_yield y) } if (supplied_etag) { - strncpy(supplied_md5, supplied_etag, sizeof(supplied_md5) - 1); - supplied_md5[sizeof(supplied_md5) - 1] = '\0'; + supplied_md5 = supplied_etag; } const bool multipart = !multipart_upload_id.empty(); @@ -4925,11 +4927,11 @@ void RGWPutObj::execute(optional_yield y) } } - buf_to_hex(m, CEPH_CRYPTO_MD5_DIGESTSIZE, calc_md5); + buf_to_hex(m, std::back_inserter(calc_md5)); etag = calc_md5; - if (supplied_md5_b64 && strcmp(calc_md5, supplied_md5)) { + if (supplied_md5_b64 && (calc_md5 != supplied_md5)) { op_ret = -ERR_BAD_DIGEST; return; } @@ -5123,7 +5125,8 @@ void RGWPostObj::execute(optional_yield y) { boost::optional compressor; CompressorRef plugin; - char supplied_md5[CEPH_CRYPTO_MD5_DIGESTSIZE * 2 + 1]; + std::string supplied_md5; + supplied_md5.reserve(CEPH_CRYPTO_MD5_DIGESTSIZE * 2); // make reservation for notification if needed std::unique_ptr res @@ -5137,7 +5140,6 @@ void RGWPostObj::execute(optional_yield y) /* Start iteration over data fields. It's necessary as Swift's FormPost * is capable to handle multiple files in single form. */ do { - char calc_md5[CEPH_CRYPTO_MD5_DIGESTSIZE * 2 + 1]; unsigned char m[CEPH_CRYPTO_MD5_DIGESTSIZE]; MD5 hash; // Allow use of MD5 digest in FIPS mode for non-cryptographic purposes @@ -5160,7 +5162,8 @@ void RGWPostObj::execute(optional_yield y) return; } - buf_to_hex((const unsigned char *)supplied_md5_bin, CEPH_CRYPTO_MD5_DIGESTSIZE, supplied_md5); + supplied_md5.clear(); + buf_to_hex(std::span{supplied_md5_bin, CEPH_CRYPTO_MD5_DIGESTSIZE}, std::back_inserter(supplied_md5)); ldpp_dout(this, 15) << "supplied_md5=" << supplied_md5 << dendl; } @@ -5273,11 +5276,11 @@ void RGWPostObj::execute(optional_yield y) } hash.Final(m); - buf_to_hex(m, CEPH_CRYPTO_MD5_DIGESTSIZE, calc_md5); + etag.clear(); + etag.reserve(CEPH_CRYPTO_MD5_DIGESTSIZE * 2); + buf_to_hex(m, std::back_inserter(etag)); - etag = calc_md5; - - if (supplied_md5_b64 && strcmp(calc_md5, supplied_md5)) { + if (supplied_md5_b64 && etag != supplied_md5) { op_ret = -ERR_BAD_DIGEST; return; } @@ -7651,11 +7654,11 @@ bool RGWCompleteMultipart::check_previously_completed(const RGWMultiCompleteUplo } unsigned char final_etag[CEPH_CRYPTO_MD5_DIGESTSIZE]; - char final_etag_str[CEPH_CRYPTO_MD5_DIGESTSIZE * 2 + 16]; + std::string final_etag_str; + final_etag_str.reserve(CEPH_CRYPTO_MD5_DIGESTSIZE * 2 + 16); hash.Final(final_etag); - buf_to_hex(final_etag, CEPH_CRYPTO_MD5_DIGESTSIZE, final_etag_str); - snprintf(&final_etag_str[CEPH_CRYPTO_MD5_DIGESTSIZE * 2], sizeof(final_etag_str) - CEPH_CRYPTO_MD5_DIGESTSIZE * 2, - "-%lld", (long long)parts->parts.size()); + buf_to_hex(final_etag, std::back_inserter(final_etag_str)); + fmt::format_to(std::back_inserter(final_etag_str), "-{}", parts->parts.size()); if (oetag.compare(final_etag_str) != 0) { ldpp_dout(this, 1) << __func__ << "() NOTICE: etag mismatch: object etag:" @@ -8666,17 +8669,18 @@ int RGWBulkUploadOp::handle_file(const std::string_view path, return op_ret; } - char calc_md5[CEPH_CRYPTO_MD5_DIGESTSIZE * 2 + 1]; unsigned char m[CEPH_CRYPTO_MD5_DIGESTSIZE]; hash.Final(m); - buf_to_hex(m, CEPH_CRYPTO_MD5_DIGESTSIZE, calc_md5); + ceph::bufferlist etag_bl; + append_bl(etag_bl, CEPH_CRYPTO_MD5_DIGESTSIZE * 2 + 1, [&](auto iter) { + iter = buf_to_hex(m, iter); + *iter++ = '\0'; + return iter; + }); /* Create metadata: ETAG. */ std::map attrs; - std::string etag = calc_md5; - ceph::bufferlist etag_bl; - etag_bl.append(etag.c_str(), etag.size() + 1); - attrs.emplace(RGW_ATTR_ETAG, std::move(etag_bl)); + attrs.emplace(RGW_ATTR_ETAG, etag_bl); /* Create metadata: ACLs. */ RGWAccessControlPolicy policy; @@ -8704,7 +8708,7 @@ int RGWBulkUploadOp::handle_file(const std::string_view path, /* Complete the transaction. */ const req_context rctx{this, s->yield, s->trace.get()}; - op_ret = processor->complete(size, etag, nullptr, ceph::real_time(), + op_ret = processor->complete(size, etag_bl.c_str(), nullptr, ceph::real_time(), attrs, rgw::cksum::no_cksum, ceph::real_time() /* delete_at */, nullptr, nullptr, nullptr, nullptr, nullptr, diff --git a/src/rgw/rgw_op.h b/src/rgw/rgw_op.h index 4408bbb2e868..6748f6b08fab 100644 --- a/src/rgw/rgw_op.h +++ b/src/rgw/rgw_op.h @@ -2495,16 +2495,17 @@ inline int encode_dlo_manifest_attr(const char * const dlo_manifest, return 0; } /* encode_dlo_manifest_attr */ -inline void complete_etag(MD5& hash, std::string *etag) +inline void complete_etag(MD5& hash, std::string* etag) { + if (!etag) { + return; + } char etag_buf[CEPH_CRYPTO_MD5_DIGESTSIZE]; - char etag_buf_str[CEPH_CRYPTO_MD5_DIGESTSIZE * 2 + 16]; hash.Final((unsigned char *)etag_buf); - buf_to_hex((const unsigned char *)etag_buf, CEPH_CRYPTO_MD5_DIGESTSIZE, - etag_buf_str); - - *etag = etag_buf_str; + etag->clear(); + etag->reserve(CEPH_CRYPTO_MD5_DIGESTSIZE * 2); + buf_to_hex(etag_buf, std::back_inserter(*etag)); } /* complete_etag */ using boost::container::flat_map; diff --git a/src/rgw/rgw_sal_dbstore.cc b/src/rgw/rgw_sal_dbstore.cc index f9be92fda676..17a9aa8dc625 100644 --- a/src/rgw/rgw_sal_dbstore.cc +++ b/src/rgw/rgw_sal_dbstore.cc @@ -948,9 +948,6 @@ namespace rgw::sal { const char *if_nomatch) { char final_etag[CEPH_CRYPTO_MD5_DIGESTSIZE]; - char final_etag_str[CEPH_CRYPTO_MD5_DIGESTSIZE * 2 + 16]; - std::string etag; - bufferlist etag_bl; MD5 hash; bool truncated; int ret; @@ -1018,16 +1015,17 @@ namespace rgw::sal { } while (truncated); hash.Final((unsigned char *)final_etag); - buf_to_hex((unsigned char *)final_etag, sizeof(final_etag), final_etag_str); - snprintf(&final_etag_str[CEPH_CRYPTO_MD5_DIGESTSIZE * 2], - sizeof(final_etag_str) - CEPH_CRYPTO_MD5_DIGESTSIZE * 2, - "-%lld", (long long)part_etags.size()); - etag = final_etag_str; - ldpp_dout(dpp, 10) << "calculated etag: " << etag << dendl; - - etag_bl.append(etag); - - attrs[RGW_ATTR_ETAG] = etag_bl; + bufferlist etag_bl; + append_bl(etag_bl, CEPH_CRYPTO_MD5_DIGESTSIZE * 2 + 16, [&](auto iter) { + const auto start = iter; + iter = buf_to_hex(final_etag, iter); + iter = fmt::format_to(iter, "-{}", part_etags.size()); + ldpp_dout(dpp, 10) << "calculated etag: " << std::string_view{start, iter} + << dendl; + return iter; + }); + + attrs[RGW_ATTR_ETAG] = std::move(etag_bl); /* XXX: handle compression ? */ diff --git a/src/rgw/rgw_swift_auth.cc b/src/rgw/rgw_swift_auth.cc index df4ebfc1629f..c89bcbd3569e 100644 --- a/src/rgw/rgw_swift_auth.cc +++ b/src/rgw/rgw_swift_auth.cc @@ -3,6 +3,7 @@ #include #include +#include #include #include @@ -20,7 +21,6 @@ #include "rgw_client_io.h" #include "rgw_http_client.h" -#include "rgw_sal_rados.h" #include "include/str_list.h" #define dout_context g_ceph_context @@ -541,8 +541,9 @@ static int build_token(const string& swift_user, bufferptr p(CEPH_CRYPTO_HMACSHA1_DIGESTSIZE); - char buf[bl.length() * 2 + 1]; - buf_to_hex((const unsigned char *)bl.c_str(), bl.length(), buf); + std::string buf; + buf.reserve(bl.length() * 2); + buf_to_hex(std::span{bl.c_str(), bl.length()}, std::back_inserter(buf)); dout(20) << "build_token token=" << buf << dendl; char k[CEPH_CRYPTO_HMACSHA1_DIGESTSIZE]; @@ -675,10 +676,12 @@ SignedTokenEngine::authenticate(const DoutPrefixProvider* dpp, if (memcmp(local_tok_bl.c_str(), tok_bl.c_str(), local_tok_bl.length()) != 0) { - char buf[local_tok_bl.length() * 2 + 1]; + std::string buf; + buf.reserve(local_tok_bl.length() * 2); - buf_to_hex(reinterpret_cast(local_tok_bl.c_str()), - local_tok_bl.length(), buf); + buf_to_hex( + std::span{local_tok_bl.c_str(), local_tok_bl.length()}, + std::back_inserter(buf)); ldpp_dout(dpp, 0) << "NOTICE: tokens mismatch tok=" << buf << dendl; return result_t::deny(-EPERM); @@ -800,12 +803,12 @@ void RGW_SWIFT_Auth_Get::execute(optional_yield y) goto done; { - static constexpr size_t PREFIX_LEN = sizeof("AUTH_rgwtk") - 1; - char token_val[PREFIX_LEN + bl.length() * 2 + 1]; + static constexpr auto PREFIX = "AUTH_rgwtk"sv; + std::string token_val; + token_val.reserve(PREFIX.size() + bl.length() * 2 + 1); - snprintf(token_val, PREFIX_LEN + 1, "AUTH_rgwtk"); - buf_to_hex((const unsigned char *)bl.c_str(), bl.length(), - token_val + PREFIX_LEN); + token_val.append(PREFIX); + buf_to_hex(std::span{bl.c_str(), bl.length()}, std::back_inserter(token_val)); dump_header(s, "X-Storage-Token", token_val); dump_header(s, "X-Auth-Token", token_val); diff --git a/src/rgw/rgw_swift_auth.h b/src/rgw/rgw_swift_auth.h index 060137337105..ea946dbf7bf1 100644 --- a/src/rgw/rgw_swift_auth.h +++ b/src/rgw/rgw_swift_auth.h @@ -3,6 +3,15 @@ #pragma once +#include +#include +#include +#include +#include +#include +#include +#include + #include "rgw_common.h" #include "driver/rados/rgw_user.h" #include "rgw_op.h" @@ -393,9 +402,9 @@ class FormatSignature : public SignatureH using base_t = SignatureHelperT; public: const char *result() { - buf_to_hex((UCHARPTR) base_t::dest, - signature_hash_size, - base_t::dest_str); + auto i = buf_to_hex(std::span{(UCHARPTR) base_t::dest, signature_hash_size}, + std::begin(base_t::dest_str)); + *i++ = '\0'; base_t::dest_size = strlen(base_t::dest_str); return base_t::dest_str; }; diff --git a/src/test/rgw/test_rgw_cksum.cc b/src/test/rgw/test_rgw_cksum.cc index 22b11972bcf4..19aa0476b68c 100644 --- a/src/test/rgw/test_rgw_cksum.cc +++ b/src/test/rgw/test_rgw_cksum.cc @@ -14,17 +14,15 @@ */ #include -#include #include #include +#include #include #include #include "gtest/gtest.h" -#include "common/config.h" #include "common/ceph_argparse.h" -#include "common/debug.h" #include "rgw/rgw_cksum.h" #include "rgw/rgw_cksum_pipe.h" #include @@ -176,9 +174,8 @@ TEST(RGWCksum, DigestSha1) ::SHA1((unsigned char *)input_str->c_str(), input_str->length(), sha1_hash); // do some stuff with the hash - char buf[20 * 2 + 1]; - memset(buf, 0, sizeof(buf)); - buf_to_hex(sha1_hash, SHA_DIGEST_LENGTH, buf); + std::string buf; + buf_to_hex(std::span{sha1_hash, SHA_DIGEST_LENGTH}, std::back_inserter(buf)); if (verbose) { std::cout << "byhand sha1 " << buf << std::endl; } @@ -189,7 +186,7 @@ TEST(RGWCksum, DigestSha1) } /* check match with direct OpenSSL mech */ - ASSERT_TRUE(memcmp(buf, cksum.hex().c_str(), + ASSERT_TRUE(memcmp(buf.c_str(), cksum.hex().c_str(), cksum.hex().length()) == 0); if (input_str == &lorem) { diff --git a/src/test/rgw/test_rgw_hex.cc b/src/test/rgw/test_rgw_hex.cc index 94c5795a22bf..10a8a543021a 100644 --- a/src/test/rgw/test_rgw_hex.cc +++ b/src/test/rgw/test_rgw_hex.cc @@ -6,14 +6,15 @@ #include "rgw/rgw_hex.h" -#include -#include +#include +#include using std::end; using std::begin; using std::array; using std::string; using std::string_view; +using std::vector; TEST_CASE("hexdigit", "[rgw]") { @@ -29,25 +30,25 @@ TEST_CASE("hexdigit", "[rgw]") { TEST_CASE("buf_to_hex", "[rgw]") { - constexpr auto in = "AabcdefghA"; + constexpr string_view in = "AabcdefghA"; constexpr string_view out_expected { "41616263646566676841" }; SECTION("C character array overload") { SECTION("empty input") { - constexpr auto in_empty = ""; - const string_view out_expected_empty { "" }; - char out_data[] = {}; + constexpr string_view in_empty{}; + const vector out_expected_empty{}; + vector out_data; - buf_to_hex((const unsigned char* const)(in_empty), std::strlen(in_empty), out_data); + buf_to_hex(in_empty, std::back_inserter(out_data)); CHECK(out_expected_empty == out_data); } SECTION("heuristic (known value)") { - char out_data[1 + (3*sizeof(in))] = {}; + std::string out_data; CAPTURE(out_data); - buf_to_hex((const unsigned char* const)(in), std::strlen(in), out_data); + buf_to_hex(in, std::back_inserter(out_data)); CAPTURE(out_data); CHECK(out_expected == out_data);