From 7da903db83b2b50bf2b16624a578bb50ad1157e4 Mon Sep 17 00:00:00 2001 From: Kefu Chai Date: Tue, 11 Jun 2019 15:47:37 +0800 Subject: [PATCH] buffer,crypto,osd,tools: extract digest methods out of bufferlist they were consumers of ceph::list::buffers(), and there is no need to make them member functions of ceph::list, so move them into ceph::crypto namespace. and update the unit test and ceph_dedup_tool.cc accordingly. Signed-off-by: Kefu Chai --- src/common/buffer.cc | 38 ----------- src/common/ceph_crypto.h | 17 +++++ src/include/buffer.h | 8 --- src/include/types.h | 2 + src/osd/PrimaryLogPG.cc | 34 ++++----- src/test/bufferlist.cc | 120 -------------------------------- src/test/ceph_crypto.cc | 129 +++++++++++++++++++++++++++++++++++ src/tools/ceph_dedup_tool.cc | 15 ++-- 8 files changed, 169 insertions(+), 194 deletions(-) diff --git a/src/common/buffer.cc b/src/common/buffer.cc index 8782a7b6f6c..d89a9817ecd 100644 --- a/src/common/buffer.cc +++ b/src/common/buffer.cc @@ -2041,44 +2041,6 @@ void buffer::list::invalidate_crc() } } -#include "common/ceph_crypto.h" -using ceph::crypto::SHA1; -using ceph::crypto::SHA256; -using ceph::crypto::SHA512; - -sha1_digest_t buffer::list::sha1() -{ - unsigned char fingerprint[CEPH_CRYPTO_SHA1_DIGESTSIZE]; - SHA1 sha1_gen; - for (auto& p : _buffers) { - sha1_gen.Update((const unsigned char *)p.c_str(), p.length()); - } - sha1_gen.Final(fingerprint); - return sha1_digest_t(fingerprint); -} - -sha256_digest_t buffer::list::sha256() -{ - unsigned char fingerprint[CEPH_CRYPTO_SHA256_DIGESTSIZE]; - SHA256 sha256_gen; - for (auto& p : _buffers) { - sha256_gen.Update((const unsigned char *)p.c_str(), p.length()); - } - sha256_gen.Final(fingerprint); - return sha256_digest_t(fingerprint); -} - -sha512_digest_t buffer::list::sha512() -{ - unsigned char fingerprint[CEPH_CRYPTO_SHA512_DIGESTSIZE]; - SHA512 sha512_gen; - for (auto& p : _buffers) { - sha512_gen.Update((const unsigned char *)p.c_str(), p.length()); - } - sha512_gen.Final(fingerprint); - return sha512_digest_t(fingerprint); -} - /** * Binary write all contents to a C++ stream */ diff --git a/src/common/ceph_crypto.h b/src/common/ceph_crypto.h index 2c064b64477..737454b53e7 100644 --- a/src/common/ceph_crypto.h +++ b/src/common/ceph_crypto.h @@ -5,6 +5,9 @@ #include "acconfig.h" #include +#include "include/buffer.h" +#include "include/types.h" + #define CEPH_CRYPTO_MD5_DIGESTSIZE 16 #define CEPH_CRYPTO_HMACSHA1_DIGESTSIZE 20 #define CEPH_CRYPTO_SHA1_DIGESTSIZE 20 @@ -376,4 +379,18 @@ namespace ceph { # error "No supported crypto implementation found." #endif +namespace ceph::crypto { +template +auto digest(const ceph::buffer::list& bl) +{ + unsigned char fingerprint[Digest::digest_size]; + Digest gen; + for (auto& p : bl.buffers()) { + gen.Update((const unsigned char *)p.c_str(), p.length()); + } + gen.Final(fingerprint); + return sha_digest_t{fingerprint}; +} +} + #endif diff --git a/src/include/buffer.h b/src/include/buffer.h index c9aeb896c0a..642fce0a857 100644 --- a/src/include/buffer.h +++ b/src/include/buffer.h @@ -73,11 +73,6 @@ class packet; } #endif // HAVE_SEASTAR class deleter; -template -struct sha_digest_t; -using sha1_digest_t = sha_digest_t<20>; -using sha256_digest_t = sha_digest_t<32>; -using sha512_digest_t = sha_digest_t<64>; template class DencDumper; @@ -1232,9 +1227,6 @@ inline namespace v14_2_0 { } uint32_t crc32c(uint32_t crc) const; void invalidate_crc(); - sha1_digest_t sha1(); - sha256_digest_t sha256(); - sha512_digest_t sha512(); // These functions return a bufferlist with a pointer to a single // static buffer. They /must/ not outlive the memory they diff --git a/src/include/types.h b/src/include/types.h index 461cb981931..be83e8f89db 100644 --- a/src/include/types.h +++ b/src/include/types.h @@ -623,6 +623,8 @@ WRITE_CLASS_ENCODER(sha1_digest_t) using sha256_digest_t = sha_digest_t<32>; WRITE_CLASS_ENCODER(sha256_digest_t) +using sha512_digest_t = sha_digest_t<64>; + using md5_digest_t = sha_digest_t<16>; WRITE_CLASS_ENCODER(md5_digest_t) diff --git a/src/osd/PrimaryLogPG.cc b/src/osd/PrimaryLogPG.cc index e3cfbccda94..bc104a64c89 100644 --- a/src/osd/PrimaryLogPG.cc +++ b/src/osd/PrimaryLogPG.cc @@ -25,6 +25,7 @@ #include "Session.h" #include "objclass/objclass.h" +#include "common/ceph_crypto.h" #include "common/errno.h" #include "common/scrub_types.h" #include "common/perf_counters.h" @@ -2469,32 +2470,23 @@ int PrimaryLogPG::do_manifest_flush(OpRequestRef op, ObjectContextRef obc, Flush unsigned flags = CEPH_OSD_FLAG_IGNORE_CACHE | CEPH_OSD_FLAG_IGNORE_OVERLAY | CEPH_OSD_FLAG_RWORDERED; tgt_length = chunk_data.length(); - pg_pool_t::fingerprint_t fp_algo_t = pool.info.get_fingerprint_type(); - if (iter->second.has_reference() && - fp_algo_t != pg_pool_t::TYPE_FINGERPRINT_NONE) { - object_t fp_oid; - bufferlist in; - switch (fp_algo_t) { + if (pg_pool_t::fingerprint_t fp_algo = pool.info.get_fingerprint_type(); + iter->second.has_reference() && + fp_algo != pg_pool_t::TYPE_FINGERPRINT_NONE) { + object_t fp_oid = [fp_algo, &chunk_data]() -> string { + switch (fp_algo) { case pg_pool_t::TYPE_FINGERPRINT_SHA1: - { - sha1_digest_t sha1r = chunk_data.sha1(); - fp_oid = sha1r.to_str(); - break; - } + return crypto::digest(chunk_data).to_str(); case pg_pool_t::TYPE_FINGERPRINT_SHA256: - { - sha256_digest_t sha256r = chunk_data.sha256(); - fp_oid = sha256r.to_str(); - } + return crypto::digest(chunk_data).to_str(); case pg_pool_t::TYPE_FINGERPRINT_SHA512: - { - sha512_digest_t sha512r = chunk_data.sha512(); - fp_oid = sha512r.to_str(); - } + return crypto::digest(chunk_data).to_str(); default: assert(0 == "unrecognized fingerprint type"); - break; - } + return {}; + } + }(); + bufferlist in; if (fp_oid != tgt_soid.oid) { // decrement old chunk's reference count ObjectOperation dec_op; diff --git a/src/test/bufferlist.cc b/src/test/bufferlist.cc index abc53f11a7c..7a65471fa75 100644 --- a/src/test/bufferlist.cc +++ b/src/test/bufferlist.cc @@ -2871,126 +2871,6 @@ TEST(BufferList, TestIsProvidedBuffer) { ASSERT_FALSE(bl.is_provided_buffer(buff)); } -TEST(BufferList, TestSHA1) { - { - bufferlist bl; - sha1_digest_t sha1 = bl.sha1(); - EXPECT_EQ("da39a3ee5e6b4b0d3255bfef95601890afd80709", sha1.to_str()); - } - { - bufferlist bl; - bl.append(""); - sha1_digest_t sha1 = bl.sha1(); - EXPECT_EQ("da39a3ee5e6b4b0d3255bfef95601890afd80709", sha1.to_str()); - } - { - bufferlist bl; - bl.append("Hello"); - sha1_digest_t sha1 = bl.sha1(); - EXPECT_EQ("f7ff9e8b7bb2e09b70935a5d785e0cc5d9d0abf0", sha1.to_str()); - } - { - bufferlist bl, bl2; - bl.append("Hello"); - bl2.append(", world!"); - bl.claim_append(bl2); - sha1_digest_t sha1 = bl.sha1(); - EXPECT_EQ("943a702d06f34599aee1f8da8ef9f7296031d699", sha1.to_str()); - bl2.append(" How are you today?"); - bl.claim_append(bl2); - sha1 = bl.sha1(); - EXPECT_EQ("778b5d10e5133aa28fb8de71d35b6999b9a25eb4", sha1.to_str()); - } - { - bufferptr p(65536); - memset(p.c_str(), 0, 65536); - bufferlist bl; - bl.append(p); - sha1_digest_t sha1 = bl.sha1(); - EXPECT_EQ("1adc95bebe9eea8c112d40cd04ab7a8d75c4f961", sha1.to_str()); - } -} - -TEST(BufferList, TestSHA256) { - { - bufferlist bl; - sha256_digest_t sha256 = bl.sha256(); - EXPECT_EQ("e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", sha256.to_str()); - } - { - bufferlist bl; - bl.append(""); - sha256_digest_t sha256 = bl.sha256(); - EXPECT_EQ("e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", sha256.to_str()); - } - { - bufferlist bl; - bl.append("Hello"); - sha256_digest_t sha256 = bl.sha256(); - EXPECT_EQ("185f8db32271fe25f561a6fc938b2e264306ec304eda518007d1764826381969", sha256.to_str()); - } - { - bufferlist bl, bl2; - bl.append("Hello"); - bl2.append(", world!"); - bl.claim_append(bl2); - sha256_digest_t sha256 = bl.sha256(); - EXPECT_EQ("315f5bdb76d078c43b8ac0064e4a0164612b1fce77c869345bfc94c75894edd3", sha256.to_str()); - bl2.append(" How are you today?"); - bl.claim_append(bl2); - sha256 = bl.sha256(); - EXPECT_EQ("e85f57f8bb018bd4f7beed6f27488cef22b13d5e06e8b8a27cac8b087c2a549e", sha256.to_str()); - } - { - bufferptr p(65536); - memset(p.c_str(), 0, 65536); - bufferlist bl; - bl.append(p); - sha256_digest_t sha256 = bl.sha256(); - EXPECT_EQ("de2f256064a0af797747c2b97505dc0b9f3df0de4f489eac731c23ae9ca9cc31", sha256.to_str()); - } -} - -TEST(BufferList, TestSHA512) { - { - bufferlist bl; - sha512_digest_t sha512 = bl.sha512(); - EXPECT_EQ("cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e", sha512.to_str()); - } - { - bufferlist bl; - bl.append(""); - sha512_digest_t sha512 = bl.sha512(); - EXPECT_EQ("cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e", sha512.to_str()); - } - { - bufferlist bl; - bl.append("Hello"); - sha512_digest_t sha512 = bl.sha512(); - EXPECT_EQ("3615f80c9d293ed7402687f94b22d58e529b8cc7916f8fac7fddf7fbd5af4cf777d3d795a7a00a16bf7e7f3fb9561ee9baae480da9fe7a18769e71886b03f315", sha512.to_str()); - } - { - bufferlist bl, bl2; - bl.append("Hello"); - bl2.append(", world!"); - bl.claim_append(bl2); - sha512_digest_t sha512 = bl.sha512(); - EXPECT_EQ("c1527cd893c124773d811911970c8fe6e857d6df5dc9226bd8a160614c0cd963a4ddea2b94bb7d36021ef9d865d5cea294a82dd49a0bb269f51f6e7a57f79421", sha512.to_str()); - bl2.append(" How are you today?"); - bl.claim_append(bl2); - sha512 = bl.sha512(); - EXPECT_EQ("7d50e299496754f9a0d158e018d4b733f2ef51c487b43b50719ffdabe3c3da5a347029741056887b4ffa2ddd0aa9e0dd358b8ed9da9a4f3455f44896fc8e5395", sha512.to_str()); - } - { - bufferptr p(65536); - memset(p.c_str(), 0, 65536); - bufferlist bl; - bl.append(p); - sha512_digest_t sha512 = bl.sha512(); - EXPECT_EQ("73e4153936dab198397b74ee9efc26093dda721eaab2f8d92786891153b45b04265a161b169c988edb0db2c53124607b6eaaa816559c5ce54f3dbc9fa6a7a4b2", sha512.to_str()); - } -} - TEST(BufferHash, all) { { bufferlist bl; diff --git a/src/test/ceph_crypto.cc b/src/test/ceph_crypto.cc index fc8f4ed8b45..78214f15c45 100644 --- a/src/test/ceph_crypto.cc +++ b/src/test/ceph_crypto.cc @@ -108,6 +108,135 @@ TEST(HMACSHA1, Restart) { ASSERT_EQ(0, err); } +TEST(Digest, SHA1) { + auto digest = [](const bufferlist& bl) { + return ceph::crypto::digest(bl); + }; + { + bufferlist bl; + sha1_digest_t sha1 = digest(bl); + EXPECT_EQ("da39a3ee5e6b4b0d3255bfef95601890afd80709", sha1.to_str()); + } + { + bufferlist bl; + bl.append(""); + sha1_digest_t sha1 = digest(bl); + EXPECT_EQ("da39a3ee5e6b4b0d3255bfef95601890afd80709", sha1.to_str()); + } + { + bufferlist bl; + bl.append("Hello"); + sha1_digest_t sha1 = digest(bl); + EXPECT_EQ("f7ff9e8b7bb2e09b70935a5d785e0cc5d9d0abf0", sha1.to_str()); + } + { + bufferlist bl, bl2; + bl.append("Hello"); + bl2.append(", world!"); + bl.claim_append(bl2); + sha1_digest_t sha1 = digest(bl); + EXPECT_EQ("943a702d06f34599aee1f8da8ef9f7296031d699", sha1.to_str()); + bl2.append(" How are you today?"); + bl.claim_append(bl2); + sha1 = digest(bl); + EXPECT_EQ("778b5d10e5133aa28fb8de71d35b6999b9a25eb4", sha1.to_str()); + } + { + bufferptr p(65536); + memset(p.c_str(), 0, 65536); + bufferlist bl; + bl.append(p); + sha1_digest_t sha1 = digest(bl); + EXPECT_EQ("1adc95bebe9eea8c112d40cd04ab7a8d75c4f961", sha1.to_str()); + } +} + +TEST(Digest, SHA256) { + auto digest = [](const bufferlist& bl) { + return ceph::crypto::digest(bl); + }; + { + bufferlist bl; + sha256_digest_t sha256 = digest(bl); + EXPECT_EQ("e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", sha256.to_str()); + } + { + bufferlist bl; + bl.append(""); + sha256_digest_t sha256 = digest(bl); + EXPECT_EQ("e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", sha256.to_str()); + } + { + bufferlist bl; + bl.append("Hello"); + sha256_digest_t sha256 = digest(bl); + EXPECT_EQ("185f8db32271fe25f561a6fc938b2e264306ec304eda518007d1764826381969", sha256.to_str()); + } + { + bufferlist bl, bl2; + bl.append("Hello"); + bl2.append(", world!"); + bl.claim_append(bl2); + sha256_digest_t sha256 = digest(bl); + EXPECT_EQ("315f5bdb76d078c43b8ac0064e4a0164612b1fce77c869345bfc94c75894edd3", sha256.to_str()); + bl2.append(" How are you today?"); + bl.claim_append(bl2); + sha256 = digest(bl); + EXPECT_EQ("e85f57f8bb018bd4f7beed6f27488cef22b13d5e06e8b8a27cac8b087c2a549e", sha256.to_str()); + } + { + bufferptr p(65536); + memset(p.c_str(), 0, 65536); + bufferlist bl; + bl.append(p); + sha256_digest_t sha256 = digest(bl); + EXPECT_EQ("de2f256064a0af797747c2b97505dc0b9f3df0de4f489eac731c23ae9ca9cc31", sha256.to_str()); + } +} + +TEST(Digest, SHA512) { + auto digest = [](const bufferlist& bl) { + return ceph::crypto::digest(bl); + }; + { + bufferlist bl; + sha512_digest_t sha512 = digest(bl); + EXPECT_EQ("cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e", sha512.to_str()); + } + { + bufferlist bl; + bl.append(""); + sha512_digest_t sha512 = digest(bl); + EXPECT_EQ("cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e", sha512.to_str()); + } + { + bufferlist bl; + bl.append("Hello"); + sha512_digest_t sha512 = digest(bl); + EXPECT_EQ("3615f80c9d293ed7402687f94b22d58e529b8cc7916f8fac7fddf7fbd5af4cf777d3d795a7a00a16bf7e7f3fb9561ee9baae480da9fe7a18769e71886b03f315", sha512.to_str()); + } + { + bufferlist bl, bl2; + bl.append("Hello"); + bl2.append(", world!"); + bl.claim_append(bl2); + sha512_digest_t sha512 = digest(bl); + EXPECT_EQ("c1527cd893c124773d811911970c8fe6e857d6df5dc9226bd8a160614c0cd963a4ddea2b94bb7d36021ef9d865d5cea294a82dd49a0bb269f51f6e7a57f79421", sha512.to_str()); + bl2.append(" How are you today?"); + bl.claim_append(bl2); + sha512 = digest(bl); + EXPECT_EQ("7d50e299496754f9a0d158e018d4b733f2ef51c487b43b50719ffdabe3c3da5a347029741056887b4ffa2ddd0aa9e0dd358b8ed9da9a4f3455f44896fc8e5395", sha512.to_str()); + } + { + bufferptr p(65536); + memset(p.c_str(), 0, 65536); + bufferlist bl; + bl.append(p); + sha512_digest_t sha512 = digest(bl); + EXPECT_EQ("73e4153936dab198397b74ee9efc26093dda721eaab2f8d92786891153b45b04265a161b169c988edb0db2c53124607b6eaaa816559c5ce54f3dbc9fa6a7a4b2", sha512.to_str()); + } +} + class ForkDeathTest : public ::testing::Test { protected: void SetUp() override { diff --git a/src/tools/ceph_dedup_tool.cc b/src/tools/ceph_dedup_tool.cc index a2c310c0203..eab1a5a243d 100644 --- a/src/tools/ceph_dedup_tool.cc +++ b/src/tools/ceph_dedup_tool.cc @@ -19,14 +19,15 @@ #include "acconfig.h" -#include "common/config.h" -#include "common/ceph_argparse.h" -#include "global/global_init.h" #include "common/Cond.h" +#include "common/Formatter.h" +#include "common/ceph_argparse.h" +#include "common/ceph_crypto.h" +#include "common/config.h" #include "common/debug.h" #include "common/errno.h" -#include "common/Formatter.h" #include "common/obj_bencher.h" +#include "global/global_init.h" #include #include @@ -369,13 +370,13 @@ void EstimateDedupRatio::add_chunk_fp_to_stat(bufferlist &chunk) { string fp; if (fp_algo == "sha1") { - sha1_digest_t sha1_val = chunk.sha1(); + sha1_digest_t sha1_val = crypto::digest(chunk); fp = sha1_val.to_str(); } else if (fp_algo == "sha256") { - sha256_digest_t sha256_val = chunk.sha256(); + sha256_digest_t sha256_val = crypto::digest(chunk); fp = sha256_val.to_str(); } else if (fp_algo == "sha512") { - sha512_digest_t sha512_val = chunk.sha512(); + sha512_digest_t sha512_val = crypto::digest(chunk); fp = sha512_val.to_str(); } else if (chunk_algo == "rabin") { uint64_t hash = rabin.gen_rabin_hash(chunk.c_str(), 0, chunk.length()); -- 2.39.5