buffer,crypto,osd,tools: extract digest methods out of bufferlist

they were consumers of ceph::list::buffers(), and there is no need to
make them member functions of ceph::list, so move them into ceph::crypto
namespace. and update the unit test and ceph_dedup_tool.cc accordingly.

Signed-off-by: Kefu Chai <kchai@redhat.com>

diff --git a/src/common/buffer.cc b/src/common/buffer.cc
index 8782a7b6f6cb9cd48ca1eca2ab39233c9a48f8db..d89a9817ecd9a964a5ce29b9096c59b9459c3aac 100644 (file)
--- a/src/common/buffer.cc
+++ b/src/common/buffer.cc
@@ -2041,44 +2041,6 @@ void buffer::list::invalidate_crc()
   }
 }
 
-#include "common/ceph_crypto.h"
-using ceph::crypto::SHA1;
-using ceph::crypto::SHA256;
-using ceph::crypto::SHA512;
-
-sha1_digest_t buffer::list::sha1()
-{
-  unsigned char fingerprint[CEPH_CRYPTO_SHA1_DIGESTSIZE];
-  SHA1 sha1_gen;
-  for (auto& p : _buffers) {
-    sha1_gen.Update((const unsigned char *)p.c_str(), p.length());
-  }
-  sha1_gen.Final(fingerprint);
-  return sha1_digest_t(fingerprint);
-}
-
-sha256_digest_t buffer::list::sha256()
-{
-  unsigned char fingerprint[CEPH_CRYPTO_SHA256_DIGESTSIZE];
-  SHA256 sha256_gen;
-  for (auto& p : _buffers) {
-    sha256_gen.Update((const unsigned char *)p.c_str(), p.length());
-  }
-  sha256_gen.Final(fingerprint);
-  return sha256_digest_t(fingerprint);
-}
-
-sha512_digest_t buffer::list::sha512()
-{
-  unsigned char fingerprint[CEPH_CRYPTO_SHA512_DIGESTSIZE];
-  SHA512 sha512_gen;
-  for (auto& p : _buffers) {
-    sha512_gen.Update((const unsigned char *)p.c_str(), p.length());
-  }
-  sha512_gen.Final(fingerprint);
-  return sha512_digest_t(fingerprint);
-}
-
 /**
  * Binary write all contents to a C++ stream
  */

diff --git a/src/common/ceph_crypto.h b/src/common/ceph_crypto.h
index 2c064b644775c2807c9f5d8f1fe8616a56966738..737454b53e770a9d1418dc4c46fe03db0b6233fa 100644 (file)
--- a/src/common/ceph_crypto.h
+++ b/src/common/ceph_crypto.h
@@ -5,6 +5,9 @@
 #include "acconfig.h"
 #include <stdexcept>
 
+#include "include/buffer.h"
+#include "include/types.h"
+
 #define CEPH_CRYPTO_MD5_DIGESTSIZE 16
 #define CEPH_CRYPTO_HMACSHA1_DIGESTSIZE 20
 #define CEPH_CRYPTO_SHA1_DIGESTSIZE 20
@@ -376,4 +379,18 @@ namespace ceph {
 # error "No supported crypto implementation found."
 #endif
 
+namespace ceph::crypto {
+template<class Digest>
+auto digest(const ceph::buffer::list& bl)
+{
+  unsigned char fingerprint[Digest::digest_size];
+  Digest gen;
+  for (auto& p : bl.buffers()) {
+    gen.Update((const unsigned char *)p.c_str(), p.length());
+  }
+  gen.Final(fingerprint);
+  return sha_digest_t<Digest::digest_size>{fingerprint};
+}
+}
+
 #endif

diff --git a/src/include/buffer.h b/src/include/buffer.h
index c9aeb896c0aa15b7c3e982c49884b9fa75ab3191..642fce0a857dcf35a8ee524a21fe0fbbb30e7a5e 100644 (file)
--- a/src/include/buffer.h
+++ b/src/include/buffer.h
@@ -73,11 +73,6 @@ class packet;
 }
 #endif // HAVE_SEASTAR
 class deleter;
-template<uint8_t S>
-struct sha_digest_t;
-using sha1_digest_t = sha_digest_t<20>;
-using sha256_digest_t = sha_digest_t<32>;
-using sha512_digest_t = sha_digest_t<64>;
 
 template<typename T> class DencDumper;
 
@@ -1232,9 +1227,6 @@ inline namespace v14_2_0 {
     }
     uint32_t crc32c(uint32_t crc) const;
     void invalidate_crc();
-    sha1_digest_t sha1();
-    sha256_digest_t sha256();
-    sha512_digest_t sha512();
 
     // These functions return a bufferlist with a pointer to a single
     // static buffer. They /must/ not outlive the memory they

diff --git a/src/include/types.h b/src/include/types.h
index 461cb9819311f2a7e1eedd48865b088375a9440d..be83e8f89db4aee214cbf03a40123874ae018115 100644 (file)
--- a/src/include/types.h
+++ b/src/include/types.h
@@ -623,6 +623,8 @@ WRITE_CLASS_ENCODER(sha1_digest_t)
 using sha256_digest_t = sha_digest_t<32>;
 WRITE_CLASS_ENCODER(sha256_digest_t)
 
+using sha512_digest_t = sha_digest_t<64>;
+
 using md5_digest_t = sha_digest_t<16>;
 WRITE_CLASS_ENCODER(md5_digest_t)
 

diff --git a/src/osd/PrimaryLogPG.cc b/src/osd/PrimaryLogPG.cc
index e3cfbccda9402438fb2b751a4ff239915203e695..bc104a64c8977bb78a6ea5c2aa12ef909af43277 100644 (file)
--- a/src/osd/PrimaryLogPG.cc
+++ b/src/osd/PrimaryLogPG.cc
@@ -25,6 +25,7 @@
 #include "Session.h"
 #include "objclass/objclass.h"
 
+#include "common/ceph_crypto.h"
 #include "common/errno.h"
 #include "common/scrub_types.h"
 #include "common/perf_counters.h"
@@ -2469,32 +2470,23 @@ int PrimaryLogPG::do_manifest_flush(OpRequestRef op, ObjectContextRef obc, Flush
     unsigned flags = CEPH_OSD_FLAG_IGNORE_CACHE | CEPH_OSD_FLAG_IGNORE_OVERLAY |
                     CEPH_OSD_FLAG_RWORDERED;
     tgt_length = chunk_data.length();
-    pg_pool_t::fingerprint_t fp_algo_t = pool.info.get_fingerprint_type();
-    if (iter->second.has_reference() &&
-       fp_algo_t != pg_pool_t::TYPE_FINGERPRINT_NONE) {
-      object_t fp_oid;
-      bufferlist in;
-      switch (fp_algo_t) {
+    if (pg_pool_t::fingerprint_t fp_algo = pool.info.get_fingerprint_type();
+       iter->second.has_reference() &&
+       fp_algo != pg_pool_t::TYPE_FINGERPRINT_NONE) {
+      object_t fp_oid = [fp_algo, &chunk_data]() -> string {
+        switch (fp_algo) {
        case pg_pool_t::TYPE_FINGERPRINT_SHA1:
-         {
-           sha1_digest_t sha1r = chunk_data.sha1();
-           fp_oid = sha1r.to_str();
-           break;
-         }
+         return crypto::digest<crypto::SHA1>(chunk_data).to_str();
        case pg_pool_t::TYPE_FINGERPRINT_SHA256:
-         {
-           sha256_digest_t sha256r = chunk_data.sha256();
-           fp_oid = sha256r.to_str();
-         }
+         return crypto::digest<crypto::SHA256>(chunk_data).to_str();
        case pg_pool_t::TYPE_FINGERPRINT_SHA512:
-         {
-           sha512_digest_t sha512r = chunk_data.sha512();
-           fp_oid = sha512r.to_str();
-         }
+         return crypto::digest<crypto::SHA512>(chunk_data).to_str();
        default:
          assert(0 == "unrecognized fingerprint type");
-         break;
-      }
+         return {};
+       }
+      }();
+      bufferlist in;
       if (fp_oid != tgt_soid.oid) {
        // decrement old chunk's reference count 
        ObjectOperation dec_op;

diff --git a/src/test/bufferlist.cc b/src/test/bufferlist.cc
index abc53f11a7cf74606fa5ed3985209335829210f2..7a65471fa7548717bbdb62be5441b2ca59630e0c 100644 (file)
--- a/src/test/bufferlist.cc
+++ b/src/test/bufferlist.cc
@@ -2871,126 +2871,6 @@ TEST(BufferList, TestIsProvidedBuffer) {
   ASSERT_FALSE(bl.is_provided_buffer(buff));
 }
 
-TEST(BufferList, TestSHA1) {
-  {
-    bufferlist bl;
-    sha1_digest_t sha1 = bl.sha1();
-    EXPECT_EQ("da39a3ee5e6b4b0d3255bfef95601890afd80709", sha1.to_str());
-  }
-  {
-    bufferlist bl;
-    bl.append("");
-    sha1_digest_t sha1 = bl.sha1();
-    EXPECT_EQ("da39a3ee5e6b4b0d3255bfef95601890afd80709", sha1.to_str());
-  }
-  {
-    bufferlist bl;
-    bl.append("Hello");
-    sha1_digest_t sha1 = bl.sha1();
-    EXPECT_EQ("f7ff9e8b7bb2e09b70935a5d785e0cc5d9d0abf0", sha1.to_str());
-  }
-  {
-    bufferlist bl, bl2;
-    bl.append("Hello");
-    bl2.append(", world!");
-    bl.claim_append(bl2);
-    sha1_digest_t sha1 = bl.sha1();
-    EXPECT_EQ("943a702d06f34599aee1f8da8ef9f7296031d699", sha1.to_str());
-    bl2.append("  How are you today?");
-    bl.claim_append(bl2);
-    sha1 = bl.sha1();
-    EXPECT_EQ("778b5d10e5133aa28fb8de71d35b6999b9a25eb4", sha1.to_str());
-  }
-  {
-    bufferptr p(65536);
-    memset(p.c_str(), 0, 65536);
-    bufferlist bl;
-    bl.append(p);
-    sha1_digest_t sha1 = bl.sha1();
-    EXPECT_EQ("1adc95bebe9eea8c112d40cd04ab7a8d75c4f961", sha1.to_str());
-  }
-}
-
-TEST(BufferList, TestSHA256) {
-  {
-    bufferlist bl;
-    sha256_digest_t sha256 = bl.sha256();
-    EXPECT_EQ("e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", sha256.to_str());
-  }
-  {
-    bufferlist bl;
-    bl.append("");
-    sha256_digest_t sha256 = bl.sha256();
-    EXPECT_EQ("e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", sha256.to_str());
-  }
-  {
-    bufferlist bl;
-    bl.append("Hello");
-    sha256_digest_t sha256 = bl.sha256();
-    EXPECT_EQ("185f8db32271fe25f561a6fc938b2e264306ec304eda518007d1764826381969", sha256.to_str());
-  }
-  {
-    bufferlist bl, bl2;
-    bl.append("Hello");
-    bl2.append(", world!");
-    bl.claim_append(bl2);
-    sha256_digest_t sha256 = bl.sha256();
-    EXPECT_EQ("315f5bdb76d078c43b8ac0064e4a0164612b1fce77c869345bfc94c75894edd3", sha256.to_str());
-    bl2.append("  How are you today?");
-    bl.claim_append(bl2);
-    sha256 = bl.sha256();
-    EXPECT_EQ("e85f57f8bb018bd4f7beed6f27488cef22b13d5e06e8b8a27cac8b087c2a549e", sha256.to_str());
-  }
-  {
-    bufferptr p(65536);
-    memset(p.c_str(), 0, 65536);
-    bufferlist bl;
-    bl.append(p);
-    sha256_digest_t sha256 = bl.sha256();
-    EXPECT_EQ("de2f256064a0af797747c2b97505dc0b9f3df0de4f489eac731c23ae9ca9cc31", sha256.to_str());
-  }
-}
-
-TEST(BufferList, TestSHA512) {
-  {
-    bufferlist bl;
-    sha512_digest_t sha512 = bl.sha512();
-    EXPECT_EQ("cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e", sha512.to_str());
-  }
-  {
-    bufferlist bl;
-    bl.append("");
-    sha512_digest_t sha512 = bl.sha512();
-    EXPECT_EQ("cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e", sha512.to_str());
-  }
-  {
-    bufferlist bl;
-    bl.append("Hello");
-    sha512_digest_t sha512 = bl.sha512();
-    EXPECT_EQ("3615f80c9d293ed7402687f94b22d58e529b8cc7916f8fac7fddf7fbd5af4cf777d3d795a7a00a16bf7e7f3fb9561ee9baae480da9fe7a18769e71886b03f315", sha512.to_str());
-  }
-  {
-    bufferlist bl, bl2;
-    bl.append("Hello");
-    bl2.append(", world!");
-    bl.claim_append(bl2);
-    sha512_digest_t sha512 = bl.sha512();
-    EXPECT_EQ("c1527cd893c124773d811911970c8fe6e857d6df5dc9226bd8a160614c0cd963a4ddea2b94bb7d36021ef9d865d5cea294a82dd49a0bb269f51f6e7a57f79421", sha512.to_str());
-    bl2.append("  How are you today?");
-    bl.claim_append(bl2);
-    sha512 = bl.sha512();
-    EXPECT_EQ("7d50e299496754f9a0d158e018d4b733f2ef51c487b43b50719ffdabe3c3da5a347029741056887b4ffa2ddd0aa9e0dd358b8ed9da9a4f3455f44896fc8e5395", sha512.to_str());
-  }
-  {
-    bufferptr p(65536);
-    memset(p.c_str(), 0, 65536);
-    bufferlist bl;
-    bl.append(p);
-    sha512_digest_t sha512 = bl.sha512();
-    EXPECT_EQ("73e4153936dab198397b74ee9efc26093dda721eaab2f8d92786891153b45b04265a161b169c988edb0db2c53124607b6eaaa816559c5ce54f3dbc9fa6a7a4b2", sha512.to_str());
-  }
-}
-
 TEST(BufferHash, all) {
   {
     bufferlist bl;

diff --git a/src/test/ceph_crypto.cc b/src/test/ceph_crypto.cc
index fc8f4ed8b455ac6de98bc6e94074bc2278cb0aa3..78214f15c456bb98c91792b2420c56ba0c2d4806 100644 (file)
--- a/src/test/ceph_crypto.cc
+++ b/src/test/ceph_crypto.cc
@@ -108,6 +108,135 @@ TEST(HMACSHA1, Restart) {
   ASSERT_EQ(0, err);
 }
 
+TEST(Digest, SHA1) {
+  auto digest = [](const bufferlist& bl) {
+    return ceph::crypto::digest<ceph::crypto::SHA1>(bl);
+  };
+  {
+    bufferlist bl;
+    sha1_digest_t sha1 = digest(bl);
+    EXPECT_EQ("da39a3ee5e6b4b0d3255bfef95601890afd80709", sha1.to_str());
+  }
+  {
+    bufferlist bl;
+    bl.append("");
+    sha1_digest_t sha1 = digest(bl);
+    EXPECT_EQ("da39a3ee5e6b4b0d3255bfef95601890afd80709", sha1.to_str());
+  }
+  {
+    bufferlist bl;
+    bl.append("Hello");
+    sha1_digest_t sha1 = digest(bl);
+    EXPECT_EQ("f7ff9e8b7bb2e09b70935a5d785e0cc5d9d0abf0", sha1.to_str());
+  }
+  {
+    bufferlist bl, bl2;
+    bl.append("Hello");
+    bl2.append(", world!");
+    bl.claim_append(bl2);
+    sha1_digest_t sha1 = digest(bl);
+    EXPECT_EQ("943a702d06f34599aee1f8da8ef9f7296031d699", sha1.to_str());
+    bl2.append("  How are you today?");
+    bl.claim_append(bl2);
+    sha1 = digest(bl);
+    EXPECT_EQ("778b5d10e5133aa28fb8de71d35b6999b9a25eb4", sha1.to_str());
+  }
+  {
+    bufferptr p(65536);
+    memset(p.c_str(), 0, 65536);
+    bufferlist bl;
+    bl.append(p);
+    sha1_digest_t sha1 = digest(bl);
+    EXPECT_EQ("1adc95bebe9eea8c112d40cd04ab7a8d75c4f961", sha1.to_str());
+  }
+}
+
+TEST(Digest, SHA256) {
+  auto digest = [](const bufferlist& bl) {
+    return ceph::crypto::digest<ceph::crypto::SHA256>(bl);
+  };
+  {
+    bufferlist bl;
+    sha256_digest_t sha256 = digest(bl);
+    EXPECT_EQ("e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", sha256.to_str());
+  }
+  {
+    bufferlist bl;
+    bl.append("");
+    sha256_digest_t sha256 = digest(bl);
+    EXPECT_EQ("e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", sha256.to_str());
+  }
+  {
+    bufferlist bl;
+    bl.append("Hello");
+    sha256_digest_t sha256 = digest(bl);
+    EXPECT_EQ("185f8db32271fe25f561a6fc938b2e264306ec304eda518007d1764826381969", sha256.to_str());
+  }
+  {
+    bufferlist bl, bl2;
+    bl.append("Hello");
+    bl2.append(", world!");
+    bl.claim_append(bl2);
+    sha256_digest_t sha256 = digest(bl);
+    EXPECT_EQ("315f5bdb76d078c43b8ac0064e4a0164612b1fce77c869345bfc94c75894edd3", sha256.to_str());
+    bl2.append("  How are you today?");
+    bl.claim_append(bl2);
+    sha256 = digest(bl);
+    EXPECT_EQ("e85f57f8bb018bd4f7beed6f27488cef22b13d5e06e8b8a27cac8b087c2a549e", sha256.to_str());
+  }
+  {
+    bufferptr p(65536);
+    memset(p.c_str(), 0, 65536);
+    bufferlist bl;
+    bl.append(p);
+    sha256_digest_t sha256 = digest(bl);
+    EXPECT_EQ("de2f256064a0af797747c2b97505dc0b9f3df0de4f489eac731c23ae9ca9cc31", sha256.to_str());
+  }
+}
+
+TEST(Digest, SHA512) {
+  auto digest = [](const bufferlist& bl) {
+    return ceph::crypto::digest<ceph::crypto::SHA512>(bl);
+  };
+  {
+    bufferlist bl;
+    sha512_digest_t sha512 = digest(bl);
+    EXPECT_EQ("cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e", sha512.to_str());
+  }
+  {
+    bufferlist bl;
+    bl.append("");
+    sha512_digest_t sha512 = digest(bl);
+    EXPECT_EQ("cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e", sha512.to_str());
+  }
+  {
+    bufferlist bl;
+    bl.append("Hello");
+    sha512_digest_t sha512 = digest(bl);
+    EXPECT_EQ("3615f80c9d293ed7402687f94b22d58e529b8cc7916f8fac7fddf7fbd5af4cf777d3d795a7a00a16bf7e7f3fb9561ee9baae480da9fe7a18769e71886b03f315", sha512.to_str());
+  }
+  {
+    bufferlist bl, bl2;
+    bl.append("Hello");
+    bl2.append(", world!");
+    bl.claim_append(bl2);
+    sha512_digest_t sha512 = digest(bl);
+    EXPECT_EQ("c1527cd893c124773d811911970c8fe6e857d6df5dc9226bd8a160614c0cd963a4ddea2b94bb7d36021ef9d865d5cea294a82dd49a0bb269f51f6e7a57f79421", sha512.to_str());
+    bl2.append("  How are you today?");
+    bl.claim_append(bl2);
+    sha512 = digest(bl);
+    EXPECT_EQ("7d50e299496754f9a0d158e018d4b733f2ef51c487b43b50719ffdabe3c3da5a347029741056887b4ffa2ddd0aa9e0dd358b8ed9da9a4f3455f44896fc8e5395", sha512.to_str());
+  }
+  {
+    bufferptr p(65536);
+    memset(p.c_str(), 0, 65536);
+    bufferlist bl;
+    bl.append(p);
+    sha512_digest_t sha512 = digest(bl);
+    EXPECT_EQ("73e4153936dab198397b74ee9efc26093dda721eaab2f8d92786891153b45b04265a161b169c988edb0db2c53124607b6eaaa816559c5ce54f3dbc9fa6a7a4b2", sha512.to_str());
+  }
+}
+
 class ForkDeathTest : public ::testing::Test {
  protected:
   void SetUp() override {

diff --git a/src/tools/ceph_dedup_tool.cc b/src/tools/ceph_dedup_tool.cc
index a2c310c0203c85474082a1f2e696c1550d5c6fa5..eab1a5a243d5b87672baa85745f743030019a06b 100644 (file)
--- a/src/tools/ceph_dedup_tool.cc
+++ b/src/tools/ceph_dedup_tool.cc
@@ -19,14 +19,15 @@
 
 #include "acconfig.h"
 
-#include "common/config.h"
-#include "common/ceph_argparse.h"
-#include "global/global_init.h"
 #include "common/Cond.h"
+#include "common/Formatter.h"
+#include "common/ceph_argparse.h"
+#include "common/ceph_crypto.h"
+#include "common/config.h"
 #include "common/debug.h"
 #include "common/errno.h"
-#include "common/Formatter.h"
 #include "common/obj_bencher.h"
+#include "global/global_init.h"
 
 #include <iostream>
 #include <fstream>
@@ -369,13 +370,13 @@ void EstimateDedupRatio::add_chunk_fp_to_stat(bufferlist &chunk)
 {
   string fp;
   if (fp_algo == "sha1") {
-    sha1_digest_t sha1_val = chunk.sha1();
+    sha1_digest_t sha1_val = crypto::digest<crypto::SHA1>(chunk);
     fp = sha1_val.to_str();
   } else if (fp_algo == "sha256") {
-    sha256_digest_t sha256_val = chunk.sha256();
+    sha256_digest_t sha256_val = crypto::digest<crypto::SHA256>(chunk);
     fp = sha256_val.to_str();
   } else if (fp_algo == "sha512") {
-    sha512_digest_t sha512_val = chunk.sha512();
+    sha512_digest_t sha512_val = crypto::digest<crypto::SHA512>(chunk);
     fp = sha512_val.to_str();
   } else if (chunk_algo == "rabin") {
     uint64_t hash = rabin.gen_rabin_hash(chunk.c_str(), 0, chunk.length());