From 72a0b59799a3fa95a11ef56c206541f6554e1b2f Mon Sep 17 00:00:00 2001 From: myoungwon oh Date: Tue, 11 Dec 2018 02:16:59 +0900 Subject: [PATCH] src/tools: make room for cdc Signed-off-by: Myoungwon Oh --- src/test/test_dedup_tool.sh | 4 +- src/tools/ceph_dedup_tool.cc | 127 ++++++++++++++++++++--------------- 2 files changed, 74 insertions(+), 57 deletions(-) diff --git a/src/test/test_dedup_tool.sh b/src/test/test_dedup_tool.sh index 961d7a42d2d..25994401776 100755 --- a/src/test/test_dedup_tool.sh +++ b/src/test/test_dedup_tool.sh @@ -81,9 +81,9 @@ function test_dedup_ratio_fixed() RESULT=$($DEDUP_TOOL --op estimate --pool $POOL --chunk-size 4096 --chunk-algorithm fixed --fingerprint-algorithm sha1 --max-thread 4 --debug | grep result | awk '{print$2}') - if [ 10485760 -ne $RESULT ]; + if [ 98566144 -ne $RESULT ]; then - die "Estimate failed expecting 10485760 result $RESULT" + die "Estimate failed expecting 98566144 result $RESULT" fi rm -rf ./dedup_object_1k ./dedup_object_100k ./dedup_object_10m diff --git a/src/tools/ceph_dedup_tool.cc b/src/tools/ceph_dedup_tool.cc index 9ffd0637840..9dbdd80fe2e 100644 --- a/src/tools/ceph_dedup_tool.cc +++ b/src/tools/ceph_dedup_tool.cc @@ -97,7 +97,6 @@ class EstimateThread : public Thread int32_t timeout; bool m_stop = false; uint64_t total_bytes = 0; - uint64_t deduped_bytes = 0; uint64_t examined_objects = 0; uint64_t total_objects = 0; #define COND_WAIT_INTERVAL 10 @@ -115,7 +114,6 @@ public: uint64_t get_examined_objects() { return examined_objects; } uint64_t get_total_bytes() { return total_bytes; } uint64_t get_total_objects() { return total_objects; } - uint64_t get_deduped_bytes() { return deduped_bytes; } friend class EstimateDedupRatio; friend class ChunkScrub; }; @@ -140,20 +138,18 @@ public: void estimate_dedup_ratio(); void print_status(Formatter *f, ostream &out); map< string, pair > &get_chunk_statistics() { return local_chunk_statistics; } + uint64_t fixed_chunk(string oid, uint64_t offset); }; class ChunkScrub: public EstimateThread { IoCtx chunk_io_ctx; - uint64_t examined_objects = 0; - uint64_t total_objects = 0; public: ChunkScrub(IoCtx& io_ctx, int n, int m, ObjectCursor begin, ObjectCursor end, IoCtx& chunk_io_ctx, int32_t timeout): EstimateThread(io_ctx, n, m, begin, end, timeout), chunk_io_ctx(chunk_io_ctx) { } - void* entry() { chunk_scrub_common(); return NULL; @@ -170,27 +166,39 @@ static void print_dedup_estimate(bool debug = false) uint64_t dedup_size = 0; uint64_t examined_objects = 0; EstimateDedupRatio *ratio = NULL; - if (debug) { - for (auto &et : estimate_threads) { - Mutex::Locker l(glock); - ratio = dynamic_cast(et.get()); - chunk_statistics.insert(ratio->get_chunk_statistics().begin(), - ratio->get_chunk_statistics().end()); + for (auto &et : estimate_threads) { + Mutex::Locker l(glock); + ratio = dynamic_cast(et.get()); + assert(ratio); + for (auto p : ratio->get_chunk_statistics()) { + auto c = chunk_statistics.find(p.first); + if (c != chunk_statistics.end()) { + c->second.first += p.second.first; + } else { + chunk_statistics.insert(p); + } } + } + if (debug) { for (auto p : chunk_statistics) { cout << " -- " << std::endl; cout << " key: " << p.first << std::endl; cout << " count: " << p.second.first << std::endl; cout << " chunk_size: " << p.second.second << std::endl; + dedup_size += p.second.second; cout << " -- " << std::endl; } + } else { + for (auto p : chunk_statistics) { + dedup_size += p.second.second; + } + } for (auto &et : estimate_threads) { total_size += et->get_total_bytes(); examined_objects += et->get_examined_objects(); - dedup_size += et->get_deduped_bytes(); } cout << " result: " << total_size << " / " << dedup_size << " (total size / dedup size) " << std::endl; @@ -220,7 +228,6 @@ void EstimateDedupRatio::print_status(Formatter *f, ostream &out) f->close_section(); f->open_object_section("Status"); f->dump_string("Total bytes", stringify(total_bytes)); - f->dump_string("Deduped bytes", stringify(deduped_bytes)); f->dump_string("Examined objectes", stringify(examined_objects)); f->close_section(); f->flush(out); @@ -232,8 +239,6 @@ void EstimateDedupRatio::estimate_dedup_ratio() { ObjectCursor shard_start; ObjectCursor shard_end; - unsigned op_size = default_op_size; - int ret; utime_t cur_time = ceph_clock_now(); io_ctx.object_list_slice( @@ -265,51 +270,19 @@ void EstimateDedupRatio::estimate_dedup_ratio() delete formatter; return; } - bufferlist outdata; - ret = io_ctx.read(oid, outdata, op_size, offset); - if (ret <= 0) { - break; - } + uint64_t next_offset; if (chunk_algo == "fixed") { - if (fp_algo == "sha1") { - uint64_t c_offset = 0; - while (c_offset < outdata.length()) { - bufferlist chunk; - if (outdata.length() - c_offset > chunk_size) { - bufferptr bptr(chunk_size); - chunk.push_back(std::move(bptr)); - chunk.copy_in(0, chunk_size, outdata.c_str()); - } else { - bufferptr bptr(outdata.length() - c_offset); - chunk.push_back(std::move(bptr)); - chunk.copy_in(0, outdata.length() - c_offset, outdata.c_str()); - } - sha1_digest_t sha1_val = chunk.sha1(); - string fp = sha1_val.to_str(); - auto p = local_chunk_statistics.find(fp); - if (p != local_chunk_statistics.end()) { - uint64_t count = p->second.first; - count++; - local_chunk_statistics[fp] = make_pair(count, chunk.length()); - } else { - local_chunk_statistics[fp] = make_pair(1, chunk.length()); - deduped_bytes += chunk.length(); - } - total_bytes += chunk.length(); - c_offset = c_offset + chunk_size; - } - } else { - ceph_assert(0 == "no support fingerprint algorithm"); - } + next_offset = fixed_chunk(oid, offset); } else { + // CDC .. ceph_assert(0 == "no support chunk algorithm"); } - if (outdata.length() < op_size) { + if (!next_offset) { break; } - offset += outdata.length(); + offset += next_offset; m_cond.WaitInterval(m_lock,utime_t(0, COND_WAIT_INTERVAL)); if (cur_time + utime_t(timeout, 0) < ceph_clock_now()) { Formatter *formatter = Formatter::create("json-pretty"); @@ -324,6 +297,52 @@ void EstimateDedupRatio::estimate_dedup_ratio() } } +uint64_t EstimateDedupRatio::fixed_chunk(string oid, uint64_t offset) +{ + unsigned op_size = default_op_size; + int ret; + bufferlist outdata; + ret = io_ctx.read(oid, outdata, op_size, offset); + if (ret <= 0) { + return 0; + } + + if (fp_algo == "sha1") { + uint64_t c_offset = 0; + while (c_offset < outdata.length()) { + bufferlist chunk; + if (outdata.length() - c_offset > chunk_size) { + bufferptr bptr(chunk_size); + chunk.push_back(std::move(bptr)); + chunk.copy_in(0, chunk_size, outdata.c_str()); + } else { + bufferptr bptr(outdata.length() - c_offset); + chunk.push_back(std::move(bptr)); + chunk.copy_in(0, outdata.length() - c_offset, outdata.c_str()); + } + sha1_digest_t sha1_val = chunk.sha1(); + string fp = sha1_val.to_str(); + auto p = local_chunk_statistics.find(fp); + if (p != local_chunk_statistics.end()) { + uint64_t count = p->second.first; + count++; + local_chunk_statistics[fp] = make_pair(count, chunk.length()); + } else { + local_chunk_statistics[fp] = make_pair(1, chunk.length()); + } + total_bytes += chunk.length(); + c_offset = c_offset + chunk_size; + } + } else { + ceph_assert(0 == "no support fingerperint algorithm"); + } + + if (outdata.length() < op_size) { + return 0; + } + return outdata.length(); +} + void ChunkScrub::chunk_scrub_common() { ObjectCursor shard_start; @@ -391,8 +410,6 @@ void ChunkScrub::chunk_scrub_common() } total_objects++; } - cout << " Total object : " << total_objects << std::endl; - cout << " Completed object : " << examined_objects << std::endl; } void ChunkScrub::print_status(Formatter *f, ostream &out) @@ -534,7 +551,7 @@ static void print_chunk_scrub() } cout << " Total object : " << total_objects << std::endl; - cout << " Completed object : " << examined_objects << std::endl; + cout << " Examined object : " << examined_objects << std::endl; } int chunk_scrub_common(const std::map < std::string, std::string > &opts, -- 2.39.5