From: myoungwon oh Date: Sat, 15 Dec 2018 10:51:44 +0000 (+0900) Subject: src/tools: add stats (fixed objects,total objects) X-Git-Tag: v14.1.0~83^2~2 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=e11fb7ce4d4a305244ea8da0fd3fec7d9c565e19;p=ceph.git src/tools: add stats (fixed objects,total objects) Signed-off-by: Myoungwon Oh --- diff --git a/qa/workunits/rados/test_dedup_tool.sh b/qa/workunits/rados/test_dedup_tool.sh new file mode 100755 index 000000000000..259944017769 --- /dev/null +++ b/qa/workunits/rados/test_dedup_tool.sh @@ -0,0 +1,156 @@ +#!/usr/bin/env bash + +set -x + +die() { + echo "$@" + exit 1 +} + +do_run() { + if [ "$1" == "--tee" ]; then + shift + tee_out="$1" + shift + "$@" | tee $tee_out + else + "$@" + fi +} + +run_expect_succ() { + echo "RUN_EXPECT_SUCC: " "$@" + do_run "$@" + [ $? -ne 0 ] && die "expected success, but got failure! cmd: $@" +} + +run() { + echo "RUN: " $@ + do_run "$@" +} + +if [ -n "$CEPH_BIN" ] ; then + # CMake env + RADOS_TOOL="$CEPH_BIN/rados" + CEPH_TOOL="$CEPH_BIN/ceph" + DEDUP_TOOL="$CEPH_BIN/cephdeduptool" +else + # executables should be installed by the QA env + RADOS_TOOL=$(which rados) + CEPH_TOOL=$(which ceph) + DEDUP_TOOL=$(which cephdeduptool) +fi + +POOL=dedup_pool +OBJ=test_rados_obj + +[ -x "$RADOS_TOOL" ] || die "couldn't find $RADOS_TOOL binary to test" +[ -x "$CEPH_TOOL" ] || die "couldn't find $CEPH_TOOL binary to test" + +run_expect_succ "$CEPH_TOOL" osd pool create "$POOL" 8 + +function test_dedup_ratio_fixed() +{ + # case 1 + dd if=/dev/urandom of=dedup_object_1k bs=1K count=1 + dd if=dedup_object_1k of=dedup_object_100k bs=1K count=100 + + $RADOS_TOOL -p $POOL put $OBJ ./dedup_object_100k + RESULT=$($DEDUP_TOOL --op estimate --pool $POOL --chunk-size 1024 --chunk-algorithm fixed --fingerprint-algorithm sha1 --debug | grep result | awk '{print$4}') + if [ 1024 -ne $RESULT ]; + then + die "Estimate failed expecting 1024 result $RESULT" + fi + + # case 2 + dd if=/dev/zero of=dedup_object_10m bs=10M count=1 + + $RADOS_TOOL -p $POOL put $OBJ ./dedup_object_10m + RESULT=$($DEDUP_TOOL --op estimate --pool $POOL --chunk-size 4096 --chunk-algorithm fixed --fingerprint-algorithm sha1 --debug | grep result | awk '{print$4}') + if [ 4096 -ne $RESULT ]; + then + die "Estimate failed expecting 4096 result $RESULT" + fi + + # case 3 max_thread + for num in `seq 0 20` + do + dd if=/dev/zero of=dedup_object_$num bs=4M count=1 + $RADOS_TOOL -p $POOL put dedup_object_$num ./dedup_object_$num + done + + RESULT=$($DEDUP_TOOL --op estimate --pool $POOL --chunk-size 4096 --chunk-algorithm fixed --fingerprint-algorithm sha1 --max-thread 4 --debug | grep result | awk '{print$2}') + + if [ 98566144 -ne $RESULT ]; + then + die "Estimate failed expecting 98566144 result $RESULT" + fi + + rm -rf ./dedup_object_1k ./dedup_object_100k ./dedup_object_10m + for num in `seq 0 20` + do + rm -rf ./dedup_object_$num + done +} + +function test_dedup_chunk_scrub() +{ + + CHUNK_POOL=dedup_chunk_pool + run_expect_succ "$CEPH_TOOL" osd pool create "$CHUNK_POOL" 8 + + echo "hi there" > foo + + echo "hi there" > bar + + echo "there" > foo-chunk + + echo "CHUNK" > bar-chunk + + $CEPH_TOOL osd pool set $POOL fingerprint_algorithm sha1 --yes-i-really-mean-it + + $RADOS_TOOL -p $POOL put foo ./foo + $RADOS_TOOL -p $POOL put bar ./bar + + $RADOS_TOOL -p $CHUNK_POOL put bar-chunk ./bar-chunk + $RADOS_TOOL -p $CHUNK_POOL put foo-chunk ./foo-chunk + + $RADOS_TOOL -p $POOL set-chunk bar 0 8 --target-pool $CHUNK_POOL bar-chunk 0 --with-reference + $RADOS_TOOL -p $POOL set-chunk foo 0 8 --target-pool $CHUNK_POOL foo-chunk 0 --with-reference + + echo "There hi" > test_obj + # dirty + $RADOS_TOOL -p $POOL put foo ./test_obj + # flush + $RADOS_TOOL -p $POOL put foo ./test_obj + sleep 2 + + rados ls -p $CHUNK_POOL + CHUNK_OID=$(echo -n "There hi" | sha1sum) + + $DEDUP_TOOL --op add_chunk_ref --pool $POOL --chunk_pool $CHUNK_POOL --object $CHUNK_OID --target_ref bar + RESULT=$($DEDUP_TOOL --op get_chunk_ref --pool $POOL --chunk_pool $CHUNK_POOL --object $CHUNK_OID) + + $DEDUP_TOOL --op chunk_scrub --pool $POOL --chunk_pool $CHUNK_POOL + + RESULT=$($DEDUP_TOOL --op get_chunk_ref --pool $POOL --chunk_pool $CHUNK_POOL --object $CHUNK_OID | grep bar) + if [ -n "$RESULT" ] ; then + $CEPH_TOOL osd pool delete $POOL $POOL --yes-i-really-really-mean-it + $CEPH_TOOL osd pool delete $CHUNK_POOL $CHUNK_POOL --yes-i-really-really-mean-it + die "Scrub failed expecting bar is removed" + fi + + $CEPH_TOOL osd pool delete $CHUNK_POOL $CHUNK_POOL --yes-i-really-really-mean-it + + rm -rf ./foo ./bar ./foo-chunk ./bar-chunk ./test_obj +} + +test_dedup_ratio_fixed +test_dedup_chunk_scrub + +$CEPH_TOOL osd pool delete $POOL $POOL --yes-i-really-really-mean-it + +echo "SUCCESS!" +exit 0 + + diff --git a/src/test/test_dedup_tool.sh b/src/test/test_dedup_tool.sh deleted file mode 100755 index 259944017769..000000000000 --- a/src/test/test_dedup_tool.sh +++ /dev/null @@ -1,156 +0,0 @@ -#!/usr/bin/env bash - -set -x - -die() { - echo "$@" - exit 1 -} - -do_run() { - if [ "$1" == "--tee" ]; then - shift - tee_out="$1" - shift - "$@" | tee $tee_out - else - "$@" - fi -} - -run_expect_succ() { - echo "RUN_EXPECT_SUCC: " "$@" - do_run "$@" - [ $? -ne 0 ] && die "expected success, but got failure! cmd: $@" -} - -run() { - echo "RUN: " $@ - do_run "$@" -} - -if [ -n "$CEPH_BIN" ] ; then - # CMake env - RADOS_TOOL="$CEPH_BIN/rados" - CEPH_TOOL="$CEPH_BIN/ceph" - DEDUP_TOOL="$CEPH_BIN/cephdeduptool" -else - # executables should be installed by the QA env - RADOS_TOOL=$(which rados) - CEPH_TOOL=$(which ceph) - DEDUP_TOOL=$(which cephdeduptool) -fi - -POOL=dedup_pool -OBJ=test_rados_obj - -[ -x "$RADOS_TOOL" ] || die "couldn't find $RADOS_TOOL binary to test" -[ -x "$CEPH_TOOL" ] || die "couldn't find $CEPH_TOOL binary to test" - -run_expect_succ "$CEPH_TOOL" osd pool create "$POOL" 8 - -function test_dedup_ratio_fixed() -{ - # case 1 - dd if=/dev/urandom of=dedup_object_1k bs=1K count=1 - dd if=dedup_object_1k of=dedup_object_100k bs=1K count=100 - - $RADOS_TOOL -p $POOL put $OBJ ./dedup_object_100k - RESULT=$($DEDUP_TOOL --op estimate --pool $POOL --chunk-size 1024 --chunk-algorithm fixed --fingerprint-algorithm sha1 --debug | grep result | awk '{print$4}') - if [ 1024 -ne $RESULT ]; - then - die "Estimate failed expecting 1024 result $RESULT" - fi - - # case 2 - dd if=/dev/zero of=dedup_object_10m bs=10M count=1 - - $RADOS_TOOL -p $POOL put $OBJ ./dedup_object_10m - RESULT=$($DEDUP_TOOL --op estimate --pool $POOL --chunk-size 4096 --chunk-algorithm fixed --fingerprint-algorithm sha1 --debug | grep result | awk '{print$4}') - if [ 4096 -ne $RESULT ]; - then - die "Estimate failed expecting 4096 result $RESULT" - fi - - # case 3 max_thread - for num in `seq 0 20` - do - dd if=/dev/zero of=dedup_object_$num bs=4M count=1 - $RADOS_TOOL -p $POOL put dedup_object_$num ./dedup_object_$num - done - - RESULT=$($DEDUP_TOOL --op estimate --pool $POOL --chunk-size 4096 --chunk-algorithm fixed --fingerprint-algorithm sha1 --max-thread 4 --debug | grep result | awk '{print$2}') - - if [ 98566144 -ne $RESULT ]; - then - die "Estimate failed expecting 98566144 result $RESULT" - fi - - rm -rf ./dedup_object_1k ./dedup_object_100k ./dedup_object_10m - for num in `seq 0 20` - do - rm -rf ./dedup_object_$num - done -} - -function test_dedup_chunk_scrub() -{ - - CHUNK_POOL=dedup_chunk_pool - run_expect_succ "$CEPH_TOOL" osd pool create "$CHUNK_POOL" 8 - - echo "hi there" > foo - - echo "hi there" > bar - - echo "there" > foo-chunk - - echo "CHUNK" > bar-chunk - - $CEPH_TOOL osd pool set $POOL fingerprint_algorithm sha1 --yes-i-really-mean-it - - $RADOS_TOOL -p $POOL put foo ./foo - $RADOS_TOOL -p $POOL put bar ./bar - - $RADOS_TOOL -p $CHUNK_POOL put bar-chunk ./bar-chunk - $RADOS_TOOL -p $CHUNK_POOL put foo-chunk ./foo-chunk - - $RADOS_TOOL -p $POOL set-chunk bar 0 8 --target-pool $CHUNK_POOL bar-chunk 0 --with-reference - $RADOS_TOOL -p $POOL set-chunk foo 0 8 --target-pool $CHUNK_POOL foo-chunk 0 --with-reference - - echo "There hi" > test_obj - # dirty - $RADOS_TOOL -p $POOL put foo ./test_obj - # flush - $RADOS_TOOL -p $POOL put foo ./test_obj - sleep 2 - - rados ls -p $CHUNK_POOL - CHUNK_OID=$(echo -n "There hi" | sha1sum) - - $DEDUP_TOOL --op add_chunk_ref --pool $POOL --chunk_pool $CHUNK_POOL --object $CHUNK_OID --target_ref bar - RESULT=$($DEDUP_TOOL --op get_chunk_ref --pool $POOL --chunk_pool $CHUNK_POOL --object $CHUNK_OID) - - $DEDUP_TOOL --op chunk_scrub --pool $POOL --chunk_pool $CHUNK_POOL - - RESULT=$($DEDUP_TOOL --op get_chunk_ref --pool $POOL --chunk_pool $CHUNK_POOL --object $CHUNK_OID | grep bar) - if [ -n "$RESULT" ] ; then - $CEPH_TOOL osd pool delete $POOL $POOL --yes-i-really-really-mean-it - $CEPH_TOOL osd pool delete $CHUNK_POOL $CHUNK_POOL --yes-i-really-really-mean-it - die "Scrub failed expecting bar is removed" - fi - - $CEPH_TOOL osd pool delete $CHUNK_POOL $CHUNK_POOL --yes-i-really-really-mean-it - - rm -rf ./foo ./bar ./foo-chunk ./bar-chunk ./test_obj -} - -test_dedup_ratio_fixed -test_dedup_chunk_scrub - -$CEPH_TOOL osd pool delete $POOL $POOL --yes-i-really-really-mean-it - -echo "SUCCESS!" -exit 0 - - diff --git a/src/tools/ceph_dedup_tool.cc b/src/tools/ceph_dedup_tool.cc index 9dbdd80fe2e3..1713cde49a02 100644 --- a/src/tools/ceph_dedup_tool.cc +++ b/src/tools/ceph_dedup_tool.cc @@ -111,6 +111,7 @@ public: m_cond.Signal(); } virtual void print_status(Formatter *f, ostream &out) = 0; + uint64_t count_objects(IoCtx &ioctx, ObjectCursor &begin, ObjectCursor &end); uint64_t get_examined_objects() { return examined_objects; } uint64_t get_total_bytes() { return total_bytes; } uint64_t get_total_objects() { return total_objects; } @@ -132,6 +133,7 @@ public: chunk_size(chunk_size) { } void* entry() { + count_objects(io_ctx, begin, end); estimate_dedup_ratio(); return NULL; } @@ -144,6 +146,7 @@ public: class ChunkScrub: public EstimateThread { IoCtx chunk_io_ctx; + int fixed_objects = 0; public: ChunkScrub(IoCtx& io_ctx, int n, int m, ObjectCursor begin, ObjectCursor end, @@ -151,20 +154,52 @@ public: EstimateThread(io_ctx, n, m, begin, end, timeout), chunk_io_ctx(chunk_io_ctx) { } void* entry() { + count_objects(chunk_io_ctx, begin, end); chunk_scrub_common(); return NULL; } void chunk_scrub_common(); + int get_fixed_objects() { return fixed_objects; } void print_status(Formatter *f, ostream &out); }; vector> estimate_threads; +uint64_t EstimateThread::count_objects(IoCtx &ioctx, ObjectCursor &begin, ObjectCursor &end) +{ + ObjectCursor shard_start; + ObjectCursor shard_end; + uint64_t count = 0; + + ioctx.object_list_slice( + begin, + end, + n, + m, + &shard_start, + &shard_end); + + ObjectCursor c(shard_start); + while(c < shard_end) + { + std::vector result; + int r = ioctx.object_list(c, shard_end, 12, {}, &result, &c); + if (r < 0 ) { + cerr << "error object_list : " << cpp_strerror(r) << std::endl; + return 0; + } + count += result.size(); + total_objects += result.size(); + } + return count; +} + static void print_dedup_estimate(bool debug = false) { uint64_t total_size = 0; uint64_t dedup_size = 0; uint64_t examined_objects = 0; + uint64_t total_objects = 0; EstimateDedupRatio *ratio = NULL; for (auto &et : estimate_threads) { Mutex::Locker l(glock); @@ -199,11 +234,13 @@ static void print_dedup_estimate(bool debug = false) for (auto &et : estimate_threads) { total_size += et->get_total_bytes(); examined_objects += et->get_examined_objects(); + total_objects += et->get_total_objects(); } - cout << " result: " << total_size << " / " << dedup_size << " (total size / dedup size) " << std::endl; - cout << " dedup ratio: " << dedup_size/total_size << std::endl; - cout << " examined objects: " << examined_objects << std::endl; + cout << " result: " << total_size << " | " << dedup_size << " (total size | deduped size) " << std::endl; + cout << " Dedup ratio: " << (100 - (double)(dedup_size)/total_size*100) << " % " << std::endl; + cout << " Examined objects: " << examined_objects << std::endl; + cout << " Total objects: " << total_objects << std::endl; } static void handle_signal(int signum) @@ -293,7 +330,6 @@ void EstimateDedupRatio::estimate_dedup_ratio() } examined_objects++; } - total_objects++; } } @@ -398,6 +434,7 @@ void ChunkScrub::chunk_scrub_common() if (ret < 0) { continue; } + fixed_objects++; } examined_objects++; m_cond.WaitInterval(m_lock,utime_t(0, COND_WAIT_INTERVAL)); @@ -408,7 +445,6 @@ void ChunkScrub::chunk_scrub_common() cur_time = ceph_clock_now(); } } - total_objects++; } } @@ -420,6 +456,7 @@ void ChunkScrub::print_status(Formatter *f, ostream &out) f->open_object_section("Status"); f->dump_string("Total object", stringify(total_objects)); f->dump_string("Examined objectes", stringify(examined_objects)); + f->dump_string("Fixed objectes", stringify(fixed_objects)); f->close_section(); f->flush(out); cout << std::endl; @@ -440,6 +477,8 @@ int estimate_dedup_ratio(const std::map < std::string, std::string > &opts, int ret; std::map::const_iterator i; bool debug = false; + ObjectCursor begin; + ObjectCursor end; i = opts.find("pool"); if (i != opts.end()) { @@ -519,9 +558,9 @@ int estimate_dedup_ratio(const std::map < std::string, std::string > &opts, } glock.Lock(); + begin = io_ctx.object_list_begin(); + end = io_ctx.object_list_end(); for (unsigned i = 0; i < max_thread; i++) { - ObjectCursor begin = io_ctx.object_list_begin(); - ObjectCursor end = io_ctx.object_list_end(); std::unique_ptr ptr (new EstimateDedupRatio(io_ctx, i, max_thread, begin, end, chunk_algo, fp_algo, chunk_size, report_period)); @@ -544,14 +583,18 @@ static void print_chunk_scrub() { uint64_t total_objects = 0; uint64_t examined_objects = 0; + int fixed_objects = 0; for (auto &et : estimate_threads) { total_objects += et->get_total_objects(); examined_objects += et->get_examined_objects(); + ChunkScrub *ptr = static_cast(et.get()); + fixed_objects += ptr->get_fixed_objects(); } cout << " Total object : " << total_objects << std::endl; cout << " Examined object : " << examined_objects << std::endl; + cout << " Fixed object : " << fixed_objects << std::endl; } int chunk_scrub_common(const std::map < std::string, std::string > &opts, @@ -565,6 +608,8 @@ int chunk_scrub_common(const std::map < std::string, std::string > &opts, unsigned max_thread = default_max_thread; std::map::const_iterator i; uint32_t report_period = default_report_period; + ObjectCursor begin; + ObjectCursor end; i = opts.find("pool"); if (i != opts.end()) { @@ -690,9 +735,9 @@ int chunk_scrub_common(const std::map < std::string, std::string > &opts, } glock.Lock(); + begin = io_ctx.object_list_begin(); + end = io_ctx.object_list_end(); for (unsigned i = 0; i < max_thread; i++) { - ObjectCursor begin = io_ctx.object_list_begin(); - ObjectCursor end = io_ctx.object_list_end(); std::unique_ptr ptr (new ChunkScrub(io_ctx, i, max_thread, begin, end, chunk_io_ctx, report_period)); ptr->create("estimate_thread");