--- /dev/null
+#!/usr/bin/env bash
+
+set -x
+
+die() {
+ echo "$@"
+ exit 1
+}
+
+do_run() {
+ if [ "$1" == "--tee" ]; then
+ shift
+ tee_out="$1"
+ shift
+ "$@" | tee $tee_out
+ else
+ "$@"
+ fi
+}
+
+run_expect_succ() {
+ echo "RUN_EXPECT_SUCC: " "$@"
+ do_run "$@"
+ [ $? -ne 0 ] && die "expected success, but got failure! cmd: $@"
+}
+
+run() {
+ echo "RUN: " $@
+ do_run "$@"
+}
+
+if [ -n "$CEPH_BIN" ] ; then
+ # CMake env
+ RADOS_TOOL="$CEPH_BIN/rados"
+ CEPH_TOOL="$CEPH_BIN/ceph"
+ DEDUP_TOOL="$CEPH_BIN/cephdeduptool"
+else
+ # executables should be installed by the QA env
+ RADOS_TOOL=$(which rados)
+ CEPH_TOOL=$(which ceph)
+ DEDUP_TOOL=$(which cephdeduptool)
+fi
+
+POOL=dedup_pool
+OBJ=test_rados_obj
+
+[ -x "$RADOS_TOOL" ] || die "couldn't find $RADOS_TOOL binary to test"
+[ -x "$CEPH_TOOL" ] || die "couldn't find $CEPH_TOOL binary to test"
+
+run_expect_succ "$CEPH_TOOL" osd pool create "$POOL" 8
+
+function test_dedup_ratio_fixed()
+{
+ # case 1
+ dd if=/dev/urandom of=dedup_object_1k bs=1K count=1
+ dd if=dedup_object_1k of=dedup_object_100k bs=1K count=100
+
+ $RADOS_TOOL -p $POOL put $OBJ ./dedup_object_100k
+ RESULT=$($DEDUP_TOOL --op estimate --pool $POOL --chunk-size 1024 --chunk-algorithm fixed --fingerprint-algorithm sha1 --debug | grep result | awk '{print$4}')
+ if [ 1024 -ne $RESULT ];
+ then
+ die "Estimate failed expecting 1024 result $RESULT"
+ fi
+
+ # case 2
+ dd if=/dev/zero of=dedup_object_10m bs=10M count=1
+
+ $RADOS_TOOL -p $POOL put $OBJ ./dedup_object_10m
+ RESULT=$($DEDUP_TOOL --op estimate --pool $POOL --chunk-size 4096 --chunk-algorithm fixed --fingerprint-algorithm sha1 --debug | grep result | awk '{print$4}')
+ if [ 4096 -ne $RESULT ];
+ then
+ die "Estimate failed expecting 4096 result $RESULT"
+ fi
+
+ # case 3 max_thread
+ for num in `seq 0 20`
+ do
+ dd if=/dev/zero of=dedup_object_$num bs=4M count=1
+ $RADOS_TOOL -p $POOL put dedup_object_$num ./dedup_object_$num
+ done
+
+ RESULT=$($DEDUP_TOOL --op estimate --pool $POOL --chunk-size 4096 --chunk-algorithm fixed --fingerprint-algorithm sha1 --max-thread 4 --debug | grep result | awk '{print$2}')
+
+ if [ 98566144 -ne $RESULT ];
+ then
+ die "Estimate failed expecting 98566144 result $RESULT"
+ fi
+
+ rm -rf ./dedup_object_1k ./dedup_object_100k ./dedup_object_10m
+ for num in `seq 0 20`
+ do
+ rm -rf ./dedup_object_$num
+ done
+}
+
+function test_dedup_chunk_scrub()
+{
+
+ CHUNK_POOL=dedup_chunk_pool
+ run_expect_succ "$CEPH_TOOL" osd pool create "$CHUNK_POOL" 8
+
+ echo "hi there" > foo
+
+ echo "hi there" > bar
+
+ echo "there" > foo-chunk
+
+ echo "CHUNK" > bar-chunk
+
+ $CEPH_TOOL osd pool set $POOL fingerprint_algorithm sha1 --yes-i-really-mean-it
+
+ $RADOS_TOOL -p $POOL put foo ./foo
+ $RADOS_TOOL -p $POOL put bar ./bar
+
+ $RADOS_TOOL -p $CHUNK_POOL put bar-chunk ./bar-chunk
+ $RADOS_TOOL -p $CHUNK_POOL put foo-chunk ./foo-chunk
+
+ $RADOS_TOOL -p $POOL set-chunk bar 0 8 --target-pool $CHUNK_POOL bar-chunk 0 --with-reference
+ $RADOS_TOOL -p $POOL set-chunk foo 0 8 --target-pool $CHUNK_POOL foo-chunk 0 --with-reference
+
+ echo "There hi" > test_obj
+ # dirty
+ $RADOS_TOOL -p $POOL put foo ./test_obj
+ # flush
+ $RADOS_TOOL -p $POOL put foo ./test_obj
+ sleep 2
+
+ rados ls -p $CHUNK_POOL
+ CHUNK_OID=$(echo -n "There hi" | sha1sum)
+
+ $DEDUP_TOOL --op add_chunk_ref --pool $POOL --chunk_pool $CHUNK_POOL --object $CHUNK_OID --target_ref bar
+ RESULT=$($DEDUP_TOOL --op get_chunk_ref --pool $POOL --chunk_pool $CHUNK_POOL --object $CHUNK_OID)
+
+ $DEDUP_TOOL --op chunk_scrub --pool $POOL --chunk_pool $CHUNK_POOL
+
+ RESULT=$($DEDUP_TOOL --op get_chunk_ref --pool $POOL --chunk_pool $CHUNK_POOL --object $CHUNK_OID | grep bar)
+ if [ -n "$RESULT" ] ; then
+ $CEPH_TOOL osd pool delete $POOL $POOL --yes-i-really-really-mean-it
+ $CEPH_TOOL osd pool delete $CHUNK_POOL $CHUNK_POOL --yes-i-really-really-mean-it
+ die "Scrub failed expecting bar is removed"
+ fi
+
+ $CEPH_TOOL osd pool delete $CHUNK_POOL $CHUNK_POOL --yes-i-really-really-mean-it
+
+ rm -rf ./foo ./bar ./foo-chunk ./bar-chunk ./test_obj
+}
+
+test_dedup_ratio_fixed
+test_dedup_chunk_scrub
+
+$CEPH_TOOL osd pool delete $POOL $POOL --yes-i-really-really-mean-it
+
+echo "SUCCESS!"
+exit 0
+
+
+++ /dev/null
-#!/usr/bin/env bash
-
-set -x
-
-die() {
- echo "$@"
- exit 1
-}
-
-do_run() {
- if [ "$1" == "--tee" ]; then
- shift
- tee_out="$1"
- shift
- "$@" | tee $tee_out
- else
- "$@"
- fi
-}
-
-run_expect_succ() {
- echo "RUN_EXPECT_SUCC: " "$@"
- do_run "$@"
- [ $? -ne 0 ] && die "expected success, but got failure! cmd: $@"
-}
-
-run() {
- echo "RUN: " $@
- do_run "$@"
-}
-
-if [ -n "$CEPH_BIN" ] ; then
- # CMake env
- RADOS_TOOL="$CEPH_BIN/rados"
- CEPH_TOOL="$CEPH_BIN/ceph"
- DEDUP_TOOL="$CEPH_BIN/cephdeduptool"
-else
- # executables should be installed by the QA env
- RADOS_TOOL=$(which rados)
- CEPH_TOOL=$(which ceph)
- DEDUP_TOOL=$(which cephdeduptool)
-fi
-
-POOL=dedup_pool
-OBJ=test_rados_obj
-
-[ -x "$RADOS_TOOL" ] || die "couldn't find $RADOS_TOOL binary to test"
-[ -x "$CEPH_TOOL" ] || die "couldn't find $CEPH_TOOL binary to test"
-
-run_expect_succ "$CEPH_TOOL" osd pool create "$POOL" 8
-
-function test_dedup_ratio_fixed()
-{
- # case 1
- dd if=/dev/urandom of=dedup_object_1k bs=1K count=1
- dd if=dedup_object_1k of=dedup_object_100k bs=1K count=100
-
- $RADOS_TOOL -p $POOL put $OBJ ./dedup_object_100k
- RESULT=$($DEDUP_TOOL --op estimate --pool $POOL --chunk-size 1024 --chunk-algorithm fixed --fingerprint-algorithm sha1 --debug | grep result | awk '{print$4}')
- if [ 1024 -ne $RESULT ];
- then
- die "Estimate failed expecting 1024 result $RESULT"
- fi
-
- # case 2
- dd if=/dev/zero of=dedup_object_10m bs=10M count=1
-
- $RADOS_TOOL -p $POOL put $OBJ ./dedup_object_10m
- RESULT=$($DEDUP_TOOL --op estimate --pool $POOL --chunk-size 4096 --chunk-algorithm fixed --fingerprint-algorithm sha1 --debug | grep result | awk '{print$4}')
- if [ 4096 -ne $RESULT ];
- then
- die "Estimate failed expecting 4096 result $RESULT"
- fi
-
- # case 3 max_thread
- for num in `seq 0 20`
- do
- dd if=/dev/zero of=dedup_object_$num bs=4M count=1
- $RADOS_TOOL -p $POOL put dedup_object_$num ./dedup_object_$num
- done
-
- RESULT=$($DEDUP_TOOL --op estimate --pool $POOL --chunk-size 4096 --chunk-algorithm fixed --fingerprint-algorithm sha1 --max-thread 4 --debug | grep result | awk '{print$2}')
-
- if [ 98566144 -ne $RESULT ];
- then
- die "Estimate failed expecting 98566144 result $RESULT"
- fi
-
- rm -rf ./dedup_object_1k ./dedup_object_100k ./dedup_object_10m
- for num in `seq 0 20`
- do
- rm -rf ./dedup_object_$num
- done
-}
-
-function test_dedup_chunk_scrub()
-{
-
- CHUNK_POOL=dedup_chunk_pool
- run_expect_succ "$CEPH_TOOL" osd pool create "$CHUNK_POOL" 8
-
- echo "hi there" > foo
-
- echo "hi there" > bar
-
- echo "there" > foo-chunk
-
- echo "CHUNK" > bar-chunk
-
- $CEPH_TOOL osd pool set $POOL fingerprint_algorithm sha1 --yes-i-really-mean-it
-
- $RADOS_TOOL -p $POOL put foo ./foo
- $RADOS_TOOL -p $POOL put bar ./bar
-
- $RADOS_TOOL -p $CHUNK_POOL put bar-chunk ./bar-chunk
- $RADOS_TOOL -p $CHUNK_POOL put foo-chunk ./foo-chunk
-
- $RADOS_TOOL -p $POOL set-chunk bar 0 8 --target-pool $CHUNK_POOL bar-chunk 0 --with-reference
- $RADOS_TOOL -p $POOL set-chunk foo 0 8 --target-pool $CHUNK_POOL foo-chunk 0 --with-reference
-
- echo "There hi" > test_obj
- # dirty
- $RADOS_TOOL -p $POOL put foo ./test_obj
- # flush
- $RADOS_TOOL -p $POOL put foo ./test_obj
- sleep 2
-
- rados ls -p $CHUNK_POOL
- CHUNK_OID=$(echo -n "There hi" | sha1sum)
-
- $DEDUP_TOOL --op add_chunk_ref --pool $POOL --chunk_pool $CHUNK_POOL --object $CHUNK_OID --target_ref bar
- RESULT=$($DEDUP_TOOL --op get_chunk_ref --pool $POOL --chunk_pool $CHUNK_POOL --object $CHUNK_OID)
-
- $DEDUP_TOOL --op chunk_scrub --pool $POOL --chunk_pool $CHUNK_POOL
-
- RESULT=$($DEDUP_TOOL --op get_chunk_ref --pool $POOL --chunk_pool $CHUNK_POOL --object $CHUNK_OID | grep bar)
- if [ -n "$RESULT" ] ; then
- $CEPH_TOOL osd pool delete $POOL $POOL --yes-i-really-really-mean-it
- $CEPH_TOOL osd pool delete $CHUNK_POOL $CHUNK_POOL --yes-i-really-really-mean-it
- die "Scrub failed expecting bar is removed"
- fi
-
- $CEPH_TOOL osd pool delete $CHUNK_POOL $CHUNK_POOL --yes-i-really-really-mean-it
-
- rm -rf ./foo ./bar ./foo-chunk ./bar-chunk ./test_obj
-}
-
-test_dedup_ratio_fixed
-test_dedup_chunk_scrub
-
-$CEPH_TOOL osd pool delete $POOL $POOL --yes-i-really-really-mean-it
-
-echo "SUCCESS!"
-exit 0
-
-
m_cond.Signal();
}
virtual void print_status(Formatter *f, ostream &out) = 0;
+ uint64_t count_objects(IoCtx &ioctx, ObjectCursor &begin, ObjectCursor &end);
uint64_t get_examined_objects() { return examined_objects; }
uint64_t get_total_bytes() { return total_bytes; }
uint64_t get_total_objects() { return total_objects; }
chunk_size(chunk_size) { }
void* entry() {
+ count_objects(io_ctx, begin, end);
estimate_dedup_ratio();
return NULL;
}
class ChunkScrub: public EstimateThread
{
IoCtx chunk_io_ctx;
+ int fixed_objects = 0;
public:
ChunkScrub(IoCtx& io_ctx, int n, int m, ObjectCursor begin, ObjectCursor end,
EstimateThread(io_ctx, n, m, begin, end, timeout), chunk_io_ctx(chunk_io_ctx)
{ }
void* entry() {
+ count_objects(chunk_io_ctx, begin, end);
chunk_scrub_common();
return NULL;
}
void chunk_scrub_common();
+ int get_fixed_objects() { return fixed_objects; }
void print_status(Formatter *f, ostream &out);
};
vector<std::unique_ptr<EstimateThread>> estimate_threads;
+uint64_t EstimateThread::count_objects(IoCtx &ioctx, ObjectCursor &begin, ObjectCursor &end)
+{
+ ObjectCursor shard_start;
+ ObjectCursor shard_end;
+ uint64_t count = 0;
+
+ ioctx.object_list_slice(
+ begin,
+ end,
+ n,
+ m,
+ &shard_start,
+ &shard_end);
+
+ ObjectCursor c(shard_start);
+ while(c < shard_end)
+ {
+ std::vector<ObjectItem> result;
+ int r = ioctx.object_list(c, shard_end, 12, {}, &result, &c);
+ if (r < 0 ) {
+ cerr << "error object_list : " << cpp_strerror(r) << std::endl;
+ return 0;
+ }
+ count += result.size();
+ total_objects += result.size();
+ }
+ return count;
+}
+
static void print_dedup_estimate(bool debug = false)
{
uint64_t total_size = 0;
uint64_t dedup_size = 0;
uint64_t examined_objects = 0;
+ uint64_t total_objects = 0;
EstimateDedupRatio *ratio = NULL;
for (auto &et : estimate_threads) {
Mutex::Locker l(glock);
for (auto &et : estimate_threads) {
total_size += et->get_total_bytes();
examined_objects += et->get_examined_objects();
+ total_objects += et->get_total_objects();
}
- cout << " result: " << total_size << " / " << dedup_size << " (total size / dedup size) " << std::endl;
- cout << " dedup ratio: " << dedup_size/total_size << std::endl;
- cout << " examined objects: " << examined_objects << std::endl;
+ cout << " result: " << total_size << " | " << dedup_size << " (total size | deduped size) " << std::endl;
+ cout << " Dedup ratio: " << (100 - (double)(dedup_size)/total_size*100) << " % " << std::endl;
+ cout << " Examined objects: " << examined_objects << std::endl;
+ cout << " Total objects: " << total_objects << std::endl;
}
static void handle_signal(int signum)
}
examined_objects++;
}
- total_objects++;
}
}
if (ret < 0) {
continue;
}
+ fixed_objects++;
}
examined_objects++;
m_cond.WaitInterval(m_lock,utime_t(0, COND_WAIT_INTERVAL));
cur_time = ceph_clock_now();
}
}
- total_objects++;
}
}
f->open_object_section("Status");
f->dump_string("Total object", stringify(total_objects));
f->dump_string("Examined objectes", stringify(examined_objects));
+ f->dump_string("Fixed objectes", stringify(fixed_objects));
f->close_section();
f->flush(out);
cout << std::endl;
int ret;
std::map<std::string, std::string>::const_iterator i;
bool debug = false;
+ ObjectCursor begin;
+ ObjectCursor end;
i = opts.find("pool");
if (i != opts.end()) {
}
glock.Lock();
+ begin = io_ctx.object_list_begin();
+ end = io_ctx.object_list_end();
for (unsigned i = 0; i < max_thread; i++) {
- ObjectCursor begin = io_ctx.object_list_begin();
- ObjectCursor end = io_ctx.object_list_end();
std::unique_ptr<EstimateThread> ptr (new EstimateDedupRatio(io_ctx, i, max_thread, begin, end,
chunk_algo, fp_algo, chunk_size,
report_period));
{
uint64_t total_objects = 0;
uint64_t examined_objects = 0;
+ int fixed_objects = 0;
for (auto &et : estimate_threads) {
total_objects += et->get_total_objects();
examined_objects += et->get_examined_objects();
+ ChunkScrub *ptr = static_cast<ChunkScrub*>(et.get());
+ fixed_objects += ptr->get_fixed_objects();
}
cout << " Total object : " << total_objects << std::endl;
cout << " Examined object : " << examined_objects << std::endl;
+ cout << " Fixed object : " << fixed_objects << std::endl;
}
int chunk_scrub_common(const std::map < std::string, std::string > &opts,
unsigned max_thread = default_max_thread;
std::map<std::string, std::string>::const_iterator i;
uint32_t report_period = default_report_period;
+ ObjectCursor begin;
+ ObjectCursor end;
i = opts.find("pool");
if (i != opts.end()) {
}
glock.Lock();
+ begin = io_ctx.object_list_begin();
+ end = io_ctx.object_list_end();
for (unsigned i = 0; i < max_thread; i++) {
- ObjectCursor begin = io_ctx.object_list_begin();
- ObjectCursor end = io_ctx.object_list_end();
std::unique_ptr<EstimateThread> ptr (new ChunkScrub(io_ctx, i, max_thread, begin, end, chunk_io_ctx,
report_period));
ptr->create("estimate_thread");
projects / ceph-ci.git / commitdiff