From 565320f04d60b03385dc0824af5a8de1485279e5 Mon Sep 17 00:00:00 2001 From: myoungwon oh Date: Mon, 2 May 2022 23:14:45 +0900 Subject: [PATCH] tool/ceph-dedup-tool: add performing dedup option on cloned object Signed-off-by: Myoungwon Oh --- qa/workunits/rados/test_dedup_tool.sh | 25 +++++ src/tools/ceph_dedup_tool.cc | 128 ++++++++++++++++---------- 2 files changed, 106 insertions(+), 47 deletions(-) diff --git a/qa/workunits/rados/test_dedup_tool.sh b/qa/workunits/rados/test_dedup_tool.sh index 85351f8a806..ecfaeb1411e 100755 --- a/qa/workunits/rados/test_dedup_tool.sh +++ b/qa/workunits/rados/test_dedup_tool.sh @@ -299,6 +299,31 @@ function test_dedup_object() die "Comparing failed expecting chunk mismatch" fi + echo -n "THERE HIHIHI" > bar + $RADOS_TOOL -p $POOL put bar ./bar + $RADOS_TOOL -p $POOL mksnap mysnap + + echo -n "There HIHIHI" > bar + $RADOS_TOOL -p $POOL put bar ./bar + + RESULT=$($DEDUP_TOOL --pool $POOL --op object-dedup --object bar --chunk-pool $CHUNK_POOL --fingerprint-algorithm sha1 --dedup-cdc-chunk-size 4096 --snap) + + CHUNK_OID=$(echo -n "THERE HIHIHI" | sha1sum | awk '{print $1}') + RESULT=$($DEDUP_TOOL --op dump-chunk-refs --chunk-pool $CHUNK_POOL --object $CHUNK_OID | grep bar) + if [ -z "$RESULT" ] ; then + $CEPH_TOOL osd pool delete $POOL $POOL --yes-i-really-really-mean-it + $CEPH_TOOL osd pool delete $CHUNK_POOL $CHUNK_POOL --yes-i-really-really-mean-it + die "Scrub failed expecting bar is removed" + fi + + CHUNK_OID=$(echo -n "There HIHIHI" | sha1sum | awk '{print $1}') + RESULT=$($DEDUP_TOOL --op dump-chunk-refs --chunk-pool $CHUNK_POOL --object $CHUNK_OID | grep bar) + if [ -z "$RESULT" ] ; then + $CEPH_TOOL osd pool delete $POOL $POOL --yes-i-really-really-mean-it + $CEPH_TOOL osd pool delete $CHUNK_POOL $CHUNK_POOL --yes-i-really-really-mean-it + die "Scrub failed expecting bar is removed" + fi + $CEPH_TOOL osd pool delete $CHUNK_POOL $CHUNK_POOL --yes-i-really-really-mean-it rm -rf ./foo ./bar ./chunk diff --git a/src/tools/ceph_dedup_tool.cc b/src/tools/ceph_dedup_tool.cc index 4b8c926f75e..ba2a9f518c8 100644 --- a/src/tools/ceph_dedup_tool.cc +++ b/src/tools/ceph_dedup_tool.cc @@ -1100,6 +1100,7 @@ int make_dedup_object(const std::map < std::string, std::string > &opts, } } else if (op_name == "object-dedup") { unsigned chunk_size; + bool snap = false; i = opts.find("dedup-cdc-chunk-size"); if (i != opts.end()) { if (rados_sistrtoll(i, &chunk_size)) { @@ -1107,6 +1108,10 @@ int make_dedup_object(const std::map < std::string, std::string > &opts, return -EINVAL; } } + i = opts.find("snap"); + if (i != opts.end()) { + snap = true; + } bufferlist inbl; ret = rados.mon_command( @@ -1145,57 +1150,84 @@ int make_dedup_object(const std::map < std::string, std::string > &opts, * it and replace it with the real contents. */ // convert object to manifest object - ObjectWriteOperation op; - bufferlist temp; - temp.append("temp"); - op.write_full(temp); - - auto gen_r_num = [] () -> string { - std::random_device rd; - std::mt19937 gen(rd()); - std::uniform_int_distribution dist; - uint64_t r_num = dist(gen); - return to_string(r_num); - }; - string temp_oid = gen_r_num(); - // create temp chunk object for set-chunk - ret = chunk_io_ctx.operate(temp_oid, &op); - if (ret == -EEXIST) { - // one more try - temp_oid = gen_r_num(); + auto create_new_deduped_object = + [&chunk_io_ctx, &io_ctx](string object_name) -> int { + + int ret = 0; + ObjectWriteOperation op; + bufferlist temp; + temp.append("temp"); + op.write_full(temp); + + auto gen_r_num = [] () -> string { + std::random_device rd; + std::mt19937 gen(rd()); + std::uniform_int_distribution dist; + uint64_t r_num = dist(gen); + return to_string(r_num); + }; + string temp_oid = gen_r_num(); + // create temp chunk object for set-chunk ret = chunk_io_ctx.operate(temp_oid, &op); - } - if (ret < 0) { - cerr << " operate fail : " << cpp_strerror(ret) << std::endl; - goto out; - } + if (ret == -EEXIST) { + // one more try + temp_oid = gen_r_num(); + ret = chunk_io_ctx.operate(temp_oid, &op); + } + if (ret < 0) { + cerr << " operate fail : " << cpp_strerror(ret) << std::endl; + return ret; + } - // set-chunk to make manifest object - ObjectReadOperation chunk_op; - chunk_op.set_chunk(0, 4, chunk_io_ctx, temp_oid, 0, - CEPH_OSD_OP_FLAG_WITH_REFERENCE); - ret = io_ctx.operate(object_name, &chunk_op, NULL); - if (ret < 0) { - cerr << " set_chunk fail : " << cpp_strerror(ret) << std::endl; - goto out; - } + // set-chunk to make manifest object + ObjectReadOperation chunk_op; + chunk_op.set_chunk(0, 4, chunk_io_ctx, temp_oid, 0, + CEPH_OSD_OP_FLAG_WITH_REFERENCE); + ret = io_ctx.operate(object_name, &chunk_op, NULL); + if (ret < 0) { + cerr << " set_chunk fail : " << cpp_strerror(ret) << std::endl; + return ret; + } - // tier-flush to perform deduplication - ObjectReadOperation flush_op; - flush_op.tier_flush(); - ret = io_ctx.operate(object_name, &flush_op, NULL); - if (ret < 0) { - cerr << " tier_flush fail : " << cpp_strerror(ret) << std::endl; - goto out; - } + // tier-flush to perform deduplication + ObjectReadOperation flush_op; + flush_op.tier_flush(); + ret = io_ctx.operate(object_name, &flush_op, NULL); + if (ret < 0) { + cerr << " tier_flush fail : " << cpp_strerror(ret) << std::endl; + return ret; + } - // tier-evict - ObjectReadOperation evict_op; - evict_op.tier_evict(); - ret = io_ctx.operate(object_name, &evict_op, NULL); - if (ret < 0) { - cerr << " tier_evict fail : " << cpp_strerror(ret) << std::endl; - goto out; + // tier-evict + ObjectReadOperation evict_op; + evict_op.tier_evict(); + ret = io_ctx.operate(object_name, &evict_op, NULL); + if (ret < 0) { + cerr << " tier_evict fail : " << cpp_strerror(ret) << std::endl; + return ret; + } + return ret; + }; + + if (snap) { + io_ctx.snap_set_read(librados::SNAP_DIR); + snap_set_t snap_set; + int snap_ret; + ObjectReadOperation op; + op.list_snaps(&snap_set, &snap_ret); + io_ctx.operate(object_name, &op, NULL); + + for (vector::const_iterator r = snap_set.clones.begin(); + r != snap_set.clones.end(); + ++r) { + io_ctx.snap_set_read(r->cloneid); + ret = create_new_deduped_object(object_name); + if (ret < 0) { + goto out; + } + } + } else { + ret = create_new_deduped_object(object_name); } } @@ -1269,6 +1301,8 @@ int main(int argc, const char **argv) opts["source-length"] = val; } else if (ceph_argparse_witharg(args, i, &val, "--dedup-cdc-chunk-size", (char*)NULL)) { opts["dedup-cdc-chunk-size"] = val; + } else if (ceph_argparse_flag(args, i, "--snap", (char*)NULL)) { + opts["snap"] = "true"; } else if (ceph_argparse_flag(args, i, "--debug", (char*)NULL)) { opts["debug"] = "true"; } else { -- 2.39.5