From c1b69f48d29efd22e9411b142267b46843ba415d Mon Sep 17 00:00:00 2001 From: myoungwon oh Date: Thu, 11 Jul 2019 20:24:11 +0900 Subject: [PATCH] src/test, qa/suites/rados/thrash: add dedup test Signed-off-by: Myoungwon Oh --- .../rados/thrash/workloads/dedup_tier.yaml | 14 +++++++++ qa/tasks/rados.py | 2 ++ src/test/osd/RadosModel.h | 31 ++++++++++++++++--- src/test/osd/TestRados.cc | 17 +++++++--- 4 files changed, 54 insertions(+), 10 deletions(-) create mode 100644 qa/suites/rados/thrash/workloads/dedup_tier.yaml diff --git a/qa/suites/rados/thrash/workloads/dedup_tier.yaml b/qa/suites/rados/thrash/workloads/dedup_tier.yaml new file mode 100644 index 0000000000000..26c3c2b7092a1 --- /dev/null +++ b/qa/suites/rados/thrash/workloads/dedup_tier.yaml @@ -0,0 +1,14 @@ +tasks: +- exec: + client.0: + - sudo ceph osd pool create low_tier 4 +- rados: + clients: [client.0] + low_tier_pool: 'low_tier' + ops: 1500 + objects: 50 + set_chunk: true + enable_dedup: true + op_weights: + read: 100 + write: 50 diff --git a/qa/tasks/rados.py b/qa/tasks/rados.py index 26a84d734af1b..64d21d11e90b5 100644 --- a/qa/tasks/rados.py +++ b/qa/tasks/rados.py @@ -147,6 +147,8 @@ def task(ctx, config): args.extend(['--set_redirect']) if config.get('set_chunk', False): args.extend(['--set_chunk']) + if config.get('enable_dedup', False): + args.extend(['--enable_dedup']) if config.get('low_tier_pool', None): args.extend(['--low_tier_pool', config.get('low_tier_pool', None)]) if config.get('pool_snaps', False): diff --git a/src/test/osd/RadosModel.h b/src/test/osd/RadosModel.h index f7cac8e16be74..2253b846695ce 100644 --- a/src/test/osd/RadosModel.h +++ b/src/test/osd/RadosModel.h @@ -198,6 +198,7 @@ public: librados::IoCtx low_tier_io_ctx; int snapname_num; map redirect_objs; + bool enable_dedup; RadosTestContext(const string &pool_name, int max_in_flight, @@ -209,6 +210,7 @@ public: bool pool_snaps, bool write_fadvise_dontneed, const string &low_tier_pool_name, + bool enable_dedup, const char *id = 0) : state_lock("Context Lock"), pool_obj_cont(), @@ -227,7 +229,8 @@ public: pool_snaps(pool_snaps), write_fadvise_dontneed(write_fadvise_dontneed), low_tier_pool_name(low_tier_pool_name), - snapname_num(0) + snapname_num(0), + enable_dedup(enable_dedup) { } @@ -266,6 +269,17 @@ public: rados.shutdown(); return r; } + if (enable_dedup) { + r = rados.mon_command( + "{\"prefix\": \"osd pool set\", \"pool\": \"" + pool_name + + "\", \"var\": \"fingerprint_algorithm\", \"val\": \"" + "sha256" + "\"}", + inbl, NULL, NULL); + if (r < 0) { + rados.shutdown(); + return r; + } + } + char hostname_cstr[100]; gethostname(hostname_cstr, 100); stringstream hostpid; @@ -2305,6 +2319,7 @@ public: uint64_t offset; uint32_t length; uint64_t tgt_offset; + bool enable_with_reference; SetChunkOp(int n, RadosTestContext *context, const string &oid, @@ -2313,12 +2328,13 @@ public: const string &oid_tgt, const string &tgt_pool_name, uint64_t tgt_offset, - TestOpStat *stat = 0) + TestOpStat *stat = 0, + bool enable_with_reference = false) : TestOp(n, context, stat), oid(oid), oid_tgt(oid_tgt), tgt_pool_name(tgt_pool_name), comp(NULL), done(0), r(0), offset(offset), length(length), - tgt_offset(tgt_offset) + tgt_offset(tgt_offset), enable_with_reference(enable_with_reference) {} void _begin() override @@ -2334,8 +2350,13 @@ public: if (src_value.version != 0 && !src_value.deleted()) op.assert_version(src_value.version); - op.set_chunk(offset, length, context->low_tier_io_ctx, - context->prefix+oid_tgt, tgt_offset); + if (enable_with_reference) { + op.set_chunk(offset, length, context->low_tier_io_ctx, + context->prefix+oid_tgt, tgt_offset, CEPH_OSD_OP_FLAG_WITH_REFERENCE); + } else { + op.set_chunk(offset, length, context->low_tier_io_ctx, + context->prefix+oid_tgt, tgt_offset); + } pair *cb_arg = new pair(this, diff --git a/src/test/osd/TestRados.cc b/src/test/osd/TestRados.cc index 78f067c1d7bc4..d70c5274f1754 100644 --- a/src/test/osd/TestRados.cc +++ b/src/test/osd/TestRados.cc @@ -31,14 +31,16 @@ public: bool ec_pool, bool balance_reads, bool set_redirect, - bool set_chunk) : + bool set_chunk, + bool enable_dedup) : m_nextop(NULL), m_op(0), m_ops(ops), m_seconds(max_seconds), m_objects(objects), m_stats(stats), m_total_weight(0), m_ec_pool(ec_pool), m_balance_reads(balance_reads), m_set_redirect(set_redirect), - m_set_chunk(set_chunk) + m_set_chunk(set_chunk), + m_enable_dedup(enable_dedup) { m_start = time(0); for (map::const_iterator it = op_weights.begin(); @@ -243,7 +245,7 @@ public: << " length: " << rand_length << " target oid " << oid2.str() << " tgt_offset: " << rand_tgt_offset << std::endl; op = new SetChunkOp(m_op, &context, oid.str(), rand_offset, rand_length, oid2.str(), - context.low_tier_pool_name, rand_tgt_offset, m_stats); + context.low_tier_pool_name, rand_tgt_offset, m_stats, m_enable_dedup); return true; } } else if (m_op == make_manifest_end + 1) { @@ -266,7 +268,7 @@ public: cout << " redirect_not_in_use: " << oid.str() << std::endl; context.oid_redirect_not_in_use.insert(oid.str()); } - } + } return false; } @@ -451,6 +453,7 @@ private: bool m_balance_reads; bool m_set_redirect; bool m_set_chunk; + bool m_enable_dedup; }; int main(int argc, char **argv) @@ -505,6 +508,7 @@ int main(int argc, char **argv) bool balance_reads = false; bool set_redirect = false; bool set_chunk = false; + bool enable_dedup = false; for (int i = 1; i < argc; ++i) { if (strcmp(argv[i], "--max-ops") == 0) @@ -585,6 +589,8 @@ int main(int argc, char **argv) * to prevent the race. see https://github.com/ceph/ceph/pull/20096 */ low_tier_pool_name = argv[++i]; + } else if (strcmp(argv[i], "--enable_dedup") == 0) { + enable_dedup = true; } else { cerr << "unknown arg " << argv[i] << std::endl; exit(1); @@ -647,13 +653,14 @@ int main(int argc, char **argv) pool_snaps, write_fadvise_dontneed, low_tier_pool_name, + enable_dedup, id); TestOpStat stats; WeightedTestGenerator gen = WeightedTestGenerator( ops, objects, op_weights, &stats, max_seconds, - ec_pool, balance_reads, set_redirect, set_chunk); + ec_pool, balance_reads, set_redirect, set_chunk, enable_dedup); int r = context.init(); if (r < 0) { cerr << "Error initializing rados test context: " -- 2.39.5