src/test, qa/suites/rados/thrash: add dedup test

Signed-off-by: Myoungwon Oh <omwmw@sk.com>

diff --git a/qa/suites/rados/thrash/workloads/dedup_tier.yaml b/qa/suites/rados/thrash/workloads/dedup_tier.yaml
new file mode 100644 (file)
index 0000000..26c3c2b
--- /dev/null
+++ b/qa/suites/rados/thrash/workloads/dedup_tier.yaml
@@ -0,0 +1,14 @@
+tasks:
+- exec:
+    client.0:
+      - sudo ceph osd pool create low_tier 4
+- rados:
+    clients: [client.0]
+    low_tier_pool: 'low_tier'
+    ops: 1500
+    objects: 50
+    set_chunk: true
+    enable_dedup: true
+    op_weights:
+      read: 100
+      write: 50

diff --git a/qa/tasks/rados.py b/qa/tasks/rados.py
index 26a84d734af1bb47c04a55fc230f6490fdce3528..64d21d11e90b54e8a5adc1de8d145cc5b9f92a8d 100644 (file)
--- a/qa/tasks/rados.py
+++ b/qa/tasks/rados.py
@@ -147,6 +147,8 @@ def task(ctx, config):
         args.extend(['--set_redirect'])
     if config.get('set_chunk', False):
         args.extend(['--set_chunk'])
+    if config.get('enable_dedup', False):
+        args.extend(['--enable_dedup'])
     if config.get('low_tier_pool', None):
         args.extend(['--low_tier_pool', config.get('low_tier_pool', None)])
     if config.get('pool_snaps', False):

diff --git a/src/test/osd/RadosModel.h b/src/test/osd/RadosModel.h
index f7cac8e16be7424cf2ab4354b0f6d403b5f38265..2253b846695ceda60956c8377c145b1f5778eb1c 100644 (file)
--- a/src/test/osd/RadosModel.h
+++ b/src/test/osd/RadosModel.h
@@ -198,6 +198,7 @@ public:
   librados::IoCtx low_tier_io_ctx;
   int snapname_num;
   map<string,string > redirect_objs;
+  bool enable_dedup;
 
   RadosTestContext(const string &pool_name, 
                   int max_in_flight,
@@ -209,6 +210,7 @@ public:
                   bool pool_snaps,
                   bool write_fadvise_dontneed,
                   const string &low_tier_pool_name,
+                  bool enable_dedup,
                   const char *id = 0) :
     state_lock("Context Lock"),
     pool_obj_cont(),
@@ -227,7 +229,8 @@ public:
     pool_snaps(pool_snaps),
     write_fadvise_dontneed(write_fadvise_dontneed),
     low_tier_pool_name(low_tier_pool_name),
-    snapname_num(0)
+    snapname_num(0),
+    enable_dedup(enable_dedup)
   {
   }
 
@@ -266,6 +269,17 @@ public:
       rados.shutdown();
       return r;
     }
+    if (enable_dedup) {
+      r = rados.mon_command(
+       "{\"prefix\": \"osd pool set\", \"pool\": \"" + pool_name +
+       "\", \"var\": \"fingerprint_algorithm\", \"val\": \"" + "sha256" + "\"}",
+       inbl, NULL, NULL);
+      if (r < 0) {
+       rados.shutdown();
+       return r;
+      }
+    }
+
     char hostname_cstr[100];
     gethostname(hostname_cstr, 100);
     stringstream hostpid;
@@ -2305,6 +2319,7 @@ public:
   uint64_t offset;
   uint32_t length;
   uint64_t tgt_offset;
+  bool enable_with_reference;
   SetChunkOp(int n,
             RadosTestContext *context,
             const string &oid,
@@ -2313,12 +2328,13 @@ public:
             const string &oid_tgt,
             const string &tgt_pool_name,
             uint64_t tgt_offset,
-            TestOpStat *stat = 0)
+            TestOpStat *stat = 0,
+            bool enable_with_reference = false)
     : TestOp(n, context, stat),
       oid(oid), oid_tgt(oid_tgt), tgt_pool_name(tgt_pool_name),
       comp(NULL), done(0), 
       r(0), offset(offset), length(length), 
-      tgt_offset(tgt_offset)
+      tgt_offset(tgt_offset), enable_with_reference(enable_with_reference)
   {}
 
   void _begin() override
@@ -2334,8 +2350,13 @@ public:
 
     if (src_value.version != 0 && !src_value.deleted())
       op.assert_version(src_value.version);
-    op.set_chunk(offset, length, context->low_tier_io_ctx, 
-                context->prefix+oid_tgt, tgt_offset);
+    if (enable_with_reference) {
+      op.set_chunk(offset, length, context->low_tier_io_ctx, 
+                  context->prefix+oid_tgt, tgt_offset, CEPH_OSD_OP_FLAG_WITH_REFERENCE);
+    } else {
+      op.set_chunk(offset, length, context->low_tier_io_ctx, 
+                  context->prefix+oid_tgt, tgt_offset);
+    }
 
     pair<TestOp*, TestOp::CallbackInfo*> *cb_arg =
       new pair<TestOp*, TestOp::CallbackInfo*>(this,

diff --git a/src/test/osd/TestRados.cc b/src/test/osd/TestRados.cc
index 78f067c1d7bc4d06dcad513c9a786cb742ea2b3a..d70c5274f1754f211f26845428288d01b78eeb24 100644 (file)
--- a/src/test/osd/TestRados.cc
+++ b/src/test/osd/TestRados.cc
@@ -31,14 +31,16 @@ public:
                        bool ec_pool,
                        bool balance_reads,
                        bool set_redirect,
-                       bool set_chunk) :
+                       bool set_chunk,
+                       bool enable_dedup) :
     m_nextop(NULL), m_op(0), m_ops(ops), m_seconds(max_seconds),
     m_objects(objects), m_stats(stats),
     m_total_weight(0),
     m_ec_pool(ec_pool),
     m_balance_reads(balance_reads),
     m_set_redirect(set_redirect),
-    m_set_chunk(set_chunk)
+    m_set_chunk(set_chunk),
+    m_enable_dedup(enable_dedup)
   {
     m_start = time(0);
     for (map<TestOpType, unsigned int>::const_iterator it = op_weights.begin();
@@ -243,7 +245,7 @@ public:
             << " length: " << rand_length <<  " target oid " << oid2.str() 
             << " tgt_offset: " << rand_tgt_offset << std::endl;
        op = new SetChunkOp(m_op, &context, oid.str(), rand_offset, rand_length, oid2.str(), 
-                             context.low_tier_pool_name, rand_tgt_offset, m_stats);
+                             context.low_tier_pool_name, rand_tgt_offset, m_stats, m_enable_dedup);
        return true;
       }
     } else if (m_op == make_manifest_end + 1) {
@@ -266,7 +268,7 @@ public:
        cout << " redirect_not_in_use: " << oid.str() << std::endl;
        context.oid_redirect_not_in_use.insert(oid.str());
       }
-    }
+    } 
 
     return false;
   }
@@ -451,6 +453,7 @@ private:
   bool m_balance_reads;
   bool m_set_redirect;
   bool m_set_chunk;
+  bool m_enable_dedup;
 };
 
 int main(int argc, char **argv)
@@ -505,6 +508,7 @@ int main(int argc, char **argv)
   bool balance_reads = false;
   bool set_redirect = false;
   bool set_chunk = false;
+  bool enable_dedup = false;
 
   for (int i = 1; i < argc; ++i) {
     if (strcmp(argv[i], "--max-ops") == 0)
@@ -585,6 +589,8 @@ int main(int argc, char **argv)
        * to prevent the race. see https://github.com/ceph/ceph/pull/20096
        */
       low_tier_pool_name = argv[++i];
+    } else if (strcmp(argv[i], "--enable_dedup") == 0) {
+      enable_dedup = true;
     } else {
       cerr << "unknown arg " << argv[i] << std::endl;
       exit(1);
@@ -647,13 +653,14 @@ int main(int argc, char **argv)
     pool_snaps,
     write_fadvise_dontneed,
     low_tier_pool_name,
+    enable_dedup,
     id);
 
   TestOpStat stats;
   WeightedTestGenerator gen = WeightedTestGenerator(
     ops, objects,
     op_weights, &stats, max_seconds,
-    ec_pool, balance_reads, set_redirect, set_chunk);
+    ec_pool, balance_reads, set_redirect, set_chunk, enable_dedup);
   int r = context.init();
   if (r < 0) {
     cerr << "Error initializing rados test context: "