osd, mon : add dedup options

add dedup_tier, dedup_chunk_algorithm, dedup_cdc_window_size
and dedup_cdc_chunk_size.

dedup_tier: a pool where chunks are stored.
dedup_chunk_algorithm: chunking algorithm.
dedup_cdc_window_size: the maximum data size to run CDC.
 Dedup process reads amount of dedup_cdc_window_size,
 then calculate chunks by using CDC.
dedup_cdc_chunk_size: chunk size for CDC.

Signed-off-by: Myoungwon Oh <myoungwon.oh@samsumg.com>

diff --git a/src/mon/MonCommands.h b/src/mon/MonCommands.h
index 779f76f9c2421e9e58319680b8d8c97737752c26..93d315b63495f8121e170d192451303c8da44c91 100644 (file)
--- a/src/mon/MonCommands.h
+++ b/src/mon/MonCommands.h
@@ -1090,11 +1090,11 @@ COMMAND("osd pool rename "
        "rename <srcpool> to <destpool>", "osd", "rw")
 COMMAND("osd pool get "
        "name=pool,type=CephPoolname "
-       "name=var,type=CephChoices,strings=size|min_size|pg_num|pgp_num|crush_rule|hashpspool|nodelete|nopgchange|nosizechange|write_fadvise_dontneed|noscrub|nodeep-scrub|hit_set_type|hit_set_period|hit_set_count|hit_set_fpp|use_gmt_hitset|target_max_objects|target_max_bytes|cache_target_dirty_ratio|cache_target_dirty_high_ratio|cache_target_full_ratio|cache_min_flush_age|cache_min_evict_age|erasure_code_profile|min_read_recency_for_promote|all|min_write_recency_for_promote|fast_read|hit_set_grade_decay_rate|hit_set_search_last_n|scrub_min_interval|scrub_max_interval|deep_scrub_interval|recovery_priority|recovery_op_priority|scrub_priority|compression_mode|compression_algorithm|compression_required_ratio|compression_max_blob_size|compression_min_blob_size|csum_type|csum_min_block|csum_max_block|allow_ec_overwrites|fingerprint_algorithm|pg_autoscale_mode|pg_autoscale_bias|pg_num_min|target_size_bytes|target_size_ratio",
+       "name=var,type=CephChoices,strings=size|min_size|pg_num|pgp_num|crush_rule|hashpspool|nodelete|nopgchange|nosizechange|write_fadvise_dontneed|noscrub|nodeep-scrub|hit_set_type|hit_set_period|hit_set_count|hit_set_fpp|use_gmt_hitset|target_max_objects|target_max_bytes|cache_target_dirty_ratio|cache_target_dirty_high_ratio|cache_target_full_ratio|cache_min_flush_age|cache_min_evict_age|erasure_code_profile|min_read_recency_for_promote|all|min_write_recency_for_promote|fast_read|hit_set_grade_decay_rate|hit_set_search_last_n|scrub_min_interval|scrub_max_interval|deep_scrub_interval|recovery_priority|recovery_op_priority|scrub_priority|compression_mode|compression_algorithm|compression_required_ratio|compression_max_blob_size|compression_min_blob_size|csum_type|csum_min_block|csum_max_block|allow_ec_overwrites|fingerprint_algorithm|pg_autoscale_mode|pg_autoscale_bias|pg_num_min|target_size_bytes|target_size_ratio|dedup_tier|dedup_chunk_algorithm|dedup_cdc_window_size|dedup_cdc_chunk_size",
        "get pool parameter <var>", "osd", "r")
 COMMAND("osd pool set "
        "name=pool,type=CephPoolname "
-       "name=var,type=CephChoices,strings=size|min_size|pg_num|pgp_num|pgp_num_actual|crush_rule|hashpspool|nodelete|nopgchange|nosizechange|write_fadvise_dontneed|noscrub|nodeep-scrub|hit_set_type|hit_set_period|hit_set_count|hit_set_fpp|use_gmt_hitset|target_max_bytes|target_max_objects|cache_target_dirty_ratio|cache_target_dirty_high_ratio|cache_target_full_ratio|cache_min_flush_age|cache_min_evict_age|min_read_recency_for_promote|min_write_recency_for_promote|fast_read|hit_set_grade_decay_rate|hit_set_search_last_n|scrub_min_interval|scrub_max_interval|deep_scrub_interval|recovery_priority|recovery_op_priority|scrub_priority|compression_mode|compression_algorithm|compression_required_ratio|compression_max_blob_size|compression_min_blob_size|csum_type|csum_min_block|csum_max_block|allow_ec_overwrites|fingerprint_algorithm|pg_autoscale_mode|pg_autoscale_bias|pg_num_min|target_size_bytes|target_size_ratio "
+       "name=var,type=CephChoices,strings=size|min_size|pg_num|pgp_num|pgp_num_actual|crush_rule|hashpspool|nodelete|nopgchange|nosizechange|write_fadvise_dontneed|noscrub|nodeep-scrub|hit_set_type|hit_set_period|hit_set_count|hit_set_fpp|use_gmt_hitset|target_max_bytes|target_max_objects|cache_target_dirty_ratio|cache_target_dirty_high_ratio|cache_target_full_ratio|cache_min_flush_age|cache_min_evict_age|min_read_recency_for_promote|min_write_recency_for_promote|fast_read|hit_set_grade_decay_rate|hit_set_search_last_n|scrub_min_interval|scrub_max_interval|deep_scrub_interval|recovery_priority|recovery_op_priority|scrub_priority|compression_mode|compression_algorithm|compression_required_ratio|compression_max_blob_size|compression_min_blob_size|csum_type|csum_min_block|csum_max_block|allow_ec_overwrites|fingerprint_algorithm|pg_autoscale_mode|pg_autoscale_bias|pg_num_min|target_size_bytes|target_size_ratio|dedup_tier|dedup_chunk_algorithm|dedup_cdc_window_size|dedup_cdc_chunk_size "
        "name=val,type=CephString "
        "name=yes_i_really_mean_it,type=CephBool,req=false",
        "set pool parameter <var> to <val>", "osd", "rw")

diff --git a/src/mon/OSDMonitor.cc b/src/mon/OSDMonitor.cc
index e154ed72e004422f6ee0610aa203f1b2d34d6066..b55e7461a64cc97b23cbdcf1bf872e2821f0e33a 100644 (file)
--- a/src/mon/OSDMonitor.cc
+++ b/src/mon/OSDMonitor.cc
@@ -5373,7 +5373,8 @@ namespace {
     COMPRESSION_MAX_BLOB_SIZE, COMPRESSION_MIN_BLOB_SIZE,
     CSUM_TYPE, CSUM_MAX_BLOCK, CSUM_MIN_BLOCK, FINGERPRINT_ALGORITHM,
     PG_AUTOSCALE_MODE, PG_NUM_MIN, TARGET_SIZE_BYTES, TARGET_SIZE_RATIO,
-    PG_AUTOSCALE_BIAS };
+    PG_AUTOSCALE_BIAS, DEDUP_TIER, DEDUP_CHUNK_ALGORITHM, DEDUP_CDC_WINDOW_SIZE,
+    DEDUP_CDC_CHUNK_SIZE };
 
   std::set<osd_pool_get_choices>
     subtract_second_from_first(const std::set<osd_pool_get_choices>& first,
@@ -6108,6 +6109,10 @@ bool OSDMonitor::preprocess_command(MonOpRequestRef op)
       {"target_size_bytes", TARGET_SIZE_BYTES},
       {"target_size_ratio", TARGET_SIZE_RATIO},
       {"pg_autoscale_bias", PG_AUTOSCALE_BIAS},
+      {"dedup_tier", DEDUP_TIER},
+      {"dedup_chunk_algorithm", DEDUP_CHUNK_ALGORITHM},
+      {"dedup_cdc_window_size", DEDUP_CDC_WINDOW_SIZE},
+      {"dedup_cdc_chunk_size", DEDUP_CDC_CHUNK_SIZE},
     };
 
     typedef std::set<osd_pool_get_choices> choices_set_t;
@@ -6324,6 +6329,10 @@ bool OSDMonitor::preprocess_command(MonOpRequestRef op)
          case TARGET_SIZE_BYTES:
          case TARGET_SIZE_RATIO:
          case PG_AUTOSCALE_BIAS:
+         case DEDUP_TIER:
+         case DEDUP_CHUNK_ALGORITHM:
+         case DEDUP_CDC_WINDOW_SIZE:
+         case DEDUP_CDC_CHUNK_SIZE:
             pool_opts_t::key_t key = pool_opts_t::get_opt_desc(i->first).key;
             if (p->opts.is_set(key)) {
               if(*it == CSUM_TYPE) {
@@ -6481,6 +6490,10 @@ bool OSDMonitor::preprocess_command(MonOpRequestRef op)
          case TARGET_SIZE_BYTES:
          case TARGET_SIZE_RATIO:
          case PG_AUTOSCALE_BIAS:
+         case DEDUP_TIER:
+         case DEDUP_CHUNK_ALGORITHM:
+         case DEDUP_CDC_WINDOW_SIZE:
+         case DEDUP_CDC_CHUNK_SIZE:
            for (i = ALL_CHOICES.begin(); i != ALL_CHOICES.end(); ++i) {
              if (i->second == *it)
                break;
@@ -8648,6 +8661,39 @@ int OSDMonitor::prepare_command_pool_set(const cmdmap_t& cmdmap,
        ss << "pg_autoscale_bias must be between 0 and 1000";
        return -EINVAL;
       }
+    } else if (var == "dedup_tier") {
+      int64_t lowtierpool_id = osdmap.lookup_pg_pool_name(val);
+      if (lowtierpool_id < 0) {
+       ss << "unrecognized pool '" << val << "'";
+       return -ENOENT;
+      }
+      const pg_pool_t *tp = osdmap.get_pg_pool(lowtierpool_id);
+      ceph_assert(tp);
+      if (p.is_erasure()) {
+       ss << "pool '" << poolstr
+          << "' is an ec pool, which cannot be a tier";
+       return -ENOTSUP;
+      }
+      n = lowtierpool_id;
+      interr.clear(); 
+    } else if (var == "dedup_chunk_algorithm") {
+      if (!unset) {
+        auto alg = pg_pool_t::get_dedup_chunk_algorithm_from_str(val);
+        if (!alg) {
+          ss << "unrecognized fingerprint_algorithm '" << val << "'";
+         return -EINVAL;
+        }
+      }
+    } else if (var == "dedup_cdc_window_size") {
+      if (interr.length()) {
+        ss << "error parsing int value '" << val << "': " << interr;
+        return -EINVAL;
+      }
+    } else if (var == "dedup_cdc_chunk_size") {
+      if (interr.length()) {
+        ss << "error parsing int value '" << val << "': " << interr;
+        return -EINVAL;
+      }
     }
 
     pool_opts_t::opt_desc_t desc = pool_opts_t::get_opt_desc(var);
@@ -13517,7 +13563,7 @@ bool OSDMonitor::prepare_command_impl(MonOpRequestRef op,
     ss << "Triggering recovery stretch mode";
     err = 0;
     goto reply;
-} else {
+  } else {
     err = -EINVAL;
   }
 

diff --git a/src/osd/osd_types.cc b/src/osd/osd_types.cc
index 30f1c94ff6f49755214f778d0751554bf7a3e885..5e3f7abdba44f75b01befb25d62a44a23451fa67 100644 (file)
--- a/src/osd/osd_types.cc
+++ b/src/osd/osd_types.cc
@@ -1354,7 +1354,15 @@ static opt_mapping_t opt_mapping = boost::assign::map_list_of
            ("pg_autoscale_bias", pool_opts_t::opt_desc_t(
             pool_opts_t::PG_AUTOSCALE_BIAS, pool_opts_t::DOUBLE))
            ("read_lease_interval", pool_opts_t::opt_desc_t(
-            pool_opts_t::READ_LEASE_INTERVAL, pool_opts_t::DOUBLE));
+            pool_opts_t::READ_LEASE_INTERVAL, pool_opts_t::DOUBLE))
+           ("dedup_tier", pool_opts_t::opt_desc_t(
+            pool_opts_t::DEDUP_TIER, pool_opts_t::INT))
+           ("dedup_chunk_algorithm", pool_opts_t::opt_desc_t(
+            pool_opts_t::DEDUP_CHUNK_ALGORITHM, pool_opts_t::STR))
+           ("dedup_cdc_chunk_size", pool_opts_t::opt_desc_t(
+            pool_opts_t::DEDUP_CDC_CHUNK_SIZE, pool_opts_t::INT))
+           ("dedup_cdc_window_size", pool_opts_t::opt_desc_t(
+            pool_opts_t::DEDUP_CDC_WINDOW_SIZE, pool_opts_t::INT));
 
 bool pool_opts_t::is_opt_name(const std::string& name)
 {

diff --git a/src/osd/osd_types.h b/src/osd/osd_types.h
index 636c3e710c5a8541dc59ebc03620561da9e30717..0a879f6fcf8efeb058df7ff510d3d376c058ae47 100644 (file)
--- a/src/osd/osd_types.h
+++ b/src/osd/osd_types.h
@@ -1060,6 +1060,10 @@ public:
     TARGET_SIZE_RATIO,  // fraction of total cluster
     PG_AUTOSCALE_BIAS,
     READ_LEASE_INTERVAL,
+    DEDUP_TIER,
+    DEDUP_CHUNK_ALGORITHM,
+    DEDUP_CDC_WINDOW_SIZE,
+    DEDUP_CDC_CHUNK_SIZE,
   };
 
   enum type_t {
@@ -1545,6 +1549,57 @@ public:
     }
   }
 
+  typedef enum {
+    TYPE_DEDUP_CHUNK_NONE = 0,
+    TYPE_DEDUP_CHUNK_FASTCDC = 1,     
+    TYPE_DEDUP_CHUNK_FIXEDCDC = 2,     
+  } dedup_chunk_algo_t;
+  static dedup_chunk_algo_t get_dedup_chunk_algorithm_from_str(const std::string& s) {
+    if (s == "none")
+      return TYPE_DEDUP_CHUNK_NONE;
+    if (s == "fastcdc")
+      return TYPE_DEDUP_CHUNK_FASTCDC;
+    if (s == "fixed")
+      return TYPE_DEDUP_CHUNK_FIXEDCDC;
+    return (dedup_chunk_algo_t)-1;
+  }
+  const dedup_chunk_algo_t get_dedup_chunk_algorithm_type() const {
+    std::string algo_str;
+    opts.get(pool_opts_t::DEDUP_CHUNK_ALGORITHM, &algo_str);
+    return get_dedup_chunk_algorithm_from_str(algo_str);
+  }
+  const char *get_dedup_chunk_algorithm_name() const {
+    std::string dedup_chunk_algo_str;
+    dedup_chunk_algo_t dedup_chunk_algo_t;
+    opts.get(pool_opts_t::DEDUP_CHUNK_ALGORITHM, &dedup_chunk_algo_str);
+    dedup_chunk_algo_t = get_dedup_chunk_algorithm_from_str(dedup_chunk_algo_str);
+    return get_dedup_chunk_algorithm_name(dedup_chunk_algo_t);
+  }
+  static const char *get_dedup_chunk_algorithm_name(dedup_chunk_algo_t m) {
+    switch (m) {
+    case TYPE_DEDUP_CHUNK_NONE: return "none";
+    case TYPE_DEDUP_CHUNK_FASTCDC: return "fastcdc";
+    case TYPE_DEDUP_CHUNK_FIXEDCDC: return "fixed";
+    default: return "unknown";
+    }
+  }
+
+  int64_t get_dedup_tier() const {
+    int64_t tier_id;
+    opts.get(pool_opts_t::DEDUP_TIER, &tier_id);
+    return tier_id;
+  }
+  int64_t get_dedup_cdc_chunk_size() const {
+    int64_t chunk_size;
+    opts.get(pool_opts_t::DEDUP_CDC_CHUNK_SIZE, &chunk_size);
+    return chunk_size;
+  }
+  int64_t get_dedup_cdc_window_size() const {
+    int64_t window_size;
+    opts.get(pool_opts_t::DEDUP_CDC_WINDOW_SIZE, &window_size);
+    return window_size;
+  }
+
   /// application -> key/value metadata
   std::map<std::string, std::map<std::string, std::string>> application_metadata;