From: myoungwon oh Date: Mon, 5 Oct 2020 08:12:27 +0000 (+0900) Subject: osd, mon : add dedup options X-Git-Tag: v16.1.0~374^2~36 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=dd8a88bdffb6392ef2405930a387e7dab377a44d;p=ceph.git osd, mon : add dedup options add dedup_tier, dedup_chunk_algorithm, dedup_cdc_window_size and dedup_cdc_chunk_size. dedup_tier: a pool where chunks are stored. dedup_chunk_algorithm: chunking algorithm. dedup_cdc_window_size: the maximum data size to run CDC. Dedup process reads amount of dedup_cdc_window_size, then calculate chunks by using CDC. dedup_cdc_chunk_size: chunk size for CDC. Signed-off-by: Myoungwon Oh --- diff --git a/src/mon/MonCommands.h b/src/mon/MonCommands.h index 779f76f9c242..93d315b63495 100644 --- a/src/mon/MonCommands.h +++ b/src/mon/MonCommands.h @@ -1090,11 +1090,11 @@ COMMAND("osd pool rename " "rename to ", "osd", "rw") COMMAND("osd pool get " "name=pool,type=CephPoolname " - "name=var,type=CephChoices,strings=size|min_size|pg_num|pgp_num|crush_rule|hashpspool|nodelete|nopgchange|nosizechange|write_fadvise_dontneed|noscrub|nodeep-scrub|hit_set_type|hit_set_period|hit_set_count|hit_set_fpp|use_gmt_hitset|target_max_objects|target_max_bytes|cache_target_dirty_ratio|cache_target_dirty_high_ratio|cache_target_full_ratio|cache_min_flush_age|cache_min_evict_age|erasure_code_profile|min_read_recency_for_promote|all|min_write_recency_for_promote|fast_read|hit_set_grade_decay_rate|hit_set_search_last_n|scrub_min_interval|scrub_max_interval|deep_scrub_interval|recovery_priority|recovery_op_priority|scrub_priority|compression_mode|compression_algorithm|compression_required_ratio|compression_max_blob_size|compression_min_blob_size|csum_type|csum_min_block|csum_max_block|allow_ec_overwrites|fingerprint_algorithm|pg_autoscale_mode|pg_autoscale_bias|pg_num_min|target_size_bytes|target_size_ratio", + "name=var,type=CephChoices,strings=size|min_size|pg_num|pgp_num|crush_rule|hashpspool|nodelete|nopgchange|nosizechange|write_fadvise_dontneed|noscrub|nodeep-scrub|hit_set_type|hit_set_period|hit_set_count|hit_set_fpp|use_gmt_hitset|target_max_objects|target_max_bytes|cache_target_dirty_ratio|cache_target_dirty_high_ratio|cache_target_full_ratio|cache_min_flush_age|cache_min_evict_age|erasure_code_profile|min_read_recency_for_promote|all|min_write_recency_for_promote|fast_read|hit_set_grade_decay_rate|hit_set_search_last_n|scrub_min_interval|scrub_max_interval|deep_scrub_interval|recovery_priority|recovery_op_priority|scrub_priority|compression_mode|compression_algorithm|compression_required_ratio|compression_max_blob_size|compression_min_blob_size|csum_type|csum_min_block|csum_max_block|allow_ec_overwrites|fingerprint_algorithm|pg_autoscale_mode|pg_autoscale_bias|pg_num_min|target_size_bytes|target_size_ratio|dedup_tier|dedup_chunk_algorithm|dedup_cdc_window_size|dedup_cdc_chunk_size", "get pool parameter ", "osd", "r") COMMAND("osd pool set " "name=pool,type=CephPoolname " - "name=var,type=CephChoices,strings=size|min_size|pg_num|pgp_num|pgp_num_actual|crush_rule|hashpspool|nodelete|nopgchange|nosizechange|write_fadvise_dontneed|noscrub|nodeep-scrub|hit_set_type|hit_set_period|hit_set_count|hit_set_fpp|use_gmt_hitset|target_max_bytes|target_max_objects|cache_target_dirty_ratio|cache_target_dirty_high_ratio|cache_target_full_ratio|cache_min_flush_age|cache_min_evict_age|min_read_recency_for_promote|min_write_recency_for_promote|fast_read|hit_set_grade_decay_rate|hit_set_search_last_n|scrub_min_interval|scrub_max_interval|deep_scrub_interval|recovery_priority|recovery_op_priority|scrub_priority|compression_mode|compression_algorithm|compression_required_ratio|compression_max_blob_size|compression_min_blob_size|csum_type|csum_min_block|csum_max_block|allow_ec_overwrites|fingerprint_algorithm|pg_autoscale_mode|pg_autoscale_bias|pg_num_min|target_size_bytes|target_size_ratio " + "name=var,type=CephChoices,strings=size|min_size|pg_num|pgp_num|pgp_num_actual|crush_rule|hashpspool|nodelete|nopgchange|nosizechange|write_fadvise_dontneed|noscrub|nodeep-scrub|hit_set_type|hit_set_period|hit_set_count|hit_set_fpp|use_gmt_hitset|target_max_bytes|target_max_objects|cache_target_dirty_ratio|cache_target_dirty_high_ratio|cache_target_full_ratio|cache_min_flush_age|cache_min_evict_age|min_read_recency_for_promote|min_write_recency_for_promote|fast_read|hit_set_grade_decay_rate|hit_set_search_last_n|scrub_min_interval|scrub_max_interval|deep_scrub_interval|recovery_priority|recovery_op_priority|scrub_priority|compression_mode|compression_algorithm|compression_required_ratio|compression_max_blob_size|compression_min_blob_size|csum_type|csum_min_block|csum_max_block|allow_ec_overwrites|fingerprint_algorithm|pg_autoscale_mode|pg_autoscale_bias|pg_num_min|target_size_bytes|target_size_ratio|dedup_tier|dedup_chunk_algorithm|dedup_cdc_window_size|dedup_cdc_chunk_size " "name=val,type=CephString " "name=yes_i_really_mean_it,type=CephBool,req=false", "set pool parameter to ", "osd", "rw") diff --git a/src/mon/OSDMonitor.cc b/src/mon/OSDMonitor.cc index e154ed72e004..b55e7461a64c 100644 --- a/src/mon/OSDMonitor.cc +++ b/src/mon/OSDMonitor.cc @@ -5373,7 +5373,8 @@ namespace { COMPRESSION_MAX_BLOB_SIZE, COMPRESSION_MIN_BLOB_SIZE, CSUM_TYPE, CSUM_MAX_BLOCK, CSUM_MIN_BLOCK, FINGERPRINT_ALGORITHM, PG_AUTOSCALE_MODE, PG_NUM_MIN, TARGET_SIZE_BYTES, TARGET_SIZE_RATIO, - PG_AUTOSCALE_BIAS }; + PG_AUTOSCALE_BIAS, DEDUP_TIER, DEDUP_CHUNK_ALGORITHM, DEDUP_CDC_WINDOW_SIZE, + DEDUP_CDC_CHUNK_SIZE }; std::set subtract_second_from_first(const std::set& first, @@ -6108,6 +6109,10 @@ bool OSDMonitor::preprocess_command(MonOpRequestRef op) {"target_size_bytes", TARGET_SIZE_BYTES}, {"target_size_ratio", TARGET_SIZE_RATIO}, {"pg_autoscale_bias", PG_AUTOSCALE_BIAS}, + {"dedup_tier", DEDUP_TIER}, + {"dedup_chunk_algorithm", DEDUP_CHUNK_ALGORITHM}, + {"dedup_cdc_window_size", DEDUP_CDC_WINDOW_SIZE}, + {"dedup_cdc_chunk_size", DEDUP_CDC_CHUNK_SIZE}, }; typedef std::set choices_set_t; @@ -6324,6 +6329,10 @@ bool OSDMonitor::preprocess_command(MonOpRequestRef op) case TARGET_SIZE_BYTES: case TARGET_SIZE_RATIO: case PG_AUTOSCALE_BIAS: + case DEDUP_TIER: + case DEDUP_CHUNK_ALGORITHM: + case DEDUP_CDC_WINDOW_SIZE: + case DEDUP_CDC_CHUNK_SIZE: pool_opts_t::key_t key = pool_opts_t::get_opt_desc(i->first).key; if (p->opts.is_set(key)) { if(*it == CSUM_TYPE) { @@ -6481,6 +6490,10 @@ bool OSDMonitor::preprocess_command(MonOpRequestRef op) case TARGET_SIZE_BYTES: case TARGET_SIZE_RATIO: case PG_AUTOSCALE_BIAS: + case DEDUP_TIER: + case DEDUP_CHUNK_ALGORITHM: + case DEDUP_CDC_WINDOW_SIZE: + case DEDUP_CDC_CHUNK_SIZE: for (i = ALL_CHOICES.begin(); i != ALL_CHOICES.end(); ++i) { if (i->second == *it) break; @@ -8648,6 +8661,39 @@ int OSDMonitor::prepare_command_pool_set(const cmdmap_t& cmdmap, ss << "pg_autoscale_bias must be between 0 and 1000"; return -EINVAL; } + } else if (var == "dedup_tier") { + int64_t lowtierpool_id = osdmap.lookup_pg_pool_name(val); + if (lowtierpool_id < 0) { + ss << "unrecognized pool '" << val << "'"; + return -ENOENT; + } + const pg_pool_t *tp = osdmap.get_pg_pool(lowtierpool_id); + ceph_assert(tp); + if (p.is_erasure()) { + ss << "pool '" << poolstr + << "' is an ec pool, which cannot be a tier"; + return -ENOTSUP; + } + n = lowtierpool_id; + interr.clear(); + } else if (var == "dedup_chunk_algorithm") { + if (!unset) { + auto alg = pg_pool_t::get_dedup_chunk_algorithm_from_str(val); + if (!alg) { + ss << "unrecognized fingerprint_algorithm '" << val << "'"; + return -EINVAL; + } + } + } else if (var == "dedup_cdc_window_size") { + if (interr.length()) { + ss << "error parsing int value '" << val << "': " << interr; + return -EINVAL; + } + } else if (var == "dedup_cdc_chunk_size") { + if (interr.length()) { + ss << "error parsing int value '" << val << "': " << interr; + return -EINVAL; + } } pool_opts_t::opt_desc_t desc = pool_opts_t::get_opt_desc(var); @@ -13517,7 +13563,7 @@ bool OSDMonitor::prepare_command_impl(MonOpRequestRef op, ss << "Triggering recovery stretch mode"; err = 0; goto reply; -} else { + } else { err = -EINVAL; } diff --git a/src/osd/osd_types.cc b/src/osd/osd_types.cc index 30f1c94ff6f4..5e3f7abdba44 100644 --- a/src/osd/osd_types.cc +++ b/src/osd/osd_types.cc @@ -1354,7 +1354,15 @@ static opt_mapping_t opt_mapping = boost::assign::map_list_of ("pg_autoscale_bias", pool_opts_t::opt_desc_t( pool_opts_t::PG_AUTOSCALE_BIAS, pool_opts_t::DOUBLE)) ("read_lease_interval", pool_opts_t::opt_desc_t( - pool_opts_t::READ_LEASE_INTERVAL, pool_opts_t::DOUBLE)); + pool_opts_t::READ_LEASE_INTERVAL, pool_opts_t::DOUBLE)) + ("dedup_tier", pool_opts_t::opt_desc_t( + pool_opts_t::DEDUP_TIER, pool_opts_t::INT)) + ("dedup_chunk_algorithm", pool_opts_t::opt_desc_t( + pool_opts_t::DEDUP_CHUNK_ALGORITHM, pool_opts_t::STR)) + ("dedup_cdc_chunk_size", pool_opts_t::opt_desc_t( + pool_opts_t::DEDUP_CDC_CHUNK_SIZE, pool_opts_t::INT)) + ("dedup_cdc_window_size", pool_opts_t::opt_desc_t( + pool_opts_t::DEDUP_CDC_WINDOW_SIZE, pool_opts_t::INT)); bool pool_opts_t::is_opt_name(const std::string& name) { diff --git a/src/osd/osd_types.h b/src/osd/osd_types.h index 636c3e710c5a..0a879f6fcf8e 100644 --- a/src/osd/osd_types.h +++ b/src/osd/osd_types.h @@ -1060,6 +1060,10 @@ public: TARGET_SIZE_RATIO, // fraction of total cluster PG_AUTOSCALE_BIAS, READ_LEASE_INTERVAL, + DEDUP_TIER, + DEDUP_CHUNK_ALGORITHM, + DEDUP_CDC_WINDOW_SIZE, + DEDUP_CDC_CHUNK_SIZE, }; enum type_t { @@ -1545,6 +1549,57 @@ public: } } + typedef enum { + TYPE_DEDUP_CHUNK_NONE = 0, + TYPE_DEDUP_CHUNK_FASTCDC = 1, + TYPE_DEDUP_CHUNK_FIXEDCDC = 2, + } dedup_chunk_algo_t; + static dedup_chunk_algo_t get_dedup_chunk_algorithm_from_str(const std::string& s) { + if (s == "none") + return TYPE_DEDUP_CHUNK_NONE; + if (s == "fastcdc") + return TYPE_DEDUP_CHUNK_FASTCDC; + if (s == "fixed") + return TYPE_DEDUP_CHUNK_FIXEDCDC; + return (dedup_chunk_algo_t)-1; + } + const dedup_chunk_algo_t get_dedup_chunk_algorithm_type() const { + std::string algo_str; + opts.get(pool_opts_t::DEDUP_CHUNK_ALGORITHM, &algo_str); + return get_dedup_chunk_algorithm_from_str(algo_str); + } + const char *get_dedup_chunk_algorithm_name() const { + std::string dedup_chunk_algo_str; + dedup_chunk_algo_t dedup_chunk_algo_t; + opts.get(pool_opts_t::DEDUP_CHUNK_ALGORITHM, &dedup_chunk_algo_str); + dedup_chunk_algo_t = get_dedup_chunk_algorithm_from_str(dedup_chunk_algo_str); + return get_dedup_chunk_algorithm_name(dedup_chunk_algo_t); + } + static const char *get_dedup_chunk_algorithm_name(dedup_chunk_algo_t m) { + switch (m) { + case TYPE_DEDUP_CHUNK_NONE: return "none"; + case TYPE_DEDUP_CHUNK_FASTCDC: return "fastcdc"; + case TYPE_DEDUP_CHUNK_FIXEDCDC: return "fixed"; + default: return "unknown"; + } + } + + int64_t get_dedup_tier() const { + int64_t tier_id; + opts.get(pool_opts_t::DEDUP_TIER, &tier_id); + return tier_id; + } + int64_t get_dedup_cdc_chunk_size() const { + int64_t chunk_size; + opts.get(pool_opts_t::DEDUP_CDC_CHUNK_SIZE, &chunk_size); + return chunk_size; + } + int64_t get_dedup_cdc_window_size() const { + int64_t window_size; + opts.get(pool_opts_t::DEDUP_CDC_WINDOW_SIZE, &window_size); + return window_size; + } + /// application -> key/value metadata std::map> application_metadata;