From e6dcf1419a45c6a94c1c543269fa70483f163b41 Mon Sep 17 00:00:00 2001 From: Mykola Golub Date: Tue, 29 Sep 2015 09:35:32 +0300 Subject: [PATCH] osd: store per pool scrub intervals in pool options ceph osd pool set $POOL scrub_min_interval N ceph osd pool set $POOL scrub_max_interval N ceph osd pool set $POOL deep_scrub_interval N If N > 0, this value is used for the pool instead of the corresponding global parameter from the config (osd_scrub_min_interval, osd_scrub_max_interval or osd_deep_scrub_interval). Fixes: #13077 Signed-off-by: Mykola Golub --- doc/rados/operations/pools.rst | 55 +++++++++++++++++++++++++++ qa/workunits/cephtool/test.sh | 18 +++++++++ src/mon/MonCommands.h | 4 +- src/mon/OSDMonitor.cc | 33 +++++++++++++++- src/osd/OSD.cc | 15 ++++++-- src/osd/OSD.h | 10 +++-- src/osd/PG.cc | 14 ++++++- src/osd/osd_types.cc | 8 +++- src/osd/osd_types.h | 3 ++ src/test/mon/misc.sh | 14 +++++++ src/test/osd/types.cc | 54 ++++++++++++++++++++++++++ src/test/pybind/test_ceph_argparse.py | 8 +++- 12 files changed, 220 insertions(+), 16 deletions(-) diff --git a/doc/rados/operations/pools.rst b/doc/rados/operations/pools.rst index 0babf071c300..f79cebee5b27 100644 --- a/doc/rados/operations/pools.rst +++ b/doc/rados/operations/pools.rst @@ -488,6 +488,39 @@ You may set values for the following keys: :Type: Boolean :Defaults: ``0`` +.. _scrub_min_interval: + +``scrub_min_interval`` + +:Description: The maximum interval in seconds for pool scrubbing when + load is low. If it is 0, the value osd_scrub_min_interval + from config is used. + +:Type: Double +:Default: ``0`` + +.. _scrub_max_interval: + +``scrub_max_interval`` + +:Description: The maximum interval in seconds for pool scrubbing + irrespective of cluster load. If it is 0, the value + osd_scrub_max_interval from config is used. + +:Type: Double +:Default: ``0`` + +.. _deep_scrub_interval: + +``deep_scrub_interval`` + +:Description: The interval in seconds for pool “deep” scrubbing. If it + is 0, the value osd_deep_scrub_interval from config is used. + +:Type: Double +:Default: ``0`` + + Get Pool Values =============== @@ -613,6 +646,28 @@ You may get values for the following keys: :Type: Boolean + +``scrub_min_interval`` + +:Description: see scrub_min_interval_ + +:Type: Double + + +``scrub_max_interval`` + +:Description: see scrub_max_interval_ + +:Type: Double + + +``deep_scrub_interval`` + +:Description: see deep_scrub_interval_ + +:Type: Double + + Set the Number of Object Replicas ================================= diff --git a/qa/workunits/cephtool/test.sh b/qa/workunits/cephtool/test.sh index c00fa19cd0b6..b33f1bec4374 100755 --- a/qa/workunits/cephtool/test.sh +++ b/qa/workunits/cephtool/test.sh @@ -1389,6 +1389,24 @@ function test_mon_osd_pool_set() expect_false ceph osd pool set $TEST_POOL_GETSET $flag 2 done + expect_false "ceph osd pool get $TEST_POOL_GETSET scrub_min_interval | grep '.'" + ceph osd pool set $TEST_POOL_GETSET scrub_min_interval 123456 + ceph osd pool get $TEST_POOL_GETSET scrub_min_interval | grep 'scrub_min_interval: 123456' + ceph osd pool set $TEST_POOL_GETSET scrub_min_interval 0 + expect_false "ceph osd pool get $TEST_POOL_GETSET scrub_min_interval | grep '.'" + + expect_false "ceph osd pool get $TEST_POOL_GETSET scrub_max_interval | grep '.'" + ceph osd pool set $TEST_POOL_GETSET scrub_max_interval 123456 + ceph osd pool get $TEST_POOL_GETSET scrub_max_interval | grep 'scrub_max_interval: 123456' + ceph osd pool set $TEST_POOL_GETSET scrub_max_interval 0 + expect_false "ceph osd pool get $TEST_POOL_GETSET scrub_max_interval | grep '.'" + + expect_false "ceph osd pool get $TEST_POOL_GETSET deep_scrub_interval | grep '.'" + ceph osd pool set $TEST_POOL_GETSET deep_scrub_interval 123456 + ceph osd pool get $TEST_POOL_GETSET deep_scrub_interval | grep 'deep_scrub_interval: 123456' + ceph osd pool set $TEST_POOL_GETSET deep_scrub_interval 0 + expect_false "ceph osd pool get $TEST_POOL_GETSET deep_scrub_interval | grep '.'" + ceph osd pool set $TEST_POOL_GETSET nopgchange 1 expect_false ceph osd pool set $TEST_POOL_GETSET pg_num 10 expect_false ceph osd pool set $TEST_POOL_GETSET pgp_num 10 diff --git a/src/mon/MonCommands.h b/src/mon/MonCommands.h index 67936d5d3465..0e82188a7eab 100644 --- a/src/mon/MonCommands.h +++ b/src/mon/MonCommands.h @@ -674,11 +674,11 @@ COMMAND("osd pool rename " \ "rename to ", "osd", "rw", "cli,rest") COMMAND("osd pool get " \ "name=pool,type=CephPoolname " \ - "name=var,type=CephChoices,strings=size|min_size|crash_replay_interval|pg_num|pgp_num|crush_ruleset|hashpspool|nodelete|nopgchange|nosizechange|write_fadvise_dontneed|noscrub|nodeep-scrub|hit_set_type|hit_set_period|hit_set_count|hit_set_fpp|auid|target_max_objects|target_max_bytes|cache_target_dirty_ratio|cache_target_dirty_high_ratio|cache_target_full_ratio|cache_min_flush_age|cache_min_evict_age|erasure_code_profile|min_read_recency_for_promote|all|min_write_recency_for_promote|fast_read|hit_set_grade_decay_rate|hit_set_search_last_n", \ + "name=var,type=CephChoices,strings=size|min_size|crash_replay_interval|pg_num|pgp_num|crush_ruleset|hashpspool|nodelete|nopgchange|nosizechange|write_fadvise_dontneed|noscrub|nodeep-scrub|hit_set_type|hit_set_period|hit_set_count|hit_set_fpp|auid|target_max_objects|target_max_bytes|cache_target_dirty_ratio|cache_target_dirty_high_ratio|cache_target_full_ratio|cache_min_flush_age|cache_min_evict_age|erasure_code_profile|min_read_recency_for_promote|all|min_write_recency_for_promote|fast_read|hit_set_grade_decay_rate|hit_set_search_last_n|scrub_min_interval|scrub_max_interval|deep_scrub_interval", \ "get pool parameter ", "osd", "r", "cli,rest") COMMAND("osd pool set " \ "name=pool,type=CephPoolname " \ - "name=var,type=CephChoices,strings=size|min_size|crash_replay_interval|pg_num|pgp_num|crush_ruleset|hashpspool|nodelete|nopgchange|nosizechange|write_fadvise_dontneed|noscrub|nodeep-scrub|hit_set_type|hit_set_period|hit_set_count|hit_set_fpp|use_gmt_hitset|debug_fake_ec_pool|target_max_bytes|target_max_objects|cache_target_dirty_ratio|cache_target_dirty_high_ratio|cache_target_full_ratio|cache_min_flush_age|cache_min_evict_age|auid|min_read_recency_for_promote|min_write_recency_for_promote|fast_read|hit_set_grade_decay_rate|hit_set_search_last_n " \ + "name=var,type=CephChoices,strings=size|min_size|crash_replay_interval|pg_num|pgp_num|crush_ruleset|hashpspool|nodelete|nopgchange|nosizechange|write_fadvise_dontneed|noscrub|nodeep-scrub|hit_set_type|hit_set_period|hit_set_count|hit_set_fpp|use_gmt_hitset|debug_fake_ec_pool|target_max_bytes|target_max_objects|cache_target_dirty_ratio|cache_target_dirty_high_ratio|cache_target_full_ratio|cache_min_flush_age|cache_min_evict_age|auid|min_read_recency_for_promote|min_write_recency_for_promote|fast_read|hit_set_grade_decay_rate|hit_set_search_last_n|scrub_min_interval|scrub_max_interval|deep_scrub_interval " \ "name=val,type=CephString " \ "name=force,type=CephChoices,strings=--yes-i-really-mean-it,req=false", \ "set pool parameter to ", "osd", "rw", "cli,rest") diff --git a/src/mon/OSDMonitor.cc b/src/mon/OSDMonitor.cc index 2fcb07d49d73..3578b7933bdf 100644 --- a/src/mon/OSDMonitor.cc +++ b/src/mon/OSDMonitor.cc @@ -2906,7 +2906,8 @@ namespace { CACHE_MIN_FLUSH_AGE, CACHE_MIN_EVICT_AGE, ERASURE_CODE_PROFILE, MIN_READ_RECENCY_FOR_PROMOTE, MIN_WRITE_RECENCY_FOR_PROMOTE, FAST_READ, - HIT_SET_GRADE_DECAY_RATE, HIT_SET_SEARCH_LAST_N}; + HIT_SET_GRADE_DECAY_RATE, HIT_SET_SEARCH_LAST_N, + SCRUB_MIN_INTERVAL, SCRUB_MAX_INTERVAL, DEEP_SCRUB_INTERVAL}; std::set subtract_second_from_first(const std::set& first, @@ -3382,7 +3383,10 @@ bool OSDMonitor::preprocess_command(MonOpRequestRef op) ("min_write_recency_for_promote", MIN_WRITE_RECENCY_FOR_PROMOTE) ("fast_read", FAST_READ) ("hit_set_grade_decay_rate", HIT_SET_GRADE_DECAY_RATE) - ("hit_set_search_last_n", HIT_SET_SEARCH_LAST_N); + ("hit_set_search_last_n", HIT_SET_SEARCH_LAST_N) + ("scrub_min_interval", SCRUB_MIN_INTERVAL) + ("scrub_max_interval", SCRUB_MAX_INTERVAL) + ("deep_scrub_interval", DEEP_SCRUB_INTERVAL); typedef std::set choices_set_t; @@ -3561,6 +3565,16 @@ bool OSDMonitor::preprocess_command(MonOpRequestRef op) f->dump_int("hit_set_search_last_n", p->hit_set_search_last_n); break; + case SCRUB_MIN_INTERVAL: + case SCRUB_MAX_INTERVAL: + case DEEP_SCRUB_INTERVAL: + for (i = ALL_CHOICES.begin(); i != ALL_CHOICES.end(); ++i) { + if (i->second == *it) + break; + } + assert(i != ALL_CHOICES.end()); + p->opts.dump(i->first, f.get()); + break; } f->close_section(); f->flush(rdata); @@ -3682,6 +3696,21 @@ bool OSDMonitor::preprocess_command(MonOpRequestRef op) case FAST_READ: ss << "fast_read: " << p->fast_read << "\n"; break; + case SCRUB_MIN_INTERVAL: + case SCRUB_MAX_INTERVAL: + case DEEP_SCRUB_INTERVAL: + for (i = ALL_CHOICES.begin(); i != ALL_CHOICES.end(); ++i) { + if (i->second == *it) + break; + } + assert(i != ALL_CHOICES.end()); + { + pool_opts_t::key_t key = pool_opts_t::get_opt_desc(i->first).key; + if (p->opts.is_set(key)) { + ss << i->first << ": " << p->opts.get(key) << "\n"; + } + } + break; } rdata.append(ss.str()); ss.str(""); diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc index 1e2fd698f6e6..0c195c406522 100644 --- a/src/osd/OSD.cc +++ b/src/osd/OSD.cc @@ -6059,19 +6059,26 @@ bool OSD::scrub_random_backoff() return false; } -OSDService::ScrubJob::ScrubJob(const spg_t& pg, const utime_t& timestamp, bool must) +OSDService::ScrubJob::ScrubJob(const spg_t& pg, const utime_t& timestamp, + double pool_scrub_min_interval, + double pool_scrub_max_interval, bool must) : pgid(pg), sched_time(timestamp), deadline(timestamp) { // if not explicitly requested, postpone the scrub with a random delay if (!must) { - sched_time += g_conf->osd_scrub_min_interval; + double scrub_min_interval = pool_scrub_min_interval > 0 ? + pool_scrub_min_interval : g_conf->osd_scrub_min_interval; + double scrub_max_interval = pool_scrub_max_interval > 0 ? + pool_scrub_max_interval : g_conf->osd_scrub_max_interval; + + sched_time += scrub_min_interval; if (g_conf->osd_scrub_interval_randomize_ratio > 0) { - sched_time += rand() % (int)(g_conf->osd_scrub_min_interval * + sched_time += rand() % (int)(scrub_min_interval * g_conf->osd_scrub_interval_randomize_ratio); } - deadline += g_conf->osd_scrub_max_interval; + deadline += scrub_max_interval; } } diff --git a/src/osd/OSD.h b/src/osd/OSD.h index e34cc333bf55..35617184c500 100644 --- a/src/osd/OSD.h +++ b/src/osd/OSD.h @@ -594,15 +594,19 @@ public: /// the hard upper bound of scrub time utime_t deadline; ScrubJob() {} - explicit ScrubJob(const spg_t& pg, const utime_t& timestamp, bool must = true); + explicit ScrubJob(const spg_t& pg, const utime_t& timestamp, + double pool_scrub_min_interval = 0, + double pool_scrub_max_interval = 0, bool must = true); /// order the jobs by sched_time bool operator<(const ScrubJob& rhs) const; }; set sched_scrub_pg; /// @returns the scrub_reg_stamp used for unregister the scrub job - utime_t reg_pg_scrub(spg_t pgid, utime_t t, bool must) { - ScrubJob scrub(pgid, t, must); + utime_t reg_pg_scrub(spg_t pgid, utime_t t, double pool_scrub_min_interval, + double pool_scrub_max_interval, bool must) { + ScrubJob scrub(pgid, t, pool_scrub_min_interval, pool_scrub_max_interval, + must); Mutex::Locker l(sched_scrub_lock); sched_scrub_pg.insert(scrub); return scrub.sched_time; diff --git a/src/osd/PG.cc b/src/osd/PG.cc index a7f88370aaac..263c88bed5f0 100644 --- a/src/osd/PG.cc +++ b/src/osd/PG.cc @@ -3243,8 +3243,13 @@ bool PG::sched_scrub() return false; } - bool time_for_deep = (ceph_clock_now(cct) >= - info.history.last_deep_scrub_stamp + cct->_conf->osd_deep_scrub_interval); + double deep_scrub_interval = 0; + pool.info.opts.get(pool_opts_t::DEEP_SCRUB_INTERVAL, &deep_scrub_interval); + if (deep_scrub_interval <= 0) { + deep_scrub_interval = cct->_conf->osd_deep_scrub_interval; + } + bool time_for_deep = ceph_clock_now(cct) >= + info.history.last_deep_scrub_stamp + deep_scrub_interval; bool deep_coin_flip = false; // Only add random deep scrubs when NOT user initiated scrub @@ -3332,8 +3337,13 @@ void PG::reg_next_scrub() } // note down the sched_time, so we can locate this scrub, and remove it // later on. + double scrub_min_interval = 0, scrub_max_interval = 0; + pool.info.opts.get(pool_opts_t::SCRUB_MIN_INTERVAL, &scrub_min_interval); + pool.info.opts.get(pool_opts_t::SCRUB_MAX_INTERVAL, &scrub_max_interval); scrubber.scrub_reg_stamp = osd->reg_pg_scrub(info.pgid, reg_stamp, + scrub_min_interval, + scrub_max_interval, scrubber.must_scrub); } diff --git a/src/osd/osd_types.cc b/src/osd/osd_types.cc index 8b54993c687a..464cb197cb02 100644 --- a/src/osd/osd_types.cc +++ b/src/osd/osd_types.cc @@ -894,7 +894,13 @@ void pool_snap_info_t::generate_test_instances(list& o) // -- pool_opts_t -- typedef std::map opt_mapping_t; -static opt_mapping_t opt_mapping; +static opt_mapping_t opt_mapping = boost::assign::map_list_of + ("scrub_min_interval", pool_opts_t::opt_desc_t( + pool_opts_t::SCRUB_MIN_INTERVAL, pool_opts_t::DOUBLE)) + ("scrub_max_interval", pool_opts_t::opt_desc_t( + pool_opts_t::SCRUB_MAX_INTERVAL, pool_opts_t::DOUBLE)) + ("deep_scrub_interval", pool_opts_t::opt_desc_t( + pool_opts_t::DEEP_SCRUB_INTERVAL, pool_opts_t::DOUBLE)); bool pool_opts_t::is_opt_name(const std::string& name) { return opt_mapping.find(name) != opt_mapping.end(); diff --git a/src/osd/osd_types.h b/src/osd/osd_types.h index 643fb48fbd0f..9810e6b76dc4 100644 --- a/src/osd/osd_types.h +++ b/src/osd/osd_types.h @@ -899,6 +899,9 @@ inline ostream& operator<<(ostream& out, const pool_snap_info_t& si) { class pool_opts_t { public: enum key_t { + SCRUB_MIN_INTERVAL, + SCRUB_MAX_INTERVAL, + DEEP_SCRUB_INTERVAL, }; enum type_t { diff --git a/src/test/mon/misc.sh b/src/test/mon/misc.sh index b7d823339b51..196b5ab2132f 100755 --- a/src/test/mon/misc.sh +++ b/src/test/mon/misc.sh @@ -64,6 +64,20 @@ function TEST_osd_pool_get_set() { local size=$(ceph osd pool get $TEST_POOL size|awk '{print $2}') local min_size=$(ceph osd pool get $TEST_POOL min_size|awk '{print $2}') + + ceph osd pool set $TEST_POOL scrub_min_interval 123456 || return 1 + ceph osd dump | grep 'pool ' | grep 'scrub_min_interval 123456' || return 1 + ceph osd pool set $TEST_POOL scrub_min_interval 0 || return 1 + ceph osd dump | grep 'pool ' | grep 'scrub_min_interval' && return 1 + ceph osd pool set $TEST_POOL scrub_max_interval 123456 || return 1 + ceph osd dump | grep 'pool ' | grep 'scrub_max_interval 123456' || return 1 + ceph osd pool set $TEST_POOL scrub_max_interval 0 || return 1 + ceph osd dump | grep 'pool ' | grep 'scrub_max_interval' && return 1 + ceph osd pool set $TEST_POOL deep_scrub_interval 123456 || return 1 + ceph osd dump | grep 'pool ' | grep 'deep_scrub_interval 123456' || return 1 + ceph osd pool set $TEST_POOL deep_scrub_interval 0 || return 1 + ceph osd dump | grep 'pool ' | grep 'deep_scrub_interval' && return 1 + #replicated pool size restrict in 1 and 10 ! ceph osd pool set $TEST_POOL 11 || return 1 #replicated pool min_size must be between in 1 and size diff --git a/src/test/osd/types.cc b/src/test/osd/types.cc index 14f07c81073d..1652ac5efbee 100644 --- a/src/test/osd/types.cc +++ b/src/test/osd/types.cc @@ -1393,6 +1393,60 @@ TEST(pool_opts_t, invalid_opt) { EXPECT_THROW(pool_opts_t::get_opt_desc("INVALID_OPT"), FailedAssertion); } +TEST(pool_opts_t, scrub_min_interval) { + EXPECT_TRUE(pool_opts_t::is_opt_name("scrub_min_interval")); + EXPECT_EQ(pool_opts_t::get_opt_desc("scrub_min_interval"), + pool_opts_t::opt_desc_t(pool_opts_t::SCRUB_MIN_INTERVAL, + pool_opts_t::DOUBLE)); + + pool_opts_t opts; + EXPECT_FALSE(opts.is_set(pool_opts_t::SCRUB_MIN_INTERVAL)); + EXPECT_THROW(opts.get(pool_opts_t::SCRUB_MIN_INTERVAL), FailedAssertion); + double val; + EXPECT_FALSE(opts.get(pool_opts_t::SCRUB_MIN_INTERVAL, &val)); + opts.set(pool_opts_t::SCRUB_MIN_INTERVAL, static_cast(2015)); + EXPECT_TRUE(opts.get(pool_opts_t::SCRUB_MIN_INTERVAL, &val)); + EXPECT_EQ(val, 2015); + opts.unset(pool_opts_t::SCRUB_MIN_INTERVAL); + EXPECT_FALSE(opts.is_set(pool_opts_t::SCRUB_MIN_INTERVAL)); +} + +TEST(pool_opts_t, scrub_max_interval) { + EXPECT_TRUE(pool_opts_t::is_opt_name("scrub_max_interval")); + EXPECT_EQ(pool_opts_t::get_opt_desc("scrub_max_interval"), + pool_opts_t::opt_desc_t(pool_opts_t::SCRUB_MAX_INTERVAL, + pool_opts_t::DOUBLE)); + + pool_opts_t opts; + EXPECT_FALSE(opts.is_set(pool_opts_t::SCRUB_MAX_INTERVAL)); + EXPECT_THROW(opts.get(pool_opts_t::SCRUB_MAX_INTERVAL), FailedAssertion); + double val; + EXPECT_FALSE(opts.get(pool_opts_t::SCRUB_MAX_INTERVAL, &val)); + opts.set(pool_opts_t::SCRUB_MAX_INTERVAL, static_cast(2015)); + EXPECT_TRUE(opts.get(pool_opts_t::SCRUB_MAX_INTERVAL, &val)); + EXPECT_EQ(val, 2015); + opts.unset(pool_opts_t::SCRUB_MAX_INTERVAL); + EXPECT_FALSE(opts.is_set(pool_opts_t::SCRUB_MAX_INTERVAL)); +} + +TEST(pool_opts_t, deep_scrub_interval) { + EXPECT_TRUE(pool_opts_t::is_opt_name("deep_scrub_interval")); + EXPECT_EQ(pool_opts_t::get_opt_desc("deep_scrub_interval"), + pool_opts_t::opt_desc_t(pool_opts_t::DEEP_SCRUB_INTERVAL, + pool_opts_t::DOUBLE)); + + pool_opts_t opts; + EXPECT_FALSE(opts.is_set(pool_opts_t::DEEP_SCRUB_INTERVAL)); + EXPECT_THROW(opts.get(pool_opts_t::DEEP_SCRUB_INTERVAL), FailedAssertion); + double val; + EXPECT_FALSE(opts.get(pool_opts_t::DEEP_SCRUB_INTERVAL, &val)); + opts.set(pool_opts_t::DEEP_SCRUB_INTERVAL, static_cast(2015)); + EXPECT_TRUE(opts.get(pool_opts_t::DEEP_SCRUB_INTERVAL, &val)); + EXPECT_EQ(val, 2015); + opts.unset(pool_opts_t::DEEP_SCRUB_INTERVAL); + EXPECT_FALSE(opts.is_set(pool_opts_t::DEEP_SCRUB_INTERVAL)); +} + /* * Local Variables: * compile-command: "cd ../.. ; diff --git a/src/test/pybind/test_ceph_argparse.py b/src/test/pybind/test_ceph_argparse.py index fd449776ceea..9ac47c1582ed 100755 --- a/src/test/pybind/test_ceph_argparse.py +++ b/src/test/pybind/test_ceph_argparse.py @@ -1032,7 +1032,9 @@ class TestOSD(TestArgparse): def test_pool_get(self): for var in ('size', 'min_size', 'crash_replay_interval', - 'pg_num', 'pgp_num', 'crush_ruleset', 'auid', 'fast_read'): + 'pg_num', 'pgp_num', 'crush_ruleset', 'auid', 'fast_read', + 'scrub_min_interval', 'scrub_max_interval', + 'deep_scrub_interval'): self.assert_valid_command(['osd', 'pool', 'get', 'poolname', var]) assert_equal({}, validate_command(sigdict, ['osd', 'pool'])) assert_equal({}, validate_command(sigdict, ['osd', 'pool', @@ -1049,7 +1051,9 @@ class TestOSD(TestArgparse): def test_pool_set(self): for var in ('size', 'min_size', 'crash_replay_interval', 'pg_num', 'pgp_num', 'crush_ruleset', - 'hashpspool', 'auid', 'fast_read'): + 'hashpspool', 'auid', 'fast_read', + 'scrub_min_interval', 'scrub_max_interval', + 'deep_scrub_interval'): self.assert_valid_command(['osd', 'pool', 'set', 'poolname', var, 'value']) assert_equal({}, validate_command(sigdict, ['osd', 'pool', -- 2.47.3