From: Brad Hubbard Date: Tue, 23 Jul 2019 07:42:54 +0000 (+1000) Subject: luminous: tests: Convert to boost::regex and add blocking scrub X-Git-Tag: v12.2.13~147^2 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=4b0bdb4ceaf169be73340df61490aa49eb7a7064;p=ceph.git luminous: tests: Convert to boost::regex and add blocking scrub Convert lazy_omap_stats_test to boost::regex since std::regex is not fully implemented in the C++ libs used to compile luminous. Also add the ability to block until scrub completes as that is not available in luminous. Signed-off-by: Brad Hubbard --- diff --git a/src/test/lazy-omap-stats/CMakeLists.txt b/src/test/lazy-omap-stats/CMakeLists.txt index fad71f135ce..d7143a7546d 100644 --- a/src/test/lazy-omap-stats/CMakeLists.txt +++ b/src/test/lazy-omap-stats/CMakeLists.txt @@ -4,7 +4,7 @@ add_executable(ceph_test_lazy_omap_stats main.cc lazy_omap_stats_test.cc) target_link_libraries(ceph_test_lazy_omap_stats - librados ${UNITTEST_LIBS} Boost::system) + librados ${UNITTEST_LIBS} ceph-common) install(TARGETS ceph_test_lazy_omap_stats DESTINATION ${CMAKE_INSTALL_BINDIR}) diff --git a/src/test/lazy-omap-stats/lazy_omap_stats_test.cc b/src/test/lazy-omap-stats/lazy_omap_stats_test.cc index dd461429f30..ea4e06b89c0 100644 --- a/src/test/lazy-omap-stats/lazy_omap_stats_test.cc +++ b/src/test/lazy-omap-stats/lazy_omap_stats_test.cc @@ -15,6 +15,7 @@ #include #include +#include "boost/date_time/posix_time/posix_time.hpp" #include #include #include // uuid class @@ -27,8 +28,10 @@ #include "lazy_omap_stats_test.h" +using namespace boost; using namespace std; namespace bp = boost::process; +namespace pt = boost::posix_time; void LazyOmapStatsTest::init(const int argc, const char** argv) { @@ -166,27 +169,81 @@ void LazyOmapStatsTest::create_payload() << endl; } -void LazyOmapStatsTest::scrub() const +void LazyOmapStatsTest::get_scrub_stamps(map& scrub_stamps) { - // Use CLI because we need to block + string command = R"({"prefix": "pg dump", "dumpcontents": ["pgs"]})"; + string dump_output = get_output(command); - cout << "Scrubbing" << endl; - error_code ec; - bp::ipstream is; - bp::system("ceph osd deep-scrub all --block", bp::std_out > is, ec); - if (ec) { - cout << "Deep scrub command failed! Error: " << ec.value() << " " - << ec.message() << endl; - exit(ec.value()); + regex reg(R"(^(PG_STAT\s.*))" + "\n"); + + int pg_index = find_index(dump_output, reg, "PG_STAT"); + int stamp_date_index = find_index(dump_output, reg, "LAST_DEEP_SCRUB"); + int stamp_time_index = find_index(dump_output, reg, "DEEP_SCRUB_STAMP"); + reg = R"(^(PG_STAT[\s\S]*))" + "\n\n"; + smatch match; + regex_search(dump_output, match, reg); + auto table = match[1].str(); + istringstream buffer(table); + string line; + bool header = true; + while (std::getline(buffer, line)) { + if (header) { + header = false; + continue; + } + boost::char_separator sep{" "}; + boost::tokenizer> tok(line, sep); + vector tokens(tok.begin(), tok.end()); + //cout << tokens.at(pg_index) << " " << tokens.at(stamp_date_index) + // << " " << tokens.at(stamp_time_index) << endl; + string date_time_stamp = tokens.at(stamp_date_index) + " " + + tokens.at(stamp_time_index); + scrub_stamps.insert({tokens.at(pg_index), date_time_stamp}); } - cout << is.rdbuf() << endl; + +} + +void LazyOmapStatsTest::wait_for_scrub_stamps(map& intial_scrub_stamps) +{ + while (!intial_scrub_stamps.empty()) + { + map scrub_stamps; + get_scrub_stamps(scrub_stamps); + for (auto it = intial_scrub_stamps.begin(); + it != intial_scrub_stamps.end();) { + auto initial_ts = pt::time_from_string(it->second); + auto current_ts = pt::time_from_string(scrub_stamps.at(it->first)); + if (current_ts > initial_ts) { + cout << "Scrub stamp updated: " << it->first << endl; + intial_scrub_stamps.erase(it++); + } else { + ++it; + } + } + this_thread::sleep_for(chrono::milliseconds(500)); + } +} + +void LazyOmapStatsTest::scrub() +{ + cout << "Getting pre-scrub timestamps" << endl; + map scrub_stamps; + get_scrub_stamps(scrub_stamps); + cout << "Scrubbing" << endl; + string command = R"({"prefix": "osd deep-scrub", "who": "all", "target": ["mgr", ""]})"; + string dump_output = get_output(command); + cout << dump_output << endl; + cout << "Waiting for updated scrub timestamps" << endl; + wait_for_scrub_stamps(scrub_stamps); } const int LazyOmapStatsTest::find_matches(string& output, regex& reg) const { sregex_iterator cur(output.begin(), output.end(), reg); uint x = 0; - for (auto end = std::sregex_iterator(); cur != end; ++cur) { + for (auto end = sregex_iterator(); cur != end; ++cur) { cout << (*cur)[1].str() << endl; x++; } @@ -225,9 +282,9 @@ void LazyOmapStatsTest::check_one() cout << truncated_output << endl; reg = regex( "\n" - R"(([0-9,s].*\s)" + + R"(([0-9,s].*?\s)" + to_string(conf.keys) + - R"(\s.*))" + R"(\s.*?))" "\n"); cout << "Checking number of keys " << conf.keys << endl; @@ -240,9 +297,9 @@ void LazyOmapStatsTest::check_one() reg = regex( "\n" - R"(([0-9,s].*\s)" + + R"(([0-9,s].*?\s)" + to_string(conf.payload_size * conf.keys) + - R"(\s.*))" + R"(\s.*?))" "\n"); cout << "Checking number of bytes " << conf.payload_size * conf.keys << endl; @@ -406,7 +463,7 @@ void LazyOmapStatsTest::check_pg_dump_summary() reg = "\n" - R"((sum\s.*))" + R"((sum\s.*?))" "\n"; smatch match; regex_search(dump_output, match, reg); @@ -464,7 +521,7 @@ void LazyOmapStatsTest::check_pg_dump_pools() "\n" R"(()" + pool_id + - R"(\s.*))" + R"(\s.*?))" "\n"; smatch match; regex_search(dump_output, match, reg); @@ -486,11 +543,11 @@ void LazyOmapStatsTest::check_pg_ls() string dump_output = get_output(command); cout << dump_output << endl; - regex reg(R"(^(PG\s.*))" + regex reg(R"(^(PG_STAT\s.*))" "\n"); index_t indexes = get_indexes(reg, dump_output); - reg = R"(^(PG[\s\S]*))" + reg = R"(^(PG_STAT[\s\S]*?))" "\n\n"; smatch match; regex_search(dump_output, match, reg); diff --git a/src/test/lazy-omap-stats/lazy_omap_stats_test.h b/src/test/lazy-omap-stats/lazy_omap_stats_test.h index 5399012eae7..c242427cab7 100644 --- a/src/test/lazy-omap-stats/lazy_omap_stats_test.h +++ b/src/test/lazy-omap-stats/lazy_omap_stats_test.h @@ -16,7 +16,7 @@ #define CEPH_LAZY_OMAP_STATS_TEST_H #include -#include +#include #include #include "include/rados/librados.hpp" @@ -50,16 +50,18 @@ class LazyOmapStatsTest const std::string get_name() const; void create_payload(); void write_many(const uint how_many); - void scrub() const; - const int find_matches(std::string& output, std::regex& reg) const; + void get_scrub_stamps(std::map& scrub_stamps); + void wait_for_scrub_stamps(std::map& intial_scrub_stamps); + void scrub(); + const int find_matches(std::string& output, boost::regex& reg) const; void check_one(); - const int find_index(std::string& haystack, std::regex& needle, + const int find_index(std::string& haystack, boost::regex& needle, std::string label) const; const uint tally_column(const uint omap_bytes_index, const std::string& table, bool header) const; void check_column(const int index, const std::string& table, const std::string& type, bool header = true) const; - index_t get_indexes(std::regex& reg, std::string& output) const; + index_t get_indexes(boost::regex& reg, std::string& output) const; const std::string get_pool_id(std::string& pool); void check_pg_dump(); void check_pg_dump_summary();