From: Sridhar Seshasayee Date: Tue, 9 Jul 2019 11:17:49 +0000 (+0530) Subject: mon/OSDMonitor: Add standalone test for mon_memory_target X-Git-Tag: v15.1.0~1912^2 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=1034782d4c26326e351cceb0c11f63b6039ecf4d;p=ceph.git mon/OSDMonitor: Add standalone test for mon_memory_target Signed-off-by: Sridhar Seshasayee --- diff --git a/qa/suites/rados/singleton/all/mon-memory-target-compliance.yaml.disabled b/qa/suites/rados/singleton/all/mon-memory-target-compliance.yaml.disabled new file mode 100644 index 000000000000..56de322ebc4e --- /dev/null +++ b/qa/suites/rados/singleton/all/mon-memory-target-compliance.yaml.disabled @@ -0,0 +1,152 @@ +roles: +- - mon.a + - mgr.x + - osd.0 + - osd.1 + - osd.2 + - osd.3 + - osd.4 + - osd.5 + - osd.6 + - osd.7 + - osd.8 + - osd.9 + - osd.10 + - osd.11 + - osd.12 + - osd.13 + - osd.14 + - client.0 +openstack: + - volumes: # attached to each instance + count: 4 + size: 1 # GB +overrides: + ceph: + conf: + mon: + mon memory target: 134217728 # reduced to 128_M + rocksdb cache size: 67108864 # reduced to 64_M + mon osd cache size: 100000 + mon osd cache size min: 134217728 + osd: + osd memory target: 1610612736 # reduced to 1.5_G + osd objectstore: bluestore + debug bluestore: 20 + osd scrub min interval: 60 + osd scrub max interval: 120 + osd max backfills: 9 + +tasks: +- install: + branch: wip-sseshasa2-testing-2019-07-30-1825 # change as appropriate +- ceph: + create_rbd_pool: false + log-whitelist: + - overall HEALTH_ + - \(OSDMAP_FLAGS\) + - \(OSD_ + - \(PG_ + - \(POOL_ + - \(CACHE_POOL_ + - \(OBJECT_ + - \(SLOW_OPS\) + - \(REQUEST_SLOW\) + - \(TOO_FEW_PGS\) + - slow requests +- interactive: +- parallel: + - log-mon-rss + - stress-tasks + - benchload +- exec: + client.0: + - "ceph_test_mon_memory_target 134217728" # mon memory target + - "ceph_test_mon_rss_usage 134217728" +log-mon-rss: +- background_exec: + client.0: + - while true + - do /usr/bin/ceph_test_log_rss_usage ceph-mon >> /var/log/ceph/ceph-mon-rss-usage.log + - sleep 300 # log rss usage every 5 mins. May be modified accordingly + - done +- exec: + client.0: + - sleep 37860 # sum total of the radosbench test times below plus 60 secs +benchload: # The total radosbench test below translates to 10.5 hrs +- full_sequential: + - radosbench: + clients: [client.0] + time: 1800 + - radosbench: + clients: [client.0] + time: 1800 + - radosbench: + clients: [client.0] + time: 1800 + - radosbench: + clients: [client.0] + time: 1800 + - radosbench: + clients: [client.0] + time: 1800 + - radosbench: + clients: [client.0] + time: 1800 + - radosbench: + clients: [client.0] + time: 1800 + - radosbench: + clients: [client.0] + time: 1800 + - radosbench: + clients: [client.0] + time: 1800 + - radosbench: + clients: [client.0] + time: 1800 + - radosbench: + clients: [client.0] + time: 1800 + - radosbench: + clients: [client.0] + time: 1800 + - radosbench: + clients: [client.0] + time: 1800 + - radosbench: + clients: [client.0] + time: 1800 + - radosbench: + clients: [client.0] + time: 1800 + - radosbench: + clients: [client.0] + time: 1800 + - radosbench: + clients: [client.0] + time: 1800 + - radosbench: + clients: [client.0] + time: 1800 + - radosbench: + clients: [client.0] + time: 1800 + - radosbench: + clients: [client.0] + time: 1800 + - radosbench: + clients: [client.0] + time: 1800 +stress-tasks: +- thrashosds: + op_delay: 1 + bdev_inject_crash: 1 + bdev_inject_crash_probability: .8 + chance_down: 80 + chance_pgnum_grow: 3 + chance_pgpnum_fix: 1 + chance_thrash_cluster_full: 0 + chance_thrash_pg_upmap: 3 + chance_thrash_pg_upmap_items: 3 + min_in: 2 diff --git a/src/test/mon/CMakeLists.txt b/src/test/mon/CMakeLists.txt index 53b57dfe4d27..b712e95be7d4 100644 --- a/src/test/mon/CMakeLists.txt +++ b/src/test/mon/CMakeLists.txt @@ -39,3 +39,27 @@ add_executable(unittest_mon_montypes ) add_ceph_unittest(unittest_mon_montypes) target_link_libraries(unittest_mon_montypes mon global) + +# ceph_test_mon_memory_target +add_executable(ceph_test_mon_memory_target + test_mon_memory_target.cc + ) +target_link_libraries(ceph_test_mon_memory_target ${UNITTEST_LIBS} Boost::system) +install(TARGETS ceph_test_mon_memory_target + DESTINATION ${CMAKE_INSTALL_BINDIR}) + +# ceph_test_mon_log_rss_usage +add_executable(ceph_test_log_rss_usage + test_log_rss_usage.cc + ) +target_link_libraries(ceph_test_log_rss_usage ${UNITTEST_LIBS}) +install(TARGETS ceph_test_log_rss_usage + DESTINATION ${CMAKE_INSTALL_BINDIR}) + +# ceph_test_mon_rss_usage +add_executable(ceph_test_mon_rss_usage + test_mon_rss_usage.cc + ) +target_link_libraries(ceph_test_mon_rss_usage ${UNITTEST_LIBS}) +install(TARGETS ceph_test_mon_rss_usage + DESTINATION ${CMAKE_INSTALL_BINDIR}) diff --git a/src/test/mon/test_log_rss_usage.cc b/src/test/mon/test_log_rss_usage.cc new file mode 100644 index 000000000000..f6e85f414c8e --- /dev/null +++ b/src/test/mon/test_log_rss_usage.cc @@ -0,0 +1,101 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace std; + +int getPidByName(string procName) +{ + int pid = -1; + + // Open the /proc directory + DIR *dp = opendir("/proc"); + if (dp != NULL) + { + // Enumerate all entries in '/proc' until process is found + struct dirent *dirp; + while (pid < 0 && (dirp = readdir(dp))) + { + // Skip non-numeric entries + int id = atoi(dirp->d_name); + if (id > 0) + { + // Read contents of virtual /proc/{pid}/cmdline file + string cmdPath = string("/proc/") + dirp->d_name + "/cmdline"; + ifstream cmdFile(cmdPath.c_str()); + string cmdLine; + getline(cmdFile, cmdLine); + if (!cmdLine.empty()) + { + // Keep first cmdline item which contains the program path + size_t pos = cmdLine.find('\0'); + if (pos != string::npos) { + cmdLine = cmdLine.substr(0, pos); + } + // Get program name only, removing the path + pos = cmdLine.rfind('/'); + if (pos != string::npos) { + cmdLine = cmdLine.substr(pos + 1); + } + // Compare against requested process name + if (procName == cmdLine) { + pid = id; + } + } + } + } + } + + closedir(dp); + + return pid; +} + +uint64_t getRssUsage(string pid) +{ + int totalSize = 0; + int resSize = 0; + + string statmPath = string("/proc/") + pid + "/statm"; + ifstream buffer(statmPath); + buffer >> totalSize >> resSize; + buffer.close(); + + long page_size = sysconf(_SC_PAGE_SIZE); + uint64_t rss = resSize * page_size; + + return rss; +} + +int main(int argc, char* argv[]) +{ + if (argc != 2) { + cout << "Syntax: " + << "ceph_test_log_rss_usage " + << endl; + exit(EINVAL); + } + uint64_t rss = 0; + int pid = getPidByName(argv[1]); + string rssUsage; + + // Use the pid to get RSS memory usage + // and print it to stdout + if (pid != -1) { + rss = getRssUsage(to_string(pid)); + } else { + cout << "Process " << argv[1] << " NOT FOUND!\n" << endl; + exit(ESRCH); + } + + rssUsage = to_string(rss) + ":" + to_string(pid) + ":"; + cout << rssUsage.c_str() << endl; + return 0; +} diff --git a/src/test/mon/test_mon_memory_target.cc b/src/test/mon/test_mon_memory_target.cc new file mode 100644 index 000000000000..da3a86d6517d --- /dev/null +++ b/src/test/mon/test_mon_memory_target.cc @@ -0,0 +1,78 @@ +#include +#include +#include +#include +#include +#include + +#include +#include + +namespace bp = boost::process; +using namespace std; + +int main(int argc, char** argv) +{ + cout << "Mon Memory Target Test" << endl; + + if (argc != 2) { + cout << "Syntax: " + << "ceph_test_mon_memory_target " + << endl; + exit(EINVAL); + } + + string target_directory("/var/log/ceph/"); + unsigned long maxallowed = stoul(argv[1], nullptr, 10); + regex reg(R"(cache_size:(\d*)\s)"); + + string grep_command("grep _set_new_cache_sizes " + target_directory + + "ceph-mon.a.log"); + bp::ipstream is; + error_code ec; + bp::child grep(grep_command, bp::std_out > is, ec); + if (ec) { + cout << "Error grepping logs! Exiting" << endl; + cout << "Error: " << ec.value() << " " << ec.message() << endl; + exit(ec.value()); + } + + string line; + vector results; + while (grep.running() && getline(is, line) && !line.empty()) { + smatch match; + if (regex_search(line, match, reg)) { + results.push_back(stoul(match[1].str())); + } + } + + if (results.empty()) { + cout << "Error: No grep results found!" << endl; + exit(ENOENT); + } + + auto maxe = *(max_element(results.begin(), results.end())); + cout << "Results for mon_memory_target:" << endl; + cout << "Max: " << maxe << endl; + cout << "Min: " << *(min_element(results.begin(), results.end())) << endl; + auto sum = accumulate(results.begin(), results.end(), + static_cast(0)); + auto mean = sum / results.size(); + cout << "Mean average: " << mean << endl; + vector diff(results.size()); + transform(results.begin(), results.end(), diff.begin(), + [mean](unsigned long x) { return x - mean; }); + auto sump = inner_product(diff.begin(), diff.end(), diff.begin(), 0.0); + auto stdev = sqrt(sump / results.size()); + cout << "Standard deviation: " << stdev << endl; + + if (maxe > maxallowed) { + cout << "Error: Mon memory consumption exceeds maximum allowed!" << endl; + exit(ENOMEM); + } + + grep.wait(); + + cout << "Completed successfully" << endl; + return 0; +} diff --git a/src/test/mon/test_mon_rss_usage.cc b/src/test/mon/test_mon_rss_usage.cc new file mode 100644 index 000000000000..76b5856f6dc4 --- /dev/null +++ b/src/test/mon/test_mon_rss_usage.cc @@ -0,0 +1,72 @@ +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace std; + +int main(int argc, char **argv) +{ + cout << "Mon RSS Usage Test" << endl; + + if (argc != 2) { + cout << "Syntax: " + << "ceph_test_mon_rss_usage " + << endl; + exit(EINVAL); + } + + unsigned long maxallowed = stoul(argv[1], nullptr, 10); + // Set max allowed RSS usage to be 125% of mon-memory-target + maxallowed *= 1.25; + + string target_directory("/var/log/ceph/"); + string filePath = target_directory + "ceph-mon-rss-usage.log"; + ifstream buffer(filePath.c_str()); + string line; + vector results; + while(getline(buffer, line) && !line.empty()) { + string rssUsage; + size_t pos = line.find(':'); + if (pos != string::npos) { + rssUsage = line.substr(0, pos); + } + if (!rssUsage.empty()) { + results.push_back(stoul(rssUsage)); + } + } + + buffer.close(); + if (results.empty()) { + cout << "Error: No grep results found!" << endl; + exit(ENOENT); + } + + auto maxe = *(max_element(results.begin(), results.end())); + cout << "Stats for mon RSS Memory Usage:" << endl; + cout << "Parsed " << results.size() << " entries." << endl; + cout << "Max: " << maxe << endl; + cout << "Min: " << *(min_element(results.begin(), results.end())) << endl; + auto sum = accumulate(results.begin(), results.end(), + static_cast(0)); + auto mean = sum / results.size(); + cout << "Mean average: " << mean << endl; + vector diff(results.size()); + transform(results.begin(), results.end(), diff.begin(), + [mean](unsigned long x) { return x - mean; }); + auto sump = inner_product(diff.begin(), diff.end(), diff.begin(), 0.0); + auto stdev = sqrt(sump / results.size()); + cout << fixed << "Standard deviation: " << stdev << endl; + + if (maxe > maxallowed) { + cout << "Error: Mon RSS memory usage exceeds maximum allowed!" << endl; + exit(ENOMEM); + } + + cout << "Completed successfully" << endl; + return 0; +}