--- /dev/null
+roles:
+- - mon.a
+ - mgr.x
+ - osd.0
+ - osd.1
+ - osd.2
+ - osd.3
+ - osd.4
+ - osd.5
+ - osd.6
+ - osd.7
+ - osd.8
+ - osd.9
+ - osd.10
+ - osd.11
+ - osd.12
+ - osd.13
+ - osd.14
+ - client.0
+openstack:
+ - volumes: # attached to each instance
+ count: 4
+ size: 1 # GB
+overrides:
+ ceph:
+ conf:
+ mon:
+ mon memory target: 134217728 # reduced to 128_M
+ rocksdb cache size: 67108864 # reduced to 64_M
+ mon osd cache size: 100000
+ mon osd cache size min: 134217728
+ osd:
+ osd memory target: 1610612736 # reduced to 1.5_G
+ osd objectstore: bluestore
+ debug bluestore: 20
+ osd scrub min interval: 60
+ osd scrub max interval: 120
+ osd max backfills: 9
+
+tasks:
+- install:
+ branch: wip-sseshasa2-testing-2019-07-30-1825 # change as appropriate
+- ceph:
+ create_rbd_pool: false
+ log-whitelist:
+ - overall HEALTH_
+ - \(OSDMAP_FLAGS\)
+ - \(OSD_
+ - \(PG_
+ - \(POOL_
+ - \(CACHE_POOL_
+ - \(OBJECT_
+ - \(SLOW_OPS\)
+ - \(REQUEST_SLOW\)
+ - \(TOO_FEW_PGS\)
+ - slow requests
+- interactive:
+- parallel:
+ - log-mon-rss
+ - stress-tasks
+ - benchload
+- exec:
+ client.0:
+ - "ceph_test_mon_memory_target 134217728" # mon memory target
+ - "ceph_test_mon_rss_usage 134217728"
+log-mon-rss:
+- background_exec:
+ client.0:
+ - while true
+ - do /usr/bin/ceph_test_log_rss_usage ceph-mon >> /var/log/ceph/ceph-mon-rss-usage.log
+ - sleep 300 # log rss usage every 5 mins. May be modified accordingly
+ - done
+- exec:
+ client.0:
+ - sleep 37860 # sum total of the radosbench test times below plus 60 secs
+benchload: # The total radosbench test below translates to 10.5 hrs
+- full_sequential:
+ - radosbench:
+ clients: [client.0]
+ time: 1800
+ - radosbench:
+ clients: [client.0]
+ time: 1800
+ - radosbench:
+ clients: [client.0]
+ time: 1800
+ - radosbench:
+ clients: [client.0]
+ time: 1800
+ - radosbench:
+ clients: [client.0]
+ time: 1800
+ - radosbench:
+ clients: [client.0]
+ time: 1800
+ - radosbench:
+ clients: [client.0]
+ time: 1800
+ - radosbench:
+ clients: [client.0]
+ time: 1800
+ - radosbench:
+ clients: [client.0]
+ time: 1800
+ - radosbench:
+ clients: [client.0]
+ time: 1800
+ - radosbench:
+ clients: [client.0]
+ time: 1800
+ - radosbench:
+ clients: [client.0]
+ time: 1800
+ - radosbench:
+ clients: [client.0]
+ time: 1800
+ - radosbench:
+ clients: [client.0]
+ time: 1800
+ - radosbench:
+ clients: [client.0]
+ time: 1800
+ - radosbench:
+ clients: [client.0]
+ time: 1800
+ - radosbench:
+ clients: [client.0]
+ time: 1800
+ - radosbench:
+ clients: [client.0]
+ time: 1800
+ - radosbench:
+ clients: [client.0]
+ time: 1800
+ - radosbench:
+ clients: [client.0]
+ time: 1800
+ - radosbench:
+ clients: [client.0]
+ time: 1800
+stress-tasks:
+- thrashosds:
+ op_delay: 1
+ bdev_inject_crash: 1
+ bdev_inject_crash_probability: .8
+ chance_down: 80
+ chance_pgnum_grow: 3
+ chance_pgpnum_fix: 1
+ chance_thrash_cluster_full: 0
+ chance_thrash_pg_upmap: 3
+ chance_thrash_pg_upmap_items: 3
+ min_in: 2
)
add_ceph_unittest(unittest_mon_montypes)
target_link_libraries(unittest_mon_montypes mon global)
+
+# ceph_test_mon_memory_target
+add_executable(ceph_test_mon_memory_target
+ test_mon_memory_target.cc
+ )
+target_link_libraries(ceph_test_mon_memory_target ${UNITTEST_LIBS} Boost::system)
+install(TARGETS ceph_test_mon_memory_target
+ DESTINATION ${CMAKE_INSTALL_BINDIR})
+
+# ceph_test_mon_log_rss_usage
+add_executable(ceph_test_log_rss_usage
+ test_log_rss_usage.cc
+ )
+target_link_libraries(ceph_test_log_rss_usage ${UNITTEST_LIBS})
+install(TARGETS ceph_test_log_rss_usage
+ DESTINATION ${CMAKE_INSTALL_BINDIR})
+
+# ceph_test_mon_rss_usage
+add_executable(ceph_test_mon_rss_usage
+ test_mon_rss_usage.cc
+ )
+target_link_libraries(ceph_test_mon_rss_usage ${UNITTEST_LIBS})
+install(TARGETS ceph_test_mon_rss_usage
+ DESTINATION ${CMAKE_INSTALL_BINDIR})
--- /dev/null
+#include <sys/types.h>
+#include <dirent.h>
+#include <errno.h>
+#include <vector>
+#include <string>
+#include <iostream>
+#include <fstream>
+#include <stdlib.h>
+#include <stdio.h>
+#include <unistd.h>
+
+using namespace std;
+
+int getPidByName(string procName)
+{
+ int pid = -1;
+
+ // Open the /proc directory
+ DIR *dp = opendir("/proc");
+ if (dp != NULL)
+ {
+ // Enumerate all entries in '/proc' until process is found
+ struct dirent *dirp;
+ while (pid < 0 && (dirp = readdir(dp)))
+ {
+ // Skip non-numeric entries
+ int id = atoi(dirp->d_name);
+ if (id > 0)
+ {
+ // Read contents of virtual /proc/{pid}/cmdline file
+ string cmdPath = string("/proc/") + dirp->d_name + "/cmdline";
+ ifstream cmdFile(cmdPath.c_str());
+ string cmdLine;
+ getline(cmdFile, cmdLine);
+ if (!cmdLine.empty())
+ {
+ // Keep first cmdline item which contains the program path
+ size_t pos = cmdLine.find('\0');
+ if (pos != string::npos) {
+ cmdLine = cmdLine.substr(0, pos);
+ }
+ // Get program name only, removing the path
+ pos = cmdLine.rfind('/');
+ if (pos != string::npos) {
+ cmdLine = cmdLine.substr(pos + 1);
+ }
+ // Compare against requested process name
+ if (procName == cmdLine) {
+ pid = id;
+ }
+ }
+ }
+ }
+ }
+
+ closedir(dp);
+
+ return pid;
+}
+
+uint64_t getRssUsage(string pid)
+{
+ int totalSize = 0;
+ int resSize = 0;
+
+ string statmPath = string("/proc/") + pid + "/statm";
+ ifstream buffer(statmPath);
+ buffer >> totalSize >> resSize;
+ buffer.close();
+
+ long page_size = sysconf(_SC_PAGE_SIZE);
+ uint64_t rss = resSize * page_size;
+
+ return rss;
+}
+
+int main(int argc, char* argv[])
+{
+ if (argc != 2) {
+ cout << "Syntax: "
+ << "ceph_test_log_rss_usage <process name>"
+ << endl;
+ exit(EINVAL);
+ }
+ uint64_t rss = 0;
+ int pid = getPidByName(argv[1]);
+ string rssUsage;
+
+ // Use the pid to get RSS memory usage
+ // and print it to stdout
+ if (pid != -1) {
+ rss = getRssUsage(to_string(pid));
+ } else {
+ cout << "Process " << argv[1] << " NOT FOUND!\n" << endl;
+ exit(ESRCH);
+ }
+
+ rssUsage = to_string(rss) + ":" + to_string(pid) + ":";
+ cout << rssUsage.c_str() << endl;
+ return 0;
+}
--- /dev/null
+#include <algorithm>
+#include <iostream>
+#include <string>
+#include <numeric>
+#include <regex>
+#include <system_error>
+
+#include <boost/process.hpp>
+#include <boost/tokenizer.hpp>
+
+namespace bp = boost::process;
+using namespace std;
+
+int main(int argc, char** argv)
+{
+ cout << "Mon Memory Target Test" << endl;
+
+ if (argc != 2) {
+ cout << "Syntax: "
+ << "ceph_test_mon_memory_target <mon-memory-target-bytes>"
+ << endl;
+ exit(EINVAL);
+ }
+
+ string target_directory("/var/log/ceph/");
+ unsigned long maxallowed = stoul(argv[1], nullptr, 10);
+ regex reg(R"(cache_size:(\d*)\s)");
+
+ string grep_command("grep _set_new_cache_sizes " + target_directory
+ + "ceph-mon.a.log");
+ bp::ipstream is;
+ error_code ec;
+ bp::child grep(grep_command, bp::std_out > is, ec);
+ if (ec) {
+ cout << "Error grepping logs! Exiting" << endl;
+ cout << "Error: " << ec.value() << " " << ec.message() << endl;
+ exit(ec.value());
+ }
+
+ string line;
+ vector<unsigned long> results;
+ while (grep.running() && getline(is, line) && !line.empty()) {
+ smatch match;
+ if (regex_search(line, match, reg)) {
+ results.push_back(stoul(match[1].str()));
+ }
+ }
+
+ if (results.empty()) {
+ cout << "Error: No grep results found!" << endl;
+ exit(ENOENT);
+ }
+
+ auto maxe = *(max_element(results.begin(), results.end()));
+ cout << "Results for mon_memory_target:" << endl;
+ cout << "Max: " << maxe << endl;
+ cout << "Min: " << *(min_element(results.begin(), results.end())) << endl;
+ auto sum = accumulate(results.begin(), results.end(),
+ static_cast<unsigned long long>(0));
+ auto mean = sum / results.size();
+ cout << "Mean average: " << mean << endl;
+ vector<unsigned long> diff(results.size());
+ transform(results.begin(), results.end(), diff.begin(),
+ [mean](unsigned long x) { return x - mean; });
+ auto sump = inner_product(diff.begin(), diff.end(), diff.begin(), 0.0);
+ auto stdev = sqrt(sump / results.size());
+ cout << "Standard deviation: " << stdev << endl;
+
+ if (maxe > maxallowed) {
+ cout << "Error: Mon memory consumption exceeds maximum allowed!" << endl;
+ exit(ENOMEM);
+ }
+
+ grep.wait();
+
+ cout << "Completed successfully" << endl;
+ return 0;
+}
--- /dev/null
+#include <algorithm>
+#include <iostream>
+#include <fstream>
+#include <string>
+#include <numeric>
+#include <regex>
+#include <cmath>
+#include <system_error>
+
+using namespace std;
+
+int main(int argc, char **argv)
+{
+ cout << "Mon RSS Usage Test" << endl;
+
+ if (argc != 2) {
+ cout << "Syntax: "
+ << "ceph_test_mon_rss_usage <mon-memory-target-bytes>"
+ << endl;
+ exit(EINVAL);
+ }
+
+ unsigned long maxallowed = stoul(argv[1], nullptr, 10);
+ // Set max allowed RSS usage to be 125% of mon-memory-target
+ maxallowed *= 1.25;
+
+ string target_directory("/var/log/ceph/");
+ string filePath = target_directory + "ceph-mon-rss-usage.log";
+ ifstream buffer(filePath.c_str());
+ string line;
+ vector<unsigned long> results;
+ while(getline(buffer, line) && !line.empty()) {
+ string rssUsage;
+ size_t pos = line.find(':');
+ if (pos != string::npos) {
+ rssUsage = line.substr(0, pos);
+ }
+ if (!rssUsage.empty()) {
+ results.push_back(stoul(rssUsage));
+ }
+ }
+
+ buffer.close();
+ if (results.empty()) {
+ cout << "Error: No grep results found!" << endl;
+ exit(ENOENT);
+ }
+
+ auto maxe = *(max_element(results.begin(), results.end()));
+ cout << "Stats for mon RSS Memory Usage:" << endl;
+ cout << "Parsed " << results.size() << " entries." << endl;
+ cout << "Max: " << maxe << endl;
+ cout << "Min: " << *(min_element(results.begin(), results.end())) << endl;
+ auto sum = accumulate(results.begin(), results.end(),
+ static_cast<unsigned long long>(0));
+ auto mean = sum / results.size();
+ cout << "Mean average: " << mean << endl;
+ vector<unsigned long> diff(results.size());
+ transform(results.begin(), results.end(), diff.begin(),
+ [mean](unsigned long x) { return x - mean; });
+ auto sump = inner_product(diff.begin(), diff.end(), diff.begin(), 0.0);
+ auto stdev = sqrt(sump / results.size());
+ cout << fixed << "Standard deviation: " << stdev << endl;
+
+ if (maxe > maxallowed) {
+ cout << "Error: Mon RSS memory usage exceeds maximum allowed!" << endl;
+ exit(ENOMEM);
+ }
+
+ cout << "Completed successfully" << endl;
+ return 0;
+}
projects / ceph.git / commitdiff