From ca6abca63de813c83a6960f83624be8e1a86a1f8 Mon Sep 17 00:00:00 2001 From: =?utf8?q?Piotr=20Da=C5=82ek?= Date: Wed, 20 May 2015 12:41:22 +0200 Subject: [PATCH] tools: add --no-verify option to rados bench MIME-Version: 1.0 Content-Type: text/plain; charset=utf8 Content-Transfer-Encoding: 8bit When doing seq and rand read benchmarks using rados bench, a quite large portion of cpu time is consumed by doing object verification. This patch adds an option to disable this verification when it's not needed, in turn giving better cluster utilization. rados -p storage bench 600 rand scores without --no-verification: Total time run: 600.228901 Total reads made: 144982 Read size: 4194304 Bandwidth (MB/sec): 966 Average IOPS: 241 Stddev IOPS: 38 Max IOPS: 909522486 Min IOPS: 0 Average Latency: 0.0662 Max latency: 1.51 Min latency: 0.004 real 10m1.173s user 5m41.162s sys 11m42.961s Same command, but with --no-verify: Total time run: 600.161379 Total reads made: 174142 Read size: 4194304 Bandwidth (MB/sec): 1.16e+03 Average IOPS: 290 Stddev IOPS: 20 Max IOPS: 909522486 Min IOPS: 0 Average Latency: 0.0551 Max latency: 1.12 Min latency: 0.00343 real 10m1.172s user 4m13.792s sys 13m38.556s Note the decreased latencies, increased bandwidth and more reads performed. Signed-off-by: Piotr Dałek --- src/common/obj_bencher.cc | 67 ++++++++++++++++++++++++--------------- src/common/obj_bencher.h | 6 ++-- src/tools/rados/rados.cc | 11 ++++++- 3 files changed, 55 insertions(+), 29 deletions(-) diff --git a/src/common/obj_bencher.cc b/src/common/obj_bencher.cc index 128a544a5442..8fd963ba4db4 100644 --- a/src/common/obj_bencher.cc +++ b/src/common/obj_bencher.cc @@ -169,7 +169,7 @@ void *ObjBencher::status_printer(void *_bencher) { int ObjBencher::aio_bench( int operation, int secondsToRun, - int concurrentios, int object_size, bool cleanup, const char* run_name) { + int concurrentios, int object_size, bool cleanup, const char* run_name, bool no_verify) { if (concurrentios <= 0) return -EINVAL; @@ -214,11 +214,11 @@ int ObjBencher::aio_bench( if (r != 0) goto out; } else if (OP_SEQ_READ == operation) { - r = seq_read_bench(secondsToRun, num_objects, concurrentios, prevPid); + r = seq_read_bench(secondsToRun, num_objects, concurrentios, prevPid, no_verify); if (r != 0) goto out; } else if (OP_RAND_READ == operation) { - r = rand_read_bench(secondsToRun, num_objects, concurrentios, prevPid); + r = rand_read_bench(secondsToRun, num_objects, concurrentios, prevPid, no_verify); if (r != 0) goto out; } @@ -501,7 +501,7 @@ int ObjBencher::write_bench(int secondsToRun, return r; } -int ObjBencher::seq_read_bench(int seconds_to_run, int num_objects, int concurrentios, int pid) { +int ObjBencher::seq_read_bench(int seconds_to_run, int num_objects, int concurrentios, int pid, bool no_verify) { lock_cond lc(&lock); if (concurrentios <= 0) @@ -617,12 +617,17 @@ int ObjBencher::seq_read_bench(int seconds_to_run, int num_objects, int concurre lock.Lock(); ++data.started; ++data.in_flight; - snprintf(data.object_contents, data.object_size, "I'm the %16dth object!", current_index); - lock.Unlock(); - if (memcmp(data.object_contents, cur_contents->c_str(), data.object_size) != 0) { - cerr << name[slot] << " is not correct!" << std::endl; - ++errors; + if (!no_verify) { + snprintf(data.object_contents, data.object_size, "I'm the %16dth object!", current_index); + lock.Unlock(); + if (memcmp(data.object_contents, cur_contents->c_str(), data.object_size) != 0) { + cerr << name[slot] << " is not correct!" << std::endl; + ++errors; + } + } else { + lock.Unlock(); } + name[slot] = newName; delete cur_contents; } @@ -646,11 +651,15 @@ int ObjBencher::seq_read_bench(int seconds_to_run, int num_objects, int concurre data.avg_latency = total_latency / data.finished; --data.in_flight; release_completion(slot); - snprintf(data.object_contents, data.object_size, "I'm the %16dth object!", index[slot]); - lock.Unlock(); - if (memcmp(data.object_contents, contents[slot]->c_str(), data.object_size) != 0) { - cerr << name[slot] << " is not correct!" << std::endl; - ++errors; + if (!no_verify) { + snprintf(data.object_contents, data.object_size, "I'm the %16dth object!", index[slot]); + lock.Unlock(); + if (memcmp(data.object_contents, contents[slot]->c_str(), data.object_size) != 0) { + cerr << name[slot] << " is not correct!" << std::endl; + ++errors; + } + } else { + lock.Unlock(); } delete contents[slot]; } @@ -690,7 +699,7 @@ int ObjBencher::seq_read_bench(int seconds_to_run, int num_objects, int concurre return r; } -int ObjBencher::rand_read_bench(int seconds_to_run, int num_objects, int concurrentios, int pid) +int ObjBencher::rand_read_bench(int seconds_to_run, int num_objects, int concurrentios, int pid, bool no_verify) { lock_cond lc(&lock); @@ -810,11 +819,15 @@ int ObjBencher::rand_read_bench(int seconds_to_run, int num_objects, int concurr lock.Lock(); ++data.started; ++data.in_flight; - snprintf(data.object_contents, data.object_size, "I'm the %16dth object!", current_index); - lock.Unlock(); - if (memcmp(data.object_contents, cur_contents->c_str(), data.object_size) != 0) { - cerr << name[slot] << " is not correct!" << std::endl; - ++errors; + if (!no_verify) { + snprintf(data.object_contents, data.object_size, "I'm the %16dth object!", current_index); + lock.Unlock(); + if (memcmp(data.object_contents, cur_contents->c_str(), data.object_size) != 0) { + cerr << name[slot] << " is not correct!" << std::endl; + ++errors; + } + } else { + lock.Unlock(); } name[slot] = newName; delete cur_contents; @@ -839,11 +852,15 @@ int ObjBencher::rand_read_bench(int seconds_to_run, int num_objects, int concurr data.avg_latency = total_latency / data.finished; --data.in_flight; release_completion(slot); - snprintf(data.object_contents, data.object_size, "I'm the %16dth object!", index[slot]); - lock.Unlock(); - if (memcmp(data.object_contents, contents[slot]->c_str(), data.object_size) != 0) { - cerr << name[slot] << " is not correct!" << std::endl; - ++errors; + if (!no_verify) { + snprintf(data.object_contents, data.object_size, "I'm the %16dth object!", index[slot]); + lock.Unlock(); + if (memcmp(data.object_contents, contents[slot]->c_str(), data.object_size) != 0) { + cerr << name[slot] << " is not correct!" << std::endl; + ++errors; + } + } else { + lock.Unlock(); } delete contents[slot]; } diff --git a/src/common/obj_bencher.h b/src/common/obj_bencher.h index c67d4292d2ab..e1d5e9de7fdd 100644 --- a/src/common/obj_bencher.h +++ b/src/common/obj_bencher.h @@ -67,8 +67,8 @@ protected: int fetch_bench_metadata(const std::string& metadata_file, int* object_size, int* num_objects, int* prevPid); int write_bench(int secondsToRun, int concurrentios, const string& run_name_meta); - int seq_read_bench(int secondsToRun, int num_objects, int concurrentios, int writePid); - int rand_read_bench(int secondsToRun, int num_objects, int concurrentios, int writePid); + int seq_read_bench(int secondsToRun, int num_objects, int concurrentios, int writePid, bool no_verify=false); + int rand_read_bench(int secondsToRun, int num_objects, int concurrentios, int writePid, bool no_verify=false); int clean_up(int num_objects, int prevPid, int concurrentios); int clean_up_slow(const std::string& prefix, int concurrentios); @@ -100,7 +100,7 @@ public: virtual ~ObjBencher() {} int aio_bench( int operation, int secondsToRun, - int concurrentios, int op_size, bool cleanup, const char* run_name); + int concurrentios, int op_size, bool cleanup, const char* run_name, bool no_verify=false); int clean_up(const char* prefix, int concurrentios, const char* run_name); void set_show_time(bool dt) { diff --git a/src/tools/rados/rados.cc b/src/tools/rados/rados.cc index c5d8ff52387d..f9e4ee6637f4 100644 --- a/src/tools/rados/rados.cc +++ b/src/tools/rados/rados.cc @@ -185,6 +185,8 @@ void usage(ostream& out) " Set number of concurrent I/O operations\n" " --show-time\n" " prefix output with date/time\n" +" --no-verify\n" +" do not verify contents of read objects\n" "\n" "LOAD GEN OPTIONS:\n" " --num-objects total number of objects\n" @@ -1144,6 +1146,7 @@ static int rados_tool_common(const std::map < std::string, std::string > &opts, unsigned op_size = default_op_size; bool block_size_specified = false; bool cleanup = true; + bool no_verify = false; const char *snapname = NULL; snap_t snapid = CEPH_NOSNAP; std::map::const_iterator i; @@ -1314,6 +1317,10 @@ static int rados_tool_common(const std::map < std::string, std::string > &opts, if (i != opts.end()) { nspace = i->second; } + i = opts.find("no-verify"); + if (i != opts.end()) { + no_verify = true; + } // open rados @@ -2281,7 +2288,7 @@ static int rados_tool_common(const std::map < std::string, std::string > &opts, RadosBencher bencher(g_ceph_context, rados, io_ctx); bencher.set_show_time(show_time); ret = bencher.aio_bench(operation, seconds, - concurrent_ios, op_size, cleanup, run_name); + concurrent_ios, op_size, cleanup, run_name, no_verify); if (ret != 0) cerr << "error during benchmark: " << ret << std::endl; } @@ -2660,6 +2667,8 @@ int main(int argc, const char **argv) opts["show-time"] = "true"; } else if (ceph_argparse_flag(args, i, "--no-cleanup", (char*)NULL)) { opts["no-cleanup"] = "true"; + } else if (ceph_argparse_flag(args, i, "--no-verify", (char*)NULL)) { + opts["no-verify"] = "true"; } else if (ceph_argparse_witharg(args, i, &val, "--run-name", (char*)NULL)) { opts["run-name"] = val; } else if (ceph_argparse_witharg(args, i, &val, "--prefix", (char*)NULL)) { -- 2.47.3