From: Matty Williams Date: Mon, 20 Oct 2025 15:46:43 +0000 (+0100) Subject: test/osd: Add balanced read flags to io_sequence exerciser X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=98f36a1e02e805e02a23ebc395baad352d8d586a;p=ceph.git test/osd: Add balanced read flags to io_sequence exerciser Added optional "-b"/"balanced" flag to the end of read/read2/read3 operations in interactive mode, to make them balanced reads. Balanced read percentage is not used in interactive mode. Add command line argument to specify percentage of read ops that should use the balanced reads flag. Default is 100%. Signed-off-by: Matty Williams Signed-off-by: Alex Ainscow Signed-off-by: Jon Bailey --- diff --git a/src/common/io_exerciser/IoOp.cc b/src/common/io_exerciser/IoOp.cc index 4bb6fb761eb..11c1b3ca4c3 100644 --- a/src/common/io_exerciser/IoOp.cc +++ b/src/common/io_exerciser/IoOp.cc @@ -117,6 +117,15 @@ ceph::io_exerciser::ReadWriteOp::ReadWriteOp( } } +template +ceph::io_exerciser::ReadWriteOp::ReadWriteOp( + std::array&& offset, + std::array&& length, + std::optional balanced_read) + : ReadWriteOp(std::move(offset), std::move(length)) { + this->balanced_read = balanced_read; + } + ConsistencyOp::ConsistencyOp() : TestOp() {} std::unique_ptr ConsistencyOp::generate() { @@ -175,35 +184,57 @@ std::string ceph::io_exerciser::ReadWriteOp::to_string( } } -SingleReadOp::SingleReadOp(uint64_t offset, uint64_t length) - : ReadWriteOp({offset}, {length}) {} +SingleReadOp::SingleReadOp(uint64_t offset, uint64_t length, std::optional balanced_read) + : ReadWriteOp({offset}, {length}, balanced_read) {} std::unique_ptr SingleReadOp::generate(uint64_t offset, uint64_t length) { - return std::make_unique(offset, length); + return std::make_unique(offset, length, std::nullopt); +} + +std::unique_ptr SingleReadOp::generate(uint64_t offset, + uint64_t length, bool balanced_read) { + return std::make_unique(offset, length, balanced_read); } DoubleReadOp::DoubleReadOp(uint64_t offset1, uint64_t length1, uint64_t offset2, - uint64_t length2) - : ReadWriteOp({offset1, offset2}, {length1, length2}) {} + uint64_t length2, std::optional balanced_read) + : ReadWriteOp({offset1, offset2}, {length1, length2}, balanced_read) {} std::unique_ptr DoubleReadOp::generate(uint64_t offset1, uint64_t length1, uint64_t offset2, uint64_t length2) { - return std::make_unique(offset1, length1, offset2, length2); + return std::make_unique(offset1, length1, offset2, length2, std::nullopt); +} + +std::unique_ptr DoubleReadOp::generate(uint64_t offset1, + uint64_t length1, + uint64_t offset2, + uint64_t length2, + bool balanced_read) { + return std::make_unique(offset1, length1, offset2, length2, balanced_read); } TripleReadOp::TripleReadOp(uint64_t offset1, uint64_t length1, uint64_t offset2, - uint64_t length2, uint64_t offset3, uint64_t length3) + uint64_t length2, uint64_t offset3, uint64_t length3, + std::optional balanced_read) : ReadWriteOp({offset1, offset2, offset3}, - {length1, length2, length3}) {} + {length1, length2, length3}, + balanced_read) {} std::unique_ptr TripleReadOp::generate( uint64_t offset1, uint64_t length1, uint64_t offset2, uint64_t length2, uint64_t offset3, uint64_t length3) { return std::make_unique(offset1, length1, offset2, length2, - offset3, length3); + offset3, length3, std::nullopt); +} + +std::unique_ptr TripleReadOp::generate( + uint64_t offset1, uint64_t length1, uint64_t offset2, uint64_t length2, + uint64_t offset3, uint64_t length3, bool balanced_read) { + return std::make_unique(offset1, length1, offset2, length2, + offset3, length3, balanced_read); } SingleWriteOp::SingleWriteOp(uint64_t offset, uint64_t length) diff --git a/src/common/io_exerciser/IoOp.h b/src/common/io_exerciser/IoOp.h index 88165db8180..603cbcb7e2d 100644 --- a/src/common/io_exerciser/IoOp.h +++ b/src/common/io_exerciser/IoOp.h @@ -87,37 +87,52 @@ class ReadWriteOp : public TestOp { public: std::array offset; std::array length; + std::optional balanced_read = std::nullopt; protected: ReadWriteOp(std::array&& offset, std::array&& length); + ReadWriteOp(std::array&& offset, + std::array&& length, + std::optional balanced_read); std::string to_string(uint64_t block_size) const override; }; class SingleReadOp : public ReadWriteOp { public: - SingleReadOp(uint64_t offset, uint64_t length); + SingleReadOp(uint64_t offset, uint64_t length, std::optional balanced_read); static std::unique_ptr generate(uint64_t offset, uint64_t length); + static std::unique_ptr generate(uint64_t offset, + uint64_t length, bool balanced_read); }; class DoubleReadOp : public ReadWriteOp { public: DoubleReadOp(uint64_t offset1, uint64_t length1, uint64_t offset2, - uint64_t length2); + uint64_t length2, std::optional balanced_read); static std::unique_ptr generate(uint64_t offset1, uint64_t length1, uint64_t offset2, uint64_t length2); + static std::unique_ptr generate(uint64_t offset1, + uint64_t length1, + uint64_t offset2, + uint64_t length2, + bool balanced_read); }; class TripleReadOp : public ReadWriteOp { public: TripleReadOp(uint64_t offset1, uint64_t length1, uint64_t offset2, - uint64_t length2, uint64_t offset3, uint64_t length3); + uint64_t length2, uint64_t offset3, uint64_t length3, + std::optional balanced_read); static std::unique_ptr generate( uint64_t offset1, uint64_t length1, uint64_t offset2, uint64_t length2, uint64_t offset3, uint64_t length3); + static std::unique_ptr generate( + uint64_t offset1, uint64_t length1, uint64_t offset2, uint64_t length2, + uint64_t offset3, uint64_t length3, bool balanced_read); }; class SingleWriteOp : public ReadWriteOp { diff --git a/src/common/io_exerciser/RadosIo.cc b/src/common/io_exerciser/RadosIo.cc index 6881a4ccc3c..f98348ce3e8 100644 --- a/src/common/io_exerciser/RadosIo.cc +++ b/src/common/io_exerciser/RadosIo.cc @@ -49,7 +49,8 @@ RadosIo::RadosIo(librados::Rados& rados, boost::asio::io_context& asio, const std::string& pool, const std::string& primary_oid, const std::string& secondary_oid, uint64_t block_size, int seed, int threads, ceph::mutex& lock, ceph::condition_variable& cond, bool is_replicated_pool, - bool ec_optimizations, GenerationType data_generation_type, + bool ec_optimizations, int balanced_read_percentage, + GenerationType data_generation_type, std::shared_ptr seq, bool delete_objects) : Model(primary_oid, secondary_oid, block_size, delete_objects), rados(rados), @@ -62,7 +63,9 @@ RadosIo::RadosIo(librados::Rados& rados, boost::asio::io_context& asio, lock(lock), cond(cond), outstanding_io(0), - seq(seq) { + seq(seq), + rng(seed), + balanced_read_percentage(balanced_read_percentage) { int rc; rc = rados.ioctx_create(pool.c_str(), io); ceph_assert(rc == 0); @@ -271,8 +274,25 @@ void RadosIo::applyReadWriteOp(IoOp& op) { } finish_io(); }; + + int flags = 0; + if (readOp.balanced_read.has_value()) { + if (*readOp.balanced_read) { + flags = librados::OPERATION_BALANCE_READS; + } + // Else: keep flags == 0 + } else { + ceph_assert(balanced_read_percentage >= 0); + ceph_assert(balanced_read_percentage <= 100); + uint64_t range = 100; + uint64_t rand_value = rng(); + int index = rand_value % range; + if (index <= balanced_read_percentage) { + flags = librados::OPERATION_BALANCE_READS; + } + } librados::async_operate(asio.get_executor(), io, primary_oid, - std::move(rop), 0, nullptr, read_cb); + std::move(rop), flags, nullptr, read_cb); num_io++; }; diff --git a/src/common/io_exerciser/RadosIo.h b/src/common/io_exerciser/RadosIo.h index 5ed082b593d..d79847dfa86 100644 --- a/src/common/io_exerciser/RadosIo.h +++ b/src/common/io_exerciser/RadosIo.h @@ -6,6 +6,8 @@ #include "librados/AioCompletionImpl.h" #include "common/ceph_mutex.h" +#include + namespace boost::asio { class io_context; } /* Overview @@ -40,6 +42,8 @@ class RadosIo : public Model { librados::IoCtx io; int outstanding_io; std::shared_ptr seq; + std::mt19937_64 rng; + int balanced_read_percentage; void start_io(); void finish_io(); @@ -50,7 +54,8 @@ class RadosIo : public Model { const std::string& pool, const std::string& primary_oid, const std::string& secondary_oid, uint64_t block_size, int seed, int threads, ceph::mutex& lock, ceph::condition_variable& cond, bool is_replicated_pool, - bool ec_optimizations, ceph::io_exerciser::data_generation::GenerationType data_generation_type, + bool ec_optimizations, int balanced_read_percentage, + ceph::io_exerciser::data_generation::GenerationType data_generation_type, std::shared_ptr seq = nullptr, bool delete_objects = true); ~RadosIo(); diff --git a/src/test/osd/ceph_test_rados_io_sequence/ceph_test_rados_io_sequence.cc b/src/test/osd/ceph_test_rados_io_sequence/ceph_test_rados_io_sequence.cc index d7540919f3f..0297312c656 100644 --- a/src/test/osd/ceph_test_rados_io_sequence/ceph_test_rados_io_sequence.cc +++ b/src/test/osd/ceph_test_rados_io_sequence/ceph_test_rados_io_sequence.cc @@ -166,9 +166,12 @@ constexpr std::string_view usage[] = { "\t\t remove", "\t\t swap", "\t\t copy", - "\t\t read|write|failedwrite ", - "\t\t read2|write2|failedwrite2 ", - "\t\t read3|write3|failedwrite3 ", + "\t\t read [balanced]", + "\t\t write|failedwrite ", + "\t\t read2 [balanced]", + "\t\t write2|failedwrite2 ", + "\t\t read3 [balanced]", + "\t\t write3|failedwrite3 ", "\t\t append", "\t\t truncate", "\t\t injecterror ", @@ -225,6 +228,9 @@ po::options_description get_options_description() { "dont_delete_objects", "Stops the IO exerciser from deleting the object it was running the test " "against once the test finishes. Does not affect interactive mode")( + "balanced_read_percentage", po::value(), + "The percentage of read operations that should be performed with " + "balanced reads enabled. 100 by default. Doesn't affect interactive mode.")( "data_generation_type", po::value(), "Data generation type to use for write IOs. Default is HeaderedSeededRandom"); @@ -1025,6 +1031,8 @@ void ceph::io_sequence::tester::SelectErasurePool::configureServices( "the specified plugin type may not " "support them"); } + + ceph_assert(rc == 0); } if (allow_pool_ec_overwrites) { @@ -1060,10 +1068,10 @@ ceph::io_sequence::tester::TestObject::TestObject( SelectObjectSize& sos, SelectNumThreads& snt, SelectSeqRange& ssr, std::mt19937_64& rng, ceph::mutex& lock, ceph::condition_variable& cond, bool dryrun, bool verbose, - std::optional seqseed, bool testrecovery, bool checkconsistency, - bool delete_objects, GenerationType data_generation_type) - : rng(rng), verbose(verbose), seqseed(seqseed), testrecovery(testrecovery), - checkconsistency(checkconsistency), delete_objects(delete_objects), + std::optional seqseed, bool testrecovery, bool checkconsistency, bool delete_objects, + int balanced_read_percentage, GenerationType data_generation_type) + : rng(rng), verbose(verbose), seqseed(seqseed), testrecovery(testrecovery), checkconsistency(checkconsistency), + delete_objects(delete_objects), balanced_read_percentage(balanced_read_percentage), data_generation_type(data_generation_type) { if (dryrun) { int model_seed = rng(); @@ -1090,7 +1098,8 @@ ceph::io_sequence::tester::TestObject::TestObject( exerciser_model = std::make_unique( rados, asio, pool, primary_oid, secondary_oid, sbs.select(), model_seed, threads, lock, cond, spo.is_replicated_pool(), - spo.get_allow_pool_ec_optimizations(), data_generation_type, seq, delete_objects); + spo.get_allow_pool_ec_optimizations(), balanced_read_percentage, + data_generation_type, seq, delete_objects); dout(0) << "= " << primary_oid << " pool=" << pool << " threads=" << threads << " blocksize=" << exerciser_model->get_block_size() << " =" << dendl; @@ -1211,6 +1220,21 @@ ceph::io_sequence::tester::TestRunner::TestRunner( allow_pool_deep_scrubbing = vm.contains("allow_pool_deep_scrubbing"); allow_pool_scrubbing = vm.contains("allow_pool_scrubbing"); + if (vm.contains("balanced_read_percentage")) { + balanced_read_percentage = vm["balanced_read_percentage"].as(); + if (balanced_read_percentage > 100) { + balanced_read_percentage = 100; + } else if (balanced_read_percentage < 0) { + balanced_read_percentage = 0; + } + if (interactive) { + dout(0) << "Balanced read percentage cannot be specified in interactive mode. " + << "Use the \"-b\" or \"balanced\" flag to make a balanced read operation in the interactive terminal." << dendl; + } + } else { + balanced_read_percentage = 100; + } + if (testrecovery && (num_object_pairs > 1)) { throw std::invalid_argument("testrecovery option not allowed if parallel is" " specified, except when parallel=1 is used"); @@ -1362,7 +1386,7 @@ bool ceph::io_sequence::tester::TestRunner::run_interactive_test() { rados, asio, pool, primary_object_name, secondary_object_name, sbs.select(), model_seed, 1, // 1 thread lock, cond, spo.is_replicated_pool(), - spo.get_allow_pool_ec_optimizations(), + spo.get_allow_pool_ec_optimizations(), balanced_read_percentage, data_generation_type); } @@ -1385,13 +1409,23 @@ bool ceph::io_sequence::tester::TestRunner::run_interactive_test() { } else if (op == "read") { uint64_t offset = get_numeric_token(); uint64_t length = get_numeric_token(); - ioop = ceph::io_exerciser::SingleReadOp::generate(offset, length); + std::optional token = get_optional_token(); + if (token.has_value() && (*token == "-b" || *token == "balanced")) { + ioop = ceph::io_exerciser::SingleReadOp::generate(offset, length, true); + } else { + ioop = ceph::io_exerciser::SingleReadOp::generate(offset, length, false); + } } else if (op == "read2") { uint64_t offset1 = get_numeric_token(); uint64_t length1 = get_numeric_token(); uint64_t offset2 = get_numeric_token(); uint64_t length2 = get_numeric_token(); - ioop = DoubleReadOp::generate(offset1, length1, offset2, length2); + std::optional token = get_optional_token(); + if (token.has_value() && (*token == "-b" || *token == "balanced")) { + ioop = DoubleReadOp::generate(offset1, length1, offset2, length2, true); + } else { + ioop = DoubleReadOp::generate(offset1, length1, offset2, length2, false); + } } else if (op == "read3") { uint64_t offset1 = get_numeric_token(); uint64_t length1 = get_numeric_token(); @@ -1399,8 +1433,14 @@ bool ceph::io_sequence::tester::TestRunner::run_interactive_test() { uint64_t length2 = get_numeric_token(); uint64_t offset3 = get_numeric_token(); uint64_t length3 = get_numeric_token(); - ioop = TripleReadOp::generate(offset1, length1, offset2, length2, offset3, - length3); + std::optional token = get_optional_token(); + if (token.has_value() && (*token == "-b" || *token == "balanced")) { + ioop = TripleReadOp::generate(offset1, length1, offset2, length2, offset3, + length3, true); + } else { + ioop = TripleReadOp::generate(offset1, length1, offset2, length2, offset3, + length3, false); + } } else if (op == "write") { uint64_t offset = get_numeric_token(); uint64_t length = get_numeric_token(); @@ -1519,8 +1559,8 @@ bool ceph::io_sequence::tester::TestRunner::run_automated_test() { test_objects.push_back( std::make_shared( primary_name, secondary_name, rados, asio, sbs, spo, sos, snt, ssr, rng, lock, cond, - dryrun, verbose, seqseed, testrecovery, checkconsistency, - delete_objects, data_generation_type)); + dryrun, verbose, seqseed, testrecovery, checkconsistency, delete_objects, + balanced_read_percentage, data_generation_type)); } catch (const std::runtime_error &e) { std::cerr << "Error: " << e.what() << std::endl; diff --git a/src/test/osd/ceph_test_rados_io_sequence/ceph_test_rados_io_sequence.h b/src/test/osd/ceph_test_rados_io_sequence/ceph_test_rados_io_sequence.h index e592d56fc1a..f4adf0a9174 100644 --- a/src/test/osd/ceph_test_rados_io_sequence/ceph_test_rados_io_sequence.h +++ b/src/test/osd/ceph_test_rados_io_sequence/ceph_test_rados_io_sequence.h @@ -466,6 +466,7 @@ class TestObject { bool testRecovery, bool checkConsistency, bool delete_objects, + int balanced_read_percentage, GenerationType data_generation_type); int get_num_io(); @@ -491,6 +492,7 @@ class TestObject { bool testrecovery; bool checkconsistency; bool delete_objects; + int balanced_read_percentage; GenerationType data_generation_type; }; @@ -551,6 +553,8 @@ class TestRunner { ceph::split split = ceph::split(""); ceph::spliterator tokens; + int balanced_read_percentage; + void clear_tokens(); std::string get_token(bool allow_eof = false); std::optional get_optional_token();