From 347ba63e83f77c6ba870a246b5c3836d3cd331d6 Mon Sep 17 00:00:00 2001 From: JonBailey1993 Date: Fri, 11 Oct 2024 13:44:17 +0100 Subject: [PATCH] common/io_exerciser: Add injecterror commands to ceph_test_rados_io_sequence interactive mode Add injecterror commands that can be used in interactive mode to inject read and write errors as well as clear them Signed-off-by: Jon Bailey --- src/common/io_exerciser/CMakeLists.txt | 1 + src/common/io_exerciser/IoOp.cc | 174 +++++++- src/common/io_exerciser/IoOp.h | 132 +++++- src/common/io_exerciser/JsonStructures.cc | 425 ++++++++++++++++++++ src/common/io_exerciser/JsonStructures.h | 264 ++++++++++++ src/common/io_exerciser/ObjectModel.cc | 40 ++ src/common/io_exerciser/OpType.h | 46 ++- src/common/io_exerciser/RadosIo.cc | 365 ++++++++++++----- src/common/io_exerciser/RadosIo.h | 6 + src/test/osd/ceph_test_rados_io_sequence.cc | 409 +++++++++++++++---- src/test/osd/ceph_test_rados_io_sequence.h | 34 +- 11 files changed, 1721 insertions(+), 175 deletions(-) create mode 100644 src/common/io_exerciser/JsonStructures.cc create mode 100644 src/common/io_exerciser/JsonStructures.h diff --git a/src/common/io_exerciser/CMakeLists.txt b/src/common/io_exerciser/CMakeLists.txt index 07091df86e1..930cfec9ab5 100644 --- a/src/common/io_exerciser/CMakeLists.txt +++ b/src/common/io_exerciser/CMakeLists.txt @@ -5,6 +5,7 @@ add_library(object_io_exerciser STATIC Model.cc ObjectModel.cc RadosIo.cc + JsonStructures.cc ) target_link_libraries(object_io_exerciser diff --git a/src/common/io_exerciser/IoOp.cc b/src/common/io_exerciser/IoOp.cc index 2a5a60e4a24..22773a00960 100644 --- a/src/common/io_exerciser/IoOp.cc +++ b/src/common/io_exerciser/IoOp.cc @@ -2,6 +2,8 @@ #include "fmt/format.h" +#include "include/ceph_assert.h" + using IoOp = ceph::io_exerciser::IoOp; using OpType = ceph::io_exerciser::OpType; @@ -15,6 +17,9 @@ using TripleReadOp = ceph::io_exerciser::TripleReadOp; using SingleWriteOp = ceph::io_exerciser::SingleWriteOp; using DoubleWriteOp = ceph::io_exerciser::DoubleWriteOp; using TripleWriteOp = ceph::io_exerciser::TripleWriteOp; +using SingleFailedWriteOp = ceph::io_exerciser::SingleFailedWriteOp; +using DoubleFailedWriteOp = ceph::io_exerciser::DoubleFailedWriteOp; +using TripleFailedWriteOp = ceph::io_exerciser::TripleFailedWriteOp; namespace { @@ -164,17 +169,23 @@ std::string ceph::io_exerciser::ReadWriteOp switch(opType) { case OpType::Read: - [[fallthrough]]; + [[ fallthrough ]]; case OpType::Read2: - [[fallthrough]]; + [[ fallthrough ]]; case OpType::Read3: return fmt::format("Read{} ({})", numIOs, offset_length_desc); case OpType::Write: - [[fallthrough]]; + [[ fallthrough ]]; case OpType::Write2: - [[fallthrough]]; + [[ fallthrough ]]; case OpType::Write3: return fmt::format("Write{} ({})", numIOs, offset_length_desc); + case OpType::FailedWrite: + [[ fallthrough ]]; + case OpType::FailedWrite2: + [[ fallthrough ]]; + case OpType::FailedWrite3: + return fmt::format("FailedWrite{} ({})", numIOs, offset_length_desc); default: ceph_abort_msg(fmt::format("Unsupported op type by ReadWriteOp ({})", opType)); } @@ -262,4 +273,159 @@ std::unique_ptr TripleWriteOp::generate(uint64_t offset1, uint64_ return std::make_unique(offset1, length1, offset2, length2, offset3, length3); +} + +SingleFailedWriteOp::SingleFailedWriteOp(uint64_t offset, uint64_t length) : + ReadWriteOp({offset}, {length}) +{ + +} + +std::unique_ptr SingleFailedWriteOp::generate(uint64_t offset, + uint64_t length) +{ + return std::make_unique(offset, length); +} + +DoubleFailedWriteOp::DoubleFailedWriteOp(uint64_t offset1, uint64_t length1, + uint64_t offset2, uint64_t length2) : + ReadWriteOp({offset1, offset2}, {length1, length2}) +{ + +} + +std::unique_ptr DoubleFailedWriteOp::generate(uint64_t offset1, + uint64_t length1, + uint64_t offset2, + uint64_t length2) +{ + return std::make_unique(offset1, length1, offset2, length2); +} + +TripleFailedWriteOp::TripleFailedWriteOp(uint64_t offset1, uint64_t length1, + uint64_t offset2, uint64_t length2, + uint64_t offset3, uint64_t length3) : + ReadWriteOp({offset1, offset2, offset3}, + {length1, length2, length3}) +{ + +} + +std::unique_ptr TripleFailedWriteOp::generate(uint64_t offset1, + uint64_t length1, + uint64_t offset2, + uint64_t length2, + uint64_t offset3, + uint64_t length3) +{ + return std::make_unique(offset1, length1, + offset2, length2, + offset3, length3); +} + +template +ceph::io_exerciser::InjectErrorOp + ::InjectErrorOp(int shard, + const std::optional& type, + const std::optional& when, + const std::optional& duration) : + TestOp(), + shard(shard), + type(type), + when(when), + duration(duration) +{ + +} + +template +std::string ceph::io_exerciser::InjectErrorOp::to_string(uint64_t blocksize) const +{ + std::string_view inject_type = get_inject_type_string(); + return fmt::format("Inject {} error on shard {} of type {}" + " after {} successful inject(s) lasting {} inject(s)", + inject_type, shard, type.value_or(0), + when.value_or(0), duration.value_or(1)); +} + +ceph::io_exerciser::InjectReadErrorOp::InjectReadErrorOp(int shard, + const std::optional& type, + const std::optional& when, + const std::optional& duration) : + InjectErrorOp(shard, type, when, duration) +{ + +} + +std::unique_ptr ceph::io_exerciser + ::InjectReadErrorOp::generate(int shard, + const std::optional& type, + const std::optional& when, + const std::optional& duration) +{ + return std::make_unique(shard, type, when, duration); +} + +ceph::io_exerciser::InjectWriteErrorOp::InjectWriteErrorOp(int shard, + const std::optional& type, + const std::optional& when, + const std::optional& duration) : + InjectErrorOp(shard, type, when, duration) +{ + +} + +std::unique_ptr ceph::io_exerciser + ::InjectWriteErrorOp::generate(int shard, + const std::optional& type, + const std::optional& when, + const std::optional& duration) +{ + return std::make_unique(shard, type, when, duration); +} + + + +template +ceph::io_exerciser::ClearErrorInjectOp + ::ClearErrorInjectOp(int shard, const std::optional& type) : + TestOp(), + shard(shard), + type(type) +{ + +} + +template +std::string ceph::io_exerciser::ClearErrorInjectOp::to_string(uint64_t blocksize) const +{ + std::string_view inject_type = get_inject_type_string(); + return fmt::format("Clear {} injects on shard {} of type {}", + inject_type, shard, type.value_or(0)); +} + +ceph::io_exerciser::ClearReadErrorInjectOp::ClearReadErrorInjectOp(int shard, + const std::optional& type) : + ClearErrorInjectOp(shard, type) +{ + +} + +std::unique_ptr ceph::io_exerciser + ::ClearReadErrorInjectOp::generate(int shard, const std::optional& type) +{ + return std::make_unique(shard, type); +} + +ceph::io_exerciser::ClearWriteErrorInjectOp::ClearWriteErrorInjectOp(int shard, + const std::optional& type) : + ClearErrorInjectOp(shard, type) +{ + +} + +std::unique_ptr ceph::io_exerciser + ::ClearWriteErrorInjectOp::generate(int shard, const std::optional& type) +{ + return std::make_unique(shard, type); } \ No newline at end of file diff --git a/src/common/io_exerciser/IoOp.h b/src/common/io_exerciser/IoOp.h index 2f9f169d954..45b4ab64678 100644 --- a/src/common/io_exerciser/IoOp.h +++ b/src/common/io_exerciser/IoOp.h @@ -1,9 +1,9 @@ #pragma once #include +#include #include #include -#include "include/ceph_assert.h" #include "OpType.h" @@ -145,5 +145,135 @@ namespace ceph { uint64_t offset3, uint64_t length3); }; + + class SingleFailedWriteOp : public ReadWriteOp + { + public: + SingleFailedWriteOp(uint64_t offset, uint64_t length); + static std::unique_ptr generate(uint64_t offset, + uint64_t length); + }; + + class DoubleFailedWriteOp : public ReadWriteOp + { + public: + DoubleFailedWriteOp(uint64_t offset1, uint64_t length1, + uint64_t offset2, uint64_t length2); + static std::unique_ptr generate(uint64_t offset1, + uint64_t length1, + uint64_t offset2, + uint64_t length2); + }; + + class TripleFailedWriteOp : public ReadWriteOp + { + public: + TripleFailedWriteOp(uint64_t offset1, uint64_t length1, + uint64_t offset2, uint64_t length2, + uint64_t offset3, uint64_t length3); + static std::unique_ptr generate(uint64_t offset1, + uint64_t length1, + uint64_t offset2, + uint64_t length2, + uint64_t offset3, + uint64_t length3); + }; + + template + class InjectErrorOp : public TestOp + { + public: + InjectErrorOp(int shard, + const std::optional& type, + const std::optional& when, + const std::optional& duration); + + std::string to_string(uint64_t block_size) const override; + + int shard; + std::optional type; + std::optional when; + std::optional duration; + + protected: + virtual inline constexpr std::string_view get_inject_type_string() const = 0; + }; + + class InjectReadErrorOp : public InjectErrorOp + { + public: + InjectReadErrorOp(int shard, + const std::optional& type, + const std::optional& when, + const std::optional& duration); + + static std::unique_ptr generate(int shard, + const std::optional& type, + const std::optional& when, + const std::optional& duration); + + protected: + inline constexpr std::string_view get_inject_type_string() const override + { return "read"; } + }; + + class InjectWriteErrorOp : public InjectErrorOp + { + public: + InjectWriteErrorOp(int shard, + const std::optional& type, + const std::optional& when, + const std::optional& duration); + + static std::unique_ptr generate(int shard, + const std::optional& type, + const std::optional& when, + const std::optional& duration); + + protected: + inline constexpr std::string_view get_inject_type_string() const override + { return "write"; } + }; + + template + class ClearErrorInjectOp : public TestOp + { + public: + ClearErrorInjectOp(int shard, const std::optional& type); + + std::string to_string(uint64_t block_size) const override; + + int shard; + std::optional type; + + protected: + virtual inline constexpr std::string_view get_inject_type_string() const = 0; + }; + + class ClearReadErrorInjectOp : public ClearErrorInjectOp + { + public: + ClearReadErrorInjectOp(int shard, const std::optional& type); + + static std::unique_ptr generate(int shard, + const std::optional& type); + + protected: + inline constexpr std::string_view get_inject_type_string() const override + { return "read"; } + }; + + class ClearWriteErrorInjectOp : public ClearErrorInjectOp + { + public: + ClearWriteErrorInjectOp(int shard, const std::optional& type); + + static std::unique_ptr generate(int shard, + const std::optional& type); + + protected: + inline constexpr std::string_view get_inject_type_string() const override + { return "write"; } + }; } } \ No newline at end of file diff --git a/src/common/io_exerciser/JsonStructures.cc b/src/common/io_exerciser/JsonStructures.cc new file mode 100644 index 00000000000..12d792a0cbb --- /dev/null +++ b/src/common/io_exerciser/JsonStructures.cc @@ -0,0 +1,425 @@ +#include "JsonStructures.h" + +#include "common/ceph_json.h" + +using namespace ceph::io_exerciser::json; + +JSONStructure::JSONStructure(std::shared_ptr formatter) : + formatter(formatter) +{ + +} + +std::string JSONStructure::encode_json() +{ + oss.clear(); + + dump(); + formatter->flush(oss); + return oss.str(); +} + +OSDMapRequest::OSDMapRequest(const std::string& pool_name, + const std::string& object, + const std::string& nspace, + std::shared_ptr formatter) : + JSONStructure(formatter), + pool(pool_name), + object(object), + nspace(nspace) +{ + +} + +OSDMapRequest::OSDMapRequest(std::shared_ptr formatter) : + JSONStructure(formatter) +{ + +} + +void OSDMapRequest::decode_json(JSONObj* obj) +{ + JSONDecoder::decode_json("prefix", prefix, obj); + JSONDecoder::decode_json("pool", pool, obj); + JSONDecoder::decode_json("object", object, obj); + JSONDecoder::decode_json("nspace", nspace, obj); + JSONDecoder::decode_json("format", format, obj); +} + +void OSDMapRequest::dump() const +{ + formatter->open_object_section("OSDMapRequest"); + ::encode_json("prefix", prefix, formatter.get()); + ::encode_json("pool", pool, formatter.get()); + ::encode_json("object", object, formatter.get()); + ::encode_json("nspace", nspace, formatter.get()); + ::encode_json("format", format, formatter.get()); + formatter->close_section(); +} + +OSDMapReply::OSDMapReply(std::shared_ptr formatter) : + JSONStructure(formatter) +{ + +} + +void OSDMapReply::decode_json(JSONObj* obj) +{ + JSONDecoder::decode_json("epoch", epoch, obj); + JSONDecoder::decode_json("pool", pool, obj); + JSONDecoder::decode_json("pool_id", pool_id, obj); + JSONDecoder::decode_json("objname", objname, obj); + JSONDecoder::decode_json("raw_pgid", raw_pgid, obj); + JSONDecoder::decode_json("pgid", pgid, obj); + JSONDecoder::decode_json("up", up, obj); + JSONDecoder::decode_json("up_primary", up_primary, obj); + JSONDecoder::decode_json("acting", acting, obj); + JSONDecoder::decode_json("acting_primary", acting_primary, obj); +} + +void OSDMapReply::dump() const +{ + formatter->open_object_section("OSDMapReply"); + ::encode_json("epoch", epoch, formatter.get()); + ::encode_json("pool", pool, formatter.get()); + ::encode_json("pool_id", pool_id, formatter.get()); + ::encode_json("objname", objname, formatter.get()); + ::encode_json("raw_pgid", raw_pgid, formatter.get()); + ::encode_json("pgid", pgid, formatter.get()); + ::encode_json("up", up, formatter.get()); + ::encode_json("up_primary", up_primary, formatter.get()); + ::encode_json("acting", acting, formatter.get()); + ::encode_json("acting_primary", acting_primary, formatter.get()); + formatter->close_section(); +} + +OSDECProfileSetRequest::OSDECProfileSetRequest(const std::string& name, + std::vector profile, + std::shared_ptr formatter) : + JSONStructure(formatter), + name(name), + profile(profile) +{ + +} + +OSDECProfileSetRequest + ::OSDECProfileSetRequest(std::shared_ptr formatter) : + JSONStructure(formatter) +{ + +} + +void OSDECProfileSetRequest::decode_json(JSONObj* obj) +{ + JSONDecoder::decode_json("prefix", prefix, obj); + JSONDecoder::decode_json("name", name, obj); + JSONDecoder::decode_json("profile", profile, obj); +} + +void OSDECProfileSetRequest::dump() const +{ + formatter->open_object_section("OSDECProfileSetRequest"); + ::encode_json("prefix", prefix, formatter.get()); + ::encode_json("name", name, formatter.get()); + ::encode_json("profile", profile, formatter.get()); + formatter->close_section(); +} + +OSDECPoolCreateRequest::OSDECPoolCreateRequest(const std::string& pool, + const std::string& erasure_code_profile, + std::shared_ptr formatter) : + JSONStructure(formatter), + pool(pool), + erasure_code_profile(erasure_code_profile) +{ + +} + +OSDECPoolCreateRequest + ::OSDECPoolCreateRequest(std::shared_ptr formatter) : + JSONStructure(formatter) +{ + +} + +void OSDECPoolCreateRequest::decode_json(JSONObj* obj) +{ + JSONDecoder::decode_json("prefix", prefix, obj); + JSONDecoder::decode_json("pool", pool, obj); + JSONDecoder::decode_json("pool_type", pool_type, obj); + JSONDecoder::decode_json("pg_num", pg_num, obj); + JSONDecoder::decode_json("pgp_num", pgp_num, obj); + JSONDecoder::decode_json("erasure_code_profile", erasure_code_profile, obj); +} + +void OSDECPoolCreateRequest::dump() const +{ + formatter->open_object_section("OSDECPoolCreateRequest"); + ::encode_json("prefix", prefix, formatter.get()); + ::encode_json("pool", pool, formatter.get()); + ::encode_json("pool_type", pool_type, formatter.get()); + ::encode_json("pg_num", pg_num, formatter.get()); + ::encode_json("pgp_num", pgp_num, formatter.get()); + ::encode_json("erasure_code_profile", erasure_code_profile, formatter.get()); + formatter->close_section(); +} + +OSDSetRequest::OSDSetRequest(const std::string& key, + const std::optional& yes_i_really_mean_it, + std::shared_ptr formatter) : + JSONStructure(formatter), + key(key), + yes_i_really_mean_it(yes_i_really_mean_it) +{ + +} + +OSDSetRequest::OSDSetRequest(std::shared_ptr formatter) : + JSONStructure(formatter) +{ + +} + +void OSDSetRequest::decode_json(JSONObj* obj) +{ + JSONDecoder::decode_json("prefix", prefix, obj); + JSONDecoder::decode_json("key", key, obj); + JSONDecoder::decode_json("yes_i_really_mean_it", yes_i_really_mean_it, obj); +} + +void OSDSetRequest::dump() const +{ + formatter->open_object_section("OSDSetRequest"); + ::encode_json("prefix", prefix, formatter.get()); + ::encode_json("key", key, formatter.get()); + ::encode_json("yes_i_really_mean_it", yes_i_really_mean_it, formatter.get()); + formatter->close_section(); +} + +BalancerOffRequest::BalancerOffRequest(std::shared_ptr formatter) : + JSONStructure(formatter) +{ + +} + +void BalancerOffRequest::decode_json(JSONObj* obj) +{ + JSONDecoder::decode_json("prefix", prefix, obj); +} + +void BalancerOffRequest::dump() const +{ + formatter->open_object_section("BalancerOffRequest"); + ::encode_json("prefix", prefix, formatter.get()); + formatter->close_section(); +} + +BalancerStatusRequest + ::BalancerStatusRequest(std::shared_ptr formatter) : + JSONStructure(formatter) +{ + +} + +void BalancerStatusRequest::decode_json(JSONObj* obj) +{ + JSONDecoder::decode_json("prefix", prefix, obj); +} + +void BalancerStatusRequest::dump() const +{ + formatter->open_object_section("BalancerStatusRequest"); + ::encode_json("prefix", prefix, formatter.get()); + formatter->close_section(); +} + +BalancerStatusReply::BalancerStatusReply(std::shared_ptr formatter) : + JSONStructure(formatter) +{ + +} + +void BalancerStatusReply::decode_json(JSONObj* obj) +{ + JSONDecoder::decode_json("active", active, obj); + JSONDecoder::decode_json("last_optimization_duration", + last_optimization_duration, obj); + JSONDecoder::decode_json("last_optimization_started", + last_optimization_started, obj); + JSONDecoder::decode_json("mode", mode, obj); + JSONDecoder::decode_json("no_optimization_needed", + no_optimization_needed, obj); + JSONDecoder::decode_json("optimize_result", optimize_result, obj); + +} + +void BalancerStatusReply::dump() const +{ + formatter->open_object_section("BalancerStatusReply"); + ::encode_json("active", active, formatter.get()); + ::encode_json("last_optimization_duration", + last_optimization_duration, + formatter.get()); + ::encode_json("last_optimization_started", + last_optimization_started, + formatter.get()); + ::encode_json("mode", mode, formatter.get()); + ::encode_json("no_optimization_needed", + no_optimization_needed, + formatter.get()); + ::encode_json("optimize_result", + optimize_result, + formatter.get()); + formatter->close_section(); +} + +ConfigSetRequest::ConfigSetRequest(const std::string& who, + const std::string& name, + const std::string& value, + const std::optional& force, + std::shared_ptr formatter) : + JSONStructure(formatter), + who(who), + name(name), + value(value), + force(force) +{ + +} + +ConfigSetRequest::ConfigSetRequest(std::shared_ptr formatter) : + JSONStructure(formatter) +{ + +} + +void ConfigSetRequest::decode_json(JSONObj* obj) +{ + JSONDecoder::decode_json("prefix", prefix, obj); + JSONDecoder::decode_json("who", who, obj); + JSONDecoder::decode_json("name", name, obj); + JSONDecoder::decode_json("value", value, obj); + JSONDecoder::decode_json("force", force, obj); +} + +void ConfigSetRequest::dump() const +{ + formatter->open_object_section("ConfigSetRequest"); + ::encode_json("prefix", prefix, formatter.get()); + ::encode_json("who", who, formatter.get()); + ::encode_json("name", name, formatter.get()); + ::encode_json("value", value, formatter.get()); + ::encode_json("force", force, formatter.get()); + formatter->close_section(); +} + +InjectECErrorRequest::InjectECErrorRequest(InjectOpType injectOpType, + const std::string& pool, + const std::string& objname, + int shardid, + const std::optional& type, + const std::optional& when, + const std::optional& duration, + std::shared_ptr formatter) : + JSONStructure(formatter), + pool(pool), + objname(objname), + shardid(shardid), + type(type), + when(when), + duration(duration) +{ + switch(injectOpType) + { + case InjectOpType::Read: + prefix = "injectecreaderr"; + break; + case InjectOpType::Write: + prefix = "injectecwriteerr"; + break; + } +} + +InjectECErrorRequest + ::InjectECErrorRequest(std::shared_ptr formatter) : + JSONStructure(formatter) +{ + +} + +void InjectECErrorRequest::dump() const +{ + formatter->open_object_section("InjectECErrorRequest"); + ::encode_json("prefix", prefix, formatter.get()); + ::encode_json("pool", pool, formatter.get()); + ::encode_json("objname", objname, formatter.get()); + ::encode_json("shardid", shardid, formatter.get()); + ::encode_json("type", type, formatter.get()); + ::encode_json("when", when, formatter.get()); + ::encode_json("duration", duration, formatter.get()); + formatter->close_section(); +} + +void InjectECErrorRequest::decode_json(JSONObj* obj) +{ + JSONDecoder::decode_json("prefix", prefix, obj); + JSONDecoder::decode_json("pool", pool, obj); + JSONDecoder::decode_json("objname", objname, obj); + JSONDecoder::decode_json("shardid", shardid, obj); + JSONDecoder::decode_json("type", type, obj); + JSONDecoder::decode_json("when", when, obj); + JSONDecoder::decode_json("duration", duration, obj); +} + + + +InjectECClearErrorRequest::InjectECClearErrorRequest(InjectOpType injectOpType, + const std::string& pool, + const std::string& objname, + int shardid, + const std::optional& type, + std::shared_ptr formatter) : + JSONStructure(formatter), + pool(pool), + objname(objname), + shardid(shardid), + type(type) +{ + switch(injectOpType) + { + case InjectOpType::Read: + prefix = "injectecclearreaderr"; + break; + case InjectOpType::Write: + prefix = "injectecclearwriteerr"; + break; + } +} + +InjectECClearErrorRequest + ::InjectECClearErrorRequest(std::shared_ptr formatter) : + JSONStructure(formatter) +{ + +} + +void InjectECClearErrorRequest::dump() const +{ + formatter->open_object_section("InjectECErrorRequest"); + ::encode_json("prefix", prefix, formatter.get()); + ::encode_json("pool", pool, formatter.get()); + ::encode_json("objname", objname, formatter.get()); + ::encode_json("shardid", shardid, formatter.get()); + ::encode_json("type", type, formatter.get()); + formatter->close_section(); +} + +void InjectECClearErrorRequest::decode_json(JSONObj* obj) +{ + JSONDecoder::decode_json("prefix", prefix, obj); + JSONDecoder::decode_json("pool", pool, obj); + JSONDecoder::decode_json("objname", objname, obj); + JSONDecoder::decode_json("shardid", shardid, obj); + JSONDecoder::decode_json("type", type, obj); +} \ No newline at end of file diff --git a/src/common/io_exerciser/JsonStructures.h b/src/common/io_exerciser/JsonStructures.h new file mode 100644 index 00000000000..7e1b295919d --- /dev/null +++ b/src/common/io_exerciser/JsonStructures.h @@ -0,0 +1,264 @@ +#include +#include +#include + +#include "include/types.h" + +/* Overview + * + * class JSONStructure + * Stores elements of a JSONStructure in C++ friendly format so they do not + * have to be parsed from strings. Includes encode and decode functions to + * provide easy ways to convert from c++ structures to json structures. + * + */ + +class JSONObj; + +namespace ceph +{ + namespace io_exerciser + { + namespace json + { + class JSONStructure + { + public: + JSONStructure(std::shared_ptr formatter + = std::make_shared(false)); + + virtual ~JSONStructure() = default; + + std::string encode_json(); + virtual void decode_json(JSONObj* obj)=0; + virtual void dump() const = 0; + + protected: + std::shared_ptr formatter; + + private: + std::ostringstream oss; + }; + + class OSDMapRequest : public JSONStructure + { + public: + OSDMapRequest(const std::string& pool_name, + const std::string& object, + const std::string& nspace, + std::shared_ptr formatter + = std::make_shared(false)); + OSDMapRequest(std::shared_ptr formatter + = std::make_shared(false)); + + std::string prefix = "osd map"; + std::string pool; + std::string object; + std::string nspace; + std::string format = "json"; + + void decode_json(JSONObj* obj) override; + void dump() const override; + }; + + class OSDMapReply : public JSONStructure + { + public: + OSDMapReply(std::shared_ptr formatter + = std::make_shared(false)); + + epoch_t epoch; + std::string pool; + uint64_t pool_id; + std::string objname; + std::string raw_pgid; + std::string pgid; + std::vector up; + int up_primary; + std::vector acting; + int acting_primary; + + void decode_json(JSONObj *obj); + void dump() const; + }; + + class OSDECProfileSetRequest : public JSONStructure + { + public: + OSDECProfileSetRequest(const std::string& name, + std::vector profile, + std::shared_ptr formatter + = std::make_shared(false)); + OSDECProfileSetRequest(std::shared_ptr formatter + = std::make_shared(false)); + + std::string prefix = "osd erasure-code-profile set"; + std::string name; + std::vector profile; + + void decode_json(JSONObj* obj) override; + void dump() const override; + }; + + class OSDECPoolCreateRequest : public JSONStructure + { + public: + OSDECPoolCreateRequest(const std::string& pool, + const std::string& erasure_code_profile, + std::shared_ptr formatter + = std::make_shared(false)); + OSDECPoolCreateRequest(std::shared_ptr formatter + = std::make_shared(false)); + + std::string prefix = "osd pool create"; + std::string pool; + std::string pool_type = "erasure"; + int pg_num = 8; + int pgp_num = 8; + std::string erasure_code_profile; + + void decode_json(JSONObj* obj) override; + void dump() const override; + }; + + class OSDSetRequest : public JSONStructure + { + public: + OSDSetRequest(const std::string& key, + const std::optional& yes_i_really_mean_it + = std::nullopt, + std::shared_ptr formatter + = std::make_shared(false)); + OSDSetRequest(std::shared_ptr formatter + = std::make_shared(false)); + + std::string prefix = "osd set"; + std::string key; + std::optional yes_i_really_mean_it = std::nullopt; + + void decode_json(JSONObj* obj) override; + void dump() const override; + }; + + class BalancerOffRequest : public JSONStructure + { + public: + BalancerOffRequest(std::shared_ptr formatter + = std::make_shared(false)); + + std::string prefix = "balancer off"; + + void decode_json(JSONObj* obj) override; + void dump() const override; + }; + + class BalancerStatusRequest : public JSONStructure + { + public: + BalancerStatusRequest(std::shared_ptr formatter + = std::make_shared(false)); + + std::string prefix = "balancer status"; + + void decode_json(JSONObj* obj) override; + void dump() const override; + }; + + class BalancerStatusReply : public JSONStructure + { + public: + BalancerStatusReply(std::shared_ptr formatter + = std::make_shared(false)); + + bool active; + std::string last_optimization_duration; + std::string last_optimization_started; + std::string mode; + bool no_optimization_needed; + std::string optimize_result; + + void decode_json(JSONObj* obj) override; + void dump() const override; + }; + + class ConfigSetRequest : public JSONStructure + { + public: + ConfigSetRequest(const std::string& who, + const std::string& name, + const std::string& value, + const std::optional& force = std::nullopt, + std::shared_ptr formatter + = std::make_shared(false)); + ConfigSetRequest(std::shared_ptr formatter + = std::make_shared(false)); + + std::string prefix = "config set"; + std::string who; + std::string name; + std::string value; + std::optional force; + + void decode_json(JSONObj* obj) override; + void dump() const override; + }; + + enum class InjectOpType + { + Read, + Write + }; + + class InjectECErrorRequest : public JSONStructure + { + public: + InjectECErrorRequest(InjectOpType injectOpType, + const std::string& pool, + const std::string& objname, + int shardid, + const std::optional& type, + const std::optional& when, + const std::optional& duration, + std::shared_ptr formatter + = std::make_shared(false)); + InjectECErrorRequest(std::shared_ptr formatter + = std::make_shared(false)); + + std::string prefix; + std::string pool; + std::string objname; + int shardid; + std::optional type; + std::optional when; + std::optional duration; + + void decode_json(JSONObj* obj) override; + void dump() const override; + }; + + class InjectECClearErrorRequest : public JSONStructure + { + public: + InjectECClearErrorRequest(InjectOpType injectOpType, + const std::string& pool, + const std::string& objname, + int shardid, + const std::optional& type, + std::shared_ptr formatter + = std::make_shared(false)); + + InjectECClearErrorRequest(std::shared_ptr formatter + = std::make_shared(false)); + + std::string prefix; + std::string pool; + std::string objname; + int shardid; + std::optional type; + + void decode_json(JSONObj* obj) override; + void dump() const override; + }; + } + } +} \ No newline at end of file diff --git a/src/common/io_exerciser/ObjectModel.cc b/src/common/io_exerciser/ObjectModel.cc index 350b8e5756b..4f2c295af72 100644 --- a/src/common/io_exerciser/ObjectModel.cc +++ b/src/common/io_exerciser/ObjectModel.cc @@ -104,6 +104,27 @@ void ObjectModel::applyIoOp(IoOp& op) num_io++; }; + auto verify_failed_write_and_record = [ &contents = contents, + &created = created, + &num_io = num_io, + &reads = reads, + &writes = writes ] + + (ReadWriteOp writeOp) + { + // Ensure write should still be valid, even though we are expecting OSD failure + ceph_assert(created); + for (int i = 0; i < N; i++) + { + // Not allowed: write overlapping with parallel read or write + ceph_assert(!reads.intersects(writeOp.offset[i], writeOp.length[i])); + ceph_assert(!writes.intersects(writeOp.offset[i], writeOp.length[i])); + writes.union_insert(writeOp.offset[i], writeOp.length[i]); + ceph_assert(writeOp.offset[i] + writeOp.length[i] <= contents.size()); + } + num_io++; + }; + switch (op.getOpType()) { case OpType::Barrier: reads.clear(); @@ -166,6 +187,25 @@ void ObjectModel::applyIoOp(IoOp& op) verify_write_and_record_and_generate_seed(writeOp); } break; + case OpType::FailedWrite: + { + ceph_assert(created); + SingleWriteOp& writeOp = static_cast(op); + verify_failed_write_and_record(writeOp); + } + break; + case OpType::FailedWrite2: + { + DoubleWriteOp& writeOp = static_cast(op); + verify_failed_write_and_record(writeOp); + } + break; + case OpType::FailedWrite3: + { + TripleWriteOp& writeOp = static_cast(op); + verify_failed_write_and_record(writeOp); + } + break; default: break; } diff --git a/src/common/io_exerciser/OpType.h b/src/common/io_exerciser/OpType.h index 408645f6c25..737973e0455 100644 --- a/src/common/io_exerciser/OpType.h +++ b/src/common/io_exerciser/OpType.h @@ -2,6 +2,8 @@ #include +#include + /* Overview * * enum OpType @@ -15,16 +17,23 @@ namespace ceph { enum class OpType { - Done, // End of I/O sequence - Barrier, // Barrier - all prior I/Os must complete - Create, // Create object and pattern with data - Remove, // Remove object - Read, // Read - Read2, // Two reads in a single op - Read3, // Three reads in a single op - Write, // Write - Write2, // Two writes in a single op - Write3 // Three writes in a single op + Done, // End of I/O sequence + Barrier, // Barrier - all prior I/Os must complete + Create, // Create object and pattern with data + Remove, // Remove object + Read, // Read + Read2, // Two reads in a single op + Read3, // Three reads in a single op + Write, // Write + Write2, // Two writes in a single op + Write3, // Three writes in a single op + FailedWrite, // A write which should fail + FailedWrite2, // Two writes in one op which should fail + FailedWrite3, // Three writes in one op which should fail + InjectReadError, // Op to tell OSD to inject read errors + InjectWriteError, // Op to tell OSD to inject write errors + ClearReadErrorInject, // Op to tell OSD to clear read error injects + ClearWriteErrorInject // Op to tell OSD to clear write error injects }; } } @@ -61,6 +70,23 @@ struct fmt::formatter return fmt::format_to(ctx.out(), "Write2"); case ceph::io_exerciser::OpType::Write3: return fmt::format_to(ctx.out(), "Write3"); + case ceph::io_exerciser::OpType::FailedWrite: + return fmt::format_to(ctx.out(), "FailedWrite"); + case ceph::io_exerciser::OpType::FailedWrite2: + return fmt::format_to(ctx.out(), "FailedWrite2"); + case ceph::io_exerciser::OpType::FailedWrite3: + return fmt::format_to(ctx.out(), "FailedWrite3"); + case ceph::io_exerciser::OpType::InjectReadError: + return fmt::format_to(ctx.out(), "InjectReadError"); + case ceph::io_exerciser::OpType::InjectWriteError: + return fmt::format_to(ctx.out(), "InjectWriteError"); + case ceph::io_exerciser::OpType::ClearReadErrorInject: + return fmt::format_to(ctx.out(), "ClearReadErrorInject"); + case ceph::io_exerciser::OpType::ClearWriteErrorInject: + return fmt::format_to(ctx.out(), "ClearWriteErrorInject"); + default: + ceph_abort_msg("Unknown OpType"); + return fmt::format_to(ctx.out(), "Unknown OpType"); } } }; \ No newline at end of file diff --git a/src/common/io_exerciser/RadosIo.cc b/src/common/io_exerciser/RadosIo.cc index 4d68c60626e..be91b3c27ca 100644 --- a/src/common/io_exerciser/RadosIo.cc +++ b/src/common/io_exerciser/RadosIo.cc @@ -2,6 +2,13 @@ #include "DataGenerator.h" +#include +#include + +#include "common/ceph_json.h" + +#include "JsonStructures.h" + #include using RadosIo = ceph::io_exerciser::RadosIo; @@ -10,6 +17,7 @@ RadosIo::RadosIo(librados::Rados& rados, boost::asio::io_context& asio, const std::string& pool, const std::string& oid, + const std::optional>& cached_shard_order, uint64_t block_size, int seed, int threads, @@ -20,8 +28,9 @@ RadosIo::RadosIo(librados::Rados& rados, asio(asio), om(std::make_unique(oid, block_size, seed)), db(data_generation::DataGenerator::create_generator( - data_generation::GenerationType::HeaderedSeededRandom, *om)), + data_generation::GenerationType::HeaderedSeededRandom, *om)), pool(pool), + cached_shard_order(cached_shard_order), threads(threads), lock(lock), cond(cond), @@ -72,7 +81,8 @@ void RadosIo::allow_ec_overwrites(bool allow) } template -RadosIo::AsyncOpInfo::AsyncOpInfo(const std::array& offset, const std::array& length) : +RadosIo::AsyncOpInfo::AsyncOpInfo(const std::array& offset, + const std::array& length) : offset(offset), length(length) { @@ -104,32 +114,112 @@ void RadosIo::applyIoOp(IoOp& op) // at least one I/O to complete wait_for_io(threads-1); + switch (op.getOpType()) + { + case OpType::Done: + [[ fallthrough ]]; + case OpType::Barrier: + // Wait for all outstanding I/O to complete + wait_for_io(0); + break; + + case OpType::Create: + { + start_io(); + uint64_t opSize = static_cast(op).size; + std::shared_ptr> op_info + = std::make_shared>(std::array{0}, + std::array{opSize}); + op_info->bufferlist[0] = db->generate_data(0, opSize); + op_info->wop.write_full(op_info->bufferlist[0]); + auto create_cb = [this](boost::system::error_code ec, + version_t ver) + { + ceph_assert(ec == boost::system::errc::success); + finish_io(); + }; + librados::async_operate(asio, io, oid, + &op_info->wop, 0, nullptr, create_cb); + break; + } + + case OpType::Remove: + { + start_io(); + auto op_info = std::make_shared>(); + op_info->wop.remove(); + auto remove_cb = [this] (boost::system::error_code ec, + version_t ver) + { + ceph_assert(ec == boost::system::errc::success); + finish_io(); + }; + librados::async_operate(asio, io, oid, + &op_info->wop, 0, nullptr, remove_cb); + break; + } + case OpType::Read: + [[ fallthrough ]]; + case OpType::Read2: + [[ fallthrough ]]; + case OpType::Read3: + [[ fallthrough ]]; + case OpType::Write: + [[ fallthrough ]]; + case OpType::Write2: + [[ fallthrough ]]; + case OpType::Write3: + [[ fallthrough ]]; + case OpType::FailedWrite: + [[ fallthrough ]]; + case OpType::FailedWrite2: + [[ fallthrough ]]; + case OpType::FailedWrite3: + applyReadWriteOp(op); + break; + case OpType::InjectReadError: + [[ fallthrough ]]; + case OpType::InjectWriteError: + [[ fallthrough ]]; + case OpType::ClearReadErrorInject: + [[ fallthrough ]]; + case OpType::ClearWriteErrorInject: + applyInjectOp(op); + break; + default: + ceph_abort_msg("Unrecognised Op"); + break; + } +} + +void RadosIo::applyReadWriteOp(IoOp& op) +{ auto applyReadOp = [this](ReadWriteOp readOp) { auto op_info = std::make_shared>(readOp.offset, readOp.length); + for (int i = 0; i < N; i++) + { + op_info->rop.read(readOp.offset[i] * block_size, + readOp.length[i] * block_size, + &op_info->bufferlist[i], nullptr); + } + auto read_cb = [this, op_info] (boost::system::error_code ec, + version_t ver, + bufferlist bl) + { + ceph_assert(ec == boost::system::errc::success); for (int i = 0; i < N; i++) { - op_info->rop.read(readOp.offset[i] * block_size, - readOp.length[i] * block_size, - &op_info->bufferlist[i], nullptr); + ceph_assert(db->validate(op_info->bufferlist[i], + op_info->offset[i], + op_info->length[i])); } - auto read_cb = [this, op_info] (boost::system::error_code ec, - version_t ver, - bufferlist bl) - { - ceph_assert(ec == boost::system::errc::success); - for (int i = 0; i < N; i++) - { - ceph_assert(db->validate(op_info->bufferlist[i], - op_info->offset[i], - op_info->length[i])); - } - finish_io(); - }; - librados::async_operate(asio, io, oid, - &op_info->rop, 0, nullptr, read_cb); - num_io++; + finish_io(); + }; + librados::async_operate(asio, io, oid, + &op_info->rop, 0, nullptr, read_cb); + num_io++; }; auto applyWriteOp = [this](ReadWriteOp writeOp) @@ -151,96 +241,191 @@ void RadosIo::applyIoOp(IoOp& op) num_io++; }; - switch (op.getOpType()) + auto applyFailedWriteOp = [this](ReadWriteOp writeOp) { - case OpType::Done: - [[ fallthrough ]]; - case OpType::Barrier: - // Wait for all outstanding I/O to complete - wait_for_io(0); - break; - - case OpType::Create: + auto op_info = std::make_shared>(writeOp.offset, writeOp.length); + for (int i = 0; i < N; i++) { - start_io(); - uint64_t opSize = static_cast(op).size; - std::shared_ptr> op_info = std::make_shared>(std::array{0}, std::array{opSize}); - op_info->bufferlist[0] = db->generate_data(0, opSize); - op_info->wop.write_full(op_info->bufferlist[0]); - auto create_cb = [this] (boost::system::error_code ec, - version_t ver) { - ceph_assert(ec == boost::system::errc::success); - finish_io(); - }; - librados::async_operate(asio, io, oid, - &op_info->wop, 0, nullptr, create_cb); + op_info->bufferlist[i] = db->generate_data(writeOp.offset[i], writeOp.length[i]); + op_info->wop.write(writeOp.offset[i] * block_size, op_info->bufferlist[i]); } - break; - - case OpType::Remove: + auto write_cb = [this, writeOp] (boost::system::error_code ec, + version_t ver) { - start_io(); - auto op_info = std::make_shared>(); - op_info->wop.remove(); - auto remove_cb = [this] (boost::system::error_code ec, - version_t ver) { - ceph_assert(ec == boost::system::errc::success); - finish_io(); - }; - librados::async_operate(asio, io, oid, - &op_info->wop, 0, nullptr, remove_cb); - } - break; + ceph_assert(ec != boost::system::errc::success); + finish_io(); + }; + librados::async_operate(asio, io, oid, + &op_info->wop, 0, nullptr, write_cb); + num_io++; + }; + switch (op.getOpType()) + { case OpType::Read: - { + { start_io(); SingleReadOp& readOp = static_cast(op); applyReadOp(readOp); - } - break; - + break; + } case OpType::Read2: - { - start_io(); - DoubleReadOp& readOp = static_cast(op); - applyReadOp(readOp); - } + { + start_io(); + DoubleReadOp& readOp = static_cast(op); + applyReadOp(readOp); break; - + } case OpType::Read3: - { - start_io(); - TripleReadOp& readOp = static_cast(op); - applyReadOp(readOp); - } + { + start_io(); + TripleReadOp& readOp = static_cast(op); + applyReadOp(readOp); break; - + } case OpType::Write: - { - start_io(); - SingleWriteOp& writeOp = static_cast(op); - applyWriteOp(writeOp); - } + { + start_io(); + SingleWriteOp& writeOp = static_cast(op); + applyWriteOp(writeOp); break; - + } case OpType::Write2: - { - start_io(); - DoubleWriteOp& writeOp = static_cast(op); - applyWriteOp(writeOp); - } + { + start_io(); + DoubleWriteOp& writeOp = static_cast(op); + applyWriteOp(writeOp); break; - + } case OpType::Write3: - { - start_io(); - TripleWriteOp& writeOp = static_cast(op); - applyWriteOp(writeOp); - } + { + start_io(); + TripleWriteOp& writeOp = static_cast(op); + applyWriteOp(writeOp); + break; + } + + case OpType::FailedWrite: + { + start_io(); + SingleFailedWriteOp& writeOp = static_cast(op); + applyFailedWriteOp(writeOp); + break; + } + case OpType::FailedWrite2: + { + start_io(); + DoubleFailedWriteOp& writeOp = static_cast(op); + applyFailedWriteOp(writeOp); + break; + } + case OpType::FailedWrite3: + { + start_io(); + TripleFailedWriteOp& writeOp = static_cast(op); + applyFailedWriteOp(writeOp); break; + } default: + ceph_abort_msg(fmt::format("Unsupported Read/Write operation ({})", + op.getOpType())); break; } } + +void RadosIo::applyInjectOp(IoOp& op) +{ + bufferlist osdmap_inbl, inject_inbl, osdmap_outbl, inject_outbl; + auto formatter = std::make_shared(false); + + int osd = -1; + + ceph::io_exerciser::json::OSDMapRequest osdMapRequest(pool, + get_oid(), + "", + formatter); + int rc = rados.mon_command(osdMapRequest.encode_json(), + osdmap_inbl, + &osdmap_outbl, + nullptr); + ceph_assert(rc == 0); + + JSONParser p; + bool success = p.parse(osdmap_outbl.c_str(), osdmap_outbl.length()); + ceph_assert(success); + + ceph::io_exerciser::json::OSDMapReply reply{formatter}; + reply.decode_json(&p); + + osd = reply.acting_primary; + + switch(op.getOpType()) + { + case OpType::InjectReadError: + { + InjectReadErrorOp& errorOp = static_cast(op); + + ceph::io_exerciser::json::InjectECErrorRequest injectErrorRequest(json::InjectOpType::Read, + pool, + oid, + errorOp.shard, + errorOp.type, + errorOp.when, + errorOp.duration, + formatter); + + int rc = rados.osd_command(osd, injectErrorRequest.encode_json(), inject_inbl, &inject_outbl, nullptr); + ceph_assert(rc == 0); + break; + } + case OpType::InjectWriteError: + { + InjectWriteErrorOp& errorOp = static_cast(op); + + ceph::io_exerciser::json::InjectECErrorRequest injectErrorRequest(json::InjectOpType::Write, + pool, + oid, + errorOp.shard, + errorOp.type, + errorOp.when, + errorOp.duration, + formatter); + + int rc = rados.osd_command(osd, injectErrorRequest.encode_json(), inject_inbl, &inject_outbl, nullptr); + ceph_assert(rc == 0); + break; + } + case OpType::ClearReadErrorInject: + { + ClearReadErrorInjectOp& errorOp = static_cast(op); + + ceph::io_exerciser::json::InjectECClearErrorRequest clearErrorInject(json::InjectOpType::Read, + pool, + oid, + errorOp.shard, + errorOp.type); + + int rc = rados.osd_command(osd, clearErrorInject.encode_json(), inject_inbl, &inject_outbl, nullptr); + ceph_assert(rc == 0); + break; + } + case OpType::ClearWriteErrorInject: + { + ClearReadErrorInjectOp& errorOp = static_cast(op); + + ceph::io_exerciser::json::InjectECClearErrorRequest clearErrorInject(json::InjectOpType::Write, + pool, + oid, + errorOp.shard, + errorOp.type); + + int rc = rados.osd_command(osd, clearErrorInject.encode_json(), inject_inbl, &inject_outbl, nullptr); + ceph_assert(rc == 0); + break; + } + default: + ceph_abort_msg(fmt::format("Unsupported inject operation ({})", op.getOpType())); + break; + } +} \ No newline at end of file diff --git a/src/common/io_exerciser/RadosIo.h b/src/common/io_exerciser/RadosIo.h index eba80a24368..9e5316b7a73 100644 --- a/src/common/io_exerciser/RadosIo.h +++ b/src/common/io_exerciser/RadosIo.h @@ -26,6 +26,7 @@ namespace ceph { std::unique_ptr om; std::unique_ptr db; std::string pool; + std::optional> cached_shard_order; int threads; ceph::mutex& lock; ceph::condition_variable& cond; @@ -41,6 +42,7 @@ namespace ceph { boost::asio::io_context& asio, const std::string& pool, const std::string& oid, + const std::optional>& cached_shard_order, uint64_t block_size, int seed, int threads, @@ -68,6 +70,10 @@ namespace ceph { // Must be called with lock held bool readyForIoOp(IoOp& op); void applyIoOp(IoOp& op); + + private: + void applyReadWriteOp(IoOp& op); + void applyInjectOp(IoOp& op); }; } } \ No newline at end of file diff --git a/src/test/osd/ceph_test_rados_io_sequence.cc b/src/test/osd/ceph_test_rados_io_sequence.cc index 7d473382dc8..0a45739a43d 100644 --- a/src/test/osd/ceph_test_rados_io_sequence.cc +++ b/src/test/osd/ceph_test_rados_io_sequence.cc @@ -17,6 +17,8 @@ #include "common/dout.h" #include "common/split.h" #include "common/strtol.h" // for strict_iecstrtoll() +#include "common/ceph_json.h" +#include "common/Formatter.h" #include "common/io_exerciser/DataGenerator.h" #include "common/io_exerciser/Model.h" @@ -24,6 +26,11 @@ #include "common/io_exerciser/RadosIo.h" #include "common/io_exerciser/IoOp.h" #include "common/io_exerciser/IoSequence.h" +#include "common/io_exerciser/JsonStructures.h" + +#include "json_spirit/json_spirit.h" + +#include "fmt/format.h" #define dout_subsys ceph_subsys_rados #define dout_context g_ceph_context @@ -40,6 +47,9 @@ using TripleReadOp = ceph::io_exerciser::TripleReadOp; using SingleWriteOp = ceph::io_exerciser::SingleWriteOp; using DoubleWriteOp = ceph::io_exerciser::DoubleWriteOp; using TripleWriteOp = ceph::io_exerciser::TripleWriteOp; +using SingleFailedWriteOp = ceph::io_exerciser::SingleFailedWriteOp; +using DoubleFailedWriteOp = ceph::io_exerciser::DoubleFailedWriteOp; +using TripleFailedWriteOp = ceph::io_exerciser::TripleFailedWriteOp; namespace { struct Size {}; @@ -132,9 +142,11 @@ namespace { "\t are specified with unit of blocksize. Supported commands:", "\t\t create ", "\t\t remove", - "\t\t read|write ", - "\t\t read2|write2 ", - "\t\t read3|write3 ", + "\t\t read|write|failedwrite ", + "\t\t read2|write2|failedwrite2 ", + "\t\t read3|write3|failedwrite3 ", + "\t\t injecterror ", + "\t\t clearinject ", "\t\t done" }; @@ -175,7 +187,15 @@ namespace { ("parallel,p", po::value()->default_value(1), "number of objects to exercise in parallel") ("interactive", - "interactive mode, execute IO commands from stdin"); + "interactive mode, execute IO commands from stdin") + ("allow_pool_autoscaling", + "Allows pool autoscaling. Disabled by default.") + ("allow_pool_balancer", + "Enables pool balancing. Disabled by default.") + ("allow_pool_deep_scrubbing", + "Enables pool deep scrub. Disabled by default.") + ("allow_pool_scrubbing", + "Enables pool scrubbing. Disabled by default."); return desc; } @@ -336,8 +356,10 @@ ceph::io_sequence::tester::SelectErasurePlugin::SelectErasurePlugin( -ceph::io_sequence::tester::SelectErasureChunkSize::SelectErasureChunkSize(ceph::util::random_number_generator& rng, po::variables_map vm) - : ProgramOptionSelector(rng, vm, "stripe_unit", true, false) +ceph::io_sequence::tester::SelectErasureChunkSize::SelectErasureChunkSize( + ceph::util::random_number_generator& rng, + po::variables_map vm) + : ProgramOptionSelector(rng, vm, "chunksize", true, false) { } @@ -347,10 +369,18 @@ ceph::io_sequence::tester::SelectECPool::SelectECPool( ceph::util::random_number_generator& rng, po::variables_map vm, librados::Rados& rados, - bool dry_run) + bool dry_run, + bool allow_pool_autoscaling, + bool allow_pool_balancer, + bool allow_pool_deep_scrubbing, + bool allow_pool_scrubbing) : ProgramOptionSelector(rng, vm, "pool", false, false), rados(rados), dry_run(dry_run), + allow_pool_autoscaling(allow_pool_autoscaling), + allow_pool_balancer(allow_pool_balancer), + allow_pool_deep_scrubbing(allow_pool_deep_scrubbing), + allow_pool_scrubbing(allow_pool_scrubbing), skm(SelectErasureKM(rng, vm)), spl(SelectErasurePlugin(rng, vm)), scs(SelectErasureChunkSize(rng, vm)) @@ -396,28 +426,95 @@ void ceph::io_sequence::tester::SelectECPool::create_pool( { int rc; bufferlist inbl, outbl; - std::string profile_create = - "{\"prefix\": \"osd erasure-code-profile set\", \ - \"name\": \"testprofile-" + pool_name + "\", \ - \"profile\": [ \"plugin=" + plugin + "\", \ - \"k=" + std::to_string(k) + "\", \ - \"m=" + std::to_string(m) + "\", \ - \"stripe_unit=" + std::to_string(chunk_size) + "\", \ - \"crush-failure-domain=osd\"]}"; - rc = rados.mon_command(profile_create, inbl, &outbl, nullptr); + auto formatter = std::make_shared(false); + + ceph::io_exerciser::json::OSDECProfileSetRequest ecProfileSetRequest( + fmt::format("testprofile-{}", pool_name), + { fmt::format("plugin={}", plugin), + fmt::format("k={}", k), + fmt::format("m={}", m), + fmt::format("stripe_unit={}", chunk_size), + fmt::format("crush-failure-domain=osd")}, + formatter); + rc = rados.mon_command(ecProfileSetRequest.encode_json(), inbl, &outbl, nullptr); ceph_assert(rc == 0); - std::string cmdstr = - "{\"prefix\": \"osd pool create\", \ - \"pool\": \"" + pool_name + "\", \ - \"pool_type\": \"erasure\", \ - \"pg_num\": 8, \ - \"pgp_num\": 8, \ - \"erasure_code_profile\": \"testprofile-" + pool_name + "\"}"; - rc = rados.mon_command(cmdstr, inbl, &outbl, nullptr); + + ceph::io_exerciser::json::OSDECPoolCreateRequest poolCreateRequest(pool_name, + fmt::format("testprofile-{}", pool_name), + formatter); + rc = rados.mon_command(poolCreateRequest.encode_json(), inbl, &outbl, nullptr); ceph_assert(rc == 0); -} + if (allow_pool_autoscaling) + { + ceph::io_exerciser::json::OSDSetRequest setNoAutoscaleRequest("noautoscale", + std::nullopt, + formatter); + rc = rados.mon_command(setNoAutoscaleRequest.encode_json(), inbl, &outbl, nullptr); + ceph_assert(rc == 0); + } + + if (allow_pool_balancer) + { + ceph::io_exerciser::json::BalancerOffRequest balancerOffRequest(formatter); + rc = rados.mon_command(balancerOffRequest.encode_json(), inbl, &outbl, nullptr); + ceph_assert(rc == 0); + + ceph::io_exerciser::json::BalancerStatusRequest balancerStatusRequest(formatter); + rc = rados.mon_command(balancerStatusRequest.encode_json(), inbl, &outbl, nullptr); + ceph_assert(rc == 0); + + JSONParser p; + bool success = p.parse(outbl.c_str(), outbl.length()); + ceph_assert(success); + + ceph::io_exerciser::json::BalancerStatusReply reply{formatter}; + reply.decode_json(&p); + ceph_assert(!reply.active); + } + + if (allow_pool_deep_scrubbing) + { + ceph::io_exerciser::json::OSDSetRequest setNoDeepScrubRequest("nodeep-scrub", + std::nullopt, + formatter); + rc = rados.mon_command(setNoDeepScrubRequest.encode_json(), inbl, &outbl, nullptr); + ceph_assert(rc == 0); + } + + if (allow_pool_scrubbing) + { + ceph::io_exerciser::json::OSDSetRequest setNoScrubRequest("noscrub", + std::nullopt, + formatter); + rc = rados.mon_command(setNoScrubRequest.encode_json(), inbl, &outbl, nullptr); + ceph_assert(rc == 0); + } + ceph::io_exerciser::json + ::ConfigSetRequest configSetBluestoreDebugRequest("global", + "bluestore_debug_inject_read_err", + "true", + std::nullopt, + formatter); + rc = rados.mon_command(configSetBluestoreDebugRequest.encode_json(), + inbl, + &outbl, + nullptr); + ceph_assert(rc == 0); + + ceph::io_exerciser::json + ::ConfigSetRequest configSetMaxMarkdownRequest("global", + "osd_max_markdown_count", + "99999999", + std::nullopt, + formatter); + rc = rados.mon_command(configSetMaxMarkdownRequest.encode_json(), + inbl, + &outbl, + nullptr); + ceph_assert(rc == 0); +} ceph::io_sequence::tester::TestObject::TestObject( const std::string oid, librados::Rados& rados, @@ -443,10 +540,34 @@ ceph::io_sequence::tester::TestObject::TestObject( const std::string oid, } else { const std::string pool = spo.choose(); int threads = snt.choose(); + + bufferlist inbl, outbl; + + std::optional> cached_shard_order = std::nullopt; + + if (!spo.get_allow_pool_autoscaling() && + !spo.get_allow_pool_balancer() && + !spo.get_allow_pool_deep_scrubbing() && + !spo.get_allow_pool_scrubbing()) + { + ceph::io_exerciser::json::OSDMapRequest osdMapRequest(pool, oid, ""); + int rc = rados.mon_command(osdMapRequest.encode_json(), inbl, &outbl, nullptr); + ceph_assert(rc == 0); + + JSONParser p; + bool success = p.parse(outbl.c_str(), outbl.length()); + ceph_assert(success); + + ceph::io_exerciser::json::OSDMapReply reply{}; + reply.decode_json(&p); + cached_shard_order = reply.acting; + } + exerciser_model = std::make_unique(rados, asio, pool, oid, + cached_shard_order, sbs.choose(), rng(), threads, @@ -539,7 +660,12 @@ ceph::io_sequence::tester::TestRunner::TestRunner(po::variables_map& vm, rng(ceph::util::random_number_generator(seed)), sbs{rng, vm}, sos{rng, vm}, - spo{rng, vm, rados, vm.contains("dryrun")}, + spo{rng, vm, rados, + vm.contains("dryrun"), + vm.contains("allow_pool_autoscaling"), + vm.contains("allow_pool_balancer"), + vm.contains("allow_pool_deep_scrubbing"), + vm.contains("allow_pool_scrubbing")}, snt{rng, vm}, ssr{rng, vm} { @@ -556,6 +682,11 @@ ceph::io_sequence::tester::TestRunner::TestRunner(po::variables_map& vm, object_name = vm["object"].as(); interactive = vm.contains("interactive"); + allow_pool_autoscaling = vm.contains("allow_pool_autoscaling"); + allow_pool_balancer = vm.contains("allow_pool_balancer"); + allow_pool_deep_scrubbing = vm.contains("allow_pool_deep_scrubbing"); + allow_pool_scrubbing = vm.contains("allow_pool_scrubbing"); + if (!dryrun) { guard.emplace(boost::asio::make_work_guard(asio)); @@ -601,13 +732,17 @@ void ceph::io_sequence::tester::TestRunner::list_sequence() } } +void ceph::io_sequence::tester::TestRunner::clear_tokens() +{ + tokens = split.end(); +} + std::string ceph::io_sequence::tester::TestRunner::get_token() { - static std::string line; - static ceph::split split = ceph::split(""); - static ceph::spliterator tokens; - while (line.empty() || tokens == split.end()) { - if (!std::getline(std::cin, line)) { + while (line.empty() || tokens == split.end()) + { + if (!std::getline(std::cin, line)) + { throw std::runtime_error("End of input"); } split = ceph::split(line); @@ -616,17 +751,47 @@ std::string ceph::io_sequence::tester::TestRunner::get_token() return std::string(*tokens++); } +std::optional ceph::io_sequence::tester::TestRunner + ::get_optional_token() +{ + std::optional ret = std::nullopt; + if (tokens != split.end()) + { + ret = std::string(*tokens++); + } + return ret; +} + uint64_t ceph::io_sequence::tester::TestRunner::get_numeric_token() { std::string parse_error; std::string token = get_token(); uint64_t num = strict_iecstrtoll(token, &parse_error); - if (!parse_error.empty()) { + if (!parse_error.empty()) + { throw std::runtime_error("Invalid number "+token); } return num; } +std::optional ceph::io_sequence::tester::TestRunner + ::get_optional_numeric_token() +{ + std::string parse_error; + std::optional token = get_optional_token(); + if (token) + { + uint64_t num = strict_iecstrtoll(*token, &parse_error); + if (!parse_error.empty()) + { + throw std::runtime_error("Invalid number "+*token); + } + return num; + } + + return std::optional(std::nullopt); +} + bool ceph::io_sequence::tester::TestRunner::run_test() { if (show_help) @@ -664,43 +829,60 @@ bool ceph::io_sequence::tester::TestRunner::run_interactive_test() else { const std::string pool = spo.choose(); + + bufferlist inbl, outbl; + + ceph::io_exerciser::json::OSDMapRequest osdMapRequest(pool, object_name, ""); + int rc = rados.mon_command(osdMapRequest.encode_json(), + inbl, + &outbl, + nullptr); + ceph_assert(rc == 0); + + JSONParser p; + bool success = p.parse(outbl.c_str(), outbl.length()); + ceph_assert(success); + + ceph::io_exerciser::json::OSDMapReply reply{}; + reply.decode_json(&p); + model = std::make_unique(rados, asio, pool, - object_name, sbs.choose(), - rng(), 1, // 1 thread + object_name, reply.acting, + sbs.choose(), rng(), + 1, // 1 thread lock, cond); } while (!done) { const std::string op = get_token(); - if (!op.compare("done") || !op.compare("q") || !op.compare("quit")) + if (op == "done" || op == "q" || op == "quit") { - ioop = DoneOp::generate(); + ioop = ceph::io_exerciser::DoneOp::generate(); } - else if (!op.compare("create")) + else if (op == "create") { - ioop = CreateOp::generate(get_numeric_token()); + ioop = ceph::io_exerciser::CreateOp::generate(get_numeric_token()); } - else if (!op.compare("remove") || !op.compare("delete")) + else if (op == "remove" || op == "delete") { - ioop = RemoveOp::generate(); + ioop = ceph::io_exerciser::RemoveOp::generate(); } - else if (!op.compare("read")) + else if (op == "read") { uint64_t offset = get_numeric_token(); uint64_t length = get_numeric_token(); - ioop = SingleReadOp::generate(offset, length); + ioop = ceph::io_exerciser::SingleReadOp::generate(offset, length); } - else if (!op.compare("read2")) + else if (op == "read2") { uint64_t offset1 = get_numeric_token(); uint64_t length1 = get_numeric_token(); uint64_t offset2 = get_numeric_token(); uint64_t length2 = get_numeric_token(); - ioop = DoubleReadOp::generate(offset1, length1, - offset2, length2); + ioop = DoubleReadOp::generate(offset1, length1, offset2, length2); } - else if (!op.compare("read3")) + else if (op == "read3") { uint64_t offset1 = get_numeric_token(); uint64_t length1 = get_numeric_token(); @@ -709,25 +891,24 @@ bool ceph::io_sequence::tester::TestRunner::run_interactive_test() uint64_t offset3 = get_numeric_token(); uint64_t length3 = get_numeric_token(); ioop = TripleReadOp::generate(offset1, length1, - offset2, length2, - offset3, length3); + offset2, length2, + offset3, length3); } - else if (!op.compare("write")) + else if (op == "write") { uint64_t offset = get_numeric_token(); uint64_t length = get_numeric_token(); ioop = SingleWriteOp::generate(offset, length); } - else if (!op.compare("write2")) + else if (op == "write2") { uint64_t offset1 = get_numeric_token(); uint64_t length1 = get_numeric_token(); uint64_t offset2 = get_numeric_token(); uint64_t length2 = get_numeric_token(); - ioop = DoubleWriteOp::generate(offset1, length1, - offset2, length2); + ioop = DoubleWriteOp::generate(offset1, length1, offset2, length2); } - else if (!op.compare("write3")) + else if (op == "write3") { uint64_t offset1 = get_numeric_token(); uint64_t length1 = get_numeric_token(); @@ -736,19 +917,104 @@ bool ceph::io_sequence::tester::TestRunner::run_interactive_test() uint64_t offset3 = get_numeric_token(); uint64_t length3 = get_numeric_token(); ioop = TripleWriteOp::generate(offset1, length1, - offset2, length2, - offset3, length3); + offset2, length2, + offset3, length3); + } + else if (op == "failedwrite") + { + uint64_t offset = get_numeric_token(); + uint64_t length = get_numeric_token(); + ioop = SingleFailedWriteOp::generate(offset, length); + } + else if (op == "failedwrite2") + { + uint64_t offset1 = get_numeric_token(); + uint64_t length1 = get_numeric_token(); + uint64_t offset2 = get_numeric_token(); + uint64_t length2 = get_numeric_token(); + ioop = DoubleFailedWriteOp::generate(offset1, length1, offset2, length2); + } + else if (op == "failedwrite3") + { + uint64_t offset1 = get_numeric_token(); + uint64_t length1 = get_numeric_token(); + uint64_t offset2 = get_numeric_token(); + uint64_t length2 = get_numeric_token(); + uint64_t offset3 = get_numeric_token(); + uint64_t length3 = get_numeric_token(); + ioop = TripleFailedWriteOp::generate(offset1, length1, + offset2, length2, + offset3, length3); + } + else if (op == "injecterror") + { + std::string inject_type = get_token(); + int shard = get_numeric_token(); + std::optional type = get_optional_numeric_token(); + std::optional when = get_optional_numeric_token(); + std::optional duration = get_optional_numeric_token(); + if (inject_type == "read") + { + ioop = ceph::io_exerciser::InjectReadErrorOp::generate(shard, + type, + when, + duration); + } + else if (inject_type == "write") + { + ioop = ceph::io_exerciser::InjectWriteErrorOp::generate(shard, + type, + when, + duration); + } + else + { + clear_tokens(); + ioop.reset(); + dout(0) << fmt::format("Invalid error inject {}. No action performed.", + inject_type) << dendl; + } + } + else if (op == "clearinject") + { + std::string inject_type = get_token(); + int shard = get_numeric_token(); + std::optional type = get_optional_numeric_token(); + if (inject_type == "read") + { + ioop = ceph::io_exerciser::ClearReadErrorInjectOp::generate(shard, + type); + } + else if (inject_type == "write") + { + ioop = ceph::io_exerciser::ClearWriteErrorInjectOp::generate(shard, + type); + } + else + { + clear_tokens(); + ioop.reset(); + dout(0) << fmt::format("Invalid error inject {}. No action performed.", + inject_type) << dendl; + } } else { - throw std::runtime_error("Invalid operation "+op); + clear_tokens(); + ioop.reset(); + dout(0) << fmt::format("Invalid op {}. No action performed.", + op) << dendl; } - dout(0) << ioop->to_string(model->get_block_size()) << dendl; - model->applyIoOp(*ioop); - done = ioop->getOpType() == OpType::Done; - if (!done) { - ioop = BarrierOp::generate(); + if (ioop) + { + dout(0) << ioop->to_string(model->get_block_size()) << dendl; model->applyIoOp(*ioop); + done = ioop->getOpType() == ceph::io_exerciser::OpType::Done; + if (!done) + { + ioop = ceph::io_exerciser::BarrierOp::generate(); + model->applyIoOp(*ioop); + } } } @@ -791,12 +1057,15 @@ bool ceph::io_sequence::tester::TestRunner::run_automated_test() bool started_io = true; bool need_wait = true; - while (started_io || need_wait) { + while (started_io || need_wait) + { started_io = false; need_wait = false; - for (auto obj = test_objects.begin(); obj != test_objects.end(); ++obj) { + for (auto obj = test_objects.begin(); obj != test_objects.end(); ++obj) + { std::shared_ptr to = *obj; - if (!to->finished()) { + if (!to->finished()) + { lock.lock(); bool ready = to->readyForIo(); lock.unlock(); @@ -809,12 +1078,15 @@ bool ceph::io_sequence::tester::TestRunner::run_automated_test() } } } - if (!started_io && need_wait) { + if (!started_io && need_wait) + { std::unique_lock l(lock); // Recheck with lock incase anything has changed - for (auto obj = test_objects.begin(); obj != test_objects.end(); ++obj) { + for (auto obj = test_objects.begin(); obj != test_objects.end(); ++obj) + { std::shared_ptr to = *obj; - if (!to->finished()) { + if (!to->finished()) + { need_wait = !to->readyForIo(); if (!need_wait) { @@ -827,7 +1099,8 @@ bool ceph::io_sequence::tester::TestRunner::run_automated_test() } int total_io = 0; - for (auto obj = test_objects.begin(); obj != test_objects.end(); ++obj) { + for (auto obj = test_objects.begin(); obj != test_objects.end(); ++obj) + { std::shared_ptr to = *obj; total_io += to->get_num_io(); ceph_assert(to->finished()); diff --git a/src/test/osd/ceph_test_rados_io_sequence.h b/src/test/osd/ceph_test_rados_io_sequence.h index 4e21d025700..e34fdf32d0b 100644 --- a/src/test/osd/ceph_test_rados_io_sequence.h +++ b/src/test/osd/ceph_test_rados_io_sequence.h @@ -9,11 +9,15 @@ #include "common/io_exerciser/IoSequence.h" #include "common/io_exerciser/Model.h" +#include "common/split.h" + #include "librados/librados_asio.h" #include #include +#include + /* Overview * * class ProgramOptionSelector @@ -223,7 +227,8 @@ namespace ceph io_sequence::tester::chunkSizeChoices> { public: - SelectErasureChunkSize(ceph::util::random_number_generator& rng, po::variables_map vm); + SelectErasureChunkSize(ceph::util::random_number_generator& rng, + po::variables_map vm); }; class SelectECPool @@ -235,9 +240,18 @@ namespace ceph SelectECPool(ceph::util::random_number_generator& rng, po::variables_map vm, librados::Rados& rados, - bool dry_run); + bool dry_run, + bool allow_pool_autoscaling, + bool allow_pool_balancer, + bool allow_pool_deep_scrubbing, + bool allow_pool_scrubbing); const std::string choose() override; + bool get_allow_pool_autoscaling() { return allow_pool_autoscaling; } + bool get_allow_pool_balancer() { return allow_pool_balancer; } + bool get_allow_pool_deep_scrubbing() { return allow_pool_deep_scrubbing; } + bool get_allow_pool_scrubbing() { return allow_pool_scrubbing; } + private: void create_pool(librados::Rados& rados, const std::string& pool_name, @@ -248,6 +262,10 @@ namespace ceph protected: librados::Rados& rados; bool dry_run; + bool allow_pool_autoscaling; + bool allow_pool_balancer; + bool allow_pool_deep_scrubbing; + bool allow_pool_scrubbing; SelectErasureKM skm; SelectErasurePlugin spl; @@ -324,14 +342,26 @@ namespace ceph std::optional seqseed; bool interactive; + bool allow_pool_autoscaling; + bool allow_pool_balancer; + bool allow_pool_deep_scrubbing; + bool allow_pool_scrubbing; + bool show_sequence; bool show_help; int num_objects; std::string object_name; + std::string line; + ceph::split split = ceph::split(""); + ceph::spliterator tokens; + + void clear_tokens(); std::string get_token(); + std::optional get_optional_token(); uint64_t get_numeric_token(); + std::optional get_optional_numeric_token(); bool run_automated_test(); -- 2.39.5