From: Casey Bodley Date: Wed, 19 Aug 2020 19:15:56 +0000 (-0400) Subject: common: add generic FaultInjector X-Git-Url: http://git.apps.os.sepia.ceph.com/?a=commitdiff_plain;h=fbd3a9b439fac87aa3e9f9a4773d733fa4d61ee5;p=ceph.git common: add generic FaultInjector Signed-off-by: Casey Bodley --- diff --git a/src/common/fault_injector.h b/src/common/fault_injector.h new file mode 100644 index 0000000000000..53b247e08b002 --- /dev/null +++ b/src/common/fault_injector.h @@ -0,0 +1,135 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab ft=cpp +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2020 Red Hat, Inc. + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + */ + +#pragma once + +#include +#include +#include +#include +#include "include/ceph_assert.h" +#include "common/dout.h" + +/// @file + +/// A failure type that aborts the process with a failed assertion. +struct InjectAbort {}; + +/// A failure type that injects an error code and optionally logs a message. +struct InjectError { + /// error code to inject + int error; + /// an optional log channel to print an error message + const DoutPrefixProvider* dpp = nullptr; +}; + +/** @class FaultInjector + * @brief Used to instrument a code path with deterministic fault injection + * by making one or more calls to check(). + * + * A default-constructed FaultInjector contains no failure. It can also be + * constructed with a failure of type InjectAbort or InjectError, along with + * a location to inject that failure. + * + * The contained failure can be overwritten with a call to inject() or clear(). + * This is not thread-safe with respect to other member functions on the same + * instance. + * + * @tparam Key The location can be represented by any Key type that is + * movable, default-constructible, inequality-comparable and stream-outputable. + * A string or string_view Key may be preferable when the location comes from + * user input, or to describe the steps like "before-foo" and "after-foo". + * An integer Key may be preferable for a code path with many steps, where you + * just want to check 1, 2, 3, etc. without inventing names for each. + */ +template +class FaultInjector { + public: + /// Default-construct with no injected failure. + constexpr FaultInjector() noexcept : location() {} + + /// Construct with an injected assertion failure at the given location. + constexpr FaultInjector(Key location, InjectAbort a) + : location(std::move(location)), failure(a) {} + + /// Construct with an injected error code at the given location. + constexpr FaultInjector(Key location, InjectError e) + : location(std::move(location)), failure(e) {} + + /// Inject an assertion failure at the given location. + void inject(Key location, InjectAbort a) { + this->location = std::move(location); + this->failure = a; + } + + /// Inject an error at the given location. + void inject(Key location, InjectError e) { + this->location = std::move(location); + this->failure = e; + } + + /// Clear any injected failure. + void clear() { + this->failure = Empty{}; + } + + /// Check for an injected failure at the given location. If the location + /// matches an InjectAbort failure, the process aborts here with an assertion + /// failure. + /// @returns 0 or InjectError::error if the location matches an InjectError + /// failure + [[nodiscard]] constexpr int check(const Key& location) const { + struct visitor { + const Key& check_location; + const Key& this_location; + constexpr int operator()(const std::monostate&) const { + return 0; + } + int operator()(const InjectAbort&) const { + if (check_location == this_location) { + ceph_assert_always(!"FaultInjector"); + } + return 0; + } + int operator()(const InjectError& e) const { + if (check_location == this_location) { + ldpp_dout(e.dpp, -1) << "Injecting error=" << e.error + << " at location=" << this_location << dendl; + return e.error; + } + return 0; + } + }; + return std::visit(visitor{location, this->location}, failure); + } + + private: + // Key requirements: + static_assert(std::is_default_constructible_v, + "Key must be default-constrible"); + static_assert(std::is_move_constructible_v, + "Key must be move-constructible"); + static_assert(std::is_move_assignable_v, + "Key must be move-assignable"); + static_assert(boost::has_equal_to::value, + "Key must be equality-comparable"); + static_assert(boost::has_left_shift::value, + "Key must have an ostream operator<<"); + + Key location; // location of the check that should fail + + using Empty = std::monostate; // empty state for std::variant + + std::variant failure; +}; diff --git a/src/test/common/CMakeLists.txt b/src/test/common/CMakeLists.txt index 0e84b3d722815..1763f69290889 100644 --- a/src/test/common/CMakeLists.txt +++ b/src/test/common/CMakeLists.txt @@ -364,10 +364,16 @@ add_ceph_unittest(unittest_cdc) add_executable(unittest_ceph_timer test_ceph_timer.cc) add_ceph_unittest(unittest_ceph_timer) + add_executable(unittest_option test_option.cc) target_link_libraries(unittest_option ceph-common GTest::Main) add_ceph_unittest(unittest_option) +add_executable(unittest_fault_injector test_fault_injector.cc + $) +target_link_libraries(unittest_fault_injector global) +add_ceph_unittest(unittest_fault_injector) + add_executable(unittest_blocked_completion test_blocked_completion.cc) add_ceph_unittest(unittest_blocked_completion) target_link_libraries(unittest_blocked_completion Boost::system GTest::GTest) diff --git a/src/test/common/test_fault_injector.cc b/src/test/common/test_fault_injector.cc new file mode 100644 index 0000000000000..92dc2ec7abd83 --- /dev/null +++ b/src/test/common/test_fault_injector.cc @@ -0,0 +1,224 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab ft=cpp +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2020 Red Hat, Inc. + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + */ + +#include "common/fault_injector.h" +#include + +static const DoutPrefixProvider* dpp() { + static NoDoutPrefix d{g_ceph_context, ceph_subsys_context}; + return &d; +} + +TEST(FaultInjectorDeathTest, InjectAbort) +{ + constexpr FaultInjector f{false, InjectAbort{}}; + EXPECT_EQ(f.check(true), 0); + EXPECT_DEATH([[maybe_unused]] int r = f.check(false), "FaultInjector"); +} + +TEST(FaultInjectorDeathTest, AssignAbort) +{ + FaultInjector f; + ASSERT_EQ(f.check(false), 0); + f.inject(false, InjectAbort{}); + EXPECT_DEATH([[maybe_unused]] int r = f.check(false), "FaultInjector"); +} + +// test int as a Key type +TEST(FaultInjectorInt, Default) +{ + constexpr FaultInjector f; + EXPECT_EQ(f.check(0), 0); + EXPECT_EQ(f.check(1), 0); + EXPECT_EQ(f.check(2), 0); + EXPECT_EQ(f.check(3), 0); +} + +TEST(FaultInjectorInt, InjectError) +{ + constexpr FaultInjector f{2, InjectError{-EINVAL}}; + EXPECT_EQ(f.check(0), 0); + EXPECT_EQ(f.check(1), 0); + EXPECT_EQ(f.check(2), -EINVAL); + EXPECT_EQ(f.check(3), 0); +} + +TEST(FaultInjectorInt, InjectErrorMessage) +{ + FaultInjector f{2, InjectError{-EINVAL, dpp()}}; + EXPECT_EQ(f.check(0), 0); + EXPECT_EQ(f.check(1), 0); + EXPECT_EQ(f.check(2), -EINVAL); + EXPECT_EQ(f.check(3), 0); +} + +TEST(FaultInjectorInt, AssignError) +{ + FaultInjector f; + ASSERT_EQ(f.check(0), 0); + f.inject(0, InjectError{-EINVAL}); + EXPECT_EQ(f.check(0), -EINVAL); +} + +TEST(FaultInjectorInt, AssignErrorMessage) +{ + FaultInjector f; + ASSERT_EQ(f.check(0), 0); + f.inject(0, InjectError{-EINVAL, dpp()}); + EXPECT_EQ(f.check(0), -EINVAL); +} + +// test std::string_view as a Key type +TEST(FaultInjectorString, Default) +{ + constexpr FaultInjector f; + EXPECT_EQ(f.check("Red"), 0); + EXPECT_EQ(f.check("Green"), 0); + EXPECT_EQ(f.check("Blue"), 0); +} + +TEST(FaultInjectorString, InjectError) +{ + FaultInjector f{"Red", InjectError{-EIO}}; + EXPECT_EQ(f.check("Red"), -EIO); + EXPECT_EQ(f.check("Green"), 0); + EXPECT_EQ(f.check("Blue"), 0); +} + +TEST(FaultInjectorString, InjectErrorMessage) +{ + FaultInjector f{"Red", InjectError{-EIO, dpp()}}; + EXPECT_EQ(f.check("Red"), -EIO); + EXPECT_EQ(f.check("Green"), 0); + EXPECT_EQ(f.check("Blue"), 0); +} + +TEST(FaultInjectorString, AssignError) +{ + FaultInjector f; + ASSERT_EQ(f.check("Red"), 0); + f.inject("Red", InjectError{-EINVAL}); + EXPECT_EQ(f.check("Red"), -EINVAL); +} + +TEST(FaultInjectorString, AssignErrorMessage) +{ + FaultInjector f; + ASSERT_EQ(f.check("Red"), 0); + f.inject("Red", InjectError{-EINVAL, dpp()}); + EXPECT_EQ(f.check("Red"), -EINVAL); +} + +// test enum class as a Key type +enum class Color { Red, Green, Blue }; + +static std::ostream& operator<<(std::ostream& out, const Color& c) { + switch (c) { + case Color::Red: return out << "Red"; + case Color::Green: return out << "Green"; + case Color::Blue: return out << "Blue"; + } + return out; +} + +TEST(FaultInjectorEnum, Default) +{ + constexpr FaultInjector f; + EXPECT_EQ(f.check(Color::Red), 0); + EXPECT_EQ(f.check(Color::Green), 0); + EXPECT_EQ(f.check(Color::Blue), 0); +} + +TEST(FaultInjectorEnum, InjectError) +{ + FaultInjector f{Color::Red, InjectError{-EIO}}; + EXPECT_EQ(f.check(Color::Red), -EIO); + EXPECT_EQ(f.check(Color::Green), 0); + EXPECT_EQ(f.check(Color::Blue), 0); +} + +TEST(FaultInjectorEnum, InjectErrorMessage) +{ + FaultInjector f{Color::Red, InjectError{-EIO, dpp()}}; + EXPECT_EQ(f.check(Color::Red), -EIO); + EXPECT_EQ(f.check(Color::Green), 0); + EXPECT_EQ(f.check(Color::Blue), 0); +} + +TEST(FaultInjectorEnum, AssignError) +{ + FaultInjector f; + ASSERT_EQ(f.check(Color::Red), 0); + f.inject(Color::Red, InjectError{-EINVAL}); + EXPECT_EQ(f.check(Color::Red), -EINVAL); +} + +TEST(FaultInjectorEnum, AssignErrorMessage) +{ + FaultInjector f; + ASSERT_EQ(f.check(Color::Red), 0); + f.inject(Color::Red, InjectError{-EINVAL, dpp()}); + EXPECT_EQ(f.check(Color::Red), -EINVAL); +} + +// test custom move-only Key type +struct MoveOnlyKey { + MoveOnlyKey() = default; + MoveOnlyKey(const MoveOnlyKey&) = delete; + MoveOnlyKey& operator=(const MoveOnlyKey&) = delete; + MoveOnlyKey(MoveOnlyKey&&) = default; + MoveOnlyKey& operator=(MoveOnlyKey&&) = default; + ~MoveOnlyKey() = default; +}; + +static bool operator==(const MoveOnlyKey&, const MoveOnlyKey&) { + return true; // all keys are equal +} +static std::ostream& operator<<(std::ostream& out, const MoveOnlyKey&) { + return out; +} + +TEST(FaultInjectorMoveOnly, Default) +{ + constexpr FaultInjector f; + EXPECT_EQ(f.check(MoveOnlyKey{}), 0); +} + +TEST(FaultInjectorMoveOnly, InjectError) +{ + FaultInjector f{MoveOnlyKey{}, InjectError{-EIO}}; + EXPECT_EQ(f.check(MoveOnlyKey{}), -EIO); +} + +TEST(FaultInjectorMoveOnly, InjectErrorMessage) +{ + FaultInjector f{MoveOnlyKey{}, InjectError{-EIO, dpp()}}; + EXPECT_EQ(f.check(MoveOnlyKey{}), -EIO); +} + +TEST(FaultInjectorMoveOnly, AssignError) +{ + FaultInjector f; + ASSERT_EQ(f.check({}), 0); + f.inject({}, InjectError{-EINVAL}); + EXPECT_EQ(f.check({}), -EINVAL); +} + +TEST(FaultInjectorMoveOnly, AssignErrorMessage) +{ + FaultInjector f; + ASSERT_EQ(f.check({}), 0); + f.inject({}, InjectError{-EINVAL, dpp()}); + EXPECT_EQ(f.check({}), -EINVAL); +}