common: add generic FaultInjector

Signed-off-by: Casey Bodley <cbodley@redhat.com>

diff --git a/src/common/fault_injector.h b/src/common/fault_injector.h
new file mode 100644 (file)
index 0000000..53b247e
--- /dev/null
+++ b/src/common/fault_injector.h
@@ -0,0 +1,135 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab ft=cpp
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2020 Red Hat, Inc.
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation. See file COPYING.
+ *
+ */
+
+#pragma once
+
+#include <type_traits>
+#include <boost/type_traits/has_equal_to.hpp>
+#include <boost/type_traits/has_left_shift.hpp>
+#include <variant>
+#include "include/ceph_assert.h"
+#include "common/dout.h"
+
+/// @file
+
+/// A failure type that aborts the process with a failed assertion.
+struct InjectAbort {};
+
+/// A failure type that injects an error code and optionally logs a message.
+struct InjectError {
+  /// error code to inject
+  int error;
+  /// an optional log channel to print an error message
+  const DoutPrefixProvider* dpp = nullptr;
+};
+
+/** @class FaultInjector
+ * @brief Used to instrument a code path with deterministic fault injection
+ * by making one or more calls to check().
+ *
+ * A default-constructed FaultInjector contains no failure. It can also be
+ * constructed with a failure of type InjectAbort or InjectError, along with
+ * a location to inject that failure.
+ *
+ * The contained failure can be overwritten with a call to inject() or clear().
+ * This is not thread-safe with respect to other member functions on the same
+ * instance.
+ *
+ * @tparam Key  The location can be represented by any Key type that is
+ * movable, default-constructible, inequality-comparable and stream-outputable.
+ * A string or string_view Key may be preferable when the location comes from
+ * user input, or to describe the steps like "before-foo" and "after-foo".
+ * An integer Key may be preferable for a code path with many steps, where you
+ * just want to check 1, 2, 3, etc. without inventing names for each.
+ */
+template <typename Key>
+class FaultInjector {
+ public:
+  /// Default-construct with no injected failure.
+  constexpr FaultInjector() noexcept : location() {}
+
+  /// Construct with an injected assertion failure at the given location.
+  constexpr FaultInjector(Key location, InjectAbort a)
+    : location(std::move(location)), failure(a) {}
+
+  /// Construct with an injected error code at the given location.
+  constexpr FaultInjector(Key location, InjectError e)
+    : location(std::move(location)), failure(e) {}
+
+  /// Inject an assertion failure at the given location.
+  void inject(Key location, InjectAbort a) {
+    this->location = std::move(location);
+    this->failure = a;
+  }
+
+  /// Inject an error at the given location.
+  void inject(Key location, InjectError e) {
+    this->location = std::move(location);
+    this->failure = e;
+  }
+
+  /// Clear any injected failure.
+  void clear() {
+    this->failure = Empty{};
+  }
+
+  /// Check for an injected failure at the given location. If the location
+  /// matches an InjectAbort failure, the process aborts here with an assertion
+  /// failure.
+  /// @returns 0 or InjectError::error if the location matches an InjectError
+  /// failure
+  [[nodiscard]] constexpr int check(const Key& location) const {
+    struct visitor {
+      const Key& check_location;
+      const Key& this_location;
+      constexpr int operator()(const std::monostate&) const {
+        return 0;
+      }
+      int operator()(const InjectAbort&) const {
+        if (check_location == this_location) {
+          ceph_assert_always(!"FaultInjector");
+        }
+        return 0;
+      }
+      int operator()(const InjectError& e) const {
+        if (check_location == this_location) {
+          ldpp_dout(e.dpp, -1) << "Injecting error=" << e.error
+              << " at location=" << this_location << dendl;
+          return e.error;
+        }
+        return 0;
+      }
+    };
+    return std::visit(visitor{location, this->location}, failure);
+  }
+
+ private:
+  // Key requirements:
+  static_assert(std::is_default_constructible_v<Key>,
+                "Key must be default-constrible");
+  static_assert(std::is_move_constructible_v<Key>,
+                "Key must be move-constructible");
+  static_assert(std::is_move_assignable_v<Key>,
+                "Key must be move-assignable");
+  static_assert(boost::has_equal_to<Key, Key, bool>::value,
+                "Key must be equality-comparable");
+  static_assert(boost::has_left_shift<std::ostream, Key, std::ostream&>::value,
+                "Key must have an ostream operator<<");
+
+  Key location; // location of the check that should fail
+
+  using Empty = std::monostate; // empty state for std::variant
+
+  std::variant<Empty, InjectAbort, InjectError> failure;
+};

diff --git a/src/test/common/CMakeLists.txt b/src/test/common/CMakeLists.txt
index 0e84b3d722815a691a9142a0ab365ea75ffc834d..1763f69290889843317b069bfab2c5ae54113609 100644 (file)
--- a/src/test/common/CMakeLists.txt
+++ b/src/test/common/CMakeLists.txt
@@ -364,10 +364,16 @@ add_ceph_unittest(unittest_cdc)
 add_executable(unittest_ceph_timer test_ceph_timer.cc)
 add_ceph_unittest(unittest_ceph_timer)
 
+
 add_executable(unittest_option test_option.cc)
 target_link_libraries(unittest_option ceph-common GTest::Main)
 add_ceph_unittest(unittest_option)
 
+add_executable(unittest_fault_injector test_fault_injector.cc
+  $<TARGET_OBJECTS:unit-main>)
+target_link_libraries(unittest_fault_injector global)
+add_ceph_unittest(unittest_fault_injector)
+
 add_executable(unittest_blocked_completion test_blocked_completion.cc)
 add_ceph_unittest(unittest_blocked_completion)
 target_link_libraries(unittest_blocked_completion Boost::system GTest::GTest)

diff --git a/src/test/common/test_fault_injector.cc b/src/test/common/test_fault_injector.cc
new file mode 100644 (file)
index 0000000..92dc2ec
--- /dev/null
+++ b/src/test/common/test_fault_injector.cc
@@ -0,0 +1,224 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab ft=cpp
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2020 Red Hat, Inc.
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation. See file COPYING.
+ *
+ */
+
+#include "common/fault_injector.h"
+#include <gtest/gtest.h>
+
+static const DoutPrefixProvider* dpp() {
+  static NoDoutPrefix d{g_ceph_context, ceph_subsys_context};
+  return &d;
+}
+
+TEST(FaultInjectorDeathTest, InjectAbort)
+{
+  constexpr FaultInjector f{false, InjectAbort{}};
+  EXPECT_EQ(f.check(true), 0);
+  EXPECT_DEATH([[maybe_unused]] int r = f.check(false), "FaultInjector");
+}
+
+TEST(FaultInjectorDeathTest, AssignAbort)
+{
+  FaultInjector<bool> f;
+  ASSERT_EQ(f.check(false), 0);
+  f.inject(false, InjectAbort{});
+  EXPECT_DEATH([[maybe_unused]] int r = f.check(false), "FaultInjector");
+}
+
+// test int as a Key type
+TEST(FaultInjectorInt, Default)
+{
+  constexpr FaultInjector<int> f;
+  EXPECT_EQ(f.check(0), 0);
+  EXPECT_EQ(f.check(1), 0);
+  EXPECT_EQ(f.check(2), 0);
+  EXPECT_EQ(f.check(3), 0);
+}
+
+TEST(FaultInjectorInt, InjectError)
+{
+  constexpr FaultInjector f{2, InjectError{-EINVAL}};
+  EXPECT_EQ(f.check(0), 0);
+  EXPECT_EQ(f.check(1), 0);
+  EXPECT_EQ(f.check(2), -EINVAL);
+  EXPECT_EQ(f.check(3), 0);
+}
+
+TEST(FaultInjectorInt, InjectErrorMessage)
+{
+  FaultInjector f{2, InjectError{-EINVAL, dpp()}};
+  EXPECT_EQ(f.check(0), 0);
+  EXPECT_EQ(f.check(1), 0);
+  EXPECT_EQ(f.check(2), -EINVAL);
+  EXPECT_EQ(f.check(3), 0);
+}
+
+TEST(FaultInjectorInt, AssignError)
+{
+  FaultInjector<int> f;
+  ASSERT_EQ(f.check(0), 0);
+  f.inject(0, InjectError{-EINVAL});
+  EXPECT_EQ(f.check(0), -EINVAL);
+}
+
+TEST(FaultInjectorInt, AssignErrorMessage)
+{
+  FaultInjector<int> f;
+  ASSERT_EQ(f.check(0), 0);
+  f.inject(0, InjectError{-EINVAL, dpp()});
+  EXPECT_EQ(f.check(0), -EINVAL);
+}
+
+// test std::string_view as a Key type
+TEST(FaultInjectorString, Default)
+{
+  constexpr FaultInjector<std::string_view> f;
+  EXPECT_EQ(f.check("Red"), 0);
+  EXPECT_EQ(f.check("Green"), 0);
+  EXPECT_EQ(f.check("Blue"), 0);
+}
+
+TEST(FaultInjectorString, InjectError)
+{
+  FaultInjector<std::string_view> f{"Red", InjectError{-EIO}};
+  EXPECT_EQ(f.check("Red"), -EIO);
+  EXPECT_EQ(f.check("Green"), 0);
+  EXPECT_EQ(f.check("Blue"), 0);
+}
+
+TEST(FaultInjectorString, InjectErrorMessage)
+{
+  FaultInjector<std::string_view> f{"Red", InjectError{-EIO, dpp()}};
+  EXPECT_EQ(f.check("Red"), -EIO);
+  EXPECT_EQ(f.check("Green"), 0);
+  EXPECT_EQ(f.check("Blue"), 0);
+}
+
+TEST(FaultInjectorString, AssignError)
+{
+  FaultInjector<std::string_view> f;
+  ASSERT_EQ(f.check("Red"), 0);
+  f.inject("Red", InjectError{-EINVAL});
+  EXPECT_EQ(f.check("Red"), -EINVAL);
+}
+
+TEST(FaultInjectorString, AssignErrorMessage)
+{
+  FaultInjector<std::string_view> f;
+  ASSERT_EQ(f.check("Red"), 0);
+  f.inject("Red", InjectError{-EINVAL, dpp()});
+  EXPECT_EQ(f.check("Red"), -EINVAL);
+}
+
+// test enum class as a Key type
+enum class Color { Red, Green, Blue };
+
+static std::ostream& operator<<(std::ostream& out, const Color& c) {
+  switch (c) {
+    case Color::Red: return out << "Red";
+    case Color::Green: return out << "Green";
+    case Color::Blue: return out << "Blue";
+  }
+  return out;
+}
+
+TEST(FaultInjectorEnum, Default)
+{
+  constexpr FaultInjector<Color> f;
+  EXPECT_EQ(f.check(Color::Red), 0);
+  EXPECT_EQ(f.check(Color::Green), 0);
+  EXPECT_EQ(f.check(Color::Blue), 0);
+}
+
+TEST(FaultInjectorEnum, InjectError)
+{
+  FaultInjector f{Color::Red, InjectError{-EIO}};
+  EXPECT_EQ(f.check(Color::Red), -EIO);
+  EXPECT_EQ(f.check(Color::Green), 0);
+  EXPECT_EQ(f.check(Color::Blue), 0);
+}
+
+TEST(FaultInjectorEnum, InjectErrorMessage)
+{
+  FaultInjector f{Color::Red, InjectError{-EIO, dpp()}};
+  EXPECT_EQ(f.check(Color::Red), -EIO);
+  EXPECT_EQ(f.check(Color::Green), 0);
+  EXPECT_EQ(f.check(Color::Blue), 0);
+}
+
+TEST(FaultInjectorEnum, AssignError)
+{
+  FaultInjector<Color> f;
+  ASSERT_EQ(f.check(Color::Red), 0);
+  f.inject(Color::Red, InjectError{-EINVAL});
+  EXPECT_EQ(f.check(Color::Red), -EINVAL);
+}
+
+TEST(FaultInjectorEnum, AssignErrorMessage)
+{
+  FaultInjector<Color> f;
+  ASSERT_EQ(f.check(Color::Red), 0);
+  f.inject(Color::Red, InjectError{-EINVAL, dpp()});
+  EXPECT_EQ(f.check(Color::Red), -EINVAL);
+}
+
+// test custom move-only Key type
+struct MoveOnlyKey {
+  MoveOnlyKey() = default;
+  MoveOnlyKey(const MoveOnlyKey&) = delete;
+  MoveOnlyKey& operator=(const MoveOnlyKey&) = delete;
+  MoveOnlyKey(MoveOnlyKey&&) = default;
+  MoveOnlyKey& operator=(MoveOnlyKey&&) = default;
+  ~MoveOnlyKey() = default;
+};
+
+static bool operator==(const MoveOnlyKey&, const MoveOnlyKey&) {
+  return true; // all keys are equal
+}
+static std::ostream& operator<<(std::ostream& out, const MoveOnlyKey&) {
+  return out;
+}
+
+TEST(FaultInjectorMoveOnly, Default)
+{
+  constexpr FaultInjector<MoveOnlyKey> f;
+  EXPECT_EQ(f.check(MoveOnlyKey{}), 0);
+}
+
+TEST(FaultInjectorMoveOnly, InjectError)
+{
+  FaultInjector f{MoveOnlyKey{}, InjectError{-EIO}};
+  EXPECT_EQ(f.check(MoveOnlyKey{}), -EIO);
+}
+
+TEST(FaultInjectorMoveOnly, InjectErrorMessage)
+{
+  FaultInjector f{MoveOnlyKey{}, InjectError{-EIO, dpp()}};
+  EXPECT_EQ(f.check(MoveOnlyKey{}), -EIO);
+}
+
+TEST(FaultInjectorMoveOnly, AssignError)
+{
+  FaultInjector<MoveOnlyKey> f;
+  ASSERT_EQ(f.check({}), 0);
+  f.inject({}, InjectError{-EINVAL});
+  EXPECT_EQ(f.check({}), -EINVAL);
+}
+
+TEST(FaultInjectorMoveOnly, AssignErrorMessage)
+{
+  FaultInjector<MoveOnlyKey> f;
+  ASSERT_EQ(f.check({}), 0);
+  f.inject({}, InjectError{-EINVAL, dpp()});
+  EXPECT_EQ(f.check({}), -EINVAL);
+}