]> git.apps.os.sepia.ceph.com Git - ceph.git/commitdiff
test/os/bluestore: unit tests for common/regression_utils.cc and os/bluestore/BlueSto... 43413/head
authorEsmaeil Mirvakili <smirvaki@ucsc.edu>
Tue, 1 Feb 2022 10:23:14 +0000 (02:23 -0800)
committeresmaeil-mirvakili <smirvaki@ucsc.edu>
Tue, 22 Mar 2022 14:22:12 +0000 (07:22 -0700)
Signed-off-by: Esmaeil Mirvakili <smirvaki@ucsc.edu>d
src/test/common/CMakeLists.txt
src/test/common/test_regression_utils.cc [new file with mode: 0644]
src/test/os/CMakeLists.txt
src/test/os/bluestore/CMakeLists.txt [new file with mode: 0644]
src/test/os/bluestore/TestBlueStoreSlowFastCoDel.cc [new file with mode: 0644]

index 0e84b3d722815a691a9142a0ab365ea75ffc834d..3b6354062243cd14e6360a217fa17a6a1300763b 100644 (file)
@@ -155,6 +155,11 @@ add_executable(unittest_random
 add_ceph_unittest(unittest_random)
 target_link_libraries(unittest_random Boost::random)
 
+# unittest_regression_utils
+add_executable(unittest_regression_utils test_regression_utils.cc)
+add_ceph_unittest(unittest_regression_utils)
+target_link_libraries(unittest_regression_utils ceph-common global)
+
 # unittest_throttle
 add_executable(unittest_throttle
   Throttle.cc
diff --git a/src/test/common/test_regression_utils.cc b/src/test/common/test_regression_utils.cc
new file mode 100644 (file)
index 0000000..6304e5d
--- /dev/null
@@ -0,0 +1,89 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include <gtest/gtest.h>
+#include <vector>
+#include <cmath>
+#include "common/regression_utils.h"
+#include <boost/numeric/ublas/matrix.hpp>
+#include <boost/numeric/ublas/io.hpp>
+
+using namespace boost::numeric::ublas;
+
+std::vector<double> generate_rand_vector(int size, int max_value) {
+  std::srand(std::time(0));
+  std::vector<double> rand_vec;
+  for (int i = 0; i < size; i++) {
+    double rand_value = std::rand() % max_value;
+    rand_vec.push_back(rand_value);
+  }
+  return rand_vec;
+}
+
+matrix<double> generate_rand_matrix(int size1, int size2, int max_value) {
+  std::srand(std::time(0));
+  matrix<double> rand_m(size1, size2);
+  for (int i = 0; i < size1; i++) {
+    for (int j = 0; j < size2; j++) {
+      double rand_value = std::rand() % max_value;
+      rand_m(i, j) = rand_value;
+    }
+  }
+  return rand_m;
+}
+
+std::vector<double> exp_vector(std::vector<double> x) {
+  std::vector<double> exp_vec;
+  for (int i = 0; i < x.size(); i++) {
+    exp_vec.push_back(std::exp(x[i]));
+  }
+  return exp_vec;
+}
+
+bool is_almost_equal(double x1, double x2, double precision) {
+  if (std::abs(x1 - x2) < precision) {
+    return true;
+  }
+  return false;
+}
+
+TEST(matrix_op, matrix_inverse) {
+  int matrix_size = 2;    // has to be 2x2
+  matrix<double> random_square_m = generate_rand_matrix(matrix_size, matrix_size, 1000);
+  matrix<double> random_square_m_inv = ceph::matrix_inverse(random_square_m);
+  // the inverse matrix should have the same size
+  ASSERT_EQ(random_square_m_inv.size1(), random_square_m.size1());
+  ASSERT_EQ(random_square_m_inv.size2(), random_square_m.size2());
+  matrix<double> matrix_prod = prod(random_square_m, random_square_m_inv);
+  // the product should be an identity matrix
+  for ( int i = 0; i < matrix_prod.size1(); i++){
+    for (int j = 0; j < matrix_prod.size2(); j++){
+      if (i == j) {
+        ASSERT_TRUE(is_almost_equal(matrix_prod(i, j), 1, 1e-9));    // i == j -> 1
+      } else {
+        ASSERT_TRUE(is_almost_equal(matrix_prod(i, j), 0, 1e-9));    // i <> j -> 0
+      }
+    }
+  }
+}
+
+TEST(regression, log_regression) {
+  // y = ln(x)
+  std::vector<double> y = generate_rand_vector(200, 100);
+  std::vector<double> x = exp_vector(y);
+
+  double theta[2];    // y = theta[0] + theta[1] * ln(x)
+  ceph::regression(x, y, theta);
+  ASSERT_TRUE(is_almost_equal(theta[0], 0, 1e-9)); // theta[0] = 0
+  ASSERT_TRUE(is_almost_equal(theta[1], 1, 1e-9)); // theta[1] = 1
+}
+
+TEST(regression, find_slope_location) {
+  // y = ln(x)
+  std::vector<double> y = generate_rand_vector(200, 100);
+  std::vector<double> x = exp_vector(y);
+
+  double target_slope = 5;
+  double x_target = ceph::find_slope_on_curve(x, y, target_slope);
+  ASSERT_TRUE(is_almost_equal(x_target, 0.2, 1e-9));  // y'(0.2) = 5
+}
index 35eb8f1178026d8746a289d2eeb10c9672692d59..c7baa16447664237c81d855a129e4a24018992aa 100644 (file)
@@ -5,3 +5,4 @@ add_executable(unittest_lfnindex
 add_ceph_unittest(unittest_lfnindex)
 target_link_libraries(unittest_lfnindex os global)
 
+add_subdirectory(bluestore)
diff --git a/src/test/os/bluestore/CMakeLists.txt b/src/test/os/bluestore/CMakeLists.txt
new file mode 100644 (file)
index 0000000..86de05a
--- /dev/null
@@ -0,0 +1,15 @@
+# unittest_slow_fast_codel
+add_executable(unittest_slow_fast_codel
+  TestBlueStoreSlowFastCoDel.cc
+  )
+add_ceph_unittest(unittest_slow_fast_codel)
+target_link_libraries(unittest_slow_fast_codel
+  rados_test_stub
+  librados
+  global
+  radostest-cxx
+  GTest::GTest
+  ceph-common
+  os
+  global
+  )
diff --git a/src/test/os/bluestore/TestBlueStoreSlowFastCoDel.cc b/src/test/os/bluestore/TestBlueStoreSlowFastCoDel.cc
new file mode 100644 (file)
index 0000000..04e3318
--- /dev/null
@@ -0,0 +1,191 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include <filesystem>
+#include <iostream>
+#include <unistd.h>
+#include <mutex>
+#include <cmath>
+#include <vector>
+#include <condition_variable>
+#include <cmath>
+#include <cstdlib>
+
+#include "gtest/gtest.h"
+#include "include/Context.h"
+
+#include "common/ceph_time.h"
+#include "os/bluestore/BlueStoreSlowFastCoDel.h"
+
+
+static int64_t milliseconds_to_nanoseconds(int64_t ms) {
+  return ms * 1000.0 * 1000.0;
+}
+
+static double nanoseconds_to_milliseconds(int64_t ms) {
+  return ms / (1000.0 * 1000.0);
+}
+
+class BlueStoreSlowFastCoDelMock : public BlueStoreSlowFastCoDel {
+public:
+  BlueStoreSlowFastCoDelMock(
+    CephContext *_cct,
+    std::function<void(int64_t)> _bluestore_budget_reset_callback,
+    std::function<int64_t()> _get_kv_throttle_current,
+    std::mutex &_iteration_mutex,
+    std::condition_variable &_iteration_cond,
+    int64_t _target_latency,
+    int64_t _fast_interval,
+    int64_t _slow_interval,
+    double _target_slope
+  ) : BlueStoreSlowFastCoDel(_cct, _bluestore_budget_reset_callback,
+                             _get_kv_throttle_current),
+      iteration_mutex(_iteration_mutex), iteration_cond(_iteration_cond),
+      test_target_latency(_target_latency), test_fast_interval(_fast_interval),
+      test_slow_interval(_slow_interval), test_target_slope(_target_slope) {
+    init_test();
+  }
+
+  void init_test() {
+    std::lock_guard l(register_lock);
+    activated = true;
+    target_slope = test_target_slope;
+    slow_interval = test_slow_interval;
+    initial_fast_interval = test_fast_interval;
+    min_target_latency = milliseconds_to_nanoseconds(1);
+    initial_target_latency = test_target_latency;
+    max_target_latency = milliseconds_to_nanoseconds(500);
+    initial_bluestore_budget = 100 * 1024;
+    min_bluestore_budget = 10 * 1024;
+    bluestore_budget_increment = 1024;
+    regression_history_size = 5;
+    bluestore_budget = initial_bluestore_budget;
+    min_bluestore_budget = initial_bluestore_budget;
+    max_queue_length = min_bluestore_budget;
+    fast_interval = initial_fast_interval;
+    target_latency = initial_target_latency;
+    min_latency = INITIAL_LATENCY_VALUE;
+    slow_interval_registered_bytes = 0;
+    regression_throughput_history.clear();
+    regression_target_latency_history.clear();
+    slow_interval_start = ceph::mono_clock::zero();
+  }
+
+  std::vector <int64_t> target_latency_vector;
+
+protected:
+  std::mutex &iteration_mutex;
+  std::condition_variable &iteration_cond;
+  int64_t test_target_latency;
+  int64_t test_fast_interval;
+  int64_t test_slow_interval;
+  double test_target_slope;
+
+  void on_fast_interval_finished() override {
+    std::unique_lock <std::mutex> locker(iteration_mutex);
+    iteration_cond.notify_one();
+  }
+
+  void on_slow_interval_finished() override {
+    target_latency_vector.push_back(target_latency);
+  }
+};
+
+class TestSlowFastCoDel : public ::testing::Test {
+public:
+  CephContext *ceph_context = nullptr;
+  BlueStoreSlowFastCoDelMock *slow_fast_codel = nullptr;
+  int64_t test_throttle_budget = 0;
+  std::mutex iteration_mutex;
+  std::condition_variable iteration_cond;
+  int64_t target_latency = milliseconds_to_nanoseconds(50);
+  int64_t fast_interval = milliseconds_to_nanoseconds(100);
+  int64_t slow_interval = milliseconds_to_nanoseconds(400);
+  double target_slope = 1;
+
+  std::vector <int64_t> target_latency_vector;
+  std::vector <int64_t> txc_size_vector;
+
+  TestSlowFastCoDel() {}
+
+  ~TestSlowFastCoDel() {}
+
+  static void SetUpTestCase() {}
+
+  static void TearDownTestCase() {}
+
+  void SetUp() override {
+    ceph_context = (new CephContext(CEPH_ENTITY_TYPE_ANY))->get();
+  }
+
+  void create_bluestore_slow_fast_codel() {
+    slow_fast_codel = new BlueStoreSlowFastCoDelMock(
+            ceph_context,
+            [this](int64_t x) mutable {
+                this->test_throttle_budget = x;
+            },
+            [this]() mutable {
+                return this->test_throttle_budget;
+            },
+            iteration_mutex,
+            iteration_cond,
+            target_latency,
+            fast_interval,
+            slow_interval,
+            target_slope);
+  }
+
+  void TearDown() override {
+    if (slow_fast_codel)
+      delete slow_fast_codel;
+  }
+
+  void test_codel() {
+    int64_t max_iterations = 50;
+    int iteration_timeout = 1; // 1 sec
+    int txc_num = 4;
+    for (int iteration = 0; iteration < max_iterations; iteration++) {
+      std::unique_lock <std::mutex> locker(iteration_mutex);
+      bool violation = iteration % 2 == 1;
+      auto budget_tmp = test_throttle_budget;
+      auto target = slow_fast_codel->get_target_latency();
+      double target_throughput =
+        (target_slope * nanoseconds_to_milliseconds(target_latency)) *
+        std::log(nanoseconds_to_milliseconds(target) * 1.0);
+      int64_t txc_size =
+              (nanoseconds_to_milliseconds(slow_interval) *
+               target_throughput) /
+              (1000 * txc_num * (slow_interval / fast_interval));
+      txc_size *= 1024 * 1024;
+      txc_size_vector.push_back(txc_size);
+      target_latency_vector.push_back(target);
+      for (int i = 0; i < txc_num; i++) {
+        auto time = ceph::mono_clock::now();
+        if (violation) {
+          int rand_ms = std::rand() % 1000 + 1000;
+          int64_t time_diff = milliseconds_to_nanoseconds(rand_ms);
+          time = time - std::chrono::nanoseconds(target + time_diff);
+        }
+        slow_fast_codel->update_from_txc_info(time, txc_size);
+      }
+      if (iteration_cond.wait_for(
+              locker, std::chrono::seconds(iteration_timeout)) ==
+          std::cv_status::timeout) {
+        ASSERT_TRUE(false) << "Test timeout.";
+        return;
+      }
+      if (violation) {
+        ASSERT_LT(test_throttle_budget, budget_tmp);
+      } else {
+        ASSERT_GT(test_throttle_budget, budget_tmp);
+      }
+    }
+
+    ASSERT_TRUE(slow_fast_codel->target_latency_vector.size() > 0);
+  }
+};
+
+TEST_F(TestSlowFastCoDel, test1) {
+  create_bluestore_slow_fast_codel();
+  test_codel();
+}