crimson/onode-staged-tree: validate insert/lookup/erase with eagain

Signed-off-by: Yingxin Cheng <yingxin.cheng@intel.com>

diff --git a/src/crimson/os/seastore/onode_manager/staged-fltree/tree_utils.h b/src/crimson/os/seastore/onode_manager/staged-fltree/tree_utils.h
index 1f0d284659b5850b09611f201f7ae04940d1a8c7..31b4270db5ae659726bfe2dea343107c79134005 100644 (file)
--- a/src/crimson/os/seastore/onode_manager/staged-fltree/tree_utils.h
+++ b/src/crimson/os/seastore/onode_manager/staged-fltree/tree_utils.h
@@ -290,6 +290,34 @@ class TreeBuilder {
     return tree->mkfs(t);
   }
 
+  eagain_future<BtreeCursor> insert_one(
+      Transaction& t, const iterator_t& iter_rd) {
+    auto p_kv = *iter_rd;
+    logger().debug("[{}] insert {} -> {}",
+                   iter_rd - kvs.random_begin(),
+                   key_hobj_t{p_kv->key},
+                   p_kv->value);
+    return tree->insert(
+        t, p_kv->key, {p_kv->value.get_payload_size()}
+    ).safe_then([&t, this, p_kv](auto ret) {
+      auto& [cursor, success] = ret;
+      initialize_cursor_from_item(t, p_kv->key, p_kv->value, cursor, success);
+#ifndef NDEBUG
+      validate_cursor_from_item(p_kv->key, p_kv->value, cursor);
+      return tree->find(t, p_kv->key
+      ).safe_then([this, cursor, p_kv](auto cursor_) mutable {
+        assert(!cursor_.is_end());
+        ceph_assert(cursor_.get_ghobj() == p_kv->key);
+        ceph_assert(cursor_.value() == cursor.value());
+        validate_cursor_from_item(p_kv->key, p_kv->value, cursor_);
+        return cursor;
+      });
+#else
+      return eagain_ertr::make_ready_future<BtreeCrusor>(cursor);
+#endif
+    });
+  }
+
   eagain_future<> insert(Transaction& t) {
     auto ref_kv_iter = seastar::make_lw_shared<iterator_t>();
     *ref_kv_iter = kvs.random_begin();
@@ -302,38 +330,16 @@ class TreeBuilder {
         std::chrono::duration<double> duration = mono_clock::now() - start_time;
         logger().warn("Insert done! {}s", duration.count());
         return seastar::make_ready_future<bool>(true);
-      }
-      auto p_kv = **ref_kv_iter;
-      logger().debug("[{}] insert {} -> {}",
-                     (*ref_kv_iter) - kvs.random_begin(),
-                     key_hobj_t{p_kv->key},
-                     p_kv->value);
-      return tree->insert(
-          t, p_kv->key, {p_kv->value.get_payload_size()}
-      ).safe_then([&t, this, cursors, ref_kv_iter](auto ret) {
-        auto p_kv = **ref_kv_iter;
-        auto& [cursor, success] = ret;
-        initialize_cursor_from_item(t, p_kv->key, p_kv->value, cursor, success);
-        if constexpr (TRACK) {
-          cursors->emplace_back(cursor);
-        }
-#ifndef NDEBUG
-        validate_cursor_from_item(p_kv->key, p_kv->value, cursor);
-        return tree->find(t, p_kv->key
-        ).safe_then([this, cursor, ref_kv_iter](auto cursor_) mutable {
-          assert(!cursor_.is_end());
-          auto p_kv = **ref_kv_iter;
-          ceph_assert(cursor_.get_ghobj() == p_kv->key);
-          ceph_assert(cursor_.value() == cursor.value());
-          validate_cursor_from_item(p_kv->key, p_kv->value, cursor_);
+      } else {
+        return insert_one(t, *ref_kv_iter
+        ).safe_then([cursors, ref_kv_iter] (auto cursor) {
+          if constexpr (TRACK) {
+            cursors->emplace_back(cursor);
+          }
           ++(*ref_kv_iter);
           return seastar::make_ready_future<bool>(false);
         });
-#else
-        ++(*ref_kv_iter);
-        return seastar::make_ready_future<bool>(false);
-#endif
-      });
+      }
     }).safe_then([&t, this, cursors, ref_kv_iter] {
       if (!cursors->empty()) {
         logger().info("Verifing tracked cursors ...");
@@ -367,11 +373,31 @@ class TreeBuilder {
     });
   }
 
+  eagain_future<> erase_one(
+      Transaction& t, const iterator_t& iter_rd) {
+    auto p_kv = *iter_rd;
+    logger().debug("[{}] erase {} -> {}",
+                   iter_rd - kvs.random_begin(),
+                   key_hobj_t{p_kv->key},
+                   p_kv->value);
+    return tree->erase(t, p_kv->key
+    ).safe_then([&t, this, p_kv] (auto size) {
+      ceph_assert(size == 1);
+#ifndef NDEBUG
+      return tree->contains(t, p_kv->key
+      ).safe_then([] (bool ret) {
+        ceph_assert(ret == false);
+      });
+#else
+      return eagain_ertr::now();
+#endif
+    });
+  }
+
   eagain_future<> erase(Transaction& t, std::size_t erase_size) {
     assert(erase_size <= kvs.size());
     kvs.shuffle();
-    auto begin = kvs.random_begin();
-    auto end = begin + erase_size;
+    auto erase_end = kvs.random_begin() + erase_size;
     auto ref_kv_iter = seastar::make_lw_shared<iterator_t>();
     auto cursors = seastar::make_lw_shared<std::map<ghobject_t, BtreeCursor>>();
     return seastar::now().then([&t, this, cursors, ref_kv_iter] {
@@ -398,42 +424,30 @@ class TreeBuilder {
       } else {
         return eagain_ertr::now();
       }
-    }).safe_then([&t, this, ref_kv_iter, begin, end] {
-      *ref_kv_iter = begin;
-      logger().warn("start erasing {}/{} kvs ...", end - begin, kvs.size());
+    }).safe_then([&t, this, ref_kv_iter, erase_end] {
+      *ref_kv_iter = kvs.random_begin();
+      logger().warn("start erasing {}/{} kvs ...",
+                    erase_end - kvs.random_begin(), kvs.size());
       auto start_time = mono_clock::now();
       return crimson::do_until([&t, this, ref_kv_iter,
-                                start_time, begin, end] () -> eagain_future<bool> {
-        if (*ref_kv_iter == end) {
+                                start_time, erase_end] () -> eagain_future<bool> {
+        if (*ref_kv_iter == erase_end) {
           std::chrono::duration<double> duration = mono_clock::now() - start_time;
           logger().warn("Erase done! {}s", duration.count());
           return seastar::make_ready_future<bool>(true);
-        }
-        auto p_kv = **ref_kv_iter;
-        logger().debug("[{}] erase {} -> {}",
-                       (*ref_kv_iter) - begin,
-                       key_hobj_t{p_kv->key},
-                       p_kv->value);
-        return tree->erase(t, p_kv->key).safe_then([&t, this, ref_kv_iter] (auto size) {
-          ceph_assert(size == 1);
-#ifndef NDEBUG
-          auto p_kv = **ref_kv_iter;
-          return tree->contains(t, p_kv->key).safe_then([ref_kv_iter] (bool ret) {
-            ceph_assert(ret == false);
+        } else {
+          return erase_one(t, *ref_kv_iter
+          ).safe_then([ref_kv_iter] {
             ++(*ref_kv_iter);
             return seastar::make_ready_future<bool>(false);
           });
-#else
-          ++(*ref_kv_iter);
-          return seastar::make_ready_future<bool>(false);
-#endif
-        });
+        }
       });
-    }).safe_then([this, cursors, ref_kv_iter, begin, end] {
+    }).safe_then([this, cursors, ref_kv_iter, erase_end] {
       if constexpr (TRACK) {
         logger().info("Verifing tracked cursors ...");
-        *ref_kv_iter = begin;
-        while (*ref_kv_iter != end) {
+        *ref_kv_iter = kvs.random_begin();
+        while (*ref_kv_iter != erase_end) {
           auto p_kv = **ref_kv_iter;
           auto c_it = cursors->find(p_kv->key);
           ceph_assert(c_it != cursors->end());
@@ -442,7 +456,7 @@ class TreeBuilder {
           ++(*ref_kv_iter);
         }
       }
-      kvs.erase_from_random(begin, end);
+      kvs.erase_from_random(kvs.random_begin(), erase_end);
       if constexpr (TRACK) {
         *ref_kv_iter = kvs.begin();
         for (auto& [k, c] : *cursors) {
@@ -471,23 +485,33 @@ class TreeBuilder {
     tree.emplace(std::move(nm));
   }
 
-  eagain_future<> validate(Transaction& t) {
-    return seastar::async([this, &t] {
-      logger().info("Verifing inserted ...");
-      for (auto& p_kv : kvs) {
-        auto cursor = tree->find(t, p_kv->key).unsafe_get0();
+  eagain_future<> validate_one(
+      Transaction& t, const iterator_t& iter_seq) {
+    assert(iter_seq != kvs.end());
+    auto next_iter = iter_seq + 1;
+    auto p_kv = *iter_seq;
+    return tree->find(t, p_kv->key
+    ).safe_then([p_kv, &t] (auto cursor) {
+      validate_cursor_from_item(p_kv->key, p_kv->value, cursor);
+      return cursor.get_next(t);
+    }).safe_then([next_iter, this] (auto cursor) {
+      if (next_iter == kvs.end()) {
+        ceph_assert(cursor.is_end());
+      } else {
+        auto p_kv = *next_iter;
         validate_cursor_from_item(p_kv->key, p_kv->value, cursor);
       }
+    });
+  }
 
-      logger().info("Verifing range query ...");
-      auto cursor = tree->begin(t).unsafe_get0();
-      for (auto& p_kv : kvs) {
-        assert(!cursor.is_end());
-        validate_cursor_from_item(p_kv->key, p_kv->value, cursor);
-        cursor = cursor.get_next(t).unsafe_get0();
+  eagain_future<> validate(Transaction& t) {
+    return seastar::async([this, &t] {
+      logger().info("Verifing inserted ...");
+      auto iter = kvs.begin();
+      while (iter != kvs.end()) {
+        validate_one(t, iter).unsafe_get0();
+        ++iter;
       }
-      assert(cursor.is_end());
-
       logger().info("Verify done!");
     });
   }

diff --git a/src/test/crimson/seastore/onode_tree/test_staged_fltree.cc b/src/test/crimson/seastore/onode_tree/test_staged_fltree.cc
index 5d11f8c9764e19af47234803e08410e0994eca6a..68beedc274f0d83c4ef3318d445d166257ac7c37 100644 (file)
--- a/src/test/crimson/seastore/onode_tree/test_staged_fltree.cc
+++ b/src/test/crimson/seastore/onode_tree/test_staged_fltree.cc
@@ -10,8 +10,8 @@
 
 #include "crimson/common/log.h"
 #include "crimson/os/seastore/onode_manager/staged-fltree/node.h"
-#include "crimson/os/seastore/onode_manager/staged-fltree/node_extent_manager.h"
 #include "crimson/os/seastore/onode_manager/staged-fltree/node_extent_manager/dummy.h"
+#include "crimson/os/seastore/onode_manager/staged-fltree/node_extent_manager/seastore.h"
 #include "crimson/os/seastore/onode_manager/staged-fltree/node_layout.h"
 #include "crimson/os/seastore/onode_manager/staged-fltree/tree.h"
 #include "crimson/os/seastore/onode_manager/staged-fltree/tree_utils.h"
@@ -1609,3 +1609,113 @@ TEST_F(d_seastore_tm_test_t, 6_random_tree_insert_erase)
     tree.reset();
   });
 }
+
+TEST_F(d_seastore_tm_test_t, 7_tree_insert_erase_eagain)
+{
+  run_async([this] {
+    constexpr double EAGAIN_PROBABILITY = 0.1;
+    constexpr bool TRACK_CURSORS = false;
+    auto kvs = KVPool<test_item_t>::create_raw_range(
+        {8, 11, 64, 256, 301, 320},
+        {8, 16, 128, 512, 576, 640},
+        {0, 8}, {0, 10}, {0, 4});
+    auto moved_nm = NodeExtentManager::create_seastore(
+        *tm, L_ADDR_MIN, EAGAIN_PROBABILITY);
+    auto p_nm = static_cast<SeastoreNodeExtentManager<true>*>(moved_nm.get());
+    auto tree = std::make_unique<TreeBuilder<TRACK_CURSORS, test_item_t>>(
+        kvs, std::move(moved_nm));
+    unsigned num_ops = 0;
+    unsigned num_ops_eagain = 0;
+
+    // bootstrap
+    ++num_ops;
+    repeat_eagain([this, &tree, &num_ops_eagain] {
+      ++num_ops_eagain;
+      auto t = tm->create_transaction();
+      return tree->bootstrap(*t
+      ).safe_then([this, t = std::move(t)] () mutable {
+        return tm->submit_transaction(std::move(t));
+      });
+    }).unsafe_get0();
+    segment_cleaner->run_until_halt().get0();
+
+    // insert
+    logger().warn("start inserting {} kvs ...", kvs.size());
+    {
+      auto iter = kvs.random_begin();
+      while (iter != kvs.random_end()) {
+        ++num_ops;
+        repeat_eagain([this, &tree, &num_ops_eagain, &iter] {
+          ++num_ops_eagain;
+          auto t = tm->create_transaction();
+          return tree->insert_one(*t, iter
+          ).safe_then([this, t = std::move(t)] (auto cursor) mutable {
+            cursor.invalidate();
+            return tm->submit_transaction(std::move(t));
+          });
+        }).unsafe_get0();
+        segment_cleaner->run_until_halt().get0();
+        ++iter;
+      }
+    }
+
+    {
+      p_nm->set_generate_eagain(false);
+      auto t = tm->create_transaction();
+      tree->get_stats(*t).unsafe_get0();
+      p_nm->set_generate_eagain(true);
+    }
+
+    // lookup
+    logger().warn("start lookup {} kvs ...", kvs.size());
+    {
+      auto iter = kvs.begin();
+      while (iter != kvs.end()) {
+        ++num_ops;
+        repeat_eagain2([this, &tree, &num_ops_eagain, &iter] {
+          ++num_ops_eagain;
+          auto t = tm->create_transaction();
+          return tree->validate_one(*t, iter
+          ).safe_then([t=std::move(t)]{});
+        }).get0();
+        ++iter;
+      }
+    }
+
+    // erase
+    logger().warn("start erase {} kvs ...", kvs.size());
+    {
+      kvs.shuffle();
+      auto iter = kvs.random_begin();
+      while (iter != kvs.random_end()) {
+        ++num_ops;
+        repeat_eagain([this, &tree, &num_ops_eagain, &iter] {
+          ++num_ops_eagain;
+          auto t = tm->create_transaction();
+          return tree->erase_one(*t, iter
+          ).safe_then([this, t = std::move(t)] () mutable {
+            return tm->submit_transaction(std::move(t));
+          });
+        }).unsafe_get0();
+        segment_cleaner->run_until_halt().get0();
+        ++iter;
+      }
+      kvs.erase_from_random(kvs.random_begin(), kvs.random_end());
+    }
+
+    {
+      p_nm->set_generate_eagain(false);
+      auto t = tm->create_transaction();
+      tree->get_stats(*t).unsafe_get0();
+      tree->validate(*t).unsafe_get0();
+      EXPECT_EQ(tree->height(*t).unsafe_get0(), 1);
+    }
+
+    // we can adjust EAGAIN_PROBABILITY to get a proper eagain_rate
+    double eagain_rate = num_ops_eagain;
+    eagain_rate /= num_ops;
+    logger().info("eagain rate: {}", eagain_rate);
+
+    tree.reset();
+  });
+}