crimson/os/seastore: track per-transaction conflict/replay counts

Add a num_replays counter to Transaction, incremented in
Cache::mark_transaction_conflicted whenever a transaction is marked
conflicted.

Only relevant for user-MUTATE path (do_transaction_no_callbacks).

Signed-off-by: Matan Breizman <mbreizma@redhat.com>

diff --git a/src/crimson/os/seastore/cache.cc b/src/crimson/os/seastore/cache.cc
index 8a6b29ca5b957b6f45f2d17660a68a5a298c3132..be90decca48a62f62c9d39fc9e4726f6aef16d85 100644 (file)
--- a/src/crimson/os/seastore/cache.cc
+++ b/src/crimson/os/seastore/cache.cc
@@ -1023,6 +1023,9 @@ void Cache::mark_transaction_conflicted(
   SUBTRACET(seastore_t, "", t);
   assert(!t.conflicted);
   t.conflicted = true;
+  // count is only *sampled* for the user-MUTATE do_transaction path,
+  // where the transaction is reused across retries
+  ++t.num_replays;
 
   auto& efforts = get_by_src(stats.invalidated_efforts_by_src,
                              t.get_src());

diff --git a/src/crimson/os/seastore/seastore.cc b/src/crimson/os/seastore/seastore.cc
index f66fc9fd964ea2d318e50cacf99234179bc3bddb..ffa49c02283311f266f191978f94a3daa69cd943 100644 (file)
--- a/src/crimson/os/seastore/seastore.cc
+++ b/src/crimson/os/seastore/seastore.cc
@@ -203,6 +203,28 @@ void SeaStore::Shard::register_metrics(store_index_t store_index)
     );
   }
 
+  stats.conflict_replays.buckets.resize(REPLAY_BUCKETS);
+  for (std::size_t i = 0; i < REPLAY_BUCKETS; ++i) {
+    stats.conflict_replays.buckets[i].upper_bound = i;
+    stats.conflict_replays.buckets[i].count = 0;
+  }
+  metrics.add_group(
+    "seastore",
+    {
+      sm::make_histogram(
+        "conflict_replay_distribution",
+        [this]() -> seastar::metrics::histogram& {
+          return stats.conflict_replays;
+        },
+        sm::description("distribution of per-transaction conflict/replay counts "
+                        "before commit, for user transactions submitted via "
+                        "do_transaction (the reused-transaction / "
+                        "with_repeat_trans_intr path); not all MUTATE transactions"),
+        {sm::label_instance("shard_store_index", std::to_string(store_index))}
+      )
+    }
+  );
+
   metrics.add_group(
     "seastore",
     {
@@ -1711,6 +1733,7 @@ seastar::future<> SeaStore::Shard::do_transaction_no_callbacks(
   );
 
   DEBUGT("done", *ctx.transaction);
+  add_conflict_replay_sample(ctx.transaction->get_num_replays());
   add_latency_sample(
     op_type_t::DO_TRANSACTION,
     std::chrono::steady_clock::now() - ctx.begin_timestamp);

diff --git a/src/crimson/os/seastore/seastore.h b/src/crimson/os/seastore/seastore.h
index 54dd67c783fdff7e8c41c4268721244ed2d19344..e2bbc576a4361884c03a0a3426f46ad2671429c0 100644 (file)
--- a/src/crimson/os/seastore/seastore.h
+++ b/src/crimson/os/seastore/seastore.h
@@ -437,9 +437,12 @@ public:
       100000
     };
 
+    // Buckets for the per-transaction conflict/replay distribution.
+    static constexpr std::size_t REPLAY_BUCKETS = 16;
 
     struct {
       std::array<seastar::metrics::histogram, LAT_MAX> op_lat;
+      seastar::metrics::histogram conflict_replays;
     } stats;
 
     seastar::metrics::histogram& get_latency(
@@ -467,6 +470,21 @@ public:
       }
     }
 
+    // Record how many times a just-completed transaction was conflicted/replayed.
+    // Called only from the do_transaction_no_callbacks() completion path.
+    void add_conflict_replay_sample(std::size_t num_replays) {
+      auto& hist = stats.conflict_replays;
+      if (hist.buckets.empty()) {
+        // register_metrics() did not run (store inactive); nothing to record.
+        return;
+      }
+      std::size_t idx = num_replays < REPLAY_BUCKETS ?
+        num_replays : REPLAY_BUCKETS - 1;
+      ++hist.buckets[idx].count;
+      ++hist.sample_count;
+      hist.sample_sum += num_replays;
+    }
+
     /*
      * omaptree interfaces
      */

diff --git a/src/crimson/os/seastore/transaction.h b/src/crimson/os/seastore/transaction.h
index a6e2e1c72722449a21eb15f68bebf361e2970f83..8fa2d011f5e12dcf055b00c85742d09f2a85b01c 100644 (file)
--- a/src/crimson/os/seastore/transaction.h
+++ b/src/crimson/os/seastore/transaction.h
@@ -464,6 +464,12 @@ public:
     return conflicted;
   }
 
+  // Number of times this transaction was conflicted and replayed before
+  // finally committing. do_transaction_no_callbacks() (user MUTATE writes)
+  std::size_t get_num_replays() const {
+    return num_replays;
+  }
+
   auto &get_handle() {
     return handle;
   }
@@ -882,6 +888,8 @@ private:
 
   bool conflicted = false;
 
+  std::size_t num_replays = 0;
+
   bool has_reset = false;
 
   OrderingHandle handle;