crimson/os/alienstore: improve alienstore's write parallelism

replace the grand per store tp_mutex with a finer grained per-collection
lock for better concurrency

Signed-off-by: Xuehan Xu <xxhdx1985126@gmail.com>

diff --git a/src/crimson/os/alienstore/alien_collection.h b/src/crimson/os/alienstore/alien_collection.h
index 98b8fdef44ddeda8ffd3188727556e0543b9a318..538db3dae669d29c63092ca5668e88b009b05338 100644 (file)
--- a/src/crimson/os/alienstore/alien_collection.h
+++ b/src/crimson/os/alienstore/alien_collection.h
@@ -19,8 +19,14 @@ public:
 
   ~AlienCollection() {}
 
+  template <typename Func, typename Result = std::invoke_result_t<Func>>
+  seastar::futurize_t<Result> with_lock(Func&& func) {
+    return seastar::with_lock(mutex, std::forward<Func>(func));
+  }
+
 private:
   ObjectStore::CollectionHandle collection;
+  seastar::shared_mutex mutex;
   friend AlienStore;
 };
 }

diff --git a/src/crimson/os/alienstore/alien_store.cc b/src/crimson/os/alienstore/alien_store.cc
index 8b4e666b3f68d43eb6ae56f8d84885ed3616ef4b..31b29d29acba714dca734669e00136d34537723a 100644 (file)
--- a/src/crimson/os/alienstore/alien_store.cc
+++ b/src/crimson/os/alienstore/alien_store.cc
@@ -363,7 +363,8 @@ seastar::future<> AlienStore::do_transaction(CollectionRef ch,
     std::move(done),
     [this, ch, id] (auto &txn, auto &done) {
       return seastar::with_gate(transaction_gate, [this, ch, id, &txn, &done] {
-       return tp_mutex.lock().then ([this, ch, id, &txn, &done] {
+       AlienCollection* alien_coll = static_cast<AlienCollection*>(ch.get());
+       return alien_coll->with_lock([this, ch, id, &txn, &done] {
          Context *crimson_wrapper =
            ceph::os::Transaction::collect_all_contexts(txn);
          return tp->submit([this, ch, id, crimson_wrapper, &txn, &done] {
@@ -373,7 +374,6 @@ seastar::future<> AlienStore::do_transaction(CollectionRef ch,
          });
        }).then([this, &done] (int r) {
          assert(r == 0);
-         tp_mutex.unlock();
          return done.get_future();
        });
       });

diff --git a/src/crimson/os/alienstore/alien_store.h b/src/crimson/os/alienstore/alien_store.h
index 92739340e782c8360a1c4b272c9291863313e85b..8e67d08f24ecf79f6cd7719b61649567ef52022d 100644 (file)
--- a/src/crimson/os/alienstore/alien_store.h
+++ b/src/crimson/os/alienstore/alien_store.h
@@ -120,6 +120,5 @@ private:
   std::unique_ptr<CephContext> cct;
   seastar::gate transaction_gate;
   std::unordered_map<coll_t, CollectionRef> coll_map;
-  seastar::shared_mutex tp_mutex;
 };
 }