mds: optimize batch backtrace store

Call backtrace stored callback in batch. This avoids taking mds_lock
for each inode.

Signed-off-by: Erqi Chen <chenerqi@gmail.com>

diff --git a/src/mds/CInode.cc b/src/mds/CInode.cc
index b2c25d57831270c7cb568728fdf4cd6ff5d012ba..a9724952fd72989ee32a9eac3f6a89432d5abb8f 100644 (file)
--- a/src/mds/CInode.cc
+++ b/src/mds/CInode.cc
@@ -52,14 +52,14 @@
 #undef dout_prefix
 #define dout_prefix *_dout << "mds." << mdcache->mds->get_nodeid() << ".cache.ino(" << ino() << ") "
 
-void CInodeCommitOperation::update(ObjectOperation &op, inode_backtrace_t *bt) {
+void CInodeCommitOperation::update(ObjectOperation &op, inode_backtrace_t &bt) {
   using ceph::encode;
 
   op.priority = priority;
   op.create(false);
 
   bufferlist parent_bl;
-  encode(*bt, parent_bl);
+  encode(bt, parent_bl);
   op.setxattr("parent", parent_bl);
 
   // for the old pool there is no need to update the layout
@@ -1318,23 +1318,10 @@ struct C_IO_Inode_StoredBacktrace : public CInodeIOContext {
   }
 };
 
-struct C_IO_Inode_CommitBacktrace : public Context {
-  CInode *in;
-  version_t version;
-  MDSContext *fin;
-  std::vector<CInodeCommitOperation> ops_vec;
-  inode_backtrace_t bt;
-
-  C_IO_Inode_CommitBacktrace(CInode *i, version_t v, MDSContext *f) :
-    in(i), version(v), fin(f) { }
-  void finish(int r) override {
-    in->_commit_ops(r, version, fin, ops_vec, &bt);
-  }
-};
 
-void CInode::_commit_ops(int r, version_t version, MDSContext *fin,
+void CInode::_commit_ops(int r, C_GatherBuilder &gather_bld,
                          std::vector<CInodeCommitOperation> &ops_vec,
-                         inode_backtrace_t *bt)
+                         inode_backtrace_t &bt)
 {
   dout(10) << __func__ << dendl;
 
@@ -1343,12 +1330,6 @@ void CInode::_commit_ops(int r, version_t version, MDSContext *fin,
     return;
   }
 
-  C_GatherBuilder gather(g_ceph_context,
-                         new C_OnFinisher(new C_IO_Inode_StoredBacktrace(this,
-                                                                         version,
-                                                                         fin),
-                         mdcache->mds->finisher));
-
   SnapContext snapc;
   object_t oid = get_object_name(ino(), frag_t(), "");
 
@@ -1358,9 +1339,8 @@ void CInode::_commit_ops(int r, version_t version, MDSContext *fin,
     op.update(obj_op, bt);
     mdcache->mds->objecter->mutate(oid, oloc, obj_op, snapc,
                                    ceph::real_clock::now(),
-                                   0, gather.new_sub());
+                                   0, gather_bld.new_sub());
   }
-  gather.activate();
 }
 
 void CInode::_store_backtrace(std::vector<CInodeCommitOperation> &ops_vec,
@@ -1401,11 +1381,18 @@ void CInode::_store_backtrace(std::vector<CInodeCommitOperation> &ops_vec,
 void CInode::store_backtrace(MDSContext *fin, int op_prio)
 {
   std::vector<CInodeCommitOperation> ops_vec;
+  inode_backtrace_t bt;
   auto version = get_inode()->backtrace_version;
 
-  auto c = new C_IO_Inode_CommitBacktrace(this, version, fin);
-  _store_backtrace(c->ops_vec, c->bt, op_prio);
-  mdcache->mds->finisher->queue(c);
+  _store_backtrace(ops_vec, bt, op_prio);
+
+  C_GatherBuilder gather(g_ceph_context,
+                        new C_OnFinisher(
+                          new C_IO_Inode_StoredBacktrace(this, version, fin),
+                          mdcache->mds->finisher));
+  _commit_ops(0, gather, ops_vec, bt);
+  ceph_assert(gather.has_subs());
+  gather.activate();
 }
 
 void CInode::store_backtrace(CInodeCommitOperations &op, int op_prio)

diff --git a/src/mds/CInode.h b/src/mds/CInode.h
index f792e9f133e311e7bde37e69b8f56c343ba048dc..66ea728cc2d50da802e273d2279bc20384fd9be3 100644 (file)
--- a/src/mds/CInode.h
+++ b/src/mds/CInode.h
@@ -70,7 +70,7 @@ public:
       update_layout = true;
   }
 
-  void update(ObjectOperation &op, inode_backtrace_t *bt);
+  void update(ObjectOperation &op, inode_backtrace_t &bt);
   int64_t get_pool() { return pool; }
 
 private:
@@ -791,9 +791,9 @@ class CInode : public MDSCacheObject, public InodeStoreBase, public Counter<CIno
   void fetch(MDSContext *fin);
   void _fetched(ceph::buffer::list& bl, ceph::buffer::list& bl2, Context *fin);  
 
-  void _commit_ops(int r, version_t version, MDSContext *fin,
+  void _commit_ops(int r, C_GatherBuilder &gather_bld,
                    std::vector<CInodeCommitOperation> &ops_vec,
-                   inode_backtrace_t *bt);
+                   inode_backtrace_t &bt);
   void build_backtrace(int64_t pool, inode_backtrace_t& bt);
   void _store_backtrace(std::vector<CInodeCommitOperation> &ops_vec,
                         inode_backtrace_t &bt, int op_prio);

diff --git a/src/mds/inode_backtrace.h b/src/mds/inode_backtrace.h
index 3674a716bcf661cd96054d37a9885b549022ce4f..5a6cc4ca76791a649fce00980cac613faa957f60 100644 (file)
--- a/src/mds/inode_backtrace.h
+++ b/src/mds/inode_backtrace.h
@@ -73,6 +73,11 @@ struct inode_backtrace_t {
   int compare(const inode_backtrace_t& other,
                bool *equivalent, bool *divergent) const;
 
+  void clear() {
+    ancestors.clear();
+    old_pools.clear();
+  }
+
   inodeno_t ino;       // my ino
   std::vector<inode_backpointer_t> ancestors;
   int64_t pool = -1;

diff --git a/src/mds/journal.cc b/src/mds/journal.cc
index 5f1253037cead932f7b6f3de89e261ed492750ad..d9118ca73144bc5ed9a04e4d7f0ee17ecc549881 100644 (file)
--- a/src/mds/journal.cc
+++ b/src/mds/journal.cc
@@ -62,37 +62,45 @@
 // -----------------------
 // LogSegment
 
-struct BatchStoredBacktrace : public MDSContext {
+struct BatchStoredBacktrace : public MDSIOContext {
   MDSContext *fin;
-  MDSRank *mds;
+  std::vector<CInodeCommitOperations> ops_vec;
 
-  BatchStoredBacktrace(MDSContext *f, MDSRank *m) : fin(f), mds(m) {}
+  BatchStoredBacktrace(MDSRank *m, MDSContext *f,
+                      std::vector<CInodeCommitOperations>&& ops) :
+    MDSIOContext(m), fin(f), ops_vec(std::move(ops)) {}
   void finish(int r) override {
+    for (auto& op : ops_vec) {
+      op.in->_stored_backtrace(r, op.version, nullptr);
+    }
     fin->complete(r);
   }
-  MDSRank *get_mds() override { return mds; };
+  void print(ostream& out) const override {
+    out << "batch backtrace_store";
+  }
 };
 
 struct BatchCommitBacktrace : public Context {
-  std::vector<CInodeCommitOperations> ops_vec;
-  MDSContext *con;
   MDSRank *mds;
+  MDSContext *fin;
+  std::vector<CInodeCommitOperations> ops_vec;
 
-  BatchCommitBacktrace(std::vector<CInodeCommitOperations> &ops, MDSContext *c,
-                       MDSRank *m) : con(c), mds(m) {
-    ops_vec.swap(ops);
-  }
+  BatchCommitBacktrace(MDSRank *m, MDSContext *f,
+                      std::vector<CInodeCommitOperations>&& ops) :
+    mds(m), fin(f), ops_vec(std::move(ops)) {}
   void finish(int r) override {
-    MDSGatherBuilder gather(g_ceph_context);
+    C_GatherBuilder gather(g_ceph_context);
 
     for (auto &op : ops_vec) {
-      op.in->_commit_ops(r, op.version, gather.new_sub(), op.ops_vec, &op.bt);
-    }
-    if (gather.has_subs()) {
-      gather.set_finisher(new BatchStoredBacktrace(con, mds));
-      std::scoped_lock l(mds->mds_lock);
-      gather.activate();
+      op.in->_commit_ops(r, gather, op.ops_vec, op.bt);
+      op.ops_vec.clear();
+      op.bt.clear();
     }
+    ceph_assert(gather.has_subs());
+    gather.set_finisher(new C_OnFinisher(
+                         new BatchStoredBacktrace(mds, fin, std::move(ops_vec)),
+                         mds->finisher));
+    gather.activate();
   }
 };
 
@@ -241,7 +249,7 @@ void LogSegment::try_to_expire(MDSRank *mds, MDSGatherBuilder &gather_bld, int o
     }
   }
   if (!ops_vec.empty())
-    mds->finisher->queue(new BatchCommitBacktrace(ops_vec, gather_bld.new_sub(), mds));
+    mds->finisher->queue(new BatchCommitBacktrace(mds, gather_bld.new_sub(), std::move(ops_vec)));
 
   ceph_assert(g_conf()->mds_kill_journal_expire_at != 4);