From 04904b9a5d4f42727b02a47479e4255f3aa6e175 Mon Sep 17 00:00:00 2001 From: Erqi Chen Date: Tue, 27 Oct 2020 10:14:35 +0800 Subject: [PATCH] mds: optimize batch backtrace store Call backtrace stored callback in batch. This avoids taking mds_lock for each inode. Signed-off-by: Erqi Chen --- src/mds/CInode.cc | 43 +++++++++++++------------------------- src/mds/CInode.h | 6 +++--- src/mds/inode_backtrace.h | 5 +++++ src/mds/journal.cc | 44 +++++++++++++++++++++++---------------- 4 files changed, 49 insertions(+), 49 deletions(-) diff --git a/src/mds/CInode.cc b/src/mds/CInode.cc index b2c25d5783127..a9724952fd729 100644 --- a/src/mds/CInode.cc +++ b/src/mds/CInode.cc @@ -52,14 +52,14 @@ #undef dout_prefix #define dout_prefix *_dout << "mds." << mdcache->mds->get_nodeid() << ".cache.ino(" << ino() << ") " -void CInodeCommitOperation::update(ObjectOperation &op, inode_backtrace_t *bt) { +void CInodeCommitOperation::update(ObjectOperation &op, inode_backtrace_t &bt) { using ceph::encode; op.priority = priority; op.create(false); bufferlist parent_bl; - encode(*bt, parent_bl); + encode(bt, parent_bl); op.setxattr("parent", parent_bl); // for the old pool there is no need to update the layout @@ -1318,23 +1318,10 @@ struct C_IO_Inode_StoredBacktrace : public CInodeIOContext { } }; -struct C_IO_Inode_CommitBacktrace : public Context { - CInode *in; - version_t version; - MDSContext *fin; - std::vector ops_vec; - inode_backtrace_t bt; - - C_IO_Inode_CommitBacktrace(CInode *i, version_t v, MDSContext *f) : - in(i), version(v), fin(f) { } - void finish(int r) override { - in->_commit_ops(r, version, fin, ops_vec, &bt); - } -}; -void CInode::_commit_ops(int r, version_t version, MDSContext *fin, +void CInode::_commit_ops(int r, C_GatherBuilder &gather_bld, std::vector &ops_vec, - inode_backtrace_t *bt) + inode_backtrace_t &bt) { dout(10) << __func__ << dendl; @@ -1343,12 +1330,6 @@ void CInode::_commit_ops(int r, version_t version, MDSContext *fin, return; } - C_GatherBuilder gather(g_ceph_context, - new C_OnFinisher(new C_IO_Inode_StoredBacktrace(this, - version, - fin), - mdcache->mds->finisher)); - SnapContext snapc; object_t oid = get_object_name(ino(), frag_t(), ""); @@ -1358,9 +1339,8 @@ void CInode::_commit_ops(int r, version_t version, MDSContext *fin, op.update(obj_op, bt); mdcache->mds->objecter->mutate(oid, oloc, obj_op, snapc, ceph::real_clock::now(), - 0, gather.new_sub()); + 0, gather_bld.new_sub()); } - gather.activate(); } void CInode::_store_backtrace(std::vector &ops_vec, @@ -1401,11 +1381,18 @@ void CInode::_store_backtrace(std::vector &ops_vec, void CInode::store_backtrace(MDSContext *fin, int op_prio) { std::vector ops_vec; + inode_backtrace_t bt; auto version = get_inode()->backtrace_version; - auto c = new C_IO_Inode_CommitBacktrace(this, version, fin); - _store_backtrace(c->ops_vec, c->bt, op_prio); - mdcache->mds->finisher->queue(c); + _store_backtrace(ops_vec, bt, op_prio); + + C_GatherBuilder gather(g_ceph_context, + new C_OnFinisher( + new C_IO_Inode_StoredBacktrace(this, version, fin), + mdcache->mds->finisher)); + _commit_ops(0, gather, ops_vec, bt); + ceph_assert(gather.has_subs()); + gather.activate(); } void CInode::store_backtrace(CInodeCommitOperations &op, int op_prio) diff --git a/src/mds/CInode.h b/src/mds/CInode.h index f792e9f133e31..66ea728cc2d50 100644 --- a/src/mds/CInode.h +++ b/src/mds/CInode.h @@ -70,7 +70,7 @@ public: update_layout = true; } - void update(ObjectOperation &op, inode_backtrace_t *bt); + void update(ObjectOperation &op, inode_backtrace_t &bt); int64_t get_pool() { return pool; } private: @@ -791,9 +791,9 @@ class CInode : public MDSCacheObject, public InodeStoreBase, public Counter &ops_vec, - inode_backtrace_t *bt); + inode_backtrace_t &bt); void build_backtrace(int64_t pool, inode_backtrace_t& bt); void _store_backtrace(std::vector &ops_vec, inode_backtrace_t &bt, int op_prio); diff --git a/src/mds/inode_backtrace.h b/src/mds/inode_backtrace.h index 3674a716bcf66..5a6cc4ca76791 100644 --- a/src/mds/inode_backtrace.h +++ b/src/mds/inode_backtrace.h @@ -73,6 +73,11 @@ struct inode_backtrace_t { int compare(const inode_backtrace_t& other, bool *equivalent, bool *divergent) const; + void clear() { + ancestors.clear(); + old_pools.clear(); + } + inodeno_t ino; // my ino std::vector ancestors; int64_t pool = -1; diff --git a/src/mds/journal.cc b/src/mds/journal.cc index 5f1253037cead..d9118ca73144b 100644 --- a/src/mds/journal.cc +++ b/src/mds/journal.cc @@ -62,37 +62,45 @@ // ----------------------- // LogSegment -struct BatchStoredBacktrace : public MDSContext { +struct BatchStoredBacktrace : public MDSIOContext { MDSContext *fin; - MDSRank *mds; + std::vector ops_vec; - BatchStoredBacktrace(MDSContext *f, MDSRank *m) : fin(f), mds(m) {} + BatchStoredBacktrace(MDSRank *m, MDSContext *f, + std::vector&& ops) : + MDSIOContext(m), fin(f), ops_vec(std::move(ops)) {} void finish(int r) override { + for (auto& op : ops_vec) { + op.in->_stored_backtrace(r, op.version, nullptr); + } fin->complete(r); } - MDSRank *get_mds() override { return mds; }; + void print(ostream& out) const override { + out << "batch backtrace_store"; + } }; struct BatchCommitBacktrace : public Context { - std::vector ops_vec; - MDSContext *con; MDSRank *mds; + MDSContext *fin; + std::vector ops_vec; - BatchCommitBacktrace(std::vector &ops, MDSContext *c, - MDSRank *m) : con(c), mds(m) { - ops_vec.swap(ops); - } + BatchCommitBacktrace(MDSRank *m, MDSContext *f, + std::vector&& ops) : + mds(m), fin(f), ops_vec(std::move(ops)) {} void finish(int r) override { - MDSGatherBuilder gather(g_ceph_context); + C_GatherBuilder gather(g_ceph_context); for (auto &op : ops_vec) { - op.in->_commit_ops(r, op.version, gather.new_sub(), op.ops_vec, &op.bt); - } - if (gather.has_subs()) { - gather.set_finisher(new BatchStoredBacktrace(con, mds)); - std::scoped_lock l(mds->mds_lock); - gather.activate(); + op.in->_commit_ops(r, gather, op.ops_vec, op.bt); + op.ops_vec.clear(); + op.bt.clear(); } + ceph_assert(gather.has_subs()); + gather.set_finisher(new C_OnFinisher( + new BatchStoredBacktrace(mds, fin, std::move(ops_vec)), + mds->finisher)); + gather.activate(); } }; @@ -241,7 +249,7 @@ void LogSegment::try_to_expire(MDSRank *mds, MDSGatherBuilder &gather_bld, int o } } if (!ops_vec.empty()) - mds->finisher->queue(new BatchCommitBacktrace(ops_vec, gather_bld.new_sub(), mds)); + mds->finisher->queue(new BatchCommitBacktrace(mds, gather_bld.new_sub(), std::move(ops_vec))); ceph_assert(g_conf()->mds_kill_journal_expire_at != 4); -- 2.39.5