From: Sage Weil Date: Sun, 20 Jul 2014 21:51:28 +0000 (-0700) Subject: mds: mark objecter completions with _IO_, take mds_lock X-Git-Tag: v0.86~213^2~64 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=641364e70599e625c2fbbcba88c45c67a002b3a5;p=ceph.git mds: mark objecter completions with _IO_, take mds_lock For any completion we pass directly to Objecter, make sure we take the mds_lock in finish(), and mark the class with _IO_ in the name. Note that this doesn't address the use of Journaler. And this assumes that we are not holding the mds_lock already when Objecter::handle_osd_op_reply is called. Signed-off-by: Sage Weil --- diff --git a/src/mds/CDir.cc b/src/mds/CDir.cc index 12179af84200..efe9bea4a4cc 100644 --- a/src/mds/CDir.cc +++ b/src/mds/CDir.cc @@ -1342,15 +1342,16 @@ void CDir::fetch(Context *c, const string& want_dn, bool ignore_authpinnability) _omap_fetch(want_dn); } -class C_Dir_TMAP_Fetched : public Context { +class C_IO_Dir_TMAP_Fetched : public Context { protected: CDir *dir; string want_dn; public: bufferlist bl; - C_Dir_TMAP_Fetched(CDir *d, const string& w) : dir(d), want_dn(w) { } + C_IO_Dir_TMAP_Fetched(CDir *d, const string& w) : dir(d), want_dn(w) { } void finish(int r) { + Mutex::Locker l(dir->cache->mds->mds_lock); dir->_tmap_fetched(bl, want_dn, r); } }; @@ -1358,7 +1359,7 @@ class C_Dir_TMAP_Fetched : public Context { void CDir::_tmap_fetch(const string& want_dn) { // start by reading the first hunk of it - C_Dir_TMAP_Fetched *fin = new C_Dir_TMAP_Fetched(this, want_dn); + C_IO_Dir_TMAP_Fetched *fin = new C_IO_Dir_TMAP_Fetched(this, want_dn); object_t oid = get_ondisk_object(); object_locator_t oloc(cache->mds->mdsmap->get_metadata_pool()); ObjectOperation rd; @@ -1396,7 +1397,7 @@ void CDir::_tmap_fetched(bufferlist& bl, const string& want_dn, int r) _omap_fetched(header, omap, want_dn, r); } -class C_Dir_OMAP_Fetched : public Context { +class C_IO_Dir_OMAP_Fetched : public Context { protected: CDir *dir; string want_dn; @@ -1405,8 +1406,9 @@ class C_Dir_OMAP_Fetched : public Context { map omap; int ret1, ret2; - C_Dir_OMAP_Fetched(CDir *d, const string& w) : dir(d), want_dn(w) { } + C_IO_Dir_OMAP_Fetched(CDir *d, const string& w) : dir(d), want_dn(w) { } void finish(int r) { + Mutex::Locker l(dir->cache->mds->mds_lock); if (r >= 0) r = ret1; if (r >= 0) r = ret2; dir->_omap_fetched(hdrbl, omap, want_dn, r); @@ -1415,7 +1417,7 @@ class C_Dir_OMAP_Fetched : public Context { void CDir::_omap_fetch(const string& want_dn) { - C_Dir_OMAP_Fetched *fin = new C_Dir_OMAP_Fetched(this, want_dn); + C_IO_Dir_OMAP_Fetched *fin = new C_IO_Dir_OMAP_Fetched(this, want_dn); object_t oid = get_ondisk_object(); object_locator_t oloc(cache->mds->mdsmap->get_metadata_pool()); ObjectOperation rd; @@ -1747,12 +1749,13 @@ void CDir::commit(version_t want, Context *c, bool ignore_authpinnability, int o _commit(want, op_prio); } -class C_Dir_Committed : public Context { +class C_IO_Dir_Committed : public Context { CDir *dir; version_t version; public: - C_Dir_Committed(CDir *d, version_t v) : dir(d), version(v) { } + C_IO_Dir_Committed(CDir *d, version_t v) : dir(d), version(v) { } void finish(int r) { + Mutex::Locker l(dir->cache->mds->mds_lock); assert(r == 0); dir->_committed(version); } @@ -1787,7 +1790,8 @@ void CDir::_omap_commit(int op_prio) set to_remove; map to_set; - C_GatherBuilder gather(g_ceph_context, new C_Dir_Committed(this, get_version())); + C_GatherBuilder gather(g_ceph_context, + new C_IO_Dir_Committed(this, get_version())); SnapContext snapc; object_t oid = get_ondisk_object(); diff --git a/src/mds/CDir.h b/src/mds/CDir.h index 051a725e7e4d..30b61a309a74 100644 --- a/src/mds/CDir.h +++ b/src/mds/CDir.h @@ -276,9 +276,9 @@ protected: friend class CDirDiscover; friend class CDirExport; - friend class C_Dir_TMAP_Fetched; - friend class C_Dir_OMAP_Fetched; - friend class C_Dir_Committed; + friend class C_IO_Dir_TMAP_Fetched; + friend class C_IO_Dir_OMAP_Fetched; + friend class C_IO_Dir_Committed; bloom_filter *bloom; /* If you set up the bloom filter, you must keep it accurate! diff --git a/src/mds/CInode.cc b/src/mds/CInode.cc index cc8e7997954e..469c68b5dce9 100644 --- a/src/mds/CInode.cc +++ b/src/mds/CInode.cc @@ -885,12 +885,13 @@ void CInode::mark_clean() // per-inode storage // (currently for root inode only) -struct C_Inode_Stored : public Context { +struct C_IO_Inode_Stored : public Context { CInode *in; version_t version; Context *fin; - C_Inode_Stored(CInode *i, version_t v, Context *f) : in(i), version(v), fin(f) {} + C_IO_Inode_Stored(CInode *i, version_t v, Context *f) : in(i), version(v), fin(f) {} void finish(int r) { + Mutex::Locker l(in->mdcache->mds->mds_lock); assert(r == 0); in->_stored(version, fin); } @@ -923,7 +924,7 @@ void CInode::store(Context *fin) object_locator_t oloc(mdcache->mds->mdsmap->get_metadata_pool()); mdcache->mds->objecter->mutate(oid, oloc, m, snapc, ceph_clock_now(g_ceph_context), 0, - NULL, new C_Inode_Stored(this, get_version(), fin) ); + NULL, new C_IO_Inode_Stored(this, get_version(), fin) ); } void CInode::_stored(version_t v, Context *fin) @@ -935,12 +936,13 @@ void CInode::_stored(version_t v, Context *fin) fin->complete(0); } -struct C_Inode_Fetched : public Context { +struct C_IO_Inode_Fetched : public Context { CInode *in; bufferlist bl, bl2; Context *fin; - C_Inode_Fetched(CInode *i, Context *f) : in(i), fin(f) {} + C_IO_Inode_Fetched(CInode *i, Context *f) : in(i), fin(f) {} void finish(int r) { + Mutex::Locker l(in->mdcache->mds->mds_lock); in->_fetched(bl, bl2, fin); } }; @@ -949,7 +951,7 @@ void CInode::fetch(Context *fin) { dout(10) << "fetch" << dendl; - C_Inode_Fetched *c = new C_Inode_Fetched(this, fin); + C_IO_Inode_Fetched *c = new C_IO_Inode_Fetched(this, fin); C_GatherBuilder gather(g_ceph_context, c); object_t oid = CInode::get_object_name(ino(), frag_t(), ""); @@ -1015,12 +1017,13 @@ void CInode::build_backtrace(int64_t pool, inode_backtrace_t& bt) } } -struct C_Inode_StoredBacktrace : public Context { +struct C_IO_Inode_StoredBacktrace : public Context { CInode *in; version_t version; Context *fin; - C_Inode_StoredBacktrace(CInode *i, version_t v, Context *f) : in(i), version(v), fin(f) {} + C_IO_Inode_StoredBacktrace(CInode *i, version_t v, Context *f) : in(i), version(v), fin(f) {} void finish(int r) { + Mutex::Locker l(in->mdcache->mds->mds_lock); assert(r == 0); in->_stored_backtrace(version, fin); } @@ -1055,7 +1058,7 @@ void CInode::store_backtrace(Context *fin, int op_prio) SnapContext snapc; object_t oid = get_object_name(ino(), frag_t(), ""); object_locator_t oloc(pool); - Context *fin2 = new C_Inode_StoredBacktrace(this, inode.backtrace_version, fin); + Context *fin2 = new C_IO_Inode_StoredBacktrace(this, inode.backtrace_version, fin); if (!state_test(STATE_DIRTYPOOL) || inode.old_pools.empty()) { mdcache->mds->objecter->mutate(oid, oloc, op, snapc, ceph_clock_now(g_ceph_context), diff --git a/src/mds/MDCache.cc b/src/mds/MDCache.cc index 46f18dc76fdf..541ed21d0d64 100644 --- a/src/mds/MDCache.cc +++ b/src/mds/MDCache.cc @@ -7892,14 +7892,15 @@ void MDCache::make_trace(vector& trace, CInode *in) // ------------------------------------------------------------------------------- // Open inode by inode number -class C_MDC_OpenInoBacktraceFetched : public Context { +class C_IO_MDC_OpenInoBacktraceFetched : public Context { MDCache *cache; inodeno_t ino; public: bufferlist bl; - C_MDC_OpenInoBacktraceFetched(MDCache *c, inodeno_t i) : + C_IO_MDC_OpenInoBacktraceFetched(MDCache *c, inodeno_t i) : cache(c), ino(i) {} void finish(int r) { + Mutex::Locker l(cache->mds->mds_lock); cache->_open_ino_backtrace_fetched(ino, bl, r); } }; @@ -7946,7 +7947,7 @@ void MDCache::_open_ino_backtrace_fetched(inodeno_t ino, bufferlist& bl, int err dout(10) << " old object in pool " << info.pool << ", retrying pool " << backtrace.pool << dendl; info.pool = backtrace.pool; - C_MDC_OpenInoBacktraceFetched *fin = new C_MDC_OpenInoBacktraceFetched(this, ino); + C_IO_MDC_OpenInoBacktraceFetched *fin = new C_IO_MDC_OpenInoBacktraceFetched(this, ino); fetch_backtrace(ino, info.pool, fin->bl, fin); return; } @@ -7956,7 +7957,7 @@ void MDCache::_open_ino_backtrace_fetched(inodeno_t ino, bufferlist& bl, int err dout(10) << " no object in pool " << info.pool << ", retrying pool " << meta_pool << dendl; info.pool = meta_pool; - C_MDC_OpenInoBacktraceFetched *fin = new C_MDC_OpenInoBacktraceFetched(this, ino); + C_IO_MDC_OpenInoBacktraceFetched *fin = new C_IO_MDC_OpenInoBacktraceFetched(this, ino); fetch_backtrace(ino, info.pool, fin->bl, fin); return; } @@ -8181,7 +8182,7 @@ void MDCache::do_open_ino(inodeno_t ino, open_ino_info_t& info, int err) info.checking = mds->get_nodeid(); info.checked.clear(); info.checked.insert(mds->get_nodeid()); - C_MDC_OpenInoBacktraceFetched *fin = new C_MDC_OpenInoBacktraceFetched(this, ino); + C_IO_MDC_OpenInoBacktraceFetched *fin = new C_IO_MDC_OpenInoBacktraceFetched(this, ino); fetch_backtrace(ino, info.pool, fin->bl, fin); } else { assert(!info.ancestors.empty()); @@ -9096,13 +9097,14 @@ void MDCache::fetch_backtrace(inodeno_t ino, int64_t pool, bufferlist& bl, Conte mds->objecter->getxattr(oid, object_locator_t(pool), "parent", CEPH_NOSNAP, &bl, 0, fin); } -class C_MDC_PurgeStrayPurged : public Context { +class C_IO_MDC_PurgeStrayPurged : public Context { MDCache *cache; CDentry *dn; public: - C_MDC_PurgeStrayPurged(MDCache *c, CDentry *d) : + C_IO_MDC_PurgeStrayPurged(MDCache *c, CDentry *d) : cache(c), dn(d) { } void finish(int r) { + Mutex::Locker l(cache->mds->mds_lock); assert(r == 0 || r == -ENOENT); cache->_purge_stray_purged(dn, r); } @@ -9130,7 +9132,7 @@ void MDCache::purge_stray(CDentry *dn) // dir. on recovery, we'll need to re-eval all strays anyway. SnapContext nullsnapc; - C_GatherBuilder gather(g_ceph_context, new C_MDC_PurgeStrayPurged(this, dn)); + C_GatherBuilder gather(g_ceph_context, new C_IO_MDC_PurgeStrayPurged(this, dn)); if (in->is_dir()) { object_locator_t oloc(mds->mdsmap->get_metadata_pool()); @@ -10941,16 +10943,17 @@ public: } }; -class C_MDC_FragmentFinish : public Context { +class C_IO_MDC_FragmentFinish : public Context { MDCache *mdcache; dirfrag_t basedirfrag; list resultfrags; public: - C_MDC_FragmentFinish(MDCache *m, dirfrag_t f, list& l) : + C_IO_MDC_FragmentFinish(MDCache *m, dirfrag_t f, list& l) : mdcache(m), basedirfrag(f) { resultfrags.swap(l); } virtual void finish(int r) { + Mutex::Locker l(mdcache->mds->mds_lock); assert(r == 0 || r == -ENOENT); mdcache->_fragment_finish(basedirfrag, resultfrags); } @@ -11183,7 +11186,7 @@ void MDCache::_fragment_committed(dirfrag_t basedirfrag, list& resultfrag ufragment &uf = it->second; // remove old frags - C_GatherBuilder gather(g_ceph_context, new C_MDC_FragmentFinish(this, basedirfrag, resultfrags)); + C_GatherBuilder gather(g_ceph_context, new C_IO_MDC_FragmentFinish(this, basedirfrag, resultfrags)); SnapContext nullsnapc; object_locator_t oloc(mds->mdsmap->get_metadata_pool()); diff --git a/src/mds/MDCache.h b/src/mds/MDCache.h index e6ed1cf21aad..432a3ab27ed7 100644 --- a/src/mds/MDCache.h +++ b/src/mds/MDCache.h @@ -811,7 +811,7 @@ protected: void do_open_ino_peer(inodeno_t ino, open_ino_info_t& info); void handle_open_ino(MMDSOpenIno *m); void handle_open_ino_reply(MMDSOpenInoReply *m); - friend class C_MDC_OpenInoBacktraceFetched; + friend class C_IO_MDC_OpenInoBacktraceFetched; friend struct C_MDC_OpenInoTraverseDir; friend struct C_MDC_OpenInoParentOpened; @@ -870,10 +870,10 @@ protected: void _purge_stray_logged(CDentry *dn, version_t pdv, LogSegment *ls); void _purge_stray_logged_truncate(CDentry *dn, LogSegment *ls); friend struct C_MDC_RetryScanStray; - friend class C_MDC_FetchedBacktrace; + friend class C_IO_MDC_FetchedBacktrace; friend class C_MDC_PurgeStrayLogged; friend class C_MDC_PurgeStrayLoggedTruncate; - friend class C_MDC_PurgeStrayPurged; + friend class C_IO_MDC_PurgeStrayPurged; void reintegrate_stray(CDentry *dn, CDentry *rlink); void migrate_stray(CDentry *dn, int dest); @@ -978,7 +978,7 @@ private: friend class C_MDC_FragmentPrep; friend class C_MDC_FragmentStore; friend class C_MDC_FragmentCommit; - friend class C_MDC_FragmentFinish; + friend class C_IO_MDC_FragmentFinish; void handle_fragment_notify(MMDSFragmentNotify *m); diff --git a/src/mds/MDSTable.cc b/src/mds/MDSTable.cc index 967c9ee52aaf..4cd5e60457f5 100644 --- a/src/mds/MDSTable.cc +++ b/src/mds/MDSTable.cc @@ -30,12 +30,13 @@ #define dout_prefix *_dout << "mds." << (mds ? mds->get_nodeid() : -1) << "." << table_name << ": " -class C_MT_Save : public Context { +class C_IO_MT_Save : public Context { MDSTable *ida; version_t version; public: - C_MT_Save(MDSTable *i, version_t v) : ida(i), version(v) {} + C_IO_MT_Save(MDSTable *i, version_t v) : ida(i), version(v) {} void finish(int r) { + Mutex::Locker l(ida->mds->mds_lock); ida->save_2(r, version); } }; @@ -68,7 +69,7 @@ void MDSTable::save(Context *onfinish, version_t v) mds->objecter->write_full(oid, oloc, snapc, bl, ceph_clock_now(g_ceph_context), 0, - NULL, new C_MT_Save(this, version)); + NULL, new C_IO_MT_Save(this, version)); } void MDSTable::save_2(int r, version_t v) @@ -105,13 +106,14 @@ void MDSTable::reset() // ----------------------- -class C_MT_Load : public Context { +class C_IO_MT_Load : public Context { public: MDSTable *ida; Context *onfinish; bufferlist bl; - C_MT_Load(MDSTable *i, Context *o) : ida(i), onfinish(o) {} + C_IO_MT_Load(MDSTable *i, Context *o) : ida(i), onfinish(o) {} void finish(int r) { + Mutex::Locker l(ida->mds->mds_lock); ida->load_2(r, bl, onfinish); } }; @@ -133,7 +135,7 @@ void MDSTable::load(Context *onfinish) assert(is_undef()); state = STATE_OPENING; - C_MT_Load *c = new C_MT_Load(this, onfinish); + C_IO_MT_Load *c = new C_IO_MT_Load(this, onfinish); object_t oid = get_object_name(); object_locator_t oloc(mds->mdsmap->get_metadata_pool()); mds->objecter->read_full(oid, oloc, CEPH_NOSNAP, &c->bl, 0, c); diff --git a/src/mds/MDSTable.h b/src/mds/MDSTable.h index edcb203201be..0b50d699e3b5 100644 --- a/src/mds/MDSTable.h +++ b/src/mds/MDSTable.h @@ -23,9 +23,9 @@ class MDS; class MDSTable { - protected: +public: MDS *mds; - +protected: const char *table_name; bool per_mds; diff --git a/src/mds/SessionMap.cc b/src/mds/SessionMap.cc index 8b797f763c97..f1aa68bd51ca 100644 --- a/src/mds/SessionMap.cc +++ b/src/mds/SessionMap.cc @@ -53,12 +53,13 @@ object_t SessionMap::get_object_name() return object_t(s); } -class C_SM_Load : public Context { +class C_IO_SM_Load : public Context { SessionMap *sessionmap; public: bufferlist bl; - C_SM_Load(SessionMap *cm) : sessionmap(cm) {} + C_IO_SM_Load(SessionMap *cm) : sessionmap(cm) {} void finish(int r) { + Mutex::Locker l(sessionmap->mds->mds_lock); sessionmap->_load_finish(r, bl); } }; @@ -70,7 +71,7 @@ void SessionMap::load(Context *onload) if (onload) waiting_for_load.push_back(onload); - C_SM_Load *c = new C_SM_Load(this); + C_IO_SM_Load *c = new C_IO_SM_Load(this); object_t oid = get_object_name(); object_locator_t oloc(mds->mdsmap->get_metadata_pool()); mds->objecter->read_full(oid, oloc, CEPH_NOSNAP, &c->bl, 0, c); @@ -98,12 +99,13 @@ void SessionMap::_load_finish(int r, bufferlist &bl) // ---------------- // SAVE -class C_SM_Save : public Context { +class C_IO_SM_Save : public Context { SessionMap *sessionmap; version_t version; public: - C_SM_Save(SessionMap *cm, version_t v) : sessionmap(cm), version(v) {} + C_IO_SM_Save(SessionMap *cm, version_t v) : sessionmap(cm), version(v) {} void finish(int r) { + Mutex::Locker l(sessionmap->mds->mds_lock); assert(r == 0); sessionmap->_save_finish(version); } @@ -132,7 +134,7 @@ void SessionMap::save(Context *onsave, version_t needv) mds->objecter->write_full(oid, oloc, snapc, bl, ceph_clock_now(g_ceph_context), 0, - NULL, new C_SM_Save(this, version)); + NULL, new C_IO_SM_Save(this, version)); } void SessionMap::_save_finish(version_t v) diff --git a/src/mds/SessionMap.h b/src/mds/SessionMap.h index fefda91fdd22..43197355649b 100644 --- a/src/mds/SessionMap.h +++ b/src/mds/SessionMap.h @@ -233,8 +233,9 @@ public: class MDS; class SessionMap { -private: +public: MDS *mds; +private: ceph::unordered_map session_map; public: map* > by_state;