mds: mark objecter completions with _IO_, take mds_lock

For any completion we pass directly to Objecter, make sure we take the
mds_lock in finish(), and mark the class with _IO_ in the name.

Note that this doesn't address the use of Journaler.  And this assumes that
we are not holding the mds_lock already when Objecter::handle_osd_op_reply
is called.

Signed-off-by: Sage Weil <sage@redhat.com>

diff --git a/src/mds/CDir.cc b/src/mds/CDir.cc
index 12179af84200641265567a0b367e04d601f20d48..efe9bea4a4cc577e846b1b6b6d1169a357fcec8b 100644 (file)
--- a/src/mds/CDir.cc
+++ b/src/mds/CDir.cc
@@ -1342,15 +1342,16 @@ void CDir::fetch(Context *c, const string& want_dn, bool ignore_authpinnability)
   _omap_fetch(want_dn);
 }
 
-class C_Dir_TMAP_Fetched : public Context {
+class C_IO_Dir_TMAP_Fetched : public Context {
  protected:
   CDir *dir;
   string want_dn;
  public:
   bufferlist bl;
 
-  C_Dir_TMAP_Fetched(CDir *d, const string& w) : dir(d), want_dn(w) { }
+  C_IO_Dir_TMAP_Fetched(CDir *d, const string& w) : dir(d), want_dn(w) { }
   void finish(int r) {
+    Mutex::Locker l(dir->cache->mds->mds_lock);
     dir->_tmap_fetched(bl, want_dn, r);
   }
 };
@@ -1358,7 +1359,7 @@ class C_Dir_TMAP_Fetched : public Context {
 void CDir::_tmap_fetch(const string& want_dn)
 {
   // start by reading the first hunk of it
-  C_Dir_TMAP_Fetched *fin = new C_Dir_TMAP_Fetched(this, want_dn);
+  C_IO_Dir_TMAP_Fetched *fin = new C_IO_Dir_TMAP_Fetched(this, want_dn);
   object_t oid = get_ondisk_object();
   object_locator_t oloc(cache->mds->mdsmap->get_metadata_pool());
   ObjectOperation rd;
@@ -1396,7 +1397,7 @@ void CDir::_tmap_fetched(bufferlist& bl, const string& want_dn, int r)
   _omap_fetched(header, omap, want_dn, r);
 }
 
-class C_Dir_OMAP_Fetched : public Context {
+class C_IO_Dir_OMAP_Fetched : public Context {
  protected:
   CDir *dir;
   string want_dn;
@@ -1405,8 +1406,9 @@ class C_Dir_OMAP_Fetched : public Context {
   map<string, bufferlist> omap;
   int ret1, ret2;
 
-  C_Dir_OMAP_Fetched(CDir *d, const string& w) : dir(d), want_dn(w) { }
+  C_IO_Dir_OMAP_Fetched(CDir *d, const string& w) : dir(d), want_dn(w) { }
   void finish(int r) {
+    Mutex::Locker l(dir->cache->mds->mds_lock);
     if (r >= 0) r = ret1;
     if (r >= 0) r = ret2;
     dir->_omap_fetched(hdrbl, omap, want_dn, r);
@@ -1415,7 +1417,7 @@ class C_Dir_OMAP_Fetched : public Context {
 
 void CDir::_omap_fetch(const string& want_dn)
 {
-  C_Dir_OMAP_Fetched *fin = new C_Dir_OMAP_Fetched(this, want_dn);
+  C_IO_Dir_OMAP_Fetched *fin = new C_IO_Dir_OMAP_Fetched(this, want_dn);
   object_t oid = get_ondisk_object();
   object_locator_t oloc(cache->mds->mdsmap->get_metadata_pool());
   ObjectOperation rd;
@@ -1747,12 +1749,13 @@ void CDir::commit(version_t want, Context *c, bool ignore_authpinnability, int o
   _commit(want, op_prio);
 }
 
-class C_Dir_Committed : public Context {
+class C_IO_Dir_Committed : public Context {
   CDir *dir;
   version_t version;
 public:
-  C_Dir_Committed(CDir *d, version_t v) : dir(d), version(v) { }
+  C_IO_Dir_Committed(CDir *d, version_t v) : dir(d), version(v) { }
   void finish(int r) {
+    Mutex::Locker l(dir->cache->mds->mds_lock);
     assert(r == 0);
     dir->_committed(version);
   }
@@ -1787,7 +1790,8 @@ void CDir::_omap_commit(int op_prio)
   set<string> to_remove;
   map<string, bufferlist> to_set;
 
-  C_GatherBuilder gather(g_ceph_context, new C_Dir_Committed(this, get_version()));
+  C_GatherBuilder gather(g_ceph_context,
+                        new C_IO_Dir_Committed(this, get_version()));
 
   SnapContext snapc;
   object_t oid = get_ondisk_object();

diff --git a/src/mds/CDir.h b/src/mds/CDir.h
index 051a725e7e4df70745b130b8c15ee7ce5f5fbac7..30b61a309a74e0f82a6129831c492384098f9648 100644 (file)
--- a/src/mds/CDir.h
+++ b/src/mds/CDir.h
@@ -276,9 +276,9 @@ protected:
 
   friend class CDirDiscover;
   friend class CDirExport;
-  friend class C_Dir_TMAP_Fetched;
-  friend class C_Dir_OMAP_Fetched;
-  friend class C_Dir_Committed;
+  friend class C_IO_Dir_TMAP_Fetched;
+  friend class C_IO_Dir_OMAP_Fetched;
+  friend class C_IO_Dir_Committed;
 
   bloom_filter *bloom;
   /* If you set up the bloom filter, you must keep it accurate!

diff --git a/src/mds/CInode.cc b/src/mds/CInode.cc
index cc8e7997954e97866dd45cd7ecbbb7b56b38ec2f..469c68b5dce980075a632c983204953fb3e1de04 100644 (file)
--- a/src/mds/CInode.cc
+++ b/src/mds/CInode.cc
@@ -885,12 +885,13 @@ void CInode::mark_clean()
 // per-inode storage
 // (currently for root inode only)
 
-struct C_Inode_Stored : public Context {
+struct C_IO_Inode_Stored : public Context {
   CInode *in;
   version_t version;
   Context *fin;
-  C_Inode_Stored(CInode *i, version_t v, Context *f) : in(i), version(v), fin(f) {}
+  C_IO_Inode_Stored(CInode *i, version_t v, Context *f) : in(i), version(v), fin(f) {}
   void finish(int r) {
+    Mutex::Locker l(in->mdcache->mds->mds_lock);
     assert(r == 0);
     in->_stored(version, fin);
   }
@@ -923,7 +924,7 @@ void CInode::store(Context *fin)
   object_locator_t oloc(mdcache->mds->mdsmap->get_metadata_pool());
 
   mdcache->mds->objecter->mutate(oid, oloc, m, snapc, ceph_clock_now(g_ceph_context), 0,
-                                NULL, new C_Inode_Stored(this, get_version(), fin) );
+                                NULL, new C_IO_Inode_Stored(this, get_version(), fin) );
 }
 
 void CInode::_stored(version_t v, Context *fin)
@@ -935,12 +936,13 @@ void CInode::_stored(version_t v, Context *fin)
   fin->complete(0);
 }
 
-struct C_Inode_Fetched : public Context {
+struct C_IO_Inode_Fetched : public Context {
   CInode *in;
   bufferlist bl, bl2;
   Context *fin;
-  C_Inode_Fetched(CInode *i, Context *f) : in(i), fin(f) {}
+  C_IO_Inode_Fetched(CInode *i, Context *f) : in(i), fin(f) {}
   void finish(int r) {
+    Mutex::Locker l(in->mdcache->mds->mds_lock);
     in->_fetched(bl, bl2, fin);
   }
 };
@@ -949,7 +951,7 @@ void CInode::fetch(Context *fin)
 {
   dout(10) << "fetch" << dendl;
 
-  C_Inode_Fetched *c = new C_Inode_Fetched(this, fin);
+  C_IO_Inode_Fetched *c = new C_IO_Inode_Fetched(this, fin);
   C_GatherBuilder gather(g_ceph_context, c);
 
   object_t oid = CInode::get_object_name(ino(), frag_t(), "");
@@ -1015,12 +1017,13 @@ void CInode::build_backtrace(int64_t pool, inode_backtrace_t& bt)
   }
 }
 
-struct C_Inode_StoredBacktrace : public Context {
+struct C_IO_Inode_StoredBacktrace : public Context {
   CInode *in;
   version_t version;
   Context *fin;
-  C_Inode_StoredBacktrace(CInode *i, version_t v, Context *f) : in(i), version(v), fin(f) {}
+  C_IO_Inode_StoredBacktrace(CInode *i, version_t v, Context *f) : in(i), version(v), fin(f) {}
   void finish(int r) {
+    Mutex::Locker l(in->mdcache->mds->mds_lock);
     assert(r == 0);
     in->_stored_backtrace(version, fin);
   }
@@ -1055,7 +1058,7 @@ void CInode::store_backtrace(Context *fin, int op_prio)
   SnapContext snapc;
   object_t oid = get_object_name(ino(), frag_t(), "");
   object_locator_t oloc(pool);
-  Context *fin2 = new C_Inode_StoredBacktrace(this, inode.backtrace_version, fin);
+  Context *fin2 = new C_IO_Inode_StoredBacktrace(this, inode.backtrace_version, fin);
 
   if (!state_test(STATE_DIRTYPOOL) || inode.old_pools.empty()) {
     mdcache->mds->objecter->mutate(oid, oloc, op, snapc, ceph_clock_now(g_ceph_context),

diff --git a/src/mds/MDCache.cc b/src/mds/MDCache.cc
index 46f18dc76fdf7f87fe04e1c8dc4e1befe7c1cbf7..541ed21d0d6434acc18b4339d45e113ec9729fa3 100644 (file)
--- a/src/mds/MDCache.cc
+++ b/src/mds/MDCache.cc
@@ -7892,14 +7892,15 @@ void MDCache::make_trace(vector<CDentry*>& trace, CInode *in)
 // -------------------------------------------------------------------------------
 // Open inode by inode number
 
-class C_MDC_OpenInoBacktraceFetched : public Context {
+class C_IO_MDC_OpenInoBacktraceFetched : public Context {
   MDCache *cache;
   inodeno_t ino;
   public:
   bufferlist bl;
-  C_MDC_OpenInoBacktraceFetched(MDCache *c, inodeno_t i) :
+  C_IO_MDC_OpenInoBacktraceFetched(MDCache *c, inodeno_t i) :
     cache(c), ino(i) {}
   void finish(int r) {
+    Mutex::Locker l(cache->mds->mds_lock);
     cache->_open_ino_backtrace_fetched(ino, bl, r);
   }
 };
@@ -7946,7 +7947,7 @@ void MDCache::_open_ino_backtrace_fetched(inodeno_t ino, bufferlist& bl, int err
       dout(10) << " old object in pool " << info.pool
               << ", retrying pool " << backtrace.pool << dendl;
       info.pool = backtrace.pool;
-      C_MDC_OpenInoBacktraceFetched *fin = new C_MDC_OpenInoBacktraceFetched(this, ino);
+      C_IO_MDC_OpenInoBacktraceFetched *fin = new C_IO_MDC_OpenInoBacktraceFetched(this, ino);
       fetch_backtrace(ino, info.pool, fin->bl, fin);
       return;
     }
@@ -7956,7 +7957,7 @@ void MDCache::_open_ino_backtrace_fetched(inodeno_t ino, bufferlist& bl, int err
       dout(10) << " no object in pool " << info.pool
               << ", retrying pool " << meta_pool << dendl;
       info.pool = meta_pool;
-      C_MDC_OpenInoBacktraceFetched *fin = new C_MDC_OpenInoBacktraceFetched(this, ino);
+      C_IO_MDC_OpenInoBacktraceFetched *fin = new C_IO_MDC_OpenInoBacktraceFetched(this, ino);
       fetch_backtrace(ino, info.pool, fin->bl, fin);
       return;
     }
@@ -8181,7 +8182,7 @@ void MDCache::do_open_ino(inodeno_t ino, open_ino_info_t& info, int err)
     info.checking = mds->get_nodeid();
     info.checked.clear();
     info.checked.insert(mds->get_nodeid());
-    C_MDC_OpenInoBacktraceFetched *fin = new C_MDC_OpenInoBacktraceFetched(this, ino);
+    C_IO_MDC_OpenInoBacktraceFetched *fin = new C_IO_MDC_OpenInoBacktraceFetched(this, ino);
     fetch_backtrace(ino, info.pool, fin->bl, fin);
   } else {
     assert(!info.ancestors.empty());
@@ -9096,13 +9097,14 @@ void MDCache::fetch_backtrace(inodeno_t ino, int64_t pool, bufferlist& bl, Conte
   mds->objecter->getxattr(oid, object_locator_t(pool), "parent", CEPH_NOSNAP, &bl, 0, fin);
 }
 
-class C_MDC_PurgeStrayPurged : public Context {
+class C_IO_MDC_PurgeStrayPurged : public Context {
   MDCache *cache;
   CDentry *dn;
 public:
-  C_MDC_PurgeStrayPurged(MDCache *c, CDentry *d) : 
+  C_IO_MDC_PurgeStrayPurged(MDCache *c, CDentry *d) : 
     cache(c), dn(d) { }
   void finish(int r) {
+    Mutex::Locker l(cache->mds->mds_lock);
     assert(r == 0 || r == -ENOENT);
     cache->_purge_stray_purged(dn, r);
   }
@@ -9130,7 +9132,7 @@ void MDCache::purge_stray(CDentry *dn)
   // dir.  on recovery, we'll need to re-eval all strays anyway.
   
   SnapContext nullsnapc;
-  C_GatherBuilder gather(g_ceph_context, new C_MDC_PurgeStrayPurged(this, dn));
+  C_GatherBuilder gather(g_ceph_context, new C_IO_MDC_PurgeStrayPurged(this, dn));
 
   if (in->is_dir()) {
     object_locator_t oloc(mds->mdsmap->get_metadata_pool());
@@ -10941,16 +10943,17 @@ public:
   }
 };
 
-class C_MDC_FragmentFinish : public Context {
+class C_IO_MDC_FragmentFinish : public Context {
   MDCache *mdcache;
   dirfrag_t basedirfrag;
   list<CDir*> resultfrags;
 public:
-  C_MDC_FragmentFinish(MDCache *m, dirfrag_t f, list<CDir*>& l) :
+  C_IO_MDC_FragmentFinish(MDCache *m, dirfrag_t f, list<CDir*>& l) :
     mdcache(m), basedirfrag(f) {
     resultfrags.swap(l);
   }
   virtual void finish(int r) {
+    Mutex::Locker l(mdcache->mds->mds_lock);
     assert(r == 0 || r == -ENOENT);
     mdcache->_fragment_finish(basedirfrag, resultfrags);
   }
@@ -11183,7 +11186,7 @@ void MDCache::_fragment_committed(dirfrag_t basedirfrag, list<CDir*>& resultfrag
   ufragment &uf = it->second;
 
   // remove old frags
-  C_GatherBuilder gather(g_ceph_context, new C_MDC_FragmentFinish(this, basedirfrag, resultfrags));
+  C_GatherBuilder gather(g_ceph_context, new C_IO_MDC_FragmentFinish(this, basedirfrag, resultfrags));
 
   SnapContext nullsnapc;
   object_locator_t oloc(mds->mdsmap->get_metadata_pool());

diff --git a/src/mds/MDCache.h b/src/mds/MDCache.h
index e6ed1cf21aad86801bd149223810df29dd05f87a..432a3ab27ed7f9b5debe3aa89410e5a8b64b5106 100644 (file)
--- a/src/mds/MDCache.h
+++ b/src/mds/MDCache.h
@@ -811,7 +811,7 @@ protected:
   void do_open_ino_peer(inodeno_t ino, open_ino_info_t& info);
   void handle_open_ino(MMDSOpenIno *m);
   void handle_open_ino_reply(MMDSOpenInoReply *m);
-  friend class C_MDC_OpenInoBacktraceFetched;
+  friend class C_IO_MDC_OpenInoBacktraceFetched;
   friend struct C_MDC_OpenInoTraverseDir;
   friend struct C_MDC_OpenInoParentOpened;
 
@@ -870,10 +870,10 @@ protected:
   void _purge_stray_logged(CDentry *dn, version_t pdv, LogSegment *ls);
   void _purge_stray_logged_truncate(CDentry *dn, LogSegment *ls);
   friend struct C_MDC_RetryScanStray;
-  friend class C_MDC_FetchedBacktrace;
+  friend class C_IO_MDC_FetchedBacktrace;
   friend class C_MDC_PurgeStrayLogged;
   friend class C_MDC_PurgeStrayLoggedTruncate;
-  friend class C_MDC_PurgeStrayPurged;
+  friend class C_IO_MDC_PurgeStrayPurged;
   void reintegrate_stray(CDentry *dn, CDentry *rlink);
   void migrate_stray(CDentry *dn, int dest);
 
@@ -978,7 +978,7 @@ private:
   friend class C_MDC_FragmentPrep;
   friend class C_MDC_FragmentStore;
   friend class C_MDC_FragmentCommit;
-  friend class C_MDC_FragmentFinish;
+  friend class C_IO_MDC_FragmentFinish;
 
   void handle_fragment_notify(MMDSFragmentNotify *m);
 

diff --git a/src/mds/MDSTable.cc b/src/mds/MDSTable.cc
index 967c9ee52aafbb4e71cd4585d8fbba393fc9eacd..4cd5e60457f535b499bdabdd9473257ec9481df0 100644 (file)
--- a/src/mds/MDSTable.cc
+++ b/src/mds/MDSTable.cc
@@ -30,12 +30,13 @@
 #define dout_prefix *_dout << "mds." << (mds ? mds->get_nodeid() : -1) << "." << table_name << ": "
 
 
-class C_MT_Save : public Context {
+class C_IO_MT_Save : public Context {
   MDSTable *ida;
   version_t version;
 public:
-  C_MT_Save(MDSTable *i, version_t v) : ida(i), version(v) {}
+  C_IO_MT_Save(MDSTable *i, version_t v) : ida(i), version(v) {}
   void finish(int r) {
+    Mutex::Locker l(ida->mds->mds_lock);
     ida->save_2(r, version);
   }
 };
@@ -68,7 +69,7 @@ void MDSTable::save(Context *onfinish, version_t v)
   mds->objecter->write_full(oid, oloc,
                            snapc,
                            bl, ceph_clock_now(g_ceph_context), 0,
-                           NULL, new C_MT_Save(this, version));
+                           NULL, new C_IO_MT_Save(this, version));
 }
 
 void MDSTable::save_2(int r, version_t v)
@@ -105,13 +106,14 @@ void MDSTable::reset()
 
 // -----------------------
 
-class C_MT_Load : public Context {
+class C_IO_MT_Load : public Context {
 public:
   MDSTable *ida;
   Context *onfinish;
   bufferlist bl;
-  C_MT_Load(MDSTable *i, Context *o) : ida(i), onfinish(o) {}
+  C_IO_MT_Load(MDSTable *i, Context *o) : ida(i), onfinish(o) {}
   void finish(int r) {
+    Mutex::Locker l(ida->mds->mds_lock);
     ida->load_2(r, bl, onfinish);
   }
 };
@@ -133,7 +135,7 @@ void MDSTable::load(Context *onfinish)
   assert(is_undef());
   state = STATE_OPENING;
 
-  C_MT_Load *c = new C_MT_Load(this, onfinish);
+  C_IO_MT_Load *c = new C_IO_MT_Load(this, onfinish);
   object_t oid = get_object_name();
   object_locator_t oloc(mds->mdsmap->get_metadata_pool());
   mds->objecter->read_full(oid, oloc, CEPH_NOSNAP, &c->bl, 0, c);

diff --git a/src/mds/MDSTable.h b/src/mds/MDSTable.h
index edcb203201be5ec251ed781186c503545e9ae835..0b50d699e3b52929efc266bd4abbfb13f417776b 100644 (file)
--- a/src/mds/MDSTable.h
+++ b/src/mds/MDSTable.h
@@ -23,9 +23,9 @@
 class MDS;
 
 class MDSTable {
- protected:
+public:
   MDS *mds;
-
+protected:
   const char *table_name;
   bool per_mds;
 

diff --git a/src/mds/SessionMap.cc b/src/mds/SessionMap.cc
index 8b797f763c97640f3b440f50e1578b9ad3e75aef..f1aa68bd51ca5c19466eb98b4c1a29f41d391e91 100644 (file)
--- a/src/mds/SessionMap.cc
+++ b/src/mds/SessionMap.cc
@@ -53,12 +53,13 @@ object_t SessionMap::get_object_name()
   return object_t(s);
 }
 
-class C_SM_Load : public Context {
+class C_IO_SM_Load : public Context {
   SessionMap *sessionmap;
 public:
   bufferlist bl;
-  C_SM_Load(SessionMap *cm) : sessionmap(cm) {}
+  C_IO_SM_Load(SessionMap *cm) : sessionmap(cm) {}
   void finish(int r) {
+    Mutex::Locker l(sessionmap->mds->mds_lock);
     sessionmap->_load_finish(r, bl);
   }
 };
@@ -70,7 +71,7 @@ void SessionMap::load(Context *onload)
   if (onload)
     waiting_for_load.push_back(onload);
   
-  C_SM_Load *c = new C_SM_Load(this);
+  C_IO_SM_Load *c = new C_IO_SM_Load(this);
   object_t oid = get_object_name();
   object_locator_t oloc(mds->mdsmap->get_metadata_pool());
   mds->objecter->read_full(oid, oloc, CEPH_NOSNAP, &c->bl, 0, c);
@@ -98,12 +99,13 @@ void SessionMap::_load_finish(int r, bufferlist &bl)
 // ----------------
 // SAVE
 
-class C_SM_Save : public Context {
+class C_IO_SM_Save : public Context {
   SessionMap *sessionmap;
   version_t version;
 public:
-  C_SM_Save(SessionMap *cm, version_t v) : sessionmap(cm), version(v) {}
+  C_IO_SM_Save(SessionMap *cm, version_t v) : sessionmap(cm), version(v) {}
   void finish(int r) {
+    Mutex::Locker l(sessionmap->mds->mds_lock);
     assert(r == 0);
     sessionmap->_save_finish(version);
   }
@@ -132,7 +134,7 @@ void SessionMap::save(Context *onsave, version_t needv)
   mds->objecter->write_full(oid, oloc,
                            snapc,
                            bl, ceph_clock_now(g_ceph_context), 0,
-                           NULL, new C_SM_Save(this, version));
+                           NULL, new C_IO_SM_Save(this, version));
 }
 
 void SessionMap::_save_finish(version_t v)

diff --git a/src/mds/SessionMap.h b/src/mds/SessionMap.h
index fefda91fdd22f415a78f5201b38749226af86d44..43197355649be6a1d3c14e34e1afd8e16a63a7ff 100644 (file)
--- a/src/mds/SessionMap.h
+++ b/src/mds/SessionMap.h
@@ -233,8 +233,9 @@ public:
 class MDS;
 
 class SessionMap {
-private:
+public:
   MDS *mds;
+private:
   ceph::unordered_map<entity_name_t, Session*> session_map;
 public:
   map<int,xlist<Session*>* > by_state;