osdc/Filer: use finisher to execute C_Probe and C_PurgeRange

Currently contexts C_Probe/C_PurgeRange are executed while holding
OSDSession::completion_lock. C_Probe and C_PurgeRange may call
Objecter::stat() and Objecter::remove() respectively, which acquire
Objecter::rwlock. This can cause deadlock because there is intermediate
dependency between Objecter::rwlock and OSDSession::completion_lock:

 Objecter::rwlock -> OSDSession::lock -> OSDSession::completion_lock

The fix is exexcute C_Probe/C_PurgeRange in finisher thread.

Fixes: #10229
Signed-off-by: Yan, Zheng <zyan@redhat.com>

diff --git a/src/client/Client.cc b/src/client/Client.cc
index 42611f0f93da894af0fb7752d4f8344c94079bd1..5cd6e861c30c0a170b22e8e9147902e606e26322 100644 (file)
--- a/src/client/Client.cc
+++ b/src/client/Client.cc
@@ -216,7 +216,7 @@ Client::Client(Messenger *m, MonClient *mc)
                                  cct->_conf->client_oc_max_dirty_age,
                                  true);
   objecter_finisher.start();
-  filer = new Filer(objecter);
+  filer = new Filer(objecter, &objecter_finisher);
 }
 
 

diff --git a/src/mds/MDS.cc b/src/mds/MDS.cc
index a4d27945e2f985f3ca4bba0a94ade64d80f6f7f7..a9e0a10eeb2d8c0410d1ac4b5ed622d2813891ca 100644 (file)
--- a/src/mds/MDS.cc
+++ b/src/mds/MDS.cc
@@ -129,7 +129,7 @@ MDS::MDS(const std::string &n, Messenger *m, MonClient *mc) :
   objecter = new Objecter(m->cct, messenger, monc, 0, 0);
   objecter->unset_honor_osdmap_full();
 
-  filer = new Filer(objecter);
+  filer = new Filer(objecter, &finisher);
 
   mdcache = new MDCache(this);
   mdlog = new MDLog(this);

diff --git a/src/osdc/Filer.cc b/src/osdc/Filer.cc
index 84392de53b59a877b1a2f48ebe5242314cb32f8a..d89c8781ea0f0f614fbcca79dc2d2dd5ea013db6 100644 (file)
--- a/src/osdc/Filer.cc
+++ b/src/osdc/Filer.cc
@@ -25,6 +25,7 @@
 
 #include "include/Context.h"
 
+#include "common/Finisher.h"
 #include "common/config.h"
 
 #define dout_subsys ceph_subsys_filer
@@ -134,8 +135,9 @@ void Filer::_probe(Probe *probe)
   for (std::vector<ObjectExtent>::iterator i = stat_extents.begin();
       i != stat_extents.end(); ++i) {
     C_Probe *c = new C_Probe(this, probe, i->oid);
-    objecter->stat(i->oid, i->oloc, probe->snapid, &c->size, &c->mtime, 
-                   probe->flags | CEPH_OSD_FLAG_RWORDERED, c);
+    objecter->stat(i->oid, i->oloc, probe->snapid, &c->size, &c->mtime,
+                  probe->flags | CEPH_OSD_FLAG_RWORDERED,
+                  new C_OnFinisher(c, finisher));
   }
 }
 
@@ -339,8 +341,8 @@ void Filer::_do_purge_range(PurgeRange *pr, int fin)
     const OSDMap *osdmap = objecter->get_osdmap_read();
     const object_locator_t oloc = osdmap->file_to_object_locator(pr->layout);
     objecter->put_osdmap_read();
-    objecter->remove(oid, oloc, pr->snapc, pr->mtime, pr->flags,
-                    NULL, new C_PurgeRange(this, pr));
+    objecter->remove(oid, oloc, pr->snapc, pr->mtime, pr->flags, NULL,
+                    new C_OnFinisher(new C_PurgeRange(this, pr), finisher));
   }
 }
 

diff --git a/src/osdc/Filer.h b/src/osdc/Filer.h
index 1164d95873a04477ff33ff64ce2c26705c09e630..cbf3fa25a914df939ddf674a3e7de748fedb59d7 100644 (file)
--- a/src/osdc/Filer.h
+++ b/src/osdc/Filer.h
@@ -35,7 +35,7 @@
 class Context;
 class Messenger;
 class OSDMap;
-
+class Finisher;
 
 
 /**** Filer interface ***/
@@ -43,6 +43,7 @@ class OSDMap;
 class Filer {
   CephContext *cct;
   Objecter   *objecter;
+  Finisher   *finisher;
   
   // probes
   struct Probe {
@@ -88,7 +89,7 @@ class Filer {
   Filer(const Filer& other);
   const Filer operator=(const Filer& other);
 
-  Filer(Objecter *o) : cct(o->cct), objecter(o) {}
+  Filer(Objecter *o, Finisher *f) : cct(o->cct), objecter(o), finisher(f) {}
   ~Filer() {}
 
   bool is_active() {

diff --git a/src/osdc/Journaler.h b/src/osdc/Journaler.h
index d171c263663ce0ce099176fbd9f2d64dc4ee70c1..c4e9b2fbd8b0d4d75a26048e85698f03083dd4c5 100644 (file)
--- a/src/osdc/Journaler.h
+++ b/src/osdc/Journaler.h
@@ -377,7 +377,7 @@ public:
     ino(ino_), pg_pool(pool), readonly(true),
     stream_format(-1), journal_stream(-1),
     magic(mag),
-    objecter(obj), filer(objecter), logger(l), logger_key_lat(lkey),
+    objecter(obj), filer(objecter, f), logger(l), logger_key_lat(lkey),
     timer(tim), delay_flush_event(0),
     state(STATE_UNDEF), error(0),
     prezeroing_pos(0), prezero_pos(0), write_pos(0), flush_pos(0), safe_pos(0),

diff --git a/src/tools/cephfs/Dumper.cc b/src/tools/cephfs/Dumper.cc
index c8e7f8468528fba5903060dd072a862e73ac1376..857fb33231d05ce097e32fb09782be9eb0d6c8e4 100644 (file)
--- a/src/tools/cephfs/Dumper.cc
+++ b/src/tools/cephfs/Dumper.cc
@@ -85,7 +85,7 @@ int Dumper::dump(const char *dump_file)
 
   cout << "journal is " << start << "~" << len << std::endl;
 
-  Filer filer(objecter);
+  Filer filer(objecter, &finisher);
   bufferlist bl;
 
   C_SaferCond cond;
@@ -234,7 +234,7 @@ int Dumper::undump(const char *dump_file)
     return r;
   }
 
-  Filer filer(objecter);
+  Filer filer(objecter, &finisher);
 
   /* Erase any objects at the end of the region to which we shall write
    * the new log data.  This is to avoid leaving trailing junk after