From: Yan, Zheng <zyan@redhat.com>
Date: Thu, 4 Dec 2014 04:18:47 +0000 (+0800)
Subject: osdc/Filer: use finisher to execute C_Probe and C_PurgeRange
X-Git-Tag: v0.90~5^2^2
X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=d3ee89ace660161df7796affbf9a70f3d0dedce1;p=ceph.git

osdc/Filer: use finisher to execute C_Probe and C_PurgeRange

Currently contexts C_Probe/C_PurgeRange are executed while holding
OSDSession::completion_lock. C_Probe and C_PurgeRange may call
Objecter::stat() and Objecter::remove() respectively, which acquire
Objecter::rwlock. This can cause deadlock because there is intermediate
dependency between Objecter::rwlock and OSDSession::completion_lock:

 Objecter::rwlock -> OSDSession::lock -> OSDSession::completion_lock

The fix is exexcute C_Probe/C_PurgeRange in finisher thread.

Fixes: #10229
Signed-off-by: Yan, Zheng <zyan@redhat.com>
---

diff --git a/src/client/Client.cc b/src/client/Client.cc
index 42611f0f93da..5cd6e861c30c 100644
--- a/src/client/Client.cc
+++ b/src/client/Client.cc
@@ -216,7 +216,7 @@ Client::Client(Messenger *m, MonClient *mc)
 				  cct->_conf->client_oc_max_dirty_age,
 				  true);
   objecter_finisher.start();
-  filer = new Filer(objecter);
+  filer = new Filer(objecter, &objecter_finisher);
 }
 
 
diff --git a/src/mds/MDS.cc b/src/mds/MDS.cc
index a4d27945e2f9..a9e0a10eeb2d 100644
--- a/src/mds/MDS.cc
+++ b/src/mds/MDS.cc
@@ -129,7 +129,7 @@ MDS::MDS(const std::string &n, Messenger *m, MonClient *mc) :
   objecter = new Objecter(m->cct, messenger, monc, 0, 0);
   objecter->unset_honor_osdmap_full();
 
-  filer = new Filer(objecter);
+  filer = new Filer(objecter, &finisher);
 
   mdcache = new MDCache(this);
   mdlog = new MDLog(this);
diff --git a/src/osdc/Filer.cc b/src/osdc/Filer.cc
index 84392de53b59..d89c8781ea0f 100644
--- a/src/osdc/Filer.cc
+++ b/src/osdc/Filer.cc
@@ -25,6 +25,7 @@
 
 #include "include/Context.h"
 
+#include "common/Finisher.h"
 #include "common/config.h"
 
 #define dout_subsys ceph_subsys_filer
@@ -134,8 +135,9 @@ void Filer::_probe(Probe *probe)
   for (std::vector<ObjectExtent>::iterator i = stat_extents.begin();
       i != stat_extents.end(); ++i) {
     C_Probe *c = new C_Probe(this, probe, i->oid);
-    objecter->stat(i->oid, i->oloc, probe->snapid, &c->size, &c->mtime, 
-                   probe->flags | CEPH_OSD_FLAG_RWORDERED, c);
+    objecter->stat(i->oid, i->oloc, probe->snapid, &c->size, &c->mtime,
+		   probe->flags | CEPH_OSD_FLAG_RWORDERED,
+		   new C_OnFinisher(c, finisher));
   }
 }
 
@@ -339,8 +341,8 @@ void Filer::_do_purge_range(PurgeRange *pr, int fin)
     const OSDMap *osdmap = objecter->get_osdmap_read();
     const object_locator_t oloc = osdmap->file_to_object_locator(pr->layout);
     objecter->put_osdmap_read();
-    objecter->remove(oid, oloc, pr->snapc, pr->mtime, pr->flags,
-		     NULL, new C_PurgeRange(this, pr));
+    objecter->remove(oid, oloc, pr->snapc, pr->mtime, pr->flags, NULL,
+		     new C_OnFinisher(new C_PurgeRange(this, pr), finisher));
   }
 }
 
diff --git a/src/osdc/Filer.h b/src/osdc/Filer.h
index 1164d95873a0..cbf3fa25a914 100644
--- a/src/osdc/Filer.h
+++ b/src/osdc/Filer.h
@@ -35,7 +35,7 @@
 class Context;
 class Messenger;
 class OSDMap;
-
+class Finisher;
 
 
 /**** Filer interface ***/
@@ -43,6 +43,7 @@ class OSDMap;
 class Filer {
   CephContext *cct;
   Objecter   *objecter;
+  Finisher   *finisher;
   
   // probes
   struct Probe {
@@ -88,7 +89,7 @@ class Filer {
   Filer(const Filer& other);
   const Filer operator=(const Filer& other);
 
-  Filer(Objecter *o) : cct(o->cct), objecter(o) {}
+  Filer(Objecter *o, Finisher *f) : cct(o->cct), objecter(o), finisher(f) {}
   ~Filer() {}
 
   bool is_active() {
diff --git a/src/osdc/Journaler.h b/src/osdc/Journaler.h
index d171c263663c..c4e9b2fbd8b0 100644
--- a/src/osdc/Journaler.h
+++ b/src/osdc/Journaler.h
@@ -377,7 +377,7 @@ public:
     ino(ino_), pg_pool(pool), readonly(true),
     stream_format(-1), journal_stream(-1),
     magic(mag),
-    objecter(obj), filer(objecter), logger(l), logger_key_lat(lkey),
+    objecter(obj), filer(objecter, f), logger(l), logger_key_lat(lkey),
     timer(tim), delay_flush_event(0),
     state(STATE_UNDEF), error(0),
     prezeroing_pos(0), prezero_pos(0), write_pos(0), flush_pos(0), safe_pos(0),
diff --git a/src/tools/cephfs/Dumper.cc b/src/tools/cephfs/Dumper.cc
index c8e7f8468528..857fb33231d0 100644
--- a/src/tools/cephfs/Dumper.cc
+++ b/src/tools/cephfs/Dumper.cc
@@ -85,7 +85,7 @@ int Dumper::dump(const char *dump_file)
 
   cout << "journal is " << start << "~" << len << std::endl;
 
-  Filer filer(objecter);
+  Filer filer(objecter, &finisher);
   bufferlist bl;
 
   C_SaferCond cond;
@@ -234,7 +234,7 @@ int Dumper::undump(const char *dump_file)
     return r;
   }
 
-  Filer filer(objecter);
+  Filer filer(objecter, &finisher);
 
   /* Erase any objects at the end of the region to which we shall write
    * the new log data.  This is to avoid leaving trailing junk after