From a64caec1ee6ab0b4d93af25ef87f0017f07caa64 Mon Sep 17 00:00:00 2001
From: Sage Weil <sage@newdream.net>
Date: Fri, 26 Feb 2010 12:30:07 -0800
Subject: [PATCH] osd: use onreadable_sync finishers to drop ondisk locks

This fixes a deadlock where we are holding pg->lock, block
waiting for the ondisk lock, but the unlock completion is stuck
behind something else waiting on pg->lock in the finisher queue.
---
 src/osd/ReplicatedPG.cc | 11 ++++++-----
 src/osd/ReplicatedPG.h  |  7 +++++++
 2 files changed, 13 insertions(+), 5 deletions(-)

diff --git a/src/osd/ReplicatedPG.cc b/src/osd/ReplicatedPG.cc
index 60292769a907f..48faa96e8819e 100644
--- a/src/osd/ReplicatedPG.cc
+++ b/src/osd/ReplicatedPG.cc
@@ -1768,7 +1768,8 @@ void ReplicatedPG::apply_repop(RepGather *repop)
 
   Context *oncommit = new C_OSD_OpCommit(this, repop);
   Context *onapplied = new C_OSD_OpApplied(this, repop);
-  int r = osd->store->queue_transactions(repop->tls, onapplied, oncommit);
+  Context *onapplied_sync = new C_OSD_OndiskWriteUnlock(repop->obc);
+  int r = osd->store->queue_transactions(repop->tls, onapplied, oncommit, onapplied_sync);
   if (r) {
     dout(-10) << "apply_repop  queue_transactions returned " << r << " on " << *repop << dendl;
     assert(0);
@@ -1777,8 +1778,6 @@ void ReplicatedPG::apply_repop(RepGather *repop)
 
 void ReplicatedPG::op_applied(RepGather *repop)
 {
-  repop->obc->ondisk_write_unlock();
-
   lock();
   dout(10) << "op_applied " << *repop << dendl;
 
@@ -2965,7 +2964,6 @@ void ReplicatedPG::_committed(epoch_t same_since, eversion_t last_complete)
 void ReplicatedPG::_wrote_pushed_object(ObjectStore::Transaction *t, ObjectContext *obc)
 {
   dout(10) << "_wrote_pushed_object " << *obc << dendl;
-  obc->ondisk_write_unlock();
   lock();
   put_object_context(obc);
   unlock();
@@ -3126,6 +3124,7 @@ void ReplicatedPG::sub_op_push(MOSDSubOp *op)
 
   // track ObjectContext
   Context *onreadable = 0;
+  Context *onreadable_sync = 0;
   if (is_primary()) {
     dout(10) << " setting up obc for " << soid << dendl;
     ObjectContext *obc = 0;
@@ -3136,6 +3135,7 @@ void ReplicatedPG::sub_op_push(MOSDSubOp *op)
     obc->obs.oi.decode(oibl);
 
     onreadable = new C_OSD_WrotePushedObject(this, t, obc);
+    onreadable_sync = new C_OSD_OndiskWriteUnlock(obc);
   } else {
     onreadable = new ObjectStore::C_DeleteTransaction(t);
   }
@@ -3145,7 +3145,8 @@ void ReplicatedPG::sub_op_push(MOSDSubOp *op)
   int r = osd->store->queue_transaction(t,
 					onreadable,
 					new C_OSD_Commit(this, info.history.same_acting_since,
-							 info.last_complete));
+							 info.last_complete),
+					onreadable_sync);
   assert(r == 0);
 
   osd->logger->inc(l_osd_r_pull);
diff --git a/src/osd/ReplicatedPG.h b/src/osd/ReplicatedPG.h
index f6c54f7fe17c7..226b94acba7ff 100644
--- a/src/osd/ReplicatedPG.h
+++ b/src/osd/ReplicatedPG.h
@@ -523,6 +523,13 @@ protected:
       rm->pg->sub_op_modify_commit(rm);
     }
   };
+  struct C_OSD_OndiskWriteUnlock : public Context {
+    ObjectContext *obc;
+    C_OSD_OndiskWriteUnlock(ObjectContext *o) : obc(o) {}
+    void finish(int r) {
+      obc->ondisk_write_unlock();
+    }
+  };
   struct C_OSD_WrotePushedObject : public Context {
     ReplicatedPG *pg;
     ObjectStore::Transaction *t;
-- 
2.39.5