From 88377be21db9f6255dfefbeb7a18bdda8bb4481e Mon Sep 17 00:00:00 2001
From: Sage Weil <sage@newdream.net>
Date: Tue, 17 Nov 2009 16:26:33 -0800
Subject: [PATCH] osd: eval access mode before try_*

The problem is the try_* functions may change the mode, but a start_write
may not follow if the op doesn't execute (for any number of reasons).  So
the next time we come around, we may need to reset back to IDLE if
num_wr == 0.  Be careful about the wake flag.

This fixes osd op hangs (particularly after mds restart, and lots of
failed stats on objects during file size/mtime recovery).
---
 src/osd/ReplicatedPG.cc |  1 +
 src/osd/ReplicatedPG.h  | 33 +++++++++++++++++----------------
 2 files changed, 18 insertions(+), 16 deletions(-)

diff --git a/src/osd/ReplicatedPG.cc b/src/osd/ReplicatedPG.cc
index ea88fab7a5783..0c1db3dc6a4f8 100644
--- a/src/osd/ReplicatedPG.cc
+++ b/src/osd/ReplicatedPG.cc
@@ -487,6 +487,7 @@ void ReplicatedPG::do_op(MOSDOp *op)
   
   bool ok;
   dout(10) << "do_op mode is " << mode << dendl;
+  assert(!mode.wake);   // we should never have woken waiters here.
   if (op->may_read() && op->may_write())
     ok = mode.try_rmw(client);
   else if (op->may_write())
diff --git a/src/osd/ReplicatedPG.h b/src/osd/ReplicatedPG.h
index 0ca4051182dbd..cdfd023a355e2 100644
--- a/src/osd/ReplicatedPG.h
+++ b/src/osd/ReplicatedPG.h
@@ -102,7 +102,13 @@ public:
     AccessMode() : state(IDLE),
 		   num_wr(0), wake(false) {}
 
+    void check_mode() {
+      if (num_wr == 0)
+	state = IDLE;
+    }
+
     bool try_read(entity_inst_t& c) {
+      check_mode();
       switch (state) {
       case IDLE:
       case DELAYED:
@@ -120,6 +126,7 @@ public:
       }
     }
     bool try_write(entity_inst_t& c) {
+      check_mode();
       switch (state) {
       case IDLE:
 	state = DELAYED;
@@ -138,6 +145,7 @@ public:
       }
     }
     bool try_rmw(entity_inst_t& c) {
+      check_mode();
       switch (state) {
       case IDLE:
 	state = RMW;
@@ -173,21 +181,10 @@ public:
     void finish_write() {
       assert(num_wr > 0);
       --num_wr;
-      if (num_wr == 0)
-	switch (state) {
-	case DELAYED:
-	  state = IDLE;
-	  wake = true;
-	  break;
-	case RMW:
-	case DELAYED_FLUSHING:
-	case RMW_FLUSHING:
-	  state = IDLE;
-	  wake = true;
-	  break;
-	default:
-	  assert(0);
-	}
+      if (num_wr == 0) {
+	state = IDLE;
+	wake = true;
+      }
     }
   };
 
@@ -491,7 +488,11 @@ inline ostream& operator<<(ostream& out, ReplicatedPG::RepGather& repop)
 
 inline ostream& operator<<(ostream& out, ReplicatedPG::AccessMode& mode)
 {
-  return out << mode.get_state_name(mode.state) << "(wr=" << mode.num_wr << ")";    
+  out << mode.get_state_name(mode.state) << "(wr=" << mode.num_wr;
+  if (mode.wake)
+    out << " WAKE";
+  out << ")";
+  return out;
 }
 
 #endif
-- 
2.39.5