From 310032ee8128f6417ac302c0f5ecd27c691cbcc7 Mon Sep 17 00:00:00 2001
From: Sage Weil <sage@newdream.net>
Date: Wed, 29 Jun 2011 15:23:40 -0700
Subject: [PATCH] mds: fix mds scatter_writebehind starvation

scatter_writebehind is called by eval_gather on dirty locks, and
eval_gather is called by wrlock_finish on unstable locks when you
drop the last wrlock...and scatter_writebehind force-takes a wrlock.

This meant that a workload like:
seq 3000|xargs -i mkdir a/b/{} &
mkdir a/c
could cause the mkdir a/c to wait until after the other process
finished because rstats can propagate upwards asynchronously, but
mark the directory dirty synchronously, while the mkdir a/c requires
an actual wrlock in order to modify the rstats.

Signed-off-by: Sage Weil <sage@newdream.net>
Signed-off-by: Greg Farnum <gregory.farnum@dreamhost.com>
---
 src/mds/Locker.cc     |  3 ++-
 src/mds/ScatterLock.h | 19 +++++++++++++++----
 src/mds/SimpleLock.h  |  2 ++
 3 files changed, 19 insertions(+), 5 deletions(-)

diff --git a/src/mds/Locker.cc b/src/mds/Locker.cc
index 020864dcf2945..9f939a568d064 100644
--- a/src/mds/Locker.cc
+++ b/src/mds/Locker.cc
@@ -615,11 +615,12 @@ void Locker::eval_gather(SimpleLock *lock, bool first, bool *pneed_issue, list<C
     } else {
       // auth
 
-      if (lock->is_dirty()) {
+      if (lock->is_dirty() && !lock->is_flushed()) {
 	scatter_writebehind((ScatterLock*)lock);
 	mds->mdlog->flush();
 	return;
       }
+      lock->clear_flushed();
       
       switch (lock->get_state()) {
 	// to mixed
diff --git a/src/mds/ScatterLock.h b/src/mds/ScatterLock.h
index 9afd21a8dd439..9c64fa1f0ed2e 100644
--- a/src/mds/ScatterLock.h
+++ b/src/mds/ScatterLock.h
@@ -20,7 +20,7 @@
 
 class ScatterLock : public SimpleLock {
   struct more_bits_t {
-    bool dirty, flushing;
+    bool dirty, flushing, flushed;
     bool scatter_wanted;
     utime_t last_scatter;
     xlist<ScatterLock*>::item item_updated;
@@ -28,13 +28,15 @@ class ScatterLock : public SimpleLock {
     bool stale;
 
     more_bits_t(ScatterLock *lock) :
-      dirty(false), flushing(false), scatter_wanted(false),
+      dirty(false), flushing(false), flushed(false), scatter_wanted(false),
       item_updated(lock), stale(false)
     {}
 
     bool empty() const {
-      return dirty == false &&
+      return
+	dirty == false &&
 	flushing == false &&
+	flushed == false &&
 	scatter_wanted == false &&
 	!item_updated.is_on_list() &&
 	!stale;
@@ -116,6 +118,9 @@ public:
   bool is_flushing() const {
     return have_more() ? _more->flushing : false;
   }
+  bool is_flushed() const {
+    return have_more() ? _more->flushed : false;
+  }
   bool is_dirty_or_flushing() const {
     return have_more() ? (_more->dirty || _more->flushing) : false;
   }
@@ -134,17 +139,23 @@ public:
   void finish_flush() {
     if (more()->flushing) {
       _more->flushing = false;
+      _more->flushed = true;
       if (!_more->dirty) {
 	parent->put(MDSCacheObject::PIN_DIRTYSCATTERED);
 	parent->clear_dirty_scattered(get_type());
       }
-      try_clear_more();
     }
   }
   void clear_dirty() {
     start_flush();
     finish_flush();
   }
+  void clear_flushed() {
+    if (_more) {
+      _more->flushed = false;
+      try_clear_more();
+    }
+  }
 
   void set_last_scatter(utime_t t) { more()->last_scatter = t; }
   utime_t get_last_scatter() {
diff --git a/src/mds/SimpleLock.h b/src/mds/SimpleLock.h
index d11de69b51389..d35cc6314a0c9 100644
--- a/src/mds/SimpleLock.h
+++ b/src/mds/SimpleLock.h
@@ -386,6 +386,8 @@ public:
   virtual bool is_dirty() const { return false; }
   virtual bool is_stale() const { return false; }
   virtual bool is_flushing() const { return false; }
+  virtual bool is_flushed() const { return false; }
+  virtual void clear_flushed() { }
 
   // can_*
   bool can_lease(client_t client) const {
-- 
2.39.5