From a38f9e5104a6e08e130dc4f15ad19a06d9e63719 Mon Sep 17 00:00:00 2001
From: Jason Dillaman <dillaman@redhat.com>
Date: Thu, 7 May 2015 14:06:16 -0400
Subject: [PATCH] osdc: ObjectCacher flusher might needs additional locks

librbd requires the ObjectCacher flusher thread to acquire
an additional lock in order to maintain lock ordering
constraints.

Signed-off-by: Jason Dillaman <dillaman@redhat.com>
---
 src/librbd/LibrbdWriteback.cc |  9 +++++++++
 src/librbd/LibrbdWriteback.h  |  3 +++
 src/osdc/ObjectCacher.cc      | 10 ++++++++++
 src/osdc/WritebackHandler.h   |  3 +++
 4 files changed, 25 insertions(+)

diff --git a/src/librbd/LibrbdWriteback.cc b/src/librbd/LibrbdWriteback.cc
index cd2e1de2aa8a6..058760e6d9799 100644
--- a/src/librbd/LibrbdWriteback.cc
+++ b/src/librbd/LibrbdWriteback.cc
@@ -157,6 +157,7 @@ namespace librbd {
 			       uint64_t trunc_size, __u32 trunc_seq,
 			       Context *oncommit)
   {
+    assert(m_ictx->owner_lock.is_locked());
     uint64_t object_no = oid_to_object_no(oid.name, m_ictx->object_prefix);
     
     write_result_d *result = new write_result_d(oid.name, oncommit);
@@ -169,6 +170,14 @@ namespace librbd {
     return ++m_tid;
   }
 
+  void LibrbdWriteback::get_client_lock() {
+    m_ictx->owner_lock.get_read();
+  }
+
+  void LibrbdWriteback::put_client_lock() {
+    m_ictx->owner_lock.put_read();
+  }
+
   void LibrbdWriteback::complete_writes(const std::string& oid)
   {
     assert(m_lock.is_locked());
diff --git a/src/librbd/LibrbdWriteback.h b/src/librbd/LibrbdWriteback.h
index 2c71e8434cb9d..b5578ae62f5c3 100644
--- a/src/librbd/LibrbdWriteback.h
+++ b/src/librbd/LibrbdWriteback.h
@@ -38,6 +38,9 @@ namespace librbd {
 			const bufferlist &bl, utime_t mtime, uint64_t trunc_size,
 			__u32 trunc_seq, Context *oncommit);
 
+    virtual void get_client_lock();
+    virtual void put_client_lock();
+
     struct write_result_d {
       bool done;
       int ret;
diff --git a/src/osdc/ObjectCacher.cc b/src/osdc/ObjectCacher.cc
index 5465706370fcd..db6779c7dbf0c 100644
--- a/src/osdc/ObjectCacher.cc
+++ b/src/osdc/ObjectCacher.cc
@@ -1520,6 +1520,7 @@ int ObjectCacher::_wait_for_write(OSDWrite *wr, uint64_t len, ObjectSet *oset, C
 void ObjectCacher::flusher_entry()
 {
   ldout(cct, 10) << "flusher start" << dendl;
+  writeback_handler.get_client_lock();
   lock.Lock();
   while (!flusher_stop) {
     loff_t all = get_stat_tx() + get_stat_rx() + get_stat_clean() + get_stat_dirty();
@@ -1556,13 +1557,21 @@ void ObjectCacher::flusher_entry()
       if (!max) {
 	// back off the lock to avoid starving other threads
 	lock.Unlock();
+        writeback_handler.put_client_lock();
+        writeback_handler.get_client_lock();
 	lock.Lock();
 	continue;
       }
     }
     if (flusher_stop)
       break;
+
+    writeback_handler.put_client_lock();
     flusher_cond.WaitInterval(cct, lock, utime_t(1,0));
+    lock.Unlock();
+
+    writeback_handler.get_client_lock();
+    lock.Lock();
   }
 
   /* Wait for reads to finish. This is only possible if handling
@@ -1578,6 +1587,7 @@ void ObjectCacher::flusher_entry()
   }
 
   lock.Unlock();
+  writeback_handler.put_client_lock();
   ldout(cct, 10) << "flusher finish" << dendl;
 }
 
diff --git a/src/osdc/WritebackHandler.h b/src/osdc/WritebackHandler.h
index 466f84e77983b..fe7d977de26e2 100644
--- a/src/osdc/WritebackHandler.h
+++ b/src/osdc/WritebackHandler.h
@@ -37,6 +37,9 @@ class WritebackHandler {
 			  int op, int flags, Context *onack, Context *oncommit) {
     assert(0 == "this WritebackHandler does not support the lock operation");
   }
+
+  virtual void get_client_lock() {}
+  virtual void put_client_lock() {}
 };
 
 #endif
-- 
2.39.5