From 91bf3bb39f8b7f449e804a9d876922f8b4cd7ba0 Mon Sep 17 00:00:00 2001
From: Haomai Wang <haomai@xsky.com>
Date: Sun, 17 Jan 2016 23:21:29 +0800
Subject: [PATCH] NVMEDevice: make flush wait for all write completed

Signed-off-by: Haomai Wang <haomai@xsky.com>
---
 src/os/bluestore/NVMEDevice.cc | 17 +++++++++++++++++
 src/os/bluestore/NVMEDevice.h  |  4 ++++
 2 files changed, 21 insertions(+)

diff --git a/src/os/bluestore/NVMEDevice.cc b/src/os/bluestore/NVMEDevice.cc
index af78b5cfe0366..623c3b0b4a698 100644
--- a/src/os/bluestore/NVMEDevice.cc
+++ b/src/os/bluestore/NVMEDevice.cc
@@ -317,6 +317,8 @@ NVMEDevice::NVMEDevice(aio_callback_t cb, void *cbpriv)
       queue_empty(1),
       queue_lock("NVMEDevice::queue_lock"),
       aio_thread(this),
+      flush_lock("NVMEDevice::flush_lock"),
+      flush_waiters(0),
       logger(nullptr),
       inflight_ops(0),
       aio_callback(cb),
@@ -418,6 +420,7 @@ void NVMEDevice::_aio_thread()
       if (queue_empty.read()) {
         if (aio_stop)
           break;
+        assert(flush_waiters.read() == 0);
         queue_cond.Wait(queue_lock);
       }
     }
@@ -477,6 +480,10 @@ void NVMEDevice::_aio_thread()
       }
     } else if (!inflight_ops.read()) {
       dout(20) << __func__ << " idle, have a pause" << dendl;
+      if (flush_waiters.read()) {
+        Mutex::Locker l(flush_lock);
+        flush_cond.Signal();
+      }
 #ifdef HAVE_SSE
       _mm_pause();
 #else
@@ -494,6 +501,16 @@ void NVMEDevice::_aio_thread()
 int NVMEDevice::flush()
 {
   dout(10) << __func__ << " start" << dendl;
+  if (inflight_ops.read()) {
+    // TODO: this may contains read op
+    dout(1) << __func__ << " existed inflight ops " << inflight_ops.read() << dendl;
+    Mutex::Locker l(flush_lock);
+    flush_waiters.inc();
+    while (inflight_ops.read()) {
+      flush_cond.Wait(flush_lock);
+    }
+    flush_waiters.dec();
+  }
   return 0;
   // nvme device will cause terriable performance degraded
   // while issuing flush command
diff --git a/src/os/bluestore/NVMEDevice.h b/src/os/bluestore/NVMEDevice.h
index 96cfd1c8e07ef..3825c590b6db4 100644
--- a/src/os/bluestore/NVMEDevice.h
+++ b/src/os/bluestore/NVMEDevice.h
@@ -92,6 +92,10 @@ class NVMEDevice : public BlockDevice {
 
   void _aio_thread();
 
+  Mutex flush_lock;
+  Cond flush_cond;
+  atomic_t flush_waiters;
+
   static void init();
 
  public:
-- 
2.39.5