From 0b179be31ed9f4462704826f23f84287f15b9ab5 Mon Sep 17 00:00:00 2001
From: Mykola Golub <mgolub@mirantis.com>
Date: Tue, 24 Jan 2017 12:53:51 +0100
Subject: [PATCH] librbd: track in-fly break_lock and get_locker requests in
 managed lock

Signed-off-by: Mykola Golub <mgolub@mirantis.com>
---
 src/librbd/ManagedLock.cc | 68 +++++++++++++++++++++++++++++++++++----
 src/librbd/ManagedLock.h  |  6 ++--
 2 files changed, 65 insertions(+), 9 deletions(-)

diff --git a/src/librbd/ManagedLock.cc b/src/librbd/ManagedLock.cc
index 5861da7bd34a2..4325d620730d3 100644
--- a/src/librbd/ManagedLock.cc
+++ b/src/librbd/ManagedLock.cc
@@ -38,6 +38,21 @@ struct C_SendLockRequest : public Context {
   }
 };
 
+struct C_Tracked : public Context {
+  AsyncOpTracker &tracker;
+  Context *ctx;
+  C_Tracked(AsyncOpTracker &tracker, Context *ctx)
+    : tracker(tracker), ctx(ctx) {
+    tracker.start_op();
+  }
+  virtual ~C_Tracked() {
+    tracker.finish_op();
+  }
+  virtual void finish(int r) override {
+    ctx->complete(r);
+  }
+};
+
 } // anonymous namespace
 
 using librbd::util::create_context_callback;
@@ -66,6 +81,14 @@ ManagedLock<I>::~ManagedLock() {
   Mutex::Locker locker(m_lock);
   assert(m_state == STATE_SHUTDOWN || m_state == STATE_UNLOCKED ||
          m_state == STATE_UNINITIALIZED);
+  if (m_state == STATE_UNINITIALIZED) {
+    // never initialized -- ensure any in-flight ops are complete
+    // since we wouldn't expect shut_down to be invoked
+    C_SaferCond ctx;
+    m_async_op_tracker.wait_for_ops(&ctx);
+    ctx.wait();
+  }
+  assert(m_async_op_tracker.empty());
 }
 
 template <typename I>
@@ -200,17 +223,37 @@ void ManagedLock<I>::get_locker(managed_lock::Locker *locker,
                                 Context *on_finish) {
   ldout(m_cct, 10) << dendl;
 
-  auto req = managed_lock::GetLockerRequest<I>::create(
-    m_ioctx, m_oid, m_mode == EXCLUSIVE, locker, on_finish);
-  req->send();
+  int r;
+  {
+    Mutex::Locker l(m_lock);
+    if (is_state_shutdown()) {
+      r = -ESHUTDOWN;
+    } else {
+      on_finish = new C_Tracked(m_async_op_tracker, on_finish);
+      auto req = managed_lock::GetLockerRequest<I>::create(
+        m_ioctx, m_oid, m_mode == EXCLUSIVE, locker, on_finish);
+      req->send();
+      return;
+    }
+  }
+
+  on_finish->complete(r);
 }
 
 template <typename I>
 void ManagedLock<I>::break_lock(const managed_lock::Locker &locker,
                                 bool force_break_lock, Context *on_finish) {
+  ldout(m_cct, 10) << dendl;
+
+  int r;
   {
     Mutex::Locker l(m_lock);
-    if (!is_lock_owner(m_lock)) {
+    if (is_state_shutdown()) {
+      r = -ESHUTDOWN;
+    } else if (is_lock_owner(m_lock)) {
+      r = -EBUSY;
+    } else {
+      on_finish = new C_Tracked(m_async_op_tracker, on_finish);
       auto req = managed_lock::BreakRequest<I>::create(
         m_ioctx, m_work_queue, m_oid, locker, m_blacklist_on_break_lock,
         m_blacklist_expire_seconds, force_break_lock, on_finish);
@@ -219,7 +262,7 @@ void ManagedLock<I>::break_lock(const managed_lock::Locker &locker,
     }
   }
 
-  on_finish->complete(-EBUSY);
+  on_finish->complete(r);
 }
 
 template <typename I>
@@ -633,7 +676,7 @@ template <typename I>
 void ManagedLock<I>::handle_shutdown(int r) {
   ldout(m_cct, 10) << ": r=" << r << dendl;
 
-  complete_shutdown(r);
+  wait_for_tracked_ops(r);
 }
 
 template <typename I>
@@ -676,7 +719,18 @@ template <typename I>
 void ManagedLock<I>::handle_shutdown_post_release(int r) {
   ldout(m_cct, 10) << ": r=" << r << dendl;
 
-  complete_shutdown(r);
+  wait_for_tracked_ops(r);
+}
+
+template <typename I>
+void ManagedLock<I>::wait_for_tracked_ops(int r) {
+  ldout(m_cct, 10) << ": r=" << r << dendl;
+
+  Context *ctx = new FunctionContext([this, r](int ret) {
+      complete_shutdown(r);
+    });
+
+  m_async_op_tracker.wait_for_ops(ctx);
 }
 
 template <typename I>
diff --git a/src/librbd/ManagedLock.h b/src/librbd/ManagedLock.h
index b8bec661b918e..075b8f758495c 100644
--- a/src/librbd/ManagedLock.h
+++ b/src/librbd/ManagedLock.h
@@ -7,10 +7,11 @@
 #include "include/int_types.h"
 #include "include/Context.h"
 #include "include/rados/librados.hpp"
+#include "common/AsyncOpTracker.h"
+#include "common/Mutex.h"
 #include "cls/lock/cls_lock_types.h"
 #include "librbd/watcher/Types.h"
 #include "librbd/managed_lock/Types.h"
-#include "common/Mutex.h"
 #include <list>
 #include <string>
 #include <utility>
@@ -219,6 +220,7 @@ private:
   State m_post_next_state;
 
   ActionsContexts m_actions_contexts;
+  AsyncOpTracker m_async_op_tracker;
 
   bool is_lock_owner(Mutex &lock) const;
   bool is_transition_state() const;
@@ -229,7 +231,6 @@ private:
   Action get_active_action() const;
   void complete_active_action(State next_state, int r);
 
-
   void send_acquire_lock();
   void handle_pre_acquire_lock(int r);
   void handle_acquire_lock(int r);
@@ -249,6 +250,7 @@ private:
   void send_shutdown_release();
   void handle_shutdown_pre_release(int r);
   void handle_shutdown_post_release(int r);
+  void wait_for_tracked_ops(int r);
   void complete_shutdown(int r);
 };
 
-- 
2.39.5