librbd: ImageWatcher should cancel in-flight ops on watch error

Upon an watch error notification from librados, pending requests
should be canceled and in-flight IO should be flushed prior to
unlocking the image.

Fixes: #11363
Signed-off-by: Jason Dillaman <dillaman@redhat.com>
(cherry picked from commit fc2e511b2aed4d40eff5101a4c9e513b34e5e58e)

diff --git a/src/librbd/ImageWatcher.cc b/src/librbd/ImageWatcher.cc
index 142c2db84233df8693bcb4cd10bf49d415b033e6..8233633ceb59cd7429140c440c5c3cf0c3ba358d 100644 (file)
--- a/src/librbd/ImageWatcher.cc
+++ b/src/librbd/ImageWatcher.cc
@@ -363,22 +363,21 @@ int ImageWatcher::unlock()
   return 0;
 }
 
-void ImageWatcher::release_lock()
+bool ImageWatcher::release_lock()
 {
+  assert(m_image_ctx.owner_lock.is_wlocked());
   ldout(m_image_ctx.cct, 10) << "releasing exclusive lock by request" << dendl;
-  {
-    RWLock::WLocker l(m_image_ctx.owner_lock);
-    if (!is_lock_owner()) {
-      return;
-    }
-    prepare_unlock();
+  if (!is_lock_owner()) {
+    return false;
   }
+  prepare_unlock();
 
+  m_image_ctx.owner_lock.put_write();
   m_image_ctx.cancel_async_requests();
+  m_image_ctx.owner_lock.get_write();
 
-  RWLock::WLocker l(m_image_ctx.owner_lock);
   if (!is_lock_owner()) {
-    return;
+    return false;
   }
 
   {
@@ -387,6 +386,7 @@ void ImageWatcher::release_lock()
   }
 
   unlock();
+  return true;
 }
 
 void ImageWatcher::finalize_header_update() {
@@ -563,6 +563,11 @@ ClientId ImageWatcher::get_client_id() {
   return ClientId(m_image_ctx.md_ctx.get_instance_id(), m_watch_handle);
 }
 
+void ImageWatcher::notify_release_lock() {
+  RWLock::WLocker owner_locker(m_image_ctx.owner_lock);
+  release_lock();
+}
+
 void ImageWatcher::notify_released_lock() {
   ldout(m_image_ctx.cct, 10) << "notify released lock" << dendl;
   bufferlist bl;
@@ -776,7 +781,7 @@ void ImageWatcher::handle_payload(const RequestLockPayload &payload,
 
     ldout(m_image_ctx.cct, 10) << "queuing release of exclusive lock" << dendl;
     FunctionContext *ctx = new FunctionContext(
-      boost::bind(&ImageWatcher::release_lock, this));
+      boost::bind(&ImageWatcher::notify_release_lock, this));
     m_task_finisher->queue(TASK_CODE_RELEASING_LOCK, ctx);
   }
 }
@@ -970,9 +975,10 @@ void ImageWatcher::reregister_watch() {
 
   {
     RWLock::WLocker l(m_image_ctx.owner_lock);
-    bool lock_owner = (m_lock_owner_state == LOCK_OWNER_STATE_LOCKED);
-    if (lock_owner) {
-      unlock();
+    bool was_lock_owner = false;
+    if (m_lock_owner_state == LOCK_OWNER_STATE_LOCKED) {
+      // ensure all async requests are canceled and IO is flushed
+      was_lock_owner = release_lock();
     }
 
     int r;
@@ -1000,7 +1006,7 @@ void ImageWatcher::reregister_watch() {
     }
     handle_payload(HeaderUpdatePayload(), NULL);
 
-    if (lock_owner) {
+    if (was_lock_owner) {
       r = try_lock();
       if (r == -EBUSY) {
         ldout(m_image_ctx.cct, 5) << "lost image lock while re-registering "

diff --git a/src/librbd/ImageWatcher.h b/src/librbd/ImageWatcher.h
index 2ffe96569723aa9b27c21b4bc28fded053a6f343..b20d6b1b7a3f1347b441bd5018ed1fa700a17c17 100644 (file)
--- a/src/librbd/ImageWatcher.h
+++ b/src/librbd/ImageWatcher.h
@@ -210,7 +210,7 @@ namespace librbd {
     int get_lock_owner_info(entity_name_t *locker, std::string *cookie,
                            std::string *address, uint64_t *handle);
     int lock();
-    void release_lock();
+    bool release_lock();
     bool try_request_lock();
     void finalize_header_update();
 
@@ -222,6 +222,7 @@ namespace librbd {
 
     WatchNotify::ClientId get_client_id();
 
+    void notify_release_lock();
     void notify_released_lock();
     void notify_request_lock();
     int notify_lock_owner(bufferlist &bl);