librbd: ImageWatcher should cancel in-flight ops on watch error

Upon an watch error notification from librados, pending requests
should be canceled and in-flight IO should be flushed prior to
unlocking the image.

Fixes: #11363
Signed-off-by: Jason Dillaman <dillaman@redhat.com>

diff --git a/src/librbd/ImageWatcher.cc b/src/librbd/ImageWatcher.cc
index 218f1a394e467c08e97a89cd273fd095d833d368..214db128508efd0ea471adf4e1b7a136bb0b14bc 100644 (file)
--- a/src/librbd/ImageWatcher.cc
+++ b/src/librbd/ImageWatcher.cc
@@ -361,22 +361,21 @@ int ImageWatcher::unlock()
   return 0;
 }
 
-void ImageWatcher::release_lock()
+bool ImageWatcher::release_lock()
 {
+  assert(m_image_ctx.owner_lock.is_wlocked());
   ldout(m_image_ctx.cct, 10) << "releasing exclusive lock by request" << dendl;
-  {
-    RWLock::WLocker l(m_image_ctx.owner_lock);
-    if (!is_lock_owner()) {
-      return;
-    }
-    prepare_unlock();
+  if (!is_lock_owner()) {
+    return false;
   }
+  prepare_unlock();
 
+  m_image_ctx.owner_lock.put_write();
   m_image_ctx.cancel_async_requests();
+  m_image_ctx.owner_lock.get_write();
 
-  RWLock::WLocker l(m_image_ctx.owner_lock);
   if (!is_lock_owner()) {
-    return;
+    return false;
   }
 
   {
@@ -385,6 +384,7 @@ void ImageWatcher::release_lock()
   }
 
   unlock();
+  return true;
 }
 
 void ImageWatcher::assert_header_locked(librados::ObjectWriteOperation *op) {
@@ -556,6 +556,11 @@ ClientId ImageWatcher::get_client_id() {
   return ClientId(m_image_ctx.md_ctx.get_instance_id(), m_watch_handle);
 }
 
+void ImageWatcher::notify_release_lock() {
+  RWLock::WLocker owner_locker(m_image_ctx.owner_lock);
+  release_lock();
+}
+
 void ImageWatcher::notify_released_lock() {
   ldout(m_image_ctx.cct, 10) << "notify released lock" << dendl;
   bufferlist bl;
@@ -769,7 +774,7 @@ void ImageWatcher::handle_payload(const RequestLockPayload &payload,
 
     ldout(m_image_ctx.cct, 10) << "queuing release of exclusive lock" << dendl;
     FunctionContext *ctx = new FunctionContext(
-      boost::bind(&ImageWatcher::release_lock, this));
+      boost::bind(&ImageWatcher::notify_release_lock, this));
     m_task_finisher->queue(TASK_CODE_RELEASING_LOCK, ctx);
   }
 }
@@ -953,9 +958,10 @@ void ImageWatcher::reregister_watch() {
 
   {
     RWLock::WLocker l(m_image_ctx.owner_lock);
-    bool lock_owner = (m_lock_owner_state == LOCK_OWNER_STATE_LOCKED);
-    if (lock_owner) {
-      unlock();
+    bool was_lock_owner = false;
+    if (m_lock_owner_state == LOCK_OWNER_STATE_LOCKED) {
+      // ensure all async requests are canceled and IO is flushed
+      was_lock_owner = release_lock();
     }
 
     int r;
@@ -983,7 +989,7 @@ void ImageWatcher::reregister_watch() {
     }
     handle_payload(HeaderUpdatePayload(), NULL);
 
-    if (lock_owner) {
+    if (was_lock_owner) {
       r = try_lock();
       if (r == -EBUSY) {
         ldout(m_image_ctx.cct, 5) << "lost image lock while re-registering "

diff --git a/src/librbd/ImageWatcher.h b/src/librbd/ImageWatcher.h
index d75e2fe22223fa8f26b919baf1ae1d07b3413afb..ba126ac2ee6f18f9cb89bbde9de8c5e2310d020a 100644 (file)
--- a/src/librbd/ImageWatcher.h
+++ b/src/librbd/ImageWatcher.h
@@ -209,7 +209,7 @@ namespace librbd {
     int get_lock_owner_info(entity_name_t *locker, std::string *cookie,
                            std::string *address, uint64_t *handle);
     int lock();
-    void release_lock();
+    bool release_lock();
     bool try_request_lock();
 
     void schedule_retry_aio_requests(bool use_timer);
@@ -220,6 +220,7 @@ namespace librbd {
 
     WatchNotify::ClientId get_client_id();
 
+    void notify_release_lock();
     void notify_released_lock();
     void notify_request_lock();
     int notify_lock_owner(bufferlist &bl);