librbd: attempt to recover lost image watcher upon all failures

For example, if an image is blacklisted and the blacklist eventually
expires, the image should recover its watch.

Fixes: http://tracker.ceph.com/issues/34534
Signed-off-by: Jason Dillaman <dillaman@redhat.com>
(cherry picked from commit 23b7447f6be87a14f84664f29431d2fdd2af4512)

Conflicts:
	src/librbd/watcher/RewatchRequest.cc: trivial resolution
	src/test/librbd/test_mock_Watcher.cc: trivial resolution

diff --git a/src/librbd/Watcher.cc b/src/librbd/Watcher.cc
index 9a72fc13a76b1a722cd8621c583b48ff262ca927..1865eaaf84201425d6d164e9dbd590d898376928 100644 (file)
--- a/src/librbd/Watcher.cc
+++ b/src/librbd/Watcher.cc
@@ -314,9 +314,9 @@ void Watcher::handle_rewatch_callback(int r) {
     if (m_unregister_watch_ctx != nullptr) {
       m_watch_state = WATCH_STATE_IDLE;
       std::swap(unregister_watch_ctx, m_unregister_watch_ctx);
-    } else if (r  == -EBLACKLISTED || r == -ENOENT) {
+    } else if (r == -ENOENT) {
       m_watch_state = WATCH_STATE_IDLE;
-    } else if (m_watch_error) {
+    } else if (r < 0 || m_watch_error) {
       watch_error = true;
     } else {
       m_watch_state = WATCH_STATE_IDLE;

diff --git a/src/librbd/watcher/RewatchRequest.cc b/src/librbd/watcher/RewatchRequest.cc
index 22d2bb8e567888312790f3ae6e4fb968cf8f7b08..48b965110ccde8bea2b29431e40b9e6a20e7df01 100644 (file)
--- a/src/librbd/watcher/RewatchRequest.cc
+++ b/src/librbd/watcher/RewatchRequest.cc
@@ -35,7 +35,11 @@ void RewatchRequest::send() {
 
 void RewatchRequest::unwatch() {
   assert(m_watch_lock.is_wlocked());
-  assert(*m_watch_handle != 0);
+  ceph_assert(m_watch_lock.is_wlocked());
+  if (*m_watch_handle == 0) {
+    rewatch();
+    return;
+  }
 
   CephContext *cct = reinterpret_cast<CephContext *>(m_ioctx.cct());
   ldout(cct, 10) << dendl;
@@ -81,8 +85,7 @@ void RewatchRequest::handle_rewatch(int r) {
   if (r < 0) {
     lderr(cct) << "failed to watch object: " << cpp_strerror(r)
                << dendl;
-    finish(r);
-    return;
+    m_rewatch_handle = 0;
   }
 
   {
@@ -90,7 +93,7 @@ void RewatchRequest::handle_rewatch(int r) {
     *m_watch_handle = m_rewatch_handle;
   }
 
-  finish(0);
+  finish(r);
 }
 
 void RewatchRequest::finish(int r) {

diff --git a/src/test/librbd/CMakeLists.txt b/src/test/librbd/CMakeLists.txt
index 667cbdfab00b1e980325734685fb2cb6ed7f66b8..924daeecaf6300c187e27d94f14f93cff8b889f6 100644 (file)
--- a/src/test/librbd/CMakeLists.txt
+++ b/src/test/librbd/CMakeLists.txt
@@ -34,6 +34,7 @@ set(unittest_librbd_srcs
   test_mock_ManagedLock.cc
   test_mock_ObjectMap.cc
   test_mock_TrashWatcher.cc
+  test_mock_Watcher.cc
   deep_copy/test_mock_ImageCopyRequest.cc
   deep_copy/test_mock_MetadataCopyRequest.cc
   deep_copy/test_mock_ObjectCopyRequest.cc

diff --git a/src/test/librbd/test_mock_ObjectWatcher.cc b/src/test/librbd/test_mock_ObjectWatcher.cc
deleted file mode 100644 (file)
index b17a55f..0000000
--- a/src/test/librbd/test_mock_ObjectWatcher.cc
+++ /dev/null
@@ -1,404 +0,0 @@
-// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
-// vim: ts=8 sw=2 smarttab
-
-#include "test/librbd/test_mock_fixture.h"
-#include "test/librbd/test_support.h"
-#include "test/librbd/mock/MockImageCtx.h"
-#include "test/librados_test_stub/MockTestMemIoCtxImpl.h"
-#include "test/librados_test_stub/MockTestMemRadosClient.h"
-#include "common/Cond.h"
-#include "common/Mutex.h"
-#include "librados/AioCompletionImpl.h"
-#include "librbd/ObjectWatcher.h"
-#include "gmock/gmock.h"
-#include "gtest/gtest.h"
-#include <list>
-
-namespace librbd {
-
-namespace {
-
-struct MockObjectWatcher : public ObjectWatcher<MockImageCtx> {
-  std::string oid;
-
-  MockObjectWatcher(MockImageCtx &mock_image_ctx, const std::string &oid)
-    : ObjectWatcher<MockImageCtx>(mock_image_ctx.md_ctx,
-                                  mock_image_ctx.op_work_queue),
-      oid(oid) {
-  }
-
-  virtual std::string get_oid() const override {
-    return oid;
-  }
-
-  virtual void handle_notify(uint64_t notify_id, uint64_t handle,
-                             bufferlist &bl) {
-  }
-};
-
-} // anonymous namespace
-
-} // namespace librbd
-
-// template definitions
-#include "librbd/ObjectWatcher.cc"
-template class librbd::ObjectWatcher<librbd::MockImageCtx>;
-
-namespace librbd {
-
-using ::testing::_;
-using ::testing::DoDefault;
-using ::testing::Invoke;
-using ::testing::InSequence;
-using ::testing::Return;
-using ::testing::SaveArg;
-using ::testing::WithArg;
-
-class TestMockObjectWatcher : public TestMockFixture {
-public:
-  TestMockObjectWatcher() : m_lock("TestMockObjectWatcher::m_lock") {
-  }
-
-  virtual void SetUp() {
-    TestMockFixture::SetUp();
-
-    m_oid = get_temp_image_name();
-
-    bufferlist bl;
-    ASSERT_EQ(0, m_ioctx.write_full(m_oid, bl));
-  }
-
-  void expect_aio_watch(MockImageCtx &mock_image_ctx, int r,
-                        const std::function<void()> &action = std::function<void()>()) {
-    librados::MockTestMemIoCtxImpl &mock_io_ctx(get_mock_io_ctx(
-      mock_image_ctx.md_ctx));
-    librados::MockTestMemRadosClient *mock_rados_client(
-      mock_io_ctx.get_mock_rados_client());
-
-    auto &expect = EXPECT_CALL(mock_io_ctx, aio_watch(m_oid, _, _, _));
-    if (r < 0) {
-      expect.WillOnce(DoAll(WithArg<1>(Invoke([this, mock_rados_client, r, action](librados::AioCompletionImpl *c) {
-                                if (action) {
-                                  action();
-                                }
-
-                                c->get();
-                                mock_rados_client->finish_aio_completion(c, r);
-                                notify_watch();
-                              })),
-                            Return(0)));
-    } else {
-      expect.WillOnce(DoAll(SaveArg<3>(&m_watch_ctx),
-                            Invoke([this, &mock_io_ctx, action](const std::string& o,
-                                                                librados::AioCompletionImpl *c,
-                                                                uint64_t *handle,
-                                                                librados::WatchCtx2 *ctx) {
-                                if (action) {
-                                  action();
-                                }
-
-                                mock_io_ctx.do_aio_watch(o, c, handle, ctx);
-                                notify_watch();
-                              }),
-                            Return(0)));
-    }
-  }
-
-  void expect_aio_unwatch(MockImageCtx &mock_image_ctx, int r,
-                          const std::function<void()> &action = std::function<void()>()) {
-    librados::MockTestMemIoCtxImpl &mock_io_ctx(get_mock_io_ctx(
-      mock_image_ctx.md_ctx));
-
-    auto &expect = EXPECT_CALL(mock_io_ctx, aio_unwatch(_, _));
-    if (r < 0) {
-      expect.WillOnce(DoAll(Invoke([this, &mock_io_ctx, r, action](uint64_t handle,
-                                                                   librados::AioCompletionImpl *c) {
-                                if (action) {
-                                  action();
-                                }
-
-                                librados::AioCompletionImpl *dummy_c = new librados::AioCompletionImpl();
-                                mock_io_ctx.do_aio_unwatch(handle, dummy_c);
-                                ASSERT_EQ(0, dummy_c->wait_for_complete());
-                                dummy_c->release();
-
-                                c->get();
-                                mock_io_ctx.get_mock_rados_client()->finish_aio_completion(c, r);
-                                notify_watch();
-                              }),
-                            Return(0)));
-    } else {
-      expect.WillOnce(DoAll(Invoke([this, &mock_io_ctx, action](uint64_t handle,
-                                                                librados::AioCompletionImpl *c) {
-                                if (action) {
-                                  action();
-                                }
-
-                                mock_io_ctx.do_aio_unwatch(handle, c);
-                                notify_watch();
-                              }),
-                            Return(0)));
-    }
-  }
-
-  std::string m_oid;
-  librados::WatchCtx2 *m_watch_ctx = nullptr;
-
-  void notify_watch() {
-    Mutex::Locker locker(m_lock);
-    ++m_watch_count;
-    m_cond.Signal();
-  }
-
-  bool wait_for_watch(MockImageCtx &mock_image_ctx, size_t count) {
-    Mutex::Locker locker(m_lock);
-    while (m_watch_count < count) {
-      if (m_cond.WaitInterval(m_lock, utime_t(10, 0)) != 0) {
-        return false;
-      }
-    }
-    return true;
-  }
-
-  Mutex m_lock;
-  Cond m_cond;
-  size_t m_watch_count = 0;
-};
-
-TEST_F(TestMockObjectWatcher, Success) {
-  librbd::ImageCtx *ictx;
-  ASSERT_EQ(0, open_image(m_image_name, &ictx));
-
-  MockImageCtx mock_image_ctx(*ictx);
-  MockObjectWatcher mock_image_watcher(mock_image_ctx, m_oid);
-
-  InSequence seq;
-  expect_aio_watch(mock_image_ctx, 0);
-  expect_aio_unwatch(mock_image_ctx, 0);
-
-  C_SaferCond register_ctx;
-  mock_image_watcher.register_watch(&register_ctx);
-  ASSERT_EQ(0, register_ctx.wait());
-
-  C_SaferCond unregister_ctx;
-  mock_image_watcher.unregister_watch(&unregister_ctx);
-  ASSERT_EQ(0, unregister_ctx.wait());
-}
-
-TEST_F(TestMockObjectWatcher, RegisterError) {
-  librbd::ImageCtx *ictx;
-  ASSERT_EQ(0, open_image(m_image_name, &ictx));
-
-  MockImageCtx mock_image_ctx(*ictx);
-  MockObjectWatcher mock_image_watcher(mock_image_ctx, m_oid);
-
-  InSequence seq;
-  expect_aio_watch(mock_image_ctx, -EINVAL);
-
-  C_SaferCond register_ctx;
-  mock_image_watcher.register_watch(&register_ctx);
-  ASSERT_EQ(-EINVAL, register_ctx.wait());
-}
-
-TEST_F(TestMockObjectWatcher, UnregisterError) {
-  librbd::ImageCtx *ictx;
-  ASSERT_EQ(0, open_image(m_image_name, &ictx));
-
-  MockImageCtx mock_image_ctx(*ictx);
-  MockObjectWatcher mock_image_watcher(mock_image_ctx, m_oid);
-
-  InSequence seq;
-  expect_aio_watch(mock_image_ctx, 0);
-  expect_aio_unwatch(mock_image_ctx, -EINVAL);
-
-  C_SaferCond register_ctx;
-  mock_image_watcher.register_watch(&register_ctx);
-  ASSERT_EQ(0, register_ctx.wait());
-
-  C_SaferCond unregister_ctx;
-  mock_image_watcher.unregister_watch(&unregister_ctx);
-  ASSERT_EQ(-EINVAL, unregister_ctx.wait());
-}
-
-TEST_F(TestMockObjectWatcher, Reregister) {
-  librbd::ImageCtx *ictx;
-  ASSERT_EQ(0, open_image(m_image_name, &ictx));
-
-  MockImageCtx mock_image_ctx(*ictx);
-  MockObjectWatcher mock_image_watcher(mock_image_ctx, m_oid);
-
-  expect_op_work_queue(mock_image_ctx);
-
-  InSequence seq;
-  expect_aio_watch(mock_image_ctx, 0);
-  expect_aio_unwatch(mock_image_ctx, 0);
-  expect_aio_watch(mock_image_ctx, 0);
-  expect_aio_unwatch(mock_image_ctx, 0);
-
-  C_SaferCond register_ctx;
-  mock_image_watcher.register_watch(&register_ctx);
-  ASSERT_EQ(0, register_ctx.wait());
-
-  assert(m_watch_ctx != nullptr);
-  m_watch_ctx->handle_error(0, -ESHUTDOWN);
-
-  // wait for recovery unwatch/watch
-  ASSERT_TRUE(wait_for_watch(mock_image_ctx, 3));
-
-  C_SaferCond unregister_ctx;
-  mock_image_watcher.unregister_watch(&unregister_ctx);
-  ASSERT_EQ(0, unregister_ctx.wait());
-}
-
-TEST_F(TestMockObjectWatcher, ReregisterUnwatchError) {
-  librbd::ImageCtx *ictx;
-  ASSERT_EQ(0, open_image(m_image_name, &ictx));
-
-  MockImageCtx mock_image_ctx(*ictx);
-  MockObjectWatcher mock_image_watcher(mock_image_ctx, m_oid);
-
-  expect_op_work_queue(mock_image_ctx);
-
-  InSequence seq;
-  expect_aio_watch(mock_image_ctx, 0);
-  expect_aio_unwatch(mock_image_ctx, -EINVAL);
-  expect_aio_watch(mock_image_ctx, 0);
-  expect_aio_unwatch(mock_image_ctx, 0);
-
-  C_SaferCond register_ctx;
-  mock_image_watcher.register_watch(&register_ctx);
-  ASSERT_EQ(0, register_ctx.wait());
-
-  assert(m_watch_ctx != nullptr);
-  m_watch_ctx->handle_error(0, -ESHUTDOWN);
-
-  // wait for recovery unwatch/watch
-  ASSERT_TRUE(wait_for_watch(mock_image_ctx, 3));
-
-  C_SaferCond unregister_ctx;
-  mock_image_watcher.unregister_watch(&unregister_ctx);
-  ASSERT_EQ(0, unregister_ctx.wait());
-}
-
-TEST_F(TestMockObjectWatcher, ReregisterWatchError) {
-  librbd::ImageCtx *ictx;
-  ASSERT_EQ(0, open_image(m_image_name, &ictx));
-
-  MockImageCtx mock_image_ctx(*ictx);
-  MockObjectWatcher mock_image_watcher(mock_image_ctx, m_oid);
-
-  expect_op_work_queue(mock_image_ctx);
-
-  InSequence seq;
-  expect_aio_watch(mock_image_ctx, 0);
-  expect_aio_unwatch(mock_image_ctx, 0);
-  expect_aio_watch(mock_image_ctx, -ESHUTDOWN);
-  expect_aio_watch(mock_image_ctx, 0);
-  expect_aio_unwatch(mock_image_ctx, 0);
-
-  C_SaferCond register_ctx;
-  mock_image_watcher.register_watch(&register_ctx);
-  ASSERT_EQ(0, register_ctx.wait());
-
-  assert(m_watch_ctx != nullptr);
-  m_watch_ctx->handle_error(0, -ESHUTDOWN);
-
-  // wait for recovery unwatch/watch
-  ASSERT_TRUE(wait_for_watch(mock_image_ctx, 4));
-
-  C_SaferCond unregister_ctx;
-  mock_image_watcher.unregister_watch(&unregister_ctx);
-  ASSERT_EQ(0, unregister_ctx.wait());
-}
-
-TEST_F(TestMockObjectWatcher, ReregisterUnwatchPendingUnregister) {
-  librbd::ImageCtx *ictx;
-  ASSERT_EQ(0, open_image(m_image_name, &ictx));
-
-  MockImageCtx mock_image_ctx(*ictx);
-  MockObjectWatcher mock_image_watcher(mock_image_ctx, m_oid);
-
-  expect_op_work_queue(mock_image_ctx);
-
-  InSequence seq;
-  expect_aio_watch(mock_image_ctx, 0);
-
-  // inject an unregister
-  C_SaferCond unregister_ctx;
-  expect_aio_unwatch(mock_image_ctx, 0, [&mock_image_watcher, &unregister_ctx]() {
-      mock_image_watcher.unregister_watch(&unregister_ctx);
-    });
-
-  C_SaferCond register_ctx;
-  mock_image_watcher.register_watch(&register_ctx);
-  ASSERT_EQ(0, register_ctx.wait());
-
-  assert(m_watch_ctx != nullptr);
-  m_watch_ctx->handle_error(0, -ESHUTDOWN);
-
-  ASSERT_EQ(0, unregister_ctx.wait());
-}
-
-TEST_F(TestMockObjectWatcher, ReregisterWatchPendingUnregister) {
-  librbd::ImageCtx *ictx;
-  ASSERT_EQ(0, open_image(m_image_name, &ictx));
-
-  MockImageCtx mock_image_ctx(*ictx);
-  MockObjectWatcher mock_image_watcher(mock_image_ctx, m_oid);
-
-  expect_op_work_queue(mock_image_ctx);
-
-  InSequence seq;
-  expect_aio_watch(mock_image_ctx, 0);
-  expect_aio_unwatch(mock_image_ctx, 0);
-
-  // inject an unregister
-  C_SaferCond unregister_ctx;
-  expect_aio_watch(mock_image_ctx, -ESHUTDOWN,
-                   [&mock_image_watcher, &unregister_ctx]() {
-      mock_image_watcher.unregister_watch(&unregister_ctx);
-    });
-
-  C_SaferCond register_ctx;
-  mock_image_watcher.register_watch(&register_ctx);
-  ASSERT_EQ(0, register_ctx.wait());
-
-  assert(m_watch_ctx != nullptr);
-  m_watch_ctx->handle_error(0, -ESHUTDOWN);
-
-  ASSERT_EQ(0, unregister_ctx.wait());
-}
-
-TEST_F(TestMockObjectWatcher, ReregisterPendingUnregister) {
-  librbd::ImageCtx *ictx;
-  ASSERT_EQ(0, open_image(m_image_name, &ictx));
-
-  MockImageCtx mock_image_ctx(*ictx);
-  MockObjectWatcher mock_image_watcher(mock_image_ctx, m_oid);
-
-  expect_op_work_queue(mock_image_ctx);
-
-  InSequence seq;
-  expect_aio_watch(mock_image_ctx, 0);
-  expect_aio_unwatch(mock_image_ctx, 0);
-
-  // inject an unregister
-  C_SaferCond unregister_ctx;
-  expect_aio_watch(mock_image_ctx, 0,
-                   [&mock_image_watcher, &unregister_ctx]() {
-      mock_image_watcher.unregister_watch(&unregister_ctx);
-    });
-
-  expect_aio_unwatch(mock_image_ctx, 0);
-
-  C_SaferCond register_ctx;
-  mock_image_watcher.register_watch(&register_ctx);
-  ASSERT_EQ(0, register_ctx.wait());
-
-  assert(m_watch_ctx != nullptr);
-  m_watch_ctx->handle_error(0, -ESHUTDOWN);
-
-  ASSERT_EQ(0, unregister_ctx.wait());
-}
-
-} // namespace librbd

diff --git a/src/test/librbd/test_mock_Watcher.cc b/src/test/librbd/test_mock_Watcher.cc
new file mode 100644 (file)
index 0000000..4feffc6
--- /dev/null
+++ b/src/test/librbd/test_mock_Watcher.cc
@@ -0,0 +1,403 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include "test/librbd/test_mock_fixture.h"
+#include "test/librbd/test_support.h"
+#include "test/librbd/mock/MockImageCtx.h"
+#include "test/librados_test_stub/MockTestMemIoCtxImpl.h"
+#include "test/librados_test_stub/MockTestMemRadosClient.h"
+#include "common/Cond.h"
+#include "common/Mutex.h"
+#include "librados/AioCompletionImpl.h"
+#include "librbd/Watcher.h"
+#include "librbd/watcher/RewatchRequest.h"
+#include "gmock/gmock.h"
+#include "gtest/gtest.h"
+#include <list>
+
+namespace librbd {
+
+namespace {
+
+struct MockWatcher : public Watcher {
+  std::string oid;
+
+  MockWatcher(librados::IoCtx& ioctx, ContextWQ *work_queue,
+              const std::string& oid)
+    : Watcher(ioctx, work_queue, oid) {
+  }
+
+  virtual void handle_notify(uint64_t notify_id, uint64_t handle,
+                             uint64_t notifier_id, bufferlist &bl) {
+  }
+};
+
+} // anonymous namespace
+} // namespace librbd
+
+namespace librbd {
+
+using ::testing::_;
+using ::testing::DoDefault;
+using ::testing::Invoke;
+using ::testing::InSequence;
+using ::testing::Return;
+using ::testing::SaveArg;
+using ::testing::WithArg;
+using ::testing::WithArgs;
+
+class TestMockWatcher : public TestMockFixture {
+public:
+  TestMockWatcher() : m_lock("TestMockWatcher::m_lock") {
+  }
+
+  virtual void SetUp() {
+    TestMockFixture::SetUp();
+
+    m_oid = get_temp_image_name();
+
+    bufferlist bl;
+    ASSERT_EQ(0, m_ioctx.write_full(m_oid, bl));
+  }
+
+  void expect_aio_watch(MockImageCtx &mock_image_ctx, int r,
+                        const std::function<void()> &action = std::function<void()>()) {
+    librados::MockTestMemIoCtxImpl &mock_io_ctx(get_mock_io_ctx(m_ioctx));
+    librados::MockTestMemRadosClient *mock_rados_client(
+      mock_io_ctx.get_mock_rados_client());
+
+    EXPECT_CALL(mock_io_ctx, aio_watch(m_oid, _, _, _))
+      .WillOnce(DoAll(WithArgs<1, 2, 3>(Invoke([this, &mock_image_ctx, mock_rados_client, r, action](
+              librados::AioCompletionImpl *c, uint64_t *cookie,
+              librados::WatchCtx2 *watch_ctx) {
+          if (r == 0) {
+            *cookie = 234U;
+            m_watch_ctx = watch_ctx;
+          }
+
+          c->get();
+          mock_image_ctx.image_ctx->op_work_queue->queue(new FunctionContext([mock_rados_client, action, c](int r) {
+              if (action) {
+                action();
+              }
+
+              mock_rados_client->finish_aio_completion(c, r);
+            }), r);
+          notify_watch();
+        })), Return(0)));
+  }
+
+  void expect_aio_unwatch(MockImageCtx &mock_image_ctx, int r,
+                          const std::function<void()> &action = std::function<void()>()) {
+    librados::MockTestMemIoCtxImpl &mock_io_ctx(get_mock_io_ctx(m_ioctx));
+    librados::MockTestMemRadosClient *mock_rados_client(
+      mock_io_ctx.get_mock_rados_client());
+
+    EXPECT_CALL(mock_io_ctx, aio_unwatch(_, _))
+      .WillOnce(DoAll(Invoke([this, &mock_image_ctx, mock_rados_client, r, action](
+              uint64_t handle, librados::AioCompletionImpl *c) {
+          c->get();
+          mock_image_ctx.image_ctx->op_work_queue->queue(new FunctionContext([mock_rados_client, action, c](int r) {
+              if (action) {
+                action();
+              }
+
+              mock_rados_client->finish_aio_completion(c, r);
+            }), r);
+          notify_watch();
+        }), Return(0)));
+  }
+
+  std::string m_oid;
+  librados::WatchCtx2 *m_watch_ctx = nullptr;
+
+  void notify_watch() {
+    Mutex::Locker locker(m_lock);
+    ++m_watch_count;
+    m_cond.Signal();
+  }
+
+  bool wait_for_watch(MockImageCtx &mock_image_ctx, size_t count) {
+    Mutex::Locker locker(m_lock);
+    while (m_watch_count < count) {
+      if (m_cond.WaitInterval(m_lock, utime_t(10, 0)) != 0) {
+        return false;
+      }
+    }
+    return true;
+  }
+
+  Mutex m_lock;
+  Cond m_cond;
+  size_t m_watch_count = 0;
+};
+
+TEST_F(TestMockWatcher, Success) {
+  librbd::ImageCtx *ictx;
+  ASSERT_EQ(0, open_image(m_image_name, &ictx));
+
+  MockImageCtx mock_image_ctx(*ictx);
+  MockWatcher mock_image_watcher(m_ioctx, ictx->op_work_queue, m_oid);
+
+  InSequence seq;
+  expect_aio_watch(mock_image_ctx, 0);
+  expect_aio_unwatch(mock_image_ctx, 0);
+
+  C_SaferCond register_ctx;
+  mock_image_watcher.register_watch(&register_ctx);
+  ASSERT_EQ(0, register_ctx.wait());
+
+  C_SaferCond unregister_ctx;
+  mock_image_watcher.unregister_watch(&unregister_ctx);
+  ASSERT_EQ(0, unregister_ctx.wait());
+}
+
+TEST_F(TestMockWatcher, RegisterError) {
+  librbd::ImageCtx *ictx;
+  ASSERT_EQ(0, open_image(m_image_name, &ictx));
+
+  MockImageCtx mock_image_ctx(*ictx);
+  MockWatcher mock_image_watcher(m_ioctx, ictx->op_work_queue, m_oid);
+
+  InSequence seq;
+  expect_aio_watch(mock_image_ctx, -EINVAL);
+
+  C_SaferCond register_ctx;
+  mock_image_watcher.register_watch(&register_ctx);
+  ASSERT_EQ(-EINVAL, register_ctx.wait());
+}
+
+TEST_F(TestMockWatcher, UnregisterError) {
+  librbd::ImageCtx *ictx;
+  ASSERT_EQ(0, open_image(m_image_name, &ictx));
+
+  MockImageCtx mock_image_ctx(*ictx);
+  MockWatcher mock_image_watcher(m_ioctx, ictx->op_work_queue, m_oid);
+
+  InSequence seq;
+  expect_aio_watch(mock_image_ctx, 0);
+  expect_aio_unwatch(mock_image_ctx, -EINVAL);
+
+  C_SaferCond register_ctx;
+  mock_image_watcher.register_watch(&register_ctx);
+  ASSERT_EQ(0, register_ctx.wait());
+
+  C_SaferCond unregister_ctx;
+  mock_image_watcher.unregister_watch(&unregister_ctx);
+  ASSERT_EQ(-EINVAL, unregister_ctx.wait());
+}
+
+TEST_F(TestMockWatcher, Reregister) {
+  librbd::ImageCtx *ictx;
+  ASSERT_EQ(0, open_image(m_image_name, &ictx));
+
+  MockImageCtx mock_image_ctx(*ictx);
+  MockWatcher mock_image_watcher(m_ioctx, ictx->op_work_queue, m_oid);
+
+  expect_op_work_queue(mock_image_ctx);
+
+  InSequence seq;
+  expect_aio_watch(mock_image_ctx, 0);
+  expect_aio_unwatch(mock_image_ctx, 0);
+  expect_aio_watch(mock_image_ctx, 0);
+  expect_aio_unwatch(mock_image_ctx, 0);
+
+  C_SaferCond register_ctx;
+  mock_image_watcher.register_watch(&register_ctx);
+  ASSERT_EQ(0, register_ctx.wait());
+
+  assert(m_watch_ctx != nullptr);
+  m_watch_ctx->handle_error(0, -ESHUTDOWN);
+
+  // wait for recovery unwatch/watch
+  ASSERT_TRUE(wait_for_watch(mock_image_ctx, 3));
+
+  C_SaferCond unregister_ctx;
+  mock_image_watcher.unregister_watch(&unregister_ctx);
+  ASSERT_EQ(0, unregister_ctx.wait());
+}
+
+TEST_F(TestMockWatcher, ReregisterUnwatchError) {
+  librbd::ImageCtx *ictx;
+  ASSERT_EQ(0, open_image(m_image_name, &ictx));
+
+  MockImageCtx mock_image_ctx(*ictx);
+  MockWatcher mock_image_watcher(m_ioctx, ictx->op_work_queue, m_oid);
+
+  expect_op_work_queue(mock_image_ctx);
+
+  InSequence seq;
+  expect_aio_watch(mock_image_ctx, 0);
+  expect_aio_unwatch(mock_image_ctx, -EINVAL);
+  expect_aio_watch(mock_image_ctx, 0);
+  expect_aio_unwatch(mock_image_ctx, 0);
+
+  C_SaferCond register_ctx;
+  mock_image_watcher.register_watch(&register_ctx);
+  ASSERT_EQ(0, register_ctx.wait());
+
+  assert(m_watch_ctx != nullptr);
+  m_watch_ctx->handle_error(0, -ESHUTDOWN);
+
+  // wait for recovery unwatch/watch
+  ASSERT_TRUE(wait_for_watch(mock_image_ctx, 3));
+
+  C_SaferCond unregister_ctx;
+  mock_image_watcher.unregister_watch(&unregister_ctx);
+  ASSERT_EQ(0, unregister_ctx.wait());
+}
+
+TEST_F(TestMockWatcher, ReregisterWatchError) {
+  librbd::ImageCtx *ictx;
+  ASSERT_EQ(0, open_image(m_image_name, &ictx));
+
+  MockImageCtx mock_image_ctx(*ictx);
+  MockWatcher mock_image_watcher(m_ioctx, ictx->op_work_queue, m_oid);
+
+  expect_op_work_queue(mock_image_ctx);
+
+  InSequence seq;
+  expect_aio_watch(mock_image_ctx, 0);
+  expect_aio_unwatch(mock_image_ctx, 0);
+  expect_aio_watch(mock_image_ctx, -EPERM);
+  expect_aio_watch(mock_image_ctx, 0);
+  expect_aio_unwatch(mock_image_ctx, 0);
+
+  C_SaferCond register_ctx;
+  mock_image_watcher.register_watch(&register_ctx);
+  ASSERT_EQ(0, register_ctx.wait());
+
+  assert(m_watch_ctx != nullptr);
+  m_watch_ctx->handle_error(0, -ESHUTDOWN);
+
+  // wait for recovery unwatch/watch
+  ASSERT_TRUE(wait_for_watch(mock_image_ctx, 4));
+
+  C_SaferCond unregister_ctx;
+  mock_image_watcher.unregister_watch(&unregister_ctx);
+  ASSERT_EQ(0, unregister_ctx.wait());
+}
+
+TEST_F(TestMockWatcher, ReregisterWatchBlacklist) {
+  librbd::ImageCtx *ictx;
+  ASSERT_EQ(0, open_image(m_image_name, &ictx));
+
+  MockImageCtx mock_image_ctx(*ictx);
+  MockWatcher mock_image_watcher(m_ioctx, ictx->op_work_queue, m_oid);
+
+  expect_op_work_queue(mock_image_ctx);
+
+  InSequence seq;
+  expect_aio_watch(mock_image_ctx, 0);
+  expect_aio_unwatch(mock_image_ctx, 0);
+  expect_aio_watch(mock_image_ctx, -EBLACKLISTED);
+  expect_aio_watch(mock_image_ctx, 0);
+  expect_aio_unwatch(mock_image_ctx, 0);
+
+  C_SaferCond register_ctx;
+  mock_image_watcher.register_watch(&register_ctx);
+  ASSERT_EQ(0, register_ctx.wait());
+
+  ceph_assert(m_watch_ctx != nullptr);
+  m_watch_ctx->handle_error(0, -EBLACKLISTED);
+
+  // wait for recovery unwatch/watch
+  ASSERT_TRUE(wait_for_watch(mock_image_ctx, 4));
+
+  C_SaferCond unregister_ctx;
+  mock_image_watcher.unregister_watch(&unregister_ctx);
+  ASSERT_EQ(0, unregister_ctx.wait());
+}
+
+TEST_F(TestMockWatcher, ReregisterUnwatchPendingUnregister) {
+  librbd::ImageCtx *ictx;
+  ASSERT_EQ(0, open_image(m_image_name, &ictx));
+
+  MockImageCtx mock_image_ctx(*ictx);
+  MockWatcher mock_image_watcher(m_ioctx, ictx->op_work_queue, m_oid);
+
+  expect_op_work_queue(mock_image_ctx);
+
+  InSequence seq;
+  expect_aio_watch(mock_image_ctx, 0);
+
+  // inject an unregister
+  C_SaferCond unregister_ctx;
+  expect_aio_unwatch(mock_image_ctx, -EBLACKLISTED,
+                     [&mock_image_watcher, &unregister_ctx]() {
+      mock_image_watcher.unregister_watch(&unregister_ctx);
+    });
+
+  C_SaferCond register_ctx;
+  mock_image_watcher.register_watch(&register_ctx);
+  ASSERT_EQ(0, register_ctx.wait());
+
+  assert(m_watch_ctx != nullptr);
+  m_watch_ctx->handle_error(0, -EBLACKLISTED);
+
+  ASSERT_EQ(0, unregister_ctx.wait());
+}
+
+TEST_F(TestMockWatcher, ReregisterWatchPendingUnregister) {
+  librbd::ImageCtx *ictx;
+  ASSERT_EQ(0, open_image(m_image_name, &ictx));
+
+  MockImageCtx mock_image_ctx(*ictx);
+  MockWatcher mock_image_watcher(m_ioctx, ictx->op_work_queue, m_oid);
+
+  expect_op_work_queue(mock_image_ctx);
+
+  InSequence seq;
+  expect_aio_watch(mock_image_ctx, 0);
+  expect_aio_unwatch(mock_image_ctx, 0);
+
+  // inject an unregister
+  C_SaferCond unregister_ctx;
+  expect_aio_watch(mock_image_ctx, -ESHUTDOWN,
+                   [&mock_image_watcher, &unregister_ctx]() {
+      mock_image_watcher.unregister_watch(&unregister_ctx);
+    });
+
+  C_SaferCond register_ctx;
+  mock_image_watcher.register_watch(&register_ctx);
+  ASSERT_EQ(0, register_ctx.wait());
+
+  assert(m_watch_ctx != nullptr);
+  m_watch_ctx->handle_error(0, -ESHUTDOWN);
+
+  ASSERT_EQ(0, unregister_ctx.wait());
+}
+
+TEST_F(TestMockWatcher, ReregisterPendingUnregister) {
+  librbd::ImageCtx *ictx;
+  ASSERT_EQ(0, open_image(m_image_name, &ictx));
+
+  MockImageCtx mock_image_ctx(*ictx);
+  MockWatcher mock_image_watcher(m_ioctx, ictx->op_work_queue, m_oid);
+
+  expect_op_work_queue(mock_image_ctx);
+
+  InSequence seq;
+  expect_aio_watch(mock_image_ctx, 0);
+  expect_aio_unwatch(mock_image_ctx, 0);
+
+  // inject an unregister
+  C_SaferCond unregister_ctx;
+  expect_aio_watch(mock_image_ctx, 0,
+                   [&mock_image_watcher, &unregister_ctx]() {
+      mock_image_watcher.unregister_watch(&unregister_ctx);
+    });
+
+  expect_aio_unwatch(mock_image_ctx, 0);
+
+  C_SaferCond register_ctx;
+  mock_image_watcher.register_watch(&register_ctx);
+  ASSERT_EQ(0, register_ctx.wait());
+
+  assert(m_watch_ctx != nullptr);
+  m_watch_ctx->handle_error(0, -ESHUTDOWN);
+
+  ASSERT_EQ(0, unregister_ctx.wait());
+}
+
+} // namespace librbd

diff --git a/src/test/librbd/watcher/test_mock_RewatchRequest.cc b/src/test/librbd/watcher/test_mock_RewatchRequest.cc
index cd4dd297765bdc4459d9430a096bfd183ac82180..58069d0a201cf36c1ebf347c43706df86c9803d5 100644 (file)
--- a/src/test/librbd/watcher/test_mock_RewatchRequest.cc
+++ b/src/test/librbd/watcher/test_mock_RewatchRequest.cc
@@ -19,6 +19,7 @@ using ::testing::InSequence;
 using ::testing::Invoke;
 using ::testing::Return;
 using ::testing::WithArg;
+using ::testing::WithArgs;
 
 struct TestMockWatcherRewatchRequest : public TestMockFixture {
   typedef RewatchRequest MockRewatchRequest;
@@ -32,7 +33,8 @@ struct TestMockWatcherRewatchRequest : public TestMockFixture {
       mock_image_ctx.md_ctx));
 
     EXPECT_CALL(mock_io_ctx, aio_watch(mock_image_ctx.header_oid, _, _, _))
-      .WillOnce(DoAll(WithArg<1>(Invoke([&mock_image_ctx, &mock_io_ctx, r](librados::AioCompletionImpl *c) {
+      .WillOnce(DoAll(WithArgs<1, 2>(Invoke([&mock_image_ctx, &mock_io_ctx, r](librados::AioCompletionImpl *c, uint64_t *cookie) {
+                                   *cookie = 234;
                                    c->get();
                                    mock_image_ctx.image_ctx->op_work_queue->queue(new FunctionContext([&mock_io_ctx, c](int r) {
                                        mock_io_ctx.get_mock_rados_client()->finish_aio_completion(c, r);
@@ -93,6 +95,7 @@ TEST_F(TestMockWatcherRewatchRequest, Success) {
     req->send();
   }
   ASSERT_EQ(0, ctx.wait());
+  ASSERT_EQ(234U, m_watch_handle);
 }
 
 TEST_F(TestMockWatcherRewatchRequest, UnwatchError) {
@@ -117,6 +120,7 @@ TEST_F(TestMockWatcherRewatchRequest, UnwatchError) {
     req->send();
   }
   ASSERT_EQ(0, ctx.wait());
+  ASSERT_EQ(234U, m_watch_handle);
 }
 
 TEST_F(TestMockWatcherRewatchRequest, WatchBlacklist) {
@@ -141,6 +145,7 @@ TEST_F(TestMockWatcherRewatchRequest, WatchBlacklist) {
     req->send();
   }
   ASSERT_EQ(-EBLACKLISTED, ctx.wait());
+  ASSERT_EQ(0U, m_watch_handle);
 }
 
 TEST_F(TestMockWatcherRewatchRequest, WatchDNE) {
@@ -165,6 +170,7 @@ TEST_F(TestMockWatcherRewatchRequest, WatchDNE) {
     req->send();
   }
   ASSERT_EQ(-ENOENT, ctx.wait());
+  ASSERT_EQ(0U, m_watch_handle);
 }
 
 TEST_F(TestMockWatcherRewatchRequest, WatchError) {
@@ -189,6 +195,34 @@ TEST_F(TestMockWatcherRewatchRequest, WatchError) {
     req->send();
   }
   ASSERT_EQ(-EINVAL, ctx.wait());
+  ASSERT_EQ(0U, m_watch_handle);
+}
+
+TEST_F(TestMockWatcherRewatchRequest, InvalidWatchHandler) {
+  librbd::ImageCtx *ictx;
+  ASSERT_EQ(0, open_image(m_image_name, &ictx));
+
+  MockImageCtx mock_image_ctx(*ictx);
+
+  InSequence seq;
+  expect_aio_watch(mock_image_ctx, 0);
+
+  m_watch_handle = 0;
+
+  C_SaferCond ctx;
+  MockRewatchRequest *req = MockRewatchRequest::create(mock_image_ctx.md_ctx,
+                                                       mock_image_ctx.header_oid,
+                                                       m_watch_lock,
+                                                       &m_watch_ctx,
+                                                       &m_watch_handle,
+                                                       &ctx);
+  {
+    RWLock::WLocker watch_locker(m_watch_lock);
+    req->send();
+  }
+
+  ASSERT_EQ(0, ctx.wait());
+  ASSERT_EQ(234U, m_watch_handle);
 }
 
 } // namespace watcher