librbd: use aio watch/unwatch during open/close state machines

Removed the place holders in the open/close async state machines with
the newly added aio watch/unwatch methods.

Signed-off-by: Jason Dillaman <dillaman@redhat.com>

diff --git a/src/librbd/ImageCtx.cc b/src/librbd/ImageCtx.cc
index e91000357a0503f31edb9f2947c90c4b37d9d2ce..65d1d9e37aad53b9f4165c2a8bc9c7bfbdd058b1 100644 (file)
--- a/src/librbd/ImageCtx.cc
+++ b/src/librbd/ImageCtx.cc
@@ -287,9 +287,8 @@ struct C_InvalidateCache : public Context {
   }
 
   void ImageCtx::shutdown() {
-    if (image_watcher != nullptr) {
-      unregister_watch();
-    }
+    delete image_watcher;
+    image_watcher = nullptr;
 
     delete asok_hook;
     asok_hook = nullptr;
@@ -780,17 +779,10 @@ struct C_InvalidateCache : public Context {
     object_cacher->clear_nonexistence(object_set);
   }
 
-  int ImageCtx::register_watch() {
+  void ImageCtx::register_watch(Context *on_finish) {
     assert(image_watcher == NULL);
     image_watcher = new ImageWatcher(*this);
-    return image_watcher->register_watch();
-  }
-
-  void ImageCtx::unregister_watch() {
-    assert(image_watcher != NULL);
-    image_watcher->unregister_watch();
-    delete image_watcher;
-    image_watcher = NULL;
+    image_watcher->register_watch(on_finish);
   }
 
   uint64_t ImageCtx::prune_parent_extents(vector<pair<uint64_t,uint64_t> >& objectx,

diff --git a/src/librbd/ImageCtx.h b/src/librbd/ImageCtx.h
index 0d84f0caad3709e5459bbbfd4b85638eeaeeb600..e914491d1439daa6c7be550b241e3056f8b93ea6 100644 (file)
--- a/src/librbd/ImageCtx.h
+++ b/src/librbd/ImageCtx.h
@@ -255,8 +255,7 @@ namespace librbd {
     int invalidate_cache(bool purge_on_error=false);
     void invalidate_cache(Context *on_finish);
     void clear_nonexistence_cache();
-    int register_watch();
-    void unregister_watch();
+    void register_watch(Context *on_finish);
     uint64_t prune_parent_extents(vector<pair<uint64_t,uint64_t> >& objectx,
                                  uint64_t overlap);
 

diff --git a/src/librbd/ImageWatcher.cc b/src/librbd/ImageWatcher.cc
index 3510028e281d30782964ad5aae25ec8b676624ed..2bd3c0a61cb7abcd01c4ac9fff9cc25537d968b2 100644 (file)
--- a/src/librbd/ImageWatcher.cc
+++ b/src/librbd/ImageWatcher.cc
@@ -30,6 +30,43 @@ namespace librbd {
 using namespace image_watcher;
 using namespace watch_notify;
 using util::create_context_callback;
+using util::create_rados_safe_callback;
+
+namespace {
+
+struct C_UnwatchAndFlush : public Context {
+  librados::Rados rados;
+  Context *on_finish;
+  bool flushing = false;
+  int ret_val = 0;
+
+  C_UnwatchAndFlush(librados::IoCtx &io_ctx, Context *on_finish)
+    : rados(io_ctx), on_finish(on_finish) {
+  }
+
+  virtual void complete(int r) override {
+    if (ret_val == 0 && r < 0) {
+      ret_val = r;
+    }
+
+    if (!flushing) {
+      flushing = true;
+
+      librados::AioCompletion *aio_comp = create_rados_safe_callback(this);
+      r = rados.aio_watch_flush(aio_comp);
+      assert(r == 0);
+      aio_comp->release();
+    } else {
+      Context::complete(ret_val);
+    }
+  }
+
+  virtual void finish(int r) override {
+    on_finish->complete(r);
+  }
+};
+
+} // anonymous namespace
 
 static const double    RETRY_DELAY_SECONDS = 1.0;
 
@@ -54,40 +91,46 @@ ImageWatcher::~ImageWatcher()
   }
 }
 
-int ImageWatcher::register_watch() {
+void ImageWatcher::register_watch(Context *on_finish) {
   ldout(m_image_ctx.cct, 10) << this << " registering image watcher" << dendl;
 
-  RWLock::WLocker l(m_watch_lock);
+  RWLock::RLocker watch_locker(m_watch_lock);
+  assert(m_watch_state == WATCH_STATE_UNREGISTERED);
+  librados::AioCompletion *aio_comp = create_rados_safe_callback(
+    new C_RegisterWatch(this, on_finish));
+  int r = m_image_ctx.md_ctx.aio_watch(m_image_ctx.header_oid, aio_comp,
+                                       &m_watch_handle, &m_watch_ctx);
+  assert(r == 0);
+  aio_comp->release();
+}
+
+void ImageWatcher::handle_register_watch(int r) {
+  RWLock::WLocker watch_locker(m_watch_lock);
   assert(m_watch_state == WATCH_STATE_UNREGISTERED);
-  int r = m_image_ctx.md_ctx.watch2(m_image_ctx.header_oid,
-                                   &m_watch_handle,
-                                   &m_watch_ctx);
   if (r < 0) {
-    return r;
+    m_watch_handle = 0;
+  } else if (r >= 0) {
+    m_watch_state = WATCH_STATE_REGISTERED;
   }
-
-  m_watch_state = WATCH_STATE_REGISTERED;
-  return 0;
 }
 
-int ImageWatcher::unregister_watch() {
+void ImageWatcher::unregister_watch(Context *on_finish) {
   ldout(m_image_ctx.cct, 10) << this << " unregistering image watcher" << dendl;
 
   cancel_async_requests();
   m_task_finisher->cancel_all();
 
-  int r = 0;
   {
     RWLock::WLocker l(m_watch_lock);
     if (m_watch_state == WATCH_STATE_REGISTERED) {
-      r = m_image_ctx.md_ctx.unwatch2(m_watch_handle);
+      librados::AioCompletion *aio_comp = create_rados_safe_callback(
+        new C_UnwatchAndFlush(m_image_ctx.md_ctx, on_finish));
+      int r = m_image_ctx.md_ctx.aio_unwatch(m_watch_handle, aio_comp);
+      assert(r == 0);
+      aio_comp->release();
     }
     m_watch_state = WATCH_STATE_UNREGISTERED;
   }
-
-  librados::Rados rados(m_image_ctx.md_ctx);
-  rados.watch_flush();
-  return r;
 }
 
 void ImageWatcher::flush(Context *on_finish) {

diff --git a/src/librbd/ImageWatcher.h b/src/librbd/ImageWatcher.h
index 26738a6b728cbfbd21dd2cba75c870ab63cd1be1..89edb35df2c63f50039362006524e84723d7592a 100644 (file)
--- a/src/librbd/ImageWatcher.h
+++ b/src/librbd/ImageWatcher.h
@@ -30,8 +30,8 @@ public:
   ImageWatcher(ImageCtx& image_ctx);
   ~ImageWatcher();
 
-  int register_watch();
-  int unregister_watch();
+  void register_watch(Context *on_finish);
+  void unregister_watch(Context *on_finish);
   void flush(Context *on_finish);
 
   int notify_flatten(uint64_t request_id, ProgressContext &prog_ctx);
@@ -149,6 +149,18 @@ private:
     ProgressContext *m_prog_ctx;
   };
 
+  struct C_RegisterWatch : public Context {
+    ImageWatcher *image_watcher;
+    Context *on_finish;
+
+    C_RegisterWatch(ImageWatcher *image_watcher, Context *on_finish)
+       : image_watcher(image_watcher), on_finish(on_finish) {
+    }
+    virtual void finish(int r) override {
+      image_watcher->handle_register_watch(r);
+      on_finish->complete(r);
+    }
+  };
   struct C_NotifyAck : public Context {
     ImageWatcher *image_watcher;
     uint64_t notify_id;
@@ -224,6 +236,8 @@ private:
 
   image_watcher::Notifier m_notifier;
 
+  void handle_register_watch(int r);
+
   void schedule_cancel_async_requests();
   void cancel_async_requests();
 

diff --git a/src/librbd/image/CloseRequest.cc b/src/librbd/image/CloseRequest.cc
index 19916396a071510b0526539b95366e4fbf33974a..fa75e9426204acf937ff047175493fd039e8bbbd 100644 (file)
--- a/src/librbd/image/CloseRequest.cc
+++ b/src/librbd/image/CloseRequest.cc
@@ -33,18 +33,22 @@ CloseRequest<I>::CloseRequest(I *image_ctx, Context *on_finish)
 
 template <typename I>
 void CloseRequest<I>::send() {
-  // TODO
-  send_shut_down_aio_queue();
-  //send_unregister_image_watcher();
+  send_unregister_image_watcher();
 }
 
 template <typename I>
 void CloseRequest<I>::send_unregister_image_watcher() {
+  if (m_image_ctx->image_watcher == nullptr) {
+    send_shut_down_aio_queue();
+    return;
+  }
+
   CephContext *cct = m_image_ctx->cct;
   ldout(cct, 10) << this << " " << __func__ << dendl;
 
   // prevent incoming requests from our peers
-
+  m_image_ctx->image_watcher->unregister_watch(create_context_callback<
+    CloseRequest<I>, &CloseRequest<I>::handle_unregister_image_watcher>(this));
 }
 
 template <typename I>

diff --git a/src/librbd/image/OpenRequest.cc b/src/librbd/image/OpenRequest.cc
index 68abbba620abb23b264498d3505b745bd3651b97..fa1311c7863ad8c870dd9643251cefa9aea95de5 100644 (file)
--- a/src/librbd/image/OpenRequest.cc
+++ b/src/librbd/image/OpenRequest.cc
@@ -19,26 +19,6 @@
 namespace librbd {
 namespace image {
 
-namespace {
-
-template <typename I>
-class C_RegisterWatch : public Context {
-public:
-  I &image_ctx;
-  Context *on_finish;
-
-  C_RegisterWatch(I &image_ctx, Context *on_finish)
-    : image_ctx(image_ctx), on_finish(on_finish) {
-  }
-
-  virtual void finish(int r) {
-    assert(r == 0);
-    on_finish->complete(image_ctx.register_watch());
-  }
-};
-
-} // anonymous namespace
-
 using util::create_context_callback;
 using util::create_rados_ack_callback;
 
@@ -291,12 +271,10 @@ void OpenRequest<I>::send_register_watch() {
     CephContext *cct = m_image_ctx->cct;
     ldout(cct, 10) << this << " " << __func__ << dendl;
 
-    // no librados async version of watch
     using klass = OpenRequest<I>;
-    Context *ctx = new C_RegisterWatch<I>(
-      *m_image_ctx,
-      create_context_callback<klass, &klass::handle_register_watch>(this));
-    m_image_ctx->op_work_queue->queue(ctx);
+    Context *ctx = create_context_callback<
+      klass, &klass::handle_register_watch>(this);
+    m_image_ctx->register_watch(ctx);
   } else {
     send_refresh();
   }