From 22572a39777be12661750147fd2828223f178ac6 Mon Sep 17 00:00:00 2001
From: Sage Weil <sage@redhat.com>
Date: Thu, 16 Oct 2014 20:11:43 -0700
Subject: [PATCH] librados: add infrastructure to deliver an error notification

Use a reusable context in the WatchNotifyInfo to trigger an error
event, delivered via the existing Finisher thread.  Re-lookup the cookie
in the thread to cope with races with unregister (just as we do with
notify events).

Signed-off-by: Sage Weil <sage@redhat.com>
---
 src/librados/IoCtxImpl.h    | 17 ++++++++++-----
 src/librados/RadosClient.cc | 41 +++++++++++++++++++++++++++++++++++++
 src/librados/RadosClient.h  |  2 ++
 3 files changed, 55 insertions(+), 5 deletions(-)

diff --git a/src/librados/IoCtxImpl.h b/src/librados/IoCtxImpl.h
index fe6c90fc656ea..6b3c7b5330ae2 100644
--- a/src/librados/IoCtxImpl.h
+++ b/src/librados/IoCtxImpl.h
@@ -218,11 +218,11 @@ struct librados::IoCtxImpl {
 
 namespace librados {
 
-  /**
-   * watch/notify info
-   *
-   * Capture state about a watch or an in-progress notify
-   */
+/**
+ * watch/notify info
+ *
+ * Capture state about a watch or an in-progress notify
+ */
 struct WatchNotifyInfo : public RefCountedWaitObject {
   IoCtxImpl *io_ctx_impl;  // parent
   const object_t oid;      // the object
@@ -242,6 +242,12 @@ struct WatchNotifyInfo : public RefCountedWaitObject {
   size_t *notify_reply_buf_len;
   int *notify_rval;
 
+  struct OnError : public Context {
+    WatchNotifyInfo *info;
+    void finish(int r) { assert(0); }
+    void complete(int r);
+  } on_error;
+
   WatchNotifyInfo(IoCtxImpl *io_ctx_impl_,
 		  const object_t& _oc)
     : io_ctx_impl(io_ctx_impl_),
@@ -258,6 +264,7 @@ struct WatchNotifyInfo : public RefCountedWaitObject {
       notify_reply_buf_len(NULL),
       notify_rval(NULL) {
     io_ctx_impl->get();
+    on_error.info = this;
   }
 
   ~WatchNotifyInfo() {
diff --git a/src/librados/RadosClient.cc b/src/librados/RadosClient.cc
index 579d2362d164f..ba4f5a89b1fd3 100644
--- a/src/librados/RadosClient.cc
+++ b/src/librados/RadosClient.cc
@@ -683,6 +683,47 @@ struct C_DoWatchNotify : public Context {
   }
 };
 
+struct C_DoWatchError : public Context {
+  librados::RadosClient *rados;
+  uint64_t cookie;
+  int err;
+  C_DoWatchError(librados::RadosClient *r, uint64_t cookie, int err)
+    : rados(r), cookie(cookie), err(err) {}
+  void finish(int r) {
+    rados->do_watch_error(cookie, err);
+  }
+};
+
+void librados::WatchNotifyInfo::OnError::complete(int r)
+{
+  RadosClient *client = info->io_ctx_impl->client;
+  client->finisher.queue(new C_DoWatchError(client, info->cookie, r));
+}
+
+void librados::RadosClient::do_watch_error(uint64_t cookie, int err)
+{
+  Mutex::Locker l(lock);
+  map<uint64_t, WatchNotifyInfo *>::iterator iter =
+    watch_notify_info.find(cookie);
+  if (iter != watch_notify_info.end()) {
+    WatchNotifyInfo *wc = iter->second;
+    assert(wc);
+    if (wc->watch_ctx2) {
+      wc->get();
+      ldout(cct,10) << __func__ << " cookie " << cookie
+		    << " handle_error " << err << dendl;
+      lock.Unlock();
+      wc->watch_ctx2->handle_error(cookie, err);
+      lock.Lock();
+      ldout(cct,10) << __func__ << " cookie " << cookie
+		    << " handle_error " << err << " done" << dendl;
+      wc->put();
+    }
+  } else {
+    ldout(cct,10) << __func__ << " cookie " << cookie << " not found" << dendl;
+  }
+}
+
 void librados::RadosClient::handle_watch_notify(MWatchNotify *m)
 {
   Mutex::Locker l(lock);
diff --git a/src/librados/RadosClient.h b/src/librados/RadosClient.h
index 23960622521bf..2f9bb2f46fa8c 100644
--- a/src/librados/RadosClient.h
+++ b/src/librados/RadosClient.h
@@ -33,6 +33,7 @@ class Message;
 class MWatchNotify;
 class MLog;
 class Messenger;
+struct WatchNotifyInfo;
 
 class librados::RadosClient : public Dispatcher
 {
@@ -116,6 +117,7 @@ public:
   void unregister_watch_notify_callback(uint64_t cookie);
   void handle_watch_notify(MWatchNotify *m);
   void do_watch_notify(MWatchNotify *m);
+  void do_watch_error(uint64_t cookie, int err);
 
   int mon_command(const vector<string>& cmd, const bufferlist &inbl,
 	          bufferlist *outbl, string *outs);
-- 
2.39.5