From: Sage Weil <sage@redhat.com>
Date: Thu, 13 Nov 2014 21:08:27 +0000 (-0800)
Subject: ceph_test_rados_api_watch_notify: test watch disconnect, reconnect
X-Git-Tag: v0.91~113
X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=cdc9f5f261281757d171452e9fbe021eaca2b436;p=ceph.git

ceph_test_rados_api_watch_notify: test watch disconnect, reconnect

Signed-off-by: Sage Weil <sage@redhat.com>
---

diff --git a/src/include/rados/librados.hpp b/src/include/rados/librados.hpp
index f90632519c2..edb509a03ee 100644
--- a/src/include/rados/librados.hpp
+++ b/src/include/rados/librados.hpp
@@ -186,6 +186,11 @@ namespace librados
     /**
      * Callback activated when we encounter an error with the watch.
      *
+     * Errors we may see:
+     *   -ENOTCONN  : our watch was disconnected
+     *   -ETIMEDOUT : our watch is still valid, but we may have missed
+     *                a notify event.
+     *
      * @param cookie the watcher with the problem
      * @param err error
      */
diff --git a/src/librados/RadosClient.cc b/src/librados/RadosClient.cc
index 69a006b9ae4..2be7db1faf2 100644
--- a/src/librados/RadosClient.cc
+++ b/src/librados/RadosClient.cc
@@ -805,6 +805,22 @@ void librados::RadosClient::do_watch_notify(MWatchNotify *m)
       lock.Lock();
       ldout(cct,10) << __func__ << " failed notify done" << dendl;
       wc->put();
+    } else if (m->opcode == CEPH_WATCH_EVENT_DISCONNECT) {
+      // we failed to ping or reconnect and our watch was canceled.
+      ldout(cct,10) << __func__ << " disconnect " << *m << dendl;
+      if (wc->watch_ctx2) {
+	wc->get();
+	// trigger the callback
+	lock.Unlock();
+	wc->watch_ctx2->handle_error(m->cookie, -ENOTCONN);
+	lock.Lock();
+	ldout(cct,10) << __func__ << " disconnect done" << dendl;
+	wc->put();
+      } else {
+	lderr(cct) << __func__ << " watch disconnect on "
+		   << wc->oid << " on old API user, silently ignoring"
+		   << dendl;
+      }
     } else {
       lderr(cct) << __func__ << " got unknown event " << m->opcode
 		 << " " << ceph_watch_event_name(m->opcode) << dendl;
diff --git a/src/test/librados/watch_notify.cc b/src/test/librados/watch_notify.cc
index 877a1e9c196..8c436d050e1 100644
--- a/src/test/librados/watch_notify.cc
+++ b/src/test/librados/watch_notify.cc
@@ -219,6 +219,75 @@ TEST_F(LibRadosWatchNotifyECPP, WatchNotifyTimeout) {
   ASSERT_EQ(0, ioctx.unwatch("foo", handle));
 }
 
+
+// --
+
+TEST_F(LibRadosWatchNotify, Watch2Timeout) {
+  notify_io = ioctx;
+  notify_oid = "foo";
+  notify_cookies.clear();
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  ASSERT_EQ(0, rados_write(ioctx, notify_oid, buf, sizeof(buf), 0));
+  uint64_t handle;
+  ASSERT_EQ(0,
+	    rados_watch2(ioctx, notify_oid, &handle,
+			 watch_notify2_test_cb,
+			 watch_notify2_test_failcb,
+			 watch_notify2_test_errcb, NULL));
+  rados_conf_set(cluster, "objecter_inject_no_watch_ping", "true");
+  int left = 180;
+  while (notify_err == 0 && --left) {
+    sleep(1);
+  }
+  ASSERT_TRUE(left > 0);
+  rados_conf_set(cluster, "objecter_inject_no_watch_ping", "false");
+  ASSERT_EQ(-ENOTCONN, notify_err);
+
+  // a subsequent notify should not reach us
+  char *reply_buf;
+  size_t reply_buf_len;
+  ASSERT_EQ(0, rados_notify2(ioctx, notify_oid,
+			     "notify", 6, 0,
+			     &reply_buf, &reply_buf_len));
+  {
+    bufferlist reply;
+    reply.append(reply_buf, reply_buf_len);
+    std::multimap<uint64_t, bufferlist> reply_map;
+    bufferlist::iterator reply_p = reply.begin();
+    ::decode(reply_map, reply_p);
+    ASSERT_EQ(0u, reply_map.size());
+  }
+  ASSERT_EQ(0u, notify_cookies.size());
+
+  // re-watch
+  rados_unwatch(ioctx, notify_oid, handle);
+  handle = 0;
+  ASSERT_EQ(0,
+	    rados_watch2(ioctx, notify_oid, &handle,
+			 watch_notify2_test_cb,
+			 watch_notify2_test_failcb,
+			 watch_notify2_test_errcb, NULL));
+  // and now a notify will work.
+  ASSERT_EQ(0, rados_notify2(ioctx, notify_oid,
+			     "notify", 6, 0,
+			     &reply_buf, &reply_buf_len));
+  {
+    bufferlist reply;
+    reply.append(reply_buf, reply_buf_len);
+    std::multimap<uint64_t, bufferlist> reply_map;
+    bufferlist::iterator reply_p = reply.begin();
+    ::decode(reply_map, reply_p);
+    ASSERT_EQ(1u, reply_map.size());
+    ASSERT_EQ(1, notify_cookies.count(handle));
+    ASSERT_EQ(5, reply_map.begin()->second.length());
+    ASSERT_EQ(0, strncmp("reply", reply_map.begin()->second.c_str(), 5));
+  }
+  ASSERT_EQ(1u, notify_cookies.size());
+
+  rados_unwatch(ioctx, notify_oid, handle);
+}
+
 // --
 
 TEST_F(LibRadosWatchNotify, WatchNotify2) {