ceph_test_rados_api_watch_notify: test watch disconnect, reconnect

Signed-off-by: Sage Weil <sage@redhat.com>

diff --git a/src/include/rados/librados.hpp b/src/include/rados/librados.hpp
index f90632519c223ec34d0e8b0e6fb98095d097b73b..edb509a03ee1263461daf1184d10513419d33ef1 100644 (file)
--- a/src/include/rados/librados.hpp
+++ b/src/include/rados/librados.hpp
@@ -186,6 +186,11 @@ namespace librados
     /**
      * Callback activated when we encounter an error with the watch.
      *
+     * Errors we may see:
+     *   -ENOTCONN  : our watch was disconnected
+     *   -ETIMEDOUT : our watch is still valid, but we may have missed
+     *                a notify event.
+     *
      * @param cookie the watcher with the problem
      * @param err error
      */

diff --git a/src/librados/RadosClient.cc b/src/librados/RadosClient.cc
index 69a006b9ae4bbd94a97adb5c07f80c4c8c0bb29f..2be7db1faf2f392c0178ed59a28b3fc523746e9f 100644 (file)
--- a/src/librados/RadosClient.cc
+++ b/src/librados/RadosClient.cc
@@ -805,6 +805,22 @@ void librados::RadosClient::do_watch_notify(MWatchNotify *m)
       lock.Lock();
       ldout(cct,10) << __func__ << " failed notify done" << dendl;
       wc->put();
+    } else if (m->opcode == CEPH_WATCH_EVENT_DISCONNECT) {
+      // we failed to ping or reconnect and our watch was canceled.
+      ldout(cct,10) << __func__ << " disconnect " << *m << dendl;
+      if (wc->watch_ctx2) {
+       wc->get();
+       // trigger the callback
+       lock.Unlock();
+       wc->watch_ctx2->handle_error(m->cookie, -ENOTCONN);
+       lock.Lock();
+       ldout(cct,10) << __func__ << " disconnect done" << dendl;
+       wc->put();
+      } else {
+       lderr(cct) << __func__ << " watch disconnect on "
+                  << wc->oid << " on old API user, silently ignoring"
+                  << dendl;
+      }
     } else {
       lderr(cct) << __func__ << " got unknown event " << m->opcode
                 << " " << ceph_watch_event_name(m->opcode) << dendl;

diff --git a/src/test/librados/watch_notify.cc b/src/test/librados/watch_notify.cc
index 877a1e9c196acd0be997994a3ea7d345513f6e22..8c436d050e15e3141f98580788defe55aa51754f 100644 (file)
--- a/src/test/librados/watch_notify.cc
+++ b/src/test/librados/watch_notify.cc
@@ -219,6 +219,75 @@ TEST_F(LibRadosWatchNotifyECPP, WatchNotifyTimeout) {
   ASSERT_EQ(0, ioctx.unwatch("foo", handle));
 }
 
+
+// --
+
+TEST_F(LibRadosWatchNotify, Watch2Timeout) {
+  notify_io = ioctx;
+  notify_oid = "foo";
+  notify_cookies.clear();
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  ASSERT_EQ(0, rados_write(ioctx, notify_oid, buf, sizeof(buf), 0));
+  uint64_t handle;
+  ASSERT_EQ(0,
+           rados_watch2(ioctx, notify_oid, &handle,
+                        watch_notify2_test_cb,
+                        watch_notify2_test_failcb,
+                        watch_notify2_test_errcb, NULL));
+  rados_conf_set(cluster, "objecter_inject_no_watch_ping", "true");
+  int left = 180;
+  while (notify_err == 0 && --left) {
+    sleep(1);
+  }
+  ASSERT_TRUE(left > 0);
+  rados_conf_set(cluster, "objecter_inject_no_watch_ping", "false");
+  ASSERT_EQ(-ENOTCONN, notify_err);
+
+  // a subsequent notify should not reach us
+  char *reply_buf;
+  size_t reply_buf_len;
+  ASSERT_EQ(0, rados_notify2(ioctx, notify_oid,
+                            "notify", 6, 0,
+                            &reply_buf, &reply_buf_len));
+  {
+    bufferlist reply;
+    reply.append(reply_buf, reply_buf_len);
+    std::multimap<uint64_t, bufferlist> reply_map;
+    bufferlist::iterator reply_p = reply.begin();
+    ::decode(reply_map, reply_p);
+    ASSERT_EQ(0u, reply_map.size());
+  }
+  ASSERT_EQ(0u, notify_cookies.size());
+
+  // re-watch
+  rados_unwatch(ioctx, notify_oid, handle);
+  handle = 0;
+  ASSERT_EQ(0,
+           rados_watch2(ioctx, notify_oid, &handle,
+                        watch_notify2_test_cb,
+                        watch_notify2_test_failcb,
+                        watch_notify2_test_errcb, NULL));
+  // and now a notify will work.
+  ASSERT_EQ(0, rados_notify2(ioctx, notify_oid,
+                            "notify", 6, 0,
+                            &reply_buf, &reply_buf_len));
+  {
+    bufferlist reply;
+    reply.append(reply_buf, reply_buf_len);
+    std::multimap<uint64_t, bufferlist> reply_map;
+    bufferlist::iterator reply_p = reply.begin();
+    ::decode(reply_map, reply_p);
+    ASSERT_EQ(1u, reply_map.size());
+    ASSERT_EQ(1, notify_cookies.count(handle));
+    ASSERT_EQ(5, reply_map.begin()->second.length());
+    ASSERT_EQ(0, strncmp("reply", reply_map.begin()->second.c_str(), 5));
+  }
+  ASSERT_EQ(1u, notify_cookies.size());
+
+  rados_unwatch(ioctx, notify_oid, handle);
+}
+
 // --
 
 TEST_F(LibRadosWatchNotify, WatchNotify2) {