librados: separate ::notify return values

There is a return code from objecter for committing
the notify linger op, and then later a code in the
CEPH_MSG_WATCH_NOTIFY handled by RadosClient directly.
Afaict there isn't any nice ordering guarantee here,
so they could stamp on each other.  Use a SaferCond
for the submit one.

I don't think this was related to #9112 but while
I'm here...

Signed-off-by: John Spray <john.spray@redhat.com>

diff --git a/src/librados/IoCtxImpl.cc b/src/librados/IoCtxImpl.cc
index d02a7de7c24e4bdbfc3c0d4f36569d85a555abff..15e74e05947e84596ab4c86a875956b2c7769a75 100644 (file)
--- a/src/librados/IoCtxImpl.cc
+++ b/src/librados/IoCtxImpl.cc
@@ -1089,41 +1089,42 @@ int librados::IoCtxImpl::notify(const object_t& oid, uint64_t ver, bufferlist& b
 {
   bufferlist inbl, outbl;
 
-  Mutex mylock("IoCtxImpl::notify::mylock");
+  // Construct WatchNotifyInfo
+  Cond cond_all;
   Mutex mylock_all("IoCtxImpl::notify::mylock_all");
-  Cond cond, cond_all;
-  bool done, done_all;
-  int r;
-  Context *onack = new C_SafeCond(&mylock, &cond, &done, &r);
-  version_t objver;
-  uint64_t cookie;
-
-  ::ObjectOperation rd;
-  prepare_assert_ops(&rd);
-
-  lock->Lock();
+  bool done_all;
+  int r_notify;
   WatchNotifyInfo *wc = new WatchNotifyInfo(this, oid);
   wc->notify_done = &done_all;
   wc->notify_lock = &mylock_all;
   wc->notify_cond = &cond_all;
-  wc->notify_rval = &r;
+  wc->notify_rval = &r_notify;
+
+  lock->Lock();
+
+  // Acquire cookie
+  uint64_t cookie;
   client->register_watch_notify_callback(wc, &cookie);
   uint32_t prot_ver = 1;
   uint32_t timeout = notify_timeout;
   ::encode(prot_ver, inbl);
   ::encode(timeout, inbl);
   ::encode(bl, inbl);
+
+  // Construct RADOS op
+  ::ObjectOperation rd;
+  prepare_assert_ops(&rd);
   rd.notify(cookie, ver, inbl);
+
+  // Issue RADOS op
+  C_SaferCond onack;
+  version_t objver;
   wc->linger_id = objecter->linger_read(oid, oloc, rd, snap_seq, inbl, NULL, 0,
-                                       onack, &objver);
+                                       &onack, &objver);
   lock->Unlock();
 
-  mylock.Lock();
-  while (!done)
-    cond.Wait(mylock);
-  mylock.Unlock();
-
-  if (r == 0) {
+  int r_issue = onack.wait();
+  if (r_issue == 0) {
     mylock_all.Lock();
     while (!done_all)
       cond_all.Wait(mylock_all);
@@ -1136,7 +1137,7 @@ int librados::IoCtxImpl::notify(const object_t& oid, uint64_t ver, bufferlist& b
 
   set_sync_op_version(objver);
 
-  return r;
+  return r_issue == 0 ? r_notify : r_issue;
 }
 
 int librados::IoCtxImpl::set_alloc_hint(const object_t& oid,

diff --git a/src/librados/RadosClient.cc b/src/librados/RadosClient.cc
index faf8ea7a9c4c45957bfb0ac24eb4b3b0ba1198bc..bf964648c7635ac658011cf09ecdd467e73295d4 100644 (file)
--- a/src/librados/RadosClient.cc
+++ b/src/librados/RadosClient.cc
@@ -703,6 +703,8 @@ void librados::RadosClient::do_watch_notify(MWatchNotify *m)
       ldout(cct,10) << __func__ << " notify done" << dendl;
       wc->put();
     }
+  } else {
+    ldout(cct, 4) << __func__ << " unknown cookie " << m->cookie << dendl;
   }
   m->put();
 }