osdc/Objecter: do notify completion callback in fast-dispatch context

The notify completion has exactly one user, the librados caller which
does nothing but take a local (inner) lock and signal a Cond.  Do this
in the fast-dispatch context for simplicity.

Notably, this makes the notify completion (and timeout) trigger a
notify2() return (with ETIMEDOUT) even when the finisher queue that
normally delivers notify is busy.. for example with a notify that is
being very slow.  In our case, the unit test is doing a sleep(3) to
test timeouts but also prevented the ETIMEDOUT notification from
being delivered to the caller.  This patch resolves that.

Signed-off-by: Sage Weil <sage@redhat.com>

diff --git a/src/osdc/Objecter.cc b/src/osdc/Objecter.cc
index 152560c5aa8449e5adbc55d0200655bf4b61abb3..c11b0929d320d57287be8950d1afd8c6b845be47 100644 (file)
--- a/src/osdc/Objecter.cc
+++ b/src/osdc/Objecter.cc
@@ -770,6 +770,12 @@ void Objecter::handle_watch_notify(MWatchNotify *m)
       if (info->watch_context)
        finisher->queue(new C_DoWatchError(info, -ENOTCONN));
     }
+  } else if (!info->is_watch) {
+    // notify completion; we can do this inline since we know the only user
+    // (librados) is safe to call in fast-dispatch context
+    assert(info->on_notify_finish);
+    info->notify_result_bl->claim(m->get_data());
+    info->on_notify_finish->complete(m->return_code);
   } else {
     finisher->queue(new C_DoWatchNotify(this, info, m));
   }
@@ -791,14 +797,6 @@ void Objecter::_do_watch_notify(LingerOp *info, MWatchNotify *m)
   }
 
   // notify completion?
-  if (!info->is_watch) {
-    assert(info->on_notify_finish);
-    info->notify_result_bl->claim(m->get_data());
-    rwlock.put_read();
-    info->on_notify_finish->complete(m->return_code);
-    goto out;
-  }
-
   assert(info->is_watch);
   assert(info->watch_context);
   assert(m->opcode != CEPH_WATCH_EVENT_DISCONNECT);