From 5c0a8c411f3b5ac0ec37c2859a0a11a697f9e2a3 Mon Sep 17 00:00:00 2001
From: Sage Weil <sage@redhat.com>
Date: Mon, 17 Nov 2014 11:28:21 -0800
Subject: [PATCH] osdc/Objecter: make watch_check time reflect async delivery

When a librados user calls watch_check(), the age they get back should
establish a timestamp for which:

 1) we know that the watch was registered at least through this time,
 2) we have received and processed notifies sent as of this time,
 3) we have processed any async error events as of this time.

We already accomplish 1 by updating watch_valid_thru based on the time
the ping is sent (not received).

This patch gets us 2 and 3 by using the MIN of watch_valid_thru and the
oldest queued async event for the watch_check result.

Signed-off-by: Sage Weil <sage@redhat.com>
---
 src/osdc/Objecter.cc | 12 +++++++++++-
 src/osdc/Objecter.h  | 14 ++++++++++++++
 2 files changed, 25 insertions(+), 1 deletion(-)

diff --git a/src/osdc/Objecter.cc b/src/osdc/Objecter.cc
index 52885791aab01..856486cf8e9e9 100644
--- a/src/osdc/Objecter.cc
+++ b/src/osdc/Objecter.cc
@@ -495,9 +495,11 @@ struct C_DoWatchError : public Context {
   int err;
   C_DoWatchError(Objecter::LingerOp *i, int r) : info(i), err(r) {
     info->get();
+    info->queued_async();
   }
   void finish(int r) {
     info->watch_context->handle_error(info->linger_id, err);
+    info->finished_async();
     info->put();
   }
 };
@@ -583,7 +585,13 @@ void Objecter::_linger_ping(LingerOp *info, int r, utime_t sent,
 int Objecter::linger_check(LingerOp *info)
 {
   RWLock::WLocker wl(rwlock);
-  utime_t age = ceph_clock_now(NULL) - info->watch_valid_thru;
+  Mutex::Locker l(info->watch_lock);
+
+  utime_t stamp = info->watch_valid_thru;
+  if (!info->watch_pending_async.empty())
+    stamp = MIN(info->watch_valid_thru, info->watch_pending_async.front());
+  utime_t age = ceph_clock_now(NULL) - stamp;
+
   ldout(cct, 10) << __func__ << " " << info->linger_id
 		 << " err " << info->last_error
 		 << " age " << age << dendl;
@@ -726,6 +734,7 @@ struct C_DoWatchNotify : public Context {
   C_DoWatchNotify(Objecter *o, Objecter::LingerOp *i, MWatchNotify *m)
     : objecter(o), info(i), msg(m) {
     info->get();
+    info->queued_async();
     msg->get();
   }
   void finish(int r) {
@@ -800,6 +809,7 @@ void Objecter::_do_watch_notify(LingerOp *info, MWatchNotify *m)
   }
 
  out:
+  info->finished_async();
   info->put();
   m->put();
 }
diff --git a/src/osdc/Objecter.h b/src/osdc/Objecter.h
index eeb7fe8cff41d..942eefdd9dbeb 100644
--- a/src/osdc/Objecter.h
+++ b/src/osdc/Objecter.h
@@ -1480,6 +1480,10 @@ public:
     Mutex watch_lock;
     Cond watch_cond;
 
+    // queue of pending async operations, with the timestamp of
+    // when they were queued.
+    list<utime_t> watch_pending_async;
+
     uint32_t register_gen;
     bool registered;
     bool canceled;
@@ -1497,6 +1501,16 @@ public:
     ceph_tid_t ping_tid;
     epoch_t map_dne_bound;
 
+    void queued_async() {
+      Mutex::Locker l(watch_lock);
+      watch_pending_async.push_back(ceph_clock_now(NULL));
+    }
+    void finished_async() {
+      Mutex::Locker l(watch_lock);
+      assert(!watch_pending_async.empty());
+      watch_pending_async.pop_front();
+    }
+
     LingerOp() : linger_id(0),
 		 target(object_t(), object_locator_t(), 0),
 		 snap(CEPH_NOSNAP),
-- 
2.39.5