From: Greg Farnum <gfarnum@redhat.com>
Date: Mon, 12 Feb 2018 20:20:00 +0000 (-0800)
Subject: osdc: create and use take_linger_budget for prefetching watch/notify budgets
X-Git-Tag: v13.0.2~86^2
X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=aba55b5fb626005dd16cfa7934730c7c1c5ec9ce;p=ceph.git

osdc: create and use take_linger_budget for prefetching watch/notify budgets

We don't actually take any budget, as we don't want to hit the
total op limits (especially on eg rbd-mirror), but this prevents
our per-message budgeting from causing a deadlock.

Fixes: http://tracker.ceph.com/issues/22882

Signed-off-by: Greg Farnum <gfarnum@redhat.com>
---

diff --git a/src/osdc/Objecter.cc b/src/osdc/Objecter.cc
index f23537f45ad2..dd3657ad2c0c 100644
--- a/src/osdc/Objecter.cc
+++ b/src/osdc/Objecter.cc
@@ -581,9 +581,10 @@ void Objecter::_send_linger(LingerOp *info,
 
   // do not resend this; we will send a new op to reregister
   o->should_resend = false;
+  o->ctx_budgeted = true;
 
   if (info->register_tid) {
-    // repeat send.  cancel old registeration op, if any.
+    // repeat send.  cancel old registration op, if any.
     OSDSession::unique_lock sl(info->session->lock);
     if (info->session->ops.count(info->register_tid)) {
       Op *o = info->session->ops[info->register_tid];
@@ -591,13 +592,10 @@ void Objecter::_send_linger(LingerOp *info,
       _cancel_linger_op(o);
     }
     sl.unlock();
-
-    _op_submit(o, sul, &info->register_tid);
-  } else {
-    // first send
-    _op_submit_with_budget(o, sul, &info->register_tid);
   }
 
+  _op_submit_with_budget(o, sul, &info->register_tid, &info->ctx_budget);
+
   logger->inc(l_osdc_linger_send);
 }
 
@@ -793,7 +791,7 @@ Objecter::LingerOp *Objecter::linger_register(const object_t& oid,
 					      const object_locator_t& oloc,
 					      int flags)
 {
-  LingerOp *info = new LingerOp;
+  LingerOp *info = new LingerOp(this);
   info->target.base_oid = oid;
   info->target.base_oloc = oloc;
   if (info->target.base_oloc.key == oid)
@@ -835,6 +833,8 @@ ceph_tid_t Objecter::linger_watch(LingerOp *info,
   info->pobjver = objver;
   info->on_reg_commit = oncommit;
 
+  info->ctx_budget = take_linger_budget(info);
+
   shunique_lock sul(rwlock, ceph::acquire_unique);
   _linger_submit(info, sul);
   logger->inc(l_osdc_linger_active);
@@ -857,6 +857,8 @@ ceph_tid_t Objecter::linger_notify(LingerOp *info,
   info->pobjver = objver;
   info->on_reg_commit = onfinish;
 
+  info->ctx_budget = take_linger_budget(info);
+  
   shunique_lock sul(rwlock, ceph::acquire_unique);
   _linger_submit(info, sul);
   logger->inc(l_osdc_linger_active);
@@ -868,6 +870,7 @@ void Objecter::_linger_submit(LingerOp *info, shunique_lock& sul)
 {
   assert(sul.owns_lock() && sul.mutex() == &rwlock);
   assert(info->linger_id);
+  assert(info->ctx_budget != -1); // caller needs to have taken budget already!
 
   // Populate Op::target
   OSDSession *s = NULL;
@@ -3347,6 +3350,11 @@ void Objecter::_throttle_op(Op *op,
   }
 }
 
+int Objecter::take_linger_budget(LingerOp *info)
+{
+  return 1;
+}
+
 void Objecter::unregister_op(Op *op)
 {
   OSDSession::unique_lock sl(op->session->lock);
diff --git a/src/osdc/Objecter.h b/src/osdc/Objecter.h
index 2efe5986b593..4a820a5c877c 100644
--- a/src/osdc/Objecter.h
+++ b/src/osdc/Objecter.h
@@ -1719,6 +1719,8 @@ public:
 
     OSDSession *session;
 
+    Objecter *objecter;
+    int ctx_budget;
     ceph_tid_t register_tid;
     ceph_tid_t ping_tid;
     epoch_t map_dne_bound;
@@ -1733,22 +1735,24 @@ public:
       watch_pending_async.pop_front();
     }
 
-    LingerOp() : linger_id(0),
-		 target(object_t(), object_locator_t(), 0),
-		 snap(CEPH_NOSNAP), poutbl(NULL), pobjver(NULL),
-		 is_watch(false), last_error(0),
-		 register_gen(0),
-		 registered(false),
-		 canceled(false),
-		 on_reg_commit(NULL),
-		 on_notify_finish(NULL),
-		 notify_result_bl(NULL),
-		 notify_id(0),
-		 watch_context(NULL),
-		 session(NULL),
-		 register_tid(0),
-		 ping_tid(0),
-		 map_dne_bound(0) {}
+    LingerOp(Objecter *o) : linger_id(0),
+			    target(object_t(), object_locator_t(), 0),
+			    snap(CEPH_NOSNAP), poutbl(NULL), pobjver(NULL),
+			    is_watch(false), last_error(0),
+			    register_gen(0),
+			    registered(false),
+			    canceled(false),
+			    on_reg_commit(NULL),
+			    on_notify_finish(NULL),
+			    notify_result_bl(NULL),
+			    notify_id(0),
+			    watch_context(NULL),
+			    session(NULL),
+			    objecter(o),
+			    ctx_budget(-1),
+			    register_tid(0),
+			    ping_tid(0),
+			    map_dne_bound(0) {}
 
     const LingerOp &operator=(const LingerOp& r) = delete;
     LingerOp(const LingerOp& o) = delete;
@@ -1996,13 +2000,15 @@ private:
     int op_budget = calc_op_budget(op->ops);
     if (keep_balanced_budget) {
       _throttle_op(op, sul, op_budget);
-    } else {
+    } else { // update take_linger_budget to match this!
       op_throttle_bytes.take(op_budget);
       op_throttle_ops.take(1);
     }
     op->budgeted = true;
     return op_budget;
   }
+  int take_linger_budget(LingerOp *info);
+  friend class WatchContext; // to invoke put_up_budget_bytes
   void put_op_budget_bytes(int op_budget) {
     assert(op_budget >= 0);
     op_throttle_bytes.put(op_budget);