objecter: add in-flight ops throttling

In addition to ops length, we also want to throttle it by
actual number of ops.

Signed-off-by: Yehuda Sadeh <yehuda@hq.newdream.net>

diff --git a/src/common/config_opts.h b/src/common/config_opts.h
index 3a8ae15ee31717219727875f9eed9389cc2a123a..53a413332db05e117a2655dbc7bcec464c745773 100644 (file)
--- a/src/common/config_opts.h
+++ b/src/common/config_opts.h
@@ -148,7 +148,8 @@ OPTION(fuse_big_writes, OPT_BOOL, true)
 OPTION(objecter_tick_interval, OPT_DOUBLE, 5.0)
 OPTION(objecter_mon_retry_interval, OPT_DOUBLE, 5.0)
 OPTION(objecter_timeout, OPT_DOUBLE, 10.0)    // before we ask for a map
-OPTION(objecter_inflight_op_bytes, OPT_U64, 1024*1024*100) //max in-flight data (both directions)
+OPTION(objecter_inflight_op_bytes, OPT_U64, 1024*1024*100) // max in-flight data (both directions)
+OPTION(objecter_inflight_ops, OPT_U64, 1024)               // max in-flight ios
 OPTION(journaler_allow_split_entries, OPT_BOOL, true)
 OPTION(journaler_write_head_interval, OPT_INT, 15)
 OPTION(journaler_prefetch_periods, OPT_INT, 10)   // * journal object size

diff --git a/src/osdc/Objecter.cc b/src/osdc/Objecter.cc
index 0caa2ba453b5cb1309ad391494391ef67d07738c..24b95d2c8422e3db57b7c901ef674151f57cf272 100644 (file)
--- a/src/osdc/Objecter.cc
+++ b/src/osdc/Objecter.cc
@@ -1137,9 +1137,14 @@ void Objecter::throttle_op(Op *op, int op_budget)
 {
   if (!op_budget)
     op_budget = calc_op_budget(op);
-  if (!op_throttler.get_or_fail(op_budget)) { //couldn't take right now
+  if (!op_throttle_bytes.get_or_fail(op_budget)) { //couldn't take right now
     client_lock.Unlock();
-    op_throttler.get(op_budget);
+    op_throttle_bytes.get(op_budget);
+    client_lock.Lock();
+  }
+  if (!op_throttle_ops.get_or_fail(1)) { //couldn't take right now
+    client_lock.Unlock();
+    op_throttle_ops.get(1);
     client_lock.Lock();
   }
 }

diff --git a/src/osdc/Objecter.h b/src/osdc/Objecter.h
index 5ab242ca80f18145dbc871c5f9d9507ba6b02c4e..a5a3c0d0a19feef38c0bffbc84b3135d84d9ef4f 100644 (file)
--- a/src/osdc/Objecter.h
+++ b/src/osdc/Objecter.h
@@ -894,13 +894,15 @@ public:
     if (keep_balanced_budget)
       throttle_op(op, op_budget);
     else
-      op_throttler.take(op_budget);
+      op_throttle_bytes.take(op_budget);
+    op_throttle_ops.take(1);
   }
   void put_op_budget(Op *op) {
     int op_budget = calc_op_budget(op);
-    op_throttler.put(op_budget);
+    op_throttle_bytes.put(op_budget);
+    op_throttle_ops.put(1);
   }
-  Throttle op_throttler;
+  Throttle op_throttle_bytes, op_throttle_ops;
 
  public:
   Objecter(CephContext *cct_, Messenger *m, MonClient *mc,
@@ -917,7 +919,8 @@ public:
     logger(NULL), tick_event(NULL),
     m_request_state_hook(NULL),
     num_homeless_ops(0),
-    op_throttler(cct->_conf->objecter_inflight_op_bytes)
+    op_throttle_bytes(cct->_conf->objecter_inflight_op_bytes),
+    op_throttle_ops(cct->_conf->objecter_inflight_ops)
   { }
   ~Objecter() {
     assert(!tick_event);