osd: throttle client messages by count, not just by bytes

This lets us put a cap on outstanding client IOs.  This is particularly
important for clients issuing lots of small IOs.

Fixes: #4579
Signed-off-by: Sage Weil <sage@inktank.com>

diff --git a/src/ceph_osd.cc b/src/ceph_osd.cc
index d7735a7a83afc1a79008362d67223faf51606366..33a107c1dc03748430d9546b9e49e91c00a4665b 100644 (file)
--- a/src/ceph_osd.cc
+++ b/src/ceph_osd.cc
@@ -338,9 +338,12 @@ int main(int argc, const char **argv)
                    "(no journal)" : g_conf->osd_journal)
        << std::endl;
 
-  boost::scoped_ptr<Throttle> client_throttler(
+  boost::scoped_ptr<Throttle> client_byte_throttler(
     new Throttle(g_ceph_context, "osd_client_bytes",
                 g_conf->osd_client_message_size_cap));
+  boost::scoped_ptr<Throttle> client_msg_throttler(
+    new Throttle(g_ceph_context, "osd_client_messages",
+                g_conf->osd_client_message_cap));
 
   uint64_t supported =
     CEPH_FEATURE_UID | 
@@ -350,8 +353,8 @@ int main(int argc, const char **argv)
 
   client_messenger->set_default_policy(Messenger::Policy::stateless_server(supported, 0));
   client_messenger->set_policy_throttlers(entity_name_t::TYPE_CLIENT,
-                                         client_throttler.get(),
-                                         NULL);
+                                         client_byte_throttler.get(),
+                                         client_msg_throttler.get());
   client_messenger->set_policy(entity_name_t::TYPE_MON,
                                Messenger::Policy::lossy_client(supported,
                                                               CEPH_FEATURE_UID |
@@ -462,7 +465,8 @@ int main(int argc, const char **argv)
   delete messenger_hbclient;
   delete messenger_hbserver;
   delete cluster_messenger;
-  client_throttler.reset();
+  client_byte_throttler.reset();
+  client_msg_throttler.reset();
   g_ceph_context->put();
 
   // cd on exit, so that gmon.out (if any) goes into a separate directory for each node.

diff --git a/src/common/config_opts.h b/src/common/config_opts.h
index cb2fd391fc9130f62f19a36257ebb3139514abc9..17e6ec4c8c41a3dc09c4348eb55d41307807700d 100644 (file)
--- a/src/common/config_opts.h
+++ b/src/common/config_opts.h
@@ -349,6 +349,7 @@ OPTION(osd_journal_size, OPT_INT, 5120)         // in mb
 OPTION(osd_max_write_size, OPT_INT, 90)
 OPTION(osd_max_pgls, OPT_U64, 1024) // max number of pgls entries to return
 OPTION(osd_client_message_size_cap, OPT_U64, 500*1024L*1024L) // client data allowed in-memory (in bytes)
+OPTION(osd_client_message_cap, OPT_U64, 100)              // client messages allowed in-memory (in bytes)
 OPTION(osd_pg_bits, OPT_INT, 6)  // bits per osd
 OPTION(osd_pgp_bits, OPT_INT, 6)  // bits per osd
 OPTION(osd_crush_chooseleaf_type, OPT_INT, 1) // 1 = host