From: Sage Weil <sage@redhat.com>
Date: Sun, 26 Feb 2017 19:17:38 +0000 (-0500)
Subject: mon: forward mgr commands; cap them
X-Git-Tag: v12.0.2~252^2~50
X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=89735b78f67182aa44003e61532ba7e750f14d10;p=ceph.git

mon: forward mgr commands; cap them

Put the completions on a finisher to avoid deadlock.
(MgrClient lock is inside mon_lock due to start_command;
completion cannot retake mon_lock while holding MgrClient's
internal lock.)

The mon has a cap on the nubmer of client message bytes it hold
in memory.  We do not want to eat those up and DOS the mon
because the mgr is not available and all mgr commands are
blocked.

Define the limit as a fraction of mon_client_bytes to avoid user
misconfiguration.

Return EAGAIN if we hit the limit.

Signed-off-by: Sage Weil <sage@redhat.com>
---

diff --git a/src/common/config_opts.h b/src/common/config_opts.h
index a0b77264aa3f..7682a6a80226 100644
--- a/src/common/config_opts.h
+++ b/src/common/config_opts.h
@@ -329,6 +329,7 @@ OPTION(mon_probe_timeout, OPT_DOUBLE, 2.0)
 OPTION(mon_slurp_timeout, OPT_DOUBLE, 10.0)
 OPTION(mon_slurp_bytes, OPT_INT, 256*1024)    // limit size of slurp messages
 OPTION(mon_client_bytes, OPT_U64, 100ul << 20)  // client msg data allowed in memory (in bytes)
+OPTION(mon_mgr_proxy_client_bytes_ratio, OPT_FLOAT, .3) // ratio of mon_client_bytes that can be consumed by proxied mgr commands before we error out to client
 OPTION(mon_daemon_bytes, OPT_U64, 400ul << 20)  // mds, osd message memory cap (in bytes)
 OPTION(mon_max_log_entries_per_event, OPT_INT, 4096)
 OPTION(mon_reweight_min_pgs_per_osd, OPT_U64, 10)   // min pgs per osd for reweight-by-pg command
diff --git a/src/mon/Monitor.cc b/src/mon/Monitor.cc
index 461dc6ad6c37..46a7ed55f293 100644
--- a/src/mon/Monitor.cc
+++ b/src/mon/Monitor.cc
@@ -2743,6 +2743,21 @@ bool Monitor::is_keyring_required()
     auth_cluster_required == "cephx";
 }
 
+struct C_MgrProxyCommand : public Context {
+  Monitor *mon;
+  MonOpRequestRef op;
+  uint64_t size;
+  bufferlist outbl;
+  string outs;
+  C_MgrProxyCommand(Monitor *mon, MonOpRequestRef op, uint64_t s)
+    : mon(mon), op(op), size(s) { }
+  void finish(int r) {
+    Mutex::Locker l(mon->lock);
+    mon->mgr_proxy_bytes -= size;
+    mon->reply_command(op, r, outs, outbl, 0);
+  }
+};
+
 void Monitor::handle_command(MonOpRequestRef op)
 {
   assert(op->is_type_command());
@@ -2919,6 +2934,30 @@ void Monitor::handle_command(MonOpRequestRef op)
     << "entity='" << session->entity_name << "' "
     << "cmd=" << m->cmd << ": dispatch";
 
+  if (mon_cmd->is_mgr() &&
+      osdmon()->osdmap.test_flag(CEPH_OSDMAP_REQUIRE_LUMINOUS)) {
+    const auto& hdr = m->get_header();
+    uint64_t size = hdr.front_len + hdr.middle_len + hdr.data_len;
+    uint64_t max =
+      g_conf->mon_client_bytes * g_conf->mon_mgr_proxy_client_bytes_ratio;
+    if (mgr_proxy_bytes + size > max) {
+      dout(10) << __func__ << " current mgr proxy bytes " << mgr_proxy_bytes
+	       << " + " << size << " > max " << max << dendl;
+      reply_command(op, -EAGAIN, "hit limit on proxied mgr commands", rdata, 0);
+      return;
+    }
+    mgr_proxy_bytes += size;
+    dout(10) << __func__ << " proxying mgr command (+" << size
+	     << " -> " << mgr_proxy_bytes << ")" << dendl;
+    C_MgrProxyCommand *fin = new C_MgrProxyCommand(this, op, size);
+    mgr_client.start_command(m->cmd,
+			     m->get_data(),
+			     &fin->outbl,
+			     &fin->outs,
+			     new C_OnFinisher(fin, &finisher));
+    return;
+  }
+
   if (module == "mds" || module == "fs") {
     mdsmon()->dispatch(op);
     return;
diff --git a/src/mon/Monitor.h b/src/mon/Monitor.h
index 6fb6229b3691..77741ad6e4dd 100644
--- a/src/mon/Monitor.h
+++ b/src/mon/Monitor.h
@@ -159,6 +159,8 @@ public:
 
   Messenger *mgr_messenger;
   MgrClient mgr_client;
+  uint64_t mgr_proxy_bytes = 0;  // in-flight proxied mgr command message bytes
+
 private:
   void new_tick();
   friend class C_Mon_Tick;