mon: forward mgr commands; cap them

Put the completions on a finisher to avoid deadlock.
(MgrClient lock is inside mon_lock due to start_command;
completion cannot retake mon_lock while holding MgrClient's
internal lock.)

The mon has a cap on the nubmer of client message bytes it hold
in memory.  We do not want to eat those up and DOS the mon
because the mgr is not available and all mgr commands are
blocked.

Define the limit as a fraction of mon_client_bytes to avoid user
misconfiguration.

Return EAGAIN if we hit the limit.

Signed-off-by: Sage Weil <sage@redhat.com>

diff --git a/src/common/config_opts.h b/src/common/config_opts.h
index a0b77264aa3f8d6d1da2435896a6e7e5be538857..7682a6a8022658e508fe485ac5734fc98d9cdf48 100644 (file)
--- a/src/common/config_opts.h
+++ b/src/common/config_opts.h
@@ -329,6 +329,7 @@ OPTION(mon_probe_timeout, OPT_DOUBLE, 2.0)
 OPTION(mon_slurp_timeout, OPT_DOUBLE, 10.0)
 OPTION(mon_slurp_bytes, OPT_INT, 256*1024)    // limit size of slurp messages
 OPTION(mon_client_bytes, OPT_U64, 100ul << 20)  // client msg data allowed in memory (in bytes)
+OPTION(mon_mgr_proxy_client_bytes_ratio, OPT_FLOAT, .3) // ratio of mon_client_bytes that can be consumed by proxied mgr commands before we error out to client
 OPTION(mon_daemon_bytes, OPT_U64, 400ul << 20)  // mds, osd message memory cap (in bytes)
 OPTION(mon_max_log_entries_per_event, OPT_INT, 4096)
 OPTION(mon_reweight_min_pgs_per_osd, OPT_U64, 10)   // min pgs per osd for reweight-by-pg command

diff --git a/src/mon/Monitor.cc b/src/mon/Monitor.cc
index 461dc6ad6c37b25ee65e0d1712c1d3597c557908..46a7ed55f29338e214c444b33e915fa7cd28c437 100644 (file)
--- a/src/mon/Monitor.cc
+++ b/src/mon/Monitor.cc
@@ -2743,6 +2743,21 @@ bool Monitor::is_keyring_required()
     auth_cluster_required == "cephx";
 }
 
+struct C_MgrProxyCommand : public Context {
+  Monitor *mon;
+  MonOpRequestRef op;
+  uint64_t size;
+  bufferlist outbl;
+  string outs;
+  C_MgrProxyCommand(Monitor *mon, MonOpRequestRef op, uint64_t s)
+    : mon(mon), op(op), size(s) { }
+  void finish(int r) {
+    Mutex::Locker l(mon->lock);
+    mon->mgr_proxy_bytes -= size;
+    mon->reply_command(op, r, outs, outbl, 0);
+  }
+};
+
 void Monitor::handle_command(MonOpRequestRef op)
 {
   assert(op->is_type_command());
@@ -2919,6 +2934,30 @@ void Monitor::handle_command(MonOpRequestRef op)
     << "entity='" << session->entity_name << "' "
     << "cmd=" << m->cmd << ": dispatch";
 
+  if (mon_cmd->is_mgr() &&
+      osdmon()->osdmap.test_flag(CEPH_OSDMAP_REQUIRE_LUMINOUS)) {
+    const auto& hdr = m->get_header();
+    uint64_t size = hdr.front_len + hdr.middle_len + hdr.data_len;
+    uint64_t max =
+      g_conf->mon_client_bytes * g_conf->mon_mgr_proxy_client_bytes_ratio;
+    if (mgr_proxy_bytes + size > max) {
+      dout(10) << __func__ << " current mgr proxy bytes " << mgr_proxy_bytes
+              << " + " << size << " > max " << max << dendl;
+      reply_command(op, -EAGAIN, "hit limit on proxied mgr commands", rdata, 0);
+      return;
+    }
+    mgr_proxy_bytes += size;
+    dout(10) << __func__ << " proxying mgr command (+" << size
+            << " -> " << mgr_proxy_bytes << ")" << dendl;
+    C_MgrProxyCommand *fin = new C_MgrProxyCommand(this, op, size);
+    mgr_client.start_command(m->cmd,
+                            m->get_data(),
+                            &fin->outbl,
+                            &fin->outs,
+                            new C_OnFinisher(fin, &finisher));
+    return;
+  }
+
   if (module == "mds" || module == "fs") {
     mdsmon()->dispatch(op);
     return;

diff --git a/src/mon/Monitor.h b/src/mon/Monitor.h
index 6fb6229b3691465074594fc78b535a26ac9523de..77741ad6e4ddf3e754b16d03217b28cf63ba0621 100644 (file)
--- a/src/mon/Monitor.h
+++ b/src/mon/Monitor.h
@@ -159,6 +159,8 @@ public:
 
   Messenger *mgr_messenger;
   MgrClient mgr_client;
+  uint64_t mgr_proxy_bytes = 0;  // in-flight proxied mgr command message bytes
+
 private:
   void new_tick();
   friend class C_Mon_Tick;