From 89735b78f67182aa44003e61532ba7e750f14d10 Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Sun, 26 Feb 2017 14:17:38 -0500 Subject: [PATCH] mon: forward mgr commands; cap them Put the completions on a finisher to avoid deadlock. (MgrClient lock is inside mon_lock due to start_command; completion cannot retake mon_lock while holding MgrClient's internal lock.) The mon has a cap on the nubmer of client message bytes it hold in memory. We do not want to eat those up and DOS the mon because the mgr is not available and all mgr commands are blocked. Define the limit as a fraction of mon_client_bytes to avoid user misconfiguration. Return EAGAIN if we hit the limit. Signed-off-by: Sage Weil --- src/common/config_opts.h | 1 + src/mon/Monitor.cc | 39 +++++++++++++++++++++++++++++++++++++++ src/mon/Monitor.h | 2 ++ 3 files changed, 42 insertions(+) diff --git a/src/common/config_opts.h b/src/common/config_opts.h index a0b77264aa3..7682a6a8022 100644 --- a/src/common/config_opts.h +++ b/src/common/config_opts.h @@ -329,6 +329,7 @@ OPTION(mon_probe_timeout, OPT_DOUBLE, 2.0) OPTION(mon_slurp_timeout, OPT_DOUBLE, 10.0) OPTION(mon_slurp_bytes, OPT_INT, 256*1024) // limit size of slurp messages OPTION(mon_client_bytes, OPT_U64, 100ul << 20) // client msg data allowed in memory (in bytes) +OPTION(mon_mgr_proxy_client_bytes_ratio, OPT_FLOAT, .3) // ratio of mon_client_bytes that can be consumed by proxied mgr commands before we error out to client OPTION(mon_daemon_bytes, OPT_U64, 400ul << 20) // mds, osd message memory cap (in bytes) OPTION(mon_max_log_entries_per_event, OPT_INT, 4096) OPTION(mon_reweight_min_pgs_per_osd, OPT_U64, 10) // min pgs per osd for reweight-by-pg command diff --git a/src/mon/Monitor.cc b/src/mon/Monitor.cc index 461dc6ad6c3..46a7ed55f29 100644 --- a/src/mon/Monitor.cc +++ b/src/mon/Monitor.cc @@ -2743,6 +2743,21 @@ bool Monitor::is_keyring_required() auth_cluster_required == "cephx"; } +struct C_MgrProxyCommand : public Context { + Monitor *mon; + MonOpRequestRef op; + uint64_t size; + bufferlist outbl; + string outs; + C_MgrProxyCommand(Monitor *mon, MonOpRequestRef op, uint64_t s) + : mon(mon), op(op), size(s) { } + void finish(int r) { + Mutex::Locker l(mon->lock); + mon->mgr_proxy_bytes -= size; + mon->reply_command(op, r, outs, outbl, 0); + } +}; + void Monitor::handle_command(MonOpRequestRef op) { assert(op->is_type_command()); @@ -2919,6 +2934,30 @@ void Monitor::handle_command(MonOpRequestRef op) << "entity='" << session->entity_name << "' " << "cmd=" << m->cmd << ": dispatch"; + if (mon_cmd->is_mgr() && + osdmon()->osdmap.test_flag(CEPH_OSDMAP_REQUIRE_LUMINOUS)) { + const auto& hdr = m->get_header(); + uint64_t size = hdr.front_len + hdr.middle_len + hdr.data_len; + uint64_t max = + g_conf->mon_client_bytes * g_conf->mon_mgr_proxy_client_bytes_ratio; + if (mgr_proxy_bytes + size > max) { + dout(10) << __func__ << " current mgr proxy bytes " << mgr_proxy_bytes + << " + " << size << " > max " << max << dendl; + reply_command(op, -EAGAIN, "hit limit on proxied mgr commands", rdata, 0); + return; + } + mgr_proxy_bytes += size; + dout(10) << __func__ << " proxying mgr command (+" << size + << " -> " << mgr_proxy_bytes << ")" << dendl; + C_MgrProxyCommand *fin = new C_MgrProxyCommand(this, op, size); + mgr_client.start_command(m->cmd, + m->get_data(), + &fin->outbl, + &fin->outs, + new C_OnFinisher(fin, &finisher)); + return; + } + if (module == "mds" || module == "fs") { mdsmon()->dispatch(op); return; diff --git a/src/mon/Monitor.h b/src/mon/Monitor.h index 6fb6229b369..77741ad6e4d 100644 --- a/src/mon/Monitor.h +++ b/src/mon/Monitor.h @@ -159,6 +159,8 @@ public: Messenger *mgr_messenger; MgrClient mgr_client; + uint64_t mgr_proxy_bytes = 0; // in-flight proxied mgr command message bytes + private: void new_tick(); friend class C_Mon_Tick; -- 2.39.5