mgr/MgrClient: throttle connection attempts

Max of 1 per second.

Signed-off-by: Sage Weil <sage@redhat.com>

diff --git a/src/common/config_opts.h b/src/common/config_opts.h
index 7682a6a8022658e508fe485ac5734fc98d9cdf48..aa6b16baee2c6eb9a7d07068e38942559f2b7e84 100644 (file)
--- a/src/common/config_opts.h
+++ b/src/common/config_opts.h
@@ -1644,6 +1644,8 @@ OPTION(mgr_mds_messages, OPT_U64, 128)        // messages from mdss
 OPTION(mgr_mon_bytes, OPT_U64, 128*1048576)   // bytes from mons
 OPTION(mgr_mon_messages, OPT_U64, 128)        // messages from mons
 
+OPTION(mgr_connect_retry_interval, OPT_DOUBLE, 1.0)
+
 OPTION(mon_mgr_digest_period, OPT_INT, 5)  // How frequently to send digests
 OPTION(mon_mgr_beacon_grace, OPT_INT, 30)  // How long to wait to failover
 

diff --git a/src/mgr/MgrClient.cc b/src/mgr/MgrClient.cc
index 49d752054c16e8e9bcf716fdd23db667402276b1..8079aa8012a3f3de14133c44cc53df6132a584a3 100644 (file)
--- a/src/mgr/MgrClient.cc
+++ b/src/mgr/MgrClient.cc
@@ -32,8 +32,7 @@
 MgrClient::MgrClient(CephContext *cct_, Messenger *msgr_)
     : Dispatcher(cct_), cct(cct_), msgr(msgr_),
       lock("mgrc"),
-      timer(cct_, lock),
-      report_callback(nullptr)
+      timer(cct_, lock)
 {
   assert(cct != nullptr);
 }
@@ -93,33 +92,57 @@ void MgrClient::reconnect()
     }
   }
 
-  if (map.get_available()) {
-    ldout(cct, 4) << "Starting new session with " << map.get_active_addr()
-                 << dendl;
-    entity_inst_t inst;
-    inst.addr = map.get_active_addr();
-    inst.name = entity_name_t::MGR(map.get_active_gid());
-
-    session.reset(new MgrSessionState());
-    session->con = msgr->get_connection(inst);
-
-    // Don't send an open if we're just a client (i.e. doing
-    // command-sending, not stats etc)
-    if (g_conf && !g_conf->name.is_client()) {
-      auto open = new MMgrOpen();
-      open->daemon_name = g_conf->name.get_id();
-      session->con->send_message(open);
-    }
+  if (!map.get_available()) {
+    ldout(cct, 4) << "No active mgr available yet" << dendl;
+    return;
+  }
 
-    // resend any pending commands
-    for (const auto &p : command_table.get_commands()) {
-      MCommand *m = p.second.get_message({});
-      assert(session);
-      assert(session->con);
-      session->con->send_message(m);
+  if (last_connect_attempt != utime_t()) {
+    utime_t now = ceph_clock_now();
+    utime_t when = last_connect_attempt;
+    when += cct->_conf->mgr_connect_retry_interval;
+    if (now < when) {
+      if (!connect_retry_callback) {
+       connect_retry_callback = new FunctionContext([this](int r){
+           connect_retry_callback = nullptr;
+           reconnect();
+         });
+       timer.add_event_at(when, connect_retry_callback);
+      }
+      ldout(cct, 4) << "waiting to retry connect until " << when << dendl;
+      return;
     }
-  } else {
-    ldout(cct, 4) << "No active mgr available yet" << dendl;
+  }
+
+  if (connect_retry_callback) {
+    timer.cancel_event(connect_retry_callback);
+    connect_retry_callback = nullptr;
+  }
+
+  ldout(cct, 4) << "Starting new session with " << map.get_active_addr()
+               << dendl;
+  entity_inst_t inst;
+  inst.addr = map.get_active_addr();
+  inst.name = entity_name_t::MGR(map.get_active_gid());
+  last_connect_attempt = ceph_clock_now();
+
+  session.reset(new MgrSessionState());
+  session->con = msgr->get_connection(inst);
+
+  // Don't send an open if we're just a client (i.e. doing
+  // command-sending, not stats etc)
+  if (g_conf && !g_conf->name.is_client()) {
+    auto open = new MMgrOpen();
+    open->daemon_name = g_conf->name.get_id();
+    session->con->send_message(open);
+  }
+
+  // resend any pending commands
+  for (const auto &p : command_table.get_commands()) {
+    MCommand *m = p.second.get_message({});
+    assert(session);
+    assert(session->con);
+    session->con->send_message(m);
   }
 }
 

diff --git a/src/mgr/MgrClient.h b/src/mgr/MgrClient.h
index 4190a58ea79af68cfd3c8eba53bb612468b36f71..fce73cb7e58b664e7d48668ffb63b591c22c6f83 100644 (file)
--- a/src/mgr/MgrClient.h
+++ b/src/mgr/MgrClient.h
@@ -63,7 +63,10 @@ protected:
 
   CommandTable<MgrCommand> command_table;
 
-  Context *report_callback;
+  utime_t last_connect_attempt;
+
+  Context *report_callback = nullptr;
+  Context *connect_retry_callback = nullptr;
 
   // If provided, use this to compose an MPGStats to send with
   // our reports (hook for use by OSD)