mon/MonClient: give up targetted mon_commands after one attempt

If one of the monitors is down, we need to give up trying to reconnect to
that mon and send it a MonCommand.

Fixes: https://tracker.ceph.com/issues/40792
Signed-off-by: Sage Weil <sage@redhat.com>
Signed-off-by: Greg Farnum <gfarnum@redhat.com>

diff --git a/src/mon/MonClient.cc b/src/mon/MonClient.cc
index 853dac05b6916efc6c7f2bfb02b35895db5969f9..d868555b520eba1a59bc57ec01d020424b91314e 100644 (file)
--- a/src/mon/MonClient.cc
+++ b/src/mon/MonClient.cc
@@ -1031,6 +1031,8 @@ int MonClient::wait_auth_rotating(double timeout)
 
 void MonClient::_send_command(MonCommand *r)
 {
+  ++r->send_attempts;
+
   entity_addr_t peer;
   if (active_con) {
     peer = active_con->get_con()->get_peer_addr();
@@ -1038,6 +1040,10 @@ void MonClient::_send_command(MonCommand *r)
 
   if (r->target_rank >= 0 &&
       r->target_rank != monmap.get_rank(peer)) {
+    if (r->send_attempts > 1) {
+      _finish_command(r, -ENXIO, "mon unavailable");
+      return;
+    }
     ldout(cct, 10) << __func__ << " " << r->tid << " " << r->cmd
                   << " wants rank " << r->target_rank
                   << ", reopening session"
@@ -1053,6 +1059,10 @@ void MonClient::_send_command(MonCommand *r)
 
   if (r->target_name.length() &&
       r->target_name != monmap.get_name(peer)) {
+    if (r->send_attempts > 1) {
+      _finish_command(r, -ENXIO, "mon unavailable");
+      return;
+    }
     ldout(cct, 10) << __func__ << " " << r->tid << " " << r->cmd
                   << " wants mon " << r->target_name
                   << ", reopening session"
@@ -1078,8 +1088,11 @@ void MonClient::_send_command(MonCommand *r)
 void MonClient::_resend_mon_commands()
 {
   // resend any requests
-  for (auto p = mon_commands.begin(); p != mon_commands.end(); ++p) {
-    _send_command(p->second);
+  auto p = mon_commands.begin();
+  while (p != mon_commands.end()) {
+    auto cmd = p->second;
+    ++p;
+    _send_command(cmd); // might remove cmd from mon_commands
   }
 }
 

diff --git a/src/mon/MonClient.h b/src/mon/MonClient.h
index 2fa4fdcf8643ad15f513c95a58342b563c51aa0e..bee891ce9334449e2d5ae6e45befcd2b1582defb 100644 (file)
--- a/src/mon/MonClient.h
+++ b/src/mon/MonClient.h
@@ -501,6 +501,7 @@ private:
   struct MonCommand {
     std::string target_name;
     int target_rank;
+    unsigned send_attempts = 0;
     uint64_t tid;
     std::vector<std::string> cmd;
     ceph::buffer::list inbl;