mon/MonClient: give up targetted mon_commands after one attempt

If one of the monitors is down, we need to give up trying to reconnect to
that mon and send it a MonCommand.

Fixes: https://tracker.ceph.com/issues/40792
Signed-off-by: Sage Weil <sage@redhat.com>
Signed-off-by: Greg Farnum <gfarnum@redhat.com>
(cherry picked from commit 134f6a843ce6a9c9aeea59cd43c8c60103e635a7)

Conflicts:
	src/mon/MonClient.cc
- 5475f67bf66d1d7faf20b91dd6e629051996a0e9 is not being backported

diff --git a/src/mon/MonClient.cc b/src/mon/MonClient.cc
index 6c9722698c5dba361b9afcbb41e906b9f5557c9e..c08d95dcdfaec5d739a10f60bf0d98a122e457cc 100644 (file)
--- a/src/mon/MonClient.cc
+++ b/src/mon/MonClient.cc
@@ -1001,6 +1001,8 @@ int MonClient::wait_auth_rotating(double timeout)
 
 void MonClient::_send_command(MonCommand *r)
 {
+  ++r->send_attempts;
+
   entity_addr_t peer;
   if (active_con) {
     peer = active_con->get_con()->get_peer_addr();
@@ -1008,6 +1010,10 @@ void MonClient::_send_command(MonCommand *r)
 
   if (r->target_rank >= 0 &&
       r->target_rank != monmap.get_rank(peer)) {
+    if (r->send_attempts > 1) {
+      _finish_command(r, -ENXIO, "mon unavailable");
+      return;
+    }
     ldout(cct, 10) << __func__ << " " << r->tid << " " << r->cmd
                   << " wants rank " << r->target_rank
                   << ", reopening session"
@@ -1023,6 +1029,10 @@ void MonClient::_send_command(MonCommand *r)
 
   if (r->target_name.length() &&
       r->target_name != monmap.get_name(peer)) {
+    if (r->send_attempts > 1) {
+      _finish_command(r, -ENXIO, "mon unavailable");
+      return;
+    }
     ldout(cct, 10) << __func__ << " " << r->tid << " " << r->cmd
                   << " wants mon " << r->target_name
                   << ", reopening session"
@@ -1048,10 +1058,11 @@ void MonClient::_send_command(MonCommand *r)
 void MonClient::_resend_mon_commands()
 {
   // resend any requests
-  for (map<uint64_t,MonCommand*>::iterator p = mon_commands.begin();
-       p != mon_commands.end();
-       ++p) {
-    _send_command(p->second);
+  map<uint64_t,MonCommand*>::iterator p = mon_commands.begin();
+  while (p != mon_commands.end()) {
+    auto cmd = p->second;
+    ++p;
+    _send_command(cmd); // might remove cmd from mon_commands
   }
 }
 

diff --git a/src/mon/MonClient.h b/src/mon/MonClient.h
index e4c018289bee5334e87ac27a569cb592c8be8a6b..a7874b72a5862cd18f82f0d493e90665e21dcfb1 100644 (file)
--- a/src/mon/MonClient.h
+++ b/src/mon/MonClient.h
@@ -499,6 +499,7 @@ private:
   struct MonCommand {
     string target_name;
     int target_rank;
+    unsigned send_attempts = 0;
     uint64_t tid;
     vector<string> cmd;
     bufferlist inbl;