From: Sage Weil <sage@redhat.com>
Date: Wed, 17 Jul 2019 15:20:25 +0000 (-0500)
Subject: mon/MonClient: give up targetted mon_commands after one attempt
X-Git-Tag: v15.1.0~1758^2~2
X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=134f6a843ce6a9c9aeea59cd43c8c60103e635a7;p=ceph.git

mon/MonClient: give up targetted mon_commands after one attempt

If one of the monitors is down, we need to give up trying to reconnect to
that mon and send it a MonCommand.

Fixes: https://tracker.ceph.com/issues/40792
Signed-off-by: Sage Weil <sage@redhat.com>
Signed-off-by: Greg Farnum <gfarnum@redhat.com>
---

diff --git a/src/mon/MonClient.cc b/src/mon/MonClient.cc
index 853dac05b69..d868555b520 100644
--- a/src/mon/MonClient.cc
+++ b/src/mon/MonClient.cc
@@ -1031,6 +1031,8 @@ int MonClient::wait_auth_rotating(double timeout)
 
 void MonClient::_send_command(MonCommand *r)
 {
+  ++r->send_attempts;
+
   entity_addr_t peer;
   if (active_con) {
     peer = active_con->get_con()->get_peer_addr();
@@ -1038,6 +1040,10 @@ void MonClient::_send_command(MonCommand *r)
 
   if (r->target_rank >= 0 &&
       r->target_rank != monmap.get_rank(peer)) {
+    if (r->send_attempts > 1) {
+      _finish_command(r, -ENXIO, "mon unavailable");
+      return;
+    }
     ldout(cct, 10) << __func__ << " " << r->tid << " " << r->cmd
 		   << " wants rank " << r->target_rank
 		   << ", reopening session"
@@ -1053,6 +1059,10 @@ void MonClient::_send_command(MonCommand *r)
 
   if (r->target_name.length() &&
       r->target_name != monmap.get_name(peer)) {
+    if (r->send_attempts > 1) {
+      _finish_command(r, -ENXIO, "mon unavailable");
+      return;
+    }
     ldout(cct, 10) << __func__ << " " << r->tid << " " << r->cmd
 		   << " wants mon " << r->target_name
 		   << ", reopening session"
@@ -1078,8 +1088,11 @@ void MonClient::_send_command(MonCommand *r)
 void MonClient::_resend_mon_commands()
 {
   // resend any requests
-  for (auto p = mon_commands.begin(); p != mon_commands.end(); ++p) {
-    _send_command(p->second);
+  auto p = mon_commands.begin();
+  while (p != mon_commands.end()) {
+    auto cmd = p->second;
+    ++p;
+    _send_command(cmd); // might remove cmd from mon_commands
   }
 }
 
diff --git a/src/mon/MonClient.h b/src/mon/MonClient.h
index 2fa4fdcf864..bee891ce933 100644
--- a/src/mon/MonClient.h
+++ b/src/mon/MonClient.h
@@ -501,6 +501,7 @@ private:
   struct MonCommand {
     std::string target_name;
     int target_rank;
+    unsigned send_attempts = 0;
     uint64_t tid;
     std::vector<std::string> cmd;
     ceph::buffer::list inbl;