osd: fix hb con failure handler

Fix a few bugs introduced by 27381c0c6259ac89f5f9c592b4bfb585937a1cfc:

- check against both front and back cons; either one may have failed.
- close *both* front and back before reopening either.  this is
  overkill, but slightly simpler code.
- fix leak of con when marking down
- handle race against osdmap update and note_down_osd

Fixes: #5172
Signed-off-by: Sage Weil <sage@inktank.com>

diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc
index e725e97e8221ab4330dabe49365973fd0662bdd7..0915a08190c79fab20c9e5df6536951b35e72fa9 100644 (file)
--- a/src/osd/OSD.cc
+++ b/src/osd/OSD.cc
@@ -2622,29 +2622,37 @@ bool OSD::heartbeat_reset(Connection *con)
     }
     map<int,HeartbeatInfo>::iterator p = heartbeat_peers.find(s->peer);
     if (p != heartbeat_peers.end() &&
-       p->second.con_back == con) {
-      pair<ConnectionRef,ConnectionRef> newcon = service.get_con_osd_hb(p->second.peer, p->second.epoch);
-      if (!newcon.first) {
-       dout(10) << "heartbeat_reset reopen failed hb con " << con << " but failed to reopen" << dendl;
-      } else {
-       dout(10) << "heartbeat_reset reopen failed hb con " << con << dendl;
+       (p->second.con_back == con ||
+        p->second.con_front == con)) {
+      dout(10) << "heartbeat_reset failed hb con " << con << " for osd." << p->second.peer
+              << ", reopening" << dendl;
+      if (con != p->second.con_back) {
        hbclient_messenger->mark_down(p->second.con_back);
+       p->second.con_back->put();
+      }
+      p->second.con_back = NULL;
+      if (p->second.con_front && con != p->second.con_front) {
+       hbclient_messenger->mark_down(p->second.con_front);
+       p->second.con_front->put();
+      }
+      p->second.con_front = NULL;
+      pair<ConnectionRef,ConnectionRef> newcon = service.get_con_osd_hb(p->second.peer, p->second.epoch);
+      if (newcon.first) {
        p->second.con_back = newcon.first.get();
        p->second.con_back->get();
        p->second.con_back->set_priv(s);
-       if (p->second.con_front)
-         hbclient_messenger->mark_down(p->second.con_front);
        if (newcon.second) {
          p->second.con_front = newcon.second.get();
          p->second.con_front->get();
          p->second.con_front->set_priv(s->get());
-       } else {
-         p->second.con_front = NULL;
        }
+      } else {
+       dout(10) << "heartbeat_reset failed hb con " << con << " for osd." << p->second.peer
+                << ", raced with osdmap update, closing out peer" << dendl;
+       heartbeat_peers.erase(p);
       }
     } else {
       dout(10) << "heartbeat_reset closing (old) failed hb con " << con << dendl;
-      hbclient_messenger->mark_down(con);
     }
     heartbeat_lock.Unlock();
     s->put();