osd: always mark down old hb peers; send map update via cluster link

If we don't mark down the hb link immediately, we'll forget about it
because it won't be in the from or to set anymore, and if it does go down
later we'll end up with garbage in the logs.

Instead, always mark it down.  Since we want to share our map with old
peers that are still up, do that via the cluster link instead, which is
reliably marked down if/when the peer goes down.

Signed-off-by: Sage Weil <sage@newdream.net>

diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc
index 2e12390c1dfeaca8ceb71945c758e681123d09bb..f5fc78eb11c60384f769e8a61997093364b8fe4f 100644 (file)
--- a/src/osd/OSD.cc
+++ b/src/osd/OSD.cc
@@ -1223,17 +1223,20 @@ void OSD::update_heartbeat_peers()
               << " as of " << p->second << dendl;
       heartbeat_to[p->first] = p->second;
       heartbeat_inst[p->first] = old_inst[p->first];
-    } else if (osdmap->is_down(p->first) ||
-              osdmap->get_hb_inst(p->first) != old_inst[p->first]) {
-      dout(10) << "update_heartbeat_peers: marking down old down _to peer " << old_inst[p->first] 
-              << " as of " << p->second << dendl;
-      heartbeat_messenger->mark_down(old_inst[p->first].addr);
     } else {
-      dout(10) << "update_heartbeat_peers: sharing map with old _to peer " << old_inst[p->first] 
+      dout(10) << "update_heartbeat_peers: marking down old _to peer " << old_inst[p->first] 
               << " as of " << p->second << dendl;
-      // share latest map with this peer, so they know not to expect
-      // heartbeats from us.  otherwise they may mark us down!
-      _share_map_outgoing(old_inst[p->first]);
+      heartbeat_messenger->mark_down(old_inst[p->first].addr);
+
+      if (!osdmap->is_down(p->first) &&
+         osdmap->get_hb_inst(p->first) == old_inst[p->first]) {
+       dout(10) << "update_heartbeat_peers: sharing map with old _to peer " << old_inst[p->first] 
+                << " as of " << p->second << dendl;
+       // share latest map with this peer (via the cluster link, NOT
+       // the heartbeat link), so they know not to expect heartbeats
+       // from us.  otherwise they may mark us down!
+       _share_map_outgoing(osdmap->get_inst(p->first));
+      }
     }
   }
   for (map<int,epoch_t>::iterator p = old_from.begin();