PG: track purged pgs during active

See bug #2462.

The following sequence could cause a log assuming a non-empty pg
to an empty replica:

1. primary sends query to stray
2. stray sends notify to primary
3. primary sends purge to stray removing stray from peer_info
4. stray recieves query and sends a notify
5. stray recieves purge and purges its pg
6. primary recieves notify from stray and adds it to peer_info
	note: peer_info[stray] is now wrong
7. acting set changes, primary is still primary, stray is replica
8. primary sends log to replica based on incorrect info from 6.

This patch adds a purged_peer set which is populated during purge_strays
and cleared during start_peering_interval.  The primary will ignore
notifies from the peer once the peer is in this set.

Signed-off-by: Samuel Just <sam.just@inktank.com>

diff --git a/src/osd/PG.cc b/src/osd/PG.cc
index 0a0c82344856ef09838f8a529faafaf5132b8126..abf33d204ba7e334264e2d963ce6cbc0f1aa71e7 100644 (file)
--- a/src/osd/PG.cc
+++ b/src/osd/PG.cc
@@ -1707,6 +1707,7 @@ void PG::purge_strays()
       dout(10) << "not sending PGRemove to down osd." << *p << dendl;
     }
     peer_info.erase(*p);
+    peer_purged.insert(*p);
     removed = true;
   }
 
@@ -3596,6 +3597,7 @@ void PG::start_peering_interval(const OSDMapRef lastmap,
   state_clear(PG_STATE_RECOVERING);
 
   peer_missing.clear();
+  peer_purged.clear();
 
   // reset primary state?
   if (oldrole == 0 || get_role() == 0)
@@ -4147,6 +4149,10 @@ boost::statechart::result PG::RecoveryState::Active::react(const MNotifyRec& not
     dout(10) << "Active: got notify from " << notevt.from 
             << ", already have info from that osd, ignoring" 
             << dendl;
+  } else if (pg->peer_purged.count(notevt.from)) {
+    dout(10) << "Active: got notify from " << notevt.from
+            << ", already purged that peer, ignoring"
+            << dendl;
   } else {
     dout(10) << "Active: got notify from " << notevt.from 
             << ", calling proc_replica_info and discover_all_missing"

diff --git a/src/osd/PG.h b/src/osd/PG.h
index eec356775bef3e5c071c3483170effd8fde75493..fa4be911298a6d902594e3a83e31f8698d972005 100644 (file)
--- a/src/osd/PG.h
+++ b/src/osd/PG.h
@@ -508,6 +508,7 @@ protected:
   set<int>    stray_set;   // non-acting osds that have PG data.
   eversion_t  oldest_update; // acting: lowest (valid) last_update in active set
   map<int,pg_info_t>    peer_info;   // info from peers (stray or prior)
+  set<int> peer_purged; // peers purged
   map<int,pg_missing_t> peer_missing;
   set<int>             peer_log_requested;  // logs i've requested (and start stamps)
   set<int>             peer_missing_requested;