objecter: resend all writes after osdmap loses the full flag

Now that the osd does not respond if it gets a map with the full flag
set first, clients need to resend all writes.

Clients talking to old osds are still subject to the race condition,
so both sides must be upgraded to avoid it.

Refs: #6938
Backport: dumpling, emperor
Signed-off-by: Josh Durgin <josh.durgin@inktank.com>

diff --git a/src/osdc/Objecter.cc b/src/osdc/Objecter.cc
index 18e313ece739aee1f01b0ac958716a6032beaa52..4083e93fe34625d5e782ed2ffffdfb90ed9f02d0 100644 (file)
--- a/src/osdc/Objecter.cc
+++ b/src/osdc/Objecter.cc
@@ -447,7 +447,8 @@ void Objecter::dispatch(Message *m)
   }
 }
 
-void Objecter::scan_requests(bool skipped_map,
+void Objecter::scan_requests(bool force_resend,
+                            bool force_resend_writes,
                             map<tid_t, Op*>& need_resend,
                             list<LingerOp*>& need_resend_linger,
                             map<tid_t, CommandOp*>& need_resend_command)
@@ -461,8 +462,7 @@ void Objecter::scan_requests(bool skipped_map,
     int r = recalc_linger_op_target(op);
     switch (r) {
     case RECALC_OP_TARGET_NO_ACTION:
-      // resend if skipped map; otherwise do nothing.
-      if (!skipped_map)
+      if (!force_resend && !force_resend_writes)
        break;
       // -- fall-thru --
     case RECALC_OP_TARGET_NEED_RESEND:
@@ -484,8 +484,8 @@ void Objecter::scan_requests(bool skipped_map,
     int r = recalc_op_target(op);
     switch (r) {
     case RECALC_OP_TARGET_NO_ACTION:
-      // resend if skipped map; otherwise do nothing.
-      if (!skipped_map)
+      if (!force_resend &&
+         (!force_resend_writes || !(op->flags & CEPH_OSD_FLAG_WRITE)))
        break;
       // -- fall-thru --
     case RECALC_OP_TARGET_NEED_RESEND:
@@ -508,7 +508,7 @@ void Objecter::scan_requests(bool skipped_map,
     switch (r) {
     case RECALC_OP_TARGET_NO_ACTION:
       // resend if skipped map; otherwise do nothing.
-      if (!skipped_map)
+      if (!force_resend && !force_resend_writes)
        break;
       // -- fall-thru --
     case RECALC_OP_TARGET_NEED_RESEND:
@@ -537,8 +537,9 @@ void Objecter::handle_osd_map(MOSDMap *m)
   }
 
   bool was_pauserd = osdmap->test_flag(CEPH_OSDMAP_PAUSERD);
-  bool was_pausewr = osdmap->test_flag(CEPH_OSDMAP_PAUSEWR) || osdmap->test_flag(CEPH_OSDMAP_FULL);
-  
+  bool was_full = osdmap->test_flag(CEPH_OSDMAP_FULL);
+  bool was_pausewr = osdmap->test_flag(CEPH_OSDMAP_PAUSEWR) || was_full;
+
   list<LingerOp*> need_resend_linger;
   map<tid_t, Op*> need_resend;
   map<tid_t, CommandOp*> need_resend_command;
@@ -586,8 +587,10 @@ void Objecter::handle_osd_map(MOSDMap *m)
          continue;
        }
        logger->set(l_osdc_map_epoch, osdmap->get_epoch());
-       
-       scan_requests(skipped_map, need_resend, need_resend_linger, need_resend_command);
+
+       was_full = was_full || osdmap->test_flag(CEPH_OSDMAP_FULL);
+       scan_requests(skipped_map, was_full, need_resend, need_resend_linger,
+                     need_resend_command);
 
        // osd addr changes?
        for (map<int,OSDSession*>::iterator p = osd_sessions.begin();
@@ -611,7 +614,8 @@ void Objecter::handle_osd_map(MOSDMap *m)
        ldout(cct, 3) << "handle_osd_map decoding full epoch " << m->get_last() << dendl;
        osdmap->decode(m->maps[m->get_last()]);
 
-       scan_requests(false, need_resend, need_resend_linger, need_resend_command);
+       scan_requests(false, false, need_resend, need_resend_linger,
+                     need_resend_command);
       } else {
        ldout(cct, 3) << "handle_osd_map hmm, i want a full map, requesting" << dendl;
        monc->sub_want("osdmap", 0, CEPH_SUBSCRIBE_ONETIME);

diff --git a/src/osdc/Objecter.h b/src/osdc/Objecter.h
index 3f9c7f2a5f92067bcc186117d69d0939dd27c5fe..8774a3ab5af43798506129b0ff6d20f4a2710708 100644 (file)
--- a/src/osdc/Objecter.h
+++ b/src/osdc/Objecter.h
@@ -1326,7 +1326,8 @@ public:
   void set_honor_cache_redirects() { honor_cache_redirects = true; }
   void unset_honor_cache_redirects() { honor_cache_redirects = false; }
 
-  void scan_requests(bool skipped_map,
+  void scan_requests(bool force_resend,
+                    bool force_resend_writes,
                     map<tid_t, Op*>& need_resend,
                     list<LingerOp*>& need_resend_linger,
                     map<tid_t, CommandOp*>& need_resend_command);