ReplicatedPG: copy: conditionally requeue copy ops when cancelled

We may need to requeue copy ops which are cancelled as part of an acting
set change but don't change the primary. To support this, add a
"requeue" flag to cancel_copy_ops() and copy_ops(), as well as to
CopyResults. The CopyCallback is then responsible for requeuing (the
higher layers can't do so as they can't know which request actually
triggered the copy).

Signed-off-by: Greg Farnum <greg@inktank.com>

diff --git a/src/osd/ReplicatedPG.cc b/src/osd/ReplicatedPG.cc
index c4dccf68442fa894f3bde8a5cf902171e72502d6..fd4ffb77485b62746b7f270bc6c16e05fd6336b0 100644 (file)
--- a/src/osd/ReplicatedPG.cc
+++ b/src/osd/ReplicatedPG.cc
@@ -4336,7 +4336,7 @@ int ReplicatedPG::start_copy(CopyCallback *cb, ObjectContextRef obc,
     // FIXME: if the src etc match, we could avoid restarting from the
     // beginning.
     CopyOpRef cop = copy_ops[dest];
-    cancel_copy(cop);
+    cancel_copy(cop, false);
   }
 
   CopyOpRef cop(new CopyOp(cb, obc, src, oloc, version, temp_dest_oid));
@@ -4429,6 +4429,7 @@ void ReplicatedPG::process_copy_chunk(hobject_t oid, tid_t tid, int r)
 
   dout(20) << __func__ << " complete; committing" << dendl;
   results.get<0>() = r;
+  results.get<4>() = false;
   cop->cb->complete(results);
 
   copy_ops.erase(obc->obs.oi.soid);
@@ -4514,7 +4515,7 @@ int ReplicatedPG::finish_copyfrom(OpContext *ctx)
   return 0;
 }
 
-void ReplicatedPG::cancel_copy(CopyOpRef cop)
+void ReplicatedPG::cancel_copy(CopyOpRef cop, bool requeue)
 {
   dout(10) << __func__ << " " << cop->obc->obs.oi.soid
           << " from " << cop->src << " " << cop->oloc << " v" << cop->version
@@ -4531,16 +4532,18 @@ void ReplicatedPG::cancel_copy(CopyOpRef cop)
 
   kick_object_context_blocked(cop->obc);
   bool temp_obj_created = !cop->cursor.is_initial();
-  CopyResults result(-ECANCELED, 0, temp_obj_created, ObjectStore::Transaction());
+  CopyResults result(-ECANCELED, 0, temp_obj_created,
+                     ObjectStore::Transaction(), requeue);
   cop->cb->complete(result);
 }
 
-void ReplicatedPG::cancel_copy_ops()
+void ReplicatedPG::cancel_copy_ops(bool requeue)
 {
   dout(10) << __func__ << dendl;
   map<hobject_t,CopyOpRef>::iterator p = copy_ops.begin();
   while (p != copy_ops.end()) {
-    cancel_copy((p++)->second);
+    // requeue this op? can I queue up all of them?
+    cancel_copy((p++)->second, requeue);
   }
 }
 
@@ -7292,7 +7295,7 @@ void ReplicatedPG::on_shutdown()
   deleting = true;
 
   unreg_next_scrub();
-  cancel_copy_ops();
+  cancel_copy_ops(false);
   apply_and_flush_repops(false);
   context_registry_on_change();
 
@@ -7329,7 +7332,7 @@ void ReplicatedPG::on_change(ObjectStore::Transaction *t)
 
   context_registry_on_change();
 
-  cancel_copy_ops();
+  cancel_copy_ops(is_primary());
 
   // requeue object waiters
   if (is_primary()) {

diff --git a/src/osd/ReplicatedPG.h b/src/osd/ReplicatedPG.h
index 1292780d0440f578ba0dcf7ecd4f5b9c11fbd9c0..00216170516979a7ebc1e5ef6c2292a8110f3457 100644 (file)
--- a/src/osd/ReplicatedPG.h
+++ b/src/osd/ReplicatedPG.h
@@ -140,8 +140,8 @@ public:
    * transactions would not allow); if you are doing the copy for a read
    * op you will have to generate a separate op to finish the copy with.
    */
-  /// return code, total object size, data in temp object?, final Transaction
-  typedef boost::tuple<int, size_t, bool, ObjectStore::Transaction> CopyResults;
+  /// return code, total object size, data in temp object?, final Transaction, should requeue Op
+  typedef boost::tuple<int, size_t, bool, ObjectStore::Transaction, bool> CopyResults;
   class CopyCallback : public GenContext<CopyResults&> {
   protected:
     CopyCallback() {}
@@ -155,6 +155,9 @@ public:
      * results.get<3>() is a Transaction; if non-empty you need to perform
      * its results before any other accesses to the object in order to
      * complete the copy.
+     * results.get<4>() is a bool; if true you must requeue the client Op
+     * after processing the rest of the results (this will only be true
+     * in conjunction with an ECANCELED return code).
      */
     virtual void finish(CopyResults& results_) = 0;
 
@@ -182,6 +185,8 @@ public:
       if (r < 0) {
        if (r != -ECANCELED) { // on cancel just toss it out; client resends
          ctx->pg->osd->reply_op_error(ctx->op, r);
+       } else if (results_.get<4>()) {
+         ctx->pg->requeue_op(ctx->op);
        }
        ctx->pg->close_op_ctx(ctx);
       }
@@ -978,8 +983,8 @@ protected:
   void _build_finish_copy_transaction(CopyOpRef cop,
                                       ObjectStore::Transaction& t);
   int finish_copyfrom(OpContext *ctx);
-  void cancel_copy(CopyOpRef cop);
-  void cancel_copy_ops();
+  void cancel_copy(CopyOpRef cop, bool requeue);
+  void cancel_copy_ops(bool requeue);
 
   friend class C_Copyfrom;