ReplicatePG,PG: SnapTrimmer state machine skeleton

Currently, snap_trimmer does not wait for replicas to apply the object
removal repops before updating the info with the removed snapshot and
sending out infos.  Thus, there is a race between the replica applying
the object removal transactions and recieving the info prompting it to
remove its local snap collection.  In the cases where the info beats the
transaction to the file system, we get the infamous -39 ENOTEMPTY error
crashing the OSD.

One solution would be to block in snap_trimmer until we get the
responses.  This would, unfortunately, tie up a disk_tp thread.  Moving
to a state machine would:
a) guarrantee that the snap_trimmer state is cleaned up when the pg
resets (using on_change())
b) remove the necessity of blocking in a disk_tp thread (we currently
unlock and relock the pg lock between operations causing us to block on
another thread using the pg).  Rather than unlocking and relocking, we
would instead unlock and requeue ourselves in the snap_trim_wq while we
wait allowing the thread to do something else.

Signed-off-by: Samuel Just <samuel.just@dreamhost.com>

diff --git a/src/osd/PG.h b/src/osd/PG.h
index 6549b47427f2d87047e344426153ddb28d96a776..4b67aae27003ec59220239767470da4eaff3869c 100644 (file)
--- a/src/osd/PG.h
+++ b/src/osd/PG.h
@@ -860,6 +860,15 @@ public:
        context_list(context_list), transaction(transaction) {}
   };
 
+  struct NamedState {
+    const char *state_name;
+    utime_t enter_time;
+    const char *get_state_name() { return state_name; }
+    NamedState() : enter_time(ceph_clock_now(g_ceph_context)) {}
+    virtual ~NamedState() {}
+  };
+
+
   /* Encapsulates PG recovery process */
   class RecoveryState {
     void start_handle(RecoveryCtx *new_ctx) {

diff --git a/src/osd/ReplicatedPG.cc b/src/osd/ReplicatedPG.cc
index 0c5d2281939d892543c61eaf1b74996c4aa93a6c..d71e17b4eef6525ff1260e4be30bedd02dccfcba 100644 (file)
--- a/src/osd/ReplicatedPG.cc
+++ b/src/osd/ReplicatedPG.cc
@@ -4319,6 +4319,9 @@ void ReplicatedPG::on_change()
   pushing.clear();
   pulling.clear();
   pull_from_peer.clear();
+
+  // clear snap_trimmer state
+  snap_trimmer_machine.process_event(SnapTrimmer::Reset());
 }
 
 void ReplicatedPG::on_role_change()
@@ -4872,3 +4875,70 @@ int ReplicatedPG::_scrub(ScrubMap& scrubmap, int& errors, int& fixed)
   dout(10) << "_scrub (" << mode << ") finish" << dendl;
   return errors;
 }
+
+/*---SnapTrimmer states---*/
+#undef dout_prefix
+#define dout_prefix (*_dout << context< SnapTrimmer >().pg->gen_prefix() \
+                    << "SnapTrimmer state<" << get_state_name() << ">: ")
+
+/* NotTrimming */
+ReplicatedPG::SnapTrimmer::NotTrimming::NotTrimming(my_context ctx) : my_base(ctx)
+{
+  state_name = "Started/Primary/Active/NotTrimming";
+  //context< SnapTrimmer >().log_enter(state_name);
+}
+
+void ReplicatedPG::SnapTrimmer::NotTrimming::exit()
+{
+  //context< SnapTrimmer >().log_exit(state_name, enter_time);
+}
+
+boost::statechart::result ReplicatedPG::SnapTrimmer::NotTrimming::react(const SnapTrim&)
+{
+  return discard_event();
+}
+
+/* Trimming */
+ReplicatedPG::SnapTrimmer::Trimming::Trimming(my_context ctx) : my_base(ctx)
+{
+  state_name = "Started/Primary/Active/Trimming";
+  //context< SnapTrimmer >().log_enter(state_name);
+}
+
+void ReplicatedPG::SnapTrimmer::Trimming::exit()
+{
+  //context< SnapTrimmer >().log_exit(state_name, enter_time);
+}
+
+/* TrimmingObjects */
+ReplicatedPG::SnapTrimmer::TrimmingObjects::TrimmingObjects(my_context ctx) : my_base(ctx)
+{
+  state_name = "Started/Primary/Active/Trimming/TrimmingObjects";
+  //context< SnapTrimmer >().log_enter(state_name);
+}
+
+void ReplicatedPG::SnapTrimmer::TrimmingObjects::exit()
+{
+  //context< SnapTrimmer >().log_exit(state_name, enter_time);
+}
+
+boost::statechart::result ReplicatedPG::SnapTrimmer::TrimmingObjects::react(const SnapTrim&)
+{
+  return discard_event();
+}
+/* WaitingOnReplicasObjects */
+ReplicatedPG::SnapTrimmer::WaitingOnReplicas::WaitingOnReplicas(my_context ctx) : my_base(ctx)
+{
+  state_name = "Started/Primary/Active/Trimming/WaitingOnReplicas";
+  //context< SnapTrimmer >().log_enter(state_name);
+}
+
+void ReplicatedPG::SnapTrimmer::WaitingOnReplicas::exit()
+{
+  //context< SnapTrimmer >().log_exit(state_name, enter_time);
+}
+
+boost::statechart::result ReplicatedPG::SnapTrimmer::WaitingOnReplicas::react(const SnapTrim&)
+{
+  return discard_event();
+}

diff --git a/src/osd/ReplicatedPG.h b/src/osd/ReplicatedPG.h
index 345b24d83655e2ca248baf349f5ccdac122bcf6c..640658c6055adc4a9c8d58ea499fb08cefabbf1d 100644 (file)
--- a/src/osd/ReplicatedPG.h
+++ b/src/osd/ReplicatedPG.h
@@ -665,7 +665,7 @@ protected:
 
 public:
   ReplicatedPG(OSD *o, PGPool *_pool, pg_t p, const sobject_t& oid, const sobject_t& ioid) : 
-    PG(o, _pool, p, oid, ioid)
+    PG(o, _pool, p, oid, ioid), snap_trimmer_machine(this)
   { }
   ~ReplicatedPG() {}
 
@@ -683,6 +683,57 @@ public:
                 bufferlist& odata);
   void do_osd_op_effects(OpContext *ctx);
 private:
+  class NotTrimming;
+  class SnapTrimmer : public boost::statechart::state_machine< SnapTrimmer, NotTrimming > {
+    ReplicatedPG *pg;
+
+  public:
+    SnapTrimmer(ReplicatedPG *pg) : pg(pg) {}
+
+    struct SnapTrim : boost::statechart::event< SnapTrim > {};
+    struct Reset : boost::statechart::event< SnapTrim > {};
+  private:
+    struct NotTrimming : boost::statechart::state< NotTrimming, SnapTrimmer >, NamedState {
+      typedef boost::mpl::list <
+       boost::statechart::custom_reaction< SnapTrim >
+       > reactions;
+      NotTrimming(my_context ctx);
+      void exit();
+      boost::statechart::result react(const SnapTrim&);
+    };
+
+    struct TrimmingObjects;
+    struct RepGather;
+    struct Trimming : boost::statechart::state< Trimming, SnapTrimmer, TrimmingObjects >, NamedState {
+      typedef boost::mpl::list <
+       boost::statechart::transition< Reset, NotTrimming >
+       > reactions;
+      Trimming(my_context ctx);
+      void exit();
+      set< RepGather* > pending_repops;
+      set< sobject_t > remaining_objects;
+      coll_t col_trimming;
+      snapid_t snap_trimming;
+    };
+
+    struct TrimmingObjects : boost::statechart::state< TrimmingObjects, Trimming >, NamedState {
+      typedef boost::mpl::list <
+       boost::statechart::custom_reaction< SnapTrim >
+       > reactions;
+      TrimmingObjects(my_context ctx);
+      void exit();
+      boost::statechart::result react(const SnapTrim&);
+    };
+
+    struct WaitingOnReplicas : boost::statechart::state< WaitingOnReplicas, Trimming >, NamedState {
+      typedef boost::mpl::list <
+       boost::statechart::custom_reaction< SnapTrim >
+       > reactions;
+      WaitingOnReplicas(my_context ctx);
+      void exit();
+      boost::statechart::result react(const SnapTrim&);
+    };
+  } snap_trimmer_machine;
   void _delete_head(OpContext *ctx);
   int _rollback_to(OpContext *ctx, ceph_osd_op& op);
 public: