ReplicatedPG: refactor OpContext to use ObcLockManager

Signed-off-by: Samuel Just <sjust@redhat.com>

diff --git a/src/osd/ReplicatedPG.cc b/src/osd/ReplicatedPG.cc
index e0c2718698175f56747311a2b2e858d814c3b647..1cdaa4561a97f53ec0c9035d3e20e042bedfd2a1 100644 (file)
--- a/src/osd/ReplicatedPG.cc
+++ b/src/osd/ReplicatedPG.cc
@@ -3301,28 +3301,12 @@ ReplicatedPG::OpContextUPtr ReplicatedPG::trim_object(const hobject_t &coid)
   }
   assert(obc->ssc);
 
-  if (!obc->get_snaptrimmer_write()) {
-    dout(10) << __func__ << ": Unable to get a wlock on " << coid << dendl;
-    return NULL;
-  }
-
   hobject_t snapoid(
     coid.oid, coid.get_key(),
     obc->ssc->snapset.head_exists ? CEPH_NOSNAP:CEPH_SNAPDIR, coid.get_hash(),
     info.pgid.pool(), coid.get_namespace());
   ObjectContextRef snapset_obc = get_object_context(snapoid, false);
 
-  if (!snapset_obc->get_snaptrimmer_write()) {
-    dout(10) << __func__ << ": Unable to get a wlock on " << snapoid << dendl;
-    list<OpRequestRef> to_wake;
-    bool requeue_recovery = false;
-    bool requeue_snaptrimmer = false;
-    obc->put_write(&to_wake, &requeue_recovery, &requeue_snaptrimmer);
-    assert(to_wake.empty());
-    assert(!requeue_recovery);
-    return NULL;
-  }
-
   object_info_t &coi = obc->obs.oi;
   set<snapid_t> old_snaps(coi.snaps.begin(), coi.snaps.end());
   if (old_snaps.empty()) {
@@ -3359,8 +3343,23 @@ ReplicatedPG::OpContextUPtr ReplicatedPG::trim_object(const hobject_t &coid)
 
   OpContextUPtr ctx = simple_opc_create(obc);
   ctx->snapset_obc = snapset_obc;
-  ctx->lock_to_release = OpContext::W_LOCK;
-  ctx->release_snapset_obc = true;
+
+  if (!ctx->lock_manager.get_snaptrimmer_write(
+       coid,
+       obc)) {
+    close_op_ctx(ctx.release());
+    dout(10) << __func__ << ": Unable to get a wlock on " << coid << dendl;
+    return NULL;
+  }
+
+  if (!ctx->lock_manager.get_snaptrimmer_write(
+       snapoid,
+       snapset_obc)) {
+    close_op_ctx(ctx.release());
+    dout(10) << __func__ << ": Unable to get a wlock on " << snapoid << dendl;
+    return NULL;
+  }
+
   ctx->at_version = get_next_version();
 
   PGBackend::PGTransaction *t = ctx->op_t.get();
@@ -4439,7 +4438,7 @@ int ReplicatedPG::do_osd_ops(OpContext *ctx, vector<OSDOp>& ops)
       ++ctx->num_write;
       {
        tracepoint(osd, do_osd_op_pre_try_flush, soid.oid.name.c_str(), soid.snap.val);
-       if (ctx->lock_to_release != OpContext::NONE) {
+       if (ctx->lock_type != ObjectContext::RWState::RWNONE) {
          dout(10) << "cache-try-flush without SKIPRWLOCKS flag set" << dendl;
          result = -EINVAL;
          break;
@@ -4471,7 +4470,7 @@ int ReplicatedPG::do_osd_ops(OpContext *ctx, vector<OSDOp>& ops)
       ++ctx->num_write;
       {
        tracepoint(osd, do_osd_op_pre_cache_flush, soid.oid.name.c_str(), soid.snap.val);
-       if (ctx->lock_to_release == OpContext::NONE) {
+       if (ctx->lock_type == ObjectContext::RWState::RWNONE) {
          dout(10) << "cache-flush with SKIPRWLOCKS flag set" << dendl;
          result = -EINVAL;
          break;
@@ -6261,7 +6260,10 @@ void ReplicatedPG::make_writeable(OpContext *ctx)
       if (pool.info.require_rollback())
        ctx->clone_obc->attr_cache = ctx->obc->attr_cache;
       snap_oi = &ctx->clone_obc->obs.oi;
-      bool got = ctx->clone_obc->get_write_greedy(ctx->op);
+      bool got = ctx->lock_manager.get_write_greedy(
+       coid,
+       ctx->clone_obc,
+       ctx->op);
       assert(got);
       dout(20) << " got greedy write on clone_obc " << *ctx->clone_obc << dendl;
     } else {
@@ -6600,15 +6602,21 @@ void ReplicatedPG::finish_ctx(OpContext *ctx, int log_op_type, bool maintain_ssc
       if (!ctx->snapset_obc)
        ctx->snapset_obc = get_object_context(snapoid, true);
       bool got = false;
-      if (ctx->lock_to_release == OpContext::W_LOCK) {
-       got = ctx->snapset_obc->get_write_greedy(ctx->op);
+      if (ctx->lock_type == ObjectContext::RWState::RWWRITE) {
+       got = ctx->lock_manager.get_write_greedy(
+         snapoid,
+         ctx->snapset_obc,
+         ctx->op);
       } else {
-       assert(ctx->lock_to_release == OpContext::E_LOCK);
-       got = ctx->snapset_obc->get_excl(ctx->op);
+       assert(ctx->lock_type == ObjectContext::RWState::RWEXCL);
+       got = ctx->lock_manager.get_lock_type(
+         ObjectContext::RWState::RWEXCL,
+         snapoid,
+         ctx->snapset_obc,
+         ctx->op);
       }
       assert(got);
       dout(20) << " got greedy write on snapset_obc " << *ctx->snapset_obc << dendl;
-      ctx->release_snapset_obc = true;
       if (pool.info.require_rollback() && !ctx->snapset_obc->obs.exists) {
        ctx->log.back().mod_desc.create();
       } else if (!pool.info.require_rollback()) {
@@ -7548,10 +7556,11 @@ void ReplicatedPG::finish_promote(int r, CopyResults *results,
     tctx->new_snapset.clone_size.erase(soid.snap);
 
     // take RWWRITE lock for duration of our local write.  ignore starvation.
-    if (!obc->rwstate.take_write_lock()) {
+    if (!tctx->lock_manager.take_write_lock(
+         head,
+         obc)) {
       assert(0 == "problem!");
     }
-    tctx->lock_to_release = OpContext::W_LOCK;
     dout(20) << __func__ << " took lock on obc, " << obc->rwstate << dendl;
 
     finish_ctx(tctx.get(), pg_log_entry_t::PROMOTE);
@@ -7646,10 +7655,11 @@ void ReplicatedPG::finish_promote(int r, CopyResults *results,
   dout(20) << __func__ << " new_snapset " << tctx->new_snapset << dendl;
 
   // take RWWRITE lock for duration of our local write.  ignore starvation.
-  if (!obc->rwstate.take_write_lock()) {
+  if (!tctx->lock_manager.take_write_lock(
+       obc->obs.oi.soid,
+       obc)) {
     assert(0 == "problem!");
   }
-  tctx->lock_to_release = OpContext::W_LOCK;
   dout(20) << __func__ << " took lock on obc, " << obc->rwstate << dendl;
 
   finish_ctx(tctx.get(), pg_log_entry_t::PROMOTE);
@@ -8063,30 +8073,37 @@ int ReplicatedPG::try_flush_mark_clean(FlushOpRef fop)
     return 0;
   }
 
+  dout(10) << __func__ << " clearing DIRTY flag for " << oid << dendl;
+  OpContextUPtr ctx = simple_opc_create(fop->obc);
+
   // successfully flushed; can we clear the dirty bit?
   // try to take the lock manually, since we don't
   // have a ctx yet.
-  if (obc->get_write(fop->op)) {
+  if (ctx->lock_manager.get_lock_type(
+       ObjectContext::RWState::RWWRITE,
+       oid,
+       obc,
+       fop->op)) {
     dout(20) << __func__ << " took write lock" << dendl;
   } else if (fop->op) {
     dout(10) << __func__ << " waiting on write lock" << dendl;
+    close_op_ctx(ctx.release());
     requeue_op(fop->op);
     requeue_ops(fop->dup_ops);
     return -EAGAIN;    // will retry
   } else {
     dout(10) << __func__ << " failed write lock, no op; failing" << dendl;
+    close_op_ctx(ctx.release());
     osd->logger->inc(l_osd_tier_try_flush_fail);
     cancel_flush(fop, false);
     return -ECANCELED;
   }
 
-  dout(10) << __func__ << " clearing DIRTY flag for " << oid << dendl;
-  OpContextUPtr ctx = simple_opc_create(fop->obc);
-
-  ctx->register_on_finish(*(fop->on_flush));
-  fop->on_flush = boost::none;
+  if (fop->on_flush) {
+    ctx->register_on_finish(*(fop->on_flush));
+    fop->on_flush = boost::none;
+  }
 
-  ctx->lock_to_release = OpContext::W_LOCK;  // we took it above
   ctx->at_version = get_next_version();
 
   ctx->new_obs = obc->obs;
@@ -11866,21 +11883,25 @@ bool ReplicatedPG::agent_maybe_evict(ObjectContextRef& obc, bool after_flush)
       return false;
   }
 
-  if (!obc->get_write(OpRequestRef())) {
+  dout(10) << __func__ << " evicting " << obc->obs.oi << dendl;
+  OpContextUPtr ctx = simple_opc_create(obc);
+
+  if (!ctx->lock_manager.get_lock_type(
+       ObjectContext::RWState::RWWRITE,
+       obc->obs.oi.soid,
+       obc,
+       OpRequestRef())) {
+    close_op_ctx(ctx.release());
     dout(20) << __func__ << " skip (cannot get lock) " << obc->obs.oi << dendl;
     return false;
   }
 
-  dout(10) << __func__ << " evicting " << obc->obs.oi << dendl;
-  OpContextUPtr ctx = simple_opc_create(obc);
-
   osd->agent_start_evict_op();
   ctx->register_on_finish(
     [this]() {
       osd->agent_finish_evict_op();
     });
 
-  ctx->lock_to_release = OpContext::W_LOCK;
   ctx->at_version = get_next_version();
   assert(ctx->new_obs.exists);
   int r = _delete_oid(ctx.get(), true);

diff --git a/src/osd/ReplicatedPG.h b/src/osd/ReplicatedPG.h
index 2e09ebd1b8ce0b782ab4546b979ca5c7730bd525..4d19eb5c2d086c2237b99b0d7290a445b8abb459 100644 (file)
--- a/src/osd/ReplicatedPG.h
+++ b/src/osd/ReplicatedPG.h
@@ -383,6 +383,7 @@ public:
     map<string, bufferlist> &attrs) {
     return get_object_context(hoid, true, &attrs);
   }
+
   void log_operation(
     const vector<pg_log_entry_t> &logv,
     boost::optional<pg_hit_set_history_t> &hset_history,
@@ -622,13 +623,12 @@ public:
 
     ObjectModDesc mod_desc;
 
-    enum { W_LOCK, R_LOCK, E_LOCK, NONE } lock_to_release;
+    ObjectContext::RWState::State lock_type;
+    ObcLockManager lock_manager;
 
     OpContext(const OpContext& other);
     const OpContext& operator=(const OpContext& other);
 
-    bool release_snapset_obc;
-
     OpContext(OpRequestRef _op, osd_reqid_t _reqid, vector<OSDOp>& _ops,
              ObjectContextRef& obc,
              ReplicatedPG *_pg) :
@@ -648,8 +648,7 @@ public:
       sent_ack(false), sent_disk(false),
       async_read_result(0),
       inflightreads(0),
-      lock_to_release(NONE),
-      release_snapset_obc(false) {
+      lock_type(ObjectContext::RWState::RWNONE) {
       if (obc->ssc) {
        new_snapset = obc->ssc->snapset;
        snapset = &obc->ssc->snapset;
@@ -668,8 +667,7 @@ public:
       copy_cb(NULL),
       async_read_result(0),
       inflightreads(0),
-      lock_to_release(NONE),
-      release_snapset_obc(false) { }
+      lock_type(ObjectContext::RWState::RWNONE) {}
     void reset_obs(ObjectContextRef obc) {
       new_obs = ObjectState(obc->obs.oi, obc->obs.exists);
       if (obc->ssc) {
@@ -679,7 +677,7 @@ public:
     }
     ~OpContext() {
       assert(!op_t);
-      assert(lock_to_release == NONE);
+      assert(lock_type == ObjectContext::RWState::RWNONE);
       if (reply)
        reply->put();
       for (list<pair<boost::tuple<uint64_t, uint64_t, unsigned>,
@@ -768,33 +766,35 @@ protected:
      * this (read or write) if we get the first we will be guaranteed
      * to get the second.
      */
-    ObjectContext::RWState::State type = ObjectContext::RWState::RWNONE;
     if (write_ordered && ctx->op->may_read()) {
-      type = ObjectContext::RWState::RWEXCL;
-      ctx->lock_to_release = OpContext::E_LOCK;
+      ctx->lock_type = ObjectContext::RWState::RWEXCL;
     } else if (write_ordered) {
-      type = ObjectContext::RWState::RWWRITE;
-      ctx->lock_to_release = OpContext::W_LOCK;
+      ctx->lock_type = ObjectContext::RWState::RWWRITE;
     } else {
       assert(ctx->op->may_read());
-      type = ObjectContext::RWState::RWREAD;
-      ctx->lock_to_release = OpContext::R_LOCK;
+      ctx->lock_type = ObjectContext::RWState::RWREAD;
     }
 
     if (ctx->snapset_obc) {
       assert(!ctx->obc->obs.exists);
-      if (ctx->snapset_obc->get_lock_type(ctx->op, type)) {
-       ctx->release_snapset_obc = true;
-      } else {
-       ctx->lock_to_release = OpContext::NONE;
+      if (!ctx->lock_manager.get_lock_type(
+           ctx->lock_type,
+           ctx->snapset_obc->obs.oi.soid,
+           ctx->snapset_obc,
+           ctx->op)) {
+       ctx->lock_type = ObjectContext::RWState::RWNONE;
        return false;
       }
     }
-    if (ctx->obc->get_lock_type(ctx->op, type)) {
+    if (ctx->lock_manager.get_lock_type(
+         ctx->lock_type,
+         ctx->obc->obs.oi.soid,
+         ctx->obc,
+         ctx->op)) {
       return true;
     } else {
       assert(!ctx->snapset_obc);
-      ctx->lock_to_release = OpContext::NONE;
+      ctx->lock_type = ObjectContext::RWState::RWNONE;
       return false;
     }
   }
@@ -823,74 +823,23 @@ protected:
   void release_op_ctx_locks(OpContext *ctx) {
     list<OpRequestRef> to_req;
     bool requeue_recovery = false;
-    bool requeue_recovery_clone = false;
-    bool requeue_recovery_snapset = false;
-    bool requeue_snaptrimmer = false;
-    bool requeue_snaptrimmer_clone = false;
-    bool requeue_snaptrimmer_snapset = false;
-    switch (ctx->lock_to_release) {
-    case OpContext::W_LOCK:
-      if (ctx->snapset_obc && ctx->release_snapset_obc) {
-       ctx->snapset_obc->put_write(
-         &to_req,
-         &requeue_recovery_snapset,
-         &requeue_snaptrimmer_snapset);
-       ctx->release_snapset_obc = false;
-      }
-      ctx->obc->put_write(
-       &to_req,
-       &requeue_recovery,
-       &requeue_snaptrimmer);
-      if (ctx->clone_obc)
-       ctx->clone_obc->put_write(
-         &to_req,
-         &requeue_recovery_clone,
-         &requeue_snaptrimmer_clone);
-      break;
-    case OpContext::E_LOCK:
-      if (ctx->snapset_obc && ctx->release_snapset_obc) {
-       ctx->snapset_obc->put_excl(
-         &to_req,
-         &requeue_recovery_snapset,
-         &requeue_snaptrimmer_snapset);
-       ctx->release_snapset_obc = false;
-      }
-      ctx->obc->put_excl(
-       &to_req,
-       &requeue_recovery,
-       &requeue_snaptrimmer);
-      if (ctx->clone_obc)
-       ctx->clone_obc->put_write(
-         &to_req,
-         &requeue_recovery_clone,
-         &requeue_snaptrimmer_clone);
-      break;
-    case OpContext::R_LOCK:
-      if (ctx->snapset_obc && ctx->release_snapset_obc) {
-       ctx->snapset_obc->put_read(&to_req);
-       ctx->release_snapset_obc = false;
-      }
-      ctx->obc->put_read(&to_req);
-      break;
-    case OpContext::NONE:
-      break;
-    default:
-      assert(0);
-    };
-    assert(ctx->release_snapset_obc == false);
-    ctx->lock_to_release = OpContext::NONE;
-    if (requeue_recovery || requeue_recovery_clone || requeue_recovery_snapset)
+    bool requeue_snaptrim = false;
+    ctx->lock_manager.put_locks(
+      &to_req,
+      &requeue_recovery,
+      &requeue_snaptrim);
+    ctx->lock_type = ObjectContext::RWState::RWNONE;
+    if (requeue_recovery)
       osd->recovery_wq.queue(this);
-    if (requeue_snaptrimmer ||
-       requeue_snaptrimmer_clone ||
-       requeue_snaptrimmer_snapset)
+    if (requeue_snaptrim)
       queue_snap_trim();
 
     if (!to_req.empty()) {
       assert(ctx->obc);
       // requeue at front of scrub blocking queue if we are blocked by scrub
-      if (scrubber.write_blocked_by_scrub(ctx->obc->obs.oi.soid.get_head(),
-                                         get_sort_bitwise())) {
+      if (scrubber.write_blocked_by_scrub(
+           ctx->obc->obs.oi.soid.get_head(),
+           get_sort_bitwise())) {
        waiting_for_active.splice(
          waiting_for_active.begin(),
          to_req,
@@ -1687,8 +1636,8 @@ inline ostream& operator<<(ostream& out, ReplicatedPG::RepGather& repop)
       << " rep_tid=" << repop.rep_tid 
       << " committed?=" << repop.all_committed
       << " applied?=" << repop.all_applied;
-  if (repop.ctx->lock_to_release != ReplicatedPG::OpContext::NONE)
-    out << " lock=" << (int)repop.ctx->lock_to_release;
+  if (repop.ctx->lock_type != ObjectContext::RWState::RWNONE)
+    out << " lock=" << (int)repop.ctx->lock_type;
   if (repop.ctx->op)
     out << " op=" << *(repop.ctx->op->get_req());
   out << ")";