From: Samuel Just Date: Wed, 13 Jan 2016 19:29:24 +0000 (-0800) Subject: ReplicatedPG: refactor OpContext to use ObcLockManager X-Git-Tag: v10.1.0~277^2~11 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=e68b5507ecfbb24500b3fc86c40eeaa378f014bb;p=ceph.git ReplicatedPG: refactor OpContext to use ObcLockManager Signed-off-by: Samuel Just --- diff --git a/src/osd/ReplicatedPG.cc b/src/osd/ReplicatedPG.cc index e0c271869817..1cdaa4561a97 100644 --- a/src/osd/ReplicatedPG.cc +++ b/src/osd/ReplicatedPG.cc @@ -3301,28 +3301,12 @@ ReplicatedPG::OpContextUPtr ReplicatedPG::trim_object(const hobject_t &coid) } assert(obc->ssc); - if (!obc->get_snaptrimmer_write()) { - dout(10) << __func__ << ": Unable to get a wlock on " << coid << dendl; - return NULL; - } - hobject_t snapoid( coid.oid, coid.get_key(), obc->ssc->snapset.head_exists ? CEPH_NOSNAP:CEPH_SNAPDIR, coid.get_hash(), info.pgid.pool(), coid.get_namespace()); ObjectContextRef snapset_obc = get_object_context(snapoid, false); - if (!snapset_obc->get_snaptrimmer_write()) { - dout(10) << __func__ << ": Unable to get a wlock on " << snapoid << dendl; - list to_wake; - bool requeue_recovery = false; - bool requeue_snaptrimmer = false; - obc->put_write(&to_wake, &requeue_recovery, &requeue_snaptrimmer); - assert(to_wake.empty()); - assert(!requeue_recovery); - return NULL; - } - object_info_t &coi = obc->obs.oi; set old_snaps(coi.snaps.begin(), coi.snaps.end()); if (old_snaps.empty()) { @@ -3359,8 +3343,23 @@ ReplicatedPG::OpContextUPtr ReplicatedPG::trim_object(const hobject_t &coid) OpContextUPtr ctx = simple_opc_create(obc); ctx->snapset_obc = snapset_obc; - ctx->lock_to_release = OpContext::W_LOCK; - ctx->release_snapset_obc = true; + + if (!ctx->lock_manager.get_snaptrimmer_write( + coid, + obc)) { + close_op_ctx(ctx.release()); + dout(10) << __func__ << ": Unable to get a wlock on " << coid << dendl; + return NULL; + } + + if (!ctx->lock_manager.get_snaptrimmer_write( + snapoid, + snapset_obc)) { + close_op_ctx(ctx.release()); + dout(10) << __func__ << ": Unable to get a wlock on " << snapoid << dendl; + return NULL; + } + ctx->at_version = get_next_version(); PGBackend::PGTransaction *t = ctx->op_t.get(); @@ -4439,7 +4438,7 @@ int ReplicatedPG::do_osd_ops(OpContext *ctx, vector& ops) ++ctx->num_write; { tracepoint(osd, do_osd_op_pre_try_flush, soid.oid.name.c_str(), soid.snap.val); - if (ctx->lock_to_release != OpContext::NONE) { + if (ctx->lock_type != ObjectContext::RWState::RWNONE) { dout(10) << "cache-try-flush without SKIPRWLOCKS flag set" << dendl; result = -EINVAL; break; @@ -4471,7 +4470,7 @@ int ReplicatedPG::do_osd_ops(OpContext *ctx, vector& ops) ++ctx->num_write; { tracepoint(osd, do_osd_op_pre_cache_flush, soid.oid.name.c_str(), soid.snap.val); - if (ctx->lock_to_release == OpContext::NONE) { + if (ctx->lock_type == ObjectContext::RWState::RWNONE) { dout(10) << "cache-flush with SKIPRWLOCKS flag set" << dendl; result = -EINVAL; break; @@ -6261,7 +6260,10 @@ void ReplicatedPG::make_writeable(OpContext *ctx) if (pool.info.require_rollback()) ctx->clone_obc->attr_cache = ctx->obc->attr_cache; snap_oi = &ctx->clone_obc->obs.oi; - bool got = ctx->clone_obc->get_write_greedy(ctx->op); + bool got = ctx->lock_manager.get_write_greedy( + coid, + ctx->clone_obc, + ctx->op); assert(got); dout(20) << " got greedy write on clone_obc " << *ctx->clone_obc << dendl; } else { @@ -6600,15 +6602,21 @@ void ReplicatedPG::finish_ctx(OpContext *ctx, int log_op_type, bool maintain_ssc if (!ctx->snapset_obc) ctx->snapset_obc = get_object_context(snapoid, true); bool got = false; - if (ctx->lock_to_release == OpContext::W_LOCK) { - got = ctx->snapset_obc->get_write_greedy(ctx->op); + if (ctx->lock_type == ObjectContext::RWState::RWWRITE) { + got = ctx->lock_manager.get_write_greedy( + snapoid, + ctx->snapset_obc, + ctx->op); } else { - assert(ctx->lock_to_release == OpContext::E_LOCK); - got = ctx->snapset_obc->get_excl(ctx->op); + assert(ctx->lock_type == ObjectContext::RWState::RWEXCL); + got = ctx->lock_manager.get_lock_type( + ObjectContext::RWState::RWEXCL, + snapoid, + ctx->snapset_obc, + ctx->op); } assert(got); dout(20) << " got greedy write on snapset_obc " << *ctx->snapset_obc << dendl; - ctx->release_snapset_obc = true; if (pool.info.require_rollback() && !ctx->snapset_obc->obs.exists) { ctx->log.back().mod_desc.create(); } else if (!pool.info.require_rollback()) { @@ -7548,10 +7556,11 @@ void ReplicatedPG::finish_promote(int r, CopyResults *results, tctx->new_snapset.clone_size.erase(soid.snap); // take RWWRITE lock for duration of our local write. ignore starvation. - if (!obc->rwstate.take_write_lock()) { + if (!tctx->lock_manager.take_write_lock( + head, + obc)) { assert(0 == "problem!"); } - tctx->lock_to_release = OpContext::W_LOCK; dout(20) << __func__ << " took lock on obc, " << obc->rwstate << dendl; finish_ctx(tctx.get(), pg_log_entry_t::PROMOTE); @@ -7646,10 +7655,11 @@ void ReplicatedPG::finish_promote(int r, CopyResults *results, dout(20) << __func__ << " new_snapset " << tctx->new_snapset << dendl; // take RWWRITE lock for duration of our local write. ignore starvation. - if (!obc->rwstate.take_write_lock()) { + if (!tctx->lock_manager.take_write_lock( + obc->obs.oi.soid, + obc)) { assert(0 == "problem!"); } - tctx->lock_to_release = OpContext::W_LOCK; dout(20) << __func__ << " took lock on obc, " << obc->rwstate << dendl; finish_ctx(tctx.get(), pg_log_entry_t::PROMOTE); @@ -8063,30 +8073,37 @@ int ReplicatedPG::try_flush_mark_clean(FlushOpRef fop) return 0; } + dout(10) << __func__ << " clearing DIRTY flag for " << oid << dendl; + OpContextUPtr ctx = simple_opc_create(fop->obc); + // successfully flushed; can we clear the dirty bit? // try to take the lock manually, since we don't // have a ctx yet. - if (obc->get_write(fop->op)) { + if (ctx->lock_manager.get_lock_type( + ObjectContext::RWState::RWWRITE, + oid, + obc, + fop->op)) { dout(20) << __func__ << " took write lock" << dendl; } else if (fop->op) { dout(10) << __func__ << " waiting on write lock" << dendl; + close_op_ctx(ctx.release()); requeue_op(fop->op); requeue_ops(fop->dup_ops); return -EAGAIN; // will retry } else { dout(10) << __func__ << " failed write lock, no op; failing" << dendl; + close_op_ctx(ctx.release()); osd->logger->inc(l_osd_tier_try_flush_fail); cancel_flush(fop, false); return -ECANCELED; } - dout(10) << __func__ << " clearing DIRTY flag for " << oid << dendl; - OpContextUPtr ctx = simple_opc_create(fop->obc); - - ctx->register_on_finish(*(fop->on_flush)); - fop->on_flush = boost::none; + if (fop->on_flush) { + ctx->register_on_finish(*(fop->on_flush)); + fop->on_flush = boost::none; + } - ctx->lock_to_release = OpContext::W_LOCK; // we took it above ctx->at_version = get_next_version(); ctx->new_obs = obc->obs; @@ -11866,21 +11883,25 @@ bool ReplicatedPG::agent_maybe_evict(ObjectContextRef& obc, bool after_flush) return false; } - if (!obc->get_write(OpRequestRef())) { + dout(10) << __func__ << " evicting " << obc->obs.oi << dendl; + OpContextUPtr ctx = simple_opc_create(obc); + + if (!ctx->lock_manager.get_lock_type( + ObjectContext::RWState::RWWRITE, + obc->obs.oi.soid, + obc, + OpRequestRef())) { + close_op_ctx(ctx.release()); dout(20) << __func__ << " skip (cannot get lock) " << obc->obs.oi << dendl; return false; } - dout(10) << __func__ << " evicting " << obc->obs.oi << dendl; - OpContextUPtr ctx = simple_opc_create(obc); - osd->agent_start_evict_op(); ctx->register_on_finish( [this]() { osd->agent_finish_evict_op(); }); - ctx->lock_to_release = OpContext::W_LOCK; ctx->at_version = get_next_version(); assert(ctx->new_obs.exists); int r = _delete_oid(ctx.get(), true); diff --git a/src/osd/ReplicatedPG.h b/src/osd/ReplicatedPG.h index 2e09ebd1b8ce..4d19eb5c2d08 100644 --- a/src/osd/ReplicatedPG.h +++ b/src/osd/ReplicatedPG.h @@ -383,6 +383,7 @@ public: map &attrs) { return get_object_context(hoid, true, &attrs); } + void log_operation( const vector &logv, boost::optional &hset_history, @@ -622,13 +623,12 @@ public: ObjectModDesc mod_desc; - enum { W_LOCK, R_LOCK, E_LOCK, NONE } lock_to_release; + ObjectContext::RWState::State lock_type; + ObcLockManager lock_manager; OpContext(const OpContext& other); const OpContext& operator=(const OpContext& other); - bool release_snapset_obc; - OpContext(OpRequestRef _op, osd_reqid_t _reqid, vector& _ops, ObjectContextRef& obc, ReplicatedPG *_pg) : @@ -648,8 +648,7 @@ public: sent_ack(false), sent_disk(false), async_read_result(0), inflightreads(0), - lock_to_release(NONE), - release_snapset_obc(false) { + lock_type(ObjectContext::RWState::RWNONE) { if (obc->ssc) { new_snapset = obc->ssc->snapset; snapset = &obc->ssc->snapset; @@ -668,8 +667,7 @@ public: copy_cb(NULL), async_read_result(0), inflightreads(0), - lock_to_release(NONE), - release_snapset_obc(false) { } + lock_type(ObjectContext::RWState::RWNONE) {} void reset_obs(ObjectContextRef obc) { new_obs = ObjectState(obc->obs.oi, obc->obs.exists); if (obc->ssc) { @@ -679,7 +677,7 @@ public: } ~OpContext() { assert(!op_t); - assert(lock_to_release == NONE); + assert(lock_type == ObjectContext::RWState::RWNONE); if (reply) reply->put(); for (list, @@ -768,33 +766,35 @@ protected: * this (read or write) if we get the first we will be guaranteed * to get the second. */ - ObjectContext::RWState::State type = ObjectContext::RWState::RWNONE; if (write_ordered && ctx->op->may_read()) { - type = ObjectContext::RWState::RWEXCL; - ctx->lock_to_release = OpContext::E_LOCK; + ctx->lock_type = ObjectContext::RWState::RWEXCL; } else if (write_ordered) { - type = ObjectContext::RWState::RWWRITE; - ctx->lock_to_release = OpContext::W_LOCK; + ctx->lock_type = ObjectContext::RWState::RWWRITE; } else { assert(ctx->op->may_read()); - type = ObjectContext::RWState::RWREAD; - ctx->lock_to_release = OpContext::R_LOCK; + ctx->lock_type = ObjectContext::RWState::RWREAD; } if (ctx->snapset_obc) { assert(!ctx->obc->obs.exists); - if (ctx->snapset_obc->get_lock_type(ctx->op, type)) { - ctx->release_snapset_obc = true; - } else { - ctx->lock_to_release = OpContext::NONE; + if (!ctx->lock_manager.get_lock_type( + ctx->lock_type, + ctx->snapset_obc->obs.oi.soid, + ctx->snapset_obc, + ctx->op)) { + ctx->lock_type = ObjectContext::RWState::RWNONE; return false; } } - if (ctx->obc->get_lock_type(ctx->op, type)) { + if (ctx->lock_manager.get_lock_type( + ctx->lock_type, + ctx->obc->obs.oi.soid, + ctx->obc, + ctx->op)) { return true; } else { assert(!ctx->snapset_obc); - ctx->lock_to_release = OpContext::NONE; + ctx->lock_type = ObjectContext::RWState::RWNONE; return false; } } @@ -823,74 +823,23 @@ protected: void release_op_ctx_locks(OpContext *ctx) { list to_req; bool requeue_recovery = false; - bool requeue_recovery_clone = false; - bool requeue_recovery_snapset = false; - bool requeue_snaptrimmer = false; - bool requeue_snaptrimmer_clone = false; - bool requeue_snaptrimmer_snapset = false; - switch (ctx->lock_to_release) { - case OpContext::W_LOCK: - if (ctx->snapset_obc && ctx->release_snapset_obc) { - ctx->snapset_obc->put_write( - &to_req, - &requeue_recovery_snapset, - &requeue_snaptrimmer_snapset); - ctx->release_snapset_obc = false; - } - ctx->obc->put_write( - &to_req, - &requeue_recovery, - &requeue_snaptrimmer); - if (ctx->clone_obc) - ctx->clone_obc->put_write( - &to_req, - &requeue_recovery_clone, - &requeue_snaptrimmer_clone); - break; - case OpContext::E_LOCK: - if (ctx->snapset_obc && ctx->release_snapset_obc) { - ctx->snapset_obc->put_excl( - &to_req, - &requeue_recovery_snapset, - &requeue_snaptrimmer_snapset); - ctx->release_snapset_obc = false; - } - ctx->obc->put_excl( - &to_req, - &requeue_recovery, - &requeue_snaptrimmer); - if (ctx->clone_obc) - ctx->clone_obc->put_write( - &to_req, - &requeue_recovery_clone, - &requeue_snaptrimmer_clone); - break; - case OpContext::R_LOCK: - if (ctx->snapset_obc && ctx->release_snapset_obc) { - ctx->snapset_obc->put_read(&to_req); - ctx->release_snapset_obc = false; - } - ctx->obc->put_read(&to_req); - break; - case OpContext::NONE: - break; - default: - assert(0); - }; - assert(ctx->release_snapset_obc == false); - ctx->lock_to_release = OpContext::NONE; - if (requeue_recovery || requeue_recovery_clone || requeue_recovery_snapset) + bool requeue_snaptrim = false; + ctx->lock_manager.put_locks( + &to_req, + &requeue_recovery, + &requeue_snaptrim); + ctx->lock_type = ObjectContext::RWState::RWNONE; + if (requeue_recovery) osd->recovery_wq.queue(this); - if (requeue_snaptrimmer || - requeue_snaptrimmer_clone || - requeue_snaptrimmer_snapset) + if (requeue_snaptrim) queue_snap_trim(); if (!to_req.empty()) { assert(ctx->obc); // requeue at front of scrub blocking queue if we are blocked by scrub - if (scrubber.write_blocked_by_scrub(ctx->obc->obs.oi.soid.get_head(), - get_sort_bitwise())) { + if (scrubber.write_blocked_by_scrub( + ctx->obc->obs.oi.soid.get_head(), + get_sort_bitwise())) { waiting_for_active.splice( waiting_for_active.begin(), to_req, @@ -1687,8 +1636,8 @@ inline ostream& operator<<(ostream& out, ReplicatedPG::RepGather& repop) << " rep_tid=" << repop.rep_tid << " committed?=" << repop.all_committed << " applied?=" << repop.all_applied; - if (repop.ctx->lock_to_release != ReplicatedPG::OpContext::NONE) - out << " lock=" << (int)repop.ctx->lock_to_release; + if (repop.ctx->lock_type != ObjectContext::RWState::RWNONE) + out << " lock=" << (int)repop.ctx->lock_type; if (repop.ctx->op) out << " op=" << *(repop.ctx->op->get_req()); out << ")";