From 8259d87493d06a7628a862ff63e3d1a12961eb7f Mon Sep 17 00:00:00 2001 From: Samuel Just Date: Mon, 14 Apr 2014 16:20:39 -0700 Subject: [PATCH] ReplicatedPG: in trim, grab w locks on obc and snapset_obc Signed-off-by: Samuel Just --- src/osd/ReplicatedPG.cc | 46 +++++++++++++++++++++++++++++------------ src/osd/ReplicatedPG.h | 22 +++++++++++++++++--- src/osd/osd_types.h | 18 +++++++++++++++- 3 files changed, 69 insertions(+), 17 deletions(-) diff --git a/src/osd/ReplicatedPG.cc b/src/osd/ReplicatedPG.cc index 25bec4106a973..9274893a597a7 100644 --- a/src/osd/ReplicatedPG.cc +++ b/src/osd/ReplicatedPG.cc @@ -2223,18 +2223,34 @@ ReplicatedPG::RepGather *ReplicatedPG::trim_object(const hobject_t &coid) derr << __func__ << "could not find coid " << coid << dendl; assert(0); } + assert(obc->ssc); + + if (!obc->get_snaptrimmer_write()) { + dout(10) << __func__ << ": Unable to get a wlock on " << coid << dendl; + return NULL; + } + + hobject_t snapoid( + coid.oid, coid.get_key(), + obc->ssc->snapset.head_exists ? CEPH_NOSNAP:CEPH_SNAPDIR, coid.hash, + info.pgid.pool(), coid.get_namespace()); + ObjectContextRef snapset_obc = get_object_context(snapoid, false); + + if (!snapset_obc->get_snaptrimmer_write()) { + dout(10) << __func__ << ": Unable to get a wlock on " << snapoid << dendl; + list to_wake; + bool requeue_recovery = false; + bool requeue_snaptrimmer = false; + obc->put_write(&to_wake, &requeue_recovery, &requeue_snaptrimmer); + assert(to_wake.empty()); + assert(!requeue_recovery); + return NULL; + } object_info_t &coi = obc->obs.oi; set old_snaps(coi.snaps.begin(), coi.snaps.end()); assert(old_snaps.size()); - // get snap set context - if (!obc->ssc) - obc->ssc = get_snapset_context( - coid, - false); - - assert(obc->ssc); SnapSet& snapset = obc->ssc->snapset; dout(10) << coid << " old_snaps " << old_snaps @@ -2243,6 +2259,9 @@ ReplicatedPG::RepGather *ReplicatedPG::trim_object(const hobject_t &coid) RepGather *repop = simple_repop_create(obc); OpContext *ctx = repop->ctx; + ctx->snapset_obc = snapset_obc; + ctx->lock_to_release = OpContext::W_LOCK; + ctx->release_snapset_obc = true; ctx->at_version = get_next_version(); PGBackend::PGTransaction *t = ctx->op_t; @@ -2368,12 +2387,6 @@ ReplicatedPG::RepGather *ReplicatedPG::trim_object(const hobject_t &coid) // save head snapset dout(10) << coid << " new snapset " << snapset << dendl; - hobject_t snapoid( - coid.oid, coid.get_key(), - snapset.head_exists ? CEPH_NOSNAP:CEPH_SNAPDIR, coid.hash, - info.pgid.pool(), coid.get_namespace()); - ctx->snapset_obc = get_object_context(snapoid, false); - if (snapset.clones.empty() && !snapset.head_exists) { dout(10) << coid << " removing " << snapoid << dendl; ctx->log.push_back( @@ -11740,6 +11753,7 @@ boost::statechart::result ReplicatedPG::TrimmingObjects::react(const SnapTrim&) dout(10) << "TrimmingObjects: trimming snap " << snap_to_trim << dendl; // Get next + hobject_t old_pos = pos; int r = pg->snap_mapper.get_next_object_to_trim(snap_to_trim, &pos); if (r != 0 && r != -ENOENT) { derr << __func__ << ": get_next returned " << cpp_strerror(r) << dendl; @@ -11753,6 +11767,12 @@ boost::statechart::result ReplicatedPG::TrimmingObjects::react(const SnapTrim&) dout(10) << "TrimmingObjects react trimming " << pos << dendl; RepGather *repop = pg->trim_object(pos); + if (!repop) { + dout(10) << __func__ << " could not get write lock on obj " + << pos << dendl; + pos = old_pos; + return discard_event(); + } assert(repop); repop->queue_snap_trimmer = true; diff --git a/src/osd/ReplicatedPG.h b/src/osd/ReplicatedPG.h index 76d2d1a8f13ed..a59242d329434 100644 --- a/src/osd/ReplicatedPG.h +++ b/src/osd/ReplicatedPG.h @@ -714,15 +714,27 @@ protected: bool requeue_recovery = false; bool requeue_recovery_clone = false; bool requeue_recovery_snapset = false; + bool requeue_snaptrimmer = false; + bool requeue_snaptrimmer_clone = false; + bool requeue_snaptrimmer_snapset = false; switch (ctx->lock_to_release) { case OpContext::W_LOCK: if (ctx->snapset_obc && ctx->release_snapset_obc) { - ctx->snapset_obc->put_write(&to_req, &requeue_recovery_snapset); + ctx->snapset_obc->put_write( + &to_req, + &requeue_recovery_snapset, + &requeue_snaptrimmer_snapset); ctx->release_snapset_obc = false; } - ctx->obc->put_write(&to_req, &requeue_recovery); + ctx->obc->put_write( + &to_req, + &requeue_recovery, + &requeue_snaptrimmer); if (ctx->clone_obc) - ctx->clone_obc->put_write(&to_req, &requeue_recovery_clone); + ctx->clone_obc->put_write( + &to_req, + &requeue_recovery_clone, + &requeue_snaptrimmer_clone); break; case OpContext::R_LOCK: if (ctx->snapset_obc && ctx->release_snapset_obc) { @@ -740,6 +752,10 @@ protected: ctx->lock_to_release = OpContext::NONE; if (requeue_recovery || requeue_recovery_clone || requeue_recovery_snapset) osd->recovery_wq.queue(this); + if (requeue_snaptrimmer || + requeue_snaptrimmer_clone || + requeue_snaptrimmer_snapset) + queue_snap_trim(); requeue_ops(to_req); } diff --git a/src/osd/osd_types.h b/src/osd/osd_types.h index 410e02ce98600..7c18e70fe583e 100644 --- a/src/osd/osd_types.h +++ b/src/osd/osd_types.h @@ -2671,6 +2671,9 @@ public: /// if set, restart backfill when we can get a read lock bool backfill_read_marker; + /// if set, requeue snaptrim on lock release + bool snaptrimmer_write_marker; + RWState() : state(RWNONE), count(0), backfill_read_marker(false) {} bool get_read(OpRequestRef op) { if (get_read_lock()) { @@ -2764,6 +2767,14 @@ public: bool get_write(OpRequestRef op) { return rwstate.get_write(op); } + bool get_snaptrimmer_write() { + if (rwstate.get_write_lock()) { + return true; + } else { + rwstate.snaptrimmer_write_marker = true; + return false; + } + } bool get_backfill_read() { rwstate.backfill_read_marker = true; if (rwstate.get_read_lock()) { @@ -2780,12 +2791,17 @@ public: rwstate.put_read(to_wake); } void put_write(list *to_wake, - bool *requeue_recovery) { + bool *requeue_recovery, + bool *requeue_snaptrimmer) { rwstate.put_write(to_wake); if (rwstate.empty() && rwstate.backfill_read_marker) { rwstate.backfill_read_marker = false; *requeue_recovery = true; } + if (rwstate.empty() && rwstate.snaptrimmer_write_marker) { + rwstate.snaptrimmer_write_marker = false; + *requeue_snaptrimmer = true; + } } ObjectContext() -- 2.39.5