From bfb5b8d9bd66089978fe092182a113631ff315cf Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Wed, 10 Sep 2008 15:53:00 -0700 Subject: [PATCH] osd: some snap_trimmer fixes --- src/TODO | 4 +++ src/include/object.h | 2 +- src/osd/OSD.cc | 6 ++-- src/osd/PG.cc | 14 +++++---- src/osd/PG.h | 10 +++---- src/osd/ReplicatedPG.cc | 65 +++++++++++++++++++++++------------------ 6 files changed, 59 insertions(+), 42 deletions(-) diff --git a/src/TODO b/src/TODO index 5923164c5b475..5f19bb4f5c31a 100644 --- a/src/TODO +++ b/src/TODO @@ -1,4 +1,8 @@ v0.4 +- btrfs latency. update howto. +- snap garbage collection + +v0.5 - ENOSPC - finish client failure recovery (reconnect after long eviction; and slow delayed reconnect) - make kclient use ->sendpage? diff --git a/src/include/object.h b/src/include/object.h index bdb31cf5540e4..4f0d4864a6d62 100644 --- a/src/include/object.h +++ b/src/include/object.h @@ -139,7 +139,7 @@ struct coll_t { WRITE_CLASS_ENCODER(coll_t) inline ostream& operator<<(ostream& out, const coll_t& c) { - return out << c.high << '.' << c.low; + return out << hex << c.high << '.' << c.low << dec; } inline bool operator<(const coll_t& l, const coll_t& r) { diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc index b3fad737e5390..d6a0071ad57e4 100644 --- a/src/osd/OSD.cc +++ b/src/osd/OSD.cc @@ -1704,8 +1704,8 @@ void OSD::advance_map(ObjectStore::Transaction& t, interval_set& remov p != removed_snaps.m.end(); p++) for (snapid_t t = 0; t < p->second; ++t) - pg->info.removed_snaps.insert(p->first + t); - dout(10) << *pg << " removed_snaps now " << pg->info.removed_snaps << dendl; + pg->info.dead_snaps.insert(p->first + t); + dout(10) << *pg << " dead_snaps now " << pg->info.dead_snaps << dendl; bufferlist bl; ::encode(pg->info, bl); t.collection_setattr(pg->info.pgid.to_coll(), "info", bl); @@ -1850,7 +1850,7 @@ void OSD::activate_map(ObjectStore::Transaction& t) if (pg->is_active()) { // update started counter pg->info.history.last_epoch_started = osdmap->get_epoch(); - if (!pg->info.removed_snaps.empty()) + if (!pg->info.dead_snaps.empty()) pg->queue_snap_trim(); } else if (pg->is_primary() && !pg->is_active()) { diff --git a/src/osd/PG.cc b/src/osd/PG.cc index 2435c0c082615..c1d2022d3f3a7 100644 --- a/src/osd/PG.cc +++ b/src/osd/PG.cc @@ -968,9 +968,12 @@ void PG::activate(ObjectStore::Transaction& t, write_info(t); write_log(t); - // clean up stray objects + // clean up stray objects, snaps clean_up_local(t); + if (!info.dead_snaps.empty()) + queue_snap_trim(); + // init complete pointer if (missing.num_missing() == 0 && info.last_complete != info.last_update) { @@ -1122,6 +1125,11 @@ void PG::activate(ObjectStore::Transaction& t, void PG::queue_snap_trim() { + if (state_test(PG_STATE_SNAPTRIMMING)) { + dout(10) << "queue_snap_trim -- already trimming" << dendl; + return; + } + state_set(PG_STATE_SNAPTRIMQUEUE); osd->snap_trimmer_lock.Lock(); @@ -1166,10 +1174,6 @@ void PG::_finish_recovery(Context *c) finish_sync_event = 0; dout(10) << "_finish_recovery" << dendl; purge_strays(); - - if (!info.removed_snaps.empty()) - queue_snap_trim(); - update_stats(); } osd->osd_lock.Unlock(); diff --git a/src/osd/PG.h b/src/osd/PG.h index 9bb5b19c08c23..3626494ad816a 100644 --- a/src/osd/PG.h +++ b/src/osd/PG.h @@ -67,7 +67,7 @@ public: eversion_t log_bottom; // oldest log entry. bool log_backlog; // do we store a complete log? - set removed_snaps; // snaps we need to trim + set dead_snaps; // snaps we need to trim struct History { epoch_t epoch_created; // epoch in which PG was created @@ -118,7 +118,7 @@ public: ::encode(log_bottom, bl); ::encode(log_backlog, bl); history.encode(bl); - ::encode(removed_snaps, bl); + ::encode(dead_snaps, bl); } void decode(bufferlist::iterator &bl) { ::decode(pgid, bl); @@ -127,7 +127,7 @@ public: ::decode(log_bottom, bl); ::decode(log_backlog, bl); history.decode(bl); - ::decode(removed_snaps, bl); + ::decode(dead_snaps, bl); } }; WRITE_CLASS_ENCODER(Info::History) @@ -817,8 +817,8 @@ inline ostream& operator<<(ostream& out, const PG& pg) //out << " (" << pg.log.bottom << "," << pg.log.top << "]"; if (pg.missing.num_missing()) out << " m=" << pg.missing.num_missing(); if (pg.missing.num_lost()) out << " l=" << pg.missing.num_lost(); - if (pg.info.removed_snaps.size()) - out << " rs=" << pg.info.removed_snaps; + if (pg.info.dead_snaps.size()) + out << " dead=" << pg.info.dead_snaps; out << "]"; diff --git a/src/osd/ReplicatedPG.cc b/src/osd/ReplicatedPG.cc index ba7a5473635db..6d79bc1b873dc 100644 --- a/src/osd/ReplicatedPG.cc +++ b/src/osd/ReplicatedPG.cc @@ -409,15 +409,15 @@ void ReplicatedPG::do_sub_op_reply(MOSDSubOpReply *r) bool ReplicatedPG::snap_trimmer() { lock(); - dout(10) << "snap_trimmer" << dendl; + dout(10) << "snap_trimmer start" << dendl; state_clear(PG_STATE_SNAPTRIMQUEUE); state_set(PG_STATE_SNAPTRIMMING); update_stats(); - while (info.removed_snaps.size() && + while (info.dead_snaps.size() && is_active()) { - snapid_t sn = *info.removed_snaps.begin(); + snapid_t sn = *info.dead_snaps.begin(); coll_t c = info.pgid.to_snap_coll(sn); vector ls; osd->store->collection_list(c, ls); @@ -429,34 +429,42 @@ bool ReplicatedPG::snap_trimmer() for (vector::iterator p = ls.begin(); p != ls.end(); p++) { pobject_t coid = *p; + // load clone snap list bufferlist bl; osd->store->getattr(info.pgid.to_coll(), coid, "snaps", bl); bufferlist::iterator blp = bl.begin(); vector snaps; ::decode(snaps, blp); + + // load head snapset + pobject_t head = coid; + head.oid.snap = CEPH_NOSNAP; + bl.clear(); + osd->store->getattr(info.pgid.to_coll(), head, "snapset", bl); + blp = bl.begin(); + SnapSet snapset; + ::decode(snapset, blp); + dout(10) << coid << " old head " << head << " snapset " << snapset << dendl; + + // remove snaps vector newsnaps; for (unsigned i=0; iosdmap->is_removed_snap(snaps[i])) - newsnaps.push_back(i); - + newsnaps.push_back(snaps[i]); + else { + vector::iterator q = snapset.snaps.begin(); + while (*q != snaps[i]) q++; // it should be in there + snapset.snaps.erase(q); + } if (newsnaps.empty()) { - // remove + // remove clone dout(10) << coid << " snaps " << snaps << " -> " << newsnaps << " ... deleting" << dendl; t.remove(info.pgid.to_coll(), coid); t.collection_remove(info.pgid.to_snap_coll(snaps[0]), coid); if (snaps.size() > 1) t.collection_remove(info.pgid.to_snap_coll(snaps[snaps.size()-1]), coid); - // adjust head snapset - pobject_t head = coid; - head.oid.snap = CEPH_NOSNAP; - bufferlist bl; - osd->store->getattr(info.pgid.to_coll(), head, "snapset", bl); - bufferlist::iterator blp = bl.begin(); - SnapSet snapset; - ::decode(snapset, blp); - dout(10) << coid << " old head " << head << " snapset " << snapset << dendl; - + // ...from snapset snapid_t last = coid.oid.snap; vector::iterator p; for (p = snapset.clones.begin(); p != snapset.clones.end(); p++) @@ -475,17 +483,6 @@ bool ReplicatedPG::snap_trimmer() } snapset.clones.erase(p); snapset.clone_overlap.erase(last); - - dout(10) << coid << " new head " << head << " snapset " << snapset << dendl; - - if (snapset.clones.empty() && !snapset.head_exists) { - dout(10) << coid << " removing head " << head << dendl; - t.remove(info.pgid.to_coll(), head); - } else { - bl.clear(); - ::encode(snapset, bl); - t.setattr(info.pgid.to_coll(), head, "snapset", bl); - } } else { // save adjusted snaps for this object dout(10) << coid << " snaps " << snaps << " -> " << newsnaps << dendl; @@ -503,7 +500,19 @@ bool ReplicatedPG::snap_trimmer() t.collection_add(info.pgid.to_snap_coll(newsnaps[newsnaps.size()-1]), info.pgid.to_coll(), coid); } } + + // save head snapset + dout(10) << coid << " new head " << head << " snapset " << snapset << dendl; + if (snapset.clones.empty() && !snapset.head_exists) { + dout(10) << coid << " removing head " << head << dendl; + t.remove(info.pgid.to_coll(), head); + } else { + bl.clear(); + ::encode(snapset, bl); + t.setattr(info.pgid.to_coll(), head, "snapset", bl); + } + osd->store->apply_transaction(t); // give other threads a chance at this pg @@ -511,7 +520,7 @@ bool ReplicatedPG::snap_trimmer() lock(); } - info.removed_snaps.erase(sn); + info.dead_snaps.erase(sn); } // done -- 2.39.5