osd: some snap_trimmer fixes

diff --git a/src/TODO b/src/TODO
index 5923164c5b475957c630b96bf12d837c2100f052..5f19bb4f5c31ae14c8dc9f24af6ef40358af7062 100644 (file)
--- a/src/TODO
+++ b/src/TODO
@@ -1,4 +1,8 @@
 v0.4
+- btrfs latency.  update howto.
+- snap garbage collection
+
+v0.5
 - ENOSPC
 - finish client failure recovery (reconnect after long eviction; and slow delayed reconnect)
 - make kclient use ->sendpage?

diff --git a/src/include/object.h b/src/include/object.h
index bdb31cf5540e442fa414ec56646992133a494ae9..4f0d4864a6d62b981edd75e8301082fcb49b128d 100644 (file)
--- a/src/include/object.h
+++ b/src/include/object.h
@@ -139,7 +139,7 @@ struct coll_t {
 WRITE_CLASS_ENCODER(coll_t)
 
 inline ostream& operator<<(ostream& out, const coll_t& c) {
-  return out << c.high << '.' << c.low;
+  return out << hex << c.high << '.' << c.low << dec;
 }
 
 inline bool operator<(const coll_t& l, const coll_t& r) {

diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc
index b3fad737e5390ae1af1a3f32e20725ecc2c07061..d6a0071ad57e41947b1aa52f17cce3187df40fe7 100644 (file)
--- a/src/osd/OSD.cc
+++ b/src/osd/OSD.cc
@@ -1704,8 +1704,8 @@ void OSD::advance_map(ObjectStore::Transaction& t, interval_set<snapid_t>& remov
           p != removed_snaps.m.end();
           p++)
        for (snapid_t t = 0; t < p->second; ++t)
-         pg->info.removed_snaps.insert(p->first + t);
-      dout(10) << *pg << " removed_snaps now " << pg->info.removed_snaps << dendl;
+         pg->info.dead_snaps.insert(p->first + t);
+      dout(10) << *pg << " dead_snaps now " << pg->info.dead_snaps << dendl;
       bufferlist bl;
       ::encode(pg->info, bl);
       t.collection_setattr(pg->info.pgid.to_coll(), "info", bl);
@@ -1850,7 +1850,7 @@ void OSD::activate_map(ObjectStore::Transaction& t)
     if (pg->is_active()) {
       // update started counter
       pg->info.history.last_epoch_started = osdmap->get_epoch();
-      if (!pg->info.removed_snaps.empty())
+      if (!pg->info.dead_snaps.empty())
        pg->queue_snap_trim();
     }
     else if (pg->is_primary() && !pg->is_active()) {

diff --git a/src/osd/PG.cc b/src/osd/PG.cc
index 2435c0c082615065e92f91ca02544f67b7f6ef6e..c1d2022d3f3a7368df8b8bc01dfe7b9ea79cf6be 100644 (file)
--- a/src/osd/PG.cc
+++ b/src/osd/PG.cc
@@ -968,9 +968,12 @@ void PG::activate(ObjectStore::Transaction& t,
   write_info(t);
   write_log(t);
 
-  // clean up stray objects
+  // clean up stray objects, snaps
   clean_up_local(t); 
 
+  if (!info.dead_snaps.empty())
+    queue_snap_trim();
+
   // init complete pointer
   if (missing.num_missing() == 0 &&
       info.last_complete != info.last_update) {
@@ -1122,6 +1125,11 @@ void PG::activate(ObjectStore::Transaction& t,
 
 void PG::queue_snap_trim()
 {
+  if (state_test(PG_STATE_SNAPTRIMMING)) {
+    dout(10) << "queue_snap_trim -- already trimming" << dendl;
+    return;
+  }
+
   state_set(PG_STATE_SNAPTRIMQUEUE);
 
   osd->snap_trimmer_lock.Lock();
@@ -1166,10 +1174,6 @@ void PG::_finish_recovery(Context *c)
     finish_sync_event = 0;
     dout(10) << "_finish_recovery" << dendl;
     purge_strays();
-
-    if (!info.removed_snaps.empty())
-      queue_snap_trim();
-
     update_stats();
   }
   osd->osd_lock.Unlock();

diff --git a/src/osd/PG.h b/src/osd/PG.h
index 9bb5b19c08c2348878f12168ad9e0ded4c54afd6..3626494ad816a3e4d14deaf766e2a4aeda3bb524 100644 (file)
--- a/src/osd/PG.h
+++ b/src/osd/PG.h
@@ -67,7 +67,7 @@ public:
     eversion_t log_bottom;     // oldest log entry.
     bool       log_backlog;    // do we store a complete log?
 
-    set<snapid_t> removed_snaps; // snaps we need to trim
+    set<snapid_t> dead_snaps; // snaps we need to trim
 
     struct History {
       epoch_t epoch_created;       // epoch in which PG was created
@@ -118,7 +118,7 @@ public:
       ::encode(log_bottom, bl);
       ::encode(log_backlog, bl);
       history.encode(bl);
-      ::encode(removed_snaps, bl);
+      ::encode(dead_snaps, bl);
     }
     void decode(bufferlist::iterator &bl) {
       ::decode(pgid, bl);
@@ -127,7 +127,7 @@ public:
       ::decode(log_bottom, bl);
       ::decode(log_backlog, bl);
       history.decode(bl);
-      ::decode(removed_snaps, bl);
+      ::decode(dead_snaps, bl);
     }
   };
   WRITE_CLASS_ENCODER(Info::History)
@@ -817,8 +817,8 @@ inline ostream& operator<<(ostream& out, const PG& pg)
   //out << " (" << pg.log.bottom << "," << pg.log.top << "]";
   if (pg.missing.num_missing()) out << " m=" << pg.missing.num_missing();
   if (pg.missing.num_lost()) out << " l=" << pg.missing.num_lost();
-  if (pg.info.removed_snaps.size())
-    out << " rs=" << pg.info.removed_snaps;
+  if (pg.info.dead_snaps.size())
+    out << " dead=" << pg.info.dead_snaps;
   out << "]";
 
 

diff --git a/src/osd/ReplicatedPG.cc b/src/osd/ReplicatedPG.cc
index ba7a5473635db7ff3ed1119ac329557e30c8e282..6d79bc1b873dc3194dec4132dde01ffa148988ee 100644 (file)
--- a/src/osd/ReplicatedPG.cc
+++ b/src/osd/ReplicatedPG.cc
@@ -409,15 +409,15 @@ void ReplicatedPG::do_sub_op_reply(MOSDSubOpReply *r)
 bool ReplicatedPG::snap_trimmer()
 {
   lock();
-  dout(10) << "snap_trimmer" << dendl;
+  dout(10) << "snap_trimmer start" << dendl;
   
   state_clear(PG_STATE_SNAPTRIMQUEUE);
   state_set(PG_STATE_SNAPTRIMMING);
   update_stats();
 
-  while (info.removed_snaps.size() &&
+  while (info.dead_snaps.size() &&
         is_active()) {
-    snapid_t sn = *info.removed_snaps.begin();
+    snapid_t sn = *info.dead_snaps.begin();
     coll_t c = info.pgid.to_snap_coll(sn);
     vector<pobject_t> ls;
     osd->store->collection_list(c, ls);
@@ -429,34 +429,42 @@ bool ReplicatedPG::snap_trimmer()
     for (vector<pobject_t>::iterator p = ls.begin(); p != ls.end(); p++) {
       pobject_t coid = *p;
 
+      // load clone snap list
       bufferlist bl;
       osd->store->getattr(info.pgid.to_coll(), coid, "snaps", bl);
       bufferlist::iterator blp = bl.begin();
       vector<snapid_t> snaps;
       ::decode(snaps, blp);
+
+      // load head snapset     
+      pobject_t head = coid;
+      head.oid.snap = CEPH_NOSNAP;
+      bl.clear();
+      osd->store->getattr(info.pgid.to_coll(), head, "snapset", bl);
+      blp = bl.begin();
+      SnapSet snapset;
+      ::decode(snapset, blp);
+      dout(10) << coid << " old head " << head << " snapset " << snapset << dendl;
+
+      // remove snaps
       vector<snapid_t> newsnaps;
       for (unsigned i=0; i<snaps.size(); i++)
        if (!osd->osdmap->is_removed_snap(snaps[i]))
-         newsnaps.push_back(i);
-
+         newsnaps.push_back(snaps[i]);
+       else {
+         vector<snapid_t>::iterator q = snapset.snaps.begin();
+         while (*q != snaps[i]) q++;  // it should be in there
+         snapset.snaps.erase(q);
+       }
       if (newsnaps.empty()) {
-       // remove
+       // remove clone
        dout(10) << coid << " snaps " << snaps << " -> " << newsnaps << " ... deleting" << dendl;
        t.remove(info.pgid.to_coll(), coid);
        t.collection_remove(info.pgid.to_snap_coll(snaps[0]), coid);
        if (snaps.size() > 1)
          t.collection_remove(info.pgid.to_snap_coll(snaps[snaps.size()-1]), coid);
        
-       // adjust head snapset  
-       pobject_t head = coid;
-       head.oid.snap = CEPH_NOSNAP;
-       bufferlist bl;
-       osd->store->getattr(info.pgid.to_coll(), head, "snapset", bl);
-       bufferlist::iterator blp = bl.begin();
-       SnapSet snapset;
-       ::decode(snapset, blp);
-       dout(10) << coid << " old head " << head << " snapset " << snapset << dendl;
-
+       // ...from snapset
        snapid_t last = coid.oid.snap;
        vector<snapid_t>::iterator p;
        for (p = snapset.clones.begin(); p != snapset.clones.end(); p++)
@@ -475,17 +483,6 @@ bool ReplicatedPG::snap_trimmer()
        }
        snapset.clones.erase(p);
        snapset.clone_overlap.erase(last);
-
-       dout(10) << coid << " new head " << head << " snapset " << snapset << dendl;
-
-       if (snapset.clones.empty() && !snapset.head_exists) {
-         dout(10) << coid << " removing head " << head << dendl;
-         t.remove(info.pgid.to_coll(), head);
-       } else {
-         bl.clear();
-         ::encode(snapset, bl);
-         t.setattr(info.pgid.to_coll(), head, "snapset", bl);
-       }
       } else {
        // save adjusted snaps for this object
        dout(10) << coid << " snaps " << snaps << " -> " << newsnaps << dendl;
@@ -503,7 +500,19 @@ bool ReplicatedPG::snap_trimmer()
            t.collection_add(info.pgid.to_snap_coll(newsnaps[newsnaps.size()-1]), info.pgid.to_coll(), coid);
        }             
       }
+
+      // save head snapset
+      dout(10) << coid << " new head " << head << " snapset " << snapset << dendl;
       
+      if (snapset.clones.empty() && !snapset.head_exists) {
+       dout(10) << coid << " removing head " << head << dendl;
+       t.remove(info.pgid.to_coll(), head);
+      } else {
+       bl.clear();
+       ::encode(snapset, bl);
+       t.setattr(info.pgid.to_coll(), head, "snapset", bl);
+      }
+
       osd->store->apply_transaction(t);
 
       // give other threads a chance at this pg
@@ -511,7 +520,7 @@ bool ReplicatedPG::snap_trimmer()
       lock();
     }
     
-    info.removed_snaps.erase(sn);
+    info.dead_snaps.erase(sn);
   }  
 
   // done