ObjectCacher: add a method to clear -ENOENT caching

Clear the exists and complete flags for any objects that have exists
set to false, and force any in-flight reads to retry if they get
-ENOENT instead of generating zeros.

This is useful for getting the cache into a consistent state for rbd
after an image has been flattened, since many objects which previously
did not exist and went up to the parent to retrieve data may now exist
in the child.

Signed-off-by: Josh Durgin <josh.durgin@inktank.com>

diff --git a/src/osdc/ObjectCacher.cc b/src/osdc/ObjectCacher.cc
index c583b4c2491671b910089d4c2774afe84dfea973..c45c6afff10a471de38c7b16e7afd732d6e4a439 100644 (file)
--- a/src/osdc/ObjectCacher.cc
+++ b/src/osdc/ObjectCacher.cc
@@ -607,7 +607,8 @@ void ObjectCacher::bh_read(BufferHead *bh)
 }
 
 void ObjectCacher::bh_read_finish(int64_t poolid, sobject_t oid, loff_t start,
-                                 uint64_t length, bufferlist &bl, int r)
+                                 uint64_t length, bufferlist &bl, int r,
+                                 bool trust_enoent)
 {
   assert(lock.is_locked());
   ldout(cct, 7) << "bh_read_finish " 
@@ -634,9 +635,13 @@ void ObjectCacher::bh_read_finish(int64_t poolid, sobject_t oid, loff_t start,
     Object *ob = objects[poolid][oid];
     
     if (r == -ENOENT && !ob->complete) {
-      ldout(cct, 7) << "bh_read_finish ENOENT, marking complete and !exists on " << *ob << dendl;
-      ob->complete = true;
-      ob->exists = false;
+      // just pass through and retry all waiters if we don't trust
+      // -ENOENT for this read
+      if (trust_enoent) {
+       ldout(cct, 7) << "bh_read_finish ENOENT, marking complete and !exists on " << *ob << dendl;
+       ob->complete = true;
+       ob->exists = false;
+      }
 
       // wake up *all* rx waiters, or else we risk reordering identical reads. e.g.
       //   read 1~1
@@ -701,9 +706,14 @@ void ObjectCacher::bh_read_finish(int64_t poolid, sobject_t oid, loff_t start,
       opos = bh->end();
 
       if (r == -ENOENT) {
-       ldout(cct, 10) << "bh_read_finish removing " << *bh << dendl;
-       bh_remove(ob, bh);
-       delete bh;
+       if (trust_enoent) {
+         ldout(cct, 10) << "bh_read_finish removing " << *bh << dendl;
+         bh_remove(ob, bh);
+         delete bh;
+       } else {
+         ldout(cct, 10) << "skipping unstrusted -ENOENT and will retry for "
+                        << *bh << dendl;
+       }
        continue;
       }
 
@@ -1681,7 +1691,26 @@ uint64_t ObjectCacher::release_all()
   return unclean;
 }
 
+void ObjectCacher::clear_nonexistence(ObjectSet *oset)
+{
+  assert(lock.is_locked());
+  ldout(cct, 10) << "clear_nonexistence() " << oset << dendl;
 
+  for (xlist<Object*>::iterator p = oset->objects.begin();
+       !p.end(); ++p) {
+    Object *ob = *p;
+    if (!ob->exists) {
+      ldout(cct, 10) << " setting exists and complete on " << *ob << dendl;
+      ob->exists = true;
+      ob->complete = false;
+    }
+    for (xlist<C_ReadFinish*>::iterator q = ob->reads.begin();
+        !q.end(); ++q) {
+      C_ReadFinish *comp = *q;
+      comp->distrust_enoent();
+    }
+  }
+}
 
 /**
  * discard object extents from an ObjectSet by removing the objects in exls from the in-memory oset.

diff --git a/src/osdc/ObjectCacher.h b/src/osdc/ObjectCacher.h
index a49afe44ad7efa962e10d1323031329cace3db99..80b92d9a3625a5a19a7b852fa06f055fef943ab6 100644 (file)
--- a/src/osdc/ObjectCacher.h
+++ b/src/osdc/ObjectCacher.h
@@ -446,7 +446,8 @@ class ObjectCacher {
 
  public:
   void bh_read_finish(int64_t poolid, sobject_t oid, loff_t offset,
-                     uint64_t length, bufferlist &bl, int r);
+                     uint64_t length, bufferlist &bl, int r,
+                     bool trust_enoent);
   void bh_write_commit(int64_t poolid, sobject_t oid, loff_t offset,
                       uint64_t length, tid_t t, int r);
 
@@ -457,21 +458,26 @@ class ObjectCacher {
     loff_t start;
     uint64_t length;
     xlist<C_ReadFinish*>::item set_item;
+    bool trust_enoent;
 
   public:
     bufferlist bl;
     C_ReadFinish(ObjectCacher *c, Object *ob, loff_t s, uint64_t l) :
       oc(c), poolid(ob->oloc.pool), oid(ob->get_soid()), start(s), length(l),
-      set_item(this) {
+      set_item(this), trust_enoent(true) {
       ob->reads.push_back(&set_item);
     }
 
     void finish(int r) {
-      oc->bh_read_finish(poolid, oid, start, length, bl, r);
+      oc->bh_read_finish(poolid, oid, start, length, bl, r, trust_enoent);
       // object destructor clears the list
       if (set_item.is_on_list())
        set_item.remove_myself();
     }
+
+    void distrust_enoent() {
+      trust_enoent = false;
+    }
   };
 
   class C_WriteCommit : public Context {
@@ -565,6 +571,15 @@ public:
 
   void discard_set(ObjectSet *oset, vector<ObjectExtent>& ex);
 
+  /**
+   * Retry any in-flight reads that get -ENOENT instead of marking
+   * them zero, and get rid of any cached -ENOENTs.
+   * After this is called and the cache's lock is unlocked,
+   * any new requests will treat -ENOENT normally.
+   */
+  void clear_nonexistence(ObjectSet *oset);
+
+
   // cache sizes
   void set_max_dirty(int64_t v) {
     max_dirty = v;