ObjectCacher: add a method to clear -ENOENT caching

Clear the exists and complete flags for any objects that have exists
set to false, and force any in-flight reads to retry if they get
-ENOENT instead of generating zeros.

This is useful for getting the cache into a consistent state for rbd
after an image has been flattened, since many objects which previously
did not exist and went up to the parent to retrieve data may now exist
in the child.

Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
(cherry picked from commit f2a23dc0b092c5ac081893e8f28c6d4bcabd0c2e)

diff --git a/src/osdc/ObjectCacher.cc b/src/osdc/ObjectCacher.cc
index 63e9dd17b8500332e14ffacbe4ebb8566e13d0d8..540453650ee005e753d68ed1ccd93132b4a21131 100644 (file)
--- a/src/osdc/ObjectCacher.cc
+++ b/src/osdc/ObjectCacher.cc
@@ -607,7 +607,8 @@ void ObjectCacher::bh_read(BufferHead *bh)
 }
 
 void ObjectCacher::bh_read_finish(int64_t poolid, sobject_t oid, loff_t start,
-                                 uint64_t length, bufferlist &bl, int r)
+                                 uint64_t length, bufferlist &bl, int r,
+                                 bool trust_enoent)
 {
   assert(lock.is_locked());
   ldout(cct, 7) << "bh_read_finish " 
@@ -634,9 +635,13 @@ void ObjectCacher::bh_read_finish(int64_t poolid, sobject_t oid, loff_t start,
     Object *ob = objects[poolid][oid];
     
     if (r == -ENOENT && !ob->complete) {
-      ldout(cct, 7) << "bh_read_finish ENOENT, marking complete and !exists on " << *ob << dendl;
-      ob->complete = true;
-      ob->exists = false;
+      // just pass through and retry all waiters if we don't trust
+      // -ENOENT for this read
+      if (trust_enoent) {
+       ldout(cct, 7) << "bh_read_finish ENOENT, marking complete and !exists on " << *ob << dendl;
+       ob->complete = true;
+       ob->exists = false;
+      }
 
       // wake up *all* rx waiters, or else we risk reordering identical reads. e.g.
       //   read 1~1
@@ -701,9 +706,14 @@ void ObjectCacher::bh_read_finish(int64_t poolid, sobject_t oid, loff_t start,
       opos = bh->end();
 
       if (r == -ENOENT) {
-       ldout(cct, 10) << "bh_read_finish removing " << *bh << dendl;
-       bh_remove(ob, bh);
-       delete bh;
+       if (trust_enoent) {
+         ldout(cct, 10) << "bh_read_finish removing " << *bh << dendl;
+         bh_remove(ob, bh);
+         delete bh;
+       } else {
+         ldout(cct, 10) << "skipping unstrusted -ENOENT and will retry for "
+                        << *bh << dendl;
+       }
        continue;
       }
 
@@ -1729,7 +1739,26 @@ uint64_t ObjectCacher::release_all()
   return unclean;
 }
 
+void ObjectCacher::clear_nonexistence(ObjectSet *oset)
+{
+  assert(lock.is_locked());
+  ldout(cct, 10) << "clear_nonexistence() " << oset << dendl;
 
+  for (xlist<Object*>::iterator p = oset->objects.begin();
+       !p.end(); ++p) {
+    Object *ob = *p;
+    if (!ob->exists) {
+      ldout(cct, 10) << " setting exists and complete on " << *ob << dendl;
+      ob->exists = true;
+      ob->complete = false;
+    }
+    for (xlist<C_ReadFinish*>::iterator q = ob->reads.begin();
+        !q.end(); ++q) {
+      C_ReadFinish *comp = *q;
+      comp->distrust_enoent();
+    }
+  }
+}
 
 /**
  * discard object extents from an ObjectSet by removing the objects in exls from the in-memory oset.

diff --git a/src/osdc/ObjectCacher.h b/src/osdc/ObjectCacher.h
index 27f4cf9737987a38305752ad9c4e8de154cec164..21c6d9deec3f7845cb826b4683bd9772d3209e7b 100644 (file)
--- a/src/osdc/ObjectCacher.h
+++ b/src/osdc/ObjectCacher.h
@@ -446,7 +446,8 @@ class ObjectCacher {
 
  public:
   void bh_read_finish(int64_t poolid, sobject_t oid, loff_t offset,
-                     uint64_t length, bufferlist &bl, int r);
+                     uint64_t length, bufferlist &bl, int r,
+                     bool trust_enoent);
   void bh_write_commit(int64_t poolid, sobject_t oid, loff_t offset,
                       uint64_t length, tid_t t, int r);
 
@@ -457,21 +458,26 @@ class ObjectCacher {
     loff_t start;
     uint64_t length;
     xlist<C_ReadFinish*>::item set_item;
+    bool trust_enoent;
 
   public:
     bufferlist bl;
     C_ReadFinish(ObjectCacher *c, Object *ob, loff_t s, uint64_t l) :
       oc(c), poolid(ob->oloc.pool), oid(ob->get_soid()), start(s), length(l),
-      set_item(this) {
+      set_item(this), trust_enoent(true) {
       ob->reads.push_back(&set_item);
     }
 
     void finish(int r) {
-      oc->bh_read_finish(poolid, oid, start, length, bl, r);
+      oc->bh_read_finish(poolid, oid, start, length, bl, r, trust_enoent);
       // object destructor clears the list
       if (set_item.is_on_list())
        set_item.remove_myself();
     }
+
+    void distrust_enoent() {
+      trust_enoent = false;
+    }
   };
 
   class C_WriteCommit : public Context {
@@ -567,6 +573,15 @@ public:
 
   void discard_set(ObjectSet *oset, vector<ObjectExtent>& ex);
 
+  /**
+   * Retry any in-flight reads that get -ENOENT instead of marking
+   * them zero, and get rid of any cached -ENOENTs.
+   * After this is called and the cache's lock is unlocked,
+   * any new requests will treat -ENOENT normally.
+   */
+  void clear_nonexistence(ObjectSet *oset);
+
+
   // cache sizes
   void set_max_dirty(int64_t v) {
     max_dirty = v;