kclient: changing around the rdcache invalidation stuff

diff --git a/src/kernel/caps.c b/src/kernel/caps.c
index f202691e7bb2decc9f6e2be4df34b6d8d17f28cb..14e2f0552833cd26491fe5ef0f2a74c978b506db 100644 (file)
--- a/src/kernel/caps.c
+++ b/src/kernel/caps.c
@@ -879,9 +879,9 @@ static int handle_cap_grant(struct inode *inode, struct ceph_mds_caps *grant,
        struct timespec mtime, atime, ctime;
        int wake = 0;
        int writeback = 0;
+       int revoked_rdcache = 0;
        int invalidate = 0;
        int tried_invalidate = 0;
-       u32 inv_gen = 0;
        int ret;
 
        dout(10, "handle_cap_grant inode %p ci %p mds%d seq %d\n",
@@ -906,37 +906,42 @@ start:
        dout(10, " cap %p\n", cap);
        cap->gen = session->s_cap_gen;
 
+       /*
+        * Each time we receive RDCACHE anew, we increment i_rdcache_gen.
+        */
+       if ((newcaps & CEPH_CAP_RDCACHE) &&            /* we just got RDCACHE */
+           (cap->issued & CEPH_CAP_RDCACHE) == 0 &&    /* and didn't have it */
+           (__ceph_caps_issued(ci, 0) & CEPH_CAP_RDCACHE) == 0)
+               ci->i_rdcache_gen++;
+
+       /*
+        * If RDCACHE is being revoked, and we have no dirty buffers,
+        * try to invalidate (once).  (If there are dirty buffers, we
+        * will invalidate _after_ writeback.)
+        */
        if (((cap->issued & ~newcaps) & CEPH_CAP_RDCACHE)
-           && !ci->i_wrbuffer_ref){
+           && !ci->i_wrbuffer_ref && !tried_invalidate) {
                dout(10, "RDCACHE invalidation\n");
-               if (!tried_invalidate) {
-                       inv_gen = ci->i_rdcache_gen;
-                       spin_unlock(&inode->i_lock);
+               spin_unlock(&inode->i_lock);
 
-                       tried_invalidate = 1;
-                       ret = invalidate_inode_pages2(&inode->i_data);
-                       if (ret < 0)
+               tried_invalidate = 1;
+               ret = invalidate_inode_pages2(&inode->i_data);
+               if (ret < 0) {
+                       /* there were locked pages.. invalidate later
+                          in a separate thread. */
+                       if (ci->i_rdcache_revoking != ci->i_rdcache_gen) {
                                invalidate = 1;
-                       goto start;
+                               ci->i_rdcache_revoking = ci->i_rdcache_gen;
+                       }
                } else {
-                       if (ci->i_rdcache_gen != inv_gen) /* was there a race? */
-                               invalidate = 1;
+                       /* we successfully invalidated those pages */
+                       revoked_rdcache = 1;
+                       ci->i_rdcache_gen = 0;
+                       ci->i_rdcache_revoking = 0;
                }
+               goto start;
        }
 
-       if ((cap->issued & ~newcaps) & CEPH_CAP_RDCACHE & __ceph_caps_issued(ci, 0)) {
-               if (!ci->i_rdcache_pending)
-                       ci->i_rdcache_gen++;
-               else
-                       invalidate = 0; /* ok, we're already taking care of it */
-       }
-
-       if (invalidate && !(ci->i_rdcache_pending))
-               ci->i_rdcache_pending = 1;
-
-
-       dout(10, "invalidate=%d ci->i_rdcache_pending=%d gen=%d\n", invalidate, ci->i_rdcache_pending, ci->i_rdcache_gen);
-
 
        /* size/ctime/mtime/atime? */
        issued = __ceph_caps_issued(ci, NULL);
@@ -979,7 +984,8 @@ start:
                dout(10, "revocation: %d -> %d\n", cap->issued, newcaps);
                if ((used & ~newcaps) & CEPH_CAP_WRBUFFER) {
                        writeback = 1; /* will delay ack */
-               } else if (!invalidate) {
+               } else if (((used & ~newcaps) & CEPH_CAP_RDCACHE) == 0 ||
+                          revoked_rdcache) {
                        /*
                         * we're not using revoked caps.. ack now.
                         * re-use incoming message.

diff --git a/src/kernel/inode.c b/src/kernel/inode.c
index 863e2be623cb3ba53d0eba80471bc3976fced7e7..db304b9b731eddb78aef6281655020358510a4cb 100644 (file)
--- a/src/kernel/inode.c
+++ b/src/kernel/inode.c
@@ -277,12 +277,13 @@ struct inode *ceph_alloc_inode(struct super_block *sb)
        ci->i_cap_exporting_mseq = 0;
        ci->i_cap_exporting_issued = 0;
 
-       ci->i_rd_ref = ci->i_rdcache_ref = 0;
-       ci->i_rdcache_pending = 0;
-       ci->i_rdcache_gen = 0;
+       ci->i_rd_ref = 0;
+       ci->i_rdcache_ref = 0;
        ci->i_wr_ref = 0;
        ci->i_wrbuffer_ref = 0;
        ci->i_wrbuffer_ref_head = 0;
+       ci->i_rdcache_gen = 0;
+       ci->i_rdcache_revoking = 0;
        ci->i_hold_caps_until = 0;
        INIT_LIST_HEAD(&ci->i_cap_delay_list);
 
@@ -1295,25 +1296,35 @@ static void ceph_inode_invalidate_pages(struct work_struct *work)
                                                  i_pg_inv_work);
        struct inode *inode = &ci->vfs_inode;
        u32 orig_gen;
-
-       dout(10, "invalidate_pages %p\n", inode);
+       int check = 0;
 
        spin_lock(&inode->i_lock);
 start:
+       dout(10, "invalidate_pages %p gen %d revoking %d\n", inode,
+            ci->i_rdcache_gen, ci->i_rdcache_revoking);
+       if (ci->i_rdcache_gen == 0 ||
+           ci->i_rdcache_revoking != ci->i_rdcache_gen) {
+               BUG_ON(ci->i_rdcache_revoking > ci->i_rdcache_gen);
+               /* nevermind! */
+               ci->i_rdcache_revoking = 0;
+               spin_unlock(&inode->i_lock);
+               return;
+       }
        orig_gen = ci->i_rdcache_gen;
        spin_unlock(&inode->i_lock);
 
        truncate_inode_pages(&inode->i_data, 0);
-       ci->i_rdcache_pending = 0;
-
-       ceph_check_caps(ci, 1);
 
        spin_lock(&inode->i_lock);
-
-       if (orig_gen != ci->i_rdcache_gen)
-               goto start;
-
+       if (orig_gen == ci->i_rdcache_gen) {
+               ci->i_rdcache_gen = 0;
+               ci->i_rdcache_revoking = 0;
+               check = 1;
+       }
        spin_unlock(&inode->i_lock);
+
+       if (check)
+               ceph_check_caps(ci, 0);
 }
 
 

diff --git a/src/kernel/super.h b/src/kernel/super.h
index d9dfa851f1c68232dda3a3ac6e33878439ace8c4..6f5325a98b3038853e1142b27088023664eedda0 100644 (file)
--- a/src/kernel/super.h
+++ b/src/kernel/super.h
@@ -234,9 +234,10 @@ struct ceph_inode_info {
 
        /* held references to caps */
        int i_rd_ref, i_rdcache_ref, i_wr_ref;
-       int i_rdcache_pending;
        int i_wrbuffer_ref, i_wrbuffer_ref_head;
-       u32 i_rdcache_gen;
+       /* these are used for async page invalidates when RDCACHE caps are
+        * revoked. */
+       u32 i_rdcache_gen, i_rdcache_revoking;
 
        struct ceph_snap_realm *i_snap_realm; /* snap realm (if caps) */
        struct list_head i_snap_realm_item;
@@ -372,7 +373,7 @@ static inline int __ceph_caps_used(struct ceph_inode_info *ci)
        int used = 0;
        if (ci->i_rd_ref)
                used |= CEPH_CAP_RD;
-       if (ci->i_rdcache_ref || ci->i_rdcache_pending)
+       if (ci->i_rdcache_ref || ci->i_rdcache_gen)
                used |= CEPH_CAP_RDCACHE;
        if (ci->i_wr_ref)
                used |= CEPH_CAP_WR;