kclient: trim expired rdcaps; ignored issued but timed out caps

Introduce new CEPH_CAP_EXPIREABLE mask to control which caps (if
alone) time out.

diff --git a/src/TODO b/src/TODO
index ea8eec21f81c80507957a8f246f08c698802becb..e33d84f0fddee6859dc2a0a6caab2fcace4f671d 100644 (file)
--- a/src/TODO
+++ b/src/TODO
@@ -46,13 +46,11 @@ timer
 
 
 caps
-- issue excl leases to loner if open for write...
-
-
 - kclient
   - only pin caps with dirty metadata? and/or wanted != 0?
   - and/or, put unwanted caps on an lru list, and expire?
 - mds
+  - trim expired rdcaps
   - segregate wanted/unwanted caps?  rd/wr caps?
 
 

diff --git a/src/config.cc b/src/config.cc
index 77b0b8183e02da929c34211a549e41f2a1186589..e279ce469b24a7aaf5907789928018e40f591759 100644 (file)
--- a/src/config.cc
+++ b/src/config.cc
@@ -343,7 +343,7 @@ md_config_t g_conf = {
   mds_client_prealloc_inos: 20,
   mds_early_reply: true,
 
-  mds_rdcap_ttl_ms: 30 * 10000,
+  mds_rdcap_ttl_ms: 60*1000,
 
   mds_log: true,
   mds_log_max_events: -1,

diff --git a/src/include/ceph_fs.h b/src/include/ceph_fs.h
index a8a44ec19cb4732f079ca57be4976750d9ef5ae4..98111e96eb2bb18e9dec285bd9f1ef44f2656052 100644 (file)
--- a/src/include/ceph_fs.h
+++ b/src/include/ceph_fs.h
@@ -952,6 +952,12 @@ static inline int ceph_flags_to_mode(int flags)
 #define CEPH_CAP_ANY_FILE_WR ((CEPH_CAP_GWR|CEPH_CAP_GWRBUFFER) << CEPH_CAP_SFILE)
 #define CEPH_CAP_ANY_WR   (CEPH_CAP_ANY_EXCL | CEPH_CAP_ANY_FILE_WR)
 
+/*
+ * these cap bits time out, if no others are held and nothing is
+ * registered as 'wanted' by the client.
+ */
+#define CEPH_CAP_EXPIREABLE (CEPH_CAP_PIN|CEPH_CAP_ANY_RDCACHE)
+
 
 static inline int ceph_caps_for_mode(int mode)
 {

diff --git a/src/kernel/caps.c b/src/kernel/caps.c
index 344cddc1d5aaf42d6915658420779d8ab701df3c..4a6528f77de4f48c59207ac259476448199a7344 100644 (file)
--- a/src/kernel/caps.c
+++ b/src/kernel/caps.c
@@ -226,7 +226,7 @@ retry:
                session->s_nr_caps++;
                INIT_LIST_HEAD(&cap->session_rdcaps);
        }
-       if ((cap->issued & ~(CEPH_CAP_ANY_RDCACHE|CEPH_CAP_PIN)) == 0) {
+       if ((cap->issued & ~CEPH_CAP_EXPIREABLE) == 0) {
                /* move to tail of session rdcaps lru */
                if (!list_empty(&cap->session_rdcaps))
                    list_del(&cap->session_rdcaps);
@@ -284,6 +284,15 @@ int __ceph_caps_issued(struct ceph_inode_info *ci, int *implemented)
                             cap, ceph_cap_string(cap->issued), cap->gen, gen);
                        continue;
                }
+
+               if (time_after_eq(jiffies, cap->expires) &&
+                   (cap->issued & ~CEPH_CAP_EXPIREABLE) == 0) {
+                       dout(30, "__ceph_caps_issued %p cap %p issued %s "
+                            "but readonly and expired\n", &ci->vfs_inode,
+                            cap, ceph_cap_string(cap->issued));
+                       continue;
+               }
+
                dout(30, "__ceph_caps_issued %p cap %p issued %s\n",
                     &ci->vfs_inode, cap, ceph_cap_string(cap->issued));
                have |= cap->issued;
@@ -1665,3 +1674,43 @@ void ceph_check_delayed_caps(struct ceph_mds_client *mdsc)
        spin_unlock(&mdsc->cap_delay_lock);
 }
 
+
+/*
+ * caller must hold session s_mutex
+ */
+void ceph_trim_session_rdcaps(struct ceph_mds_session *session)
+{
+       struct inode *inode;
+       struct ceph_cap *cap;
+       struct list_head *p, *n;
+       int wanted;
+
+       dout(10, "trim_rdcaps for mds%d\n", session->s_mds);
+       list_for_each_safe(p, n, &session->s_rdcaps) {
+               cap = list_entry(p, struct ceph_cap, session_rdcaps);
+
+               inode = &cap->ci->vfs_inode;
+               igrab(inode);
+               spin_lock(&inode->i_lock);
+
+               if (time_before(jiffies, cap->expires)) {
+                       dout(20, " stopping at %p cap %p expires %lu > %lu\n",
+                            inode, cap, cap->expires, jiffies);
+                       spin_unlock(&inode->i_lock);
+                       iput(inode);
+                       break;
+               }
+
+               /* wanted? */
+               wanted = __ceph_caps_wanted(cap->ci);
+               if (wanted == 0) {
+                       dout(20, " dropping %p cap %p\n", inode, cap);
+                       __ceph_remove_cap(cap);
+               } else {
+                       dout(20, " keeping %p cap %p (wanted %s)\n", inode, cap,
+                            ceph_cap_string(wanted));
+               }
+               spin_unlock(&inode->i_lock);
+               iput(inode);
+       }
+}

diff --git a/src/kernel/mds_client.c b/src/kernel/mds_client.c
index e1c9013f8f33b4d51d4f79027785a28dc78b63b8..a4a803fa416a7aae87967e66dd33ac94d245630a 100644 (file)
--- a/src/kernel/mds_client.c
+++ b/src/kernel/mds_client.c
@@ -1930,6 +1930,7 @@ static void delayed_work(struct work_struct *work)
                if (renew_caps)
                        send_renew_caps(mdsc, s);
                trim_session_leases(s);
+               ceph_trim_session_rdcaps(s);
                mutex_unlock(&s->s_mutex);
                ceph_put_mds_session(s);
 

diff --git a/src/kernel/super.h b/src/kernel/super.h
index 9fda42f28d7113daeba9d8f461cceabe87697e86..d46b4448040b72fb08b844d3843d50a5eead4749 100644 (file)
--- a/src/kernel/super.h
+++ b/src/kernel/super.h
@@ -669,7 +669,9 @@ extern void __ceph_flush_snaps(struct ceph_inode_info *ci,
                               struct ceph_mds_session **psession);
 extern void ceph_check_caps(struct ceph_inode_info *ci, int delayed, int drop);
 extern void ceph_check_delayed_caps(struct ceph_mds_client *mdsc);
-extern inline void ceph_release_caps(struct inode *inode, int mask)
+void ceph_trim_session_rdcaps(struct ceph_mds_session *session);
+
+static inline void ceph_release_caps(struct inode *inode, int mask)
 {
        ceph_check_caps(ceph_inode(inode), 1, mask);
 }