kclient: make clean_caps lru; explicitly release old caps

Use check_caps for now.  Eventually we want something more
efficient.

diff --git a/src/TODO b/src/TODO
index 199f8286600f76b8ba8f7382e09b898f3a7a600c..4af77ed9736453e2ed0ec51b9ec584eeb9542804 100644 (file)
--- a/src/TODO
+++ b/src/TODO
@@ -42,11 +42,11 @@ v0.8
 - allow chmod/whatever of root inode
 
 kclient caps
-- explicit release
+/- two session lists: dirty_caps, clean_caps
+/- time out and release clean caps explicitly
   - this makes open on client work better, since we care less about the mds 'wanted' value.
     (we only need to update it if/when we release the caps we still want, or async on open
      if we don't have them yet)
-- two session lists: dirty_caps, clean_caps
 
 alternative #1:
 - keep CAP_PIN on _all_ inodes, and don't pin inodes with caps.

diff --git a/src/include/ceph_fs.h b/src/include/ceph_fs.h
index ee5ac7a73e8e82bef95d14550c1d555b600cab3b..237c0e9629de43e9e90fccf007497c6c117ed5d6 100644 (file)
--- a/src/include/ceph_fs.h
+++ b/src/include/ceph_fs.h
@@ -1005,6 +1005,7 @@ static inline int ceph_flags_to_mode(int flags)
                           CEPH_CAP_FILE_EXCL)
 #define CEPH_CAP_ANY_FILE_WR (CEPH_CAP_FILE_WR|CEPH_CAP_FILE_WRBUFFER)
 #define CEPH_CAP_ANY_WR   (CEPH_CAP_ANY_EXCL | CEPH_CAP_ANY_FILE_WR)
+#define CEPH_CAP_ANY      (CEPH_CAP_ANY_RD|CEPH_CAP_ANY_EXCL|CEPH_CAP_ANY_FILE_WR|CEPH_CAP_PIN)
 
 /*
  * these cap bits time out, if no others are held and nothing is

diff --git a/src/kernel/caps.c b/src/kernel/caps.c
index 09b0c072fd5dea1269cc57a8753272a3acf6262e..9e2f1dea34f693f6a84732f1936eb88c7c1876cd 100644 (file)
--- a/src/kernel/caps.c
+++ b/src/kernel/caps.c
@@ -260,7 +260,6 @@ int ceph_add_cap(struct inode *inode,
        struct ceph_cap *cap;
        int mds = session->s_mds;
        int is_first = 0;
-       unsigned long duration;
 
        dout(10, "add_cap %p mds%d cap %llx %s seq %d\n", inode,
             session->s_mds, cap_id, ceph_cap_string(issued), seq);
@@ -356,12 +355,6 @@ retry:
        cap->mseq = mseq;
        cap->gen = session->s_cap_gen;
 
-       duration = ttl_ms * HZ / 1000;
-       cap->expires = ttl_from + duration;
-       cap->renew_after = ttl_from + (duration >> 1);
-       dout(30, " expires %lu, renew_after %lu (duration %lu, %u ms)\n",
-            cap->expires, cap->renew_after, duration, ttl_ms);
-
        if (fmode >= 0)
                __ceph_get_fmode(ci, fmode);
        spin_unlock(&inode->i_lock);
@@ -396,6 +389,20 @@ static int __cap_is_valid(struct ceph_cap *cap)
        return 1;
 }
 
+/*
+ * Touch cap timestamp and, if clean, move to end of lru.
+ */
+void __touch_cap(struct ceph_cap *cap)
+{
+       cap->last_used = jiffies;
+       if (!list_empty(&cap->session_clean_caps)) {
+               spin_lock(&cap->session->s_clean_caps_lock);
+               list_del_init(&cap->session_clean_caps);
+               list_add(&cap->session_clean_caps, &cap->session->s_clean_caps);
+               spin_unlock(&cap->session->s_clean_caps_lock);
+       }
+}
+
 /*
  * Return set of valid cap bits issued to us.  Note that caps time
  * out, and may be invalidated in bulk if the client session times out
@@ -418,6 +425,8 @@ int __ceph_caps_issued(struct ceph_inode_info *ci, int *implemented)
                have |= cap->issued;
                if (implemented)
                        *implemented |= cap->implemented;
+
+               __touch_cap(cap);
        }
        return have;
 }
@@ -825,6 +834,7 @@ void ceph_check_caps(struct ceph_inode_info *ci, int is_delayed, int drop,
        struct ceph_cap *cap;
        int file_wanted, used;
        int took_snap_rwsem = 0;             /* true if mdsc->snap_rwsem held */
+       int drop_session_lock = session ? 0:1;
        int want, retain, revoking, dirty, op;
        int mds = -1;   /* keep track of how far we've gone through i_caps list
                           to avoid an infinite loop on retry */
@@ -1011,7 +1021,7 @@ ack:
        }
        spin_unlock(&inode->i_lock);
 
-       if (session)
+       if (session && drop_session_lock)
                mutex_unlock(&session->s_mutex);
        if (took_snap_rwsem)
                up_read(&mdsc->snap_rwsem);
@@ -1788,7 +1798,6 @@ void ceph_handle_caps(struct ceph_mds_client *mdsc,
                        ceph_send_msg_mds(mdsc, msg, mds);
                } else if (r == 2) {
                        ceph_check_caps(ceph_inode(inode), 1, 0, session);
-                       goto done_unlocked;
                }
                break;
 
@@ -1809,7 +1818,6 @@ void ceph_handle_caps(struct ceph_mds_client *mdsc,
 
 done:
        mutex_unlock(&session->s_mutex);
-done_unlocked:
        ceph_put_mds_session(session);
 
        kfree(xattr_data);
@@ -1867,40 +1875,30 @@ void ceph_trim_session_clean_caps(struct ceph_mds_session *session)
        struct inode *inode;
        struct ceph_cap *cap;
        struct list_head *p, *n;
+       unsigned long cutoff = jiffies - HZ * 30;  /* fixme */
 
        dout(10, "trim_clean_caps for mds%d\n", session->s_mds);
-       /*
-       spin_lock(&session->s_rdcaps_lock);
-       list_for_each_safe(p, n, &session->s_rdcaps) {
-               int wanted, last_cap;
-
-               cap = list_entry(p, struct ceph_cap, session_rdcaps);
-               spin_unlock(&session->s_rdcaps_lock);
-
+       spin_lock(&session->s_clean_caps_lock);
+       list_for_each_safe(p, n, &session->s_clean_caps) {
+               cap = list_entry(p, struct ceph_cap, session_clean_caps);
                inode = &cap->ci->vfs_inode;
-               spin_lock(&inode->i_lock);
+               igrab(inode);
+               spin_unlock(&session->s_clean_caps_lock);
 
-               if (time_before(jiffies, cap->expires)) {
-                       dout(20, " stopping at %p cap %p expires %lu > %lu\n",
-                            inode, cap, cap->expires, jiffies);
+               spin_lock(&inode->i_lock);
+               if (time_before(cutoff, cap->last_used)) {
+                       dout(20, " stopping at %p cap %p last_used %lu > %lu\n",
+                            inode, cap, cap->last_used, cutoff);
                        spin_unlock(&inode->i_lock);
+                       iput(inode);
                        return;
                }
-
-               wanted = __ceph_caps_wanted(cap->ci);
-               dout(20, " dropping %p cap %p %s wanted %s\n", inode,
-                    cap, ceph_cap_string(cap->issued),
-                    ceph_cap_string(wanted));
-               BUG_ON(wanted);
-               last_cap = __ceph_remove_cap(cap);
                spin_unlock(&inode->i_lock);
-               if (last_cap)
-                       iput(inode);
-
-               spin_lock(&session->s_rdcaps_lock);
+               ceph_check_caps(cap->ci, 1, CEPH_CAP_ANY, session);
+               iput(inode);
+               spin_lock(&session->s_clean_caps_lock);
        }
-       spin_unlock(&session->s_rdcaps_lock);
-       */
+       spin_unlock(&session->s_clean_caps_lock);
 }
 
 /*

diff --git a/src/kernel/mds_client.c b/src/kernel/mds_client.c
index b35ecbf94dd929a63222a82fa7c6966f4ba0e10a..7e606bd618c1941a2874c5dc08fda88eae05646a 100644 (file)
--- a/src/kernel/mds_client.c
+++ b/src/kernel/mds_client.c
@@ -296,6 +296,7 @@ static struct ceph_mds_session *register_session(struct ceph_mds_client *mdsc,
        s->s_renew_requested = 0;
        INIT_LIST_HEAD(&s->s_caps);
        INIT_LIST_HEAD(&s->s_clean_caps);
+       spin_lock_init(&s->s_clean_caps_lock);
        s->s_nr_caps = 0;
        atomic_set(&s->s_ref, 1);
        INIT_LIST_HEAD(&s->s_waiting);

diff --git a/src/kernel/mds_client.h b/src/kernel/mds_client.h
index b88dc0eb651f35e7b5eb618feb7f28c17c550497..09886f05ee5dc0ea5be84c1e970604a2d5608242 100644 (file)
--- a/src/kernel/mds_client.h
+++ b/src/kernel/mds_client.h
@@ -116,7 +116,8 @@ struct ceph_mds_session {
        unsigned long     s_cap_ttl;  /* when session caps expire */
        unsigned long     s_renew_requested; /* last time we sent a renew req */
        struct list_head  s_caps;     /* all caps issued by this session */
-       struct list_head  s_clean_caps;   /* just the readonly caps */
+       struct list_head  s_clean_caps;       /* just the clean caps */
+       spinlock_t        s_clean_caps_lock;  /* protect s_clean_caps_lock */
        int               s_nr_caps;
        atomic_t          s_ref;
        struct list_head  s_waiting;  /* waiting requests */

diff --git a/src/kernel/super.h b/src/kernel/super.h
index 7ce8fdc2a08d56feab8cc9a78a17bb475236b6d5..d22743cc3fdd6f7040be679dee544015b8f5f4a5 100644 (file)
--- a/src/kernel/super.h
+++ b/src/kernel/super.h
@@ -154,8 +154,7 @@ struct ceph_cap {
        int flushing;     /* dirty fields being written back to mds */
        int mds_wanted;
        u32 seq, mseq, gen;
-       unsigned long expires;  /* if readonly and unwanted (jiffies) */
-       unsigned long renew_after, renew_from;  /* for _rdcap_ renewal */
+       unsigned long last_used;
 };
 
 /*