kclient: trim expired dentry and inode leases

diff --git a/src/kernel/inode.c b/src/kernel/inode.c
index 28de8e38f8b45e6c5603ce6e98cb4f855442e291..0129d0db70bd39d2054897835b4546c4793e9e66 100644 (file)
--- a/src/kernel/inode.c
+++ b/src/kernel/inode.c
@@ -193,7 +193,7 @@ no_change:
 }
 
 /* 
- * caller should hold session s_mutex.
+ * caller must hold session s_mutex.
  */
 void ceph_update_inode_lease(struct inode *inode, 
                             struct ceph_mds_reply_lease *lease,
@@ -215,15 +215,19 @@ void ceph_update_inode_lease(struct inode *inode,
                return;
 
        spin_lock(&inode->i_lock);
-       if (ttl > ci->i_lease_ttl) {
+       /*
+        * be careful: we can't remove lease from a different session
+        * without holding that other session's s_mutex.  so don't.
+        */
+       if (ttl > ci->i_lease_ttl &&
+           (!ci->i_lease_session || ci->i_lease_session == session)) {
                ci->i_lease_ttl = ttl;
                ci->i_lease_mask = le16_to_cpu(lease->mask);
-               if (ci->i_lease_session) {
-                       list_del(&ci->i_lease_item);
-               } else
+               if (!ci->i_lease_session) {
+                       ci->i_lease_session = session;
                        is_new = 1;
-               ci->i_lease_session = session;
-               list_add(&ci->i_lease_item, &session->s_inode_leases);
+               } 
+               list_move_tail(&ci->i_lease_item, &session->s_inode_leases);
        }
        spin_unlock(&inode->i_lock);
        if (is_new) {
@@ -232,26 +236,6 @@ void ceph_update_inode_lease(struct inode *inode,
        }
 }
 
-void ceph_revoke_inode_lease(struct ceph_inode_info *ci, int mask)
-{
-       int drop = 0;
-
-       spin_lock(&ci->vfs_inode.i_lock);
-       dout(10, "revoke_inode_lease on inode %p, mask %d -> %d\n", 
-            &ci->vfs_inode, ci->i_lease_mask, ci->i_lease_mask & ~mask);
-       ci->i_lease_mask &= ~mask;
-       if (ci->i_lease_mask == 0) {
-               list_del(&ci->i_lease_item);
-               ci->i_lease_session = 0;
-               drop = 1;
-       }
-       spin_unlock(&ci->vfs_inode.i_lock);
-       if (drop) {
-               dout(10, "lease iput on %p\n", &ci->vfs_inode);
-               iput(&ci->vfs_inode);
-       }
-}
-
 /*
  * check if inode lease is valid for a given mask
  */
@@ -320,20 +304,18 @@ void ceph_update_dentry_lease(struct dentry *dentry,
                        kfree(di);
                        goto fail_unlock;
                }
-               is_new = 1;
                di->dentry = dentry;
                dentry->d_fsdata = di;
+               di->lease_session = session;
+               list_add(&di->lease_item, &session->s_dentry_leases);
+               is_new = 1;
+       } else {
+               /* touch existing */
+               if (di->lease_session != session)
+                       goto fail_unlock;
+               list_move_tail(&di->lease_item, &session->s_dentry_leases);
        }
-
-       /* update */
        dentry->d_time = ttl;
-
-       /* (re)add to session lru */
-       if (!is_new && di->lease_session)
-               list_del(&di->lease_item);
-       di->lease_session = session;          
-       list_add(&di->lease_item, &session->s_dentry_leases);
-       
        spin_unlock(&dentry->d_lock);
        if (is_new) {
                dout(10, "lease dget on %p\n", dentry);
@@ -345,29 +327,6 @@ fail_unlock:
        spin_unlock(&dentry->d_lock);    
 }
 
-void ceph_revoke_dentry_lease(struct dentry *dentry)
-{
-       struct ceph_dentry_info *di;
-       struct ceph_mds_session *session;
-       int drop = 0;
-
-       /* detach from dentry */
-       spin_lock(&dentry->d_lock);
-       di = ceph_dentry(dentry);
-       if (di) {
-               session = di->lease_session;
-               list_del(&di->lease_item);
-               kfree(di);
-               drop = 1;
-               dentry->d_fsdata = 0;
-       }
-       spin_unlock(&dentry->d_lock);
-       if (drop) {
-               dout(10, "lease dput on %p\n", dentry);
-               dput(dentry);
-       }
-}
-
 /*
  * check if dentry lease is valid
  */

diff --git a/src/kernel/mds_client.c b/src/kernel/mds_client.c
index e1b95968e42883d4cf0c6302b31d57da98f9cabe..2b8a29f719c103c4e5581270f51a8a875c98ade5 100644 (file)
--- a/src/kernel/mds_client.c
+++ b/src/kernel/mds_client.c
@@ -264,12 +264,10 @@ static struct ceph_mds_session *__get_session(struct ceph_mds_client *mdsc,
 void put_session(struct ceph_mds_session *s)
 {
        BUG_ON(s == NULL);
-       dout(10, "put_session %p %d -> %d\n", s,
+       dout(30, "put_session %p %d -> %d\n", s,
             atomic_read(&s->s_ref), atomic_read(&s->s_ref)-1);
-       if (atomic_dec_and_test(&s->s_ref)) {
-               dout(10, "KFREE %p\n", s);
+       if (atomic_dec_and_test(&s->s_ref))
                kfree(s);
-       }
 }
 
 /*
@@ -591,6 +589,9 @@ static int close_session(struct ceph_mds_client *mdsc,
        return 0;
 }
 
+/*
+ * caller must hold session s_mutex
+ */
 static void remove_session_caps(struct ceph_mds_session *session)
 {
        struct ceph_inode_cap *cap;
@@ -612,6 +613,107 @@ static void remove_session_caps(struct ceph_mds_session *session)
        BUG_ON(session->s_nr_caps > 0);
 }
 
+/*
+ * caller must hold session s_mutex
+ */
+void revoke_dentry_lease(struct dentry *dentry)
+{
+       struct ceph_dentry_info *di;
+       struct ceph_mds_session *session;
+       int drop = 0;
+
+       /* detach from dentry */
+       spin_lock(&dentry->d_lock);
+       di = ceph_dentry(dentry);
+       if (di) {
+               session = di->lease_session;
+               list_del(&di->lease_item);
+               kfree(di);
+               drop = 1;
+               dentry->d_fsdata = 0;
+       }
+       spin_unlock(&dentry->d_lock);
+       if (drop) {
+               dout(10, "lease dput on %p\n", dentry);
+               dput(dentry);
+       }
+}
+
+/*
+ * caller must hold session s_mutex
+ */
+void revoke_inode_lease(struct ceph_inode_info *ci, int mask)
+{
+       int drop = 0;
+
+       spin_lock(&ci->vfs_inode.i_lock);
+       dout(10, "revoke_inode_lease on inode %p, mask %d -> %d\n", 
+            &ci->vfs_inode, ci->i_lease_mask, ci->i_lease_mask & ~mask);
+       ci->i_lease_mask &= ~mask;
+       if (ci->i_lease_mask == 0) {
+               list_del_init(&ci->i_lease_item);
+               ci->i_lease_session = 0;
+               drop = 1;
+       }
+       spin_unlock(&ci->vfs_inode.i_lock);
+       if (drop) {
+               dout(10, "lease iput on %p\n", &ci->vfs_inode);
+               iput(&ci->vfs_inode);
+       }
+}
+
+/*
+ * remove expire leases for this session.  unpint parent
+ * inode/dentries, so that [di]cache can prune them.
+ */
+static void trim_session_leases(struct ceph_mds_session *session)
+{
+       struct ceph_inode_info *ci;
+       struct ceph_dentry_info *di;
+       struct dentry *dentry;
+       
+       dout(20, "trim_session_leases on session %p\n", session);
+
+       /* inodes */
+       while (!list_empty(&session->s_inode_leases)) {
+               ci = list_first_entry(&session->s_inode_leases, 
+                                     struct ceph_inode_info, i_lease_item);
+               spin_lock(&ci->vfs_inode.i_lock);
+               if (ci->i_lease_ttl > jiffies) {
+                       spin_unlock(&ci->vfs_inode.i_lock);
+                       break;
+               }
+               dout(20, "trim_session_leases inode %p mask %d\n", 
+                    &ci->vfs_inode, ci->i_lease_mask);
+               ci->i_lease_session = 0;
+               ci->i_lease_mask = 0;
+               list_del_init(&ci->i_lease_item);
+               spin_unlock(&ci->vfs_inode.i_lock);
+               iput(&ci->vfs_inode);
+       }
+
+       /* dentries */
+       while (!list_empty(&session->s_dentry_leases)) {
+               di = list_first_entry(&session->s_dentry_leases, 
+                                     struct ceph_dentry_info, lease_item);
+               dentry = di->dentry;
+               spin_lock(&dentry->d_lock);
+               if (dentry->d_time > jiffies) {
+                       spin_unlock(&dentry->d_lock);
+                       break;
+               }
+               dout(20, "trim_session_leases dentry %p\n", dentry);
+               list_del(&di->lease_item);
+               kfree(di);
+               dentry->d_fsdata = 0;
+               spin_unlock(&dentry->d_lock);
+               dput(dentry);
+       }
+}
+
+/*
+ * caller must hold session s_mutex
+ */
 static void remove_session_leases(struct ceph_mds_session *session)
 {
        struct ceph_inode_info *ci;
@@ -624,7 +726,7 @@ static void remove_session_leases(struct ceph_mds_session *session)
                ci = list_entry(session->s_inode_leases.next, 
                                struct ceph_inode_info, i_lease_item);
                dout(10, "removing lease from inode %p\n", &ci->vfs_inode);
-               ceph_revoke_inode_lease(ci, ci->i_lease_mask);
+               revoke_inode_lease(ci, ci->i_lease_mask);
        }
 
        /* dentries */
@@ -632,7 +734,7 @@ static void remove_session_leases(struct ceph_mds_session *session)
                di = list_entry(session->s_dentry_leases.next, 
                                struct ceph_dentry_info, lease_item);
                dout(10, "removing lease from dentry %p\n", di->dentry);
-               ceph_revoke_dentry_lease(di->dentry);
+               revoke_dentry_lease(di->dentry);
        }
 }
 
@@ -1370,20 +1472,6 @@ int send_renewcaps(struct ceph_mds_client *mdsc, int mds)
        return 0;
 }
 
-int ceph_mdsc_renew_caps(struct ceph_mds_client *mdsc)
-{
-       int i;
-       dout(10, "renew_caps\n");
-       for (i = 0; i < mdsc->max_sessions; i++) {
-               if (mdsc->sessions[i] == 0 ||
-                   mdsc->sessions[i]->s_state < CEPH_MDS_SESSION_OPEN)
-                       continue;
-               send_renewcaps(mdsc, i);
-       }
-       return 0;
-}
-
-
 /*
  * leases
  */
@@ -1442,7 +1530,7 @@ void ceph_mdsc_handle_lease(struct ceph_mds_client *mdsc, struct ceph_msg *msg)
 
        /* inode */
        ci = ceph_inode(inode);
-       ceph_revoke_inode_lease(ci, mask);
+       revoke_inode_lease(ci, mask);
 
        /* dentry */
        if (mask & CEPH_LOCK_DN) {
@@ -1455,7 +1543,7 @@ void ceph_mdsc_handle_lease(struct ceph_mds_client *mdsc, struct ceph_msg *msg)
                dentry = d_lookup(parent, &dname);
                if (!dentry)
                        goto release;
-               ceph_revoke_dentry_lease(dentry);
+               revoke_dentry_lease(dentry);
                dout(10, "lease revoked on dentry %p\n", dentry);
                dput(dentry);
        } 
@@ -1541,11 +1629,10 @@ void schedule_delayed(struct ceph_mds_client *mdsc)
        /*
         * renew at 1/2 the advertised timeout period.
         */
-       int delay = mdsc->mdsmap->m_cap_bit_timeout >> 1;
+       int delay = 5;
        unsigned hz = round_jiffies_relative(HZ * delay);
-       int r;
        dout(-10, "schedule_delayed for %d seconds (%u hz)\n", delay, hz);
-       r = schedule_delayed_work(&mdsc->delayed_work, hz);
+       schedule_delayed_work(&mdsc->delayed_work, hz);
 }
 
 void delayed_work(struct work_struct *work)
@@ -1553,20 +1640,34 @@ void delayed_work(struct work_struct *work)
        int i;
        struct ceph_mds_client *mdsc =
                container_of(work, struct ceph_mds_client, delayed_work.work);
-
-       dout(-10, "delayed_work on %p\n", mdsc);
+       struct ceph_mds_session *session;
+       int renew_interval = mdsc->mdsmap->m_cap_bit_timeout >> 1;
+       int renew_caps = (jiffies >= HZ*renew_interval + mdsc->last_renew_caps);
+       
+       dout(-10, "delayed_work on %p renew_caps=%d\n", mdsc, renew_caps);
 
        /* renew caps */
        spin_lock(&mdsc->lock);
+       if (renew_caps)
+               mdsc->last_renew_caps = jiffies;
        for (i = 0; i < mdsc->max_sessions; i++) {
-               if (mdsc->sessions[i] == 0 ||
-                   mdsc->sessions[i]->s_state < CEPH_MDS_SESSION_OPEN)
+               session = __get_session(mdsc, i);
+               if (session == 0 ||
+                   session->s_state < CEPH_MDS_SESSION_OPEN) {
+                       put_session(session);
                        continue;
+               }
                spin_unlock(&mdsc->lock);
-               send_renewcaps(mdsc, i);
+               down(&session->s_mutex);
+
+               if (renew_caps)
+                       send_renewcaps(mdsc, i);
+               trim_session_leases(session);
+
+               up(&session->s_mutex);
+               put_session(session);
                spin_lock(&mdsc->lock);
        }
-
        spin_unlock(&mdsc->lock);
        schedule_delayed(mdsc);
 }
@@ -1585,6 +1686,7 @@ void ceph_mdsc_init(struct ceph_mds_client *mdsc, struct ceph_client *client)
        init_completion(&mdsc->map_waiters);
        init_completion(&mdsc->session_close_waiters);
        INIT_DELAYED_WORK(&mdsc->delayed_work, delayed_work);
+       mdsc->last_renew_caps = jiffies;
 }
 
 void ceph_mdsc_stop(struct ceph_mds_client *mdsc)

diff --git a/src/kernel/mds_client.h b/src/kernel/mds_client.h
index 50a83036338a5950ab947038f6e59b9e65747bb2..96aa5f40a717316fa6283e86435987ca8922bb24 100644 (file)
--- a/src/kernel/mds_client.h
+++ b/src/kernel/mds_client.h
@@ -105,6 +105,7 @@ struct ceph_mds_client {
        __u64                   last_requested_map;
        struct completion       map_waiters, session_close_waiters;
        struct delayed_work     delayed_work;  /* delayed work */
+       unsigned long last_renew_caps;
 };
 
 extern const char* ceph_mds_op_name(int op);

diff --git a/src/kernel/super.h b/src/kernel/super.h
index 2f45705e326cf9da25d131c5295b4a56d14cfec9..08a9f47356ddf987b47ab71843c80e2318777c41 100644 (file)
--- a/src/kernel/super.h
+++ b/src/kernel/super.h
@@ -198,9 +198,6 @@ static inline struct ceph_dentry_info *ceph_dentry(struct dentry *dentry)
        return (struct ceph_dentry_info *)dentry->d_fsdata;
 }
 
-extern void ceph_revoke_inode_lease(struct ceph_inode_info *ci, int mask);
-extern void ceph_revoke_dentry_lease(struct dentry *dentry);
-
 static inline void ceph_queue_writeback(struct ceph_client *cl, 
                                        struct ceph_inode_info *ci)
 {