kclient: flush write caps on umount; refactor send_cap code out of ceph_check_cap

diff --git a/src/kernel/inode.c b/src/kernel/inode.c
index 62c031c35c2c5cc241f96340ad5c26ef87bd51c2..db6c8b727cb259660ce31ae6d026fd330c4c9e9c 100644 (file)
--- a/src/kernel/inode.c
+++ b/src/kernel/inode.c
@@ -778,13 +778,8 @@ void ceph_check_caps(struct ceph_inode_info *ci, int is_delayed)
        struct ceph_inode_cap *cap;
        struct list_head *p;
        int wanted, used;
-       int keep;
-       __u64 seq;
-       __u64 size, max_size;
-       struct timespec mtime, atime;
-       int mds;
        struct ceph_mds_session *session = 0;  /* if non-NULL, i hold s_mutex */
-       int removed_last = 0;
+       int removed_last;
 
 retry:
        spin_lock(&inode->i_lock);
@@ -805,7 +800,7 @@ retry:
        }
 
        list_for_each(p, &ci->i_caps) {
-               int revoking, dropping;
+               int revoking;
                cap = list_entry(p, struct ceph_inode_cap, ci_caps);
 
                /* note: no side-effects allowed, until we take s_mutex */
@@ -857,46 +852,10 @@ ack:
                }
 
                /* ok */
-               dropping = cap->issued & ~wanted;
-               dout(10, " cap %p %d -> %d\n", cap, cap->issued,
-                    cap->issued & wanted);
-               cap->issued &= wanted;  /* drop bits we don't want */
-
-               if (revoking && (revoking && used) == 0)
-                       cap->implemented = cap->issued;
-
-               keep = cap->issued;
-               seq = cap->seq;
-               size = inode->i_size;
-               ci->i_reported_size = size;
-               max_size = ci->i_wanted_max_size;
-               ci->i_requested_max_size = max_size;
-               mtime = inode->i_mtime;
-               atime = inode->i_atime;
-               mds = cap->mds;
-               if (wanted == 0) {
-                       __ceph_remove_cap(cap);
-                       removed_last = list_empty(&ci->i_caps);
-               }
-               spin_unlock(&inode->i_lock);
-
-               if (dropping & CEPH_CAP_RDCACHE) {
-                       dout(20, "invalidating pages on %p\n", inode);
-                       invalidate_mapping_pages(&inode->i_data, 0, -1);
-                       dout(20, "done invalidating pages on %p\n", inode);
-               }
-
-               ceph_mdsc_send_cap_ack(mdsc, ceph_ino(inode),
-                                      keep, wanted, seq,
-                                      size, max_size, &mtime, &atime, mds);
-
-               if (wanted == 0) {
-                       if (removed_last && !is_delayed)
-                               cancel_delayed_work_sync(&ci->i_cap_dwork);
-                       iput(inode);  /* removed cap */
-                       if (removed_last)
-                               goto out;
-               }
+               removed_last = __ceph_mdsc_send_cap(mdsc, session, cap,
+                                                   used, wanted, !is_delayed);
+               if (removed_last)
+                       goto out;
                up(&session->s_mutex);
                goto retry;
        }

diff --git a/src/kernel/mds_client.c b/src/kernel/mds_client.c
index 4a13c6cf35c37d691bd2ecf20dc880da91ed4f92..bd575f4d13e816251e785336a82d5c251d705594 100644 (file)
--- a/src/kernel/mds_client.c
+++ b/src/kernel/mds_client.c
@@ -573,26 +573,6 @@ static int resume_session(struct ceph_mds_client *mdsc,
        return 0;
 }
 
-static int close_session(struct ceph_mds_client *mdsc,
-                        struct ceph_mds_session *session)
-{
-       int mds = session->s_mds;
-       struct ceph_msg *msg;
-
-       dout(10, "close_session to mds%d\n", mds);
-       session->s_state = CEPH_MDS_SESSION_CLOSING;
-       spin_unlock(&mdsc->lock);
-       msg = create_session_msg(CEPH_SESSION_REQUEST_CLOSE,
-                                session->s_cap_seq);
-       if (IS_ERR(msg)) {
-               spin_lock(&mdsc->lock);
-               return PTR_ERR(msg);
-       }
-       send_msg_mds(mdsc, msg, mds);
-       spin_lock(&mdsc->lock);
-       return 0;
-}
-
 /*
  * caller must hold session s_mutex
  */
@@ -1379,10 +1359,10 @@ void check_new_map(struct ceph_mds_client *mdsc,
 
 /* caps */
 
-void ceph_mdsc_send_cap_ack(struct ceph_mds_client *mdsc, __u64 ino, int caps,
-                           int wanted, __u32 seq, __u64 size, __u64 max_size,
-                           struct timespec *mtime, struct timespec *atime,
-                           int mds)
+void send_cap_ack(struct ceph_mds_client *mdsc, __u64 ino, int caps,
+                 int wanted, __u32 seq, __u64 size, __u64 max_size,
+                 struct timespec *mtime, struct timespec *atime,
+                 int mds)
 {
        struct ceph_mds_file_caps *fc;
        struct ceph_msg *msg;
@@ -1463,8 +1443,7 @@ void ceph_mdsc_handle_filecaps(struct ceph_mds_client *mdsc,
        if (!inode) {
                dout(10, "wtf, i don't have ino %lu=%llx?  closing out cap\n",
                     inot, ino);
-               ceph_mdsc_send_cap_ack(mdsc, ino, 0, 0, seq,
-                                      size, 0, 0, 0, mds);
+               send_cap_ack(mdsc, ino, 0, 0, seq, size, 0, 0, 0, mds);
                goto no_inode;
        }
 
@@ -1516,6 +1495,115 @@ int send_renewcaps(struct ceph_mds_client *mdsc,
        return 0;
 }
 
+
+/*
+ * called with i_lock, then drops it.
+ * 
+ * returns true if we removed the last cap on this inode.
+ */
+int __ceph_mdsc_send_cap(struct ceph_mds_client *mdsc,
+                        struct ceph_mds_session *session,
+                        struct ceph_inode_cap *cap,
+                        int used, int wanted, int cancel_work)
+{
+       struct ceph_inode_info *ci = cap->ci;
+       struct inode *inode = &ci->vfs_inode;
+       int revoking = cap->implemented & ~cap->issued;
+       int dropping = cap->issued & ~wanted;
+       int keep;
+       __u64 seq;
+       __u64 size, max_size;
+       struct timespec mtime, atime;
+       int removed_last = 0;
+
+       dout(10, "__send_cap cap %p session %p %d -> %d\n", cap, cap->session,
+            cap->issued, cap->issued & wanted);
+       cap->issued &= wanted;  /* drop bits we don't want */
+       
+       if (revoking && (revoking && used) == 0)
+               cap->implemented = cap->issued;
+       
+       keep = cap->issued;
+       seq = cap->seq;
+       size = inode->i_size;
+       ci->i_reported_size = size;
+       max_size = ci->i_wanted_max_size;
+       ci->i_requested_max_size = max_size;
+       mtime = inode->i_mtime;
+       atime = inode->i_atime;
+       if (wanted == 0) {
+               __ceph_remove_cap(cap);
+               removed_last = list_empty(&ci->i_caps);
+       }
+       spin_unlock(&inode->i_lock);
+
+       if (dropping & CEPH_CAP_RDCACHE) {
+               dout(20, "invalidating pages on %p\n", inode);
+               invalidate_mapping_pages(&inode->i_data, 0, -1);
+               dout(20, "done invalidating pages on %p\n", inode);
+       }
+
+       send_cap_ack(mdsc, ceph_ino(inode),
+                    keep, wanted, seq,
+                    size, max_size, &mtime, &atime, session->s_mds);
+       
+       if (wanted == 0) {
+               if (removed_last && cancel_work)
+                       cancel_delayed_work_sync(&ci->i_cap_dwork);
+               iput(inode);  /* removed cap */
+       }
+       return removed_last;
+}
+
+
+static void flush_write_caps(struct ceph_mds_client *mdsc,
+                            struct ceph_mds_session *session)
+{
+       struct list_head *p, *n;
+       
+       list_for_each_safe (p, n, &session->s_caps) {
+               struct ceph_inode_cap *cap = 
+                       list_entry(p, struct ceph_inode_cap, session_caps);
+               struct inode *inode = &cap->ci->vfs_inode;
+               int used, wanted;
+
+               spin_lock(&inode->i_lock);
+               if ((cap->implemented & (CEPH_CAP_WR|CEPH_CAP_WRBUFFER)) == 0) {
+                       spin_unlock(&inode->i_lock);
+                       continue;
+               }
+               used = __ceph_caps_used(cap->ci);
+               wanted = __ceph_caps_wanted(cap->ci);
+               __ceph_mdsc_send_cap(mdsc, session, cap, used, wanted, 1);
+       }
+}
+
+static int close_session(struct ceph_mds_client *mdsc,
+                        struct ceph_mds_session *session)
+{
+       int mds = session->s_mds;
+       struct ceph_msg *msg;
+
+       dout(10, "close_session mds%d\n", mds);
+       down(&session->s_mutex);
+       
+       if (session->s_state >= CEPH_MDS_SESSION_CLOSING)
+               goto done;
+
+       flush_write_caps(mdsc, session);
+       
+       session->s_state = CEPH_MDS_SESSION_CLOSING;
+       msg = create_session_msg(CEPH_SESSION_REQUEST_CLOSE,
+                                session->s_cap_seq);
+       if (IS_ERR(msg))
+               return PTR_ERR(msg);
+       send_msg_mds(mdsc, msg, mds);
+
+done:
+       up(&session->s_mutex);
+       return 0;
+}
+
 /*
  * leases
  */
@@ -1737,6 +1825,7 @@ void ceph_mdsc_init(struct ceph_mds_client *mdsc, struct ceph_client *client)
 
 void ceph_mdsc_stop(struct ceph_mds_client *mdsc)
 {
+       struct ceph_mds_session *session;
        int i;
        int n;
 
@@ -1751,11 +1840,12 @@ void ceph_mdsc_stop(struct ceph_mds_client *mdsc)
                dout(10, "closing sessions\n");
                n = 0;
                for (i = 0; i < mdsc->max_sessions; i++) {
-                       if (mdsc->sessions[i] == 0 ||
-                           mdsc->sessions[i]->s_state >=
-                            CEPH_MDS_SESSION_CLOSING)
+                       session = __get_session(mdsc, i);
+                       if (!session)
                                continue;
-                       close_session(mdsc, mdsc->sessions[i]);
+                       spin_unlock(&mdsc->lock);
+                       close_session(mdsc, session);
+                       spin_lock(&mdsc->lock);
                        n++;
                }
                if (n == 0)

diff --git a/src/kernel/mds_client.h b/src/kernel/mds_client.h
index 5bd203e9441b2226f8b51039aaf13ceb35064a30..63ec1f5786e7cc902aa39850938bb08432f6c0e7 100644 (file)
--- a/src/kernel/mds_client.h
+++ b/src/kernel/mds_client.h
@@ -141,11 +141,9 @@ extern int ceph_mdsc_do_request(struct ceph_mds_client *mdsc,
                                struct ceph_mds_request *req);
 extern void ceph_mdsc_put_request(struct ceph_mds_request *req);
 
-extern void ceph_mdsc_send_cap_ack(struct ceph_mds_client *mdsc, __u64 ino,
-                                  int caps, int wanted, __u32 seq,
-                                  __u64 size, __u64 max_size,
-                                  struct timespec *mtime,
-                                  struct timespec *atime,
-                                  int mds);
+extern int __ceph_mdsc_send_cap(struct ceph_mds_client *mdsc,
+                               struct ceph_mds_session *session,
+                               struct ceph_inode_cap *cap,
+                               int used, int wanted, int cancel_work);
 
 #endif

diff --git a/src/kernel/super.h b/src/kernel/super.h
index 44598f35a5c9ab745892d7303d49550209a4f46d..0d29ea534d6699f91726889284ac1d25c80ca9e5 100644 (file)
--- a/src/kernel/super.h
+++ b/src/kernel/super.h
@@ -378,6 +378,7 @@ extern struct ceph_inode_cap *ceph_add_cap(struct inode *inode,
                                           struct ceph_mds_session *session,
                                           int fmode,
                                           u32 cap, u32 seq);
+extern void __ceph_remove_cap(struct ceph_inode_cap *cap);
 extern void ceph_remove_cap(struct ceph_inode_cap *cap);
 extern void ceph_remove_all_caps(struct ceph_inode_info *ci);
 extern int ceph_handle_cap_grant(struct inode *inode,