kclient: fix mds reconnect message encoding; include snaprealms

diff --git a/src/include/ceph_fs.h b/src/include/ceph_fs.h
index 415ecabd8bcb1ebec97af6252dda56e2f13af216..6d78d6df73576f1bce0e31fa49f8bb29691da18b 100644 (file)
--- a/src/include/ceph_fs.h
+++ b/src/include/ceph_fs.h
@@ -872,6 +872,7 @@ struct ceph_mds_cap_reconnect {
 /* followed by encoded string */
 
 struct ceph_mds_snaprealm_reconnect {
+       __le64 ino;
        __le64 seq;
        __le64 parent;  /* parent realm */
 } __attribute__ ((packed));

diff --git a/src/kernel/mds_client.c b/src/kernel/mds_client.c
index 5378b7633b48207fbd60d3b49775b037b56274b3..8f1b33d4c50d9f2e47b10345445da1788b25f0c2 100644 (file)
--- a/src/kernel/mds_client.c
+++ b/src/kernel/mds_client.c
@@ -1293,7 +1293,10 @@ static void send_mds_reconnect(struct ceph_mds_client *mdsc, int mds)
        struct dentry *dentry;
        struct ceph_inode_info *ci;
        struct ceph_mds_cap_reconnect *rec;
-       int count;
+       int num_caps, num_realms = 0;
+       int got;
+       u64 next_snap_ino = 0;
+       u32 *pnum_realms;
 
        dout(1, "reconnect to recovering mds%d\n", mds);
 
@@ -1314,8 +1317,8 @@ static void send_mds_reconnect(struct ceph_mds_client *mdsc, int mds)
                     mds);
        }
 
-       mutex_unlock(&mdsc->mutex);  /* drop lock for duration */
-
+       down_read(&mdsc->snap_rwsem);
+       mutex_unlock(&mdsc->mutex);    /* drop lock for duration */
        if (session)
                mutex_lock(&session->s_mutex);
 
@@ -1339,29 +1342,20 @@ retry:
        /* traverse this session's caps */
        ceph_encode_8(&p, 0);
        ceph_encode_32(&p, session->s_nr_caps);
-       count = 0;
+       num_caps = 0;
        list_for_each(cp, &session->s_caps) {
+               struct inode *inode;
+
                cap = list_entry(cp, struct ceph_cap, session_caps);
                ci = cap->ci;
-               ceph_decode_need(&p, end, sizeof(u64) +
-                                sizeof(struct ceph_mds_cap_reconnect),
-                                needmore);
+               inode = &ci->vfs_inode;
+
                dout(10, " adding cap %p on ino %llx.%llx inode %p\n", cap,
-                    ceph_vinop(&ci->vfs_inode), &ci->vfs_inode);
-               ceph_encode_64(&p, ceph_ino(&ci->vfs_inode));
-               rec = p;
-               p += sizeof(*rec);
-               BUG_ON(p > end);
-               spin_lock(&ci->vfs_inode.i_lock);
-               cap->seq = 0;  /* reset cap seq */
-               rec->wanted = cpu_to_le32(__ceph_caps_wanted(ci));
-               rec->issued = cpu_to_le32(__ceph_caps_issued(ci, 0));
-               rec->size = cpu_to_le64(ci->vfs_inode.i_size);
-               ceph_encode_timespec(&rec->mtime, &ci->vfs_inode.i_mtime);
-               ceph_encode_timespec(&rec->atime, &ci->vfs_inode.i_atime);
-               spin_unlock(&ci->vfs_inode.i_lock);
+                    ceph_vinop(inode), inode);
+               ceph_decode_need(&p, end, sizeof(u64), needmore);
+               ceph_encode_64(&p, ceph_ino(inode));
 
-               dentry = d_find_alias(&ci->vfs_inode);
+               dentry = d_find_alias(inode);
                if (dentry) {
                        path = ceph_build_dentry_path(dentry, &pathlen,
                                                      &pathbase, 9999);
@@ -1375,10 +1369,52 @@ retry:
                }
                ceph_decode_need(&p, end, pathlen+4, needmore);
                ceph_encode_string(&p, end, path, pathlen);
+
+               ceph_decode_need(&p, end, sizeof(*rec), needmore);
+               rec = p;
+               p += sizeof(*rec);
+               BUG_ON(p > end);
+               spin_lock(&inode->i_lock);
+               cap->seq = 0;  /* reset cap seq */
+               rec->wanted = cpu_to_le32(__ceph_caps_wanted(ci));
+               rec->issued = cpu_to_le32(__ceph_caps_issued(ci, 0));
+               rec->size = cpu_to_le64(inode->i_size);
+               ceph_encode_timespec(&rec->mtime, &inode->i_mtime);
+               ceph_encode_timespec(&rec->atime, &inode->i_atime);
+               rec->snaprealm = cpu_to_le64(ci->i_snap_realm->ino);
+               spin_unlock(&inode->i_lock);
+
                kfree(path);
                dput(dentry);
-               count++;
+               num_caps++;
+       }
+
+       /* snaprealms */
+       next_snap_ino = 0;
+       pnum_realms = p;
+       ceph_decode_need(&p, end, sizeof(pnum_realms), needmore);
+       p += sizeof(*pnum_realms);
+       num_realms = 0;
+       while (1) {
+               struct ceph_snap_realm *realm;
+               struct ceph_mds_snaprealm_reconnect *rec;
+               got = radix_tree_gang_lookup(&mdsc->snap_realms,
+                                            (void **)&realm, next_snap_ino, 1);
+               if (!got)
+                       break;
+
+               dout(10, " adding snap realm %llx seq %lld parent %llx\n",
+                    realm->ino, realm->seq, realm->parent_ino);
+               ceph_decode_need(&p, end, sizeof(*rec), needmore);
+               rec = p;
+               rec->ino = cpu_to_le64(realm->ino);
+               rec->seq = cpu_to_le64(realm->seq);
+               rec->parent = cpu_to_le64(realm->parent_ino);
+               p += sizeof(*rec);
+               num_realms++;
+               next_snap_ino = realm->ino + 1;
        }
+       *pnum_realms = cpu_to_le32(num_realms);
 
 send:
        reply->front.iov_len = p - reply->front.iov_base;
@@ -1401,14 +1437,18 @@ out:
                mutex_unlock(&session->s_mutex);
                ceph_put_mds_session(session);
        }
+       up_read(&mdsc->snap_rwsem);
        mutex_lock(&mdsc->mutex);
        return;
 
 needmore:
+       /* bleh, this doesn't very accurately factor in snap realms,
+          but it's safe. */
+       num_caps += num_realms;
        newlen = len * (session->s_nr_caps+1);
-       do_div(newlen, (count+1));
+       do_div(newlen, (num_caps+1));
        dout(30, "i guessed %d, and did %d of %d, retrying with %d\n",
-            len, count, session->s_nr_caps, newlen);
+            len, num_caps, session->s_nr_caps, newlen);
        len = newlen;
        ceph_msg_put(reply);
        goto retry;

diff --git a/src/mds/Server.cc b/src/mds/Server.cc
index 02a61dbc2d745f8b074544b3d196d199d4187234..a44fb65dbd2ac95dee01bab0394ea799c041d200 100644 (file)
--- a/src/mds/Server.cc
+++ b/src/mds/Server.cc
@@ -380,23 +380,23 @@ void Server::handle_client_reconnect(MClientReconnect *m)
   } else {
     
     // snaprealms
-    for (map<inodeno_t, ceph_mds_snaprealm_reconnect>::iterator p = m->realms.begin();
+    for (vector<ceph_mds_snaprealm_reconnect>::iterator p = m->realms.begin();
         p != m->realms.end();
         p++) {
-      CInode *in = mdcache->get_inode(p->first);
+      CInode *in = mdcache->get_inode(inodeno_t(p->ino));
       if (in) {
        assert(in->snaprealm);
        if (in->snaprealm->have_past_parents_open()) {
          dout(15) << "open snaprealm (w/ past parents) on " << *in << dendl;
-         mdcache->finish_snaprealm_reconnect(from, in->snaprealm, snapid_t(p->second.seq));
+         mdcache->finish_snaprealm_reconnect(from, in->snaprealm, snapid_t(p->seq));
        } else {
          dout(15) << "open snaprealm (w/o past parents) on " << *in << dendl;
-         mdcache->add_reconnected_snaprealm(from, p->first, snapid_t(p->second.seq));
+         mdcache->add_reconnected_snaprealm(from, inodeno_t(p->ino), snapid_t(p->seq));
        }
       } else {
-       dout(15) << "open snaprealm (w/o inode) on " << p->first
-                << " seq " << p->second.seq << dendl;
-       mdcache->add_reconnected_snaprealm(from, p->first, snapid_t(p->second.seq));
+       dout(15) << "open snaprealm (w/o inode) on " << inodeno_t(p->ino)
+                << " seq " << p->seq << dendl;
+       mdcache->add_reconnected_snaprealm(from, inodeno_t(p->ino), snapid_t(p->seq));
       }
     }
 

diff --git a/src/messages/MClientReconnect.h b/src/messages/MClientReconnect.h
index 9c6bae72a204e3e8914af764e06bb5d4c14e4ca2..edc83e2180ce3b2efb75b2eb7fc80226a58afd53 100644 (file)
--- a/src/messages/MClientReconnect.h
+++ b/src/messages/MClientReconnect.h
@@ -22,7 +22,7 @@ class MClientReconnect : public Message {
 public:
   __u8 closed;  // true if this session was closed by the client.
   map<inodeno_t, cap_reconnect_t>  caps;   // only head inodes
-  map<inodeno_t, ceph_mds_snaprealm_reconnect> realms;
+  vector<ceph_mds_snaprealm_reconnect> realms;
 
   MClientReconnect() : Message(CEPH_MSG_CLIENT_RECONNECT),
                       closed(false) { }
@@ -41,8 +41,11 @@ public:
     caps[ino] = cap_reconnect_t(path, wanted, issued, sz, mt, at, sr);
   }
   void add_snaprealm(inodeno_t ino, snapid_t seq, inodeno_t parent) {
-    realms[ino].seq = seq;
-    realms[ino].parent = parent;
+    ceph_mds_snaprealm_reconnect r;
+    r.ino = ino;
+    r.seq = seq;
+    r.parent = parent;
+    realms.push_back(r);
   }
 
   void encode_payload() {

diff --git a/src/vstartnew.sh b/src/vstartnew.sh
index 42295005d271b3fc5467960d7c052dd88a2f50b3..57c040bca264b9ec33a4bf296a527352cd071e5a 100755 (executable)
--- a/src/vstartnew.sh
+++ b/src/vstartnew.sh
@@ -41,17 +41,18 @@ $CEPH_BIN/cmonctl osd setmap -i .ceph_osdmap
 for osd in 0 #1 #2 3 #4 5 6 7 8 9 10 11 12 13 14 15
 do
  echo mkfs osd$osd
- $SUDO $CEPH_BIN/cosd --debug_journal 20 --mkfs_for_osd $osd dev/osd$osd  --debug_journal 20 --debug_osd 20 --debug_filestore 20 --debug_ebofs 20 # initialize empty object store
-# echo valgrind --leak-check=full --show-reachable=yes $CEPH_BIN/cosd dev/osd$osd --debug_ms 1 --debug_osd 20 --debug_filestore 10 --debug_ebofs 20 #1>out/o$osd #& #--debug_osd 40
+ $SUDO $CEPH_BIN/cosd --debug_journal 20 --mkfs_for_osd $osd dev/osd$osd # --debug_journal 20 --debug_osd 20 --debug_filestore 20 --debug_ebofs 20 
  echo start osd$osd
  $SUDO $CEPH_BIN/cosd -m $IP:$CEPH_PORT dev/osd$osd -d --debug_ms 1 --debug_journal 20 --debug_osd 20 --debug_filestore 20 --debug_ebofs 20
+# echo valgrind --leak-check=full --show-reachable=yes $CEPH_BIN/cosd dev/osd$osd --debug_ms 1 --debug_osd 20 --debug_filestore 10 --debug_ebofs 20 #1>out/o$osd #& #--debug_osd 40
 done
 
 # mds
 ARGS="--mds_cache_size 500 --mds_log_max_segments 2 --debug_ms 1 --debug_mds 20"
+#valgrind --tool=massif $CEPH_BIN/cmds $ARGS --mds_log_max_segments 2 --mds_thrash_fragments 0 --mds_thrash_exports 0 > m  #--debug_ms 20
 $CEPH_BIN/cmds -d $ARGS --mds_log_max_segments 2 --mds_thrash_fragments 0 --mds_thrash_exports 0 #--debug_ms 20
-$CEPH_BIN/cmds -d $ARGS --mds_thrash_fragments 0 --mds_thrash_exports 0 #--debug_ms 20
-$CEPH_BIN/cmonctl mds set_max_mds 2
+#$CEPH_BIN/cmds -d $ARGS --mds_thrash_fragments 0 --mds_thrash_exports 0 #--debug_ms 20
+#$CEPH_BIN/cmonctl mds set_max_mds 2
 
 echo "started.  stop.sh to stop.  see out/* (e.g. 'tail -f out/????') for debug output."