monmap epoch fix; mon sends monmap to client on mount

git-svn-id: https://ceph.svn.sf.net/svnroot/ceph@2164 29311d96-e01e-0410-9327-a35deaab8ce9

diff --git a/trunk/ceph/kernel/Makefile b/trunk/ceph/kernel/Makefile
index 3be93433beb56254704f1623f9d477e79f33e0ee..3f62291a3c1d3b33e497e0c14b0da6f4b56f841c 100644 (file)
--- a/trunk/ceph/kernel/Makefile
+++ b/trunk/ceph/kernel/Makefile
@@ -8,5 +8,5 @@ ceph-objs := super.o inode.o \
        bufferlist.o ktcp.o messenger.o \
        client.o \
        mds_client.o mdsmap.o \
-       mon_client.o monmap.o \
+       mon_client.o \
        osd_client.o crush/crush.o crush/mapper.o

diff --git a/trunk/ceph/kernel/client.c b/trunk/ceph/kernel/client.c
index a397bcce40f3f27d2704a76d13974ad18cf85ec4..03ffb907a7d688da531bf02a88fc06d99c256485 100644 (file)
--- a/trunk/ceph/kernel/client.c
+++ b/trunk/ceph/kernel/client.c
@@ -98,7 +98,7 @@ static int mount(struct ceph_client *client, struct ceph_mount_args *args)
        int which;
        char r;
        
-       client->mounting = 07;  /* wait for mon+mds+osd */
+       client->mounting = 0;  /* wait for mon+mds+osd */
 
        /* send mount request */
 trymount:
@@ -117,11 +117,11 @@ trymount:
        /* wait */
        dout(10, "mount waiting\n");
        err = wait_event_interruptible_timeout(client->mount_wq, 
-                                              (client->mounting == 0),
+                                              (find_first_zero_bit(&client->mounting, 4) == 4),
                                               6*HZ);
        if (err == -EINTR)
                return err; 
-       if (client->mounting) {
+       if (client->mounting < 7) {
                dout(10, "mount still waiting for mount, attempts=%d\n", attempts);
                if (--attempts)
                        goto trymount;
@@ -129,6 +129,7 @@ trymount:
        }
 
        /* get handle for mount path */
+       dout(10, "mount got all maps; opening root directory\n");
        err = ceph_mdsc_do(&client->mdsc, CEPH_MDS_OP_OPEN,
                           CEPH_INO_ROOT, args->path, 0, 0);
        if (err < 0)
@@ -142,16 +143,17 @@ trymount:
  * the monitor responds to monmap to indicate mount success.
  * (or, someday, to indicate a change in the monitor cluster?)
  */
-static void handle_mon_map(struct ceph_client *client, struct ceph_msg *msg)
+static void handle_monmap(struct ceph_client *client, struct ceph_msg *msg)
 {
        int err;
        
-       dout(1, "handle_mon_map");
+       dout(1, "handle_monmap had epoch %d\n", client->monc.monmap.epoch);
 
        /* parse */
-       err = ceph_monmap_decode(&client->monc.monmap, msg->front.iov_base, 
+       err = ceph_monmap_decode(&client->monc.monmap, 
+                                msg->front.iov_base, 
                                 msg->front.iov_base + msg->front.iov_len);
-       if (err != 0) 
+       if (err != 0)
                return;
        
        if (client->whoami < 0) {
@@ -220,11 +222,11 @@ void ceph_put_client(struct ceph_client *cl)
 }
 
 
-void got_first_map(struct ceph_client *client, int type)
+void got_first_map(struct ceph_client *client, int num)
 {
-       dout(10, "got_first_map type %d\n", type);
-       clear_bit(type, &client->mounting);
-       if (client->mounting == 0)
+       set_bit(num, &client->mounting);
+       dout(10, "got_first_map num %d mounting now %lu\n", num, client->mounting);
+       if (find_first_bit(&client->mounting, 4) == 4)
                wake_up(&client->mount_wq);
 }
 
@@ -247,9 +249,9 @@ void ceph_dispatch(struct ceph_client *client, struct ceph_msg *msg)
                /* me */
        case CEPH_MSG_MON_MAP:
                had = client->monc.monmap.epoch ? 1:0;
-               handle_mon_map(client, msg);
+               handle_monmap(client, msg);
                if (!had && client->monc.monmap.epoch)
-                       got_first_map(client, 4);
+                       got_first_map(client, 0);
                break;
 
                /* mds client */
@@ -257,7 +259,7 @@ void ceph_dispatch(struct ceph_client *client, struct ceph_msg *msg)
                had = client->mdsc.mdsmap ? 1:0;
                ceph_mdsc_handle_map(&client->mdsc, msg);
                if (!had && client->mdsc.mdsmap) 
-                       got_first_map(client, 2);
+                       got_first_map(client, 1);
                break;
        case CEPH_MSG_CLIENT_REPLY:
                ceph_mdsc_handle_reply(&client->mdsc, msg);
@@ -271,7 +273,7 @@ void ceph_dispatch(struct ceph_client *client, struct ceph_msg *msg)
                had = client->osdc.osdmap ? 1:0;
                ceph_osdc_handle_map(&client->osdc, msg);
                if (!had && client->osdc.osdmap) 
-                       got_first_map(client, 1);
+                       got_first_map(client, 2);
                break;
        case CEPH_MSG_OSD_OPREPLY:
                ceph_osdc_handle_reply(&client->osdc, msg);

diff --git a/trunk/ceph/kernel/client.h b/trunk/ceph/kernel/client.h
index 41e87847adee170218ef8bc4b6f8ef432d76f72b..9c2b02c9dd92086715f29c39cf107a49e39df0e7 100644 (file)
--- a/trunk/ceph/kernel/client.h
+++ b/trunk/ceph/kernel/client.h
@@ -11,9 +11,9 @@
 
 #include <linux/ceph_fs.h>
 #include <linux/wait.h>
+#include <linux/completion.h>
 
 #include "messenger.h"
-#include "monmap.h"
 
 #include "mon_client.h"
 #include "mds_client.h"
@@ -39,7 +39,7 @@ struct ceph_client {
        struct ceph_fsid fsid;
        atomic_t nref;
 
-       int mounting;   /* map bitset; 4=mon, 2=mds, 1=osd map */
+       unsigned long mounting;   /* map bitset; 4=mon, 2=mds, 1=osd map */
        wait_queue_head_t mount_wq;
 
        struct ceph_messenger *msgr;   /* messenger instance */

diff --git a/trunk/ceph/kernel/mds_client.c b/trunk/ceph/kernel/mds_client.c
index 63e4464e5bd804d5ec4e801363e76fbf65f691ed..1c7c26e5f03c6c9554f952da51a1151d1d783d97 100644 (file)
--- a/trunk/ceph/kernel/mds_client.c
+++ b/trunk/ceph/kernel/mds_client.c
@@ -87,8 +87,9 @@ static void register_session(struct ceph_mds_client *mdsc, int mds)
 
        /* register */
        if (mds >= mdsc->max_sessions) {
-               /* realloc */
                struct ceph_mds_session **sa;
+               /* realloc */
+               dout(50, "mdsc register_session realloc to %d\n", mds);
                sa = kzalloc(mds * sizeof(struct ceph_mds_session), GFP_KERNEL);
                BUG_ON(sa == NULL);  /* i am lazy */
                if (mdsc->sessions) {
@@ -293,14 +294,18 @@ retry:
        if (mds < 0) {
                /* wait for new mdsmap */
                spin_unlock(&mdsc->lock);
+               dout(30, "mdsc_do_request waiting for new mdsmap\n");
                wait_for_new_map(mdsc);
                spin_lock(&mdsc->lock);
                goto retry;
        }
+       dout(30, "mdsc_do_request chose mds%d\n", mds);
 
        /* get session */
        session = get_session(mdsc, mds);
        
+       dout(30, "mdsc_do_request got session %p\n", session);
+
        /* open? */
        if (mdsc->sessions[mds]->s_state == CEPH_MDS_SESSION_IDLE) 
                open_session(mdsc, session, mds);

diff --git a/trunk/ceph/kernel/messenger.c b/trunk/ceph/kernel/messenger.c
index 64ecfdd3edcd971f5acbe3392470d50d24a9f247..25c6df1a47ea386e2209ca365c1bee0d36d94f3e 100644 (file)
--- a/trunk/ceph/kernel/messenger.c
+++ b/trunk/ceph/kernel/messenger.c
@@ -392,7 +392,9 @@ static void try_write(struct work_struct *work)
        struct ceph_messenger *msgr;
        int ret = 1;
 
+       dout(30, "try_write start\n");
        con = container_of(work, struct ceph_connection, swork);
+       spin_lock(&con->lock);
        msgr = con->msgr;
 
 more:
@@ -437,6 +439,8 @@ more:
        clear_bit(WRITE_PENDING, &con->state);
 
 done:
+       dout(30, "try_write done\n");
+       spin_unlock(&con->lock);
        return;
 }
 
@@ -919,6 +923,7 @@ int ceph_msg_send(struct ceph_messenger *msgr, struct ceph_msg *msg)
                prepare_write_connect(msgr, con);
                dout(5, "ceph_msg_send initiating connect on %p new state %u\n", con, con->state);
                ret = ceph_tcp_connect(con);
+               dout(5, "ceph_msg_send done initiating connect on %p new state %u\n", con, con->state);
                if (ret < 0) {
                        derr(1, "connection failure to peer %x:%d\n",
                             ntohl(msg->hdr.dst.addr.ipaddr.sin_addr.s_addr),
@@ -965,7 +970,6 @@ struct ceph_msg *ceph_msg_new(int type, int front_len, int page_len, int page_of
                m->front.iov_base = kmalloc(front_len, GFP_KERNEL);
                if (m->front.iov_base == NULL)
                        goto out2;
-               dout(50, "ceph_msg_new front is %p len %d\n", m->front.iov_base, front_len);
        } else {
                m->front.iov_base = 0;
        }

diff --git a/trunk/ceph/kernel/mon_client.c b/trunk/ceph/kernel/mon_client.c
index f379955804ad9387b7cb577a08506d0595d049aa..35984ac636a979e249a6d5de69e7d20832665007 100644 (file)
--- a/trunk/ceph/kernel/mon_client.c
+++ b/trunk/ceph/kernel/mon_client.c
@@ -4,6 +4,41 @@
 #include "mon_client.h"
 
 
+int ceph_monmap_decode(struct ceph_monmap *m, void *p, void *end)
+{
+       int err;
+       void *old;
+
+       dout(30, "monmap_decode %p %p\n", p, end);
+
+       if ((err = ceph_decode_32(&p, end, &m->epoch)) < 0)
+               goto bad;
+       if ((err = ceph_decode_64(&p, end, &m->fsid.major)) < 0)
+               goto bad;
+       if ((err = ceph_decode_64(&p, end, &m->fsid.minor)) < 0)
+               goto bad;
+       if ((err = ceph_decode_32(&p, end, &m->num_mon)) < 0)
+               return err;
+
+       old = m->mon_inst;
+       m->mon_inst = kmalloc(m->num_mon*sizeof(*m->mon_inst), GFP_KERNEL);
+       if (m->mon_inst == NULL) {
+               m->mon_inst = old;
+               return -ENOMEM;
+       }
+       kfree(old);
+
+       if ((err = ceph_decode_copy(&p, end, m->mon_inst, m->num_mon*sizeof(m->mon_inst[0]))) < 0)
+               goto bad;
+
+       dout(30, "monmap_decode got epoch %d, num_mon %d\n", m->epoch, m->num_mon);
+       return 0;
+
+bad:
+       dout(30, "monmap_decode failed with %d\n", err);
+       return err;
+}
+
 static int pick_mon(struct ceph_mon_client *monc, int notmon)
 {
        char r;
@@ -22,6 +57,7 @@ void ceph_monc_init(struct ceph_mon_client *monc)
 }
 
 
+
 void ceph_monc_request_mdsmap(struct ceph_mon_client *monc, __u64 have)
 {
        dout(5, "ceph_monc_request_mdsmap\n");

diff --git a/trunk/ceph/kernel/mon_client.h b/trunk/ceph/kernel/mon_client.h
index d44b9c5e687ec250442aabb7acf2292514ae62d5..478ed1c10100a00f215e4f5e819479ac64c60e4a 100644 (file)
--- a/trunk/ceph/kernel/mon_client.h
+++ b/trunk/ceph/kernel/mon_client.h
@@ -1,18 +1,26 @@
 #ifndef _FS_CEPH_MON_CLIENT_H
 #define _FS_CEPH_MON_CLIENT_H
 
-#include "monmap.h"
 #include "messenger.h"
 
 struct ceph_mount_args;
 
+struct ceph_monmap {
+       ceph_epoch_t epoch;
+       struct ceph_fsid fsid;
+       __u32 num_mon;
+       struct ceph_entity_inst *mon_inst;
+};
+
+
 struct ceph_mon_client {
        int last_mon;  /* last monitor i contacted */
        struct ceph_monmap monmap;
 };
 
+extern int ceph_monmap_decode(struct ceph_monmap *m, void *p, void *end);
+
 extern void ceph_monc_init(struct ceph_mon_client *monc);
-extern void ceph_monc_handle_monmap(struct ceph_mon_client *monc, struct ceph_msg *m);
 
 extern void ceph_monc_request_mdsmap(struct ceph_mon_client *monc, __u64 have);
 extern void ceph_monc_request_osdmap(struct ceph_mon_client *monc, __u64 have);

diff --git a/trunk/ceph/kernel/monmap.c b/trunk/ceph/kernel/monmap.c
deleted file mode 100644 (file)
index 05765ef..0000000
--- a/trunk/ceph/kernel/monmap.c
+++ /dev/null
@@ -1,26 +0,0 @@
-#include <linux/slab.h>
-#include "monmap.h"
-#include "messenger.h"
-
-int ceph_monmap_decode(struct ceph_monmap *m, void **p, void *end)
-{
-       int err;
-
-       if ((err = ceph_decode_64(p, end, &m->epoch)) < 0)
-               return err;
-       if ((err = ceph_decode_32(p, end, &m->num_mon)) < 0)
-               return err;
-
-       m->mon_inst = kmalloc(m->num_mon*sizeof(*m->mon_inst), GFP_KERNEL);
-       if (m->mon_inst == NULL)
-               return -ENOMEM;
-       if ((err = ceph_decode_copy(p, end, m->mon_inst, m->num_mon*sizeof(m->mon_inst[0]))) < 0)
-               goto bad;
-
-       return 0;
-
-bad:
-       kfree(m->mon_inst);
-       m->mon_inst = 0;
-       return err;
-}

diff --git a/trunk/ceph/kernel/monmap.h b/trunk/ceph/kernel/monmap.h
deleted file mode 100644 (file)
index 5e623b1..0000000
--- a/trunk/ceph/kernel/monmap.h
+++ /dev/null
@@ -1,17 +0,0 @@
-#ifndef _FS_CEPH_MONMAP_H
-#define _FS_CEPH_MONMAP_H
-
-#include <linux/ceph_fs.h>
-
-/*
- * monitor map
- */
-struct ceph_monmap {
-  __u64 epoch;
-  __u32 num_mon;
-  struct ceph_entity_inst *mon_inst;
-};
-
-extern int ceph_monmap_decode(struct ceph_monmap *m, void **p, void *end);
-
-#endif

diff --git a/trunk/ceph/kernel/osd_client.c b/trunk/ceph/kernel/osd_client.c
index fc1c6a610dab226c4a040d024c74f7257f7d8bcf..0a3081c53cf227c6475d8beefec86d6b564f2190 100644 (file)
--- a/trunk/ceph/kernel/osd_client.c
+++ b/trunk/ceph/kernel/osd_client.c
@@ -295,7 +295,8 @@ static int osdmap_set_max_osd(struct ceph_osdmap *map, int max)
 static struct ceph_osdmap *osdmap_decode(void **p, void *end)
 {
        struct ceph_osdmap *map;
-       __u32 crushlen, max;
+       __u32 len, max;
+       int i;
        int err;
        void *start = *p;
 
@@ -345,10 +346,28 @@ static struct ceph_osdmap *osdmap_decode(void **p, void *end)
        if ((err = ceph_decode_copy(p, end, map->osd_addr, map->max_osd*sizeof(*map->osd_addr))) < 0)
                goto bad;
 
+       /* pg primary swapping */
+       if ((err = ceph_decode_32(p, end, &len)) < 0)
+               goto bad;
+       if (len) {
+               map->pg_swap_primary = kmalloc(len * sizeof(*map->pg_swap_primary), GFP_KERNEL);
+               if (map->pg_swap_primary == NULL) {
+                       err = -ENOMEM;
+                       goto bad;
+               }
+               map->num_pg_swap_primary = len;
+               for (i=0; i<len; i++) {
+                       if ((err = ceph_decode_64(p, end, &map->pg_swap_primary[i].pg.pg64)) < 0)
+                               goto bad;
+                       if ((err = ceph_decode_32(p, end, &map->pg_swap_primary[i].osd)) < 0)
+                               goto bad;
+               }
+       }
+
        /* crush */
-       if ((err = ceph_decode_32(p, end, &crushlen)) < 0)
+       if ((err = ceph_decode_32(p, end, &len)) < 0)
                goto bad;
-       dout(30, "osdmap_decode crush len %d from off %x\n", crushlen, (int)(*p - start));
+       dout(30, "osdmap_decode crush len %d from off %x\n", len, (int)(*p - start));
        map->crush = crush_decode(p, end);
        if (IS_ERR(map->crush)) {
                err = PTR_ERR(map->crush);
@@ -550,13 +569,13 @@ void ceph_osdc_handle_map(struct ceph_osd_client *osdc, struct ceph_msg *msg)
                        osdc->osdmap = newmap;
                }
        }
-       dout(1, "done\n");
+       dout(1, "osdc handle_map done\n");
        
 out:
        return;
 
 bad:
-       derr(1, "corrupt osd map message\n");
+       derr(1, "osdc handle_map corrupt msg\n");
        goto out;
 }
 

diff --git a/trunk/ceph/kernel/osd_client.h b/trunk/ceph/kernel/osd_client.h
index 598d58bae0ac1a017eddfa27e6c89281956f2e70..3277ee15e1987fe74a42eae32edd22789f62c667 100644 (file)
--- a/trunk/ceph/kernel/osd_client.h
+++ b/trunk/ceph/kernel/osd_client.h
@@ -23,6 +23,12 @@ struct ceph_osdmap {
        __u32 *osd_offload;  /* 0 = normal, 0x10000 = 100% offload (failed) */
        struct ceph_entity_addr *osd_addr;
        struct crush_map *crush;
+
+       __u32 num_pg_swap_primary;
+       struct {
+               ceph_pg_t pg;
+               __u32 osd;
+       } *pg_swap_primary;
 };
 
 enum {

diff --git a/trunk/ceph/mkmonmap.cc b/trunk/ceph/mkmonmap.cc
index 1e2dd2050c4fa217b8dc7742f26bec0ec66a4555..32d88a0f3b2f1ff19360fe878e0d1d61c70cd58c 100644 (file)
--- a/trunk/ceph/mkmonmap.cc
+++ b/trunk/ceph/mkmonmap.cc
@@ -60,7 +60,7 @@ int main(int argc, char **argv)
   }
 
   // write it out
-  cout << "mkmonmap: writing monmap to " << outfn << " (" << monmap.size() << " monitors)" << std::endl;
+  cout << "mkmonmap: writing monmap epoch " << monmap.epoch << " to " << outfn << " (" << monmap.size() << " monitors)" << std::endl;
   int r = monmap.write(outfn);
   assert(r >= 0);
   

diff --git a/trunk/ceph/mon/ClientMonitor.cc b/trunk/ceph/mon/ClientMonitor.cc
index 9082db0b2db8f479b6d6693f395f39ac7290930f..80d0c24cec2dc8f6a2921420ce5fa3702faf2e98 100644 (file)
--- a/trunk/ceph/mon/ClientMonitor.cc
+++ b/trunk/ceph/mon/ClientMonitor.cc
@@ -19,6 +19,7 @@
 #include "OSDMonitor.h"
 #include "MonitorStore.h"
 
+#include "messages/MMonMap.h"
 #include "messages/MClientMount.h"
 #include "messages/MClientUnmount.h"
 
@@ -222,6 +223,11 @@ void ClientMonitor::_mounted(int client, MClientMount *m)
   mon->mdsmon->send_latest(to);
   mon->osdmon->send_latest(to);
 
+  dout(10) << "sending monmap to " << to << dendl;
+  bufferlist bl;
+  mon->monmap->encode(bl);
+  mon->messenger->send_message(new MMonMap(bl), to);
+
   delete m;
 }
 

diff --git a/trunk/ceph/mon/MonMap.h b/trunk/ceph/mon/MonMap.h
index 3b6bad339716d28528e232ccbfa524b4730ba13f..f8adebd3f405da4c2da84a23888b640fbfb1304b 100644 (file)
--- a/trunk/ceph/mon/MonMap.h
+++ b/trunk/ceph/mon/MonMap.h
@@ -30,7 +30,7 @@ class MonMap {
 
   int       last_mon;    // last mon i talked to
 
-  MonMap(int s=0) : epoch(0), mon_inst(s), last_mon(-1) {
+  MonMap(int s=0) : epoch(s?1:0), mon_inst(s), last_mon(-1) {
     generate_fsid();
   }
 
@@ -39,6 +39,7 @@ class MonMap {
   }
 
   void add_mon(entity_inst_t inst) {
+    if (!epoch) epoch = 1;
     mon_inst.push_back(inst);
   }