protocol: add mount_ack to client mount protocol

diff --git a/src/client/Client.cc b/src/client/Client.cc
index c3d065294e113111ed8b96e438dd37c5e352ca07..774604374ee95e6db1f6dbb08193eb9d57d4c58f 100644 (file)
--- a/src/client/Client.cc
+++ b/src/client/Client.cc
@@ -39,6 +39,7 @@ using namespace std;
 #include "messages/MMonMap.h"
 
 #include "messages/MClientMount.h"
+#include "messages/MClientMountAck.h"
 #include "messages/MClientUnmount.h"
 #include "messages/MClientSession.h"
 #include "messages/MClientReconnect.h"
@@ -1075,6 +1076,9 @@ bool Client::dispatch_impl(Message *m)
   case CEPH_MSG_MON_MAP:
     handle_mon_map((MMonMap*)m);
     break;
+  case CEPH_MSG_CLIENT_MOUNT_ACK:
+    handle_mount_ack((MClientMountAck*)m);
+    break;
 
     // osd
   case CEPH_MSG_OSD_OPREPLY:
@@ -2309,7 +2313,8 @@ int Client::mount()
   }
   mounters++;
   
-  while (!mdsmap ||
+  while (signed_ticket.length() == 0 ||
+        !mdsmap ||
         !osdmap || 
         osdmap->get_epoch() == 0 ||
         (!itsme && !mounted))       // non-doers wait a little longer
@@ -2369,6 +2374,17 @@ void Client::handle_mon_map(MMonMap *m)
   delete m;
 }
 
+void Client::handle_mount_ack(MClientMountAck* m)
+{
+  dout(10) << "handle_mount_ack " << *m << dendl;
+
+  signed_ticket = m->signed_ticket;
+  bufferlist::iterator p = signed_ticket.begin();
+  ::decode(ticket, p);
+
+  mount_cond.Signal();
+}
+
 
 // UNMOUNT
 

diff --git a/src/client/Client.h b/src/client/Client.h
index 3d23831cd48ce61186856bd502ef54c9178211ef..fd129a981ddf6d443dfcd51e57d1651af8c4129d 100644 (file)
--- a/src/client/Client.h
+++ b/src/client/Client.h
@@ -63,6 +63,7 @@ using namespace __gnu_cxx;
 class MStatfsReply;
 class MClientSession;
 class MClientRequest;
+class MClientMountAck;
 class MClientRequestForward;
 class MClientLease;
 class MMonMap;
@@ -554,6 +555,9 @@ public:
   Messenger *messenger;  
   int whoami;
   MonMap *monmap;
+
+  ceph_client_ticket ticket;
+  bufferlist signed_ticket;
   
   // mds sessions
   struct MDSSession {
@@ -831,6 +835,7 @@ protected:
 
   // messaging
   void handle_mon_map(MMonMap *m);
+  void handle_mount_ack(MClientMountAck *m);
   void handle_unmount(Message*);
   void handle_mds_map(class MMDSMap *m);
 

diff --git a/src/include/ceph_fs.h b/src/include/ceph_fs.h
index d3a8cd3f15e80bcdbfd4245b4a0cf36bc4804803..029379a949be5e2a59a372bc46fcf82b696b6ff4 100644 (file)
--- a/src/include/ceph_fs.h
+++ b/src/include/ceph_fs.h
@@ -45,7 +45,7 @@
 #define CEPH_MON_PROTOCOL     4 /* cluster internal */
 #define CEPH_OSDC_PROTOCOL    5 /* public/client */
 #define CEPH_MDSC_PROTOCOL   11 /* public/client */
-#define CEPH_MONC_PROTOCOL    8 /* public/client */
+#define CEPH_MONC_PROTOCOL    9 /* public/client */
 
 
 /*
@@ -523,9 +523,10 @@ struct ceph_msg_footer {
 #define CEPH_MSG_MON_MAP                4
 #define CEPH_MSG_MON_GET_MAP            5
 #define CEPH_MSG_CLIENT_MOUNT           10
-#define CEPH_MSG_CLIENT_UNMOUNT         11
-#define CEPH_MSG_STATFS                 12
-#define CEPH_MSG_STATFS_REPLY           13
+#define CEPH_MSG_CLIENT_MOUNT_ACK       11
+#define CEPH_MSG_CLIENT_UNMOUNT         12
+#define CEPH_MSG_STATFS                 13
+#define CEPH_MSG_STATFS_REPLY           14
 
 /* client <-> mds */
 #define CEPH_MSG_MDS_GETMAP             20

diff --git a/src/kernel/super.c b/src/kernel/super.c
index 2fb731c33c6f53e183ec90647df97b51dda95a69..29e8b182773947944070a297f8c6e13488130312 100644 (file)
--- a/src/kernel/super.c
+++ b/src/kernel/super.c
@@ -15,6 +15,7 @@
 #include "ceph_debug.h"
 #include "ceph_ver.h"
 #include "bookkeeper.h"
+#include "decode.h"
 
 /*
  * global debug value.
@@ -228,34 +229,74 @@ static const struct super_operations ceph_super_ops = {
 
 
 /*
- * the monitor responds with monmap to indicate mount success.
- * (or, someday, to indicate a change in the monitor cluster)
+ * The monitor responds with mount ack indicate mount success.  The
+ * included client ticket allows the client to talk to MDSs and OSDs.
  */
-static void handle_monmap(struct ceph_client *client, struct ceph_msg *msg)
+static int handle_mount_ack(struct ceph_client *client, struct ceph_msg *msg)
 {
-       int err;
-       int first = (client->monc.monmap->epoch == 0);
-       struct ceph_monmap *new, *old = client->monc.monmap;
-
-       dout(2, "handle_monmap had epoch %d\n", client->monc.monmap->epoch);
-       new = ceph_monmap_decode(msg->front.iov_base,
-                                msg->front.iov_base + msg->front.iov_len);
-       if (IS_ERR(new)) {
-               err = PTR_ERR(new);
-               derr(0, "problem decoding monmap, %d\n", err);
-               return;
+       struct ceph_monmap *monmap = NULL, *old = client->monc.monmap;
+       void *p, *end;
+       s32 result;
+       u32 len;
+       int err = -EINVAL;
+
+       if (client->signed_ticket) {
+               dout(2, "handle_mount_ack - already mounted\n");
+               return 0;
        }
-       client->monc.monmap = new;
-       kfree(old);
 
-       if (first) {
-               client->whoami = le32_to_cpu(msg->hdr.dst.name.num);
-               client->msgr->inst.name = msg->hdr.dst.name;
-               dout(1, "i am client%d, fsid is %llx.%llx\n", client->whoami,
-                   le64_to_cpu(__ceph_fsid_major(&client->monc.monmap->fsid)),
-                   le64_to_cpu(__ceph_fsid_minor(&client->monc.monmap->fsid)));
-               ceph_sysfs_client_init(client);
+       dout(2, "handle_mount_ack\n");
+       p = msg->front.iov_base;
+       end = p + msg->front.iov_len;
+
+       ceph_decode_32_safe(&p, end, result, bad);
+       ceph_decode_32_safe(&p, end, len, bad);
+       if (result) {
+               dout(0, "mount denied: %.*s (%d)\n", len, (char *)p, result);
+               return result;
        }
+       p += len;
+
+       ceph_decode_32_safe(&p, end, len, bad);
+       ceph_decode_need(&p, end, len, bad);
+       monmap = ceph_monmap_decode(p, p + len);
+       if (IS_ERR(monmap)) {
+               derr(0, "problem decoding monmap, %d\n", (int)PTR_ERR(monmap));
+               return -EINVAL;
+       }
+       p += len;
+
+       ceph_decode_32_safe(&p, end, len, bad);
+       dout(0, "ticket len %d\n", len);
+       ceph_decode_need(&p, end, len, bad);
+
+       client->signed_ticket = kmalloc(len, GFP_KERNEL);
+       if (!client->signed_ticket) {
+               derr(0, "problem allocating %d bytes for client ticket\n",
+                    len);
+               err = -ENOMEM;
+               goto out;
+       }
+
+       memcpy(client->signed_ticket, p, len);
+       client->signed_ticket_len = len;
+
+       client->monc.monmap = monmap;
+       kfree(old);
+
+       client->whoami = le32_to_cpu(msg->hdr.dst.name.num);
+       client->msgr->inst.name = msg->hdr.dst.name;
+       dout(1, "i am client%d, fsid is %llx.%llx\n", client->whoami,
+            le64_to_cpu(__ceph_fsid_major(&client->monc.monmap->fsid)),
+            le64_to_cpu(__ceph_fsid_minor(&client->monc.monmap->fsid)));
+       ceph_sysfs_client_init(client);
+       return 0;
+
+bad:
+       derr(0, "error decoding mount_ack message\n");
+out:
+       kfree(monmap);
+       return err;
 }
 
 const char *ceph_msg_type_name(int type)
@@ -266,6 +307,7 @@ const char *ceph_msg_type_name(int type)
        case CEPH_MSG_MON_MAP: return "mon_map";
        case CEPH_MSG_MON_GET_MAP: return "mon_get_map";
        case CEPH_MSG_CLIENT_MOUNT: return "client_mount";
+       case CEPH_MSG_CLIENT_MOUNT_ACK: return "client_mount_ack";
        case CEPH_MSG_CLIENT_UNMOUNT: return "client_unmount";
        case CEPH_MSG_STATFS: return "statfs";
        case CEPH_MSG_STATFS_REPLY: return "statfs_reply";
@@ -636,6 +678,10 @@ static struct ceph_client *ceph_create_client(void)
 
        client->msgr = NULL;
 
+       client->mount_err = 0;
+       client->signed_ticket = NULL;
+       client->signed_ticket_len = 0;
+
        client->wb_wq = create_workqueue("ceph-writeback");
        if (client->wb_wq == NULL)
                goto fail;
@@ -668,6 +714,8 @@ static void ceph_destroy_client(struct ceph_client *client)
        ceph_monc_stop(&client->monc);
        ceph_osdc_stop(&client->osdc);
 
+       kfree(client->signed_ticket);
+
        ceph_sysfs_client_cleanup(client);
        if (client->wb_wq)
                destroy_workqueue(client->wb_wq);
@@ -788,6 +836,10 @@ static int ceph_mount(struct ceph_client *client, struct vfsmount *mnt,
                                                       request_interval);
                if (err == -EINTR)
                        goto out;
+               if (client->mount_err) {
+                       err = client->mount_err;
+                       goto out;
+               }
        }
 
        dout(30, "mount opening base mountpoint\n");
@@ -822,10 +874,11 @@ void ceph_dispatch(void *p, struct ceph_msg *msg)
        int type = le16_to_cpu(msg->hdr.type);
 
        switch (type) {
-       case CEPH_MSG_MON_MAP:
-               had = client->monc.monmap->epoch ? 1 : 0;
-               handle_monmap(client, msg);
-               if (!had && client->monc.monmap->epoch && have_all_maps(client))
+       case CEPH_MSG_CLIENT_MOUNT_ACK:
+               had = client->signed_ticket ? 1 : 0;
+               client->mount_err = handle_mount_ack(client, msg);
+               if (client->mount_err ||
+                   (!had && client->signed_ticket && have_all_maps(client)))
                        wake_up(&client->mount_wq);
                break;
 

diff --git a/src/kernel/super.h b/src/kernel/super.h
index 28e50e6d9c2190b3cec932a1f3df0f05505d28e8..ebd179e9ec6ecc17f65169ff24ef158ef3f278ea 100644 (file)
--- a/src/kernel/super.h
+++ b/src/kernel/super.h
@@ -107,6 +107,10 @@ struct ceph_client {
        unsigned long mount_state;
        wait_queue_head_t mount_wq;
 
+       int mount_err;
+       void *signed_ticket;           /* our keys to the kingdom */
+       int signed_ticket_len;
+
        struct ceph_messenger *msgr;   /* messenger instance */
        struct ceph_mon_client monc;
        struct ceph_mds_client mdsc;

diff --git a/src/mon/ClientMap.h b/src/mon/ClientMap.h
index 96050e9c29edc01975d5ff3a63eebce39400e33c..48a7fa097a8367ea5bbd92c5ddfe74c85d6ab603 100644 (file)
--- a/src/mon/ClientMap.h
+++ b/src/mon/ClientMap.h
@@ -32,16 +32,19 @@ using namespace std;
 #include "PaxosService.h"
 #endif
 struct client_info_t {
-  entity_addr_t addr;
-  utime_t mount_time;
+  ceph_client_ticket ticket;
+  bufferlist signed_ticket;
+
+  entity_addr_t addr() { return entity_addr_t(ticket.addr); }
+  utime_t created() { return utime_t(ticket.created); }
 
   void encode(bufferlist& bl) const {
-    ::encode(addr, bl);
-    ::encode(mount_time, bl);
+    ::encode(ticket, bl);
+    ::encode(signed_ticket, bl);
   }
   void decode(bufferlist::iterator& bl) {
-    ::decode(addr, bl);
-    ::decode(mount_time, bl);
+    ::decode(ticket, bl);
+    ::decode(signed_ticket, bl);
   }
 };
 
@@ -96,12 +99,12 @@ public:
 
   void reverse() {
     addr_client.clear();
-      for (map<uint32_t,client_info_t>::iterator p = client_info.begin();
-          p != client_info.end();
-          ++p) {
-       addr_client[p->second.addr] = p->first;
-      }
+    for (map<uint32_t,client_info_t>::iterator p = client_info.begin();
+        p != client_info.end();
+        ++p) {
+      addr_client[p->second.addr()] = p->first;
     }
+  }
   void apply_incremental(Incremental &inc) {
     assert(inc.version == version+1);
     version = inc.version;
@@ -110,14 +113,14 @@ public:
           p != inc.mount.end();
           ++p) {
        client_info[p->first] = p->second;
-       addr_client[p->second.addr] = p->first;
+       addr_client[p->second.addr()] = p->first;
     }
 
     for (set<int32_t>::iterator p = inc.unmount.begin();
           p != inc.unmount.end();
           ++p) {
        assert(client_info.count(*p));
-       addr_client.erase(client_info[*p].addr);
+       addr_client.erase(client_info[*p].addr());
        client_info.erase(*p);
     }
   }

diff --git a/src/mon/ClientMonitor.cc b/src/mon/ClientMonitor.cc
index d06a475c614a0c2e4fe862fbb397ccc36fe95943..2cb69cbcc21bef1f511c4102b53bae033370e637 100644 (file)
--- a/src/mon/ClientMonitor.cc
+++ b/src/mon/ClientMonitor.cc
@@ -21,6 +21,7 @@
 
 #include "messages/MMonMap.h"
 #include "messages/MClientMount.h"
+#include "messages/MClientMountAck.h"
 #include "messages/MClientUnmount.h"
 #include "messages/MMonCommand.h"
 
@@ -196,15 +197,18 @@ bool ClientMonitor::prepare_update(Message *m)
       } else {
        dout(10) << "mount: client" << client << " requested by " << addr << dendl;
        if (client_map.client_info.count(client)) {
-         assert(client_map.client_info[client].addr != addr);
+         assert(client_map.client_info[client].addr() != addr);
          dout(0) << "mount: WARNING: client" << client << " requested by " << addr
-                 << ", which used to be "  << client_map.client_info[client].addr << dendl;
+                 << ", which used to be "  << client_map.client_info[client].addr() << dendl;
        }
       }
       
       client_info_t info;
-      info.addr = addr;
-      info.mount_time = g_clock.now();
+      info.ticket.client = client;
+      info.ticket.addr = addr;
+      info.ticket.created = g_clock.now();
+      info.ticket.expires = utime_t();
+      ::encode(info.ticket, info.signed_ticket);
       pending_inc.add_mount(client, info);
       paxos->wait_for_commit(new C_Mounted(this, client, (MClientMount*)m));
     }
@@ -260,8 +264,8 @@ bool ClientMonitor::preprocess_command(MMonCommand *m)
           p != client_map.client_info.end();
           p++) {
        ss << "client" << p->first
-          << "\t" << p->second.addr
-          << "\t" << p->second.mount_time
+          << "\t" << p->second.addr()
+          << "\t" << p->second.created()
           << std::endl;
       }
       while (!ss.eof()) {
@@ -311,11 +315,14 @@ void ClientMonitor::_mounted(int client, MClientMount *m)
 
   dout(10) << "_mounted client" << client << " at " << to << dendl;
   
-  // reply with latest mon, mds, osd maps
-  bufferlist bl;
-  mon->monmap->encode(bl);
-  mon->messenger->send_message(new MMonMap(bl), to);
+  // reply with client ticket
+  MClientMountAck *ack = new MClientMountAck;
+  mon->monmap->encode(ack->monmap_bl);
+  ack->signed_ticket = client_map.client_info[client].signed_ticket;
+
+  mon->messenger->send_message(ack, to);
 
+  // also send latest mds and osd maps
   mon->mdsmon()->send_latest(to);
   mon->osdmon()->send_latest(to);
 

diff --git a/src/msg/Message.cc b/src/msg/Message.cc
index 14da63b26f125343bf9e82c7e44cfc74e1689f51..ad2f1779fe1dff77408d78a0c777e8b9c7860fde 100644 (file)
--- a/src/msg/Message.cc
+++ b/src/msg/Message.cc
@@ -54,6 +54,7 @@ using namespace std;
 #include "messages/MMonGetMap.h"
 
 #include "messages/MClientMount.h"
+#include "messages/MClientMountAck.h"
 #include "messages/MClientUnmount.h"
 #include "messages/MClientSession.h"
 #include "messages/MClientReconnect.h"
@@ -260,6 +261,9 @@ Message *decode_message(ceph_msg_header& header, ceph_msg_footer& footer,
   case CEPH_MSG_CLIENT_MOUNT:
     m = new MClientMount;
     break;
+  case CEPH_MSG_CLIENT_MOUNT_ACK:
+    m = new MClientMountAck;
+    break;
   case CEPH_MSG_CLIENT_UNMOUNT:
     m = new MClientUnmount;
     break;