From 13eb18fcd7da3f011560263c9469823c8b2a4a67 Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Sat, 26 Jan 2008 08:41:15 -0800 Subject: [PATCH] allow client to initially bind to INADDR_ANY and learn its ip from the monitor --- src/.gitignore | 2 +- src/kernel/client.c | 5 ++++- src/kernel/ktcp.c | 7 +------ src/kernel/ktcp.h | 2 +- src/kernel/messenger.c | 26 +++++++++++++++++++++++++- src/kernel/super.c | 1 + src/kernel/super.h | 1 + src/msg/SimpleMessenger.cc | 17 +++++++++++++++++ src/start.sh | 19 ++++++++++++------- 9 files changed, 63 insertions(+), 17 deletions(-) diff --git a/src/.gitignore b/src/.gitignore index 3c46f97334705..1b438ed694bde 100644 --- a/src/.gitignore +++ b/src/.gitignore @@ -16,4 +16,4 @@ log mnt TAGS tags - +out diff --git a/src/kernel/client.c b/src/kernel/client.c index af8b4d5ef40f9..b770e0300ffea 100644 --- a/src/kernel/client.c +++ b/src/kernel/client.c @@ -258,6 +258,7 @@ void got_first_map(struct ceph_client *client, int num) struct ceph_client *ceph_create_client(struct ceph_mount_args *args, struct super_block *sb) { struct ceph_client *cl; + struct ceph_entity_addr *myaddr = 0; int err; cl = kzalloc(sizeof(*cl), GFP_KERNEL); @@ -269,7 +270,9 @@ struct ceph_client *ceph_create_client(struct ceph_mount_args *args, struct supe get_client_counter(); /* messenger */ - cl->msgr = ceph_messenger_create(&args->my_addr); + if (args->flags & CEPH_MOUNT_MYIP) + myaddr = &args->my_addr; + cl->msgr = ceph_messenger_create(myaddr); if (IS_ERR(cl->msgr)) { err = PTR_ERR(cl->msgr); goto fail; diff --git a/src/kernel/ktcp.c b/src/kernel/ktcp.c index 4dbf48b981945..11b7d1dc7168d 100644 --- a/src/kernel/ktcp.c +++ b/src/kernel/ktcp.c @@ -140,7 +140,7 @@ done: /* * setup listening socket */ -int ceph_tcp_listen(struct ceph_messenger *msgr, int port) +int ceph_tcp_listen(struct ceph_messenger *msgr) { int ret; struct socket *sock = NULL; @@ -163,11 +163,6 @@ int ceph_tcp_listen(struct ceph_messenger *msgr, int port) /* set user_data to be the messenger */ sock->sk->sk_user_data = msgr; - /* currently no user specified address given so create */ - /* if (!*myaddr) */ - myaddr->sin_family = AF_INET; - myaddr->sin_addr.s_addr = htonl(INADDR_ANY); - myaddr->sin_port = htons(port); /* any port */ ret = sock->ops->bind(sock, (struct sockaddr *)myaddr, sizeof(*myaddr)); if (ret < 0) { derr(0, "Failed to bind: %d\n", ret); diff --git a/src/kernel/ktcp.h b/src/kernel/ktcp.h index be3a12028f8cd..dcc1c3de2a013 100644 --- a/src/kernel/ktcp.h +++ b/src/kernel/ktcp.h @@ -6,7 +6,7 @@ extern struct workqueue_struct *send_wq; /* send work queue */ /* prototype definitions */ int ceph_tcp_connect(struct ceph_connection *); -int ceph_tcp_listen(struct ceph_messenger *, int port); +int ceph_tcp_listen(struct ceph_messenger *); int ceph_tcp_accept(struct socket *, struct ceph_connection *); int ceph_tcp_recvmsg(struct socket *, void *, size_t ); int ceph_tcp_sendmsg(struct socket *, struct kvec *, size_t, size_t); diff --git a/src/kernel/messenger.c b/src/kernel/messenger.c index ef3ac9334a54d..c0361c35ea3a6 100644 --- a/src/kernel/messenger.c +++ b/src/kernel/messenger.c @@ -667,6 +667,20 @@ static int read_message_partial(struct ceph_connection *con) done: dout(20, "read_message_partial got msg %p\n", m); + + /* did i learn my ip? */ + if (con->msgr->inst.addr.ipaddr.sin_addr.s_addr == htonl(INADDR_ANY)) { + /* + * in practice, we learn our ip from the first incoming mon + * message, before anyone else knows we exist, so this is + * safe. + */ + con->msgr->inst.addr.ipaddr = con->in_msg->hdr.dst.addr.ipaddr; + dout(10, "read_message_partial learned my addr is %x:%d\n", + ntohl(con->msgr->inst.addr.ipaddr.sin_addr.s_addr), + ntohs(con->msgr->inst.addr.ipaddr.sin_port)); + } + return 1; /* done! */ } @@ -1015,8 +1029,18 @@ struct ceph_messenger *ceph_messenger_create(struct ceph_entity_addr *myaddr) INIT_LIST_HEAD(&msgr->con_accepting); INIT_RADIX_TREE(&msgr->con_open, GFP_ATOMIC); /* we insert under spinlock */ + /* pick listening address */ + if (myaddr) { + msgr->inst.addr = *myaddr; + } else { + dout(10, "create my ip not specified, binding to INADDR_ANY\n"); + msgr->inst.addr.ipaddr.sin_addr.s_addr = htonl(INADDR_ANY); + msgr->inst.addr.ipaddr.sin_port = htons(0); /* any port */ + } + msgr->inst.addr.ipaddr.sin_family = AF_INET; + /* create listening socket */ - ret = ceph_tcp_listen(msgr, myaddr ? ntohs(myaddr->ipaddr.sin_port):0); + ret = ceph_tcp_listen(msgr); if (ret < 0) { kfree(msgr); return ERR_PTR(ret); diff --git a/src/kernel/super.c b/src/kernel/super.c index be3613fa76995..d69c85e190dcc 100644 --- a/src/kernel/super.c +++ b/src/kernel/super.c @@ -339,6 +339,7 @@ static int parse_mount_args(int flags, char *options, const char *dev_name, &args->my_addr); if (err < 0) return err; + args->flags |= CEPH_MOUNT_MYIP; break; default: derr(1, "parse_mount_args bad token %d\n", token); diff --git a/src/kernel/super.h b/src/kernel/super.h index c3fb6226e350b..2683a31dd4ef2 100644 --- a/src/kernel/super.h +++ b/src/kernel/super.h @@ -35,6 +35,7 @@ extern int ceph_debug_osdc; */ #define CEPH_MOUNT_FSID 1 #define CEPH_MOUNT_NOSHARE 2 /* don't share client with other mounts */ +#define CEPH_MOUNT_MYIP 4 /* specified my ip */ struct ceph_mount_args { int mntflags; diff --git a/src/msg/SimpleMessenger.cc b/src/msg/SimpleMessenger.cc index 6df3de5cf474e..a57e9f2c219b1 100644 --- a/src/msg/SimpleMessenger.cc +++ b/src/msg/SimpleMessenger.cc @@ -721,6 +721,18 @@ int Rank::Pipe::accept() state = STATE_CLOSED; return -1; } + if (peer_addr.v.ipaddr.sin_addr.s_addr == htonl(INADDR_ANY)) { + // peer apparently doesn't know what ip they have; figure it out for them. + entity_addr_t old_addr = peer_addr; + socklen_t len = sizeof(peer_addr.v.ipaddr); + int r = ::getpeername(sd, (sockaddr*)&peer_addr.v.ipaddr, &len); + if (r < 0) { + dout(0) << "accept failed to getpeername " << errno << " " << strerror(errno) << dendl; + state = STATE_CLOSED; + return -1; + } + dout(2) << "accept peer says " << old_addr << ", socket says " << peer_addr << dendl; + } __u32 cseq; rc = tcp_read(sd, (char*)&cseq, sizeof(cseq)); @@ -1365,6 +1377,11 @@ Message *Rank::Pipe::read_message() << " data=" << env.data_len << " at " << env.data_off << dendl; + if (env.src.addr.ipaddr.sin_addr.s_addr == htonl(INADDR_ANY)) { + dout(10) << "reader munging src addr " << env.src << " to be " << peer_addr << dendl; + env.src.addr.ipaddr = peer_addr.v.ipaddr; + } + // read front bufferlist front; bufferptr bp; diff --git a/src/start.sh b/src/start.sh index aa006cae01147..cd7037530b50f 100755 --- a/src/start.sh +++ b/src/start.sh @@ -1,12 +1,17 @@ #!/bin/sh + rm core* ./stop.sh -./mkmonmap `host \`hostname -f\`|cut -d ' ' -f 4`:12345 # your IP here + test -d out || mkdir out rm out/* -./cmon --mkfs --mon 0 --doutdir out & -./cosd --mkfs --osd 0 --doutdir out & -./cosd --mkfs --osd 1 --doutdir out & -./cosd --mkfs --osd 2 --doutdir out & -./cosd --mkfs --osd 3 --doutdir out & -./cmds --doutdir out & + +ARGS="--doutdir out" +./mkmonmap `host \`hostname -f\`|cut -d ' ' -f 4`:12345 # your IP here +./cmon --mkfs --mon 0 $ARGS & +./cosd --mkfs --osd 0 $ARGS & +./cosd --mkfs --osd 1 $ARGS & +./cosd --mkfs --osd 2 $ARGS & +./cosd --mkfs --osd 3 $ARGS & +./cmds $ARGS +echo "started. stop.sh to stop. see out/* for debug output." -- 2.39.5