From dc38de9b14c5386f9f446124ca6d6673eb8a1e20 Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Mon, 8 Dec 2008 11:44:21 -0800 Subject: [PATCH] ceph: new default mon port; try to bind to port in known range New monitor port in unused region (according to nmap-services). Try to bind to a port in a known range, so that tools can easily identify the protocol in use. Remove some old .sh cruft. --- src/dstart.sh | 2 +- src/include/ceph_fs.h | 11 +++++++- src/mkcephfs.sh | 2 +- src/mkfs.sh | 44 ----------------------------- src/msg/SimpleMessenger.cc | 28 +++++++++++++++---- src/startnew.sh | 57 -------------------------------------- src/vstart.sh | 2 +- 7 files changed, 36 insertions(+), 110 deletions(-) delete mode 100755 src/mkfs.sh delete mode 100755 src/startnew.sh diff --git a/src/dstart.sh b/src/dstart.sh index 7d89893ea1c18..5e2f4beca7fa0 100755 --- a/src/dstart.sh +++ b/src/dstart.sh @@ -65,7 +65,7 @@ if [ $new -eq 1 ]; then fi # build a fresh fs monmap, mon fs - ./monmaptool --create --clobber --add $IP:12345 --print .ceph_monmap + ./monmaptool --create --clobber --add $IP:6789 --print .ceph_monmap ./mkmonfs --clobber mondata/mon0 --mon 0 --monmap .ceph_monmap fi diff --git a/src/include/ceph_fs.h b/src/include/ceph_fs.h index 004f1adb357a1..4eba54853c978 100644 --- a/src/include/ceph_fs.h +++ b/src/include/ceph_fs.h @@ -9,7 +9,16 @@ #define _FS_CEPH_CEPH_FS_H -#define CEPH_MON_PORT 12345 +#define CEPH_MON_PORT 6789 /* default monitor port */ + +/* + * client-side processes will try to bind to ports in this + * range, simply for the benefit of tools like nmap or wireshark + * that would like to identify the protocol. + */ +#define CEPH_PORT_FIRST 6789 +#define CEPH_PORT_START 6800 /* non-monitors start here */ +#define CEPH_PORT_LAST 6900 /* * Max file size is a policy choice; in reality we are limited diff --git a/src/mkcephfs.sh b/src/mkcephfs.sh index 7f952ccb68ad6..748979ed30222 100755 --- a/src/mkcephfs.sh +++ b/src/mkcephfs.sh @@ -17,7 +17,7 @@ then fi # build a fresh fs monmap, mon fs -$CEPH_BIN/monmaptool --create --clobber --add $IP:12345 --print .ceph_monmap +$CEPH_BIN/monmaptool --create --clobber --add $IP:6789 --print .ceph_monmap $CEPH_BIN/mkmonfs --clobber mondata/mon0 --mon 0 --monmap .ceph_monmap # shared args diff --git a/src/mkfs.sh b/src/mkfs.sh deleted file mode 100755 index 056742b1f2f74..0000000000000 --- a/src/mkfs.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/bin/sh - -# figure machine's ip -HOSTNAME=`hostname` -IP=`host $HOSTNAME | cut -d ' ' -f 4` -[ "$CEPH_BIN" == "" ] && CEPH_BIN=. - -echo hostname $HOSTNAME -echo "ip $IP" -if [ `echo $IP | grep '^127\\.'` ] -then - echo - echo "WARNING: hostname resolves to loopback; remote hosts will not be able to" - echo " connect. either adjust /etc/hosts, or edit this script to use your" - echo " machine's real IP." - echo -fi - -# build a fresh fs monmap, mon fs -$CEPH_BIN/monmaptool --create --clobber --add $IP:12345 --print .ceph_monmap -$CEPH_BIN/mkmonfs --clobber mondata/mon0 --mon 0 --monmap .ceph_monmap - -# shared args -ARGS="-d --debug_ms 1" - -# start monitor -$CEPH_BIN/cmon $ARGS mondata/mon0 --debug_mon 20 --debug_ms 1 - -# build and inject an initial osd map -$CEPH_BIN/osdmaptool --clobber --createsimple .ceph_monmap 4 --print .ceph_osdmap -$CEPH_BIN/cmonctl osd setmap -i .ceph_osdmap - -# initialize osd stores -for osd in 0 1 2 3 -do - $CEPH_BIN/cosd --mkfs_for_osd $osd dev/osd$osd # initialize empty object store -done - -# stop monitor -killall cmon -#$CEPH_BIN/cmonctl stop - -echo "mkfs done." - diff --git a/src/msg/SimpleMessenger.cc b/src/msg/SimpleMessenger.cc index 97f9b5a27f476..33bbac79cd5cf 100644 --- a/src/msg/SimpleMessenger.cc +++ b/src/msg/SimpleMessenger.cc @@ -121,11 +121,29 @@ int Rank::Accepter::bind(int64_t force_nonce) ::setsockopt(listen_sd, SOL_SOCKET, SO_REUSEADDR, &on, sizeof(on)); /* bind to port */ - int rc = ::bind(listen_sd, (struct sockaddr *) &listen_addr, sizeof(listen_addr)); - if (rc < 0) { - derr(0) << "accepter.bind unable to bind to " << g_my_addr.ipaddr - << ": " << strerror(errno) << dendl; - return -errno; + int rc; + if (listen_addr.sin_port) { + // specific port + rc = ::bind(listen_sd, (struct sockaddr *) &listen_addr, sizeof(listen_addr)); + if (rc < 0) { + derr(0) << "accepter.bind unable to bind to " << g_my_addr.ipaddr + << ": " << strerror(errno) << dendl; + return -errno; + } + } else { + // try a range of ports + for (int port = CEPH_PORT_START; port <= CEPH_PORT_LAST; port++) { + listen_addr.sin_port = htons(port); + rc = ::bind(listen_sd, (struct sockaddr *) &listen_addr, sizeof(listen_addr)); + if (rc == 0) + break; + } + if (rc < 0) { + derr(0) << "accepter.bind unable to bind to " << g_my_addr.ipaddr + << " on any port in range " << CEPH_PORT_START << "-" << CEPH_PORT_LAST + << ": " << strerror(errno) << dendl; + return -errno; + } } // what port did we get? diff --git a/src/startnew.sh b/src/startnew.sh deleted file mode 100755 index b9238f4a5307b..0000000000000 --- a/src/startnew.sh +++ /dev/null @@ -1,57 +0,0 @@ -#!/bin/bash - -# sudo if btrfs -test -d dev/osd0 && SUDO="sudo" - -$SUDO ./stop.sh -$SUDO rm core* - -test -d out || mkdir out -$SUDO rm out/* - -# figure machine's ip -HOSTNAME=`hostname` -IP=`host $HOSTNAME | cut -d ' ' -f 4` -[ "$CEPH_BIN" == "" ] && CEPH_BIN=. -[ "$CEPH_PORT" == "" ] && CEPH_PORT=12345 - -echo hostname $HOSTNAME -echo "ip $IP" -if [ `echo $IP | grep '^127\\.'` ] -then - echo - echo "WARNING: hostname resolves to loopback; remote hosts will not be able to" - echo " connect. either adjust /etc/hosts, or edit this script to use your" - echo " machine's real IP." - echo -fi - -# build a fresh fs monmap, mon fs -$CEPH_BIN/monmaptool --create --clobber --add $IP:$CEPH_PORT --print .ceph_monmap -$CEPH_BIN/mkmonfs --clobber mondata/mon0 --mon 0 --monmap .ceph_monmap - -# shared args -ARGS="-f" -CMON_ARGS="" -CMDS_ARGS="" -COSD_ARGS="" - -# start monitor -$CEPH_BIN/crun $CEPH_BIN/cmon $ARGS $CMON_ARGS mondata/mon0 --debug_mon 10 --debug_ms 1 & - - -# build and inject an initial osd map -$CEPH_BIN/osdmaptool --clobber --createsimple .ceph_monmap 4 .ceph_osdmap -$CEPH_BIN/cmonctl osd setmap -i .ceph_osdmap - -for osd in 0 #1 2 3 -do - $SUDO $CEPH_BIN/cosd --mkfs_for_osd $osd dev/osd$osd # initialize empty object store - $CEPH_BIN/crun $SUDO $CEPH_BIN/cosd $ARGS $COSD_ARGS dev/osd$osd & -done - -# mds -$CEPH_BIN/crun $CEPH_BIN/cmds $ARGS $CMDS_ARGS & # --debug_ms 1 #--debug_mds 20 --debug_ms 20 - -echo "started. stop.sh to stop. see out/* (e.g. 'tail -f out/????') for debug output." - diff --git a/src/vstart.sh b/src/vstart.sh index 976f2ba9099f6..3705f60597d0c 100755 --- a/src/vstart.sh +++ b/src/vstart.sh @@ -60,7 +60,7 @@ test -d gmon && $SUDO rm -rf gmon/* HOSTNAME=`hostname` IP=`host $HOSTNAME | grep $HOSTNAME | cut -d ' ' -f 4` [ "$CEPH_BIN" == "" ] && CEPH_BIN=. -[ "$CEPH_PORT" == "" ] && CEPH_PORT=12345 +[ "$CEPH_PORT" == "" ] && CEPH_PORT=6789 echo hostname $HOSTNAME echo "ip $IP" -- 2.39.5