From: Sage Weil <sage@newdream.net>
Date: Sun, 1 Mar 2009 00:06:27 +0000 (-0800)
Subject: initscripts: make mkcephfs and initscript behave on cosd*
X-Git-Tag: v0.7~94
X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=28d5d5125dac697c8b5d5052e16db4b7422057fa;p=ceph.git

initscripts: make mkcephfs and initscript behave on cosd*
---

diff --git a/src/ceph-daemons b/src/ceph-daemons
index ddc7793a1879..feb2bb3f4ef5 100755
--- a/src/ceph-daemons
+++ b/src/ceph-daemons
@@ -27,25 +27,25 @@ stop_daemon() {
     daemon=$2
     pidfile=$3
     echo -n "Stopping ceph $name on $host..."
-    while [ 1 ]; do
+    do_cmd "while [ 1 ]; do 
 	[ -e $pidfile ] || break
 	pid=`cat $pidfile`
-	while [ -e /proc/$pid ] && grep -q $daemon /proc/$pid/cmdline ; do
-	    cmd="kill $pid"
-	    [[ $host != $hostname ]] && cmd="ssh $host $cmd"
-	    echo -n $cmd...
-	    $cmd
+	while [ -e /proc/\$pid ] && grep -q $daemon /proc/\$pid/cmdline ; do
+	    cmd=\"kill \$pid\"
+	    echo -n \$cmd...
+	    \$cmd
 	    sleep 1
 	    continue
 	done
 	break
-    done
+    done"
     echo done
 }
 
 ## command line options
 options=
 
+version=0
 dovalgrind=0
 docrun=1
 allhosts=0
@@ -54,13 +54,16 @@ monaddr=
 
 while [[ $1 =~ '-' ]]; do     # FIXME: why not '^-'?
 case $1 in
+    -v | --verbose)
+	    verbose=1
+	    ;;
     --valgrind)
 	    dovalgrind=1
 	    ;;
     --novalgrind)
 	    dovalgrind=0
 	    ;;
-    --allhosts)
+    --allhosts | -a)
 	    allhosts=1;
 	    ;;
     --restart)
@@ -100,39 +103,23 @@ for name in $what; do
     num=`echo $name | cut -c 4-`
     sections="$name $type global"
 
-    # this host?
-    host=`$CCONF --conf_file $conf -s $name -s $type -s global host`
-    ssh=""
-    cmd=""
-    if [[ $host != "" ]]; then
-	echo host for $name is $host, i am $hostname
-	if [[ $host != $hostname ]]; then
-	    # skip, unless we're starting remote daemons too
-	    if [[ $allhosts -eq 0 ]]; then
-		continue;
-	    fi
-
-	    # we'll need to ssh into that host
-	    ssh="ssh root@$host"
-	    cd_path=`$CCONF --conf_file $conf -s $name -s $type -s global "ssh path"`
-	fi
-    else
-	host=$hostname
-    fi
+    check_host || continue
 
     get_conf pid_file "/var/run/ceph/$name.pid" "pid file" $sections
     get_conf conf_file "$runtime_conf" "conf file" $sections
+
     get_conf mon_addr "$monaddr" "mon addr" $sections
     [ "$mon_addr" != "" ] && mon_addr_cmd="-m $mon_addr"
+
     # extract name-specific options from $conf
     if [[ $name =~ "mon" ]]; then
-	get_conf mon_path "" "mon path" $sections
-	module_opt="$mon_path"
+	get_conf mon_data "" "mon data" $sections
+	module_opt="$mon_data"
 	module_bin="$BINDIR/cmon"
     fi
 
     if [[ $name =~ "mds" ]]; then
-	module_opt="--conf_file $conf_file $mon_addr_cmd"
+	module_opt="$mon_addr_cmd"
 	module_bin="$BINDIR/cmds"
     fi
 
@@ -140,32 +127,31 @@ for name in $what; do
 	get_conf osd_data "" "osd data" $sections
 	get_conf osd_journal "" "osd journal" $sections
 	[ "$osd_journal" != "" ] && osd_journal_cmd="-j $osd_journal" || osd_journal_cmd=""
-	module_opt="--conf_file $conf_file \
-			$mon_addr_cmd $osd_data $osd_journal_cmd"
+	module_opt="$mon_addr_cmd $osd_data $osd_journal_cmd"
 	module_bin="$BINDIR/cosd"
     fi
 
-    # build final command
-    wrap=""
-    runflags="-d"
-    runmode=""
-
-    get_conf_bool crun "$docrun" "restart on core dump" $sections
-    [[ $crun -eq 1 ]] && wrap="$BINDIR/crun"
-
-    get_conf_bool valgrind "$dovalgrind" "valgrind" $sections
-    echo valgrind=$valgrind
-    [[ $valgrind -eq 1 ]] && wrap="$wrap valgrind"
-
-    [[ $wrap != "" ]] && runflags="-f" && runmode="&"
-
-    cmd="$ssh $wrap $module_bin $runflags $module_opt $runmode"
+    module_opt="-p $pid_file -c $conf_file $module_opt"
 
     case "$command" in
 	start)
+            # build final command
+	    wrap=""
+	    runflags="-d"
+	    runmode=""
+	    
+	    get_conf_bool crun "$docrun" "restart on core dump" $sections
+	    [[ $crun -eq 1 ]] && wrap="$BINDIR/crun"
+	    
+	    get_conf_bool valgrind "$dovalgrind" "valgrind" $sections
+	    [[ $valgrind -eq 1 ]] && wrap="$wrap valgrind"
+	    
+	    [[ $wrap != "" ]] && runflags="-f" && runmode="&"
+	    
+	    cmd="$wrap $module_bin $runflags $module_opt $runmode"
+	    
 	    echo Starting ceph $name on $host...
-	    echo $cmd
-            eval $cmd
+            do_cmd "$cmd"
 	    ;;
 	
 	stop)
diff --git a/src/ceph_common.sh b/src/ceph_common.sh
index 28508187ace2..58e517b3f048 100644
--- a/src/ceph_common.sh
+++ b/src/ceph_common.sh
@@ -7,6 +7,42 @@ runtime_conf=$ETCDIR"/ceph.conf"
 hostname=`hostname | cut -d . -f 1`
 
 
+check_host() {
+    # what host is this daemon assigned to?
+    host=`$CCONF -c $conf -s $name -s $type host`
+    ssh=""
+    dir=$PWD
+    if [[ $host != "" ]]; then
+	#echo host for $name is $host, i am $hostname
+	if [[ $host != $hostname ]]; then
+	    # skip, unless we're starting remote daemons too
+	    if [[ $allhosts -eq 0 ]]; then
+		return 1
+	    fi
+
+	    # we'll need to ssh into that host
+	    ssh="ssh root@$host"
+	    get_conf dir "$dir" "ssh path" $sections
+	fi
+    else
+	host=$hostname
+    fi
+
+    echo -n "=== $name === "
+
+    return 0
+}
+
+do_cmd() {
+    [[ $verbose = 1 ]] && echo "--- $host:$dir# $1"
+    if [[ $ssh = "" ]]; then
+	ulimit -c unlimited
+	bash -c "$1" || (echo failed. ; exit 1)
+    else
+	$ssh "cd $dir ; ulimit -c unlimited ; $1" || (echo failed. ; exit 1)
+    fi
+}
+
 get_name_list() {
     orig=$1
 
diff --git a/src/cosd.conf b/src/cosd.conf
new file mode 100644
index 000000000000..38c76de54adf
--- /dev/null
+++ b/src/cosd.conf
@@ -0,0 +1,35 @@
+[global]
+	pid file = /home/sage/ceph/src/out/$name.pid
+	restart on core dump = false
+	conf file = cosdruntime.conf
+
+[mon]
+[mon0]
+	mon addr = 10.3.14.95:6789
+	mon data = /home/sage/ceph/src/mondata/mon$num
+
+[osd]
+[osd0]
+	host = cosd1
+	osd data = devm/osd0
+	btrfs devs = /dev/disk/by-path/pci-0000:05:01.0-scsi-2:0:0:0
+	btrfs path = devm/osd0
+[osd4]
+	host = cosd2
+	osd data = devm/osd4
+	btrfs devs = /dev/disk/by-path/pci-0000:05:01.0-scsi-2:0:0:0
+
+[osd9]
+	host = cosd3
+	osd data = devm/osd9
+	btrfs devs = /dev/disk/by-path/pci-0000:05:01.0-scsi-3:0:0:0
+[osd13]
+	host = cosd4
+	osd data = devm/osd13
+	btrfs devs = /dev/disk/by-path/pci-0000:05:01.0-scsi-3:0:0:0
+[osd17]
+	host = cosd5
+	osd data = devm/osd17
+	btrfs devs = /dev/disk/by-path/pci-0000:05:01.0-scsi-3:0:0:0
+
+[mds0]
\ No newline at end of file
diff --git a/src/cosdruntime.conf b/src/cosdruntime.conf
new file mode 100644
index 000000000000..67c97f97cc29
--- /dev/null
+++ b/src/cosdruntime.conf
@@ -0,0 +1,9 @@
+; runtime options
+
+[global]
+	logger dir = /home/sage/ceph/src/log
+	dout dir = /home/sage/ceph/src/out
+	dout sym dir = /home/sage/ceph/src/out
+	monmap file = /home/sage/ceph/src/mondata/mon0/monmap/1
+	chdir root = false
+
diff --git a/src/mkcephfs b/src/mkcephfs
index 27e1898a8afb..bc4aaf84f704 100755
--- a/src/mkcephfs
+++ b/src/mkcephfs
@@ -26,7 +26,7 @@ numosd=
 
 while [ $# -ge 1 ]; do
 case $1 in
-    --allhosts)
+    --allhosts | -a)
 	    allhosts=1
 	    ;;
     --clobber_old_data)
@@ -84,37 +84,33 @@ for name in $what; do
     num=`echo $name | cut -c 4-`
     sections="$name $type global"
 
-    # what host is this daemon assigned to?
-    host=`$CCONF -c $conf -s $name -s $type host`
-    ssh=""
-    if [[ $host != "" ]]; then
-	#echo host for $name is $host, i am $hostname
-	if [[ $host != $hostname ]]; then
-	    # skip, unless we're starting remote daemons too
-	    if [[ $allhosts -eq 0 ]]; then
-		continue;
-	    fi
-
-	    # we'll need to ssh into that host
-	    ssh="ssh root@$host"
-	fi
-    else
-	host=$hostname
-    fi
+    check_host || continue
 
     get_conf conf_file "$runtime_conf" "conf file" $sections
 
+    if [[ $ssh = 1 ]] && [[ ! $pushed_to =~ " $host " ]]; then
+	scp $osdmap $host:$osdmap
+	scp $monmap $host:$monmap
+	pushed_to="$pushed_to $host "
+    fi
+
     if [[ $type = "mon" ]]; then
-	get_conf mon_path "" "mon data" $sections
-	$BINDIR/mkmonfs -c $conf_file $clobber $mon_path --mon $num \
-	    --monmap $monmap --osdmap $osdmap || exit 1
+	get_conf mon_data "" "mon data" $sections
+	do_cmd "$BINDIR/mkmonfs $clobber $mon_data --mon $num --monmap $monmap --osdmap $osdmap"
     fi
 
     if [[ $type = "osd" ]]; then
 	get_conf osd_path "" "osd data" $sections
+	get_conf btrfs_path "$osd_path" "btrfs path" $sections  # mount point defaults so osd path
+	get_conf btrfs_devs "" "btrfs devs" $sections
+	first_dev=`echo $btrfs_devs | cut '-d ' -f 1`
+	
+	if [ $mkbtrfs -eq 1 ]; then
+	    do_cmd "modprobe btrfs ; umount $btrfs_path ; mkfs.btrfs $btrfs_devs ; mount -t btrfs $first_dev $btrfs_path"
+	fi
+
 	[[ $ssh != "" ]] && scp $monmap $host:$monmap
-	$ssh $BINDIR/cosd -c $conf_file --monmap_file $monmap \
-	    --mkfs_for_osd $num $osd_path || exit 1
+	do_cmd "$BINDIR/cosd -c $conf_file --monmap_file $monmap --mkfs_for_osd $num $osd_path"
     fi
 
 done