From 302d8d5f61bbce4d68cbb41e106d5dae4617a0f4 Mon Sep 17 00:00:00 2001
From: Kefu Chai <kchai@redhat.com>
Date: Wed, 15 Mar 2017 15:28:09 +0800
Subject: [PATCH] vstart.sh: extract start_{osd,mon,mgr,mds} into functions

Signed-off-by: Kefu Chai <kchai@redhat.com>
---
 src/vstart.sh | 477 ++++++++++++++++++++++++++------------------------
 1 file changed, 250 insertions(+), 227 deletions(-)

diff --git a/src/vstart.sh b/src/vstart.sh
index cfa08fe1b77d5..758f6a4bf6a82 100755
--- a/src/vstart.sh
+++ b/src/vstart.sh
@@ -370,141 +370,16 @@ wconf() {
 	fi
 }
 
-if [ "$debug" -eq 0 ]; then
-    CMONDEBUG='
-	debug mon = 10
-        debug ms = 1'
-    COSDDEBUG='
-        debug ms = 1'
-    CMDSDEBUG='
-        debug ms = 1'
-else
-    echo "** going verbose **"
-    CMONDEBUG='
-	debug mon = 20
-        debug paxos = 20
-        debug auth = 20
-        debug ms = 1'
-    COSDDEBUG='
-        debug ms = 1
-        debug osd = 25
-        debug objecter = 20
-        debug monc = 20
-        debug mgrc = 20
-        debug journal = 20
-        debug filestore = 20
-        debug bluestore = 30
-        debug bluefs = 20
-        debug rocksdb = 10
-        debug bdev = 20
-        debug rgw = 20
-        debug objclass = 20'
-    CMDSDEBUG='
-        debug ms = 1
-        debug mds = 20
-        debug auth = 20
-        debug monc = 20
-        debug mgrc = 20
-        mds debug scatterstat = true
-        mds verify scatter = true
-        mds log max segments = 2'
-    CMGRDEBUG='
-        debug ms = 1
-        debug monc = 20
-        debug mgr = 20'
-fi
-
-if [ -n "$MON_ADDR" ]; then
-	CMON_ARGS=" -m "$MON_ADDR
-	COSD_ARGS=" -m "$MON_ADDR
-	CMDS_ARGS=" -m "$MON_ADDR
-fi
-
-if [ "$memstore" -eq 1 ]; then
-    COSDMEMSTORE='
-	osd objectstore = memstore'
-fi
-if [ "$bluestore" -eq 1 ]; then
-    COSDMEMSTORE='
-	osd objectstore = bluestore'
-fi
-
-if [ -z "$CEPH_PORT" ]; then
-    CEPH_PORT=6789
-    [ -e ".ceph_port" ] && CEPH_PORT=`cat .ceph_port`
-fi
-
-[ -z "$INIT_CEPH" ] && INIT_CEPH=$CEPH_BIN/init-ceph
-
-# sudo if btrfs
-test -d $CEPH_DEV_DIR/osd0/. && test -e $CEPH_DEV_DIR/sudo && SUDO="sudo"
-
-prun $SUDO rm -f core*
-
-test -d $CEPH_OUT_DIR || mkdir $CEPH_OUT_DIR
-test -d $CEPH_DEV_DIR || mkdir $CEPH_DEV_DIR
-$SUDO rm -rf $CEPH_OUT_DIR/*
-test -d gmon && $SUDO rm -rf gmon/*
-
-[ "$cephx" -eq 1 ] && [ "$new" -eq 1 ] && test -e $keyring_fn && rm $keyring_fn
-
-
-# figure machine's ip
-HOSTNAME=`hostname -s`
-if [ -n "$ip" ]; then
-    IP="$ip"
-else
-    echo hostname $HOSTNAME
-    if [ -x "$(which ip 2>/dev/null)" ]; then
-	IP_CMD="ip addr"
-    else
-	IP_CMD="ifconfig"
-    fi
-    # filter out IPv6 and localhost addresses
-    IP="$($IP_CMD | sed -En 's/127.0.0.1//;s/.*inet (addr:)?(([0-9]*\.){3}[0-9]*).*/\2/p' | head -n1)"
-    # if nothing left, try using localhost address, it might work
-    if [ -z "$IP" ]; then IP="127.0.0.1"; fi
-fi
-echo "ip $IP"
-echo "port $CEPH_PORT"
-
-
-[ -z $CEPH_ADM ] && CEPH_ADM=$CEPH_BIN/ceph
-
-ceph_adm() {
-    if [ "$cephx" -eq 1 ]; then
-        prun $SUDO "$CEPH_ADM" -c "$conf_fn" -k "$keyring_fn" "$@"
-    else
-        prun $SUDO "$CEPH_ADM" -c "$conf_fn" "$@"
-    fi
-}
-
-MONS=""
-count=0
-for f in a b c d e f g h i j k l m n o p q r s t u v w x y z
-do
-    if [ -z "$MONS" ];
-    then
-	MONS="$f"
-    else
-	MONS="$MONS $f"
-    fi
-    count=$(($count + 1))
-    [ $count -eq $CEPH_NUM_MON ] && break;
-done
-
-DAEMONOPTS="
-	log file = $CEPH_OUT_DIR/\$name.log
+prepare_conf() {
+    local DAEMONOPTS="
+        log file = $CEPH_OUT_DIR/\$name.log
         admin socket = $CEPH_OUT_DIR/\$name.asok
-	chdir = \"\"
-	pid file = $CEPH_OUT_DIR/\$name.pid
+        chdir = \"\"
+        pid file = $CEPH_OUT_DIR/\$name.pid
         heartbeat file = $CEPH_OUT_DIR/\$name.heartbeat
 "
 
-
-if [ "$start_mon" -eq 1 ]; then
-	if [ "$new" -eq 1 ]; then
-        wconf <<EOF
+    wconf <<EOF
 ; generated by vstart.sh on `date`
 [$VSTART_SEC]
         num mon = $CEPH_NUM_MON
@@ -533,29 +408,29 @@ if [ "$start_mon" -eq 1 ]; then
         run dir = $CEPH_OUT_DIR
         enable experimental unrecoverable data corrupting features = *
 EOF
-		if [ "$lockdep" -eq 1 ] ; then
-			wconf <<EOF
+	if [ "$lockdep" -eq 1 ] ; then
+		wconf <<EOF
         lockdep = true
 EOF
-		fi
-		if [ "$cephx" -eq 1 ] ; then
-			wconf <<EOF
+	fi
+	if [ "$cephx" -eq 1 ] ; then
+		wconf <<EOF
         auth cluster required = cephx
         auth service required = cephx
         auth client required = cephx
 EOF
-		else
-			wconf <<EOF
+	else
+		wconf <<EOF
 	auth cluster required = none
 	auth service required = none
 	auth client required = none
 EOF
-		fi
-		if [ "$short" -eq 1 ]; then
-			COSDSHORT="        osd max object name len = 460
+	fi
+	if [ "$short" -eq 1 ]; then
+		COSDSHORT="        osd max object name len = 460
         osd max object namespace len = 64"
-		fi
-		wconf <<EOF
+	fi
+	wconf <<EOF
 [client]
         keyring = $keyring_fn
         log file = $CEPH_OUT_DIR/\$name.\$pid.log
@@ -598,12 +473,12 @@ $DAEMONOPTS
         filestore wbthrottle btrfs ios hard limit = 20
         filestore wbthrottle btrfs inodes hard limit = 30
         osd copyfrom max chunk = 524288
-	bluestore fsck on mount = true
-	bluestore block create = true
-	bluestore block db size = 67108864
-	bluestore block db create = true
-	bluestore block wal size = 1048576000
-	bluestore block wal create = true
+        bluestore fsck on mount = true
+        bluestore block create = true
+        bluestore block db size = 67108864
+        bluestore block db create = true
+        bluestore block wal size = 1048576000
+        bluestore block wal create = true
 $COSDDEBUG
 $COSDMEMSTORE
 $COSDSHORT
@@ -622,6 +497,24 @@ $extra_conf
 [global]
 $extra_conf
 EOF
+}
+
+start_mon() {
+    local MONS=""
+    local count=0
+    for f in a b c d e f g h i j k l m n o p q r s t u v w x y z
+    do
+        if [ -z "$MONS" ];
+        then
+	        MONS="$f"
+        else
+	        MONS="$MONS $f"
+        fi
+        count=$(($count + 1))
+        [ $count -eq $CEPH_NUM_MON ] && break;
+    done
+
+    if [ "$new" -eq 1 ]; then
 		if [ `echo $IP | grep '^127\\.'` ]
 		then
 			echo
@@ -639,8 +532,8 @@ EOF
 			"$keyring_fn"
 
 		# build a fresh fs monmap, mon fs
-		str=""
-		count=0
+		local str=""
+		local count=0
 		for f in $MONS
 		do
 			str="$str --add $f $IP:$(($CEPH_PORT+$count))"
@@ -671,55 +564,69 @@ EOF
 		    run 'mon' $CEPH_BIN/ceph-mon -i $f $ARGS $CMON_ARGS
 		done
 	fi
-fi
+}
 
-#osd
-if [ "$start_osd" -eq 1 ]; then
+start_osd() {
     for osd in `seq 0 $((CEPH_NUM_OSD-1))`
     do
-	if [ "$new" -eq 1 ]; then
-		wconf <<EOF
+	    if [ "$new" -eq 1 ]; then
+		    wconf <<EOF
 [osd.$osd]
         host = $HOSTNAME
 EOF
 
-	    rm -rf $CEPH_DEV_DIR/osd$osd || true
+            rm -rf $CEPH_DEV_DIR/osd$osd || true
             if command -v btrfs > /dev/null; then
-	        for f in $CEPH_DEV_DIR/osd$osd/*; do btrfs sub delete $f &> /dev/null || true; done
+                for f in $CEPH_DEV_DIR/osd$osd/*; do btrfs sub delete $f &> /dev/null || true; done
             fi
-	    mkdir -p $CEPH_DEV_DIR/osd$osd
+            mkdir -p $CEPH_DEV_DIR/osd$osd
 
-	    uuid=`uuidgen`
-	    echo "add osd$osd $uuid"
-	    ceph_adm osd create $uuid
-	    ceph_adm osd crush add osd.$osd 1.0 host=$HOSTNAME root=default
-	    $SUDO $CEPH_BIN/ceph-osd -i $osd $ARGS --mkfs --mkkey --osd-uuid $uuid
+            local uuid=`uuidgen`
+            echo "add osd$osd $uuid"
+            ceph_adm osd create $uuid
+            ceph_adm osd crush add osd.$osd 1.0 host=$HOSTNAME root=default
+            $SUDO $CEPH_BIN/ceph-osd -i $osd $ARGS --mkfs --mkkey --osd-uuid $uuid
 
-	    key_fn=$CEPH_DEV_DIR/osd$osd/keyring
-	    echo adding osd$osd key to auth repository
-	    ceph_adm -i "$key_fn" auth add osd.$osd osd "allow *" mon "allow profile osd" mgr "allow"
-	fi
-	echo start osd$osd
-	run 'osd' $SUDO $CEPH_BIN/ceph-osd -i $osd $ARGS $COSD_ARGS
+            local key_fn=$CEPH_DEV_DIR/osd$osd/keyring
+            echo adding osd$osd key to auth repository
+            ceph_adm -i "$key_fn" auth add osd.$osd osd "allow *" mon "allow profile osd" mgr "allow"
+        fi
+        echo start osd$osd
+        run 'osd' $SUDO $CEPH_BIN/ceph-osd -i $osd $ARGS $COSD_ARGS
     done
-fi
+}
 
-# mds
-if [ "$smallmds" -eq 1 ]; then
-    wconf <<EOF
-[mds]
-	mds log max segments = 2
-	mds cache size = 10000
+start_mgr() {
+    local mgr=0
+    for name in x y z a b c d e f g h i j k l m n o p
+    do
+        if [ "$new" -eq 1 ]; then
+            mkdir -p $CEPH_DEV_DIR/mgr.$name
+            key_fn=$CEPH_DEV_DIR/mgr.$name/keyring
+            $SUDO $CEPH_BIN/ceph-authtool --create-keyring --gen-key --name=mgr.$name $key_fn
+            ceph_adm -i $key_fn auth add mgr.$name mon 'allow *'
+        fi
+
+        wconf <<EOF
+[mgr.$name]
+        host = $HOSTNAME
 EOF
-fi
 
-if [ "$start_mds" -eq 1 -a "$CEPH_NUM_MDS" -gt 0 ]; then
+        echo "Starting mgr.${name}"
+        run 'mgr' $CEPH_BIN/ceph-mgr -i $name
+
+        mgr=$(($mgr + 1))
+        [ $mgr -eq $CEPH_NUM_MGR ] && break
+    done
+}
+
+start_mds() {
     if [ "$CEPH_NUM_FS" -gt "0" ] ; then
         if [ "$CEPH_NUM_FS" -gt "1" ] ; then
             ceph_adm fs flag set enable_multiple true --yes-i-really-mean-it
         fi
 
-        fs=0
+        local fs=0
         for name in a b c d e f g h i j k l m n o p
         do
             ceph_adm osd pool create "cephfs_data_${name}" 8
@@ -730,48 +637,182 @@ if [ "$start_mds" -eq 1 -a "$CEPH_NUM_MDS" -gt 0 ]; then
         done
     fi
 
-    mds=0
+    local mds=0
     for name in a b c d e f g h i j k l m n o p
     do
-	if [ "$new" -eq 1 ]; then
-	    prun mkdir -p "$CEPH_DEV_DIR/mds.$name"
-	    key_fn=$CEPH_DEV_DIR/mds.$name/keyring
-	    wconf <<EOF
+	    if [ "$new" -eq 1 ]; then
+	        prun mkdir -p "$CEPH_DEV_DIR/mds.$name"
+	        key_fn=$CEPH_DEV_DIR/mds.$name/keyring
+	        wconf <<EOF
 [mds.$name]
         host = $HOSTNAME
 EOF
-		if [ "$standby" -eq 1 ]; then
-		    mkdir -p $CEPH_DEV_DIR/mds.${name}s
-			wconf <<EOF
+		    if [ "$standby" -eq 1 ]; then
+		        mkdir -p $CEPH_DEV_DIR/mds.${name}s
+			    wconf <<EOF
        mds standby for rank = $mds
 [mds.${name}s]
         mds standby replay = true
         mds standby for name = ${name}
 EOF
+	        fi
+	        prun $SUDO "$CEPH_BIN/ceph-authtool" --create-keyring --gen-key --name="mds.$name" "$key_fn"
+	        ceph_adm -i "$key_fn" auth add "mds.$name" mon 'allow profile mds' osd 'allow *' mds 'allow' mgr 'allow'
+	        if [ "$standby" -eq 1 ]; then
+			    prun $SUDO "$CEPH_BIN/ceph-authtool" --create-keyring --gen-key --name="mds.${name}s" \
+				     "$CEPH_DEV_DIR/mds.${name}s/keyring"
+			    ceph_adm -i "$CEPH_DEV_DIR/mds.${name}s/keyring" auth add "mds.${name}s" \
+				         mon 'allow *' osd 'allow *' mds 'allow' mgr 'allow'
+	        fi
 	    fi
-	    prun $SUDO "$CEPH_BIN/ceph-authtool" --create-keyring --gen-key --name="mds.$name" "$key_fn"
-	    ceph_adm -i "$key_fn" auth add "mds.$name" mon 'allow profile mds' osd 'allow *' mds 'allow' mgr 'allow'
+
+	    run 'mds' $CEPH_BIN/ceph-mds -i $name $ARGS $CMDS_ARGS
 	    if [ "$standby" -eq 1 ]; then
-			prun $SUDO "$CEPH_BIN/ceph-authtool" --create-keyring --gen-key --name="mds.${name}s" \
-				"$CEPH_DEV_DIR/mds.${name}s/keyring"
-			ceph_adm -i "$CEPH_DEV_DIR/mds.${name}s/keyring" auth add "mds.${name}s" \
-				mon 'allow *' osd 'allow *' mds 'allow' mgr 'allow'
+	        run 'mds' $CEPH_BIN/ceph-mds -i ${name}s $ARGS $CMDS_ARGS
 	    fi
 
-	fi
-	
-	run 'mds' $CEPH_BIN/ceph-mds -i $name $ARGS $CMDS_ARGS
-	if [ "$standby" -eq 1 ]; then
-	    run 'mds' $CEPH_BIN/ceph-mds -i ${name}s $ARGS $CMDS_ARGS
-	fi
-	
-	mds=$(($mds + 1))
-	[ $mds -eq $CEPH_NUM_MDS ] && break
+	    mds=$(($mds + 1))
+	    [ $mds -eq $CEPH_NUM_MDS ] && break
 
-#valgrind --tool=massif $CEPH_BIN/ceph-mds $ARGS --mds_log_max_segments 2 --mds_thrash_fragments 0 --mds_thrash_exports 0 > m  #--debug_ms 20
-#$CEPH_BIN/ceph-mds -d $ARGS --mds_thrash_fragments 0 --mds_thrash_exports 0 #--debug_ms 20
-#ceph_adm mds set max_mds 2
+        #valgrind --tool=massif $CEPH_BIN/ceph-mds $ARGS --mds_log_max_segments 2 --mds_thrash_fragments 0 --mds_thrash_exports 0 > m  #--debug_ms 20
+        #$CEPH_BIN/ceph-mds -d $ARGS --mds_thrash_fragments 0 --mds_thrash_exports 0 #--debug_ms 20
+        #ceph_adm mds set max_mds 2
     done
+}
+
+if [ "$debug" -eq 0 ]; then
+    CMONDEBUG='
+        debug mon = 10
+        debug ms = 1'
+    COSDDEBUG='
+        debug ms = 1'
+    CMDSDEBUG='
+        debug ms = 1'
+else
+    echo "** going verbose **"
+    CMONDEBUG='
+        debug mon = 20
+        debug paxos = 20
+        debug auth = 20
+        debug ms = 1'
+    COSDDEBUG='
+        debug ms = 1
+        debug osd = 25
+        debug objecter = 20
+        debug monc = 20
+        debug mgrc = 20
+        debug journal = 20
+        debug filestore = 20
+        debug bluestore = 30
+        debug bluefs = 20
+        debug rocksdb = 10
+        debug bdev = 20
+        debug rgw = 20
+        debug objclass = 20'
+    CMDSDEBUG='
+        debug ms = 1
+        debug mds = 20
+        debug auth = 20
+        debug monc = 20
+        debug mgrc = 20
+        mds debug scatterstat = true
+        mds verify scatter = true
+        mds log max segments = 2'
+    CMGRDEBUG='
+        debug ms = 1
+        debug monc = 20
+        debug mgr = 20'
+fi
+
+if [ -n "$MON_ADDR" ]; then
+	CMON_ARGS=" -m "$MON_ADDR
+	COSD_ARGS=" -m "$MON_ADDR
+	CMDS_ARGS=" -m "$MON_ADDR
+fi
+
+if [ "$memstore" -eq 1 ]; then
+    COSDMEMSTORE='
+	osd objectstore = memstore'
+fi
+if [ "$bluestore" -eq 1 ]; then
+    COSDMEMSTORE='
+	osd objectstore = bluestore'
+fi
+
+if [ -z "$CEPH_PORT" ]; then
+    CEPH_PORT=6789
+    [ -e ".ceph_port" ] && CEPH_PORT=`cat .ceph_port`
+fi
+
+[ -z "$INIT_CEPH" ] && INIT_CEPH=$CEPH_BIN/init-ceph
+
+# sudo if btrfs
+test -d $CEPH_DEV_DIR/osd0/. && test -e $CEPH_DEV_DIR/sudo && SUDO="sudo"
+
+prun $SUDO rm -f core*
+
+test -d $CEPH_OUT_DIR || mkdir $CEPH_OUT_DIR
+test -d $CEPH_DEV_DIR || mkdir $CEPH_DEV_DIR
+$SUDO rm -rf $CEPH_OUT_DIR/*
+test -d gmon && $SUDO rm -rf gmon/*
+
+[ "$cephx" -eq 1 ] && [ "$new" -eq 1 ] && test -e $keyring_fn && rm $keyring_fn
+
+
+# figure machine's ip
+HOSTNAME=`hostname -s`
+if [ -n "$ip" ]; then
+    IP="$ip"
+else
+    echo hostname $HOSTNAME
+    if [ -x "$(which ip 2>/dev/null)" ]; then
+	IP_CMD="ip addr"
+    else
+	IP_CMD="ifconfig"
+    fi
+    # filter out IPv6 and localhost addresses
+    IP="$($IP_CMD | sed -En 's/127.0.0.1//;s/.*inet (addr:)?(([0-9]*\.){3}[0-9]*).*/\2/p' | head -n1)"
+    # if nothing left, try using localhost address, it might work
+    if [ -z "$IP" ]; then IP="127.0.0.1"; fi
+fi
+echo "ip $IP"
+echo "port $CEPH_PORT"
+
+
+[ -z $CEPH_ADM ] && CEPH_ADM=$CEPH_BIN/ceph
+
+ceph_adm() {
+    if [ "$cephx" -eq 1 ]; then
+        prun $SUDO "$CEPH_ADM" -c "$conf_fn" -k "$keyring_fn" "$@"
+    else
+        prun $SUDO "$CEPH_ADM" -c "$conf_fn" "$@"
+    fi
+}
+
+if [ "$new" -eq 1 ]; then
+    prepare_conf
+fi
+
+if [ "$start_mon" -eq 1 ]; then
+    start_mon
+fi
+
+# osd
+if [ "$start_osd" -eq 1 ]; then
+    start_osd
+fi
+
+# mds
+if [ "$smallmds" -eq 1 ]; then
+    wconf <<EOF
+[mds]
+	mds log max segments = 2
+	mds cache size = 10000
+EOF
+fi
+
+if [ "$start_mds" -eq 1 -a "$CEPH_NUM_MDS" -gt 0 ]; then
+    start_mds
 fi
 
 # Don't set max_mds until all the daemons are started, otherwise
@@ -790,28 +831,10 @@ do
     [ $fs -eq $CEPH_NUM_FS ] && break
 done
 
-if [ "$CEPH_NUM_MGR" -gt 0 ]; then
-    mgr=0
-    for name in x y z a b c d e f g h i j k l m n o p
-    do
-        if [ "$new" -eq 1 ]; then
-            mkdir -p $CEPH_DEV_DIR/mgr.$name
-            key_fn=$CEPH_DEV_DIR/mgr.$name/keyring
-            $SUDO $CEPH_BIN/ceph-authtool --create-keyring --gen-key --name=mgr.$name $key_fn
-            ceph_adm -i $key_fn auth add mgr.$name mon 'allow *'
-        fi
-
-        wconf <<EOF
-[mgr.$name]
-        host = $HOSTNAME
-EOF
-
-        echo "Starting mgr.${name}"
-        run 'mgr' $CEPH_BIN/ceph-mgr -i $name
+# mgr
 
-        mgr=$(($mgr + 1))
-        [ $mgr -eq $CEPH_NUM_MGR ] && break
-    done
+if [ "$CEPH_NUM_MGR" -gt 0 ]; then
+    start_mgr
 fi
 
 if [ "$ec" -eq 1 ]; then
-- 
2.39.5