vstart.sh: run multiple rgws with different ids

We should expect being able to run multiple rgws with different ids
using vstart, and yet that was not the case as they were sharing the
same name (and same key).

We now mimic each rgw running on a different host, with ids being their
ports; as such, we'll have 'client.rgw.X', 'client.rgw.X+1', and so
forth. These will match the ports the daemons are running on.

Signed-off-by: Joao Eduardo Luis <joao@suse.com>

diff --git a/src/mrgw.sh b/src/mrgw.sh
index 7579f663beef0b74b43452f42b754a21d568fc5f..54e8039e643d6ab8b1b0e8f589102dc5c656e75b 100755 (executable)
--- a/src/mrgw.sh
+++ b/src/mrgw.sh
@@ -28,4 +28,10 @@ logfile=$run_root/out/radosgw.${port}.log
 
 $vstart_path/mstop.sh $name radosgw $port
 
-$vstart_path/mrun $name radosgw --rgw-frontends="$rgw_frontend port=$port" -n client.rgw --pid-file=$pidfile --admin-socket=$asokfile "$@" --log-file=$logfile
+$vstart_path/mrun $name ceph -c $run_root/ceph.conf \
+       -k $run_root/keyring auth get-or-create client.rgw.$port mon \
+       'allow rw' osd 'allow rwx' mgr 'allow rw' >> $run_root/keyring
+
+$vstart_path/mrun $name radosgw --rgw-frontends="$rgw_frontend port=$port" \
+       -n client.rgw.$port --pid-file=$pidfile \
+       --admin-socket=$asokfile "$@" --log-file=$logfile

diff --git a/src/vstart.sh b/src/vstart.sh
index 4f600120d2eb04dc582b03ace2e493013a2d1dfb..f36a0820bdbb9bd2b7ec5e7d1464d97df0d7117e 100755 (executable)
--- a/src/vstart.sh
+++ b/src/vstart.sh
@@ -466,6 +466,13 @@ prun() {
     "$@"
 }
 
+prun_to_file() {
+       f=$1
+       shift
+       quoted_print "$@"
+       "$@" >> $f
+}
+
 run() {
     type=$1
     shift
@@ -503,6 +510,32 @@ get_pci_selector_num() {
     lspci -mm -n -D -d $pci_id | cut -d' ' -f 1 | wc -l
 }
 
+do_rgw_conf() {
+
+    if [ $CEPH_NUM_RGW -eq 0 ]; then
+        return 0
+    fi
+
+    # setup each rgw on a sequential port, starting at $CEPH_RGW_PORT.
+    # individual rgw's ids will be their ports.
+    current_port=$CEPH_RGW_PORT
+    for n in $(seq 1 $CEPH_NUM_RGW); do
+        wconf << EOF
+[client.rgw.${current_port}]
+        rgw frontends = $rgw_frontend port=${current_port}
+        admin socket = ${CEPH_OUT_DIR}/radosgw.${current_port}.asok
+        ; needed for s3tests
+        rgw crypt s3 kms encryption keys = testkey-1=YmluCmJvb3N0CmJvb3N0LWJ1aWxkCmNlcGguY29uZgo= testkey-2=aWIKTWFrZWZpbGUKbWFuCm91dApzcmMKVGVzdGluZwo=
+        rgw crypt require ssl = false
+        ; uncomment the following to set LC days as the value in seconds;
+        ; needed for passing lc time based s3-tests (can be verbose)
+        ; rgw lc debug interval = 10
+EOF
+        current_port=$((current_port + 1))
+done
+
+}
+
 prepare_conf() {
     local DAEMONOPTS="
         log file = $CEPH_OUT_DIR/\$name.log
@@ -624,15 +657,10 @@ EOF
         log file = $CEPH_OUT_DIR/\$name.\$pid.log
         admin socket = $CEPH_ASOK_DIR/\$name.\$pid.asok
 $extra_conf
-[client.rgw]
-        rgw frontends = $rgw_frontend port=$CEPH_RGW_PORT
-        admin socket = ${CEPH_OUT_DIR}/radosgw.${CEPH_RGW_PORT}.asok
-        ; needed for s3tests
-        rgw crypt s3 kms encryption keys = testkey-1=YmluCmJvb3N0CmJvb3N0LWJ1aWxkCmNlcGguY29uZgo= testkey-2=aWIKTWFrZWZpbGUKbWFuCm91dApzcmMKVGVzdGluZwo=
-        rgw crypt require ssl = false
-        ; uncomment the following to set LC days as the value in seconds;
-        ; needed for passing lc time based s3-tests (can be verbose)
-        ; rgw lc debug interval = 10
+EOF
+
+       do_rgw_conf
+       wconf << EOF
 [mds]
 $DAEMONOPTS
         mds data = $CEPH_DEV_DIR/mds.\$id
@@ -717,12 +745,6 @@ start_mon() {
              --cap mds 'allow rwp' \
              "$keyring_fn"
 
-        prun $SUDO "$CEPH_BIN/ceph-authtool" --gen-key --name=client.rgw \
-             --cap mon 'allow rw' \
-             --cap osd 'allow rwx' \
-             --cap mgr 'allow rw' \
-             "$keyring_fn"
-
         # build a fresh fs monmap, mon fs
         local params=()
         local count=0
@@ -1036,6 +1058,17 @@ ceph_adm() {
     fi
 }
 
+ceph_adm_to_file() {
+       f=$1
+       shift
+
+       if [ "$cephx" -eq 1 ]; then
+               prun_to_file $f $SUDO "$CEPH_ADM" -c "$conf_fn" -k "$keyring_fn" "$@"
+       else
+               prun_to_file $f $SUDO "$CEPH_ADM" -c "$conf_fn" "$@"
+       fi
+}
+
 if [ "$new" -eq 1 ]; then
     prepare_conf
 fi
@@ -1252,14 +1285,29 @@ do_rgw()
     fi
     RGWSUDO=
     [ $CEPH_RGW_PORT_NUM -lt 1024 ] && RGWSUDO=sudo
-    n=$(($CEPH_NUM_RGW - 1))
-    i=0
-    for rgw in j k l m n o p q r s t u v; do
-        current_port=$((CEPH_RGW_PORT_NUM + i))
+
+    current_port=$CEPH_RGW_PORT
+    for n in $(seq 1 $CEPH_NUM_RGW); do
+        rgw_name="client.rgw.${current_port}"
+
+        ceph_adm_to_file $keyring_fn auth get-or-create $rgw_name \
+            mon 'allow rw' \
+            osd 'allow rwx' \
+            mgr 'allow rw' \
+
         echo start rgw on http${CEPH_RGW_HTTPS}://localhost:${current_port}
-        run 'rgw' $current_port $RGWSUDO $CEPH_BIN/radosgw -c $conf_fn --log-file=${CEPH_OUT_DIR}/radosgw.${current_port}.log --admin-socket=${CEPH_OUT_DIR}/radosgw.${current_port}.asok --pid-file=${CEPH_OUT_DIR}/radosgw.${current_port}.pid ${RGWDEBUG} -n client.rgw "--rgw_frontends=${rgw_frontend} port=${current_port}${CEPH_RGW_HTTPS}"
+        run 'rgw' $current_port $RGWSUDO $CEPH_BIN/radosgw -c $conf_fn \
+            --log-file=${CEPH_OUT_DIR}/radosgw.${current_port}.log \
+            --admin-socket=${CEPH_OUT_DIR}/radosgw.${current_port}.asok \
+            --pid-file=${CEPH_OUT_DIR}/radosgw.${current_port}.pid \
+            ${RGWDEBUG} \
+            -n ${rgw_name} \
+            "--rgw_frontends=${rgw_frontend} port=${current_port}${CEPH_RGW_HTTPS}"
+
         i=$(($i + 1))
         [ $i -eq $CEPH_NUM_RGW ] && break
+
+        current_port=$((current_port+1))
     done
 }
 if [ "$CEPH_NUM_RGW" -gt 0 ]; then