From 65d6ba87130a6bf36122050ac1b2a15831ca6587 Mon Sep 17 00:00:00 2001 From: Joao Eduardo Luis Date: Tue, 16 Jul 2019 14:53:27 +0000 Subject: [PATCH] vstart.sh: run multiple rgws with different ids We should expect being able to run multiple rgws with different ids using vstart, and yet that was not the case as they were sharing the same name (and same key). We now mimic each rgw running on a different host, with ids being their ports; as such, we'll have 'client.rgw.X', 'client.rgw.X+1', and so forth. These will match the ports the daemons are running on. Signed-off-by: Joao Eduardo Luis --- src/mrgw.sh | 8 ++++- src/vstart.sh | 88 +++++++++++++++++++++++++++++++++++++++------------ 2 files changed, 75 insertions(+), 21 deletions(-) diff --git a/src/mrgw.sh b/src/mrgw.sh index 7579f663bee..54e8039e643 100755 --- a/src/mrgw.sh +++ b/src/mrgw.sh @@ -28,4 +28,10 @@ logfile=$run_root/out/radosgw.${port}.log $vstart_path/mstop.sh $name radosgw $port -$vstart_path/mrun $name radosgw --rgw-frontends="$rgw_frontend port=$port" -n client.rgw --pid-file=$pidfile --admin-socket=$asokfile "$@" --log-file=$logfile +$vstart_path/mrun $name ceph -c $run_root/ceph.conf \ + -k $run_root/keyring auth get-or-create client.rgw.$port mon \ + 'allow rw' osd 'allow rwx' mgr 'allow rw' >> $run_root/keyring + +$vstart_path/mrun $name radosgw --rgw-frontends="$rgw_frontend port=$port" \ + -n client.rgw.$port --pid-file=$pidfile \ + --admin-socket=$asokfile "$@" --log-file=$logfile diff --git a/src/vstart.sh b/src/vstart.sh index 4f600120d2e..f36a0820bdb 100755 --- a/src/vstart.sh +++ b/src/vstart.sh @@ -466,6 +466,13 @@ prun() { "$@" } +prun_to_file() { + f=$1 + shift + quoted_print "$@" + "$@" >> $f +} + run() { type=$1 shift @@ -503,6 +510,32 @@ get_pci_selector_num() { lspci -mm -n -D -d $pci_id | cut -d' ' -f 1 | wc -l } +do_rgw_conf() { + + if [ $CEPH_NUM_RGW -eq 0 ]; then + return 0 + fi + + # setup each rgw on a sequential port, starting at $CEPH_RGW_PORT. + # individual rgw's ids will be their ports. + current_port=$CEPH_RGW_PORT + for n in $(seq 1 $CEPH_NUM_RGW); do + wconf << EOF +[client.rgw.${current_port}] + rgw frontends = $rgw_frontend port=${current_port} + admin socket = ${CEPH_OUT_DIR}/radosgw.${current_port}.asok + ; needed for s3tests + rgw crypt s3 kms encryption keys = testkey-1=YmluCmJvb3N0CmJvb3N0LWJ1aWxkCmNlcGguY29uZgo= testkey-2=aWIKTWFrZWZpbGUKbWFuCm91dApzcmMKVGVzdGluZwo= + rgw crypt require ssl = false + ; uncomment the following to set LC days as the value in seconds; + ; needed for passing lc time based s3-tests (can be verbose) + ; rgw lc debug interval = 10 +EOF + current_port=$((current_port + 1)) +done + +} + prepare_conf() { local DAEMONOPTS=" log file = $CEPH_OUT_DIR/\$name.log @@ -624,15 +657,10 @@ EOF log file = $CEPH_OUT_DIR/\$name.\$pid.log admin socket = $CEPH_ASOK_DIR/\$name.\$pid.asok $extra_conf -[client.rgw] - rgw frontends = $rgw_frontend port=$CEPH_RGW_PORT - admin socket = ${CEPH_OUT_DIR}/radosgw.${CEPH_RGW_PORT}.asok - ; needed for s3tests - rgw crypt s3 kms encryption keys = testkey-1=YmluCmJvb3N0CmJvb3N0LWJ1aWxkCmNlcGguY29uZgo= testkey-2=aWIKTWFrZWZpbGUKbWFuCm91dApzcmMKVGVzdGluZwo= - rgw crypt require ssl = false - ; uncomment the following to set LC days as the value in seconds; - ; needed for passing lc time based s3-tests (can be verbose) - ; rgw lc debug interval = 10 +EOF + + do_rgw_conf + wconf << EOF [mds] $DAEMONOPTS mds data = $CEPH_DEV_DIR/mds.\$id @@ -717,12 +745,6 @@ start_mon() { --cap mds 'allow rwp' \ "$keyring_fn" - prun $SUDO "$CEPH_BIN/ceph-authtool" --gen-key --name=client.rgw \ - --cap mon 'allow rw' \ - --cap osd 'allow rwx' \ - --cap mgr 'allow rw' \ - "$keyring_fn" - # build a fresh fs monmap, mon fs local params=() local count=0 @@ -1036,6 +1058,17 @@ ceph_adm() { fi } +ceph_adm_to_file() { + f=$1 + shift + + if [ "$cephx" -eq 1 ]; then + prun_to_file $f $SUDO "$CEPH_ADM" -c "$conf_fn" -k "$keyring_fn" "$@" + else + prun_to_file $f $SUDO "$CEPH_ADM" -c "$conf_fn" "$@" + fi +} + if [ "$new" -eq 1 ]; then prepare_conf fi @@ -1252,14 +1285,29 @@ do_rgw() fi RGWSUDO= [ $CEPH_RGW_PORT_NUM -lt 1024 ] && RGWSUDO=sudo - n=$(($CEPH_NUM_RGW - 1)) - i=0 - for rgw in j k l m n o p q r s t u v; do - current_port=$((CEPH_RGW_PORT_NUM + i)) + + current_port=$CEPH_RGW_PORT + for n in $(seq 1 $CEPH_NUM_RGW); do + rgw_name="client.rgw.${current_port}" + + ceph_adm_to_file $keyring_fn auth get-or-create $rgw_name \ + mon 'allow rw' \ + osd 'allow rwx' \ + mgr 'allow rw' \ + echo start rgw on http${CEPH_RGW_HTTPS}://localhost:${current_port} - run 'rgw' $current_port $RGWSUDO $CEPH_BIN/radosgw -c $conf_fn --log-file=${CEPH_OUT_DIR}/radosgw.${current_port}.log --admin-socket=${CEPH_OUT_DIR}/radosgw.${current_port}.asok --pid-file=${CEPH_OUT_DIR}/radosgw.${current_port}.pid ${RGWDEBUG} -n client.rgw "--rgw_frontends=${rgw_frontend} port=${current_port}${CEPH_RGW_HTTPS}" + run 'rgw' $current_port $RGWSUDO $CEPH_BIN/radosgw -c $conf_fn \ + --log-file=${CEPH_OUT_DIR}/radosgw.${current_port}.log \ + --admin-socket=${CEPH_OUT_DIR}/radosgw.${current_port}.asok \ + --pid-file=${CEPH_OUT_DIR}/radosgw.${current_port}.pid \ + ${RGWDEBUG} \ + -n ${rgw_name} \ + "--rgw_frontends=${rgw_frontend} port=${current_port}${CEPH_RGW_HTTPS}" + i=$(($i + 1)) [ $i -eq $CEPH_NUM_RGW ] && break + + current_port=$((current_port+1)) done } if [ "$CEPH_NUM_RGW" -gt 0 ]; then -- 2.39.5