From 639ea51a716b9c37c3dd95d8c1264a95584e9924 Mon Sep 17 00:00:00 2001 From: Ali Maredia Date: Mon, 5 Feb 2024 11:52:38 -0500 Subject: [PATCH] qa: enable test_awssdkv4_sig workunit to run in teuthology Signed-off-by: Ali Maredia --- qa/suites/rgw/verify/tasks/cls.yaml | 2 +- qa/tasks/rgw.py | 11 ++++ .../test/java/io/ceph/jcksum/PutObjects.java | 7 +++ qa/workunits/rgw/test_awssdkv4_sig.sh | 54 ++++++++++--------- 4 files changed, 47 insertions(+), 27 deletions(-) diff --git a/qa/suites/rgw/verify/tasks/cls.yaml b/qa/suites/rgw/verify/tasks/cls.yaml index 71cca69d603..568395a0dd0 100644 --- a/qa/suites/rgw/verify/tasks/cls.yaml +++ b/qa/suites/rgw/verify/tasks/cls.yaml @@ -13,4 +13,4 @@ tasks: - rgw/test_rgw_gc_log.sh - rgw/test_rgw_obj.sh - rgw/test_librgw_file.sh - - rgw/test_awssdkv4_sig.sh + - rgw/test_awssdkv4_sig.sh diff --git a/qa/tasks/rgw.py b/qa/tasks/rgw.py index 1adf7f7ee14..e733b73c61e 100644 --- a/qa/tasks/rgw.py +++ b/qa/tasks/rgw.py @@ -61,6 +61,16 @@ def start_rgw(ctx, config, clients): log.info("Using %s as radosgw frontend", ctx.rgw.frontend) endpoint = ctx.rgw.role_endpoints[client] + + # create a file with rgw endpoint in it for test_awssdkv4 workunit + url = endpoint.url() + # remove trailing slash from the url + if url[-1] == '/': + url = url[:-1] + url_file = '{tdir}/url_file'.format(tdir=testdir) + ctx.cluster.only(client).run(args=['sudo', 'echo', '-n', '{url}'.format(url=url), run.Raw('|'), 'sudo', 'tee', url_file]) + ctx.cluster.only(client).run(args=['sudo', 'chown', 'ceph', url_file]) + frontends = ctx.rgw.frontend frontend_prefix = client_config.get('frontend_prefix', None) if frontend_prefix: @@ -239,6 +249,7 @@ def start_rgw(ctx, config, clients): ], ) ctx.cluster.only(client).run(args=['sudo', 'rm', '-f', token_path]) + ctx.cluster.only(client).run(args=['sudo', 'rm', '-f', url_file]) rgwadmin(ctx, client, cmd=['gc', 'process', '--include-all'], check_status=True) def assign_endpoints(ctx, config, default_cert): diff --git a/qa/workunits/rgw/jcksum/src/test/java/io/ceph/jcksum/PutObjects.java b/qa/workunits/rgw/jcksum/src/test/java/io/ceph/jcksum/PutObjects.java index 5a3b788b924..9f9d3475c04 100644 --- a/qa/workunits/rgw/jcksum/src/test/java/io/ceph/jcksum/PutObjects.java +++ b/qa/workunits/rgw/jcksum/src/test/java/io/ceph/jcksum/PutObjects.java @@ -50,6 +50,7 @@ class PutObjects { void generateFile(String in_file_path, String out_file_path, long length) { try { + System.out.println("DEBUG: Generating File"); Path ifp = Paths.get(in_file_path); File f = ifp.toFile(); @@ -98,6 +99,7 @@ class PutObjects { if (wch != null) { wch.close(); } + System.out.println("DEBUG: File Generated"); } catch (IOException e) { System.err.println(e.getMessage()); } @@ -146,6 +148,7 @@ class PutObjects { /* https://sdk.amazonaws.com/java/api/latest/software/amazon/awssdk/services/s3/S3Client.html */ + System.out.println("DEBUG: Environment Variables Read"); try { client = S3Client.builder() .endpointOverride(http_uri) @@ -158,7 +161,9 @@ class PutObjects { System.exit(1); } + System.out.println("DEBUG: S3 Client Initialized"); generateBigFiles(); + System.out.println("DEBUG: Generated Big Files"); /* create test bucket if it doesn't exist yet */ try { @@ -167,6 +172,8 @@ class PutObjects { System.err.println(e.getMessage()); System.exit(1); } + + System.out.println("DEBUG: Test Bucket Created"); } /* setup */ /* TODO: zap */ diff --git a/qa/workunits/rgw/test_awssdkv4_sig.sh b/qa/workunits/rgw/test_awssdkv4_sig.sh index 5c56a46285f..0f4782260c4 100755 --- a/qa/workunits/rgw/test_awssdkv4_sig.sh +++ b/qa/workunits/rgw/test_awssdkv4_sig.sh @@ -1,45 +1,47 @@ -#!/bin/sh -e +#!/bin/sh # # To run this test script with a cluster created via vstart.sh: -# $PATH needs to be set for radosgw-admin and ceph_test_librgw executables. -# $KEYRING need to be set as the path for a vstart clusters Ceph keyring. +# $PATH needs to be set for radosgw-admin executables. +# $CEPH_ROOT needs to be set to the path of the Ceph source code +# $RGW_HTTP_ENDPOINT_URL needs to be set to the endpoint of the RGW # # Example when ceph source is cloned into $HOME and a vstart cluster is already running with a radosgw: -# $ PATH=~/ceph/build/bin/:$PATH KEYRING=~/ceph/build/keyring ~/ceph/qa/workunits/rgw/test_awssdkv4_sig.sh +# $ PATH=~/ceph/build/bin/:$PATH CEPH_ROOT=~/ceph RGW_HTTP_ENDPOINT=http://localhost:8000 ~/ceph/qa/workunits/rgw/test_awssdkv4_sig.sh +# + +set -x if [ -z ${AWS_ACCESS_KEY_ID} ] then - export AWS_ACCESS_KEY_ID=`openssl rand -base64 20` - export AWS_SECRET_ACCESS_KEY=`openssl rand -base64 40` + export AWS_ACCESS_KEY_ID="lNCnR47C2g+ZidCWBAUuwfSAA7Q=" + export AWS_SECRET_ACCESS_KEY="tYuA2Y+Uu1ow2l9Xe59tWKVml3gMuVfyhUjjJwfwEI0vFFONIcqf4g==" radosgw-admin user create --uid ceph-test-maven \ --access-key $AWS_ACCESS_KEY_ID \ --secret $AWS_SECRET_ACCESS_KEY \ --display-name "maven test user" \ --email sigv4@example.com || echo "sigv4 maven user exists" - - # keyring override for teuthology env - if [ -z ${KEYRING} ] - then - KEYRING="/etc/ceph/ceph.keyring" - fi - K="-k ${KEYRING}" fi -# the required S3 access_key and secret_key are already exported above, but -# we need to hook up the S3 endpoints - -# the following are taken from - -# XXXX ok, so I think there should be only RGW_HTTP_ENDPOINT_URL and that -# it may already be set before this script runs, though if we can figure it -# out, it's ok if we set it... - -export RGW_HTTP_ENDPOINT_URL="http://localhost:80" -#export RGW_HTTPS_ENDPOINT_URL="https://localhost:443" +if [ -z ${RGW_HTTP_ENDPOINT_URL} ] +then + # TESTDIR and this block are meant for when this script is run in a teuthology environment + if [ -z ${TESTDIR} ] + then + echo "TESTDIR is not defined, cannot set RGW_HTTP_ENDPOINT_URL in teuthology" + exit + else + export RGW_HTTP_ENDPOINT_URL=$(cat ${TESTDIR}/url_file) + fi +fi -# rgw/test_awssdkv4_sig.sh -pushd jcksum +if [ -z ${CEPH_ROOT} ] +then + echo "CEPH_ROOT is not defined" + exit +else + cd $CEPH_ROOT/qa/workunits/rgw/jcksum +fi ./mvnw clean package ./mvnw test -Dtest=PutObjects -- 2.39.5