From 54ef383698a9e8256a709f8e9dbeeb9dbdc28854 Mon Sep 17 00:00:00 2001 From: Matt Benjamin Date: Fri, 30 May 2025 17:56:10 -0400 Subject: [PATCH] rgw: framework shell of gosdk tests Contains two golang functions based on the checksum failure reproducer provided by Fred Heinecke. Signed-off-by: Matt Benjamin --- qa/suites/rgw/verify/0-install.yaml | 5 +- qa/suites/rgw/verify/tasks/cls.yaml | 1 + qa/workunits/rgw/gcksum/gcksum.go | 175 +++++++++++++++++++++++++ qa/workunits/rgw/gcksum/gcksum_test.go | 47 +++++++ qa/workunits/rgw/gcksum/go.mod | 27 ++++ qa/workunits/rgw/gcksum/go.sum | 36 +++++ qa/workunits/rgw/test_gosdk2.sh | 74 +++++++++++ 7 files changed, 363 insertions(+), 2 deletions(-) create mode 100644 qa/workunits/rgw/gcksum/gcksum.go create mode 100644 qa/workunits/rgw/gcksum/gcksum_test.go create mode 100644 qa/workunits/rgw/gcksum/go.mod create mode 100644 qa/workunits/rgw/gcksum/go.sum create mode 100755 qa/workunits/rgw/test_gosdk2.sh diff --git a/qa/suites/rgw/verify/0-install.yaml b/qa/suites/rgw/verify/0-install.yaml index 85b25e7ef8697..075a12fe576f6 100644 --- a/qa/suites/rgw/verify/0-install.yaml +++ b/qa/suites/rgw/verify/0-install.yaml @@ -2,9 +2,10 @@ tasks: - install: # extra packages added for the rgw-datacache task # java and maven needed for S3 trailer signature tests + # golang is needed by aws-sdk-go-v2 tests extra_system_packages: - deb: ['s3cmd', 'maven'] - rpm: ['s3cmd', 'maven'] + deb: ['s3cmd', 'maven', 'golang'] + rpm: ['s3cmd', 'maven', 'golang'] - ceph: - openssl_keys: - rgw: diff --git a/qa/suites/rgw/verify/tasks/cls.yaml b/qa/suites/rgw/verify/tasks/cls.yaml index 5800293484ef4..4d66790663c57 100644 --- a/qa/suites/rgw/verify/tasks/cls.yaml +++ b/qa/suites/rgw/verify/tasks/cls.yaml @@ -22,3 +22,4 @@ tasks: - rgw/test_rgw_datalog.sh - rgw/test_librgw_file.sh - rgw/test_awssdkv4_sig.sh + - rgw/test_gosdk2.sh diff --git a/qa/workunits/rgw/gcksum/gcksum.go b/qa/workunits/rgw/gcksum/gcksum.go new file mode 100644 index 0000000000000..2e9655616be53 --- /dev/null +++ b/qa/workunits/rgw/gcksum/gcksum.go @@ -0,0 +1,175 @@ +package main + +import ( + "bytes" + "context" + "strings" + "crypto/tls" + "fmt" + "io" + "net/http" + "os" + "sync" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" +) + +var region string = "us-east-1" +var bucketName string = "caijupepe" + +var ctx context.Context +var s3Client *s3.Client +var wg sync.WaitGroup + +func getenv(k string, defv string) string { + v := os.Getenv(k) + if (v == "") { + v = defv + } + return v +} + +func main() { + fmt.Println("gcksum main() (moar cowbell)") + + // Setup + ctx := context.Background() + s3Client := setupS3Client(ctx) + + wg.Add(1) + setupBucket(ctx, s3Client, bucketName) + wg.Wait() + + // Tests + putObject(ctx, s3Client, bucketName); + fmt.Println() + fmt.Println() + fmt.Println() + + demonstrateChunkedUpload(ctx, s3Client, bucketName) + fmt.Println() + fmt.Println() + fmt.Println() + demonstrateFixedLengthUpload(ctx, s3Client, bucketName) +} + +func setupS3Client(ctx context.Context) *s3.Client { + + access_key := getenv("AWS_ACCESS_KEY_ID", "0555b35654ad1656d804") + secret_key := getenv("AWS_SECRET_ACCESS_KEY", + "h7GhxuBLTrlhVUyxSPUKUV8r/2EI4ngqJxD7iBdBYLhwluN30JaT3Q==") + + endpoint_url := getenv("RGW_HTTP_ENDPOINT_URL", "https://localhost:8443") + session_token := "" + + staticProvider := credentials.NewStaticCredentialsProvider( + access_key, + secret_key, + session_token, + ) + + tr := &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + } + + client := &http.Client{Transport: tr} + + awsConfig, err := config.LoadDefaultConfig(ctx, + config.WithClientLogMode(aws.LogRequestWithBody|aws.LogResponseWithBody), + config.WithCredentialsProvider(staticProvider), + ) + + if err != nil { + fmt.Printf("failed to load AWS config: %v\n", err) + os.Exit(1) + } + + s3_client := s3.NewFromConfig( + awsConfig, func(o *s3.Options) { + o.HTTPClient = client + o.Region = region + o.BaseEndpoint = aws.String(endpoint_url) + o.UsePathStyle = true + o.RetryMaxAttempts = 1 + }) + + return s3_client +} + +func setupBucket(ctx context.Context, client *s3.Client, bucketName string) { + _, err := client.CreateBucket(ctx, &s3.CreateBucketInput{ + Bucket: aws.String(bucketName), + CreateBucketConfiguration: &types.CreateBucketConfiguration{ + }, + }) + err = err + defer wg.Done() +} + +func putObject(ctx context.Context, client *s3.Client, + bucketName string) error { + // simplest possible S3 PUT operation + obj_ix := 1 + key := fmt.Sprintf("object%d", obj_ix) + body := fmt.Sprintf("body for %s/%s", bucketName, key) + poinput := &s3.PutObjectInput{ + Body: strings.NewReader(body), + Bucket: aws.String(bucketName), + Key: aws.String(key), + } + _, err := client.PutObject(context.TODO(), poinput) + return err +} + +func demonstrateChunkedUpload(ctx context.Context, s3Client *s3.Client, + bucketName string) error { + // Create an IO pipe. The total amount of data read isn't known to the + // reader (S3 PutObject), so the PutObject call will use a chunked upload. + pipeReader, pipeWriter := io.Pipe() + + dataToUpload := []byte("This is some example chunked data to upload to S3.") + key := "chunked-upload-example" + + // Start a goroutine to write data to the pipe + go func() { + pipeWriter.Write(dataToUpload) + pipeWriter.Close() + }() + + // Upload the data from the pipe to S3 using a chunked upload + _, err := s3Client.PutObject(ctx, &s3.PutObjectInput{ + Bucket: &bucketName, + Key: &key, + Body: pipeReader, + }) + + fmt.Printf("Uploaded chunked data to S3 bucket %s with key %s\n", bucketName, key) + fmt.Printf("Error: %v\n", err) + return err +} + +func demonstrateFixedLengthUpload(ctx context.Context, s3Client *s3.Client, + bucketName string) error { + // Create a fixed-length byte slice to upload + dataToUpload := []byte("This is some example fixed-length data to upload to S3.") + key := "fixed-length-upload-example" + + // Using a reader-seeker ensures that the data will be uploaded as fixed length, with the + // content length set to the size of the byte slice. + var readerSeeker io.ReadSeeker = bytes.NewReader(dataToUpload) + + // Upload the data directly to S3 + _, err := s3Client.PutObject(ctx, &s3.PutObjectInput{ + Bucket: &bucketName, + Key: &key, + Body: readerSeeker, + }) + + fmt.Printf("Uploaded fixed-length data to S3 bucket %s with key %s\n", bucketName, key) + fmt.Printf("Error: %v\n", err) + return err +} diff --git a/qa/workunits/rgw/gcksum/gcksum_test.go b/qa/workunits/rgw/gcksum/gcksum_test.go new file mode 100644 index 0000000000000..7c80f98db4b29 --- /dev/null +++ b/qa/workunits/rgw/gcksum/gcksum_test.go @@ -0,0 +1,47 @@ + +package main + +import ( + "testing" + "fmt" + "context" +) + +/* tests */ + +func TestPut1(t *testing.T) { + err := putObject(ctx, s3Client, bucketName) + if (err != nil) { + t.Errorf("Failed: %v", err) + } +} + +func TestChunkedUpload(t *testing.T) { + err := demonstrateChunkedUpload(ctx, s3Client, bucketName) + if (err != nil) { + t.Errorf("Failed: %v", err) + } +} + +func TestFixedLengthUpload(t *testing.T) { + err := demonstrateFixedLengthUpload(ctx, s3Client, bucketName) + if (err != nil) { + t.Errorf("Failed: %v", err) + } +} + +func TestMain(m *testing.M) { + fmt.Println("gcksum golang testing") + fmt.Println() + fmt.Println() + + ctx = context.Background() + s3Client = setupS3Client(ctx) + + wg.Add(1) + setupBucket(ctx, s3Client, bucketName) + wg.Wait() + + // call flag.Parse() here if TestMain uses flags + m.Run() +} diff --git a/qa/workunits/rgw/gcksum/go.mod b/qa/workunits/rgw/gcksum/go.mod new file mode 100644 index 0000000000000..6e4bbcde50861 --- /dev/null +++ b/qa/workunits/rgw/gcksum/go.mod @@ -0,0 +1,27 @@ +module gcksum + +go 1.23.4 + +require ( + github.com/aws/aws-sdk-go-v2 v1.36.3 + github.com/aws/aws-sdk-go-v2/config v1.29.14 // Important: Set this to 1.28.11 or lower to avoid the issue + github.com/aws/aws-sdk-go-v2/service/s3 v1.79.3 +) + +require ( + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.10 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.17.67 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.30 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.34 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.34 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.3 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.7.1 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.15 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.15 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.25.3 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.1 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.33.19 // indirect + github.com/aws/smithy-go v1.22.2 // indirect +) diff --git a/qa/workunits/rgw/gcksum/go.sum b/qa/workunits/rgw/gcksum/go.sum new file mode 100644 index 0000000000000..9b9162a03a8e3 --- /dev/null +++ b/qa/workunits/rgw/gcksum/go.sum @@ -0,0 +1,36 @@ +github.com/aws/aws-sdk-go-v2 v1.36.3 h1:mJoei2CxPutQVxaATCzDUjcZEjVRdpsiiXi2o38yqWM= +github.com/aws/aws-sdk-go-v2 v1.36.3/go.mod h1:LLXuLpgzEbD766Z5ECcRmi8AzSwfZItDtmABVkRLGzg= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.10 h1:zAybnyUQXIZ5mok5Jqwlf58/TFE7uvd3IAsa1aF9cXs= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.10/go.mod h1:qqvMj6gHLR/EXWZw4ZbqlPbQUyenf4h82UQUlKc+l14= +github.com/aws/aws-sdk-go-v2/config v1.29.14 h1:f+eEi/2cKCg9pqKBoAIwRGzVb70MRKqWX4dg1BDcSJM= +github.com/aws/aws-sdk-go-v2/config v1.29.14/go.mod h1:wVPHWcIFv3WO89w0rE10gzf17ZYy+UVS1Geq8Iei34g= +github.com/aws/aws-sdk-go-v2/credentials v1.17.67 h1:9KxtdcIA/5xPNQyZRgUSpYOE6j9Bc4+D7nZua0KGYOM= +github.com/aws/aws-sdk-go-v2/credentials v1.17.67/go.mod h1:p3C44m+cfnbv763s52gCqrjaqyPikj9Sg47kUVaNZQQ= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.30 h1:x793wxmUWVDhshP8WW2mlnXuFrO4cOd3HLBroh1paFw= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.30/go.mod h1:Jpne2tDnYiFascUEs2AWHJL9Yp7A5ZVy3TNyxaAjD6M= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34 h1:ZK5jHhnrioRkUNOc+hOgQKlUL5JeC3S6JgLxtQ+Rm0Q= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34/go.mod h1:p4VfIceZokChbA9FzMbRGz5OV+lekcVtHlPKEO0gSZY= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.34 h1:SZwFm17ZUNNg5Np0ioo/gq8Mn6u9w19Mri8DnJ15Jf0= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.34/go.mod h1:dFZsC0BLo346mvKQLWmoJxT+Sjp+qcVR1tRVHQGOH9Q= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 h1:bIqFDwgGXXN1Kpp99pDOdKMTTb5d2KyU5X/BZxjOkRo= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3/go.mod h1:H5O/EsxDWyU+LP/V8i5sm8cxoZgc2fdNR9bxlOFrQTo= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.34 h1:ZNTqv4nIdE/DiBfUUfXcLZ/Spcuz+RjeziUtNJackkM= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.34/go.mod h1:zf7Vcd1ViW7cPqYWEHLHJkS50X0JS2IKz9Cgaj6ugrs= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.3 h1:eAh2A4b5IzM/lum78bZ590jy36+d/aFLgKF/4Vd1xPE= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.3/go.mod h1:0yKJC/kb8sAnmlYa6Zs3QVYqaC8ug2AbnNChv5Ox3uA= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.7.1 h1:4nm2G6A4pV9rdlWzGMPv4BNtQp22v1hg3yrtkYpeLl8= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.7.1/go.mod h1:iu6FSzgt+M2/x3Dk8zhycdIcHjEFb36IS8HVUVFoMg0= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.15 h1:dM9/92u2F1JbDaGooxTq18wmmFzbJRfXfVfy96/1CXM= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.15/go.mod h1:SwFBy2vjtA0vZbjjaFtfN045boopadnoVPhu4Fv66vY= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.15 h1:moLQUoVq91LiqT1nbvzDukyqAlCv89ZmwaHw/ZFlFZg= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.15/go.mod h1:ZH34PJUc8ApjBIfgQCFvkWcUDBtl/WTD+uiYHjd8igA= +github.com/aws/aws-sdk-go-v2/service/s3 v1.79.3 h1:BRXS0U76Z8wfF+bnkilA2QwpIch6URlm++yPUt9QPmQ= +github.com/aws/aws-sdk-go-v2/service/s3 v1.79.3/go.mod h1:bNXKFFyaiVvWuR6O16h/I1724+aXe/tAkA9/QS01t5k= +github.com/aws/aws-sdk-go-v2/service/sso v1.25.3 h1:1Gw+9ajCV1jogloEv1RRnvfRFia2cL6c9cuKV2Ps+G8= +github.com/aws/aws-sdk-go-v2/service/sso v1.25.3/go.mod h1:qs4a9T5EMLl/Cajiw2TcbNt2UNo/Hqlyp+GiuG4CFDI= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.1 h1:hXmVKytPfTy5axZ+fYbR5d0cFmC3JvwLm5kM83luako= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.1/go.mod h1:MlYRNmYu/fGPoxBQVvBYr9nyr948aY/WLUvwBMBJubs= +github.com/aws/aws-sdk-go-v2/service/sts v1.33.19 h1:1XuUZ8mYJw9B6lzAkXhqHlJd/XvaX32evhproijJEZY= +github.com/aws/aws-sdk-go-v2/service/sts v1.33.19/go.mod h1:cQnB8CUnxbMU82JvlqjKR2HBOm3fe9pWorWBza6MBJ4= +github.com/aws/smithy-go v1.22.2 h1:6D9hW43xKFrRx/tXXfAlIZc4JI+yQe6snnWcQyxSyLQ= +github.com/aws/smithy-go v1.22.2/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg= diff --git a/qa/workunits/rgw/test_gosdk2.sh b/qa/workunits/rgw/test_gosdk2.sh new file mode 100755 index 0000000000000..3514df91b264e --- /dev/null +++ b/qa/workunits/rgw/test_gosdk2.sh @@ -0,0 +1,74 @@ +#!/bin/sh +# +# To run this test script with a cluster created via vstart.sh: +# $PATH needs to be set for radosgw-admin executables. +# $CEPH_ROOT needs to be set to the path of the Ceph source code +# $RGW_HTTP_ENDPOINT_URL needs to be set to the endpoint of the RGW +# +# Example when ceph source is cloned into $HOME and a vstart cluster is already running with a radosgw: +# $ PATH=~/ceph/build/bin/:$PATH CEPH_ROOT=~/ceph RGW_HTTP_ENDPOINT=http://localhost:8000 ~/ceph/qa/workunits/rgw/test_gosdk2.sh +# + +set -x + +if [ -z ${AWS_ACCESS_KEY_ID} ] +then + export AWS_ACCESS_KEY_ID="lNCnR47C2g+ZidCWBAUuwfSAA7Q=" + export AWS_SECRET_ACCESS_KEY="tYuA2Y+Uu1ow2l9Xe59tWKVml3gMuVfyhUjjJwfwEI0vFFONIcqf4g==" + + radosgw-admin user create --uid ceph-test-gosdk2 \ + --access-key $AWS_ACCESS_KEY_ID \ + --secret $AWS_SECRET_ACCESS_KEY \ + --display-name "gosdk2 test user" \ + --email gosdk2@example.com || echo "gosdk2 user exists" +fi + +if [ -z ${RGW_HTTP_ENDPOINT_URL} ] +then + # TESTDIR and this block are meant for when this script is run in a teuthology environment + if [ -z ${TESTDIR} ] + then + echo "TESTDIR is not defined, cannot set RGW_HTTP_ENDPOINT_URL in teuthology" + exit + else + export RGW_HTTP_ENDPOINT_URL=$(cat ${TESTDIR}/url_file) + fi +fi + +if [ -z ${CEPH_ROOT} ] +then + echo "CEPH_ROOT is not defined" + exit +fi + +# https://stackoverflow.com/questions/7978517/how-do-i-get-cmake-to-work-with-the-go-programming-language + +# operations using addiitonal checksums +cd $CEPH_ROOT/qa/workunits/rgw/gcksum + +go mod download github.com/aws/aws-sdk-go-v2 +go mod download github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream +go mod download github.com/aws/aws-sdk-go-v2/config +go mod download github.com/aws/aws-sdk-go-v2/credentials +go mod download github.com/aws/aws-sdk-go-v2/feature/ec2/imds +go mod download github.com/aws/aws-sdk-go-v2/internal/configsources +go mod download github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 +go mod download github.com/aws/aws-sdk-go-v2/internal/ini +go mod download github.com/aws/aws-sdk-go-v2/internal/v4a +go mod download github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding +go mod download github.com/aws/aws-sdk-go-v2/service/internal/checksum +go mod download github.com/aws/aws-sdk-go-v2/service/internal/presigned-url +go mod download github.com/aws/aws-sdk-go-v2/service/internal/s3shared +go mod download github.com/aws/aws-sdk-go-v2/service/s3 +go mod download github.com/aws/aws-sdk-go-v2/service/sso +go mod download github.com/aws/aws-sdk-go-v2/service/ssooidc +go mod download github.com/aws/aws-sdk-go-v2/service/sts +go mod download github.com/aws/smithy-go + +# depending on the selection of RGW_HTTP_ENDPOINT_URL in the workunit, +# this will test an HTTP or HTTP/s, which is relevant to cover all +# checksum strategies (e.g., header vs trailer, aws-chunked) + +go test + +exit 0 -- 2.39.5