- install:
# extra packages added for the rgw-datacache task
# java and maven needed for S3 trailer signature tests
+ # golang is needed by aws-sdk-go-v2 tests
extra_system_packages:
- deb: ['s3cmd', 'maven']
- rpm: ['s3cmd', 'maven']
+ deb: ['s3cmd', 'maven', 'golang']
+ rpm: ['s3cmd', 'maven', 'golang']
- ceph:
- openssl_keys:
- rgw:
- rgw/test_rgw_datalog.sh
- rgw/test_librgw_file.sh
- rgw/test_awssdkv4_sig.sh
+ - rgw/test_gosdk2.sh
--- /dev/null
+package main
+
+import (
+ "bytes"
+ "context"
+ "strings"
+ "crypto/tls"
+ "fmt"
+ "io"
+ "net/http"
+ "os"
+ "sync"
+
+ "github.com/aws/aws-sdk-go-v2/aws"
+ "github.com/aws/aws-sdk-go-v2/config"
+ "github.com/aws/aws-sdk-go-v2/credentials"
+ "github.com/aws/aws-sdk-go-v2/service/s3"
+ "github.com/aws/aws-sdk-go-v2/service/s3/types"
+)
+
+var region string = "us-east-1"
+var bucketName string = "caijupepe"
+
+var ctx context.Context
+var s3Client *s3.Client
+var wg sync.WaitGroup
+
+func getenv(k string, defv string) string {
+ v := os.Getenv(k)
+ if (v == "") {
+ v = defv
+ }
+ return v
+}
+
+func main() {
+ fmt.Println("gcksum main() (moar cowbell)")
+
+ // Setup
+ ctx := context.Background()
+ s3Client := setupS3Client(ctx)
+
+ wg.Add(1)
+ setupBucket(ctx, s3Client, bucketName)
+ wg.Wait()
+
+ // Tests
+ putObject(ctx, s3Client, bucketName);
+ fmt.Println()
+ fmt.Println()
+ fmt.Println()
+
+ demonstrateChunkedUpload(ctx, s3Client, bucketName)
+ fmt.Println()
+ fmt.Println()
+ fmt.Println()
+ demonstrateFixedLengthUpload(ctx, s3Client, bucketName)
+}
+
+func setupS3Client(ctx context.Context) *s3.Client {
+
+ access_key := getenv("AWS_ACCESS_KEY_ID", "0555b35654ad1656d804")
+ secret_key := getenv("AWS_SECRET_ACCESS_KEY",
+ "h7GhxuBLTrlhVUyxSPUKUV8r/2EI4ngqJxD7iBdBYLhwluN30JaT3Q==")
+
+ endpoint_url := getenv("RGW_HTTP_ENDPOINT_URL", "https://localhost:8443")
+ session_token := ""
+
+ staticProvider := credentials.NewStaticCredentialsProvider(
+ access_key,
+ secret_key,
+ session_token,
+ )
+
+ tr := &http.Transport{
+ TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
+ }
+
+ client := &http.Client{Transport: tr}
+
+ awsConfig, err := config.LoadDefaultConfig(ctx,
+ config.WithClientLogMode(aws.LogRequestWithBody|aws.LogResponseWithBody),
+ config.WithCredentialsProvider(staticProvider),
+ )
+
+ if err != nil {
+ fmt.Printf("failed to load AWS config: %v\n", err)
+ os.Exit(1)
+ }
+
+ s3_client := s3.NewFromConfig(
+ awsConfig, func(o *s3.Options) {
+ o.HTTPClient = client
+ o.Region = region
+ o.BaseEndpoint = aws.String(endpoint_url)
+ o.UsePathStyle = true
+ o.RetryMaxAttempts = 1
+ })
+
+ return s3_client
+}
+
+func setupBucket(ctx context.Context, client *s3.Client, bucketName string) {
+ _, err := client.CreateBucket(ctx, &s3.CreateBucketInput{
+ Bucket: aws.String(bucketName),
+ CreateBucketConfiguration: &types.CreateBucketConfiguration{
+ },
+ })
+ err = err
+ defer wg.Done()
+}
+
+func putObject(ctx context.Context, client *s3.Client,
+ bucketName string) error {
+ // simplest possible S3 PUT operation
+ obj_ix := 1
+ key := fmt.Sprintf("object%d", obj_ix)
+ body := fmt.Sprintf("body for %s/%s", bucketName, key)
+ poinput := &s3.PutObjectInput{
+ Body: strings.NewReader(body),
+ Bucket: aws.String(bucketName),
+ Key: aws.String(key),
+ }
+ _, err := client.PutObject(context.TODO(), poinput)
+ return err
+}
+
+func demonstrateChunkedUpload(ctx context.Context, s3Client *s3.Client,
+ bucketName string) error {
+ // Create an IO pipe. The total amount of data read isn't known to the
+ // reader (S3 PutObject), so the PutObject call will use a chunked upload.
+ pipeReader, pipeWriter := io.Pipe()
+
+ dataToUpload := []byte("This is some example chunked data to upload to S3.")
+ key := "chunked-upload-example"
+
+ // Start a goroutine to write data to the pipe
+ go func() {
+ pipeWriter.Write(dataToUpload)
+ pipeWriter.Close()
+ }()
+
+ // Upload the data from the pipe to S3 using a chunked upload
+ _, err := s3Client.PutObject(ctx, &s3.PutObjectInput{
+ Bucket: &bucketName,
+ Key: &key,
+ Body: pipeReader,
+ })
+
+ fmt.Printf("Uploaded chunked data to S3 bucket %s with key %s\n", bucketName, key)
+ fmt.Printf("Error: %v\n", err)
+ return err
+}
+
+func demonstrateFixedLengthUpload(ctx context.Context, s3Client *s3.Client,
+ bucketName string) error {
+ // Create a fixed-length byte slice to upload
+ dataToUpload := []byte("This is some example fixed-length data to upload to S3.")
+ key := "fixed-length-upload-example"
+
+ // Using a reader-seeker ensures that the data will be uploaded as fixed length, with the
+ // content length set to the size of the byte slice.
+ var readerSeeker io.ReadSeeker = bytes.NewReader(dataToUpload)
+
+ // Upload the data directly to S3
+ _, err := s3Client.PutObject(ctx, &s3.PutObjectInput{
+ Bucket: &bucketName,
+ Key: &key,
+ Body: readerSeeker,
+ })
+
+ fmt.Printf("Uploaded fixed-length data to S3 bucket %s with key %s\n", bucketName, key)
+ fmt.Printf("Error: %v\n", err)
+ return err
+}
--- /dev/null
+
+package main
+
+import (
+ "testing"
+ "fmt"
+ "context"
+)
+
+/* tests */
+
+func TestPut1(t *testing.T) {
+ err := putObject(ctx, s3Client, bucketName)
+ if (err != nil) {
+ t.Errorf("Failed: %v", err)
+ }
+}
+
+func TestChunkedUpload(t *testing.T) {
+ err := demonstrateChunkedUpload(ctx, s3Client, bucketName)
+ if (err != nil) {
+ t.Errorf("Failed: %v", err)
+ }
+}
+
+func TestFixedLengthUpload(t *testing.T) {
+ err := demonstrateFixedLengthUpload(ctx, s3Client, bucketName)
+ if (err != nil) {
+ t.Errorf("Failed: %v", err)
+ }
+}
+
+func TestMain(m *testing.M) {
+ fmt.Println("gcksum golang testing")
+ fmt.Println()
+ fmt.Println()
+
+ ctx = context.Background()
+ s3Client = setupS3Client(ctx)
+
+ wg.Add(1)
+ setupBucket(ctx, s3Client, bucketName)
+ wg.Wait()
+
+ // call flag.Parse() here if TestMain uses flags
+ m.Run()
+}
--- /dev/null
+module gcksum
+
+go 1.23.4
+
+require (
+ github.com/aws/aws-sdk-go-v2 v1.36.3
+ github.com/aws/aws-sdk-go-v2/config v1.29.14 // Important: Set this to 1.28.11 or lower to avoid the issue
+ github.com/aws/aws-sdk-go-v2/service/s3 v1.79.3
+)
+
+require (
+ github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.10 // indirect
+ github.com/aws/aws-sdk-go-v2/credentials v1.17.67 // indirect
+ github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.30 // indirect
+ github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34 // indirect
+ github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.34 // indirect
+ github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 // indirect
+ github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.34 // indirect
+ github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.3 // indirect
+ github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.7.1 // indirect
+ github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.15 // indirect
+ github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.15 // indirect
+ github.com/aws/aws-sdk-go-v2/service/sso v1.25.3 // indirect
+ github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.1 // indirect
+ github.com/aws/aws-sdk-go-v2/service/sts v1.33.19 // indirect
+ github.com/aws/smithy-go v1.22.2 // indirect
+)
--- /dev/null
+github.com/aws/aws-sdk-go-v2 v1.36.3 h1:mJoei2CxPutQVxaATCzDUjcZEjVRdpsiiXi2o38yqWM=
+github.com/aws/aws-sdk-go-v2 v1.36.3/go.mod h1:LLXuLpgzEbD766Z5ECcRmi8AzSwfZItDtmABVkRLGzg=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.10 h1:zAybnyUQXIZ5mok5Jqwlf58/TFE7uvd3IAsa1aF9cXs=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.10/go.mod h1:qqvMj6gHLR/EXWZw4ZbqlPbQUyenf4h82UQUlKc+l14=
+github.com/aws/aws-sdk-go-v2/config v1.29.14 h1:f+eEi/2cKCg9pqKBoAIwRGzVb70MRKqWX4dg1BDcSJM=
+github.com/aws/aws-sdk-go-v2/config v1.29.14/go.mod h1:wVPHWcIFv3WO89w0rE10gzf17ZYy+UVS1Geq8Iei34g=
+github.com/aws/aws-sdk-go-v2/credentials v1.17.67 h1:9KxtdcIA/5xPNQyZRgUSpYOE6j9Bc4+D7nZua0KGYOM=
+github.com/aws/aws-sdk-go-v2/credentials v1.17.67/go.mod h1:p3C44m+cfnbv763s52gCqrjaqyPikj9Sg47kUVaNZQQ=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.30 h1:x793wxmUWVDhshP8WW2mlnXuFrO4cOd3HLBroh1paFw=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.30/go.mod h1:Jpne2tDnYiFascUEs2AWHJL9Yp7A5ZVy3TNyxaAjD6M=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34 h1:ZK5jHhnrioRkUNOc+hOgQKlUL5JeC3S6JgLxtQ+Rm0Q=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34/go.mod h1:p4VfIceZokChbA9FzMbRGz5OV+lekcVtHlPKEO0gSZY=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.34 h1:SZwFm17ZUNNg5Np0ioo/gq8Mn6u9w19Mri8DnJ15Jf0=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.34/go.mod h1:dFZsC0BLo346mvKQLWmoJxT+Sjp+qcVR1tRVHQGOH9Q=
+github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 h1:bIqFDwgGXXN1Kpp99pDOdKMTTb5d2KyU5X/BZxjOkRo=
+github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3/go.mod h1:H5O/EsxDWyU+LP/V8i5sm8cxoZgc2fdNR9bxlOFrQTo=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.34 h1:ZNTqv4nIdE/DiBfUUfXcLZ/Spcuz+RjeziUtNJackkM=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.34/go.mod h1:zf7Vcd1ViW7cPqYWEHLHJkS50X0JS2IKz9Cgaj6ugrs=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.3 h1:eAh2A4b5IzM/lum78bZ590jy36+d/aFLgKF/4Vd1xPE=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.3/go.mod h1:0yKJC/kb8sAnmlYa6Zs3QVYqaC8ug2AbnNChv5Ox3uA=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.7.1 h1:4nm2G6A4pV9rdlWzGMPv4BNtQp22v1hg3yrtkYpeLl8=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.7.1/go.mod h1:iu6FSzgt+M2/x3Dk8zhycdIcHjEFb36IS8HVUVFoMg0=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.15 h1:dM9/92u2F1JbDaGooxTq18wmmFzbJRfXfVfy96/1CXM=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.15/go.mod h1:SwFBy2vjtA0vZbjjaFtfN045boopadnoVPhu4Fv66vY=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.15 h1:moLQUoVq91LiqT1nbvzDukyqAlCv89ZmwaHw/ZFlFZg=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.15/go.mod h1:ZH34PJUc8ApjBIfgQCFvkWcUDBtl/WTD+uiYHjd8igA=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.79.3 h1:BRXS0U76Z8wfF+bnkilA2QwpIch6URlm++yPUt9QPmQ=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.79.3/go.mod h1:bNXKFFyaiVvWuR6O16h/I1724+aXe/tAkA9/QS01t5k=
+github.com/aws/aws-sdk-go-v2/service/sso v1.25.3 h1:1Gw+9ajCV1jogloEv1RRnvfRFia2cL6c9cuKV2Ps+G8=
+github.com/aws/aws-sdk-go-v2/service/sso v1.25.3/go.mod h1:qs4a9T5EMLl/Cajiw2TcbNt2UNo/Hqlyp+GiuG4CFDI=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.1 h1:hXmVKytPfTy5axZ+fYbR5d0cFmC3JvwLm5kM83luako=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.1/go.mod h1:MlYRNmYu/fGPoxBQVvBYr9nyr948aY/WLUvwBMBJubs=
+github.com/aws/aws-sdk-go-v2/service/sts v1.33.19 h1:1XuUZ8mYJw9B6lzAkXhqHlJd/XvaX32evhproijJEZY=
+github.com/aws/aws-sdk-go-v2/service/sts v1.33.19/go.mod h1:cQnB8CUnxbMU82JvlqjKR2HBOm3fe9pWorWBza6MBJ4=
+github.com/aws/smithy-go v1.22.2 h1:6D9hW43xKFrRx/tXXfAlIZc4JI+yQe6snnWcQyxSyLQ=
+github.com/aws/smithy-go v1.22.2/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg=
--- /dev/null
+#!/bin/sh
+#
+# To run this test script with a cluster created via vstart.sh:
+# $PATH needs to be set for radosgw-admin executables.
+# $CEPH_ROOT needs to be set to the path of the Ceph source code
+# $RGW_HTTP_ENDPOINT_URL needs to be set to the endpoint of the RGW
+#
+# Example when ceph source is cloned into $HOME and a vstart cluster is already running with a radosgw:
+# $ PATH=~/ceph/build/bin/:$PATH CEPH_ROOT=~/ceph RGW_HTTP_ENDPOINT=http://localhost:8000 ~/ceph/qa/workunits/rgw/test_gosdk2.sh
+#
+
+set -x
+
+if [ -z ${AWS_ACCESS_KEY_ID} ]
+then
+ export AWS_ACCESS_KEY_ID="lNCnR47C2g+ZidCWBAUuwfSAA7Q="
+ export AWS_SECRET_ACCESS_KEY="tYuA2Y+Uu1ow2l9Xe59tWKVml3gMuVfyhUjjJwfwEI0vFFONIcqf4g=="
+
+ radosgw-admin user create --uid ceph-test-gosdk2 \
+ --access-key $AWS_ACCESS_KEY_ID \
+ --secret $AWS_SECRET_ACCESS_KEY \
+ --display-name "gosdk2 test user" \
+ --email gosdk2@example.com || echo "gosdk2 user exists"
+fi
+
+if [ -z ${RGW_HTTP_ENDPOINT_URL} ]
+then
+ # TESTDIR and this block are meant for when this script is run in a teuthology environment
+ if [ -z ${TESTDIR} ]
+ then
+ echo "TESTDIR is not defined, cannot set RGW_HTTP_ENDPOINT_URL in teuthology"
+ exit
+ else
+ export RGW_HTTP_ENDPOINT_URL=$(cat ${TESTDIR}/url_file)
+ fi
+fi
+
+if [ -z ${CEPH_ROOT} ]
+then
+ echo "CEPH_ROOT is not defined"
+ exit
+fi
+
+# https://stackoverflow.com/questions/7978517/how-do-i-get-cmake-to-work-with-the-go-programming-language
+
+# operations using addiitonal checksums
+cd $CEPH_ROOT/qa/workunits/rgw/gcksum
+
+go mod download github.com/aws/aws-sdk-go-v2
+go mod download github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream
+go mod download github.com/aws/aws-sdk-go-v2/config
+go mod download github.com/aws/aws-sdk-go-v2/credentials
+go mod download github.com/aws/aws-sdk-go-v2/feature/ec2/imds
+go mod download github.com/aws/aws-sdk-go-v2/internal/configsources
+go mod download github.com/aws/aws-sdk-go-v2/internal/endpoints/v2
+go mod download github.com/aws/aws-sdk-go-v2/internal/ini
+go mod download github.com/aws/aws-sdk-go-v2/internal/v4a
+go mod download github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding
+go mod download github.com/aws/aws-sdk-go-v2/service/internal/checksum
+go mod download github.com/aws/aws-sdk-go-v2/service/internal/presigned-url
+go mod download github.com/aws/aws-sdk-go-v2/service/internal/s3shared
+go mod download github.com/aws/aws-sdk-go-v2/service/s3
+go mod download github.com/aws/aws-sdk-go-v2/service/sso
+go mod download github.com/aws/aws-sdk-go-v2/service/ssooidc
+go mod download github.com/aws/aws-sdk-go-v2/service/sts
+go mod download github.com/aws/smithy-go
+
+# depending on the selection of RGW_HTTP_ENDPOINT_URL in the workunit,
+# this will test an HTTP or HTTP/s, which is relevant to cover all
+# checksum strategies (e.g., header vs trailer, aws-chunked)
+
+go test
+
+exit 0
projects / ceph.git / commitdiff