test/rgw/logging: add bucket logging test suite to teuthology

this is depended with this s3test PR:
https://github.com/ceph/s3-tests/pull/576

Signed-off-by: Yuval Lifshitz <ylifshit@ibm.com>

diff --git a/qa/rgw/s3tests-branch.yaml b/qa/rgw/s3tests-branch.yaml
index ef6819c87e06a4c1df50c8021ab245d27ca25225..8710ce35893b1e378e1d48ed146891b91332f49c 100644 (file)
--- a/qa/rgw/s3tests-branch.yaml
+++ b/qa/rgw/s3tests-branch.yaml
@@ -1,4 +1,4 @@
 overrides:
   s3tests:
-    force-branch: ceph-master
-    # git_remote: https://github.com/ceph/
+     force-branch: ceph-master
+     # git_remote: https://github.com/ceph/

diff --git a/qa/suites/rgw/bucket-logging/% b/qa/suites/rgw/bucket-logging/%
new file mode 100644 (file)
index 0000000..e69de29

diff --git a/qa/suites/rgw/bucket-logging/.qa b/qa/suites/rgw/bucket-logging/.qa
new file mode 120000 (symlink)
index 0000000..a602a03
--- /dev/null
+++ b/qa/suites/rgw/bucket-logging/.qa
@@ -0,0 +1 @@
+../.qa/
\ No newline at end of file

diff --git a/qa/suites/rgw/bucket-logging/0-install.yaml b/qa/suites/rgw/bucket-logging/0-install.yaml
new file mode 100644 (file)
index 0000000..6cf82f5
--- /dev/null
+++ b/qa/suites/rgw/bucket-logging/0-install.yaml
@@ -0,0 +1,13 @@
+tasks:
+- install:
+- ceph:
+- openssl_keys:
+- rgw: [client.0]
+- tox: [client.0]
+
+overrides:
+  ceph:
+    conf:
+      global:
+        osd_min_pg_log_entries: 10
+        osd_max_pg_log_entries: 10

diff --git a/qa/suites/rgw/bucket-logging/beast.yaml b/qa/suites/rgw/bucket-logging/beast.yaml
new file mode 120000 (symlink)
index 0000000..09ced62
--- /dev/null
+++ b/qa/suites/rgw/bucket-logging/beast.yaml
@@ -0,0 +1 @@
+.qa/rgw_frontend/beast.yaml
\ No newline at end of file

diff --git a/qa/suites/rgw/bucket-logging/fixed-1.yaml b/qa/suites/rgw/bucket-logging/fixed-1.yaml
new file mode 120000 (symlink)
index 0000000..02df5dd
--- /dev/null
+++ b/qa/suites/rgw/bucket-logging/fixed-1.yaml
@@ -0,0 +1 @@
+.qa/clusters/fixed-1.yaml
\ No newline at end of file

diff --git a/qa/suites/rgw/bucket-logging/ignore-pg-availability.yaml b/qa/suites/rgw/bucket-logging/ignore-pg-availability.yaml
new file mode 120000 (symlink)
index 0000000..32340b1
--- /dev/null
+++ b/qa/suites/rgw/bucket-logging/ignore-pg-availability.yaml
@@ -0,0 +1 @@
+.qa/rgw/ignore-pg-availability.yaml
\ No newline at end of file

diff --git a/qa/suites/rgw/bucket-logging/overrides.yaml b/qa/suites/rgw/bucket-logging/overrides.yaml
new file mode 100644 (file)
index 0000000..a448a32
--- /dev/null
+++ b/qa/suites/rgw/bucket-logging/overrides.yaml
@@ -0,0 +1,10 @@
+overrides:
+  ceph:
+    conf:
+      client:
+        setuser: ceph
+        setgroup: ceph
+        debug rgw: 20
+        rgw bucket logging obj roll time: 5
+  rgw:
+    storage classes: LUKEWARM, FROZEN

diff --git a/qa/suites/rgw/bucket-logging/s3tests-branch.yaml b/qa/suites/rgw/bucket-logging/s3tests-branch.yaml
new file mode 120000 (symlink)
index 0000000..bdcaca4
--- /dev/null
+++ b/qa/suites/rgw/bucket-logging/s3tests-branch.yaml
@@ -0,0 +1 @@
+.qa/rgw/s3tests-branch.yaml
\ No newline at end of file

diff --git a/qa/suites/rgw/bucket-logging/supported-distros b/qa/suites/rgw/bucket-logging/supported-distros
new file mode 120000 (symlink)
index 0000000..78f2991
--- /dev/null
+++ b/qa/suites/rgw/bucket-logging/supported-distros
@@ -0,0 +1 @@
+.qa/distros/supported-random-distro$/
\ No newline at end of file

diff --git a/qa/suites/rgw/bucket-logging/tasks/+ b/qa/suites/rgw/bucket-logging/tasks/+
new file mode 100644 (file)
index 0000000..e69de29

diff --git a/qa/suites/rgw/bucket-logging/tasks/s3tests.yaml b/qa/suites/rgw/bucket-logging/tasks/s3tests.yaml
new file mode 100644 (file)
index 0000000..c1d3b71
--- /dev/null
+++ b/qa/suites/rgw/bucket-logging/tasks/s3tests.yaml
@@ -0,0 +1,6 @@
+tasks:
+- s3tests:
+    client.0:
+      boto3_extensions: True
+      rgw_server: client.0
+      extra_attrs: ["bucket_logging"]

diff --git a/qa/tasks/s3tests.py b/qa/tasks/s3tests.py
index 6d7b39d58921034206e53f6a1381208061d43ab4..85ab97d23cd26c31e02a6204d82954d872042c0d 100644 (file)
--- a/qa/tasks/s3tests.py
+++ b/qa/tasks/s3tests.py
@@ -57,6 +57,17 @@ def download(ctx, config):
                     'git', 'reset', '--hard', sha1,
                     ],
                 )
+        if client_config.get('boto3_extensions'):
+            ctx.cluster.only(client).run(
+                    args=['mkdir',
+                          '-p',
+                          '/home/ubuntu/.aws/models/s3/2006-03-01/']
+                    )
+            (remote,) = ctx.cluster.only(client).remotes.keys()
+            remote_file = '/home/ubuntu/.aws/models/s3/2006-03-01/service-2.sdk-extras.json'
+            local_file = '{qadir}/../examples/rgw/boto3/service-2.sdk-extras.json'.format(qadir=ctx.config.get('suite_path'))
+            remote.put_file(local_file, remote_file)
+
     try:
         yield
     finally:
@@ -70,6 +81,17 @@ def download(ctx, config):
                     '{tdir}/s3-tests-{client}'.format(tdir=testdir, client=client),
                     ],
                 )
+            if client_config.get('boto3_extensions'):
+                ctx.cluster.only(client).run(
+                        args=[
+                            'rm', '-rf', '/home/ubuntu/.aws/models/s3/2006-03-01/service-2.sdk-extras.json',
+                            ],
+                        )
+                ctx.cluster.only(client).run(
+                        args=[
+                            'cd', '/home/ubuntu/', run.Raw('&&'), 'rmdir', '-p', '.aws/models/s3/2006-03-01/',
+                            ],
+                        )
 
 
 def _config_user(s3tests_conf, section, user, email):
@@ -444,8 +466,10 @@ def run_tests(ctx, config):
             attrs += ['not fails_with_subdomain']
         if not client_config.get('with-sse-s3'):
             attrs += ['not sse_s3']
-       
+
         attrs += client_config.get('extra_attrs', [])
+        if 'bucket_logging' not in attrs:
+            attrs += ['not bucket_logging']
         if 'unit_test_scan' in client_config and client_config['unit_test_scan']:
             xmlfile_id = datetime.datetime.now().strftime("%Y-%m-%d-%H:%M:%S--") + str(uuid.uuid4())
             xmlpath= f'{testdir}/archive/s3test-{xmlfile_id}.xml'