mimic: update s3-test download code for s3-test tasks

Fixes: https://tracker.ceph.com/issues/43077
- Ensure the download code for all tasks running
s3-tests is consistent.
- Simplify download code to only use the config
variable 'force-branch' for the branch being
cloned.
- Make ceph-mimic the force-branch for all
suites using s3-tests.
- Add force-branch to suites running s3readwrite
& s3roundtrip tasks

Signed-off-by: Ali Maredia <amaredia@redhat.com>

diff --git a/qa/suites/rados/basic/tasks/rgw_snaps.yaml b/qa/suites/rados/basic/tasks/rgw_snaps.yaml
index 2a9746391e9e2eeab50076a622d1f30fa1777df6..3a75bfc12dfe5003150c79896a40b6091a2951f2 100644 (file)
--- a/qa/suites/rados/basic/tasks/rgw_snaps.yaml
+++ b/qa/suites/rados/basic/tasks/rgw_snaps.yaml
@@ -28,6 +28,7 @@ tasks:
     - default.rgw.log
 - s3readwrite:
     client.0:
+      force-branch: ceph-mimic
       rgw_server: client.0
       readwrite:
         bucket: rwtest

diff --git a/qa/suites/rgw/multifs/tasks/rgw_readwrite.yaml b/qa/suites/rgw/multifs/tasks/rgw_readwrite.yaml
index c7efaa1c757916993c8967ed4569426923a39d87..cc5fa8046fae394fa36f5e2cea6c19b9272d8215 100644 (file)
--- a/qa/suites/rgw/multifs/tasks/rgw_readwrite.yaml
+++ b/qa/suites/rgw/multifs/tasks/rgw_readwrite.yaml
@@ -4,6 +4,7 @@ tasks:
 - rgw: [client.0]
 - s3readwrite:
     client.0:
+      force-branch: ceph-mimic
       rgw_server: client.0
       readwrite:
         bucket: rwtest

diff --git a/qa/suites/rgw/multifs/tasks/rgw_roundtrip.yaml b/qa/suites/rgw/multifs/tasks/rgw_roundtrip.yaml
index 47b3c1894a22337e75532190e9b4031022fd780a..8139d950e8ecc03340d58c498b546ed1228ba1fd 100644 (file)
--- a/qa/suites/rgw/multifs/tasks/rgw_roundtrip.yaml
+++ b/qa/suites/rgw/multifs/tasks/rgw_roundtrip.yaml
@@ -4,6 +4,7 @@ tasks:
 - rgw: [client.0]
 - s3roundtrip:
     client.0:
+      force-branch: ceph-mimic
       rgw_server: client.0
       roundtrip:
         bucket: rttest

diff --git a/qa/suites/rgw/thrash/workload/rgw_readwrite.yaml b/qa/suites/rgw/thrash/workload/rgw_readwrite.yaml
index e4e68286848ed3822d8b3bf0fc7cbdeb9630bdb0..0f586c506eef877b3579b7b7269fb9d65332e487 100644 (file)
--- a/qa/suites/rgw/thrash/workload/rgw_readwrite.yaml
+++ b/qa/suites/rgw/thrash/workload/rgw_readwrite.yaml
@@ -1,6 +1,7 @@
 tasks:
 - s3readwrite:
     client.0:
+      force-branch: ceph-mimic
       rgw_server: client.0
       readwrite:
         bucket: rwtest

diff --git a/qa/suites/rgw/thrash/workload/rgw_roundtrip.yaml b/qa/suites/rgw/thrash/workload/rgw_roundtrip.yaml
index 621683aeb1b852de218cbdbfd8e1b22c395ae530..9d2f1f59cae3210fc16540fcc5a5257b7aad2ab1 100644 (file)
--- a/qa/suites/rgw/thrash/workload/rgw_roundtrip.yaml
+++ b/qa/suites/rgw/thrash/workload/rgw_roundtrip.yaml
@@ -1,6 +1,7 @@
 tasks:
 - s3roundtrip:
     client.0:
+      force-branch: ceph-mimic
       rgw_server: client.0
       roundtrip:
         bucket: rttest

diff --git a/qa/suites/smoke/basic/tasks/rgw_ec_s3tests.yaml b/qa/suites/smoke/basic/tasks/rgw_ec_s3tests.yaml
index dda9aad0d3bb97dee372ecab45a2941ae4b4c875..292b0d9c9eeeb68486259f2105c657381820801e 100644 (file)
--- a/qa/suites/smoke/basic/tasks/rgw_ec_s3tests.yaml
+++ b/qa/suites/smoke/basic/tasks/rgw_ec_s3tests.yaml
@@ -9,6 +9,7 @@ tasks:
 - rgw: [client.0]
 - s3tests:
     client.0:
+      force-branch: ceph-mimic
       rgw_server: client.0
 overrides:
   ceph:

diff --git a/qa/suites/smoke/basic/tasks/rgw_s3tests.yaml b/qa/suites/smoke/basic/tasks/rgw_s3tests.yaml
index 4fca78c883f5590e61fb1c17e312376a6b9a1c79..427dfe6ad5b1ea4dfe7cf909a4aa5bb0b00aafcf 100644 (file)
--- a/qa/suites/smoke/basic/tasks/rgw_s3tests.yaml
+++ b/qa/suites/smoke/basic/tasks/rgw_s3tests.yaml
@@ -5,6 +5,7 @@ tasks:
 - rgw: [client.0]
 - s3tests:
     client.0:
+      force-branch: ceph-mimic
       rgw_server: client.0
 overrides:
   ceph:

diff --git a/qa/suites/teuthology/rgw/tasks/s3tests-civetweb.yaml b/qa/suites/teuthology/rgw/tasks/s3tests-civetweb.yaml
index 01580e38f066021eec6cfcc2d52993981c0b9579..0423ab41949b068bbea80504ba30813d8c7499e3 100644 (file)
--- a/qa/suites/teuthology/rgw/tasks/s3tests-civetweb.yaml
+++ b/qa/suites/teuthology/rgw/tasks/s3tests-civetweb.yaml
@@ -11,7 +11,7 @@ tasks:
 - s3tests:
     client.0:
       rgw_server: client.0
-      force-branch: master
+      force-branch: ceph-mimic
 overrides:
   ceph:
     fs: xfs

diff --git a/qa/suites/teuthology/rgw/tasks/s3tests-fastcgi.yaml b/qa/suites/teuthology/rgw/tasks/s3tests-fastcgi.yaml
index d8a5050385cce3b2ee7e9f43fd51d995859fd5e9..bb434a6ab24392ff75b891d5def522892e05ba71 100644 (file)
--- a/qa/suites/teuthology/rgw/tasks/s3tests-fastcgi.yaml
+++ b/qa/suites/teuthology/rgw/tasks/s3tests-fastcgi.yaml
@@ -11,7 +11,7 @@ tasks:
 - s3tests:
     client.0:
       rgw_server: client.0
-      force-branch: master
+      force-branch: ceph-mimic
 overrides:
   ceph:
     fs: xfs

diff --git a/qa/suites/teuthology/rgw/tasks/s3tests-fcgi.yaml b/qa/suites/teuthology/rgw/tasks/s3tests-fcgi.yaml
index 1def7b042bde9ba6a51e52c9e201f81420a73729..14193da0a1a8f46f2a7af4843a17ab6814f6fa89 100644 (file)
--- a/qa/suites/teuthology/rgw/tasks/s3tests-fcgi.yaml
+++ b/qa/suites/teuthology/rgw/tasks/s3tests-fcgi.yaml
@@ -12,7 +12,7 @@ tasks:
 - s3tests:
     client.0:
       rgw_server: client.0
-      force-branch: master
+      force-branch: ceph-mimic
 overrides:
   ceph:
     fs: xfs

diff --git a/qa/tasks/s3readwrite.py b/qa/tasks/s3readwrite.py
index 6f98e59b073348d59485d1d35c82429c66ae25d9..1a6b2b6d56bb44a35e137b49f918d07a9efeaa31 100644 (file)
--- a/qa/tasks/s3readwrite.py
+++ b/qa/tasks/s3readwrite.py
@@ -18,29 +18,32 @@ from teuthology.orchestra.connection import split_user
 
 log = logging.getLogger(__name__)
 
-
 @contextlib.contextmanager
 def download(ctx, config):
     """
     Download the s3 tests from the git builder.
     Remove downloaded s3 file upon exit.
-    
+
     The context passed in should be identical to the context
     passed in to the main task.
     """
     assert isinstance(config, dict)
     log.info('Downloading s3-tests...')
     testdir = teuthology.get_testdir(ctx)
-    for (client, cconf) in config.items():
-        branch = cconf.get('force-branch', None)
-        if not branch:
-            branch = cconf.get('branch', 'master')
-        sha1 = cconf.get('sha1')
+    for (client, client_config) in config.items():
+        s3tests_branch = client_config.get('force-branch', None)
+        if not s3tests_branch:
+            raise ValueError(
+                "Could not determine what branch to use for s3-tests. Please add 'force-branch: {s3-tests branch name}' to the .yaml config for this s3readwrite task.")
+
+        log.info("Using branch '%s' for s3tests", s3tests_branch)
+        sha1 = client_config.get('sha1')
+        git_remote = client_config.get('git_remote', teuth_config.ceph_git_base_url)
         ctx.cluster.only(client).run(
             args=[
                 'git', 'clone',
-                '-b', branch,
-                teuth_config.ceph_git_base_url + 's3-tests.git',
+                '-b', s3tests_branch,
+                git_remote + 's3-tests.git',
                 '{tdir}/s3-tests'.format(tdir=testdir),
                 ],
             )

diff --git a/qa/tasks/s3roundtrip.py b/qa/tasks/s3roundtrip.py
index f2632c9b1e509b1832a08138de37eb06b35357b0..9ed3ed44b0c6cff4a1400e966e11a7dee864b272 100644 (file)
--- a/qa/tasks/s3roundtrip.py
+++ b/qa/tasks/s3roundtrip.py
@@ -18,35 +18,48 @@ from teuthology.orchestra.connection import split_user
 
 log = logging.getLogger(__name__)
 
-
 @contextlib.contextmanager
 def download(ctx, config):
     """
     Download the s3 tests from the git builder.
     Remove downloaded s3 file upon exit.
-    
+
     The context passed in should be identical to the context
     passed in to the main task.
     """
     assert isinstance(config, dict)
     log.info('Downloading s3-tests...')
     testdir = teuthology.get_testdir(ctx)
-    for (client, cconf) in config.iteritems():
-        branch = cconf.get('force-branch', None)
-        if not branch:
-            branch = cconf.get('branch', 'master')
+    for (client, client_config) in config.items():
+        s3tests_branch = client_config.get('force-branch', None)
+        if not s3tests_branch:
+            raise ValueError(
+                "Could not determine what branch to use for s3-tests. Please add 'force-branch: {s3-tests branch name}' to the .yaml config for this s3roundtrip task.")
+
+        log.info("Using branch '%s' for s3tests", s3tests_branch)
+        sha1 = client_config.get('sha1')
+        git_remote = client_config.get('git_remote', teuth_config.ceph_git_base_url)
         ctx.cluster.only(client).run(
             args=[
                 'git', 'clone',
-                '-b', branch,
-                teuth_config.ceph_git_base_url + 's3-tests.git',
+                '-b', s3tests_branch,
+                git_remote + 's3-tests.git',
                 '{tdir}/s3-tests'.format(tdir=testdir),
                 ],
             )
+        if sha1 is not None:
+            ctx.cluster.only(client).run(
+                args=[
+                    'cd', '{tdir}/s3-tests'.format(tdir=testdir),
+                    run.Raw('&&'),
+                    'git', 'reset', '--hard', sha1,
+                    ],
+                )
     try:
         yield
     finally:
         log.info('Removing s3-tests...')
+        testdir = teuthology.get_testdir(ctx)
         for client in config:
             ctx.cluster.only(client).run(
                 args=[
@@ -56,6 +69,7 @@ def download(ctx, config):
                     ],
                 )
 
+
 def _config_user(s3tests_conf, section, user):
     """
     Configure users for this section by stashing away keys, ids, and

diff --git a/qa/tasks/s3tests.py b/qa/tasks/s3tests.py
index 11e139df4a41682ab7bc3281b2f680a4f22fc52c..305de1e6e15e89c0b09aafe37b535e116c381ff5 100644 (file)
--- a/qa/tasks/s3tests.py
+++ b/qa/tasks/s3tests.py
@@ -30,27 +30,19 @@ def download(ctx, config):
     assert isinstance(config, dict)
     log.info('Downloading s3-tests...')
     testdir = teuthology.get_testdir(ctx)
-    s3_branches = [ 'giant', 'firefly', 'firefly-original', 'hammer' ]
-    for (client, cconf) in config.items():
-        branch = cconf.get('force-branch', None)
-        if not branch:
-            ceph_branch = ctx.config.get('branch')
-            suite_branch = ctx.config.get('suite_branch', ceph_branch)
-            if suite_branch in s3_branches:
-                branch = cconf.get('branch', suite_branch)
-           else:
-                branch = cconf.get('branch', 'ceph-' + suite_branch)
-        if not branch:
+    for (client, client_config) in config.items():
+        s3tests_branch = client_config.get('force-branch', None)
+        if not s3tests_branch:
             raise ValueError(
-                "Could not determine what branch to use for s3tests!")
-        else:
-            log.info("Using branch '%s' for s3tests", branch)
-        sha1 = cconf.get('sha1')
-        git_remote = cconf.get('git_remote', None) or teuth_config.ceph_git_base_url
+                "Could not determine what branch to use for s3-tests. Please add 'force-branch: {s3-tests branch name}' to the .yaml config for this s3tests task.")
+
+        log.info("Using branch '%s' for s3tests", s3tests_branch)
+        sha1 = client_config.get('sha1')
+        git_remote = client_config.get('git_remote', teuth_config.ceph_git_base_url)
         ctx.cluster.only(client).run(
             args=[
                 'git', 'clone',
-                '-b', branch,
+                '-b', s3tests_branch,
                 git_remote + 's3-tests.git',
                 '{tdir}/s3-tests'.format(tdir=testdir),
                 ],
@@ -340,7 +332,9 @@ def task(ctx, config):
         tasks:
         - ceph:
         - rgw: [client.0]
-        - s3tests: [client.0]
+        - s3tests:
+            client.0:
+              force-branch: ceph-mimic
 
     To run against a server on client.1 and increase the boto timeout to 10m::
 
@@ -349,6 +343,7 @@ def task(ctx, config):
         - rgw: [client.1]
         - s3tests:
             client.0:
+              force-branch: ceph-mimic
               rgw_server: client.1
               idle_timeout: 600
 
@@ -359,8 +354,10 @@ def task(ctx, config):
         - rgw: [client.0]
         - s3tests:
             client.0:
+              force-branch: ceph-mimic
               extra_args: ['test_s3:test_object_acl_grand_public_read']
             client.1:
+              force-branch: ceph-mimic
               extra_args: ['--exclude', 'test_100_continue']
     """
     assert hasattr(ctx, 'rgw'), 's3tests must run after the rgw task'