qa: update s3-test download code for s3-test tasks

- Ensure the download code for all tasks running
s3-tests is consistent.
- Simplify download code to only use the config
variable 'force-branch' for the branch being
cloned.
- make ceph-master the force-branch for all
suites using s3-tests.

Fixes: https://tracker.ceph.com/issues/43077
Signed-off-by: Ali Maredia <amaredia@redhat.com>

diff --git a/qa/machine_types/vps.yaml b/qa/machine_types/vps.yaml
index 64a3da47d4e3c799b34f2f053792905643894d24..9a38e25170e9a60fc3e849523e7166de27d87e98 100644 (file)
--- a/qa/machine_types/vps.yaml
+++ b/qa/machine_types/vps.yaml
@@ -7,6 +7,7 @@ overrides:
         mon lease: 15
         mon lease ack timeout: 25
   s3tests:
+    force-branch: ceph-master
     idle_timeout: 1200
   ceph-fuse:
     client.0:

diff --git a/qa/rgw_pool_type/ec-profile.yaml b/qa/rgw_pool_type/ec-profile.yaml
index 05384cb5309a5f5f4b51adc0ec3d701e7ae25311..1103039c539da7c32c2229198bf0825a1f58fb1f 100644 (file)
--- a/qa/rgw_pool_type/ec-profile.yaml
+++ b/qa/rgw_pool_type/ec-profile.yaml
@@ -7,4 +7,5 @@ overrides:
       m: 1
       crush-failure-domain: osd
   s3tests:
+    force-branch: ceph-master
     slow_backend: true

diff --git a/qa/rgw_pool_type/ec.yaml b/qa/rgw_pool_type/ec.yaml
index 7c99b7f85c8ee27510f10b15fb293a0bfab86f78..d290e38ed4005a23dbadc335407919d197d03d9b 100644 (file)
--- a/qa/rgw_pool_type/ec.yaml
+++ b/qa/rgw_pool_type/ec.yaml
@@ -2,4 +2,5 @@ overrides:
   rgw:
     ec-data-pool: true
   s3tests:
+    force-branch: ceph-master
     slow_backend: true

diff --git a/qa/suites/smoke/basic/tasks/rgw_ec_s3tests.yaml b/qa/suites/smoke/basic/tasks/rgw_ec_s3tests.yaml
index 1455b51b4dc020f89bae6b83de3f633199275fa7..29a6cec129c0cde00ebfe402adc2a20e8d572059 100644 (file)
--- a/qa/suites/smoke/basic/tasks/rgw_ec_s3tests.yaml
+++ b/qa/suites/smoke/basic/tasks/rgw_ec_s3tests.yaml
@@ -9,6 +9,7 @@ tasks:
 - rgw: [client.0]
 - s3tests:
     client.0:
+      force-branch: ceph-master
       rgw_server: client.0
 overrides:
   ceph:

diff --git a/qa/suites/smoke/basic/tasks/rgw_s3tests.yaml b/qa/suites/smoke/basic/tasks/rgw_s3tests.yaml
index 3d30774ed96bdf54d54855e9510336c2d9542e3e..2ba6a4b497e04ae235ed6fb0cb88e65848146839 100644 (file)
--- a/qa/suites/smoke/basic/tasks/rgw_s3tests.yaml
+++ b/qa/suites/smoke/basic/tasks/rgw_s3tests.yaml
@@ -5,6 +5,7 @@ tasks:
 - rgw: [client.0]
 - s3tests:
     client.0:
+      force-branch: ceph-master
       rgw_server: client.0
 overrides:
   ceph:

diff --git a/qa/suites/teuthology/rgw/tasks/s3tests-civetweb.yaml b/qa/suites/teuthology/rgw/tasks/s3tests-civetweb.yaml
index 01580e38f066021eec6cfcc2d52993981c0b9579..7d39cfbe9202a3a5c70dcd583b50f3687e38b444 100644 (file)
--- a/qa/suites/teuthology/rgw/tasks/s3tests-civetweb.yaml
+++ b/qa/suites/teuthology/rgw/tasks/s3tests-civetweb.yaml
@@ -11,7 +11,7 @@ tasks:
 - s3tests:
     client.0:
       rgw_server: client.0
-      force-branch: master
+      force-branch: ceph-master
 overrides:
   ceph:
     fs: xfs

diff --git a/qa/suites/teuthology/rgw/tasks/s3tests-fastcgi.yaml b/qa/suites/teuthology/rgw/tasks/s3tests-fastcgi.yaml
index d8a5050385cce3b2ee7e9f43fd51d995859fd5e9..d76121fad1029c864b21329a4cb2e96e828e965a 100644 (file)
--- a/qa/suites/teuthology/rgw/tasks/s3tests-fastcgi.yaml
+++ b/qa/suites/teuthology/rgw/tasks/s3tests-fastcgi.yaml
@@ -11,7 +11,7 @@ tasks:
 - s3tests:
     client.0:
       rgw_server: client.0
-      force-branch: master
+      force-branch: ceph-master
 overrides:
   ceph:
     fs: xfs

diff --git a/qa/suites/teuthology/rgw/tasks/s3tests-fcgi.yaml b/qa/suites/teuthology/rgw/tasks/s3tests-fcgi.yaml
index 1def7b042bde9ba6a51e52c9e201f81420a73729..8228501ca4b7b0036741156b29c682332e78d414 100644 (file)
--- a/qa/suites/teuthology/rgw/tasks/s3tests-fcgi.yaml
+++ b/qa/suites/teuthology/rgw/tasks/s3tests-fcgi.yaml
@@ -12,7 +12,7 @@ tasks:
 - s3tests:
     client.0:
       rgw_server: client.0
-      force-branch: master
+      force-branch: ceph-master
 overrides:
   ceph:
     fs: xfs

diff --git a/qa/tasks/s3readwrite.py b/qa/tasks/s3readwrite.py
index 5e28513121889906cd3ece15fa0dda8f595dde7b..8ed7828c61cceca759fa4505052d2304bec02dee 100644 (file)
--- a/qa/tasks/s3readwrite.py
+++ b/qa/tasks/s3readwrite.py
@@ -18,29 +18,32 @@ from teuthology.orchestra.connection import split_user
 
 log = logging.getLogger(__name__)
 
-
 @contextlib.contextmanager
 def download(ctx, config):
     """
     Download the s3 tests from the git builder.
     Remove downloaded s3 file upon exit.
-    
+
     The context passed in should be identical to the context
     passed in to the main task.
     """
     assert isinstance(config, dict)
     log.info('Downloading s3-tests...')
     testdir = teuthology.get_testdir(ctx)
-    for (client, cconf) in config.items():
-        branch = cconf.get('force-branch', None)
-        if not branch:
-            branch = cconf.get('branch', 'master')
-        sha1 = cconf.get('sha1')
+    for (client, client_config) in config.items():
+        s3tests_branch = client_config.get('force-branch', None)
+        if not s3tests_branch:
+            raise ValueError(
+                "Could not determine what branch to use for s3-tests. Please add 'force-branch: {s3-tests branch name}' to the .yaml config for this s3readwrite task.")
+
+        log.info("Using branch '%s' for s3tests", s3tests_branch)
+        sha1 = client_config.get('sha1')
+        git_remote = client_config.get('git_remote', teuth_config.ceph_git_base_url)
         ctx.cluster.only(client).run(
             args=[
                 'git', 'clone',
-                '-b', branch,
-                teuth_config.ceph_git_base_url + 's3-tests.git',
+                '-b', s3tests_branch,
+                git_remote + 's3-tests.git',
                 '{tdir}/s3-tests'.format(tdir=testdir),
                 ],
             )

diff --git a/qa/tasks/s3roundtrip.py b/qa/tasks/s3roundtrip.py
index 01b56b1102f915fb60b9e772decf9e106811711b..7823defd7c6e3bedd71446cff0501f1944dc67be 100644 (file)
--- a/qa/tasks/s3roundtrip.py
+++ b/qa/tasks/s3roundtrip.py
@@ -18,35 +18,48 @@ from teuthology.orchestra.connection import split_user
 
 log = logging.getLogger(__name__)
 
-
 @contextlib.contextmanager
 def download(ctx, config):
     """
     Download the s3 tests from the git builder.
     Remove downloaded s3 file upon exit.
-    
+
     The context passed in should be identical to the context
     passed in to the main task.
     """
     assert isinstance(config, dict)
     log.info('Downloading s3-tests...')
     testdir = teuthology.get_testdir(ctx)
-    for (client, cconf) in config.items():
-        branch = cconf.get('force-branch', None)
-        if not branch:
-            branch = cconf.get('branch', 'master')
+    for (client, client_config) in config.items():
+        s3tests_branch = client_config.get('force-branch', None)
+        if not s3tests_branch:
+            raise ValueError(
+                "Could not determine what branch to use for s3-tests. Please add 'force-branch: {s3-tests branch name}' to the .yaml config for this s3roundtrip task.")
+
+        log.info("Using branch '%s' for s3tests", s3tests_branch)
+        sha1 = client_config.get('sha1')
+        git_remote = client_config.get('git_remote', teuth_config.ceph_git_base_url)
         ctx.cluster.only(client).run(
             args=[
                 'git', 'clone',
-                '-b', branch,
-                teuth_config.ceph_git_base_url + 's3-tests.git',
+                '-b', s3tests_branch,
+                git_remote + 's3-tests.git',
                 '{tdir}/s3-tests'.format(tdir=testdir),
                 ],
             )
+        if sha1 is not None:
+            ctx.cluster.only(client).run(
+                args=[
+                    'cd', '{tdir}/s3-tests'.format(tdir=testdir),
+                    run.Raw('&&'),
+                    'git', 'reset', '--hard', sha1,
+                    ],
+                )
     try:
         yield
     finally:
         log.info('Removing s3-tests...')
+        testdir = teuthology.get_testdir(ctx)
         for client in config:
             ctx.cluster.only(client).run(
                 args=[
@@ -56,6 +69,7 @@ def download(ctx, config):
                     ],
                 )
 
+
 def _config_user(s3tests_conf, section, user):
     """
     Configure users for this section by stashing away keys, ids, and

diff --git a/qa/tasks/s3tests.py b/qa/tasks/s3tests.py
index eb6dc5a6eeb4e6982bcf92ad88b260a4a8aa801e..16d7126d9edadd2cabe2b6845dde317bb347013d 100644 (file)
--- a/qa/tasks/s3tests.py
+++ b/qa/tasks/s3tests.py
@@ -31,27 +31,19 @@ def download(ctx, config):
     assert isinstance(config, dict)
     log.info('Downloading s3-tests...')
     testdir = teuthology.get_testdir(ctx)
-    s3_branches = [ 'giant', 'firefly', 'firefly-original', 'hammer' ]
-    for (client, cconf) in config.items():
-        branch = cconf.get('force-branch', None)
-        if not branch:
-            ceph_branch = ctx.config.get('branch')
-            suite_branch = ctx.config.get('suite_branch', ceph_branch)
-            if suite_branch in s3_branches:
-                branch = cconf.get('branch', suite_branch)
-           else:
-                branch = cconf.get('branch', 'ceph-' + suite_branch)
-        if not branch:
+    for (client, client_config) in config.items():
+        s3tests_branch = client_config.get('force-branch', None)
+        if not s3tests_branch:
             raise ValueError(
-                "Could not determine what branch to use for s3tests!")
-        else:
-            log.info("Using branch '%s' for s3tests", branch)
-        sha1 = cconf.get('sha1')
-        git_remote = cconf.get('git_remote', None) or teuth_config.ceph_git_base_url
+                "Could not determine what branch to use for s3-tests. Please add 'force-branch: {s3-tests branch name}' to the .yaml config for this s3tests task.")
+
+        log.info("Using branch '%s' for s3tests", s3tests_branch)
+        sha1 = client_config.get('sha1')
+        git_remote = client_config.get('git_remote', teuth_config.ceph_git_base_url)
         ctx.cluster.only(client).run(
             args=[
                 'git', 'clone',
-                '-b', branch,
+                '-b', s3tests_branch,
                 git_remote + 's3-tests.git',
                 '{tdir}/s3-tests'.format(tdir=testdir),
                 ],