From: Ali Maredia <amaredia@redhat.com>
Date: Mon, 25 Nov 2019 02:30:03 +0000 (-0500)
Subject: qa: update s3-test download code for s3-test tasks
X-Git-Tag: v15.1.0~563^2
X-Git-Url: http://git.apps.os.sepia.ceph.com/?a=commitdiff_plain;h=refs%2Fpull%2F31839%2Fhead;p=ceph.git

qa: update s3-test download code for s3-test tasks

- Ensure the download code for all tasks running
s3-tests is consistent.
- Simplify download code to only use the config
variable 'force-branch' for the branch being
cloned.
- make ceph-master the force-branch for all
suites using s3-tests.

Fixes: https://tracker.ceph.com/issues/43077

Signed-off-by: Ali Maredia <amaredia@redhat.com>
---

diff --git a/qa/machine_types/vps.yaml b/qa/machine_types/vps.yaml
index 64a3da47d4e3c..9a38e25170e9a 100644
--- a/qa/machine_types/vps.yaml
+++ b/qa/machine_types/vps.yaml
@@ -7,6 +7,7 @@ overrides:
         mon lease: 15
         mon lease ack timeout: 25
   s3tests:
+    force-branch: ceph-master
     idle_timeout: 1200
   ceph-fuse:
     client.0:
diff --git a/qa/rgw_pool_type/ec-profile.yaml b/qa/rgw_pool_type/ec-profile.yaml
index 05384cb5309a5..1103039c539da 100644
--- a/qa/rgw_pool_type/ec-profile.yaml
+++ b/qa/rgw_pool_type/ec-profile.yaml
@@ -7,4 +7,5 @@ overrides:
       m: 1
       crush-failure-domain: osd
   s3tests:
+    force-branch: ceph-master
     slow_backend: true
diff --git a/qa/rgw_pool_type/ec.yaml b/qa/rgw_pool_type/ec.yaml
index 7c99b7f85c8ee..d290e38ed4005 100644
--- a/qa/rgw_pool_type/ec.yaml
+++ b/qa/rgw_pool_type/ec.yaml
@@ -2,4 +2,5 @@ overrides:
   rgw:
     ec-data-pool: true
   s3tests:
+    force-branch: ceph-master
     slow_backend: true
diff --git a/qa/suites/smoke/basic/tasks/rgw_ec_s3tests.yaml b/qa/suites/smoke/basic/tasks/rgw_ec_s3tests.yaml
index 1455b51b4dc02..29a6cec129c0c 100644
--- a/qa/suites/smoke/basic/tasks/rgw_ec_s3tests.yaml
+++ b/qa/suites/smoke/basic/tasks/rgw_ec_s3tests.yaml
@@ -9,6 +9,7 @@ tasks:
 - rgw: [client.0]
 - s3tests:
     client.0:
+      force-branch: ceph-master
       rgw_server: client.0
 overrides:
   ceph:
diff --git a/qa/suites/smoke/basic/tasks/rgw_s3tests.yaml b/qa/suites/smoke/basic/tasks/rgw_s3tests.yaml
index 3d30774ed96bd..2ba6a4b497e04 100644
--- a/qa/suites/smoke/basic/tasks/rgw_s3tests.yaml
+++ b/qa/suites/smoke/basic/tasks/rgw_s3tests.yaml
@@ -5,6 +5,7 @@ tasks:
 - rgw: [client.0]
 - s3tests:
     client.0:
+      force-branch: ceph-master
       rgw_server: client.0
 overrides:
   ceph:
diff --git a/qa/suites/teuthology/rgw/tasks/s3tests-civetweb.yaml b/qa/suites/teuthology/rgw/tasks/s3tests-civetweb.yaml
index 01580e38f0660..7d39cfbe9202a 100644
--- a/qa/suites/teuthology/rgw/tasks/s3tests-civetweb.yaml
+++ b/qa/suites/teuthology/rgw/tasks/s3tests-civetweb.yaml
@@ -11,7 +11,7 @@ tasks:
 - s3tests:
     client.0:
       rgw_server: client.0
-      force-branch: master
+      force-branch: ceph-master
 overrides:
   ceph:
     fs: xfs
diff --git a/qa/suites/teuthology/rgw/tasks/s3tests-fastcgi.yaml b/qa/suites/teuthology/rgw/tasks/s3tests-fastcgi.yaml
index d8a5050385cce..d76121fad1029 100644
--- a/qa/suites/teuthology/rgw/tasks/s3tests-fastcgi.yaml
+++ b/qa/suites/teuthology/rgw/tasks/s3tests-fastcgi.yaml
@@ -11,7 +11,7 @@ tasks:
 - s3tests:
     client.0:
       rgw_server: client.0
-      force-branch: master
+      force-branch: ceph-master
 overrides:
   ceph:
     fs: xfs
diff --git a/qa/suites/teuthology/rgw/tasks/s3tests-fcgi.yaml b/qa/suites/teuthology/rgw/tasks/s3tests-fcgi.yaml
index 1def7b042bde9..8228501ca4b7b 100644
--- a/qa/suites/teuthology/rgw/tasks/s3tests-fcgi.yaml
+++ b/qa/suites/teuthology/rgw/tasks/s3tests-fcgi.yaml
@@ -12,7 +12,7 @@ tasks:
 - s3tests:
     client.0:
       rgw_server: client.0
-      force-branch: master
+      force-branch: ceph-master
 overrides:
   ceph:
     fs: xfs
diff --git a/qa/tasks/s3readwrite.py b/qa/tasks/s3readwrite.py
index 5e28513121889..8ed7828c61cce 100644
--- a/qa/tasks/s3readwrite.py
+++ b/qa/tasks/s3readwrite.py
@@ -18,29 +18,32 @@ from teuthology.orchestra.connection import split_user
 
 log = logging.getLogger(__name__)
 
-
 @contextlib.contextmanager
 def download(ctx, config):
     """
     Download the s3 tests from the git builder.
     Remove downloaded s3 file upon exit.
-    
+
     The context passed in should be identical to the context
     passed in to the main task.
     """
     assert isinstance(config, dict)
     log.info('Downloading s3-tests...')
     testdir = teuthology.get_testdir(ctx)
-    for (client, cconf) in config.items():
-        branch = cconf.get('force-branch', None)
-        if not branch:
-            branch = cconf.get('branch', 'master')
-        sha1 = cconf.get('sha1')
+    for (client, client_config) in config.items():
+        s3tests_branch = client_config.get('force-branch', None)
+        if not s3tests_branch:
+            raise ValueError(
+                "Could not determine what branch to use for s3-tests. Please add 'force-branch: {s3-tests branch name}' to the .yaml config for this s3readwrite task.")
+
+        log.info("Using branch '%s' for s3tests", s3tests_branch)
+        sha1 = client_config.get('sha1')
+        git_remote = client_config.get('git_remote', teuth_config.ceph_git_base_url)
         ctx.cluster.only(client).run(
             args=[
                 'git', 'clone',
-                '-b', branch,
-                teuth_config.ceph_git_base_url + 's3-tests.git',
+                '-b', s3tests_branch,
+                git_remote + 's3-tests.git',
                 '{tdir}/s3-tests'.format(tdir=testdir),
                 ],
             )
diff --git a/qa/tasks/s3roundtrip.py b/qa/tasks/s3roundtrip.py
index 01b56b1102f91..7823defd7c6e3 100644
--- a/qa/tasks/s3roundtrip.py
+++ b/qa/tasks/s3roundtrip.py
@@ -18,35 +18,48 @@ from teuthology.orchestra.connection import split_user
 
 log = logging.getLogger(__name__)
 
-
 @contextlib.contextmanager
 def download(ctx, config):
     """
     Download the s3 tests from the git builder.
     Remove downloaded s3 file upon exit.
-    
+
     The context passed in should be identical to the context
     passed in to the main task.
     """
     assert isinstance(config, dict)
     log.info('Downloading s3-tests...')
     testdir = teuthology.get_testdir(ctx)
-    for (client, cconf) in config.items():
-        branch = cconf.get('force-branch', None)
-        if not branch:
-            branch = cconf.get('branch', 'master')
+    for (client, client_config) in config.items():
+        s3tests_branch = client_config.get('force-branch', None)
+        if not s3tests_branch:
+            raise ValueError(
+                "Could not determine what branch to use for s3-tests. Please add 'force-branch: {s3-tests branch name}' to the .yaml config for this s3roundtrip task.")
+
+        log.info("Using branch '%s' for s3tests", s3tests_branch)
+        sha1 = client_config.get('sha1')
+        git_remote = client_config.get('git_remote', teuth_config.ceph_git_base_url)
         ctx.cluster.only(client).run(
             args=[
                 'git', 'clone',
-                '-b', branch,
-                teuth_config.ceph_git_base_url + 's3-tests.git',
+                '-b', s3tests_branch,
+                git_remote + 's3-tests.git',
                 '{tdir}/s3-tests'.format(tdir=testdir),
                 ],
             )
+        if sha1 is not None:
+            ctx.cluster.only(client).run(
+                args=[
+                    'cd', '{tdir}/s3-tests'.format(tdir=testdir),
+                    run.Raw('&&'),
+                    'git', 'reset', '--hard', sha1,
+                    ],
+                )
     try:
         yield
     finally:
         log.info('Removing s3-tests...')
+        testdir = teuthology.get_testdir(ctx)
         for client in config:
             ctx.cluster.only(client).run(
                 args=[
@@ -56,6 +69,7 @@ def download(ctx, config):
                     ],
                 )
 
+
 def _config_user(s3tests_conf, section, user):
     """
     Configure users for this section by stashing away keys, ids, and
diff --git a/qa/tasks/s3tests.py b/qa/tasks/s3tests.py
index eb6dc5a6eeb4e..16d7126d9edad 100644
--- a/qa/tasks/s3tests.py
+++ b/qa/tasks/s3tests.py
@@ -31,27 +31,19 @@ def download(ctx, config):
     assert isinstance(config, dict)
     log.info('Downloading s3-tests...')
     testdir = teuthology.get_testdir(ctx)
-    s3_branches = [ 'giant', 'firefly', 'firefly-original', 'hammer' ]
-    for (client, cconf) in config.items():
-        branch = cconf.get('force-branch', None)
-        if not branch:
-            ceph_branch = ctx.config.get('branch')
-            suite_branch = ctx.config.get('suite_branch', ceph_branch)
-            if suite_branch in s3_branches:
-                branch = cconf.get('branch', suite_branch)
-	    else:
-                branch = cconf.get('branch', 'ceph-' + suite_branch)
-        if not branch:
+    for (client, client_config) in config.items():
+        s3tests_branch = client_config.get('force-branch', None)
+        if not s3tests_branch:
             raise ValueError(
-                "Could not determine what branch to use for s3tests!")
-        else:
-            log.info("Using branch '%s' for s3tests", branch)
-        sha1 = cconf.get('sha1')
-        git_remote = cconf.get('git_remote', None) or teuth_config.ceph_git_base_url
+                "Could not determine what branch to use for s3-tests. Please add 'force-branch: {s3-tests branch name}' to the .yaml config for this s3tests task.")
+
+        log.info("Using branch '%s' for s3tests", s3tests_branch)
+        sha1 = client_config.get('sha1')
+        git_remote = client_config.get('git_remote', teuth_config.ceph_git_base_url)
         ctx.cluster.only(client).run(
             args=[
                 'git', 'clone',
-                '-b', branch,
+                '-b', s3tests_branch,
                 git_remote + 's3-tests.git',
                 '{tdir}/s3-tests'.format(tdir=testdir),
                 ],