qa/tasks/cram: tasks now must live in the repository

Commit 0d8887652d53 ("qa/tasks/cram: use suite_repo repository for all
cram jobs") removed hardcoded git.ceph.com links, but as it turned out
it is still used for nightlies.  There is no good way to accommodate
the different URL schemes, so let's get rid of URLs altogether.

Fixes: https://tracker.ceph.com/issues/27211
Signed-off-by: Ilya Dryomov <idryomov@gmail.com>
(cherry picked from commit 592f566b4e270d3833ba896dabb193d5241dd5de)

Conflicts:
	qa/suites/krbd/basic/tasks/krbd_blkroset.yaml
	qa/suites/krbd/basic/tasks/krbd_huge_image.yaml
	qa/suites/krbd/basic/tasks/krbd_msgr_segments.yaml
	qa/suites/krbd/basic/tasks/krbd_parent_overlap.yaml
	qa/suites/krbd/basic/tasks/krbd_whole_object_discard.yaml
- in master, the cram task is referred to in these additional yaml
files, but in luminous it's only referred to in
qa/suites/krbd/unmap/tasks/unmap.yaml

diff --git a/qa/suites/krbd/unmap/tasks/unmap.yaml b/qa/suites/krbd/unmap/tasks/unmap.yaml
index 47153cf4d36ffef8a670b640422a64092e749791..435061b459ff43c7442dbabcc09b9b1416e9e602 100644 (file)
--- a/qa/suites/krbd/unmap/tasks/unmap.yaml
+++ b/qa/suites/krbd/unmap/tasks/unmap.yaml
@@ -2,4 +2,4 @@ tasks:
 - cram:
     clients:
       client.0:
-      - '{gh_url}/raw/{branch}/src/test/cli-integration/rbd/unmap.t'
+      - src/test/cli-integration/rbd/unmap.t

diff --git a/qa/suites/rbd/singleton/all/formatted-output.yaml b/qa/suites/rbd/singleton/all/formatted-output.yaml
index df1112d4e32af5699a5ea0614e0cfbada123556e..7be94ef23027c30d3ee8439509ed3cdb1d27208f 100644 (file)
--- a/qa/suites/rbd/singleton/all/formatted-output.yaml
+++ b/qa/suites/rbd/singleton/all/formatted-output.yaml
@@ -7,4 +7,4 @@ tasks:
 - cram:
     clients:
       client.0:
-      - '{gh_url}/raw/{branch}/src/test/cli-integration/rbd/formatted-output.t'
+      - src/test/cli-integration/rbd/formatted-output.t

diff --git a/qa/tasks/cram.py b/qa/tasks/cram.py
index bc2a7c64f690e4e1efe171665747ab864fef9f3e..9fe4fb809f573fefcf18353d0ab0a364e6e96c52 100644 (file)
--- a/qa/tasks/cram.py
+++ b/qa/tasks/cram.py
@@ -4,6 +4,8 @@ Cram tests
 import logging
 import os
 
+from util.workunit import get_refspec_after_overrides
+
 from teuthology import misc as teuthology
 from teuthology.parallel import parallel
 from teuthology.orchestra import run
@@ -13,7 +15,7 @@ log = logging.getLogger(__name__)
 
 def task(ctx, config):
     """
-    Run all cram tests from the specified urls on the specified
+    Run all cram tests from the specified paths on the specified
     clients. Each client runs tests in parallel.
 
     Limitations:
@@ -27,9 +29,9 @@ def task(ctx, config):
         - cram:
             clients:
               client.0:
-              - http://download.ceph.com/qa/test.t
-              - http://download.ceph.com/qa/test2.t]
-              client.1: [http://download.ceph.com/qa/test.t]
+              - qa/test.t
+              - qa/test2.t]
+              client.1: [qa/test.t]
             branch: foo
 
     You can also run a list of cram tests on all clients::
@@ -38,7 +40,7 @@ def task(ctx, config):
         - ceph:
         - cram:
             clients:
-              all: [http://download.ceph.com/qa/test.t]
+              all: [qa/test.t]
 
     :param ctx: Context
     :param config: Configuration
@@ -52,19 +54,10 @@ def task(ctx, config):
     testdir = teuthology.get_testdir(ctx)
 
     overrides = ctx.config.get('overrides', {})
-    teuthology.deep_merge(config, overrides.get('workunit', {}))
-
-    refspec = config.get('branch')
-    if refspec is None:
-        refspec = config.get('tag')
-    if refspec is None:
-        refspec = config.get('sha1')
-    if refspec is None:
-        refspec = 'HEAD'
+    refspec = get_refspec_after_overrides(config, overrides)
 
     git_url = teuth_config.get_ceph_qa_suite_git_url()
-    if git_url.endswith('.git'):
-        git_url = git_url[:-4]
+    log.info('Pulling tests from %s ref %s', git_url, refspec)
 
     try:
         for client, tests in clients.iteritems():
@@ -80,13 +73,14 @@ def task(ctx, config):
                     'install', 'cram==0.6',
                     ],
                 )
+            clone_dir = '{tdir}/clone.{role}'.format(tdir=testdir, role=client)
+            remote.run(args=refspec.clone(git_url, clone_dir))
+
             for test in tests:
-                url = test.format(gh_url=git_url, branch=refspec)
-                log.info('fetching test %s for %s', url, client)
                 assert test.endswith('.t'), 'tests must end in .t'
                 remote.run(
                     args=[
-                        'wget', '-nc', '-nv', '-P', client_dir, '--', url,
+                        'cp', '--', os.path.join(clone_dir, test), client_dir,
                         ],
                     )
 
@@ -117,6 +111,7 @@ def task(ctx, config):
                 args=[
                     'rm', '-rf', '--',
                     '{tdir}/virtualenv'.format(tdir=testdir),
+                    clone_dir,
                     run.Raw(';'),
                     'rmdir', '--ignore-fail-on-non-empty', client_dir,
                     ],