From: Ali Maredia Date: Mon, 25 Nov 2019 02:30:03 +0000 (-0500) Subject: nautilus: update s3-test download code for s3-test tasks X-Git-Tag: v14.2.8~28^2~2^2~24^2 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=refs%2Fpull%2F32229%2Fhead;p=ceph.git nautilus: update s3-test download code for s3-test tasks Fixes: https://tracker.ceph.com/issues/43077 - Ensure the download code for all tasks running s3-tests is consistent. - Simplify download code to only use the config variable 'force-branch' for the branch being cloned. - Make ceph-nautilus the force-branch for all suites using s3-tests. - Add force-branch to suites running s3readwrite & s3roundtrip tasks Signed-off-by: Ali Maredia --- diff --git a/qa/suites/rados/basic/tasks/rgw_snaps.yaml b/qa/suites/rados/basic/tasks/rgw_snaps.yaml index 2a9746391e9e..c94b7b2bafd9 100644 --- a/qa/suites/rados/basic/tasks/rgw_snaps.yaml +++ b/qa/suites/rados/basic/tasks/rgw_snaps.yaml @@ -28,6 +28,7 @@ tasks: - default.rgw.log - s3readwrite: client.0: + force-branch: ceph-nautilus rgw_server: client.0 readwrite: bucket: rwtest diff --git a/qa/suites/rgw/multifs/tasks/rgw_readwrite.yaml b/qa/suites/rgw/multifs/tasks/rgw_readwrite.yaml index c7efaa1c7579..40a915999d6f 100644 --- a/qa/suites/rgw/multifs/tasks/rgw_readwrite.yaml +++ b/qa/suites/rgw/multifs/tasks/rgw_readwrite.yaml @@ -4,6 +4,7 @@ tasks: - rgw: [client.0] - s3readwrite: client.0: + force-branch: ceph-nautilus rgw_server: client.0 readwrite: bucket: rwtest diff --git a/qa/suites/rgw/multifs/tasks/rgw_roundtrip.yaml b/qa/suites/rgw/multifs/tasks/rgw_roundtrip.yaml index 47b3c1894a22..8f8883130b4f 100644 --- a/qa/suites/rgw/multifs/tasks/rgw_roundtrip.yaml +++ b/qa/suites/rgw/multifs/tasks/rgw_roundtrip.yaml @@ -4,6 +4,7 @@ tasks: - rgw: [client.0] - s3roundtrip: client.0: + force-branch: ceph-nautilus rgw_server: client.0 roundtrip: bucket: rttest diff --git a/qa/suites/rgw/thrash/workload/rgw_readwrite.yaml b/qa/suites/rgw/thrash/workload/rgw_readwrite.yaml index e4e68286848e..d67d52638918 100644 --- a/qa/suites/rgw/thrash/workload/rgw_readwrite.yaml +++ b/qa/suites/rgw/thrash/workload/rgw_readwrite.yaml @@ -1,6 +1,7 @@ tasks: - s3readwrite: client.0: + force-branch: ceph-nautilus rgw_server: client.0 readwrite: bucket: rwtest diff --git a/qa/suites/rgw/thrash/workload/rgw_roundtrip.yaml b/qa/suites/rgw/thrash/workload/rgw_roundtrip.yaml index 621683aeb1b8..b4b9806d5d89 100644 --- a/qa/suites/rgw/thrash/workload/rgw_roundtrip.yaml +++ b/qa/suites/rgw/thrash/workload/rgw_roundtrip.yaml @@ -1,6 +1,7 @@ tasks: - s3roundtrip: client.0: + force-branch: ceph-nautilus rgw_server: client.0 roundtrip: bucket: rttest diff --git a/qa/suites/smoke/basic/tasks/rgw_ec_s3tests.yaml b/qa/suites/smoke/basic/tasks/rgw_ec_s3tests.yaml index dda9aad0d3bb..3871cf53d979 100644 --- a/qa/suites/smoke/basic/tasks/rgw_ec_s3tests.yaml +++ b/qa/suites/smoke/basic/tasks/rgw_ec_s3tests.yaml @@ -9,6 +9,7 @@ tasks: - rgw: [client.0] - s3tests: client.0: + force-branch: ceph-nautilus rgw_server: client.0 overrides: ceph: diff --git a/qa/suites/smoke/basic/tasks/rgw_s3tests.yaml b/qa/suites/smoke/basic/tasks/rgw_s3tests.yaml index 4fca78c883f5..011f34a4aa68 100644 --- a/qa/suites/smoke/basic/tasks/rgw_s3tests.yaml +++ b/qa/suites/smoke/basic/tasks/rgw_s3tests.yaml @@ -5,6 +5,7 @@ tasks: - rgw: [client.0] - s3tests: client.0: + force-branch: ceph-nautilus rgw_server: client.0 overrides: ceph: diff --git a/qa/suites/teuthology/rgw/tasks/s3tests-civetweb.yaml b/qa/suites/teuthology/rgw/tasks/s3tests-civetweb.yaml index 01580e38f066..00a82a70ecd5 100644 --- a/qa/suites/teuthology/rgw/tasks/s3tests-civetweb.yaml +++ b/qa/suites/teuthology/rgw/tasks/s3tests-civetweb.yaml @@ -11,7 +11,7 @@ tasks: - s3tests: client.0: rgw_server: client.0 - force-branch: master + force-branch: ceph-nautilus overrides: ceph: fs: xfs diff --git a/qa/suites/teuthology/rgw/tasks/s3tests-fastcgi.yaml b/qa/suites/teuthology/rgw/tasks/s3tests-fastcgi.yaml index d8a5050385cc..8e923d00ecd9 100644 --- a/qa/suites/teuthology/rgw/tasks/s3tests-fastcgi.yaml +++ b/qa/suites/teuthology/rgw/tasks/s3tests-fastcgi.yaml @@ -11,7 +11,7 @@ tasks: - s3tests: client.0: rgw_server: client.0 - force-branch: master + force-branch: ceph-nautilus overrides: ceph: fs: xfs diff --git a/qa/suites/teuthology/rgw/tasks/s3tests-fcgi.yaml b/qa/suites/teuthology/rgw/tasks/s3tests-fcgi.yaml index 1def7b042bde..d03248c23d53 100644 --- a/qa/suites/teuthology/rgw/tasks/s3tests-fcgi.yaml +++ b/qa/suites/teuthology/rgw/tasks/s3tests-fcgi.yaml @@ -12,7 +12,7 @@ tasks: - s3tests: client.0: rgw_server: client.0 - force-branch: master + force-branch: ceph-nautilus overrides: ceph: fs: xfs diff --git a/qa/tasks/s3readwrite.py b/qa/tasks/s3readwrite.py index 6f98e59b0733..c37420f21f19 100644 --- a/qa/tasks/s3readwrite.py +++ b/qa/tasks/s3readwrite.py @@ -18,29 +18,32 @@ from teuthology.orchestra.connection import split_user log = logging.getLogger(__name__) - @contextlib.contextmanager def download(ctx, config): """ Download the s3 tests from the git builder. Remove downloaded s3 file upon exit. - + The context passed in should be identical to the context passed in to the main task. """ assert isinstance(config, dict) log.info('Downloading s3-tests...') testdir = teuthology.get_testdir(ctx) - for (client, cconf) in config.items(): - branch = cconf.get('force-branch', None) - if not branch: - branch = cconf.get('branch', 'master') - sha1 = cconf.get('sha1') + for (client, client_config) in config.items(): + s3tests_branch = client_config.get('force-branch', None) + if not s3tests_branch: + raise ValueError( + "Could not determine what branch to use for s3-tests. Please add 'force-branch: {s3-tests branch name}' to the .yaml config for this s3readwrite task.") + + log.info("Using branch '%s' for s3tests", s3tests_branch) + sha1 = client_config.get('sha1') + git_remote = client_config.get('git_remote', teuth_config.ceph_git_base_url) ctx.cluster.only(client).run( args=[ 'git', 'clone', - '-b', branch, - teuth_config.ceph_git_base_url + 's3-tests.git', + '-b', s3tests_branch, + git_remote + 's3-tests.git', '{tdir}/s3-tests'.format(tdir=testdir), ], ) @@ -242,6 +245,7 @@ def task(ctx, config): - ceph: - rgw: - s3readwrite: + force-branch: ceph-nautilus To restrict testing to particular clients:: @@ -257,6 +261,7 @@ def task(ctx, config): - rgw: [client.1] - s3readwrite: client.0: + force-branch: ceph-nautilus rgw_server: client.1 To pass extra test arguments @@ -266,6 +271,7 @@ def task(ctx, config): - rgw: [client.0] - s3readwrite: client.0: + force-branch: ceph-nautilus readwrite: bucket: mybucket readers: 10 @@ -285,6 +291,7 @@ def task(ctx, config): - rgw: [client.0] - s3readwrite: client.0: + force-branch: ceph-nautilus s3: user_id: myuserid display_name: myname diff --git a/qa/tasks/s3roundtrip.py b/qa/tasks/s3roundtrip.py index f2632c9b1e50..51bf5fb02419 100644 --- a/qa/tasks/s3roundtrip.py +++ b/qa/tasks/s3roundtrip.py @@ -18,35 +18,48 @@ from teuthology.orchestra.connection import split_user log = logging.getLogger(__name__) - @contextlib.contextmanager def download(ctx, config): """ Download the s3 tests from the git builder. Remove downloaded s3 file upon exit. - + The context passed in should be identical to the context passed in to the main task. """ assert isinstance(config, dict) log.info('Downloading s3-tests...') testdir = teuthology.get_testdir(ctx) - for (client, cconf) in config.iteritems(): - branch = cconf.get('force-branch', None) - if not branch: - branch = cconf.get('branch', 'master') + for (client, client_config) in config.items(): + s3tests_branch = client_config.get('force-branch', None) + if not s3tests_branch: + raise ValueError( + "Could not determine what branch to use for s3-tests. Please add 'force-branch: {s3-tests branch name}' to the .yaml config for this s3roundtrip task.") + + log.info("Using branch '%s' for s3tests", s3tests_branch) + sha1 = client_config.get('sha1') + git_remote = client_config.get('git_remote', teuth_config.ceph_git_base_url) ctx.cluster.only(client).run( args=[ 'git', 'clone', - '-b', branch, - teuth_config.ceph_git_base_url + 's3-tests.git', + '-b', s3tests_branch, + git_remote + 's3-tests.git', '{tdir}/s3-tests'.format(tdir=testdir), ], ) + if sha1 is not None: + ctx.cluster.only(client).run( + args=[ + 'cd', '{tdir}/s3-tests'.format(tdir=testdir), + run.Raw('&&'), + 'git', 'reset', '--hard', sha1, + ], + ) try: yield finally: log.info('Removing s3-tests...') + testdir = teuthology.get_testdir(ctx) for client in config: ctx.cluster.only(client).run( args=[ @@ -56,6 +69,7 @@ def download(ctx, config): ], ) + def _config_user(s3tests_conf, section, user): """ Configure users for this section by stashing away keys, ids, and @@ -211,13 +225,16 @@ def task(ctx, config): - ceph: - rgw: - s3roundtrip: + force-branch: ceph-nautilus To restrict testing to particular clients:: tasks: - ceph: - rgw: [client.0] - - s3roundtrip: [client.0] + - s3roundtrip: + client.0: + force-branch: ceph-nautilus To run against a server on client.1:: @@ -226,6 +243,7 @@ def task(ctx, config): - rgw: [client.1] - s3roundtrip: client.0: + force-branch: ceph-nautilus rgw_server: client.1 To pass extra test arguments @@ -235,6 +253,7 @@ def task(ctx, config): - rgw: [client.0] - s3roundtrip: client.0: + force-branch: ceph-nautilus roundtrip: bucket: mybucket readers: 10 @@ -253,6 +272,7 @@ def task(ctx, config): - ceph: - rgw: [client.0] - s3roundtrip: + force-branch: ceph-nautilus client.0: s3: user_id: myuserid diff --git a/qa/tasks/s3tests.py b/qa/tasks/s3tests.py index 11e139df4a41..3f2c5a2b7397 100644 --- a/qa/tasks/s3tests.py +++ b/qa/tasks/s3tests.py @@ -30,27 +30,19 @@ def download(ctx, config): assert isinstance(config, dict) log.info('Downloading s3-tests...') testdir = teuthology.get_testdir(ctx) - s3_branches = [ 'giant', 'firefly', 'firefly-original', 'hammer' ] - for (client, cconf) in config.items(): - branch = cconf.get('force-branch', None) - if not branch: - ceph_branch = ctx.config.get('branch') - suite_branch = ctx.config.get('suite_branch', ceph_branch) - if suite_branch in s3_branches: - branch = cconf.get('branch', suite_branch) - else: - branch = cconf.get('branch', 'ceph-' + suite_branch) - if not branch: + for (client, client_config) in config.items(): + s3tests_branch = client_config.get('force-branch', None) + if not s3tests_branch: raise ValueError( - "Could not determine what branch to use for s3tests!") - else: - log.info("Using branch '%s' for s3tests", branch) - sha1 = cconf.get('sha1') - git_remote = cconf.get('git_remote', None) or teuth_config.ceph_git_base_url + "Could not determine what branch to use for s3-tests. Please add 'force-branch: {s3-tests branch name}' to the .yaml config for this s3tests task.") + + log.info("Using branch '%s' for s3tests", s3tests_branch) + sha1 = client_config.get('sha1') + git_remote = client_config.get('git_remote', teuth_config.ceph_git_base_url) ctx.cluster.only(client).run( args=[ 'git', 'clone', - '-b', branch, + '-b', s3tests_branch, git_remote + 's3-tests.git', '{tdir}/s3-tests'.format(tdir=testdir), ], @@ -340,7 +332,9 @@ def task(ctx, config): tasks: - ceph: - rgw: [client.0] - - s3tests: [client.0] + - s3tests: + client.0: + force-branch: ceph-nautilus To run against a server on client.1 and increase the boto timeout to 10m:: @@ -349,6 +343,7 @@ def task(ctx, config): - rgw: [client.1] - s3tests: client.0: + force-branch: ceph-nautilus rgw_server: client.1 idle_timeout: 600 @@ -359,8 +354,10 @@ def task(ctx, config): - rgw: [client.0] - s3tests: client.0: + force-branch: ceph-nautilus extra_args: ['test_s3:test_object_acl_grand_public_read'] client.1: + force-branch: ceph-nautilus extra_args: ['--exclude', 'test_100_continue'] """ assert hasattr(ctx, 'rgw'), 's3tests must run after the rgw task'