From: Jiffin Tony Thottan Date: Tue, 18 Feb 2025 07:34:00 +0000 (+0530) Subject: qa/rgw: add new teuthology for cloud restore s3 tests X-Git-Tag: v20.3.0~454^2 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=refs%2Fpull%2F61930%2Fhead;p=ceph.git qa/rgw: add new teuthology for cloud restore s3 tests Signed-off-by: Jiffin Tony Thottan --- diff --git a/qa/suites/rgw/cloud-transition/% b/qa/suites/rgw/cloud-transition/% new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/qa/suites/rgw/cloud-transition/+ b/qa/suites/rgw/cloud-transition/+ deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/qa/suites/rgw/cloud-transition/overrides.yaml b/qa/suites/rgw/cloud-transition/overrides.yaml index c0e021e71594..ed7dc40d19b0 100644 --- a/qa/suites/rgw/cloud-transition/overrides.yaml +++ b/qa/suites/rgw/cloud-transition/overrides.yaml @@ -11,6 +11,7 @@ overrides: rgw sts key: abcdefghijklmnop rgw s3 auth use sts: true rgw lc debug interval: 10 + rgw_restore_debug_interval: 20 rgw: storage classes: LUKEWARM, FROZEN frontend: beast diff --git a/qa/suites/rgw/cloud-transition/tasks/cloud_transition_s3tests.yaml b/qa/suites/rgw/cloud-transition/tasks/cloud_transition_s3tests.yaml deleted file mode 100644 index 39023173fcef..000000000000 --- a/qa/suites/rgw/cloud-transition/tasks/cloud_transition_s3tests.yaml +++ /dev/null @@ -1,63 +0,0 @@ -tasks: -- install: -- ceph: -- rgw: - storage classes: LUKEWARM, FROZEN - client.0: - port: 8000 - client.1: - port: 8001 - #client.2: - #port: 8002 - #client.3: - #port: 8003 -- rgw-cloudtier: - client.0: - # cloudtier storage class params - # retain_head_object = false - cloud_storage_class: CLOUDTIER-CLIENT0 - cloud_client: client.1 - cloud_regular_storage_class: LUKEWARM - cloud_target_storage_class: FROZEN - cloud_retain_head_object: "false" - cloud_target_path: "teuthology-client0" - cloudtier_user: - # cloud-user creds to be created on cloud-client - cloud_secret: "abcefgh" - cloud_access_key: "12345678" - #client.2: - # cloudtier storage class params - # retain_head_object = true - # - # Having multiple cloudtier storage classes - # in the same task is increasing the transition - # time and resulting in spurious failures. - # Hence disabling this until there is a - # consistent way of running the tests - # without having to depend on lc_debug_interval. - # - #cloud_storage_class: CLOUDTIER-CLIENT2 - #cloud_client: client.3 - #cloud_regular_storage_class: LUKEWARM - #cloud_retain_head_object: "true" - #cloud_target_path: "teuthology-client2" - #cloudtier_user: - # cloud-user creds to be created on cloud-client - #cloud_secret: "foobar" - #cloud_access_key: "87654321" -- tox: [client.0] -- s3tests: - client.0: - rgw_server: client.0 - storage classes: LUKEWARM, FROZEN - extra_attrs: ["cloud_transition"] - lc_debug_interval: 10 - lifecycle_tests: True - cloudtier_tests: True - #client.2: - #force-branch: ceph-master - #rgw_server: client.2 - #storage classes: LUKEWARM, FROZEN - #extra_attrs: ["cloud_transition"] - #lc_debug_interval: 10 - #cloudtier_tests: True diff --git a/qa/suites/rgw/cloud-transition/tasks/restore/cloud_restore_s3tests.yaml b/qa/suites/rgw/cloud-transition/tasks/restore/cloud_restore_s3tests.yaml new file mode 100644 index 000000000000..c0c3a02449c3 --- /dev/null +++ b/qa/suites/rgw/cloud-transition/tasks/restore/cloud_restore_s3tests.yaml @@ -0,0 +1,33 @@ +tasks: +- install: +- ceph: +- rgw: + storage classes: LUKEWARM, FROZEN + client.0: + port: 8000 + client.1: + port: 8001 +- rgw-cloudtier: + client.0: + cloud_storage_class: CLOUDTIER-CLIENT0 + cloud_client: client.1 + cloud_regular_storage_class: LUKEWARM + cloud_target_storage_class: FROZEN + cloud_retain_head_object: "true" + cloud_target_path: "teuthology-client0" + cloud_allow_read_through: "true" + cloud_read_through_restore_days: "1" + cloudtier_user: + # cloud-user creds to be created on cloud-client + cloud_secret: "abcefgh" + cloud_access_key: "12345678" +- tox: [client.0] +- s3tests: + client.0: + rgw_server: client.0 + storage classes: LUKEWARM, FROZEN + extra_attrs: ["cloud_restore"] + lc_debug_interval: 10 + rgw_restore_debug_interval: 20 + lifecycle_tests: True + cloudtier_tests: True diff --git a/qa/suites/rgw/cloud-transition/tasks/transition/cloud_transition_s3tests.yaml b/qa/suites/rgw/cloud-transition/tasks/transition/cloud_transition_s3tests.yaml new file mode 100644 index 000000000000..39023173fcef --- /dev/null +++ b/qa/suites/rgw/cloud-transition/tasks/transition/cloud_transition_s3tests.yaml @@ -0,0 +1,63 @@ +tasks: +- install: +- ceph: +- rgw: + storage classes: LUKEWARM, FROZEN + client.0: + port: 8000 + client.1: + port: 8001 + #client.2: + #port: 8002 + #client.3: + #port: 8003 +- rgw-cloudtier: + client.0: + # cloudtier storage class params + # retain_head_object = false + cloud_storage_class: CLOUDTIER-CLIENT0 + cloud_client: client.1 + cloud_regular_storage_class: LUKEWARM + cloud_target_storage_class: FROZEN + cloud_retain_head_object: "false" + cloud_target_path: "teuthology-client0" + cloudtier_user: + # cloud-user creds to be created on cloud-client + cloud_secret: "abcefgh" + cloud_access_key: "12345678" + #client.2: + # cloudtier storage class params + # retain_head_object = true + # + # Having multiple cloudtier storage classes + # in the same task is increasing the transition + # time and resulting in spurious failures. + # Hence disabling this until there is a + # consistent way of running the tests + # without having to depend on lc_debug_interval. + # + #cloud_storage_class: CLOUDTIER-CLIENT2 + #cloud_client: client.3 + #cloud_regular_storage_class: LUKEWARM + #cloud_retain_head_object: "true" + #cloud_target_path: "teuthology-client2" + #cloudtier_user: + # cloud-user creds to be created on cloud-client + #cloud_secret: "foobar" + #cloud_access_key: "87654321" +- tox: [client.0] +- s3tests: + client.0: + rgw_server: client.0 + storage classes: LUKEWARM, FROZEN + extra_attrs: ["cloud_transition"] + lc_debug_interval: 10 + lifecycle_tests: True + cloudtier_tests: True + #client.2: + #force-branch: ceph-master + #rgw_server: client.2 + #storage classes: LUKEWARM, FROZEN + #extra_attrs: ["cloud_transition"] + #lc_debug_interval: 10 + #cloudtier_tests: True diff --git a/qa/tasks/rgw_cloudtier.py b/qa/tasks/rgw_cloudtier.py index 88f7d0dc285a..dabcbd9528fd 100644 --- a/qa/tasks/rgw_cloudtier.py +++ b/qa/tasks/rgw_cloudtier.py @@ -28,6 +28,8 @@ class RGWCloudTier(Task): cloud_target_storage_class: cloud_retain_head_object: cloud_target_path: + cloud_allow_read_through: + cloud_read_through_restore_days: cloudtier_user: cloud_secret: cloud_access_key: @@ -64,6 +66,8 @@ class RGWCloudTier(Task): cloud_target_path = client_config.get('cloud_target_path') cloud_target_storage_class = client_config.get('cloud_target_storage_class') cloud_retain_head_object = client_config.get('cloud_retain_head_object') + cloud_allow_read_through = client_config.get('cloud_allow_read_through') + cloud_read_through_restore_days = client_config.get('cloud_read_through_restore_days') cloudtier_user = client_config.get('cloudtier_user') cloud_access_key = cloudtier_user.get('cloud_access_key') @@ -85,6 +89,10 @@ class RGWCloudTier(Task): tier_config_params += ",target_path=" + cloud_target_path if (cloud_target_storage_class != None): tier_config_params += ",target_storage_class=" + cloud_target_storage_class + if (cloud_allow_read_through != None): + tier_config_params += ",allow_read_through=" + cloud_allow_read_through + if (cloud_read_through_restore_days != None): + tier_config_params += ",read_through_restore_days=" + cloud_read_through_restore_days log.info('Configuring cloud-s3 tier storage class type = %s', cloud_storage_class) diff --git a/qa/tasks/s3tests.py b/qa/tasks/s3tests.py index 85ab97d23cd2..e1f27df7d6f6 100644 --- a/qa/tasks/s3tests.py +++ b/qa/tasks/s3tests.py @@ -367,6 +367,10 @@ def configure(ctx, config): if lc_debug_interval: s3tests_conf['s3 main']['lc_debug_interval'] = lc_debug_interval + rgw_restore_debug_interval = properties.get('rgw_restore_debug_interval') + if rgw_restore_debug_interval: + s3tests_conf['s3 main']['rgw_restore_debug_interval'] = rgw_restore_debug_interval + if ctx.rgw_cloudtier is not None: log.info(' ctx.rgw_cloudtier config is %s ...', ctx.rgw_cloudtier.config) client_rgw_config = ctx.rgw_cloudtier.config.get(client) @@ -388,6 +392,12 @@ def configure(ctx, config): s3tests_conf['s3 cloud']['target_path'] = cloud_target_path if (cloud_target_storage_class != None): s3tests_conf['s3 cloud']['target_storage_class'] = cloud_target_storage_class + cloud_allow_read_through = client_rgw_config.get('cloud_allow_read_through') + if (cloud_allow_read_through != None): + s3tests_conf['s3 cloud']['allow_read_through'] = cloud_allow_read_through + cloud_read_through_restore_days = client_rgw_config.get('cloud_read_through_restore_days') + if (cloud_read_through_restore_days != None): + s3tests_conf['s3 cloud']['read_through_restore_days'] = cloud_read_through_restore_days (remote,) = ctx.cluster.only(client).remotes.keys() conf_fp = BytesIO()