qa/rgw: add teuthology support for target_by_bucket cloud transition

Add cloud_target_by_bucket and cloud_target_by_bucket_prefix options
to rgw_cloudtier.py and s3tests.py. Create new test suite to run
target_by_bucket-specific s3-tests.

Signed-off-by: Matthew N. Heler <matthew.heler@hotmail.com>

diff --git a/qa/suites/rgw/cloud-transition/tasks/target_by_bucket/.qa b/qa/suites/rgw/cloud-transition/tasks/target_by_bucket/.qa
new file mode 120000 (symlink)
index 0000000..a602a03
--- /dev/null
+++ b/qa/suites/rgw/cloud-transition/tasks/target_by_bucket/.qa
@@ -0,0 +1 @@
+../.qa/
\ No newline at end of file

diff --git a/qa/suites/rgw/cloud-transition/tasks/target_by_bucket/cloud_target_by_bucket_s3tests.yaml b/qa/suites/rgw/cloud-transition/tasks/target_by_bucket/cloud_target_by_bucket_s3tests.yaml
new file mode 100644 (file)
index 0000000..b668780
--- /dev/null
+++ b/qa/suites/rgw/cloud-transition/tasks/target_by_bucket/cloud_target_by_bucket_s3tests.yaml
@@ -0,0 +1,33 @@
+tasks:
+- install:
+- ceph:
+- rgw:
+    client.0:
+      port: 8000
+    client.1:
+      port: 8001
+- rgw-cloudtier:
+    client.1:
+    client.0:
+      cloud_storage_class: CLOUDTIER-CLIENT0
+      cloud_client: client.1
+      cloud_regular_storage_class: LUKEWARM
+      cloud_target_storage_class: FROZEN
+      cloud_retain_head_object: "true"
+      cloud_target_by_bucket: "true"
+      cloud_target_by_bucket_prefix: "rgwx-${storage_class}-${bucket}"
+      cloud_allow_read_through: "true"
+      cloud_read_through_restore_days: "5"
+      cloudtier_user:
+        cloud_secret: "abcefgh"
+        cloud_access_key: "12345678"
+- tox: [client.0]
+- s3tests:
+    client.0:
+      rgw_server: client.0
+      extra_attrs: ["target_by_bucket"]
+      lc_debug_interval: 10
+      rgw_restore_debug_interval: 20
+      rgw_restore_processor_period: 10
+      lifecycle_tests: True
+      cloudtier_tests: True

diff --git a/qa/tasks/rgw_cloudtier.py b/qa/tasks/rgw_cloudtier.py
index 2bbc12f90b5d81c6c4cd6b698c7630c33ba595f9..f40f3229838a969edb470fe3aa9556fbeb3290dd 100644 (file)
--- a/qa/tasks/rgw_cloudtier.py
+++ b/qa/tasks/rgw_cloudtier.py
@@ -30,6 +30,8 @@ class RGWCloudTier(Task):
               cloud_target_path:
               cloud_allow_read_through:
               cloud_read_through_restore_days:
+              cloud_target_by_bucket:
+              cloud_target_by_bucket_prefix:
               cloudtier_user:
                 cloud_secret:
                 cloud_access_key:
@@ -38,6 +40,24 @@ class RGWCloudTier(Task):
     def __init__(self, ctx, config):
         super(RGWCloudTier, self).__init__(ctx, config)
 
+    @staticmethod
+    def _normalize_bool_option(value, option_name):
+        """
+        Normalize a boolean option to 'true' or 'false' string.
+        RGW parses these as case-sensitive s == "true".
+        """
+        if value is None:
+            return None
+        if isinstance(value, bool):
+            return "true" if value else "false"
+        if isinstance(value, str):
+            normalized = value.strip().lower()
+            if normalized not in ("true", "false"):
+                raise ConfigError(
+                    f"rgw-cloudtier: {option_name} must be 'true' or 'false', got '{value}'")
+            return normalized
+        raise ConfigError(f"rgw-cloudtier: {option_name} must be a boolean or string, got {type(value)}")
+
     def setup(self):
         super(RGWCloudTier, self).setup()
 
@@ -66,6 +86,9 @@ class RGWCloudTier(Task):
                 cloud_retain_head_object = client_config.get('cloud_retain_head_object')
                 cloud_allow_read_through = client_config.get('cloud_allow_read_through')
                 cloud_read_through_restore_days = client_config.get('cloud_read_through_restore_days')
+                cloud_target_by_bucket = self._normalize_bool_option(
+                    client_config.get('cloud_target_by_bucket'), 'cloud_target_by_bucket')
+                cloud_target_by_bucket_prefix = client_config.get('cloud_target_by_bucket_prefix')
 
                 cloudtier_user = client_config.get('cloudtier_user')
                 cloud_access_key = cloudtier_user.get('cloud_access_key')
@@ -91,6 +114,10 @@ class RGWCloudTier(Task):
                     tier_config_params += ",allow_read_through=" + cloud_allow_read_through
                 if (cloud_read_through_restore_days != None):
                     tier_config_params += ",read_through_restore_days=" + cloud_read_through_restore_days
+                if (cloud_target_by_bucket != None):
+                    tier_config_params += ",target_by_bucket=" + cloud_target_by_bucket
+                if (cloud_target_by_bucket_prefix != None):
+                    tier_config_params += ",target_by_bucket_prefix=" + cloud_target_by_bucket_prefix
 
                 log.info('Configuring cloud-s3 tier storage class type = %s', cloud_storage_class)
 

diff --git a/qa/tasks/s3tests.py b/qa/tasks/s3tests.py
index 93d370251c14c7e38789836b132dbc02e1d55213..2ef5d802eb2ca5e9761f78ff8639df1bd6cfcf37 100644 (file)
--- a/qa/tasks/s3tests.py
+++ b/qa/tasks/s3tests.py
@@ -403,6 +403,12 @@ def configure(ctx, config):
                 cloud_read_through_restore_days = client_rgw_config.get('cloud_read_through_restore_days')
                 if (cloud_read_through_restore_days != None):
                     s3tests_conf['s3 cloud']['read_through_restore_days'] = cloud_read_through_restore_days
+                cloud_target_by_bucket = client_rgw_config.get('cloud_target_by_bucket')
+                if (cloud_target_by_bucket != None):
+                    s3tests_conf['s3 cloud']['target_by_bucket'] = cloud_target_by_bucket
+                cloud_target_by_bucket_prefix = client_rgw_config.get('cloud_target_by_bucket_prefix')
+                if (cloud_target_by_bucket_prefix != None):
+                    s3tests_conf['s3 cloud']['target_by_bucket_prefix'] = cloud_target_by_bucket_prefix
 
         (remote,) = ctx.cluster.only(client).remotes.keys()
         conf_fp = BytesIO()