qa/rgw: add new teuthology for cloud restore s3 tests

Signed-off-by: Jiffin Tony Thottan <thottanjiffin@gmail.com>

diff --git a/qa/suites/rgw/cloud-transition/% b/qa/suites/rgw/cloud-transition/%
new file mode 100644 (file)
index 0000000..e69de29

diff --git a/qa/suites/rgw/cloud-transition/+ b/qa/suites/rgw/cloud-transition/+
deleted file mode 100644 (file)
index e69de29..0000000

diff --git a/qa/suites/rgw/cloud-transition/overrides.yaml b/qa/suites/rgw/cloud-transition/overrides.yaml
index c0e021e71594e6b413a502cb3b9843ef6eabe01c..ed7dc40d19b097562b5f8aef3c7bbf7437c065e2 100644 (file)
--- a/qa/suites/rgw/cloud-transition/overrides.yaml
+++ b/qa/suites/rgw/cloud-transition/overrides.yaml
@@ -11,6 +11,7 @@ overrides:
         rgw sts key: abcdefghijklmnop
         rgw s3 auth use sts: true
         rgw lc debug interval: 10
+        rgw_restore_debug_interval: 20
   rgw:
     storage classes: LUKEWARM, FROZEN
     frontend: beast

diff --git a/qa/suites/rgw/cloud-transition/tasks/cloud_transition_s3tests.yaml b/qa/suites/rgw/cloud-transition/tasks/cloud_transition_s3tests.yaml
deleted file mode 100644 (file)
index 3902317..0000000
--- a/qa/suites/rgw/cloud-transition/tasks/cloud_transition_s3tests.yaml
+++ /dev/null
@@ -1,63 +0,0 @@
-tasks:
-- install:
-- ceph:
-- rgw:
-    storage classes: LUKEWARM, FROZEN
-    client.0:
-      port: 8000
-    client.1:
-      port: 8001
-    #client.2:
-      #port: 8002
-    #client.3:
-      #port: 8003
-- rgw-cloudtier:
-    client.0:
-      # cloudtier storage class params
-      # retain_head_object = false
-      cloud_storage_class: CLOUDTIER-CLIENT0
-      cloud_client: client.1
-      cloud_regular_storage_class: LUKEWARM
-      cloud_target_storage_class: FROZEN
-      cloud_retain_head_object: "false"
-      cloud_target_path: "teuthology-client0"
-      cloudtier_user:
-        # cloud-user creds to be created on cloud-client
-        cloud_secret: "abcefgh"
-        cloud_access_key: "12345678"
-    #client.2:
-      # cloudtier storage class params
-      # retain_head_object = true
-      #
-      # Having multiple cloudtier storage classes
-      # in the same task is increasing the transition
-      # time and resulting in spurious failures.
-      # Hence disabling this until there is a
-      # consistent way of running the tests
-      # without having to depend on lc_debug_interval.
-      #
-      #cloud_storage_class: CLOUDTIER-CLIENT2
-      #cloud_client: client.3
-      #cloud_regular_storage_class: LUKEWARM
-      #cloud_retain_head_object: "true"
-      #cloud_target_path: "teuthology-client2"
-      #cloudtier_user:
-        # cloud-user creds to be created on cloud-client
-        #cloud_secret: "foobar"
-        #cloud_access_key: "87654321"
-- tox: [client.0]
-- s3tests:
-    client.0:
-      rgw_server: client.0
-      storage classes: LUKEWARM, FROZEN
-      extra_attrs: ["cloud_transition"]
-      lc_debug_interval: 10
-      lifecycle_tests: True
-      cloudtier_tests: True
-    #client.2:
-      #force-branch: ceph-master
-      #rgw_server: client.2
-      #storage classes: LUKEWARM, FROZEN
-      #extra_attrs: ["cloud_transition"]
-      #lc_debug_interval: 10
-      #cloudtier_tests: True

diff --git a/qa/suites/rgw/cloud-transition/tasks/restore/cloud_restore_s3tests.yaml b/qa/suites/rgw/cloud-transition/tasks/restore/cloud_restore_s3tests.yaml
new file mode 100644 (file)
index 0000000..c0c3a02
--- /dev/null
+++ b/qa/suites/rgw/cloud-transition/tasks/restore/cloud_restore_s3tests.yaml
@@ -0,0 +1,33 @@
+tasks:
+- install:
+- ceph:
+- rgw:
+    storage classes: LUKEWARM, FROZEN
+    client.0:
+      port: 8000
+    client.1:
+      port: 8001
+- rgw-cloudtier:
+    client.0:
+      cloud_storage_class: CLOUDTIER-CLIENT0
+      cloud_client: client.1
+      cloud_regular_storage_class: LUKEWARM
+      cloud_target_storage_class: FROZEN
+      cloud_retain_head_object: "true"
+      cloud_target_path: "teuthology-client0"
+      cloud_allow_read_through: "true"
+      cloud_read_through_restore_days: "1"
+      cloudtier_user:
+        # cloud-user creds to be created on cloud-client
+        cloud_secret: "abcefgh"
+        cloud_access_key: "12345678"
+- tox: [client.0]
+- s3tests:
+    client.0:
+      rgw_server: client.0
+      storage classes: LUKEWARM, FROZEN
+      extra_attrs: ["cloud_restore"]
+      lc_debug_interval: 10
+      rgw_restore_debug_interval: 20
+      lifecycle_tests: True
+      cloudtier_tests: True

diff --git a/qa/suites/rgw/cloud-transition/tasks/transition/cloud_transition_s3tests.yaml b/qa/suites/rgw/cloud-transition/tasks/transition/cloud_transition_s3tests.yaml
new file mode 100644 (file)
index 0000000..3902317
--- /dev/null
+++ b/qa/suites/rgw/cloud-transition/tasks/transition/cloud_transition_s3tests.yaml
@@ -0,0 +1,63 @@
+tasks:
+- install:
+- ceph:
+- rgw:
+    storage classes: LUKEWARM, FROZEN
+    client.0:
+      port: 8000
+    client.1:
+      port: 8001
+    #client.2:
+      #port: 8002
+    #client.3:
+      #port: 8003
+- rgw-cloudtier:
+    client.0:
+      # cloudtier storage class params
+      # retain_head_object = false
+      cloud_storage_class: CLOUDTIER-CLIENT0
+      cloud_client: client.1
+      cloud_regular_storage_class: LUKEWARM
+      cloud_target_storage_class: FROZEN
+      cloud_retain_head_object: "false"
+      cloud_target_path: "teuthology-client0"
+      cloudtier_user:
+        # cloud-user creds to be created on cloud-client
+        cloud_secret: "abcefgh"
+        cloud_access_key: "12345678"
+    #client.2:
+      # cloudtier storage class params
+      # retain_head_object = true
+      #
+      # Having multiple cloudtier storage classes
+      # in the same task is increasing the transition
+      # time and resulting in spurious failures.
+      # Hence disabling this until there is a
+      # consistent way of running the tests
+      # without having to depend on lc_debug_interval.
+      #
+      #cloud_storage_class: CLOUDTIER-CLIENT2
+      #cloud_client: client.3
+      #cloud_regular_storage_class: LUKEWARM
+      #cloud_retain_head_object: "true"
+      #cloud_target_path: "teuthology-client2"
+      #cloudtier_user:
+        # cloud-user creds to be created on cloud-client
+        #cloud_secret: "foobar"
+        #cloud_access_key: "87654321"
+- tox: [client.0]
+- s3tests:
+    client.0:
+      rgw_server: client.0
+      storage classes: LUKEWARM, FROZEN
+      extra_attrs: ["cloud_transition"]
+      lc_debug_interval: 10
+      lifecycle_tests: True
+      cloudtier_tests: True
+    #client.2:
+      #force-branch: ceph-master
+      #rgw_server: client.2
+      #storage classes: LUKEWARM, FROZEN
+      #extra_attrs: ["cloud_transition"]
+      #lc_debug_interval: 10
+      #cloudtier_tests: True

diff --git a/qa/tasks/rgw_cloudtier.py b/qa/tasks/rgw_cloudtier.py
index 88f7d0dc285a81824321b9590571754401ec80f6..dabcbd9528fde56f075abe2bf40f7ae6417752ee 100644 (file)
--- a/qa/tasks/rgw_cloudtier.py
+++ b/qa/tasks/rgw_cloudtier.py
@@ -28,6 +28,8 @@ class RGWCloudTier(Task):
               cloud_target_storage_class:
               cloud_retain_head_object:
               cloud_target_path:
+              cloud_allow_read_through:
+              cloud_read_through_restore_days:
               cloudtier_user:
                 cloud_secret:
                 cloud_access_key:
@@ -64,6 +66,8 @@ class RGWCloudTier(Task):
                 cloud_target_path = client_config.get('cloud_target_path')
                 cloud_target_storage_class = client_config.get('cloud_target_storage_class')
                 cloud_retain_head_object = client_config.get('cloud_retain_head_object')
+                cloud_allow_read_through = client_config.get('cloud_allow_read_through')
+                cloud_read_through_restore_days = client_config.get('cloud_read_through_restore_days')
 
                 cloudtier_user = client_config.get('cloudtier_user')
                 cloud_access_key = cloudtier_user.get('cloud_access_key')
@@ -85,6 +89,10 @@ class RGWCloudTier(Task):
                     tier_config_params += ",target_path=" + cloud_target_path
                 if (cloud_target_storage_class != None):
                     tier_config_params += ",target_storage_class=" + cloud_target_storage_class
+                if (cloud_allow_read_through != None):
+                    tier_config_params += ",allow_read_through=" + cloud_allow_read_through
+                if (cloud_read_through_restore_days != None):
+                    tier_config_params += ",read_through_restore_days=" + cloud_read_through_restore_days
 
                 log.info('Configuring cloud-s3 tier storage class type = %s', cloud_storage_class)
 

diff --git a/qa/tasks/s3tests.py b/qa/tasks/s3tests.py
index 85ab97d23cd26c31e02a6204d82954d872042c0d..e1f27df7d6f6e9c61a94abebbf865b5e98373ff5 100644 (file)
--- a/qa/tasks/s3tests.py
+++ b/qa/tasks/s3tests.py
@@ -367,6 +367,10 @@ def configure(ctx, config):
         if lc_debug_interval:
             s3tests_conf['s3 main']['lc_debug_interval'] = lc_debug_interval
 
+        rgw_restore_debug_interval = properties.get('rgw_restore_debug_interval')
+        if rgw_restore_debug_interval:
+            s3tests_conf['s3 main']['rgw_restore_debug_interval'] = rgw_restore_debug_interval
+
         if ctx.rgw_cloudtier is not None:
             log.info(' ctx.rgw_cloudtier config  is %s ...', ctx.rgw_cloudtier.config)
             client_rgw_config = ctx.rgw_cloudtier.config.get(client)
@@ -388,6 +392,12 @@ def configure(ctx, config):
                     s3tests_conf['s3 cloud']['target_path'] = cloud_target_path
                 if (cloud_target_storage_class != None):
                     s3tests_conf['s3 cloud']['target_storage_class'] = cloud_target_storage_class
+                cloud_allow_read_through = client_rgw_config.get('cloud_allow_read_through')
+                if (cloud_allow_read_through != None):
+                    s3tests_conf['s3 cloud']['allow_read_through'] = cloud_allow_read_through
+                cloud_read_through_restore_days = client_rgw_config.get('cloud_read_through_restore_days')
+                if (cloud_read_through_restore_days != None):
+                    s3tests_conf['s3 cloud']['read_through_restore_days'] = cloud_read_through_restore_days
 
         (remote,) = ctx.cluster.only(client).remotes.keys()
         conf_fp = BytesIO()