qa/workunits/rgw: add tests that reproduce bucket stats inconsistency bugs

Signed-off-by: Cory Snyder <csnyder@1111systems.com>
(cherry picked from commit b79dcf640ac2cc3dacf1b87bbe351db823c445d0)

diff --git a/qa/workunits/rgw/common.py b/qa/workunits/rgw/common.py
index 235c36c9521f88d3bef0cb22d7a2a64ccf2ce48a..2c9c5d035786d347c65bd5b39f003ec3e14b20cf 100755 (executable)
--- a/qa/workunits/rgw/common.py
+++ b/qa/workunits/rgw/common.py
@@ -5,6 +5,9 @@ import subprocess
 import logging as log
 import boto3
 import botocore.exceptions
+import random
+import json
+from time import sleep
 
 log.basicConfig(format = '%(message)s', level=log.DEBUG)
 log.getLogger('botocore').setLevel(log.CRITICAL)
@@ -55,3 +58,46 @@ def boto_connect(access_key, secret_key, config=None):
         except botocore.exceptions.ConnectionError:
             # retry with ssl
             return try_connect('443', True, 'https')
+
+def put_objects(bucket, key_list):
+    objs = []
+    for key in key_list:
+        o = bucket.put_object(Key=key, Body=b"some_data")
+        objs.append((o.key, o.version_id))
+    return objs
+
+def create_unlinked_objects(conn, bucket, key_list):
+    # creates an unlinked/unlistable object for each key in key_list
+    
+    object_versions = []
+    try:
+        exec_cmd('ceph config set client rgw_debug_inject_set_olh_err 2')
+        exec_cmd('ceph config set client rgw_debug_inject_olh_cancel_modification_err true')
+        sleep(1)
+        for key in key_list:
+            tag = str(random.randint(0, 1_000_000))
+            try:
+                bucket.put_object(Key=key, Body=b"some_data", Metadata = {
+                    'tag': tag,
+                })
+            except Exception as e:
+                log.debug(e)
+            out = exec_cmd(f'radosgw-admin bi list --bucket {bucket.name} --object {key}')
+            instance_entries = filter(
+                lambda x: x['type'] == 'instance',
+                json.loads(out.replace(b'\x80', b'0x80')))
+            found = False
+            for ie in instance_entries:
+                instance_id = ie['entry']['instance']
+                ov = conn.ObjectVersion(bucket.name, key, instance_id).head()
+                if ov['Metadata'] and ov['Metadata']['tag'] == tag:
+                    object_versions.append((key, instance_id))
+                    found = True
+                    break
+            if not found:
+                raise Exception(f'failed to create unlinked object for key={key}')
+    finally:
+        exec_cmd('ceph config rm client rgw_debug_inject_set_olh_err')
+        exec_cmd('ceph config rm client rgw_debug_inject_olh_cancel_modification_err')
+    return object_versions
+

diff --git a/qa/workunits/rgw/test_rgw_bucket_check.py b/qa/workunits/rgw/test_rgw_bucket_check.py
index 4b4f776e8488f124edf0f007d121b9a62aaf4405..0a0e084dd9be73d14b13fb6afdcda96e0b026907 100755 (executable)
--- a/qa/workunits/rgw/test_rgw_bucket_check.py
+++ b/qa/workunits/rgw/test_rgw_bucket_check.py
@@ -2,10 +2,8 @@
 
 import logging as log
 import json
-import random
 import botocore
-from common import exec_cmd, create_user, boto_connect
-from time import sleep
+from common import exec_cmd, create_user, boto_connect, put_objects, create_unlinked_objects
 from botocore.config import Config
 
 """
@@ -25,48 +23,6 @@ ACCESS_KEY = 'OJODXSLNX4LUNHQG99PA'
 SECRET_KEY = '3l6ffld34qaymfomuh832j94738aie2x4p2o8h6n'
 BUCKET_NAME = 'check-bucket'
 
-def put_objects(bucket, key_list):
-    objs = []
-    for key in key_list:
-        o = bucket.put_object(Key=key, Body=b"some_data")
-        objs.append((o.key, o.version_id))
-    return objs
-
-def create_unlinked_objects(conn, bucket, key_list):
-    # creates an unlinked/unlistable object for each key in key_list
-    
-    object_versions = []
-    try:
-        exec_cmd('ceph config set client rgw_debug_inject_set_olh_err 2')
-        exec_cmd('ceph config set client rgw_debug_inject_olh_cancel_modification_err true')
-        sleep(1)
-        for key in key_list:
-            tag = str(random.randint(0, 1_000_000))
-            try:
-                bucket.put_object(Key=key, Body=b"some_data", Metadata = {
-                    'tag': tag,
-                })
-            except Exception as e:
-                log.debug(e)
-            out = exec_cmd(f'radosgw-admin bi list --bucket {bucket.name} --object {key}')
-            instance_entries = filter(
-                lambda x: x['type'] == 'instance',
-                json.loads(out.replace(b'\x80', b'0x80')))
-            found = False
-            for ie in instance_entries:
-                instance_id = ie['entry']['instance']
-                ov = conn.ObjectVersion(bucket.name, key, instance_id).head()
-                if ov['Metadata'] and ov['Metadata']['tag'] == tag:
-                    object_versions.append((key, instance_id))
-                    found = True
-                    break
-            if not found:
-                raise Exception(f'failed to create unlinked object for key={key}')
-    finally:
-        exec_cmd('ceph config rm client rgw_debug_inject_set_olh_err')
-        exec_cmd('ceph config rm client rgw_debug_inject_olh_cancel_modification_err')
-    return object_versions
-
 def main():
     """
     execute bucket check commands
@@ -200,6 +156,27 @@ def main():
     for key in null_version_keys:
         assert (key, 'null') in all_versions
 
+    # TESTCASE 'bucket check stats are correct in the presence of unlinked entries'
+    log.debug('TEST: bucket check stats are correct in the presence of unlinked entries\n')
+    bucket.object_versions.all().delete()
+    null_version_objs = put_objects(bucket, null_version_keys)
+    ok_objs = put_objects(bucket, ok_keys)
+    unlinked_objs = create_unlinked_objects(connection, bucket, unlinked_keys)
+    exec_cmd(f'radosgw-admin bucket check --fix --bucket {BUCKET_NAME}')
+    out = exec_cmd(f'radosgw-admin bucket check unlinked --bucket {BUCKET_NAME} --fix --min-age-hours 0 --rgw-olh-pending-timeout-sec 0 --dump-keys')
+    json_out = json.loads(out)
+    assert len(json_out) == len(unlinked_keys)
+    bucket.object_versions.all().delete()
+    out = exec_cmd(f'radosgw-admin bucket stats --bucket {BUCKET_NAME}')
+    json_out = json.loads(out)
+    log.debug(json_out['usage'])
+    assert json_out['usage']['rgw.main']['size'] == 0
+    assert json_out['usage']['rgw.main']['num_objects'] == 0
+    assert json_out['usage']['rgw.main']['size_actual'] == 0
+    assert json_out['usage']['rgw.main']['size_kb'] == 0
+    assert json_out['usage']['rgw.main']['size_kb_actual'] == 0
+    assert json_out['usage']['rgw.main']['size_kb_utilized'] == 0
+
     # Clean up
     log.debug("Deleting bucket {}".format(BUCKET_NAME))
     bucket.object_versions.all().delete()

diff --git a/qa/workunits/rgw/test_rgw_reshard.py b/qa/workunits/rgw/test_rgw_reshard.py
index 842f70da14b59d813d8677c565b81b22258e068f..eed6fc75e2375ccd9814ac3e90b417caff31c852 100755 (executable)
--- a/qa/workunits/rgw/test_rgw_reshard.py
+++ b/qa/workunits/rgw/test_rgw_reshard.py
@@ -4,7 +4,7 @@ import time
 import logging as log
 import json
 import os
-from common import exec_cmd, boto_connect, create_user
+from common import exec_cmd, boto_connect, create_user, put_objects, create_unlinked_objects
 
 """
 Rgw manual and dynamic resharding  testing against a running instance
@@ -82,7 +82,7 @@ def main():
     bucket1 = connection.create_bucket(Bucket=BUCKET_NAME1)
     bucket2 = connection.create_bucket(Bucket=BUCKET_NAME2)
     ver_bucket = connection.create_bucket(Bucket=VER_BUCKET_NAME)
-    connection.BucketVersioning('ver_bucket')
+    connection.BucketVersioning(VER_BUCKET_NAME).enable()
 
     bucket_stats1 = get_bucket_stats(BUCKET_NAME1)
     bucket_stats2 = get_bucket_stats(BUCKET_NAME2)
@@ -199,6 +199,28 @@ def main():
     json_op = json.loads(cmd)
     assert len(json_op) == 0
 
+    # TESTCASE 'check that bucket stats are correct after reshard with unlinked entries'
+    log.debug('TEST: check that bucket stats are correct after reshard with unlinked entries\n')
+    ver_bucket.object_versions.all().delete()
+    ok_keys = ['a', 'b', 'c']
+    unlinked_keys = ['x', 'y', 'z']
+    put_objects(ver_bucket, ok_keys)
+    create_unlinked_objects(connection, ver_bucket, unlinked_keys)
+    cmd = exec_cmd(f'radosgw-admin bucket reshard --bucket {VER_BUCKET_NAME} --num-shards 17 --yes-i-really-mean-it')
+    out = exec_cmd(f'radosgw-admin bucket check unlinked --bucket {VER_BUCKET_NAME} --fix --min-age-hours 0 --rgw-olh-pending-timeout-sec 0 --dump-keys')
+    json_out = json.loads(out)
+    assert len(json_out) == len(unlinked_keys)
+    ver_bucket.object_versions.all().delete()
+    out = exec_cmd(f'radosgw-admin bucket stats --bucket {VER_BUCKET_NAME}')
+    json_out = json.loads(out)
+    log.debug(json_out['usage'])
+    assert json_out['usage']['rgw.main']['size'] == 0
+    assert json_out['usage']['rgw.main']['num_objects'] == 0
+    assert json_out['usage']['rgw.main']['size_actual'] == 0
+    assert json_out['usage']['rgw.main']['size_kb'] == 0
+    assert json_out['usage']['rgw.main']['size_kb_actual'] == 0
+    assert json_out['usage']['rgw.main']['size_kb_utilized'] == 0
+
     # Clean up
     log.debug("Deleting bucket %s", BUCKET_NAME1)
     bucket1.objects.all().delete()