From c013ce5796d45e1870844c0fca4fb6608e69122c Mon Sep 17 00:00:00 2001 From: Cory Snyder Date: Thu, 7 Sep 2023 14:43:23 +0000 Subject: [PATCH] qa/workunits/rgw: add tests that reproduce bucket stats inconsistency bugs Signed-off-by: Cory Snyder (cherry picked from commit b79dcf640ac2cc3dacf1b87bbe351db823c445d0) --- qa/workunits/rgw/common.py | 46 ++++++++++++++++ qa/workunits/rgw/test_rgw_bucket_check.py | 67 ++++++++--------------- qa/workunits/rgw/test_rgw_reshard.py | 26 ++++++++- 3 files changed, 92 insertions(+), 47 deletions(-) diff --git a/qa/workunits/rgw/common.py b/qa/workunits/rgw/common.py index 235c36c9521f8..2c9c5d035786d 100755 --- a/qa/workunits/rgw/common.py +++ b/qa/workunits/rgw/common.py @@ -5,6 +5,9 @@ import subprocess import logging as log import boto3 import botocore.exceptions +import random +import json +from time import sleep log.basicConfig(format = '%(message)s', level=log.DEBUG) log.getLogger('botocore').setLevel(log.CRITICAL) @@ -55,3 +58,46 @@ def boto_connect(access_key, secret_key, config=None): except botocore.exceptions.ConnectionError: # retry with ssl return try_connect('443', True, 'https') + +def put_objects(bucket, key_list): + objs = [] + for key in key_list: + o = bucket.put_object(Key=key, Body=b"some_data") + objs.append((o.key, o.version_id)) + return objs + +def create_unlinked_objects(conn, bucket, key_list): + # creates an unlinked/unlistable object for each key in key_list + + object_versions = [] + try: + exec_cmd('ceph config set client rgw_debug_inject_set_olh_err 2') + exec_cmd('ceph config set client rgw_debug_inject_olh_cancel_modification_err true') + sleep(1) + for key in key_list: + tag = str(random.randint(0, 1_000_000)) + try: + bucket.put_object(Key=key, Body=b"some_data", Metadata = { + 'tag': tag, + }) + except Exception as e: + log.debug(e) + out = exec_cmd(f'radosgw-admin bi list --bucket {bucket.name} --object {key}') + instance_entries = filter( + lambda x: x['type'] == 'instance', + json.loads(out.replace(b'\x80', b'0x80'))) + found = False + for ie in instance_entries: + instance_id = ie['entry']['instance'] + ov = conn.ObjectVersion(bucket.name, key, instance_id).head() + if ov['Metadata'] and ov['Metadata']['tag'] == tag: + object_versions.append((key, instance_id)) + found = True + break + if not found: + raise Exception(f'failed to create unlinked object for key={key}') + finally: + exec_cmd('ceph config rm client rgw_debug_inject_set_olh_err') + exec_cmd('ceph config rm client rgw_debug_inject_olh_cancel_modification_err') + return object_versions + diff --git a/qa/workunits/rgw/test_rgw_bucket_check.py b/qa/workunits/rgw/test_rgw_bucket_check.py index 4b4f776e8488f..0a0e084dd9be7 100755 --- a/qa/workunits/rgw/test_rgw_bucket_check.py +++ b/qa/workunits/rgw/test_rgw_bucket_check.py @@ -2,10 +2,8 @@ import logging as log import json -import random import botocore -from common import exec_cmd, create_user, boto_connect -from time import sleep +from common import exec_cmd, create_user, boto_connect, put_objects, create_unlinked_objects from botocore.config import Config """ @@ -25,48 +23,6 @@ ACCESS_KEY = 'OJODXSLNX4LUNHQG99PA' SECRET_KEY = '3l6ffld34qaymfomuh832j94738aie2x4p2o8h6n' BUCKET_NAME = 'check-bucket' -def put_objects(bucket, key_list): - objs = [] - for key in key_list: - o = bucket.put_object(Key=key, Body=b"some_data") - objs.append((o.key, o.version_id)) - return objs - -def create_unlinked_objects(conn, bucket, key_list): - # creates an unlinked/unlistable object for each key in key_list - - object_versions = [] - try: - exec_cmd('ceph config set client rgw_debug_inject_set_olh_err 2') - exec_cmd('ceph config set client rgw_debug_inject_olh_cancel_modification_err true') - sleep(1) - for key in key_list: - tag = str(random.randint(0, 1_000_000)) - try: - bucket.put_object(Key=key, Body=b"some_data", Metadata = { - 'tag': tag, - }) - except Exception as e: - log.debug(e) - out = exec_cmd(f'radosgw-admin bi list --bucket {bucket.name} --object {key}') - instance_entries = filter( - lambda x: x['type'] == 'instance', - json.loads(out.replace(b'\x80', b'0x80'))) - found = False - for ie in instance_entries: - instance_id = ie['entry']['instance'] - ov = conn.ObjectVersion(bucket.name, key, instance_id).head() - if ov['Metadata'] and ov['Metadata']['tag'] == tag: - object_versions.append((key, instance_id)) - found = True - break - if not found: - raise Exception(f'failed to create unlinked object for key={key}') - finally: - exec_cmd('ceph config rm client rgw_debug_inject_set_olh_err') - exec_cmd('ceph config rm client rgw_debug_inject_olh_cancel_modification_err') - return object_versions - def main(): """ execute bucket check commands @@ -200,6 +156,27 @@ def main(): for key in null_version_keys: assert (key, 'null') in all_versions + # TESTCASE 'bucket check stats are correct in the presence of unlinked entries' + log.debug('TEST: bucket check stats are correct in the presence of unlinked entries\n') + bucket.object_versions.all().delete() + null_version_objs = put_objects(bucket, null_version_keys) + ok_objs = put_objects(bucket, ok_keys) + unlinked_objs = create_unlinked_objects(connection, bucket, unlinked_keys) + exec_cmd(f'radosgw-admin bucket check --fix --bucket {BUCKET_NAME}') + out = exec_cmd(f'radosgw-admin bucket check unlinked --bucket {BUCKET_NAME} --fix --min-age-hours 0 --rgw-olh-pending-timeout-sec 0 --dump-keys') + json_out = json.loads(out) + assert len(json_out) == len(unlinked_keys) + bucket.object_versions.all().delete() + out = exec_cmd(f'radosgw-admin bucket stats --bucket {BUCKET_NAME}') + json_out = json.loads(out) + log.debug(json_out['usage']) + assert json_out['usage']['rgw.main']['size'] == 0 + assert json_out['usage']['rgw.main']['num_objects'] == 0 + assert json_out['usage']['rgw.main']['size_actual'] == 0 + assert json_out['usage']['rgw.main']['size_kb'] == 0 + assert json_out['usage']['rgw.main']['size_kb_actual'] == 0 + assert json_out['usage']['rgw.main']['size_kb_utilized'] == 0 + # Clean up log.debug("Deleting bucket {}".format(BUCKET_NAME)) bucket.object_versions.all().delete() diff --git a/qa/workunits/rgw/test_rgw_reshard.py b/qa/workunits/rgw/test_rgw_reshard.py index 842f70da14b59..eed6fc75e2375 100755 --- a/qa/workunits/rgw/test_rgw_reshard.py +++ b/qa/workunits/rgw/test_rgw_reshard.py @@ -4,7 +4,7 @@ import time import logging as log import json import os -from common import exec_cmd, boto_connect, create_user +from common import exec_cmd, boto_connect, create_user, put_objects, create_unlinked_objects """ Rgw manual and dynamic resharding testing against a running instance @@ -82,7 +82,7 @@ def main(): bucket1 = connection.create_bucket(Bucket=BUCKET_NAME1) bucket2 = connection.create_bucket(Bucket=BUCKET_NAME2) ver_bucket = connection.create_bucket(Bucket=VER_BUCKET_NAME) - connection.BucketVersioning('ver_bucket') + connection.BucketVersioning(VER_BUCKET_NAME).enable() bucket_stats1 = get_bucket_stats(BUCKET_NAME1) bucket_stats2 = get_bucket_stats(BUCKET_NAME2) @@ -199,6 +199,28 @@ def main(): json_op = json.loads(cmd) assert len(json_op) == 0 + # TESTCASE 'check that bucket stats are correct after reshard with unlinked entries' + log.debug('TEST: check that bucket stats are correct after reshard with unlinked entries\n') + ver_bucket.object_versions.all().delete() + ok_keys = ['a', 'b', 'c'] + unlinked_keys = ['x', 'y', 'z'] + put_objects(ver_bucket, ok_keys) + create_unlinked_objects(connection, ver_bucket, unlinked_keys) + cmd = exec_cmd(f'radosgw-admin bucket reshard --bucket {VER_BUCKET_NAME} --num-shards 17 --yes-i-really-mean-it') + out = exec_cmd(f'radosgw-admin bucket check unlinked --bucket {VER_BUCKET_NAME} --fix --min-age-hours 0 --rgw-olh-pending-timeout-sec 0 --dump-keys') + json_out = json.loads(out) + assert len(json_out) == len(unlinked_keys) + ver_bucket.object_versions.all().delete() + out = exec_cmd(f'radosgw-admin bucket stats --bucket {VER_BUCKET_NAME}') + json_out = json.loads(out) + log.debug(json_out['usage']) + assert json_out['usage']['rgw.main']['size'] == 0 + assert json_out['usage']['rgw.main']['num_objects'] == 0 + assert json_out['usage']['rgw.main']['size_actual'] == 0 + assert json_out['usage']['rgw.main']['size_kb'] == 0 + assert json_out['usage']['rgw.main']['size_kb_actual'] == 0 + assert json_out['usage']['rgw.main']['size_kb_utilized'] == 0 + # Clean up log.debug("Deleting bucket %s", BUCKET_NAME1) bucket1.objects.all().delete() -- 2.39.5