From: Yehuda Sadeh Date: Tue, 2 May 2017 20:57:50 +0000 (-0700) Subject: rgw/test_multi: add tests for different key types X-Git-Tag: ses5-milestone6~9^2~3^2~18 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=131f6a3cfb1910fdb99995a7fc6d1778f76f3955;p=ceph.git rgw/test_multi: add tests for different key types add int and date tests Signed-off-by: Yehuda Sadeh --- diff --git a/src/test/rgw/rgw_multi/tests_es.py b/src/test/rgw/rgw_multi/tests_es.py index 9b3ec4fda40c..4d50a7828ba0 100644 --- a/src/test/rgw/rgw_multi/tests_es.py +++ b/src/test/rgw/rgw_multi/tests_es.py @@ -5,6 +5,9 @@ import logging import boto import boto.s3.connection +import datetime +import dateutil + from nose.tools import eq_ as eq from rgw_multi.multisite import * @@ -55,34 +58,31 @@ def do_check_mdsearch(conn, bucket, src_keys, req_str, src_filter): result_keys = req.search(sort_key = lambda k: (k.bucket.name, k.name, k.version_id)) verify_search(bucket_name, src_keys, result_keys, src_filter) -def test_es_object_search(): +def init_env(create_obj, num_keys = 5, buckets_per_zone = 1, bucket_init_cb = None): check_es_configured() realm = get_realm() zonegroup = realm.master_zonegroup() zonegroup_conns = ZonegroupConns(zonegroup) - buckets, zone_bucket = create_bucket_per_zone(zonegroup_conns, buckets_per_zone = 2) + buckets, zone_bucket = create_bucket_per_zone(zonegroup_conns, buckets_per_zone = buckets_per_zone) - min_size = 10 - content = 'a' * min_size + if bucket_init_cb: + for zone_conn, bucket in zone_bucket: + bucket_init_cb(zone_conn, bucket) src_keys = [] owner = None - max_keys = 5 - - etags = [] - names = [] - obj_prefix=''.join(random.choice(string.ascii_lowercase) for _ in range(6)) # don't wait for meta sync just yet for zone, bucket in zone_bucket: - for count in xrange(0, max_keys): + for count in xrange(0, num_keys): objname = obj_prefix + str(count) k = new_key(zone, bucket.name, objname) - k.set_contents_from_string(content + 'x' * count) + # k.set_contents_from_string(content + 'x' * count) + create_obj(k, count) if not owner: for list_key in bucket.list_versions(): @@ -93,9 +93,6 @@ def test_es_object_search(): k.owner = owner # owner is not set when doing get_key() src_keys.append(k) - names.append(k.name) - - max_size = min_size + count - 1 zonegroup_meta_checkpoint(zonegroup) @@ -113,6 +110,17 @@ def test_es_object_search(): for target_conn in targets: zone_bucket_checkpoint(target_conn.zone, source_conn.zone, bucket.name) + return targets, buckets, src_keys + +def test_es_object_search(): + min_size = 10 + content = 'a' * min_size + + def create_obj(k, i): + k.set_contents_from_string(content + 'x' * i) + + targets, buckets, src_keys = init_es_test(create_obj, num_keys = 5, buckets_per_zone = 2) + for target_conn in targets: # bucket checks @@ -136,8 +144,8 @@ def test_es_object_search(): do_check_mdsearch(target_conn.conn, bucket, src_keys , 'name >= ' + key.name, lambda k: k.name >= key.name) do_check_mdsearch(target_conn.conn, bucket, src_keys , 'name > ' + key.name, lambda k: k.name > key.name) - do_check_mdsearch(target_conn.conn, bucket, src_keys , 'name == ' + names[0] + ' or name >= ' + names[2], - lambda k: k.name == names[0] or k.name >= names[2]) + do_check_mdsearch(target_conn.conn, bucket, src_keys , 'name == ' + src_keys[0].name + ' or name >= ' + src_keys[2].name, + lambda k: k.name == src_keys[0].name or k.name >= src_keys[2].name) # check etag for key in src_keys: @@ -159,3 +167,54 @@ def test_es_object_search(): for key in src_keys: do_check_mdsearch(target_conn.conn, bucket, src_keys , 'size > ' + str(key.size), lambda k: k.size > key.size) +def date_from_str(s): + return dateutil.parser.parse(s) + +def test_es_object_search_custom(): + min_size = 10 + content = 'a' * min_size + + def bucket_init(zone_conn, bucket): + req = MDSearchConfig(zone_conn.conn, bucket.name) + req.set_config('x-amz-meta-foo-str; string, x-amz-meta-foo-int; int, x-amz-meta-foo-date; date') + + def create_obj(k, i): + date = datetime.datetime.now() + datetime.timedelta(seconds=1) * i + k.set_contents_from_string(content + 'x' * i, headers = { 'X-Amz-Meta-Foo-Str': str(i * 5), + 'X-Amz-Meta-Foo-Int': str(i * 5), + 'X-Amz-Meta-Foo-Date' : date.isoformat()}) + + targets, buckets, src_keys = init_es_test(create_obj, num_keys = 5, buckets_per_zone = 1, bucket_init_cb = bucket_init) + + + for target_conn in targets: + + # bucket checks + for bucket in buckets: + for key in src_keys: + # check string values + val = key.get_metadata('foo-str') + do_check_mdsearch(target_conn.conn, bucket, src_keys , 'x-amz-meta-foo-str < ' + val, lambda k: k.get_metadata('foo-str') < val) + do_check_mdsearch(target_conn.conn, bucket, src_keys , 'x-amz-meta-foo-str <= ' + val, lambda k: k.get_metadata('foo-str') <= val) + do_check_mdsearch(target_conn.conn, bucket, src_keys , 'x-amz-meta-foo-str == ' + val, lambda k: k.get_metadata('foo-str') == val) + do_check_mdsearch(target_conn.conn, bucket, src_keys , 'x-amz-meta-foo-str >= ' + val, lambda k: k.get_metadata('foo-str') >= val) + do_check_mdsearch(target_conn.conn, bucket, src_keys , 'x-amz-meta-foo-str > ' + val, lambda k: k.get_metadata('foo-str') > val) + + # check int values + sval = key.get_metadata('foo-int') + val = int(sval) + do_check_mdsearch(target_conn.conn, bucket, src_keys , 'x-amz-meta-foo-int < ' + sval, lambda k: int(k.get_metadata('foo-int')) < val) + do_check_mdsearch(target_conn.conn, bucket, src_keys , 'x-amz-meta-foo-int <= ' + sval, lambda k: int(k.get_metadata('foo-int')) <= val) + do_check_mdsearch(target_conn.conn, bucket, src_keys , 'x-amz-meta-foo-int == ' + sval, lambda k: int(k.get_metadata('foo-int')) == val) + do_check_mdsearch(target_conn.conn, bucket, src_keys , 'x-amz-meta-foo-int >= ' + sval, lambda k: int(k.get_metadata('foo-int')) >= val) + do_check_mdsearch(target_conn.conn, bucket, src_keys , 'x-amz-meta-foo-int > ' + sval, lambda k: int(k.get_metadata('foo-int')) > val) + + # check int values + sval = key.get_metadata('foo-date') + val = date_from_str(sval) + do_check_mdsearch(target_conn.conn, bucket, src_keys , 'x-amz-meta-foo-date < ' + sval, lambda k: date_from_str(k.get_metadata('foo-date')) < val) + do_check_mdsearch(target_conn.conn, bucket, src_keys , 'x-amz-meta-foo-date <= ' + sval, lambda k: date_from_str(k.get_metadata('foo-date')) <= val) + do_check_mdsearch(target_conn.conn, bucket, src_keys , 'x-amz-meta-foo-date == ' + sval, lambda k: date_from_str(k.get_metadata('foo-date')) == val) + do_check_mdsearch(target_conn.conn, bucket, src_keys , 'x-amz-meta-foo-date >= ' + sval, lambda k: date_from_str(k.get_metadata('foo-date')) >= val) + do_check_mdsearch(target_conn.conn, bucket, src_keys , 'x-amz-meta-foo-date > ' + sval, lambda k: date_from_str(k.get_metadata('foo-date')) > val) + diff --git a/src/test/rgw/rgw_multi/zone_es.py b/src/test/rgw/rgw_multi/zone_es.py index b58d08fa6702..d89d5edfea24 100644 --- a/src/test/rgw/rgw_multi/zone_es.py +++ b/src/test/rgw/rgw_multi/zone_es.py @@ -105,7 +105,7 @@ class MDSearch: k.metadata = {} for e in entry['CustomMetadata']: - k.metadata[e['Name']] = e['Value'] + k.metadata[e['Name']] = str(e['Value']) # int values will return as int, cast to string for compatibility with object meta response l.append(k) @@ -132,6 +132,29 @@ class MDSearch: return l +class MDSearchConfig: + def __init__(self, conn, bucket_name): + self.conn = conn + self.bucket_name = bucket_name or '' + if bucket_name: + self.bucket = boto.s3.bucket.Bucket(name=bucket_name) + else: + self.bucket = None + + def send_request(self, conf, method): + query_args = 'mdsearch' + headers = { 'X-Amz-Meta-Search': conf } + + result = make_request(self.conn, method, bucket=self.bucket_name, key='', query_args=query_args, headers=headers) + if result.status / 100 != 2: + raise boto.exception.S3ResponseError(result.status, result.reason, result.read()) + + def set_config(self, conf): + self.send_request(conf, 'POST') + + def del_config(self, conf): + self.send_request(conf, 'DELETE') + class ESZoneBucket: def __init__(self, zone_conn, name, conn):