From 10cbecb87becdfa5f94e70d2f76cabf73c9ff2d0 Mon Sep 17 00:00:00 2001 From: Yehuda Sadeh Date: Wed, 18 Dec 2019 08:58:50 -0800 Subject: [PATCH 1/1] ragweed: python3 string / bytearrat fixes and other related issues Signed-off-by: Yehuda Sadeh --- LICENSE | 19 ++ bootstrap | 44 +++ ragweed-example.conf | 30 ++ ragweed/__init__.py | 1 + ragweed/framework.py | 461 +++++++++++++++++++++++++++++ ragweed/reqs.py | 110 +++++++ ragweed/tests/__init__.py | 0 ragweed/tests/tests.py | 603 ++++++++++++++++++++++++++++++++++++++ requirements.txt | 7 + setup.py | 29 ++ 10 files changed, 1304 insertions(+) create mode 100644 LICENSE create mode 100755 bootstrap create mode 100644 ragweed-example.conf create mode 100644 ragweed/__init__.py create mode 100644 ragweed/framework.py create mode 100644 ragweed/reqs.py create mode 100644 ragweed/tests/__init__.py create mode 100644 ragweed/tests/tests.py create mode 100644 requirements.txt create mode 100644 setup.py diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..afa2b82 --- /dev/null +++ b/LICENSE @@ -0,0 +1,19 @@ +Copyright (c) 2017 Red Hat, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/bootstrap b/bootstrap new file mode 100755 index 0000000..32df76a --- /dev/null +++ b/bootstrap @@ -0,0 +1,44 @@ +#!/bin/sh +set -e + +if [ -f /etc/debian_version ]; then + for package in python3-pip python3-virtualenv python3-dev libevent-dev libxml2-dev libxslt-dev zlib1g-dev; do + if [ "$(dpkg --status -- $package 2>/dev/null|sed -n 's/^Status: //p')" != "install ok installed" ]; then + # add a space after old values + missing="${missing:+$missing }$package" + fi + done + if [ -n "$missing" ]; then + echo "$0: missing required DEB packages. Installing via sudo." 1>&2 + sudo apt-get -y install $missing + fi +fi +if [ -f /etc/redhat-release ]; then + for package in python3-pip python3-virtualenv python3-devel libevent-devel libxml2-devel libxslt-devel zlib-devel; do + if [ "$(rpm -qa $package 2>/dev/null)" == "" ]; then + missing="${missing:+$missing }$package" + fi + done + if [ -n "$missing" ]; then + echo "$0: missing required RPM packages. Installing via sudo." 1>&2 + sudo yum -y install $missing + fi +fi + +virtualenv -p python3 --system-site-packages --distribute virtualenv + +# avoid pip bugs +./virtualenv/bin/pip install --upgrade pip + +# work-around change in pip 1.5 +./virtualenv/bin/pip install six +./virtualenv/bin/pip install -I nose +./virtualenv/bin/pip install setuptools + +./virtualenv/bin/pip install -U -r requirements.txt + +# forbid setuptools from using the network because it'll try to use +# easy_install, and we really wanted pip; next line will fail if pip +# requirements.txt does not match setup.py requirements -- sucky but +# good enough for now +./virtualenv/bin/python setup.py develop diff --git a/ragweed-example.conf b/ragweed-example.conf new file mode 100644 index 0000000..13393c3 --- /dev/null +++ b/ragweed-example.conf @@ -0,0 +1,30 @@ +[DEFAULT] + +[rados] + +ceph_conf = + +[rgw] +## replace with e.g. "localhost" to run against local software +host = + +## uncomment the port to use something other than 80 +port = 8000 + +## say "no" to disable TLS +is_secure = no + +# bucket prefix +bucket_prefix = ragweed + +[fixtures] + +[user regular] +user_id = yehsad +access_key = accesskey1 +secret_key = secret1 + +[user system] +access_key = accesskey2 +secret_key = secret2 + diff --git a/ragweed/__init__.py b/ragweed/__init__.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/ragweed/__init__.py @@ -0,0 +1 @@ + diff --git a/ragweed/framework.py b/ragweed/framework.py new file mode 100644 index 0000000..3dfb3e4 --- /dev/null +++ b/ragweed/framework.py @@ -0,0 +1,461 @@ +import sys +import os +import boto +import boto.s3.connection +import json +import inspect +import pickle +import munch +import yaml +import configparser +from boto.s3.key import Key +from nose.plugins.attrib import attr +from nose.tools import eq_ as eq +import rados + +from .reqs import _make_admin_request + +ragweed_env = None +suite = None + +class RGWConnection: + def __init__(self, access_key, secret_key, host, port, is_secure): + self.host = host + self.port = port + self.is_secure = is_secure + self.conn = boto.connect_s3( + aws_access_key_id = access_key, + aws_secret_access_key = secret_key, + host=host, + port=port, + is_secure=is_secure, + calling_format = boto.s3.connection.OrdinaryCallingFormat(), + ) + + def create_bucket(self, name): + return self.conn.create_bucket(name) + + def get_bucket(self, name, validate=True): + return self.conn.get_bucket(name, validate=validate) + + +class RGWRESTAdmin: + def __init__(self, connection): + self.conn = connection + + def get_resource(self, path, params): + r = _make_admin_request(self.conn, "GET", path, params) + if r.status != 200: + raise boto.exception.S3ResponseError(r.status, r.reason) + return munch.munchify(json.loads(r.read())) + + + def read_meta_key(self, key): + return self.get_resource('/admin/metadata', {'key': key}) + + def get_bucket_entrypoint(self, bucket_name): + return self.read_meta_key('bucket:' + bucket_name) + + def get_bucket_instance_info(self, bucket_name, bucket_id = None): + if not bucket_id: + ep = self.get_bucket_entrypoint(bucket_name) + print(ep) + bucket_id = ep.data.bucket.bucket_id + result = self.read_meta_key('bucket.instance:' + bucket_name + ":" + bucket_id) + return result.data.bucket_info + + def check_bucket_index(self, bucket_name): + return self.get_resource('/admin/bucket',{'index' : None, 'bucket':bucket_name}) + + def get_obj_layout(self, key): + path = '/' + key.bucket.name + '/' + key.name + params = {'layout': None} + if key.version_id is not None: + params['versionId'] = key.version_id + + print(params) + + return self.get_resource(path, params) + + def get_zone_params(self): + return self.get_resource('/admin/config', {'type': 'zone'}) + + +class RSuite: + def __init__(self, name, bucket_prefix, zone, suite_step): + self.name = name + self.bucket_prefix = bucket_prefix + self.zone = zone + self.config_bucket = None + self.rtests = [] + self.do_preparing = False + self.do_check = False + for step in suite_step.split(','): + if step == 'prepare': + self.do_preparing = True + self.config_bucket = self.zone.create_raw_bucket(self.get_bucket_name('conf')) + if step == 'check' or step == 'test': + self.do_check = True + self.config_bucket = self.zone.get_raw_bucket(self.get_bucket_name('conf')) + + def get_bucket_name(self, suffix): + return self.bucket_prefix + '-' + suffix + + def register_test(self, t): + self.rtests.append(t) + + def write_test_data(self, test): + k = Key(self.config_bucket) + k.key = 'tests/' + test._name + k.set_contents_from_string(test.to_json()) + + def read_test_data(self, test): + k = Key(self.config_bucket) + k.key = 'tests/' + test._name + s = k.get_contents_as_string() + print('read_test_data=', s) + test.from_json(s) + + def is_preparing(self): + return self.do_preparing + + def is_checking(self): + return self.do_check + + +class RTestJSONSerialize(json.JSONEncoder): + def default(self, obj): + if isinstance(obj, (list, dict, tuple, str, int, float, bool, type(None))): + return JSONEncoder.default(self, obj) + return {'__pickle': pickle.dumps(obj, 0).decode('utf-8')} + +def rtest_decode_json(d): + if '__pickle' in d: + return pickle.loads(bytearray(d['__pickle'], 'utf-8')) + return d + +class RPlacementRule: + def __init__(self, rule): + r = rule.split('/', 1) + + self.placement_id = r[0] + + if (len(r) == 2): + self.storage_class=r[1] + else: + self.storage_class = 'STANDARD' + + +class RBucket: + def __init__(self, zone, bucket, bucket_info): + self.zone = zone + self.bucket = bucket + self.name = bucket.name + self.bucket_info = bucket_info + + try: + self.placement_rule = RPlacementRule(self.bucket_info.placement_rule) + self.placement_target = self.zone.get_placement_target(self.bucket_info.placement_rule) + except: + pass + + def get_data_pool(self): + try: + # old style explicit pool + explicit_pool = self.bucket_info.bucket.pool + except: + # new style explicit pool + explicit_pool = self.bucket_info.bucket.explicit_placement.data_pool + if explicit_pool is not None and explicit_pool != '': + return explicit_pool + + return self.placement_target.get_data_pool(self.placement_rule) + + + def get_tail_pool(self, obj_layout): + try: + placement_rule = obj_layout.manifest.tail_placement.placement_rule + except: + placement_rule = '' + if placement_rule == '': + try: + # new style + return obj_layout.manifest.tail_placement.bucket.explicit_placement.data_pool + except: + pass + + try: + # old style + return obj_layout.manifest.tail_bucket.pool + except: + pass + + pr = RPlacementRule(placement_rule) + + return self.placement_target.get_data_pool(pr) + +class RStorageClasses: + def __init__(self, config): + if hasattr(config, 'storage_classes'): + self.storage_classes = config.storage_classes + else: + try: + self.storage_classes = munch.munchify({ 'STANDARD': { 'data_pool': config.data_pool }}) + except: + self.storage_classes = None + pass + + def get(self, storage_class): + assert(self.storage_classes != None) + try: + if not storage_class: + storage_class = 'STANDARD' + sc = self.storage_classes[storage_class] + except: + eq('could not find storage class ' + storage_class, 0) + + return sc + + def get_all(self): + for (name, _) in self.storage_classes.items(): + yield name + +class RPlacementTarget: + def __init__(self, name, config): + self.name = name + self.index_pool = config.index_pool + self.data_extra_pool = config.data_extra_pool + self.storage_classes = RStorageClasses(config) + + if not self.data_extra_pool: + self.data_extra_pool = self.storage_classes.get_data_pool('STANDARD') + + def get_data_pool(self, placement_rule): + return self.storage_classes.get(placement_rule.storage_class).data_pool + +class RZone: + def __init__(self, conn): + self.conn = conn + + self.rgw_rest_admin = RGWRESTAdmin(self.conn.system) + self.zone_params = self.rgw_rest_admin.get_zone_params() + + self.placement_targets = {} + + for e in self.zone_params.placement_pools: + self.placement_targets[e.key] = e.val + + print('zone_params:', self.zone_params) + + def get_placement_target(self, placement_id): + plid = placement_id + if placement_id is None or placement_id == '': + print('zone_params=', self.zone_params) + plid = self.zone_params.default_placement + + try: + return RPlacementTarget(plid, self.placement_targets[plid]) + except: + pass + + return None + + def get_default_placement(self): + return get_placement_target(self.zone_params.default_placement) + + def create_bucket(self, name): + bucket = self.create_raw_bucket(name) + bucket_info = self.rgw_rest_admin.get_bucket_instance_info(bucket.name) + print('bucket_info:', bucket_info) + return RBucket(self, bucket, bucket_info) + + def get_bucket(self, name): + bucket = self.get_raw_bucket(name) + bucket_info = self.rgw_rest_admin.get_bucket_instance_info(bucket.name) + print('bucket_info:', bucket_info) + return RBucket(self, bucket, bucket_info) + + def create_raw_bucket(self, name): + return self.conn.regular.create_bucket(name) + + def get_raw_bucket(self, name): + return self.conn.regular.get_bucket(name) + + def refresh_rbucket(self, rbucket): + rbucket.bucket = self.get_raw_bucket(rbucket.bucket.name) + rbucket.bucket_info = self.rgw_rest_admin.get_bucket_instance_info(rbucket.bucket.name) + + +class RTest: + def __init__(self): + self._name = self.__class__.__name__ + self.r_buckets = [] + self.init() + + def create_bucket(self): + bid = len(self.r_buckets) + 1 + bucket_name = suite.get_bucket_name(self._name + '.' + str(bid)) + bucket_name = bucket_name.replace("_", "-") + rb = suite.zone.create_bucket(bucket_name) + self.r_buckets.append(rb) + + return rb + + def get_buckets(self): + for rb in self.r_buckets: + yield rb + + def init(self): + pass + + def prepare(self): + pass + + def check(self): + pass + + def to_json(self): + attrs = {} + for x in dir(self): + if x.startswith('r_'): + attrs[x] = getattr(self, x) + return json.dumps(attrs, cls=RTestJSONSerialize) + + def from_json(self, s): + j = json.loads(s, object_hook=rtest_decode_json) + for e in j: + setattr(self, e, j[e]) + + def save(self): + suite.write_test_data(self) + + def load(self): + suite.read_test_data(self) + for rb in self.r_buckets: + suite.zone.refresh_rbucket(rb) + + def test(self): + suite.register_test(self) + if suite.is_preparing(): + self.prepare() + self.save() + + if suite.is_checking(): + self.load() + self.check() + +def read_config(fp): + config = munch.Munch() + g = yaml.safe_load_all(fp) + for new in g: + print(munch.munchify(new)) + config.update(munch.munchify(new)) + return config + +str_config_opts = [ + 'user_id', + 'access_key', + 'secret_key', + 'host', + 'ceph_conf', + 'bucket_prefix', + ] + +int_config_opts = [ + 'port', + ] + +bool_config_opts = [ + 'is_secure', + ] + +def dict_find(d, k): + if k in d: + return d[k] + return None + +class RagweedEnv: + def __init__(self): + self.config = munch.Munch() + + cfg = configparser.RawConfigParser() + try: + path = os.environ['RAGWEED_CONF'] + except KeyError: + raise RuntimeError( + 'To run tests, point environment ' + + 'variable RAGWEED_CONF to a config file.', + ) + with open(path, 'r') as f: + cfg.readfp(f) + + for section in cfg.sections(): + try: + (section_type, name) = section.split(None, 1) + if not section_type in self.config: + self.config[section_type] = munch.Munch() + self.config[section_type][name] = munch.Munch() + cur = self.config[section_type] + except ValueError: + section_type = '' + name = section + self.config[name] = munch.Munch() + cur = self.config + + cur[name] = munch.Munch() + + for var in str_config_opts: + try: + cur[name][var] = cfg.get(section, var) + except configparser.NoOptionError: + pass + + for var in int_config_opts: + try: + cur[name][var] = cfg.getint(section, var) + except configparser.NoOptionError: + pass + + for var in bool_config_opts: + try: + cur[name][var] = cfg.getboolean(section, var) + except configparser.NoOptionError: + pass + + print(json.dumps(self.config)) + + rgw_conf = self.config.rgw + + try: + self.bucket_prefix = rgw_conf.bucket_prefix + except: + self.bucket_prefix = 'ragweed' + + conn = munch.Munch() + for (k, u) in self.config.user.items(): + conn[k] = RGWConnection(u.access_key, u.secret_key, rgw_conf.host, dict_find(rgw_conf, 'port'), dict_find(rgw_conf, 'is_secure')) + + self.zone = RZone(conn) + self.suite = RSuite('ragweed', self.bucket_prefix, self.zone, os.environ['RAGWEED_STAGES']) + + try: + self.ceph_conf = self.config.rados.ceph_conf + except: + raise RuntimeError( + 'ceph_conf is missing under the [rados] section in ' + os.environ['RAGWEED_CONF'] + ) + + self.rados = rados.Rados(conffile=self.ceph_conf) + self.rados.connect() + + pools = self.rados.list_pools() + + for pool in pools: + print("rados pool>", pool) + +def setup_module(): + global ragweed_env + global suite + + ragweed_env = RagweedEnv() + suite = ragweed_env.suite diff --git a/ragweed/reqs.py b/ragweed/reqs.py new file mode 100644 index 0000000..f066506 --- /dev/null +++ b/ragweed/reqs.py @@ -0,0 +1,110 @@ +import boto.s3.connection +from http.client import HTTPConnection, HTTPSConnection +from urllib.parse import urlparse, urlencode + +def _make_admin_request(conn, method, path, query_dict=None, body=None, response_headers=None, request_headers=None, expires_in=100000, path_style=True, timeout=None): + """ + issue a request for a specified method, on a specified path + with a specified (optional) body (encrypted per the connection), and + return the response (status, reason). + """ + + query = '' + if query_dict is not None: + query = urlencode(query_dict) + + (bucket_str, key_str) = path.split('/', 2)[1:] + bucket = conn.get_bucket(bucket_str, validate=False) + key = bucket.get_key(key_str, validate=False) + + urlobj = None + if key is not None: + urlobj = key + elif bucket is not None: + urlobj = bucket + else: + raise RuntimeError('Unable to find bucket name') + url = urlobj.generate_url(expires_in, method=method, response_headers=response_headers, headers=request_headers) + o = urlparse(url) + req_path = o.path + '?' + o.query + '&' + query + + return _make_raw_request(host=conn.host, port=conn.port, method=method, path=req_path, body=body, request_headers=request_headers, secure=conn.is_secure, timeout=timeout) + +def _make_request(method, bucket, key, body=None, authenticated=False, response_headers=None, request_headers=None, expires_in=100000, path_style=True, timeout=None): + """ + issue a request for a specified method, on a specified , + with a specified (optional) body (encrypted per the connection), and + return the response (status, reason). + + If key is None, then this will be treated as a bucket-level request. + + If the request or response headers are None, then default values will be + provided by later methods. + """ + if not path_style: + conn = bucket.connection + request_headers['Host'] = conn.calling_format.build_host(conn.server_name(), bucket.name) + + if authenticated: + urlobj = None + if key is not None: + urlobj = key + elif bucket is not None: + urlobj = bucket + else: + raise RuntimeError('Unable to find bucket name') + url = urlobj.generate_url(expires_in, method=method, response_headers=response_headers, headers=request_headers) + o = urlparse(url) + path = o.path + '?' + o.query + else: + bucketobj = None + if key is not None: + path = '/{obj}'.format(obj=key.name) + bucketobj = key.bucket + elif bucket is not None: + path = '/' + bucketobj = bucket + else: + raise RuntimeError('Unable to find bucket name') + if path_style: + path = '/{bucket}'.format(bucket=bucketobj.name) + path + + return _make_raw_request(host=s3.main.host, port=s3.main.port, method=method, path=path, body=body, request_headers=request_headers, secure=s3.main.is_secure, timeout=timeout) + +def _make_bucket_request(method, bucket, body=None, authenticated=False, response_headers=None, request_headers=None, expires_in=100000, path_style=True, timeout=None): + """ + issue a request for a specified method, on a specified , + with a specified (optional) body (encrypted per the connection), and + return the response (status, reason) + """ + return _make_request(method=method, bucket=bucket, key=None, body=body, authenticated=authenticated, response_headers=response_headers, request_headers=request_headers, expires_in=expires_in, path_style=path_style, timeout=timeout) + +def _make_raw_request(host, port, method, path, body=None, request_headers=None, secure=False, timeout=None): + """ + issue a request to a specific host & port, for a specified method, on a + specified path with a specified (optional) body (encrypted per the + connection), and return the response (status, reason). + + This allows construction of special cases not covered by the bucket/key to + URL mapping of _make_request/_make_bucket_request. + """ + if secure: + class_ = HTTPSConnection + else: + class_ = HTTPConnection + + if request_headers is None: + request_headers = {} + + c = class_(host, port, timeout=timeout) + + # TODO: We might have to modify this in future if we need to interact with + # how http.client.request handles Accept-Encoding and Host. + c.request(method, path, body=body, headers=request_headers) + + res = c.getresponse() + #c.close() + + return res + + diff --git a/ragweed/tests/__init__.py b/ragweed/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/ragweed/tests/tests.py b/ragweed/tests/tests.py new file mode 100644 index 0000000..c57b2ca --- /dev/null +++ b/ragweed/tests/tests.py @@ -0,0 +1,603 @@ +from io import StringIO +import ragweed.framework +import binascii +import string +import random + + +from ragweed.framework import * + +class obj_placement: + def __init__(self, pool, oid, loc): + self.pool = pool + self.oid = oid + self.loc = loc + +def get_zone(): + return ragweed.framework.ragweed_env.zone + +def rgwa(): + return get_zone().rgw_rest_admin + +def get_pool_ioctx(pool_name): + return ragweed.framework.ragweed_env.rados.open_ioctx(pool_name) + + +def get_placement(obj_json): + try: + return obj_placement(obj_json.pool, obj_json.oid, obj_json.loc) + except: + oid = obj_json.bucket.marker + '_' + obj_json.object + key = '' + if obj_json.key != '': + key = obj_json.bucket.marker + '_' + obj_json.key + return obj_placement(obj_json.bucket.pool, oid, key) + +def validate_obj_location(rbucket, obj): + expected_head_pool = rbucket.get_data_pool() + head_pool_ioctx = get_pool_ioctx(expected_head_pool) + print('expected head pool: ' + expected_head_pool) + + obj_layout = rgwa().get_obj_layout(obj) + + print('layout', obj_layout) + + obj_size = obj_layout.manifest.obj_size + print('obj_size', obj_size) + + print('head', obj_layout.head) + expected_tail_pool = rbucket.get_tail_pool(obj_layout) + tail_pool_ioctx = get_pool_ioctx(expected_tail_pool) + + head_placement = get_placement(obj_layout.head) + + eq(head_placement.pool, expected_head_pool) + + # check rados object for head exists + head_pool_ioctx.set_locator_key(head_placement.loc) + (size, mtime) = head_pool_ioctx.stat(head_placement.oid) + + print('head size:', size, 'mtime:', mtime) + + # check tail + for o in obj_layout.data_location: + print('o=', o) + print('ofs=', o.ofs, 'loc', o.loc) + placement = get_placement(o.loc) + if o.ofs < obj_layout.manifest.head_size: + eq(placement.pool, expected_head_pool) + pool_ioctx = head_pool_ioctx + else: + eq(placement.pool, expected_tail_pool) + pool_ioctx = tail_pool_ioctx + + # validate rados object exists + pool_ioctx.set_locator_key(placement.loc) + (size, mtime) = pool_ioctx.stat(placement.oid) + + if o.ofs + o.loc_size > obj_size: + # radosgw get_obj_layout() request on previous versions could return bigger o.loc_size + # than actual at the end of the object. Fixed in later versions but need to adjust here + # for when running on older versions. + o.loc_size = obj_size - o.ofs + + check_size = o.loc_ofs + o.loc_size + + eq(size, check_size) + +def calc_crc(data): + crc = binascii.crc32(data) + return '{:#010x}'.format(crc) + + +def validate_obj(rbucket, obj_name, expected_crc): + b = rbucket.bucket + + obj = b.get_key(obj_name) + + validate_obj_location(rbucket, obj) + obj_crc = calc_crc(obj.get_contents_as_string()) + eq(obj_crc, expected_crc) + + +def generate_random(size): + """ + Generate random data + (actually each MB is a repetition of the first KB) + """ + chunk = 1024 + allowed = string.ascii_letters + s = '' + strpart = ''.join([allowed[random.randint(0, len(allowed) - 1)] for _ in range(chunk)]) + + for y in range((size + chunk - 1) // chunk): + this_chunk_len = chunk + + if len(s) + this_chunk_len > size: + this_chunk_len = size - len(s) + strpart = strpart[0:this_chunk_len] + + s += strpart + + return s + +def gen_rand_string(size, chars=string.ascii_uppercase + string.digits): + return ''.join(random.choice(chars) for _ in range(size)) + + +# prepare: +# create objects in multiple sizes, with various names +# check: +# verify data correctness +# verify that objects were written to the expected data pool +class r_test_small_obj_data(RTest): + def prepare(self): + self.r_obj_names = [ 'obj', '_', '__', '_ _' ] + self.r_bucket_sizes = {} + + sizes = { 0, 512 * 1024, 1024 * 1024 } + + for size in sizes: + rbucket = self.create_bucket() + self.r_bucket_sizes[rbucket.name] = size + data = '0' * size + for n in self.r_obj_names: + obj = Key(rbucket.bucket) + obj.key = n; + obj.set_contents_from_string(data) + + def check(self): + print(self.r_obj_names) + for rbucket in self.get_buckets(): + size = self.r_bucket_sizes[rbucket.name] + data = '0' * int(size) + + for n in self.r_obj_names: + obj = Key(rbucket.bucket) + obj.key = n; + obj_data = obj.get_contents_as_string() + eq(bytearray(data, 'utf-8'), obj_data) + + validate_obj_location(rbucket, obj) + +class MultipartUploaderState: + def __init__(self, mu): + self.upload_id = mu.mp.id + self.crc = mu.crc + self.cur_part = mu.cur_part + + +class MultipartUploader: + def __init__(self, rbucket, obj_name, size, part_size, storage_class=None, state=None): + self.rbucket = rbucket + self.obj_name = obj_name + self.size = size + self.part_size = part_size + self.storage_class = storage_class + self.crc = 0 + self.cur_part = 0 + + if state is not None: + self.crc = state.crc + self.cur_part = state.cur_part + + for upload in rbucket.bucket.list_multipart_uploads(): + if upload.key_name == self.obj_name and upload.id == state.upload_id: + self.mp = upload + + self.num_full_parts = self.size // self.part_size + self.last_part_size = self.size % self.part_size + + + def prepare(self): + headers = None + if self.storage_class is not None: + if not headers: + headers = {} + headers['X-Amz-Storage-Class'] = self.storage_class + + self.mp = self.rbucket.bucket.initiate_multipart_upload(self.obj_name, headers = headers) + self.crc = 0 + self.cur_part = 0 + + def upload(self): + if self.cur_part > self.num_full_parts: + return False + + if self.cur_part < self.num_full_parts: + payload=gen_rand_string(self.part_size // (1024 * 1024)) * 1024 * 1024 + + self.mp.upload_part_from_file(StringIO(payload), self.cur_part + 1) + self.crc = binascii.crc32(bytearray(payload, 'utf-8'), self.crc) + self.cur_part += 1 + + return True + + + if self.last_part_size > 0: + last_payload='1'*self.last_part_size + + self.mp.upload_part_from_file(StringIO(last_payload), self.num_full_parts + 1) + self.crc = binascii.crc32(bytearray(last_payload, 'utf-8'), self.crc) + self.cur_part += 1 + + return False + + def upload_all(self): + while self.upload(): + pass + + def complete(self): + self.mp.complete_upload() + + def hexdigest(self): + return '{:#010x}'.format(self.crc) + + def get_state(self): + return MultipartUploaderState(self) + + + +# prepare: +# init, upload, and complete a multipart object +# check: +# verify data correctness +# verify that object layout is correct +class r_test_multipart_simple(RTest): + def init(self): + self.obj_size = 18 * 1024 * 1024 + self.part_size = 5 * 1024 * 1024 + + def prepare(self): + rb = self.create_bucket() + self.r_obj = 'foo' + + uploader = MultipartUploader(rb, self.r_obj, self.obj_size, self.part_size) + + uploader.prepare() + uploader.upload_all() + uploader.complete() + + self.r_crc = uploader.hexdigest() + print('written crc: ' + self.r_crc) + + def check(self): + for rb in self.get_buckets(): + break + + k = rb.bucket.get_key(self.r_obj) + eq(k.size, self.obj_size) + + validate_obj(rb, self.r_obj, self.r_crc) + +# prepare: +# init, upload, and complete a multipart object +# check: +# part index is removed +class r_test_multipart_index_versioning(RTest): + def init(self): + self.obj_size = 18 * 1024 * 1024 + self.part_size = 5 * 1024 * 1024 + + def prepare(self): + rb = self.create_bucket() + rb.bucket.configure_versioning('true') + self.r_obj = 'foo' + + uploader = MultipartUploader(rb, self.r_obj, self.obj_size, self.part_size) + + uploader.prepare() + uploader.upload_all() + uploader.complete() + + def check(self): + for rb in self.get_buckets(): + break + index_check_result = rgwa().check_bucket_index(rb.name) + print(index_check_result) + + eq(0, len(index_check_result)) + + +# prepare: +# init, upload multipart object +# check: +# complete multipart +# verify data correctness +# verify that object layout is correct +class r_test_multipart_defer_complete(RTest): + def init(self): + self.obj_size = 18 * 1024 * 1024 + self.part_size = 5 * 1024 * 1024 + + def prepare(self): + rb = self.create_bucket() + self.r_obj = 'foo' + + uploader = MultipartUploader(rb, self.r_obj, self.obj_size, self.part_size) + + uploader.prepare() + uploader.upload_all() + + self.r_upload_state = uploader.get_state() + + + def check(self): + for rb in self.get_buckets(): + break + + uploader = MultipartUploader(rb, self.r_obj, self.obj_size, self.part_size, + state=self.r_upload_state) + + uploader.complete() + crc = uploader.hexdigest() + print('written crc: ' + crc) + + k = rb.bucket.get_key(self.r_obj) + eq(k.size, self.obj_size) + + validate_obj(rb, self.r_obj, crc) + + +# prepare: +# init, upload multipart object +# check: +# complete multipart +# verify data correctness +# verify that object layout is correct +class r_test_multipart_defer_update_complete(RTest): + def init(self): + self.obj_size = 18 * 1024 * 1024 + self.part_size = 5 * 1024 * 1024 + + def prepare(self): + rb = self.create_bucket() + self.r_obj = 'foo' + + uploader = MultipartUploader(rb, self.r_obj, self.obj_size, self.part_size) + + uploader.prepare() + ret = uploader.upload() # only upload one part + eq(ret, True) + + self.r_upload_state = uploader.get_state() + + + def check(self): + for rb in self.get_buckets(): + break + + uploader = MultipartUploader(rb, self.r_obj, self.obj_size, self.part_size, + state=self.r_upload_state) + + uploader.upload_all() # upload remaining + uploader.complete() + crc = uploader.hexdigest() + print('written crc: ' + crc) + + k = rb.bucket.get_key(self.r_obj) + eq(k.size, self.obj_size) + + validate_obj(rb, self.r_obj, crc) + +class ObjInfo(object): + pass + +# prepare: +# init, create obj in different storage classes +# check: +# verify rados objects data, and in expected location +class r_test_obj_storage_class(RTest): + def init(self): + self.obj_size = 10 * 1024 * 1024 + + def prepare(self): + rb = self.create_bucket() + zone = get_zone() + + placement_target = zone.get_placement_target(rb.placement_rule.placement_id) + + data = generate_random(self.obj_size) + crc = calc_crc(bytearray(data, 'utf-8')) + + self.r_objs = [] + + for sc in placement_target.storage_classes.get_all(): + obj_info = ObjInfo() + + obj_info.storage_class = sc + obj_info.name = 'foo-' + sc + obj_info.obj_size = self.obj_size + obj_info.crc = crc + + obj = Key(rb.bucket) + obj.key = obj_info.name + obj.storage_class = sc + obj.set_contents_from_string(data) + + self.r_objs.append(obj_info) + + + def check(self): + for rb in self.get_buckets(): + break + + for obj_info in self.r_objs: + k = rb.bucket.get_key(obj_info.name) + eq(k.size, obj_info.obj_size) + eq(k.storage_class, obj_info.storage_class) + + + validate_obj(rb, obj_info.name, obj_info.crc) + +# prepare: +# init, start multipart obj creation in different storage classes +# check: +# complete uploads, verify rados objects data, and in expected location +class r_test_obj_storage_class_multipart(RTest): + def init(self): + self.obj_size = 11 * 1024 * 1024 + self.part_size = 5 * 1024 * 1024 + + def prepare(self): + rb = self.create_bucket() + + self.r_objs = [] + + zone = get_zone() + placement_target = zone.get_placement_target(rb.placement_rule.placement_id) + + for sc in placement_target.storage_classes.get_all(): + obj_info = ObjInfo() + + obj_info.storage_class = sc + obj_info.name = 'foo-' + sc + obj_info.obj_size = self.obj_size + obj_info.part_size = self.part_size + + uploader = MultipartUploader(rb, obj_info.name, self.obj_size, self.part_size, storage_class=sc) + + uploader.prepare() + uploader.upload_all() + + obj_info.upload_state = uploader.get_state() + + self.r_objs.append(obj_info) + + + def check(self): + for rb in self.get_buckets(): + break + + for obj_info in self.r_objs: + uploader = MultipartUploader(rb, obj_info.name, obj_info.obj_size, + obj_info.part_size, state=obj_info.upload_state) + + uploader.complete() + crc = uploader.hexdigest() + print('written crc: ' + crc) + + k = rb.bucket.get_key(obj_info.name) + eq(k.size, self.obj_size) + eq(k.storage_class, obj_info.storage_class) + + validate_obj(rb, obj_info.name, crc) + + +# prepare: +# init, create obj in different storage classes, copy them +# check: +# verify rados objects data, and in expected location +class r_test_obj_storage_class_copy(RTest): + def init(self): + self.obj_size = 5 * 1024 * 1024 + + def prepare(self): + rb = self.create_bucket() + zone = get_zone() + + placement_target = zone.get_placement_target(rb.placement_rule.placement_id) + + data = generate_random(self.obj_size) + crc = calc_crc(bytearray(data, 'utf-8')) + + self.r_objs = [] + + for sc in placement_target.storage_classes.get_all(): + obj_info = ObjInfo() + obj_info.storage_class = sc + obj_info.name = 'foo-' + sc + obj_info.obj_size = self.obj_size + obj_info.crc = crc + + obj = Key(rb.bucket) + obj.key = obj_info.name + obj.storage_class = sc + obj.set_contents_from_string(data) + self.r_objs.append(obj_info) + + for sc2 in placement_target.storage_classes.get_all(): + copy_obj_info = ObjInfo() + copy_obj_info.storage_class = sc2 + copy_obj_info.name = obj_info.name + '-' + sc2 + copy_obj_info.obj_size = obj_info.obj_size + copy_obj_info.crc = obj_info.crc + + rb.bucket.copy_key(copy_obj_info.name, rb.bucket.name, obj_info.name, storage_class=copy_obj_info.storage_class) + + self.r_objs.append(copy_obj_info) + + + + def check(self): + for rb in self.get_buckets(): + break + + for obj_info in self.r_objs: + k = rb.bucket.get_key(obj_info.name) + eq(k.size, obj_info.obj_size) + eq(k.storage_class, obj_info.storage_class) + + print('validate', obj_info.name) + + validate_obj(rb, obj_info.name, obj_info.crc) + +# prepare: +# init, create multipart obj in different storage classes, copy them +# check: +# verify rados objects data, and in expected location +class r_test_obj_storage_class_multipart_copy(RTest): + def init(self): + self.obj_size = 11 * 1024 * 1024 + self.part_size = 5 * 1024 * 1024 + + def prepare(self): + rb = self.create_bucket() + zone = get_zone() + + placement_target = zone.get_placement_target(rb.placement_rule.placement_id) + + self.r_objs = [] + + for sc in placement_target.storage_classes.get_all(): + obj_info = ObjInfo() + obj_info.storage_class = sc + obj_info.name = 'foo-' + sc + obj_info.obj_size = self.obj_size + + uploader = MultipartUploader(rb, obj_info.name, self.obj_size, self.part_size, storage_class=sc) + + uploader.prepare() + uploader.upload_all() + uploader.complete() + + obj_info.crc = uploader.hexdigest() + + self.r_objs.append(obj_info) + + for sc2 in placement_target.storage_classes.get_all(): + copy_obj_info = ObjInfo() + copy_obj_info.storage_class = sc2 + copy_obj_info.name = obj_info.name + '-' + sc2 + copy_obj_info.obj_size = obj_info.obj_size + copy_obj_info.crc = obj_info.crc + + rb.bucket.copy_key(copy_obj_info.name, rb.bucket.name, obj_info.name, storage_class=copy_obj_info.storage_class) + + self.r_objs.append(copy_obj_info) + + + + def check(self): + for rb in self.get_buckets(): + break + + for obj_info in self.r_objs: + k = rb.bucket.get_key(obj_info.name) + eq(k.size, obj_info.obj_size) + eq(k.storage_class, obj_info.storage_class) + + print('validate', obj_info.name) + + validate_obj(rb, obj_info.name, obj_info.crc) + diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..178ef67 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,7 @@ +PyYAML +nose >=1.0.0 +boto >=2.6.0, != 2.46.0 +munch >=1.0.0 +gevent >=1.0 +httplib2 +lxml diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..7ee3825 --- /dev/null +++ b/setup.py @@ -0,0 +1,29 @@ +#!/usr/bin/python3 +from setuptools import setup, find_packages + +setup( + name='ragweed', + version='0.0.1', + packages=find_packages(), + + author='Yehuda Sadeh', + author_email='yehuda@redhat.com', + description='A test suite for ceph rgw', + license='MIT', + keywords='ceph rgw testing', + + install_requires=[ + 'boto >=2.0b4', + 'PyYAML', + 'munch >=1.0.0', + 'gevent >=1.0', + 'isodate >=0.4.4', + ], + + #entry_points={ + # 'console_scripts': [ + # 'ragweed = ragweed:main', + # ], + # }, + + ) -- 2.39.5