--- /dev/null
+Copyright (c) 2017 Red Hat, Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
--- /dev/null
+===============
+ Ragweed Tests
+===============
+
+This is a set of test that verify functionality for the RGW, and the way it is represented in rados.
+
+This can be used to verify functionality between upgrades.
+
+Tests are run in two phases. In the first phase, (possibly when running against the old version) data is prepared.
+
+In the second phase the representation of that data is then tested (possibly after an upgrade).
+
+Each of these phases can be executed separately.
+
+For more information on the background of the tests visit: https://www.spinics.net/lists/ceph-devel/msg34636.html
+
+The tests use the Nose test framework. To get started, ensure you have
+the ``virtualenv`` software installed; e.g. on Debian/Ubuntu::
+
+ sudo apt-get install python-virtualenv
+
+on Fedora/RHEL::
+
+ sudo yum install python3-virtualenv
+
+and then run::
+
+ ./bootstrap
+
+You will need to create a configuration file with the location of the
+service and two different credentials. A sample configuration file named
+``ragweed-example.conf`` has been provided in this repo.
+
+Once you have that file copied and edited, you can run the tests with::
+
+ RAGWEED_CONF=ragweed.conf RAGWEED_STAGES=prepare,check ./virtualenv/bin/nosetests -v
+
+The phase(s) of the tests are set via ``RAGWEED_STAGES``. The options for ``RAGWEED_STAGES`` are ``prepare`` and ``check``. ``test`` can be used instead of ``check``.
+
+=====================================
+Running Ragweed Tests with vstart.sh
+=====================================
+
+Note: This example assumes the path to the ceph source code is $HOME/ceph.
+
+The ``ragweed-example.conf`` file provided can be can be used to run the ragweed tests on a Ceph cluster started with vstart.
+
+Before the ragweed tests are run a system user must be created on the cluster first. From the ``ceph/build`` directory run::
+
+ $HOME/ceph/build/bin/radosgw-admin -c ceph.conf user create --uid=admin_user --display-name="Admin User" --access-key=accesskey2 --secret-key=secretkey2 --admin
+
+If the system user created is different than the one created above the ``[user system]`` section in the ragweed.conf file much match the created user.
+
+Then run ``$HOME/ceph/build/vstart_environment.sh`` or export the ``LD_LIBRARY_PATH`` and ``PYTHONPATH`` generated in the file ``$HOME/ceph/build/vstart_environment.sh``::
+
+ chmod 775 $HOME/ceph/build/vstart_environment.sh
+ $HOME/ceph/build/vstart_environment.sh
+
+OR::
+
+ export PYTHONPATH=$HOME/ceph/master/src/pybind:$HOME/ceph/master/build/lib/cython_modules/lib.3:$HOME/ceph/master/src/python-common:$PYTHONPATH
+ export LD_LIBRARY_PATH=$HOME/ceph/master/build/lib:$LD_LIBRARY_PATH
+
+Finally run the ragweed tests::
+
+ RAGWEED_CONF=ragweed.conf RAGWEED_STAGES=prepare,check ./virtualenv/bin/nosetests -v
--- /dev/null
+#!/bin/sh
+set -x
+set -e
+
+if [ -f /etc/debian_version ]; then
+ for package in python3-pip python3-virtualenv python3-dev libevent-dev libxml2-dev libxslt-dev zlib1g-dev libffi-dev python3-cffi python3-pycparser python3-cachecontrol; do
+ if [ "$(dpkg --status -- $package 2>/dev/null|sed -n 's/^Status: //p')" != "install ok installed" ]; then
+ # add a space after old values
+ missing="${missing:+$missing }$package"
+ fi
+ done
+ if [ -n "$missing" ]; then
+ echo "$0: missing required DEB packages. Installing via sudo." 1>&2
+ sudo apt-get -y install $missing
+ fi
+fi
+if [ -f /etc/redhat-release ]; then
+ for package in python3-pip python3-virtualenv python3-devel libevent-devel libxml2-devel libxslt-devel zlib-devel; do
+ if [ "$(rpm -qa $package 2>/dev/null)" == "" ]; then
+ missing="${missing:+$missing }$package"
+ fi
+ done
+ if [ -n "$missing" ]; then
+ echo "$0: missing required RPM packages. Installing via sudo." 1>&2
+ sudo yum -y install $missing
+ fi
+fi
+
+sudo pip3 install --upgrade --trusted-host apt-mirror.front.sepia.ceph.com setuptools cffi cachecontrol # address pip issue: https://github.com/pypa/pip/issues/6264
+virtualenv -p python3 --system-site-packages --download virtualenv
+
+# avoid pip bugs
+./virtualenv/bin/pip install --upgrade pip
+
+# work-around change in pip 1.5
+./virtualenv/bin/pip install six
+./virtualenv/bin/pip install -I nose
+./virtualenv/bin/pip install setuptools
+
+./virtualenv/bin/pip install -U -r requirements.txt
+
+# forbid setuptools from using the network because it'll try to use
+# easy_install, and we really wanted pip; next line will fail if pip
+# requirements.txt does not match setup.py requirements -- sucky but
+# good enough for now
+./virtualenv/bin/python setup.py develop
--- /dev/null
+[DEFAULT]
+
+[rados]
+
+# config file for ceph cluster, ex: ceph_conf = /home/user_name/ceph/build/ceph.conf
+ceph_conf = <path to ceph.conf>
+
+[rgw]
+host = localhost
+
+## uncomment the port to use something other than 80
+port = 8000
+
+## say "no" to disable TLS
+is_secure = no
+
+# bucket prefix
+bucket_prefix = ragweed
+
+[fixtures]
+
+[user regular]
+user_id = testid
+access_key = 0555b35654ad1656d804
+secret_key = h7GhxuBLTrlhVUyxSPUKUV8r/2EI4ngqJxD7iBdBYLhwluN30JaT3Q==
+
+[user system]
+user_id = admin_user
+access_key = accesskey2
+secret_key = secret2
--- /dev/null
+import sys
+import os
+import boto
+import boto.s3.connection
+import json
+import inspect
+import pickle
+import munch
+import yaml
+import configparser
+from boto.s3.key import Key
+from nose.plugins.attrib import attr
+from nose.tools import eq_ as eq
+import rados
+
+from .reqs import _make_admin_request
+
+ragweed_env = None
+suite = None
+
+class RGWConnection:
+ def __init__(self, access_key, secret_key, host, port, is_secure):
+ self.host = host
+ self.port = port
+ self.is_secure = is_secure
+ self.conn = boto.connect_s3(
+ aws_access_key_id = access_key,
+ aws_secret_access_key = secret_key,
+ host=host,
+ port=port,
+ is_secure=is_secure,
+ calling_format = boto.s3.connection.OrdinaryCallingFormat(),
+ )
+
+ def create_bucket(self, name):
+ return self.conn.create_bucket(name)
+
+ def get_bucket(self, name, validate=True):
+ return self.conn.get_bucket(name, validate=validate)
+
+
+class RGWRESTAdmin:
+ def __init__(self, connection):
+ self.conn = connection
+
+ def get_resource(self, path, params):
+ r = _make_admin_request(self.conn, "GET", path, params)
+ if r.status != 200:
+ raise boto.exception.S3ResponseError(r.status, r.reason)
+ return munch.munchify(json.loads(r.read()))
+
+
+ def read_meta_key(self, key):
+ return self.get_resource('/admin/metadata', {'key': key})
+
+ def get_bucket_entrypoint(self, bucket_name):
+ return self.read_meta_key('bucket:' + bucket_name)
+
+ def get_bucket_instance_info(self, bucket_name, bucket_id = None):
+ if not bucket_id:
+ ep = self.get_bucket_entrypoint(bucket_name)
+ print(ep)
+ bucket_id = ep.data.bucket.bucket_id
+ result = self.read_meta_key('bucket.instance:' + bucket_name + ":" + bucket_id)
+ return result.data.bucket_info
+
+ def check_bucket_index(self, bucket_name):
+ return self.get_resource('/admin/bucket',{'index' : None, 'bucket':bucket_name})
+
+ def get_obj_layout(self, key):
+ path = '/' + key.bucket.name + '/' + key.name
+ params = {'layout': None}
+ if key.version_id is not None:
+ params['versionId'] = key.version_id
+
+ print(params)
+
+ return self.get_resource(path, params)
+
+ def get_zone_params(self):
+ return self.get_resource('/admin/config', {'type': 'zone'})
+
+
+class RSuite:
+ def __init__(self, name, bucket_prefix, zone, suite_step):
+ self.name = name
+ self.bucket_prefix = bucket_prefix
+ self.zone = zone
+ self.config_bucket = None
+ self.rtests = []
+ self.do_preparing = False
+ self.do_check = False
+ for step in suite_step.split(','):
+ if step == 'prepare':
+ self.do_preparing = True
+ self.config_bucket = self.zone.create_raw_bucket(self.get_bucket_name('conf'))
+ if step == 'check' or step == 'test':
+ self.do_check = True
+ self.config_bucket = self.zone.get_raw_bucket(self.get_bucket_name('conf'))
+
+ def get_bucket_name(self, suffix):
+ return self.bucket_prefix + '-' + suffix
+
+ def register_test(self, t):
+ self.rtests.append(t)
+
+ def write_test_data(self, test):
+ k = Key(self.config_bucket)
+ k.key = 'tests/' + test._name
+ k.set_contents_from_string(test.to_json())
+
+ def read_test_data(self, test):
+ k = Key(self.config_bucket)
+ k.key = 'tests/' + test._name
+ s = k.get_contents_as_string()
+ print('read_test_data=', s)
+ test.from_json(s)
+
+ def is_preparing(self):
+ return self.do_preparing
+
+ def is_checking(self):
+ return self.do_check
+
+
+class RTestJSONSerialize(json.JSONEncoder):
+ def default(self, obj):
+ if isinstance(obj, (list, dict, tuple, str, int, float, bool, type(None))):
+ return JSONEncoder.default(self, obj)
+ return {'__pickle': pickle.dumps(obj, 0).decode('utf-8')}
+
+def rtest_decode_json(d):
+ if '__pickle' in d:
+ return pickle.loads(bytearray(d['__pickle'], 'utf-8'))
+ return d
+
+class RPlacementRule:
+ def __init__(self, rule):
+ r = rule.split('/', 1)
+
+ self.placement_id = r[0]
+
+ if (len(r) == 2):
+ self.storage_class=r[1]
+ else:
+ self.storage_class = 'STANDARD'
+
+
+class RBucket:
+ def __init__(self, zone, bucket, bucket_info):
+ self.zone = zone
+ self.bucket = bucket
+ self.name = bucket.name
+ self.bucket_info = bucket_info
+
+ try:
+ self.placement_rule = RPlacementRule(self.bucket_info.placement_rule)
+ self.placement_target = self.zone.get_placement_target(self.bucket_info.placement_rule)
+ except:
+ pass
+
+ def get_data_pool(self):
+ try:
+ # old style explicit pool
+ explicit_pool = self.bucket_info.bucket.pool
+ except:
+ # new style explicit pool
+ explicit_pool = self.bucket_info.bucket.explicit_placement.data_pool
+ if explicit_pool is not None and explicit_pool != '':
+ return explicit_pool
+
+ return self.placement_target.get_data_pool(self.placement_rule)
+
+
+ def get_tail_pool(self, obj_layout):
+ try:
+ placement_rule = obj_layout.manifest.tail_placement.placement_rule
+ except:
+ placement_rule = ''
+ if placement_rule == '':
+ try:
+ # new style
+ return obj_layout.manifest.tail_placement.bucket.explicit_placement.data_pool
+ except:
+ pass
+
+ try:
+ # old style
+ return obj_layout.manifest.tail_bucket.pool
+ except:
+ pass
+
+ pr = RPlacementRule(placement_rule)
+
+ return self.placement_target.get_data_pool(pr)
+
+class RStorageClasses:
+ def __init__(self, config):
+ if hasattr(config, 'storage_classes'):
+ self.storage_classes = config.storage_classes
+ else:
+ try:
+ self.storage_classes = munch.munchify({ 'STANDARD': { 'data_pool': config.data_pool }})
+ except:
+ self.storage_classes = None
+ pass
+
+ def get(self, storage_class):
+ assert(self.storage_classes != None)
+ try:
+ if not storage_class:
+ storage_class = 'STANDARD'
+ sc = self.storage_classes[storage_class]
+ except:
+ eq('could not find storage class ' + storage_class, 0)
+
+ return sc
+
+ def get_all(self):
+ for (name, _) in self.storage_classes.items():
+ yield name
+
+class RPlacementTarget:
+ def __init__(self, name, config):
+ self.name = name
+ self.index_pool = config.index_pool
+ self.data_extra_pool = config.data_extra_pool
+ self.storage_classes = RStorageClasses(config)
+
+ if not self.data_extra_pool:
+ self.data_extra_pool = self.storage_classes.get_data_pool('STANDARD')
+
+ def get_data_pool(self, placement_rule):
+ return self.storage_classes.get(placement_rule.storage_class).data_pool
+
+class RZone:
+ def __init__(self, conn):
+ self.conn = conn
+
+ self.rgw_rest_admin = RGWRESTAdmin(self.conn.system)
+ self.zone_params = self.rgw_rest_admin.get_zone_params()
+
+ self.placement_targets = {}
+
+ for e in self.zone_params.placement_pools:
+ self.placement_targets[e.key] = e.val
+
+ print('zone_params:', self.zone_params)
+
+ def get_placement_target(self, placement_id):
+ plid = placement_id
+ if placement_id is None or placement_id == '':
+ print('zone_params=', self.zone_params)
+ plid = self.zone_params.default_placement
+
+ try:
+ return RPlacementTarget(plid, self.placement_targets[plid])
+ except:
+ pass
+
+ return None
+
+ def get_default_placement(self):
+ return get_placement_target(self.zone_params.default_placement)
+
+ def create_bucket(self, name):
+ bucket = self.create_raw_bucket(name)
+ bucket_info = self.rgw_rest_admin.get_bucket_instance_info(bucket.name)
+ print('bucket_info:', bucket_info)
+ return RBucket(self, bucket, bucket_info)
+
+ def get_bucket(self, name):
+ bucket = self.get_raw_bucket(name)
+ bucket_info = self.rgw_rest_admin.get_bucket_instance_info(bucket.name)
+ print('bucket_info:', bucket_info)
+ return RBucket(self, bucket, bucket_info)
+
+ def create_raw_bucket(self, name):
+ return self.conn.regular.create_bucket(name)
+
+ def get_raw_bucket(self, name):
+ return self.conn.regular.get_bucket(name)
+
+ def refresh_rbucket(self, rbucket):
+ rbucket.bucket = self.get_raw_bucket(rbucket.bucket.name)
+ rbucket.bucket_info = self.rgw_rest_admin.get_bucket_instance_info(rbucket.bucket.name)
+
+
+class RTest:
+ def __init__(self):
+ self._name = self.__class__.__name__
+ self.r_buckets = []
+ self.init()
+
+ def create_bucket(self):
+ bid = len(self.r_buckets) + 1
+ bucket_name = suite.get_bucket_name(self._name + '.' + str(bid))
+ bucket_name = bucket_name.replace("_", "-")
+ rb = suite.zone.create_bucket(bucket_name)
+ self.r_buckets.append(rb)
+
+ return rb
+
+ def get_buckets(self):
+ for rb in self.r_buckets:
+ yield rb
+
+ def init(self):
+ pass
+
+ def prepare(self):
+ pass
+
+ def check(self):
+ pass
+
+ def to_json(self):
+ attrs = {}
+ for x in dir(self):
+ if x.startswith('r_'):
+ attrs[x] = getattr(self, x)
+ return json.dumps(attrs, cls=RTestJSONSerialize)
+
+ def from_json(self, s):
+ j = json.loads(s, object_hook=rtest_decode_json)
+ for e in j:
+ setattr(self, e, j[e])
+
+ def save(self):
+ suite.write_test_data(self)
+
+ def load(self):
+ suite.read_test_data(self)
+ for rb in self.r_buckets:
+ suite.zone.refresh_rbucket(rb)
+
+ def test(self):
+ suite.register_test(self)
+ if suite.is_preparing():
+ self.prepare()
+ self.save()
+
+ if suite.is_checking():
+ self.load()
+ self.check()
+
+def read_config(fp):
+ config = munch.Munch()
+ g = yaml.safe_load_all(fp)
+ for new in g:
+ print(munch.munchify(new))
+ config.update(munch.munchify(new))
+ return config
+
+str_config_opts = [
+ 'user_id',
+ 'access_key',
+ 'secret_key',
+ 'host',
+ 'ceph_conf',
+ 'bucket_prefix',
+ ]
+
+int_config_opts = [
+ 'port',
+ ]
+
+bool_config_opts = [
+ 'is_secure',
+ ]
+
+def dict_find(d, k):
+ if k in d:
+ return d[k]
+ return None
+
+class RagweedEnv:
+ def __init__(self):
+ self.config = munch.Munch()
+
+ cfg = configparser.RawConfigParser()
+ try:
+ path = os.environ['RAGWEED_CONF']
+ except KeyError:
+ raise RuntimeError(
+ 'To run tests, point environment '
+ + 'variable RAGWEED_CONF to a config file.',
+ )
+ with open(path, 'r') as f:
+ cfg.readfp(f)
+
+ for section in cfg.sections():
+ try:
+ (section_type, name) = section.split(None, 1)
+ if not section_type in self.config:
+ self.config[section_type] = munch.Munch()
+ self.config[section_type][name] = munch.Munch()
+ cur = self.config[section_type]
+ except ValueError:
+ section_type = ''
+ name = section
+ self.config[name] = munch.Munch()
+ cur = self.config
+
+ cur[name] = munch.Munch()
+
+ for var in str_config_opts:
+ try:
+ cur[name][var] = cfg.get(section, var)
+ except configparser.NoOptionError:
+ pass
+
+ for var in int_config_opts:
+ try:
+ cur[name][var] = cfg.getint(section, var)
+ except configparser.NoOptionError:
+ pass
+
+ for var in bool_config_opts:
+ try:
+ cur[name][var] = cfg.getboolean(section, var)
+ except configparser.NoOptionError:
+ pass
+
+ print(json.dumps(self.config))
+
+ rgw_conf = self.config.rgw
+
+ try:
+ self.bucket_prefix = rgw_conf.bucket_prefix
+ except:
+ self.bucket_prefix = 'ragweed'
+
+ conn = munch.Munch()
+ for (k, u) in self.config.user.items():
+ conn[k] = RGWConnection(u.access_key, u.secret_key, rgw_conf.host, dict_find(rgw_conf, 'port'), dict_find(rgw_conf, 'is_secure'))
+
+ self.zone = RZone(conn)
+ self.suite = RSuite('ragweed', self.bucket_prefix, self.zone, os.environ['RAGWEED_STAGES'])
+
+ try:
+ self.ceph_conf = self.config.rados.ceph_conf
+ except:
+ raise RuntimeError(
+ 'ceph_conf is missing under the [rados] section in ' + os.environ['RAGWEED_CONF']
+ )
+
+ self.rados = rados.Rados(conffile=self.ceph_conf)
+ self.rados.connect()
+
+ pools = self.rados.list_pools()
+
+ for pool in pools:
+ print("rados pool>", pool)
+
+def setup_module():
+ global ragweed_env
+ global suite
+
+ ragweed_env = RagweedEnv()
+ suite = ragweed_env.suite
--- /dev/null
+import boto.s3.connection
+from http.client import HTTPConnection, HTTPSConnection
+from urllib.parse import urlparse, urlencode
+
+def _make_admin_request(conn, method, path, query_dict=None, body=None, response_headers=None, request_headers=None, expires_in=100000, path_style=True, timeout=None):
+ """
+ issue a request for a specified method, on a specified path
+ with a specified (optional) body (encrypted per the connection), and
+ return the response (status, reason).
+ """
+
+ query = ''
+ if query_dict is not None:
+ query = urlencode(query_dict)
+
+ (bucket_str, key_str) = path.split('/', 2)[1:]
+ bucket = conn.get_bucket(bucket_str, validate=False)
+ key = bucket.get_key(key_str, validate=False)
+
+ urlobj = None
+ if key is not None:
+ urlobj = key
+ elif bucket is not None:
+ urlobj = bucket
+ else:
+ raise RuntimeError('Unable to find bucket name')
+ url = urlobj.generate_url(expires_in, method=method, response_headers=response_headers, headers=request_headers)
+ o = urlparse(url)
+ req_path = o.path + '?' + o.query + '&' + query
+
+ return _make_raw_request(host=conn.host, port=conn.port, method=method, path=req_path, body=body, request_headers=request_headers, secure=conn.is_secure, timeout=timeout)
+
+def _make_request(method, bucket, key, body=None, authenticated=False, response_headers=None, request_headers=None, expires_in=100000, path_style=True, timeout=None):
+ """
+ issue a request for a specified method, on a specified <bucket,key>,
+ with a specified (optional) body (encrypted per the connection), and
+ return the response (status, reason).
+
+ If key is None, then this will be treated as a bucket-level request.
+
+ If the request or response headers are None, then default values will be
+ provided by later methods.
+ """
+ if not path_style:
+ conn = bucket.connection
+ request_headers['Host'] = conn.calling_format.build_host(conn.server_name(), bucket.name)
+
+ if authenticated:
+ urlobj = None
+ if key is not None:
+ urlobj = key
+ elif bucket is not None:
+ urlobj = bucket
+ else:
+ raise RuntimeError('Unable to find bucket name')
+ url = urlobj.generate_url(expires_in, method=method, response_headers=response_headers, headers=request_headers)
+ o = urlparse(url)
+ path = o.path + '?' + o.query
+ else:
+ bucketobj = None
+ if key is not None:
+ path = '/{obj}'.format(obj=key.name)
+ bucketobj = key.bucket
+ elif bucket is not None:
+ path = '/'
+ bucketobj = bucket
+ else:
+ raise RuntimeError('Unable to find bucket name')
+ if path_style:
+ path = '/{bucket}'.format(bucket=bucketobj.name) + path
+
+ return _make_raw_request(host=s3.main.host, port=s3.main.port, method=method, path=path, body=body, request_headers=request_headers, secure=s3.main.is_secure, timeout=timeout)
+
+def _make_bucket_request(method, bucket, body=None, authenticated=False, response_headers=None, request_headers=None, expires_in=100000, path_style=True, timeout=None):
+ """
+ issue a request for a specified method, on a specified <bucket>,
+ with a specified (optional) body (encrypted per the connection), and
+ return the response (status, reason)
+ """
+ return _make_request(method=method, bucket=bucket, key=None, body=body, authenticated=authenticated, response_headers=response_headers, request_headers=request_headers, expires_in=expires_in, path_style=path_style, timeout=timeout)
+
+def _make_raw_request(host, port, method, path, body=None, request_headers=None, secure=False, timeout=None):
+ """
+ issue a request to a specific host & port, for a specified method, on a
+ specified path with a specified (optional) body (encrypted per the
+ connection), and return the response (status, reason).
+
+ This allows construction of special cases not covered by the bucket/key to
+ URL mapping of _make_request/_make_bucket_request.
+ """
+ if secure:
+ class_ = HTTPSConnection
+ else:
+ class_ = HTTPConnection
+
+ if request_headers is None:
+ request_headers = {}
+
+ c = class_(host, port, timeout=timeout)
+
+ # TODO: We might have to modify this in future if we need to interact with
+ # how http.client.request handles Accept-Encoding and Host.
+ c.request(method, path, body=body, headers=request_headers)
+
+ res = c.getresponse()
+ #c.close()
+
+ return res
+
+
--- /dev/null
+from io import StringIO
+import ragweed.framework
+import binascii
+import string
+import random
+
+
+from ragweed.framework import *
+
+class obj_placement:
+ def __init__(self, pool, oid, loc):
+ self.pool = pool
+ self.oid = oid
+ self.loc = loc
+
+def get_zone():
+ return ragweed.framework.ragweed_env.zone
+
+def rgwa():
+ return get_zone().rgw_rest_admin
+
+def get_pool_ioctx(pool_name):
+ return ragweed.framework.ragweed_env.rados.open_ioctx(pool_name)
+
+
+def get_placement(obj_json):
+ try:
+ return obj_placement(obj_json.pool, obj_json.oid, obj_json.loc)
+ except:
+ oid = obj_json.bucket.marker + '_' + obj_json.object
+ key = ''
+ if obj_json.key != '':
+ key = obj_json.bucket.marker + '_' + obj_json.key
+ return obj_placement(obj_json.bucket.pool, oid, key)
+
+def validate_obj_location(rbucket, obj):
+ expected_head_pool = rbucket.get_data_pool()
+ head_pool_ioctx = get_pool_ioctx(expected_head_pool)
+ print('expected head pool: ' + expected_head_pool)
+
+ obj_layout = rgwa().get_obj_layout(obj)
+
+ print('layout', obj_layout)
+
+ obj_size = obj_layout.manifest.obj_size
+ print('obj_size', obj_size)
+
+ print('head', obj_layout.head)
+ expected_tail_pool = rbucket.get_tail_pool(obj_layout)
+ tail_pool_ioctx = get_pool_ioctx(expected_tail_pool)
+
+ head_placement = get_placement(obj_layout.head)
+
+ eq(head_placement.pool, expected_head_pool)
+
+ # check rados object for head exists
+ head_pool_ioctx.set_locator_key(head_placement.loc)
+ (size, mtime) = head_pool_ioctx.stat(head_placement.oid)
+
+ print('head size:', size, 'mtime:', mtime)
+
+ # check tail
+ for o in obj_layout.data_location:
+ print('o=', o)
+ print('ofs=', o.ofs, 'loc', o.loc)
+ placement = get_placement(o.loc)
+ if o.ofs < obj_layout.manifest.head_size:
+ eq(placement.pool, expected_head_pool)
+ pool_ioctx = head_pool_ioctx
+ else:
+ eq(placement.pool, expected_tail_pool)
+ pool_ioctx = tail_pool_ioctx
+
+ # validate rados object exists
+ pool_ioctx.set_locator_key(placement.loc)
+ (size, mtime) = pool_ioctx.stat(placement.oid)
+
+ if o.ofs + o.loc_size > obj_size:
+ # radosgw get_obj_layout() request on previous versions could return bigger o.loc_size
+ # than actual at the end of the object. Fixed in later versions but need to adjust here
+ # for when running on older versions.
+ o.loc_size = obj_size - o.ofs
+
+ check_size = o.loc_ofs + o.loc_size
+
+ eq(size, check_size)
+
+def calc_crc(data):
+ crc = binascii.crc32(data)
+ return '{:#010x}'.format(crc)
+
+
+def validate_obj(rbucket, obj_name, expected_crc):
+ b = rbucket.bucket
+
+ obj = b.get_key(obj_name)
+
+ validate_obj_location(rbucket, obj)
+ obj_crc = calc_crc(obj.get_contents_as_string())
+ eq(obj_crc, expected_crc)
+
+
+def generate_random(size):
+ """
+ Generate random data
+ (actually each MB is a repetition of the first KB)
+ """
+ chunk = 1024
+ allowed = string.ascii_letters
+ s = ''
+ strpart = ''.join([allowed[random.randint(0, len(allowed) - 1)] for _ in range(chunk)])
+
+ for y in range((size + chunk - 1) // chunk):
+ this_chunk_len = chunk
+
+ if len(s) + this_chunk_len > size:
+ this_chunk_len = size - len(s)
+ strpart = strpart[0:this_chunk_len]
+
+ s += strpart
+
+ return s
+
+def gen_rand_string(size, chars=string.ascii_uppercase + string.digits):
+ return ''.join(random.choice(chars) for _ in range(size))
+
+
+# prepare:
+# create objects in multiple sizes, with various names
+# check:
+# verify data correctness
+# verify that objects were written to the expected data pool
+class r_test_small_obj_data(RTest):
+ def prepare(self):
+ self.r_obj_names = [ 'obj', '_', '__', '_ _' ]
+ self.r_bucket_sizes = {}
+
+ sizes = { 0, 512 * 1024, 1024 * 1024 }
+
+ for size in sizes:
+ rbucket = self.create_bucket()
+ self.r_bucket_sizes[rbucket.name] = size
+ data = '0' * size
+ for n in self.r_obj_names:
+ obj = Key(rbucket.bucket)
+ obj.key = n;
+ obj.set_contents_from_string(data)
+
+ def check(self):
+ print(self.r_obj_names)
+ for rbucket in self.get_buckets():
+ size = self.r_bucket_sizes[rbucket.name]
+ data = '0' * int(size)
+
+ for n in self.r_obj_names:
+ obj = Key(rbucket.bucket)
+ obj.key = n;
+ obj_data = obj.get_contents_as_string()
+ eq(bytearray(data, 'utf-8'), obj_data)
+
+ validate_obj_location(rbucket, obj)
+
+class MultipartUploaderState:
+ def __init__(self, mu):
+ self.upload_id = mu.mp.id
+ self.crc = mu.crc
+ self.cur_part = mu.cur_part
+
+
+class MultipartUploader:
+ def __init__(self, rbucket, obj_name, size, part_size, storage_class=None, state=None):
+ self.rbucket = rbucket
+ self.obj_name = obj_name
+ self.size = size
+ self.part_size = part_size
+ self.storage_class = storage_class
+ self.crc = 0
+ self.cur_part = 0
+
+ if state is not None:
+ self.crc = state.crc
+ self.cur_part = state.cur_part
+
+ for upload in rbucket.bucket.list_multipart_uploads():
+ if upload.key_name == self.obj_name and upload.id == state.upload_id:
+ self.mp = upload
+
+ self.num_full_parts = self.size // self.part_size
+ self.last_part_size = self.size % self.part_size
+
+
+ def prepare(self):
+ headers = None
+ if self.storage_class is not None:
+ if not headers:
+ headers = {}
+ headers['X-Amz-Storage-Class'] = self.storage_class
+
+ self.mp = self.rbucket.bucket.initiate_multipart_upload(self.obj_name, headers = headers)
+ self.crc = 0
+ self.cur_part = 0
+
+ def upload(self):
+ if self.cur_part > self.num_full_parts:
+ return False
+
+ if self.cur_part < self.num_full_parts:
+ payload=gen_rand_string(self.part_size // (1024 * 1024)) * 1024 * 1024
+
+ self.mp.upload_part_from_file(StringIO(payload), self.cur_part + 1)
+ self.crc = binascii.crc32(bytearray(payload, 'utf-8'), self.crc)
+ self.cur_part += 1
+
+ return True
+
+
+ if self.last_part_size > 0:
+ last_payload='1'*self.last_part_size
+
+ self.mp.upload_part_from_file(StringIO(last_payload), self.num_full_parts + 1)
+ self.crc = binascii.crc32(bytearray(last_payload, 'utf-8'), self.crc)
+ self.cur_part += 1
+
+ return False
+
+ def upload_all(self):
+ while self.upload():
+ pass
+
+ def complete(self):
+ self.mp.complete_upload()
+
+ def hexdigest(self):
+ return '{:#010x}'.format(self.crc)
+
+ def get_state(self):
+ return MultipartUploaderState(self)
+
+
+
+# prepare:
+# init, upload, and complete a multipart object
+# check:
+# verify data correctness
+# verify that object layout is correct
+class r_test_multipart_simple(RTest):
+ def init(self):
+ self.obj_size = 18 * 1024 * 1024
+ self.part_size = 5 * 1024 * 1024
+
+ def prepare(self):
+ rb = self.create_bucket()
+ self.r_obj = 'foo'
+
+ uploader = MultipartUploader(rb, self.r_obj, self.obj_size, self.part_size)
+
+ uploader.prepare()
+ uploader.upload_all()
+ uploader.complete()
+
+ self.r_crc = uploader.hexdigest()
+ print('written crc: ' + self.r_crc)
+
+ def check(self):
+ for rb in self.get_buckets():
+ break
+
+ k = rb.bucket.get_key(self.r_obj)
+ eq(k.size, self.obj_size)
+
+ validate_obj(rb, self.r_obj, self.r_crc)
+
+# prepare:
+# init, upload, and complete a multipart object
+# check:
+# part index is removed
+class r_test_multipart_index_versioning(RTest):
+ def init(self):
+ self.obj_size = 18 * 1024 * 1024
+ self.part_size = 5 * 1024 * 1024
+
+ def prepare(self):
+ rb = self.create_bucket()
+ rb.bucket.configure_versioning('true')
+ self.r_obj = 'foo'
+
+ uploader = MultipartUploader(rb, self.r_obj, self.obj_size, self.part_size)
+
+ uploader.prepare()
+ uploader.upload_all()
+ uploader.complete()
+
+ def check(self):
+ for rb in self.get_buckets():
+ break
+ index_check_result = rgwa().check_bucket_index(rb.name)
+ print(index_check_result)
+
+ eq(0, len(index_check_result))
+
+
+# prepare:
+# init, upload multipart object
+# check:
+# complete multipart
+# verify data correctness
+# verify that object layout is correct
+class r_test_multipart_defer_complete(RTest):
+ def init(self):
+ self.obj_size = 18 * 1024 * 1024
+ self.part_size = 5 * 1024 * 1024
+
+ def prepare(self):
+ rb = self.create_bucket()
+ self.r_obj = 'foo'
+
+ uploader = MultipartUploader(rb, self.r_obj, self.obj_size, self.part_size)
+
+ uploader.prepare()
+ uploader.upload_all()
+
+ self.r_upload_state = uploader.get_state()
+
+
+ def check(self):
+ for rb in self.get_buckets():
+ break
+
+ uploader = MultipartUploader(rb, self.r_obj, self.obj_size, self.part_size,
+ state=self.r_upload_state)
+
+ uploader.complete()
+ crc = uploader.hexdigest()
+ print('written crc: ' + crc)
+
+ k = rb.bucket.get_key(self.r_obj)
+ eq(k.size, self.obj_size)
+
+ validate_obj(rb, self.r_obj, crc)
+
+
+# prepare:
+# init, upload multipart object
+# check:
+# complete multipart
+# verify data correctness
+# verify that object layout is correct
+class r_test_multipart_defer_update_complete(RTest):
+ def init(self):
+ self.obj_size = 18 * 1024 * 1024
+ self.part_size = 5 * 1024 * 1024
+
+ def prepare(self):
+ rb = self.create_bucket()
+ self.r_obj = 'foo'
+
+ uploader = MultipartUploader(rb, self.r_obj, self.obj_size, self.part_size)
+
+ uploader.prepare()
+ ret = uploader.upload() # only upload one part
+ eq(ret, True)
+
+ self.r_upload_state = uploader.get_state()
+
+
+ def check(self):
+ for rb in self.get_buckets():
+ break
+
+ uploader = MultipartUploader(rb, self.r_obj, self.obj_size, self.part_size,
+ state=self.r_upload_state)
+
+ uploader.upload_all() # upload remaining
+ uploader.complete()
+ crc = uploader.hexdigest()
+ print('written crc: ' + crc)
+
+ k = rb.bucket.get_key(self.r_obj)
+ eq(k.size, self.obj_size)
+
+ validate_obj(rb, self.r_obj, crc)
+
+class ObjInfo(object):
+ pass
+
+# prepare:
+# init, create obj in different storage classes
+# check:
+# verify rados objects data, and in expected location
+class r_test_obj_storage_class(RTest):
+ def init(self):
+ self.obj_size = 10 * 1024 * 1024
+
+ def prepare(self):
+ rb = self.create_bucket()
+ zone = get_zone()
+
+ placement_target = zone.get_placement_target(rb.placement_rule.placement_id)
+
+ data = generate_random(self.obj_size)
+ crc = calc_crc(bytearray(data, 'utf-8'))
+
+ self.r_objs = []
+
+ for sc in placement_target.storage_classes.get_all():
+ obj_info = ObjInfo()
+
+ obj_info.storage_class = sc
+ obj_info.name = 'foo-' + sc
+ obj_info.obj_size = self.obj_size
+ obj_info.crc = crc
+
+ obj = Key(rb.bucket)
+ obj.key = obj_info.name
+ obj.storage_class = sc
+ obj.set_contents_from_string(data)
+
+ self.r_objs.append(obj_info)
+
+
+ def check(self):
+ for rb in self.get_buckets():
+ break
+
+ for obj_info in self.r_objs:
+ k = rb.bucket.get_key(obj_info.name)
+ eq(k.size, obj_info.obj_size)
+ eq(k.storage_class, obj_info.storage_class)
+
+
+ validate_obj(rb, obj_info.name, obj_info.crc)
+
+# prepare:
+# init, start multipart obj creation in different storage classes
+# check:
+# complete uploads, verify rados objects data, and in expected location
+class r_test_obj_storage_class_multipart(RTest):
+ def init(self):
+ self.obj_size = 11 * 1024 * 1024
+ self.part_size = 5 * 1024 * 1024
+
+ def prepare(self):
+ rb = self.create_bucket()
+
+ self.r_objs = []
+
+ zone = get_zone()
+ placement_target = zone.get_placement_target(rb.placement_rule.placement_id)
+
+ for sc in placement_target.storage_classes.get_all():
+ obj_info = ObjInfo()
+
+ obj_info.storage_class = sc
+ obj_info.name = 'foo-' + sc
+ obj_info.obj_size = self.obj_size
+ obj_info.part_size = self.part_size
+
+ uploader = MultipartUploader(rb, obj_info.name, self.obj_size, self.part_size, storage_class=sc)
+
+ uploader.prepare()
+ uploader.upload_all()
+
+ obj_info.upload_state = uploader.get_state()
+
+ self.r_objs.append(obj_info)
+
+
+ def check(self):
+ for rb in self.get_buckets():
+ break
+
+ for obj_info in self.r_objs:
+ uploader = MultipartUploader(rb, obj_info.name, obj_info.obj_size,
+ obj_info.part_size, state=obj_info.upload_state)
+
+ uploader.complete()
+ crc = uploader.hexdigest()
+ print('written crc: ' + crc)
+
+ k = rb.bucket.get_key(obj_info.name)
+ eq(k.size, self.obj_size)
+ eq(k.storage_class, obj_info.storage_class)
+
+ validate_obj(rb, obj_info.name, crc)
+
+
+# prepare:
+# init, create obj in different storage classes, copy them
+# check:
+# verify rados objects data, and in expected location
+class r_test_obj_storage_class_copy(RTest):
+ def init(self):
+ self.obj_size = 5 * 1024 * 1024
+
+ def prepare(self):
+ rb = self.create_bucket()
+ zone = get_zone()
+
+ placement_target = zone.get_placement_target(rb.placement_rule.placement_id)
+
+ data = generate_random(self.obj_size)
+ crc = calc_crc(bytearray(data, 'utf-8'))
+
+ self.r_objs = []
+
+ for sc in placement_target.storage_classes.get_all():
+ obj_info = ObjInfo()
+ obj_info.storage_class = sc
+ obj_info.name = 'foo-' + sc
+ obj_info.obj_size = self.obj_size
+ obj_info.crc = crc
+
+ obj = Key(rb.bucket)
+ obj.key = obj_info.name
+ obj.storage_class = sc
+ obj.set_contents_from_string(data)
+ self.r_objs.append(obj_info)
+
+ for sc2 in placement_target.storage_classes.get_all():
+ copy_obj_info = ObjInfo()
+ copy_obj_info.storage_class = sc2
+ copy_obj_info.name = obj_info.name + '-' + sc2
+ copy_obj_info.obj_size = obj_info.obj_size
+ copy_obj_info.crc = obj_info.crc
+
+ rb.bucket.copy_key(copy_obj_info.name, rb.bucket.name, obj_info.name, storage_class=copy_obj_info.storage_class)
+
+ self.r_objs.append(copy_obj_info)
+
+
+
+ def check(self):
+ for rb in self.get_buckets():
+ break
+
+ for obj_info in self.r_objs:
+ k = rb.bucket.get_key(obj_info.name)
+ eq(k.size, obj_info.obj_size)
+ eq(k.storage_class, obj_info.storage_class)
+
+ print('validate', obj_info.name)
+
+ validate_obj(rb, obj_info.name, obj_info.crc)
+
+# prepare:
+# init, create multipart obj in different storage classes, copy them
+# check:
+# verify rados objects data, and in expected location
+class r_test_obj_storage_class_multipart_copy(RTest):
+ def init(self):
+ self.obj_size = 11 * 1024 * 1024
+ self.part_size = 5 * 1024 * 1024
+
+ def prepare(self):
+ rb = self.create_bucket()
+ zone = get_zone()
+
+ placement_target = zone.get_placement_target(rb.placement_rule.placement_id)
+
+ self.r_objs = []
+
+ for sc in placement_target.storage_classes.get_all():
+ obj_info = ObjInfo()
+ obj_info.storage_class = sc
+ obj_info.name = 'foo-' + sc
+ obj_info.obj_size = self.obj_size
+
+ uploader = MultipartUploader(rb, obj_info.name, self.obj_size, self.part_size, storage_class=sc)
+
+ uploader.prepare()
+ uploader.upload_all()
+ uploader.complete()
+
+ obj_info.crc = uploader.hexdigest()
+
+ self.r_objs.append(obj_info)
+
+ for sc2 in placement_target.storage_classes.get_all():
+ copy_obj_info = ObjInfo()
+ copy_obj_info.storage_class = sc2
+ copy_obj_info.name = obj_info.name + '-' + sc2
+ copy_obj_info.obj_size = obj_info.obj_size
+ copy_obj_info.crc = obj_info.crc
+
+ rb.bucket.copy_key(copy_obj_info.name, rb.bucket.name, obj_info.name, storage_class=copy_obj_info.storage_class)
+
+ self.r_objs.append(copy_obj_info)
+
+
+
+ def check(self):
+ for rb in self.get_buckets():
+ break
+
+ for obj_info in self.r_objs:
+ k = rb.bucket.get_key(obj_info.name)
+ eq(k.size, obj_info.obj_size)
+ eq(k.storage_class, obj_info.storage_class)
+
+ print('validate', obj_info.name)
+
+ validate_obj(rb, obj_info.name, obj_info.crc)
+
--- /dev/null
+PyYAML
+nose >=1.0.0
+boto >=2.6.0, != 2.46.0
+munch >=1.0.0
+gevent >=1.0
+httplib2
+lxml
--- /dev/null
+#!/usr/bin/python3
+from setuptools import setup, find_packages
+
+setup(
+ name='ragweed',
+ version='0.0.1',
+ packages=find_packages(),
+
+ author='Yehuda Sadeh',
+ author_email='yehuda@redhat.com',
+ description='A test suite for ceph rgw',
+ license='MIT',
+ keywords='ceph rgw testing',
+
+ install_requires=[
+ 'boto >=2.0b4',
+ 'PyYAML',
+ 'munch >=1.0.0',
+ 'gevent >=1.0',
+ 'isodate >=0.4.4',
+ ],
+
+ #entry_points={
+ # 'console_scripts': [
+ # 'ragweed = ragweed:main',
+ # ],
+ # },
+
+ )
projects / ragweed.git / commitdiff