From: Dan Mick Date: Tue, 11 Feb 2014 04:31:33 +0000 (-0800) Subject: Add calamari setup/test tasks X-Git-Tag: 1.1.0~1666^2~4 X-Git-Url: http://git.apps.os.sepia.ceph.com/?a=commitdiff_plain;h=e26bb356497949ded23b678ce23146b45122f065;p=teuthology.git Add calamari setup/test tasks Signed-off-by: Warren Usui Signed-off-by: Dan Mick --- diff --git a/teuthology/task/calamari.py b/teuthology/task/calamari.py new file mode 100644 index 0000000000..845056eb01 --- /dev/null +++ b/teuthology/task/calamari.py @@ -0,0 +1,460 @@ +""" +calamari - set up various machines with roles for participating +in Calamari management app testing. Requires secret info for +accessing authenticated package repos to install Calamari, supplied +in an override: clause for calamari.reposetup below. Contains +five tasks: + +- calamari.reposetup: set up the calamari package repos (all targets) +- calamari.agent: install stats collection daemons (all cluster targets) +- calamari.restapi: cluster-management api access (one monitor target) +- calamari.server: main webapp/gui front end (management target) +- calamari.test: run automated test against calamari.server target (local) + +calamari.test runs on the local machine, as it accesses the Calamari +server across https using requests.py, which must be present. It uses +several external modules in calamari_test/. +""" + +from cStringIO import StringIO +import contextlib +import logging +import os +import subprocess +import teuthology.misc as teuthology +import textwrap +import time +from ..orchestra import run + +log = logging.getLogger(__name__) + + +def _edit_diamond_config(remote, serverhost): + """ Edit remote's diamond config to send stats to serverhost """ + ret = remote.run(args=['sudo', + 'sed', + '-i', + 's/calamari/{host}/'.format(host=serverhost), + '/etc/diamond/diamond.conf'], + stdout=StringIO()) + if not ret: + return False + return remote.run(args=['sudo', 'service', 'diamond', 'restart']) + + +def _disable_default_nginx(remote): + """ + Fix up nginx values + """ + script = textwrap.dedent(''' + if [ -f /etc/nginx/conf.d/default.conf ]; then + mv /etc/nginx/conf.d/default.conf \ + /etc/nginx/conf.d/default.disabled + fi + if [ -f /etc/nginx/sites-enabled/default ] ; then + rm /etc/nginx/sites-enabled/default + fi + service nginx restart + service {service} restart + ''') + service = _http_service_name(remote) + script = script.format(service=service) + teuthology.sudo_write_file(remote, '/tmp/disable.nginx', script) + return remote.run(args=['sudo', 'bash', '/tmp/disable.nginx'], + stdout=StringIO()) + + +def _setup_calamari_cluster(remote, restapi_remote): + """ + Add restapi db entry to the server. + """ + restapi_hostname = str(restapi_remote).split('@')[1] + sqlcmd = 'insert into ceph_cluster (name, api_base_url) ' \ + 'values ("{host}", "http://{host}:5000/api/v0.1/");'. \ + format(host=restapi_hostname) + teuthology.write_file(remote, '/tmp/create.cluster.sql', sqlcmd) + return remote.run(args=['cat', + '/tmp/create.cluster.sql', + run.Raw('|'), + 'sudo', + 'sqlite3', + '/opt/calamari/webapp/calamari/db.sqlite3'], + stdout=StringIO()) + + +RELEASE_MAP = { + 'Ubuntu precise': dict(flavor='deb', release='ubuntu', version='precise'), + 'Debian wheezy': dict(flavor='deb', release='debian', version='wheezy'), + 'CentOS 6.4': dict(flavor='rpm', release='centos', version='6.4'), + 'RedHatEnterpriseServer 6.4': dict(flavor='rpm', release='rhel', + version='6.4'), +} + + +def _get_relmap(rem): + relmap = getattr(rem, 'relmap', None) + if relmap is not None: + return relmap + lsb_release_out = StringIO() + rem.run(args=['lsb_release', '-ics'], stdout=lsb_release_out) + release = lsb_release_out.getvalue().replace('\n', ' ').rstrip() + if release in RELEASE_MAP: + rem.relmap = RELEASE_MAP[release] + return rem.relmap + else: + lsb_release_out = StringIO() + rem.run(args=['lsb_release', '-irs'], stdout=lsb_release_out) + release = lsb_release_out.getvalue().replace('\n', ' ').rstrip() + if release in RELEASE_MAP: + rem.relmap = RELEASE_MAP[release] + return rem.relmap + raise RuntimeError('Can\'t get release info for {}'.format(rem)) + + +def _sqlite_package_name(rem): + name = 'sqlite3' if _get_relmap(rem)['flavor'] == 'deb' else None + return name + + +def _http_service_name(rem): + name = 'httpd' if _get_relmap(rem)['flavor'] == 'rpm' else 'apache2' + return name + + +def _install_repo(remote, pkgdir, username, password): + # installing repo is assumed to be idempotent + + relmap = _get_relmap(remote) + log.info('Installing repo on %s', remote) + if relmap['flavor'] == 'deb': + contents = 'deb https://{username}:{password}@download.inktank.com/' \ + '{pkgdir}/deb {codename} main' + contents = contents.format(username=username, password=password, + pkgdir=pkgdir, codename=relmap['version'],) + teuthology.sudo_write_file(remote, + '/etc/apt/sources.list.d/inktank.list', + contents) + remote.run(args=['sudo', + 'apt-get', + 'install', + 'apt-transport-https', + '-y']) + result = remote.run(args=['sudo', 'apt-get', 'update', '-y'], + stdout=StringIO()) + return True + + elif relmap['flavor'] == 'rpm': + baseurl = 'https://{username}:{password}@download.inktank.com/' \ + '{pkgdir}/rpm/{release}{version}' + contents = textwrap.dedent(''' + [inktank] + name=Inktank Storage, Inc. + baseurl={baseurl} + gpgcheck=1 + enabled=1 + '''.format(baseurl=baseurl)) + contents = contents.format(username=username, + password=password, + pkgdir=pkgdir, + release=relmap['release'], + version=relmap['version']) + teuthology.sudo_write_file(remote, + '/etc/yum.repos.d/inktank.repo', + contents) + return remote.run(args=['sudo', 'yum', 'makecache']) + + else: + return False + + +def _remove_repo(remote): + log.info('Removing repo on %s', remote) + flavor = _get_relmap(remote)['flavor'] + if flavor == 'deb': + teuthology.delete_file(remote, '/etc/apt/sources.list.d/inktank.list', + sudo=True, force=True) + result = remote.run(args=['sudo', 'apt-get', 'update', '-y'], + stdout=StringIO()) + return True + + elif flavor == 'rpm': + teuthology.delete_file(remote, '/etc/yum.repos.d/inktank.repo', + sudo=True, force=True) + return remote.run(args=['sudo', 'yum', 'makecache']) + + else: + return False + + +def _install_repokey(remote): + # installing keys is assumed to be idempotent + log.info('Installing repo key on %s', remote) + flavor = _get_relmap(remote)['flavor'] + if flavor == 'deb': + return remote.run(args=['wget', + '-q', + '-O-', + 'http://download.inktank.com/keys/release.asc', + run.Raw('|'), + 'sudo', + 'apt-key', + 'add', + '-']) + elif flavor == 'rpm': + args = ['sudo', 'rpm', '--import', + 'http://download.inktank.com/keys/release.asc'] + return remote.run(args=args) + else: + return False + + +def _install_package(package, remote): + """ + package: name + remote: Remote() to install on + release: deb only, 'precise' or 'wheezy' + pkgdir: may or may not include a branch name, so, say, either + packages or packages-staging/master + """ + log.info('Installing package %s on %s', package, remote) + flavor = _get_relmap(remote)['flavor'] + if flavor == 'deb': + pkgcmd = ['DEBIAN_FRONTEND=noninteractive', + 'sudo', + '-E', + 'apt-get', + '-y', + 'install', + '{package}'.format(package=package)] + elif flavor == 'rpm': + pkgcmd = ['sudo', + 'yum', + '-y', + 'install', + '{package}'.format(package=package)] + else: + log.error('_install_package: bad flavor ' + flavor + '\n') + return False + return remote.run(args=pkgcmd) + + +def _remove_package(package, remote): + """ + remove package from remote + """ + flavor = _get_relmap(remote)['flavor'] + if flavor == 'deb': + pkgcmd = ['DEBIAN_FRONTEND=noninteractive', + 'sudo', + '-E', + 'apt-get', + '-y', + 'purge', + '{package}'.format(package=package)] + elif flavor == 'rpm': + pkgcmd = ['sudo', + 'yum', + '-y', + 'erase', + '{package}'.format(package=package)] + else: + log.error('_remove_package: bad flavor ' + flavor + '\n') + return False + return remote.run(args=pkgcmd) + +""" +Tasks +""" + + +@contextlib.contextmanager +def agent(ctx, config): + """ + task agent + calamari.agent: install stats collection (for each cluster host) + + For example:: + + tasks: + - calamari.agent: + roles: + - mon.0 + - osd.0 + - osd.1 + server: server.0 + """ + + log.info('calamari.agent starting') + overrides = ctx.config.get('overrides', {}) + teuthology.deep_merge(config, overrides.get('calamari.agent', {})) + + remotes = teuthology.roles_to_remotes(ctx.cluster, config) + try: + for rem in remotes: + log.info('Installing calamari-agent on %s', rem) + _install_package('calamari-agent', rem) + server_role = config.get('server') + if not server_role: + raise RuntimeError('must supply \'server\' config key') + server_remote = ctx.cluster.only(server_role).remotes.keys()[0] + # why isn't shortname available by default? + serverhost = server_remote.name.split('@')[1] + log.info('configuring Diamond for {}'.format(serverhost)) + if not _edit_diamond_config(rem, serverhost): + raise RuntimeError( + 'Diamond config edit failed on {0}'.format(rem) + ) + yield + finally: + for rem in remotes: + _remove_package('calamari-agent', rem) + + +@contextlib.contextmanager +def reposetup(ctx, config): + """ + task reposetup + Sets up calamari repository on all remotes; cleans up when done + + calamari.reposetup: + pkgdir: + username: + password: + + Supply the above in an override file if you need to manage the + secret repo credentials separately from the test definition (likely). + + pkgdir encodes package directory (possibly more than one path component) + as in https://:@SERVER//{deb,rpm}{..} + + """ + overrides = ctx.config.get('overrides', {}) + # XXX deep_merge returns the result, which matters if either is None + # make sure that doesn't happen + if config is None: + config = {'dummy': 'dummy'} + teuthology.deep_merge(config, overrides.get('calamari.reposetup', {})) + + try: + pkgdir = config['pkgdir'] + username = config['username'] + password = config['password'] + except KeyError: + raise RuntimeError('requires pkgdir, username, and password') + + remotes = ctx.cluster.remotes.keys() + + try: + for rem in remotes: + log.info(rem) + _install_repokey(rem) + _install_repo(rem, pkgdir, username, password) + yield + + finally: + for rem in remotes: + _remove_repo(rem) + + +@contextlib.contextmanager +def restapi(ctx, config): + """ + task restapi + + Calamari Rest API + + For example:: + + tasks: + - calamari.restapi: + roles: mon.0 + """ + overrides = ctx.config.get('overrides', {}) + teuthology.deep_merge(config, overrides.get('calamari.restapi', {})) + + remotes = teuthology.roles_to_remotes(ctx.cluster, config) + + try: + for rem in remotes: + log.info(rem) + _install_package('calamari-restapi', rem) + yield + + finally: + for rem in remotes: + _remove_package('calamari-restapi', rem) + + +@contextlib.contextmanager +def server(ctx, config): + """ + task server: + + Calamari server setup. "roles" is a list of roles that should run + the webapp, and "restapi_server" is a list of roles that will + be running the calamari-restapi package. Both lists probably should + have only one entry (only the first is used). + + For example:: + + roles: [[server.0], [mon.0], [osd.0, osd.1]] + tasks: + - calamari.server: + roles: [server.0] + restapi_server: [mon.0] + """ + overrides = ctx.config.get('overrides', {}) + teuthology.deep_merge(config, overrides.get('calamari.server', {})) + + remote = teuthology.roles_to_remotes(ctx.cluster, config)[0] + restapi_remote = teuthology.roles_to_remotes(ctx.cluster, config, + attrname='restapi_server')[0] + if not restapi_remote: + raise RuntimeError('Must supply restapi_server') + + try: + # sqlite3 command is required; on some platforms it's already + # there and not removable (required for, say yum) + sqlite_package = _sqlite_package_name(remote) + if sqlite_package and not _install_package(sqlite_package, remote): + raise RuntimeError('{} install failed'.format(sqlite_package)) + + if not _install_package('calamari-server', remote) or \ + not _install_package('calamari-clients', remote) or \ + not _disable_default_nginx(remote) or \ + not _setup_calamari_cluster(remote, restapi_remote): + raise RuntimeError('Server installation failure') + + log.info('client/server setup complete') + yield + finally: + _remove_package('calamari-server', remote) + _remove_package('calamari-clients', remote) + if sqlite_package: + _remove_package(sqlite_package, remote) + + +def test(ctx, config): + """ + task test + Run the calamari smoketest + Tests to run are in calamari_testdir. + delay: wait this long before starting + + tasks: + - calamari.test: + delay: 30 + server: server.0 + """ + delay = config.get('delay', 0) + if delay: + log.info("delaying %d sec", delay) + time.sleep(delay) + testhost = ctx.cluster.only(config['server']).remotes.keys()[0].name + testhost = testhost.split('@')[1] + mypath = os.path.dirname(__file__) + cmd_list = [os.path.join(mypath, 'calamari_testdir', + 'test_server_1_0.py')] + os.environ['CALAMARI_BASE_URI'] = 'http://{0}/api/v1/'.format(testhost) + log.info("testing %s", testhost) + return subprocess.call(cmd_list) diff --git a/teuthology/task/calamari_testdir/http_client.py b/teuthology/task/calamari_testdir/http_client.py new file mode 100755 index 0000000000..84a03c7bfa --- /dev/null +++ b/teuthology/task/calamari_testdir/http_client.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python + +import json +import logging +import requests + +log = logging.getLogger(__name__) + + +class AuthenticatedHttpClient(requests.Session): + """ + Client for the calamari REST API, principally exists to do + authentication, but also helpfully prefixes + URLs in requests with the API base URL and JSONizes + POST data. + """ + def __init__(self, api_url, username, password): + super(AuthenticatedHttpClient, self).__init__() + self._username = username + self._password = password + self._api_url = api_url + self.headers = { + 'Content-type': "application/json; charset=UTF-8" + } + + def request(self, method, url, **kwargs): + if not url.startswith('/'): + url = self._api_url + url + response = super(AuthenticatedHttpClient, self).request(method, url, **kwargs) + if response.status_code >= 400: + # For the benefit of test logs + print "%s: %s" % (response.status_code, response.content) + return response + + def post(self, url, data=None, **kwargs): + if isinstance(data, dict): + data = json.dumps(data) + return super(AuthenticatedHttpClient, self).post(url, data, **kwargs) + + def patch(self, url, data=None, **kwargs): + if isinstance(data, dict): + data = json.dumps(data) + return super(AuthenticatedHttpClient, self).patch(url, data, **kwargs) + + def login(self): + """ + Authenticate with the Django auth system as + it is exposed in the Calamari REST API. + """ + log.info("Logging in as %s" % self._username) + response = self.get("auth/login/") + response.raise_for_status() + self.headers['X-XSRF-TOKEN'] = response.cookies['XSRF-TOKEN'] + + self.post("auth/login/", { + 'next': "/", + 'username': self._username, + 'password': self._password + }) + response.raise_for_status() + + # Check we're allowed in now. + response = self.get("cluster") + response.raise_for_status() + +if __name__ == "__main__": + + import argparse + + p = argparse.ArgumentParser() + p.add_argument('-u', '--uri', default='http://mira035/api/v1/') + p.add_argument('--user', default='admin') + p.add_argument('--pass', dest='password', default='admin') + args, remainder = p.parse_known_args() + + c = AuthenticatedHttpClient(args.uri, args.user, args.password) + c.login() + response = c.request('GET', ''.join(remainder)).json() + print json.dumps(response, indent=2) diff --git a/teuthology/task/calamari_testdir/test_server_1_0.py b/teuthology/task/calamari_testdir/test_server_1_0.py new file mode 100755 index 0000000000..b9b07a3905 --- /dev/null +++ b/teuthology/task/calamari_testdir/test_server_1_0.py @@ -0,0 +1,269 @@ +#!/usr/bin/env python + +import datetime +import os +import logging +import logging.handlers +import requests +import uuid +import unittest +from http_client import AuthenticatedHttpClient + +log = logging.getLogger(__name__) +log.addHandler(logging.StreamHandler()) +log.setLevel(logging.INFO) + +global base_uri +global client +base_uri = None +server_uri = None +client = None + +def setUpModule(): + global base_uri + global server_uri + global client + try: + base_uri = os.environ['CALAMARI_BASE_URI'] + except KeyError: + log.error('Must define CALAMARI_BASE_URI') + os._exit(1) + if not base_uri.endswith('/'): + base_uri += '/' + if not base_uri.endswith('api/v1/'): + base_uri += 'api/v1/' + client = AuthenticatedHttpClient(base_uri, 'admin', 'admin') + server_uri = base_uri.replace('api/v1/', '') + client.login() + +class RestTest(unittest.TestCase): + 'Base class for all tests here; get class\'s data' + + def setUp(self): + # Called once for each test_* case. A bit wasteful, but we + # really like using the simple class variable self.uri + # to customize each derived TestCase + method = getattr(self, 'method', 'GET') + raw = self.uri.startswith('/') + self.response = self.get_object(method, self.uri, raw=raw) + + def get_object(self, method, url, raw=False): + global server_uri + 'Return Python object decoded from JSON response to method/url' + if not raw: + return client.request(method, url).json() + else: + return requests.request(method, server_uri + url).json() + +class TestUserMe(RestTest): + + uri = 'user/me' + + def test_me(self): + self.assertEqual(self.response['username'], 'admin') + +class TestCluster(RestTest): + + uri = 'cluster' + + def test_id(self): + self.assertEqual(self.response[0]['id'], 1) + + def test_times(self): + for time in ( + self.response[0]['cluster_update_time'], + self.response[0]['cluster_update_attempt_time'], + ): + self.assertTrue(is_datetime(time)) + + def test_api_base_url(self): + api_base_url = self.response[0]['api_base_url'] + self.assertTrue(api_base_url.startswith('http')) + self.assertIn('api/v0.1', api_base_url) + +class TestHealth(RestTest): + + uri = 'cluster/1/health' + + def test_cluster(self): + self.assertEqual(self.response['cluster'], 1) + + def test_times(self): + for time in ( + self.response['cluster_update_time'], + self.response['added'], + ): + self.assertTrue(is_datetime(time)) + + def test_report_and_overall_status(self): + self.assertIn('report', self.response) + self.assertIn('overall_status', self.response['report']) + +class TestHealthCounters(RestTest): + + uri = 'cluster/1/health_counters' + + def test_cluster(self): + self.assertEqual(self.response['cluster'], 1) + + def test_time(self): + self.assertTrue(is_datetime(self.response['cluster_update_time'])) + + def test_existence(self): + for section in ('pg', 'mon', 'osd'): + for counter in ('warn', 'critical', 'ok'): + count = self.response[section][counter]['count'] + self.assertIsInstance(count, int) + self.assertIsInstance(self.response['pool']['total'], int) + + def test_mds_sum(self): + count = self.response['mds'] + self.assertEqual( + count['up_not_in'] + count['not_up_not_in'] + count['up_in'], + count['total'] + ) + +class TestSpace(RestTest): + + uri = 'cluster/1/space' + + def test_cluster(self): + self.assertEqual(self.response['cluster'], 1) + + def test_times(self): + for time in ( + self.response['cluster_update_time'], + self.response['added'], + ): + self.assertTrue(is_datetime(time)) + + def test_space(self): + for size in ('free_bytes', 'used_bytes', 'capacity_bytes'): + self.assertIsInstance(self.response['space'][size], int) + self.assertGreater(self.response['space'][size], 0) + + def test_report(self): + for size in ('total_used', 'total_space', 'total_avail'): + self.assertIsInstance(self.response['report'][size], int) + self.assertGreater(self.response['report'][size], 0) + +class TestOSD(RestTest): + + uri = 'cluster/1/osd' + + def test_cluster(self): + self.assertEqual(self.response['cluster'], 1) + + def test_times(self): + for time in ( + self.response['cluster_update_time'], + self.response['added'], + ): + self.assertTrue(is_datetime(time)) + + def test_osd_uuid(self): + for osd in self.response['osds']: + uuidobj = uuid.UUID(osd['uuid']) + self.assertEqual(str(uuidobj), osd['uuid']) + + def test_osd_pools(self): + for osd in self.response['osds']: + if osd['up'] != 1: + continue + self.assertIsInstance(osd['pools'], list) + self.assertIsInstance(osd['pools'][0], basestring) + + def test_osd_up_in(self): + for osd in self.response['osds']: + for flag in ('up', 'in'): + self.assertIn(osd[flag], (0, 1)) + + def test_osd_0(self): + osd0 = self.get_object('GET', 'cluster/1/osd/0')['osd'] + for field in osd0.keys(): + if not field.startswith('cluster_update_time'): + self.assertEqual(self.response['osds'][0][field], osd0[field]) + +class TestPool(RestTest): + + uri = 'cluster/1/pool' + + def test_cluster(self): + for pool in self.response: + self.assertEqual(pool['cluster'], 1) + + def test_fields_are_ints(self): + for pool in self.response: + for field in ('id', 'used_objects', 'used_bytes'): + self.assertIsInstance(pool[field], int) + + def test_name_is_str(self): + for pool in self.response: + self.assertIsInstance(pool['name'], basestring) + + def test_pool_0(self): + poolid = self.response[0]['id'] + pool = self.get_object('GET', 'cluster/1/pool/{id}'.format(id=poolid)) + self.assertEqual(self.response[0], pool) + +class TestServer(RestTest): + + uri = 'cluster/1/server' + + def test_ipaddr(self): + for server in self.response: + octets = server['addr'].split('.') + self.assertEqual(len(octets), 4) + for octetstr in octets: + octet = int(octetstr) + self.assertIsInstance(octet, int) + self.assertGreaterEqual(octet, 0) + self.assertLessEqual(octet, 255) + + def test_hostname_name_strings(self): + for server in self.response: + for field in ('name', 'hostname'): + self.assertIsInstance(server[field], basestring) + + def test_services(self): + for server in self.response: + self.assertIsInstance(server['services'], list) + for service in server['services']: + self.assertIn(service['type'], ('osd', 'mon', 'mds')) + +class TestGraphitePoolIOPS(RestTest): + + uri = '/graphite/render?format=json-array&' \ + 'target=ceph.cluster.ceph.pool.0.num_read&' \ + 'target=ceph.cluster.ceph.pool.0.num_write' + + def test_targets_contain_request(self): + self.assertIn('targets', self.response) + self.assertIn('ceph.cluster.ceph.pool.0.num_read', + self.response['targets']) + self.assertIn('ceph.cluster.ceph.pool.0.num_write', + self.response['targets']) + + def test_datapoints(self): + self.assertIn('datapoints', self.response) + self.assertGreater(len(self.response['datapoints']), 0) + data = self.response['datapoints'][0] + self.assertEqual(len(data), 3) + self.assertIsInstance(data[0], int) + if data[1]: + self.assertIsInstance(data[1], float) + if data[2]: + self.assertIsInstance(data[2], float) + +# +# Utility functions +# + +DATETIME_FORMAT = '%Y-%m-%dT%H:%M:%S.%fZ' + +def is_datetime(time): + datetime.datetime.strptime(time, DATETIME_FORMAT) + return True + +if __name__ == '__main__': + unittest.main()