]> git-server-git.apps.pok.os.sepia.ceph.com Git - teuthology.git/commitdiff
Add calamari setup/test tasks
authorDan Mick <dan.mick@inktank.com>
Tue, 11 Feb 2014 04:31:33 +0000 (20:31 -0800)
committerDan Mick <dan.mick@inktank.com>
Tue, 11 Feb 2014 21:34:31 +0000 (13:34 -0800)
Signed-off-by: Warren Usui <warren.usui@inktank.com>
Signed-off-by: Dan Mick <dan.mick@inktank.com>
teuthology/task/calamari.py [new file with mode: 0644]
teuthology/task/calamari_testdir/http_client.py [new file with mode: 0755]
teuthology/task/calamari_testdir/test_server_1_0.py [new file with mode: 0755]

diff --git a/teuthology/task/calamari.py b/teuthology/task/calamari.py
new file mode 100644 (file)
index 0000000..845056e
--- /dev/null
@@ -0,0 +1,460 @@
+"""
+calamari - set up various machines with roles for participating
+in Calamari management app testing.  Requires secret info for
+accessing authenticated package repos to install Calamari, supplied
+in an override: clause for calamari.reposetup below.  Contains
+five tasks:
+
+- calamari.reposetup: set up the calamari package repos (all targets)
+- calamari.agent: install stats collection daemons (all cluster targets)
+- calamari.restapi: cluster-management api access (one monitor target)
+- calamari.server: main webapp/gui front end (management target)
+- calamari.test: run automated test against calamari.server target (local)
+
+calamari.test runs on the local machine, as it accesses the Calamari
+server across https using requests.py, which must be present.  It uses
+several external modules in calamari_test/.
+"""
+
+from cStringIO import StringIO
+import contextlib
+import logging
+import os
+import subprocess
+import teuthology.misc as teuthology
+import textwrap
+import time
+from ..orchestra import run
+
+log = logging.getLogger(__name__)
+
+
+def _edit_diamond_config(remote, serverhost):
+    """ Edit remote's diamond config to send stats to serverhost """
+    ret = remote.run(args=['sudo',
+                     'sed',
+                     '-i',
+                     's/calamari/{host}/'.format(host=serverhost),
+                     '/etc/diamond/diamond.conf'],
+                     stdout=StringIO())
+    if not ret:
+        return False
+    return remote.run(args=['sudo', 'service', 'diamond', 'restart'])
+
+
+def _disable_default_nginx(remote):
+    """
+    Fix up nginx values
+    """
+    script = textwrap.dedent('''
+        if [ -f /etc/nginx/conf.d/default.conf ]; then
+            mv /etc/nginx/conf.d/default.conf \
+                /etc/nginx/conf.d/default.disabled
+        fi
+        if [ -f /etc/nginx/sites-enabled/default ] ; then
+            rm /etc/nginx/sites-enabled/default
+        fi
+        service nginx restart
+        service {service} restart
+    ''')
+    service = _http_service_name(remote)
+    script = script.format(service=service)
+    teuthology.sudo_write_file(remote, '/tmp/disable.nginx', script)
+    return remote.run(args=['sudo', 'bash', '/tmp/disable.nginx'],
+                      stdout=StringIO())
+
+
+def _setup_calamari_cluster(remote, restapi_remote):
+    """
+    Add restapi db entry to the server.
+    """
+    restapi_hostname = str(restapi_remote).split('@')[1]
+    sqlcmd = 'insert into ceph_cluster (name, api_base_url) ' \
+             'values ("{host}", "http://{host}:5000/api/v0.1/");'. \
+             format(host=restapi_hostname)
+    teuthology.write_file(remote, '/tmp/create.cluster.sql', sqlcmd)
+    return remote.run(args=['cat',
+                            '/tmp/create.cluster.sql',
+                            run.Raw('|'),
+                            'sudo',
+                            'sqlite3',
+                            '/opt/calamari/webapp/calamari/db.sqlite3'],
+                      stdout=StringIO())
+
+
+RELEASE_MAP = {
+    'Ubuntu precise': dict(flavor='deb', release='ubuntu', version='precise'),
+    'Debian wheezy': dict(flavor='deb', release='debian', version='wheezy'),
+    'CentOS 6.4': dict(flavor='rpm', release='centos', version='6.4'),
+    'RedHatEnterpriseServer 6.4': dict(flavor='rpm', release='rhel',
+                                       version='6.4'),
+}
+
+
+def _get_relmap(rem):
+    relmap = getattr(rem, 'relmap', None)
+    if relmap is not None:
+        return relmap
+    lsb_release_out = StringIO()
+    rem.run(args=['lsb_release', '-ics'], stdout=lsb_release_out)
+    release = lsb_release_out.getvalue().replace('\n', ' ').rstrip()
+    if release in RELEASE_MAP:
+        rem.relmap = RELEASE_MAP[release]
+        return rem.relmap
+    else:
+        lsb_release_out = StringIO()
+        rem.run(args=['lsb_release', '-irs'], stdout=lsb_release_out)
+        release = lsb_release_out.getvalue().replace('\n', ' ').rstrip()
+        if release in RELEASE_MAP:
+            rem.relmap = RELEASE_MAP[release]
+            return rem.relmap
+    raise RuntimeError('Can\'t get release info for {}'.format(rem))
+
+
+def _sqlite_package_name(rem):
+    name = 'sqlite3' if _get_relmap(rem)['flavor'] == 'deb' else None
+    return name
+
+
+def _http_service_name(rem):
+    name = 'httpd' if _get_relmap(rem)['flavor'] == 'rpm' else 'apache2'
+    return name
+
+
+def _install_repo(remote, pkgdir, username, password):
+    # installing repo is assumed to be idempotent
+
+    relmap = _get_relmap(remote)
+    log.info('Installing repo on %s', remote)
+    if relmap['flavor'] == 'deb':
+        contents = 'deb https://{username}:{password}@download.inktank.com/' \
+                   '{pkgdir}/deb {codename} main'
+        contents = contents.format(username=username, password=password,
+                                   pkgdir=pkgdir, codename=relmap['version'],)
+        teuthology.sudo_write_file(remote,
+                                   '/etc/apt/sources.list.d/inktank.list',
+                                   contents)
+        remote.run(args=['sudo',
+                         'apt-get',
+                         'install',
+                         'apt-transport-https',
+                         '-y'])
+        result = remote.run(args=['sudo', 'apt-get', 'update', '-y'],
+                            stdout=StringIO())
+        return True
+
+    elif relmap['flavor'] == 'rpm':
+        baseurl = 'https://{username}:{password}@download.inktank.com/' \
+                  '{pkgdir}/rpm/{release}{version}'
+        contents = textwrap.dedent('''
+            [inktank]
+            name=Inktank Storage, Inc.
+            baseurl={baseurl}
+            gpgcheck=1
+            enabled=1
+            '''.format(baseurl=baseurl))
+        contents = contents.format(username=username,
+                                   password=password,
+                                   pkgdir=pkgdir,
+                                   release=relmap['release'],
+                                   version=relmap['version'])
+        teuthology.sudo_write_file(remote,
+                                   '/etc/yum.repos.d/inktank.repo',
+                                   contents)
+        return remote.run(args=['sudo', 'yum', 'makecache'])
+
+    else:
+        return False
+
+
+def _remove_repo(remote):
+    log.info('Removing repo on %s', remote)
+    flavor = _get_relmap(remote)['flavor']
+    if flavor == 'deb':
+        teuthology.delete_file(remote, '/etc/apt/sources.list.d/inktank.list',
+                               sudo=True, force=True)
+        result = remote.run(args=['sudo', 'apt-get', 'update', '-y'],
+                            stdout=StringIO())
+        return True
+
+    elif flavor == 'rpm':
+        teuthology.delete_file(remote, '/etc/yum.repos.d/inktank.repo',
+                               sudo=True, force=True)
+        return remote.run(args=['sudo', 'yum', 'makecache'])
+
+    else:
+        return False
+
+
+def _install_repokey(remote):
+    # installing keys is assumed to be idempotent
+    log.info('Installing repo key on %s', remote)
+    flavor = _get_relmap(remote)['flavor']
+    if flavor == 'deb':
+        return remote.run(args=['wget',
+                                '-q',
+                                '-O-',
+                                'http://download.inktank.com/keys/release.asc',
+                                run.Raw('|'),
+                                'sudo',
+                                'apt-key',
+                                'add',
+                                '-'])
+    elif flavor == 'rpm':
+        args = ['sudo', 'rpm', '--import',
+                'http://download.inktank.com/keys/release.asc']
+        return remote.run(args=args)
+    else:
+        return False
+
+
+def _install_package(package, remote):
+    """
+    package: name
+    remote: Remote() to install on
+    release: deb only, 'precise' or 'wheezy'
+    pkgdir: may or may not include a branch name, so, say, either
+            packages or packages-staging/master
+    """
+    log.info('Installing package %s on %s', package, remote)
+    flavor = _get_relmap(remote)['flavor']
+    if flavor == 'deb':
+        pkgcmd = ['DEBIAN_FRONTEND=noninteractive',
+                  'sudo',
+                  '-E',
+                  'apt-get',
+                  '-y',
+                  'install',
+                  '{package}'.format(package=package)]
+    elif flavor == 'rpm':
+        pkgcmd = ['sudo',
+                  'yum',
+                  '-y',
+                  'install',
+                  '{package}'.format(package=package)]
+    else:
+        log.error('_install_package: bad flavor ' + flavor + '\n')
+        return False
+    return remote.run(args=pkgcmd)
+
+
+def _remove_package(package, remote):
+    """
+    remove package from remote
+    """
+    flavor = _get_relmap(remote)['flavor']
+    if flavor == 'deb':
+        pkgcmd = ['DEBIAN_FRONTEND=noninteractive',
+                  'sudo',
+                  '-E',
+                  'apt-get',
+                  '-y',
+                  'purge',
+                  '{package}'.format(package=package)]
+    elif flavor == 'rpm':
+        pkgcmd = ['sudo',
+                  'yum',
+                  '-y',
+                  'erase',
+                  '{package}'.format(package=package)]
+    else:
+        log.error('_remove_package: bad flavor ' + flavor + '\n')
+        return False
+    return remote.run(args=pkgcmd)
+
+"""
+Tasks
+"""
+
+
+@contextlib.contextmanager
+def agent(ctx, config):
+    """
+    task agent
+    calamari.agent: install stats collection (for each cluster host)
+
+    For example::
+
+        tasks:
+        - calamari.agent:
+           roles:
+               - mon.0
+               - osd.0
+               - osd.1
+           server: server.0
+    """
+
+    log.info('calamari.agent starting')
+    overrides = ctx.config.get('overrides', {})
+    teuthology.deep_merge(config, overrides.get('calamari.agent', {}))
+
+    remotes = teuthology.roles_to_remotes(ctx.cluster, config)
+    try:
+        for rem in remotes:
+            log.info('Installing calamari-agent on %s', rem)
+            _install_package('calamari-agent', rem)
+            server_role = config.get('server')
+            if not server_role:
+                raise RuntimeError('must supply \'server\' config key')
+            server_remote = ctx.cluster.only(server_role).remotes.keys()[0]
+            # why isn't shortname available by default?
+            serverhost = server_remote.name.split('@')[1]
+            log.info('configuring Diamond for {}'.format(serverhost))
+            if not _edit_diamond_config(rem, serverhost):
+                raise RuntimeError(
+                    'Diamond config edit failed on {0}'.format(rem)
+                )
+        yield
+    finally:
+            for rem in remotes:
+                _remove_package('calamari-agent', rem)
+
+
+@contextlib.contextmanager
+def reposetup(ctx, config):
+    """
+    task reposetup
+    Sets up calamari repository on all remotes; cleans up when done
+
+    calamari.reposetup:
+        pkgdir:
+        username:
+        password:
+
+    Supply the above in an override file if you need to manage the
+    secret repo credentials separately from the test definition (likely).
+
+    pkgdir encodes package directory (possibly more than one path component)
+    as in https://<username>:<password>@SERVER/<pkgdir>/{deb,rpm}{..}
+
+    """
+    overrides = ctx.config.get('overrides', {})
+    # XXX deep_merge returns the result, which matters if either is None
+    # make sure that doesn't happen
+    if config is None:
+        config = {'dummy': 'dummy'}
+    teuthology.deep_merge(config, overrides.get('calamari.reposetup', {}))
+
+    try:
+        pkgdir = config['pkgdir']
+        username = config['username']
+        password = config['password']
+    except KeyError:
+        raise RuntimeError('requires pkgdir, username, and password')
+
+    remotes = ctx.cluster.remotes.keys()
+
+    try:
+        for rem in remotes:
+            log.info(rem)
+            _install_repokey(rem)
+            _install_repo(rem, pkgdir, username, password)
+        yield
+
+    finally:
+        for rem in remotes:
+            _remove_repo(rem)
+
+
+@contextlib.contextmanager
+def restapi(ctx, config):
+    """
+    task restapi
+
+    Calamari Rest API
+
+    For example::
+
+        tasks:
+        - calamari.restapi:
+          roles: mon.0
+    """
+    overrides = ctx.config.get('overrides', {})
+    teuthology.deep_merge(config, overrides.get('calamari.restapi', {}))
+
+    remotes = teuthology.roles_to_remotes(ctx.cluster, config)
+
+    try:
+        for rem in remotes:
+            log.info(rem)
+            _install_package('calamari-restapi', rem)
+        yield
+
+    finally:
+        for rem in remotes:
+            _remove_package('calamari-restapi', rem)
+
+
+@contextlib.contextmanager
+def server(ctx, config):
+    """
+    task server:
+
+    Calamari server setup.  "roles" is a list of roles that should run
+    the webapp, and "restapi_server" is a list of roles that will
+    be running the calamari-restapi package.  Both lists probably should
+    have only one entry (only the first is used).
+
+    For example::
+
+        roles: [[server.0], [mon.0], [osd.0, osd.1]]
+        tasks:
+        - calamari.server:
+            roles: [server.0]
+            restapi_server: [mon.0]
+    """
+    overrides = ctx.config.get('overrides', {})
+    teuthology.deep_merge(config, overrides.get('calamari.server', {}))
+
+    remote = teuthology.roles_to_remotes(ctx.cluster, config)[0]
+    restapi_remote = teuthology.roles_to_remotes(ctx.cluster, config,
+                                                 attrname='restapi_server')[0]
+    if not restapi_remote:
+        raise RuntimeError('Must supply restapi_server')
+
+    try:
+        # sqlite3 command is required; on some platforms it's already
+        # there and not removable (required for, say yum)
+        sqlite_package = _sqlite_package_name(remote)
+        if sqlite_package and not _install_package(sqlite_package, remote):
+            raise RuntimeError('{} install failed'.format(sqlite_package))
+
+        if not _install_package('calamari-server', remote) or \
+           not _install_package('calamari-clients', remote) or \
+           not _disable_default_nginx(remote) or \
+           not _setup_calamari_cluster(remote, restapi_remote):
+            raise RuntimeError('Server installation failure')
+
+        log.info('client/server setup complete')
+        yield
+    finally:
+        _remove_package('calamari-server', remote)
+        _remove_package('calamari-clients', remote)
+        if sqlite_package:
+            _remove_package(sqlite_package, remote)
+
+
+def test(ctx, config):
+    """
+    task test
+    Run the calamari smoketest
+    Tests to run are in calamari_testdir.
+    delay: wait this long before starting
+
+        tasks:
+        - calamari.test:
+            delay: 30
+            server: server.0
+    """
+    delay = config.get('delay', 0)
+    if delay:
+        log.info("delaying %d sec", delay)
+        time.sleep(delay)
+    testhost = ctx.cluster.only(config['server']).remotes.keys()[0].name
+    testhost = testhost.split('@')[1]
+    mypath = os.path.dirname(__file__)
+    cmd_list = [os.path.join(mypath, 'calamari_testdir',
+                             'test_server_1_0.py')]
+    os.environ['CALAMARI_BASE_URI'] = 'http://{0}/api/v1/'.format(testhost)
+    log.info("testing %s", testhost)
+    return subprocess.call(cmd_list)
diff --git a/teuthology/task/calamari_testdir/http_client.py b/teuthology/task/calamari_testdir/http_client.py
new file mode 100755 (executable)
index 0000000..84a03c7
--- /dev/null
@@ -0,0 +1,79 @@
+#!/usr/bin/env python
+
+import json
+import logging
+import requests
+
+log = logging.getLogger(__name__)
+
+
+class AuthenticatedHttpClient(requests.Session):
+    """
+    Client for the calamari REST API, principally exists to do
+    authentication, but also helpfully prefixes
+    URLs in requests with the API base URL and JSONizes
+    POST data.
+    """
+    def __init__(self, api_url, username, password):
+        super(AuthenticatedHttpClient, self).__init__()
+        self._username = username
+        self._password = password
+        self._api_url = api_url
+        self.headers = {
+            'Content-type': "application/json; charset=UTF-8"
+        }
+
+    def request(self, method, url, **kwargs):
+        if not url.startswith('/'):
+            url = self._api_url + url
+        response = super(AuthenticatedHttpClient, self).request(method, url, **kwargs)
+        if response.status_code >= 400:
+            # For the benefit of test logs
+            print "%s: %s" % (response.status_code, response.content)
+        return response
+
+    def post(self, url, data=None, **kwargs):
+        if isinstance(data, dict):
+            data = json.dumps(data)
+        return super(AuthenticatedHttpClient, self).post(url, data, **kwargs)
+
+    def patch(self, url, data=None, **kwargs):
+        if isinstance(data, dict):
+            data = json.dumps(data)
+        return super(AuthenticatedHttpClient, self).patch(url, data, **kwargs)
+
+    def login(self):
+        """
+        Authenticate with the Django auth system as
+        it is exposed in the Calamari REST API.
+        """
+        log.info("Logging in as %s" % self._username)
+        response = self.get("auth/login/")
+        response.raise_for_status()
+        self.headers['X-XSRF-TOKEN'] = response.cookies['XSRF-TOKEN']
+
+        self.post("auth/login/", {
+            'next': "/",
+            'username': self._username,
+            'password': self._password
+        })
+        response.raise_for_status()
+
+        # Check we're allowed in now.
+        response = self.get("cluster")
+        response.raise_for_status()
+
+if __name__ == "__main__":
+
+    import argparse
+
+    p = argparse.ArgumentParser()
+    p.add_argument('-u', '--uri', default='http://mira035/api/v1/')
+    p.add_argument('--user', default='admin')
+    p.add_argument('--pass', dest='password', default='admin')
+    args, remainder = p.parse_known_args()
+
+    c = AuthenticatedHttpClient(args.uri, args.user, args.password)
+    c.login()
+    response = c.request('GET', ''.join(remainder)).json()
+    print json.dumps(response, indent=2)
diff --git a/teuthology/task/calamari_testdir/test_server_1_0.py b/teuthology/task/calamari_testdir/test_server_1_0.py
new file mode 100755 (executable)
index 0000000..b9b07a3
--- /dev/null
@@ -0,0 +1,269 @@
+#!/usr/bin/env python
+
+import datetime
+import os
+import logging
+import logging.handlers
+import requests
+import uuid
+import unittest
+from http_client import AuthenticatedHttpClient
+
+log = logging.getLogger(__name__)
+log.addHandler(logging.StreamHandler())
+log.setLevel(logging.INFO)
+
+global base_uri
+global client
+base_uri = None
+server_uri = None
+client = None
+
+def setUpModule():
+    global base_uri
+    global server_uri
+    global client
+    try:
+        base_uri = os.environ['CALAMARI_BASE_URI']
+    except KeyError:
+        log.error('Must define CALAMARI_BASE_URI')
+        os._exit(1)
+    if not base_uri.endswith('/'):
+        base_uri += '/'
+    if not base_uri.endswith('api/v1/'):
+        base_uri += 'api/v1/'
+    client = AuthenticatedHttpClient(base_uri, 'admin', 'admin')
+    server_uri = base_uri.replace('api/v1/', '')
+    client.login()
+
+class RestTest(unittest.TestCase):
+    'Base class for all tests here; get class\'s data'
+
+    def setUp(self):
+        # Called once for each test_* case.  A bit wasteful, but we
+        # really like using the simple class variable self.uri
+        # to customize each derived TestCase
+        method = getattr(self, 'method', 'GET')
+        raw = self.uri.startswith('/')
+        self.response = self.get_object(method, self.uri, raw=raw)
+
+    def get_object(self, method, url, raw=False):
+        global server_uri
+        'Return Python object decoded from JSON response to method/url'
+        if not raw:
+            return client.request(method, url).json()
+        else:
+            return requests.request(method, server_uri + url).json()
+
+class TestUserMe(RestTest):
+
+    uri = 'user/me'
+
+    def test_me(self):
+        self.assertEqual(self.response['username'], 'admin')
+
+class TestCluster(RestTest):
+
+    uri = 'cluster'
+
+    def test_id(self):
+        self.assertEqual(self.response[0]['id'], 1)
+
+    def test_times(self):
+        for time in (
+            self.response[0]['cluster_update_time'],
+            self.response[0]['cluster_update_attempt_time'],
+        ):
+            self.assertTrue(is_datetime(time))
+
+    def test_api_base_url(self):
+        api_base_url = self.response[0]['api_base_url']
+        self.assertTrue(api_base_url.startswith('http'))
+        self.assertIn('api/v0.1', api_base_url)
+
+class TestHealth(RestTest):
+
+    uri = 'cluster/1/health'
+
+    def test_cluster(self):
+        self.assertEqual(self.response['cluster'], 1)
+
+    def test_times(self):
+        for time in (
+            self.response['cluster_update_time'],
+            self.response['added'],
+        ):
+            self.assertTrue(is_datetime(time))
+
+    def test_report_and_overall_status(self):
+        self.assertIn('report', self.response)
+        self.assertIn('overall_status', self.response['report'])
+
+class TestHealthCounters(RestTest):
+
+    uri = 'cluster/1/health_counters'
+
+    def test_cluster(self):
+        self.assertEqual(self.response['cluster'], 1)
+
+    def test_time(self):
+        self.assertTrue(is_datetime(self.response['cluster_update_time']))
+
+    def test_existence(self):
+        for section in ('pg', 'mon', 'osd'):
+            for counter in ('warn', 'critical', 'ok'):
+                count = self.response[section][counter]['count']
+                self.assertIsInstance(count, int)
+        self.assertIsInstance(self.response['pool']['total'], int)
+
+    def test_mds_sum(self):
+        count = self.response['mds']
+        self.assertEqual(
+            count['up_not_in'] + count['not_up_not_in'] + count['up_in'],
+            count['total']
+        )
+
+class TestSpace(RestTest):
+
+    uri = 'cluster/1/space'
+
+    def test_cluster(self):
+        self.assertEqual(self.response['cluster'], 1)
+
+    def test_times(self):
+        for time in (
+            self.response['cluster_update_time'],
+            self.response['added'],
+        ):
+            self.assertTrue(is_datetime(time))
+
+    def test_space(self):
+        for size in ('free_bytes', 'used_bytes', 'capacity_bytes'):
+            self.assertIsInstance(self.response['space'][size], int)
+            self.assertGreater(self.response['space'][size], 0)
+
+    def test_report(self):
+        for size in ('total_used', 'total_space', 'total_avail'):
+            self.assertIsInstance(self.response['report'][size], int)
+            self.assertGreater(self.response['report'][size], 0)
+
+class TestOSD(RestTest):
+
+    uri = 'cluster/1/osd'
+
+    def test_cluster(self):
+        self.assertEqual(self.response['cluster'], 1)
+
+    def test_times(self):
+        for time in (
+            self.response['cluster_update_time'],
+            self.response['added'],
+        ):
+            self.assertTrue(is_datetime(time))
+
+    def test_osd_uuid(self):
+        for osd in self.response['osds']:
+            uuidobj = uuid.UUID(osd['uuid'])
+            self.assertEqual(str(uuidobj), osd['uuid'])
+
+    def test_osd_pools(self):
+        for osd in self.response['osds']:
+            if osd['up'] != 1:
+                continue
+            self.assertIsInstance(osd['pools'], list)
+            self.assertIsInstance(osd['pools'][0], basestring)
+
+    def test_osd_up_in(self):
+        for osd in self.response['osds']:
+            for flag in ('up', 'in'):
+                self.assertIn(osd[flag], (0, 1))
+
+    def test_osd_0(self):
+        osd0 = self.get_object('GET', 'cluster/1/osd/0')['osd']
+        for field in osd0.keys():
+            if not field.startswith('cluster_update_time'):
+                self.assertEqual(self.response['osds'][0][field], osd0[field])
+
+class TestPool(RestTest):
+
+    uri = 'cluster/1/pool'
+
+    def test_cluster(self):
+        for pool in self.response:
+            self.assertEqual(pool['cluster'], 1)
+
+    def test_fields_are_ints(self):
+        for pool in self.response:
+            for field in ('id', 'used_objects', 'used_bytes'):
+                self.assertIsInstance(pool[field], int)
+
+    def test_name_is_str(self):
+        for pool in self.response:
+            self.assertIsInstance(pool['name'], basestring)
+
+    def test_pool_0(self):
+        poolid = self.response[0]['id']
+        pool = self.get_object('GET', 'cluster/1/pool/{id}'.format(id=poolid))
+        self.assertEqual(self.response[0], pool)
+
+class TestServer(RestTest):
+
+    uri = 'cluster/1/server'
+
+    def test_ipaddr(self):
+        for server in self.response:
+            octets = server['addr'].split('.')
+            self.assertEqual(len(octets), 4)
+            for octetstr in octets:
+                octet = int(octetstr)
+                self.assertIsInstance(octet, int)
+                self.assertGreaterEqual(octet, 0)
+                self.assertLessEqual(octet, 255)
+
+    def test_hostname_name_strings(self):
+        for server in self.response:
+            for field in ('name', 'hostname'):
+                self.assertIsInstance(server[field], basestring)
+
+    def test_services(self):
+        for server in self.response:
+            self.assertIsInstance(server['services'], list)
+            for service in server['services']:
+                self.assertIn(service['type'], ('osd', 'mon', 'mds'))
+
+class TestGraphitePoolIOPS(RestTest):
+
+    uri = '/graphite/render?format=json-array&' \
+          'target=ceph.cluster.ceph.pool.0.num_read&' \
+          'target=ceph.cluster.ceph.pool.0.num_write'
+
+    def test_targets_contain_request(self):
+        self.assertIn('targets', self.response)
+        self.assertIn('ceph.cluster.ceph.pool.0.num_read',
+                      self.response['targets'])
+        self.assertIn('ceph.cluster.ceph.pool.0.num_write',
+                      self.response['targets'])
+
+    def test_datapoints(self):
+        self.assertIn('datapoints', self.response)
+        self.assertGreater(len(self.response['datapoints']), 0)
+        data = self.response['datapoints'][0]
+        self.assertEqual(len(data), 3)
+        self.assertIsInstance(data[0], int)
+        if data[1]:
+            self.assertIsInstance(data[1], float)
+        if data[2]:
+            self.assertIsInstance(data[2], float)
+
+#
+# Utility functions
+#
+
+DATETIME_FORMAT = '%Y-%m-%dT%H:%M:%S.%fZ'
+
+def is_datetime(time):
+    datetime.datetime.strptime(time, DATETIME_FORMAT)
+    return True
+
+if __name__ == '__main__':
+    unittest.main()