qa/tasks: run cosbench using the CBT task

Signed-off-by: Neha Ojha <nojha@redhat.com>

diff --git a/qa/tasks/cbt.py b/qa/tasks/cbt.py
index 40b8a08a7156877a534a96ab412e8c5ab06de56d..fd2550e5865403c68a9ed5fed462d79d8af9a211 100644 (file)
--- a/qa/tasks/cbt.py
+++ b/qa/tasks/cbt.py
@@ -25,24 +25,44 @@ class CBT(Task):
         mon_hosts = self.hosts_of_type('mon')
         osd_hosts = self.hosts_of_type('osd')
         client_hosts = self.hosts_of_type('client')
+        rgw_client = {}
+        rgw_client[client_hosts[0]] = None
+        rgw_hosts = self.config.get('cluster', {}).get('rgws', rgw_client)
         cluster_config = dict(
             user=self.config.get('cluster', {}).get('user', 'ubuntu'),
             head=mon_hosts[0],
             osds=osd_hosts,
             mons=mon_hosts,
             clients=client_hosts,
+            rgws=rgw_hosts,
             osds_per_node=self.config.get('cluster', {}).get('osds_per_node', 1),
             rebuild_every_test=False,
             use_existing=True,
+            is_teuthology=self.config.get('cluster', {}).get('is_teuthology', True),
             iterations=self.config.get('cluster', {}).get('iterations', 1),
             tmp_dir='/tmp/cbt',
             pool_profiles=self.config.get('cluster', {}).get('pool_profiles'),
             )
+
         benchmark_config = self.config.get('benchmarks')
         benchmark_type = benchmark_config.keys()[0]
         if benchmark_type == 'librbdfio':
           testdir = misc.get_testdir(self.ctx)
           benchmark_config['librbdfio']['cmd_path'] = os.path.join(testdir, 'fio/fio')
+        if benchmark_type == 'cosbench':
+            # create cosbench_dir and cosbench_xml_dir
+            testdir = misc.get_testdir(self.ctx)
+            benchmark_config['cosbench']['cosbench_dir'] = os.path.join(testdir, 'cos')
+            benchmark_config['cosbench']['cosbench_xml_dir'] = os.path.join(testdir, 'xml')
+            self.ctx.cluster.run(args=['mkdir', '-p', '-m0755', '--', benchmark_config['cosbench']['cosbench_xml_dir']])
+            benchmark_config['cosbench']['controller'] = osd_hosts[0]
+
+            # set auth details
+            remotes_and_roles = self.ctx.cluster.remotes.items()
+            ips = [host for (host, port) in
+                   (remote.ssh.get_transport().getpeername() for (remote, role_list) in remotes_and_roles)]
+            benchmark_config['cosbench']['auth'] = "username=cosbench:operator;password=intel2012;url=http://%s:7280/auth/v1.0;retry=9" %(ips[0])
+
         return dict(
             cluster=cluster_config,
             benchmarks=benchmark_config,
@@ -58,23 +78,76 @@ class CBT(Task):
             install_cmd = ['sudo', 'apt-get', '-y', '--force-yes', 'install']
             cbt_depends = ['python-yaml', 'python-lxml', 'librbd-dev', 'collectl']
         self.first_mon.run(args=install_cmd + cbt_depends)
-         
-        # install fio
-        testdir = misc.get_testdir(self.ctx)
-        self.first_mon.run(
-            args=[
-                'git', 'clone', '-b', 'master',
-                'https://github.com/axboe/fio.git',
-                '{tdir}/fio'.format(tdir=testdir)
-            ]
-        )
-        self.first_mon.run(
-            args=[
-                'cd', os.path.join(testdir, 'fio'), run.Raw('&&'),
-                './configure', run.Raw('&&'),
-                'make'
-            ]
-        )
+
+        benchmark_type = self.cbt_config.get('benchmarks').keys()[0]
+        self.log.info('benchmark: %s', benchmark_type)
+
+        if benchmark_type == 'librbdfio':
+            # install fio
+            testdir = misc.get_testdir(self.ctx)
+            self.first_mon.run(
+                args=[
+                    'git', 'clone', '-b', 'master',
+                    'https://github.com/axboe/fio.git',
+                    '{tdir}/fio'.format(tdir=testdir)
+                ]
+            )
+            self.first_mon.run(
+                args=[
+                    'cd', os.path.join(testdir, 'fio'), run.Raw('&&'),
+                    './configure', run.Raw('&&'),
+                    'make'
+                ]
+            )
+
+        if benchmark_type == 'cosbench':
+            # install cosbench
+            self.log.info('install dependecies for cosbench')
+            if system_type == 'rpm':
+                cosbench_depends = ['wget', 'unzip', 'java-1.7.0-openjdk', 'curl']
+            else:
+                cosbench_depends = ['wget', 'unzip', 'openjdk-8-jre', 'curl']
+            self.first_mon.run(args=install_cmd + cosbench_depends)
+            testdir = misc.get_testdir(self.ctx)
+            cosbench_version = '0.4.2.c3'
+            self.first_mon.run(
+                args=[
+                    'cd', testdir, run.Raw('&&'),
+                    'wget',
+                    'https://github.com/intel-cloud/cosbench/releases/download/v0.4.2.c3/0.4.2.c3.zip', run.Raw('&&'),
+                    'unzip', '0.4.2.c3.zip',
+                ]
+            )
+            self.first_mon.run(
+                args=[
+                    'cd', testdir, run.Raw('&&'),
+                    'ln', '-s', cosbench_version, 'cos',
+                ]
+            )
+            self.first_mon.run(
+                args=[
+                    'cd', os.path.join(testdir, 'cos'), run.Raw('&&'),
+                    'chmod', '+x', run.Raw('*.sh'),
+                ]
+            )
+
+            # start cosbench and check info
+            self.log.info('start cosbench')
+            self.first_mon.run(
+                args=[
+                    'cd', testdir, run.Raw('&&'),
+                    'cd', 'cos', run.Raw('&&'),
+                    'sh', 'start-all.sh'
+                ]
+            )
+            self.log.info('check cosbench info')
+            self.first_mon.run(
+                args=[
+                    'cd', testdir, run.Raw('&&'),
+                    'cd', 'cos', run.Raw('&&'),
+                    'sh', 'cli.sh', 'info'
+                ]
+            )
 
     def checkout_cbt(self):
         testdir = misc.get_testdir(self.ctx)
@@ -130,11 +203,40 @@ class CBT(Task):
                 '{tdir}/cbt'.format(tdir=testdir),
             ]
         )
-        self.first_mon.run(
-            args=[
-                'rm', '--one-file-system', '-rf', '--',
-                '{tdir}/fio'.format(tdir=testdir),
-            ]
-        )
+        benchmark_type = self.cbt_config.get('benchmarks').keys()[0]
+        if benchmark_type == 'librbdfio':
+            self.first_mon.run(
+                args=[
+                    'rm', '--one-file-system', '-rf', '--',
+                    '{tdir}/fio'.format(tdir=testdir),
+                ]
+            )
+
+        if benchmark_type == 'cosbench':
+            self.first_mon.run(
+                args=[
+                    'rm', '--one-file-system', '-rf', '--',
+                    '{tdir}/cos'.format(tdir=testdir),
+                ]
+            )
+            self.first_mon.run(
+                args=[
+                    'rm', '--one-file-system', '-rf', '--',
+                    '{tdir}/0.4.2.c3'.format(tdir=testdir),
+                ]
+            )
+            self.first_mon.run(
+                args=[
+                    'rm', '--one-file-system', '-rf', '--',
+                    '{tdir}/0.4.2.c3.zip'.format(tdir=testdir),
+                ]
+            )
+            self.first_mon.run(
+                args=[
+                    'rm', '--one-file-system', '-rf', '--',
+                    '{tdir}/xml'.format(tdir=testdir),
+                ]
+            )
+
 
 task = CBT

diff --git a/qa/tasks/rgw.py b/qa/tasks/rgw.py
index 24b7e89b2aaded977265ad12afdfb3c513a600ad..13c82736338d3531258f9a0f4eaa82e05e718e83 100644 (file)
--- a/qa/tasks/rgw.py
+++ b/qa/tasks/rgw.py
@@ -194,14 +194,18 @@ def create_pools(ctx, clients):
     for client in clients:
         log.debug("Obtaining remote for client {}".format(client))
         (remote,) = ctx.cluster.only(client).remotes.iterkeys()
-        data_pool = '.rgw.buckets'
+        data_pool = 'default.rgw.buckets.data'
         cluster_name, daemon_type, client_id = teuthology.split_role(client)
 
         if ctx.rgw.ec_data_pool:
-            create_ec_pool(remote, data_pool, client, 64,
+            create_ec_pool(remote, data_pool, client, ctx.rgw.data_pool_pg_size,
                            ctx.rgw.erasure_code_profile, cluster_name, 'rgw')
         else:
-            create_replicated_pool(remote, data_pool, 64, cluster_name, 'rgw')
+            create_replicated_pool(remote, data_pool, ctx.rgw.data_pool_pg_size, cluster_name, 'rgw')
+
+        index_pool = 'default.rgw.buckets.index'
+        create_replicated_pool(remote, index_pool, ctx.rgw.index_pool_pg_size, cluster_name, 'rgw')
+
         if ctx.rgw.cache_pools:
             create_cache_pool(remote, data_pool, data_pool + '.cache', 64,
                               64*1024*1024, cluster_name)
@@ -256,6 +260,13 @@ def task(ctx, config):
               valgrind: [--tool=memcheck]
             client.3:
               valgrind: [--tool=memcheck]
+
+    To configure data or index pool pg_size:
+
+        overrides:
+          rgw:
+            data_pool_pg_size: 256
+            index_pool_pg_size: 128
     """
     if config is None:
         config = dict(('client.{id}'.format(id=id_), None)
@@ -277,6 +288,8 @@ def task(ctx, config):
     ctx.rgw.frontend = config.pop('frontend', 'civetweb')
     ctx.rgw.compression_type = config.pop('compression type', None)
     default_cert = config.pop('ssl certificate', None)
+    ctx.rgw.data_pool_pg_size = config.pop('data_pool_pg_size', 64)
+    ctx.rgw.index_pool_pg_size = config.pop('index_pool_pg_size', 64)
     ctx.rgw.config = config
 
     log.debug("config is {}".format(config))