for its `Web UI`_. But please note, shaman does not build the
packages, it just offers information on the builds.
+ceph-perf-pull-requests
+-----------------------
+
+``ceph-perf-pull-requests`` runs CBT performance regression checks on dedicated
+``performance`` Jenkins agents. The job definition lives in `ceph-build`_ and
+generates two jobs from a single template: ``ceph-perf-classic`` and
+``ceph-perf-crimson``.
+
+A pull request can trigger a run with::
+
+ jenkins test classic perf
+ jenkins test crimson perf
+
+The ``performance`` label is also whitelisted for automatic runs. Each job:
+
+#. checks out ``ceph-main`` (``origin/main``) and the PR merge ref
+#. builds both trees (classic ``vstart-base`` or Crimson ``crimson-osd``)
+#. runs the checked-in ``radosbench_4K_read.yaml`` workload via ``run-cbt.sh``
+#. compares PR results against ``main`` with ``cbt/compare.py``
+#. publishes a GitHub check named ``perf-test-{classic,crimson}``
+
+The benchmark YAML is always taken from ``ceph-main`` so PR and baseline runs use
+the same workload definition. Teuthology-to-CBT translation is handled by
+``src/test/crimson/cbt/t2c.py`` in the Ceph tree (not patched at build time).
+
+.. _ceph-build: https://github.com/ceph/ceph-build
+
As the following shows, `chacra`_ manages multiple projects whose metadata
are stored in a database. These metadata are exposed via Shaman as a web
service. `chacractl`_ is a utility to interact with the `chacra`_ service.
#!/usr/bin/env python3
+"""
+Translate a teuthology-style benchmark YAML into a CBT configuration file.
+
+The YAML files under ``src/test/crimson/cbt/`` describe workloads using
+teuthology's ``tasks`` list. ``run-cbt.sh`` invokes this module to extract the
+``cbt`` task and emit a CBT-ready configuration (cluster layout, benchmark
+definitions, monitoring profiles).
+"""
from __future__ import print_function
import argparse
rados_cmd=os.path.join(self.build_dir, 'bin', 'rados'),
pid_dir=os.path.join(self.build_dir, 'out')
))
- return conf
+ return conf
def get_cbt_tasks(path):
- with open(path) as input:
- teuthology_config = yaml.load(input)
- for task in teuthology_config['tasks']:
+ with open(path) as yaml_file:
+ teuthology_config = yaml.safe_load(yaml_file) or {}
+ if not isinstance(teuthology_config, dict):
+ teuthology_config = {}
+ for task in teuthology_config.get('tasks', []):
+ if not isinstance(task, dict):
+ continue
for name, conf in task.items():
if name == 'cbt':
yield conf
--- /dev/null
+#!/usr/bin/env python3
+
+import os
+import sys
+import tempfile
+import unittest
+import unittest.mock
+
+import yaml
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+import t2c # noqa: E402
+
+
+SAMPLE_TEUTHOLOGY_YAML = """\
+meta:
+- desc: sample radosbench workload
+tasks:
+- install:
+ extra_system_packages:
+ deb:
+ - lvm2
+- cbt:
+ benchmarks:
+ radosbench:
+ read_time: 30
+ read_only: true
+ monitoring_profiles:
+ perf:
+ nodes:
+ - osds
+ cluster:
+ osds_per_node: 3
+ iterations: 1
+ pool_profiles:
+ replicated:
+ pg_size: 128
+ pgp_size: 128
+ replication: replicated
+"""
+
+
+class TestGetCbtTasks(unittest.TestCase):
+ def _write_yaml(self, contents):
+ handle = tempfile.NamedTemporaryFile(mode='w', suffix='.yaml',
+ delete=False)
+ handle.write(contents)
+ handle.close()
+ self.addCleanup(os.unlink, handle.name)
+ return handle.name
+
+ def test_extracts_cbt_task(self):
+ path = self._write_yaml(SAMPLE_TEUTHOLOGY_YAML)
+ tasks = list(t2c.get_cbt_tasks(path))
+ self.assertEqual(len(tasks), 1)
+ self.assertIn('benchmarks', tasks[0])
+ self.assertEqual(tasks[0]['benchmarks']['radosbench']['read_time'], 30)
+ self.assertEqual(tasks[0]['cluster']['osds_per_node'], 3)
+
+ def test_ignores_non_cbt_tasks(self):
+ path = self._write_yaml("tasks:\n- install:\n version: main\n")
+ tasks = list(t2c.get_cbt_tasks(path))
+ self.assertEqual(tasks, [])
+
+ def test_empty_file_returns_no_tasks(self):
+ path = self._write_yaml("")
+ tasks = list(t2c.get_cbt_tasks(path))
+ self.assertEqual(tasks, [])
+
+ def test_missing_tasks_key_returns_no_tasks(self):
+ path = self._write_yaml("meta:\n- desc: no tasks here\n")
+ tasks = list(t2c.get_cbt_tasks(path))
+ self.assertEqual(tasks, [])
+
+ def test_multiple_cbt_tasks(self):
+ path = self._write_yaml(
+ "tasks:\n"
+ "- cbt:\n"
+ " cluster:\n"
+ " iterations: 1\n"
+ "- cbt:\n"
+ " cluster:\n"
+ " iterations: 2\n")
+ tasks = list(t2c.get_cbt_tasks(path))
+ self.assertEqual(len(tasks), 2)
+ self.assertEqual(tasks[0]['cluster']['iterations'], 1)
+ self.assertEqual(tasks[1]['cluster']['iterations'], 2)
+
+
+class TestTranslator(unittest.TestCase):
+ def test_translate_builds_cbt_cluster_section(self):
+ build_dir = '/tmp/ceph-build'
+ translator = t2c.Translator(build_dir)
+ cbt_task = {
+ 'cluster': {
+ 'osds_per_node': 4,
+ 'iterations': 2,
+ 'pool_profiles': {
+ 'replicated': {
+ 'pg_size': 64,
+ 'pgp_size': 64,
+ 'replication': 'replicated',
+ },
+ },
+ },
+ 'benchmarks': {
+ 'radosbench': {
+ 'read_time': 10,
+ },
+ },
+ 'monitoring_profiles': {
+ 'perf': {
+ 'nodes': ['osds'],
+ },
+ },
+ }
+
+ translated = translator.translate(cbt_task)
+ cluster = translated['cluster']
+ self.assertEqual(cluster['osds_per_node'], 4)
+ self.assertEqual(cluster['iterations'], 2)
+ self.assertEqual(cluster['pool_profiles'], cbt_task['cluster']['pool_profiles'])
+ self.assertEqual(cluster['conf_file'], os.path.join(build_dir, 'ceph.conf'))
+ self.assertEqual(cluster['ceph_cmd'], os.path.join(build_dir, 'bin', 'ceph'))
+ self.assertEqual(cluster['rados_cmd'], os.path.join(build_dir, 'bin', 'rados'))
+ self.assertEqual(cluster['pid_dir'], os.path.join(build_dir, 'out'))
+ self.assertFalse(cluster['rebuild_every_test'])
+ self.assertEqual(translated['benchmarks'], cbt_task['benchmarks'])
+ self.assertEqual(translated['monitoring_profiles'],
+ cbt_task['monitoring_profiles'])
+
+
+class TestMain(unittest.TestCase):
+ def test_main_writes_translated_yaml(self):
+ input_path = tempfile.NamedTemporaryFile(mode='w', suffix='.yaml',
+ delete=False)
+ input_path.write(SAMPLE_TEUTHOLOGY_YAML)
+ input_path.close()
+ self.addCleanup(os.unlink, input_path.name)
+
+ output_path = tempfile.NamedTemporaryFile(mode='w', suffix='.yaml',
+ delete=False)
+ output_path.close()
+ self.addCleanup(os.unlink, output_path.name)
+
+ build_dir = '/tmp/ceph-build'
+ argv = [
+ 't2c.py',
+ '--build-dir', build_dir,
+ '--input', input_path.name,
+ '--output', output_path.name,
+ ]
+ with unittest.mock.patch.object(sys, 'argv', argv):
+ t2c.main()
+
+ with open(output_path.name) as output:
+ translated = yaml.safe_load(output)
+ self.assertIn('cluster', translated)
+ self.assertIn('benchmarks', translated)
+ self.assertEqual(translated['cluster']['conf_file'],
+ os.path.join(build_dir, 'ceph.conf'))
+
+ def test_main_errors_when_cbt_task_missing(self):
+ input_path = tempfile.NamedTemporaryFile(mode='w', suffix='.yaml',
+ delete=False)
+ input_path.write("tasks:\n- install:\n")
+ input_path.close()
+ self.addCleanup(os.unlink, input_path.name)
+
+ output_path = tempfile.NamedTemporaryFile(mode='w', suffix='.yaml',
+ delete=False)
+ output_path.close()
+ self.addCleanup(os.unlink, output_path.name)
+
+ argv = [
+ 't2c.py',
+ '--input', input_path.name,
+ '--output', output_path.name,
+ ]
+ with unittest.mock.patch.object(sys, 'argv', argv):
+ with self.assertRaises(SystemExit) as ctx:
+ t2c.main()
+ self.assertEqual(ctx.exception.code, 1)
+
+
+if __name__ == '__main__':
+ unittest.main()
projects / ceph.git / commitdiff