From: Kautilya Tripathi Date: Tue, 9 Jun 2026 05:01:12 +0000 (+0530) Subject: crimson/cbt: use yaml.safe_load in t2c and add unit tests X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=91878740bbe9939b996df99861bf74ada3382050;p=ceph.git crimson/cbt: use yaml.safe_load in t2c and add unit tests t2c.py translates teuthology benchmark YAML into CBT configuration for run-cbt.sh. Switch input parsing from yaml.load() to yaml.safe_load() so the module works with current PyYAML and avoids unsafe deserialization. Add test_t2c.py covering get_cbt_tasks(), Translator.translate(), and main(), and register it with make check via add_ceph_test. Document the translator in doc/dev/crimson/index.rst and describe the ceph-perf-pull-requests Jenkins workflow in doc/dev/continuous-integration.rst. Signed-off-by: Kautilya Tripathi --- diff --git a/doc/dev/continuous-integration.rst b/doc/dev/continuous-integration.rst index 5c2f158236c..091f4f154a4 100644 --- a/doc/dev/continuous-integration.rst +++ b/doc/dev/continuous-integration.rst @@ -94,6 +94,33 @@ Shaman for its `Web UI`_. But please note, shaman does not build the packages, it just offers information on the builds. +ceph-perf-pull-requests +----------------------- + +``ceph-perf-pull-requests`` runs CBT performance regression checks on dedicated +``performance`` Jenkins agents. The job definition lives in `ceph-build`_ and +generates two jobs from a single template: ``ceph-perf-classic`` and +``ceph-perf-crimson``. + +A pull request can trigger a run with:: + + jenkins test classic perf + jenkins test crimson perf + +The ``performance`` label is also whitelisted for automatic runs. Each job: + +#. checks out ``ceph-main`` (``origin/main``) and the PR merge ref +#. builds both trees (classic ``vstart-base`` or Crimson ``crimson-osd``) +#. runs the checked-in ``radosbench_4K_read.yaml`` workload via ``run-cbt.sh`` +#. compares PR results against ``main`` with ``cbt/compare.py`` +#. publishes a GitHub check named ``perf-test-{classic,crimson}`` + +The benchmark YAML is always taken from ``ceph-main`` so PR and baseline runs use +the same workload definition. Teuthology-to-CBT translation is handled by +``src/test/crimson/cbt/t2c.py`` in the Ceph tree (not patched at build time). + +.. _ceph-build: https://github.com/ceph/ceph-build + As the following shows, `chacra`_ manages multiple projects whose metadata are stored in a database. These metadata are exposed via Shaman as a web service. `chacractl`_ is a utility to interact with the `chacra`_ service. diff --git a/doc/dev/crimson/index.rst b/doc/dev/crimson/index.rst index fc7ec7fa3ff..1ee5afd6fb9 100644 --- a/doc/dev/crimson/index.rst +++ b/doc/dev/crimson/index.rst @@ -253,7 +253,17 @@ In order to use ``fio`` to test ``crimson-store-nbd``, perform the below steps. CBT --- -We can use `cbt`_ for performance tests:: +We can use `cbt`_ for performance tests. Benchmark workloads are checked in under +``src/test/crimson/cbt/`` as teuthology-style YAML files. Before ``run-cbt.sh`` +invokes CBT, ``t2c.py`` translates the teuthology ``tasks`` list into a +CBT-ready configuration: it extracts the ``cbt`` task, fills in cluster paths +(``ceph.conf``, ``ceph``/``rados`` binaries, PID directory), and writes the +result to a temporary YAML file consumed by ``cbt.py``. + +Unit tests for the translator live in ``src/test/crimson/cbt/test_t2c.py`` and +are registered with ``make check`` via ``add_ceph_test``. + +:: $ git checkout main $ make crimson-osd diff --git a/src/test/crimson/CMakeLists.txt b/src/test/crimson/CMakeLists.txt index 8a8c82ae5e7..71e0b876f67 100644 --- a/src/test/crimson/CMakeLists.txt +++ b/src/test/crimson/CMakeLists.txt @@ -135,3 +135,5 @@ target_link_libraries( unittest-crimson-scrub crimson-common crimson::gtest) + +add_subdirectory(cbt) diff --git a/src/test/crimson/cbt/CMakeLists.txt b/src/test/crimson/cbt/CMakeLists.txt new file mode 100644 index 00000000000..8a969b684bd --- /dev/null +++ b/src/test/crimson/cbt/CMakeLists.txt @@ -0,0 +1,2 @@ +add_ceph_test(test_t2c.py + ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/test_t2c.py) diff --git a/src/test/crimson/cbt/t2c.py b/src/test/crimson/cbt/t2c.py index 0d4ee49e5b6..9a13eadd3f2 100755 --- a/src/test/crimson/cbt/t2c.py +++ b/src/test/crimson/cbt/t2c.py @@ -1,4 +1,12 @@ #!/usr/bin/env python3 +""" +Translate a teuthology-style benchmark YAML into a CBT configuration file. + +The YAML files under ``src/test/crimson/cbt/`` describe workloads using +teuthology's ``tasks`` list. ``run-cbt.sh`` invokes this module to extract the +``cbt`` task and emit a CBT-ready configuration (cluster layout, benchmark +definitions, monitoring profiles). +""" from __future__ import print_function import argparse @@ -39,12 +47,16 @@ class Translator(object): rados_cmd=os.path.join(self.build_dir, 'bin', 'rados'), pid_dir=os.path.join(self.build_dir, 'out') )) - return conf + return conf def get_cbt_tasks(path): - with open(path) as input: - teuthology_config = yaml.load(input) - for task in teuthology_config['tasks']: + with open(path) as yaml_file: + teuthology_config = yaml.safe_load(yaml_file) or {} + if not isinstance(teuthology_config, dict): + teuthology_config = {} + for task in teuthology_config.get('tasks', []): + if not isinstance(task, dict): + continue for name, conf in task.items(): if name == 'cbt': yield conf diff --git a/src/test/crimson/cbt/test_t2c.py b/src/test/crimson/cbt/test_t2c.py new file mode 100644 index 00000000000..02c39c1bce1 --- /dev/null +++ b/src/test/crimson/cbt/test_t2c.py @@ -0,0 +1,187 @@ +#!/usr/bin/env python3 + +import os +import sys +import tempfile +import unittest +import unittest.mock + +import yaml + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import t2c # noqa: E402 + + +SAMPLE_TEUTHOLOGY_YAML = """\ +meta: +- desc: sample radosbench workload +tasks: +- install: + extra_system_packages: + deb: + - lvm2 +- cbt: + benchmarks: + radosbench: + read_time: 30 + read_only: true + monitoring_profiles: + perf: + nodes: + - osds + cluster: + osds_per_node: 3 + iterations: 1 + pool_profiles: + replicated: + pg_size: 128 + pgp_size: 128 + replication: replicated +""" + + +class TestGetCbtTasks(unittest.TestCase): + def _write_yaml(self, contents): + handle = tempfile.NamedTemporaryFile(mode='w', suffix='.yaml', + delete=False) + handle.write(contents) + handle.close() + self.addCleanup(os.unlink, handle.name) + return handle.name + + def test_extracts_cbt_task(self): + path = self._write_yaml(SAMPLE_TEUTHOLOGY_YAML) + tasks = list(t2c.get_cbt_tasks(path)) + self.assertEqual(len(tasks), 1) + self.assertIn('benchmarks', tasks[0]) + self.assertEqual(tasks[0]['benchmarks']['radosbench']['read_time'], 30) + self.assertEqual(tasks[0]['cluster']['osds_per_node'], 3) + + def test_ignores_non_cbt_tasks(self): + path = self._write_yaml("tasks:\n- install:\n version: main\n") + tasks = list(t2c.get_cbt_tasks(path)) + self.assertEqual(tasks, []) + + def test_empty_file_returns_no_tasks(self): + path = self._write_yaml("") + tasks = list(t2c.get_cbt_tasks(path)) + self.assertEqual(tasks, []) + + def test_missing_tasks_key_returns_no_tasks(self): + path = self._write_yaml("meta:\n- desc: no tasks here\n") + tasks = list(t2c.get_cbt_tasks(path)) + self.assertEqual(tasks, []) + + def test_multiple_cbt_tasks(self): + path = self._write_yaml( + "tasks:\n" + "- cbt:\n" + " cluster:\n" + " iterations: 1\n" + "- cbt:\n" + " cluster:\n" + " iterations: 2\n") + tasks = list(t2c.get_cbt_tasks(path)) + self.assertEqual(len(tasks), 2) + self.assertEqual(tasks[0]['cluster']['iterations'], 1) + self.assertEqual(tasks[1]['cluster']['iterations'], 2) + + +class TestTranslator(unittest.TestCase): + def test_translate_builds_cbt_cluster_section(self): + build_dir = '/tmp/ceph-build' + translator = t2c.Translator(build_dir) + cbt_task = { + 'cluster': { + 'osds_per_node': 4, + 'iterations': 2, + 'pool_profiles': { + 'replicated': { + 'pg_size': 64, + 'pgp_size': 64, + 'replication': 'replicated', + }, + }, + }, + 'benchmarks': { + 'radosbench': { + 'read_time': 10, + }, + }, + 'monitoring_profiles': { + 'perf': { + 'nodes': ['osds'], + }, + }, + } + + translated = translator.translate(cbt_task) + cluster = translated['cluster'] + self.assertEqual(cluster['osds_per_node'], 4) + self.assertEqual(cluster['iterations'], 2) + self.assertEqual(cluster['pool_profiles'], cbt_task['cluster']['pool_profiles']) + self.assertEqual(cluster['conf_file'], os.path.join(build_dir, 'ceph.conf')) + self.assertEqual(cluster['ceph_cmd'], os.path.join(build_dir, 'bin', 'ceph')) + self.assertEqual(cluster['rados_cmd'], os.path.join(build_dir, 'bin', 'rados')) + self.assertEqual(cluster['pid_dir'], os.path.join(build_dir, 'out')) + self.assertFalse(cluster['rebuild_every_test']) + self.assertEqual(translated['benchmarks'], cbt_task['benchmarks']) + self.assertEqual(translated['monitoring_profiles'], + cbt_task['monitoring_profiles']) + + +class TestMain(unittest.TestCase): + def test_main_writes_translated_yaml(self): + input_path = tempfile.NamedTemporaryFile(mode='w', suffix='.yaml', + delete=False) + input_path.write(SAMPLE_TEUTHOLOGY_YAML) + input_path.close() + self.addCleanup(os.unlink, input_path.name) + + output_path = tempfile.NamedTemporaryFile(mode='w', suffix='.yaml', + delete=False) + output_path.close() + self.addCleanup(os.unlink, output_path.name) + + build_dir = '/tmp/ceph-build' + argv = [ + 't2c.py', + '--build-dir', build_dir, + '--input', input_path.name, + '--output', output_path.name, + ] + with unittest.mock.patch.object(sys, 'argv', argv): + t2c.main() + + with open(output_path.name) as output: + translated = yaml.safe_load(output) + self.assertIn('cluster', translated) + self.assertIn('benchmarks', translated) + self.assertEqual(translated['cluster']['conf_file'], + os.path.join(build_dir, 'ceph.conf')) + + def test_main_errors_when_cbt_task_missing(self): + input_path = tempfile.NamedTemporaryFile(mode='w', suffix='.yaml', + delete=False) + input_path.write("tasks:\n- install:\n") + input_path.close() + self.addCleanup(os.unlink, input_path.name) + + output_path = tempfile.NamedTemporaryFile(mode='w', suffix='.yaml', + delete=False) + output_path.close() + self.addCleanup(os.unlink, output_path.name) + + argv = [ + 't2c.py', + '--input', input_path.name, + '--output', output_path.name, + ] + with unittest.mock.patch.object(sys, 'argv', argv): + with self.assertRaises(SystemExit) as ctx: + t2c.main() + self.assertEqual(ctx.exception.code, 1) + + +if __name__ == '__main__': + unittest.main()