- name: Lint
run: uv tool run ruff check
- name: Run tests
- run: uv run --all-extras pytest teuthology scripts
+ run: uv run --all-extras pytest
- name: Run docs build
run: uv run tox -e docs
[tool.setuptools.packages.find]
exclude = [
- "*test*",
+ "tests*",
"docs*",
]
-[tool.setuptools.exclude-package-data]
-"*" = ["test*"]
-
[tool.setuptools_scm]
version_scheme = "python-simplified-semver"
lint.select = ["F", "E9"]
[tool.pytest]
-testpaths = ["teuthology", "scripts"]
+testpaths = ["tests"]
norecursedirs = [".git", "build", "virtualenv", "teuthology.egg-info", ".tox", "*/integration", "teuthology/task/tests"]
log_cli = true
log_level = "NOTSET"
+++ /dev/null
-import sys
-import pytest
-from importlib import import_module
-
-
-class Script(object):
- script_name = "teuthology"
- script_module = None # Override in subclasses, e.g., 'scripts.run'
-
- @pytest.fixture(scope="class")
- def module_name(self) -> str:
- # e.g., 'teuthology-dispatcher' -> 'scripts.dispatcher'
- return self.script_name.replace("teuthology-", "").replace("teuthology", "run")
-
- @pytest.fixture(scope="class")
- def module(self, module_name):
- return import_module(self.script_module or f"scripts.{module_name}")
-
- def test_help(self, capsys: pytest.CaptureFixture[str], module):
- # docopt
- if module.__doc__ and "usage" in module.__doc__.lower():
- return
- if hasattr(module, "doc") and module.doc and "usage" in module.doc.lower():
- return
- # argparse
- if hasattr(module, "parse_args"):
- with pytest.raises(SystemExit):
- module.parse_args([])
- captured = capsys.readouterr()
- assert "usage: " in captured.err
- return
- # If neither, fail
- raise AssertionError(
- f"{self.script_name} has neither a docstring/doc variable with usage info "
- f"nor a parse_args function"
- )
-
- def test_invalid(self, module):
- original_argv = sys.argv
- try:
- sys.argv = [self.script_name, "--invalid-option"]
- with pytest.raises(SystemExit):
- if hasattr(module, "parse_args"):
- module.parse_args(sys.argv[1:])
- elif hasattr(module, "main"):
- # For docopt-based scripts, main() will call docopt which exits on error
- module.main()
- else:
- raise NotImplementedError(
- f"Don't know how to test {self.script_name}"
- )
- finally:
- sys.argv = original_argv
+++ /dev/null
-from script import Script
-
-
-class TestDispatcher(Script):
- script_name = 'teuthology-dispatcher'
- script_module = 'scripts.dispatcher'
+++ /dev/null
-from script import Script
-
-
-class TestExporter(Script):
- script_name = 'teuthology-exporter'
- script_module = 'scripts.exporter'
+++ /dev/null
-from script import Script
-
-
-class TestLock(Script):
- script_name = 'teuthology-lock'
- script_module = 'scripts.lock'
+++ /dev/null
-import docopt
-
-from script import Script
-from scripts import ls
-
-doc = ls.__doc__
-
-
-class TestLs(Script):
- script_name = 'teuthology-ls'
- script_module = 'scripts.ls'
-
- def test_args(self):
- args = docopt.docopt(doc, ["--verbose", "some/archive/dir"])
- assert args["--verbose"]
- assert args["<archive_dir>"] == "some/archive/dir"
+++ /dev/null
-from script import Script
-
-
-class TestPruneLogs(Script):
- script_name = 'teuthology-prune-logs'
- script_module = 'scripts.prune_logs'
+++ /dev/null
-from script import Script
-
-
-class TestReport(Script):
- script_name = 'teuthology-report'
- script_module = 'scripts.report'
+++ /dev/null
-from script import Script
-
-
-class TestResults(Script):
- script_name = 'teuthology-results'
- script_module = 'scripts.results'
+++ /dev/null
-import docopt
-
-from script import Script
-from scripts import run
-
-doc = run.__doc__
-
-
-class TestRun(Script):
- script_name = 'teuthology'
- script_module = 'scripts.run'
-
- def test_all_args(self):
- args = docopt.docopt(doc, [
- "--verbose",
- "--archive", "some/archive/dir",
- "--description", "the_description",
- "--owner", "the_owner",
- "--lock",
- "--machine-type", "machine_type",
- "--os-type", "os_type",
- "--os-version", "os_version",
- "--block",
- "--name", "the_name",
- "--suite-path", "some/suite/dir",
- "path/to/config.yml",
- ])
- assert args["--verbose"]
- assert args["--archive"] == "some/archive/dir"
- assert args["--description"] == "the_description"
- assert args["--owner"] == "the_owner"
- assert args["--lock"]
- assert args["--machine-type"] == "machine_type"
- assert args["--os-type"] == "os_type"
- assert args["--os-version"] == "os_version"
- assert args["--block"]
- assert args["--name"] == "the_name"
- assert args["--suite-path"] == "some/suite/dir"
- assert args["<config>"] == ["path/to/config.yml"]
-
- def test_multiple_configs(self):
- args = docopt.docopt(doc, [
- "config1.yml",
- "config2.yml",
- ])
- assert args["<config>"] == ["config1.yml", "config2.yml"]
+++ /dev/null
-from script import Script
-
-
-class TestSchedule(Script):
- script_name = 'teuthology-schedule'
- script_module = 'scripts.schedule'
+++ /dev/null
-from script import Script
-
-
-class TestSuite(Script):
- script_name = 'teuthology-suite'
- script_module = 'scripts.suite'
+++ /dev/null
-from script import Script
-
-
-class TestSupervisor(Script):
- script_name = 'teuthology-supervisor'
- script_module = 'scripts.supervisor'
+++ /dev/null
-from script import Script
-import docopt
-from pytest import raises
-from pytest import skip
-from scripts import updatekeys
-
-
-class TestUpdatekeys(Script):
- script_name = 'teuthology-updatekeys'
- script_module = 'scripts.updatekeys'
-
- def test_invalid(self):
- skip("teuthology.lock needs to be partially refactored to allow" +
- "teuthology-updatekeys to return nonzero in all erorr cases")
-
- def test_all_and_targets(self):
- with raises(docopt.DocoptExit):
- docopt.docopt(updatekeys.doc, ['-a', '-t', 'foo'])
-
- def test_no_args(self):
- with raises(docopt.DocoptExit):
- docopt.docopt(updatekeys.doc, [])
--- /dev/null
+import os
+import pytest
+import sys
+
+skipif_teuthology_process = pytest.mark.skipif(
+ os.path.basename(sys.argv[0]) == "teuthology",
+ reason="Skipped because this test cannot pass when run in a teuthology " \
+ "process (as opposed to py.test)"
+)
\ No newline at end of file
--- /dev/null
+import datetime
+import pytest
+
+from unittest.mock import patch, Mock, MagicMock
+
+from teuthology import dispatcher
+from teuthology.config import FakeNamespace
+from teuthology.contextutil import MaxWhileTries
+from teuthology.util.time import TIMESTAMP_FMT
+
+
+class TestDispatcher(object):
+ @pytest.fixture(autouse=True)
+ def setup_method(self, tmp_path):
+ self.ctx = FakeNamespace()
+ self.ctx.verbose = True
+ self.ctx.archive_dir = str(tmp_path / "archive/dir")
+ self.ctx.log_dir = str(tmp_path / "log/dir")
+ self.ctx.tube = 'tube'
+
+ @patch("teuthology.repo_utils.ls_remote")
+ @patch("os.path.isdir")
+ @patch("teuthology.repo_utils.fetch_teuthology")
+ @patch("teuthology.dispatcher.teuth_config")
+ @patch("teuthology.repo_utils.fetch_qa_suite")
+ def test_prep_job(self, m_fetch_qa_suite, m_teuth_config,
+ m_fetch_teuthology, m_isdir, m_ls_remote):
+ config = dict(
+ name="the_name",
+ job_id="1",
+ suite_sha1="suite_hash",
+ )
+ m_fetch_teuthology.return_value = '/teuth/path'
+ m_fetch_qa_suite.return_value = '/suite/path'
+ m_ls_remote.return_value = 'teuth_hash'
+ m_isdir.return_value = True
+ m_teuth_config.teuthology_path = None
+ got_config, teuth_bin_path = dispatcher.prep_job(config)
+ assert got_config['teuthology_branch'] == 'main'
+ m_fetch_teuthology.assert_called_once_with(branch='main', commit='teuth_hash')
+ assert teuth_bin_path == '/teuth/path/.venv/bin'
+ m_fetch_qa_suite.assert_called_once_with('main', 'suite_hash')
+ assert got_config['suite_path'] == '/suite/path'
+
+ def build_fake_jobs(self, m_connection, m_job, job_bodies):
+ """
+ Given patched copies of:
+ beanstalkc.Connection
+ beanstalkc.Job
+ And a list of basic job bodies, return a list of mocked Job objects
+ """
+ # Make sure instantiating m_job returns a new object each time
+ jobs = []
+ job_id = 0
+ for job_body in job_bodies:
+ job_id += 1
+ job = MagicMock(conn=m_connection, jid=job_id, body=job_body)
+ job.jid = job_id
+ job.body = job_body
+ jobs.append(job)
+ return jobs
+
+ @patch("teuthology.dispatcher.find_dispatcher_processes")
+ @patch("teuthology.repo_utils.ls_remote")
+ @patch("teuthology.dispatcher.report.try_push_job_info")
+ @patch("teuthology.dispatcher.supervisor.run_job")
+ @patch("beanstalkc.Job", autospec=True)
+ @patch("teuthology.repo_utils.fetch_qa_suite")
+ @patch("teuthology.repo_utils.fetch_teuthology")
+ @patch("teuthology.dispatcher.beanstalk.watch_tube")
+ @patch("teuthology.dispatcher.beanstalk.connect")
+ @patch("os.path.isdir", return_value=True)
+ @patch("teuthology.dispatcher.setup_log_file")
+ def test_main_loop(
+ self, m_setup_log_file, m_isdir, m_connect, m_watch_tube,
+ m_fetch_teuthology, m_fetch_qa_suite, m_job, m_run_job,
+ m_try_push_job_info, m_ls_remote, m_find_dispatcher_processes,
+ ):
+ m_find_dispatcher_processes.return_value = {}
+ m_connection = Mock()
+ jobs = self.build_fake_jobs(
+ m_connection,
+ m_job,
+ [
+ 'name: name\nfoo: bar',
+ 'name: name\nstop_worker: true',
+ ],
+ )
+ m_connection.reserve.side_effect = jobs
+ m_connect.return_value = m_connection
+ dispatcher.main(self.ctx)
+ # There should be one reserve call per item in the jobs list
+ expected_reserve_calls = [
+ dict(timeout=60) for i in range(len(jobs))
+ ]
+ got_reserve_calls = [
+ call[1] for call in m_connection.reserve.call_args_list
+ ]
+ assert got_reserve_calls == expected_reserve_calls
+ for job in jobs:
+ job.bury.assert_called_once_with()
+ job.delete.assert_called_once_with()
+
+ @patch("teuthology.dispatcher.find_dispatcher_processes")
+ @patch("teuthology.repo_utils.ls_remote")
+ @patch("teuthology.dispatcher.report.try_push_job_info")
+ @patch("teuthology.dispatcher.supervisor.run_job")
+ @patch("beanstalkc.Job", autospec=True)
+ @patch("teuthology.repo_utils.fetch_qa_suite")
+ @patch("teuthology.repo_utils.fetch_teuthology")
+ @patch("teuthology.dispatcher.beanstalk.watch_tube")
+ @patch("teuthology.dispatcher.beanstalk.connect")
+ @patch("os.path.isdir", return_value=True)
+ @patch("teuthology.dispatcher.setup_log_file")
+ def test_main_loop_13925(
+ self, m_setup_log_file, m_isdir, m_connect, m_watch_tube,
+ m_fetch_teuthology, m_fetch_qa_suite, m_job, m_run_job,
+ m_try_push_job_info, m_ls_remote, m_find_dispatcher_processes,
+ ):
+ m_find_dispatcher_processes.return_value = {}
+ m_connection = Mock()
+ jobs = self.build_fake_jobs(
+ m_connection,
+ m_job,
+ [
+ 'name: name',
+ 'name: name\nstop_worker: true',
+ ],
+ )
+ m_connection.reserve.side_effect = jobs
+ m_connect.return_value = m_connection
+ m_fetch_qa_suite.side_effect = [
+ MaxWhileTries(),
+ MaxWhileTries(),
+ ]
+ dispatcher.main(self.ctx)
+ assert len(m_run_job.call_args_list) == 0
+ assert len(m_try_push_job_info.call_args_list) == len(jobs)
+ for i in range(len(jobs)):
+ push_call = m_try_push_job_info.call_args_list[i]
+ assert push_call[0][1]['status'] == 'dead'
+
+ @pytest.mark.parametrize(
+ ["timestamp", "expire", "skip"],
+ [
+ [datetime.timedelta(days=-1), None, False],
+ [datetime.timedelta(days=-30), None, True],
+ [None, datetime.timedelta(days=1), False],
+ [None, datetime.timedelta(days=-1), True],
+ [datetime.timedelta(days=-1), datetime.timedelta(days=1), False],
+ [datetime.timedelta(days=1), datetime.timedelta(days=-1), True],
+ ]
+ )
+ @patch("teuthology.dispatcher.report.try_push_job_info")
+ def test_check_job_expiration(self, _, timestamp, expire, skip):
+ now = datetime.datetime.now(datetime.timezone.utc)
+ job_config = dict(
+ job_id="1",
+ name="job_name",
+ )
+ if timestamp:
+ job_config["timestamp"] = (now + timestamp).strftime(TIMESTAMP_FMT)
+ if expire:
+ job_config["expire"] = (now + expire).strftime(TIMESTAMP_FMT)
+ if skip:
+ with pytest.raises(dispatcher.SkipJob):
+ dispatcher.check_job_expiration(job_config)
+ else:
+ dispatcher.check_job_expiration(job_config)
--- /dev/null
+from teuthology.dispatcher import supervisor
+from unittest.mock import patch
+
+class TestCheckReImageFailureMarkDown(object):
+ def setup_method(self):
+ self.the_function = supervisor.check_for_reimage_failures_and_mark_down
+
+ def create_n_out_of_10_reimage_failed_jobs(self, n):
+ ret_list = []
+ for i in range(n):
+ obj1 = {
+ "failure_reason":"Error reimaging machines: Manually raised error"
+ }
+ ret_list.append(obj1)
+ for j in range(10-n):
+ obj2 = {"failure_reason":"Error something else: dummy"}
+ ret_list.append(obj2)
+ return ret_list
+
+ @patch('teuthology.dispatcher.supervisor.shortname')
+ @patch('teuthology.lock.ops.update_lock')
+ @patch('teuthology.dispatcher.supervisor.requests')
+ @patch('teuthology.dispatcher.supervisor.urljoin')
+ @patch('teuthology.dispatcher.supervisor.teuth_config')
+ def test_one_machine_ten_reimage_failed_jobs(
+ self,
+ m_t_config,
+ m_urljoin,
+ m_requests,
+ mark_down,
+ shortname
+ ):
+ targets = {'fakeos@rmachine061.front.sepia.ceph.com': 'ssh-ed25519'}
+ m_requests.get.return_value.json.return_value = \
+ self.create_n_out_of_10_reimage_failed_jobs(10)
+ shortname.return_value = 'rmachine061'
+ self.the_function(targets)
+ assert mark_down.called
+
+ @patch('teuthology.dispatcher.supervisor.shortname')
+ @patch('teuthology.lock.ops.update_lock')
+ @patch('teuthology.dispatcher.supervisor.requests')
+ @patch('teuthology.dispatcher.supervisor.urljoin')
+ @patch('teuthology.dispatcher.supervisor.teuth_config')
+ def test_one_machine_seven_reimage_failed_jobs(
+ self,
+ m_t_config,
+ m_urljoin,
+ m_requests,
+ mark_down,
+ shortname,
+ ):
+ targets = {'fakeos@rmachine061.front.sepia.ceph.com': 'ssh-ed25519'}
+ m_requests.get.return_value.json.return_value = \
+ self.create_n_out_of_10_reimage_failed_jobs(7)
+ shortname.return_value = 'rmachine061'
+ self.the_function(targets)
+ assert mark_down.called is False
+
+ @patch('teuthology.dispatcher.supervisor.shortname')
+ @patch('teuthology.lock.ops.update_lock')
+ @patch('teuthology.dispatcher.supervisor.requests')
+ @patch('teuthology.dispatcher.supervisor.urljoin')
+ @patch('teuthology.dispatcher.supervisor.teuth_config')
+ def test_two_machine_all_reimage_failed_jobs(
+ self,
+ m_t_config,
+ m_urljoin,
+ m_requests,
+ mark_down,
+ shortname,
+ ):
+ targets = {'fakeos@rmachine061.front.sepia.ceph.com': 'ssh-ed25519',
+ 'fakeos@rmachine179.back.sepia.ceph.com': 'ssh-ed45333'}
+ m_requests.get.return_value.json.side_effect = \
+ [self.create_n_out_of_10_reimage_failed_jobs(10),
+ self.create_n_out_of_10_reimage_failed_jobs(10)]
+ shortname.return_value.side_effect = ['rmachine061', 'rmachine179']
+ self.the_function(targets)
+ assert mark_down.call_count == 2
+
+ @patch('teuthology.dispatcher.supervisor.shortname')
+ @patch('teuthology.lock.ops.update_lock')
+ @patch('teuthology.dispatcher.supervisor.requests')
+ @patch('teuthology.dispatcher.supervisor.urljoin')
+ @patch('teuthology.dispatcher.supervisor.teuth_config')
+ def test_two_machine_one_healthy_one_reimage_failure(
+ self,
+ m_t_config,
+ m_urljoin,
+ m_requests,
+ mark_down,
+ shortname,
+ ):
+ targets = {'fakeos@rmachine061.front.sepia.ceph.com': 'ssh-ed25519',
+ 'fakeos@rmachine179.back.sepia.ceph.com': 'ssh-ed45333'}
+ m_requests.get.return_value.json.side_effect = \
+ [self.create_n_out_of_10_reimage_failed_jobs(0),
+ self.create_n_out_of_10_reimage_failed_jobs(10)]
+ shortname.return_value.side_effect = ['rmachine061', 'rmachine179']
+ self.the_function(targets)
+ assert mark_down.call_count == 1
+ assert mark_down.call_args_list[0][0][0].startswith('rmachine179')
+
--- /dev/null
+from subprocess import DEVNULL
+from unittest.mock import patch, Mock, MagicMock
+
+from teuthology.dispatcher import supervisor
+
+
+class TestSuperviser(object):
+ @patch("teuthology.dispatcher.supervisor.run_with_watchdog")
+ @patch("teuthology.dispatcher.supervisor.teuth_config")
+ @patch("subprocess.Popen")
+ @patch("os.environ")
+ @patch("os.mkdir")
+ @patch("yaml.safe_dump")
+ @patch("tempfile.NamedTemporaryFile")
+ def test_run_job_with_watchdog(self, m_tempfile, m_safe_dump, m_mkdir,
+ m_environ, m_popen, m_t_config,
+ m_run_watchdog):
+ config = {
+ "suite_path": "suite/path",
+ "config": {"foo": "bar"},
+ "verbose": True,
+ "owner": "the_owner",
+ "archive_path": "archive/path",
+ "name": "the_name",
+ "description": "the_description",
+ "job_id": "1",
+ }
+ m_tmp = MagicMock()
+ temp_file = Mock()
+ temp_file.name = "the_name"
+ m_tmp.__enter__.return_value = temp_file
+ m_tempfile.return_value = m_tmp
+ m_p = Mock()
+ m_p.returncode = 0
+ m_popen.return_value = m_p
+ m_t_config.results_server = True
+ config_path = "archive/path/initial.config.yaml"
+ supervisor.run_job(config, config_path, "teuth/bin/path", "archive/dir", verbose=False)
+ m_run_watchdog.assert_called_with(m_p, config)
+ expected_args = [
+ 'teuth/bin/path/teuthology',
+ '-v',
+ '--owner', 'the_owner',
+ '--archive', 'archive/path',
+ '--name', 'the_name',
+ '--description',
+ 'the_description',
+ '--',
+ config_path,
+ ]
+ m_popen.assert_called_with(args=expected_args, stderr=DEVNULL, stdout=DEVNULL)
+
+ @patch("time.sleep")
+ @patch("teuthology.dispatcher.supervisor.teuth_config")
+ @patch("subprocess.Popen")
+ @patch("os.environ")
+ @patch("os.mkdir")
+ @patch("yaml.safe_dump")
+ @patch("tempfile.NamedTemporaryFile")
+ def test_run_job_no_watchdog(self, m_tempfile, m_safe_dump, m_mkdir,
+ m_environ, m_popen, m_t_config,
+ m_sleep):
+ config = {
+ "suite_path": "suite/path",
+ "config": {"foo": "bar"},
+ "verbose": True,
+ "owner": "the_owner",
+ "archive_path": "archive/path",
+ "name": "the_name",
+ "description": "the_description",
+ "job_id": "1",
+ }
+ m_tmp = MagicMock()
+ temp_file = Mock()
+ temp_file.name = "the_name"
+ m_tmp.__enter__.return_value = temp_file
+ m_tempfile.return_value = m_tmp
+ env = dict(PYTHONPATH="python/path")
+ m_environ.copy.return_value = env
+ m_p = Mock()
+ m_p.returncode = 1
+ m_popen.return_value = m_p
+ m_t_config.results_server = False
+ config_path = "archive/path/initial.config.yaml"
+ supervisor.run_job(config, config_path, "teuth/bin/path", "archive/dir", verbose=False)
+
+ @patch("teuthology.dispatcher.supervisor.report.try_push_job_info")
+ @patch("time.sleep")
+ def test_run_with_watchdog_no_reporting(self, m_sleep, m_try_push):
+ config = {
+ "name": "the_name",
+ "job_id": "1",
+ "archive_path": "archive/path",
+ "teuthology_branch": "main",
+ "owner": "the_owner",
+ }
+ process = Mock()
+ process.poll.return_value = "not None"
+ supervisor.run_with_watchdog(process, config)
+ m_try_push.assert_called_with(
+ dict(name=config["name"], job_id=config["job_id"]),
+ dict(status='dead')
+ )
+
+ @patch("subprocess.Popen")
+ @patch("time.sleep")
+ @patch("teuthology.dispatcher.supervisor.report.try_push_job_info")
+ def test_run_with_watchdog_with_reporting(self, m_tpji, m_sleep, m_popen):
+ config = {
+ "name": "the_name",
+ "job_id": "1",
+ "archive_path": "archive/path",
+ "teuthology_branch": "jewel",
+ "owner": "the_owner",
+ }
+ process = Mock()
+ process.poll.return_value = "not None"
+ m_proc = Mock()
+ m_proc.poll.return_value = "not None"
+ m_popen.return_value = m_proc
+ supervisor.run_with_watchdog(process, config)
--- /dev/null
+import os
+import shutil
+import yaml
+import random
+
+
+class FakeArchive(object):
+ def __init__(self, archive_base="./test_archive"):
+ self.archive_base = archive_base
+
+ def get_random_metadata(self, run_name, job_id=None, hung=False):
+ """
+ Generate a random info dict for a fake job. If 'hung' is not True, also
+ generate a summary dict.
+
+ :param run_name: Run name e.g. 'test_foo'
+ :param job_id: Job ID e.g. '12345'
+ :param hung: Simulate a hung job e.g. don't return a summary.yaml
+ :return: A dict with keys 'job_id', 'info' and possibly
+ 'summary', with corresponding values
+ """
+ rand = random.Random()
+
+ description = 'description for job with id %s' % job_id
+ owner = 'job@owner'
+ duration = rand.randint(1, 36000)
+ pid = rand.randint(1000, 99999)
+ job_id = rand.randint(1, 99999)
+
+ info = {
+ 'description': description,
+ 'job_id': job_id,
+ 'run_name': run_name,
+ 'owner': owner,
+ 'pid': pid,
+ }
+
+ metadata = {
+ 'info': info,
+ 'job_id': job_id,
+ }
+
+ if not hung:
+ success = True if rand.randint(0, 1) != 1 else False
+
+ summary = {
+ 'description': description,
+ 'duration': duration,
+ 'owner': owner,
+ 'success': success,
+ }
+
+ if not success:
+ summary['failure_reason'] = 'Failure reason!'
+ metadata['summary'] = summary
+
+ return metadata
+
+ def setup(self):
+ if os.path.exists(self.archive_base):
+ shutil.rmtree(self.archive_base)
+ os.mkdir(self.archive_base)
+
+ def teardown(self):
+ shutil.rmtree(self.archive_base)
+
+ def populate_archive(self, run_name, jobs):
+ run_archive_dir = os.path.join(self.archive_base, run_name)
+ os.mkdir(run_archive_dir)
+ for job in jobs:
+ archive_dir = os.path.join(run_archive_dir, str(job['job_id']))
+ os.mkdir(archive_dir)
+
+ with open(os.path.join(archive_dir, 'info.yaml'), 'w') as yfile:
+ yaml.safe_dump(job['info'], yfile)
+
+ if 'summary' in job:
+ summary_path = os.path.join(archive_dir, 'summary.yaml')
+ with open(summary_path, 'w') as yfile:
+ yaml.safe_dump(job['summary'], yfile)
+
+ def create_fake_run(self, run_name, job_count, yaml_path, num_hung=0):
+ """
+ Creates a fake run using run_name. Uses the YAML specified for each
+ job's config.yaml
+
+ Returns a list of job_ids
+ """
+ assert os.path.exists(yaml_path)
+ assert job_count > 0
+ jobs = []
+ made_hung = 0
+ for i in range(job_count):
+ if made_hung < num_hung:
+ jobs.append(self.get_random_metadata(run_name, hung=True))
+ made_hung += 1
+ else:
+ jobs.append(self.get_random_metadata(run_name, hung=False))
+ #job_config = yaml.safe_load(yaml_path)
+ self.populate_archive(run_name, jobs)
+ for job in jobs:
+ job_id = job['job_id']
+ job_yaml_path = os.path.join(self.archive_base, run_name,
+ str(job_id), 'config.yaml')
+ shutil.copyfile(yaml_path, job_yaml_path)
+ return jobs
+
--- /dev/null
+from io import BytesIO
+from contextlib import closing
+
+
+try:
+ FileNotFoundError, NotADirectoryError
+except NameError:
+ FileNotFoundError = NotADirectoryError = OSError
+
+
+def make_fake_fstools(fake_filesystem):
+ """
+ Build fake versions of os.listdir(), os.isfile(), etc. for use in
+ unit tests
+
+ An example fake_filesystem value:
+ >>> fake_fs = {\
+ 'a_directory': {\
+ 'another_directory': {\
+ 'empty_file': None,\
+ 'another_empty_file': None,\
+ },\
+ 'random_file': None,\
+ 'yet_another_directory': {\
+ 'empty_directory': {},\
+ },\
+ 'file_with_contents': 'data',\
+ },\
+ }
+ >>> fake_listdir, fake_isfile, _, _ = \
+ make_fake_fstools(fake_fs)
+ >>> fake_listdir('a_directory/yet_another_directory')
+ ['empty_directory']
+ >>> fake_isfile('a_directory/yet_another_directory')
+ False
+
+ :param fake_filesystem: A dict representing a filesystem
+ """
+ assert isinstance(fake_filesystem, dict)
+
+ def fake_listdir(path, fsdict=False):
+ if fsdict is False:
+ fsdict = fake_filesystem
+
+ remainder = path.strip('/') + '/'
+ subdict = fsdict
+ while '/' in remainder:
+ next_dir, remainder = remainder.split('/', 1)
+ if next_dir not in subdict:
+ raise FileNotFoundError(
+ '[Errno 2] No such file or directory: %s' % next_dir)
+ subdict = subdict.get(next_dir)
+ if not isinstance(subdict, dict):
+ raise NotADirectoryError('[Errno 20] Not a directory: %s' % next_dir)
+ if subdict and not remainder:
+ return list(subdict)
+ return []
+
+ def fake_isfile(path, fsdict=False):
+ if fsdict is False:
+ fsdict = fake_filesystem
+
+ components = path.strip('/').split('/')
+ subdict = fsdict
+ for component in components:
+ if component not in subdict:
+ raise FileNotFoundError('[Errno 2] No such file or directory: %s' % component)
+ subdict = subdict.get(component)
+ return subdict is None or isinstance(subdict, str)
+
+ def fake_isdir(path, fsdict=False):
+ return not fake_isfile(path)
+
+ def fake_exists(path, fsdict=False):
+ return fake_isfile(path, fsdict) or fake_isdir(path, fsdict)
+
+ def fake_open(path, mode=None, buffering=None):
+ components = path.strip('/').split('/')
+ subdict = fake_filesystem
+ for component in components:
+ if component not in subdict:
+ raise FileNotFoundError('[Errno 2] No such file or directory: %s' % component)
+ subdict = subdict.get(component)
+ if isinstance(subdict, dict):
+ raise IOError('[Errno 21] Is a directory: %s' % path)
+ elif subdict is None:
+ return closing(BytesIO(b''))
+ return closing(BytesIO(subdict.encode()))
+
+ return fake_exists, fake_listdir, fake_isfile, fake_isdir, fake_open
--- /dev/null
+import os
+import requests
+from pytest import raises, skip
+
+from teuthology.config import config
+from teuthology import suite
+
+
+class TestSuiteOnline(object):
+ def setup_method(self):
+ if 'TEST_ONLINE' not in os.environ:
+ skip("To run these sets, set the environment variable TEST_ONLINE")
+
+ def test_ceph_hash_simple(self):
+ resp = requests.get(
+ 'https://api.github.com/repos/ceph/ceph/git/refs/heads/main')
+ ref_hash = resp.json()['object']['sha']
+ assert suite.get_hash('ceph') == ref_hash
+
+ def test_kernel_hash_saya(self):
+ # We don't currently have these packages.
+ assert suite.get_hash('kernel', 'main', 'default', 'saya') is None
+
+ def test_all_main_branches(self):
+ # Don't attempt to send email
+ config.results_email = None
+ job_config = suite.create_initial_config('suite', 'main',
+ 'main', 'main', 'testing',
+ 'default', 'centos', 'plana')
+ assert ((job_config.branch, job_config.teuthology_branch,
+ job_config.suite_branch) == ('main', 'main', 'main'))
+
+ def test_config_bogus_kernel_branch(self):
+ # Don't attempt to send email
+ config.results_email = None
+ with raises(suite.ScheduleFailError):
+ suite.create_initial_config('s', None, 'main', 't',
+ 'bogus_kernel_branch', 'f', 'd', 'm')
+
+ def test_config_bogus_flavor(self):
+ # Don't attempt to send email
+ config.results_email = None
+ with raises(suite.ScheduleFailError):
+ suite.create_initial_config('s', None, 'main', 't', 'k',
+ 'bogus_flavor', 'd', 'm')
+
+ def test_config_bogus_ceph_branch(self):
+ # Don't attempt to send email
+ config.results_email = None
+ with raises(suite.ScheduleFailError):
+ suite.create_initial_config('s', None, 'bogus_ceph_branch', 't',
+ 'k', 'f', 'd', 'm')
+
+ def test_config_bogus_suite_branch(self):
+ # Don't attempt to send email
+ config.results_email = None
+ with raises(suite.ScheduleFailError):
+ suite.create_initial_config('s', 'bogus_suite_branch', 'main',
+ 't', 'k', 'f', 'd', 'm')
+
+ def test_config_bogus_teuthology_branch(self):
+ # Don't attempt to send email
+ config.results_email = None
+ with raises(suite.ScheduleFailError):
+ suite.create_initial_config('s', None, 'main',
+ 'bogus_teuth_branch', 'k', 'f', 'd',
+ 'm')
+
+ def test_config_substitution(self):
+ # Don't attempt to send email
+ config.results_email = None
+ job_config = suite.create_initial_config('MY_SUITE', 'main',
+ 'main', 'main', 'testing',
+ 'default', 'centos', 'plana')
+ assert job_config['suite'] == 'MY_SUITE'
+
+ def test_config_kernel_section(self):
+ # Don't attempt to send email
+ config.results_email = None
+ job_config = suite.create_initial_config('MY_SUITE', 'main',
+ 'main', 'main', 'testing',
+ 'default', 'centos', 'plana')
+ assert job_config['kernel']['kdb'] is True
+
+
+# maybe use notario for the above?
--- /dev/null
+import teuthology.lock.util
+
+class TestLock(object):
+
+ def test_locked_since_seconds(self):
+ node = { "locked_since": "2013-02-07 19:33:55.000000" }
+ assert teuthology.lock.util.locked_since_seconds(node) > 3600
--- /dev/null
+archive-on-error: true
--- /dev/null
+stop_worker: true
+machine_type: openstack
+os_type: ubuntu
+os_version: "14.04"
+roles:
+- - mon.a
+ - osd.0
+tasks:
+- exec:
+ mon.a:
+ - echo "Well done !"
+
--- /dev/null
+#
+# Copyright (c) 2015, 2016 Red Hat, Inc.
+#
+# Author: Loic Dachary <loic@dachary.org>
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+#
+import argparse
+import logging
+import json
+import os
+import subprocess
+import tempfile
+import shutil
+
+import teuthology.lock
+import teuthology.lock.cli
+import teuthology.lock.query
+import teuthology.lock.util
+import teuthology.misc
+import teuthology.schedule
+import teuthology.suite
+import teuthology.openstack
+import scripts.schedule
+import scripts.lock
+import scripts.suite
+from teuthology.config import config as teuth_config
+from teuthology.config import set_config_attr
+
+
+class Integration(object):
+
+ @classmethod
+ def setup_class(self):
+ teuthology.log.setLevel(logging.DEBUG)
+ set_config_attr(argparse.Namespace())
+ self.teardown_class()
+
+ @classmethod
+ def teardown_class(self):
+ os.system("sudo /etc/init.d/beanstalkd restart")
+ # if this fails it will not show the error but some weird
+ # INTERNALERROR> IndexError: list index out of range
+ # move that to def tearDown for debug and when it works move it
+ # back in tearDownClass so it is not called on every test
+ ownedby = "ownedby='" + teuth_config.openstack['ip']
+ all_instances = teuthology.openstack.OpenStack().run(
+ "server list -f json --long")
+ for instance in json.loads(all_instances):
+ if ownedby in instance['Properties']:
+ teuthology.openstack.OpenStack().run(
+ "server delete --wait " + instance['ID'])
+
+ def setup_worker(self):
+ self.logs = self.d + "/log"
+ os.mkdir(self.logs, 0o755)
+ self.archive = self.d + "/archive"
+ os.mkdir(self.archive, 0o755)
+ self.worker_cmd = ("teuthology-worker --tube openstack " +
+ "-l " + self.logs + " "
+ "--archive-dir " + self.archive + " ")
+ logging.info(self.worker_cmd)
+ self.worker = subprocess.Popen(self.worker_cmd,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ shell=True)
+
+ def wait_worker(self):
+ if not self.worker:
+ return
+
+ (stdoutdata, stderrdata) = self.worker.communicate()
+ stdoutdata = stdoutdata.decode('utf-8')
+ stderrdata = stderrdata.decode('utf-8')
+ logging.info(self.worker_cmd + ":" +
+ " stdout " + stdoutdata +
+ " stderr " + stderrdata + " end ")
+ assert self.worker.returncode == 0
+ self.worker = None
+
+ def get_teuthology_log(self):
+ # the archive is removed before each test, there must
+ # be only one run and one job
+ run = os.listdir(self.archive)[0]
+ job = os.listdir(os.path.join(self.archive, run))[0]
+ path = os.path.join(self.archive, run, job, 'teuthology.log')
+ return open(path, 'r').read()
+
+class TestSuite(Integration):
+
+ def setup_method(self):
+ self.d = tempfile.mkdtemp()
+ self.setup_worker()
+ logging.info("TestSuite: done worker")
+
+ def teardown(self):
+ self.wait_worker()
+ shutil.rmtree(self.d)
+
+ def test_suite_noop(self):
+ cwd = os.getcwd()
+ os.mkdir(self.d + '/upload', 0o755)
+ upload = 'localhost:' + self.d + '/upload'
+ args = ['--suite', 'noop',
+ '--suite-dir', cwd + '/teuthology/openstack/test',
+ '--machine-type', 'openstack',
+ '--archive-upload', upload,
+ '--verbose']
+ logging.info("TestSuite:test_suite_noop")
+ scripts.suite.main(args)
+ self.wait_worker()
+ log = self.get_teuthology_log()
+ assert "teuthology.run:pass" in log
+ assert "Well done" in log
+ upload_key = teuth_config.archive_upload_key
+ if upload_key:
+ ssh = "RSYNC_RSH='ssh -i " + upload_key + "'"
+ else:
+ ssh = ''
+ assert 'teuthology.log' in teuthology.misc.sh(ssh + " rsync -av " + upload)
+
+class TestSchedule(Integration):
+
+ def setup_method(self):
+ self.d = tempfile.mkdtemp()
+ self.setup_worker()
+
+ def teardown(self):
+ self.wait_worker()
+ shutil.rmtree(self.d)
+
+ def test_schedule_stop_worker(self):
+ job = 'teuthology/openstack/test/stop_worker.yaml'
+ args = ['--name', 'fake',
+ '--verbose',
+ '--owner', 'test@test.com',
+ '--worker', 'openstack',
+ job]
+ scripts.schedule.main(args)
+ self.wait_worker()
+
+ def test_schedule_noop(self):
+ job = 'teuthology/openstack/test/noop.yaml'
+ args = ['--name', 'fake',
+ '--verbose',
+ '--owner', 'test@test.com',
+ '--worker', 'openstack',
+ job]
+ scripts.schedule.main(args)
+ self.wait_worker()
+ log = self.get_teuthology_log()
+ assert "teuthology.run:pass" in log
+ assert "Well done" in log
+
+ def test_schedule_resources_hint(self):
+ """It is tricky to test resources hint in a provider agnostic way. The
+ best way seems to ask for at least 1GB of RAM and 10GB
+ disk. Some providers do not offer a 1GB RAM flavor (OVH for
+ instance) and the 2GB RAM will be chosen instead. It however
+ seems unlikely that a 4GB RAM will be chosen because it would
+ mean such a provider has nothing under that limit and it's a
+ little too high.
+
+ Since the default when installing is to ask for 7000 MB, we
+ can reasonably assume that the hint has been taken into
+ account if the instance has less than 4GB RAM.
+ """
+ try:
+ teuthology.openstack.OpenStack().run("volume list")
+ job = 'teuthology/openstack/test/resources_hint.yaml'
+ has_cinder = True
+ except subprocess.CalledProcessError:
+ job = 'teuthology/openstack/test/resources_hint_no_cinder.yaml'
+ has_cinder = False
+ args = ['--name', 'fake',
+ '--verbose',
+ '--owner', 'test@test.com',
+ '--worker', 'openstack',
+ job]
+ scripts.schedule.main(args)
+ self.wait_worker()
+ log = self.get_teuthology_log()
+ assert "teuthology.run:pass" in log
+ assert "RAM size ok" in log
+ if has_cinder:
+ assert "Disk size ok" in log
+
+class TestLock(Integration):
+
+ def setup_method(self):
+ self.options = ['--verbose',
+ '--machine-type', 'openstack' ]
+
+ def test_main(self):
+ args = scripts.lock.parse_args(self.options + ['--lock'])
+ assert teuthology.lock.cli.main(args) == 0
+
+ def test_lock_unlock(self):
+ default_archs = teuthology.openstack.OpenStack().get_available_archs()
+ if 'TEST_IMAGES' in os.environ:
+ images = os.environ['TEST_IMAGES'].split()
+ else:
+ images = teuthology.openstack.OpenStack.image2url.keys()
+ for image in images:
+ (os_type, os_version, arch) = image.split('-')
+ if arch not in default_archs:
+ logging.info("skipping " + image + " because arch " +
+ " is not supported (" + str(default_archs) + ")")
+ continue
+ args = scripts.lock.parse_args(self.options +
+ ['--lock-many', '1',
+ '--os-type', os_type,
+ '--os-version', os_version,
+ '--arch', arch])
+ assert teuthology.lock.cli.main(args) == 0
+ locks = teuthology.lock.query.list_locks(locked=True)
+ assert len(locks) == 1
+ args = scripts.lock.parse_args(self.options +
+ ['--unlock', locks[0]['name']])
+ assert teuthology.lock.cli.main(args) == 0
+
+ def test_list(self, capsys):
+ args = scripts.lock.parse_args(self.options + ['--list', '--all'])
+ teuthology.lock.cli.main(args)
+ out, err = capsys.readouterr()
+ assert 'machine_type' in out
+ assert 'openstack' in out
--- /dev/null
+stop_worker: true
+machine_type: openstack
+openstack:
+ - machine:
+ disk: 10 # GB
+ ram: 10000 # MB
+ cpus: 1
+ volumes:
+ count: 1
+ size: 2 # GB
+os_type: ubuntu
+os_version: "14.04"
+roles:
+- - mon.a
+ - osd.0
+tasks:
+- exec:
+ mon.a:
+ - test $(sed -n -e 's/MemTotal.* \([0-9][0-9]*\).*/\1/p' < /proc/meminfo) -ge 10000000 && echo "RAM" "size" "ok"
+ - cat /proc/meminfo
+# wait for the attached volume to show up
+ - for delay in 1 2 4 8 16 32 64 128 256 512 ; do if test -e /sys/block/vdb/size ; then break ; else sleep $delay ; fi ; done
+# 4000000 because 512 bytes sectors
+ - test $(cat /sys/block/vdb/size) -gt 4000000 && echo "Disk" "size" "ok"
+ - cat /sys/block/vdb/size
--- /dev/null
+stop_worker: true
+machine_type: openstack
+openstack:
+ - machine:
+ disk: 10 # GB
+ ram: 10000 # MB
+ cpus: 1
+ volumes:
+ count: 0
+ size: 2 # GB
+os_type: ubuntu
+os_version: "14.04"
+roles:
+- - mon.a
+ - osd.0
+tasks:
+- exec:
+ mon.a:
+ - cat /proc/meminfo
+ - test $(sed -n -e 's/MemTotal.* \([0-9][0-9]*\).*/\1/p' < /proc/meminfo) -ge 10000000 && echo "RAM" "size" "ok"
--- /dev/null
+stop_worker: true
--- /dev/null
+stop_worker: true
+roles:
+- - mon.a
+ - osd.0
+tasks:
+- exec:
+ mon.a:
+ - echo "Well done !"
+
--- /dev/null
+from teuthology.config import config
+
+
+class TestOpenStack(object):
+
+ def setup_method(self):
+ self.openstack_config = config['openstack']
+
+ def test_config_clone(self):
+ assert 'clone' in self.openstack_config
+
+ def test_config_user_data(self):
+ os_type = 'rhel'
+ os_version = '7.0'
+ template_path = self.openstack_config['user-data'].format(
+ os_type=os_type,
+ os_version=os_version)
+ assert os_type in template_path
+ assert os_version in template_path
+
+ def test_config_ip(self):
+ assert 'ip' in self.openstack_config
+
+ def test_config_machine(self):
+ assert 'machine' in self.openstack_config
+ machine_config = self.openstack_config['machine']
+ assert 'disk' in machine_config
+ assert 'ram' in machine_config
+ assert 'cpus' in machine_config
+
+ def test_config_volumes(self):
+ assert 'volumes' in self.openstack_config
+ volumes_config = self.openstack_config['volumes']
+ assert 'count' in volumes_config
+ assert 'size' in volumes_config
--- /dev/null
+#
+# Copyright (c) 2015,2016 Red Hat, Inc.
+#
+# Author: Loic Dachary <loic@dachary.org>
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+#
+import argparse
+import logging
+import os
+import pytest
+import subprocess
+import tempfile
+import time
+from mock import patch
+
+import teuthology
+from teuthology import misc
+from teuthology.config import set_config_attr
+from teuthology.openstack import TeuthologyOpenStack, OpenStack, OpenStackInstance
+from teuthology.openstack import NoFlavorException
+import scripts.openstack
+
+
+class TestOpenStackBase(object):
+
+ def setup_method(self):
+ OpenStack.token = None
+ OpenStack.token_expires = None
+ self.environ = {}
+ for k in os.environ.keys():
+ if k.startswith('OS_'):
+ self.environ[k] = os.environ[k]
+
+ def teardown_method(self):
+ OpenStack.token = None
+ OpenStack.token_expires = None
+ for k in os.environ.keys():
+ if k.startswith('OS_'):
+ if k in self.environ:
+ os.environ[k] = self.environ[k]
+ else:
+ del os.environ[k]
+
+class TestOpenStackInstance(TestOpenStackBase):
+
+ teuthology_instance = """
+{
+ "OS-EXT-STS:task_state": null,
+ "addresses": "Ext-Net=167.114.233.32",
+ "image": "Ubuntu 14.04 (0d315a8d-75e3-418a-80e4-48e62d599627)",
+ "OS-EXT-STS:vm_state": "active",
+ "OS-SRV-USG:launched_at": "2015-08-17T12:22:13.000000",
+ "flavor": "vps-ssd-1 (164fcc7e-7771-414f-a607-b388cb7b7aa0)",
+ "id": "f3ca32d7-212b-458b-a0d4-57d1085af953",
+ "security_groups": [
+ {
+ "name": "default"
+ }
+ ],
+ "user_id": "3a075820e5d24fda96cd340b87fd94e9",
+ "OS-DCF:diskConfig": "AUTO",
+ "accessIPv4": "",
+ "accessIPv6": "",
+ "progress": 0,
+ "OS-EXT-STS:power_state": 1,
+ "project_id": "62cf1be03cec403c8ed8e64df55732ea",
+ "config_drive": "",
+ "status": "ACTIVE",
+ "updated": "2015-11-03T13:48:53Z",
+ "hostId": "bcdf964b6f724e573c07156ff85b4db1707f6f0969f571cf33e0468d",
+ "OS-SRV-USG:terminated_at": null,
+ "key_name": "loic",
+ "properties": "",
+ "OS-EXT-AZ:availability_zone": "nova",
+ "name": "mrdarkdragon",
+ "created": "2015-08-17T12:21:31Z",
+ "os-extended-volumes:volumes_attached": [{"id": "627e2631-fbb3-48cd-b801-d29cd2a76f74"}, {"id": "09837649-0881-4ee2-a560-adabefc28764"}, {"id": "44e5175b-6044-40be-885a-c9ddfb6f75bb"}]
+}
+ """
+
+ teuthology_instance_no_addresses = """
+{
+ "OS-EXT-STS:task_state": null,
+ "addresses": "",
+ "image": "Ubuntu 14.04 (0d315a8d-75e3-418a-80e4-48e62d599627)",
+ "OS-EXT-STS:vm_state": "active",
+ "OS-SRV-USG:launched_at": "2015-08-17T12:22:13.000000",
+ "flavor": "vps-ssd-1 (164fcc7e-7771-414f-a607-b388cb7b7aa0)",
+ "id": "f3ca32d7-212b-458b-a0d4-57d1085af953",
+ "security_groups": [
+ {
+ "name": "default"
+ }
+ ],
+ "user_id": "3a075820e5d24fda96cd340b87fd94e9",
+ "OS-DCF:diskConfig": "AUTO",
+ "accessIPv4": "",
+ "accessIPv6": "",
+ "progress": 0,
+ "OS-EXT-STS:power_state": 1,
+ "project_id": "62cf1be03cec403c8ed8e64df55732ea",
+ "config_drive": "",
+ "status": "ACTIVE",
+ "updated": "2015-11-03T13:48:53Z",
+ "hostId": "bcdf964b6f724e573c07156ff85b4db1707f6f0969f571cf33e0468d",
+ "OS-SRV-USG:terminated_at": null,
+ "key_name": "loic",
+ "properties": "",
+ "OS-EXT-AZ:availability_zone": "nova",
+ "name": "mrdarkdragon",
+ "created": "2015-08-17T12:21:31Z",
+ "os-extended-volumes:volumes_attached": []
+}
+ """
+
+ @classmethod
+ def setup_class(self):
+ if 'OS_AUTH_URL' not in os.environ:
+ pytest.skip('no OS_AUTH_URL environment variable')
+
+ def test_init(self):
+ with patch.multiple(
+ misc,
+ sh=lambda cmd: self.teuthology_instance,
+ ):
+ o = OpenStackInstance('NAME')
+ assert o['id'] == 'f3ca32d7-212b-458b-a0d4-57d1085af953'
+ o = OpenStackInstance('NAME', {"id": "OTHER"})
+ assert o['id'] == "OTHER"
+
+ def test_get_created(self):
+ with patch.multiple(
+ misc,
+ sh=lambda cmd: self.teuthology_instance,
+ ):
+ o = OpenStackInstance('NAME')
+ assert o.get_created() > 0
+
+ def test_exists(self):
+ with patch.multiple(
+ misc,
+ sh=lambda cmd: self.teuthology_instance,
+ ):
+ o = OpenStackInstance('NAME')
+ assert o.exists()
+ def sh_raises(cmd):
+ raise subprocess.CalledProcessError('FAIL', 'BAD')
+ with patch.multiple(
+ misc,
+ sh=sh_raises,
+ ):
+ o = OpenStackInstance('NAME')
+ assert not o.exists()
+
+ def test_volumes(self):
+ with patch.multiple(
+ misc,
+ sh=lambda cmd: self.teuthology_instance,
+ ):
+ o = OpenStackInstance('NAME')
+ assert len(o.get_volumes()) == 3
+
+ def test_get_addresses(self):
+ answers = [
+ self.teuthology_instance_no_addresses,
+ self.teuthology_instance,
+ ]
+ def sh(self):
+ return answers.pop(0)
+ with patch.multiple(
+ misc,
+ sh=sh,
+ ):
+ o = OpenStackInstance('NAME')
+ assert o.get_addresses() == 'Ext-Net=167.114.233.32'
+
+ def test_get_ip_neutron(self):
+ instance_id = '8e1fd70a-3065-46f8-9c30-84dc028c1834'
+ ip = '10.10.10.4'
+ def sh(cmd):
+ if 'neutron subnet-list' in cmd:
+ return """
+[
+ {
+ "ip_version": 6,
+ "id": "c45b9661-b2ba-4817-9e3a-f8f63bf32989"
+ },
+ {
+ "ip_version": 4,
+ "id": "e03a3dbc-afc8-4b52-952e-7bf755397b50"
+ }
+]
+ """
+ elif 'neutron port-list' in cmd:
+ return ("""
+[
+ {
+ "device_id": "915504ad-368b-4cce-be7c-4f8a83902e28",
+ "fixed_ips": "{\\"subnet_id\\": \\"e03a3dbc-afc8-4b52-952e-7bf755397b50\\", \\"ip_address\\": \\"10.10.10.1\\"}\\n{\\"subnet_id\\": \\"c45b9661-b2ba-4817-9e3a-f8f63bf32989\\", \\"ip_address\\": \\"2607:f298:6050:9afc::1\\"}"
+ },
+ {
+ "device_id": "{instance_id}",
+ "fixed_ips": "{\\"subnet_id\\": \\"e03a3dbc-afc8-4b52-952e-7bf755397b50\\", \\"ip_address\\": \\"{ip}\\"}\\n{\\"subnet_id\\": \\"c45b9661-b2ba-4817-9e3a-f8f63bf32989\\", \\"ip_address\\": \\"2607:f298:6050:9afc:f816:3eff:fe07:76c1\\"}"
+ },
+ {
+ "device_id": "17e4a968-4caa-4cee-8e4b-f950683a02bd",
+ "fixed_ips": "{\\"subnet_id\\": \\"e03a3dbc-afc8-4b52-952e-7bf755397b50\\", \\"ip_address\\": \\"10.10.10.5\\"}\\n{\\"subnet_id\\": \\"c45b9661-b2ba-4817-9e3a-f8f63bf32989\\", \\"ip_address\\": \\"2607:f298:6050:9afc:f816:3eff:fe9c:37f0\\"}"
+ }
+]
+ """.replace('{instance_id}', instance_id).
+ replace('{ip}', ip))
+ else:
+ raise Exception("unexpected " + cmd)
+ with patch.multiple(
+ misc,
+ sh=sh,
+ ):
+ assert ip == OpenStackInstance(
+ instance_id,
+ { 'id': instance_id },
+ ).get_ip_neutron()
+
+class TestOpenStack(TestOpenStackBase):
+
+ flavors = """[
+ {
+ "Name": "eg-120-ssd",
+ "RAM": 120000,
+ "Ephemeral": 0,
+ "VCPUs": 32,
+ "Is Public": true,
+ "Disk": 800,
+ "ID": "008f75de-c467-4d15-8f70-79c8fbe19538"
+ },
+ {
+ "Name": "hg-60",
+ "RAM": 60000,
+ "Ephemeral": 0,
+ "VCPUs": 16,
+ "Is Public": true,
+ "Disk": 1600,
+ "ID": "0297d7ac-fe6f-4ff1-b6e7-0b8b0908c94f"
+ },
+ {
+ "Name": "win-sp-120-ssd-flex",
+ "RAM": 120000,
+ "Ephemeral": 0,
+ "VCPUs": 8,
+ "Is Public": true,
+ "Disk": 50,
+ "ID": "039e31f2-6541-46c8-85cf-7f47fab7ad78"
+ },
+ {
+ "Name": "win-sp-60",
+ "RAM": 60000,
+ "Ephemeral": 0,
+ "VCPUs": 4,
+ "Is Public": true,
+ "Disk": 400,
+ "ID": "0417a0e6-f68a-4b8b-a642-ca5ecb9652f7"
+ },
+ {
+ "Name": "hg-120-ssd",
+ "RAM": 120000,
+ "Ephemeral": 0,
+ "VCPUs": 32,
+ "Is Public": true,
+ "Disk": 800,
+ "ID": "042aefc6-b713-4a7e-ada5-3ff81daa1960"
+ },
+ {
+ "Name": "win-sp-60-flex",
+ "RAM": 60000,
+ "Ephemeral": 0,
+ "VCPUs": 4,
+ "Is Public": true,
+ "Disk": 50,
+ "ID": "0609290c-ad2a-40f0-8c66-c755dd38fe3f"
+ },
+ {
+ "Name": "win-eg-120",
+ "RAM": 120000,
+ "Ephemeral": 0,
+ "VCPUs": 32,
+ "Is Public": true,
+ "Disk": 800,
+ "ID": "0651080f-5d07-44b1-a759-7ea4594b669e"
+ },
+ {
+ "Name": "win-sp-240",
+ "RAM": 240000,
+ "Ephemeral": 0,
+ "VCPUs": 16,
+ "Is Public": true,
+ "Disk": 1600,
+ "ID": "07885848-8831-486d-8525-91484c09cc7e"
+ },
+ {
+ "Name": "win-hg-60-ssd",
+ "RAM": 60000,
+ "Ephemeral": 0,
+ "VCPUs": 16,
+ "Is Public": true,
+ "Disk": 800,
+ "ID": "079aa0a2-5e48-4e58-8205-719bc962736e"
+ },
+ {
+ "Name": "eg-120",
+ "RAM": 120000,
+ "Ephemeral": 0,
+ "VCPUs": 32,
+ "Is Public": true,
+ "Disk": 1600,
+ "ID": "090f8b8c-673c-4ab8-9a07-6e54a8776e7b"
+ },
+ {
+ "Name": "win-hg-15-ssd-flex",
+ "RAM": 15000,
+ "Ephemeral": 0,
+ "VCPUs": 4,
+ "Is Public": true,
+ "Disk": 50,
+ "ID": "10e10c58-d29f-4ff6-a1fd-085c35a3bd9b"
+ },
+ {
+ "Name": "eg-15-ssd",
+ "RAM": 15000,
+ "Ephemeral": 0,
+ "VCPUs": 4,
+ "Is Public": true,
+ "Disk": 200,
+ "ID": "1340a920-0f2f-4c1b-8d74-e2502258da73"
+ },
+ {
+ "Name": "win-eg-30-ssd-flex",
+ "RAM": 30000,
+ "Ephemeral": 0,
+ "VCPUs": 8,
+ "Is Public": true,
+ "Disk": 50,
+ "ID": "13e54752-fbd0-47a6-aa93-e5a67dfbc743"
+ },
+ {
+ "Name": "eg-120-ssd-flex",
+ "RAM": 120000,
+ "Ephemeral": 0,
+ "VCPUs": 32,
+ "Is Public": true,
+ "Disk": 50,
+ "ID": "15c07a54-2dfb-41d9-aa73-6989fd8cafc2"
+ },
+ {
+ "Name": "win-eg-120-ssd-flex",
+ "RAM": 120000,
+ "Ephemeral": 0,
+ "VCPUs": 32,
+ "Is Public": true,
+ "Disk": 50,
+ "ID": "15e0dfcc-10f4-4e70-8ac1-30bc323273e2"
+ },
+ {
+ "Name": "vps-ssd-1",
+ "RAM": 2000,
+ "Ephemeral": 0,
+ "VCPUs": 1,
+ "Is Public": true,
+ "Disk": 10,
+ "ID": "164fcc7e-7771-414f-a607-b388cb7b7aa0"
+ },
+ {
+ "Name": "win-sp-120-flex",
+ "RAM": 120000,
+ "Ephemeral": 0,
+ "VCPUs": 8,
+ "Is Public": true,
+ "Disk": 50,
+ "ID": "169415e1-0979-4527-94fb-638c885bbd8c"
+ },
+ {
+ "Name": "win-hg-60-flex",
+ "RAM": 60000,
+ "Ephemeral": 0,
+ "VCPUs": 16,
+ "Is Public": true,
+ "Disk": 50,
+ "ID": "16f13d5b-be27-4b8b-88da-959d3904d3ba"
+ },
+ {
+ "Name": "win-sp-30-ssd",
+ "RAM": 30000,
+ "Ephemeral": 0,
+ "VCPUs": 2,
+ "Is Public": true,
+ "Disk": 100,
+ "ID": "1788102b-ab80-4a0c-b819-541deaca7515"
+ },
+ {
+ "Name": "win-sp-240-flex",
+ "RAM": 240000,
+ "Ephemeral": 0,
+ "VCPUs": 16,
+ "Is Public": true,
+ "Disk": 50,
+ "ID": "17bcfa14-135f-442f-9397-a4dc25265560"
+ },
+ {
+ "Name": "win-eg-60-ssd-flex",
+ "RAM": 60000,
+ "Ephemeral": 0,
+ "VCPUs": 16,
+ "Is Public": true,
+ "Disk": 50,
+ "ID": "194ca9ba-04af-4d86-ba37-d7da883a7eab"
+ },
+ {
+ "Name": "win-eg-60-flex",
+ "RAM": 60000,
+ "Ephemeral": 0,
+ "VCPUs": 16,
+ "Is Public": true,
+ "Disk": 50,
+ "ID": "19ff8837-4751-4f6c-a82b-290bc53c83c1"
+ },
+ {
+ "Name": "win-eg-30-flex",
+ "RAM": 30000,
+ "Ephemeral": 0,
+ "VCPUs": 8,
+ "Is Public": true,
+ "Disk": 50,
+ "ID": "1aaef5e5-4df9-4462-80d3-701683ab9ff0"
+ },
+ {
+ "Name": "eg-15",
+ "RAM": 15000,
+ "Ephemeral": 0,
+ "VCPUs": 4,
+ "Is Public": true,
+ "Disk": 400,
+ "ID": "1cd85b81-5e4d-477a-a127-eb496b1d75de"
+ },
+ {
+ "Name": "hg-120",
+ "RAM": 120000,
+ "Ephemeral": 0,
+ "VCPUs": 32,
+ "Is Public": true,
+ "Disk": 1600,
+ "ID": "1f1efedf-ec91-4a42-acd7-f5cf64b02d3c"
+ },
+ {
+ "Name": "hg-15-ssd-flex",
+ "RAM": 15000,
+ "Ephemeral": 0,
+ "VCPUs": 4,
+ "Is Public": true,
+ "Disk": 50,
+ "ID": "20347a07-a289-4c07-a645-93cb5e8e2d30"
+ },
+ {
+ "Name": "win-eg-7-ssd",
+ "RAM": 7000,
+ "Ephemeral": 0,
+ "VCPUs": 2,
+ "Is Public": true,
+ "Disk": 100,
+ "ID": "20689394-bd77-4f4d-900e-52cc8a86aeb4"
+ },
+ {
+ "Name": "win-sp-60-ssd-flex",
+ "RAM": 60000,
+ "Ephemeral": 0,
+ "VCPUs": 4,
+ "Is Public": true,
+ "Disk": 50,
+ "ID": "21104d99-ba7b-47a0-9133-7e884710089b"
+ },
+ {
+ "Name": "win-sp-120-ssd",
+ "RAM": 120000,
+ "Ephemeral": 0,
+ "VCPUs": 8,
+ "Is Public": true,
+ "Disk": 400,
+ "ID": "23c21ecc-9ee8-4ad3-bd9f-aa17a3faf84e"
+ },
+ {
+ "Name": "win-hg-15-ssd",
+ "RAM": 15000,
+ "Ephemeral": 0,
+ "VCPUs": 4,
+ "Is Public": true,
+ "Disk": 200,
+ "ID": "24e293ed-bc54-4f26-8fb7-7b9457d08e66"
+ },
+ {
+ "Name": "eg-15-ssd-flex",
+ "RAM": 15000,
+ "Ephemeral": 0,
+ "VCPUs": 4,
+ "Is Public": true,
+ "Disk": 50,
+ "ID": "25f3534a-89e5-489d-aa8b-63f62e76875b"
+ },
+ {
+ "Name": "win-eg-60",
+ "RAM": 60000,
+ "Ephemeral": 0,
+ "VCPUs": 16,
+ "Is Public": true,
+ "Disk": 800,
+ "ID": "291173f1-ea1d-410b-8045-667361a4addb"
+ },
+ {
+ "Name": "sp-30-ssd-flex",
+ "RAM": 30000,
+ "Ephemeral": 0,
+ "VCPUs": 2,
+ "Is Public": true,
+ "Disk": 50,
+ "ID": "2b646463-2efa-428b-94ed-4059923c3636"
+ },
+ {
+ "Name": "win-eg-120-flex",
+ "RAM": 120000,
+ "Ephemeral": 0,
+ "VCPUs": 32,
+ "Is Public": true,
+ "Disk": 50,
+ "ID": "2c74df82-29d2-4b1a-a32c-d5633e7359b4"
+ },
+ {
+ "Name": "win-eg-15-ssd",
+ "RAM": 15000,
+ "Ephemeral": 0,
+ "VCPUs": 4,
+ "Is Public": true,
+ "Disk": 200,
+ "ID": "2fe4344f-d701-4bc4-8dcd-6d0b5d83fa13"
+ },
+ {
+ "Name": "sp-30-flex",
+ "RAM": 30000,
+ "Ephemeral": 0,
+ "VCPUs": 2,
+ "Is Public": true,
+ "Disk": 50,
+ "ID": "31487b30-eeb6-472f-a9b6-38ace6587ebc"
+ },
+ {
+ "Name": "win-sp-240-ssd",
+ "RAM": 240000,
+ "Ephemeral": 0,
+ "VCPUs": 16,
+ "Is Public": true,
+ "Disk": 800,
+ "ID": "325b602f-ecc4-4444-90bd-5a2cf4e0da53"
+ },
+ {
+ "Name": "win-hg-7",
+ "RAM": 7000,
+ "Ephemeral": 0,
+ "VCPUs": 2,
+ "Is Public": true,
+ "Disk": 200,
+ "ID": "377ded36-491f-4ad7-9eb4-876798b2aea9"
+ },
+ {
+ "Name": "sp-30-ssd",
+ "RAM": 30000,
+ "Ephemeral": 0,
+ "VCPUs": 2,
+ "Is Public": true,
+ "Disk": 100,
+ "ID": "382f2831-4dba-40c4-bb7a-6fadff71c4db"
+ },
+ {
+ "Name": "hg-30",
+ "RAM": 30000,
+ "Ephemeral": 0,
+ "VCPUs": 8,
+ "Is Public": true,
+ "Disk": 800,
+ "ID": "3c1d6170-0097-4b5c-a3b3-adff1b7a86e0"
+ },
+ {
+ "Name": "hg-60-flex",
+ "RAM": 60000,
+ "Ephemeral": 0,
+ "VCPUs": 16,
+ "Is Public": true,
+ "Disk": 50,
+ "ID": "3c669730-b5cd-4e44-8bd2-bc8d9f984ab2"
+ },
+ {
+ "Name": "sp-240-ssd-flex",
+ "RAM": 240000,
+ "Ephemeral": 0,
+ "VCPUs": 16,
+ "Is Public": true,
+ "Disk": 50,
+ "ID": "3d66fea3-26f2-4195-97ab-fdea3b836099"
+ },
+ {
+ "Name": "sp-240-flex",
+ "RAM": 240000,
+ "Ephemeral": 0,
+ "VCPUs": 16,
+ "Is Public": true,
+ "Disk": 50,
+ "ID": "40c781f7-d7a7-4b0d-bcca-5304aeabbcd9"
+ },
+ {
+ "Name": "hg-7-flex",
+ "RAM": 7000,
+ "Ephemeral": 0,
+ "VCPUs": 2,
+ "Is Public": true,
+ "Disk": 50,
+ "ID": "42730e52-147d-46b8-9546-18e31e5ac8a9"
+ },
+ {
+ "Name": "eg-30-ssd",
+ "RAM": 30000,
+ "Ephemeral": 0,
+ "VCPUs": 8,
+ "Is Public": true,
+ "Disk": 400,
+ "ID": "463f30e9-7d7a-4693-944f-142067cf553b"
+ },
+ {
+ "Name": "hg-15-flex",
+ "RAM": 15000,
+ "Ephemeral": 0,
+ "VCPUs": 4,
+ "Is Public": true,
+ "Disk": 50,
+ "ID": "534f07c6-91af-44c8-9e62-156360fe8359"
+ },
+ {
+ "Name": "win-sp-30-flex",
+ "RAM": 30000,
+ "Ephemeral": 0,
+ "VCPUs": 2,
+ "Is Public": true,
+ "Disk": 50,
+ "ID": "55533fdf-ad57-4aa7-a2c6-ee31bb94e77b"
+ },
+ {
+ "Name": "win-hg-60-ssd-flex",
+ "RAM": 60000,
+ "Ephemeral": 0,
+ "VCPUs": 16,
+ "Is Public": true,
+ "Disk": 50,
+ "ID": "58b24234-3804-4c4f-9eb6-5406a3a13758"
+ },
+ {
+ "Name": "hg-7-ssd-flex",
+ "RAM": 7000,
+ "Ephemeral": 0,
+ "VCPUs": 2,
+ "Is Public": true,
+ "Disk": 50,
+ "ID": "596c1276-8e53-40a0-b183-cdd9e9b1907d"
+ },
+ {
+ "Name": "win-hg-30-ssd",
+ "RAM": 30000,
+ "Ephemeral": 0,
+ "VCPUs": 8,
+ "Is Public": true,
+ "Disk": 400,
+ "ID": "5c54dc08-28b9-4860-9f24-a2451b2a28ec"
+ },
+ {
+ "Name": "eg-7",
+ "RAM": 7000,
+ "Ephemeral": 0,
+ "VCPUs": 2,
+ "Is Public": true,
+ "Disk": 200,
+ "ID": "5e409dbc-3f4b-46e8-a629-a418c8497922"
+ },
+ {
+ "Name": "hg-30-flex",
+ "RAM": 30000,
+ "Ephemeral": 0,
+ "VCPUs": 8,
+ "Is Public": true,
+ "Disk": 50,
+ "ID": "656423ea-0551-48c6-9e0f-ec6e15952029"
+ },
+ {
+ "Name": "hg-15",
+ "RAM": 15000,
+ "Ephemeral": 0,
+ "VCPUs": 4,
+ "Is Public": true,
+ "Disk": 400,
+ "ID": "675558ea-04fe-47a2-83de-40be9b2eacd4"
+ },
+ {
+ "Name": "eg-60-flex",
+ "RAM": 60000,
+ "Ephemeral": 0,
+ "VCPUs": 16,
+ "Is Public": true,
+ "Disk": 50,
+ "ID": "68a8e4e1-d291-46e8-a724-fbb1c4b9b051"
+ },
+ {
+ "Name": "hg-30-ssd",
+ "RAM": 30000,
+ "Ephemeral": 0,
+ "VCPUs": 8,
+ "Is Public": true,
+ "Disk": 400,
+ "ID": "6ab72807-e0a5-4e9f-bbb9-7cbbf0038b26"
+ },
+ {
+ "Name": "win-hg-30",
+ "RAM": 30000,
+ "Ephemeral": 0,
+ "VCPUs": 8,
+ "Is Public": true,
+ "Disk": 800,
+ "ID": "6e12cae3-0492-483c-aa39-54a0dcaf86dd"
+ },
+ {
+ "Name": "win-hg-7-ssd",
+ "RAM": 7000,
+ "Ephemeral": 0,
+ "VCPUs": 2,
+ "Is Public": true,
+ "Disk": 100,
+ "ID": "6ead771c-e8b9-424c-afa0-671280416422"
+ },
+ {
+ "Name": "win-hg-30-flex",
+ "RAM": 30000,
+ "Ephemeral": 0,
+ "VCPUs": 8,
+ "Is Public": true,
+ "Disk": 50,
+ "ID": "70ded741-8f58-4bb9-8cfd-5e838b66b5f3"
+ },
+ {
+ "Name": "win-sp-30-ssd-flex",
+ "RAM": 30000,
+ "Ephemeral": 0,
+ "VCPUs": 2,
+ "Is Public": true,
+ "Disk": 50,
+ "ID": "7284d104-a260-421d-8cee-6dc905107b25"
+ },
+ {
+ "Name": "win-eg-120-ssd",
+ "RAM": 120000,
+ "Ephemeral": 0,
+ "VCPUs": 32,
+ "Is Public": true,
+ "Disk": 800,
+ "ID": "72c0b262-855d-40bb-a3e9-fd989a1bc466"
+ },
+ {
+ "Name": "win-hg-7-flex",
+ "RAM": 7000,
+ "Ephemeral": 0,
+ "VCPUs": 2,
+ "Is Public": true,
+ "Disk": 50,
+ "ID": "73961591-c5f1-436f-b641-1a506eddaef4"
+ },
+ {
+ "Name": "sp-240-ssd",
+ "RAM": 240000,
+ "Ephemeral": 0,
+ "VCPUs": 16,
+ "Is Public": true,
+ "Disk": 800,
+ "ID": "7568d834-3b16-42ce-a2c1-0654e0781160"
+ },
+ {
+ "Name": "win-eg-60-ssd",
+ "RAM": 60000,
+ "Ephemeral": 0,
+ "VCPUs": 16,
+ "Is Public": true,
+ "Disk": 800,
+ "ID": "75f7fe5c-f87a-41d8-a961-a0169d02c268"
+ },
+ {
+ "Name": "eg-7-ssd-flex",
+ "RAM": 7000,
+ "Ephemeral": 0,
+ "VCPUs": 2,
+ "Is Public": true,
+ "Disk": 50,
+ "ID": "77e1db73-0b36-4e37-8e47-32c2d2437ca9"
+ },
+ {
+ "Name": "eg-60-ssd-flex",
+ "RAM": 60000,
+ "Ephemeral": 0,
+ "VCPUs": 16,
+ "Is Public": true,
+ "Disk": 50,
+ "ID": "78df4e30-98ca-4362-af68-037d958edaf0"
+ },
+ {
+ "Name": "vps-ssd-2",
+ "RAM": 4000,
+ "Ephemeral": 0,
+ "VCPUs": 1,
+ "Is Public": true,
+ "Disk": 20,
+ "ID": "7939cc5c-79b1-45c0-be2d-aa935d92faa1"
+ },
+ {
+ "Name": "sp-60",
+ "RAM": 60000,
+ "Ephemeral": 0,
+ "VCPUs": 4,
+ "Is Public": true,
+ "Disk": 400,
+ "ID": "80d8510a-79cc-4307-8db7-d1965c9e8ddb"
+ },
+ {
+ "Name": "win-hg-120-ssd-flex",
+ "RAM": 120000,
+ "Ephemeral": 0,
+ "VCPUs": 32,
+ "Is Public": true,
+ "Disk": 50,
+ "ID": "835e734a-46b6-4cb2-be68-e8678fd71059"
+ },
+ {
+ "Name": "win-eg-7",
+ "RAM": 7000,
+ "Ephemeral": 0,
+ "VCPUs": 2,
+ "Is Public": true,
+ "Disk": 200,
+ "ID": "84869b00-b43a-4523-babd-d47d206694e9"
+ },
+ {
+ "Name": "win-eg-7-ssd-flex",
+ "RAM": 7000,
+ "Ephemeral": 0,
+ "VCPUs": 2,
+ "Is Public": true,
+ "Disk": 50,
+ "ID": "852308f8-b8bf-44a4-af41-cbc27437b275"
+ },
+ {
+ "Name": "win-sp-30",
+ "RAM": 30000,
+ "Ephemeral": 0,
+ "VCPUs": 2,
+ "Is Public": true,
+ "Disk": 200,
+ "ID": "8be9dc29-3eca-499b-ae2d-e3c99699131a"
+ },
+ {
+ "Name": "win-hg-7-ssd-flex",
+ "RAM": 7000,
+ "Ephemeral": 0,
+ "VCPUs": 2,
+ "Is Public": true,
+ "Disk": 50,
+ "ID": "8d704cfd-05b2-4d4a-add2-e2868bcc081f"
+ },
+ {
+ "Name": "eg-30",
+ "RAM": 30000,
+ "Ephemeral": 0,
+ "VCPUs": 8,
+ "Is Public": true,
+ "Disk": 800,
+ "ID": "901f77c2-73f6-4fae-b28a-18b829b55a17"
+ },
+ {
+ "Name": "sp-60-ssd-flex",
+ "RAM": 60000,
+ "Ephemeral": 0,
+ "VCPUs": 4,
+ "Is Public": true,
+ "Disk": 50,
+ "ID": "944b92fb-9a0c-406d-bb9f-a1d93cda9f01"
+ },
+ {
+ "Name": "eg-30-flex",
+ "RAM": 30000,
+ "Ephemeral": 0,
+ "VCPUs": 8,
+ "Is Public": true,
+ "Disk": 50,
+ "ID": "965472c7-eb54-4d4d-bd6e-01ebb694a631"
+ },
+ {
+ "Name": "sp-120-ssd",
+ "RAM": 120000,
+ "Ephemeral": 0,
+ "VCPUs": 8,
+ "Is Public": true,
+ "Disk": 400,
+ "ID": "97824a8c-e683-49a8-a70a-ead64240395c"
+ },
+ {
+ "Name": "hg-60-ssd",
+ "RAM": 60000,
+ "Ephemeral": 0,
+ "VCPUs": 16,
+ "Is Public": true,
+ "Disk": 800,
+ "ID": "9831d7f1-3e79-483d-8958-88e3952c7ea2"
+ },
+ {
+ "Name": "eg-60",
+ "RAM": 60000,
+ "Ephemeral": 0,
+ "VCPUs": 16,
+ "Is Public": true,
+ "Disk": 1600,
+ "ID": "9e1f13d0-4fcc-4abc-a9e6-9c76d662c92d"
+ },
+ {
+ "Name": "win-eg-30-ssd",
+ "RAM": 30000,
+ "Ephemeral": 0,
+ "VCPUs": 8,
+ "Is Public": true,
+ "Disk": 400,
+ "ID": "9e6b85fa-6f37-45ce-a3d6-11ab40a28fad"
+ },
+ {
+ "Name": "hg-120-flex",
+ "RAM": 120000,
+ "Ephemeral": 0,
+ "VCPUs": 32,
+ "Is Public": true,
+ "Disk": 50,
+ "ID": "9ed787cc-a0db-400b-8cc1-49b6384a1000"
+ },
+ {
+ "Name": "sp-120-flex",
+ "RAM": 120000,
+ "Ephemeral": 0,
+ "VCPUs": 8,
+ "Is Public": true,
+ "Disk": 50,
+ "ID": "9f3cfdf7-b850-47cc-92be-33aefbfd2b05"
+ },
+ {
+ "Name": "hg-60-ssd-flex",
+ "RAM": 60000,
+ "Ephemeral": 0,
+ "VCPUs": 16,
+ "Is Public": true,
+ "Disk": 50,
+ "ID": "a37bdf17-e1b1-41cc-a67f-ed665a120446"
+ },
+ {
+ "Name": "win-hg-120-ssd",
+ "RAM": 120000,
+ "Ephemeral": 0,
+ "VCPUs": 32,
+ "Is Public": true,
+ "Disk": 800,
+ "ID": "aa753e73-dadb-4528-9c4a-24e36fc41bf4"
+ },
+ {
+ "Name": "win-sp-240-ssd-flex",
+ "RAM": 240000,
+ "Ephemeral": 0,
+ "VCPUs": 16,
+ "Is Public": true,
+ "Disk": 50,
+ "ID": "abc007b8-cc44-4b6b-9606-fd647b03e101"
+ },
+ {
+ "Name": "sp-120",
+ "RAM": 120000,
+ "Ephemeral": 0,
+ "VCPUs": 8,
+ "Is Public": true,
+ "Disk": 800,
+ "ID": "ac74cb45-d895-47dd-b9cf-c17778033d83"
+ },
+ {
+ "Name": "win-hg-15",
+ "RAM": 15000,
+ "Ephemeral": 0,
+ "VCPUs": 4,
+ "Is Public": true,
+ "Disk": 400,
+ "ID": "ae900175-72bd-4fbc-8ab2-4673b468aa5b"
+ },
+ {
+ "Name": "win-eg-15-ssd-flex",
+ "RAM": 15000,
+ "Ephemeral": 0,
+ "VCPUs": 4,
+ "Is Public": true,
+ "Disk": 50,
+ "ID": "aeb37dbf-d7c9-4fd7-93f1-f3818e488ede"
+ },
+ {
+ "Name": "hg-7-ssd",
+ "RAM": 7000,
+ "Ephemeral": 0,
+ "VCPUs": 2,
+ "Is Public": true,
+ "Disk": 100,
+ "ID": "b1dc776c-b6e3-4a96-b230-850f570db3d5"
+ },
+ {
+ "Name": "sp-60-ssd",
+ "RAM": 60000,
+ "Ephemeral": 0,
+ "VCPUs": 4,
+ "Is Public": true,
+ "Disk": 200,
+ "ID": "b24df495-10f3-466e-95ab-26f0f6839a2f"
+ },
+ {
+ "Name": "win-hg-120",
+ "RAM": 120000,
+ "Ephemeral": 0,
+ "VCPUs": 32,
+ "Is Public": true,
+ "Disk": 1600,
+ "ID": "b798e44e-bf71-488c-9335-f20bf5976547"
+ },
+ {
+ "Name": "eg-7-ssd",
+ "RAM": 7000,
+ "Ephemeral": 0,
+ "VCPUs": 2,
+ "Is Public": true,
+ "Disk": 100,
+ "ID": "b94e6623-913d-4147-b2a3-34ccf6fe7a5e"
+ },
+ {
+ "Name": "eg-15-flex",
+ "RAM": 15000,
+ "Ephemeral": 0,
+ "VCPUs": 4,
+ "Is Public": true,
+ "Disk": 50,
+ "ID": "bb5fdda8-34ec-40c8-a4e3-308b9e2c9ee2"
+ },
+ {
+ "Name": "win-eg-7-flex",
+ "RAM": 7000,
+ "Ephemeral": 0,
+ "VCPUs": 2,
+ "Is Public": true,
+ "Disk": 50,
+ "ID": "c65384f6-4665-461a-a292-2f3f5a016244"
+ },
+ {
+ "Name": "eg-60-ssd",
+ "RAM": 60000,
+ "Ephemeral": 0,
+ "VCPUs": 16,
+ "Is Public": true,
+ "Disk": 800,
+ "ID": "c678f1a8-6542-4f9d-89af-ffc98715d674"
+ },
+ {
+ "Name": "hg-30-ssd-flex",
+ "RAM": 30000,
+ "Ephemeral": 0,
+ "VCPUs": 8,
+ "Is Public": true,
+ "Disk": 50,
+ "ID": "d147a094-b653-41e7-9250-8d4da3044334"
+ },
+ {
+ "Name": "sp-30",
+ "RAM": 30000,
+ "Ephemeral": 0,
+ "VCPUs": 2,
+ "Is Public": true,
+ "Disk": 200,
+ "ID": "d1acf88d-6f55-4c5c-a914-4ecbdbd50d6b"
+ },
+ {
+ "Name": "sp-120-ssd-flex",
+ "RAM": 120000,
+ "Ephemeral": 0,
+ "VCPUs": 8,
+ "Is Public": true,
+ "Disk": 50,
+ "ID": "d2d33e8e-58b1-4661-8141-826c47f82166"
+ },
+ {
+ "Name": "hg-120-ssd-flex",
+ "RAM": 120000,
+ "Ephemeral": 0,
+ "VCPUs": 32,
+ "Is Public": true,
+ "Disk": 50,
+ "ID": "d7322c37-9881-4a57-9b40-2499fe2e8f42"
+ },
+ {
+ "Name": "win-hg-15-flex",
+ "RAM": 15000,
+ "Ephemeral": 0,
+ "VCPUs": 4,
+ "Is Public": true,
+ "Disk": 50,
+ "ID": "daf597ea-fbbc-4c71-a35e-5b41d33ccc6c"
+ },
+ {
+ "Name": "win-hg-30-ssd-flex",
+ "RAM": 30000,
+ "Ephemeral": 0,
+ "VCPUs": 8,
+ "Is Public": true,
+ "Disk": 50,
+ "ID": "dcfd834c-3932-47a3-8b4b-cdfeecdfde2c"
+ },
+ {
+ "Name": "win-hg-60",
+ "RAM": 60000,
+ "Ephemeral": 0,
+ "VCPUs": 16,
+ "Is Public": true,
+ "Disk": 1600,
+ "ID": "def75cbd-a4b1-4f82-9152-90c65df9587b"
+ },
+ {
+ "Name": "eg-30-ssd-flex",
+ "RAM": 30000,
+ "Ephemeral": 0,
+ "VCPUs": 8,
+ "Is Public": true,
+ "Disk": 50,
+ "ID": "e04c7ad6-a5de-45f5-93c9-f3343bdfe8d1"
+ },
+ {
+ "Name": "vps-ssd-3",
+ "RAM": 8000,
+ "Ephemeral": 0,
+ "VCPUs": 2,
+ "Is Public": true,
+ "Disk": 40,
+ "ID": "e43d7458-6b82-4a78-a712-3a4dc6748cf4"
+ },
+ {
+ "Name": "win-eg-15-flex",
+ "RAM": 15000,
+ "Ephemeral": 0,
+ "VCPUs": 4,
+ "Is Public": true,
+ "Disk": 50,
+ "ID": "e8bd3402-7310-4a0f-8b99-d9212359c957"
+ },
+ {
+ "Name": "win-eg-30",
+ "RAM": 30000,
+ "Ephemeral": 0,
+ "VCPUs": 8,
+ "Is Public": true,
+ "Disk": 800,
+ "ID": "ebf7a997-e2f8-42f4-84f7-33a3d53d1af9"
+ },
+ {
+ "Name": "eg-120-flex",
+ "RAM": 120000,
+ "Ephemeral": 0,
+ "VCPUs": 32,
+ "Is Public": true,
+ "Disk": 50,
+ "ID": "ec852ed3-1e42-4c59-abc3-12bcd26abec8"
+ },
+ {
+ "Name": "sp-240",
+ "RAM": 240000,
+ "Ephemeral": 0,
+ "VCPUs": 16,
+ "Is Public": true,
+ "Disk": 1600,
+ "ID": "ed286e2c-769f-4c47-ac52-b8de7a4891f6"
+ },
+ {
+ "Name": "win-sp-60-ssd",
+ "RAM": 60000,
+ "Ephemeral": 0,
+ "VCPUs": 4,
+ "Is Public": true,
+ "Disk": 200,
+ "ID": "ed835a73-d9a0-43ee-bd89-999c51d8426d"
+ },
+ {
+ "Name": "win-eg-15",
+ "RAM": 15000,
+ "Ephemeral": 0,
+ "VCPUs": 4,
+ "Is Public": true,
+ "Disk": 400,
+ "ID": "f06056c1-a2d4-40e7-a7d8-e5bfabada72e"
+ },
+ {
+ "Name": "win-sp-120",
+ "RAM": 120000,
+ "Ephemeral": 0,
+ "VCPUs": 8,
+ "Is Public": true,
+ "Disk": 800,
+ "ID": "f247dc56-395b-49de-9a62-93ccc4fff4ed"
+ },
+ {
+ "Name": "eg-7-flex",
+ "RAM": 7000,
+ "Ephemeral": 0,
+ "VCPUs": 2,
+ "Is Public": true,
+ "Disk": 50,
+ "ID": "f476f959-ffa6-46f2-94d8-72293570604d"
+ },
+ {
+ "Name": "sp-60-flex",
+ "RAM": 60000,
+ "Ephemeral": 0,
+ "VCPUs": 4,
+ "Is Public": true,
+ "Disk": 50,
+ "ID": "f52db47a-315f-49d4-bc5c-67dd118e7ac0"
+ },
+ {
+ "Name": "win-hg-120-flex",
+ "RAM": 120000,
+ "Ephemeral": 0,
+ "VCPUs": 32,
+ "Is Public": true,
+ "Disk": 50,
+ "ID": "f6cb8144-5d98-4057-b44f-46da342fb571"
+ },
+ {
+ "Name": "hg-7",
+ "RAM": 7000,
+ "Ephemeral": 0,
+ "VCPUs": 2,
+ "Is Public": true,
+ "Disk": 200,
+ "ID": "fa3cc551-0358-4170-be64-56ea432b064c"
+ },
+ {
+ "Name": "hg-15-ssd",
+ "RAM": 15000,
+ "Ephemeral": 0,
+ "VCPUs": 4,
+ "Is Public": true,
+ "Disk": 200,
+ "ID": "ff48c2cf-c17f-4682-aaf6-31d66786f808"
+ }
+ ]"""
+
+ @classmethod
+ def setup_class(self):
+ if 'OS_AUTH_URL' not in os.environ:
+ pytest.skip('no OS_AUTH_URL environment variable')
+
+ @patch('teuthology.misc.sh')
+ def test_sorted_flavors(self, m_sh):
+ o = OpenStack()
+ select = '^(vps|hg)-.*ssd'
+ m_sh.return_value = TestOpenStack.flavors
+ flavors = o.get_sorted_flavors('arch', select)
+ assert [u'vps-ssd-1',
+ u'vps-ssd-2',
+ u'hg-7-ssd-flex',
+ u'hg-7-ssd',
+ u'vps-ssd-3',
+ u'hg-15-ssd-flex',
+ u'hg-15-ssd',
+ u'hg-30-ssd-flex',
+ u'hg-30-ssd',
+ u'hg-60-ssd-flex',
+ u'hg-60-ssd',
+ u'hg-120-ssd-flex',
+ u'hg-120-ssd',
+ ] == [ f['Name'] for f in flavors ]
+ m_sh.assert_called_with("openstack --quiet flavor list -f json")
+
+ def test_flavor(self):
+ def get_sorted_flavors(self, arch, select):
+ return [
+ {
+ 'Name': 'too_small',
+ 'RAM': 2048,
+ 'Disk': 50,
+ 'VCPUs': 1,
+ },
+ ]
+ with patch.multiple(
+ OpenStack,
+ get_sorted_flavors=get_sorted_flavors,
+ ):
+ with pytest.raises(NoFlavorException):
+ hint = { 'ram': 1000, 'disk': 40, 'cpus': 2 }
+ OpenStack().flavor(hint, 'arch')
+
+ flavor = 'good-flavor'
+ def get_sorted_flavors(self, arch, select):
+ return [
+ {
+ 'Name': flavor,
+ 'RAM': 2048,
+ 'Disk': 50,
+ 'VCPUs': 2,
+ },
+ ]
+ with patch.multiple(
+ OpenStack,
+ get_sorted_flavors=get_sorted_flavors,
+ ):
+ hint = { 'ram': 1000, 'disk': 40, 'cpus': 2 }
+ assert flavor == OpenStack().flavor(hint, 'arch')
+
+ def test_flavor_range(self):
+ flavors = [
+ {
+ 'Name': 'too_small',
+ 'RAM': 2048,
+ 'Disk': 50,
+ 'VCPUs': 1,
+ },
+ ]
+ def get_sorted_flavors(self, arch, select):
+ return flavors
+
+ min = { 'ram': 1000, 'disk': 40, 'cpus': 2 }
+ good = { 'ram': 4000, 'disk': 40, 'cpus': 2 }
+
+ #
+ # there are no flavors in the required range
+ #
+ with patch.multiple(
+ OpenStack,
+ get_sorted_flavors=get_sorted_flavors,
+ ):
+ with pytest.raises(NoFlavorException):
+ OpenStack().flavor_range(min, good, 'arch')
+
+ #
+ # there is one flavor in the required range
+ #
+ flavors.append({
+ 'Name': 'min',
+ 'RAM': 2048,
+ 'Disk': 40,
+ 'VCPUs': 2,
+ })
+
+ with patch.multiple(
+ OpenStack,
+ get_sorted_flavors=get_sorted_flavors,
+ ):
+
+ assert 'min' == OpenStack().flavor_range(min, good, 'arch')
+
+ #
+ # out of the two flavors in the required range, get the bigger one
+ #
+ flavors.append({
+ 'Name': 'good',
+ 'RAM': 3000,
+ 'Disk': 40,
+ 'VCPUs': 2,
+ })
+
+ with patch.multiple(
+ OpenStack,
+ get_sorted_flavors=get_sorted_flavors,
+ ):
+
+ assert 'good' == OpenStack().flavor_range(min, good, 'arch')
+
+ #
+ # there is one flavor bigger or equal to good, get this one
+ #
+ flavors.append({
+ 'Name': 'best',
+ 'RAM': 4000,
+ 'Disk': 40,
+ 'VCPUs': 2,
+ })
+
+ with patch.multiple(
+ OpenStack,
+ get_sorted_flavors=get_sorted_flavors,
+ ):
+
+ assert 'best' == OpenStack().flavor_range(min, good, 'arch')
+
+ #
+ # there are two flavors bigger or equal to good, get the smallest one
+ #
+ flavors.append({
+ 'Name': 'too_big',
+ 'RAM': 30000,
+ 'Disk': 400,
+ 'VCPUs': 20,
+ })
+
+ with patch.multiple(
+ OpenStack,
+ get_sorted_flavors=get_sorted_flavors,
+ ):
+
+ assert 'best' == OpenStack().flavor_range(min, good, 'arch')
+
+
+ def test_interpret_hints(self):
+ defaults = {
+ 'machine': {
+ 'ram': 0,
+ 'disk': 0,
+ 'cpus': 0,
+ },
+ 'volumes': {
+ 'count': 0,
+ 'size': 0,
+ },
+ }
+ expected_disk = 10 # first hint larger than the second
+ expected_ram = 20 # second hint larger than the first
+ expected_cpus = 0 # not set, hence zero by default
+ expected_count = 30 # second hint larger than the first
+ expected_size = 40 # does not exist in the first hint
+ hints = [
+ {
+ 'machine': {
+ 'ram': 2,
+ 'disk': expected_disk,
+ },
+ 'volumes': {
+ 'count': 9,
+ 'size': expected_size,
+ },
+ },
+ {
+ 'machine': {
+ 'ram': expected_ram,
+ 'disk': 3,
+ },
+ 'volumes': {
+ 'count': expected_count,
+ },
+ },
+ ]
+ hint = OpenStack().interpret_hints(defaults, hints)
+ assert hint == {
+ 'machine': {
+ 'ram': expected_ram,
+ 'disk': expected_disk,
+ 'cpus': expected_cpus,
+ },
+ 'volumes': {
+ 'count': expected_count,
+ 'size': expected_size,
+ }
+ }
+ assert defaults == OpenStack().interpret_hints(defaults, None)
+
+ def test_get_provider(self):
+ auth = os.environ.get('OS_AUTH_URL', None)
+ os.environ['OS_AUTH_URL'] = 'cloud.ovh.net'
+ assert OpenStack().get_provider() == 'ovh'
+ if auth != None:
+ os.environ['OS_AUTH_URL'] = auth
+ else:
+ del os.environ['OS_AUTH_URL']
+
+ def test_get_os_url(self):
+ o = OpenStack()
+ #
+ # Only for OVH
+ #
+ o.provider = 'something'
+ assert "" == o.get_os_url("server ")
+ o.provider = 'ovh'
+ assert "" == o.get_os_url("unknown ")
+ type2cmd = {
+ 'compute': ('server', 'flavor'),
+ 'network': ('ip', 'security', 'network'),
+ 'image': ('image',),
+ 'volume': ('volume',),
+ }
+ os.environ['OS_REGION_NAME'] = 'REGION'
+ os.environ['OS_TENANT_ID'] = 'TENANT'
+ for (type, cmds) in type2cmd.items():
+ for cmd in cmds:
+ assert ("//" + type) in o.get_os_url(cmd + " ")
+ for type in type2cmd.keys():
+ assert ("//" + type) in o.get_os_url("whatever ", type=type)
+
+ @patch('teuthology.misc.sh')
+ def test_cache_token(self, m_sh):
+ token = 'TOKEN VALUE'
+ m_sh.return_value = token
+ OpenStack.token = None
+ o = OpenStack()
+ #
+ # Only for OVH
+ #
+ o.provider = 'something'
+ assert False == o.cache_token()
+ o.provider = 'ovh'
+ #
+ # Set the environment with the token
+ #
+ assert 'OS_TOKEN_VALUE' not in os.environ
+ assert 'OS_TOKEN_EXPIRES' not in os.environ
+ assert True == o.cache_token()
+ m_sh.assert_called_with('openstack -q token issue -c id -f value')
+ assert token == os.environ['OS_TOKEN_VALUE']
+ assert token == OpenStack.token
+ assert time.time() < int(os.environ['OS_TOKEN_EXPIRES'])
+ assert time.time() < OpenStack.token_expires
+ #
+ # Reset after it expires
+ #
+ token_expires = int(time.time()) - 2000
+ OpenStack.token_expires = token_expires
+ assert True == o.cache_token()
+ assert time.time() < int(os.environ['OS_TOKEN_EXPIRES'])
+ assert time.time() < OpenStack.token_expires
+
+ @patch('teuthology.misc.sh')
+ def test_cache_token_from_environment(self, m_sh):
+ OpenStack.token = None
+ o = OpenStack()
+ o.provider = 'ovh'
+ token = 'TOKEN VALUE'
+ os.environ['OS_TOKEN_VALUE'] = token
+ token_expires = int(time.time()) + OpenStack.token_cache_duration
+ os.environ['OS_TOKEN_EXPIRES'] = str(token_expires)
+ assert True == o.cache_token()
+ assert token == OpenStack.token
+ assert token_expires == OpenStack.token_expires
+ m_sh.assert_not_called()
+
+ @patch('teuthology.misc.sh')
+ def test_cache_token_expired_environment(self, m_sh):
+ token = 'TOKEN VALUE'
+ m_sh.return_value = token
+ OpenStack.token = None
+ o = OpenStack()
+ o.provider = 'ovh'
+ os.environ['OS_TOKEN_VALUE'] = token
+ token_expires = int(time.time()) - 2000
+ os.environ['OS_TOKEN_EXPIRES'] = str(token_expires)
+ assert True == o.cache_token()
+ m_sh.assert_called_with('openstack -q token issue -c id -f value')
+ assert token == os.environ['OS_TOKEN_VALUE']
+ assert token == OpenStack.token
+ assert time.time() < int(os.environ['OS_TOKEN_EXPIRES'])
+ assert time.time() < OpenStack.token_expires
+
+class TestTeuthologyOpenStack(TestOpenStackBase):
+
+ @classmethod
+ def setup_class(self):
+ if 'OS_AUTH_URL' not in os.environ:
+ pytest.skip('no OS_AUTH_URL environment variable')
+
+ teuthology.log.setLevel(logging.DEBUG)
+ set_config_attr(argparse.Namespace())
+
+ ip = TeuthologyOpenStack.create_floating_ip()
+ if ip:
+ ip_id = TeuthologyOpenStack.get_floating_ip_id(ip)
+ OpenStack().run("ip floating delete " + ip_id)
+ self.can_create_floating_ips = True
+ else:
+ self.can_create_floating_ips = False
+
+ def setup_method(self):
+ super(TestTeuthologyOpenStack, self).setup_method()
+ self.key_filename = tempfile.mktemp()
+ self.key_name = 'teuthology-test'
+ self.name = 'teuthology-test'
+ self.clobber()
+ misc.sh("""
+openstack keypair create {key_name} > {key_filename}
+chmod 600 {key_filename}
+ """.format(key_filename=self.key_filename,
+ key_name=self.key_name))
+ self.options = ['--key-name', self.key_name,
+ '--key-filename', self.key_filename,
+ '--name', self.name,
+ '--verbose']
+
+ def teardown_method(self):
+ super(TestTeuthologyOpenStack, self).teardown_method()
+ self.clobber()
+ os.unlink(self.key_filename)
+
+ def clobber(self):
+ misc.sh("""
+openstack server delete {name} --wait || true
+openstack keypair delete {key_name} || true
+ """.format(key_name=self.key_name,
+ name=self.name))
+
+ def test_create(self, caplog):
+ teuthology_argv = [
+ '--suite', 'upgrade/hammer',
+ '--dry-run',
+ '--ceph', 'main',
+ '--kernel', 'distro',
+ '--flavor', 'gcov',
+ '--distro', 'ubuntu',
+ '--suite-branch', 'hammer',
+ '--email', 'loic@dachary.org',
+ '--num', '10',
+ '--limit', '23',
+ '--subset', '1/2',
+ '--priority', '101',
+ '--timeout', '234',
+ '--filter', 'trasher',
+ '--filter-out', 'erasure-code',
+ '--throttle', '3',
+ ]
+ archive_upload = 'user@archive:/tmp'
+ argv = (self.options +
+ ['--teuthology-git-url', 'TEUTHOLOGY_URL',
+ '--teuthology-branch', 'TEUTHOLOGY_BRANCH',
+ '--ceph-workbench-git-url', 'CEPH_WORKBENCH_URL',
+ '--ceph-workbench-branch', 'CEPH_WORKBENCH_BRANCH',
+ '--upload',
+ '--archive-upload', archive_upload] +
+ teuthology_argv)
+ args = scripts.openstack.parse_args(argv)
+ teuthology_argv.extend([
+ '--archive-upload', archive_upload,
+ '--archive-upload-url', args.archive_upload_url,
+ ])
+ teuthology = TeuthologyOpenStack(args, None, argv)
+ teuthology.user_data = 'teuthology/openstack/test/user-data-test1.txt'
+ teuthology.teuthology_suite = 'echo --'
+
+ teuthology.main()
+ assert 0 == teuthology.ssh("lsb_release -a")
+ assert 0 == teuthology.ssh("grep 'substituded variables' /var/log/cloud-init.log")
+ l = caplog.text
+ assert 'Ubuntu 14.04' in l
+ assert "nworkers=" + str(args.simultaneous_jobs) in l
+ assert "username=" + teuthology.username in l
+ assert "upload=--archive-upload user@archive:/tmp" in l
+ assert ("ceph_workbench="
+ " --ceph-workbench-branch CEPH_WORKBENCH_BRANCH"
+ " --ceph-workbench-git-url CEPH_WORKBENCH_URL") in l
+ assert "clone=git clone -b TEUTHOLOGY_BRANCH TEUTHOLOGY_URL" in l
+ assert os.environ['OS_AUTH_URL'] in l
+ assert " ".join(teuthology_argv) in l
+
+ if self.can_create_floating_ips:
+ ip = teuthology.get_floating_ip(self.name)
+ teuthology.teardown()
+ if self.can_create_floating_ips:
+ assert teuthology.get_floating_ip_id(ip) == None
+
+ def test_floating_ip(self):
+ if not self.can_create_floating_ips:
+ pytest.skip('unable to create floating ips')
+
+ expected = TeuthologyOpenStack.create_floating_ip()
+ ip = TeuthologyOpenStack.get_unassociated_floating_ip()
+ assert expected == ip
+ ip_id = TeuthologyOpenStack.get_floating_ip_id(ip)
+ OpenStack().run("ip floating delete " + ip_id)
--- /dev/null
+#cloud-config
+system_info:
+ default_user:
+ name: ubuntu
+final_message: "teuthology is up and running after $UPTIME seconds, substituded variables nworkers=NWORKERS openrc=OPENRC username=TEUTHOLOGY_USERNAME upload=UPLOAD ceph_workbench=CEPH_WORKBENCH clone=CLONE_OPENSTACK"
--- /dev/null
+ceph 658 1 0 Jun08 ? 00:07:43 /usr/bin/ceph-mgr -f --cluster ceph --id host1 --setuser ceph --setgroup ceph
+ceph 1634 1 0 Jun08 ? 00:02:17 /usr/bin/ceph-mds -f --cluster ceph --id host1 --setuser ceph --setgroup ceph
+ceph 31555 1 0 Jun08 ? 01:13:50 /usr/bin/ceph-mon -f --cluster ceph --id host1 --setuser ceph --setgroup ceph
+ceph 31765 1 0 Jun08 ? 00:48:42 /usr/bin/radosgw -f --cluster ceph --name client.rgw.host1.rgw0 --setuser ceph --setgroup ceph
+ceph 97427 1 0 Jun17 ? 00:41:39 /usr/bin/ceph-osd -f --cluster ceph --id 0 --setuser ceph --setgroup ceph
\ No newline at end of file
--- /dev/null
+from teuthology.orchestra import monkey
+monkey.patch_all()
+
+from io import StringIO
+
+import os
+from teuthology.orchestra import connection, remote, run
+from teuthology.orchestra.test.util import assert_raises
+from teuthology.exceptions import CommandCrashedError, ConnectionLostError
+
+from pytest import skip
+
+HOST = None
+
+
+class TestIntegration():
+ def setup_method(self):
+ try:
+ host = os.environ['ORCHESTRA_TEST_HOST']
+ except KeyError:
+ skip('To run integration tests, set environment ' +
+ 'variable ORCHESTRA_TEST_HOST to user@host to use.')
+ global HOST
+ HOST = host
+
+ def test_crash(self):
+ ssh = connection.connect(HOST)
+ e = assert_raises(
+ CommandCrashedError,
+ run.run,
+ client=ssh,
+ args=['sh', '-c', 'kill -ABRT $$'],
+ )
+ assert e.command == "sh -c 'kill -ABRT $$'"
+ assert str(e) == "Command crashed: \"sh -c 'kill -ABRT $$'\""
+
+ def test_lost(self):
+ ssh = connection.connect(HOST)
+ e = assert_raises(
+ ConnectionLostError,
+ run.run,
+ client=ssh,
+ args=['sh', '-c', 'kill -ABRT $PPID'],
+ name=HOST,
+ )
+ assert e.command == "sh -c 'kill -ABRT $PPID'"
+ assert str(e) == \
+ "SSH connection to {host} was lost: ".format(host=HOST) + \
+ "\"sh -c 'kill -ABRT $PPID'\""
+
+ def test_pipe(self):
+ ssh = connection.connect(HOST)
+ r = run.run(
+ client=ssh,
+ args=['cat'],
+ stdin=run.PIPE,
+ stdout=StringIO(),
+ wait=False,
+ )
+ assert r.stdout.getvalue() == ''
+ r.stdin.write('foo\n')
+ r.stdin.write('bar\n')
+ r.stdin.close()
+
+ r.wait()
+ got = r.exitstatus
+ assert got == 0
+ assert r.stdout.getvalue() == 'foo\nbar\n'
+
+ def test_and(self):
+ ssh = connection.connect(HOST)
+ r = run.run(
+ client=ssh,
+ args=['true', run.Raw('&&'), 'echo', 'yup'],
+ stdout=StringIO(),
+ )
+ assert r.stdout.getvalue() == 'yup\n'
+
+ def test_os(self):
+ rem = remote.Remote(HOST)
+ assert rem.os.name
+ assert rem.os.version
+
+ def test_17102(self, caplog):
+ # http://tracker.ceph.com/issues/17102
+ rem = remote.Remote(HOST)
+ interval = 3
+ rem.run(args="echo before; sleep %s; echo after" % interval)
+ for record in caplog.records:
+ if record.msg == 'before':
+ before_time = record.created
+ elif record.msg == 'after':
+ after_time = record.created
+ assert int(round(after_time - before_time)) == interval
--- /dev/null
+import pytest
+
+from mock import patch, Mock
+
+from teuthology.orchestra import cluster, remote, run
+
+
+class TestCluster(object):
+ def test_init_empty(self):
+ c = cluster.Cluster()
+ assert c.remotes == {}
+
+ def test_init(self):
+ r1 = Mock()
+ r2 = Mock()
+ c = cluster.Cluster(
+ remotes=[
+ (r1, ['foo', 'bar']),
+ (r2, ['baz']),
+ ],
+ )
+ r3 = Mock()
+ c.add(r3, ['xyzzy', 'thud', 'foo'])
+ assert c.remotes == {
+ r1: ['foo', 'bar'],
+ r2: ['baz'],
+ r3: ['xyzzy', 'thud', 'foo'],
+ }
+
+ def test_repr(self):
+ r1 = remote.Remote('r1', ssh=Mock())
+ r2 = remote.Remote('r2', ssh=Mock())
+ c = cluster.Cluster(
+ remotes=[
+ (r1, ['foo', 'bar']),
+ (r2, ['baz']),
+ ],
+ )
+ assert repr(c) == \
+ "Cluster(remotes=[[Remote(name='r1'), ['foo', 'bar']], " \
+ "[Remote(name='r2'), ['baz']]])"
+
+ def test_str(self):
+ r1 = remote.Remote('r1', ssh=Mock())
+ r2 = remote.Remote('r2', ssh=Mock())
+ c = cluster.Cluster(
+ remotes=[
+ (r1, ['foo', 'bar']),
+ (r2, ['baz']),
+ ],
+ )
+ assert str(c) == "r1[foo,bar] r2[baz]"
+
+ def test_run_all(self):
+ r1 = Mock(spec=remote.Remote)
+ r1.configure_mock(name='r1')
+ ret1 = Mock(spec=run.RemoteProcess)
+ r1.run.return_value = ret1
+ r2 = Mock(spec=remote.Remote)
+ r2.configure_mock(name='r2')
+ ret2 = Mock(spec=run.RemoteProcess)
+ r2.run.return_value = ret2
+ c = cluster.Cluster(
+ remotes=[
+ (r1, ['foo', 'bar']),
+ (r2, ['baz']),
+ ],
+ )
+ got = c.run(args=['test'])
+ r1.run.assert_called_once_with(args=['test'], wait=True)
+ r2.run.assert_called_once_with(args=['test'], wait=True)
+ assert len(got) == 2
+ assert got, [ret1 == ret2]
+ # check identity not equality
+ assert got[0] is ret1
+ assert got[1] is ret2
+
+ def test_only_one(self):
+ r1 = Mock()
+ r2 = Mock()
+ r3 = Mock()
+ c = cluster.Cluster(
+ remotes=[
+ (r1, ['foo', 'bar']),
+ (r2, ['bar']),
+ (r3, ['foo']),
+ ],
+ )
+ c_foo = c.only('foo')
+ assert c_foo.remotes == {r1: ['foo', 'bar'], r3: ['foo']}
+
+ def test_only_two(self):
+ r1 = Mock()
+ r2 = Mock()
+ r3 = Mock()
+ c = cluster.Cluster(
+ remotes=[
+ (r1, ['foo', 'bar']),
+ (r2, ['bar']),
+ (r3, ['foo']),
+ ],
+ )
+ c_both = c.only('foo', 'bar')
+ assert c_both.remotes, {r1: ['foo' == 'bar']}
+
+ def test_only_none(self):
+ r1 = Mock()
+ r2 = Mock()
+ r3 = Mock()
+ c = cluster.Cluster(
+ remotes=[
+ (r1, ['foo', 'bar']),
+ (r2, ['bar']),
+ (r3, ['foo']),
+ ],
+ )
+ c_none = c.only('impossible')
+ assert c_none.remotes == {}
+
+ def test_only_match(self):
+ r1 = Mock()
+ r2 = Mock()
+ r3 = Mock()
+ c = cluster.Cluster(
+ remotes=[
+ (r1, ['foo', 'bar']),
+ (r2, ['bar']),
+ (r3, ['foo']),
+ ],
+ )
+ c_foo = c.only('foo', lambda role: role.startswith('b'))
+ assert c_foo.remotes, {r1: ['foo' == 'bar']}
+
+ def test_exclude_one(self):
+ r1 = Mock()
+ r2 = Mock()
+ r3 = Mock()
+ c = cluster.Cluster(
+ remotes=[
+ (r1, ['foo', 'bar']),
+ (r2, ['bar']),
+ (r3, ['foo']),
+ ],
+ )
+ c_foo = c.exclude('foo')
+ assert c_foo.remotes == {r2: ['bar']}
+
+ def test_exclude_two(self):
+ r1 = Mock()
+ r2 = Mock()
+ r3 = Mock()
+ c = cluster.Cluster(
+ remotes=[
+ (r1, ['foo', 'bar']),
+ (r2, ['bar']),
+ (r3, ['foo']),
+ ],
+ )
+ c_both = c.exclude('foo', 'bar')
+ assert c_both.remotes == {r2: ['bar'], r3: ['foo']}
+
+ def test_exclude_none(self):
+ r1 = Mock()
+ r2 = Mock()
+ r3 = Mock()
+ c = cluster.Cluster(
+ remotes=[
+ (r1, ['foo', 'bar']),
+ (r2, ['bar']),
+ (r3, ['foo']),
+ ],
+ )
+ c_none = c.exclude('impossible')
+ assert c_none.remotes == {r1: ['foo', 'bar'], r2: ['bar'], r3: ['foo']}
+
+ def test_exclude_match(self):
+ r1 = Mock()
+ r2 = Mock()
+ r3 = Mock()
+ c = cluster.Cluster(
+ remotes=[
+ (r1, ['foo', 'bar']),
+ (r2, ['bar']),
+ (r3, ['foo']),
+ ],
+ )
+ c_foo = c.exclude('foo', lambda role: role.startswith('b'))
+ assert c_foo.remotes == {r2: ['bar'], r3: ['foo']}
+
+ def test_filter(self):
+ r1 = Mock(_name='r1')
+ r2 = Mock(_name='r2')
+ def func(r):
+ return r._name == "r1"
+ c = cluster.Cluster(remotes=[
+ (r1, ['foo']),
+ (r2, ['bar']),
+ ])
+ assert c.filter(func).remotes == {
+ r1: ['foo']
+ }
+
+
+class TestWriteFile(object):
+ """ Tests for cluster.write_file """
+ def setup_method(self):
+ self.r1 = remote.Remote('r1', ssh=Mock())
+ self.c = cluster.Cluster(
+ remotes=[
+ (self.r1, ['foo', 'bar']),
+ ],
+ )
+
+ @patch("teuthology.orchestra.remote.RemoteShell.write_file")
+ def test_write_file(self, m_write_file):
+ self.c.write_file("filename", "content")
+ m_write_file.assert_called_with("filename", "content")
+
+ @patch("teuthology.orchestra.remote.RemoteShell.write_file")
+ def test_fails_with_invalid_perms(self, m_write_file):
+ with pytest.raises(ValueError):
+ self.c.write_file("filename", "content", sudo=False, perms="invalid")
+
+ @patch("teuthology.orchestra.remote.RemoteShell.write_file")
+ def test_fails_with_invalid_owner(self, m_write_file):
+ with pytest.raises(ValueError):
+ self.c.write_file("filename", "content", sudo=False, owner="invalid")
+
+ @patch("teuthology.orchestra.remote.RemoteShell.write_file")
+ def test_with_sudo(self, m_write_file):
+ self.c.write_file("filename", "content", sudo=True)
+ m_write_file.assert_called_with("filename", "content", sudo=True, owner=None, mode=None)
--- /dev/null
+from mock import patch, Mock
+
+from teuthology import config
+from teuthology.orchestra import connection
+from tests.orchestra.util import assert_raises
+
+
+class TestConnection(object):
+ def setup_method(self):
+ self.start_patchers()
+
+ def teardown_method(self):
+ self.stop_patchers()
+
+ def start_patchers(self):
+ self.patcher_sleep = patch(
+ 'time.sleep',
+ )
+ self.patcher_sleep.start()
+ self.m_ssh = Mock()
+ self.patcher_ssh = patch(
+ 'teuthology.orchestra.connection.paramiko.SSHClient',
+ self.m_ssh,
+ )
+ self.patcher_ssh.start()
+
+ def stop_patchers(self):
+ self.patcher_ssh.stop()
+ self.patcher_sleep.stop()
+
+ def clear_config(self):
+ config.config.teuthology_yaml = ''
+ config.config.load()
+ config.config.ssh_key = None
+
+ def test_split_user_just_host(self):
+ got = connection.split_user('somehost.example.com')
+ assert got == (None, 'somehost.example.com')
+
+ def test_split_user_both(self):
+ got = connection.split_user('jdoe@somehost.example.com')
+ assert got == ('jdoe', 'somehost.example.com')
+
+ def test_split_user_empty_user(self):
+ s = '@somehost.example.com'
+ e = assert_raises(AssertionError, connection.split_user, s)
+ assert str(e) == 'Bad input to split_user: {s!r}'.format(s=s)
+
+ def test_connect(self):
+ self.clear_config()
+ config.config.verify_host_keys = True
+ m_ssh_instance = self.m_ssh.return_value = Mock();
+ m_transport = Mock()
+ m_ssh_instance.get_transport.return_value = m_transport
+ got = connection.connect(
+ 'jdoe@orchestra.test.newdream.net.invalid',
+ _SSHClient=self.m_ssh,
+ )
+ self.m_ssh.assert_called_once()
+ m_ssh_instance.set_missing_host_key_policy.assert_called_once()
+ m_ssh_instance.load_system_host_keys.assert_called_once_with()
+ m_ssh_instance.connect.assert_called_once_with(
+ hostname='orchestra.test.newdream.net.invalid',
+ username='jdoe',
+ timeout=60,
+ )
+ m_transport.set_keepalive.assert_called_once_with(False)
+ assert got is m_ssh_instance
+
+ def test_connect_no_verify_host_keys(self):
+ self.clear_config()
+ config.config.verify_host_keys = False
+ m_ssh_instance = self.m_ssh.return_value = Mock();
+ m_transport = Mock()
+ m_ssh_instance.get_transport.return_value = m_transport
+ got = connection.connect(
+ 'jdoe@orchestra.test.newdream.net.invalid',
+ _SSHClient=self.m_ssh,
+ )
+ self.m_ssh.assert_called_once()
+ m_ssh_instance.set_missing_host_key_policy.assert_called_once()
+ assert not m_ssh_instance.load_system_host_keys.called
+ m_ssh_instance.connect.assert_called_once_with(
+ hostname='orchestra.test.newdream.net.invalid',
+ username='jdoe',
+ timeout=60,
+ )
+ m_transport.set_keepalive.assert_called_once_with(False)
+ assert got is m_ssh_instance
+
+ def test_connect_override_hostkeys(self):
+ self.clear_config()
+ m_ssh_instance = self.m_ssh.return_value = Mock();
+ m_transport = Mock()
+ m_ssh_instance.get_transport.return_value = m_transport
+ m_host_keys = Mock()
+ m_ssh_instance.get_host_keys.return_value = m_host_keys
+ m_create_key = Mock()
+ m_create_key.return_value = "frobnitz"
+ got = connection.connect(
+ 'jdoe@orchestra.test.newdream.net.invalid',
+ host_key='ssh-rsa testkey',
+ _SSHClient=self.m_ssh,
+ _create_key=m_create_key,
+ )
+ self.m_ssh.assert_called_once()
+ m_ssh_instance.get_host_keys.assert_called_once()
+ m_host_keys.add.assert_called_once_with(
+ hostname='orchestra.test.newdream.net.invalid',
+ keytype='ssh-rsa',
+ key='frobnitz',
+ )
+ m_create_key.assert_called_once_with('ssh-rsa', 'testkey')
+ m_ssh_instance.connect.assert_called_once_with(
+ hostname='orchestra.test.newdream.net.invalid',
+ username='jdoe',
+ timeout=60,
+ )
+ m_transport.set_keepalive.assert_called_once_with(False)
+ assert got is m_ssh_instance
--- /dev/null
+from mock import patch
+
+from teuthology.config import config as teuth_config
+
+from teuthology.orchestra import console
+
+
+class TestConsole(object):
+ pass
+
+
+class TestPhysicalConsole(TestConsole):
+ klass = console.PhysicalConsole
+ ipmi_cmd_templ = 'ipmitool -H {h}.{d} -I lanplus -U {u} -P {p} {c}'
+ conserver_cmd_templ = 'console -M {m} -p {p} {mode} {h}'
+
+ def setup_method(self):
+ self.hostname = 'host'
+ teuth_config.ipmi_domain = 'ipmi_domain'
+ teuth_config.ipmi_user = 'ipmi_user'
+ teuth_config.ipmi_password = 'ipmi_pass'
+ teuth_config.conserver_master = 'conserver_master'
+ teuth_config.conserver_port = 3109
+ teuth_config.use_conserver = True
+
+ def test_has_ipmi_creds(self):
+ cons = self.klass(self.hostname)
+ assert cons.has_ipmi_credentials is True
+ teuth_config.ipmi_domain = None
+ cons = self.klass(self.hostname)
+ assert cons.has_ipmi_credentials is False
+
+ def test_console_command_conserver(self):
+ cons = self.klass(
+ self.hostname,
+ teuth_config.ipmi_user,
+ teuth_config.ipmi_password,
+ teuth_config.ipmi_domain,
+ )
+ cons.has_conserver = True
+ console_cmd = cons._console_command()
+ assert console_cmd == self.conserver_cmd_templ.format(
+ m=teuth_config.conserver_master,
+ p=teuth_config.conserver_port,
+ mode='-s',
+ h=self.hostname,
+ )
+ console_cmd = cons._console_command(readonly=False)
+ assert console_cmd == self.conserver_cmd_templ.format(
+ m=teuth_config.conserver_master,
+ p=teuth_config.conserver_port,
+ mode='-f',
+ h=self.hostname,
+ )
+
+ def test_console_command_ipmi(self):
+ teuth_config.conserver_master = None
+ cons = self.klass(
+ self.hostname,
+ teuth_config.ipmi_user,
+ teuth_config.ipmi_password,
+ teuth_config.ipmi_domain,
+ )
+ sol_cmd = cons._console_command()
+ assert sol_cmd == self.ipmi_cmd_templ.format(
+ h=self.hostname,
+ d=teuth_config.ipmi_domain,
+ u=teuth_config.ipmi_user,
+ p=teuth_config.ipmi_password,
+ c='sol activate',
+ )
+
+ def test_ipmi_command_ipmi(self):
+ cons = self.klass(
+ self.hostname,
+ teuth_config.ipmi_user,
+ teuth_config.ipmi_password,
+ teuth_config.ipmi_domain,
+ )
+ pc_cmd = cons._ipmi_command('power cycle')
+ assert pc_cmd == self.ipmi_cmd_templ.format(
+ h=self.hostname,
+ d=teuth_config.ipmi_domain,
+ u=teuth_config.ipmi_user,
+ p=teuth_config.ipmi_password,
+ c='power cycle',
+ )
+
+ def test_spawn_log_conserver(self):
+ with patch(
+ 'teuthology.orchestra.console.psutil.subprocess.Popen',
+ autospec=True,
+ ) as m_popen:
+ m_popen.return_value.pid = 42
+ m_popen.return_value.returncode = 0
+ m_popen.return_value.wait.return_value = 0
+ cons = self.klass(self.hostname)
+ assert cons.has_conserver is True
+ m_popen.reset_mock()
+ m_popen.return_value.poll.return_value = None
+ cons.spawn_sol_log('/fake/path')
+ assert m_popen.call_count == 1
+ call_args = m_popen.call_args_list[0][0][0]
+ assert any(
+ [teuth_config.conserver_master in arg for arg in call_args]
+ )
+
+ def test_spawn_log_ipmi(self):
+ with patch(
+ 'teuthology.orchestra.console.psutil.subprocess.Popen',
+ autospec=True,
+ ) as m_popen:
+ m_popen.return_value.pid = 42
+ m_popen.return_value.returncode = 1
+ m_popen.return_value.wait.return_value = 1
+ cons = self.klass(self.hostname)
+ assert cons.has_conserver is False
+ m_popen.reset_mock()
+ m_popen.return_value.poll.return_value = 1
+ cons.spawn_sol_log('/fake/path')
+ assert m_popen.call_count == 1
+ call_args = m_popen.call_args_list[0][0][0]
+ assert any(
+ ['ipmitool' in arg for arg in call_args]
+ )
+
+ def test_spawn_log_fallback(self):
+ with patch(
+ 'teuthology.orchestra.console.psutil.subprocess.Popen',
+ autospec=True,
+ ) as m_popen:
+ m_popen.return_value.pid = 42
+ m_popen.return_value.returncode = 0
+ m_popen.return_value.wait.return_value = 0
+ cons = self.klass(self.hostname)
+ assert cons.has_conserver is True
+ m_popen.reset_mock()
+ m_popen.return_value.poll.return_value = 1
+ cons.spawn_sol_log('/fake/path')
+ assert cons.has_conserver is False
+ assert m_popen.call_count == 2
+ call_args = m_popen.call_args_list[1][0][0]
+ assert any(
+ ['ipmitool' in arg for arg in call_args]
+ )
+
+ def test_get_console_conserver(self):
+ with patch(
+ 'teuthology.orchestra.console.psutil.subprocess.Popen',
+ autospec=True,
+ ) as m_popen:
+ m_popen.return_value.pid = 42
+ m_popen.return_value.returncode = 0
+ m_popen.return_value.wait.return_value = 0
+ cons = self.klass(self.hostname)
+ assert cons.has_conserver is True
+ with patch(
+ 'teuthology.orchestra.console.pexpect.spawn',
+ autospec=True,
+ ) as m_spawn:
+ cons._get_console()
+ assert m_spawn.call_count == 1
+ assert teuth_config.conserver_master in \
+ m_spawn.call_args_list[0][0][0]
+
+ def test_get_console_ipmitool(self):
+ with patch(
+ 'teuthology.orchestra.console.psutil.subprocess.Popen',
+ autospec=True,
+ ) as m_popen:
+ m_popen.return_value.pid = 42
+ m_popen.return_value.returncode = 0
+ m_popen.return_value.wait.return_value = 0
+ cons = self.klass(self.hostname)
+ assert cons.has_conserver is True
+ with patch(
+ 'teuthology.orchestra.console.pexpect.spawn',
+ autospec=True,
+ ) as m_spawn:
+ cons.has_conserver = False
+ cons._get_console()
+ assert m_spawn.call_count == 1
+ assert 'ipmitool' in m_spawn.call_args_list[0][0][0]
+
+ def test_get_console_fallback(self):
+ with patch(
+ 'teuthology.orchestra.console.psutil.subprocess.Popen',
+ autospec=True,
+ ) as m_popen:
+ m_popen.return_value.pid = 42
+ m_popen.return_value.returncode = 0
+ m_popen.return_value.wait.return_value = 0
+ cons = self.klass(self.hostname)
+ assert cons.has_conserver is True
+ with patch(
+ 'teuthology.orchestra.console.pexpect.spawn',
+ autospec=True,
+ ) as m_spawn:
+ cons.has_conserver = True
+ m_spawn.return_value.isalive.return_value = False
+ cons._get_console()
+ assert m_spawn.return_value.isalive.call_count == 1
+ assert m_spawn.call_count == 2
+ assert cons.has_conserver is False
+ assert 'ipmitool' in m_spawn.call_args_list[1][0][0]
+
+ def test_disable_conserver(self):
+ with patch(
+ 'teuthology.orchestra.console.psutil.subprocess.Popen',
+ autospec=True,
+ ) as m_popen:
+ m_popen.return_value.pid = 42
+ m_popen.return_value.returncode = 0
+ m_popen.return_value.wait.return_value = 0
+ teuth_config.use_conserver = False
+ cons = self.klass(self.hostname)
+ assert cons.has_conserver is False
--- /dev/null
+from textwrap import dedent
+from teuthology.orchestra.opsys import OS
+import pytest
+
+
+class TestOS(object):
+ str_centos_9_os_release = dedent("""
+ NAME="CentOS Stream"
+ VERSION="9"
+ ID="centos"
+ ID_LIKE="rhel fedora"
+ VERSION_ID="9"
+ PLATFORM_ID="platform:el9"
+ PRETTY_NAME="CentOS Stream 9"
+ ANSI_COLOR="0;31"
+ LOGO="fedora-logo-icon"
+ CPE_NAME="cpe:/o:centos:centos:9"
+ HOME_URL="https://centos.org/"
+ BUG_REPORT_URL="https://issues.redhat.com/"
+ REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9"
+ REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream"
+ """)
+
+ str_centos_7_os_release = dedent("""
+ NAME="CentOS Linux"
+ VERSION="7 (Core)"
+ ID="centos"
+ ID_LIKE="rhel fedora"
+ VERSION_ID="7"
+ PRETTY_NAME="CentOS Linux 7 (Core)"
+ ANSI_COLOR="0;31"
+ CPE_NAME="cpe:/o:centos:centos:7"
+ HOME_URL="https://www.centos.org/"
+ BUG_REPORT_URL="https://bugs.centos.org/"
+ """)
+
+ str_centos_7_os_release_newer = dedent("""
+ NAME="CentOS Linux"
+ VERSION="7 (Core)"
+ ID="centos"
+ ID_LIKE="rhel fedora"
+ VERSION_ID="7"
+ PRETTY_NAME="CentOS Linux 7 (Core)"
+ ANSI_COLOR="0;31"
+ CPE_NAME="cpe:/o:centos:centos:7"
+ HOME_URL="https://www.centos.org/"
+ BUG_REPORT_URL="https://bugs.centos.org/"
+
+ CENTOS_MANTISBT_PROJECT="CentOS-7"
+ CENTOS_MANTISBT_PROJECT_VERSION="7"
+ REDHAT_SUPPORT_PRODUCT="centos"
+ REDHAT_SUPPORT_PRODUCT_VERSION="7"
+ """)
+
+ str_debian_7_lsb_release = dedent("""
+ Distributor ID: Debian
+ Description: Debian GNU/Linux 7.1 (wheezy)
+ Release: 7.1
+ Codename: wheezy
+ """)
+
+ str_debian_7_os_release = dedent("""
+ PRETTY_NAME="Debian GNU/Linux 7 (wheezy)"
+ NAME="Debian GNU/Linux"
+ VERSION_ID="7"
+ VERSION="7 (wheezy)"
+ ID=debian
+ ANSI_COLOR="1;31"
+ HOME_URL="http://www.debian.org/"
+ SUPPORT_URL="http://www.debian.org/support/"
+ BUG_REPORT_URL="http://bugs.debian.org/"
+ """)
+
+ str_debian_8_os_release = dedent("""
+ PRETTY_NAME="Debian GNU/Linux 8 (jessie)"
+ NAME="Debian GNU/Linux"
+ VERSION_ID="8"
+ VERSION="8 (jessie)"
+ ID=debian
+ HOME_URL="http://www.debian.org/"
+ SUPPORT_URL="http://www.debian.org/support/"
+ BUG_REPORT_URL="https://bugs.debian.org/"
+ """)
+
+ str_debian_9_os_release = dedent("""
+ PRETTY_NAME="Debian GNU/Linux 9 (stretch)"
+ NAME="Debian GNU/Linux"
+ VERSION_ID="9"
+ VERSION="9 (stretch)"
+ ID=debian
+ HOME_URL="https://www.debian.org/"
+ SUPPORT_URL="https://www.debian.org/support"
+ BUG_REPORT_URL="https://bugs.debian.org/"
+ """)
+
+ str_ubuntu_12_04_lsb_release = dedent("""
+ Distributor ID: Ubuntu
+ Description: Ubuntu 12.04.4 LTS
+ Release: 12.04
+ Codename: precise
+ """)
+
+ str_ubuntu_12_04_os_release = dedent("""
+ NAME="Ubuntu"
+ VERSION="12.04.4 LTS, Precise Pangolin"
+ ID=ubuntu
+ ID_LIKE=debian
+ PRETTY_NAME="Ubuntu precise (12.04.4 LTS)"
+ VERSION_ID="12.04"
+ """)
+
+ str_ubuntu_14_04_os_release = dedent("""
+ NAME="Ubuntu"
+ VERSION="14.04.4 LTS, Trusty Tahr"
+ ID=ubuntu
+ ID_LIKE=debian
+ PRETTY_NAME="Ubuntu 14.04.4 LTS"
+ VERSION_ID="14.04"
+ HOME_URL="http://www.ubuntu.com/"
+ SUPPORT_URL="http://help.ubuntu.com/"
+ BUG_REPORT_URL="http://bugs.launchpad.net/ubuntu/"
+ """)
+
+ str_ubuntu_16_04_os_release = dedent("""
+ NAME="Ubuntu"
+ VERSION="16.04 LTS (Xenial Xerus)"
+ ID=ubuntu
+ ID_LIKE=debian
+ PRETTY_NAME="Ubuntu 16.04 LTS"
+ VERSION_ID="16.04"
+ HOME_URL="http://www.ubuntu.com/"
+ SUPPORT_URL="http://help.ubuntu.com/"
+ BUG_REPORT_URL="http://bugs.launchpad.net/ubuntu/"
+ UBUNTU_CODENAME=xenial
+ """)
+
+ str_ubuntu_18_04_os_release = dedent("""
+ NAME="Ubuntu"
+ VERSION="18.04 LTS (Bionic Beaver)"
+ ID=ubuntu
+ ID_LIKE=debian
+ PRETTY_NAME="Ubuntu 18.04 LTS"
+ VERSION_ID="18.04"
+ HOME_URL="https://www.ubuntu.com/"
+ SUPPORT_URL="https://help.ubuntu.com/"
+ BUG_REPORT_URL="https://bugs.launchpad.net/ubuntu/"
+ PRIVACY_POLICY_URL="https://www.ubuntu.com/legal/terms-and-policies/privacy-policy"
+ VERSION_CODENAME=bionic
+ UBUNTU_CODENAME=bionic
+ """)
+
+ str_rhel_6_4_lsb_release = dedent("""
+ LSB Version: :base-4.0-amd64:base-4.0-noarch:core-4.0-amd64:core-4.0-noarch:graphics-4.0-amd64:graphics-4.0-noarch:printing-4.0-amd64:printing-4.0-noarch
+ Distributor ID: RedHatEnterpriseServer
+ Description: Red Hat Enterprise Linux Server release 6.4 (Santiago)
+ Release: 6.4
+ Codename: Santiago
+ """)
+
+ str_rhel_7_lsb_release = dedent("""
+ LSB Version: :core-4.1-amd64:core-4.1-noarch:cxx-4.1-amd64:cxx-4.1-noarch:desktop-4.1-amd64:desktop-4.1-noarch:languages-4.1-amd64:languages-4.1-noarch:printing-4.1-amd64:printing-4.1-noarch
+ Distributor ID: RedHatEnterpriseServer
+ Description: Red Hat Enterprise Linux Server release 7.0 (Maipo)
+ Release: 7.0
+ Codename: Maipo
+ """)
+
+ str_rhel_7_os_release = dedent("""
+ NAME="Red Hat Enterprise Linux Server"
+ VERSION="7.0 (Maipo)"
+ ID="rhel"
+ ID_LIKE="fedora"
+ VERSION_ID="7.0"
+ PRETTY_NAME="Red Hat Enterprise Linux Server 7.0 (Maipo)"
+ ANSI_COLOR="0;31"
+ CPE_NAME="cpe:/o:redhat:enterprise_linux:7.0:GA:server"
+ HOME_URL="https://www.redhat.com/"
+ BUG_REPORT_URL="https://bugzilla.redhat.com/"
+
+ REDHAT_BUGZILLA_PRODUCT="Red Hat Enterprise Linux 7"
+ REDHAT_BUGZILLA_PRODUCT_VERSION=7.0
+ REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux"
+ REDHAT_SUPPORT_PRODUCT_VERSION=7.0
+ """)
+
+ str_fedora_26_os_release = dedent("""
+ NAME=Fedora
+ VERSION="26 (Twenty Six)"
+ ID=fedora
+ VERSION_ID=26
+ PRETTY_NAME="Fedora 26 (Twenty Six)"
+ ANSI_COLOR="0;34"
+ CPE_NAME="cpe:/o:fedoraproject:fedora:26"
+ HOME_URL="https://fedoraproject.org/"
+ BUG_REPORT_URL="https://bugzilla.redhat.com/"
+ REDHAT_BUGZILLA_PRODUCT="Fedora"
+ REDHAT_BUGZILLA_PRODUCT_VERSION=26
+ REDHAT_SUPPORT_PRODUCT="Fedora"
+ REDHAT_SUPPORT_PRODUCT_VERSION=26
+ PRIVACY_POLICY_URL=https://fedoraproject.org/wiki/Legal:PrivacyPolicy
+ """)
+
+ str_opensuse_42_3_os_release = dedent("""
+ NAME="openSUSE Leap"
+ VERSION="42.3"
+ ID=opensuse
+ ID_LIKE="suse"
+ VERSION_ID="42.3"
+ PRETTY_NAME="openSUSE Leap 42.3"
+ ANSI_COLOR="0;32"
+ CPE_NAME="cpe:/o:opensuse:leap:42.3"
+ BUG_REPORT_URL="https://bugs.opensuse.org"
+ HOME_URL="https://www.opensuse.org/"
+ """)
+
+ str_opensuse_15_0_os_release = dedent("""
+ NAME="openSUSE Leap"
+ VERSION="15.0"
+ ID="opensuse-leap"
+ ID_LIKE="suse opensuse"
+ VERSION_ID="15.0"
+ PRETTY_NAME="openSUSE Leap 15.0"
+ ANSI_COLOR="0;32"
+ CPE_NAME="cpe:/o:opensuse:leap:15.0"
+ BUG_REPORT_URL="https://bugs.opensuse.org"
+ HOME_URL="https://www.opensuse.org/"
+ """)
+
+ str_opensuse_15_1_os_release = dedent("""
+ NAME="openSUSE Leap"
+ VERSION="15.1"
+ ID="opensuse-leap"
+ ID_LIKE="suse opensuse"
+ VERSION_ID="15.1"
+ PRETTY_NAME="openSUSE Leap 15.1"
+ ANSI_COLOR="0;32"
+ CPE_NAME="cpe:/o:opensuse:leap:15.1"
+ BUG_REPORT_URL="https://bugs.opensuse.org"
+ HOME_URL="https://www.opensuse.org/"
+ """)
+
+ def test_centos_9_os_release(self):
+ os = OS.from_os_release(self.str_centos_9_os_release)
+ assert os.name == 'centos'
+ assert os.version == '9.stream'
+ assert os.codename == 'stream'
+ assert os.package_type == 'rpm'
+
+ def test_centos_7_os_release(self):
+ os = OS.from_os_release(self.str_centos_7_os_release)
+ assert os.name == 'centos'
+ assert os.version == '7'
+ assert os.codename == 'core'
+ assert os.package_type == 'rpm'
+
+ def test_centos_7_os_release_newer(self):
+ os = OS.from_os_release(self.str_centos_7_os_release_newer)
+ assert os.name == 'centos'
+ assert os.version == '7'
+ assert os.codename == 'core'
+ assert os.package_type == 'rpm'
+
+ def test_debian_7_lsb_release(self):
+ os = OS.from_lsb_release(self.str_debian_7_lsb_release)
+ assert os.name == 'debian'
+ assert os.version == '7.1'
+ assert os.codename == 'wheezy'
+ assert os.package_type == 'deb'
+
+ def test_debian_7_os_release(self):
+ os = OS.from_os_release(self.str_debian_7_os_release)
+ assert os.name == 'debian'
+ assert os.version == '7'
+ assert os.codename == 'wheezy'
+ assert os.package_type == 'deb'
+
+ def test_debian_8_os_release(self):
+ os = OS.from_os_release(self.str_debian_8_os_release)
+ assert os.name == 'debian'
+ assert os.version == '8'
+ assert os.codename == 'jessie'
+ assert os.package_type == 'deb'
+
+ def test_debian_9_os_release(self):
+ os = OS.from_os_release(self.str_debian_9_os_release)
+ assert os.name == 'debian'
+ assert os.version == '9'
+ assert os.codename == 'stretch'
+ assert os.package_type == 'deb'
+
+ def test_ubuntu_12_04_lsb_release(self):
+ os = OS.from_lsb_release(self.str_ubuntu_12_04_lsb_release)
+ assert os.name == 'ubuntu'
+ assert os.version == '12.04'
+ assert os.codename == 'precise'
+ assert os.package_type == 'deb'
+
+ def test_ubuntu_12_04_os_release(self):
+ os = OS.from_os_release(self.str_ubuntu_12_04_os_release)
+ assert os.name == 'ubuntu'
+ assert os.version == '12.04'
+ assert os.codename == 'precise'
+ assert os.package_type == 'deb'
+
+ def test_ubuntu_14_04_os_release(self):
+ os = OS.from_os_release(self.str_ubuntu_14_04_os_release)
+ assert os.name == 'ubuntu'
+ assert os.version == '14.04'
+ assert os.codename == 'trusty'
+ assert os.package_type == 'deb'
+
+ def test_ubuntu_16_04_os_release(self):
+ os = OS.from_os_release(self.str_ubuntu_16_04_os_release)
+ assert os.name == 'ubuntu'
+ assert os.version == '16.04'
+ assert os.codename == 'xenial'
+ assert os.package_type == 'deb'
+
+ def test_ubuntu_18_04_os_release(self):
+ os = OS.from_os_release(self.str_ubuntu_18_04_os_release)
+ assert os.name == 'ubuntu'
+ assert os.version == '18.04'
+ assert os.codename == 'bionic'
+ assert os.package_type == 'deb'
+
+ def test_rhel_6_4_lsb_release(self):
+ os = OS.from_lsb_release(self.str_rhel_6_4_lsb_release)
+ assert os.name == 'rhel'
+ assert os.version == '6.4'
+ assert os.codename == 'santiago'
+ assert os.package_type == 'rpm'
+
+ def test_rhel_7_lsb_release(self):
+ os = OS.from_lsb_release(self.str_rhel_7_lsb_release)
+ assert os.name == 'rhel'
+ assert os.version == '7.0'
+ assert os.codename == 'maipo'
+ assert os.package_type == 'rpm'
+
+ def test_rhel_7_os_release(self):
+ os = OS.from_os_release(self.str_rhel_7_os_release)
+ assert os.name == 'rhel'
+ assert os.version == '7.0'
+ assert os.codename == 'maipo'
+ assert os.package_type == 'rpm'
+
+ def test_fedora_26_os_release(self):
+ os = OS.from_os_release(self.str_fedora_26_os_release)
+ assert os.name == 'fedora'
+ assert os.version == '26'
+ assert os.codename == '26'
+ assert os.package_type == 'rpm'
+
+ def test_opensuse_42_3_os_release(self):
+ os = OS.from_os_release(self.str_opensuse_42_3_os_release)
+ assert os.name == 'opensuse'
+ assert os.version == '42.3'
+ assert os.codename == 'leap'
+ assert os.package_type == 'rpm'
+
+ def test_opensuse_15_0_os_release(self):
+ os = OS.from_os_release(self.str_opensuse_15_0_os_release)
+ assert os.name == 'opensuse'
+ assert os.version == '15.0'
+ assert os.codename == 'leap'
+ assert os.package_type == 'rpm'
+
+ def test_opensuse_15_1_os_release(self):
+ os = OS.from_os_release(self.str_opensuse_15_1_os_release)
+ assert os.name == 'opensuse'
+ assert os.version == '15.1'
+ assert os.codename == 'leap'
+ assert os.package_type == 'rpm'
+
+ def test_version_codename_success(self):
+ assert OS.version_codename('ubuntu', '14.04') == ('14.04', 'trusty')
+ assert OS.version_codename('ubuntu', 'trusty') == ('14.04', 'trusty')
+
+ def test_version_codename_failure(self):
+ with pytest.raises(KeyError) as excinfo:
+ OS.version_codename('ubuntu', 'frog')
+ assert excinfo.type == KeyError
+ assert 'frog' in excinfo.value.args[0]
+
+ def test_repr(self):
+ os = OS(name='NAME', version='0.1.2', codename='code')
+ assert repr(os) == "OS(name='NAME', version='0.1.2', codename='code')"
+
+ def test_to_dict(self):
+ os = OS(name='NAME', version='0.1.2', codename='code')
+ ref_dict = dict(name='NAME', version='0.1.2', codename='code')
+ assert os.to_dict() == ref_dict
+
+ def test_version_no_codename(self):
+ os = OS(name='ubuntu', version='16.04')
+ assert os.codename == 'xenial'
+
+ def test_codename_no_version(self):
+ os = OS(name='ubuntu', codename='trusty')
+ assert os.version == '14.04'
+
+ def test_eq_equal(self):
+ os = OS(name='ubuntu', codename='trusty', version='14.04')
+ assert OS(name='ubuntu', codename='trusty', version='14.04') == os
+
+ def test_eq_not_equal(self):
+ os = OS(name='ubuntu', codename='trusty', version='16.04')
+ assert OS(name='ubuntu', codename='trusty', version='14.04') != os
--- /dev/null
+from mock import patch, Mock, MagicMock
+from pytest import raises
+
+from io import BytesIO
+
+from teuthology.orchestra import remote
+from teuthology.orchestra import opsys
+from teuthology.orchestra.run import RemoteProcess
+from teuthology.exceptions import CommandFailedError, UnitTestError
+
+
+class TestRemote(object):
+
+ def setup_method(self):
+ self.start_patchers()
+
+ def teardown_method(self):
+ self.stop_patchers()
+
+ def start_patchers(self):
+ self.m_ssh = MagicMock()
+ self.patcher_ssh = patch(
+ 'teuthology.orchestra.connection.paramiko.SSHClient',
+ self.m_ssh,
+ )
+ self.patcher_ssh.start()
+
+ def stop_patchers(self):
+ self.patcher_ssh.stop()
+
+ def test_shortname(self):
+ r = remote.Remote(
+ name='jdoe@xyzzy.example.com',
+ shortname='xyz',
+ ssh=self.m_ssh,
+ )
+ assert r.shortname == 'xyz'
+ assert str(r) == 'jdoe@xyzzy.example.com'
+
+ def test_shortname_default(self):
+ r = remote.Remote(
+ name='jdoe@xyzzy.example.com',
+ ssh=self.m_ssh,
+ )
+ assert r.shortname == 'xyzzy'
+ assert str(r) == 'jdoe@xyzzy.example.com'
+
+ def test_run(self):
+ m_transport = MagicMock()
+ m_transport.getpeername.return_value = ('name', 22)
+ self.m_ssh.get_transport.return_value = m_transport
+ m_run = MagicMock()
+ args = [
+ 'something',
+ 'more',
+ ]
+ proc = RemoteProcess(
+ client=self.m_ssh,
+ args=args,
+ )
+ m_run.return_value = proc
+ rem = remote.Remote(name='jdoe@xyzzy.example.com', ssh=self.m_ssh)
+ rem._runner = m_run
+ result = rem.run(args=args)
+ m_transport.getpeername.assert_called_once_with()
+ m_run_call_kwargs = m_run.call_args_list[0][1]
+ assert m_run_call_kwargs['args'] == args
+ assert result is proc
+ assert result.remote is rem
+
+ @patch('teuthology.util.scanner.UnitTestScanner.scan_and_write')
+ def test_run_unit_test(self, m_scan_and_write):
+ m_transport = MagicMock()
+ m_transport.getpeername.return_value = ('name', 22)
+ self.m_ssh.get_transport.return_value = m_transport
+ m_run = MagicMock(name="run", side_effect=CommandFailedError('mocked error', 1, 'smithi'))
+ args = [
+ 'something',
+ 'more',
+ ]
+ rem = remote.Remote(name='jdoe@xyzzy.example.com', ssh=self.m_ssh)
+ rem._runner = m_run
+ m_scan_and_write.return_value = "Error Message"
+ with raises(UnitTestError) as exc:
+ rem.run_unit_test(args=args, xml_path_regex="xml_path", output_yaml="yaml_path")
+ assert str(exc.value) == "Unit test failed on smithi with status 1: 'Error Message'"
+
+ def test_hostname(self):
+ m_transport = MagicMock()
+ m_transport.getpeername.return_value = ('name', 22)
+ self.m_ssh.get_transport.return_value = m_transport
+ m_run = MagicMock()
+ args = [
+ 'hostname',
+ '--fqdn',
+ ]
+ stdout = BytesIO(b'test_hostname')
+ stdout.seek(0)
+ proc = RemoteProcess(
+ client=self.m_ssh,
+ args=args,
+ )
+ proc.stdout = stdout
+ proc._stdout_buf = Mock()
+ proc._stdout_buf.channel.recv_exit_status.return_value = 0
+ r = remote.Remote(name='xyzzy.example.com', ssh=self.m_ssh)
+ m_run.return_value = proc
+ r._runner = m_run
+ assert r.hostname == 'test_hostname'
+
+ def test_arch(self):
+ m_transport = MagicMock()
+ m_transport.getpeername.return_value = ('name', 22)
+ self.m_ssh.get_transport.return_value = m_transport
+ m_run = MagicMock()
+ args = [
+ 'uname',
+ '-m',
+ ]
+ stdout = BytesIO(b'test_arch')
+ stdout.seek(0)
+ proc = RemoteProcess(
+ client=self.m_ssh,
+ args=args,
+ )
+ proc._stdout_buf = Mock()
+ proc._stdout_buf.channel = Mock()
+ proc._stdout_buf.channel.recv_exit_status.return_value = 0
+ proc._stdout_buf.channel.expects('recv_exit_status').returns(0)
+ proc.stdout = stdout
+ m_run.return_value = proc
+ r = remote.Remote(name='jdoe@xyzzy.example.com', ssh=self.m_ssh)
+ r._runner = m_run
+ assert r.arch == 'test_arch'
+ assert len(m_run.call_args_list) == 1
+ m_run_call_kwargs = m_run.call_args_list[0][1]
+ assert m_run_call_kwargs['client'] == self.m_ssh
+ assert m_run_call_kwargs['name'] == r.shortname
+ assert m_run_call_kwargs['args'] == ' '.join(args)
+
+ def test_host_key(self):
+ m_key = MagicMock()
+ m_key.get_name.return_value = 'key_type'
+ m_key.get_base64.return_value = 'test ssh key'
+ m_transport = MagicMock()
+ m_transport.get_remote_server_key.return_value = m_key
+ self.m_ssh.get_transport.return_value = m_transport
+ r = remote.Remote(name='jdoe@xyzzy.example.com', ssh=self.m_ssh)
+ assert r.host_key == 'key_type test ssh key'
+ self.m_ssh.get_transport.assert_called_once_with()
+ m_transport.get_remote_server_key.assert_called_once_with()
+
+ def test_inventory_info(self):
+ r = remote.Remote('user@host', host_key='host_key')
+ r._arch = 'arch'
+ r._os = opsys.OS(name='os_name', version='1.2.3', codename='code')
+ inv_info = r.inventory_info
+ assert inv_info == dict(
+ name='host',
+ user='user',
+ arch='arch',
+ os_type='os_name',
+ os_version='1.2',
+ ssh_pub_key='host_key',
+ up=True,
+ )
+
+ def test_sftp_open_file(self):
+ m_file_obj = MagicMock()
+ m_stat = Mock()
+ m_stat.st_size = 42
+ m_file_obj.stat.return_value = m_stat
+ m_open = MagicMock()
+ m_open.return_value = m_file_obj
+ m_open.return_value.__enter__.return_value = m_file_obj
+ with patch.object(remote.Remote, '_sftp_open_file', new=m_open):
+ rem = remote.Remote(name='jdoe@xyzzy.example.com', ssh=self.m_ssh)
+ assert rem._sftp_open_file('x') is m_file_obj
+ assert rem._sftp_open_file('x').stat() is m_stat
+ assert rem._sftp_open_file('x').stat().st_size == 42
+ with rem._sftp_open_file('x') as f:
+ assert f == m_file_obj
+
+ def test_sftp_get_size(self):
+ m_file_obj = MagicMock()
+ m_stat = Mock()
+ m_stat.st_size = 42
+ m_file_obj.stat.return_value = m_stat
+ m_open = MagicMock()
+ m_open.return_value = m_file_obj
+ m_open.return_value.__enter__.return_value = m_file_obj
+ with patch.object(remote.Remote, '_sftp_open_file', new=m_open):
+ rem = remote.Remote(name='jdoe@xyzzy.example.com', ssh=self.m_ssh)
+ assert rem._sftp_get_size('/fake/file') == 42
+
+ def test_format_size(self):
+ assert remote.Remote._format_size(1023).strip() == '1023B'
+ assert remote.Remote._format_size(1024).strip() == '1KB'
+ assert remote.Remote._format_size(1024**2).strip() == '1MB'
+ assert remote.Remote._format_size(1024**5).strip() == '1TB'
+ assert remote.Remote._format_size(1021112).strip() == '997KB'
+ assert remote.Remote._format_size(1021112**2).strip() == '971GB'
+
+ def test_is_container(self):
+ m_transport = MagicMock()
+ m_transport.getpeername.return_value = ('name', 22)
+ self.m_ssh.get_transport.return_value = m_transport
+ m_run = MagicMock()
+ args = []
+ proc = RemoteProcess(
+ client=self.m_ssh,
+ args=args,
+ )
+ proc.returncode = 0
+ m_run.return_value = proc
+ rem = remote.Remote(name='jdoe@xyzzy.example.com', ssh=self.m_ssh)
+ rem._runner = m_run
+ assert rem.is_container
+ proc.returncode = 1
+ rem2 = remote.Remote(name='jdoe@xyzzy.example.com', ssh=self.m_ssh)
+ rem2._runner = m_run
+ assert not rem2.is_container
+
+ @patch("teuthology.orchestra.remote.Remote.sh")
+ def test_resolve_ip(self, m_sh):
+ r = remote.Remote(name="jdoe@xyzzy.example.com", ssh=self.m_ssh)
+ m_sh.return_value = "smithi001.front.sepia.ceph.com has address 172.21.15.1\n"
+ ip4 = remote.Remote.resolve_ip(r, 'smithi001')
+ assert ip4 == "172.21.15.1"
+ m_sh.return_value = "\n"
+ try:
+ ip4 = remote.Remote.resolve_ip(r, 'smithi001')
+ except Exception as e:
+ assert 'Cannot get IPv4 address' in str(e)
+ try:
+ ip4 = remote.Remote.resolve_ip(r, 'smithi001', 5)
+ except Exception as e:
+ assert 'Unknown IP version' in str(e)
+
+ m_sh.return_value = ("google.com has address 142.251.37.14\n"
+ "google.com has IPv6 address 2a00:1450:4016:80b::200e\n"
+ "google.com mail is handled by 10 smtp.google.com.\n")
+ ip4 = remote.Remote.resolve_ip(r, 'google.com')
+ assert ip4 == "142.251.37.14"
+ ip6 = remote.Remote.resolve_ip(r, 'google.com', '6')
+ assert ip6 == "2a00:1450:4016:80b::200e"
+
+
+ @patch("teuthology.orchestra.remote.Remote.run")
+ def test_write_file(self, m_run):
+ file = "fakefile"
+ contents = "fakecontents"
+ rem = remote.Remote(name='jdoe@xyzzy.example.com', ssh=self.m_ssh)
+
+ remote.Remote.write_file(rem, file, contents, bs=1, offset=1024)
+ m_run.assert_called_with(args=f"set -ex\ndd of={file} bs=1 seek=1024", stdin=contents, quiet=True)
+
+ remote.Remote.write_file(rem, file, contents, sync=True)
+ m_run.assert_called_with(args=f"set -ex\ndd of={file} conv=sync", stdin=contents, quiet=True)
--- /dev/null
+from io import BytesIO
+
+import paramiko
+import socket
+
+from mock import MagicMock, patch
+from pytest import raises
+
+from teuthology.orchestra import run
+from teuthology.exceptions import (CommandCrashedError, CommandFailedError,
+ ConnectionLostError)
+
+def set_buffer_contents(buf, contents):
+ buf.seek(0)
+ if isinstance(contents, bytes):
+ buf.write(contents)
+ elif isinstance(contents, (list, tuple)):
+ buf.writelines(contents)
+ elif isinstance(contents, str):
+ buf.write(contents.encode())
+ else:
+ raise TypeError(
+ "%s is a %s; should be a byte string, list or tuple" % (
+ contents, type(contents)
+ )
+ )
+ buf.seek(0)
+
+
+class TestRun(object):
+ def setup_method(self):
+ self.start_patchers()
+
+ def teardown_method(self):
+ self.stop_patchers()
+
+ def start_patchers(self):
+ self.m_remote_process = MagicMock(wraps=run.RemoteProcess)
+ self.patcher_remote_proc = patch(
+ 'teuthology.orchestra.run.RemoteProcess',
+ self.m_remote_process,
+ )
+ self.m_channel = MagicMock(spec=paramiko.Channel)()
+ """
+ self.m_channelfile = MagicMock(wraps=paramiko.ChannelFile)
+ self.m_stdin_buf = self.m_channelfile(self.m_channel())
+ self.m_stdout_buf = self.m_channelfile(self.m_channel())
+ self.m_stderr_buf = self.m_channelfile(self.m_channel())
+ """
+ class M_ChannelFile(BytesIO):
+ channel = MagicMock(spec=paramiko.Channel)()
+
+ self.m_channelfile = M_ChannelFile
+ self.m_stdin_buf = self.m_channelfile()
+ self.m_stdout_buf = self.m_channelfile()
+ self.m_stderr_buf = self.m_channelfile()
+ self.m_ssh = MagicMock()
+ self.m_ssh.exec_command.return_value = (
+ self.m_stdin_buf,
+ self.m_stdout_buf,
+ self.m_stderr_buf,
+ )
+ self.m_transport = MagicMock()
+ self.m_transport.getpeername.return_value = ('name', 22)
+ self.m_ssh.get_transport.return_value = self.m_transport
+ self.patcher_ssh = patch(
+ 'teuthology.orchestra.connection.paramiko.SSHClient',
+ self.m_ssh,
+ )
+ self.patcher_ssh.start()
+ # Tests must start this if they wish to use it
+ # self.patcher_remote_proc.start()
+
+ def stop_patchers(self):
+ # If this patcher wasn't started, it's ok
+ try:
+ self.patcher_remote_proc.stop()
+ except RuntimeError:
+ pass
+ self.patcher_ssh.stop()
+
+ def test_exitstatus(self):
+ self.m_stdout_buf.channel.recv_exit_status.return_value = 0
+ proc = run.run(
+ client=self.m_ssh,
+ args=['foo', 'bar baz'],
+ )
+ assert proc.exitstatus == 0
+
+ def test_run_cwd(self):
+ self.m_stdout_buf.channel.recv_exit_status.return_value = 0
+ run.run(
+ client=self.m_ssh,
+ args=['foo_bar_baz'],
+ cwd='/cwd/test',
+ )
+ self.m_ssh.exec_command.assert_called_with('(cd /cwd/test && exec foo_bar_baz)')
+
+ def test_capture_stdout(self):
+ output = 'foo\nbar'
+ set_buffer_contents(self.m_stdout_buf, output)
+ self.m_stdout_buf.channel.recv_exit_status.return_value = 0
+ stdout = BytesIO()
+ proc = run.run(
+ client=self.m_ssh,
+ args=['foo', 'bar baz'],
+ stdout=stdout,
+ )
+ assert proc.stdout is stdout
+ assert proc.stdout.read().decode() == output
+ assert proc.stdout.getvalue().decode() == output
+
+ def test_capture_stderr_newline(self):
+ output = 'foo\nbar\n'
+ set_buffer_contents(self.m_stderr_buf, output)
+ self.m_stderr_buf.channel.recv_exit_status.return_value = 0
+ stderr = BytesIO()
+ proc = run.run(
+ client=self.m_ssh,
+ args=['foo', 'bar baz'],
+ stderr=stderr,
+ )
+ assert proc.stderr is stderr
+ assert proc.stderr.read().decode() == output
+ assert proc.stderr.getvalue().decode() == output
+
+ def test_status_bad(self):
+ self.m_stdout_buf.channel.recv_exit_status.return_value = 42
+ with raises(CommandFailedError) as exc:
+ run.run(
+ client=self.m_ssh,
+ args=['foo'],
+ )
+ assert str(exc.value) == "Command failed on name with status 42: 'foo'"
+
+ def test_status_bad_nocheck(self):
+ self.m_stdout_buf.channel.recv_exit_status.return_value = 42
+ proc = run.run(
+ client=self.m_ssh,
+ args=['foo'],
+ check_status=False,
+ )
+ assert proc.exitstatus == 42
+
+ def test_status_crash(self):
+ self.m_stdout_buf.channel.recv_exit_status.return_value = -1
+ with raises(CommandCrashedError) as exc:
+ run.run(
+ client=self.m_ssh,
+ args=['foo'],
+ )
+ assert str(exc.value) == "Command crashed: 'foo'"
+
+ def test_status_crash_nocheck(self):
+ self.m_stdout_buf.channel.recv_exit_status.return_value = -1
+ proc = run.run(
+ client=self.m_ssh,
+ args=['foo'],
+ check_status=False,
+ )
+ assert proc.exitstatus == -1
+
+ def test_status_lost(self):
+ m_transport = MagicMock()
+ m_transport.getpeername.return_value = ('name', 22)
+ m_transport.is_active.return_value = False
+ self.m_stdout_buf.channel.recv_exit_status.return_value = -1
+ self.m_ssh.get_transport.return_value = m_transport
+ with raises(ConnectionLostError) as exc:
+ run.run(
+ client=self.m_ssh,
+ args=['foo'],
+ )
+ assert str(exc.value) == "SSH connection to name was lost: 'foo'"
+
+ def test_status_lost_socket(self):
+ m_transport = MagicMock()
+ m_transport.getpeername.side_effect = socket.error
+ self.m_ssh.get_transport.return_value = m_transport
+ with raises(ConnectionLostError) as exc:
+ run.run(
+ client=self.m_ssh,
+ args=['foo'],
+ )
+ assert str(exc.value) == "SSH connection was lost: 'foo'"
+
+ def test_status_lost_nocheck(self):
+ m_transport = MagicMock()
+ m_transport.getpeername.return_value = ('name', 22)
+ m_transport.is_active.return_value = False
+ self.m_stdout_buf.channel.recv_exit_status.return_value = -1
+ self.m_ssh.get_transport.return_value = m_transport
+ proc = run.run(
+ client=self.m_ssh,
+ args=['foo'],
+ check_status=False,
+ )
+ assert proc.exitstatus == -1
+
+ def test_status_bad_nowait(self):
+ self.m_stdout_buf.channel.recv_exit_status.return_value = 42
+ proc = run.run(
+ client=self.m_ssh,
+ args=['foo'],
+ wait=False,
+ )
+ with raises(CommandFailedError) as exc:
+ proc.wait()
+ assert proc.returncode == 42
+ assert str(exc.value) == "Command failed on name with status 42: 'foo'"
+
+ def test_stdin_pipe(self):
+ self.m_stdout_buf.channel.recv_exit_status.return_value = 0
+ proc = run.run(
+ client=self.m_ssh,
+ args=['foo'],
+ stdin=run.PIPE,
+ wait=False
+ )
+ assert proc.poll() == 0
+ code = proc.wait()
+ assert code == 0
+ assert proc.exitstatus == 0
+
+ def test_stdout_pipe(self):
+ self.m_stdout_buf.channel.recv_exit_status.return_value = 0
+ lines = [b'one\n', b'two', b'']
+ set_buffer_contents(self.m_stdout_buf, lines)
+ proc = run.run(
+ client=self.m_ssh,
+ args=['foo'],
+ stdout=run.PIPE,
+ wait=False
+ )
+ assert proc.poll() == 0
+ assert proc.stdout.readline() == lines[0]
+ assert proc.stdout.readline() == lines[1]
+ assert proc.stdout.readline() == lines[2]
+ code = proc.wait()
+ assert code == 0
+ assert proc.exitstatus == 0
+
+ def test_stderr_pipe(self):
+ self.m_stdout_buf.channel.recv_exit_status.return_value = 0
+ lines = [b'one\n', b'two', b'']
+ set_buffer_contents(self.m_stderr_buf, lines)
+ proc = run.run(
+ client=self.m_ssh,
+ args=['foo'],
+ stderr=run.PIPE,
+ wait=False
+ )
+ assert proc.poll() == 0
+ assert proc.stderr.readline() == lines[0]
+ assert proc.stderr.readline() == lines[1]
+ assert proc.stderr.readline() == lines[2]
+ code = proc.wait()
+ assert code == 0
+ assert proc.exitstatus == 0
+
+ def test_copy_and_close(self):
+ run.copy_and_close(None, MagicMock())
+ run.copy_and_close('', MagicMock())
+ run.copy_and_close(b'', MagicMock())
+
+
+class TestQuote(object):
+ def test_quote_simple(self):
+ got = run.quote(['a b', ' c', 'd e '])
+ assert got == "'a b' ' c' 'd e '"
+
+ def test_quote_and_quote(self):
+ got = run.quote(['echo', 'this && is embedded', '&&',
+ 'that was standalone'])
+ assert got == "echo 'this && is embedded' '&&' 'that was standalone'"
+
+ def test_quote_and_raw(self):
+ got = run.quote(['true', run.Raw('&&'), 'echo', 'yay'])
+ assert got == "true && echo yay"
+
+
+class TestRaw(object):
+ def test_eq(self):
+ str_ = "I am a raw something or other"
+ raw = run.Raw(str_)
+ assert raw == run.Raw(str_)
--- /dev/null
+import argparse
+import os
+
+from logging import debug
+from teuthology import misc
+from teuthology.orchestra import cluster
+from teuthology.orchestra.run import quote
+from teuthology.orchestra.daemon.group import DaemonGroup
+import subprocess
+
+
+class FakeRemote(object):
+ pass
+
+
+def test_pid():
+ ctx = argparse.Namespace()
+ ctx.daemons = DaemonGroup(use_systemd=True)
+ remote = FakeRemote()
+
+ ps_ef_output_path = os.path.join(
+ os.path.dirname(__file__),
+ "files/daemon-systemdstate-pid-ps-ef.output"
+ )
+
+ # patching ps -ef command output using a file
+ def sh(args):
+ args[0:2] = ["cat", ps_ef_output_path]
+ debug(args)
+ return subprocess.getoutput(quote(args))
+
+ remote.sh = sh
+ remote.init_system = 'systemd'
+ remote.shortname = 'host1'
+
+ ctx.cluster = cluster.Cluster(
+ remotes=[
+ (remote, ['rgw.0', 'mon.a', 'mgr.a', 'mds.a', 'osd.0'])
+ ],
+ )
+
+ for remote, roles in ctx.cluster.remotes.items():
+ for role in roles:
+ _, rol, id_ = misc.split_role(role)
+ if any(rol.startswith(x) for x in ['mon', 'mgr', 'mds']):
+ ctx.daemons.register_daemon(remote, rol, remote.shortname)
+ else:
+ ctx.daemons.register_daemon(remote, rol, id_)
+
+ for _, daemons in ctx.daemons.daemons.items():
+ for daemon in daemons.values():
+ pid = daemon.pid
+ debug(pid)
+ assert pid
--- /dev/null
+def assert_raises(excClass, callableObj, *args, **kwargs):
+ """
+ Like unittest.TestCase.assertRaises, but returns the exception.
+ """
+ try:
+ callableObj(*args, **kwargs)
+ except excClass as e:
+ return e
+ else:
+ if hasattr(excClass,'__name__'): excName = excClass.__name__
+ else: excName = str(excClass)
+ raise AssertionError("%s not raised" % excName)
--- /dev/null
+from libcloud.compute.providers import get_driver
+from mock import patch
+
+from teuthology.config import config
+from teuthology.provision import cloud
+
+from test_cloud_init import dummy_config, dummy_drivers
+
+
+class TestBase(object):
+ def setup_method(self):
+ config.load()
+ config.libcloud = dummy_config
+ cloud.supported_drivers['dummy'] = dummy_drivers
+
+ def teardown_method(self):
+ del cloud.supported_drivers['dummy']
+
+
+class TestProvider(TestBase):
+ def test_init(self):
+ obj = cloud.get_provider('my_provider')
+ assert obj.name == 'my_provider'
+ assert obj.driver_name == 'dummy'
+ assert obj.conf == dummy_config['providers']['my_provider']
+
+ def test_driver(self):
+ obj = cloud.get_provider('my_provider')
+ assert isinstance(obj.driver, get_driver('dummy'))
+
+
+class TestProvisioner(TestBase):
+ klass = cloud.base.Provisioner
+
+ def get_obj(
+ self, name='node_name', os_type='ubuntu', os_version='ubuntu'):
+ return cloud.get_provisioner(
+ 'my_provider',
+ 'node_name',
+ 'ubuntu',
+ '16.04',
+ )
+
+ def test_init_provider_string(self):
+ obj = self.klass('my_provider', 'ubuntu', '16.04')
+ assert obj.provider.name == 'my_provider'
+
+ def test_create(self):
+ obj = self.get_obj()
+ with patch.object(
+ self.klass,
+ '_create',
+ ) as m_create:
+ for val in [True, False]:
+ m_create.return_value = val
+ res = obj.create()
+ assert res is val
+ m_create.assert_called_once_with()
+ m_create.reset_mock()
+ m_create.side_effect = RuntimeError
+ res = obj.create()
+ assert res is False
+ assert obj.create() is None
+
+ def test_destroy(self):
+ obj = self.get_obj()
+ with patch.object(
+ self.klass,
+ '_destroy',
+ ) as m_destroy:
+ for val in [True, False]:
+ m_destroy.return_value = val
+ res = obj.destroy()
+ assert res is val
+ m_destroy.assert_called_once_with()
+ m_destroy.reset_mock()
+ m_destroy.side_effect = RuntimeError
+ res = obj.destroy()
+ assert res is False
+ assert obj.destroy() is None
+
+ def test_remote(self):
+ obj = self.get_obj()
+ assert obj.remote.shortname == 'node_name'
+
+ def test_repr(self):
+ obj = self.get_obj()
+ assert repr(obj) == \
+ "Provisioner(provider='my_provider', name='node_name', os_type='ubuntu', os_version='16.04')" # noqa
+
--- /dev/null
+from teuthology.config import config
+from teuthology.provision import cloud
+
+dummy_config = dict(
+ providers=dict(
+ my_provider=dict(
+ driver='dummy',
+ driver_args=dict(
+ creds=0,
+ ),
+ conf_1='1',
+ conf_2='2',
+ )
+ )
+)
+
+
+class DummyProvider(cloud.base.Provider):
+ # For libcloud's dummy driver
+ _driver_posargs = ['creds']
+
+dummy_drivers = dict(
+ provider=DummyProvider,
+ provisioner=cloud.base.Provisioner,
+)
+
+
+class TestInit(object):
+ def setup_method(self):
+ config.load()
+ config.libcloud = dummy_config
+ cloud.supported_drivers['dummy'] = dummy_drivers
+
+ def teardown_method(self):
+ del cloud.supported_drivers['dummy']
+
+ def test_get_types(self):
+ assert list(cloud.get_types()) == ['my_provider']
+
+ def test_get_provider_conf(self):
+ expected = dummy_config['providers']['my_provider']
+ assert cloud.get_provider_conf('my_provider') == expected
+
+ def test_get_provider(self):
+ obj = cloud.get_provider('my_provider')
+ assert obj.name == 'my_provider'
+ assert obj.driver_name == 'dummy'
+
+ def test_get_provisioner(self):
+ obj = cloud.get_provisioner(
+ 'my_provider',
+ 'node_name',
+ 'ubuntu',
+ '16.04',
+ dict(foo='bar'),
+ )
+ assert obj.provider.name == 'my_provider'
+ assert obj.name == 'node_name'
+ assert obj.os_type == 'ubuntu'
+ assert obj.os_version == '16.04'
--- /dev/null
+import datetime
+import dateutil
+import json
+
+from mock import patch, mock_open
+from pytest import mark
+
+from teuthology.provision.cloud import util
+
+
+@mark.parametrize(
+ 'path, exists',
+ [
+ ('/fake/path', True),
+ ('/fake/path', False),
+ ]
+)
+def test_get_user_ssh_pubkey(path, exists):
+ with patch('os.path.exists') as m_exists:
+ m_exists.return_value = exists
+ with patch('teuthology.provision.cloud.util.open', mock_open(), create=True) as m_open:
+ util.get_user_ssh_pubkey(path)
+ if exists:
+ m_open.assert_called_once_with(path)
+
+
+@mark.parametrize(
+ 'input_, func, expected',
+ [
+ [
+ [
+ dict(sub0=dict(key0=0, key1=0)),
+ dict(sub0=dict(key1=1, key2=2)),
+ ],
+ lambda x, y: x > y,
+ dict(sub0=dict(key0=0, key1=1, key2=2))
+ ],
+ [
+ [
+ dict(),
+ dict(sub0=dict(key1=1, key2=2)),
+ ],
+ lambda x, y: x > y,
+ dict(sub0=dict(key1=1, key2=2))
+ ],
+ [
+ [
+ dict(sub0=dict(key1=1, key2=2)),
+ dict(),
+ ],
+ lambda x, y: x > y,
+ dict(sub0=dict(key1=1, key2=2))
+ ],
+ [
+ [
+ dict(sub0=dict(key0=0, key1=0, key2=0)),
+ dict(sub0=dict(key0=1, key2=3), sub1=dict(key0=0)),
+ dict(sub0=dict(key0=3, key1=2, key2=1)),
+ dict(sub0=dict(key1=3),
+ sub1=dict(key0=3, key1=0)),
+ ],
+ lambda x, y: x > y,
+ dict(sub0=dict(key0=3, key1=3, key2=3),
+ sub1=dict(key0=3, key1=0))
+ ],
+ ]
+)
+def test_combine_dicts(input_, func, expected):
+ assert util.combine_dicts(input_, func) == expected
+
+
+def get_datetime(offset_hours=0):
+ delta = datetime.timedelta(hours=offset_hours)
+ return datetime.datetime.now(dateutil.tz.tzutc()) + delta
+
+
+def get_datetime_string(offset_hours=0):
+ obj = get_datetime(offset_hours)
+ return obj.strftime(util.AuthToken.time_format)
+
+
+class TestAuthToken(object):
+ klass = util.AuthToken
+
+ def setup_method(self):
+ default_expires = get_datetime_string(0)
+ self.test_data = dict(
+ value='token_value',
+ endpoint='endpoint',
+ expires=default_expires,
+ )
+ self.patchers = dict()
+ self.patchers['m_open'] = patch(
+ 'teuthology.provision.cloud.util.open'
+ )
+ self.patchers['m_exists'] = patch(
+ 'os.path.exists'
+ )
+ self.patchers['m_file_lock'] = patch(
+ 'teuthology.provision.cloud.util.FileLock'
+ )
+ self.mocks = dict()
+ for name, patcher in self.patchers.items():
+ self.mocks[name] = patcher.start()
+
+ def teardown_method(self):
+ for patcher in self.patchers.values():
+ patcher.stop()
+
+ def get_obj(self, name='name', directory='/fake/directory'):
+ return self.klass(
+ name=name,
+ directory=directory,
+ )
+
+ def test_no_token(self):
+ obj = self.get_obj()
+ self.mocks['m_exists'].return_value = False
+ with obj:
+ assert obj.value is None
+ assert obj.expired is True
+
+ @mark.parametrize(
+ 'test_data, expired',
+ [
+ [
+ dict(
+ value='token_value',
+ endpoint='endpoint',
+ expires=get_datetime_string(-1),
+ ),
+ True
+ ],
+ [
+ dict(
+ value='token_value',
+ endpoint='endpoint',
+ expires=get_datetime_string(1),
+ ),
+ False
+ ],
+ ]
+ )
+ def test_token_read(self, test_data, expired):
+ obj = self.get_obj()
+ self.mocks['m_exists'].return_value = True
+ self.mocks['m_open'].return_value.__enter__.return_value.read.return_value = \
+ json.dumps(test_data)
+ with obj:
+ if expired:
+ assert obj.value is None
+ assert obj.expired is True
+ else:
+ assert obj.value == test_data['value']
+
+ def test_token_write(self):
+ obj = self.get_obj()
+ datetime_obj = get_datetime(0)
+ datetime_string = get_datetime_string(0)
+ self.mocks['m_exists'].return_value = False
+ with obj:
+ obj.write('value', datetime_obj, 'endpoint')
+ m_open = self.mocks['m_open']
+ write_calls = m_open.return_value.__enter__.return_value.write\
+ .call_args_list
+ assert len(write_calls) == 1
+ expected = json.dumps(dict(
+ value='value',
+ expires=datetime_string,
+ endpoint='endpoint',
+ ))
+ assert write_calls[0][0][0] == expected
--- /dev/null
+import socket
+import yaml
+import os
+
+from teuthology.util.compat import parse_qs
+
+from copy import deepcopy
+from libcloud.compute.providers import get_driver
+from mock import patch, Mock, DEFAULT
+from pytest import raises, mark
+
+from teuthology.config import config
+from teuthology.exceptions import MaxWhileTries
+from teuthology.provision import cloud
+
+test_config = dict(
+ providers=dict(
+ my_provider=dict(
+ driver='openstack',
+ driver_args=dict(
+ username='user',
+ password='password',
+ ex_force_auth_url='http://127.0.0.1:9999/v2.0/tokens',
+ ),
+ ),
+ image_exclude_provider=dict(
+ driver='openstack',
+ exclude_image=['.*-exclude1', '.*-exclude2'],
+ driver_args=dict(
+ username='user',
+ password='password',
+ ex_force_auth_url='http://127.0.0.1:9999/v2.0/tokens',
+ ),
+ )
+ )
+)
+
+
+@patch('time.sleep')
+def test_retry(m_sleep):
+ orig_exceptions = cloud.openstack.RETRY_EXCEPTIONS
+ new_exceptions = orig_exceptions + (RuntimeError, )
+
+ class test_cls(object):
+ def __init__(self, min_val):
+ self.min_val = min_val
+ self.cur_val = 0
+
+ def func(self):
+ self.cur_val += 1
+ if self.cur_val < self.min_val:
+ raise RuntimeError
+ return self.cur_val
+
+ with patch.object(
+ cloud.openstack,
+ 'RETRY_EXCEPTIONS',
+ new=new_exceptions,
+ ):
+ test_obj = test_cls(min_val=5)
+ assert cloud.openstack.retry(test_obj.func) == 5
+ test_obj = test_cls(min_val=1000)
+ with raises(MaxWhileTries):
+ cloud.openstack.retry(test_obj.func)
+
+
+def get_fake_obj(mock_args=None, attributes=None):
+ if mock_args is None:
+ mock_args = dict()
+ if attributes is None:
+ attributes = dict()
+ obj = Mock(**mock_args)
+ for name, value in attributes.items():
+ setattr(obj, name, value)
+ return obj
+
+
+class TestOpenStackBase(object):
+ def setup_method(self):
+ config.load(dict(libcloud=deepcopy(test_config)))
+ self.start_patchers()
+
+ def start_patchers(self):
+ self.patchers = dict()
+ self.patchers['m_list_images'] = patch(
+ 'libcloud.compute.drivers.openstack'
+ '.OpenStackNodeDriver.list_images'
+ )
+ self.patchers['m_list_sizes'] = patch(
+ 'libcloud.compute.drivers.openstack'
+ '.OpenStackNodeDriver.list_sizes'
+ )
+ self.patchers['m_ex_list_networks'] = patch(
+ 'libcloud.compute.drivers.openstack'
+ '.OpenStack_1_1_NodeDriver.ex_list_networks'
+ )
+ self.patchers['m_ex_list_security_groups'] = patch(
+ 'libcloud.compute.drivers.openstack'
+ '.OpenStack_1_1_NodeDriver.ex_list_security_groups'
+ )
+ self.patchers['m_get_user_ssh_pubkey'] = patch(
+ 'teuthology.provision.cloud.util.get_user_ssh_pubkey'
+ )
+ self.patchers['m_list_nodes'] = patch(
+ 'libcloud.compute.drivers.openstack'
+ '.OpenStackNodeDriver.list_nodes'
+ )
+ self.patchers['m_create_node'] = patch(
+ 'libcloud.compute.drivers.openstack'
+ '.OpenStack_1_1_NodeDriver.create_node'
+ )
+ self.patchers['m_wait_until_running'] = patch(
+ 'libcloud.compute.drivers.openstack'
+ '.OpenStackNodeDriver.wait_until_running'
+ )
+ self.patchers['m_create_volume'] = patch(
+ 'libcloud.compute.drivers.openstack'
+ '.OpenStackNodeDriver.create_volume'
+ )
+ self.patchers['m_attach_volume'] = patch(
+ 'libcloud.compute.drivers.openstack'
+ '.OpenStackNodeDriver.attach_volume'
+ )
+ self.patchers['m_detach_volume'] = patch(
+ 'libcloud.compute.drivers.openstack'
+ '.OpenStackNodeDriver.detach_volume'
+ )
+ self.patchers['m_list_volumes'] = patch(
+ 'libcloud.compute.drivers.openstack'
+ '.OpenStackNodeDriver.list_volumes'
+ )
+ self.patchers['m_destroy_volume'] = patch(
+ 'libcloud.compute.drivers.openstack'
+ '.OpenStackNodeDriver.destroy_volume'
+ )
+ self.patchers['m_get_service_catalog'] = patch(
+ 'libcloud.common.openstack'
+ '.OpenStackBaseConnection.get_service_catalog'
+ )
+ self.patchers['m_auth_token'] = patch(
+ 'teuthology.provision.cloud.util.AuthToken'
+ )
+ self.patchers['m_get_endpoint'] = patch(
+ 'libcloud.common.openstack'
+ '.OpenStackBaseConnection.get_endpoint',
+ )
+ self.patchers['m_connect'] = patch(
+ 'libcloud.common.base'
+ '.Connection.connect',
+ )
+ self.patchers['m_sleep'] = patch(
+ 'time.sleep'
+ )
+ self.patchers['m_get'] = patch(
+ 'requests.get'
+ )
+ self.mocks = dict()
+ for name, patcher in self.patchers.items():
+ self.mocks[name] = patcher.start()
+ self.mocks['m_get_endpoint'].return_value = 'endpoint'
+
+ def teardown_method(self):
+ for patcher in self.patchers.values():
+ patcher.stop()
+
+
+class TestOpenStackProvider(TestOpenStackBase):
+ klass = cloud.openstack.OpenStackProvider
+
+ def test_init(self):
+ obj = cloud.get_provider('my_provider')
+ assert obj.name == 'my_provider'
+ assert obj.driver_name == 'openstack'
+ assert obj.conf == test_config['providers']['my_provider']
+
+ def test_driver(self):
+ token = self.mocks['m_auth_token'].return_value
+ self.mocks['m_auth_token'].return_value.__enter__.return_value = token
+ token.value = None
+ obj = cloud.get_provider('my_provider')
+ assert isinstance(obj.driver, get_driver('openstack'))
+ assert obj._auth_token.value is None
+
+ def test_images(self):
+ obj = cloud.get_provider('my_provider')
+ self.mocks['m_list_images'].return_value = [
+ get_fake_obj(attributes=dict(name=_))
+ for _ in ['image0', 'image1']]
+ assert not hasattr(obj, '_images')
+ assert [_.name for _ in obj.images] == ['image0', 'image1']
+ assert hasattr(obj, '_images')
+
+ def test_exclude_image(self):
+ obj = cloud.get_provider('image_exclude_provider')
+ self.mocks['m_list_images'].return_value = [
+ get_fake_obj(attributes=dict(name=_))
+ for _ in ['image0', 'image1',
+ 'image2-exclude1', 'image3-exclude2']]
+ assert not hasattr(obj, '_images')
+ assert [_.name for _ in obj.images] == ['image0', 'image1']
+ assert hasattr(obj, '_images')
+
+ def test_sizes(self):
+ obj = cloud.get_provider('my_provider')
+ fake_sizes = [get_fake_obj(attributes=dict(name='size%s' % i)) for
+ i in range(2)]
+ self.mocks['m_list_sizes'].return_value = fake_sizes
+ assert not hasattr(obj, '_sizes')
+ assert [s.name for s in obj.sizes] == ['size0', 'size1']
+ assert hasattr(obj, '_sizes')
+
+ def test_networks(self):
+ obj = cloud.get_provider('my_provider')
+ nets = [get_fake_obj(attributes=dict(name=i)) for i in ['net0', 'net1']]
+ self.mocks['m_ex_list_networks'].return_value = nets
+ assert not hasattr(obj, '_networks')
+ assert [i.name for i in obj.networks] == [i.name for i in nets]
+ assert hasattr(obj, '_networks')
+ self.mocks['m_ex_list_networks'].side_effect = AttributeError
+ obj = cloud.get_provider('my_provider')
+ assert not hasattr(obj, '_networks')
+ assert obj.networks == list()
+ assert hasattr(obj, '_networks')
+
+ def test_security_groups(self):
+ obj = cloud.get_provider('my_provider')
+ self.mocks['m_ex_list_security_groups'].return_value = ['sg0', 'sg1']
+ assert not hasattr(obj, '_security_groups')
+ assert obj.security_groups == ['sg0', 'sg1']
+ assert hasattr(obj, '_security_groups')
+ self.mocks['m_ex_list_security_groups'].side_effect = AttributeError
+ obj = cloud.get_provider('my_provider')
+ assert not hasattr(obj, '_security_groups')
+ assert obj.security_groups == list()
+ assert hasattr(obj, '_security_groups')
+
+
+class TestOpenStackCustomProvisioner(TestOpenStackBase):
+ klass = cloud.openstack.OpenStackProvisioner
+ def get_obj(
+ self, name='node_name', os_type='ubuntu',
+ os_version='16.04', conf=None, test_conf=None):
+
+ if test_conf:
+ yaml_file = os.path.dirname(__file__) + '/' + test_conf
+ print("Reading conf: %s" % yaml_file)
+ with open(yaml_file) as f:
+ teuth_conf=yaml.safe_load(f)
+ print(teuth_conf)
+ config.libcloud = deepcopy(teuth_conf['libcloud'] or test_config)
+ else:
+ config.libcloud = deepcopy(test_config)
+ return cloud.get_provisioner(
+ node_type='my_provider',
+ name=name,
+ os_type=os_type,
+ os_version=os_version,
+ conf=conf,
+ )
+
+ @mark.parametrize(
+ "conf",
+ [
+ dict(
+ path='test_openstack_userdata_conf.yaml',
+ runcmd_head=['uptime', 'date'],
+ ssh_authorized_keys=['user_public_key1', 'user_public_key2'],
+ user_ssh_pubkey='my_ssh_key',
+ os_version='16.04',
+ os_type='ubuntu',
+ ),
+ dict(
+ path='test_openstack_userdata_conf.yaml',
+ runcmd_head=['uptime', 'date'],
+ ssh_authorized_keys=['user_public_key1', 'user_public_key2'],
+ user_ssh_pubkey=None,
+ os_version='16.04',
+ os_type='ubuntu',
+ ),
+ dict(
+ os_version='16.04',
+ os_type='ubuntu',
+ path=None,
+ user_ssh_pubkey=None,
+ ),
+ ]
+ )
+ def test_userdata_conf(self, conf):
+ self.mocks['m_get_user_ssh_pubkey'].return_value = conf['user_ssh_pubkey']
+ obj = self.get_obj(os_version=conf['os_version'],
+ os_type=conf['os_type'],
+ test_conf=conf['path'])
+ userdata = yaml.safe_load(obj.userdata)
+ print(">>>> ", obj.conf)
+ print(">>>> ", obj.provider.conf)
+ print(">>>> ", obj.provider)
+ print(obj.userdata)
+ if conf and 'path' in conf and conf['path']:
+ assert userdata['runcmd'][0:len(conf['runcmd_head'])] == conf['runcmd_head']
+ assert userdata['bootcmd'] == [
+ 'SuSEfirewall2 stop || true',
+ 'service firewalld stop || true',
+ ]
+ assert 'packages' not in userdata
+ else:
+ assert 'bootcmd' not in userdata
+ assert userdata['packages'] == ['git', 'wget', 'python', 'ntp']
+ assert userdata['user'] == obj.user
+ assert userdata['hostname'] == obj.hostname
+ if 'user_ssh_pubkey' in conf and conf['user_ssh_pubkey']:
+ assert userdata['ssh_authorized_keys'][-1] == conf['user_ssh_pubkey']
+ if 'ssh_authorized_keys' in conf:
+ keys = conf['ssh_authorized_keys']
+ assert userdata['ssh_authorized_keys'][0:len(keys)] == keys
+ else:
+ if 'ssh_authorized_keys' in conf:
+ keys = conf['ssh_authorized_keys']
+ assert userdata['ssh_authorized_keys'][0:len(keys)] == keys
+ else:
+ assert 'ssh_authorized_keys' not in userdata
+
+ @mark.parametrize(
+ "conf",
+ [
+ dict(
+ path='test_openstack_userdata_conf.yaml',
+ runcmd_head=['uptime', 'date'],
+ ),
+ dict(
+ path=None,
+ ),
+ ]
+ )
+ def test_userdata_conf_runcmd(self, conf):
+ self.mocks['m_get_user_ssh_pubkey'].return_value = None
+ obj = self.get_obj(test_conf=conf['path'])
+ userdata = yaml.safe_load(obj.userdata)
+ assert userdata['runcmd'][-2:] == [['passwd', '-d', 'ubuntu'], ['touch', '/.teuth_provisioned']]
+
+ @mark.parametrize(
+ "conf",
+ [
+ dict(
+ path='test_openstack_userdata_conf.yaml',
+ packages=None,
+ ),
+ dict(
+ path=None,
+ packages=['git', 'wget', 'python', 'ntp']
+ ),
+ ]
+ )
+ def test_userdata_conf_packages(self, conf):
+ self.mocks['m_get_user_ssh_pubkey'].return_value = None
+ obj = self.get_obj(test_conf=conf['path'])
+ userdata = yaml.safe_load(obj.userdata)
+ assert userdata.get('packages', None) == conf['packages']
+
+class TestOpenStackProvisioner(TestOpenStackBase):
+ klass = cloud.openstack.OpenStackProvisioner
+
+ def get_obj(
+ self, name='node_name', os_type='ubuntu',
+ os_version='16.04', conf=None):
+ return cloud.get_provisioner(
+ node_type='my_provider',
+ name=name,
+ os_type=os_type,
+ os_version=os_version,
+ conf=conf,
+ )
+
+ def test_init(self):
+ with patch.object(
+ self.klass,
+ '_read_conf',
+ ) as m_read_conf:
+ self.get_obj()
+ assert len(m_read_conf.call_args_list) == 1
+
+ @mark.parametrize(
+ 'input_conf',
+ [
+ dict(machine=dict(
+ disk=42,
+ ram=9001,
+ cpus=3,
+ )),
+ dict(volumes=dict(
+ count=3,
+ size=100,
+ )),
+ dict(),
+ dict(
+ machine=dict(
+ disk=1,
+ ram=2,
+ cpus=3,
+ ),
+ volumes=dict(
+ count=4,
+ size=5,
+ )
+ ),
+ dict(
+ machine=dict(
+ disk=100,
+ ),
+ ),
+ ]
+ )
+ def test_read_conf(self, input_conf):
+ obj = self.get_obj(conf=input_conf)
+ for topic in ['machine', 'volumes']:
+ combined = cloud.util.combine_dicts(
+ [input_conf, config.openstack],
+ lambda x, y: x > y,
+ )
+ assert obj.conf[topic] == combined[topic]
+
+ @mark.parametrize(
+ 'input_conf, expected_machine, expected_vols',
+ [
+ [
+ dict(openstack=[
+ dict(machine=dict(disk=64, ram=10000, cpus=3)),
+ dict(volumes=dict(count=1, size=1)),
+ ]),
+ dict(disk=64, ram=10000, cpus=3),
+ dict(count=1, size=1),
+ ],
+ [
+ dict(openstack=[
+ dict(machine=dict(cpus=3)),
+ dict(machine=dict(disk=1, ram=9000)),
+ dict(machine=dict(disk=50, ram=2, cpus=1)),
+ dict(machine=dict()),
+ dict(volumes=dict()),
+ dict(volumes=dict(count=0, size=0)),
+ dict(volumes=dict(count=1, size=0)),
+ dict(volumes=dict(size=1)),
+ ]),
+ dict(disk=50, ram=9000, cpus=3),
+ dict(count=1, size=1),
+ ],
+ [
+ dict(openstack=[
+ dict(volumes=dict(count=3, size=30)),
+ dict(volumes=dict(size=50)),
+ ]),
+ None,
+ dict(count=3, size=50),
+ ],
+ [
+ dict(openstack=[
+ dict(machine=dict(disk=100)),
+ dict(volumes=dict(count=3, size=30)),
+ ]),
+ dict(disk=100, ram=8000, cpus=1),
+ dict(count=3, size=30),
+ ],
+ ]
+ )
+ def test_read_conf_legacy(
+ self, input_conf, expected_machine, expected_vols):
+ obj = self.get_obj(conf=input_conf)
+ if expected_machine is not None:
+ assert obj.conf['machine'] == expected_machine
+ else:
+ assert obj.conf['machine'] == config.openstack['machine']
+ if expected_vols is not None:
+ assert obj.conf['volumes'] == expected_vols
+
+ @mark.parametrize(
+ "os_type, os_version, should_find",
+ [
+ ('centos', '7', True),
+ ('BeOS', '42', False),
+ ]
+ )
+ def test_image(self, os_type, os_version, should_find):
+ image_attrs = [
+ dict(name='ubuntu-14.04'),
+ dict(name='ubuntu-16.04'),
+ dict(name='centos-7.0'),
+ ]
+ fake_images = list()
+ for item in image_attrs:
+ fake_images.append(
+ get_fake_obj(attributes=item)
+ )
+ obj = self.get_obj(os_type=os_type, os_version=os_version)
+ self.mocks['m_list_images'].return_value = fake_images
+ if should_find:
+ assert obj.os_version in obj.image.name
+ assert obj.image in fake_images
+ else:
+ with raises(RuntimeError):
+ obj.image
+
+ @mark.parametrize(
+ "input_attrs, func_or_exc",
+ [
+ (dict(ram=2**16),
+ lambda s: s.ram == 2**16),
+ (dict(disk=9999),
+ lambda s: s.disk == 9999),
+ (dict(cpus=99),
+ lambda s: s.vcpus == 99),
+ (dict(ram=2**16, disk=9999, cpus=99),
+ IndexError),
+ ]
+ )
+ def test_size(self, input_attrs, func_or_exc):
+ size_attrs = [
+ dict(ram=8000, disk=9999, vcpus=99, name='s0'),
+ dict(ram=2**16, disk=20, vcpus=99, name='s1'),
+ dict(ram=2**16, disk=9999, vcpus=1, name='s2'),
+ ]
+ fake_sizes = list()
+ for item in size_attrs:
+ fake_sizes.append(
+ get_fake_obj(attributes=item)
+ )
+ base_spec = dict(machine=dict(
+ ram=1,
+ disk=1,
+ cpus=1,
+ ))
+ spec = deepcopy(base_spec)
+ spec['machine'].update(input_attrs)
+ obj = self.get_obj(conf=spec)
+ self.mocks['m_list_sizes'].return_value = fake_sizes
+ if isinstance(func_or_exc, type):
+ with raises(func_or_exc):
+ obj.size
+ else:
+ assert obj.size in fake_sizes
+ assert func_or_exc(obj.size) is True
+
+ @mark.parametrize(
+ "wanted_groups",
+ [
+ ['group1'],
+ ['group0', 'group2'],
+ [],
+ ]
+ )
+ def test_security_groups(self, wanted_groups):
+ group_names = ['group0', 'group1', 'group2']
+ fake_groups = list()
+ for name in group_names:
+ fake_groups.append(
+ get_fake_obj(attributes=dict(name=name))
+ )
+ self.mocks['m_ex_list_security_groups'].return_value = fake_groups
+ obj = self.get_obj()
+ assert obj.security_groups is None
+ obj = self.get_obj()
+ obj.provider.conf['security_groups'] = wanted_groups
+ assert [g.name for g in obj.security_groups] == wanted_groups
+
+ def test_security_groups_exc(self):
+ fake_groups = [
+ get_fake_obj(attributes=dict(name='sg')) for i in range(2)
+ ]
+ obj = self.get_obj()
+ obj.provider.conf['security_groups'] = ['sg']
+ with raises(RuntimeError):
+ obj.security_groups
+ self.mocks['m_ex_list_security_groups'].return_value = fake_groups
+ obj = self.get_obj()
+ obj.provider.conf['security_groups'] = ['sg']
+ with raises(RuntimeError):
+ obj.security_groups
+
+ @mark.parametrize(
+ "ssh_key",
+ [
+ 'my_ssh_key',
+ None,
+ ]
+ )
+ def test_userdata(self, ssh_key):
+ self.mocks['m_get_user_ssh_pubkey'].return_value = ssh_key
+ obj = self.get_obj()
+ userdata = yaml.safe_load(obj.userdata)
+ assert userdata['user'] == obj.user
+ assert userdata['hostname'] == obj.hostname
+ if ssh_key:
+ assert userdata['ssh_authorized_keys'] == [ssh_key]
+ else:
+ assert 'ssh_authorized_keys' not in userdata
+
+ @mark.parametrize(
+ 'wanted_name, should_find, exception',
+ [
+ ('node0', True, None),
+ ('node1', True, None),
+ ('node2', False, RuntimeError),
+ ('node3', False, None),
+ ]
+ )
+ def test_node(self, wanted_name, should_find, exception):
+ node_names = ['node0', 'node1', 'node2', 'node2']
+ fake_nodes = list()
+ for name in node_names:
+ fake_nodes.append(
+ get_fake_obj(attributes=dict(name=name))
+ )
+ self.mocks['m_list_nodes'].return_value = fake_nodes
+ obj = self.get_obj(name=wanted_name)
+ if should_find:
+ assert obj.node.name == wanted_name
+ elif exception:
+ with raises(exception) as excinfo:
+ obj.node
+ assert excinfo.value.message
+ else:
+ assert obj.node is None
+
+ @mark.parametrize(
+ 'networks, security_groups',
+ [
+ ([], []),
+ (['net0'], []),
+ ([], ['sg0']),
+ (['net0'], ['sg0']),
+ ]
+ )
+ def test_create(self, networks, security_groups):
+ node_name = 'node0'
+ fake_sizes = [
+ get_fake_obj(
+ attributes=dict(ram=2**16, disk=9999, vcpus=99, name='s0')),
+ ]
+ fake_security_groups = [
+ get_fake_obj(attributes=dict(name=name))
+ for name in security_groups
+ ]
+ self.mocks['m_ex_list_networks'].return_value = networks
+ self.mocks['m_ex_list_security_groups'].return_value = \
+ fake_security_groups
+ self.mocks['m_list_sizes'].return_value = fake_sizes
+ fake_images = [
+ get_fake_obj(attributes=dict(name='ubuntu-16.04')),
+ ]
+ self.mocks['m_list_images'].return_value = fake_images
+ self.mocks['m_get_user_ssh_pubkey'].return_value = 'ssh_key'
+ fake_node = get_fake_obj(attributes=dict(name=node_name))
+ fake_ips = ['555.123.4.0']
+ self.mocks['m_create_node'].return_value = fake_node
+ self.mocks['m_wait_until_running'].return_value = \
+ [(fake_node, fake_ips)]
+ obj = self.get_obj(name=node_name)
+ obj._networks = networks
+ obj.provider.conf['security_groups'] = security_groups
+ p_wait_for_ready = patch(
+ 'teuthology.provision.cloud.openstack.OpenStackProvisioner'
+ '._wait_for_ready'
+ )
+ with p_wait_for_ready:
+ res = obj.create()
+ assert res is obj.node
+ # Test once again to ensure that if volume creation/attachment fails,
+ # we destroy any remaining volumes and consider the node creation to
+ # have failed as well.
+ del obj._node
+ with p_wait_for_ready:
+ obj.conf['volumes']['count'] = 1
+ obj.provider.driver.create_volume.side_effect = Exception
+ with patch.object(obj, '_destroy_volumes'):
+ assert obj.create() is False
+ obj._destroy_volumes.assert_called_once_with()
+
+ def test_update_dns(self):
+ config.nsupdate_url = 'nsupdate_url'
+ obj = self.get_obj()
+ obj.name = 'x'
+ obj.ips = ['y']
+ obj._update_dns()
+ call_args = self.mocks['m_get'].call_args_list
+ assert len(call_args) == 1
+ url_base, query_string = call_args[0][0][0].split('?')
+ assert url_base == 'nsupdate_url'
+ parsed_query = parse_qs(query_string)
+ assert parsed_query == dict(name=['x'], ip=['y'])
+
+ @mark.parametrize(
+ 'nodes',
+ [[], [Mock()], [Mock(), Mock()]]
+ )
+ def test_destroy(self, nodes):
+ with patch(
+ 'teuthology.provision.cloud.openstack.'
+ 'OpenStackProvisioner._find_nodes'
+ ) as m_find_nodes:
+ m_find_nodes.return_value = nodes
+ obj = self.get_obj()
+ result = obj.destroy()
+ if not all(nodes):
+ assert result is True
+ else:
+ for node in nodes:
+ node.destroy.assert_called_once_with()
+
+ _volume_matrix = (
+ 'count, size, should_succeed',
+ [
+ (1, 10, True),
+ (0, 10, True),
+ (10, 1, True),
+ (1, 10, False),
+ (10, 1, False),
+ ]
+ )
+
+ @mark.parametrize(*_volume_matrix)
+ def test_create_volumes(self, count, size, should_succeed):
+ obj_conf = dict(volumes=dict(count=count, size=size))
+ obj = self.get_obj(conf=obj_conf)
+ node = get_fake_obj()
+ if not should_succeed:
+ obj.provider.driver.create_volume.side_effect = Exception
+ obj._node = node
+ result = obj._create_volumes()
+ assert result is should_succeed
+ if should_succeed:
+ create_calls = obj.provider.driver.create_volume.call_args_list
+ attach_calls = obj.provider.driver.attach_volume.call_args_list
+ assert len(create_calls) == count
+ assert len(attach_calls) == count
+ for i in range(count):
+ vol_size, vol_name = create_calls[i][0]
+ assert vol_size == size
+ assert vol_name == '%s_%s' % (obj.name, i)
+ assert attach_calls[i][0][0] is obj._node
+ assert attach_calls[i][1]['device'] is None
+
+ @mark.parametrize(*_volume_matrix)
+ def test_destroy_volumes(self, count, size, should_succeed):
+ obj_conf = dict(volumes=dict(count=count, size=size))
+ obj = self.get_obj(conf=obj_conf)
+ fake_volumes = list()
+ for i in range(count):
+ vol_name = '%s_%s' % (obj.name, i)
+ fake_volumes.append(
+ get_fake_obj(attributes=dict(name=vol_name))
+ )
+ obj.provider.driver.list_volumes.return_value = fake_volumes
+ obj._destroy_volumes()
+ detach_calls = obj.provider.driver.detach_volume.call_args_list
+ destroy_calls = obj.provider.driver.destroy_volume.call_args_list
+ assert len(detach_calls) == count
+ assert len(destroy_calls) == count
+ assert len(obj.provider.driver.detach_volume.call_args_list) == count
+ assert len(obj.provider.driver.destroy_volume.call_args_list) == count
+ obj.provider.driver.detach_volume.reset_mock()
+ obj.provider.driver.destroy_volume.reset_mock()
+ obj.provider.driver.detach_volume.side_effect = Exception
+ obj.provider.driver.destroy_volume.side_effect = Exception
+ obj._destroy_volumes()
+ assert len(obj.provider.driver.detach_volume.call_args_list) == count
+ assert len(obj.provider.driver.destroy_volume.call_args_list) == count
+
+ def test_destroy_volumes_exc(self):
+ obj = self.get_obj()
+ obj.provider.driver.detach_volume.side_effect = Exception
+
+ def test_wait_for_ready(self):
+ obj = self.get_obj()
+ obj._node = get_fake_obj(attributes=dict(name='node_name'))
+ with patch.multiple(
+ 'teuthology.orchestra.remote.Remote',
+ connect=DEFAULT,
+ run=DEFAULT,
+ ) as mocks:
+ obj._wait_for_ready()
+ mocks['connect'].side_effect = socket.error
+ with raises(MaxWhileTries):
+ obj._wait_for_ready()
--- /dev/null
+libcloud:
+ providers:
+ my_provider:
+ allow_networks:
+ - sesci
+ userdata:
+ 'ubuntu-16.04':
+ bootcmd:
+ - 'SuSEfirewall2 stop || true'
+ - 'service firewalld stop || true'
+ runcmd:
+ - 'uptime'
+ - 'date'
+ - 'zypper in -y lsb-release make gcc gcc-c++ chrony || true'
+ - 'systemctl enable chronyd.service || true'
+ - 'systemctl start chronyd.service || true'
+ ssh_authorized_keys:
+ - user_public_key1
+ - user_public_key2
+ driver: openstack
+ driver_args:
+ username: user
+ password: password
+ ex_force_auth_url: 'http://127.0.0.1:9999/v2.0/tokens'
--- /dev/null
+from mock import Mock, MagicMock, patch
+
+from teuthology import provision
+
+
+class TestDownburst(object):
+ def setup_method(self):
+ self.ctx = Mock()
+ self.ctx.os_type = 'rhel'
+ self.ctx.os_version = '7.0'
+ self.ctx.config = dict()
+ self.name = 'vpm999'
+ self.status = dict(
+ vm_host=dict(name='host999'),
+ is_vm=True,
+ machine_type='mtype',
+ locked_by='user@a',
+ description="desc",
+ )
+
+ def test_create_if_vm_success(self):
+ name = self.name
+ ctx = self.ctx
+ status = self.status
+
+ dbrst = provision.downburst.Downburst(
+ name, ctx.os_type, ctx.os_version, status)
+ dbrst.executable = '/fake/path'
+ dbrst.build_config = MagicMock(name='build_config')
+ dbrst._run_create = MagicMock(name='_run_create')
+ dbrst._run_create.return_value = (0, '', '')
+ remove_config = MagicMock(name='remove_config')
+ dbrst.remove_config = remove_config
+
+ result = provision.create_if_vm(ctx, name, dbrst)
+ assert result is True
+
+ dbrst._run_create.assert_called_with()
+ dbrst.build_config.assert_called_with()
+ del dbrst
+ remove_config.assert_called_with()
+
+ def test_destroy_if_vm_success(self):
+ name = self.name
+ ctx = self.ctx
+ status = self.status
+
+ dbrst = provision.downburst.Downburst(
+ name, ctx.os_type, ctx.os_version, status)
+ dbrst.destroy = MagicMock(name='destroy')
+ dbrst.destroy.return_value = True
+
+ result = provision.destroy_if_vm(name, user="user@a", _downburst=dbrst)
+ assert result is True
+
+ dbrst.destroy.assert_called_with()
+
+ def test_destroy_if_vm_wrong_owner(self):
+ name = self.name
+ ctx = self.ctx
+ status = self.status
+
+ dbrst = provision.downburst.Downburst(
+ name, ctx.os_type, ctx.os_version, status)
+ dbrst.destroy = MagicMock(name='destroy', side_effect=RuntimeError)
+
+ result = provision.destroy_if_vm(name, user='user@b',
+ _downburst=dbrst)
+ assert result is False
+
+ def test_destroy_if_vm_wrong_description(self):
+ name = self.name
+ ctx = self.ctx
+ status = self.status
+
+ dbrst = provision.downburst.Downburst(
+ name, ctx.os_type, ctx.os_version, status)
+ dbrst.destroy = MagicMock(name='destroy')
+ dbrst.destroy = MagicMock(name='destroy', side_effect=RuntimeError)
+
+ result = provision.destroy_if_vm(name, description='desc_b',
+ _downburst=dbrst)
+ assert result is False
+
+ @patch('teuthology.provision.downburst.downburst_executable')
+ def test_create_fails_without_executable(self, m_exec):
+ name = self.name
+ ctx = self.ctx
+ status = self.status
+ m_exec.return_value = ''
+ dbrst = provision.downburst.Downburst(
+ name, ctx.os_type, ctx.os_version, status)
+ result = dbrst.create()
+ assert result is False
+
+ @patch('teuthology.provision.downburst.downburst_executable')
+ def test_destroy_fails_without_executable(self, m_exec):
+ name = self.name
+ ctx = self.ctx
+ status = self.status
+ m_exec.return_value = ''
+ dbrst = provision.downburst.Downburst(
+ name, ctx.os_type, ctx.os_version, status)
+ result = dbrst.destroy()
+ assert result is False
--- /dev/null
+import datetime
+
+from copy import deepcopy
+from mock import patch, DEFAULT, PropertyMock
+from pytest import raises, mark
+
+from teuthology.config import config
+from teuthology.exceptions import MaxWhileTries, CommandFailedError
+from teuthology.provision import fog
+
+
+test_config = dict(
+ fog=dict(
+ endpoint='http://fog.example.com/fog',
+ api_token='API_TOKEN',
+ user_token='USER_TOKEN',
+ machine_types='type1,type2',
+ ),
+ fog_reimage_timeout=1800,
+)
+
+
+class TestFOG(object):
+ klass = fog.FOG
+
+ def setup_method(self):
+ config.load()
+ config.update(deepcopy(test_config))
+ self.start_patchers()
+
+ def start_patchers(self):
+ self.patchers = dict()
+ self.patchers['m_sleep'] = patch(
+ 'time.sleep',
+ )
+ self.patchers['m_requests_Session_send'] = patch(
+ 'requests.Session.send',
+ )
+ self.patchers['m_Remote_connect'] = patch(
+ 'teuthology.orchestra.remote.Remote.connect'
+ )
+ self.patchers['m_Remote_run'] = patch(
+ 'teuthology.orchestra.remote.Remote.run'
+ )
+ self.patchers['m_Remote_console'] = patch(
+ 'teuthology.orchestra.remote.Remote.console',
+ new_callable=PropertyMock,
+ )
+ self.patchers['m_Remote_hostname'] = patch(
+ 'teuthology.orchestra.remote.Remote.hostname',
+ new_callable=PropertyMock,
+ )
+ self.patchers['m_Remote_machine_type'] = patch(
+ 'teuthology.orchestra.remote.Remote.machine_type',
+ new_callable=PropertyMock,
+ )
+ self.mocks = dict()
+ for name, patcher in self.patchers.items():
+ self.mocks[name] = patcher.start()
+
+ def teardown_method(self):
+ for patcher in self.patchers.values():
+ patcher.stop()
+
+ @mark.parametrize('enabled', [True, False])
+ def test_get_types(self, enabled):
+ with patch('teuthology.provision.fog.enabled') as m_enabled:
+ m_enabled.return_value = enabled
+ types = fog.get_types()
+ if enabled:
+ assert types == test_config['fog']['machine_types'].split(',')
+ else:
+ assert types == []
+
+ def test_disabled(self):
+ config.fog['endpoint'] = None
+ obj = self.klass('name.fqdn', 'type', '1.0')
+ with raises(RuntimeError):
+ obj.create()
+
+ def test_init(self):
+ self.mocks['m_Remote_hostname'].return_value = 'name.fqdn'
+ obj = self.klass('name.fqdn', 'type', '1.0')
+ assert obj.name == 'name.fqdn'
+ assert obj.shortname == 'name'
+ assert obj.os_type == 'type'
+ assert obj.os_version == '1.0'
+
+ @mark.parametrize('success', [True, False])
+ def test_create(self, success):
+ self.mocks['m_Remote_hostname'].return_value = 'name.fqdn'
+ self.mocks['m_Remote_machine_type'].return_value = 'type1'
+ obj = self.klass('name.fqdn', 'type', '1.0')
+ host_id = 99
+ task_id = 1234
+ with patch.multiple(
+ 'teuthology.provision.fog.FOG',
+ get_host_data=DEFAULT,
+ set_image=DEFAULT,
+ schedule_deploy_task=DEFAULT,
+ wait_for_deploy_task=DEFAULT,
+ cancel_deploy_task=DEFAULT,
+ _wait_for_ready=DEFAULT,
+ _fix_hostname=DEFAULT,
+ _verify_installed_os=DEFAULT,
+ ) as local_mocks:
+ local_mocks['get_host_data'].return_value = dict(id=host_id)
+ local_mocks['schedule_deploy_task'].return_value = task_id
+ if not success:
+ local_mocks['wait_for_deploy_task'].side_effect = RuntimeError
+ with raises(RuntimeError):
+ obj.create()
+ else:
+ obj.create()
+ local_mocks['get_host_data'].assert_called_once_with()
+ local_mocks['set_image'].assert_called_once_with(host_id)
+ local_mocks['schedule_deploy_task'].assert_called_once_with(host_id)
+ local_mocks['wait_for_deploy_task'].assert_called_once_with(task_id)
+ if success:
+ local_mocks['_wait_for_ready'].assert_called_once_with()
+ local_mocks['_fix_hostname'].assert_called_once_with()
+ else:
+ assert len(local_mocks['cancel_deploy_task'].call_args_list) == 1
+ self.mocks['m_Remote_console'].return_value.power_off.assert_called_once_with()
+ self.mocks['m_Remote_console'].return_value.power_on.assert_called_once_with()
+
+ def test_do_request(self):
+ obj = self.klass('name.fqdn', 'type', '1.0')
+ obj.do_request('test_url', data='DATA', method='GET')
+ assert len(self.mocks['m_requests_Session_send'].call_args_list) == 1
+ req = self.mocks['m_requests_Session_send'].call_args_list[0][0][0]
+ assert req.url == test_config['fog']['endpoint'] + 'test_url'
+ assert req.method == 'GET'
+ assert req.headers['fog-api-token'] == test_config['fog']['api_token']
+ assert req.headers['fog-user-token'] == test_config['fog']['user_token']
+ assert req.body == 'DATA'
+
+ @mark.parametrize(
+ 'count',
+ [0, 1, 2],
+ )
+ def test_get_host_data(self, count):
+ host_objs = [dict(id=i) for i in range(count)]
+ resp_obj = dict(count=count, hosts=host_objs)
+ self.mocks['m_requests_Session_send']\
+ .return_value.json.return_value = resp_obj
+ obj = self.klass('name.fqdn', 'type', '1.0')
+ if count != 1:
+ with raises(RuntimeError):
+ result = obj.get_host_data()
+ return
+ result = obj.get_host_data()
+ assert len(self.mocks['m_requests_Session_send'].call_args_list) == 1
+ req = self.mocks['m_requests_Session_send'].call_args_list[0][0][0]
+ assert req.url == test_config['fog']['endpoint'] + '/host'
+ assert req.body == '{"name": "name"}'
+ assert result == host_objs[0]
+
+ @mark.parametrize(
+ 'count',
+ [0, 1, 2],
+ )
+ def test_get_image_data(self, count):
+ img_objs = [dict(id=i) for i in range(count)]
+ resp_obj = dict(count=count, images=img_objs)
+ self.mocks['m_requests_Session_send']\
+ .return_value.json.return_value = resp_obj
+ self.mocks['m_Remote_machine_type'].return_value = 'type1'
+ obj = self.klass('name.fqdn', 'windows', 'xp')
+ if count < 1:
+ with raises(RuntimeError):
+ result = obj.get_image_data()
+ return
+ result = obj.get_image_data()
+ assert len(self.mocks['m_requests_Session_send'].call_args_list) == 1
+ req = self.mocks['m_requests_Session_send'].call_args_list[0][0][0]
+ assert req.url == test_config['fog']['endpoint'] + '/image'
+ assert req.body == '{"name": "type1_windows_xp"}'
+ assert result == img_objs[0]
+
+ def test_suggest_image_names(self):
+ data = {'images': [
+ {'name': 'mira_rhel_9.1'},
+ {'name': 'mira_rhel_9.2'},
+ ]}
+ self.mocks['m_requests_Session_send']\
+ .return_value.json.return_value = data
+ self.mocks['m_Remote_machine_type'].return_value = 'mira'
+ # Not sure what this klass() is for here:
+ obj = self.klass('name.fqdn', 'mira', '1.0')
+ result = obj.suggest_image_names()
+ assert len(self.mocks['m_requests_Session_send'].call_args_list) == 1
+ req = self.mocks['m_requests_Session_send'].call_args_list[0][0][0]
+ assert req.url == test_config['fog']['endpoint'] + '/image/search/mira'
+ assert result == ['mira_rhel_9.1', 'mira_rhel_9.2']
+
+ def test_set_image(self):
+ self.mocks['m_Remote_hostname'].return_value = 'name.fqdn'
+ self.mocks['m_Remote_machine_type'].return_value = 'type1'
+ host_id = 999
+ obj = self.klass('name.fqdn', 'type', '1.0')
+ with patch.multiple(
+ 'teuthology.provision.fog.FOG',
+ get_image_data=DEFAULT,
+ do_request=DEFAULT,
+ ) as local_mocks:
+ local_mocks['get_image_data'].return_value = dict(id='13')
+ obj.set_image(host_id)
+ local_mocks['do_request'].assert_called_once_with(
+ '/host/999', method='PUT', data='{"imageID": 13}',
+ )
+
+ def test_schedule_deploy_task(self):
+ host_id = 12
+ tasktype_id = 6
+ task_id = 5
+ tasktype_result = dict(tasktypes=[dict(name='deploy', id=tasktype_id)])
+ schedule_result = dict()
+ host_tasks = [dict(
+ createdTime=datetime.datetime.strftime(
+ datetime.datetime.now(datetime.timezone.utc),
+ self.klass.timestamp_format
+ ),
+ id=task_id,
+ )]
+ self.mocks['m_requests_Session_send']\
+ .return_value.json.side_effect = [
+ tasktype_result, schedule_result,
+ ]
+ with patch.multiple(
+ 'teuthology.provision.fog.FOG',
+ get_deploy_tasks=DEFAULT,
+ ) as local_mocks:
+ local_mocks['get_deploy_tasks'].return_value = host_tasks
+ obj = self.klass('name.fqdn', 'type', '1.0')
+ result = obj.schedule_deploy_task(host_id)
+ assert len(local_mocks['get_deploy_tasks'].call_args_list) == 2
+ assert len(self.mocks['m_requests_Session_send'].call_args_list) == 3
+ assert result == task_id
+
+ def test_get_deploy_tasks(self):
+ obj = self.klass('name.fqdn', 'type', '1.0')
+ resp_obj = dict(
+ count=2,
+ tasks=[
+ dict(host=dict(name='notme')),
+ dict(host=dict(name='name')),
+ ]
+ )
+ self.mocks['m_requests_Session_send']\
+ .return_value.json.return_value = resp_obj
+ result = obj.get_deploy_tasks()
+ assert result[0]['host']['name'] == 'name'
+
+ @mark.parametrize(
+ 'active_ids',
+ [
+ [2, 4, 6, 8],
+ [1],
+ [],
+ ]
+ )
+ def test_deploy_task_active(self, active_ids):
+ our_task_id = 4
+ result_objs = [dict(id=task_id) for task_id in active_ids]
+ obj = self.klass('name.fqdn', 'type', '1.0')
+ with patch.multiple(
+ 'teuthology.provision.fog.FOG',
+ get_deploy_tasks=DEFAULT,
+ ) as local_mocks:
+ local_mocks['get_deploy_tasks'].return_value = result_objs
+ result = obj.deploy_task_active(our_task_id)
+ assert result is (our_task_id in active_ids)
+
+ @mark.parametrize(
+ 'tries',
+ [3, 121],
+ )
+ def test_wait_for_deploy_task(self, tries):
+ wait_results = [True for i in range(tries)] + [False]
+ obj = self.klass('name.fqdn', 'type', '1.0')
+ with patch.multiple(
+ 'teuthology.provision.fog.FOG',
+ deploy_task_active=DEFAULT,
+ ) as local_mocks:
+ local_mocks['deploy_task_active'].side_effect = wait_results
+ if tries >= 60:
+ with raises(MaxWhileTries):
+ obj.wait_for_deploy_task(9)
+ return
+ obj.wait_for_deploy_task(9)
+ assert len(local_mocks['deploy_task_active'].call_args_list) == \
+ tries + 1
+
+ def test_cancel_deploy_task(self):
+ obj = self.klass('name.fqdn', 'type', '1.0')
+ with patch.multiple(
+ 'teuthology.provision.fog.FOG',
+ do_request=DEFAULT,
+ ) as local_mocks:
+ obj.cancel_deploy_task(10)
+ local_mocks['do_request'].assert_called_once_with(
+ '/task/cancel',
+ method='DELETE',
+ data='{"id": 10}',
+ )
+
+ @mark.parametrize(
+ 'tries',
+ [1, 101],
+ )
+ def test_wait_for_ready_tries(self, tries):
+ connect_results = [MaxWhileTries for i in range(tries)] + [True]
+ obj = self.klass('name.fqdn', 'type', '1.0')
+ self.mocks['m_Remote_connect'].side_effect = connect_results
+ if tries >= 100:
+ with raises(MaxWhileTries):
+ obj._wait_for_ready()
+ return
+ obj._wait_for_ready()
+ assert len(self.mocks['m_Remote_connect'].call_args_list) == tries + 1
+
+ @mark.parametrize(
+ 'sentinel_present',
+ ([False, True]),
+ )
+ def test_wait_for_ready_sentinel(self, sentinel_present):
+ config.fog['sentinel_file'] = '/a_file'
+ obj = self.klass('name.fqdn', 'type', '1.0')
+ if not sentinel_present:
+ self.mocks['m_Remote_run'].side_effect = [
+ CommandFailedError(command='cmd', exitstatus=1)]
+ with raises(CommandFailedError):
+ obj._wait_for_ready()
+ else:
+ obj._wait_for_ready()
+ assert len(self.mocks['m_Remote_run'].call_args_list) == 1
+ assert "'/a_file'" in \
+ self.mocks['m_Remote_run'].call_args_list[0][1]['args']
--- /dev/null
+from copy import deepcopy
+from pytest import raises
+from teuthology.config import config
+
+import teuthology.provision
+
+test_config = dict(
+ pelagos=dict(
+ endpoint='http://pelagos.example:5000/',
+ machine_types='ptype1,ptype2,common_type',
+ ),
+ fog=dict(
+ endpoint='http://fog.example.com/fog',
+ api_token='API_TOKEN',
+ user_token='USER_TOKEN',
+ machine_types='ftype1,ftype2,common_type',
+ )
+)
+
+class TestInitProvision(object):
+
+ def setup_method(self):
+ config.load(deepcopy(test_config))
+
+ def test_get_reimage_types(self):
+ reimage_types = teuthology.provision.get_reimage_types()
+ assert reimage_types == ["ptype1", "ptype2", "common_type",
+ "ftype1", "ftype2", "common_type"]
+
+ def test_reimage(self):
+ class context:
+ pass
+ ctx = context()
+ ctx.os_type = 'sle'
+ ctx.os_version = '15.1'
+ with raises(Exception) as e_info:
+ teuthology.provision.reimage(ctx, 'f.q.d.n.org', 'not-defined-type')
+ e_str = str(e_info)
+ print("Caught exception: " + e_str)
+ assert e_str.find(r"configured\sprovisioners") == -1
+
+ with raises(Exception) as e_info:
+ teuthology.provision.reimage(ctx, 'f.q.d.n.org', 'common_type')
+ e_str = str(e_info)
+ print("Caught exception: " + e_str)
+ assert e_str.find(r"used\swith\sone\sprovisioner\sonly") == -1
--- /dev/null
+from copy import deepcopy
+from mock import patch, DEFAULT, PropertyMock
+from pytest import raises, mark
+from requests.models import Response
+
+from teuthology.config import config
+from teuthology.orchestra.opsys import OS
+from teuthology.provision import maas
+
+
+test_config = dict(
+ maas=dict(
+ api_url="http://maas.example.com:5240/MAAS/api/2.0/",
+ api_key="CONSUMER_KEY:ACCESS_TOKEN:SECRET",
+ machine_types=["typeA", "typeB"],
+ timeout=900,
+ user_data="teuthology/maas/maas-{os_type}-{os_version}-user-data.txt"
+ )
+)
+
+class TestMaas(object):
+ klass = maas.MAAS
+
+ def setup_method(self):
+ config.load()
+ config.update(deepcopy(test_config))
+ self.start_patchers()
+
+ def start_patchers(self):
+ self.patchers = dict()
+ self.patchers["m_Remote_hostname"] = patch(
+ "teuthology.orchestra.remote.Remote.hostname",
+ new_callable=PropertyMock,
+ )
+ self.patchers["m_Remote_machine_os"] = patch(
+ "teuthology.orchestra.remote.Remote.os",
+ new_callable=PropertyMock,
+ )
+ self.patchers["m_Remote_opsys_os"] = patch(
+ "teuthology.orchestra.opsys.OS",
+ new_callable=PropertyMock,
+ )
+
+ self.mocks = dict()
+ for name, patcher in self.patchers.items():
+ self.mocks[name] = patcher.start()
+
+ def teardown_method(self):
+ for patcher in self.patchers.values():
+ patcher.stop()
+
+ def _get_mock_response(self, status_code=200, content=None, headers=None):
+ response = Response()
+ response.status_code = status_code
+ response._content = content
+ if headers is not None:
+ response.headers = headers
+ return response
+
+ def test_api_url_missing(self):
+ config.maas["api_url"] = None
+ with raises(RuntimeError):
+ self.klass("name.fqdn")
+
+ def test_api_key_missing(self):
+ config.maas["api_key"] = None
+ with raises(RuntimeError):
+ self.klass("name.fqdn")
+
+ @mark.parametrize("enabled", [True, False])
+ def test_get_types(self, enabled):
+ with patch("teuthology.provision.maas.enabled") as m_enabled:
+ m_enabled.return_value = enabled
+ types = maas.get_types()
+
+ if enabled:
+ assert types == test_config["maas"]["machine_types"]
+ else:
+ assert types == []
+
+ @mark.parametrize("status_name, osystem, distro_series", [
+ ("ready", "ubuntu", "jammy"),
+ ("deployed", "ubuntu", "jammy"),
+ ])
+ def test_init(self, status_name, osystem, distro_series):
+ self.mocks["m_Remote_hostname"].return_value = "name.fqdn"
+ with patch(
+ "teuthology.provision.maas.MAAS.get_machines_data"
+ ) as maas_machine:
+ maas_machine.return_value = {
+ "status_name": status_name,
+ "system_id": "abc123",
+ "osystem": osystem,
+ "distro_series": distro_series,
+ "architecture": "amd64/generic"
+ }
+ if not (osystem and distro_series):
+ with raises(RuntimeError):
+ obj = self.klass(
+ name="name.fqdn",
+ os_type=osystem,
+ os_version=distro_series
+ )
+ else:
+ obj = self.klass(
+ name="name.fqdn",
+ os_type=osystem,
+ os_version=distro_series
+ )
+ assert obj.name == "name.fqdn"
+ assert obj.shortname == "name"
+ assert obj.os_type == osystem
+ assert obj.os_version == distro_series
+ assert obj.system_id == "abc123"
+
+ @mark.parametrize("os_name, os_version, codename", [
+ ("ubuntu", "24.04", "noble"), ("centos", "9.stream", "centos9-stream"),
+ ])
+ def test_get_image_data(self, os_name, os_version, codename):
+ with patch.multiple(
+ "teuthology.provision.maas.MAAS",
+ do_request=DEFAULT,
+ get_machines_data=DEFAULT,
+ ) as local_mocks:
+ local_mocks["do_request"].return_value = self._get_mock_response(
+ content=b'[{"name": "%s/%s", "architecture": "amd64/generic"}]' % (
+ os_name.encode(), codename.encode()
+ )
+ )
+ local_mocks["get_machines_data"].return_value = {
+ "status_name": "ready",
+ "architecture": "amd64/generic",
+ }
+ obj = self.klass(
+ name="name.fqdn", os_type=os_name, os_version=os_version
+ )
+ assert obj.get_image_data() == {
+ "name": f"{os_name}/{codename}",
+ "architecture": "amd64/generic",
+ }
+
+ def test_lock_machine(self):
+ with patch.multiple(
+ "teuthology.provision.maas.MAAS",
+ do_request=DEFAULT,
+ get_machines_data=DEFAULT,
+ ) as local_mocks:
+ local_mocks["get_machines_data"].return_value = {
+ "system_id": "1234abc",
+ "architecture": "amd64/generic",
+ }
+ local_mocks["do_request"].return_value = self._get_mock_response(
+ content=b'{"locked": "true"}'
+ )
+ assert self.klass(name="name.fqdn").lock_machine() is None
+
+ def test_unlock_machine(self):
+ self.mocks["m_Remote_hostname"].return_value = "name.fqdn"
+ with patch.multiple(
+ "teuthology.provision.maas.MAAS",
+ do_request=DEFAULT,
+ get_machines_data=DEFAULT,
+ ) as local_mocks:
+ local_mocks["get_machines_data"].return_value = {
+ "system_id": "1234abc",
+ "architecture": "amd64/generic",
+ }
+ local_mocks["do_request"].return_value = self._get_mock_response(
+ content=b'{"locked": "true"}'
+ )
+ with raises(RuntimeError):
+ self.klass(name="name.fqdn").unlock_machine()
+
+ @mark.parametrize("type, status_name", [
+ ("uploaded", ""), ("synced", "deploying"), ("synced", "ready"),
+ ])
+ def test_deploy_machine(self, type, status_name):
+ with patch.multiple(
+ "teuthology.provision.maas.MAAS",
+ get_image_data=DEFAULT,
+ _get_user_data=DEFAULT,
+ get_machines_data=DEFAULT,
+ do_request=DEFAULT,
+ ) as local_mocks:
+ local_mocks["get_machines_data"].return_value = {
+ "system_id": "1234abc",
+ "architecture": "amd64/generic",
+ }
+ local_mocks["get_image_data"].return_value = {
+ "name": "ubuntu/noble", "type": type
+ }
+ local_mocks["_get_user_data"].return_value = "init-host-data"
+ local_mocks["do_request"].return_value = self._get_mock_response(
+ content=b'{"status_name": "%s"}' % status_name.encode()
+ )
+ obj = self.klass(name="name.fqdn")
+ if (status_name != "deploying" or type != "synced"):
+ with raises(RuntimeError):
+ obj.deploy_machine()
+ else:
+ assert obj.deploy_machine() is None
+
+ @mark.parametrize("response", [b'{}', ])
+ def test_release_machine(self, response):
+ with patch.multiple(
+ "teuthology.provision.maas.MAAS",
+ get_machines_data=DEFAULT,
+ do_request=DEFAULT,
+ ) as local_mocks:
+ local_mocks["get_machines_data"].return_value = {
+ "system_id": "1234abc",
+ "architecture": "amd64/generic",
+ "status_name": "ready"
+ }
+ local_mocks["do_request"].return_value = self._get_mock_response(
+ content=b'{"status_name": "disk erasing"}'
+ )
+ assert self.klass(name="name.fqdn").release_machine() is None
+
+ @mark.parametrize("status_name", ["deploying", "ready"])
+ def test_abort_deploy(self, status_name):
+ with patch.multiple(
+ "teuthology.provision.maas.MAAS",
+ get_machines_data=DEFAULT,
+ do_request=DEFAULT,
+ ) as local_mocks:
+ local_mocks["get_machines_data"].return_value = {
+ "system_id": "1234abc",
+ "architecture": "amd64/generic",
+ }
+ if status_name == "deploying":
+ local_mocks["do_request"].return_value = self._get_mock_response(
+ content=b'{"status_name": "allocated"}'
+ )
+
+ assert self.klass(name="name.fqdn").abort_deploy() is None
+
+ @mark.parametrize("lock_status, machine_status", [
+ (False, "ready"), (False, "allocated"),
+ (True, "deployed"), (False, "deployed"),
+ (False, "releasing"),
+ ])
+ def test_release(self, lock_status, machine_status):
+ with patch.multiple(
+ "teuthology.provision.maas.MAAS",
+ get_machines_data=DEFAULT,
+ unlock_machine=DEFAULT,
+ release_machine=DEFAULT,
+ _wait_for_status=DEFAULT,
+ ) as local_mocks:
+ local_mocks["get_machines_data"].return_value = {
+ "status_name": machine_status,
+ "locked": lock_status,
+ "system_id": "1234abc",
+ "architecture": "amd64/generic",
+ }
+ obj = self.klass(name="name.fqdn")
+ if machine_status in ["ready", "allocated"]:
+ obj.release()
+
+ elif machine_status == "deployed":
+ local_mocks["unlock_machine"].return_value = None
+ local_mocks["release_machine"].return_value = None
+ local_mocks["_wait_for_status"].return_value = None
+ obj.release()
+
+ else:
+ with raises(RuntimeError):
+ obj.release()
+
+ @mark.parametrize("user_data", [True, False])
+ def test_get_user_data(self, user_data):
+ with patch(
+ "teuthology.provision.maas.MAAS.get_machines_data"
+ ) as maas_machine:
+ maas_machine.return_value = {
+ "system_id": "1234abc",
+ "architecture": "amd64/generic",
+ }
+ obj = self.klass(name="name.fqdn")
+ if user_data:
+ assert user_data is not None
+ else:
+ config.maas["user_data"] = None
+ user_data = obj._get_user_data()
+ assert user_data is None
+
+ @mark.parametrize(
+ "os_type, os_version, remote_os_type, remote_os_version", [
+ ("centos", "8", "centos", "8"),
+ ("", "", "ubuntu", "22.04")
+ ])
+ def test_verify_installed_os(
+ self, os_type, os_version, remote_os_type, remote_os_version
+ ):
+ self.mocks["m_Remote_machine_os"].return_value = OS(
+ name=remote_os_type, version=remote_os_version
+ )
+ with patch(
+ "teuthology.provision.maas.MAAS.get_machines_data"
+ ) as maas_machine:
+ maas_machine.return_value = {
+ "system_id": "1234abc",
+ "architecture": "amd64/generic"
+ }
+ if os_type and os_version:
+ self.klass(
+ name="name.fqdn", os_type=os_type, os_version=os_version
+ )._verify_installed_os()
+ else:
+ self.klass(name="name.fqdn")._verify_installed_os()
--- /dev/null
+from copy import deepcopy
+from pytest import raises
+from teuthology.config import config
+from teuthology.provision import pelagos
+
+import teuthology.provision
+
+
+test_config = dict(
+ pelagos=dict(
+ endpoint='http://pelagos.example:5000/',
+ machine_types='ptype1,ptype2',
+ ),
+)
+
+class TestPelagos(object):
+
+ def setup_method(self):
+ config.load(deepcopy(test_config))
+
+ def teardown_method(self):
+ pass
+
+ def test_get_types(self):
+ #klass = pelagos.Pelagos
+ types = pelagos.get_types()
+ assert types == ["ptype1", "ptype2"]
+
+ def test_disabled(self):
+ config.pelagos['endpoint'] = None
+ enabled = pelagos.enabled()
+ assert enabled == False
+
+ def test_pelagos(self):
+ class context:
+ pass
+
+ ctx = context()
+ ctx.os_type ='sle'
+ ctx.os_version = '15.1'
+ with raises(Exception) as e_info:
+ teuthology.provision.reimage(ctx, 'f.q.d.n.org', 'ptype1')
+ e_str = str(e_info)
+ print("Caught exception: " + e_str)
+ assert e_str.find(r"Name\sor\sservice\snot\sknown") == -1
+
--- /dev/null
+import sys
+import pytest
+from importlib import import_module
+
+
+class Script(object):
+ script_name = "teuthology"
+ script_module = None # Override in subclasses, e.g., 'scripts.run'
+
+ @pytest.fixture(scope="class")
+ def module_name(self) -> str:
+ # e.g., 'teuthology-dispatcher' -> 'scripts.dispatcher'
+ return self.script_name.replace("teuthology-", "").replace("teuthology", "run")
+
+ @pytest.fixture(scope="class")
+ def module(self, module_name):
+ return import_module(self.script_module or f"scripts.{module_name}")
+
+ def test_help(self, capsys: pytest.CaptureFixture[str], module):
+ # docopt
+ if module.__doc__ and "usage" in module.__doc__.lower():
+ return
+ if hasattr(module, "doc") and module.doc and "usage" in module.doc.lower():
+ return
+ # argparse
+ if hasattr(module, "parse_args"):
+ with pytest.raises(SystemExit):
+ module.parse_args([])
+ captured = capsys.readouterr()
+ assert "usage: " in captured.err
+ return
+ # If neither, fail
+ raise AssertionError(
+ f"{self.script_name} has neither a docstring/doc variable with usage info "
+ f"nor a parse_args function"
+ )
+
+ def test_invalid(self, module):
+ original_argv = sys.argv
+ try:
+ sys.argv = [self.script_name, "--invalid-option"]
+ with pytest.raises(SystemExit):
+ if hasattr(module, "parse_args"):
+ module.parse_args(sys.argv[1:])
+ elif hasattr(module, "main"):
+ # For docopt-based scripts, main() will call docopt which exits on error
+ module.main()
+ else:
+ raise NotImplementedError(
+ f"Don't know how to test {self.script_name}"
+ )
+ finally:
+ sys.argv = original_argv
--- /dev/null
+from script import Script
+
+
+class TestDispatcher(Script):
+ script_name = 'teuthology-dispatcher'
+ script_module = 'scripts.dispatcher'
--- /dev/null
+from script import Script
+
+
+class TestExporter(Script):
+ script_name = 'teuthology-exporter'
+ script_module = 'scripts.exporter'
--- /dev/null
+from script import Script
+
+
+class TestLock(Script):
+ script_name = 'teuthology-lock'
+ script_module = 'scripts.lock'
--- /dev/null
+import docopt
+
+from script import Script
+from scripts import ls
+
+doc = ls.__doc__
+
+
+class TestLs(Script):
+ script_name = 'teuthology-ls'
+ script_module = 'scripts.ls'
+
+ def test_args(self):
+ args = docopt.docopt(doc, ["--verbose", "some/archive/dir"])
+ assert args["--verbose"]
+ assert args["<archive_dir>"] == "some/archive/dir"
--- /dev/null
+from script import Script
+
+
+class TestPruneLogs(Script):
+ script_name = 'teuthology-prune-logs'
+ script_module = 'scripts.prune_logs'
--- /dev/null
+from script import Script
+
+
+class TestReport(Script):
+ script_name = 'teuthology-report'
+ script_module = 'scripts.report'
--- /dev/null
+from script import Script
+
+
+class TestResults(Script):
+ script_name = 'teuthology-results'
+ script_module = 'scripts.results'
--- /dev/null
+import docopt
+
+from script import Script
+from scripts import run
+
+doc = run.__doc__
+
+
+class TestRun(Script):
+ script_name = 'teuthology'
+ script_module = 'scripts.run'
+
+ def test_all_args(self):
+ args = docopt.docopt(doc, [
+ "--verbose",
+ "--archive", "some/archive/dir",
+ "--description", "the_description",
+ "--owner", "the_owner",
+ "--lock",
+ "--machine-type", "machine_type",
+ "--os-type", "os_type",
+ "--os-version", "os_version",
+ "--block",
+ "--name", "the_name",
+ "--suite-path", "some/suite/dir",
+ "path/to/config.yml",
+ ])
+ assert args["--verbose"]
+ assert args["--archive"] == "some/archive/dir"
+ assert args["--description"] == "the_description"
+ assert args["--owner"] == "the_owner"
+ assert args["--lock"]
+ assert args["--machine-type"] == "machine_type"
+ assert args["--os-type"] == "os_type"
+ assert args["--os-version"] == "os_version"
+ assert args["--block"]
+ assert args["--name"] == "the_name"
+ assert args["--suite-path"] == "some/suite/dir"
+ assert args["<config>"] == ["path/to/config.yml"]
+
+ def test_multiple_configs(self):
+ args = docopt.docopt(doc, [
+ "config1.yml",
+ "config2.yml",
+ ])
+ assert args["<config>"] == ["config1.yml", "config2.yml"]
--- /dev/null
+from script import Script
+
+
+class TestSchedule(Script):
+ script_name = 'teuthology-schedule'
+ script_module = 'scripts.schedule'
--- /dev/null
+from script import Script
+
+
+class TestSuite(Script):
+ script_name = 'teuthology-suite'
+ script_module = 'scripts.suite'
--- /dev/null
+from script import Script
+
+
+class TestSupervisor(Script):
+ script_name = 'teuthology-supervisor'
+ script_module = 'scripts.supervisor'
--- /dev/null
+from script import Script
+import docopt
+from pytest import raises
+from pytest import skip
+from scripts import updatekeys
+
+
+class TestUpdatekeys(Script):
+ script_name = 'teuthology-updatekeys'
+ script_module = 'scripts.updatekeys'
+
+ def test_invalid(self):
+ skip("teuthology.lock needs to be partially refactored to allow" +
+ "teuthology-updatekeys to return nonzero in all erorr cases")
+
+ def test_all_and_targets(self):
+ with raises(docopt.DocoptExit):
+ docopt.docopt(updatekeys.doc, ['-a', '-t', 'foo'])
+
+ def test_no_args(self):
+ with raises(docopt.DocoptExit):
+ docopt.docopt(updatekeys.doc, [])
--- /dev/null
+from teuthology.config import config
+
+def pytest_runtest_setup():
+ config.load({})
--- /dev/null
+roles:
+- - mon.a
+ - osd.0
+tasks:
+- exec:
+ mon.a:
+ - echo "Well done !"
--- /dev/null
+import os
+import random
+
+from mock import patch, MagicMock
+
+from teuthology.suite import build_matrix
+from tests.fake_fs import make_fake_fstools
+
+
+class TestBuildMatrixSimple(object):
+ def test_combine_path(self):
+ result = build_matrix.combine_path("/path/to/left", "right/side")
+ assert result == "/path/to/left/right/side"
+
+ def test_combine_path_no_right(self):
+ result = build_matrix.combine_path("/path/to/left", None)
+ assert result == "/path/to/left"
+
+
+class TestBuildMatrix(object):
+
+ patchpoints = [
+ 'os.path.exists',
+ 'os.listdir',
+ 'os.path.isfile',
+ 'os.path.isdir',
+ 'builtins.open',
+ ]
+
+ def setup_method(self):
+ self.mocks = dict()
+ self.patchers = dict()
+ for ppoint in self.__class__.patchpoints:
+ self.mocks[ppoint] = MagicMock()
+ self.patchers[ppoint] = patch(ppoint, self.mocks[ppoint])
+
+ def start_patchers(self, fake_fs):
+ fake_fns = make_fake_fstools(fake_fs)
+ # N.B.: relies on fake_fns being in same order as patchpoints
+ for ppoint, fn in zip(self.__class__.patchpoints, fake_fns):
+ self.mocks[ppoint].side_effect = fn
+ self.patchers[ppoint].start()
+
+ def stop_patchers(self):
+ for patcher in self.patchers.values():
+ patcher.stop()
+
+ def teardown_method(self):
+ self.patchers.clear()
+ self.mocks.clear()
+
+ def fragment_occurences(self, jobs, fragment):
+ # What fraction of jobs contain fragment?
+ count = 0
+ for (description, fragment_list) in jobs:
+ for item in fragment_list:
+ if item.endswith(fragment):
+ count += 1
+ return count / float(len(jobs))
+
+ def test_concatenate_1x2x3(self):
+ fake_fs = {
+ 'd0_0': {
+ '+': None,
+ 'd1_0': {
+ 'd1_0_0.yaml': None,
+ },
+ 'd1_1': {
+ 'd1_1_0.yaml': None,
+ 'd1_1_1.yaml': None,
+ },
+ 'd1_2': {
+ 'd1_2_0.yaml': None,
+ 'd1_2_1.yaml': None,
+ 'd1_2_2.yaml': None,
+ },
+ },
+ }
+ self.start_patchers(fake_fs)
+ try:
+ result = build_matrix.build_matrix('d0_0')
+ finally:
+ self.stop_patchers()
+ assert len(result) == 1
+
+ def test_convolve_2x2(self):
+ fake_fs = {
+ 'd0_0': {
+ '%': None,
+ 'd1_0': {
+ 'd1_0_0.yaml': None,
+ 'd1_0_1.yaml': None,
+ },
+ 'd1_1': {
+ 'd1_1_0.yaml': None,
+ 'd1_1_1.yaml': None,
+ },
+ },
+ }
+ self.start_patchers(fake_fs)
+ try:
+ result = build_matrix.build_matrix('d0_0')
+ finally:
+ self.stop_patchers()
+ assert len(result) == 4
+ assert self.fragment_occurences(result, 'd1_1_1.yaml') == 0.5
+
+ def test_convolve_2x2x2(self):
+ fake_fs = {
+ 'd0_0': {
+ '%': None,
+ 'd1_0': {
+ 'd1_0_0.yaml': None,
+ 'd1_0_1.yaml': None,
+ },
+ 'd1_1': {
+ 'd1_1_0.yaml': None,
+ 'd1_1_1.yaml': None,
+ },
+ 'd1_2': {
+ 'd1_2_0.yaml': None,
+ 'd1_2_1.yaml': None,
+ },
+ },
+ }
+ self.start_patchers(fake_fs)
+ try:
+ result = build_matrix.build_matrix('d0_0')
+ finally:
+ self.stop_patchers()
+ assert len(result) == 8
+ assert self.fragment_occurences(result, 'd1_2_0.yaml') == 0.5
+
+ def test_convolve_1x2x4(self):
+ fake_fs = {
+ 'd0_0': {
+ '%': None,
+ 'd1_0': {
+ 'd1_0_0.yaml': None,
+ },
+ 'd1_1': {
+ 'd1_1_0.yaml': None,
+ 'd1_1_1.yaml': None,
+ },
+ 'd1_2': {
+ 'd1_2_0.yaml': None,
+ 'd1_2_1.yaml': None,
+ 'd1_2_2.yaml': None,
+ 'd1_2_3.yaml': None,
+ },
+ },
+ }
+ self.start_patchers(fake_fs)
+ try:
+ result = build_matrix.build_matrix('d0_0')
+ finally:
+ self.stop_patchers()
+ assert len(result) == 8
+ assert self.fragment_occurences(result, 'd1_2_2.yaml') == 0.25
+
+ def test_convolve_with_concat(self):
+ fake_fs = {
+ 'd0_0': {
+ '%': None,
+ 'd1_0': {
+ 'd1_0_0.yaml': None,
+ },
+ 'd1_1': {
+ 'd1_1_0.yaml': None,
+ 'd1_1_1.yaml': None,
+ },
+ 'd1_2': {
+ '+': None,
+ 'd1_2_0.yaml': None,
+ 'd1_2_1.yaml': None,
+ 'd1_2_2.yaml': None,
+ 'd1_2_3.yaml': None,
+ },
+ },
+ }
+ self.start_patchers(fake_fs)
+ try:
+ result = build_matrix.build_matrix('d0_0')
+ finally:
+ self.stop_patchers()
+ assert len(result) == 2
+ for i in result:
+ assert 'd0_0/d1_2/d1_2_0.yaml' in i[1]
+ assert 'd0_0/d1_2/d1_2_1.yaml' in i[1]
+ assert 'd0_0/d1_2/d1_2_2.yaml' in i[1]
+ assert 'd0_0/d1_2/d1_2_3.yaml' in i[1]
+
+ def test_convolve_nested(self):
+ fake_fs = {
+ 'd0_0': {
+ '%': None,
+ 'd1_0': {
+ 'd1_0_0.yaml': None,
+ '%': '2',
+ 'd1_0_1': {
+ 'd1_0_1_0.yaml': None,
+ 'd1_0_1_1.yaml': None,
+ },
+ 'd1_0_2': {
+ 'd1_0_2_0.yaml': None,
+ 'd1_0_2_1.yaml': None,
+ },
+ },
+ 'd1_2': {
+ 'd1_2_0.yaml': None,
+ 'd1_2_1.yaml': None,
+ 'd1_2_2.yaml': None,
+ 'd1_2_3.yaml': None,
+ },
+ },
+ }
+ self.start_patchers(fake_fs)
+ try:
+ result = build_matrix.build_matrix('d0_0')
+ finally:
+ self.stop_patchers()
+ assert len(result) == 8
+ assert self.fragment_occurences(result, 'd1_0_0.yaml') == 1
+ assert self.fragment_occurences(result, 'd1_0_1_0.yaml') == 0.5
+ assert self.fragment_occurences(result, 'd1_0_1_1.yaml') == 0.5
+ assert self.fragment_occurences(result, 'd1_0_2_0.yaml') == 0.5
+ assert self.fragment_occurences(result, 'd1_0_2_1.yaml') == 0.5
+ assert self.fragment_occurences(result, 'd1_2_0.yaml') == 0.25
+ assert self.fragment_occurences(result, 'd1_2_1.yaml') == 0.25
+ assert self.fragment_occurences(result, 'd1_2_2.yaml') == 0.25
+ assert self.fragment_occurences(result, 'd1_2_3.yaml') == 0.25
+
+
+ def test_random_dollar_sign_2x2x3(self):
+ fake_fs = {
+ 'd0_0': {
+ '$': None,
+ 'd1_0': {
+ 'd1_0_0.yaml': None,
+ 'd1_0_1.yaml': None,
+ },
+ 'd1_1': {
+ 'd1_1_0.yaml': None,
+ 'd1_1_1.yaml': None,
+ },
+ 'd1_2': {
+ 'd1_2_0.yaml': None,
+ 'd1_2_1.yaml': None,
+ 'd1_2_2.yaml': None,
+ },
+ },
+ }
+ fake_fs1 = {
+ 'd0_0$': {
+ 'd1_0': {
+ 'd1_0_0.yaml': None,
+ 'd1_0_1.yaml': None,
+ },
+ 'd1_1': {
+ 'd1_1_0.yaml': None,
+ 'd1_1_1.yaml': None,
+ },
+ 'd1_2': {
+ 'd1_2_0.yaml': None,
+ 'd1_2_1.yaml': None,
+ 'd1_2_2.yaml': None,
+ },
+ },
+ }
+ self.start_patchers(fake_fs)
+ try:
+ result = build_matrix.build_matrix('d0_0')
+ finally:
+ self.stop_patchers()
+ assert len(result) == 1
+ self.start_patchers(fake_fs1)
+ try:
+ result = build_matrix.build_matrix('d0_0$')
+ finally:
+ self.stop_patchers()
+ assert len(result) == 1
+
+ def test_random_dollar_sign_with_concat(self):
+ fake_fs = {
+ 'd0_0': {
+ '$': None,
+ 'd1_0': {
+ 'd1_0_0.yaml': None,
+ },
+ 'd1_1': {
+ 'd1_1_0.yaml': None,
+ 'd1_1_1.yaml': None,
+ },
+ 'd1_2': {
+ '+': None,
+ 'd1_2_0.yaml': None,
+ 'd1_2_1.yaml': None,
+ 'd1_2_2.yaml': None,
+ 'd1_2_3.yaml': None,
+ },
+ },
+ }
+ fake_fs1 = {
+ 'd0_0$': {
+ 'd1_0': {
+ 'd1_0_0.yaml': None,
+ },
+ 'd1_1': {
+ 'd1_1_0.yaml': None,
+ 'd1_1_1.yaml': None,
+ },
+ 'd1_2': {
+ '+': None,
+ 'd1_2_0.yaml': None,
+ 'd1_2_1.yaml': None,
+ 'd1_2_2.yaml': None,
+ 'd1_2_3.yaml': None,
+ },
+ },
+ }
+ for fs, root in [(fake_fs,'d0_0'), (fake_fs1,'d0_0$')]:
+ self.start_patchers(fs)
+ try:
+ result = build_matrix.build_matrix(root)
+ finally:
+ self.stop_patchers()
+ assert len(result) == 1
+ if result[0][0][1:].startswith('d1_2'):
+ for i in result:
+ assert os.path.join(root, 'd1_2/d1_2_0.yaml') in i[1]
+ assert os.path.join(root, 'd1_2/d1_2_1.yaml') in i[1]
+ assert os.path.join(root, 'd1_2/d1_2_2.yaml') in i[1]
+ assert os.path.join(root, 'd1_2/d1_2_3.yaml') in i[1]
+
+ def test_random_dollar_sign_with_convolve(self):
+ fake_fs = {
+ 'd0_0': {
+ '%': None,
+ 'd1_0': {
+ 'd1_0_0.yaml': None,
+ 'd1_0_1.yaml': None,
+ },
+ 'd1_1': {
+ 'd1_1_0.yaml': None,
+ 'd1_1_1.yaml': None,
+ },
+ 'd1_2': {
+ '$': None,
+ 'd1_2_0.yaml': None,
+ 'd1_2_1.yaml': None,
+ 'd1_2_2.yaml': None,
+ },
+ },
+ }
+ self.start_patchers(fake_fs)
+ try:
+ result = build_matrix.build_matrix('d0_0')
+ finally:
+ self.stop_patchers()
+ assert len(result) == 4
+ fake_fs1 = {
+ 'd0_0': {
+ '%': None,
+ 'd1_0': {
+ 'd1_0_0.yaml': None,
+ 'd1_0_1.yaml': None,
+ },
+ 'd1_1': {
+ 'd1_1_0.yaml': None,
+ 'd1_1_1.yaml': None,
+ },
+ 'd1_2$': {
+ 'd1_2_0.yaml': None,
+ 'd1_2_1.yaml': None,
+ 'd1_2_2.yaml': None,
+ },
+ },
+ }
+ self.start_patchers(fake_fs1)
+ try:
+ result = build_matrix.build_matrix('d0_0')
+ finally:
+ self.stop_patchers()
+ assert len(result) == 4
+
+ def test_emulate_teuthology_noceph(self):
+ fake_fs = {
+ 'teuthology': {
+ 'no-ceph': {
+ '%': None,
+ 'clusters': {
+ 'single.yaml': None,
+ },
+ 'distros': {
+ 'baremetal.yaml': None,
+ 'rhel7.0.yaml': None,
+ 'ubuntu12.04.yaml': None,
+ 'ubuntu14.04.yaml': None,
+ 'vps.yaml': None,
+ 'vps_centos6.5.yaml': None,
+ 'vps_debian7.yaml': None,
+ 'vps_rhel6.4.yaml': None,
+ 'vps_rhel6.5.yaml': None,
+ 'vps_rhel7.0.yaml': None,
+ 'vps_ubuntu14.04.yaml': None,
+ },
+ 'tasks': {
+ 'teuthology.yaml': None,
+ },
+ },
+ },
+ }
+ self.start_patchers(fake_fs)
+ try:
+ result = build_matrix.build_matrix('teuthology/no-ceph')
+ finally:
+ self.stop_patchers()
+ assert len(result) == 11
+ assert self.fragment_occurences(result, 'vps.yaml') == 1 / 11.0
+
+ def test_empty_dirs(self):
+ fake_fs = {
+ 'teuthology': {
+ 'no-ceph': {
+ '%': None,
+ 'clusters': {
+ 'single.yaml': None,
+ },
+ 'distros': {
+ 'baremetal.yaml': None,
+ 'rhel7.0.yaml': None,
+ 'ubuntu12.04.yaml': None,
+ 'ubuntu14.04.yaml': None,
+ 'vps.yaml': None,
+ 'vps_centos6.5.yaml': None,
+ 'vps_debian7.yaml': None,
+ 'vps_rhel6.4.yaml': None,
+ 'vps_rhel6.5.yaml': None,
+ 'vps_rhel7.0.yaml': None,
+ 'vps_ubuntu14.04.yaml': None,
+ },
+ 'tasks': {
+ 'teuthology.yaml': None,
+ },
+ },
+ },
+ }
+ self.start_patchers(fake_fs)
+ try:
+ result = build_matrix.build_matrix('teuthology/no-ceph')
+ finally:
+ self.stop_patchers()
+
+ fake_fs2 = {
+ 'teuthology': {
+ 'no-ceph': {
+ '%': None,
+ 'clusters': {
+ 'single.yaml': None,
+ },
+ 'distros': {
+ 'empty': {},
+ 'baremetal.yaml': None,
+ 'rhel7.0.yaml': None,
+ 'ubuntu12.04.yaml': None,
+ 'ubuntu14.04.yaml': None,
+ 'vps.yaml': None,
+ 'vps_centos6.5.yaml': None,
+ 'vps_debian7.yaml': None,
+ 'vps_rhel6.4.yaml': None,
+ 'vps_rhel6.5.yaml': None,
+ 'vps_rhel7.0.yaml': None,
+ 'vps_ubuntu14.04.yaml': None,
+ },
+ 'tasks': {
+ 'teuthology.yaml': None,
+ },
+ 'empty': {},
+ },
+ },
+ }
+ self.start_patchers(fake_fs2)
+ try:
+ result2 = build_matrix.build_matrix('teuthology/no-ceph')
+ finally:
+ self.stop_patchers()
+ assert len(result) == 11
+ assert len(result2) == len(result)
+
+ def test_hidden(self):
+ fake_fs = {
+ 'teuthology': {
+ 'no-ceph': {
+ '%': None,
+ '.qa': None,
+ 'clusters': {
+ 'single.yaml': None,
+ '.qa': None,
+ },
+ 'distros': {
+ '.qa': None,
+ 'baremetal.yaml': None,
+ 'rhel7.0.yaml': None,
+ 'ubuntu12.04.yaml': None,
+ 'ubuntu14.04.yaml': None,
+ 'vps.yaml': None,
+ 'vps_centos6.5.yaml': None,
+ 'vps_debian7.yaml': None,
+ 'vps_rhel6.4.yaml': None,
+ 'vps_rhel6.5.yaml': None,
+ 'vps_rhel7.0.yaml': None,
+ 'vps_ubuntu14.04.yaml': None,
+ },
+ 'tasks': {
+ '.qa': None,
+ 'teuthology.yaml': None,
+ },
+ '.foo': {
+ '.qa': None,
+ 'teuthology.yaml': None,
+ },
+ },
+ },
+ }
+ self.start_patchers(fake_fs)
+ try:
+ result = build_matrix.build_matrix('teuthology/no-ceph')
+ finally:
+ self.stop_patchers()
+
+ fake_fs2 = {
+ 'teuthology': {
+ 'no-ceph': {
+ '%': None,
+ 'clusters': {
+ 'single.yaml': None,
+ },
+ 'distros': {
+ 'baremetal.yaml': None,
+ 'rhel7.0.yaml': None,
+ 'ubuntu12.04.yaml': None,
+ 'ubuntu14.04.yaml': None,
+ 'vps.yaml': None,
+ 'vps_centos6.5.yaml': None,
+ 'vps_debian7.yaml': None,
+ 'vps_rhel6.4.yaml': None,
+ 'vps_rhel6.5.yaml': None,
+ 'vps_rhel7.0.yaml': None,
+ 'vps_ubuntu14.04.yaml': None,
+ },
+ 'tasks': {
+ 'teuthology.yaml': None,
+ },
+ },
+ },
+ }
+ self.start_patchers(fake_fs2)
+ try:
+ result2 = build_matrix.build_matrix('teuthology/no-ceph')
+ finally:
+ self.stop_patchers()
+ assert len(result) == 11
+ assert len(result2) == len(result)
+
+ def test_disable_extension(self):
+ fake_fs = {
+ 'teuthology': {
+ 'no-ceph': {
+ '%': None,
+ 'clusters': {
+ 'single.yaml': None,
+ },
+ 'distros': {
+ 'baremetal.yaml': None,
+ 'rhel7.0.yaml': None,
+ 'ubuntu12.04.yaml': None,
+ 'ubuntu14.04.yaml': None,
+ 'vps.yaml': None,
+ 'vps_centos6.5.yaml': None,
+ 'vps_debian7.yaml': None,
+ 'vps_rhel6.4.yaml': None,
+ 'vps_rhel6.5.yaml': None,
+ 'vps_rhel7.0.yaml': None,
+ 'vps_ubuntu14.04.yaml': None,
+ },
+ 'tasks': {
+ 'teuthology.yaml': None,
+ },
+ },
+ },
+ }
+ self.start_patchers(fake_fs)
+ try:
+ result = build_matrix.build_matrix('teuthology/no-ceph')
+ finally:
+ self.stop_patchers()
+
+ fake_fs2 = {
+ 'teuthology': {
+ 'no-ceph': {
+ '%': None,
+ 'clusters': {
+ 'single.yaml': None,
+ },
+ 'distros': {
+ 'baremetal.yaml': None,
+ 'rhel7.0.yaml': None,
+ 'ubuntu12.04.yaml': None,
+ 'ubuntu14.04.yaml': None,
+ 'vps.yaml': None,
+ 'vps_centos6.5.yaml': None,
+ 'vps_debian7.yaml': None,
+ 'vps_rhel6.4.yaml': None,
+ 'vps_rhel6.5.yaml': None,
+ 'vps_rhel7.0.yaml': None,
+ 'vps_ubuntu14.04.yaml': None,
+ 'forcefilevps_ubuntu14.04.yaml.disable': None,
+ 'forcefilevps_ubuntu14.04.yaml.anotherextension': None,
+ },
+ 'tasks': {
+ 'teuthology.yaml': None,
+ 'forcefilevps_ubuntu14.04notyaml': None,
+ },
+ 'forcefilevps_ubuntu14.04notyaml': None,
+ 'tasks.disable': {
+ 'teuthology2.yaml': None,
+ 'forcefilevps_ubuntu14.04notyaml': None,
+ },
+ },
+ },
+ }
+ self.start_patchers(fake_fs2)
+ try:
+ result2 = build_matrix.build_matrix('teuthology/no-ceph')
+ finally:
+ self.stop_patchers()
+ assert len(result) == 11
+ assert len(result2) == len(result)
+
+ def test_sort_order(self):
+ # This test ensures that 'ceph' comes before 'ceph-thrash' when yaml
+ # fragments are sorted.
+ fake_fs = {
+ 'thrash': {
+ '%': None,
+ 'ceph-thrash': {'default.yaml': None},
+ 'ceph': {'base.yaml': None},
+ 'clusters': {'mds-1active-1standby.yaml': None},
+ 'debug': {'mds_client.yaml': None},
+ 'fs': {'btrfs.yaml': None},
+ 'msgr-failures': {'none.yaml': None},
+ 'overrides': {'allowlist_wrongly_marked_down.yaml': None},
+ 'tasks': {'cfuse_workunit_suites_fsstress.yaml': None},
+ },
+ }
+ self.start_patchers(fake_fs)
+ try:
+ result = build_matrix.build_matrix('thrash')
+ finally:
+ self.stop_patchers()
+ assert len(result) == 1
+ assert self.fragment_occurences(result, 'base.yaml') == 1
+ fragments = result[0][1]
+ assert fragments[0] == 'thrash/ceph/base.yaml'
+ assert fragments[1] == 'thrash/ceph-thrash/default.yaml'
+
+class TestSubset(object):
+ patchpoints = [
+ 'os.path.exists',
+ 'os.listdir',
+ 'os.path.isfile',
+ 'os.path.isdir',
+ 'builtins.open',
+ ]
+
+ def setup_method(self):
+ self.mocks = dict()
+ self.patchers = dict()
+ for ppoint in self.__class__.patchpoints:
+ self.mocks[ppoint] = MagicMock()
+ self.patchers[ppoint] = patch(ppoint, self.mocks[ppoint])
+
+ def start_patchers(self, fake_fs):
+ fake_fns = make_fake_fstools(fake_fs)
+ # N.B.: relies on fake_fns being in same order as patchpoints
+ for ppoint, fn in zip(self.__class__.patchpoints, fake_fns):
+ self.mocks[ppoint].side_effect = fn
+ self.patchers[ppoint].start()
+
+ def stop_patchers(self):
+ for patcher in self.patchers.values():
+ patcher.stop()
+
+ def teardown_method(self):
+ self.patchers.clear()
+ self.mocks.clear()
+
+ MAX_FACETS = 10
+ MAX_FANOUT = 3
+ MAX_DEPTH = 3
+ MAX_SUBSET = 10
+ @staticmethod
+ def generate_fake_fs(max_facets, max_fanout, max_depth):
+ def yamilify(name):
+ return name + ".yaml"
+ def name_generator():
+ x = 0
+ while True:
+ yield(str(x))
+ x += 1
+ def generate_tree(
+ max_facets, max_fanout, max_depth, namegen, top=True):
+ if max_depth == 0:
+ return None
+ if max_facets == 0:
+ return None
+ items = random.choice(range(max_fanout))
+ if items == 0 and top:
+ items = 1
+ if items == 0:
+ return None
+ sub_max_facets = max_facets / items
+ tree = {}
+ for i in range(items):
+ subtree = generate_tree(
+ sub_max_facets, max_fanout,
+ max_depth - 1, namegen, top=False)
+ if subtree is not None:
+ tree['d' + next(namegen)] = subtree
+ else:
+ tree[yamilify('f' + next(namegen))] = None
+ random.choice([
+ lambda: tree.update({'%': None}),
+ lambda: None])()
+ return tree
+ return {
+ 'root': generate_tree(
+ max_facets, max_fanout, max_depth, name_generator())
+ }
+
+ @staticmethod
+ def generate_subset(maxsub):
+ divisions = random.choice(range(maxsub-1))+1
+ return (random.choice(range(divisions)), divisions)
+
+ @staticmethod
+ def generate_description_list(tree, subset):
+ mat, first, matlimit = build_matrix._get_matrix(
+ 'root', subset=subset)
+ return [i[0] for i in build_matrix.generate_combinations(
+ 'root', mat, first, matlimit)], mat, first, matlimit
+
+ @staticmethod
+ def verify_facets(tree, description_list, subset, mat, first, matlimit):
+ def flatten(tree):
+ for k,v in tree.items():
+ if v is None and '.yaml' in k:
+ yield k
+ elif v is not None and '.disable' not in k:
+ for x in flatten(v):
+ yield x
+
+ def pptree(tree, tabs=0):
+ ret = ""
+ for k, v in tree.items():
+ if v is None:
+ ret += ('\t'*tabs) + k.ljust(10) + "\n"
+ else:
+ ret += ('\t'*tabs) + (k + ':').ljust(10) + "\n"
+ ret += pptree(v, tabs+1)
+ return ret
+ def deyamlify(name):
+ if name.endswith('.yaml'):
+ return name[:-5]
+ else:
+ return name
+ for facet in (deyamlify(_) for _ in flatten(tree)):
+ found = False
+ for i in description_list:
+ if facet in i:
+ found = True
+ break
+ if not found:
+ print("tree\n{tree}\ngenerated list\n{desc}\n\nfrom matrix\n\n{matrix}\nsubset {subset} without facet {fac}".format(
+ tree=pptree(tree),
+ desc='\n'.join(description_list),
+ subset=subset,
+ matrix=str(mat),
+ fac=facet))
+ all_desc = build_matrix.generate_combinations(
+ 'root',
+ mat,
+ 0,
+ mat.size())
+ for i, desc in zip(range(mat.size()), all_desc):
+ if i == first:
+ print('==========')
+ print("{} {}".format(i, desc))
+ if i + 1 == matlimit:
+ print('==========')
+ assert found
+
+ def test_random(self):
+ for i in range(10000):
+ tree = self.generate_fake_fs(
+ self.MAX_FACETS,
+ self.MAX_FANOUT,
+ self.MAX_DEPTH)
+ subset = self.generate_subset(self.MAX_SUBSET)
+ self.start_patchers(tree)
+ try:
+ dlist, mat, first, matlimit = self.generate_description_list(tree, subset)
+ finally:
+ self.stop_patchers()
+ self.verify_facets(tree, dlist, subset, mat, first, matlimit)
--- /dev/null
+import os
+
+from copy import deepcopy
+
+from mock import patch, Mock, DEFAULT
+
+from teuthology import suite
+from scripts.suite import main
+from teuthology.config import config
+
+import pytest
+import time
+
+from teuthology.exceptions import ScheduleFailError
+
+def get_fake_time_and_sleep():
+ # Below we set m_time.side_effect, but we also set m_time.return_value.
+ # The reason for this is that we need to store a 'fake time' that
+ # increments when m_sleep() is called; we could use any variable name we
+ # wanted for the return value, but since 'return_value' is already a
+ # standard term in mock, and since setting side_effect causes return_value
+ # to be ignored, it's safe to just reuse the name here.
+ m_time = Mock()
+ m_time.return_value = time.time()
+
+ def m_time_side_effect():
+ # Fake the slow passage of time
+ m_time.return_value += 0.1
+ return m_time.return_value
+ m_time.side_effect = m_time_side_effect
+
+ def f_sleep(seconds):
+ m_time.return_value += seconds
+ m_sleep = Mock(wraps=f_sleep)
+ return m_time, m_sleep
+
+
+def setup_module():
+ global m_time
+ global m_sleep
+ m_time, m_sleep = get_fake_time_and_sleep()
+ global patcher_time_sleep
+ patcher_time_sleep = patch.multiple(
+ 'teuthology.suite.time',
+ time=m_time,
+ sleep=m_sleep,
+ )
+ patcher_time_sleep.start()
+
+
+def teardown_module():
+ patcher_time_sleep.stop()
+
+
+@patch.object(suite.ResultsReporter, 'get_jobs')
+def test_wait_success(m_get_jobs, caplog):
+ results = [
+ [{'status': 'queued', 'job_id': '2'}],
+ [],
+ ]
+ final = [
+ {'status': 'pass', 'job_id': '1',
+ 'description': 'DESC1', 'log_href': 'http://URL1'},
+ {'status': 'fail', 'job_id': '2',
+ 'description': 'DESC2', 'log_href': 'http://URL2'},
+ {'status': 'pass', 'job_id': '3',
+ 'description': 'DESC3', 'log_href': 'http://URL3'},
+ ]
+
+ def get_jobs(name, **kwargs):
+ if kwargs['fields'] == ['job_id', 'status']:
+ return in_progress.pop(0)
+ else:
+ return final
+ m_get_jobs.side_effect = get_jobs
+ suite.Run.WAIT_PAUSE = 1
+
+ in_progress = deepcopy(results)
+ assert 0 == suite.wait('name', 1, 'http://UPLOAD_URL')
+ m_get_jobs.assert_any_call('name', fields=['job_id', 'status'])
+ assert 0 == len(in_progress)
+ assert 'fail http://UPLOAD_URL/name/2' in caplog.text
+
+ m_get_jobs.reset_mock()
+ in_progress = deepcopy(results)
+ assert 0 == suite.wait('name', 1, None)
+ m_get_jobs.assert_any_call('name', fields=['job_id', 'status'])
+ assert 0 == len(in_progress)
+ assert 'fail http://URL2' in caplog.text
+
+
+@patch.object(suite.ResultsReporter, 'get_jobs')
+def test_wait_fails(m_get_jobs):
+ results = []
+ results.append([{'status': 'queued', 'job_id': '2'}])
+ results.append([{'status': 'queued', 'job_id': '2'}])
+ results.append([{'status': 'queued', 'job_id': '2'}])
+
+ def get_jobs(name, **kwargs):
+ return results.pop(0)
+ m_get_jobs.side_effect = get_jobs
+ suite.Run.WAIT_PAUSE = 1
+ suite.Run.WAIT_MAX_JOB_TIME = 1
+ with pytest.raises(suite.WaitException):
+ suite.wait('name', 1, None)
+
+
+REPO_SHORTHAND = [
+ ['https://github.com/dude/foo', 'bar',
+ 'https://github.com/dude/bar.git'],
+ ['https://github.com/dude/foo/', 'bar',
+ 'https://github.com/dude/bar.git'],
+ ['https://github.com/ceph/ceph', 'ceph',
+ 'https://github.com/ceph/ceph.git'],
+ ['https://github.com/ceph/ceph/', 'ceph',
+ 'https://github.com/ceph/ceph.git'],
+ ['https://github.com/ceph/ceph.git', 'ceph',
+ 'https://github.com/ceph/ceph.git'],
+ ['https://github.com/ceph/ceph', 'ceph-ci',
+ 'https://github.com/ceph/ceph-ci.git'],
+ ['https://github.com/ceph/ceph-ci', 'ceph',
+ 'https://github.com/ceph/ceph.git'],
+ ['git://git.ceph.com/ceph.git', 'ceph',
+ 'git://git.ceph.com/ceph.git'],
+ ['git://git.ceph.com/ceph.git', 'ceph-ci',
+ 'git://git.ceph.com/ceph-ci.git'],
+ ['git://git.ceph.com/ceph-ci.git', 'ceph',
+ 'git://git.ceph.com/ceph.git'],
+ ['https://github.com/ceph/ceph.git', 'ceph/ceph-ci',
+ 'https://github.com/ceph/ceph-ci.git'],
+ ['https://github.com/ceph/ceph.git', 'https://github.com/ceph/ceph-ci',
+ 'https://github.com/ceph/ceph-ci'],
+ ['https://github.com/ceph/ceph.git', 'https://github.com/ceph/ceph-ci/',
+ 'https://github.com/ceph/ceph-ci/'],
+ ['https://github.com/ceph/ceph.git', 'https://github.com/ceph/ceph-ci.git',
+ 'https://github.com/ceph/ceph-ci.git'],
+]
+
+
+@pytest.mark.parametrize(['orig', 'shorthand', 'result'], REPO_SHORTHAND)
+def test_expand_short_repo_name(orig, shorthand, result):
+ assert suite.expand_short_repo_name(shorthand, orig) == result
+
+
+class TestSuiteMain(object):
+ def test_main(self):
+ suite_name = 'SUITE'
+ throttle = '3'
+ machine_type = 'burnupi'
+
+ def prepare_and_schedule(obj):
+ assert obj.base_config.suite == suite_name
+ assert obj.args.throttle == throttle
+
+ def fake_str(*args, **kwargs):
+ return 'fake'
+
+ def fake_bool(*args, **kwargs):
+ return True
+
+ def fake_false(*args, **kwargs):
+ return False
+
+ with patch.multiple(
+ 'teuthology.suite.run.util',
+ fetch_repos=DEFAULT,
+ package_version_for_hash=fake_str,
+ git_branch_exists=fake_bool,
+ git_ls_remote=fake_str,
+ ):
+ with patch.multiple(
+ 'teuthology.suite.run.Run',
+ prepare_and_schedule=prepare_and_schedule,
+ ), patch.multiple(
+ 'teuthology.suite.run.os.path',
+ exists=fake_false,
+ ):
+ main([
+ '--ceph', 'main',
+ '--suite', suite_name,
+ '--throttle', throttle,
+ '--machine-type', machine_type,
+ ])
+
+ @patch('teuthology.suite.util.smtplib.SMTP')
+ def test_machine_type_multi_error(self, m_smtp):
+ config.results_email = "example@example.com"
+ with pytest.raises(ScheduleFailError) as exc:
+ main([
+ '--ceph', 'main',
+ '--suite', 'suite_name',
+ '--throttle', '3',
+ '--machine-type', 'multi',
+ '--dry-run',
+ ])
+ assert str(exc.value) == "Scheduling failed: 'multi' is not a valid machine_type. \
+Maybe you want 'gibba,smithi,mira' or similar"
+ m_smtp.assert_not_called()
+
+ @patch('teuthology.suite.util.smtplib.SMTP')
+ def test_machine_type_none_error(self, m_smtp):
+ config.result_email = 'example@example.com'
+ with pytest.raises(ScheduleFailError) as exc:
+ main([
+ '--ceph', 'main',
+ '--suite', 'suite_name',
+ '--throttle', '3',
+ '--machine-type', 'None',
+ '--dry-run',
+ ])
+ assert str(exc.value) == "Scheduling failed: Must specify a machine_type"
+ m_smtp.assert_not_called()
+
+ def test_schedule_suite_noverify(self):
+ suite_name = 'noop'
+ suite_dir = os.path.dirname(__file__)
+ throttle = '3'
+ machine_type = 'burnupi'
+
+ with patch.multiple(
+ 'teuthology.suite.util',
+ fetch_repos=DEFAULT,
+ teuthology_schedule=DEFAULT,
+ get_arch=lambda x: 'x86_64',
+ get_gitbuilder_hash=DEFAULT,
+ git_ls_remote=lambda *args: '1234',
+ package_version_for_hash=DEFAULT,
+ ) as m:
+ m['package_version_for_hash'].return_value = 'fake-9.5'
+ config.suite_verify_ceph_hash = False
+ main([
+ '--ceph', 'main',
+ '--suite', suite_name,
+ '--suite-dir', suite_dir,
+ '--suite-relpath', '',
+ '--throttle', throttle,
+ '--machine-type', machine_type
+ ])
+ m_sleep.assert_called_with(int(throttle))
+ m['get_gitbuilder_hash'].assert_not_called()
+
+ def test_schedule_suite(self):
+ suite_name = 'noop'
+ suite_dir = os.path.dirname(__file__)
+ throttle = '3'
+ machine_type = 'burnupi'
+
+ with patch.multiple(
+ 'teuthology.suite.util',
+ fetch_repos=DEFAULT,
+ teuthology_schedule=DEFAULT,
+ get_arch=lambda x: 'x86_64',
+ get_gitbuilder_hash=DEFAULT,
+ git_ls_remote=lambda *args: '12345',
+ package_version_for_hash=DEFAULT,
+ ) as m:
+ m['package_version_for_hash'].return_value = 'fake-9.5'
+ config.suite_verify_ceph_hash = True
+ main([
+ '--ceph', 'main',
+ '--suite', suite_name,
+ '--suite-dir', suite_dir,
+ '--suite-relpath', '',
+ '--throttle', throttle,
+ '--machine-type', machine_type
+ ])
+ m_sleep.assert_called_with(int(throttle))
--- /dev/null
+from teuthology.suite import matrix
+
+
+def verify_matrix_output_diversity(res):
+ """
+ Verifies that the size of the matrix passed matches the number of unique
+ outputs from res.index
+ """
+ sz = res.size()
+ s = frozenset([matrix.generate_lists(res.index(i)) for i in range(sz)])
+ for i in range(res.size()):
+ assert sz == len(s)
+
+
+def mbs(num, l):
+ return matrix.Sum(num*10, [matrix.Base(i + (100*num)) for i in l])
+
+
+class TestMatrix(object):
+ def test_simple(self):
+ verify_matrix_output_diversity(mbs(1, range(6)))
+
+ def test_simple2(self):
+ verify_matrix_output_diversity(mbs(1, range(5)))
+
+ # The test_product* tests differ by the degree by which dimension
+ # sizes share prime factors
+ def test_product_simple(self):
+ verify_matrix_output_diversity(
+ matrix.Product(1, [mbs(1, range(6)), mbs(2, range(2))]))
+
+ def test_product_3_facets_2_prime_factors(self):
+ verify_matrix_output_diversity(matrix.Product(1, [
+ mbs(1, range(6)),
+ mbs(2, range(2)),
+ mbs(3, range(3)),
+ ]))
+
+ def test_product_3_facets_2_prime_factors_one_larger(self):
+ verify_matrix_output_diversity(matrix.Product(1, [
+ mbs(1, range(2)),
+ mbs(2, range(5)),
+ mbs(4, range(4)),
+ ]))
+
+ def test_product_4_facets_2_prime_factors(self):
+ verify_matrix_output_diversity(matrix.Sum(1, [
+ mbs(1, range(6)),
+ mbs(3, range(3)),
+ mbs(2, range(2)),
+ mbs(4, range(9)),
+ ]))
+
+ def test_product_2_facets_2_prime_factors(self):
+ verify_matrix_output_diversity(matrix.Sum(1, [
+ mbs(1, range(2)),
+ mbs(2, range(5)),
+ ]))
+
+ def test_product_with_sum(self):
+ verify_matrix_output_diversity(matrix.Sum(
+ 9,
+ [
+ mbs(10, range(6)),
+ matrix.Product(1, [
+ mbs(1, range(2)),
+ mbs(2, range(5)),
+ mbs(4, range(4))]),
+ matrix.Product(8, [
+ mbs(7, range(2)),
+ mbs(6, range(5)),
+ mbs(5, range(4))])
+ ]
+ ))
+
+ def test_product_with_pick_random(self):
+ verify_matrix_output_diversity(matrix.PickRandom(1, [
+ mbs(1, range(6)),
+ mbs(3, range(3)),
+ mbs(2, range(2)),
+ mbs(4, range(9)),
+ ]))
--- /dev/null
+import logging
+from textwrap import dedent
+
+from mock import patch, MagicMock
+
+from teuthology.suite import build_matrix
+from teuthology.suite.merge import config_merge
+from tests.fake_fs import make_fake_fstools
+
+log = logging.getLogger(__name__)
+
+class TestMerge:
+ patchpoints = [
+ 'os.path.exists',
+ 'os.listdir',
+ 'os.path.isfile',
+ 'os.path.isdir',
+ 'builtins.open',
+ ]
+
+ def setup_method(self):
+ self.mocks = dict()
+ self.patchers = dict()
+ for ppoint in self.__class__.patchpoints:
+ self.mocks[ppoint] = MagicMock()
+ self.patchers[ppoint] = patch(ppoint, self.mocks[ppoint])
+
+ def start_patchers(self, fake_fs):
+ fake_fns = make_fake_fstools(fake_fs)
+ # N.B.: relies on fake_fns being in same order as patchpoints
+ for ppoint, fn in zip(self.__class__.patchpoints, fake_fns):
+ self.mocks[ppoint].side_effect = fn
+ self.patchers[ppoint].start()
+
+ def stop_patchers(self):
+ for patcher in self.patchers.values():
+ patcher.stop()
+
+ def teardown_method(self):
+ self.patchers.clear()
+ self.mocks.clear()
+
+ def test_premerge(self):
+ fake_fs = {
+ 'd0_0': {
+ '%': None,
+ 'd1_0': {
+ 'a.yaml': dedent("""
+ teuthology:
+ premerge: reject()
+ foo: bar
+ """),
+ },
+ 'c.yaml': dedent("""
+ top: pot
+ """),
+ },
+ }
+ self.start_patchers(fake_fs)
+ try:
+ result = build_matrix.build_matrix('d0_0')
+ assert 1 == len(result)
+ configs = list(config_merge(result))
+ assert 1 == len(configs)
+ desc, frags, yaml = configs[0]
+ assert "top" in yaml
+ assert "foo" not in yaml
+ finally:
+ self.stop_patchers()
+
+ def test_postmerge(self):
+ fake_fs = {
+ 'd0_0': {
+ '%': None,
+ 'd1_0': {
+ 'a.yaml': dedent("""
+ teuthology:
+ postmerge:
+ - reject()
+ foo: bar
+ """),
+ 'b.yaml': dedent("""
+ baz: zab
+ """),
+ },
+ 'c.yaml': dedent("""
+ top: pot
+ """),
+ },
+ }
+ self.start_patchers(fake_fs)
+ try:
+ result = build_matrix.build_matrix('d0_0')
+ assert 2 == len(result)
+ configs = list(config_merge(result))
+ assert 1 == len(configs)
+ desc, frags, yaml = configs[0]
+ assert "top" in yaml
+ assert "baz" in yaml
+ assert "foo" not in yaml
+ finally:
+ self.stop_patchers()
+
+ def test_postmerge_concat(self):
+ fake_fs = {
+ 'd0_0': {
+ '%': None,
+ 'd1_0': {
+ 'a.yaml': dedent("""
+ teuthology:
+ postmerge:
+ - local a = 1
+ foo: bar
+ """),
+ 'b.yaml': dedent("""
+ teuthology:
+ postmerge:
+ - local a = 2
+ baz: zab
+ """),
+ },
+ 'z.yaml': dedent("""
+ teuthology:
+ postmerge:
+ - if a == 1 then reject() end
+ top: pot
+ """),
+ },
+ }
+ self.start_patchers(fake_fs)
+ try:
+ result = build_matrix.build_matrix('d0_0')
+ assert 2 == len(result)
+ configs = list(config_merge(result))
+ assert 1 == len(configs)
+ desc, frags, yaml = configs[0]
+ assert "top" in yaml
+ assert "baz" in yaml
+ assert "foo" not in yaml
+ finally:
+ self.stop_patchers()
+
+
+ def test_yaml_mutation(self):
+ fake_fs = {
+ 'd0_0': {
+ '%': None,
+ 'c.yaml': dedent("""
+ teuthology:
+ postmerge:
+ - |
+ yaml["test"] = py_dict()
+ top: pot
+ """),
+ },
+ }
+ self.start_patchers(fake_fs)
+ try:
+ result = build_matrix.build_matrix('d0_0')
+ assert 1 == len(result)
+ configs = list(config_merge(result))
+ assert 1 == len(configs)
+ desc, frags, yaml = configs[0]
+ assert "test" in yaml
+ assert {} == yaml["test"]
+ finally:
+ self.stop_patchers()
+
+ def test_sandbox(self):
+ fake_fs = {
+ 'd0_0': {
+ '%': None,
+ 'c.yaml': dedent("""
+ teuthology:
+ postmerge:
+ - |
+ log.debug(_ENV)
+ log.debug("_ENV contains:")
+ for k,v in pairs(_ENV) do
+ log.debug("_ENV['%s'] = %s", tostring(k), tostring(v))
+ end
+ local check = {
+ "assert",
+ "error",
+ "ipairs",
+ "next",
+ "pairs",
+ "tonumber",
+ "tostring",
+ "py_attrgetter",
+ "py_dict",
+ "py_list",
+ "py_tuple",
+ "py_enumerate",
+ "py_iterex",
+ "py_itemgetter",
+ "math",
+ "reject",
+ "accept",
+ "deep_merge",
+ "log",
+ "reject",
+ "yaml_load",
+ }
+ for _,v in ipairs(check) do
+ log.debug("checking %s", tostring(v))
+ assert(_ENV[v])
+ end
+ local block = {
+ "coroutine",
+ "debug",
+ "io",
+ "os",
+ "package",
+ }
+ for _,v in ipairs(block) do
+ log.debug("checking %s", tostring(v))
+ assert(_ENV[v] == nil)
+ end
+ top: pot
+ """),
+ },
+ }
+ self.start_patchers(fake_fs)
+ try:
+ result = build_matrix.build_matrix('d0_0')
+ assert 1 == len(result)
+ configs = list(config_merge(result))
+ assert 1 == len(configs)
+ finally:
+ self.stop_patchers()
--- /dev/null
+from teuthology.suite.placeholder import (
+ substitute_placeholders, dict_templ, Placeholder
+)
+
+
+class TestPlaceholder(object):
+ def test_substitute_placeholders(self):
+ suite_hash = 'suite_hash'
+ input_dict = dict(
+ suite='suite',
+ suite_branch='suite_branch',
+ suite_hash=suite_hash,
+ ceph_branch='ceph_branch',
+ ceph_hash='ceph_hash',
+ teuthology_branch='teuthology_branch',
+ teuthology_sha1='teuthology_sha1',
+ machine_type='machine_type',
+ distro='distro',
+ distro_version='distro_version',
+ archive_upload='archive_upload',
+ archive_upload_key='archive_upload_key',
+ suite_repo='https://example.com/ceph/suite.git',
+ suite_relpath='',
+ ceph_repo='https://example.com/ceph/ceph.git',
+ flavor='default',
+ expire='expire',
+ )
+ output_dict = substitute_placeholders(dict_templ, input_dict)
+ assert output_dict['suite'] == 'suite'
+ assert output_dict['suite_sha1'] == suite_hash
+ assert isinstance(dict_templ['suite'], Placeholder)
+ assert isinstance(
+ dict_templ['overrides']['admin_socket']['branch'],
+ Placeholder)
+
+ def test_null_placeholders_dropped(self):
+ input_dict = dict(
+ suite='suite',
+ suite_branch='suite_branch',
+ suite_hash='suite_hash',
+ ceph_branch='ceph_branch',
+ ceph_hash='ceph_hash',
+ teuthology_branch='teuthology_branch',
+ teuthology_sha1='teuthology_sha1',
+ machine_type='machine_type',
+ archive_upload='archive_upload',
+ archive_upload_key='archive_upload_key',
+ distro=None,
+ distro_version=None,
+ suite_repo='https://example.com/ceph/suite.git',
+ suite_relpath='',
+ ceph_repo='https://example.com/ceph/ceph.git',
+ flavor=None,
+ expire='expire',
+ )
+ output_dict = substitute_placeholders(dict_templ, input_dict)
+ assert 'os_type' not in output_dict
--- /dev/null
+import logging
+import os
+import pytest
+import requests
+import contextlib
+import yaml
+
+from datetime import datetime, timedelta, timezone
+from mock import patch, call, ANY
+from io import StringIO
+from io import BytesIO
+
+from teuthology.config import config, YamlConfig
+from teuthology.exceptions import ScheduleFailError
+from teuthology.suite import run
+from teuthology.util.time import TIMESTAMP_FMT
+
+log = logging.getLogger(__name__)
+
+class TestRun(object):
+ klass = run.Run
+
+ def setup_method(self):
+ self.args_dict = dict(
+ suite='suite',
+ suite_branch='suite_branch',
+ suite_relpath='',
+ ceph_branch='ceph_branch',
+ ceph_sha1='ceph_sha1',
+ email='address@example.com',
+ teuthology_branch='teuthology_branch',
+ kernel_branch=None,
+ flavor='flavor',
+ distro='ubuntu',
+ machine_type='machine_type',
+ base_yaml_paths=list(),
+ )
+ self.args = YamlConfig.from_dict(self.args_dict)
+
+ @patch('teuthology.suite.run.util.fetch_repos')
+ @patch('teuthology.suite.run.util.git_ls_remote')
+ @patch('teuthology.suite.run.util.git_validate_sha1')
+ def test_email_addr(self, m_git_validate_sha1,
+ m_git_ls_remote, _):
+ # neuter choose_X_branch
+ m_git_validate_sha1.return_value = self.args_dict['ceph_sha1']
+ self.args_dict['teuthology_branch'] = 'main'
+ self.args_dict['suite_branch'] = 'main'
+ m_git_ls_remote.return_value = 'suite_sha1'
+
+ runobj = self.klass(self.args)
+ assert runobj.base_config.email == self.args_dict['email']
+
+ @patch('teuthology.suite.run.util.fetch_repos')
+ def test_name(self, m_fetch_repos):
+ stamp = datetime.now().strftime(TIMESTAMP_FMT)
+ with patch.object(run.Run, 'create_initial_config',
+ return_value=run.JobConfig()):
+ name = run.Run(self.args).name
+ assert str(stamp) in name
+
+ @patch('teuthology.suite.run.util.fetch_repos')
+ def test_name_owner(self, m_fetch_repos):
+ self.args.owner = 'USER'
+ with patch.object(run.Run, 'create_initial_config',
+ return_value=run.JobConfig()):
+ name = run.Run(self.args).name
+ assert name.startswith('USER-')
+
+ @patch('teuthology.suite.run.util.git_branch_exists')
+ @patch('teuthology.suite.run.util.package_version_for_hash')
+ @patch('teuthology.suite.run.util.git_ls_remote')
+ def test_branch_nonexistent(
+ self,
+ m_git_ls_remote,
+ m_package_version_for_hash,
+ m_git_branch_exists,
+ ):
+ config.gitbuilder_host = 'example.com'
+ m_git_ls_remote.side_effect = [
+ # First call will be for the ceph hash
+ None,
+ # Second call will be for the suite hash
+ 'suite_hash',
+ ]
+ m_package_version_for_hash.return_value = 'a_version'
+ m_git_branch_exists.return_value = True
+ self.args.ceph_branch = 'ceph_sha1'
+ self.args.ceph_sha1 = None
+ with pytest.raises(ScheduleFailError):
+ self.klass(self.args)
+
+ @pytest.mark.parametrize(
+ ["expire", "delta", "result"],
+ [
+ [None, timedelta(), False],
+ ["1m", timedelta(), True],
+ ["1m", timedelta(minutes=-2), False],
+ ["1m", timedelta(minutes=2), True],
+ ["7d", timedelta(days=-14), False],
+ ]
+ )
+ @patch('teuthology.repo_utils.fetch_repo')
+ @patch('teuthology.suite.run.util.git_branch_exists')
+ @patch('teuthology.suite.run.util.package_version_for_hash')
+ @patch('teuthology.suite.run.util.git_ls_remote')
+ def test_get_expiration(
+ self,
+ m_git_ls_remote,
+ m_package_version_for_hash,
+ m_git_branch_exists,
+ m_fetch_repo,
+ expire,
+ delta,
+ result,
+ ):
+ m_git_ls_remote.side_effect = 'hash'
+ m_package_version_for_hash.return_value = 'a_version'
+ m_git_branch_exists.return_value = True
+ self.args.expire = expire
+ obj = self.klass(self.args)
+ now = datetime.now(timezone.utc)
+ expires_result = obj.get_expiration(_base_time=now + delta)
+ if expire is None:
+ assert expires_result is None
+ assert obj.base_config['expire'] is None
+ else:
+ assert expires_result is not None
+ assert (now < expires_result) is result
+ assert obj.base_config['expire']
+
+ @patch('teuthology.suite.run.util.fetch_repos')
+ @patch('requests.head')
+ @patch('teuthology.suite.run.util.git_branch_exists')
+ @patch('teuthology.suite.run.util.package_version_for_hash')
+ @patch('teuthology.suite.run.util.git_ls_remote')
+ def test_sha1_exists(
+ self,
+ m_git_ls_remote,
+ m_package_version_for_hash,
+ m_git_branch_exists,
+ m_requests_head,
+ m_fetch_repos,
+ ):
+ config.gitbuilder_host = 'example.com'
+ m_package_version_for_hash.return_value = 'ceph_hash'
+ m_git_branch_exists.return_value = True
+ resp = requests.Response()
+ resp.reason = 'OK'
+ resp.status_code = 200
+ m_requests_head.return_value = resp
+ # only one call to git_ls_remote in this case
+ m_git_ls_remote.return_value = "suite_branch"
+ run = self.klass(self.args)
+ assert run.base_config.sha1 == 'ceph_sha1'
+ assert run.base_config.branch == 'ceph_branch'
+
+ @patch('teuthology.suite.run.util.git_ls_remote')
+ @patch('requests.head')
+ @patch('teuthology.suite.util.git_branch_exists')
+ @patch('teuthology.suite.util.package_version_for_hash')
+ def test_sha1_nonexistent(
+ self,
+ m_git_ls_remote,
+ m_package_version_for_hash,
+ m_git_branch_exists,
+ m_requests_head,
+ ):
+ config.gitbuilder_host = 'example.com'
+ m_package_version_for_hash.return_value = 'ceph_hash'
+ m_git_branch_exists.return_value = True
+ resp = requests.Response()
+ resp.reason = 'Not Found'
+ resp.status_code = 404
+ m_requests_head.return_value = resp
+ self.args.ceph_sha1 = 'ceph_hash_dne'
+ with pytest.raises(ScheduleFailError):
+ self.klass(self.args)
+
+ @patch('teuthology.suite.util.smtplib.SMTP')
+ @patch('teuthology.suite.util.git_ls_remote')
+ @patch('teuthology.suite.util.package_version_for_hash')
+ def test_teuthology_branch_nonexistent(
+ self,
+ m_pvfh,
+ m_git_ls_remote,
+ m_smtp,
+ ):
+ m_git_ls_remote.return_value = None
+ config.teuthology_path = None
+ config.results_email = "example@example.com"
+ self.args.dry_run = True
+ self.args.teuthology_branch = 'no_branch'
+ with pytest.raises(ScheduleFailError):
+ self.klass(self.args)
+ m_smtp.assert_not_called()
+
+ @patch('teuthology.suite.run.util.fetch_repos')
+ @patch('teuthology.suite.util.git_ls_remote')
+ @patch('teuthology.suite.run.util.package_version_for_hash')
+ def test_os_type(self, m_pvfh, m_git_ls_remote, m_fetch_repos):
+ m_git_ls_remote.return_value = "sha1"
+ del self.args['distro']
+ run_ = run.Run(self.args)
+ run_.base_args = run_.build_base_args()
+ run_.base_config = run_.build_base_config()
+ configs = [
+ ["desc", [], {"os_type": "debian", "os_version": "8.0"}],
+ ["desc", [], {"os_type": "ubuntu", "os_version": "24.0"}],
+ ]
+ missing, to_schedule = run_.collect_jobs('x86_64', configs, False, False)
+ assert to_schedule[0]['yaml']['os_type'] == "debian"
+ assert to_schedule[0]['yaml']['os_version'] == "8.0"
+ assert to_schedule[1]['yaml']['os_type'] == "ubuntu"
+ assert to_schedule[1]['yaml']['os_version'] == "24.0"
+
+ @patch('teuthology.suite.run.util.fetch_repos')
+ @patch('teuthology.suite.util.git_ls_remote')
+ @patch('teuthology.suite.run.util.package_version_for_hash')
+ def test_sha1(self, m_pvfh, m_git_ls_remote, m_fetch_repos):
+ m_git_ls_remote.return_value = "sha1"
+ del self.args['distro']
+ run_ = run.Run(self.args)
+ run_.base_args = run_.build_base_args()
+ for i in range(5): # mock backtracking
+ run_.config_input['ceph_hash'] = f"boo{i}"
+ run_.config_input['suite_hash'] = f"bar{i}"
+ run_.base_config = run_.build_base_config()
+ configs = [
+ ["desc", [], {"os_type": "debian", "os_version": "8.0",
+ "sha1": "old_sha", "suite_sha1": "old_sha",
+ "overrides": { "workunit": {"sha1": "old_sha"}, "ceph": {"sha1": "old_sha"} }
+ }],
+ ]
+ missing, to_schedule = run_.collect_jobs('x86_64', configs, False, False)
+ assert to_schedule[0]['yaml']['sha1'] == "boo4"
+ assert to_schedule[0]['yaml']['suite_sha1'] == "bar4"
+ assert to_schedule[0]['yaml']['overrides']['workunit']["sha1"] == "bar4"
+ assert to_schedule[0]['yaml']['overrides']['ceph']["sha1"] == "boo4"
+
+class TestScheduleSuite(object):
+ klass = run.Run
+
+ def setup_method(self):
+ self.args_dict = dict(
+ suite='suite',
+ suite_relpath='',
+ suite_dir='suite_dir',
+ suite_branch='main',
+ suite_repo='ceph',
+ ceph_repo='ceph',
+ ceph_branch='main',
+ ceph_sha1='ceph_sha1',
+ teuthology_branch='main',
+ kernel_branch=None,
+ flavor='flavor',
+ distro='ubuntu',
+ distro_version='14.04',
+ machine_type='machine_type',
+ base_yaml_paths=list(),
+ )
+ self.args = YamlConfig.from_dict(self.args_dict)
+
+ @patch('teuthology.suite.run.Run.schedule_jobs')
+ @patch('teuthology.suite.run.Run.write_rerun_memo')
+ @patch('teuthology.suite.util.get_install_task_flavor')
+ @patch('teuthology.suite.merge.open')
+ @patch('teuthology.suite.run.build_matrix')
+ @patch('teuthology.suite.util.git_ls_remote')
+ @patch('teuthology.suite.util.package_version_for_hash')
+ @patch('teuthology.suite.util.git_validate_sha1')
+ @patch('teuthology.suite.util.get_arch')
+ def test_successful_schedule(
+ self,
+ m_get_arch,
+ m_git_validate_sha1,
+ m_package_version_for_hash,
+ m_git_ls_remote,
+ m_build_matrix,
+ m_open,
+ m_get_install_task_flavor,
+ m_write_rerun_memo,
+ m_schedule_jobs,
+ ):
+ m_get_arch.return_value = 'x86_64'
+ m_git_validate_sha1.return_value = self.args.ceph_sha1
+ m_package_version_for_hash.return_value = 'ceph_version'
+ m_git_ls_remote.return_value = 'suite_hash'
+ build_matrix_desc = 'desc'
+ build_matrix_frags = ['frag1.yml', 'frag2.yml']
+ build_matrix_output = [
+ (build_matrix_desc, build_matrix_frags),
+ ]
+ m_build_matrix.return_value = build_matrix_output
+ frag1_read_output = 'field1: val1'
+ frag2_read_output = 'field2: val2'
+ m_open.side_effect = [
+ StringIO(frag1_read_output),
+ StringIO(frag2_read_output),
+ contextlib.closing(BytesIO())
+ ]
+ m_get_install_task_flavor.return_value = 'default'
+ m_package_version_for_hash.return_value = "v1"
+ # schedule_jobs() is just neutered; check calls below
+
+ self.args.newest = 0
+ self.args.num = 42
+ runobj = self.klass(self.args)
+ runobj.base_args = list()
+ count = runobj.schedule_suite()
+ assert(count == 1)
+ assert runobj.base_config['suite_sha1'] == 'suite_hash'
+ m_package_version_for_hash.assert_has_calls(
+ [call('ceph_sha1', 'default', 'ubuntu', '14.04', 'machine_type')],
+ )
+ y = {
+ 'field1': 'val1',
+ 'field2': 'val2'
+ }
+ teuthology_keys = [
+ 'branch',
+ 'machine_type',
+ 'name',
+ 'os_type',
+ 'os_version',
+ 'overrides',
+ 'priority',
+ 'repo',
+ 'seed',
+ 'sha1',
+ 'sleep_before_teardown',
+ 'suite',
+ 'suite_branch',
+ 'suite_relpath',
+ 'suite_repo',
+ 'suite_sha1',
+ 'tasks',
+ 'teuthology_branch',
+ 'teuthology_sha1',
+ 'timestamp',
+ 'user',
+ 'teuthology',
+ 'flavor',
+ ]
+ for t in teuthology_keys:
+ y[t] = ANY
+ expected_job = dict(
+ yaml=y,
+ sha1='ceph_sha1',
+ args=[
+ '--num',
+ '42',
+ '--description',
+ os.path.join(self.args.suite, build_matrix_desc),
+ '--',
+ '-'
+ ],
+ stdin=ANY,
+ desc=os.path.join(self.args.suite, build_matrix_desc),
+ )
+
+ m_schedule_jobs.assert_has_calls(
+ [call([], [expected_job], runobj.name)],
+ )
+ args = m_schedule_jobs.call_args.args
+ log.debug("args =\n%s", args)
+ jobargs = args[1][0]
+ stdin_yaml = yaml.safe_load(jobargs['stdin'])
+ for k in y:
+ assert y[k] == stdin_yaml[k]
+ for k in teuthology_keys:
+ assert k in stdin_yaml
+ m_write_rerun_memo.assert_called_once_with()
+
+ @patch('teuthology.suite.util.find_git_parents')
+ @patch('teuthology.suite.run.Run.schedule_jobs')
+ @patch('teuthology.suite.util.get_install_task_flavor')
+ @patch('teuthology.suite.run.config_merge')
+ @patch('teuthology.suite.run.build_matrix')
+ @patch('teuthology.suite.util.git_ls_remote')
+ @patch('teuthology.suite.util.package_version_for_hash')
+ @patch('teuthology.suite.util.git_validate_sha1')
+ @patch('teuthology.suite.util.get_arch')
+ def test_newest_failure(
+ self,
+ m_get_arch,
+ m_git_validate_sha1,
+ m_package_version_for_hash,
+ m_git_ls_remote,
+ m_build_matrix,
+ m_config_merge,
+ m_get_install_task_flavor,
+ m_schedule_jobs,
+ m_find_git_parents,
+ ):
+ m_get_arch.return_value = 'x86_64'
+ m_git_validate_sha1.return_value = self.args.ceph_sha1
+ m_package_version_for_hash.return_value = None
+ m_git_ls_remote.return_value = 'suite_hash'
+ build_matrix_desc = 'desc'
+ build_matrix_frags = ['frag.yml']
+ build_matrix_output = [
+ (build_matrix_desc, build_matrix_frags),
+ ]
+ m_build_matrix.return_value = build_matrix_output
+ m_config_merge.return_value = [(a, b, {}) for a, b in build_matrix_output]
+ m_get_install_task_flavor.return_value = 'default'
+
+ m_find_git_parents.side_effect = lambda proj, sha1, count: [f"{sha1}_{i}" for i in range(11)]
+
+ self.args.newest = 10
+ runobj = self.klass(self.args)
+ runobj.base_args = list()
+ with pytest.raises(ScheduleFailError) as exc:
+ runobj.schedule_suite()
+ assert 'Exceeded 10 backtracks' in str(exc.value)
+ m_find_git_parents.assert_has_calls(
+ [call('ceph', 'ceph_sha1', 10)]
+ )
+
+ @patch('teuthology.suite.util.find_git_parents')
+ @patch('teuthology.suite.run.Run.schedule_jobs')
+ @patch('teuthology.suite.run.Run.write_rerun_memo')
+ @patch('teuthology.suite.util.get_install_task_flavor')
+ @patch('teuthology.suite.run.config_merge')
+ @patch('teuthology.suite.run.build_matrix')
+ @patch('teuthology.suite.util.git_ls_remote')
+ @patch('teuthology.suite.util.package_version_for_hash')
+ @patch('teuthology.suite.util.git_validate_sha1')
+ @patch('teuthology.suite.util.get_arch')
+ def test_newest_success_same_branch_same_repo(
+ self,
+ m_get_arch,
+ m_git_validate_sha1,
+ m_package_version_for_hash,
+ m_git_ls_remote,
+ m_build_matrix,
+ m_config_merge,
+ m_get_install_task_flavor,
+ m_write_rerun_memo,
+ m_schedule_jobs,
+ m_find_git_parents,
+ ):
+ """
+ Test that we can successfully schedule a job with newest
+ backtracking when the ceph and suite branches are the same
+ and the ceph_sha1 is not supplied. We should expect that the
+ ceph_hash and suite_hash will be updated to the working sha1
+ """
+ m_get_arch.return_value = 'x86_64'
+ # rig has_packages_for_distro to fail this many times, so
+ # everything will run NUM_FAILS+1 times
+ NUM_FAILS = 5
+ # Here we just assume that even fi ceph_sha1 is not supplied,
+ # in git_valid_sha1, util.git_ls_remote will give us ceph_sha1
+ m_git_validate_sha1.return_value = self.args.ceph_sha1
+ # Here we know that in create_initial_config, we call
+ # git_ls_remote 3 times, choose_ceph_hash, choose_suite_hash,
+ # and choose_teuthology_branch
+ sha1_side_effect = [
+ self.args.ceph_sha1, # ceph_sha1
+ 'suite_sha1', # suite_sha1
+ 'teuthology_sha1', # teuthology_sha1
+ ]
+ m_git_ls_remote.side_effect = sha1_side_effect
+ build_matrix_desc = 'desc'
+ build_matrix_frags = ['frag.yml']
+ build_matrix_output = [
+ (build_matrix_desc, build_matrix_frags),
+ ]
+ m_build_matrix.return_value = build_matrix_output
+ m_config_merge.return_value = [(a, b, {}) for a, b in build_matrix_output]
+ m_get_install_task_flavor.return_value = 'default'
+
+ # Generate backtracked parent sha1s
+ parent_sha1s = [f"ceph_sha1_{i}" for i in range(NUM_FAILS)]
+ assert len(parent_sha1s)
+ # Last sha1 will be the one that works!
+ working_sha1 = parent_sha1s[-1]
+
+ # NUM_FAILS attempts, then success on the last parent sha1
+ m_package_version_for_hash.side_effect = \
+ [None for i in range(NUM_FAILS)] + ["ceph_version"]
+
+ m_find_git_parents.return_value = parent_sha1s
+
+ self.args.newest = 10
+ runobj = self.klass(self.args)
+ runobj.base_args = list()
+
+ # Call schedule_suite()
+ count = runobj.schedule_suite()
+ # Epect only 1 job to be scheduled
+ assert count == 1
+ # Expect that we called package_version_for_hash NUM_FAILS times + 1 for the working sha1
+ m_package_version_for_hash.assert_has_calls(
+ [call(self.args.ceph_sha1, 'default', 'ubuntu', '14.04', 'machine_type')] +
+ [call(f"ceph_sha1_{i}", 'default', 'ubuntu', '14.04', 'machine_type')
+ for i in range(0, NUM_FAILS)]
+ )
+ # (ceph, base_config.sha1, newest) called once to get grab the backtrace
+ m_find_git_parents.assert_called_once_with('ceph', 'ceph_sha1', 10)
+
+ # Verify that base_config was updated with the working SHA1
+ assert runobj.base_config.sha1 == working_sha1
+
+ # Verify that config_input's ceph_hash and suite_hash was updated
+ assert runobj.config_input['ceph_hash'] == working_sha1
+ assert runobj.config_input['suite_hash'] == working_sha1
+
+ # Verify that config_input's ceph_hash and suite_hash are not the same as the original sha1s
+ assert runobj.config_input['ceph_hash'] != sha1_side_effect[0] # ceph_sha1
+ assert runobj.config_input['suite_hash'] != sha1_side_effect[1] # suite_sha1
+
+ # Verify the sha1 in scheduled jobs
+ args = m_schedule_jobs.call_args.args
+ scheduled_jobs = args[1]
+
+ # Check each job has the correct SHA1
+ for job in scheduled_jobs:
+ assert job['sha1'] == working_sha1
+
+ # Parse YAML from stdin to check for sha1 and suite_hash
+ if 'stdin' in job:
+ job_yaml = yaml.safe_load(job['stdin'])
+ assert job_yaml.get('sha1') == working_sha1
+ assert job_yaml.get('suite_sha1') == working_sha1
+
+ @patch('teuthology.suite.util.find_git_parents')
+ @patch('teuthology.suite.run.Run.schedule_jobs')
+ @patch('teuthology.suite.run.Run.write_rerun_memo')
+ @patch('teuthology.suite.util.get_install_task_flavor')
+ @patch('teuthology.suite.run.config_merge')
+ @patch('teuthology.suite.run.build_matrix')
+ @patch('teuthology.suite.util.git_ls_remote')
+ @patch('teuthology.suite.util.package_version_for_hash')
+ @patch('teuthology.suite.util.git_validate_sha1')
+ @patch('teuthology.suite.util.get_arch')
+ def test_newest_success_diff_branch_diff_repo(
+ self,
+ m_get_arch,
+ m_git_validate_sha1,
+ m_package_version_for_hash,
+ m_git_ls_remote,
+ m_build_matrix,
+ m_config_merge,
+ m_get_install_task_flavor,
+ m_write_rerun_memo,
+ m_schedule_jobs,
+ m_find_git_parents,
+ ):
+ """
+ Test that we can successfully schedule a job with newest
+ backtracking when the ceph and suite branches are different
+ and the ceph_sha1 is not supplied. We should expect that the
+ ceph_hash will be updated to the working sha1,
+ but the suite_hash will remain the original suite_sha1.
+ """
+ m_get_arch.return_value = 'x86_64'
+ # Set different branches
+ self.args.ceph_branch = 'ceph_different_branch'
+ self.args.suite_branch = 'suite_different_branch'
+
+ # rig has_packages_for_distro to fail this many times, so
+ # everything will run NUM_FAILS+1 times
+ NUM_FAILS = 5
+ # Here we just assume that even fi ceph_sha1 is not supplied,
+ # in git_valid_sha1, util.git_ls_remote will give us ceph_sha1
+ m_git_validate_sha1.return_value = self.args.ceph_sha1
+ # Here we know that in create_initial_config, we call
+ # git_ls_remote 3 times, choose_ceph_hash, choose_suite_hash,
+ # and choose_teuthology_branch
+ sha1_side_effect = [
+ self.args.ceph_sha1, # ceph_sha1
+ 'suite_sha1', # suite_sha1
+ 'teuthology_sha1', # teuthology_sha1
+ ]
+ m_git_ls_remote.side_effect = sha1_side_effect
+ build_matrix_desc = 'desc'
+ build_matrix_frags = ['frag.yml']
+ build_matrix_output = [
+ (build_matrix_desc, build_matrix_frags),
+ ]
+ m_build_matrix.return_value = build_matrix_output
+ m_config_merge.return_value = [(a, b, {}) for a, b in build_matrix_output]
+ m_get_install_task_flavor.return_value = 'default'
+
+ # Generate backtracked parent sha1s
+ parent_sha1s = [f"ceph_sha1_{i}" for i in range(NUM_FAILS)]
+ assert len(parent_sha1s)
+ # Last sha1 will be the one that works!
+ working_sha1 = parent_sha1s[-1]
+
+ # NUM_FAILS attempts, then success on the last parent sha1
+ m_package_version_for_hash.side_effect = \
+ [None for i in range(NUM_FAILS)] + ["ceph_version"]
+
+ m_find_git_parents.return_value = parent_sha1s
+
+ self.args.newest = 10
+ runobj = self.klass(self.args)
+ runobj.base_args = list()
+
+ # Call schedule_suite()
+ count = runobj.schedule_suite()
+ # Epect only 1 job to be scheduled
+ assert count == 1
+ # Expect that we called package_version_for_hash NUM_FAILS times + 1 for the working sha1
+ m_package_version_for_hash.assert_has_calls(
+ [call(self.args.ceph_sha1, 'default', 'ubuntu', '14.04', 'machine_type')] +
+ [call(f"ceph_sha1_{i}", 'default', 'ubuntu', '14.04', 'machine_type')
+ for i in range(0, NUM_FAILS)]
+ )
+ # (ceph, base_config.sha1, newest) called once to get grab the backtrace
+ m_find_git_parents.assert_called_once_with('ceph', 'ceph_sha1', 10)
+
+ # Verify that base_config was updated with the working SHA1
+ assert runobj.base_config.sha1 == working_sha1
+
+ # Verify that config_input's ceph_hash was updated,
+ # but suite_hash is still the original suite_sha1
+ assert runobj.config_input['ceph_hash'] == working_sha1
+ assert runobj.config_input['suite_hash'] != working_sha1
+
+ # Verify that config_input's ceph_hash is not the same as the original sha1s
+ # but suite_hash is still the original suite_sha1
+ assert runobj.config_input['ceph_hash'] != sha1_side_effect[0] # ceph_sha1
+ assert runobj.config_input['suite_hash'] == sha1_side_effect[1] # suite_sha1
+
+ # Verify the sha1 in scheduled jobs
+ args = m_schedule_jobs.call_args.args
+ scheduled_jobs = args[1]
+
+ # Check each job has the correct SHA1
+ for job in scheduled_jobs:
+ assert job['sha1'] == working_sha1
+
+ # Parse YAML from stdin to check for sha1 and suite_hash
+ if 'stdin' in job:
+ job_yaml = yaml.safe_load(job['stdin'])
+ assert job_yaml.get('sha1') == working_sha1
+ assert job_yaml.get('suite_sha1') == sha1_side_effect[1]
--- /dev/null
+import os
+import pytest
+import tempfile
+
+from mock import Mock, patch
+
+from teuthology.config import config
+from teuthology.orchestra.opsys import OS
+from teuthology.suite import util
+from teuthology.exceptions import BranchNotFoundError, ScheduleFailError
+
+
+REPO_PROJECTS_AND_URLS = [
+ 'ceph',
+ 'https://github.com/not_ceph/ceph.git',
+]
+
+
+@pytest.mark.parametrize('project_or_url', REPO_PROJECTS_AND_URLS)
+@patch('subprocess.check_output')
+def test_git_branch_exists(m_check_output, project_or_url):
+ m_check_output.return_value = ''
+ assert False == util.git_branch_exists(
+ project_or_url, 'nobranchnowaycanthappen')
+ m_check_output.return_value = b'HHH branch'
+ assert True == util.git_branch_exists(project_or_url, 'main')
+
+
+@pytest.fixture
+def git_repository(request):
+ d = tempfile.mkdtemp()
+ os.system("""
+ cd {d}
+ git init
+ touch A
+ git config user.email 'you@example.com'
+ git config user.name 'Your Name'
+ git add A
+ git commit -m 'A' A
+ git rev-parse --abbrev-ref main || git checkout -b main
+ """.format(d=d))
+
+ def fin():
+ os.system("rm -fr " + d)
+ request.addfinalizer(fin)
+ return d
+
+
+class TestUtil(object):
+ @patch('teuthology.suite.util.smtplib.SMTP')
+ def test_schedule_fail(self, m_smtp):
+ config.results_email = "example@example.com"
+ with pytest.raises(ScheduleFailError) as exc:
+ util.schedule_fail(message="error msg", dry_run=False)
+ assert str(exc.value) == "Scheduling failed: error msg"
+ m_smtp.assert_called()
+
+ @patch('teuthology.suite.util.smtplib.SMTP')
+ def test_schedule_fail_dryrun(self, m_smtp):
+ config.results_email = "example@example.com"
+ with pytest.raises(ScheduleFailError) as exc:
+ util.schedule_fail(message="error msg", dry_run=True)
+ assert str(exc.value) == "Scheduling failed: error msg"
+ m_smtp.assert_not_called()
+
+ @patch('teuthology.suite.util.fetch_qa_suite')
+ @patch('teuthology.suite.util.smtplib.SMTP')
+ def test_fetch_repo_no_branch(self, m_smtp, m_fetch_qa_suite):
+ m_fetch_qa_suite.side_effect = BranchNotFoundError(
+ "no-branch", "https://github.com/ceph/ceph-ci.git")
+ config.results_email = "example@example.com"
+ with pytest.raises(ScheduleFailError) as exc:
+ util.fetch_repos("no-branch", "test1", dry_run=False)
+ assert str(exc.value) == "Scheduling test1 failed: \
+Branch 'no-branch' not found in repo: https://github.com/ceph/ceph-ci.git!"
+ m_smtp.assert_called()
+
+ @patch('teuthology.suite.util.fetch_qa_suite')
+ @patch('teuthology.suite.util.smtplib.SMTP')
+ def test_fetch_repo_no_branch_dryrun(self, m_smtp, m_fetch_qa_suite):
+ m_fetch_qa_suite.side_effect = BranchNotFoundError(
+ "no-branch", "https://github.com/ceph/ceph-ci.git")
+ config.results_email = "example@example.com"
+ with pytest.raises(ScheduleFailError) as exc:
+ util.fetch_repos("no-branch", "test1", dry_run=True)
+ assert str(exc.value) == "Scheduling test1 failed: \
+Branch 'no-branch' not found in repo: https://github.com/ceph/ceph-ci.git!"
+ m_smtp.assert_not_called()
+
+ @patch('requests.get')
+ def test_get_branch_info(self, m_get):
+ mock_resp = Mock()
+ mock_resp.ok = True
+ mock_resp.json.return_value = "some json"
+ m_get.return_value = mock_resp
+ result = util.get_branch_info("teuthology", "main")
+ m_get.assert_called_with(
+ "https://api.github.com/repos/ceph/teuthology/git/refs/heads/main"
+ )
+ assert result == "some json"
+
+ @patch('teuthology.lock.query')
+ def test_get_arch_fail(self, m_query):
+ m_query.list_locks.return_value = False
+ util.get_arch('magna')
+ m_query.list_locks.assert_called_with(machine_type="magna", count=1, tries=1)
+
+ @patch('teuthology.lock.query')
+ def test_get_arch_success(self, m_query):
+ m_query.list_locks.return_value = [{"arch": "arch"}]
+ result = util.get_arch('magna')
+ m_query.list_locks.assert_called_with(
+ machine_type="magna",
+ count=1, tries=1
+ )
+ assert result == "arch"
+
+ def test_build_git_url_github(self):
+ assert 'project' in util.build_git_url('project')
+ owner = 'OWNER'
+ git_url = util.build_git_url('project', project_owner=owner)
+ assert owner in git_url
+
+ @patch('teuthology.config.TeuthologyConfig.get_ceph_qa_suite_git_url')
+ def test_build_git_url_ceph_qa_suite_custom(
+ self,
+ m_get_ceph_qa_suite_git_url):
+ url = 'http://foo.com/some'
+ m_get_ceph_qa_suite_git_url.return_value = url + '.git'
+ assert url == util.build_git_url('ceph-qa-suite')
+
+ @patch('teuthology.config.TeuthologyConfig.get_ceph_git_url')
+ def test_build_git_url_ceph_custom(self, m_get_ceph_git_url):
+ url = 'http://foo.com/some'
+ m_get_ceph_git_url.return_value = url + '.git'
+ assert url == util.build_git_url('ceph')
+
+ @patch('teuthology.config.TeuthologyConfig.get_ceph_cm_ansible_git_url')
+ def test_build_git_url_ceph_cm_ansible_custom(self, m_get_ceph_cm_ansible_git_url):
+ url = 'http://foo.com/some'
+ m_get_ceph_cm_ansible_git_url.return_value = url + '.git'
+ assert url == util.build_git_url('ceph-cm-ansible')
+
+ @patch('teuthology.config.TeuthologyConfig.get_ceph_git_url')
+ def test_git_ls_remote(self, m_get_ceph_git_url, git_repository):
+ m_get_ceph_git_url.return_value = git_repository
+ assert util.git_ls_remote('ceph', 'nobranch') is None
+ assert util.git_ls_remote('ceph', 'main') is not None
+
+ @patch('teuthology.suite.util.requests.get')
+ def test_find_git_parents(self, m_requests_get):
+ history_resp = Mock(ok=True)
+ history_resp.json.return_value = {'sha1s': ['sha1', 'sha1_p']}
+ m_requests_get.return_value = history_resp
+ parent_sha1s = util.find_git_parents('ceph', 'sha1')
+ assert m_requests_get.call_count == 1
+ assert parent_sha1s == ['sha1_p']
+
+
+class TestFlavor(object):
+
+ def test_get_install_task_flavor_bare(self):
+ config = dict(
+ tasks=[
+ dict(
+ install=dict(),
+ ),
+ ],
+ )
+ assert util.get_install_task_flavor(config) == 'default'
+
+ def test_get_install_task_flavor_simple(self):
+ config = dict(
+ tasks=[
+ dict(
+ install=dict(
+ flavor='notcmalloc',
+ ),
+ ),
+ ],
+ )
+ assert util.get_install_task_flavor(config) == 'notcmalloc'
+
+ def test_get_install_task_flavor_override_simple(self):
+ config = dict(
+ tasks=[
+ dict(install=dict()),
+ ],
+ overrides=dict(
+ install=dict(
+ flavor='notcmalloc',
+ ),
+ ),
+ )
+ assert util.get_install_task_flavor(config) == 'notcmalloc'
+
+ def test_get_install_task_flavor_override_project(self):
+ config = dict(
+ tasks=[
+ dict(install=dict()),
+ ],
+ overrides=dict(
+ install=dict(
+ ceph=dict(
+ flavor='notcmalloc',
+ ),
+ ),
+ ),
+ )
+ assert util.get_install_task_flavor(config) == 'notcmalloc'
+
+
+class TestMissingPackages(object):
+ """
+ Tests the functionality that checks to see if a
+ scheduled job will have missing packages in shaman.
+ """
+ @patch("teuthology.packaging.ShamanProject._get_package_version")
+ def test_distro_has_packages(self, m_gpv):
+ m_gpv.return_value = "v1"
+ result = util.package_version_for_hash(
+ "sha1",
+ "basic",
+ "ubuntu",
+ "14.04",
+ "mtype",
+ )
+ assert result
+
+ @patch("teuthology.packaging.ShamanProject._get_package_version")
+ def test_distro_does_not_have_packages(self, m_gpv):
+ m_gpv.return_value = None
+ result = util.package_version_for_hash(
+ "sha1",
+ "basic",
+ "rhel",
+ "7.0",
+ "mtype",
+ )
+ assert not result
+
+
+class TestDistroDefaults(object):
+ def test_distro_defaults_plana(self):
+ expected = ('x86_64', 'ubuntu/22.04',
+ OS(name='ubuntu', version='22.04', codename='jammy'))
+ assert util.get_distro_defaults('ubuntu', 'plana') == expected
+
+ def test_distro_defaults_debian(self):
+ expected = ('x86_64', 'debian/8.0',
+ OS(name='debian', version='8.0', codename='jessie'))
+ assert util.get_distro_defaults('debian', 'magna') == expected
+
+ def test_distro_defaults_centos(self):
+ expected = ('x86_64', 'centos/9',
+ OS(name='centos', version='9.stream', codename='stream'))
+ assert util.get_distro_defaults('centos', 'magna') == expected
+
+ def test_distro_defaults_fedora(self):
+ expected = ('x86_64', 'fedora/25',
+ OS(name='fedora', version='25', codename='25'))
+ assert util.get_distro_defaults('fedora', 'magna') == expected
+
+ def test_distro_defaults_default(self):
+ expected = ('x86_64', 'centos/9',
+ OS(name='centos', version='9.stream', codename='stream'))
+ assert util.get_distro_defaults('rhel', 'magna') == expected
--- /dev/null
+from mock import patch, DEFAULT
+from pytest import raises
+
+from teuthology.config import FakeNamespace
+from teuthology.orchestra.cluster import Cluster
+from teuthology.orchestra.remote import Remote
+from teuthology.task import Task
+
+
+class TestTask(object):
+ klass = Task
+ task_name = 'task'
+
+ def setup_method(self):
+ self.ctx = FakeNamespace()
+ self.ctx.config = dict()
+ self.task_config = dict()
+
+ def test_overrides(self):
+ self.ctx.config['overrides'] = dict()
+ self.ctx.config['overrides'][self.task_name] = dict(
+ key_1='overridden',
+ )
+ self.task_config.update(dict(
+ key_1='default',
+ key_2='default',
+ ))
+ with patch.multiple(
+ self.klass,
+ begin=DEFAULT,
+ end=DEFAULT,
+ ):
+ with self.klass(self.ctx, self.task_config) as task:
+ assert task.config['key_1'] == 'overridden'
+ assert task.config['key_2'] == 'default'
+
+ def test_hosts_no_filter(self):
+ self.ctx.cluster = Cluster()
+ self.ctx.cluster.add(Remote('user@remote1'), ['role1'])
+ self.ctx.cluster.add(Remote('user@remote2'), ['role2'])
+ with patch.multiple(
+ self.klass,
+ begin=DEFAULT,
+ end=DEFAULT,
+ ):
+ with self.klass(self.ctx, self.task_config) as task:
+ task_hosts = list(task.cluster.remotes)
+ assert len(task_hosts) == 2
+ assert sorted(host.shortname for host in task_hosts) == \
+ ['remote1', 'remote2']
+
+ def test_hosts_no_results(self):
+ self.ctx.cluster = Cluster()
+ self.ctx.cluster.add(Remote('user@remote1'), ['role1'])
+ self.task_config.update(dict(
+ hosts=['role2'],
+ ))
+ with patch.multiple(
+ self.klass,
+ begin=DEFAULT,
+ end=DEFAULT,
+ ):
+ with raises(RuntimeError):
+ with self.klass(self.ctx, self.task_config):
+ pass
+
+ def test_hosts_one_role(self):
+ self.ctx.cluster = Cluster()
+ self.ctx.cluster.add(Remote('user@remote1'), ['role1'])
+ self.ctx.cluster.add(Remote('user@remote2'), ['role2'])
+ self.task_config.update(dict(
+ hosts=['role1'],
+ ))
+ with patch.multiple(
+ self.klass,
+ begin=DEFAULT,
+ end=DEFAULT,
+ ):
+ with self.klass(self.ctx, self.task_config) as task:
+ task_hosts = list(task.cluster.remotes)
+ assert len(task_hosts) == 1
+ assert task_hosts[0].shortname == 'remote1'
+
+ def test_hosts_two_roles(self):
+ self.ctx.cluster = Cluster()
+ self.ctx.cluster.add(Remote('user@remote1'), ['role1'])
+ self.ctx.cluster.add(Remote('user@remote2'), ['role2'])
+ self.ctx.cluster.add(Remote('user@remote3'), ['role3'])
+ self.task_config.update(dict(
+ hosts=['role1', 'role3'],
+ ))
+ with patch.multiple(
+ self.klass,
+ begin=DEFAULT,
+ end=DEFAULT,
+ ):
+ with self.klass(self.ctx, self.task_config) as task:
+ task_hosts = list(task.cluster.remotes)
+ assert len(task_hosts) == 2
+ hostnames = [host.shortname for host in task_hosts]
+ assert sorted(hostnames) == ['remote1', 'remote3']
+
+ def test_hosts_two_hostnames(self):
+ self.ctx.cluster = Cluster()
+ self.ctx.cluster.add(Remote('user@remote1.example.com'), ['role1'])
+ self.ctx.cluster.add(Remote('user@remote2.example.com'), ['role2'])
+ self.ctx.cluster.add(Remote('user@remote3.example.com'), ['role3'])
+ self.task_config.update(dict(
+ hosts=['remote1', 'remote2.example.com'],
+ ))
+ with patch.multiple(
+ self.klass,
+ begin=DEFAULT,
+ end=DEFAULT,
+ ):
+ with self.klass(self.ctx, self.task_config) as task:
+ task_hosts = list(task.cluster.remotes)
+ assert len(task_hosts) == 2
+ hostnames = [host.hostname for host in task_hosts]
+ assert sorted(hostnames) == ['remote1.example.com',
+ 'remote2.example.com']
+
+ def test_hosts_one_role_one_hostname(self):
+ self.ctx.cluster = Cluster()
+ self.ctx.cluster.add(Remote('user@remote1.example.com'), ['role1'])
+ self.ctx.cluster.add(Remote('user@remote2.example.com'), ['role2'])
+ self.ctx.cluster.add(Remote('user@remote3.example.com'), ['role3'])
+ self.task_config.update(dict(
+ hosts=['role1', 'remote2.example.com'],
+ ))
+ with patch.multiple(
+ self.klass,
+ begin=DEFAULT,
+ end=DEFAULT,
+ ):
+ with self.klass(self.ctx, self.task_config) as task:
+ task_hosts = list(task.cluster.remotes)
+ assert len(task_hosts) == 2
+ hostnames = [host.hostname for host in task_hosts]
+ assert sorted(hostnames) == ['remote1.example.com',
+ 'remote2.example.com']
+
+ def test_setup_called(self):
+ with patch.multiple(
+ self.klass,
+ setup=DEFAULT,
+ begin=DEFAULT,
+ end=DEFAULT,
+ teardown=DEFAULT,
+ ):
+ with self.klass(self.ctx, self.task_config) as task:
+ task.setup.assert_called_once_with()
+
+ def test_begin_called(self):
+ with patch.multiple(
+ self.klass,
+ setup=DEFAULT,
+ begin=DEFAULT,
+ end=DEFAULT,
+ teardown=DEFAULT,
+ ):
+ with self.klass(self.ctx, self.task_config) as task:
+ task.begin.assert_called_once_with()
+
+ def test_end_called(self):
+ self.task_config.update(dict())
+ with patch.multiple(
+ self.klass,
+ begin=DEFAULT,
+ end=DEFAULT,
+ ):
+ with self.klass(self.ctx, self.task_config) as task:
+ pass
+ task.end.assert_called_once_with()
+
+ def test_teardown_called(self):
+ self.task_config.update(dict())
+ with patch.multiple(
+ self.klass,
+ setup=DEFAULT,
+ begin=DEFAULT,
+ end=DEFAULT,
+ teardown=DEFAULT,
+ ):
+ with self.klass(self.ctx, self.task_config) as task:
+ pass
+ task.teardown.assert_called_once_with()
+
+ def test_skip_teardown(self):
+ self.task_config.update(dict(
+ skip_teardown=True,
+ ))
+
+ def fake_teardown(self):
+ assert False
+
+ with patch.multiple(
+ self.klass,
+ setup=DEFAULT,
+ begin=DEFAULT,
+ end=DEFAULT,
+ teardown=fake_teardown,
+ ):
+ with self.klass(self.ctx, self.task_config):
+ pass
--- /dev/null
+import json
+import os
+import yaml
+
+from unittest.mock import patch, DEFAULT, Mock, mock_open
+from pytest import raises, mark
+from teuthology.util.compat import PY3
+if PY3:
+ from io import StringIO as StringIO
+else:
+ from io import BytesIO as StringIO
+
+from teuthology.config import config, FakeNamespace
+from teuthology.exceptions import CommandFailedError
+from teuthology.orchestra.cluster import Cluster
+from teuthology.orchestra.remote import Remote
+from teuthology.task import ansible
+from teuthology.task.ansible import Ansible, CephLab, FailureAnalyzer
+
+from tests.task import TestTask
+
+
+class TestFailureAnalyzer:
+ klass = FailureAnalyzer
+
+ @mark.parametrize(
+ 'line,result',
+ [
+ [
+ "W: --force-yes is deprecated, use one of the options starting with --allow instead.",
+ "",
+ ],
+ [
+ "E: Unable to fetch some archives, maybe run apt-get update or try with --fix-missing?",
+ "",
+ ],
+ [
+ "E: Failed to fetch http://security.ubuntu.com/ubuntu/pool/main/a/apache2/apache2-bin_2.4.41-4ubuntu3.14_amd64.deb Unable to connect to archive.ubuntu.com:http:",
+ "Unable to connect to archive.ubuntu.com:http:"
+ ],
+ [
+ "E: Failed to fetch http://archive.ubuntu.com/ubuntu/pool/main/libb/libb-hooks-op-check-perl/libb-hooks-op-check-perl_0.22-1build2_amd64.deb Temporary failure resolving 'archive.ubuntu.com'",
+ "Temporary failure resolving 'archive.ubuntu.com'"
+ ],
+ [
+ "Data could not be sent to remote host \"smithi068.front.sepia.ceph.com\".",
+ "Data could not be sent to remote host \"smithi068.front.sepia.ceph.com\"."
+ ],
+ [
+ "Permissions 0644 for '/root/.ssh/id_rsa' are too open.",
+ "Permissions 0644 for '/root/.ssh/id_rsa' are too open."
+ ],
+ ]
+ )
+ def test_lines(self, line, result):
+ obj = self.klass()
+ assert obj.analyze_line(line) == result
+
+
+class TestAnsibleTask(TestTask):
+ klass = Ansible
+ task_name = 'ansible'
+
+ def setup_method(self):
+ self.ctx = FakeNamespace()
+ self.ctx.cluster = Cluster()
+ self.ctx.cluster.add(Remote('user@remote1'), ['role1'])
+ self.ctx.cluster.add(Remote('user@remote2'), ['role2'])
+ self.ctx.config = dict()
+ self.ctx.summary = dict()
+ self.ctx.archive = ""
+ self.task_config = dict(playbook=[])
+ self.start_patchers()
+
+ def start_patchers(self):
+ self.patchers = dict()
+ self.mocks = dict()
+ self.patchers['mkdtemp'] = patch(
+ 'teuthology.task.ansible.mkdtemp', return_value='/tmp/'
+ )
+ m_NTF = Mock()
+ m_file = Mock()
+ m_file.name = 'file_name'
+ m_NTF.return_value = m_file
+ self.patchers['NTF'] = patch(
+ 'teuthology.task.ansible.NamedTemporaryFile',
+ m_NTF,
+ )
+ self.patchers['file'] = patch(
+ 'teuthology.task.ansible.open', create=True)
+ self.patchers['os_mkdir'] = patch(
+ 'teuthology.task.ansible.os.mkdir',
+ )
+ self.patchers['os_remove'] = patch(
+ 'teuthology.task.ansible.os.remove',
+ )
+ self.patchers['shutil_rmtree'] = patch(
+ 'teuthology.task.ansible.shutil.rmtree',
+ )
+ for name in self.patchers.keys():
+ self.start_patcher(name)
+
+ def start_patcher(self, name):
+ if name not in self.mocks.keys():
+ self.mocks[name] = self.patchers[name].start()
+
+ def teardown_method(self, method):
+ self.stop_patchers()
+
+ def stop_patchers(self):
+ for name in list(self.mocks):
+ self.stop_patcher(name)
+
+ def stop_patcher(self, name):
+ self.patchers[name].stop()
+ del self.mocks[name]
+
+ def test_setup(self):
+ self.task_config.update(dict(
+ playbook=[]
+ ))
+
+ def fake_get_playbook(self):
+ self.playbook_file = 'fake'
+
+ with patch.multiple(
+ self.klass,
+ find_repo=DEFAULT,
+ get_playbook=fake_get_playbook,
+ get_inventory=DEFAULT,
+ generate_inventory=DEFAULT,
+ generate_playbook=Mock(side_effect=Exception),
+ ):
+ task = self.klass(self.ctx, self.task_config)
+ task.setup()
+
+ def test_setup_generate_playbook(self):
+ self.task_config.update(dict(
+ playbook=[]
+ ))
+ with patch.multiple(
+ self.klass,
+ find_repo=DEFAULT,
+ get_playbook=DEFAULT,
+ get_inventory=DEFAULT,
+ generate_inventory=DEFAULT,
+ generate_playbook=DEFAULT,
+ ):
+ task = self.klass(self.ctx, self.task_config)
+ task.setup()
+ task.generate_playbook.assert_called_once_with()
+
+ def test_find_repo_path(self):
+ self.task_config.update(dict(
+ repo='~/my/repo',
+ ))
+ task = self.klass(self.ctx, self.task_config)
+ task.find_repo()
+ assert task.repo_path == os.path.expanduser(self.task_config['repo'])
+
+ @patch('teuthology.repo_utils.fetch_repo')
+ def test_find_repo_path_remote(self, m_fetch_repo):
+ self.task_config.update(dict(
+ repo='git://fake_host/repo.git',
+ ))
+ m_fetch_repo.return_value = '/tmp/repo'
+ task = self.klass(self.ctx, self.task_config)
+ task.find_repo()
+ assert task.repo_path == os.path.expanduser('/tmp/repo')
+
+ @patch('teuthology.repo_utils.fetch_repo')
+ def test_find_repo_http(self, m_fetch_repo):
+ self.task_config.update(dict(
+ repo='http://example.com/my/repo',
+ ))
+ task = self.klass(self.ctx, self.task_config)
+ task.find_repo()
+ m_fetch_repo.assert_called_once_with(self.task_config['repo'],
+ 'main')
+
+ @patch('teuthology.repo_utils.fetch_repo')
+ def test_find_repo_git(self, m_fetch_repo):
+ self.task_config.update(dict(
+ repo='git@example.com/my/repo',
+ ))
+ task = self.klass(self.ctx, self.task_config)
+ task.find_repo()
+ m_fetch_repo.assert_called_once_with(self.task_config['repo'],
+ 'main')
+
+ def test_playbook_none(self):
+ del self.task_config['playbook']
+ task = self.klass(self.ctx, self.task_config)
+ with raises(KeyError):
+ task.get_playbook()
+
+ def test_playbook_wrong_type(self):
+ self.task_config.update(dict(
+ playbook=dict(),
+ ))
+ task = self.klass(self.ctx, self.task_config)
+ with raises(TypeError):
+ task.get_playbook()
+
+ def test_playbook_list(self):
+ playbook = [
+ dict(
+ roles=['role1'],
+ ),
+ ]
+ self.task_config.update(dict(
+ playbook=playbook,
+ ))
+ task = self.klass(self.ctx, self.task_config)
+ task.get_playbook()
+ assert task.playbook == playbook
+
+ @patch.object(ansible.requests, 'get')
+ def test_playbook_http(self, m_get):
+ m_get.return_value = Mock()
+ m_get.return_value.text = 'fake playbook text'
+ playbook = "http://example.com/my_playbook.yml"
+ self.task_config.update(dict(
+ playbook=playbook,
+ ))
+ task = self.klass(self.ctx, self.task_config)
+ task.get_playbook()
+ m_get.assert_called_once_with(playbook)
+
+ def test_playbook_file(self):
+ fake_playbook = [dict(fake_playbook=True)]
+ fake_playbook_obj = StringIO(yaml.safe_dump(fake_playbook))
+ self.task_config.update(dict(
+ playbook='~/fake/playbook',
+ ))
+ task = self.klass(self.ctx, self.task_config)
+ self.mocks['file'].return_value = fake_playbook_obj
+ task.get_playbook()
+ assert task.playbook == fake_playbook
+
+ def test_playbook_file_missing(self):
+ self.task_config.update(dict(
+ playbook='~/fake/playbook',
+ ))
+ task = self.klass(self.ctx, self.task_config)
+ self.mocks['file'].side_effect = IOError
+ with raises(IOError):
+ task.get_playbook()
+
+ def test_inventory_none(self):
+ self.task_config.update(dict(
+ playbook=[]
+ ))
+ task = self.klass(self.ctx, self.task_config)
+ with patch.object(ansible.os.path, 'exists') as m_exists:
+ m_exists.return_value = False
+ task.get_inventory()
+ assert task.inventory is None
+
+ def test_inventory_path(self):
+ inventory = '/my/inventory'
+ self.task_config.update(dict(
+ playbook=[],
+ inventory=inventory,
+ ))
+ task = self.klass(self.ctx, self.task_config)
+ task.get_inventory()
+ assert task.inventory == inventory
+ assert task.generated_inventory is False
+
+ def test_inventory_etc(self):
+ self.task_config.update(dict(
+ playbook=[]
+ ))
+ task = self.klass(self.ctx, self.task_config)
+ with patch.object(ansible.os.path, 'exists') as m_exists:
+ m_exists.return_value = True
+ task.get_inventory()
+ assert task.inventory == '/etc/ansible/hosts'
+ assert task.generated_inventory is False
+
+ @mark.parametrize(
+ 'group_vars',
+ [
+ dict(),
+ dict(all=dict(var0=0, var1=1)),
+ dict(foo=dict(var0=0), bar=dict(var0=1)),
+ ]
+ )
+ def test_generate_inventory(self, group_vars):
+ self.task_config.update(dict(
+ playbook=[]
+ ))
+ if group_vars:
+ self.task_config.update(dict(group_vars=group_vars))
+ task = self.klass(self.ctx, self.task_config)
+ hosts_file_path = '/my/hosts/inventory'
+ hosts_file_obj = StringIO()
+ hosts_file_obj.name = hosts_file_path
+ inventory_dir = os.path.dirname(hosts_file_path)
+ gv_dir = os.path.join(inventory_dir, 'group_vars')
+ self.mocks['mkdtemp'].return_value = inventory_dir
+ m_file = self.mocks['file']
+ fake_files = [hosts_file_obj]
+ # Create StringIO object for each group_vars file
+ if group_vars:
+ fake_files += [StringIO() for i in sorted(group_vars)]
+ m_file.side_effect = fake_files
+ task.generate_inventory()
+ file_calls = m_file.call_args_list
+ # Verify the inventory file was created
+ assert file_calls[0][0][0] == hosts_file_path
+ # Verify each group_vars file was created
+ for gv_name, call_obj in zip(sorted(group_vars), file_calls[1:]):
+ gv_path = call_obj[0][0]
+ assert gv_path == os.path.join(gv_dir, '%s.yml' % gv_name)
+ # Verify the group_vars dir was created
+ if group_vars:
+ mkdir_call = self.mocks['os_mkdir'].call_args_list
+ assert mkdir_call[0][0][0] == gv_dir
+ assert task.generated_inventory is True
+ assert task.inventory == inventory_dir
+ # Verify the content of the inventory *file*
+ hosts_file_obj.seek(0)
+ assert hosts_file_obj.readlines() == [
+ 'remote1\n',
+ 'remote2\n',
+ ]
+ # Verify the contents of each group_vars file
+ gv_names = sorted(group_vars)
+ for i in range(len(gv_names)):
+ gv_name = gv_names[i]
+ in_val = group_vars[gv_name]
+ gv_stringio = fake_files[1 + i]
+ gv_stringio.seek(0)
+ out_val = yaml.safe_load(gv_stringio)
+ assert in_val == out_val
+
+ def test_generate_playbook(self):
+ playbook = [
+ dict(
+ roles=['role1', 'role2'],
+ ),
+ ]
+ self.task_config.update(dict(
+ playbook=playbook
+ ))
+ task = self.klass(self.ctx, self.task_config)
+ playbook_file_path = '/my/playbook/file'
+ playbook_file_obj = StringIO()
+ playbook_file_obj.name = playbook_file_path
+ with patch.object(ansible, 'NamedTemporaryFile') as m_NTF:
+ m_NTF.return_value = playbook_file_obj
+ task.find_repo()
+ task.get_playbook()
+ task.generate_playbook()
+ m_NTF.assert_called_once_with(
+ prefix="teuth_ansible_playbook_",
+ dir=task.repo_path,
+ delete=False,
+ )
+ assert task.generated_playbook is True
+ assert task.playbook_file == playbook_file_obj
+ playbook_file_obj.seek(0)
+ playbook_result = yaml.safe_load(playbook_file_obj)
+ assert playbook_result == playbook
+
+ def test_execute_playbook(self):
+ playbook = '/my/playbook'
+ self.task_config.update(dict(
+ playbook=playbook
+ ))
+ fake_playbook = [dict(fake_playbook=True)]
+ fake_playbook_obj = StringIO(yaml.safe_dump(fake_playbook))
+ fake_playbook_obj.name = playbook
+ self.mocks['mkdtemp'].return_value = '/inventory/dir'
+
+ task = self.klass(self.ctx, self.task_config)
+ self.mocks['file'].return_value = fake_playbook_obj
+ task.setup()
+ args = task._build_args()
+ logger = StringIO()
+ with patch.object(ansible.pexpect, 'run') as m_run:
+ m_run.return_value = ('', 0)
+ with patch.object(Remote, 'reconnect') as m_reconnect:
+ m_reconnect.return_value = True
+ task.execute_playbook(_logfile=logger)
+ m_run.assert_called_once_with(
+ ' '.join(args),
+ cwd=task.repo_path,
+ logfile=logger,
+ withexitstatus=True,
+ timeout=None,
+ )
+
+ def test_execute_playbook_fail(self):
+ self.task_config.update(dict(
+ playbook=[],
+ ))
+ self.mocks['mkdtemp'].return_value = '/inventory/dir'
+ task = self.klass(self.ctx, self.task_config)
+ task.setup()
+ with patch.object(ansible.pexpect, 'run') as m_run:
+ with patch('teuthology.task.ansible.open', mock_open()):
+ m_run.return_value = ('', 1)
+ with raises(CommandFailedError):
+ task.execute_playbook()
+ assert task.ctx.summary.get('status') is None
+
+ def test_build_args_no_tags(self):
+ self.task_config.update(dict(
+ playbook=[],
+ ))
+ task = self.klass(self.ctx, self.task_config)
+ task.setup()
+ args = task._build_args()
+ assert '--tags' not in args
+
+ def test_build_args_tags(self):
+ self.task_config.update(dict(
+ playbook=[],
+ tags="user,pubkeys"
+ ))
+ task = self.klass(self.ctx, self.task_config)
+ task.setup()
+ args = task._build_args()
+ assert args.count('--tags') == 1
+ assert args[args.index('--tags') + 1] == 'user,pubkeys'
+
+ def test_build_args_skip_tags(self):
+ self.task_config.update(dict(
+ playbook=[],
+ skip_tags="user,pubkeys"
+ ))
+ task = self.klass(self.ctx, self.task_config)
+ task.setup()
+ args = task._build_args()
+ assert args.count('--skip-tags') == 1
+ assert args[args.index('--skip-tags') + 1] == 'user,pubkeys'
+
+ def test_build_args_no_vars(self):
+ self.task_config.update(dict(
+ playbook=[],
+ ))
+ task = self.klass(self.ctx, self.task_config)
+ task.setup()
+ args = task._build_args()
+ assert args.count('--extra-vars') == 1
+ vars_str = args[args.index('--extra-vars') + 1].strip("'")
+ extra_vars = json.loads(vars_str)
+ assert list(extra_vars) == ['ansible_ssh_user']
+
+ def test_build_args_vars(self):
+ extra_vars = dict(
+ string1='value1',
+ list1=['item1'],
+ dict1=dict(key='value'),
+ )
+
+ self.task_config.update(dict(
+ playbook=[],
+ vars=extra_vars,
+ ))
+ task = self.klass(self.ctx, self.task_config)
+ task.setup()
+ args = task._build_args()
+ assert args.count('--extra-vars') == 1
+ vars_str = args[args.index('--extra-vars') + 1].strip("'")
+ got_extra_vars = json.loads(vars_str)
+ assert 'ansible_ssh_user' in got_extra_vars
+ assert got_extra_vars['string1'] == extra_vars['string1']
+ assert got_extra_vars['list1'] == extra_vars['list1']
+ assert got_extra_vars['dict1'] == extra_vars['dict1']
+
+ def test_teardown_inventory(self):
+ self.task_config.update(dict(
+ playbook=[],
+ ))
+ task = self.klass(self.ctx, self.task_config)
+ task.generated_inventory = True
+ task.inventory = 'fake'
+ with patch.object(ansible.shutil, 'rmtree') as m_rmtree:
+ task.teardown()
+ m_rmtree.assert_called_once_with('fake')
+
+ def test_teardown_playbook(self):
+ self.task_config.update(dict(
+ playbook=[],
+ ))
+ task = self.klass(self.ctx, self.task_config)
+ task.generated_playbook = True
+ task.playbook_file = Mock()
+ task.playbook_file.name = 'fake'
+ with patch.object(ansible.os, 'remove') as m_remove:
+ task.teardown()
+ m_remove.assert_called_once_with('fake')
+
+ def test_teardown_cleanup_with_vars(self):
+ self.task_config.update(dict(
+ playbook=[],
+ cleanup=True,
+ vars=dict(yum_repos="testing"),
+ ))
+ task = self.klass(self.ctx, self.task_config)
+ task.inventory = "fake"
+ task.generated_playbook = True
+ task.playbook_file = Mock()
+ task.playbook_file.name = 'fake'
+ with patch.object(self.klass, 'execute_playbook') as m_execute:
+ with patch.object(ansible.os, 'remove'):
+ task.teardown()
+ task._build_args()
+ assert m_execute.called
+ assert 'cleanup' in task.config['vars']
+ assert 'yum_repos' in task.config['vars']
+
+ def test_teardown_cleanup_with_no_vars(self):
+ self.task_config.update(dict(
+ playbook=[],
+ cleanup=True,
+ ))
+ task = self.klass(self.ctx, self.task_config)
+ task.inventory = "fake"
+ task.generated_playbook = True
+ task.playbook_file = Mock()
+ task.playbook_file.name = 'fake'
+ with patch.object(self.klass, 'execute_playbook') as m_execute:
+ with patch.object(ansible.os, 'remove'):
+ task.teardown()
+ task._build_args()
+ assert m_execute.called
+ assert 'cleanup' in task.config['vars']
+
+ def test_no_remotes(self):
+ self.task_config.update(dict(
+ playbook=[],
+ ))
+ self.ctx.cluster.remotes = dict()
+ task = self.klass(self.ctx, self.task_config)
+ with patch.object(ansible.pexpect, 'run') as m_run:
+ task.setup()
+ task.begin()
+ assert not m_run.called
+
+
+class TestCephLabTask(TestAnsibleTask):
+ klass = CephLab
+ task_name = 'ansible.cephlab'
+
+ def setup_method(self):
+ super(TestCephLabTask, self).setup_method()
+ self.task_config = dict()
+
+ def start_patchers(self):
+ super(TestCephLabTask, self).start_patchers()
+ self.patchers['fetch_repo'] = patch(
+ 'teuthology.repo_utils.fetch_repo',
+ )
+ self.patchers['fetch_repo'].return_value = 'PATH'
+
+ def fake_get_playbook(self):
+ self.playbook_file = Mock()
+ self.playbook_file.name = 'cephlab.yml'
+
+ self.patchers['get_playbook'] = patch(
+ 'teuthology.task.ansible.CephLab.get_playbook',
+ new=fake_get_playbook,
+ )
+ for name in self.patchers.keys():
+ self.start_patcher(name)
+
+ @patch('teuthology.repo_utils.fetch_repo')
+ def test_find_repo_http(self, m_fetch_repo):
+ repo = os.path.join(config.ceph_git_base_url,
+ 'ceph-cm-ansible.git')
+ task = self.klass(self.ctx, dict())
+ task.find_repo()
+ m_fetch_repo.assert_called_once_with(repo, 'main')
+
+ def test_playbook_file(self):
+ fake_playbook = [dict(fake_playbook=True)]
+ fake_playbook_obj = StringIO(yaml.safe_dump(fake_playbook))
+ playbook = 'cephlab.yml'
+ fake_playbook_obj.name = playbook
+ task = self.klass(self.ctx, dict())
+ task.repo_path = '/tmp/fake/repo'
+ self.mocks['file'].return_value = fake_playbook_obj
+ task.get_playbook()
+ assert task.playbook_file.name == playbook
+
+ def test_generate_inventory(self):
+ self.task_config.update(dict(
+ playbook=[]
+ ))
+ task = self.klass(self.ctx, self.task_config)
+ hosts_file_path = '/my/hosts/file'
+ hosts_file_obj = StringIO()
+ hosts_file_obj.name = hosts_file_path
+ self.mocks['mkdtemp'].return_value = os.path.dirname(hosts_file_path)
+ self.mocks['file'].return_value = hosts_file_obj
+ task.generate_inventory()
+ assert task.generated_inventory is True
+ assert task.inventory == os.path.dirname(hosts_file_path)
+ hosts_file_obj.seek(0)
+ assert hosts_file_obj.readlines() == [
+ '[testnodes]\n',
+ 'remote1\n',
+ 'remote2\n',
+ ]
+
+ def test_fail_status_dead(self):
+ self.task_config.update(dict(
+ playbook=[],
+ ))
+ self.mocks['mkdtemp'].return_value = '/inventory/dir'
+ task = self.klass(self.ctx, self.task_config)
+ task.ctx.summary = dict()
+ task.setup()
+ with patch.object(ansible.pexpect, 'run') as m_run:
+ with patch('teuthology.task.ansible.open', mock_open()):
+ m_run.return_value = ('', 1)
+ with raises(CommandFailedError):
+ task.execute_playbook()
+ assert task.ctx.summary.get('status') == 'dead'
+
+ def test_execute_playbook_fail(self):
+ self.mocks['mkdtemp'].return_value = '/inventory/dir'
+ task = self.klass(self.ctx, self.task_config)
+ task.setup()
+ with patch.object(ansible.pexpect, 'run') as m_run:
+ with patch('teuthology.task.ansible.open', mock_open()):
+ m_run.return_value = ('', 1)
+ with raises(CommandFailedError):
+ task.execute_playbook()
+ assert task.ctx.summary.get('status') == 'dead'
+
+ @mark.skip("Unsupported")
+ def test_generate_playbook(self):
+ pass
+
+ @mark.skip("Unsupported")
+ def test_playbook_http(self):
+ pass
+
+ @mark.skip("Unsupported")
+ def test_playbook_none(self):
+ pass
+
+ @mark.skip("Unsupported")
+ def test_playbook_wrong_type(self):
+ pass
+
+ @mark.skip("Unsupported")
+ def test_playbook_list(self):
+ pass
+
+ @mark.skip("Test needs to be reimplemented for this class")
+ def test_playbook_file_missing(self):
+ pass
--- /dev/null
+from mock import patch, MagicMock
+from pytest import skip
+from teuthology.util.compat import PY3
+if PY3:
+ from io import StringIO as StringIO
+else:
+ from io import BytesIO as StringIO
+
+from teuthology.config import FakeNamespace
+from teuthology.orchestra.cluster import Cluster
+from teuthology.orchestra.remote import Remote
+from teuthology.task import ceph_ansible
+from teuthology.task.ceph_ansible import CephAnsible
+
+from tests.task import TestTask
+
+SKIP_IRRELEVANT = "Not relevant to this subclass"
+
+
+class TestCephAnsibleTask(TestTask):
+ klass = CephAnsible
+ task_name = 'ceph_ansible'
+
+ def setup_method(self):
+ self.ctx = FakeNamespace()
+ self.ctx.cluster = Cluster()
+ self.ctx.cluster.add(Remote('user@remote1'), ['mon.0'])
+ self.ctx.cluster.add(Remote('user@remote2'), ['mds.0'])
+ self.ctx.cluster.add(Remote('user@remote3'), ['osd.0'])
+ self.ctx.summary = dict()
+ self.ctx.config = dict()
+ self.ctx.archive = '../'
+ self.task_config = dict()
+ self.start_patchers()
+
+ def start_patchers(self):
+ m_fetch_repo = MagicMock()
+ m_fetch_repo.return_value = 'PATH'
+
+ def fake_get_scratch_devices(remote):
+ return ['/dev/%s' % remote.shortname]
+
+ self.patcher_get_scratch_devices = patch(
+ 'teuthology.task.ceph_ansible.get_scratch_devices',
+ fake_get_scratch_devices,
+ )
+ self.patcher_get_scratch_devices.start()
+
+ self.patcher_teardown = patch(
+ 'teuthology.task.ceph_ansible.CephAnsible.teardown',
+ )
+ self.patcher_teardown.start()
+
+ def fake_set_iface_and_cidr(self):
+ self._interface = 'eth0'
+ self._cidr = '172.21.0.0/20'
+
+ self.patcher_remote = patch.multiple(
+ Remote,
+ _set_iface_and_cidr=fake_set_iface_and_cidr,
+ )
+ self.patcher_remote.start()
+
+ def stop_patchers(self):
+ self.patcher_get_scratch_devices.stop()
+ self.patcher_remote.stop()
+ self.patcher_teardown.stop()
+
+ def test_playbook_none(self):
+ skip(SKIP_IRRELEVANT)
+
+ def test_inventory_none(self):
+ skip(SKIP_IRRELEVANT)
+
+ def test_inventory_path(self):
+ skip(SKIP_IRRELEVANT)
+
+ def test_inventory_etc(self):
+ skip(SKIP_IRRELEVANT)
+
+ def test_generate_hosts_file(self):
+ self.task_config.update(dict(
+ playbook=[],
+ vars=dict(
+ osd_auto_discovery=True,
+ monitor_interface='eth0',
+ radosgw_interface='eth0',
+ public_network='172.21.0.0/20',
+ ),
+ ))
+ task = self.klass(self.ctx, self.task_config)
+ hosts_file_path = '/my/hosts/file'
+ hosts_file_obj = StringIO()
+ hosts_file_obj.name = hosts_file_path
+ with patch.object(ceph_ansible, 'NamedTemporaryFile') as m_NTF:
+ m_NTF.return_value = hosts_file_obj
+ task.generate_hosts_file()
+ m_NTF.assert_called_once_with(prefix="teuth_ansible_hosts_",
+ mode='w+',
+ delete=False)
+ assert task.generated_inventory is True
+ assert task.inventory == hosts_file_path
+ hosts_file_obj.seek(0)
+ assert hosts_file_obj.read() == '\n'.join([
+ '[mdss]',
+ 'remote2',
+ '',
+ '[mons]',
+ 'remote1',
+ '',
+ '[osds]',
+ 'remote3',
+ ])
+
+ def test_generate_hosts_file_with_devices(self):
+ self.task_config.update(dict(
+ playbook=[],
+ vars=dict(
+ monitor_interface='eth0',
+ radosgw_interface='eth0',
+ public_network='172.21.0.0/20',
+ ),
+ ))
+ task = self.klass(self.ctx, self.task_config)
+ hosts_file_path = '/my/hosts/file'
+ hosts_file_obj = StringIO()
+ hosts_file_obj.name = hosts_file_path
+ with patch.object(ceph_ansible, 'NamedTemporaryFile') as m_NTF:
+ m_NTF.return_value = hosts_file_obj
+ task.generate_hosts_file()
+ m_NTF.assert_called_once_with(prefix="teuth_ansible_hosts_",
+ mode='w+',
+ delete=False)
+ assert task.generated_inventory is True
+ assert task.inventory == hosts_file_path
+ hosts_file_obj.seek(0)
+ assert hosts_file_obj.read() == '\n'.join([
+ '[mdss]',
+ 'remote2 devices=\'[]\'',
+ '',
+ '[mons]',
+ 'remote1 devices=\'[]\'',
+ '',
+ '[osds]',
+ 'remote3 devices=\'["/dev/remote3"]\'',
+ ])
+
+ def test_generate_hosts_file_with_network(self):
+ self.task_config.update(dict(
+ playbook=[],
+ vars=dict(
+ osd_auto_discovery=True,
+ ),
+ ))
+ task = self.klass(self.ctx, self.task_config)
+ hosts_file_path = '/my/hosts/file'
+ hosts_file_obj = StringIO()
+ hosts_file_obj.name = hosts_file_path
+ with patch.object(ceph_ansible, 'NamedTemporaryFile') as m_NTF:
+ m_NTF.return_value = hosts_file_obj
+ task.generate_hosts_file()
+ m_NTF.assert_called_once_with(prefix="teuth_ansible_hosts_",
+ mode='w+',
+ delete=False)
+ assert task.generated_inventory is True
+ assert task.inventory == hosts_file_path
+ hosts_file_obj.seek(0)
+ assert hosts_file_obj.read() == '\n'.join([
+ '[mdss]',
+ "remote2 monitor_interface='eth0' public_network='172.21.0.0/20' radosgw_interface='eth0'",
+ '',
+ '[mons]',
+ "remote1 monitor_interface='eth0' public_network='172.21.0.0/20' radosgw_interface='eth0'",
+ '',
+ '[osds]',
+ "remote3 monitor_interface='eth0' public_network='172.21.0.0/20' radosgw_interface='eth0'",
+ ])
--- /dev/null
+import os
+
+from mock import patch
+
+from teuthology.config import FakeNamespace
+from teuthology.config import config as teuth_config
+from teuthology.orchestra.cluster import Cluster
+from teuthology.orchestra.remote import Remote
+from teuthology.task.console_log import ConsoleLog
+
+from tests.task import TestTask
+
+
+class TestConsoleLog(TestTask):
+ klass = ConsoleLog
+ task_name = 'console_log'
+
+ def setup_method(self):
+ teuth_config.ipmi_domain = 'ipmi.domain'
+ teuth_config.ipmi_user = 'ipmi_user'
+ teuth_config.ipmi_password = 'ipmi_pass'
+ self.ctx = FakeNamespace()
+ self.ctx.cluster = Cluster()
+ self.ctx.cluster.add(Remote('user@remote1'), ['role1'])
+ self.ctx.cluster.add(Remote('user@remote2'), ['role2'])
+ self.ctx.config = dict()
+ self.ctx.archive = '/fake/path'
+ self.task_config = dict()
+ self.start_patchers()
+
+ def start_patchers(self):
+ self.patchers = dict()
+ self.patchers['makedirs'] = patch(
+ 'teuthology.task.console_log.os.makedirs',
+ )
+ self.patchers['is_vm'] = patch(
+ 'teuthology.lock.query.is_vm',
+ )
+ self.patchers['is_vm'].return_value = False
+ self.patchers['get_status'] = patch(
+ 'teuthology.lock.query.get_status',
+ )
+ self.mocks = dict()
+ for name, patcher in self.patchers.items():
+ self.mocks[name] = patcher.start()
+ self.mocks['is_vm'].return_value = False
+
+ def teardown_method(self):
+ for patcher in self.patchers.values():
+ patcher.stop()
+
+ def test_enabled(self):
+ task = self.klass(self.ctx, self.task_config)
+ assert task.enabled is True
+
+ def test_disabled_noarchive(self):
+ self.ctx.archive = None
+ task = self.klass(self.ctx, self.task_config)
+ assert task.enabled is False
+
+ def test_has_ipmi_credentials(self):
+ for remote in self.ctx.cluster.remotes.keys():
+ remote.console.has_ipmi_credentials = False
+ remote.console.has_conserver = False
+ task = self.klass(self.ctx, self.task_config)
+ assert len(task.cluster.remotes.keys()) == 0
+
+ def test_remotes(self):
+ with self.klass(self.ctx, self.task_config) as task:
+ assert len(task.cluster.remotes) == len(self.ctx.cluster.remotes)
+
+ @patch('teuthology.orchestra.console.PhysicalConsole')
+ def test_begin(self, m_pconsole):
+ with self.klass(self.ctx, self.task_config) as task:
+ assert len(task.processes) == len(self.ctx.cluster.remotes)
+ expected_log_paths = []
+ for remote in task.cluster.remotes.keys():
+ expected_log_paths.append(
+ os.path.join(self.ctx.archive, 'console_logs', '%s.log' % remote.shortname)
+ )
+ assert len(m_pconsole().spawn_sol_log.call_args_list) == len(task.cluster.remotes)
+ got_log_paths = [c[0][0] for c in m_pconsole().spawn_sol_log.call_args_list]
+ assert got_log_paths == expected_log_paths
+
+ @patch('teuthology.orchestra.console.PhysicalConsole')
+ def test_end(self, m_pconsole):
+ m_proc = m_pconsole().spawn_sol_log.return_value
+ m_proc.poll.return_value = None
+ with self.klass(self.ctx, self.task_config):
+ pass
+ assert len(m_proc.terminate.call_args_list) == len(self.ctx.cluster.remotes)
+ assert len(m_proc.kill.call_args_list) == len(self.ctx.cluster.remotes)
--- /dev/null
+import os
+import pytest
+import yaml
+
+from mock import patch, Mock
+from pytest import mark
+
+from teuthology.task import install
+
+
+class TestInstall(object):
+
+ def _get_default_package_list(self, project='ceph', debug=False):
+ path = os.path.join(
+ os.path.dirname(__file__),
+ '..', '..', 'teuthology', 'task', 'install', 'packages.yaml',
+ )
+ pkgs = yaml.safe_load(open(path))[project]
+ if not debug:
+ pkgs['deb'] = [p for p in pkgs['deb']
+ if not p.endswith('-dbg')]
+ pkgs['rpm'] = [p for p in pkgs['rpm']
+ if not p.endswith('-debuginfo')]
+ return pkgs
+
+ def test_get_package_list_debug(self):
+ default_pkgs = self._get_default_package_list(debug=True)
+ default_pkgs['rpm'].sort()
+ default_pkgs['deb'].sort()
+ config = dict(debuginfo=True)
+ result = install.get_package_list(ctx=None, config=config)
+ result['rpm'].sort()
+ result['deb'].sort()
+ assert result == default_pkgs
+
+ def test_get_package_list_no_debug(self):
+ default_pkgs = self._get_default_package_list(debug=False)
+ default_pkgs['rpm'].sort()
+ default_pkgs['deb'].sort()
+ config = dict(debuginfo=False)
+ result = install.get_package_list(ctx=None, config=config)
+ result['rpm'].sort()
+ result['deb'].sort()
+ assert result == default_pkgs
+
+ def test_get_package_list_custom_rpm(self):
+ default_pkgs = self._get_default_package_list(debug=False)
+ default_pkgs['rpm'].sort()
+ default_pkgs['deb'].sort()
+ rpms = ['rpm1', 'rpm2', 'rpm2-debuginfo']
+ config = dict(packages=dict(rpm=rpms))
+ result = install.get_package_list(ctx=None, config=config)
+ result['rpm'].sort()
+ result['deb'].sort()
+ assert result['rpm'] == ['rpm1', 'rpm2']
+ assert result['deb'] == default_pkgs['deb']
+
+ @patch("teuthology.task.install._get_builder_project")
+ @patch("teuthology.task.install.packaging.get_package_version")
+ def test_get_upgrade_version(self, m_get_package_version,
+ m_gitbuilder_project):
+ gb = Mock()
+ gb.version = "11.0.0"
+ gb.project = "ceph"
+ m_gitbuilder_project.return_value = gb
+ m_get_package_version.return_value = "11.0.0"
+ install.get_upgrade_version(Mock(), Mock(), Mock())
+
+ @patch("teuthology.task.install._get_builder_project")
+ @patch("teuthology.task.install.packaging.get_package_version")
+ def test_verify_ceph_version_success(self, m_get_package_version,
+ m_gitbuilder_project):
+ gb = Mock()
+ gb.version = "0.89.0"
+ gb.project = "ceph"
+ m_gitbuilder_project.return_value = gb
+ m_get_package_version.return_value = "0.89.0"
+ config = dict()
+ install.verify_package_version(Mock(), config, Mock())
+
+ @patch("teuthology.task.install._get_builder_project")
+ @patch("teuthology.task.install.packaging.get_package_version")
+ def test_verify_ceph_version_failed(self, m_get_package_version,
+ m_gitbuilder_project):
+ gb = Mock()
+ gb.version = "0.89.0"
+ gb.project = "ceph"
+ m_gitbuilder_project.return_value = gb
+ m_get_package_version.return_value = "0.89.1"
+ config = dict()
+ with pytest.raises(RuntimeError):
+ install.verify_package_version(Mock(), config, Mock())
+
+ def test_get_flavor_default(self):
+ config = dict()
+ assert install.get_flavor(config) == 'default'
+
+ def test_get_flavor_simple(self):
+ config = dict(
+ flavor='notcmalloc'
+ )
+ assert install.get_flavor(config) == 'notcmalloc'
+
+ def test_get_flavor_valgrind(self):
+ config = dict(
+ valgrind=True
+ )
+ assert install.get_flavor(config) == 'notcmalloc'
+
+ def test_upgrade_is_downgrade(self):
+ assert_ok_vals = [
+ ('9.0.0', '10.0.0'),
+ ('10.2.2-63-g8542898-1trusty', '10.2.2-64-gabcdef1-1trusty'),
+ ('11.0.0-918.g13c13c7', '11.0.0-2165.gabcdef1')
+ ]
+ for t in assert_ok_vals:
+ assert install._upgrade_is_downgrade(t[0], t[1]) == False
+
+ @patch("teuthology.packaging.get_package_version")
+ @patch("teuthology.misc.get_system_type")
+ @patch("teuthology.task.install.verify_package_version")
+ @patch("teuthology.task.install.get_upgrade_version")
+ def test_upgrade_common(self,
+ m_get_upgrade_version,
+ m_verify_package_version,
+ m_get_system_type,
+ m_get_package_version):
+ expected_system_type = 'deb'
+ def make_remote():
+ remote = Mock()
+ remote.arch = 'x86_64'
+ remote.os = Mock()
+ remote.os.name = 'ubuntu'
+ remote.os.version = '14.04'
+ remote.os.codename = 'trusty'
+ remote.system_type = expected_system_type
+ return remote
+ ctx = Mock()
+ class cluster:
+ remote1 = make_remote()
+ remote2 = make_remote()
+ remotes = {
+ remote1: ['client.0'],
+ remote2: ['mon.a','osd.0'],
+ }
+ def only(self, role):
+ result = Mock()
+ if role in ('client.0',):
+ result.remotes = { cluster.remote1: None }
+ if role in ('osd.0', 'mon.a'):
+ result.remotes = { cluster.remote2: None }
+ return result
+ ctx.cluster = cluster()
+ config = {
+ 'client.0': {
+ 'sha1': 'expectedsha1',
+ },
+ }
+ ctx.config = {
+ 'roles': [ ['client.0'], ['mon.a','osd.0'] ],
+ 'tasks': [
+ {
+ 'install.upgrade': config,
+ },
+ ],
+ }
+ m_get_upgrade_version.return_value = "11.0.0"
+ m_get_package_version.return_value = "10.2.4"
+ m_get_system_type.return_value = "deb"
+ def upgrade(ctx, node, remote, pkgs, system_type):
+ assert system_type == expected_system_type
+ assert install.upgrade_common(ctx, config, upgrade) == 1
+ expected_config = {
+ 'project': 'ceph',
+ 'sha1': 'expectedsha1',
+ }
+ m_verify_package_version.assert_called_with(ctx,
+ expected_config,
+ cluster.remote1)
+ def test_upgrade_remote_to_config(self):
+ expected_system_type = 'deb'
+ def make_remote():
+ remote = Mock()
+ remote.arch = 'x86_64'
+ remote.os = Mock()
+ remote.os.name = 'ubuntu'
+ remote.os.version = '14.04'
+ remote.os.codename = 'trusty'
+ remote.system_type = expected_system_type
+ return remote
+ ctx = Mock()
+ class cluster:
+ remote1 = make_remote()
+ remote2 = make_remote()
+ remotes = {
+ remote1: ['client.0'],
+ remote2: ['mon.a','osd.0'],
+ }
+ def only(self, role):
+ result = Mock()
+ if role in ('client.0',):
+ result.remotes = { cluster.remote1: None }
+ elif role in ('osd.0', 'mon.a'):
+ result.remotes = { cluster.remote2: None }
+ else:
+ result.remotes = None
+ return result
+ ctx.cluster = cluster()
+ ctx.config = {
+ 'roles': [ ['client.0'], ['mon.a','osd.0'] ],
+ }
+
+ # nothing -> nothing
+ assert install.upgrade_remote_to_config(ctx, {}) == {}
+
+ # select the remote for the osd.0 role
+ # the 'ignored' role does not exist and is ignored
+ # the remote for mon.a is the same as for osd.0 and
+ # is silently ignored (actually it could be the other
+ # way around, depending on how the keys are hashed)
+ config = {
+ 'osd.0': {
+ 'sha1': 'expectedsha1',
+ },
+ 'ignored': None,
+ 'mon.a': {
+ 'sha1': 'expectedsha1',
+ },
+ }
+ expected_config = {
+ cluster.remote2: {
+ 'project': 'ceph',
+ 'sha1': 'expectedsha1',
+ },
+ }
+ assert install.upgrade_remote_to_config(ctx, config) == expected_config
+
+ # select all nodes, regardless
+ config = {
+ 'all': {
+ 'sha1': 'expectedsha1',
+ },
+ }
+ expected_config = {
+ cluster.remote1: {
+ 'project': 'ceph',
+ 'sha1': 'expectedsha1',
+ },
+ cluster.remote2: {
+ 'project': 'ceph',
+ 'sha1': 'expectedsha1',
+ },
+ }
+ assert install.upgrade_remote_to_config(ctx, config) == expected_config
+
+ # verify that install overrides are used as default
+ # values for the upgrade task, not as override
+ ctx.config['overrides'] = {
+ 'install': {
+ 'ceph': {
+ 'sha1': 'overridesha1',
+ 'tag': 'overridetag',
+ 'branch': 'overridebranch',
+ },
+ },
+ }
+ config = {
+ 'client.0': {
+ 'sha1': 'expectedsha1',
+ },
+ 'osd.0': {
+ },
+ }
+ expected_config = {
+ cluster.remote1: {
+ 'project': 'ceph',
+ 'sha1': 'expectedsha1',
+ },
+ cluster.remote2: {
+ 'project': 'ceph',
+ 'sha1': 'overridesha1',
+ 'tag': 'overridetag',
+ 'branch': 'overridebranch',
+ },
+ }
+ assert install.upgrade_remote_to_config(ctx, config) == expected_config
+
+
+ @patch("teuthology.task.install.packaging.get_package_version")
+ @patch("teuthology.task.install.redhat.set_deb_repo")
+ def test_rh_install_deb_pkgs(self, m_set_rh_deb_repo, m_get_pkg_version):
+ ctx = Mock()
+ remote = Mock()
+ version = '1.3.2'
+ rh_ds_yaml = dict()
+ rh_ds_yaml = {
+ 'versions': {'deb': {'mapped': {'1.3.2': '0.94.5'}}},
+ 'pkgs': {'deb': ['pkg1', 'pkg2']},
+ 'extra_system_packages': {'deb': ['es_pkg1', 'es_pkg2']},
+ 'extra_packages': {'deb': ['e_pkg1', 'e_pkg2']},
+ }
+ m_get_pkg_version.return_value = "0.94.5"
+ install.redhat.install_deb_pkgs(ctx, remote, version, rh_ds_yaml)
+
+ @patch("teuthology.task.install.packaging.get_package_version")
+ def test_rh_install_pkgs(self, m_get_pkg_version):
+ ctx = Mock()
+ remote = Mock()
+ version = '1.3.2'
+ rh_ds_yaml = dict()
+ rh_ds_yaml = {
+ 'versions': {'rpm': {'mapped': {'1.3.2': '0.94.5',
+ '1.3.1': '0.94.3'}}},
+ 'pkgs': {'rpm': ['pkg1', 'pkg2']},
+ 'extra_system_packages': {'rpm': ['es_pkg1', 'es_pkg2']},
+ 'extra_packages': {'rpm': ['e_pkg1', 'e_pkg2']},
+ }
+
+ m_get_pkg_version.return_value = "0.94.5"
+ install.redhat.install_pkgs(ctx, remote, version, rh_ds_yaml)
+ version = '1.3.1'
+ with pytest.raises(RuntimeError) as e:
+ install.redhat.install_pkgs(ctx, remote, version, rh_ds_yaml)
+ assert "Version check failed" in str(e)
+
+ @mark.parametrize(
+ 'conf, expect',
+ [
+ [
+ {
+ 'tasks': [ { 'install': { 'clean': True, }, }, ],
+ 'overrides': {
+ 'install': {
+ 'ceph': {
+ 'extra_system_packages': ['alpha'],
+ 'flavor': 'default',
+ 'sha1': '0123456789abcdef0123456789abcdef01234567',
+ },
+ 'extra_system_packages': {
+ 'deb': [],
+ 'rpm': ['xerxes', 'yellow'],
+ },
+ },
+ },
+ },
+ {
+ 'deb': ['alpha'],
+ 'rpm': ['alpha', 'xerxes', 'yellow'],
+ }
+ ],
+ [
+ {
+ 'tasks': [ { 'install': { 'clean': True, }, }, ],
+ 'overrides': {
+ 'install': {
+ 'ceph': {
+ 'extra_system_packages': {
+ 'deb': [],
+ 'rpm': ['xerxes', 'yellow'],
+ },
+ 'flavor': 'default',
+ 'sha1': '0123456789abcdef0123456789abcdef01234567',
+ },
+ 'extra_system_packages': ['alpha'],
+ },
+ },
+ },
+ {
+ 'deb': ['alpha'],
+ 'rpm': ['xerxes', 'yellow', 'alpha'],
+ }
+ ],
+ [
+ {
+ 'tasks': [ { 'install': { 'clean': True, }, }, ],
+ 'overrides': {
+ 'install': {
+ 'ceph': {
+ 'flavor': 'default',
+ 'sha1': '0123456789abcdef0123456789abcdef01234567',
+ },
+ 'extra_system_packages': {
+ 'deb': [],
+ 'rpm': ['xerxes', 'yellow'],
+ },
+ },
+ },
+ },
+ {
+ 'deb': [],
+ 'rpm': ['xerxes', 'yellow'],
+ }
+ ],
+ [
+ {
+ 'tasks': [ { 'install': { 'clean': True, }, }, ],
+ 'overrides': {
+ 'install': {
+ 'ceph': {
+ 'flavor': 'default',
+ 'sha1': '0123456789abcdef0123456789abcdef01234567',
+ },
+ 'extra_system_packages': ['xerxes', 'yellow'],
+ },
+ },
+ },
+ {
+ 'deb': ['xerxes', 'yellow'],
+ 'rpm': ['xerxes', 'yellow'],
+ }
+ ],
+ [
+ {
+ 'tasks': [ { 'install': { 'clean': True, }, }, ],
+ 'overrides': { 'install': {
+ 'ceph': { 'flavor': 'default', 'sha1': '012345', },
+ 'extra_system_packages': { 'rpm': ['xerxes', 'yellow'], },
+ },
+ },
+ },
+ { 'deb': [], 'rpm': ['xerxes', 'yellow'], }
+
+ ],
+ [
+ {
+ 'tasks': [ { 'install': { 'clean': True,
+ 'extra_system_packages': { 'deb': ['alpha'], 'rpm': ['bravo'] },
+ }, }, ],
+ 'overrides': { 'install': {
+ 'ceph': { 'flavor': 'default', 'sha1': '012345', },
+ 'extra_system_packages': { 'rpm': ['xerxes', 'yellow'], },
+ },
+ },
+ },
+ { 'deb': ['alpha'], 'rpm': ['bravo', 'xerxes', 'yellow'], }
+
+ ],
+
+ ]
+ )
+ def test_install_extra_system_packages(self, conf, expect):
+ install_task = conf.get('tasks')[0].get('install')
+ install_overrides = conf.get('overrides').get('install', {})
+ install._override_extra_system_packages(install_task, 'ceph', install_overrides)
+
+ assert install_task.get('extra_system_packages') == expect
--- /dev/null
+from teuthology.config import FakeNamespace
+from teuthology.task import internal
+
+
+class TestInternal(object):
+ def setup_method(self):
+ self.ctx = FakeNamespace()
+ self.ctx.config = dict()
+
+ def test_buildpackages_prep(self):
+ #
+ # no buildpackages nor install tasks
+ #
+ self.ctx.config = { 'tasks': [] }
+ assert internal.buildpackages_prep(self.ctx,
+ self.ctx.config) == internal.BUILDPACKAGES_NOTHING
+ #
+ # make the buildpackages tasks the first to run
+ #
+ self.ctx.config = {
+ 'tasks': [ { 'atask': None },
+ { 'internal.buildpackages_prep': None },
+ { 'btask': None },
+ { 'install': None },
+ { 'buildpackages': None } ],
+ }
+ assert internal.buildpackages_prep(self.ctx,
+ self.ctx.config) == internal.BUILDPACKAGES_FIRST
+ assert self.ctx.config == {
+ 'tasks': [ { 'atask': None },
+ { 'internal.buildpackages_prep': None },
+ { 'buildpackages': None },
+ { 'btask': None },
+ { 'install': None } ],
+ }
+ #
+ # the buildpackages task already the first task to run
+ #
+ assert internal.buildpackages_prep(self.ctx,
+ self.ctx.config) == internal.BUILDPACKAGES_OK
+ #
+ # no buildpackages task
+ #
+ self.ctx.config = {
+ 'tasks': [ { 'install': None } ],
+ }
+ assert internal.buildpackages_prep(self.ctx,
+ self.ctx.config) == internal.BUILDPACKAGES_NOTHING
+ #
+ # no install task: the buildpackages task must be removed
+ #
+ self.ctx.config = {
+ 'tasks': [ { 'buildpackages': None } ],
+ }
+ assert internal.buildpackages_prep(self.ctx,
+ self.ctx.config) == internal.BUILDPACKAGES_REMOVED
+ assert self.ctx.config == {'tasks': []}
--- /dev/null
+from teuthology.config import FakeNamespace
+from teuthology.orchestra.cluster import Cluster
+from teuthology.orchestra.remote import Remote
+from teuthology.task.kernel import (
+ normalize_and_apply_overrides,
+ CONFIG_DEFAULT,
+ TIMEOUT_DEFAULT,
+)
+
+class TestKernelNormalizeAndApplyOverrides(object):
+
+ def setup_method(self):
+ self.ctx = FakeNamespace()
+ self.ctx.cluster = Cluster()
+ self.ctx.cluster.add(Remote('remote1'), ['mon.a', 'client.0'])
+ self.ctx.cluster.add(Remote('remote2'), ['osd.0', 'osd.1', 'osd.2'])
+ self.ctx.cluster.add(Remote('remote3'), ['client.1'])
+
+ def test_default(self):
+ config = {}
+ overrides = {}
+ config, t = normalize_and_apply_overrides(self.ctx, config, overrides)
+ assert config == {
+ 'mon.a': CONFIG_DEFAULT,
+ 'osd.0': CONFIG_DEFAULT,
+ 'osd.1': CONFIG_DEFAULT,
+ 'osd.2': CONFIG_DEFAULT,
+ 'client.0': CONFIG_DEFAULT,
+ 'client.1': CONFIG_DEFAULT,
+ }
+ assert t == TIMEOUT_DEFAULT
+
+ def test_timeout_default(self):
+ config = {
+ 'client.0': {'branch': 'testing'},
+ }
+ overrides = {}
+ config, t = normalize_and_apply_overrides(self.ctx, config, overrides)
+ assert config == {
+ 'client.0': {'branch': 'testing'},
+ }
+ assert t == TIMEOUT_DEFAULT
+
+ def test_timeout(self):
+ config = {
+ 'client.0': {'branch': 'testing'},
+ 'timeout': 100,
+ }
+ overrides = {}
+ config, t = normalize_and_apply_overrides(self.ctx, config, overrides)
+ assert config == {
+ 'client.0': {'branch': 'testing'},
+ }
+ assert t == 100
+
+ def test_override_timeout(self):
+ config = {
+ 'client.0': {'branch': 'testing'},
+ 'timeout': 100,
+ }
+ overrides = {
+ 'timeout': 200,
+ }
+ config, t = normalize_and_apply_overrides(self.ctx, config, overrides)
+ assert config == {
+ 'client.0': {'branch': 'testing'},
+ }
+ assert t == 200
+
+ def test_override_same_version_key(self):
+ config = {
+ 'client.0': {'branch': 'testing'},
+ }
+ overrides = {
+ 'client.0': {'branch': 'wip-foobar'},
+ }
+ config, t = normalize_and_apply_overrides(self.ctx, config, overrides)
+ assert config == {
+ 'client.0': {'branch': 'wip-foobar'},
+ }
+ assert t == TIMEOUT_DEFAULT
+
+ def test_override_different_version_key(self):
+ config = {
+ 'client.0': {'branch': 'testing'},
+ }
+ overrides = {
+ 'client.0': {'tag': 'v4.1'},
+ }
+ config, t = normalize_and_apply_overrides(self.ctx, config, overrides)
+ assert config == {
+ 'client.0': {'tag': 'v4.1'},
+ }
+ assert t == TIMEOUT_DEFAULT
+
+ def test_override_actual(self):
+ config = {
+ 'osd.1': {'tag': 'v4.1'},
+ 'client.0': {'branch': 'testing'},
+ }
+ overrides = {
+ 'osd.1': {'koji': 1234, 'kdb': True},
+ }
+ config, t = normalize_and_apply_overrides(self.ctx, config, overrides)
+ assert config == {
+ 'osd.1': {'koji': 1234, 'kdb': True},
+ 'client.0': {'branch': 'testing'},
+ }
+ assert t == TIMEOUT_DEFAULT
+
+ def test_override_actual_with_generic(self):
+ config = {
+ 'osd.1': {'tag': 'v4.1', 'kdb': False},
+ 'client.0': {'branch': 'testing'},
+ }
+ overrides = {
+ 'osd': {'koji': 1234},
+ }
+ config, t = normalize_and_apply_overrides(self.ctx, config, overrides)
+ assert config == {
+ 'osd.0': {'koji': 1234},
+ 'osd.1': {'koji': 1234, 'kdb': False},
+ 'osd.2': {'koji': 1234},
+ 'client.0': {'branch': 'testing'},
+ }
+ assert t == TIMEOUT_DEFAULT
+
+ def test_override_actual_with_top_level(self):
+ config = {
+ 'osd.1': {'tag': 'v4.1'},
+ 'client.0': {'branch': 'testing', 'kdb': False},
+ }
+ overrides = {'koji': 1234, 'kdb': True}
+ config, t = normalize_and_apply_overrides(self.ctx, config, overrides)
+ assert config == {
+ 'mon.a': {'koji': 1234, 'kdb': True},
+ 'osd.0': {'koji': 1234, 'kdb': True},
+ 'osd.1': {'koji': 1234, 'kdb': True},
+ 'osd.2': {'koji': 1234, 'kdb': True},
+ 'client.0': {'koji': 1234, 'kdb': True},
+ 'client.1': {'koji': 1234, 'kdb': True},
+ }
+ assert t == TIMEOUT_DEFAULT
+
+ def test_override_generic(self):
+ config = {
+ 'osd': {'tag': 'v4.1'},
+ 'client': {'branch': 'testing'},
+ }
+ overrides = {
+ 'client': {'koji': 1234, 'kdb': True},
+ }
+ config, t = normalize_and_apply_overrides(self.ctx, config, overrides)
+ assert config == {
+ 'osd.0': {'tag': 'v4.1'},
+ 'osd.1': {'tag': 'v4.1'},
+ 'osd.2': {'tag': 'v4.1'},
+ 'client.0': {'koji': 1234, 'kdb': True},
+ 'client.1': {'koji': 1234, 'kdb': True},
+ }
+ assert t == TIMEOUT_DEFAULT
+
+ def test_override_generic_with_top_level(self):
+ config = {
+ 'osd': {'tag': 'v4.1'},
+ 'client': {'branch': 'testing', 'kdb': False},
+ }
+ overrides = {
+ 'client': {'koji': 1234},
+ }
+ config, t = normalize_and_apply_overrides(self.ctx, config, overrides)
+ assert config == {
+ 'osd.0': {'tag': 'v4.1'},
+ 'osd.1': {'tag': 'v4.1'},
+ 'osd.2': {'tag': 'v4.1'},
+ 'client.0': {'koji': 1234, 'kdb': False},
+ 'client.1': {'koji': 1234, 'kdb': False},
+ }
+ assert t == TIMEOUT_DEFAULT
+
+ def test_override_generic_with_actual(self):
+ config = {
+ 'osd': {'tag': 'v4.1', 'kdb': False},
+ 'client': {'branch': 'testing'},
+ }
+ overrides = {
+ 'osd.2': {'koji': 1234, 'kdb': True},
+ }
+ config, t = normalize_and_apply_overrides(self.ctx, config, overrides)
+ assert config == {
+ 'osd.0': {'tag': 'v4.1', 'kdb': False},
+ 'osd.1': {'tag': 'v4.1', 'kdb': False},
+ 'osd.2': {'koji': 1234, 'kdb': True},
+ 'client.0': {'branch': 'testing'},
+ 'client.1': {'branch': 'testing'},
+ }
+ assert t == TIMEOUT_DEFAULT
+
+ def test_override_top_level(self):
+ config = {'branch': 'testing'}
+ overrides = {'koji': 1234, 'kdb': True}
+ config, t = normalize_and_apply_overrides(self.ctx, config, overrides)
+ assert config == {
+ 'mon.a': {'koji': 1234, 'kdb': True},
+ 'osd.0': {'koji': 1234, 'kdb': True},
+ 'osd.1': {'koji': 1234, 'kdb': True},
+ 'osd.2': {'koji': 1234, 'kdb': True},
+ 'client.0': {'koji': 1234, 'kdb': True},
+ 'client.1': {'koji': 1234, 'kdb': True},
+ }
+ assert t == TIMEOUT_DEFAULT
+
+ def test_override_top_level_with_actual(self):
+ config = {'branch': 'testing', 'kdb': False}
+ overrides = {
+ 'mon.a': {'koji': 1234},
+ }
+ config, t = normalize_and_apply_overrides(self.ctx, config, overrides)
+ assert config == {
+ 'mon.a': {'koji': 1234, 'kdb': False},
+ 'osd.0': {'branch': 'testing', 'kdb': False},
+ 'osd.1': {'branch': 'testing', 'kdb': False},
+ 'osd.2': {'branch': 'testing', 'kdb': False},
+ 'client.0': {'branch': 'testing', 'kdb': False},
+ 'client.1': {'branch': 'testing', 'kdb': False},
+ }
+ assert t == TIMEOUT_DEFAULT
+
+ def test_override_top_level_with_generic(self):
+ config = {'branch': 'testing', 'kdb': False}
+ overrides = {
+ 'client': {'koji': 1234, 'kdb': True},
+ }
+ config, t = normalize_and_apply_overrides(self.ctx, config, overrides)
+ assert config == {
+ 'mon.a': {'branch': 'testing', 'kdb': False},
+ 'osd.0': {'branch': 'testing', 'kdb': False},
+ 'osd.1': {'branch': 'testing', 'kdb': False},
+ 'osd.2': {'branch': 'testing', 'kdb': False},
+ 'client.0': {'koji': 1234, 'kdb': True},
+ 'client.1': {'koji': 1234, 'kdb': True},
+ }
+ assert t == TIMEOUT_DEFAULT
--- /dev/null
+import os
+import requests
+
+from teuthology.util.compat import parse_qs, urljoin
+
+from mock import patch, DEFAULT, Mock, mock_open, call
+from pytest import raises
+
+from teuthology.config import config, FakeNamespace
+from teuthology.orchestra.cluster import Cluster
+from teuthology.orchestra.remote import Remote
+from teuthology.orchestra.run import Raw
+from teuthology.task.pcp import (PCPDataSource, PCPArchive, PCPGrapher,
+ GrafanaGrapher, GraphiteGrapher, PCP)
+
+from tests.task import TestTask
+
+pcp_host = 'http://pcp.front.sepia.ceph.com:44323/'
+
+
+class TestPCPDataSource(object):
+ klass = PCPDataSource
+
+ def setup_method(self):
+ config.pcp_host = pcp_host
+
+ def test_init(self):
+ hosts = ['host1', 'host2']
+ time_from = 'now-2h'
+ time_until = 'now'
+ obj = self.klass(
+ hosts=hosts,
+ time_from=time_from,
+ time_until=time_until,
+ )
+ assert obj.hosts == hosts
+ assert obj.time_from == time_from
+ assert obj.time_until == time_until
+
+
+class TestPCPArchive(TestPCPDataSource):
+ klass = PCPArchive
+
+ def test_get_archive_input_dir(self):
+ hosts = ['host1', 'host2']
+ time_from = 'now-1d'
+ obj = self.klass(
+ hosts=hosts,
+ time_from=time_from,
+ )
+ assert obj.get_archive_input_dir('host1') == \
+ '/var/log/pcp/pmlogger/host1'
+
+ def test_get_pmlogextract_cmd(self):
+ obj = self.klass(
+ hosts=['host1'],
+ time_from='now-3h',
+ time_until='now-1h',
+ )
+ expected = [
+ 'pmlogextract',
+ '-S', 'now-3h',
+ '-T', 'now-1h',
+ Raw('/var/log/pcp/pmlogger/host1/*.0'),
+ ]
+ assert obj.get_pmlogextract_cmd('host1') == expected
+
+ def test_format_time(self):
+ assert self.klass._format_time(1462893484) == \
+ '@ Tue May 10 15:18:04 2016'
+
+ def test_format_time_now(self):
+ assert self.klass._format_time('now-1h') == 'now-1h'
+
+
+class TestPCPGrapher(TestPCPDataSource):
+ klass = PCPGrapher
+
+ def test_init(self):
+ hosts = ['host1', 'host2']
+ time_from = 'now-2h'
+ time_until = 'now'
+ obj = self.klass(
+ hosts=hosts,
+ time_from=time_from,
+ time_until=time_until,
+ )
+ assert obj.hosts == hosts
+ assert obj.time_from == time_from
+ assert obj.time_until == time_until
+ expected_url = urljoin(config.pcp_host, self.klass._endpoint)
+ assert obj.base_url == expected_url
+
+
+class TestGrafanaGrapher(TestPCPGrapher):
+ klass = GrafanaGrapher
+
+ def test_build_graph_url(self):
+ hosts = ['host1']
+ time_from = 'now-3h'
+ time_until = 'now-1h'
+ obj = self.klass(
+ hosts=hosts,
+ time_from=time_from,
+ time_until=time_until,
+ )
+ base_url = urljoin(
+ config.pcp_host,
+ 'grafana/index.html#/dashboard/script/index.js',
+ )
+ assert obj.base_url == base_url
+ got_url = obj.build_graph_url()
+ parsed_query = parse_qs(got_url.split('?')[1])
+ assert parsed_query['hosts'] == hosts
+ assert len(parsed_query['time_from']) == 1
+ assert parsed_query['time_from'][0] == time_from
+ assert len(parsed_query['time_to']) == 1
+ assert parsed_query['time_to'][0] == time_until
+
+ def test_format_time(self):
+ assert self.klass._format_time(1462893484) == \
+ '2016-05-10T15:18:04'
+
+ def test_format_time_now(self):
+ assert self.klass._format_time('now-1h') == 'now-1h'
+
+
+class TestGraphiteGrapher(TestPCPGrapher):
+ klass = GraphiteGrapher
+
+ def test_build_graph_urls(self):
+ obj = self.klass(
+ hosts=['host1', 'host2'],
+ time_from='now-3h',
+ time_until='now-1h',
+ )
+ expected_urls = [obj.get_graph_url(m) for m in obj.metrics]
+ obj.build_graph_urls()
+ built_urls = []
+ for metric in obj.graphs.keys():
+ built_urls.append(obj.graphs[metric]['url'])
+ assert len(built_urls) == len(expected_urls)
+ assert sorted(built_urls) == sorted(expected_urls)
+
+ def test_check_dest_dir(self):
+ obj = self.klass(
+ hosts=['host1'],
+ time_from='now-3h',
+ )
+ assert obj.dest_dir is None
+ with raises(RuntimeError):
+ obj._check_dest_dir()
+
+ def test_generate_html_dynamic(self):
+ obj = self.klass(
+ hosts=['host1'],
+ time_from='now-3h',
+ )
+ html = obj.generate_html()
+ assert config.pcp_host in html
+
+ def test_download_graphs(self):
+ dest_dir = '/fake/path'
+ obj = self.klass(
+ hosts=['host1'],
+ time_from='now-3h',
+ dest_dir=dest_dir,
+ )
+ _format = obj.graph_defaults.get('format')
+ with patch('teuthology.task.pcp.requests.get', create=True) as m_get:
+ m_resp = Mock()
+ m_resp.ok = True
+ m_get.return_value = m_resp
+ with patch('teuthology.task.pcp.open', mock_open(), create=True):
+ obj.download_graphs()
+ expected_filenames = []
+ for metric in obj.metrics:
+ expected_filenames.append(
+ "{}.{}".format(
+ os.path.join(
+ dest_dir,
+ obj._sanitize_metric_name(metric),
+ ),
+ _format,
+ )
+ )
+ graph_filenames = []
+ for metric in obj.graphs.keys():
+ graph_filenames.append(obj.graphs[metric]['file'])
+ assert sorted(graph_filenames) == sorted(expected_filenames)
+
+ def test_generate_html_static(self):
+ obj = self.klass(
+ hosts=['host1'],
+ time_from='now-3h',
+ dest_dir='/fake/path',
+ )
+ with patch('teuthology.task.pcp.requests.get', create=True) as m_get:
+ m_resp = Mock()
+ m_resp.ok = True
+ m_get.return_value = m_resp
+ with patch('teuthology.task.pcp.open', mock_open(), create=True):
+ obj.download_graphs()
+ html = obj.generate_html(mode='static')
+ assert config.pcp_host not in html
+
+ def test_sanitize_metric_name(self):
+ sanitized_metrics = {
+ 'foo.bar': 'foo.bar',
+ 'foo.*': 'foo._all_',
+ 'foo.bar baz': 'foo.bar_baz',
+ 'foo.*.bar baz': 'foo._all_.bar_baz',
+ }
+ for in_, out in sanitized_metrics.items():
+ assert self.klass._sanitize_metric_name(in_) == out
+
+ def test_get_target_globs(self):
+ obj = self.klass(
+ hosts=['host1'],
+ time_from='now-3h',
+ )
+ assert obj.get_target_globs() == ['*host1*']
+ assert obj.get_target_globs('a.metric') == ['*host1*.a.metric']
+ obj.hosts.append('host2')
+ assert obj.get_target_globs() == ['*host1*', '*host2*']
+ assert obj.get_target_globs('a.metric') == \
+ ['*host1*.a.metric', '*host2*.a.metric']
+
+
+class TestPCPTask(TestTask):
+ klass = PCP
+ task_name = 'pcp'
+
+ def setup_method(self):
+ self.ctx = FakeNamespace()
+ self.ctx.cluster = Cluster()
+ self.ctx.cluster.add(Remote('user@remote1'), ['role1'])
+ self.ctx.cluster.add(Remote('user@remote2'), ['role2'])
+ self.ctx.config = dict()
+ self.task_config = dict()
+ config.pcp_host = pcp_host
+
+ def test_init(self):
+ task = self.klass(self.ctx, self.task_config)
+ assert task.stop_time == 'now'
+
+ def test_disabled(self):
+ config.pcp_host = None
+ with self.klass(self.ctx, self.task_config) as task:
+ assert task.enabled is False
+ assert not hasattr(task, 'grafana')
+ assert not hasattr(task, 'graphite')
+ assert not hasattr(task, 'archiver')
+
+ def test_setup(self):
+ with patch.multiple(
+ self.klass,
+ setup_collectors=DEFAULT,
+ begin=DEFAULT,
+ end=DEFAULT,
+ ):
+ with self.klass(self.ctx, self.task_config) as task:
+ task.setup_collectors.assert_called_once_with()
+ assert isinstance(task.start_time, int)
+
+ def test_setup_collectors(self):
+ with patch.multiple(
+ self.klass,
+ begin=DEFAULT,
+ end=DEFAULT,
+ ):
+ with self.klass(self.ctx, self.task_config) as task:
+ assert hasattr(task, 'grafana')
+ assert not hasattr(task, 'graphite')
+ assert not hasattr(task, 'archiver')
+ self.task_config['grafana'] = False
+ with self.klass(self.ctx, self.task_config) as task:
+ assert not hasattr(task, 'grafana')
+
+ @patch('os.makedirs')
+ def test_setup_grafana(self, m_makedirs):
+ with patch.multiple(
+ self.klass,
+ begin=DEFAULT,
+ end=DEFAULT,
+ ):
+ self.ctx.archive = '/fake/path'
+ with self.klass(self.ctx, self.task_config) as task:
+ assert hasattr(task, 'grafana')
+ self.task_config['grafana'] = False
+ with self.klass(self.ctx, self.task_config) as task:
+ assert not hasattr(task, 'grafana')
+
+ @patch('os.makedirs')
+ @patch('teuthology.task.pcp.GraphiteGrapher')
+ def test_setup_graphite(self, m_graphite_grapher, m_makedirs):
+ with patch.multiple(
+ self.klass,
+ begin=DEFAULT,
+ end=DEFAULT,
+ ):
+ with self.klass(self.ctx, self.task_config) as task:
+ assert not hasattr(task, 'graphite')
+ self.task_config['graphite'] = False
+ with self.klass(self.ctx, self.task_config) as task:
+ assert not hasattr(task, 'graphite')
+ self.ctx.archive = '/fake/path'
+ self.task_config['graphite'] = True
+ with self.klass(self.ctx, self.task_config) as task:
+ assert hasattr(task, 'graphite')
+ self.task_config['graphite'] = False
+ with self.klass(self.ctx, self.task_config) as task:
+ assert not hasattr(task, 'graphite')
+
+ @patch('os.makedirs')
+ @patch('teuthology.task.pcp.PCPArchive')
+ def test_setup_archiver(self, m_archive, m_makedirs):
+ with patch.multiple(
+ self.klass,
+ begin=DEFAULT,
+ end=DEFAULT,
+ ):
+ self.task_config['fetch_archives'] = True
+ with self.klass(self.ctx, self.task_config) as task:
+ assert not hasattr(task, 'archiver')
+ self.task_config['fetch_archives'] = False
+ with self.klass(self.ctx, self.task_config) as task:
+ assert not hasattr(task, 'archiver')
+ self.ctx.archive = '/fake/path'
+ self.task_config['fetch_archives'] = True
+ with self.klass(self.ctx, self.task_config) as task:
+ assert hasattr(task, 'archiver')
+ self.task_config['fetch_archives'] = False
+ with self.klass(self.ctx, self.task_config) as task:
+ assert not hasattr(task, 'archiver')
+
+ @patch('os.makedirs')
+ @patch('teuthology.task.pcp.GrafanaGrapher')
+ @patch('teuthology.task.pcp.GraphiteGrapher')
+ def test_begin(self, m_grafana, m_graphite, m_makedirs):
+ with patch.multiple(
+ self.klass,
+ end=DEFAULT,
+ ):
+ with self.klass(self.ctx, self.task_config) as task:
+ task.grafana.build_graph_url.assert_called_once_with()
+ self.task_config['graphite'] = True
+ self.ctx.archive = '/fake/path'
+ with self.klass(self.ctx, self.task_config) as task:
+ task.graphite.write_html.assert_called_once_with()
+
+ @patch('os.makedirs')
+ @patch('teuthology.task.pcp.GrafanaGrapher')
+ @patch('teuthology.task.pcp.GraphiteGrapher')
+ def test_end(self, m_grafana, m_graphite, m_makedirs):
+ self.ctx.archive = '/fake/path'
+ with self.klass(self.ctx, self.task_config) as task:
+ # begin() should have called write_html() once by now, with no args
+ task.graphite.write_html.assert_called_once_with()
+ # end() should have called write_html() a second time by now, with
+ # mode=static
+ second_call = task.graphite.write_html.call_args_list[1]
+ assert second_call[1]['mode'] == 'static'
+ assert isinstance(task.stop_time, int)
+
+ @patch('os.makedirs')
+ @patch('teuthology.task.pcp.GrafanaGrapher')
+ @patch('teuthology.task.pcp.GraphiteGrapher')
+ def test_end_16049(self, m_grafana, m_graphite, m_makedirs):
+ # http://tracker.ceph.com/issues/16049
+ # Jobs were failing if graph downloading failed. We don't want that.
+ self.ctx.archive = '/fake/path'
+ with self.klass(self.ctx, self.task_config) as task:
+ task.graphite.download_graphs.side_effect = \
+ requests.ConnectionError
+ # Even though downloading graphs failed, we should have called
+ # write_html() a second time, again with no args
+ assert task.graphite.write_html.call_args_list == [call(), call()]
+ assert isinstance(task.stop_time, int)
--- /dev/null
+from mock import patch, Mock, DEFAULT
+
+from teuthology.config import FakeNamespace
+from teuthology.orchestra.cluster import Cluster
+from teuthology.orchestra.remote import Remote
+from teuthology.task.selinux import SELinux
+
+
+class TestSELinux(object):
+ def setup_method(self):
+ self.ctx = FakeNamespace()
+ self.ctx.config = dict()
+
+ def test_host_exclusion(self):
+ with patch.multiple(
+ Remote,
+ os=DEFAULT,
+ run=DEFAULT,
+ ):
+ self.ctx.cluster = Cluster()
+ remote1 = Remote('remote1')
+ remote1.os = Mock()
+ remote1.os.package_type = 'rpm'
+ remote1._is_vm = False
+ self.ctx.cluster.add(remote1, ['role1'])
+ remote2 = Remote('remote1')
+ remote2.os = Mock()
+ remote2.os.package_type = 'deb'
+ remote2._is_vm = False
+ self.ctx.cluster.add(remote2, ['role2'])
+ task_config = dict()
+ with SELinux(self.ctx, task_config) as task:
+ remotes = list(task.cluster.remotes)
+ assert remotes == [remote1]
+
--- /dev/null
+import pytest
+
+from teuthology import config
+
+
+class TestYamlConfig(object):
+ def setup_method(self):
+ self.test_class = config.YamlConfig
+
+ def test_set_multiple(self):
+ conf_obj = self.test_class()
+ conf_obj.foo = 'foo'
+ conf_obj.bar = 'bar'
+ assert conf_obj.foo == 'foo'
+ assert conf_obj.bar == 'bar'
+ assert conf_obj.to_dict()['foo'] == 'foo'
+
+ def test_from_dict(self):
+ in_dict = dict(foo='bar')
+ conf_obj = self.test_class.from_dict(in_dict)
+ assert conf_obj.foo == 'bar'
+
+ def test_contains(self):
+ in_dict = dict(foo='bar')
+ conf_obj = self.test_class.from_dict(in_dict)
+ conf_obj.bar = "foo"
+ assert "bar" in conf_obj
+ assert "foo" in conf_obj
+ assert "baz" not in conf_obj
+
+ def test_to_dict(self):
+ in_dict = dict(foo='bar')
+ conf_obj = self.test_class.from_dict(in_dict)
+ assert conf_obj.to_dict() == in_dict
+
+ def test_from_str(self):
+ in_str = "foo: bar"
+ conf_obj = self.test_class.from_str(in_str)
+ assert conf_obj.foo == 'bar'
+
+ def test_to_str(self):
+ in_str = "foo: bar"
+ conf_obj = self.test_class.from_str(in_str)
+ assert conf_obj.to_str() == in_str
+
+ def test_update(self):
+ conf_obj = self.test_class(dict())
+ conf_obj.foo = 'foo'
+ conf_obj.bar = 'bar'
+ conf_obj.update(dict(bar='baz'))
+ assert conf_obj.foo == 'foo'
+ assert conf_obj.bar == 'baz'
+
+ def test_delattr(self):
+ conf_obj = self.test_class()
+ conf_obj.foo = 'bar'
+ assert conf_obj.foo == 'bar'
+ del conf_obj.foo
+ assert conf_obj.foo is None
+
+ def test_assignment(self):
+ conf_obj = self.test_class()
+ conf_obj["foo"] = "bar"
+ assert conf_obj["foo"] == "bar"
+ assert conf_obj.foo == "bar"
+
+ def test_used_with_update(self):
+ d = dict()
+ conf_obj = self.test_class.from_dict({"foo": "bar"})
+ d.update(conf_obj)
+ assert d["foo"] == "bar"
+
+ def test_get(self):
+ conf_obj = self.test_class()
+ assert conf_obj.get('foo') is None
+ assert conf_obj.get('foo', 'bar') == 'bar'
+ conf_obj.foo = 'baz'
+ assert conf_obj.get('foo') == 'baz'
+
+
+class TestTeuthologyConfig(TestYamlConfig):
+ def setup_method(self):
+ self.test_class = config.TeuthologyConfig
+
+ def test_get_ceph_git_base_default(self):
+ conf_obj = self.test_class()
+ conf_obj.yaml_path = ''
+ conf_obj.load()
+ assert conf_obj.ceph_git_base_url == "https://github.com/ceph/"
+
+ def test_set_ceph_git_base_via_private(self):
+ conf_obj = self.test_class()
+ conf_obj._conf['ceph_git_base_url'] = \
+ "git://git.ceph.com/"
+ assert conf_obj.ceph_git_base_url == "git://git.ceph.com/"
+
+ def test_get_reserve_machines_default(self):
+ conf_obj = self.test_class()
+ conf_obj.yaml_path = ''
+ conf_obj.load()
+ assert conf_obj.reserve_machines == 5
+
+ def test_set_reserve_machines_via_private(self):
+ conf_obj = self.test_class()
+ conf_obj._conf['reserve_machines'] = 2
+ assert conf_obj.reserve_machines == 2
+
+ def test_set_nonstandard(self):
+ conf_obj = self.test_class()
+ conf_obj.something = 'something else'
+ assert conf_obj.something == 'something else'
+
+
+class TestJobConfig(TestYamlConfig):
+ def setup_method(self):
+ self.test_class = config.JobConfig
+
+
+class TestFakeNamespace(TestYamlConfig):
+ def setup_method(self):
+ self.test_class = config.FakeNamespace
+
+ def test_docopt_dict(self):
+ """
+ Tests if a dict in the format that docopt returns can
+ be parsed correctly.
+ """
+ d = {
+ "--verbose": True,
+ "--an-option": "some_option",
+ "<an_arg>": "the_arg",
+ "something": "some_thing",
+ }
+ conf_obj = self.test_class(d)
+ assert conf_obj.verbose
+ assert conf_obj.an_option == "some_option"
+ assert conf_obj.an_arg == "the_arg"
+ assert conf_obj.something == "some_thing"
+
+ def test_config(self):
+ """
+ Tests that a teuthology_config property is automatically added
+ to the conf_obj
+ """
+ conf_obj = self.test_class(dict(foo="bar"))
+ assert conf_obj["foo"] == "bar"
+ assert conf_obj.foo == "bar"
+ assert conf_obj.teuthology_config.get("fake key") is None
+
+ def test_getattr(self):
+ conf_obj = self.test_class.from_dict({"foo": "bar"})
+ result = getattr(conf_obj, "not_there", "default")
+ assert result == "default"
+ result = getattr(conf_obj, "foo")
+ assert result == "bar"
+
+ def test_none(self):
+ conf_obj = self.test_class.from_dict(dict(null=None))
+ assert conf_obj.null is None
+
+ def test_delattr(self):
+ conf_obj = self.test_class()
+ conf_obj.foo = 'bar'
+ assert conf_obj.foo == 'bar'
+ del conf_obj.foo
+ with pytest.raises(AttributeError):
+ conf_obj.foo
+
+ def test_to_str(self):
+ in_str = "foo: bar"
+ conf_obj = self.test_class.from_str(in_str)
+ assert conf_obj.to_str() == "{'foo': 'bar'}"
+
+ def test_multiple_access(self):
+ """
+ Test that config.config and FakeNamespace.teuthology_config reflect
+ each others' modifications
+ """
+ in_str = "foo: bar"
+ conf_obj = self.test_class.from_str(in_str)
+ assert config.config.get('test_key_1') is None
+ assert conf_obj.teuthology_config.get('test_key_1') is None
+ config.config.test_key_1 = 'test value'
+ assert conf_obj.teuthology_config['test_key_1'] == 'test value'
+
+ assert config.config.get('test_key_2') is None
+ assert conf_obj.teuthology_config.get('test_key_2') is None
+ conf_obj.teuthology_config['test_key_2'] = 'test value'
+ assert config.config['test_key_2'] == 'test value'
--- /dev/null
+from pytest import raises
+from teuthology import contextutil
+from logging import ERROR
+
+
+class TestSafeWhile(object):
+
+ def setup_method(self):
+ contextutil.log.setLevel(ERROR)
+ self.fake_sleep = lambda s: True
+ self.s_while = contextutil.safe_while
+
+ def test_6_5_10_deal(self):
+ with raises(contextutil.MaxWhileTries):
+ with self.s_while(_sleeper=self.fake_sleep) as proceed:
+ while proceed():
+ pass
+
+ def test_6_0_1_deal(self):
+ with raises(contextutil.MaxWhileTries) as error:
+ with self.s_while(
+ tries=1,
+ _sleeper=self.fake_sleep
+ ) as proceed:
+ while proceed():
+ pass
+
+ assert 'waiting for 6 seconds' in str(error)
+
+ def test_1_0_10_deal(self):
+ with raises(contextutil.MaxWhileTries) as error:
+ with self.s_while(
+ sleep=1,
+ _sleeper=self.fake_sleep
+ ) as proceed:
+ while proceed():
+ pass
+
+ assert 'waiting for 10 seconds' in str(error)
+
+ def test_6_1_10_deal(self):
+ with raises(contextutil.MaxWhileTries) as error:
+ with self.s_while(
+ increment=1,
+ _sleeper=self.fake_sleep
+ ) as proceed:
+ while proceed():
+ pass
+
+ assert 'waiting for 105 seconds' in str(error)
+
+ def test_timeout(self):
+ # series of sleep, increment, timeout params to test
+ params = [(10, 0, 100),
+ (1, 2, 30),
+ (10, 0.5, 100),
+ (2, 0, 5),
+ (2, 3, 5),
+ (10, 0, 15),
+ (20, 10, 60)]
+ for sleep, increment, timeout in params:
+ print("trying ", sleep, increment, timeout)
+ with raises(contextutil.MaxWhileTries) as error:
+ with self.s_while(
+ sleep=sleep,
+ increment=increment,
+ timeout=timeout,
+ _sleeper=self.fake_sleep
+ ) as proceed:
+ while proceed():
+ pass
+
+ assert 'waiting for {timeout}'.format(timeout=timeout) in str(error)
+
+ def test_action(self):
+ with raises(contextutil.MaxWhileTries) as error:
+ with self.s_while(
+ action='doing the thing',
+ _sleeper=self.fake_sleep
+ ) as proceed:
+ while proceed():
+ pass
+
+ assert "'doing the thing' reached maximum tries" in str(error)
+
+ def test_no_raise(self):
+ with self.s_while(_raise=False, _sleeper=self.fake_sleep) as proceed:
+ while proceed():
+ pass
+
+ assert True
+
+ def test_tries(self):
+ attempts = 0
+ with self.s_while(tries=-1, _sleeper=self.fake_sleep) as proceed:
+ while attempts < 100 and proceed():
+ attempts += 1
--- /dev/null
+# -*- coding: utf-8 -*-
+import pytest
+
+from tests.fake_fs import make_fake_fstools
+from teuthology.describe_tests import (tree_with_info, extract_info,
+ get_combinations)
+from teuthology.exceptions import ParseError
+from mock import MagicMock, patch
+
+realistic_fs = {
+ 'basic': {
+ '%': None,
+ 'base': {
+ 'install.yaml':
+ """meta:
+- desc: install ceph
+install:
+"""
+ },
+ 'clusters': {
+ 'fixed-1.yaml':
+ """meta:
+- desc: single node cluster
+roles:
+- [osd.0, osd.1, osd.2, mon.a, mon.b, mon.c, client.0]
+""",
+ 'fixed-2.yaml':
+ """meta:
+- desc: couple node cluster
+roles:
+- [osd.0, osd.1, osd.2, mon.a, mon.b, mon.c]
+- [client.0]
+""",
+ 'fixed-3.yaml':
+ """meta:
+- desc: triple node cluster
+roles:
+- [osd.0, osd.1, osd.2, mon.a, mon.b, mon.c]
+- [client.0]
+- [client.1]
+"""
+ },
+ 'workloads': {
+ 'rbd_api_tests_old_format.yaml':
+ """meta:
+- desc: c/c++ librbd api tests with format 1 images
+ rbd_features: none
+overrides:
+ ceph:
+ conf:
+ client:
+ rbd default format: 1
+tasks:
+- workunit:
+ env:
+ RBD_FEATURES: 0
+ clients:
+ client.0:
+ - rbd/test_librbd.sh
+""",
+ 'rbd_api_tests.yaml':
+ """meta:
+- desc: c/c++ librbd api tests with default settings
+ rbd_features: default
+tasks:
+- workunit:
+ clients:
+ client.0:
+ - rbd/test_librbd.sh
+""",
+ },
+ },
+}
+
+
+expected_tree = """├── %
+├── base
+│ └── install.yaml
+├── clusters
+│ ├── fixed-1.yaml
+│ ├── fixed-2.yaml
+│ └── fixed-3.yaml
+└── workloads
+ ├── rbd_api_tests.yaml
+ └── rbd_api_tests_old_format.yaml""".split('\n')
+
+
+expected_facets = [
+ '',
+ '',
+ 'base',
+ '',
+ 'clusters',
+ 'clusters',
+ 'clusters',
+ '',
+ 'workloads',
+ 'workloads',
+]
+
+
+expected_desc = [
+ '',
+ '',
+ 'install ceph',
+ '',
+ 'single node cluster',
+ 'couple node cluster',
+ 'triple node cluster',
+ '',
+ 'c/c++ librbd api tests with default settings',
+ 'c/c++ librbd api tests with format 1 images',
+]
+
+
+expected_rbd_features = [
+ '',
+ '',
+ '',
+ '',
+ '',
+ '',
+ '',
+ '',
+ 'default',
+ 'none',
+]
+
+
+class TestDescribeTests(object):
+
+ def setup_method(self):
+ self.mocks = dict()
+ self.patchers = dict()
+ exists, listdir, isfile, isdir, open = make_fake_fstools(realistic_fs)
+ for ppoint, fn in {
+ 'os.listdir': listdir,
+ 'os.path.isdir': isdir,
+ 'teuthology.describe_tests.open': open,
+ 'builtins.open': open,
+ 'os.path.exists': exists,
+ 'os.path.isfile': isfile,
+ }.items():
+ mockobj = MagicMock()
+ patcher = patch(ppoint, mockobj)
+ mockobj.side_effect = fn
+ patcher.start()
+ self.mocks[ppoint] = mockobj
+ self.patchers[ppoint] = patcher
+
+ def stop_patchers(self):
+ for patcher in self.patchers.values():
+ patcher.stop()
+
+ def teardown_method(self):
+ self.stop_patchers()
+
+ @staticmethod
+ def assert_expected_combo_headers(headers):
+ assert headers == (['subsuite depth 0'] +
+ sorted(set(filter(bool, expected_facets))))
+
+ def test_no_filters(self):
+ rows = tree_with_info('basic', [], False, '', [])
+ assert rows == [[x] for x in expected_tree]
+
+ def test_single_filter(self):
+ rows = tree_with_info('basic', ['desc'], False, '', [])
+ assert rows == [list(_) for _ in zip(expected_tree, expected_desc)]
+
+ rows = tree_with_info('basic', ['rbd_features'], False, '', [])
+ assert rows == [list(_) for _ in zip(expected_tree, expected_rbd_features)]
+
+ def test_single_filter_with_facets(self):
+ rows = tree_with_info('basic', ['desc'], True, '', [])
+ assert rows == [list(_) for _ in zip(expected_tree, expected_facets,
+ expected_desc)]
+
+ rows = tree_with_info('basic', ['rbd_features'], True, '', [])
+ assert rows == [list(_) for _ in zip(expected_tree, expected_facets,
+ expected_rbd_features)]
+
+ def test_no_matching(self):
+ rows = tree_with_info('basic', ['extra'], False, '', [])
+ assert rows == [list(_) for _ in zip(expected_tree, [''] * len(expected_tree))]
+
+ rows = tree_with_info('basic', ['extra'], True, '', [])
+ assert rows == [list(_) for _ in zip(expected_tree, expected_facets,
+ [''] * len(expected_tree))]
+
+ def test_multiple_filters(self):
+ rows = tree_with_info('basic', ['desc', 'rbd_features'], False, '', [])
+ assert rows == [list(_) for _ in zip(expected_tree,
+ expected_desc,
+ expected_rbd_features)]
+
+ rows = tree_with_info('basic', ['rbd_features', 'desc'], False, '', [])
+ assert rows == [list(_) for _ in zip(expected_tree,
+ expected_rbd_features,
+ expected_desc)]
+
+ def test_multiple_filters_with_facets(self):
+ rows = tree_with_info('basic', ['desc', 'rbd_features'], True, '', [])
+ assert rows == [list(_) for _ in zip(expected_tree,
+ expected_facets,
+ expected_desc,
+ expected_rbd_features)]
+
+ rows = tree_with_info('basic', ['rbd_features', 'desc'], True, '', [])
+ assert rows == [list(_) for _ in zip(expected_tree,
+ expected_facets,
+ expected_rbd_features,
+ expected_desc)]
+
+ def test_combinations_only_facets(self):
+ headers, rows = get_combinations('basic',
+ fields=[], subset=None, limit=1,
+ filter_in=None, filter_out=None, filter_all=None,
+ include_facet=True)
+ self.assert_expected_combo_headers(headers)
+ assert rows == [['basic', 'install', 'fixed-1', 'rbd_api_tests']]
+
+ def test_combinations_desc_features(self):
+ headers, rows = get_combinations('basic',
+ fields=['desc', 'rbd_features'], subset=None, limit=1,
+ filter_in=None, filter_out=None, filter_all=None,
+ include_facet=False)
+ assert headers == ['desc', 'rbd_features']
+ descriptions = '\n'.join([
+ 'install ceph',
+ 'single node cluster',
+ 'c/c++ librbd api tests with default settings',
+ ])
+ assert rows == [[descriptions, 'default']]
+
+ def test_combinations_filter_in(self):
+ headers, rows = get_combinations('basic',
+ fields=[], subset=None, limit=0,
+ filter_in=['old_format'], filter_out=None, filter_all=None,
+ include_facet=True)
+ self.assert_expected_combo_headers(headers)
+ assert rows == [
+ ['basic', 'install', 'fixed-1', 'rbd_api_tests_old_format'],
+ ['basic', 'install', 'fixed-2', 'rbd_api_tests_old_format'],
+ ['basic', 'install', 'fixed-3', 'rbd_api_tests_old_format'],
+ ]
+
+ def test_combinations_filter_out(self):
+ headers, rows = get_combinations('basic',
+ fields=[], subset=None, limit=0,
+ filter_in=None, filter_out=['old_format'], filter_all=None,
+ include_facet=True)
+ self.assert_expected_combo_headers(headers)
+ assert rows == [
+ ['basic', 'install', 'fixed-1', 'rbd_api_tests'],
+ ['basic', 'install', 'fixed-2', 'rbd_api_tests'],
+ ['basic', 'install', 'fixed-3', 'rbd_api_tests'],
+ ]
+
+ def test_combinations_filter_all(self):
+ headers, rows = get_combinations('basic',
+ fields=[], subset=None, limit=0,
+ filter_in=None, filter_out=None,
+ filter_all=['fixed-2', 'old_format'],
+ include_facet=True)
+ self.assert_expected_combo_headers(headers)
+ assert rows == [
+ ['basic', 'install', 'fixed-2', 'rbd_api_tests_old_format']
+ ]
+
+
+@patch('teuthology.describe_tests.open')
+@patch('os.path.isdir')
+def test_extract_info_dir(m_isdir, m_open):
+ simple_fs = {'a': {'b.yaml': 'meta: [{foo: c}]'}}
+ _, _, _, m_isdir.side_effect, m_open.side_effect = \
+ make_fake_fstools(simple_fs)
+ info = extract_info('a', [])
+ assert info == {}
+
+ info = extract_info('a', ['foo', 'bar'])
+ assert info == {'foo': '', 'bar': ''}
+
+ info = extract_info('a/b.yaml', ['foo', 'bar'])
+ assert info == {'foo': 'c', 'bar': ''}
+
+
+@patch('teuthology.describe_tests.open')
+@patch('os.path.isdir')
+def check_parse_error(fs, m_isdir, m_open):
+ _, _, _, m_isdir.side_effect, m_open.side_effect = make_fake_fstools(fs)
+ with pytest.raises(ParseError):
+ a = extract_info('a.yaml', ['a'])
+ raise Exception(str(a))
+
+
+def test_extract_info_too_many_elements():
+ check_parse_error({'a.yaml': 'meta: [{a: b}, {b: c}]'})
+
+
+def test_extract_info_not_a_list():
+ check_parse_error({'a.yaml': 'meta: {a: b}'})
+
+
+def test_extract_info_not_a_dict():
+ check_parse_error({'a.yaml': 'meta: [[a, b]]'})
+
+
+@patch('teuthology.describe_tests.open')
+@patch('os.path.isdir')
+def test_extract_info_empty_file(m_isdir, m_open):
+ simple_fs = {'a.yaml': ''}
+ _, _, _, m_isdir.side_effect, m_open.side_effect = \
+ make_fake_fstools(simple_fs)
+ info = extract_info('a.yaml', [])
+ assert info == {}
--- /dev/null
+from humanfriendly import format_timespan
+from mock import Mock, patch
+from pytest import mark
+from teuthology.config import config
+from teuthology.run_tasks import build_email_body as email_body
+from textwrap import dedent
+
+class TestSleepBeforeTeardownEmail(object):
+ def setup_method(self):
+ config.results_ui_server = "http://example.com/"
+ config.archive_server = "http://qa-proxy.ceph.com/teuthology/"
+
+ @mark.parametrize(
+ ['status', 'owner', 'suite_name', 'run_name', 'job_id', 'dura'],
+ [
+ [
+ 'pass',
+ 'noreply@host',
+ 'dummy',
+ 'run-name',
+ 123,
+ 3600,
+ ],
+ [
+ 'fail',
+ 'noname',
+ 'yummy',
+ 'next-run',
+ 1000,
+ 99999,
+ ],
+ ]
+ )
+ @patch("teuthology.run_tasks.time.time")
+ def test_sleep_before_teardown_email_body(self, m_time, status, owner,
+ suite_name, run_name, job_id, dura):
+ ctx = Mock()
+ archive_path='archive/path'
+ archive_dir='/archive/dir'
+ date_sec=3661
+ date_str='1970-01-01 01:01:01'
+ m_time.return_value=float(date_sec)
+ duration_sec=dura
+ duration_str=format_timespan(duration_sec)
+ ref_body=dedent("""
+ Teuthology job {run}/{job} has fallen asleep at {date} for {duration_str}
+
+ Owner: {owner}
+ Suite Name: {suite}
+ Sleep Date: {date}
+ Sleep Time: {duration_sec} seconds ({duration_str})
+ Job Info: http://example.com/{run}/
+ Job Logs: http://qa-proxy.ceph.com/teuthology/path/{job}/
+ Task Stack: a/b/c
+ Current Status: {status}"""
+ .format(duration_sec=duration_sec, duration_str=duration_str,
+ owner=owner, suite=suite_name, run=run_name,
+ job=job_id, status=status, date=date_str))
+ print(ref_body)
+ ctx.config = dict(
+ archive_path=archive_path,
+ job_id=job_id,
+ suite=suite_name,
+ )
+ if status == 'pass':
+ ctx.summary = dict(
+ success=True,
+ )
+ elif status == 'fail':
+ ctx.summary = dict(
+ success=False,
+ )
+ else:
+ ctx.summary = dict()
+
+ ctx.owner = owner
+ ctx.name = run_name
+ ctx.archive_dir = archive_dir
+ tasks = [('a', None), ('b', None), ('c', None)]
+ (subj, body) = email_body(ctx, tasks, dura)
+ assert body == ref_body.lstrip('\n')
--- /dev/null
+import os
+import random
+
+from unittest.mock import patch, Mock
+
+from teuthology import exit
+
+
+class TestExiter(object):
+ klass = exit.Exiter
+
+ def setup_method(self):
+ self.pid = os.getpid()
+
+ # Below, we patch os.kill() in such a way that the first time it is
+ # invoked it does actually send the signal. Any subsequent invocation
+ # won't send any signal - this is so we don't kill the process running
+ # our unit tests!
+ self.patcher_kill = patch(
+ 'teuthology.exit.os.kill',
+ wraps=os.kill,
+ )
+
+ #Keep a copy of the unpatched kill and call this in place of os.kill
+ #In the Exiter objects, the os.kill calls are patched.
+ #So the call_count should be 1.
+ self.kill_unpatched = os.kill
+ self.m_kill = self.patcher_kill.start()
+
+ def m_kill_unwrap(pid, sig):
+ # Setting return_value of a mocked object disables the wrapping
+ if self.m_kill.call_count > 1:
+ self.m_kill.return_value = None
+
+ self.m_kill.side_effect = m_kill_unwrap
+
+ def teardown_method(self):
+ self.patcher_kill.stop()
+ del self.m_kill
+
+ def test_basic(self):
+ sig = 15
+ obj = self.klass()
+ m_func = Mock()
+ obj.add_handler(sig, m_func)
+ assert len(obj.handlers) == 1
+ self.kill_unpatched(self.pid, sig)
+ assert m_func.call_count == 1
+ assert self.m_kill.call_count == 1
+ for arg_list in self.m_kill.call_args_list:
+ assert arg_list[0] == (self.pid, sig)
+
+ def test_remove_handlers(self):
+ sig = [1, 15]
+ send_sig = random.choice(sig)
+ n = 3
+ obj = self.klass()
+ handlers = list()
+ for i in range(n):
+ m_func = Mock(name="handler %s" % i)
+ handlers.append(obj.add_handler(sig, m_func))
+ assert obj.handlers == handlers
+ for handler in handlers:
+ handler.remove()
+ assert obj.handlers == list()
+ self.kill_unpatched(self.pid, send_sig)
+ assert self.m_kill.call_count == 1
+ for handler in handlers:
+ assert handler.func.call_count == 0
+
+ def test_n_handlers(self, n=10, sig=11):
+ if isinstance(sig, int):
+ send_sig = sig
+ else:
+ send_sig = random.choice(sig)
+ obj = self.klass()
+ handlers = list()
+ for i in range(n):
+ m_func = Mock(name="handler %s" % i)
+ handlers.append(obj.add_handler(sig, m_func))
+ assert obj.handlers == handlers
+ self.kill_unpatched(self.pid, send_sig)
+ for i in range(n):
+ assert handlers[i].func.call_count == 1
+ assert self.m_kill.call_count == 1
+ for arg_list in self.m_kill.call_args_list:
+ assert arg_list[0] == (self.pid, send_sig)
+
+ def test_multiple_signals(self):
+ self.test_n_handlers(n=3, sig=[1, 6, 11, 15])
--- /dev/null
+from teuthology.misc import get_distro
+
+
+class Mock:
+ pass
+
+
+class TestGetDistro(object):
+
+ def setup_method(self):
+ self.fake_ctx = Mock()
+ self.fake_ctx.config = {}
+ # os_type in ctx will always default to None
+ self.fake_ctx.os_type = None
+
+ def test_default_distro(self):
+ distro = get_distro(self.fake_ctx)
+ assert distro == 'ubuntu'
+
+ def test_argument(self):
+ # we don't want fake_ctx to have a config
+ self.fake_ctx = Mock()
+ self.fake_ctx.os_type = 'centos'
+ distro = get_distro(self.fake_ctx)
+ assert distro == 'centos'
+
+ def test_teuth_config(self):
+ self.fake_ctx.config = {'os_type': 'fedora'}
+ distro = get_distro(self.fake_ctx)
+ assert distro == 'fedora'
+
+ def test_argument_takes_precedence(self):
+ self.fake_ctx.config = {'os_type': 'fedora'}
+ self.fake_ctx.os_type = "centos"
+ distro = get_distro(self.fake_ctx)
+ assert distro == 'centos'
+
+ def test_no_config_or_os_type(self):
+ self.fake_ctx = Mock()
+ self.fake_ctx.os_type = None
+ distro = get_distro(self.fake_ctx)
+ assert distro == 'ubuntu'
+
+ def test_config_os_type_is_none(self):
+ self.fake_ctx.config["os_type"] = None
+ distro = get_distro(self.fake_ctx)
+ assert distro == 'ubuntu'
--- /dev/null
+from teuthology.misc import get_distro_version
+
+
+class Mock:
+ pass
+
+
+class TestGetDistroVersion(object):
+
+ def setup_method(self):
+ self.fake_ctx = Mock()
+ self.fake_ctx.config = {}
+ self.fake_ctx_noarg = Mock()
+ self.fake_ctx_noarg.config = {}
+ self.fake_ctx_noarg.os_version = None
+ self.fake_ctx.os_type = None
+ self.fake_ctx_noarg.os_type = None
+
+ def test_default_distro_version(self):
+ # Default distro is ubuntu, default version of ubuntu is 20.04
+ self.fake_ctx.os_version = None
+ distroversion = get_distro_version(self.fake_ctx)
+ assert distroversion == '22.04'
+
+ def test_argument_version(self):
+ self.fake_ctx.os_version = '13.04'
+ distroversion = get_distro_version(self.fake_ctx)
+ assert distroversion == '13.04'
+
+ def test_teuth_config_version(self):
+ #Argument takes precidence.
+ self.fake_ctx.os_version = '13.04'
+ self.fake_ctx.config = {'os_version': '13.10'}
+ distroversion = get_distro_version(self.fake_ctx)
+ assert distroversion == '13.04'
+
+ def test_teuth_config_noarg_version(self):
+ self.fake_ctx_noarg.config = {'os_version': '13.04'}
+ distroversion = get_distro_version(self.fake_ctx_noarg)
+ assert distroversion == '13.04'
+
+ def test_no_teuth_config(self):
+ self.fake_ctx = Mock()
+ self.fake_ctx.os_type = None
+ self.fake_ctx.os_version = '13.04'
+ distroversion = get_distro_version(self.fake_ctx)
+ assert distroversion == '13.04'
--- /dev/null
+from teuthology import misc as teuthology
+
+class Mock: pass
+
+class TestGetMultiMachineTypes(object):
+
+ def test_space(self):
+ give = 'burnupi plana vps'
+ expect = ['burnupi','plana','vps']
+ assert teuthology.get_multi_machine_types(give) == expect
+
+ def test_tab(self):
+ give = 'burnupi plana vps'
+ expect = ['burnupi','plana','vps']
+ assert teuthology.get_multi_machine_types(give) == expect
+
+ def test_comma(self):
+ give = 'burnupi,plana,vps'
+ expect = ['burnupi','plana','vps']
+ assert teuthology.get_multi_machine_types(give) == expect
+
+ def test_single(self):
+ give = 'burnupi'
+ expect = ['burnupi']
+ assert teuthology.get_multi_machine_types(give) == expect
+
+
--- /dev/null
+import importlib
+import pytest
+import sys
+
+from pathlib import Path
+from typing import List
+
+root = Path("./teuthology")
+
+
+def find_modules() -> List[str]:
+ modules = []
+ for path in root.rglob("*.py"):
+ if path.name.startswith("test_"):
+ continue
+ if "-" in path.name:
+ continue
+ if path.name == "__init__.py":
+ path = path.parent
+
+ path_name = str(path).replace("/", ".")
+ if path_name.endswith(".py"):
+ path_name = path_name[:-3]
+ modules.append(path_name)
+ return sorted(modules)
+
+
+@pytest.mark.parametrize("module", find_modules())
+def test_import_modules(module):
+ importlib.import_module(module)
+ assert module in sys.modules
--- /dev/null
+from teuthology import job_status
+
+
+class TestJobStatus(object):
+ def test_get_only_success_true(self):
+ summary = dict(success=True)
+ status = job_status.get_status(summary)
+ assert status == 'pass'
+
+ def test_get_only_success_false(self):
+ summary = dict(success=False)
+ status = job_status.get_status(summary)
+ assert status == 'fail'
+
+ def test_get_status_pass(self):
+ summary = dict(status='pass')
+ status = job_status.get_status(summary)
+ assert status == 'pass'
+
+ def test_get_status_fail(self):
+ summary = dict(status='fail')
+ status = job_status.get_status(summary)
+ assert status == 'fail'
+
+ def test_get_status_dead(self):
+ summary = dict(status='dead')
+ status = job_status.get_status(summary)
+ assert status == 'dead'
+
+ def test_get_status_none(self):
+ summary = dict()
+ status = job_status.get_status(summary)
+ assert status is None
+
+ def test_set_status_pass(self):
+ summary = dict()
+ job_status.set_status(summary, 'pass')
+ assert summary == dict(status='pass', success=True)
+
+ def test_set_status_dead(self):
+ summary = dict()
+ job_status.set_status(summary, 'dead')
+ assert summary == dict(status='dead', success=False)
+
+ def test_set_then_get_status_dead(self):
+ summary = dict()
+ job_status.set_status(summary, 'dead')
+ status = job_status.get_status(summary)
+ assert status == 'dead'
+
+ def test_set_status_none(self):
+ summary = dict()
+ job_status.set_status(summary, None)
+ assert summary == dict()
+
+ def test_legacy_fail(self):
+ summary = dict(success=True)
+ summary['success'] = False
+ status = job_status.get_status(summary)
+ assert status == 'fail'
--- /dev/null
+from unittest.mock import patch
+
+from teuthology.kill import find_targets
+
+
+class TestFindTargets(object):
+ """ Tests for teuthology.kill.find_targets """
+
+ @patch('teuthology.kill.report.ResultsReporter.get_jobs')
+ def test_missing_run_find_targets(self, m_get_jobs):
+ m_get_jobs.return_value = []
+ run_targets = find_targets("run-name")
+ assert run_targets == {}
+
+ @patch('teuthology.kill.report.ResultsReporter.get_jobs')
+ def test_missing_job_find_targets(self, m_get_jobs):
+ m_get_jobs.return_value = {}
+ job_targets = find_targets("run-name", "3")
+ assert job_targets == {}
+
+ @patch('teuthology.kill.report.ResultsReporter.get_jobs')
+ def test_missing_run_targets_find_targets(self, m_get_jobs):
+ m_get_jobs.return_value = [{"targets": None, "status": "waiting"}]
+ run_targets = find_targets("run-name")
+ assert run_targets == {}
+
+ @patch('teuthology.kill.report.ResultsReporter.get_jobs')
+ def test_missing_job_targets_find_targets(self, m_get_jobs):
+ m_get_jobs.return_value = {"targets": None}
+ job_targets = find_targets("run-name", "3")
+ assert job_targets == {}
+
+ @patch('teuthology.kill.report.ResultsReporter.get_jobs')
+ def test_run_find_targets(self, m_get_jobs):
+ m_get_jobs.return_value = [{"targets": {"node1": ""}, "status": "running"}]
+ run_targets = find_targets("run-name")
+ assert run_targets == {"node1": ""}
+ m_get_jobs.return_value = [{"targets": {"node1": ""}}]
+ run_targets = find_targets("run-name")
+ assert run_targets == {}
+
+ @patch('teuthology.kill.report.ResultsReporter.get_jobs')
+ def test_job_find_targets(self, m_get_jobs):
+ m_get_jobs.return_value = {"targets": {"node1": ""}}
+ job_targets = find_targets("run-name", "3")
+ assert job_targets == {"node1": ""}
--- /dev/null
+import pytest
+
+from unittest.mock import patch, Mock
+
+from teuthology import ls
+
+
+class TestLs(object):
+ """ Tests for teuthology.ls """
+
+ @patch('os.path.isdir')
+ @patch('os.listdir')
+ def test_get_jobs(self, m_listdir, m_isdir):
+ m_listdir.return_value = ["1", "a", "3"]
+ m_isdir.return_value = True
+ results = ls.get_jobs("some/archive/dir")
+ assert results == ["1", "3"]
+
+ @patch("yaml.safe_load_all")
+ @patch("teuthology.ls.get_jobs")
+ def test_ls(self, m_get_jobs, m_safe_load_all):
+ m_get_jobs.return_value = ["1", "2"]
+ m_safe_load_all.return_value = [{"failure_reason": "reasons"}]
+ ls.ls("some/archive/div", True)
+
+ @patch("teuthology.ls.open")
+ @patch("teuthology.ls.get_jobs")
+ def test_ls_ioerror(self, m_get_jobs, m_open):
+ m_get_jobs.return_value = ["1", "2"]
+ m_open.side_effect = IOError()
+ with pytest.raises(IOError):
+ ls.ls("some/archive/dir", True)
+
+ @patch("teuthology.ls.open")
+ @patch("os.popen")
+ @patch("os.path.isdir")
+ @patch("os.path.isfile")
+ def test_print_debug_info(self, m_isfile, m_isdir, m_popen, m_open):
+ m_isfile.return_value = True
+ m_isdir.return_value = True
+ m_popen.return_value = Mock()
+ cmdline = Mock()
+ cmdline.find = Mock(return_value=0)
+ m1 = Mock()
+ m2 = Mock()
+ m2.read = Mock(return_value=cmdline)
+ m_open.side_effect = [m1, m2]
+ ls.print_debug_info("the_job", "job/dir", "some/archive/dir")
--- /dev/null
+import argparse
+import pytest
+import subprocess
+
+from unittest.mock import Mock, patch
+
+from teuthology import misc
+from teuthology.config import config
+from teuthology.orchestra import cluster
+from teuthology.orchestra.remote import Remote
+
+
+class FakeRemote(object):
+ pass
+
+
+def test_sh_normal(caplog):
+ assert misc.sh("/bin/echo ABC") == "ABC\n"
+ assert "truncated" not in caplog.text
+
+
+def test_sh_truncate(caplog):
+ assert misc.sh("/bin/echo -n AB ; /bin/echo C", 2) == "ABC\n"
+ assert "truncated" in caplog.text
+ assert "ABC" not in caplog.text
+
+
+def test_sh_fail(caplog):
+ with pytest.raises(subprocess.CalledProcessError) as excinfo:
+ misc.sh("/bin/echo -n AB ; /bin/echo C ; exit 111", 2)
+ assert excinfo.value.returncode == 111
+ for record in caplog.records:
+ if record.levelname == 'ERROR':
+ assert ('replay full' in record.message or
+ 'ABC\n' == record.message)
+
+def test_sh_progress(caplog):
+ assert misc.sh("echo AB ; sleep 0.1 ; /bin/echo C", 2) == "AB\nC\n"
+ records = caplog.records
+ assert ':sh: ' in records[0].message
+ assert 'AB' == records[1].message
+ assert 'C' == records[2].message
+ assert records[2].created > records[1].created
+
+
+def test_wait_until_osds_up():
+ ctx = argparse.Namespace()
+ ctx.daemons = Mock()
+ ctx.daemons.iter_daemons_of_role.return_value = list()
+ remote = Mock(spec=Remote)
+ remote.sh.return_value = 'IGNORED\n{"osds":[{"state":["up"]}]}'
+ ctx.cluster = cluster.Cluster(
+ remotes=[
+ (remote, ['osd.0', 'client.1'])
+ ],
+ )
+ with patch.multiple(
+ misc,
+ get_testdir=lambda _: "TESTDIR",
+ ):
+ misc.wait_until_osds_up(ctx, ctx.cluster, remote)
+
+
+def test_get_clients_simple():
+ ctx = argparse.Namespace()
+ remote = FakeRemote()
+ ctx.cluster = cluster.Cluster(
+ remotes=[
+ (remote, ['client.0', 'client.1'])
+ ],
+ )
+ g = misc.get_clients(ctx=ctx, roles=['client.1'])
+ got = next(g)
+ assert len(got) == 2
+ assert got[0] == ('1')
+ assert got[1] is remote
+ with pytest.raises(StopIteration):
+ next(g)
+
+
+def test_get_mon_names():
+ expected = [
+ ([['mon.a', 'osd.0', 'mon.c']], 'ceph', ['mon.a', 'mon.c']),
+ ([['ceph.mon.a', 'osd.0', 'ceph.mon.c']], 'ceph', ['ceph.mon.a', 'ceph.mon.c']),
+ ([['mon.a', 'osd.0', 'mon.c'], ['ceph.mon.b']], 'ceph', ['mon.a', 'mon.c', 'ceph.mon.b']),
+ ([['mon.a', 'osd.0', 'mon.c'], ['foo.mon.a']], 'ceph', ['mon.a', 'mon.c']),
+ ([['mon.a', 'osd.0', 'mon.c'], ['foo.mon.a']], 'foo', ['foo.mon.a']),
+ ]
+ for remote_roles, cluster_name, expected_mons in expected:
+ ctx = argparse.Namespace()
+ ctx.cluster = Mock()
+ ctx.cluster.remotes = {i: roles for i, roles in enumerate(remote_roles)}
+ mons = misc.get_mon_names(ctx, cluster_name)
+ assert expected_mons == mons
+
+
+def test_get_first_mon():
+ expected = [
+ ([['mon.a', 'osd.0', 'mon.c']], 'ceph', 'mon.a'),
+ ([['ceph.mon.a', 'osd.0', 'ceph.mon.c']], 'ceph', 'ceph.mon.a'),
+ ([['mon.a', 'osd.0', 'mon.c'], ['ceph.mon.b']], 'ceph', 'ceph.mon.b'),
+ ([['mon.a', 'osd.0', 'mon.c'], ['foo.mon.a']], 'ceph', 'mon.a'),
+ ([['foo.mon.b', 'osd.0', 'mon.c'], ['foo.mon.a']], 'foo', 'foo.mon.a'),
+ ]
+ for remote_roles, cluster_name, expected_mon in expected:
+ ctx = argparse.Namespace()
+ ctx.cluster = Mock()
+ ctx.cluster.remotes = {i: roles for i, roles in enumerate(remote_roles)}
+ mon = misc.get_first_mon(ctx, None, cluster_name)
+ assert expected_mon == mon
+
+
+def test_roles_of_type():
+ expected = [
+ (['client.0', 'osd.0', 'ceph.osd.1'], 'osd', ['0', '1']),
+ (['client.0', 'osd.0', 'ceph.osd.1'], 'client', ['0']),
+ (['foo.client.1', 'bar.client.2.3', 'baz.osd.1'], 'mon', []),
+ (['foo.client.1', 'bar.client.2.3', 'baz.osd.1'], 'client',
+ ['1', '2.3']),
+ ]
+ for roles_for_host, type_, expected_ids in expected:
+ ids = list(misc.roles_of_type(roles_for_host, type_))
+ assert ids == expected_ids
+
+
+def test_cluster_roles_of_type():
+ expected = [
+ (['client.0', 'osd.0', 'ceph.osd.1'], 'osd', 'ceph',
+ ['osd.0', 'ceph.osd.1']),
+ (['client.0', 'osd.0', 'ceph.osd.1'], 'client', 'ceph',
+ ['client.0']),
+ (['foo.client.1', 'bar.client.2.3', 'baz.osd.1'], 'mon', None, []),
+ (['foo.client.1', 'bar.client.2.3', 'baz.osd.1'], 'client', None,
+ ['foo.client.1', 'bar.client.2.3']),
+ (['foo.client.1', 'bar.client.2.3', 'baz.osd.1'], 'client', 'bar',
+ ['bar.client.2.3']),
+ ]
+ for roles_for_host, type_, cluster_, expected_roles in expected:
+ roles = list(misc.cluster_roles_of_type(roles_for_host, type_, cluster_))
+ assert roles == expected_roles
+
+
+def test_all_roles_of_type():
+ expected = [
+ ([['client.0', 'osd.0', 'ceph.osd.1'], ['bar.osd.2']],
+ 'osd', ['0', '1', '2']),
+ ([['client.0', 'osd.0', 'ceph.osd.1'], ['bar.osd.2', 'baz.client.1']],
+ 'client', ['0', '1']),
+ ([['foo.client.1', 'bar.client.2.3'], ['baz.osd.1']], 'mon', []),
+ ([['foo.client.1', 'bar.client.2.3'], ['baz.osd.1', 'ceph.client.bar']],
+ 'client', ['1', '2.3', 'bar']),
+ ]
+ for host_roles, type_, expected_ids in expected:
+ cluster_ = Mock()
+ cluster_.remotes = dict(enumerate(host_roles))
+ ids = list(misc.all_roles_of_type(cluster_, type_))
+ assert ids == expected_ids
+
+
+def test_get_http_log_path():
+ # Fake configuration
+ archive_server = "http://example.com/server_root"
+ config.archive_server = archive_server
+ archive_dir = "/var/www/archives"
+
+ path = misc.get_http_log_path(archive_dir)
+ assert path == "http://example.com/server_root/archives/"
+
+ job_id = '12345'
+ path = misc.get_http_log_path(archive_dir, job_id)
+ assert path == "http://example.com/server_root/archives/12345/"
+
+ # Inktank configuration
+ archive_server = "http://qa-proxy.ceph.com/teuthology/"
+ config.archive_server = archive_server
+ archive_dir = "/var/lib/teuthworker/archive/teuthology-2013-09-12_11:49:50-ceph-deploy-main-testing-basic-vps"
+ job_id = 31087
+ path = misc.get_http_log_path(archive_dir, job_id)
+ assert path == "http://qa-proxy.ceph.com/teuthology/teuthology-2013-09-12_11:49:50-ceph-deploy-main-testing-basic-vps/31087/"
+
+ path = misc.get_http_log_path(archive_dir)
+ assert path == "http://qa-proxy.ceph.com/teuthology/teuthology-2013-09-12_11:49:50-ceph-deploy-main-testing-basic-vps/"
+
+
+def test_is_type():
+ is_client = misc.is_type('client')
+ assert is_client('client.0')
+ assert is_client('ceph.client.0')
+ assert is_client('foo.client.0')
+ assert is_client('foo.client.bar.baz')
+
+ with pytest.raises(ValueError):
+ is_client('')
+ is_client('client')
+ assert not is_client('foo.bar.baz')
+ assert not is_client('ceph.client')
+ assert not is_client('hadoop.main.0')
+
+
+def test_is_type_in_cluster():
+ is_c1_osd = misc.is_type('osd', 'c1')
+ with pytest.raises(ValueError):
+ is_c1_osd('')
+ assert not is_c1_osd('osd.0')
+ assert not is_c1_osd('ceph.osd.0')
+ assert not is_c1_osd('ceph.osd.0')
+ assert not is_c1_osd('c11.osd.0')
+ assert is_c1_osd('c1.osd.0')
+ assert is_c1_osd('c1.osd.999')
+
+
+def test_get_mons():
+ ips = ['1.1.1.1', '2.2.2.2', '3.3.3.3']
+ addrs = ['1.1.1.1:6789', '1.1.1.1:6790', '1.1.1.1:6791']
+
+ mons = misc.get_mons([['mon.a']], ips)
+ assert mons == {'mon.a': addrs[0]}
+
+ mons = misc.get_mons([['cluster-a.mon.foo', 'client.b'], ['osd.0']], ips)
+ assert mons == {'cluster-a.mon.foo': addrs[0]}
+
+ mons = misc.get_mons([['mon.a', 'mon.b', 'ceph.mon.c']], ips)
+ assert mons == {'mon.a': addrs[0],
+ 'mon.b': addrs[1],
+ 'ceph.mon.c': addrs[2]}
+
+ mons = misc.get_mons([['mon.a'], ['mon.b'], ['ceph.mon.c']], ips)
+ assert mons == {'mon.a': addrs[0],
+ 'mon.b': ips[1] + ':6789',
+ 'ceph.mon.c': ips[2] + ':6789'}
+
+
+def test_split_role():
+ expected = {
+ 'client.0': ('ceph', 'client', '0'),
+ 'foo.client.0': ('foo', 'client', '0'),
+ 'bar.baz.x.y.z': ('bar', 'baz', 'x.y.z'),
+ 'mds.a-s-b': ('ceph', 'mds', 'a-s-b'),
+ }
+
+ for role, expected_split in expected.items():
+ actual_split = misc.split_role(role)
+ assert actual_split == expected_split
+
+def test_update_key():
+ a = { "sha": "foo", "workunit": { "sha": "foo" }, "tasks": [{"task1": "ceph"}], "overrides": [{"sha": "foo"}] }
+ b = { "sha": "blah", "workunit": { "sha": "bar" }, "tasks": [] }
+
+ misc.update_key("sha", a, b)
+ assert a == { "sha": "blah", "workunit": { "sha": "bar" }, "tasks": [{"task1": "ceph"}], "overrides": [{"sha": "foo"}] }
+
+class TestHostnames(object):
+ def setup_method(self):
+ config._conf = dict()
+
+ def teardown_method(self):
+ config.load()
+
+ def test_canonicalize_hostname(self):
+ host_base = 'box1'
+ result = misc.canonicalize_hostname(host_base)
+ assert result == 'ubuntu@box1.front.sepia.ceph.com'
+
+ def test_decanonicalize_hostname(self):
+ host = 'ubuntu@box1.front.sepia.ceph.com'
+ result = misc.decanonicalize_hostname(host)
+ assert result == 'box1'
+
+ def test_canonicalize_hostname_nouser(self):
+ host_base = 'box1'
+ result = misc.canonicalize_hostname(host_base, user=None)
+ assert result == 'box1.front.sepia.ceph.com'
+
+ def test_decanonicalize_hostname_nouser(self):
+ host = 'box1.front.sepia.ceph.com'
+ result = misc.decanonicalize_hostname(host)
+ assert result == 'box1'
+
+ def test_canonicalize_hostname_otherlab(self):
+ config.lab_domain = 'example.com'
+ host_base = 'box1'
+ result = misc.canonicalize_hostname(host_base)
+ assert result == 'ubuntu@box1.example.com'
+
+ def test_decanonicalize_hostname_otherlab(self):
+ config.lab_domain = 'example.com'
+ host = 'ubuntu@box1.example.com'
+ result = misc.decanonicalize_hostname(host)
+ assert result == 'box1'
+
+ def test_canonicalize_hostname_nodomain(self):
+ config.lab_domain = ''
+ host = 'box2'
+ result = misc.canonicalize_hostname(host)
+ assert result == 'ubuntu@' + host
+
+ def test_decanonicalize_hostname_nodomain(self):
+ config.lab_domain = ''
+ host = 'ubuntu@box2'
+ result = misc.decanonicalize_hostname(host)
+ assert result == 'box2'
+
+ def test_canonicalize_hostname_full_other_user(self):
+ config.lab_domain = 'example.com'
+ host = 'user1@box1.example.come'
+ result = misc.canonicalize_hostname(host)
+ assert result == 'user1@box1.example.com'
+
+ def test_decanonicalize_hostname_full_other_user(self):
+ config.lab_domain = 'example.com'
+ host = 'user1@box1.example.come'
+ result = misc.decanonicalize_hostname(host)
+ assert result == 'box1'
+
+class TestMergeConfigs(object):
+ """ Tests merge_config and deep_merge in teuthology.misc """
+
+ @patch("os.path.exists")
+ @patch("yaml.safe_load")
+ @patch("teuthology.misc.open")
+ def test_merge_configs(self, m_open, m_safe_load, m_exists):
+ """ Only tests with one yaml file being passed, mainly just to test
+ the loop logic. The actual merge will be tested in subsequent
+ tests.
+ """
+ expected = {"a": "b", "b": "c"}
+ m_exists.return_value = True
+ m_safe_load.return_value = expected
+ result = misc.merge_configs(["path/to/config1"])
+ assert result == expected
+ m_open.assert_called_once_with("path/to/config1")
+
+ def test_merge_configs_empty(self):
+ assert misc.merge_configs([]) == {}
+
+ def test_deep_merge(self):
+ a = {"a": "b"}
+ b = {"b": "c"}
+ result = misc.deep_merge(a, b)
+ assert result == {"a": "b", "b": "c"}
+
+ def test_overwrite_deep_merge(self):
+ a = {"a": "b"}
+ b = {"a": "overwritten", "b": "c"}
+ result = misc.deep_merge(a, b)
+ assert result == {"a": "overwritten", "b": "c"}
+
+ def test_list_deep_merge(self):
+ a = [1, 2]
+ b = [3, 4]
+ result = misc.deep_merge(a, b)
+ assert result == [1, 2, 3, 4]
+
+ def test_missing_list_deep_merge(self):
+ a = [1, 2]
+ b = "not a list"
+ with pytest.raises(AssertionError):
+ misc.deep_merge(a, b)
+
+ def test_missing_a_deep_merge(self):
+ result = misc.deep_merge(None, [1, 2])
+ assert result == [1, 2]
+
+ def test_missing_b_deep_merge(self):
+ result = misc.deep_merge([1, 2], None)
+ assert result == [1, 2]
+
+ def test_invalid_b_deep_merge(self):
+ with pytest.raises(AssertionError):
+ misc.deep_merge({"a": "b"}, "invalid")
+
+
+class TestIsInDict(object):
+ def test_simple_membership(self):
+ assert misc.is_in_dict('a', 'foo', {'a':'foo', 'b':'bar'})
+
+ def test_dict_membership(self):
+ assert misc.is_in_dict(
+ 'a', {'sub1':'key1', 'sub2':'key2'},
+ {'a':{'sub1':'key1', 'sub2':'key2', 'sub3':'key3'}}
+ )
+
+ def test_simple_nonmembership(self):
+ assert not misc.is_in_dict('a', 'foo', {'a':'bar', 'b':'foo'})
+
+ def test_nonmembership_with_presence_at_lower_level(self):
+ assert not misc.is_in_dict('a', 'foo', {'a':{'a': 'foo'}})
--- /dev/null
+import pytest
+
+from unittest.mock import patch, Mock, PropertyMock
+
+from teuthology import packaging
+from teuthology.exceptions import VersionNotFoundError
+
+KOJI_TASK_RPMS_MATRIX = [
+ ('tasks/6745/9666745/kernel-4.1.0-0.rc2.git2.1.fc23.x86_64.rpm', 'kernel'),
+ ('tasks/6745/9666745/kernel-modules-4.1.0-0.rc2.git2.1.fc23.x86_64.rpm', 'kernel-modules'),
+ ('tasks/6745/9666745/kernel-tools-4.1.0-0.rc2.git2.1.fc23.x86_64.rpm', 'kernel-tools'),
+ ('tasks/6745/9666745/kernel-tools-libs-devel-4.1.0-0.rc2.git2.1.fc23.x86_64.rpm', 'kernel-tools-libs-devel'),
+ ('tasks/6745/9666745/kernel-headers-4.1.0-0.rc2.git2.1.fc23.x86_64.rpm', 'kernel-headers'),
+ ('tasks/6745/9666745/kernel-tools-debuginfo-4.1.0-0.rc2.git2.1.fc23.x86_64.rpm', 'kernel-tools-debuginfo'),
+ ('tasks/6745/9666745/kernel-debuginfo-common-x86_64-4.1.0-0.rc2.git2.1.fc23.x86_64.rpm', 'kernel-debuginfo-common-x86_64'),
+ ('tasks/6745/9666745/perf-debuginfo-4.1.0-0.rc2.git2.1.fc23.x86_64.rpm', 'perf-debuginfo'),
+ ('tasks/6745/9666745/kernel-modules-extra-4.1.0-0.rc2.git2.1.fc23.x86_64.rpm', 'kernel-modules-extra'),
+ ('tasks/6745/9666745/kernel-tools-libs-4.1.0-0.rc2.git2.1.fc23.x86_64.rpm', 'kernel-tools-libs'),
+ ('tasks/6745/9666745/kernel-core-4.1.0-0.rc2.git2.1.fc23.x86_64.rpm', 'kernel-core'),
+ ('tasks/6745/9666745/kernel-debuginfo-4.1.0-0.rc2.git2.1.fc23.x86_64.rpm', 'kernel-debuginfo'),
+ ('tasks/6745/9666745/python-perf-4.1.0-0.rc2.git2.1.fc23.x86_64.rpm', 'python-perf'),
+ ('tasks/6745/9666745/kernel-devel-4.1.0-0.rc2.git2.1.fc23.x86_64.rpm', 'kernel-devel'),
+ ('tasks/6745/9666745/python-perf-debuginfo-4.1.0-0.rc2.git2.1.fc23.x86_64.rpm', 'python-perf-debuginfo'),
+ ('tasks/6745/9666745/perf-4.1.0-0.rc2.git2.1.fc23.x86_64.rpm', 'perf'),
+]
+
+KOJI_TASK_RPMS = [rpm[0] for rpm in KOJI_TASK_RPMS_MATRIX]
+
+
+class TestPackaging(object):
+
+ def test_get_package_name_deb(self):
+ remote = Mock()
+ remote.os.package_type = "deb"
+ assert packaging.get_package_name('sqlite', remote) == "sqlite3"
+
+ def test_get_package_name_rpm(self):
+ remote = Mock()
+ remote.os.package_type = "rpm"
+ assert packaging.get_package_name('sqlite', remote) is None
+
+ def test_get_package_name_not_found(self):
+ remote = Mock()
+ remote.os.package_type = "rpm"
+ assert packaging.get_package_name('notthere', remote) is None
+
+ def test_get_service_name_deb(self):
+ remote = Mock()
+ remote.os.package_type = "deb"
+ assert packaging.get_service_name('httpd', remote) == 'apache2'
+
+ def test_get_service_name_rpm(self):
+ remote = Mock()
+ remote.os.package_type = "rpm"
+ assert packaging.get_service_name('httpd', remote) == 'httpd'
+
+ def test_get_service_name_not_found(self):
+ remote = Mock()
+ remote.os.package_type = "rpm"
+ assert packaging.get_service_name('notthere', remote) is None
+
+ def test_install_package_deb(self):
+ m_remote = Mock()
+ m_remote.os.package_type = "deb"
+ expected = [
+ 'DEBIAN_FRONTEND=noninteractive',
+ 'sudo',
+ '-E',
+ 'apt-get',
+ '-y',
+ '--force-yes',
+ 'install',
+ 'apache2'
+ ]
+ packaging.install_package('apache2', m_remote)
+ m_remote.run.assert_called_with(args=expected)
+
+ def test_install_package_rpm(self):
+ m_remote = Mock()
+ m_remote.os.package_type = "rpm"
+ expected = [
+ 'sudo',
+ 'yum',
+ '-y',
+ 'install',
+ 'httpd'
+ ]
+ packaging.install_package('httpd', m_remote)
+ m_remote.run.assert_called_with(args=expected)
+
+ def test_remove_package_deb(self):
+ m_remote = Mock()
+ m_remote.os.package_type = "deb"
+ expected = [
+ 'DEBIAN_FRONTEND=noninteractive',
+ 'sudo',
+ '-E',
+ 'apt-get',
+ '-y',
+ 'purge',
+ 'apache2'
+ ]
+ packaging.remove_package('apache2', m_remote)
+ m_remote.run.assert_called_with(args=expected)
+
+ def test_remove_package_rpm(self):
+ m_remote = Mock()
+ m_remote.os.package_type = "rpm"
+ expected = [
+ 'sudo',
+ 'yum',
+ '-y',
+ 'erase',
+ 'httpd'
+ ]
+ packaging.remove_package('httpd', m_remote)
+ m_remote.run.assert_called_with(args=expected)
+
+ def test_get_koji_package_name(self):
+ build_info = dict(version="3.10.0", release="123.20.1")
+ result = packaging.get_koji_package_name("kernel", build_info)
+ assert result == "kernel-3.10.0-123.20.1.x86_64.rpm"
+
+ @patch("teuthology.packaging.config")
+ def test_get_kojiroot_base_url(self, m_config):
+ m_config.kojiroot_url = "http://kojiroot.com"
+ build_info = dict(
+ package_name="kernel",
+ version="3.10.0",
+ release="123.20.1",
+ )
+ result = packaging.get_kojiroot_base_url(build_info)
+ expected = "http://kojiroot.com/kernel/3.10.0/123.20.1/x86_64/"
+ assert result == expected
+
+ @patch("teuthology.packaging.config")
+ def test_get_koji_build_info_success(self, m_config):
+ m_config.kojihub_url = "http://kojihub.com"
+ m_proc = Mock()
+ expected = dict(foo="bar")
+ m_proc.exitstatus = 0
+ m_proc.stdout.getvalue.return_value = str(expected)
+ m_remote = Mock()
+ m_remote.run.return_value = m_proc
+ result = packaging.get_koji_build_info(1, m_remote, dict())
+ assert result == expected
+ args, kwargs = m_remote.run.call_args
+ expected_args = [
+ 'python', '-c',
+ 'import koji; '
+ 'hub = koji.ClientSession("http://kojihub.com"); '
+ 'print(hub.getBuild(1))',
+ ]
+ assert expected_args == kwargs['args']
+
+ @patch("teuthology.packaging.config")
+ def test_get_koji_build_info_fail(self, m_config):
+ m_config.kojihub_url = "http://kojihub.com"
+ m_proc = Mock()
+ m_proc.exitstatus = 1
+ m_remote = Mock()
+ m_remote.run.return_value = m_proc
+ m_ctx = Mock()
+ m_ctx.summary = dict()
+ with pytest.raises(RuntimeError):
+ packaging.get_koji_build_info(1, m_remote, m_ctx)
+
+ @patch("teuthology.packaging.config")
+ def test_get_koji_task_result_success(self, m_config):
+ m_config.kojihub_url = "http://kojihub.com"
+ m_proc = Mock()
+ expected = dict(foo="bar")
+ m_proc.exitstatus = 0
+ m_proc.stdout.getvalue.return_value = str(expected)
+ m_remote = Mock()
+ m_remote.run.return_value = m_proc
+ result = packaging.get_koji_task_result(1, m_remote, dict())
+ assert result == expected
+ args, kwargs = m_remote.run.call_args
+ expected_args = [
+ 'python', '-c',
+ 'import koji; '
+ 'hub = koji.ClientSession("http://kojihub.com"); '
+ 'print(hub.getTaskResult(1))',
+ ]
+ assert expected_args == kwargs['args']
+
+ @patch("teuthology.packaging.config")
+ def test_get_koji_task_result_fail(self, m_config):
+ m_config.kojihub_url = "http://kojihub.com"
+ m_proc = Mock()
+ m_proc.exitstatus = 1
+ m_remote = Mock()
+ m_remote.run.return_value = m_proc
+ m_ctx = Mock()
+ m_ctx.summary = dict()
+ with pytest.raises(RuntimeError):
+ packaging.get_koji_task_result(1, m_remote, m_ctx)
+
+ @patch("teuthology.packaging.config")
+ def test_get_koji_task_rpm_info_success(self, m_config):
+ m_config.koji_task_url = "http://kojihub.com/work"
+ expected = dict(
+ base_url="http://kojihub.com/work/tasks/6745/9666745/",
+ version="4.1.0-0.rc2.git2.1.fc23.x86_64",
+ rpm_name="kernel-4.1.0-0.rc2.git2.1.fc23.x86_64.rpm",
+ package_name="kernel",
+ )
+ result = packaging.get_koji_task_rpm_info('kernel', KOJI_TASK_RPMS)
+ assert expected == result
+
+ @patch("teuthology.packaging.config")
+ def test_get_koji_task_rpm_info_fail(self, m_config):
+ m_config.koji_task_url = "http://kojihub.com/work"
+ with pytest.raises(RuntimeError):
+ packaging.get_koji_task_rpm_info('ceph', KOJI_TASK_RPMS)
+
+ def test_get_package_version_deb_found(self):
+ remote = Mock()
+ remote.os.package_type = "deb"
+ proc = Mock()
+ proc.exitstatus = 0
+ proc.stdout.getvalue.return_value = "2.2"
+ remote.run.return_value = proc
+ result = packaging.get_package_version(remote, "apache2")
+ assert result == "2.2"
+
+ def test_get_package_version_deb_command(self):
+ remote = Mock()
+ remote.os.package_type = "deb"
+ packaging.get_package_version(remote, "apache2")
+ args, kwargs = remote.run.call_args
+ expected_args = ['dpkg-query', '-W', '-f', '${Version}', 'apache2']
+ assert expected_args == kwargs['args']
+
+ def test_get_package_version_rpm_found(self):
+ remote = Mock()
+ remote.os.package_type = "rpm"
+ proc = Mock()
+ proc.exitstatus = 0
+ proc.stdout.getvalue.return_value = "2.2"
+ remote.run.return_value = proc
+ result = packaging.get_package_version(remote, "httpd")
+ assert result == "2.2"
+
+ def test_get_package_version_rpm_command(self):
+ remote = Mock()
+ remote.os.package_type = "rpm"
+ packaging.get_package_version(remote, "httpd")
+ args, kwargs = remote.run.call_args
+ expected_args = ['rpm', '-q', 'httpd', '--qf', '%{VERSION}-%{RELEASE}']
+ assert expected_args == kwargs['args']
+
+ def test_get_package_version_not_found(self):
+ remote = Mock()
+ remote.os.package_type = "rpm"
+ proc = Mock()
+ proc.exitstatus = 1
+ proc.stdout.getvalue.return_value = "not installed"
+ remote.run.return_value = proc
+ result = packaging.get_package_version(remote, "httpd")
+ assert result is None
+
+ def test_get_package_version_invalid_version(self):
+ # this tests the possibility that the package is not found
+ # but the exitstatus is still 0. Not entirely sure we'll ever
+ # hit this condition, but I want to test the codepath regardless
+ remote = Mock()
+ remote.os.package_type = "rpm"
+ proc = Mock()
+ proc.exitstatus = 0
+ proc.stdout.getvalue.return_value = "not installed"
+ remote.run.return_value = proc
+ result = packaging.get_package_version(remote, "httpd")
+ assert result is None
+
+ @pytest.mark.parametrize("input, expected", KOJI_TASK_RPMS_MATRIX)
+ def test_get_koji_task_result_package_name(self, input, expected):
+ assert packaging._get_koji_task_result_package_name(input) == expected
+
+ @patch("requests.get")
+ def test_get_response_success(self, m_get):
+ resp = Mock()
+ resp.ok = True
+ m_get.return_value = resp
+ result = packaging._get_response("google.com")
+ assert result == resp
+
+ @patch("requests.get")
+ def test_get_response_failed_wait(self, m_get):
+ resp = Mock()
+ resp.ok = False
+ m_get.return_value = resp
+ packaging._get_response("google.com", wait=True, sleep=1, tries=2)
+ assert m_get.call_count == 2
+
+ @patch("requests.get")
+ def test_get_response_failed_no_wait(self, m_get):
+ resp = Mock()
+ resp.ok = False
+ m_get.return_value = resp
+ packaging._get_response("google.com", sleep=1, tries=2)
+ assert m_get.call_count == 1
+
+ @patch('teuthology.packaging.config')
+ def test_get_builder_project_pulp(self, m_config):
+ m_config.package_source = 'pulp'
+ assert packaging.get_builder_project() is packaging.PulpProject
+
+
+class TestBuilderProject(object):
+ klass = None
+
+ def setup_method(self):
+ if self.klass is None:
+ pytest.skip()
+
+ def _get_remote(self, arch="x86_64", system_type="deb", distro="ubuntu",
+ codename="focal", version="20.04"):
+ rem = Mock()
+ rem.system_type = system_type
+ rem.os.name = distro
+ rem.os.codename = codename
+ rem.os.version = version
+ rem.arch = arch
+
+ return rem
+
+ def test_init_from_remote_base_url(self, expected=None):
+ assert expected is not None
+ rem = self._get_remote()
+ ctx = dict(foo="bar")
+ gp = self.klass("ceph", {}, ctx=ctx, remote=rem)
+ result = gp.base_url
+ assert result == expected
+
+ def test_init_from_remote_base_url_debian(self, expected=None):
+ assert expected is not None
+ # remote.os.codename returns and empty string on debian
+ rem = self._get_remote(distro="debian", codename='', version="7.1")
+ ctx = dict(foo="bar")
+ gp = self.klass("ceph", {}, ctx=ctx, remote=rem)
+ result = gp.base_url
+ assert result == expected
+
+ def test_init_from_config_base_url(self, expected=None):
+ assert expected is not None
+ config = dict(
+ os_type="ubuntu",
+ os_version="20.04",
+ sha1="sha1",
+ )
+ gp = self.klass("ceph", config)
+ result = gp.base_url
+ print(self.m_get.call_args_list)
+ assert result == expected
+
+ def test_init_from_config_branch_ref(self):
+ config = dict(
+ os_type="ubuntu",
+ os_version="20.04",
+ branch='jewel',
+ )
+ gp = self.klass("ceph", config)
+ result = gp.uri_reference
+ expected = 'ref/jewel'
+ assert result == expected
+
+ def test_init_from_config_tag_ref(self):
+ config = dict(
+ os_type="ubuntu",
+ os_version="20.04",
+ tag='v10.0.1',
+ )
+ gp = self.klass("ceph", config)
+ result = gp.uri_reference
+ expected = 'ref/v10.0.1'
+ assert result == expected
+
+ def test_init_from_config_tag_overrides_branch_ref(self, caplog):
+ config = dict(
+ os_type="ubuntu",
+ os_version="20.04",
+ branch='jewel',
+ tag='v10.0.1',
+ )
+ gp = self.klass("ceph", config)
+ result = gp.uri_reference
+ expected = 'ref/v10.0.1'
+ assert result == expected
+ expected_log = 'More than one of ref, tag, branch, or sha1 supplied; using tag'
+ assert expected_log in caplog.text
+ return gp
+
+ def test_init_from_config_branch_overrides_sha1(self, caplog):
+ config = dict(
+ os_type="ubuntu",
+ os_version="20.04",
+ branch='jewel',
+ sha1='sha1',
+ )
+ gp = self.klass("ceph", config)
+ result = gp.uri_reference
+ expected = 'ref/jewel'
+ assert result == expected
+ expected_log = 'More than one of ref, tag, branch, or sha1 supplied; using branch'
+ assert expected_log in caplog.text
+ return gp
+
+ REFERENCE_MATRIX = [
+ ('the_ref', 'the_tag', 'the_branch', 'the_sha1', dict(ref='the_ref')),
+ (None, 'the_tag', 'the_branch', 'the_sha1', dict(tag='the_tag')),
+ (None, None, 'the_branch', 'the_sha1', dict(branch='the_branch')),
+ (None, None, None, 'the_sha1', dict(sha1='the_sha1')),
+ (None, None, 'the_branch', None, dict(branch='the_branch')),
+ ]
+
+ @pytest.mark.parametrize(
+ "ref, tag, branch, sha1, expected",
+ REFERENCE_MATRIX,
+ )
+ def test_choose_reference(self, ref, tag, branch, sha1, expected):
+ config = dict(
+ os_type='ubuntu',
+ os_version='18.04',
+ )
+ if ref:
+ config['ref'] = ref
+ if tag:
+ config['tag'] = tag
+ if branch:
+ config['branch'] = branch
+ if sha1:
+ config['sha1'] = sha1
+ gp = self.klass("ceph", config)
+ assert gp._choose_reference() == expected
+
+ def test_get_package_version_found(self):
+ rem = self._get_remote()
+ ctx = dict(foo="bar")
+ gp = self.klass("ceph", {}, ctx=ctx, remote=rem)
+ assert gp.version == "0.90.0"
+
+ @patch("teuthology.packaging._get_response")
+ def test_get_package_version_not_found(self, m_get_response):
+ rem = self._get_remote()
+ ctx = dict(foo="bar")
+ resp = Mock()
+ resp.ok = False
+ m_get_response.return_value = resp
+ gp = self.klass("ceph", {}, ctx=ctx, remote=rem)
+ with pytest.raises(VersionNotFoundError):
+ gp.version
+
+ def test_get_package_sha1_fetched_found(self):
+ rem = self._get_remote()
+ ctx = dict(foo="bar")
+ gp = self.klass("ceph", {}, ctx=ctx, remote=rem)
+ assert gp.sha1 == "the_sha1"
+
+ def test_get_package_sha1_fetched_not_found(self):
+ rem = self._get_remote()
+ ctx = dict(foo="bar")
+ gp = self.klass("ceph", {}, ctx=ctx, remote=rem)
+ assert not gp.sha1
+
+ DISTRO_MATRIX = [None] * 12
+
+ @pytest.mark.parametrize(
+ "matrix_index",
+ range(len(DISTRO_MATRIX)),
+ )
+ def test_get_distro_remote(self, matrix_index):
+ (distro, version, codename, expected) = \
+ self.DISTRO_MATRIX[matrix_index]
+ rem = self._get_remote(distro=distro, version=version,
+ codename=codename)
+ ctx = dict(foo="bar")
+ gp = self.klass("ceph", {}, ctx=ctx, remote=rem)
+ assert gp.distro == expected
+
+ DISTRO_MATRIX_NOVER = [
+ ('rhel', None, None, 'centos8'),
+ ('centos', None, None, 'centos8'),
+ ('fedora', None, None, 'fedora25'),
+ ('ubuntu', None, None, 'focal'),
+ ('debian', None, None, 'jessie'),
+ ]
+
+ @pytest.mark.parametrize(
+ "matrix_index",
+ range(len(DISTRO_MATRIX) + len(DISTRO_MATRIX_NOVER)),
+ )
+ def test_get_distro_config(self, matrix_index):
+ (distro, version, codename, expected) = \
+ (self.DISTRO_MATRIX + self.DISTRO_MATRIX_NOVER)[matrix_index]
+ config = dict(
+ os_type=distro,
+ os_version=version
+ )
+ gp = self.klass("ceph", config)
+ assert gp.distro == expected
+
+ DIST_RELEASE_MATRIX = [
+ ('rhel', '7.0', None, 'el7'),
+ ('centos', '6.5', None, 'el6'),
+ ('centos', '7.0', None, 'el7'),
+ ('centos', '7.1', None, 'el7'),
+ ('centos', '8.1', None, 'el8'),
+ ('fedora', '20', None, 'fc20'),
+ ('debian', '7.0', None, 'debian'),
+ ('debian', '7', None, 'debian'),
+ ('debian', '7.1', None, 'debian'),
+ ('ubuntu', '12.04', None, 'ubuntu'),
+ ('ubuntu', '14.04', None, 'ubuntu'),
+ ('ubuntu', '16.04', None, 'ubuntu'),
+ ('ubuntu', '18.04', None, 'ubuntu'),
+ ('ubuntu', '20.04', None, 'ubuntu'),
+ ]
+
+ @pytest.mark.parametrize(
+ "matrix_index",
+ range(len(DIST_RELEASE_MATRIX)),
+ )
+ def test_get_dist_release(self, matrix_index):
+ (distro, version, codename, expected) = \
+ (self.DIST_RELEASE_MATRIX)[matrix_index]
+ rem = self._get_remote(distro=distro, version=version,
+ codename=codename)
+ ctx = dict(foo="bar")
+ gp = self.klass("ceph", {}, ctx=ctx, remote=rem)
+ assert gp.dist_release == expected
+
+
+class TestShamanProject(TestBuilderProject):
+ klass = packaging.ShamanProject
+
+ def setup_method(self):
+ self.p_config = patch('teuthology.packaging.config')
+ self.m_config = self.p_config.start()
+ self.m_config.package_source = 'shaman'
+ self.m_config.shaman = {'endpoint': 'https://shaman.ceph.com/api/'}
+ self.p_get_config_value = \
+ patch('teuthology.packaging._get_config_value_for_remote')
+ self.m_get_config_value = self.p_get_config_value.start()
+ self.m_get_config_value.return_value = None
+ self.p_get = patch('requests.get')
+ self.m_get = self.p_get.start()
+
+ def teardown_method(self):
+ self.p_config.stop()
+ self.p_get_config_value.stop()
+ self.p_get.stop()
+
+ def test_init_from_remote_base_url(self):
+ # Here, we really just need to make sure ShamanProject._search()
+ # queries the right URL. So let's make _get_base_url() just pass that
+ # URL through and test that value.
+ def m_get_base_url(obj):
+ obj._search()
+ return self.m_get.call_args_list[0][0][0]
+ with patch(
+ 'teuthology.packaging.ShamanProject._get_base_url',
+ new=m_get_base_url,
+ ):
+ super(TestShamanProject, self)\
+ .test_init_from_remote_base_url(
+ "https://shaman.ceph.com/api/search?status=ready"
+ "&project=ceph&flavor=default"
+ "&distros=ubuntu%2F20.04%2Fx86_64&ref=main"
+ )
+
+ def test_init_from_remote_base_url_debian(self):
+ # Here, we really just need to make sure ShamanProject._search()
+ # queries the right URL. So let's make _get_base_url() just pass that
+ # URL through and test that value.
+ def m_get_base_url(obj):
+ obj._search()
+ return self.m_get.call_args_list[0][0][0]
+ with patch(
+ 'teuthology.packaging.ShamanProject._get_base_url',
+ new=m_get_base_url,
+ ):
+ super(TestShamanProject, self)\
+ .test_init_from_remote_base_url_debian(
+ "https://shaman.ceph.com/api/search?status=ready"
+ "&project=ceph&flavor=default"
+ "&distros=debian%2F7.1%2Fx86_64&ref=main"
+ )
+
+ def test_init_from_config_base_url(self):
+ # Here, we really just need to make sure ShamanProject._search()
+ # queries the right URL. So let's make _get_base_url() just pass that
+ # URL through and test that value.
+ def m_get_base_url(obj):
+ obj._search()
+ return self.m_get.call_args_list[0][0][0]
+ with patch(
+ 'teuthology.packaging.ShamanProject._get_base_url',
+ new=m_get_base_url,
+ ):
+ super(TestShamanProject, self).test_init_from_config_base_url(
+ "https://shaman.ceph.com/api/search?status=ready&project=ceph" \
+ "&flavor=default&distros=ubuntu%2F20.04%2Fx86_64&sha1=sha1"
+ )
+
+ @patch('teuthology.packaging.ShamanProject._get_package_sha1')
+ def test_init_from_config_tag_ref(self, m_get_package_sha1):
+ m_get_package_sha1.return_value = 'the_sha1'
+ super(TestShamanProject, self).test_init_from_config_tag_ref()
+
+ def test_init_from_config_tag_overrides_branch_ref(self, caplog):
+ with patch(
+ 'teuthology.packaging.repo_utils.ls_remote',
+ ) as m_ls_remote:
+ m_ls_remote.return_value = 'sha1_from_my_tag'
+ obj = super(TestShamanProject, self)\
+ .test_init_from_config_tag_overrides_branch_ref(caplog)
+ search_uri = obj._search_uri
+ assert 'sha1=sha1_from_my_tag' in search_uri
+ assert 'jewel' not in search_uri
+
+ def test_init_from_config_branch_overrides_sha1(self, caplog):
+ obj = super(TestShamanProject, self)\
+ .test_init_from_config_branch_overrides_sha1(caplog)
+ search_uri = obj._search_uri
+ assert 'jewel' in search_uri
+ assert 'sha1' not in search_uri
+
+ def test_get_package_version_found(self):
+ resp = Mock()
+ resp.ok = True
+ resp.json.return_value = [
+ dict(
+ sha1='the_sha1',
+ extra=dict(package_manager_version='0.90.0'),
+ )
+ ]
+ self.m_get.return_value = resp
+ super(TestShamanProject, self)\
+ .test_get_package_version_found()
+
+ def test_get_package_sha1_fetched_found(self):
+ resp = Mock()
+ resp.ok = True
+ resp.json.return_value = [dict(sha1='the_sha1')]
+ self.m_get.return_value = resp
+ super(TestShamanProject, self)\
+ .test_get_package_sha1_fetched_found()
+
+ def test_get_package_sha1_fetched_not_found(self):
+ resp = Mock()
+ resp.json.return_value = []
+ self.m_get.return_value = resp
+ super(TestShamanProject, self)\
+ .test_get_package_sha1_fetched_not_found()
+
+ SHAMAN_SEARCH_RESPONSE = [
+ {
+ "status": "ready",
+ "sha1": "534fc6d936bd506119f9e0921ff8cf8d47caa323",
+ "extra": {
+ "build_url": "https://jenkins.ceph.com/job/ceph-dev-build/ARCH=x86_64,AVAILABLE_ARCH=x86_64,AVAILABLE_DIST=centos8,DIST=centos8,MACHINE_SIZE=gigantic/48556/",
+ "root_build_cause": "SCMTRIGGER",
+ "version": "17.0.0-8856-g534fc6d9",
+ "node_name": "172.21.2.7+braggi07",
+ "job_name": "ceph-dev-build/ARCH=x86_64,AVAILABLE_ARCH=x86_64,AVAILABLE_DIST=centos8,DIST=centos8,MACHINE_SIZE=gigantic",
+ "package_manager_version": "17.0.0-8856.g534fc6d9"
+ },
+ "url": "https://3.chacra.ceph.com/r/ceph/main/534fc6d936bd506119f9e0921ff8cf8d47caa323/centos/8/flavors/default/",
+ "modified": "2021-11-06 21:40:40.669823",
+ "distro_version": "8",
+ "project": "ceph",
+ "flavor": "default",
+ "ref": "main",
+ "chacra_url": "https://3.chacra.ceph.com/repos/ceph/main/534fc6d936bd506119f9e0921ff8cf8d47caa323/centos/8/flavors/default/",
+ "archs": [
+ "x86_64",
+ "arm64",
+ "source"
+ ],
+ "distro": "centos"
+ }
+ ]
+
+ SHAMAN_BUILDS_RESPONSE = [
+ {
+ "status": "completed",
+ "sha1": "534fc6d936bd506119f9e0921ff8cf8d47caa323",
+ "distro_arch": "arm64",
+ "started": "2021-11-06 20:20:15.121203",
+ "completed": "2021-11-06 22:36:27.115950",
+ "extra": {
+ "node_name": "172.21.4.66+confusa04",
+ "version": "17.0.0-8856-g534fc6d9",
+ "build_user": "",
+ "root_build_cause": "SCMTRIGGER",
+
+ "job_name": "ceph-dev-build/ARCH=arm64,AVAILABLE_ARCH=arm64,AVAILABLE_DIST=centos8,DIST=centos8,MACHINE_SIZE=gigantic"
+ },
+ "modified": "2021-11-06 22:36:27.118043",
+ "distro_version": "8",
+ "project": "ceph",
+ "url": "https://jenkins.ceph.com/job/ceph-dev-build/ARCH=arm64,AVAILABLE_ARCH=arm64,AVAILABLE_DIST=centos8,DIST=centos8,MACHINE_SIZE=gigantic/48556/",
+ "log_url": "https://jenkins.ceph.com/job/ceph-dev-build/ARCH=arm64,AVAILABLE_ARCH=arm64,AVAILABLE_DIST=centos8,DIST=centos8,MACHINE_SIZE=gigantic/48556//consoleFull",
+ "flavor": "default",
+ "ref": "main",
+ "distro": "centos"
+ },
+ {
+ "status": "completed",
+ "sha1": "534fc6d936bd506119f9e0921ff8cf8d47caa323",
+ "distro_arch": "x86_64",
+ "started": "2021-11-06 20:20:06.740692",
+ "completed": "2021-11-06 21:43:51.711970",
+ "extra": {
+ "node_name": "172.21.2.7+braggi07",
+ "version": "17.0.0-8856-g534fc6d9",
+ "build_user": "",
+ "root_build_cause": "SCMTRIGGER",
+ "job_name": "ceph-dev-build/ARCH=x86_64,AVAILABLE_ARCH=x86_64,AVAILABLE_DIST=centos8,DIST=centos8,MACHINE_SIZE=gigantic"
+ },
+ "modified": "2021-11-06 21:43:51.713487",
+ "distro_version": "8",
+ "project": "ceph",
+ "url": "https://jenkins.ceph.com/job/ceph-dev-build/ARCH=x86_64,AVAILABLE_ARCH=x86_64,AVAILABLE_DIST=centos8,DIST=centos8,MACHINE_SIZE=gigantic/48556/",
+ "log_url": "https://jenkins.ceph.com/job/ceph-dev-build/ARCH=x86_64,AVAILABLE_ARCH=x86_64,AVAILABLE_DIST=centos8,DIST=centos8,MACHINE_SIZE=gigantic/48556//consoleFull",
+ "flavor": "default",
+ "ref": "main",
+ "distro": "centos"
+ }
+ ]
+
+ def test_build_complete_success(self):
+ config = dict(
+ os_type="centos",
+ os_version="8",
+ branch='main',
+ arch='x86_64',
+ flavor='default',
+ )
+ builder = self.klass("ceph", config)
+
+ search_resp = Mock()
+ search_resp.ok = True
+ search_resp.json.return_value = self.SHAMAN_SEARCH_RESPONSE
+ self.m_get.return_value = search_resp
+ # cause builder to call requests.get and cache search_resp
+ builder.assert_result()
+
+ build_resp = Mock()
+ build_resp.ok = True
+ self.m_get.return_value = build_resp
+
+ # both archs completed, so x86_64 build is complete
+ builds = build_resp.json.return_value = self.SHAMAN_BUILDS_RESPONSE
+ assert builder.build_complete
+
+ # mark the arm64 build failed, x86_64 should still be complete
+ builds[0]['status'] = "failed"
+ build_resp.json.return_value = builds
+ assert builder.build_complete
+
+ # mark the x86_64 build failed, should show incomplete
+ builds[1]['status'] = "failed"
+ build_resp.json.return_value = builds
+ assert not builder.build_complete
+
+ # mark the arm64 build complete again, x86_64 still incomplete
+ builds[0]['status'] = "completed"
+ build_resp.json.return_value = builds
+ assert not builder.build_complete
+
+ DISTRO_MATRIX = [
+ ('rhel', '7.0', None, 'centos/7'),
+ ('alma', '9.7', None, 'alma/9'),
+ ('rocky', '9.7', None, 'rocky/9'),
+ ('rocky', '10.1', None, 'rocky/10'),
+ ('centos', '6.5', None, 'centos/6'),
+ ('centos', '7.0', None, 'centos/7'),
+ ('centos', '7.1', None, 'centos/7'),
+ ('centos', '8.1', None, 'centos/8'),
+ ('fedora', '20', None, 'fedora/20'),
+ ('ubuntu', '14.04', 'trusty', 'ubuntu/14.04'),
+ ('ubuntu', '14.04', None, 'ubuntu/14.04'),
+ ('debian', '7.0', None, 'debian/7.0'),
+ ('debian', '7', None, 'debian/7'),
+ ('debian', '7.1', None, 'debian/7.1'),
+ ('ubuntu', '12.04', None, 'ubuntu/12.04'),
+ ('ubuntu', '14.04', None, 'ubuntu/14.04'),
+ ('ubuntu', '16.04', None, 'ubuntu/16.04'),
+ ('ubuntu', '18.04', None, 'ubuntu/18.04'),
+ ('ubuntu', '20.04', None, 'ubuntu/20.04'),
+ ('opensuse', '15.6', None, 'opensuse/15.6'),
+ ('sle', '15.5', None, 'sle/15.5'),
+ ]
+ @pytest.mark.parametrize(
+ "matrix_index",
+ range(len(DISTRO_MATRIX)),
+ )
+ def test_get_distro_remote(self, matrix_index):
+ super().test_get_distro_remote(matrix_index)
+
+ DISTRO_MATRIX_NOVER = [
+ ('rhel', None, None, 'centos/8'),
+ ('centos', None, None, 'centos/9'),
+ ('fedora', None, None, 'fedora/25'),
+ ('ubuntu', None, None, 'ubuntu/22.04'),
+ ('alma', None, None, 'alma/9'),
+ ('rocky', None, None, 'rocky/9'),
+ ]
+ @pytest.mark.parametrize(
+ "matrix_index",
+ range(len(DISTRO_MATRIX) + len(DISTRO_MATRIX_NOVER)),
+ )
+ def test_get_distro_config(self, matrix_index):
+ super().test_get_distro_config(matrix_index)
+
+
+class TestPulpProject(TestBuilderProject):
+ klass = packaging.PulpProject
+
+ def setup_method(self):
+ self.p_config = patch('teuthology.packaging.config')
+ self.m_config = self.p_config.start()
+ self.m_config.package_source = 'pulp'
+ self.m_config.pulp = {
+ 'endpoint': 'https://pulp.example.com/pulp/api/v3/',
+ 'username': 'u',
+ 'password': 'p',
+ }
+ self.p_get_config_value = \
+ patch('teuthology.packaging._get_config_value_for_remote')
+ self.m_get_config_value = self.p_get_config_value.start()
+ self.m_get_config_value.return_value = None
+ self.p_get = patch('requests.get')
+ self.m_get = self.p_get.start()
+
+ resp = Mock()
+ resp.ok = True
+ resp.json.return_value = {
+ 'results': [{
+ 'base_url': 'dist/ceph/main/sha/centos/8/flavors/default/',
+ 'pulp_labels': {
+ 'sha1': 'the_sha1',
+ 'version': '0.90.0',
+ },
+ }],
+ }
+ self.m_get.return_value = resp
+
+ def teardown_method(self):
+ self.p_config.stop()
+ self.p_get_config_value.stop()
+ self.p_get.stop()
+
+ def test_init_requires_pulp_credentials(self):
+ self.m_config.pulp = {'username': '', 'password': 'p'}
+ with pytest.raises(ValueError, match='Pulp username and password are required'):
+ packaging.PulpProject('ceph', {}, ctx=dict(foo='bar'),
+ remote=self._get_remote())
+
+ self.m_config.pulp = {'username': 'u', 'password': ''}
+ with pytest.raises(ValueError, match='Pulp username and password are required'):
+ packaging.PulpProject('ceph', {}, ctx=dict(foo='bar'),
+ remote=self._get_remote())
+
+ def test_init_from_remote_base_url(self):
+ def m_get_base_url(obj):
+ obj._search()
+ return self.m_get.call_args_list[0][0][0]
+ with patch(
+ 'teuthology.packaging.PulpProject._get_base_url',
+ new=m_get_base_url,
+ ):
+ super(TestPulpProject, self).test_init_from_remote_base_url(
+ 'https://pulp.example.com/pulp/api/v3/distributions/deb/apt',
+ )
+
+ def test_init_from_remote_base_url_debian(self):
+ def m_get_base_url(obj):
+ obj._search()
+ return self.m_get.call_args_list[0][0][0]
+ with patch(
+ 'teuthology.packaging.PulpProject._get_base_url',
+ new=m_get_base_url,
+ ):
+ super(TestPulpProject, self).test_init_from_remote_base_url_debian(
+ 'https://pulp.example.com/pulp/api/v3/distributions/deb/apt',
+ )
+
+ def test_init_from_config_base_url(self):
+ def m_get_base_url(obj):
+ obj._search()
+ return self.m_get.call_args_list[0][0][0]
+ with patch(
+ 'teuthology.packaging.PulpProject._get_base_url',
+ new=m_get_base_url,
+ ):
+ super(TestPulpProject, self).test_init_from_config_base_url(
+ 'https://pulp.example.com/pulp/api/v3/distributions/deb/apt',
+ )
+
+ @patch('teuthology.packaging.PulpProject._get_package_sha1')
+ def test_init_from_config_tag_ref(self, m_get_package_sha1):
+ m_get_package_sha1.return_value = 'the_sha1'
+ super(TestPulpProject, self).test_init_from_config_tag_ref()
+
+ def test_init_from_config_tag_overrides_branch_ref(self, caplog):
+ obj = super(TestPulpProject, self)\
+ .test_init_from_config_tag_overrides_branch_ref(caplog)
+ obj._search()
+ labels = self.m_get.call_args[1]['params']['pulp_label_select']
+ assert 'tag=v10.0.1' in labels
+ assert 'branch=jewel' not in labels
+
+ def test_init_from_config_branch_overrides_sha1(self, caplog):
+ obj = super(TestPulpProject, self)\
+ .test_init_from_config_branch_overrides_sha1(caplog)
+ obj._search()
+ labels = self.m_get.call_args[1]['params']['pulp_label_select']
+ assert 'branch=jewel' in labels
+ assert 'sha1=sha1' not in labels
+
+ def test_get_package_version_found(self):
+ resp = Mock()
+ resp.ok = True
+ resp.json.return_value = {
+ 'results': [{
+ 'base_url': 'a/b/c/d',
+ 'pulp_labels': {'version': '0.90.0'},
+ }],
+ }
+ self.m_get.return_value = resp
+ super(TestPulpProject, self).test_get_package_version_found()
+
+ def test_get_package_version_not_found(self):
+ rem = self._get_remote()
+ ctx = dict(foo="bar")
+ resp = Mock()
+ resp.ok = False
+ self.m_get.return_value = resp
+ gp = self.klass("ceph", {}, ctx=ctx, remote=rem)
+ with pytest.raises(VersionNotFoundError):
+ gp.version
+
+ def test_get_package_sha1_fetched_found(self):
+ resp = Mock()
+ resp.ok = True
+ resp.json.return_value = {
+ 'results': [{'base_url': 'a/b/c/d', 'pulp_labels': {'sha1': 'the_sha1'}}],
+ }
+ self.m_get.return_value = resp
+ super(TestPulpProject, self).test_get_package_sha1_fetched_found()
+
+ def test_get_package_sha1_fetched_not_found(self):
+ resp = Mock()
+ resp.ok = True
+ resp.json.return_value = {
+ 'results': [{'base_url': 'a/b/c/d', 'pulp_labels': {}}],
+ }
+ self.m_get.return_value = resp
+ super(TestPulpProject, self).test_get_package_sha1_fetched_not_found()
+
+ DISTRO_MATRIX = [
+ ('rhel', '7.0', None, 'centos/7'),
+ ('alma', '9.7', None, 'alma/9'),
+ ('rocky', '10.1', None, 'rocky/10'),
+ ('centos', '8.1', None, 'centos/8'),
+ ('fedora', '20', None, 'fedora/20'),
+ ('ubuntu', '20.04', 'focal', 'ubuntu/focal'),
+ ]
+
+ @pytest.mark.parametrize(
+ "matrix_index",
+ range(len(DISTRO_MATRIX)),
+ )
+ def test_get_distro_remote(self, matrix_index):
+ super().test_get_distro_remote(matrix_index)
+
+ DISTRO_MATRIX_NOVER = [
+ ('rhel', None, None, 'centos/8'),
+ ('centos', None, None, 'centos/9'),
+ ('fedora', None, None, 'fedora/25'),
+ ('ubuntu', None, None, 'ubuntu/jammy'),
+ ('alma', None, None, 'alma/9'),
+ ('rocky', None, None, 'rocky/9'),
+ ]
+
+ DISTRO_MATRIX_CONFIG = [
+ ('rhel', '7.0', None, 'centos/7'),
+ ('alma', '9.7', None, 'alma/9'),
+ ('rocky', '10.1', None, 'rocky/10'),
+ ('centos', '8.1', None, 'centos/8'),
+ ('fedora', '20', None, 'fedora/20'),
+ ('ubuntu', '20.04', None, 'ubuntu/focal'),
+ ]
+
+ @pytest.mark.parametrize(
+ "matrix_index",
+ range(len(DISTRO_MATRIX_CONFIG) + len(DISTRO_MATRIX_NOVER)),
+ )
+ def test_get_distro_config(self, matrix_index):
+ (distro, version, _, expected) = (
+ self.DISTRO_MATRIX_CONFIG + self.DISTRO_MATRIX_NOVER
+ )[matrix_index]
+ config = dict(
+ os_type=distro,
+ os_version=version
+ )
+ gp = self.klass("ceph", config)
+ assert gp.distro == expected
+
+ def test_get_rpm_repo_content_uses_pulp_base_url(self):
+ pulp_base = (
+ 'https://pulp.example.com/pulp/content/repos/ceph/tentacle/'
+ 'abc123/rocky/10/flavors/default'
+ )
+ rem = self._get_remote()
+ with patch(
+ 'teuthology.packaging.PulpProject.base_url',
+ new_callable=PropertyMock,
+ return_value=pulp_base,
+ ):
+ gp = self.klass('ceph', {}, ctx=dict(foo='bar'), remote=rem)
+ content = gp._get_rpm_repo_content()
+ assert f'baseurl={pulp_base}/$basearch' in content
+ assert f'baseurl={pulp_base}/noarch' in content
+ assert f'baseurl={pulp_base}/SRPMS' in content
+ assert 'chacra' not in content
+
+ @patch('teuthology.packaging.sudo_write_file')
+ def test_install_rpm_repo_writes_repo_file(self, m_sudo_write_file):
+ pulp_base = (
+ 'https://pulp.example.com/pulp/content/repos/ceph/tentacle/'
+ 'abc123/rocky/10/flavors/default'
+ )
+ rem = self._get_remote()
+ with patch(
+ 'teuthology.packaging.PulpProject.base_url',
+ new_callable=PropertyMock,
+ return_value=pulp_base,
+ ):
+ gp = self.klass('ceph', {}, ctx=dict(foo='bar'), remote=rem)
+ gp._install_rpm_repo()
+ m_sudo_write_file.assert_called_once()
+ assert m_sudo_write_file.call_args[0][1] == '/etc/yum.repos.d/ceph.repo'
+ assert pulp_base in m_sudo_write_file.call_args[0][2]
+
+ def test_get_deb_repo_content_uses_pulp_distribution_url(self):
+ pulp_dist = (
+ 'https://pulp.example.com/pulp/content/repos/ceph/tentacle/'
+ 'ff309bba/ubuntu/jammy/flavors/default/x86_64/'
+ )
+ rem = self._get_remote(distro='ubuntu', version='22.04', codename='jammy')
+ with patch(
+ 'teuthology.packaging.PulpProject.repo_url',
+ new_callable=PropertyMock,
+ return_value=pulp_dist,
+ ):
+ gp = self.klass('ceph', {}, ctx=dict(foo='bar'), remote=rem)
+ content = gp._get_deb_repo_content()
+ assert content == (
+ 'deb [trusted=yes] '
+ 'https://pulp.example.com/pulp/content/repos/ceph/tentacle/'
+ 'ff309bba/ubuntu/jammy/flavors/default/x86_64/ default all\n'
+ )
+
+ @patch('teuthology.packaging.sudo_write_file')
+ def test_install_deb_repo_writes_sources_list(self, m_sudo_write_file):
+ pulp_dist = (
+ 'https://pulp.example.com/pulp/content/repos/ceph/tentacle/'
+ 'ff309bba/ubuntu/jammy/flavors/default/x86_64/'
+ )
+ rem = self._get_remote(distro='ubuntu', version='22.04', codename='jammy')
+ with patch(
+ 'teuthology.packaging.PulpProject.repo_url',
+ new_callable=PropertyMock,
+ return_value=pulp_dist,
+ ):
+ gp = self.klass('ceph', {}, ctx=dict(foo='bar'), remote=rem)
+ gp._install_deb_repo()
+ m_sudo_write_file.assert_called_once()
+ assert m_sudo_write_file.call_args[0][1] == '/etc/apt/sources.list.d/ceph.list'
+ assert '[trusted=yes]' in m_sudo_write_file.call_args[0][2]
+ assert '/x86_64/ default all' in m_sudo_write_file.call_args[0][2]
--- /dev/null
+from teuthology.parallel import parallel
+
+
+def identity(item, input_set=None, remove=False):
+ if input_set is not None:
+ assert item in input_set
+ if remove:
+ input_set.remove(item)
+ return item
+
+
+class TestParallel(object):
+ def test_basic(self):
+ in_set = set(range(10))
+ with parallel() as para:
+ for i in in_set:
+ para.spawn(identity, i, in_set, remove=True)
+ assert para.any_spawned is True
+ assert para.count == len(in_set)
+
+ def test_result(self):
+ in_set = set(range(10))
+ with parallel() as para:
+ for i in in_set:
+ para.spawn(identity, i, in_set)
+ for result in para:
+ in_set.remove(result)
+
--- /dev/null
+import logging
+import unittest.mock as mock
+import os
+import os.path
+from pytest import raises, mark
+import re
+import shutil
+import subprocess
+import tempfile
+from packaging.version import parse
+
+from teuthology.exceptions import BranchNotFoundError, CommitNotFoundError
+from teuthology import repo_utils
+from teuthology import parallel
+repo_utils.log.setLevel(logging.WARNING)
+
+
+class TestRepoUtils(object):
+
+ @classmethod
+ def setup_class(cls):
+ cls.temp_path = tempfile.mkdtemp(prefix='test_repo-')
+ cls.dest_path = cls.temp_path + '/empty_dest'
+ cls.src_path = cls.temp_path + '/empty_src'
+
+ if 'TEST_ONLINE' in os.environ:
+ cls.repo_url = 'https://github.com/ceph/empty.git'
+ cls.commit = '71245d8e454a06a38a00bff09d8f19607c72e8bf'
+ else:
+ cls.repo_url = 'file://' + cls.src_path
+ cls.commit = None
+
+ cls.git_version = parse(cls.get_system_git_version())
+
+ @classmethod
+ def teardown_class(cls):
+ shutil.rmtree(cls.temp_path)
+
+ @classmethod
+ def get_system_git_version(cls):
+ # parsing following patterns
+ # 1) git version 2.45.2
+ # 2) git version 2.39.3 (Apple Git-146)
+ git_version = subprocess.check_output(('git', 'version')).decode()
+ m = re.match(r"git version (?P<ver>\d+.\d+.\d+) ?", git_version)
+ return m['ver']
+
+ def setup_method(self, method):
+ # In git 2.28.0, the --initial-branch flag was added.
+ if self.git_version >= parse("2.28.0"):
+ subprocess.check_call(
+ ('git', 'init', '--initial-branch', 'main', self.src_path)
+ )
+ else:
+ subprocess.check_call(('git', 'init', self.src_path))
+ subprocess.check_call(
+ ('git', 'checkout', '-b', 'main'),
+ cwd=self.src_path,
+ )
+ proc = subprocess.Popen(
+ ('git', 'config', 'user.email', 'test@ceph.com'),
+ cwd=self.src_path,
+ stdout=subprocess.PIPE,
+ )
+ assert proc.wait() == 0
+ proc = subprocess.Popen(
+ ('git', 'config', 'user.name', 'Test User'),
+ cwd=self.src_path,
+ stdout=subprocess.PIPE,
+ )
+ assert proc.wait() == 0
+ proc = subprocess.Popen(
+ ('git', 'commit', '--allow-empty', '--allow-empty-message',
+ '--no-edit'),
+ cwd=self.src_path,
+ stdout=subprocess.PIPE,
+ )
+ assert proc.wait() == 0
+ if not self.commit:
+ result = subprocess.check_output(
+ 'git rev-parse HEAD',
+ shell=True,
+ cwd=self.src_path,
+ ).split()
+ assert result
+ self.commit = result[0].decode()
+
+ def teardown_method(self, method):
+ shutil.rmtree(self.src_path, ignore_errors=True)
+ shutil.rmtree(self.dest_path, ignore_errors=True)
+
+ def test_clone_repo_existing_branch(self):
+ repo_utils.clone_repo(self.repo_url, self.dest_path, 'main', self.commit)
+ assert os.path.exists(self.dest_path)
+
+ def test_clone_repo_non_existing_branch(self):
+ with raises(BranchNotFoundError):
+ repo_utils.clone_repo(self.repo_url, self.dest_path, 'nobranch', self.commit)
+ assert not os.path.exists(self.dest_path)
+
+ def test_fetch_no_repo(self):
+ fake_dest_path = self.temp_path + '/not_a_repo'
+ assert not os.path.exists(fake_dest_path)
+ with raises(OSError):
+ repo_utils.fetch(fake_dest_path)
+ assert not os.path.exists(fake_dest_path)
+
+ def test_fetch_noop(self):
+ repo_utils.clone_repo(self.repo_url, self.dest_path, 'main', self.commit)
+ repo_utils.fetch(self.dest_path)
+ assert os.path.exists(self.dest_path)
+
+ def test_fetch_branch_no_repo(self):
+ fake_dest_path = self.temp_path + '/not_a_repo'
+ assert not os.path.exists(fake_dest_path)
+ with raises(OSError):
+ repo_utils.fetch_branch(fake_dest_path, 'main')
+ assert not os.path.exists(fake_dest_path)
+
+ def test_fetch_branch_fake_branch(self):
+ repo_utils.clone_repo(self.repo_url, self.dest_path, 'main', self.commit)
+ with raises(BranchNotFoundError):
+ repo_utils.fetch_branch(self.dest_path, 'nobranch')
+
+ @mark.parametrize('git_str',
+ ["fatal: couldn't find remote ref",
+ "fatal: Couldn't find remote ref"])
+ @mock.patch('subprocess.Popen')
+ def test_fetch_branch_different_git_versions(self, mock_popen, git_str):
+ """
+ Newer git versions return a lower case string
+ See: https://github.com/git/git/commit/0b9c3afdbfb629363
+ """
+ branch_name = 'nobranch'
+ process_mock = mock.Mock()
+ attrs = {
+ 'wait.return_value': 1,
+ 'stdout.read.return_value': f"{git_str} {branch_name}".encode(),
+ }
+ process_mock.configure_mock(**attrs)
+ mock_popen.return_value = process_mock
+ with raises(BranchNotFoundError):
+ repo_utils.fetch_branch('', branch_name)
+
+ def test_enforce_existing_branch(self):
+ repo_utils.enforce_repo_state(self.repo_url, self.dest_path,
+ 'main')
+ assert os.path.exists(self.dest_path)
+
+ def test_enforce_existing_commit(self):
+ import logging
+ logging.getLogger().info(subprocess.check_output("git branch", shell=True, cwd=self.src_path))
+ repo_utils.enforce_repo_state(self.repo_url, self.dest_path,
+ 'main', self.commit)
+ assert os.path.exists(self.dest_path)
+
+ def test_enforce_non_existing_branch(self):
+ with raises(BranchNotFoundError):
+ repo_utils.enforce_repo_state(self.repo_url, self.dest_path,
+ 'blah', self.commit)
+ assert not os.path.exists(self.dest_path)
+
+ def test_enforce_non_existing_commit(self):
+ with raises(CommitNotFoundError):
+ repo_utils.enforce_repo_state(self.repo_url, self.dest_path,
+ 'main', 'c69e90807d222c1719c45c8c758bf6fac3d985f1')
+ assert not os.path.exists(self.dest_path)
+
+ def test_enforce_multiple_calls_same_branch(self):
+ repo_utils.enforce_repo_state(self.repo_url, self.dest_path,
+ 'main', self.commit)
+ assert os.path.exists(self.dest_path)
+ repo_utils.enforce_repo_state(self.repo_url, self.dest_path,
+ 'main', self.commit)
+ assert os.path.exists(self.dest_path)
+ repo_utils.enforce_repo_state(self.repo_url, self.dest_path,
+ 'main', self.commit)
+ assert os.path.exists(self.dest_path)
+
+ def test_enforce_multiple_calls_different_branches(self):
+ with raises(BranchNotFoundError):
+ repo_utils.enforce_repo_state(self.repo_url, self.dest_path,
+ 'blah1')
+ assert not os.path.exists(self.dest_path)
+ repo_utils.enforce_repo_state(self.repo_url, self.dest_path,
+ 'main', self.commit)
+ assert os.path.exists(self.dest_path)
+ repo_utils.enforce_repo_state(self.repo_url, self.dest_path,
+ 'main', self.commit)
+ assert os.path.exists(self.dest_path)
+ with raises(BranchNotFoundError):
+ repo_utils.enforce_repo_state(self.repo_url, self.dest_path,
+ 'blah2')
+ assert not os.path.exists(self.dest_path)
+ repo_utils.enforce_repo_state(self.repo_url, self.dest_path,
+ 'main', self.commit)
+ assert os.path.exists(self.dest_path)
+
+ def test_enforce_invalid_branch(self):
+ with raises(ValueError):
+ repo_utils.enforce_repo_state(self.repo_url, self.dest_path, 'a b', self.commit)
+
+ def test_simultaneous_access(self):
+ count = 5
+ with parallel.parallel() as p:
+ for i in range(count):
+ p.spawn(repo_utils.enforce_repo_state, self.repo_url,
+ self.dest_path, 'main', self.commit)
+ for result in p:
+ assert result is None
+
+ def test_simultaneous_access_different_branches(self):
+ branches = [('main', self.commit), ('main', self.commit), ('nobranch', 'nocommit'),
+ ('nobranch', 'nocommit'), ('main', self.commit), ('nobranch', 'nocommit')]
+
+ with parallel.parallel() as p:
+ for branch, commit in branches:
+ if branch == 'main':
+ p.spawn(repo_utils.enforce_repo_state, self.repo_url,
+ self.dest_path, branch, commit)
+ else:
+ dest_path = self.dest_path + '_' + branch
+
+ def func():
+ repo_utils.enforce_repo_state(
+ self.repo_url, dest_path,
+ branch, commit)
+ p.spawn(
+ raises,
+ BranchNotFoundError,
+ func,
+ )
+ for result in p:
+ pass
+
+ URLS_AND_DIRNAMES = [
+ ('git@git.ceph.com/ceph-qa-suite.git', 'git.ceph.com_ceph-qa-suite'),
+ ('git://git.ceph.com/ceph-qa-suite.git', 'git.ceph.com_ceph-qa-suite'),
+ ('https://github.com/ceph/ceph', 'github.com_ceph_ceph'),
+ ('https://github.com/liewegas/ceph.git', 'github.com_liewegas_ceph'),
+ ('file:///my/dir/has/ceph.git', 'my_dir_has_ceph'),
+ ]
+
+ @mark.parametrize("input_, expected", URLS_AND_DIRNAMES)
+ def test_url_to_dirname(self, input_, expected):
+ assert repo_utils.url_to_dirname(input_) == expected
+
+ def test_current_branch(self):
+ repo_utils.clone_repo(self.repo_url, self.dest_path, 'main', self.commit)
+ assert repo_utils.current_branch(self.dest_path) == "main"
--- /dev/null
+import json
+import pytest
+import yaml
+
+from tests import fake_archive
+from teuthology import report
+
+
+@pytest.fixture
+def archive(tmp_path):
+ archive = fake_archive.FakeArchive(archive_base=str(tmp_path))
+ yield archive
+ archive.teardown()
+
+
+@pytest.fixture(autouse=True)
+def reporter(archive):
+ archive.setup()
+ return report.ResultsReporter(archive_base=archive.archive_base)
+
+
+def test_all_runs_one_run(archive, reporter):
+ run_name = "test_all_runs"
+ yaml_path = "examples/3node_ceph.yaml"
+ job_count = 3
+ archive.create_fake_run(run_name, job_count, yaml_path)
+ assert [run_name] == reporter.serializer.all_runs
+
+
+def test_all_runs_three_runs(archive, reporter):
+ run_count = 3
+ runs = {}
+ for i in range(run_count):
+ run_name = "run #%s" % i
+ yaml_path = "examples/3node_ceph.yaml"
+ job_count = 3
+ job_ids = archive.create_fake_run(
+ run_name,
+ job_count,
+ yaml_path)
+ runs[run_name] = job_ids
+ assert sorted(runs.keys()) == sorted(reporter.serializer.all_runs)
+
+
+def test_jobs_for_run(archive, reporter):
+ run_name = "test_jobs_for_run"
+ yaml_path = "examples/3node_ceph.yaml"
+ job_count = 3
+ jobs = archive.create_fake_run(run_name, job_count, yaml_path)
+ job_ids = [str(job['job_id']) for job in jobs]
+
+ got_jobs = reporter.serializer.jobs_for_run(run_name)
+ assert sorted(job_ids) == sorted(got_jobs.keys())
+
+
+def test_running_jobs_for_run(archive, reporter):
+ run_name = "test_jobs_for_run"
+ yaml_path = "examples/3node_ceph.yaml"
+ job_count = 10
+ num_hung = 3
+ archive.create_fake_run(run_name, job_count, yaml_path,
+ num_hung=num_hung)
+
+ got_jobs = reporter.serializer.running_jobs_for_run(run_name)
+ assert len(got_jobs) == num_hung
+
+
+def test_json_for_job(archive, reporter):
+ run_name = "test_json_for_job"
+ yaml_path = "examples/3node_ceph.yaml"
+ job_count = 1
+ jobs = archive.create_fake_run(run_name, job_count, yaml_path)
+ job = jobs[0]
+
+ with open(yaml_path) as yaml_file:
+ obj_from_yaml = yaml.safe_load(yaml_file)
+ full_obj = obj_from_yaml.copy()
+ full_obj.update(job['info'])
+ full_obj.update(job['summary'])
+
+ out_json = reporter.serializer.json_for_job(
+ run_name, str(job['job_id']))
+ out_obj = json.loads(out_json)
+ assert full_obj == out_obj
+
+
--- /dev/null
+import textwrap
+from teuthology.config import config
+from teuthology import results
+from teuthology import report
+
+from unittest.mock import patch, DEFAULT
+
+
+class TestResultsEmail(object):
+ reference = {
+ 'run_name': 'test_name',
+ 'jobs': [
+ # Running
+ {'description': 'description for job with name test_name',
+ 'job_id': 30481,
+ 'name': 'test_name',
+ 'log_href': 'http://qa-proxy.ceph.com/teuthology/test_name/30481/teuthology.log', # noqa
+ 'owner': 'job@owner',
+ 'duration': None,
+ 'status': 'running',
+ },
+ # Waiting
+ {'description': 'description for job with name test_name',
+ 'job_id': 62965,
+ 'name': 'test_name',
+ 'log_href': 'http://qa-proxy.ceph.com/teuthology/test_name/30481/teuthology.log', # noqa
+ 'owner': 'job@owner',
+ 'duration': None,
+ 'status': 'waiting',
+ },
+ # Queued
+ {'description': 'description for job with name test_name',
+ 'job_id': 79063,
+ 'name': 'test_name',
+ 'log_href': 'http://qa-proxy.ceph.com/teuthology/test_name/30481/teuthology.log', # noqa
+ 'owner': 'job@owner',
+ 'duration': None,
+ 'status': 'queued',
+ },
+ # Failed
+ {'description': 'description for job with name test_name',
+ 'job_id': 88979,
+ 'name': 'test_name',
+ 'log_href': 'http://qa-proxy.ceph.com/teuthology/test_name/88979/teuthology.log', # noqa
+ 'owner': 'job@owner',
+ 'duration': 35190,
+ 'success': False,
+ 'status': 'fail',
+ 'failure_reason': 'Failure reason!',
+ },
+ # Dead
+ {'description': 'description for job with name test_name',
+ 'job_id': 69152,
+ 'name': 'test_name',
+ 'log_href': 'http://qa-proxy.ceph.com/teuthology/test_name/69152/teuthology.log', # noqa
+ 'owner': 'job@owner',
+ 'duration': 5225,
+ 'success': False,
+ 'status': 'dead',
+ 'failure_reason': 'Dead reason!',
+ },
+ # Passed
+ {'description': 'description for job with name test_name',
+ 'job_id': 68369,
+ 'name': 'test_name',
+ 'log_href': 'http://qa-proxy.ceph.com/teuthology/test_name/68369/teuthology.log', # noqa
+ 'owner': 'job@owner',
+ 'duration': 33771,
+ 'success': True,
+ 'status': 'pass',
+ },
+ ],
+ 'subject': '1 fail, 1 dead, 1 running, 1 waiting, 1 queued, 1 pass in test_name', # noqa
+ 'body': textwrap.dedent("""
+ Test Run: test_name
+ =================================================================
+ info: http://example.com/test_name/
+ logs: http://qa-proxy.ceph.com/teuthology/test_name/
+ failed: 1
+ dead: 1
+ running: 1
+ waiting: 1
+ queued: 1
+ passed: 1
+
+
+ Fail
+ =================================================================
+ [88979] description for job with name test_name
+ -----------------------------------------------------------------
+ time: 09:46:30
+ info: http://example.com/test_name/88979/
+ log: http://qa-proxy.ceph.com/teuthology/test_name/88979/
+
+ Failure reason!
+
+
+
+ Dead
+ =================================================================
+ [69152] description for job with name test_name
+ -----------------------------------------------------------------
+ time: 01:27:05
+ info: http://example.com/test_name/69152/
+ log: http://qa-proxy.ceph.com/teuthology/test_name/69152/
+
+ Dead reason!
+
+
+
+ Running
+ =================================================================
+ [30481] description for job with name test_name
+ info: http://example.com/test_name/30481/
+
+
+
+ Waiting
+ =================================================================
+ [62965] description for job with name test_name
+ info: http://example.com/test_name/62965/
+
+
+
+ Queued
+ =================================================================
+ [79063] description for job with name test_name
+ info: http://example.com/test_name/79063/
+
+
+
+ Pass
+ =================================================================
+ [68369] description for job with name test_name
+ time: 09:22:51
+ info: http://example.com/test_name/68369/
+ """).strip(),
+ }
+
+ def setup_method(self):
+ config.results_ui_server = "http://example.com/"
+ config.archive_server = "http://qa-proxy.ceph.com/teuthology/"
+
+ def test_build_email_body(self):
+ run_name = self.reference['run_name']
+ with patch.multiple(
+ report,
+ ResultsReporter=DEFAULT,
+ ):
+ reporter = report.ResultsReporter()
+ reporter.get_jobs.return_value = self.reference['jobs']
+ (subject, body) = results.build_email_body(
+ run_name, _reporter=reporter)
+ assert subject == self.reference['subject']
+ assert body == self.reference['body']
--- /dev/null
+import os
+import pytest
+import docopt
+
+from unittest.mock import patch, call
+
+from teuthology import run
+from scripts import run as scripts_run
+from tests import skipif_teuthology_process
+
+
+class TestRun(object):
+ """ Tests for teuthology.run """
+
+ @patch("teuthology.log.setLevel")
+ @patch("teuthology.setup_log_file")
+ @patch("os.mkdir")
+ def test_set_up_logging(self, m_mkdir, m_setup_log_file, m_setLevel):
+ run.set_up_logging(True, "path/to/archive")
+ m_mkdir.assert_called_with("path/to/archive")
+ m_setup_log_file.assert_called_with("path/to/archive/teuthology.log")
+ assert m_setLevel.called
+
+ # because of how we import things, mock merge_configs from run - where it's used
+ # see: http://www.voidspace.org.uk/python/mock/patch.html#where-to-patch
+ @patch("teuthology.run.merge_configs")
+ def test_setup_config(self, m_merge_configs):
+ config = {"job_id": 1, "foo": "bar"}
+ m_merge_configs.return_value = config
+ result = run.setup_config(["some/config.yaml"])
+ assert m_merge_configs.called
+ assert result["job_id"] == "1"
+ assert result["foo"] == "bar"
+
+ @patch("teuthology.run.merge_configs")
+ def test_setup_config_targets_ok(self, m_merge_configs):
+ config = {"targets": list(range(4)), "roles": list(range(2))}
+ m_merge_configs.return_value = config
+ result = run.setup_config(["some/config.yaml"])
+ assert result["targets"] == [0, 1, 2, 3]
+ assert result["roles"] == [0, 1]
+
+ @patch("teuthology.run.merge_configs")
+ def test_setup_config_targets_invalid(self, m_merge_configs):
+ config = {"targets": range(2), "roles": range(4)}
+ m_merge_configs.return_value = config
+ with pytest.raises(AssertionError):
+ run.setup_config(["some/config.yaml"])
+
+ @patch("teuthology.run.open")
+ def test_write_initial_metadata(self, m_open):
+ config = {"job_id": "123", "foo": "bar"}
+ run.write_initial_metadata(
+ "some/archive/dir",
+ config,
+ "the_name",
+ "the_description",
+ "the_owner",
+ )
+ expected = [
+ call('some/archive/dir/pid', 'w'),
+ call('some/archive/dir/owner', 'w'),
+ call('some/archive/dir/orig.config.yaml', 'w'),
+ call('some/archive/dir/info.yaml', 'w')
+ ]
+ assert m_open.call_args_list == expected
+
+ def test_get_machine_type(self):
+ result = run.get_machine_type(None, {"machine-type": "the_machine_type"})
+ assert result == "the_machine_type"
+
+ def test_get_summary(self):
+ result = run.get_summary("the_owner", "the_description")
+ assert result == {"owner": "the_owner", "description": "the_description", "success": True}
+ result = run.get_summary("the_owner", None)
+ assert result == {"owner": "the_owner", "success": True}
+
+ def test_validate_tasks_invalid(self):
+ config = {"tasks": [{"kernel": "can't be here"}]}
+ with pytest.raises(AssertionError) as excinfo:
+ run.validate_tasks(config)
+ assert excinfo.value.args[0].startswith("kernel installation")
+
+ def test_validate_task_no_tasks(self):
+ result = run.validate_tasks({})
+ assert result == []
+
+ def test_validate_tasks_valid(self):
+ expected = [{"foo": "bar"}, {"bar": "foo"}]
+ result = run.validate_tasks({"tasks": expected})
+ assert result == expected
+
+ def test_validate_tasks_is_list(self):
+ with pytest.raises(AssertionError) as excinfo:
+ run.validate_tasks({"tasks": {"foo": "bar"}})
+ assert excinfo.value.args[0].startswith("Expected list")
+
+ def test_get_initial_tasks_invalid(self):
+ with pytest.raises(AssertionError) as excinfo:
+ run.get_initial_tasks(True, {"targets": "can't be here",
+ "roles": "roles" }, "machine_type")
+ assert excinfo.value.args[0].startswith("You cannot")
+
+ def test_get_inital_tasks(self):
+ config = {"roles": range(2), "kernel": "the_kernel", "use_existing_cluster": False}
+ result = run.get_initial_tasks(True, config, "machine_type")
+ assert {"internal.lock_machines": (2, "machine_type")} in result
+ assert {"kernel": "the_kernel"} in result
+ # added because use_existing_cluster == False
+ assert {'internal.vm_setup': None} in result
+ assert {'internal.buildpackages_prep': None} in result
+
+ # When tests are run in a teuthology process using the py.test
+ # API, tasks will have already been imported. Patching sys.path
+ # (and even calling sys.path_importer_cache.clear()) doesn't seem
+ # to help "forget" where the tasks are, keeping this test from
+ # passing. The test isn't critical to run in every single
+ # environment, so skip.
+ @skipif_teuthology_process
+ @patch("teuthology.run.fetch_qa_suite")
+ def test_fetch_tasks_if_needed(self, m_fetch_qa_suite):
+ config = {"suite_path": "/some/suite/path", "suite_branch": "feature_branch",
+ "suite_sha1": "commit"}
+ m_fetch_qa_suite.return_value = "/some/other/suite/path"
+ result = run.fetch_tasks_if_needed(config)
+ m_fetch_qa_suite.assert_called_with("feature_branch", commit="commit")
+ assert result == "/some/other/suite/path/qa"
+
+ @patch("teuthology.run.get_status")
+ @patch("yaml.safe_dump")
+ @patch("teuthology.report.try_push_job_info")
+ @patch("teuthology.run.email_results")
+ @patch("teuthology.run.open")
+ @patch("sys.exit")
+ def test_report_outcome(self, m_sys_exit, m_open, m_email_results, m_try_push_job_info, m_safe_dump, m_get_status):
+ m_get_status.return_value = "fail"
+ summary = {"failure_reason": "reasons"}
+ summary_dump = "failure_reason: reasons\n"
+ config = {"email-on-error": True}
+ config_dump = "email-on-error: true\n"
+ m_safe_dump.side_effect = [None, summary_dump, config_dump]
+ run.report_outcome(config, "the/archive/path", summary)
+ m_try_push_job_info.assert_called_with(config, summary)
+ m_open.assert_called_with("the/archive/path/summary.yaml", "w")
+ assert m_email_results.called
+ assert m_open.called
+ assert m_sys_exit.called
+
+ @patch("teuthology.run.set_up_logging")
+ @patch("teuthology.run.setup_config")
+ @patch("teuthology.run.get_user")
+ @patch("teuthology.run.write_initial_metadata")
+ @patch("teuthology.report.try_push_job_info")
+ @patch("teuthology.run.get_machine_type")
+ @patch("teuthology.run.get_summary")
+ @patch("yaml.safe_dump")
+ @patch("teuthology.run.validate_tasks")
+ @patch("teuthology.run.get_initial_tasks")
+ @patch("teuthology.run.fetch_tasks_if_needed")
+ @patch("teuthology.run.run_tasks")
+ @patch("teuthology.run.report_outcome")
+ def test_main(self, m_report_outcome, m_run_tasks, m_fetch_tasks_if_needed, m_get_initial_tasks, m_validate_tasks,
+ m_safe_dump, m_get_summary, m_get_machine_type, m_try_push_job_info, m_write_initial_metadata,
+ m_get_user, m_setup_config, m_set_up_logging):
+ """ This really should be an integration test of some sort. """
+ config = {"job_id": 1}
+ m_setup_config.return_value = config
+ m_get_machine_type.return_value = "machine_type"
+ doc = scripts_run.__doc__
+ args = docopt.docopt(doc, [
+ "--verbose",
+ "--archive", "some/archive/dir",
+ "--description", "the_description",
+ "--lock",
+ "--os-type", "os_type",
+ "--os-version", "os_version",
+ "--block",
+ "--name", "the_name",
+ "--suite-path", "some/suite/dir",
+ "path/to/config.yml",
+ ])
+ m_get_user.return_value = "the_owner"
+ m_get_summary.return_value = dict(success=True, owner="the_owner", description="the_description")
+ m_validate_tasks.return_value = ['task3']
+ m_get_initial_tasks.return_value = ['task1', 'task2']
+ m_fetch_tasks_if_needed.return_value = "some/suite/dir"
+ run.main(args)
+ m_set_up_logging.assert_called_with(True, "some/archive/dir")
+ m_setup_config.assert_called_with(["path/to/config.yml"])
+ m_write_initial_metadata.assert_called_with(
+ "some/archive/dir",
+ config,
+ "the_name",
+ "the_description",
+ "the_owner"
+ )
+ m_try_push_job_info.assert_called_with(config, dict(status='running', pid=os.getpid()))
+ m_get_machine_type.assert_called_with(None, config)
+ m_get_summary.assert_called_with("the_owner", "the_description")
+ m_get_initial_tasks.assert_called_with(True, config, "machine_type")
+ m_fetch_tasks_if_needed.assert_called_with(config)
+ assert m_report_outcome.called
+ args, kwargs = m_run_tasks.call_args
+ fake_ctx = kwargs["ctx"]._conf
+ # fields that must be in ctx for the tasks to behave
+ expected_ctx = ["verbose", "archive", "description", "owner", "lock", "machine_type", "os_type", "os_version",
+ "block", "name", "suite_path", "config", "summary"]
+ for key in expected_ctx:
+ assert key in fake_ctx
+ assert isinstance(fake_ctx["config"], dict)
+ assert isinstance(fake_ctx["summary"], dict)
+ assert "tasks" in fake_ctx["config"]
+ # ensures that values missing in args are added with the correct value
+ assert fake_ctx["owner"] == "the_owner"
+ assert fake_ctx["machine_type"] == "machine_type"
+ # ensures os_type and os_version are property overwritten
+ assert fake_ctx["config"]["os_type"] == "os_type"
+ assert fake_ctx["config"]["os_version"] == "os_version"
+
+ @patch("teuthology.run.set_up_logging")
+ @patch("teuthology.run.setup_config")
+ @patch("teuthology.run.get_user")
+ @patch("teuthology.run.write_initial_metadata")
+ @patch("teuthology.report.try_push_job_info")
+ @patch("teuthology.run.get_machine_type")
+ @patch("teuthology.run.get_summary")
+ @patch("yaml.safe_dump")
+ @patch("teuthology.run.validate_tasks")
+ @patch("teuthology.run.get_initial_tasks")
+ @patch("teuthology.run.fetch_tasks_if_needed")
+ @patch("teuthology.run.run_tasks")
+ @patch("teuthology.run.report_outcome")
+ def test_main_interactive(
+ self,
+ m_report_outcome,
+ m_run_tasks,
+ m_fetch_tasks_if_needed,
+ m_get_initial_tasks,
+ m_validate_tasks,
+ m_safe_dump,
+ m_get_summary,
+ m_get_machine_type,
+ m_try_push_job_info,
+ m_write_initial_metadata,
+ m_get_user,
+ m_setup_config,
+ m_set_up_logging,
+ ):
+ config = {"job_id": 1}
+ m_setup_config.return_value = config
+ m_get_machine_type.return_value = "machine_type"
+ doc = scripts_run.__doc__
+ args = docopt.docopt(doc, [
+ "--interactive-on-error",
+ "path/to/config.yml",
+ ])
+ run.main(args)
+ args, kwargs = m_run_tasks.call_args
+ fake_ctx = kwargs["ctx"]._conf
+ assert fake_ctx['interactive_on_error'] is True
+
+ def test_get_teuthology_command(self):
+ doc = scripts_run.__doc__
+ args = docopt.docopt(doc, [
+ "--archive", "some/archive/dir",
+ "--description", "the_description",
+ "--lock",
+ "--block",
+ "--name", "the_name",
+ "--suite-path", "some/suite/dir",
+ "path/to/config.yml", "path/to/config2.yaml",
+ ])
+ result = run.get_teuthology_command(args)
+ result = result.split()
+ expected = [
+ "teuthology",
+ "path/to/config.yml", "path/to/config2.yaml",
+ "--suite-path", "some/suite/dir",
+ "--lock",
+ "--description", "the_description",
+ "--name", "the_name",
+ "--block",
+ "--archive", "some/archive/dir",
+ ]
+ assert len(result) == len(expected)
+ for arg in expected:
+ assert arg in result
--- /dev/null
+from teuthology import safepath
+
+class TestSafepath(object):
+ def test_simple(self):
+ got = safepath.munge('foo')
+ assert got == 'foo'
+
+ def test_empty(self):
+ # really odd corner case
+ got = safepath.munge('')
+ assert got == '_'
+
+ def test_slash(self):
+ got = safepath.munge('/')
+ assert got == '_'
+
+ def test_slashslash(self):
+ got = safepath.munge('//')
+ assert got == '_'
+
+ def test_absolute(self):
+ got = safepath.munge('/evil')
+ assert got == 'evil'
+
+ def test_absolute_subdir(self):
+ got = safepath.munge('/evil/here')
+ assert got == 'evil/here'
+
+ def test_dot_leading(self):
+ got = safepath.munge('./foo')
+ assert got == 'foo'
+
+ def test_dot_middle(self):
+ got = safepath.munge('evil/./foo')
+ assert got == 'evil/foo'
+
+ def test_dot_trailing(self):
+ got = safepath.munge('evil/foo/.')
+ assert got == 'evil/foo'
+
+ def test_dotdot(self):
+ got = safepath.munge('../evil/foo')
+ assert got == '_./evil/foo'
+
+ def test_dotdot_subdir(self):
+ got = safepath.munge('evil/../foo')
+ assert got == 'evil/_./foo'
+
+ def test_hidden(self):
+ got = safepath.munge('.evil')
+ assert got == '_evil'
+
+ def test_hidden_subdir(self):
+ got = safepath.munge('foo/.evil')
+ assert got == 'foo/_evil'
--- /dev/null
+from teuthology.schedule import build_config
+from teuthology.misc import get_user
+
+
+class TestSchedule(object):
+ basic_args = {
+ '--verbose': False,
+ '--owner': 'OWNER',
+ '--description': 'DESC',
+ '--email': 'EMAIL',
+ '--first-in-suite': False,
+ '--last-in-suite': True,
+ '--name': 'NAME',
+ '--worker': 'tala',
+ '--timeout': '6',
+ '--priority': '99',
+ # TODO: make this work regardless of $PWD
+ #'<conf_file>': ['../../examples/3node_ceph.yaml',
+ # '../../examples/3node_rgw.yaml'],
+ }
+
+ def test_basic(self):
+ expected = {
+ 'description': 'DESC',
+ 'email': 'EMAIL',
+ 'first_in_suite': False,
+ 'last_in_suite': True,
+ 'machine_type': 'tala',
+ 'name': 'NAME',
+ 'owner': 'OWNER',
+ 'priority': 99,
+ 'results_timeout': '6',
+ 'verbose': False,
+ 'tube': 'tala',
+ }
+
+ job_dict = build_config(self.basic_args)
+ assert job_dict == expected
+
+ def test_owner(self):
+ args = self.basic_args
+ args['--owner'] = None
+ job_dict = build_config(self.basic_args)
+ assert job_dict['owner'] == 'scheduled_%s' % get_user()
+
--- /dev/null
+from __future__ import with_statement
+
+import glob
+import gzip
+import os
+import shutil
+import tempfile
+import yaml
+from teuthology import scrape
+
+class FakeResultDir(object):
+ """Mocks a Result Directory"""
+
+ def __init__(self,
+ failure_reason="Dummy reason",
+ assertion="FAILED assert 1 == 2\n",
+ blank_backtrace=False,
+ assertion_osd=False,
+ ):
+ self.failure_reason = failure_reason
+ self.assertion = assertion
+ self.blank_backtrace = blank_backtrace
+ self.path = tempfile.mkdtemp()
+
+ with open(os.path.join(self.path, "config.yaml"), "w") as f:
+ yaml.dump({"description": "Dummy test"}, f)
+
+ with open(os.path.join(self.path, "summary.yaml"), "w") as f:
+ yaml.dump({
+ "success": "false",
+ "failure_reason": self.failure_reason
+ }, f)
+
+ with open(os.path.join(self.path, "teuthology.log"), "w") as f:
+ if not self.blank_backtrace:
+ f.write(" ceph version 1000\n")
+ f.write(".stderr: Dummy error\n")
+ f.write(self.assertion)
+ f.write(" NOTE: a copy of the executable dummy text\n")
+
+ if assertion_osd:
+ host = "host1"
+ rem_log_dir = os.path.join(self.path, "remote", host, "log")
+ os.makedirs(rem_log_dir, exist_ok=True)
+ ceph_mon_log = os.path.join(rem_log_dir, "ceph-osd.0.log")
+ with open(ceph_mon_log, "w") as f:
+ f.write("ceph version 1000\n")
+ f.write(self.assertion)
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, exc_typ, exc_val, exc_tb):
+ shutil.rmtree(self.path)
+
+class TestScrape(object):
+ """Tests for teuthology.scrape"""
+
+ def test_grep(self):
+ with FakeResultDir() as d:
+ filepath = os.path.join(d.path, "scrapetest.txt")
+ with open(filepath, 'w') as f:
+ f.write("Ceph is an open-source software storage platform\n\
+ Teuthology is used for testing.")
+
+ #System level grep is called
+ value1 = scrape.grep(filepath, "software")
+ value2 = scrape.grep(filepath, "device")
+
+ assert value1 ==\
+ ['Ceph is an open-source software storage platform', '']
+ assert value2 == []
+
+ def test_job(self):
+ with FakeResultDir() as d:
+ job = scrape.Job(d.path, 1)
+ assert job.get_success() == "false"
+ assert job.get_assertion() == "FAILED assert 1 == 2"
+ assert job.get_last_tlog_line() ==\
+ b"NOTE: a copy of the executable dummy text"
+ assert job.get_failure_reason() == "Dummy reason"
+
+ def test_timeoutreason(self):
+ with FakeResultDir(failure_reason=\
+ "status 124: timeout '123 /home/ubuntu/cephtest/workunit.client.0/cephtool/test.sh'") as d:
+ job = scrape.Job(d.path, 1)
+ assert scrape.TimeoutReason.could_be(job)
+ assert scrape.TimeoutReason(job).match(job)
+
+ def test_deadreason(self):
+ with FakeResultDir() as d:
+ job = scrape.Job(d.path, 1)
+ #Summary is present
+ #So this cannot be a DeadReason
+ assert not scrape.DeadReason.could_be(job)
+
+ def test_lockdepreason(self):
+ lkReason = None
+ with FakeResultDir(assertion=\
+ "FAILED assert common/lockdep reason\n") as d:
+ job = scrape.Job(d.path, 1)
+ assert scrape.LockdepReason.could_be(job)
+
+ lkReason = scrape.LockdepReason(job)
+ #Backtraces of same jobs must match 100%
+ assert lkReason.match(job)
+ with FakeResultDir(blank_backtrace=True) as d:
+ #Corresponding to 0% match
+ assert not lkReason.match(scrape.Job(d.path, 2))
+
+ def test_assertionreason(self):
+ with FakeResultDir() as d:
+ job = scrape.Job(d.path, 1)
+ assert scrape.AssertionReason.could_be(job)
+
+ def test_genericreason(self):
+ d1 = FakeResultDir(blank_backtrace=True)
+ d2 = FakeResultDir(failure_reason="Dummy dummy")
+ d3 = FakeResultDir()
+
+ job1 = scrape.Job(d1.path, 1)
+ job2 = scrape.Job(d2.path, 2)
+ job3 = scrape.Job(d3.path, 3)
+
+ reason = scrape.GenericReason(job3)
+
+ assert reason.match(job2)
+ assert not reason.match(job1)
+
+ shutil.rmtree(d1.path)
+ shutil.rmtree(d2.path)
+ shutil.rmtree(d3.path)
+
+ def test_valgrindreason(self):
+ vreason = None
+ with FakeResultDir(
+ failure_reason="saw valgrind issues",
+ assertion="2014-08-22T20:07:18.668 ERROR:tasks.ceph:saw valgrind issue <kind>Leak_DefinitelyLost</kind> in /var/log/ceph/valgrind/osd.3.log.gz\n"
+ ) as d:
+ job = scrape.Job(d.path, 1)
+ assert scrape.ValgrindReason.could_be(job)
+
+ vreason = scrape.ValgrindReason(job)
+ assert vreason.match(job)
+
+ def test_give_me_a_reason(self):
+ with FakeResultDir() as d:
+ job = scrape.Job(d.path, 1)
+
+ assert type(scrape.give_me_a_reason(job)) == scrape.AssertionReason
+
+ #Test the lockdep ordering
+ with FakeResultDir(assertion=\
+ "FAILED assert common/lockdep reason\n") as d:
+ job = scrape.Job(d.path, 1)
+ assert type(scrape.give_me_a_reason(job)) == scrape.LockdepReason
+
+ def test_scraper(self):
+ d = FakeResultDir()
+ os.mkdir(os.path.join(d.path, "test"))
+ shutil.move(
+ os.path.join(d.path, "config.yaml"),
+ os.path.join(d.path, "test", "config.yaml")
+ )
+ shutil.move(
+ os.path.join(d.path, "summary.yaml"),
+ os.path.join(d.path, "test", "summary.yaml")
+ )
+ shutil.move(
+ os.path.join(d.path, "teuthology.log"),
+ os.path.join(d.path, "test", "teuthology.log")
+ )
+
+ scrape.Scraper(d.path).analyze()
+
+ #scrape.log should be created
+ assert os.path.exists(os.path.join(d.path, "scrape.log"))
+
+ shutil.rmtree(d.path)
+
+ def test_gzip_backtrace_decode(self):
+ with FakeResultDir(assertion="FAILED assert dummy backtrace line",
+ blank_backtrace=True,
+ assertion_osd=True) as d:
+
+ with open(os.path.join(d.path, "teuthology.log"), "a") as root_log:
+ root_log.write(
+ "command crashed with signal SIGSEGV tasks.ceph.osd.0.host1.stderr\n"
+ )
+
+ pattern = os.path.join(d.path, "**", "ceph-osd.0.log")
+ raws = glob.glob(pattern, recursive=True)
+ assert len(raws) == 1, f"expected one raw log, found: {raws}"
+ raw_log = raws[0]
+ gz_log = raw_log + ".gz"
+
+ with gzip.open(gz_log, "wb") as out:
+ out.write(open(raw_log, "rb").read())
+ os.remove(raw_log)
+
+ assert not os.path.exists(raw_log)
+ assert os.path.exists(gz_log)
+
+ job = scrape.Job(d.path, 1)
+ assert job.get_assertion() == "FAILED assert dummy backtrace line"
\ No newline at end of file
--- /dev/null
+from teuthology import timer
+
+from unittest.mock import MagicMock, patch, mock_open
+from time import time
+
+
+class TestTimer(object):
+ def test_data_empty(self):
+ self.timer = timer.Timer()
+ assert self.timer.data == dict()
+
+ def test_data_one_mark(self):
+ self.timer = timer.Timer()
+ # Avoid failing if ~1ms elapses between these two calls
+ self.timer.precision = 2
+ self.timer.mark()
+ assert len(self.timer.data['marks']) == 1
+ assert self.timer.data['marks'][0]['interval'] == 0
+ assert self.timer.data['marks'][0]['message'] == ''
+
+ def test_data_five_marks(self):
+ self.timer = timer.Timer()
+ for i in range(5):
+ self.timer.mark(str(i))
+ assert len(self.timer.data['marks']) == 5
+ assert [m['message'] for m in self.timer.data['marks']] == \
+ ['0', '1', '2', '3', '4']
+
+ def test_intervals(self):
+ fake_time = MagicMock()
+ with patch('teuthology.timer.time.time', fake_time):
+ self.timer = timer.Timer()
+ now = start_time = fake_time.return_value = time()
+ intervals = [0, 1, 1, 2, 3, 5, 8]
+ for i in intervals:
+ now += i
+ fake_time.return_value = now
+ self.timer.mark(str(i))
+
+ summed_intervals = [sum(intervals[:x + 1]) for x in range(len(intervals))]
+ result_intervals = [m['interval'] for m in self.timer.data['marks']]
+ assert result_intervals == summed_intervals
+ assert self.timer.data['start'] == \
+ self.timer.get_datetime_string(start_time)
+ assert self.timer.data['end'] == \
+ self.timer.get_datetime_string(start_time + summed_intervals[-1])
+ assert [m['message'] for m in self.timer.data['marks']] == \
+ [str(i) for i in intervals]
+ assert self.timer.data['elapsed'] == summed_intervals[-1]
+
+ def test_write(self):
+ _path = '/path'
+ _safe_dump = MagicMock(name='safe_dump')
+ with patch('teuthology.timer.yaml.safe_dump', _safe_dump):
+ with patch('teuthology.timer.open', mock_open(), create=True) as _open:
+ self.timer = timer.Timer(path=_path)
+ assert self.timer.path == _path
+ self.timer.write()
+ _open.assert_called_once_with(_path, 'w')
+ _safe_dump.assert_called_once_with(
+ dict(),
+ _open.return_value.__enter__.return_value,
+ default_flow_style=False,
+ )
+
+ def test_sync(self):
+ _path = '/path'
+ _safe_dump = MagicMock(name='safe_dump')
+ with patch('teuthology.timer.yaml.safe_dump', _safe_dump):
+ with patch('teuthology.timer.open', mock_open(), create=True) as _open:
+ self.timer = timer.Timer(path=_path, sync=True)
+ assert self.timer.path == _path
+ assert self.timer.sync is True
+ self.timer.mark()
+ _open.assert_called_once_with(_path, 'w')
+ _safe_dump.assert_called_once_with(
+ self.timer.data,
+ _open.return_value.__enter__.return_value,
+ default_flow_style=False,
+ )
--- /dev/null
+from unittest.mock import patch, Mock
+
+import teuthology.lock.util
+from teuthology import provision
+
+
+class TestVpsOsVersionParamCheck(object):
+
+ def setup_method(self):
+ self.fake_ctx = Mock()
+ self.fake_ctx.machine_type = 'vps'
+ self.fake_ctx.num_to_lock = 1
+ self.fake_ctx.lock = False
+
+ def fake_downburst_executable():
+ return ''
+
+ self.fake_downburst_executable = fake_downburst_executable
+
+ def test_ubuntu_noble(self):
+ self.fake_ctx.os_type = 'ubuntu'
+ self.fake_ctx.os_version = 'noble'
+ with patch.multiple(
+ provision.downburst,
+ downburst_executable=self.fake_downburst_executable,
+ ):
+ check_value = teuthology.lock.util.vps_version_or_type_valid(
+ self.fake_ctx.machine_type,
+ self.fake_ctx.os_type,
+ self.fake_ctx.os_version)
+
+ assert check_value
+
+ def test_ubuntu_number(self):
+ self.fake_ctx.os_type = 'ubuntu'
+ self.fake_ctx.os_version = '24.04'
+ with patch.multiple(
+ provision.downburst,
+ downburst_executable=self.fake_downburst_executable,
+ ):
+ check_value = teuthology.lock.util.vps_version_or_type_valid(
+ self.fake_ctx.machine_type,
+ self.fake_ctx.os_type,
+ self.fake_ctx.os_version)
+ assert check_value
+
+ def test_mixup(self):
+ self.fake_ctx.os_type = '6.5'
+ self.fake_ctx.os_version = 'rhel'
+ with patch.multiple(
+ provision.downburst,
+ downburst_executable=self.fake_downburst_executable,
+ ):
+ check_value = teuthology.lock.util.vps_version_or_type_valid(
+ self.fake_ctx.machine_type,
+ self.fake_ctx.os_type,
+ self.fake_ctx.os_version)
+ assert not check_value
+
+ def test_bad_type(self):
+ self.fake_ctx.os_type = 'aardvark'
+ self.fake_ctx.os_version = '6.5'
+ with patch.multiple(
+ provision.downburst,
+ downburst_executable=self.fake_downburst_executable,
+ ):
+ check_value = teuthology.lock.util.vps_version_or_type_valid(
+ self.fake_ctx.machine_type,
+ self.fake_ctx.os_type,
+ self.fake_ctx.os_version)
+ assert not check_value
+
+ def test_bad_version(self):
+ self.fake_ctx.os_type = 'ubuntu'
+ self.fake_ctx.os_version = 'vampire_bat'
+ with patch.multiple(
+ provision.downburst,
+ downburst_executable=self.fake_downburst_executable,
+ ):
+ check_value = teuthology.lock.util.vps_version_or_type_valid(
+ self.fake_ctx.machine_type,
+ self.fake_ctx.os_type,
+ self.fake_ctx.os_version)
+ assert not check_value
--- /dev/null
+<?xml version="1.0" encoding="UTF-8"?>
+<testsuite name="nosetests" tests="644" errors="0" failures="1" skip="79">
+<testcase classname="s3tests_boto3.functional.test_s3" name="test_cors_origin_response" time="3.205"></testcase>
+<testcase classname="s3tests_boto3.functional.test_s3" name="test_cors_origin_wildcard" time="3.081"></testcase>
+<testcase classname="s3tests_boto3.functional.test_s3" name="test_cors_header_option" time="3.119"></testcase>
+<testcase classname="s3tests_boto3.functional.test_s3" name="test_set_bucket_tagging" time="0.059"><failure type="builtins.AssertionError" message="'NoSuchTagSetError' != 'NoSuchTagSet'"></failure></testcase>
+</testsuite>
\ No newline at end of file
--- /dev/null
+<?xml version="1.0"?>
+<valgrindoutput>
+<error>
+ <unique>0x870fc</unique>
+ <tid>1</tid>
+ <kind>Leak_DefinitelyLost</kind>
+ <xwhat>
+ <text>1,234 bytes in 1 blocks are definitely lost in loss record 198 of 201</text>
+ <leakedbytes>1234</leakedbytes>
+ <leakedblocks>1</leakedblocks>
+ </xwhat>
+ <stack>
+ <frame>
+ <ip>0x4C39B6F</ip>
+ <obj>/usr/libexec/valgrind/vgpreload_memcheck-amd64-linux.so</obj>
+ <fn>operator new[](unsigned long)</fn>
+ <dir>/builddir/build/BUILD/valgrind-3.19.0/coregrind/m_replacemalloc</dir>
+ <file>vg_replace_malloc.c</file>
+ <line>640</line>
+ </frame>
+ <frame>
+ <ip>0xF3F4B5</ip>
+ <obj>/usr/bin/ceph-osd</obj>
+ <fn>ceph::common::leak_some_memory()</fn>
+ <dir>/usr/src/debug/ceph-18.0.0-5567.g64a4fc94.el8.x86_64/src/common</dir>
+ <file>ceph_context.cc</file>
+ <line>510</line>
+ </frame>
+ </stack>
+</error>
+</valgrindoutput>
--- /dev/null
+from mock import patch, MagicMock
+
+from io import BytesIO
+import os, io
+
+from teuthology.orchestra import remote
+from teuthology.util.scanner import UnitTestScanner, ValgrindScanner
+
+
+class MockFile(io.StringIO):
+ def close(self):
+ pass
+
+
+class TestUnitTestScanner(object):
+
+ def setup_method(self):
+ self.remote = remote.Remote(
+ name='jdoe@xyzzy.example.com', ssh=MagicMock())
+ self.test_values = {
+ "xml_path": os.path.dirname(__file__) + "/files/test_unit_test.xml",
+ "error_msg": "FAILURE: Test `test_set_bucket_tagging` of `s3tests_boto3.functional.test_s3`. \
+Reason: 'NoSuchTagSetError' != 'NoSuchTagSet'.",
+ "summary_data": [{'failed_testsuites': {'s3tests_boto3.functional.test_s3':
+ [{'kind': 'failure', 'testcase': 'test_set_bucket_tagging',
+ 'message': "'NoSuchTagSetError' != 'NoSuchTagSet'"}]},
+ 'num_of_failures': 1,
+ 'file_path': f'{os.path.dirname(__file__)}/files/test_unit_test.xml'}],
+ "yaml_data": r"""- failed_testsuites:
+ s3tests_boto3.functional.test_s3:
+ - kind: failure
+ message: '''NoSuchTagSetError'' != ''NoSuchTagSet'''
+ testcase: test_set_bucket_tagging
+ file_path: {file_dir}/files/test_unit_test.xml
+ num_of_failures: 1
+""".format(file_dir=os.path.dirname(__file__))
+ }
+
+ @patch('teuthology.util.scanner.UnitTestScanner.write_summary')
+ def test_scan_and_write(self, m_write_summary):
+ xml_path = self.test_values["xml_path"]
+ self.remote.ssh.exec_command.return_value = (None, BytesIO(xml_path.encode('utf-8')), None)
+ m_open = MagicMock()
+ m_open.return_value = open(xml_path, "rb")
+ self.remote._sftp_open_file = m_open
+ result = UnitTestScanner(remote=self.remote).scan_and_write(xml_path, "test_summary.yaml")
+ assert result == "(total 1 failed) " + self.test_values["error_msg"]
+
+ def test_parse(self):
+ xml_content = b'<?xml version="1.0" encoding="UTF-8"?>\n<testsuite name="xyz" tests="1" \
+errors="0" failures="1">\n<testcase classname="xyz" name="abc" time="0.059"><failure \
+type="builtins.AssertionError" message="error_msg"></failure></testcase>\n</testsuite>'
+ scanner = UnitTestScanner(self.remote)
+ result = scanner._parse(xml_content)
+ assert result == (
+ 'FAILURE: Test `abc` of `xyz`. Reason: error_msg.',
+ {'failed_testsuites': {'xyz':
+ [{'kind': 'failure','message': 'error_msg','testcase': 'abc'}]},
+ 'num_of_failures': 1
+ }
+ )
+
+ def test_scan_file(self):
+ xml_path = self.test_values["xml_path"]
+ m_open = MagicMock()
+ m_open.return_value = open(xml_path, "rb")
+ self.remote._sftp_open_file = m_open
+ scanner = UnitTestScanner(remote=self.remote)
+ result = scanner.scan_file(xml_path)
+ assert result == self.test_values["error_msg"]
+ assert scanner.summary_data == self.test_values["summary_data"]
+
+ def test_scan_all_files(self):
+ xml_path = self.test_values["xml_path"]
+ self.remote.ssh.exec_command.return_value = (None, BytesIO(xml_path.encode('utf-8')), None)
+ m_open = MagicMock()
+ m_open.return_value = open(xml_path, "rb")
+ self.remote._sftp_open_file = m_open
+ scanner = UnitTestScanner(remote=self.remote)
+ result = scanner.scan_all_files(xml_path)
+ assert result == [self.test_values["error_msg"]]
+
+ @patch('builtins.open')
+ def test_write_summary(self, m_open):
+ scanner = UnitTestScanner(self.remote)
+ mock_yaml_file = MockFile()
+ scanner.summary_data = self.test_values["summary_data"]
+ m_open.return_value = mock_yaml_file
+ scanner.write_summary("path/file.yaml")
+ written_content = mock_yaml_file.getvalue()
+ assert written_content == self.test_values["yaml_data"]
+
+
+class TestValgrindScanner(object):
+
+ def setup_method(self):
+ self.remote = remote.Remote(
+ name='jdoe@xyzzy.example.com', ssh=MagicMock())
+ self.test_values = {
+ "xml_path": os.path.dirname(__file__) + "/files/test_valgrind.xml",
+ "error_msg": "valgrind error: Leak_DefinitelyLost\noperator new[]\
+(unsigned long)\nceph::common::leak_some_memory()",
+ "summary_data": [{'kind': 'Leak_DefinitelyLost', 'traceback': [{'file':
+ '/builddir/build/BUILD/valgrind-3.19.0/coregrind/m_replacemalloc/vg_replace_malloc.c',
+ 'line': '640', 'function': 'operator new[](unsigned long)'},
+ {'file': '/usr/src/debug/ceph-18.0.0-5567.g64a4fc94.el8.x86_64/src/common/ceph_context.cc',
+ 'line': '510', 'function': 'ceph::common::leak_some_memory()'}], 'file_path':
+ f'{os.path.dirname(__file__)}/files/test_valgrind.xml'}],
+ "yaml_data": r"""- file_path: {file_dir}/files/test_valgrind.xml
+ kind: Leak_DefinitelyLost
+ traceback:
+ - file: /builddir/build/BUILD/valgrind-3.19.0/coregrind/m_replacemalloc/vg_replace_malloc.c
+ function: operator new[](unsigned long)
+ line: '640'
+ - file: /usr/src/debug/ceph-18.0.0-5567.g64a4fc94.el8.x86_64/src/common/ceph_context.cc
+ function: ceph::common::leak_some_memory()
+ line: '510'
+""".format(file_dir=os.path.dirname(__file__))
+ }
+
+ def test_parse_with_traceback(self):
+ xml_content = b'''<?xml version="1.0"?>
+<valgrindoutput>
+<error>
+ <kind>Leak_DefinitelyLost</kind>
+ <stack>
+ <frame>
+ <fn>func()</fn>
+ <dir>/dir</dir>
+ <file>file1.ext</file>
+ <line>640</line>
+ </frame>
+ </stack>
+</error>
+</valgrindoutput>
+'''
+ scanner = ValgrindScanner(self.remote)
+ result = scanner._parse(xml_content)
+ assert result == (
+ 'valgrind error: Leak_DefinitelyLost\nfunc()',
+ {'kind': 'Leak_DefinitelyLost', 'traceback':
+ [{'file': '/dir/file1.ext', 'line': '640', 'function': 'func()'}]
+ }
+ )
+
+ def test_parse_without_trackback(self):
+ xml_content = b'''<?xml version="1.0"?>
+<valgrindoutput>
+<error>
+ <kind>Leak_DefinitelyLost</kind>
+ <stack>
+ </stack>
+</error>
+</valgrindoutput>
+'''
+ scanner = ValgrindScanner(self.remote)
+ result = scanner._parse(xml_content)
+ assert result == (
+ 'valgrind error: Leak_DefinitelyLost\n',
+ {'kind': 'Leak_DefinitelyLost', 'traceback': []}
+ )
+
+ def test_scan_file(self):
+ xml_path = self.test_values["xml_path"]
+ m_open = MagicMock()
+ m_open.return_value = open(xml_path, "rb")
+ self.remote._sftp_open_file = m_open
+ scanner = ValgrindScanner(remote=self.remote)
+ result = scanner.scan_file(xml_path)
+ assert result == self.test_values["error_msg"]
+ assert scanner.summary_data == self.test_values["summary_data"]
+
+ def test_scan_all_files(self):
+ xml_path = self.test_values["xml_path"]
+ self.remote.ssh.exec_command.return_value = (None, BytesIO(xml_path.encode('utf-8')), None)
+ m_open = MagicMock()
+ m_open.return_value = open(xml_path, "rb")
+ self.remote._sftp_open_file = m_open
+ scanner = ValgrindScanner(remote=self.remote)
+ result = scanner.scan_all_files(xml_path)
+ assert result == [self.test_values["error_msg"]]
+
+ @patch('builtins.open')
+ def test_write_summary(self, m_open):
+ scanner = ValgrindScanner(self.remote)
+ mock_yaml_file = MockFile()
+ scanner.summary_data = self.test_values["summary_data"]
+ m_open.return_value = mock_yaml_file
+ scanner.write_summary("path/file.yaml")
+ written_content = mock_yaml_file.getvalue()
+ assert written_content == self.test_values["yaml_data"]
\ No newline at end of file
--- /dev/null
+import pytest
+
+from datetime import datetime, timedelta, timezone
+from typing import Type
+
+from teuthology.util import time
+
+
+@pytest.mark.parametrize(
+ ["timestamp", "result"],
+ [
+ ["1999-12-31_23:59:59", datetime(1999, 12, 31, 23, 59, 59, tzinfo=timezone.utc)],
+ ["1999-12-31_23:59", datetime(1999, 12, 31, 23, 59, 0, tzinfo=timezone.utc)],
+ ["1999-12-31T23:59:59", datetime(1999, 12, 31, 23, 59, 59, tzinfo=timezone.utc)],
+ ["1999-12-31T23:59:59+00:00", datetime(1999, 12, 31, 23, 59, 59, tzinfo=timezone.utc)],
+ ["1999-12-31T17:59:59-06:00", datetime(1999, 12, 31, 23, 59, 59, tzinfo=timezone.utc)],
+ ["2024-01-01", datetime(2024, 1, 1, 0, 0, tzinfo=timezone.utc)],
+ ["tomorrow", ValueError],
+ ["1d", ValueError],
+ ["", ValueError],
+ ["2024", ValueError],
+
+ ]
+)
+def test_parse_timestamp(timestamp: str, result: datetime | Type[Exception]):
+ if isinstance(result, datetime):
+ assert time.parse_timestamp(timestamp) == result
+ else:
+ with pytest.raises(result):
+ time.parse_timestamp(timestamp)
+
+
+@pytest.mark.parametrize(
+ ["offset", "result"],
+ [
+ ["1s", timedelta(seconds=1)],
+ ["1m", timedelta(minutes=1)],
+ ["1h", timedelta(hours=1)],
+ ["1d", timedelta(days=1)],
+ ["1w", timedelta(weeks=1)],
+ ["365d", timedelta(days=365)],
+ ["1x", ValueError],
+ ["-1m", ValueError],
+ ["0xde", ValueError],
+ ["frog", ValueError],
+ ["7dwarfs", ValueError],
+ ]
+)
+def test_parse_offset(offset: str, result: timedelta | Type[Exception]):
+ if isinstance(result, timedelta):
+ assert time.parse_offset(offset) == result
+ else:
+ with pytest.raises(result):
+ time.parse_offset(offset)
+++ /dev/null
-import datetime
-import pytest
-
-from unittest.mock import patch, Mock, MagicMock
-
-from teuthology import dispatcher
-from teuthology.config import FakeNamespace
-from teuthology.contextutil import MaxWhileTries
-from teuthology.util.time import TIMESTAMP_FMT
-
-
-class TestDispatcher(object):
- @pytest.fixture(autouse=True)
- def setup_method(self, tmp_path):
- self.ctx = FakeNamespace()
- self.ctx.verbose = True
- self.ctx.archive_dir = str(tmp_path / "archive/dir")
- self.ctx.log_dir = str(tmp_path / "log/dir")
- self.ctx.tube = 'tube'
-
- @patch("teuthology.repo_utils.ls_remote")
- @patch("os.path.isdir")
- @patch("teuthology.repo_utils.fetch_teuthology")
- @patch("teuthology.dispatcher.teuth_config")
- @patch("teuthology.repo_utils.fetch_qa_suite")
- def test_prep_job(self, m_fetch_qa_suite, m_teuth_config,
- m_fetch_teuthology, m_isdir, m_ls_remote):
- config = dict(
- name="the_name",
- job_id="1",
- suite_sha1="suite_hash",
- )
- m_fetch_teuthology.return_value = '/teuth/path'
- m_fetch_qa_suite.return_value = '/suite/path'
- m_ls_remote.return_value = 'teuth_hash'
- m_isdir.return_value = True
- m_teuth_config.teuthology_path = None
- got_config, teuth_bin_path = dispatcher.prep_job(config)
- assert got_config['teuthology_branch'] == 'main'
- m_fetch_teuthology.assert_called_once_with(branch='main', commit='teuth_hash')
- assert teuth_bin_path == '/teuth/path/.venv/bin'
- m_fetch_qa_suite.assert_called_once_with('main', 'suite_hash')
- assert got_config['suite_path'] == '/suite/path'
-
- def build_fake_jobs(self, m_connection, m_job, job_bodies):
- """
- Given patched copies of:
- beanstalkc.Connection
- beanstalkc.Job
- And a list of basic job bodies, return a list of mocked Job objects
- """
- # Make sure instantiating m_job returns a new object each time
- jobs = []
- job_id = 0
- for job_body in job_bodies:
- job_id += 1
- job = MagicMock(conn=m_connection, jid=job_id, body=job_body)
- job.jid = job_id
- job.body = job_body
- jobs.append(job)
- return jobs
-
- @patch("teuthology.dispatcher.find_dispatcher_processes")
- @patch("teuthology.repo_utils.ls_remote")
- @patch("teuthology.dispatcher.report.try_push_job_info")
- @patch("teuthology.dispatcher.supervisor.run_job")
- @patch("beanstalkc.Job", autospec=True)
- @patch("teuthology.repo_utils.fetch_qa_suite")
- @patch("teuthology.repo_utils.fetch_teuthology")
- @patch("teuthology.dispatcher.beanstalk.watch_tube")
- @patch("teuthology.dispatcher.beanstalk.connect")
- @patch("os.path.isdir", return_value=True)
- @patch("teuthology.dispatcher.setup_log_file")
- def test_main_loop(
- self, m_setup_log_file, m_isdir, m_connect, m_watch_tube,
- m_fetch_teuthology, m_fetch_qa_suite, m_job, m_run_job,
- m_try_push_job_info, m_ls_remote, m_find_dispatcher_processes,
- ):
- m_find_dispatcher_processes.return_value = {}
- m_connection = Mock()
- jobs = self.build_fake_jobs(
- m_connection,
- m_job,
- [
- 'name: name\nfoo: bar',
- 'name: name\nstop_worker: true',
- ],
- )
- m_connection.reserve.side_effect = jobs
- m_connect.return_value = m_connection
- dispatcher.main(self.ctx)
- # There should be one reserve call per item in the jobs list
- expected_reserve_calls = [
- dict(timeout=60) for i in range(len(jobs))
- ]
- got_reserve_calls = [
- call[1] for call in m_connection.reserve.call_args_list
- ]
- assert got_reserve_calls == expected_reserve_calls
- for job in jobs:
- job.bury.assert_called_once_with()
- job.delete.assert_called_once_with()
-
- @patch("teuthology.dispatcher.find_dispatcher_processes")
- @patch("teuthology.repo_utils.ls_remote")
- @patch("teuthology.dispatcher.report.try_push_job_info")
- @patch("teuthology.dispatcher.supervisor.run_job")
- @patch("beanstalkc.Job", autospec=True)
- @patch("teuthology.repo_utils.fetch_qa_suite")
- @patch("teuthology.repo_utils.fetch_teuthology")
- @patch("teuthology.dispatcher.beanstalk.watch_tube")
- @patch("teuthology.dispatcher.beanstalk.connect")
- @patch("os.path.isdir", return_value=True)
- @patch("teuthology.dispatcher.setup_log_file")
- def test_main_loop_13925(
- self, m_setup_log_file, m_isdir, m_connect, m_watch_tube,
- m_fetch_teuthology, m_fetch_qa_suite, m_job, m_run_job,
- m_try_push_job_info, m_ls_remote, m_find_dispatcher_processes,
- ):
- m_find_dispatcher_processes.return_value = {}
- m_connection = Mock()
- jobs = self.build_fake_jobs(
- m_connection,
- m_job,
- [
- 'name: name',
- 'name: name\nstop_worker: true',
- ],
- )
- m_connection.reserve.side_effect = jobs
- m_connect.return_value = m_connection
- m_fetch_qa_suite.side_effect = [
- MaxWhileTries(),
- MaxWhileTries(),
- ]
- dispatcher.main(self.ctx)
- assert len(m_run_job.call_args_list) == 0
- assert len(m_try_push_job_info.call_args_list) == len(jobs)
- for i in range(len(jobs)):
- push_call = m_try_push_job_info.call_args_list[i]
- assert push_call[0][1]['status'] == 'dead'
-
- @pytest.mark.parametrize(
- ["timestamp", "expire", "skip"],
- [
- [datetime.timedelta(days=-1), None, False],
- [datetime.timedelta(days=-30), None, True],
- [None, datetime.timedelta(days=1), False],
- [None, datetime.timedelta(days=-1), True],
- [datetime.timedelta(days=-1), datetime.timedelta(days=1), False],
- [datetime.timedelta(days=1), datetime.timedelta(days=-1), True],
- ]
- )
- @patch("teuthology.dispatcher.report.try_push_job_info")
- def test_check_job_expiration(self, _, timestamp, expire, skip):
- now = datetime.datetime.now(datetime.timezone.utc)
- job_config = dict(
- job_id="1",
- name="job_name",
- )
- if timestamp:
- job_config["timestamp"] = (now + timestamp).strftime(TIMESTAMP_FMT)
- if expire:
- job_config["expire"] = (now + expire).strftime(TIMESTAMP_FMT)
- if skip:
- with pytest.raises(dispatcher.SkipJob):
- dispatcher.check_job_expiration(job_config)
- else:
- dispatcher.check_job_expiration(job_config)
+++ /dev/null
-from teuthology.dispatcher import supervisor
-from unittest.mock import patch
-
-class TestCheckReImageFailureMarkDown(object):
- def setup_method(self):
- self.the_function = supervisor.check_for_reimage_failures_and_mark_down
-
- def create_n_out_of_10_reimage_failed_jobs(self, n):
- ret_list = []
- for i in range(n):
- obj1 = {
- "failure_reason":"Error reimaging machines: Manually raised error"
- }
- ret_list.append(obj1)
- for j in range(10-n):
- obj2 = {"failure_reason":"Error something else: dummy"}
- ret_list.append(obj2)
- return ret_list
-
- @patch('teuthology.dispatcher.supervisor.shortname')
- @patch('teuthology.lock.ops.update_lock')
- @patch('teuthology.dispatcher.supervisor.requests')
- @patch('teuthology.dispatcher.supervisor.urljoin')
- @patch('teuthology.dispatcher.supervisor.teuth_config')
- def test_one_machine_ten_reimage_failed_jobs(
- self,
- m_t_config,
- m_urljoin,
- m_requests,
- mark_down,
- shortname
- ):
- targets = {'fakeos@rmachine061.front.sepia.ceph.com': 'ssh-ed25519'}
- m_requests.get.return_value.json.return_value = \
- self.create_n_out_of_10_reimage_failed_jobs(10)
- shortname.return_value = 'rmachine061'
- self.the_function(targets)
- assert mark_down.called
-
- @patch('teuthology.dispatcher.supervisor.shortname')
- @patch('teuthology.lock.ops.update_lock')
- @patch('teuthology.dispatcher.supervisor.requests')
- @patch('teuthology.dispatcher.supervisor.urljoin')
- @patch('teuthology.dispatcher.supervisor.teuth_config')
- def test_one_machine_seven_reimage_failed_jobs(
- self,
- m_t_config,
- m_urljoin,
- m_requests,
- mark_down,
- shortname,
- ):
- targets = {'fakeos@rmachine061.front.sepia.ceph.com': 'ssh-ed25519'}
- m_requests.get.return_value.json.return_value = \
- self.create_n_out_of_10_reimage_failed_jobs(7)
- shortname.return_value = 'rmachine061'
- self.the_function(targets)
- assert mark_down.called is False
-
- @patch('teuthology.dispatcher.supervisor.shortname')
- @patch('teuthology.lock.ops.update_lock')
- @patch('teuthology.dispatcher.supervisor.requests')
- @patch('teuthology.dispatcher.supervisor.urljoin')
- @patch('teuthology.dispatcher.supervisor.teuth_config')
- def test_two_machine_all_reimage_failed_jobs(
- self,
- m_t_config,
- m_urljoin,
- m_requests,
- mark_down,
- shortname,
- ):
- targets = {'fakeos@rmachine061.front.sepia.ceph.com': 'ssh-ed25519',
- 'fakeos@rmachine179.back.sepia.ceph.com': 'ssh-ed45333'}
- m_requests.get.return_value.json.side_effect = \
- [self.create_n_out_of_10_reimage_failed_jobs(10),
- self.create_n_out_of_10_reimage_failed_jobs(10)]
- shortname.return_value.side_effect = ['rmachine061', 'rmachine179']
- self.the_function(targets)
- assert mark_down.call_count == 2
-
- @patch('teuthology.dispatcher.supervisor.shortname')
- @patch('teuthology.lock.ops.update_lock')
- @patch('teuthology.dispatcher.supervisor.requests')
- @patch('teuthology.dispatcher.supervisor.urljoin')
- @patch('teuthology.dispatcher.supervisor.teuth_config')
- def test_two_machine_one_healthy_one_reimage_failure(
- self,
- m_t_config,
- m_urljoin,
- m_requests,
- mark_down,
- shortname,
- ):
- targets = {'fakeos@rmachine061.front.sepia.ceph.com': 'ssh-ed25519',
- 'fakeos@rmachine179.back.sepia.ceph.com': 'ssh-ed45333'}
- m_requests.get.return_value.json.side_effect = \
- [self.create_n_out_of_10_reimage_failed_jobs(0),
- self.create_n_out_of_10_reimage_failed_jobs(10)]
- shortname.return_value.side_effect = ['rmachine061', 'rmachine179']
- self.the_function(targets)
- assert mark_down.call_count == 1
- assert mark_down.call_args_list[0][0][0].startswith('rmachine179')
-
+++ /dev/null
-from subprocess import DEVNULL
-from unittest.mock import patch, Mock, MagicMock
-
-from teuthology.dispatcher import supervisor
-
-
-class TestSuperviser(object):
- @patch("teuthology.dispatcher.supervisor.run_with_watchdog")
- @patch("teuthology.dispatcher.supervisor.teuth_config")
- @patch("subprocess.Popen")
- @patch("os.environ")
- @patch("os.mkdir")
- @patch("yaml.safe_dump")
- @patch("tempfile.NamedTemporaryFile")
- def test_run_job_with_watchdog(self, m_tempfile, m_safe_dump, m_mkdir,
- m_environ, m_popen, m_t_config,
- m_run_watchdog):
- config = {
- "suite_path": "suite/path",
- "config": {"foo": "bar"},
- "verbose": True,
- "owner": "the_owner",
- "archive_path": "archive/path",
- "name": "the_name",
- "description": "the_description",
- "job_id": "1",
- }
- m_tmp = MagicMock()
- temp_file = Mock()
- temp_file.name = "the_name"
- m_tmp.__enter__.return_value = temp_file
- m_tempfile.return_value = m_tmp
- m_p = Mock()
- m_p.returncode = 0
- m_popen.return_value = m_p
- m_t_config.results_server = True
- config_path = "archive/path/initial.config.yaml"
- supervisor.run_job(config, config_path, "teuth/bin/path", "archive/dir", verbose=False)
- m_run_watchdog.assert_called_with(m_p, config)
- expected_args = [
- 'teuth/bin/path/teuthology',
- '-v',
- '--owner', 'the_owner',
- '--archive', 'archive/path',
- '--name', 'the_name',
- '--description',
- 'the_description',
- '--',
- config_path,
- ]
- m_popen.assert_called_with(args=expected_args, stderr=DEVNULL, stdout=DEVNULL)
-
- @patch("time.sleep")
- @patch("teuthology.dispatcher.supervisor.teuth_config")
- @patch("subprocess.Popen")
- @patch("os.environ")
- @patch("os.mkdir")
- @patch("yaml.safe_dump")
- @patch("tempfile.NamedTemporaryFile")
- def test_run_job_no_watchdog(self, m_tempfile, m_safe_dump, m_mkdir,
- m_environ, m_popen, m_t_config,
- m_sleep):
- config = {
- "suite_path": "suite/path",
- "config": {"foo": "bar"},
- "verbose": True,
- "owner": "the_owner",
- "archive_path": "archive/path",
- "name": "the_name",
- "description": "the_description",
- "job_id": "1",
- }
- m_tmp = MagicMock()
- temp_file = Mock()
- temp_file.name = "the_name"
- m_tmp.__enter__.return_value = temp_file
- m_tempfile.return_value = m_tmp
- env = dict(PYTHONPATH="python/path")
- m_environ.copy.return_value = env
- m_p = Mock()
- m_p.returncode = 1
- m_popen.return_value = m_p
- m_t_config.results_server = False
- config_path = "archive/path/initial.config.yaml"
- supervisor.run_job(config, config_path, "teuth/bin/path", "archive/dir", verbose=False)
-
- @patch("teuthology.dispatcher.supervisor.report.try_push_job_info")
- @patch("time.sleep")
- def test_run_with_watchdog_no_reporting(self, m_sleep, m_try_push):
- config = {
- "name": "the_name",
- "job_id": "1",
- "archive_path": "archive/path",
- "teuthology_branch": "main",
- "owner": "the_owner",
- }
- process = Mock()
- process.poll.return_value = "not None"
- supervisor.run_with_watchdog(process, config)
- m_try_push.assert_called_with(
- dict(name=config["name"], job_id=config["job_id"]),
- dict(status='dead')
- )
-
- @patch("subprocess.Popen")
- @patch("time.sleep")
- @patch("teuthology.dispatcher.supervisor.report.try_push_job_info")
- def test_run_with_watchdog_with_reporting(self, m_tpji, m_sleep, m_popen):
- config = {
- "name": "the_name",
- "job_id": "1",
- "archive_path": "archive/path",
- "teuthology_branch": "jewel",
- "owner": "the_owner",
- }
- process = Mock()
- process.poll.return_value = "not None"
- m_proc = Mock()
- m_proc.poll.return_value = "not None"
- m_popen.return_value = m_proc
- supervisor.run_with_watchdog(process, config)
+++ /dev/null
-import teuthology.lock.util
-
-class TestLock(object):
-
- def test_locked_since_seconds(self):
- node = { "locked_since": "2013-02-07 19:33:55.000000" }
- assert teuthology.lock.util.locked_since_seconds(node) > 3600
+++ /dev/null
-archive-on-error: true
+++ /dev/null
-stop_worker: true
-machine_type: openstack
-os_type: ubuntu
-os_version: "14.04"
-roles:
-- - mon.a
- - osd.0
-tasks:
-- exec:
- mon.a:
- - echo "Well done !"
-
+++ /dev/null
-#
-# Copyright (c) 2015, 2016 Red Hat, Inc.
-#
-# Author: Loic Dachary <loic@dachary.org>
-#
-# Permission is hereby granted, free of charge, to any person obtaining a copy
-# of this software and associated documentation files (the "Software"), to deal
-# in the Software without restriction, including without limitation the rights
-# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-# copies of the Software, and to permit persons to whom the Software is
-# furnished to do so, subject to the following conditions:
-#
-# The above copyright notice and this permission notice shall be included in
-# all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-# THE SOFTWARE.
-#
-import argparse
-import logging
-import json
-import os
-import subprocess
-import tempfile
-import shutil
-
-import teuthology.lock
-import teuthology.lock.cli
-import teuthology.lock.query
-import teuthology.lock.util
-import teuthology.misc
-import teuthology.schedule
-import teuthology.suite
-import teuthology.openstack
-import scripts.schedule
-import scripts.lock
-import scripts.suite
-from teuthology.config import config as teuth_config
-from teuthology.config import set_config_attr
-
-
-class Integration(object):
-
- @classmethod
- def setup_class(self):
- teuthology.log.setLevel(logging.DEBUG)
- set_config_attr(argparse.Namespace())
- self.teardown_class()
-
- @classmethod
- def teardown_class(self):
- os.system("sudo /etc/init.d/beanstalkd restart")
- # if this fails it will not show the error but some weird
- # INTERNALERROR> IndexError: list index out of range
- # move that to def tearDown for debug and when it works move it
- # back in tearDownClass so it is not called on every test
- ownedby = "ownedby='" + teuth_config.openstack['ip']
- all_instances = teuthology.openstack.OpenStack().run(
- "server list -f json --long")
- for instance in json.loads(all_instances):
- if ownedby in instance['Properties']:
- teuthology.openstack.OpenStack().run(
- "server delete --wait " + instance['ID'])
-
- def setup_worker(self):
- self.logs = self.d + "/log"
- os.mkdir(self.logs, 0o755)
- self.archive = self.d + "/archive"
- os.mkdir(self.archive, 0o755)
- self.worker_cmd = ("teuthology-worker --tube openstack " +
- "-l " + self.logs + " "
- "--archive-dir " + self.archive + " ")
- logging.info(self.worker_cmd)
- self.worker = subprocess.Popen(self.worker_cmd,
- stdout=subprocess.PIPE,
- stderr=subprocess.PIPE,
- shell=True)
-
- def wait_worker(self):
- if not self.worker:
- return
-
- (stdoutdata, stderrdata) = self.worker.communicate()
- stdoutdata = stdoutdata.decode('utf-8')
- stderrdata = stderrdata.decode('utf-8')
- logging.info(self.worker_cmd + ":" +
- " stdout " + stdoutdata +
- " stderr " + stderrdata + " end ")
- assert self.worker.returncode == 0
- self.worker = None
-
- def get_teuthology_log(self):
- # the archive is removed before each test, there must
- # be only one run and one job
- run = os.listdir(self.archive)[0]
- job = os.listdir(os.path.join(self.archive, run))[0]
- path = os.path.join(self.archive, run, job, 'teuthology.log')
- return open(path, 'r').read()
-
-class TestSuite(Integration):
-
- def setup_method(self):
- self.d = tempfile.mkdtemp()
- self.setup_worker()
- logging.info("TestSuite: done worker")
-
- def teardown(self):
- self.wait_worker()
- shutil.rmtree(self.d)
-
- def test_suite_noop(self):
- cwd = os.getcwd()
- os.mkdir(self.d + '/upload', 0o755)
- upload = 'localhost:' + self.d + '/upload'
- args = ['--suite', 'noop',
- '--suite-dir', cwd + '/teuthology/openstack/test',
- '--machine-type', 'openstack',
- '--archive-upload', upload,
- '--verbose']
- logging.info("TestSuite:test_suite_noop")
- scripts.suite.main(args)
- self.wait_worker()
- log = self.get_teuthology_log()
- assert "teuthology.run:pass" in log
- assert "Well done" in log
- upload_key = teuth_config.archive_upload_key
- if upload_key:
- ssh = "RSYNC_RSH='ssh -i " + upload_key + "'"
- else:
- ssh = ''
- assert 'teuthology.log' in teuthology.misc.sh(ssh + " rsync -av " + upload)
-
-class TestSchedule(Integration):
-
- def setup_method(self):
- self.d = tempfile.mkdtemp()
- self.setup_worker()
-
- def teardown(self):
- self.wait_worker()
- shutil.rmtree(self.d)
-
- def test_schedule_stop_worker(self):
- job = 'teuthology/openstack/test/stop_worker.yaml'
- args = ['--name', 'fake',
- '--verbose',
- '--owner', 'test@test.com',
- '--worker', 'openstack',
- job]
- scripts.schedule.main(args)
- self.wait_worker()
-
- def test_schedule_noop(self):
- job = 'teuthology/openstack/test/noop.yaml'
- args = ['--name', 'fake',
- '--verbose',
- '--owner', 'test@test.com',
- '--worker', 'openstack',
- job]
- scripts.schedule.main(args)
- self.wait_worker()
- log = self.get_teuthology_log()
- assert "teuthology.run:pass" in log
- assert "Well done" in log
-
- def test_schedule_resources_hint(self):
- """It is tricky to test resources hint in a provider agnostic way. The
- best way seems to ask for at least 1GB of RAM and 10GB
- disk. Some providers do not offer a 1GB RAM flavor (OVH for
- instance) and the 2GB RAM will be chosen instead. It however
- seems unlikely that a 4GB RAM will be chosen because it would
- mean such a provider has nothing under that limit and it's a
- little too high.
-
- Since the default when installing is to ask for 7000 MB, we
- can reasonably assume that the hint has been taken into
- account if the instance has less than 4GB RAM.
- """
- try:
- teuthology.openstack.OpenStack().run("volume list")
- job = 'teuthology/openstack/test/resources_hint.yaml'
- has_cinder = True
- except subprocess.CalledProcessError:
- job = 'teuthology/openstack/test/resources_hint_no_cinder.yaml'
- has_cinder = False
- args = ['--name', 'fake',
- '--verbose',
- '--owner', 'test@test.com',
- '--worker', 'openstack',
- job]
- scripts.schedule.main(args)
- self.wait_worker()
- log = self.get_teuthology_log()
- assert "teuthology.run:pass" in log
- assert "RAM size ok" in log
- if has_cinder:
- assert "Disk size ok" in log
-
-class TestLock(Integration):
-
- def setup_method(self):
- self.options = ['--verbose',
- '--machine-type', 'openstack' ]
-
- def test_main(self):
- args = scripts.lock.parse_args(self.options + ['--lock'])
- assert teuthology.lock.cli.main(args) == 0
-
- def test_lock_unlock(self):
- default_archs = teuthology.openstack.OpenStack().get_available_archs()
- if 'TEST_IMAGES' in os.environ:
- images = os.environ['TEST_IMAGES'].split()
- else:
- images = teuthology.openstack.OpenStack.image2url.keys()
- for image in images:
- (os_type, os_version, arch) = image.split('-')
- if arch not in default_archs:
- logging.info("skipping " + image + " because arch " +
- " is not supported (" + str(default_archs) + ")")
- continue
- args = scripts.lock.parse_args(self.options +
- ['--lock-many', '1',
- '--os-type', os_type,
- '--os-version', os_version,
- '--arch', arch])
- assert teuthology.lock.cli.main(args) == 0
- locks = teuthology.lock.query.list_locks(locked=True)
- assert len(locks) == 1
- args = scripts.lock.parse_args(self.options +
- ['--unlock', locks[0]['name']])
- assert teuthology.lock.cli.main(args) == 0
-
- def test_list(self, capsys):
- args = scripts.lock.parse_args(self.options + ['--list', '--all'])
- teuthology.lock.cli.main(args)
- out, err = capsys.readouterr()
- assert 'machine_type' in out
- assert 'openstack' in out
+++ /dev/null
-stop_worker: true
-machine_type: openstack
-openstack:
- - machine:
- disk: 10 # GB
- ram: 10000 # MB
- cpus: 1
- volumes:
- count: 1
- size: 2 # GB
-os_type: ubuntu
-os_version: "14.04"
-roles:
-- - mon.a
- - osd.0
-tasks:
-- exec:
- mon.a:
- - test $(sed -n -e 's/MemTotal.* \([0-9][0-9]*\).*/\1/p' < /proc/meminfo) -ge 10000000 && echo "RAM" "size" "ok"
- - cat /proc/meminfo
-# wait for the attached volume to show up
- - for delay in 1 2 4 8 16 32 64 128 256 512 ; do if test -e /sys/block/vdb/size ; then break ; else sleep $delay ; fi ; done
-# 4000000 because 512 bytes sectors
- - test $(cat /sys/block/vdb/size) -gt 4000000 && echo "Disk" "size" "ok"
- - cat /sys/block/vdb/size
+++ /dev/null
-stop_worker: true
-machine_type: openstack
-openstack:
- - machine:
- disk: 10 # GB
- ram: 10000 # MB
- cpus: 1
- volumes:
- count: 0
- size: 2 # GB
-os_type: ubuntu
-os_version: "14.04"
-roles:
-- - mon.a
- - osd.0
-tasks:
-- exec:
- mon.a:
- - cat /proc/meminfo
- - test $(sed -n -e 's/MemTotal.* \([0-9][0-9]*\).*/\1/p' < /proc/meminfo) -ge 10000000 && echo "RAM" "size" "ok"
+++ /dev/null
-stop_worker: true
+++ /dev/null
-stop_worker: true
-roles:
-- - mon.a
- - osd.0
-tasks:
-- exec:
- mon.a:
- - echo "Well done !"
-
+++ /dev/null
-from teuthology.config import config
-
-
-class TestOpenStack(object):
-
- def setup_method(self):
- self.openstack_config = config['openstack']
-
- def test_config_clone(self):
- assert 'clone' in self.openstack_config
-
- def test_config_user_data(self):
- os_type = 'rhel'
- os_version = '7.0'
- template_path = self.openstack_config['user-data'].format(
- os_type=os_type,
- os_version=os_version)
- assert os_type in template_path
- assert os_version in template_path
-
- def test_config_ip(self):
- assert 'ip' in self.openstack_config
-
- def test_config_machine(self):
- assert 'machine' in self.openstack_config
- machine_config = self.openstack_config['machine']
- assert 'disk' in machine_config
- assert 'ram' in machine_config
- assert 'cpus' in machine_config
-
- def test_config_volumes(self):
- assert 'volumes' in self.openstack_config
- volumes_config = self.openstack_config['volumes']
- assert 'count' in volumes_config
- assert 'size' in volumes_config
+++ /dev/null
-#
-# Copyright (c) 2015,2016 Red Hat, Inc.
-#
-# Author: Loic Dachary <loic@dachary.org>
-#
-# Permission is hereby granted, free of charge, to any person obtaining a copy
-# of this software and associated documentation files (the "Software"), to deal
-# in the Software without restriction, including without limitation the rights
-# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-# copies of the Software, and to permit persons to whom the Software is
-# furnished to do so, subject to the following conditions:
-#
-# The above copyright notice and this permission notice shall be included in
-# all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-# THE SOFTWARE.
-#
-import argparse
-import logging
-import os
-import pytest
-import subprocess
-import tempfile
-import time
-from mock import patch
-
-import teuthology
-from teuthology import misc
-from teuthology.config import set_config_attr
-from teuthology.openstack import TeuthologyOpenStack, OpenStack, OpenStackInstance
-from teuthology.openstack import NoFlavorException
-import scripts.openstack
-
-
-class TestOpenStackBase(object):
-
- def setup_method(self):
- OpenStack.token = None
- OpenStack.token_expires = None
- self.environ = {}
- for k in os.environ.keys():
- if k.startswith('OS_'):
- self.environ[k] = os.environ[k]
-
- def teardown_method(self):
- OpenStack.token = None
- OpenStack.token_expires = None
- for k in os.environ.keys():
- if k.startswith('OS_'):
- if k in self.environ:
- os.environ[k] = self.environ[k]
- else:
- del os.environ[k]
-
-class TestOpenStackInstance(TestOpenStackBase):
-
- teuthology_instance = """
-{
- "OS-EXT-STS:task_state": null,
- "addresses": "Ext-Net=167.114.233.32",
- "image": "Ubuntu 14.04 (0d315a8d-75e3-418a-80e4-48e62d599627)",
- "OS-EXT-STS:vm_state": "active",
- "OS-SRV-USG:launched_at": "2015-08-17T12:22:13.000000",
- "flavor": "vps-ssd-1 (164fcc7e-7771-414f-a607-b388cb7b7aa0)",
- "id": "f3ca32d7-212b-458b-a0d4-57d1085af953",
- "security_groups": [
- {
- "name": "default"
- }
- ],
- "user_id": "3a075820e5d24fda96cd340b87fd94e9",
- "OS-DCF:diskConfig": "AUTO",
- "accessIPv4": "",
- "accessIPv6": "",
- "progress": 0,
- "OS-EXT-STS:power_state": 1,
- "project_id": "62cf1be03cec403c8ed8e64df55732ea",
- "config_drive": "",
- "status": "ACTIVE",
- "updated": "2015-11-03T13:48:53Z",
- "hostId": "bcdf964b6f724e573c07156ff85b4db1707f6f0969f571cf33e0468d",
- "OS-SRV-USG:terminated_at": null,
- "key_name": "loic",
- "properties": "",
- "OS-EXT-AZ:availability_zone": "nova",
- "name": "mrdarkdragon",
- "created": "2015-08-17T12:21:31Z",
- "os-extended-volumes:volumes_attached": [{"id": "627e2631-fbb3-48cd-b801-d29cd2a76f74"}, {"id": "09837649-0881-4ee2-a560-adabefc28764"}, {"id": "44e5175b-6044-40be-885a-c9ddfb6f75bb"}]
-}
- """
-
- teuthology_instance_no_addresses = """
-{
- "OS-EXT-STS:task_state": null,
- "addresses": "",
- "image": "Ubuntu 14.04 (0d315a8d-75e3-418a-80e4-48e62d599627)",
- "OS-EXT-STS:vm_state": "active",
- "OS-SRV-USG:launched_at": "2015-08-17T12:22:13.000000",
- "flavor": "vps-ssd-1 (164fcc7e-7771-414f-a607-b388cb7b7aa0)",
- "id": "f3ca32d7-212b-458b-a0d4-57d1085af953",
- "security_groups": [
- {
- "name": "default"
- }
- ],
- "user_id": "3a075820e5d24fda96cd340b87fd94e9",
- "OS-DCF:diskConfig": "AUTO",
- "accessIPv4": "",
- "accessIPv6": "",
- "progress": 0,
- "OS-EXT-STS:power_state": 1,
- "project_id": "62cf1be03cec403c8ed8e64df55732ea",
- "config_drive": "",
- "status": "ACTIVE",
- "updated": "2015-11-03T13:48:53Z",
- "hostId": "bcdf964b6f724e573c07156ff85b4db1707f6f0969f571cf33e0468d",
- "OS-SRV-USG:terminated_at": null,
- "key_name": "loic",
- "properties": "",
- "OS-EXT-AZ:availability_zone": "nova",
- "name": "mrdarkdragon",
- "created": "2015-08-17T12:21:31Z",
- "os-extended-volumes:volumes_attached": []
-}
- """
-
- @classmethod
- def setup_class(self):
- if 'OS_AUTH_URL' not in os.environ:
- pytest.skip('no OS_AUTH_URL environment variable')
-
- def test_init(self):
- with patch.multiple(
- misc,
- sh=lambda cmd: self.teuthology_instance,
- ):
- o = OpenStackInstance('NAME')
- assert o['id'] == 'f3ca32d7-212b-458b-a0d4-57d1085af953'
- o = OpenStackInstance('NAME', {"id": "OTHER"})
- assert o['id'] == "OTHER"
-
- def test_get_created(self):
- with patch.multiple(
- misc,
- sh=lambda cmd: self.teuthology_instance,
- ):
- o = OpenStackInstance('NAME')
- assert o.get_created() > 0
-
- def test_exists(self):
- with patch.multiple(
- misc,
- sh=lambda cmd: self.teuthology_instance,
- ):
- o = OpenStackInstance('NAME')
- assert o.exists()
- def sh_raises(cmd):
- raise subprocess.CalledProcessError('FAIL', 'BAD')
- with patch.multiple(
- misc,
- sh=sh_raises,
- ):
- o = OpenStackInstance('NAME')
- assert not o.exists()
-
- def test_volumes(self):
- with patch.multiple(
- misc,
- sh=lambda cmd: self.teuthology_instance,
- ):
- o = OpenStackInstance('NAME')
- assert len(o.get_volumes()) == 3
-
- def test_get_addresses(self):
- answers = [
- self.teuthology_instance_no_addresses,
- self.teuthology_instance,
- ]
- def sh(self):
- return answers.pop(0)
- with patch.multiple(
- misc,
- sh=sh,
- ):
- o = OpenStackInstance('NAME')
- assert o.get_addresses() == 'Ext-Net=167.114.233.32'
-
- def test_get_ip_neutron(self):
- instance_id = '8e1fd70a-3065-46f8-9c30-84dc028c1834'
- ip = '10.10.10.4'
- def sh(cmd):
- if 'neutron subnet-list' in cmd:
- return """
-[
- {
- "ip_version": 6,
- "id": "c45b9661-b2ba-4817-9e3a-f8f63bf32989"
- },
- {
- "ip_version": 4,
- "id": "e03a3dbc-afc8-4b52-952e-7bf755397b50"
- }
-]
- """
- elif 'neutron port-list' in cmd:
- return ("""
-[
- {
- "device_id": "915504ad-368b-4cce-be7c-4f8a83902e28",
- "fixed_ips": "{\\"subnet_id\\": \\"e03a3dbc-afc8-4b52-952e-7bf755397b50\\", \\"ip_address\\": \\"10.10.10.1\\"}\\n{\\"subnet_id\\": \\"c45b9661-b2ba-4817-9e3a-f8f63bf32989\\", \\"ip_address\\": \\"2607:f298:6050:9afc::1\\"}"
- },
- {
- "device_id": "{instance_id}",
- "fixed_ips": "{\\"subnet_id\\": \\"e03a3dbc-afc8-4b52-952e-7bf755397b50\\", \\"ip_address\\": \\"{ip}\\"}\\n{\\"subnet_id\\": \\"c45b9661-b2ba-4817-9e3a-f8f63bf32989\\", \\"ip_address\\": \\"2607:f298:6050:9afc:f816:3eff:fe07:76c1\\"}"
- },
- {
- "device_id": "17e4a968-4caa-4cee-8e4b-f950683a02bd",
- "fixed_ips": "{\\"subnet_id\\": \\"e03a3dbc-afc8-4b52-952e-7bf755397b50\\", \\"ip_address\\": \\"10.10.10.5\\"}\\n{\\"subnet_id\\": \\"c45b9661-b2ba-4817-9e3a-f8f63bf32989\\", \\"ip_address\\": \\"2607:f298:6050:9afc:f816:3eff:fe9c:37f0\\"}"
- }
-]
- """.replace('{instance_id}', instance_id).
- replace('{ip}', ip))
- else:
- raise Exception("unexpected " + cmd)
- with patch.multiple(
- misc,
- sh=sh,
- ):
- assert ip == OpenStackInstance(
- instance_id,
- { 'id': instance_id },
- ).get_ip_neutron()
-
-class TestOpenStack(TestOpenStackBase):
-
- flavors = """[
- {
- "Name": "eg-120-ssd",
- "RAM": 120000,
- "Ephemeral": 0,
- "VCPUs": 32,
- "Is Public": true,
- "Disk": 800,
- "ID": "008f75de-c467-4d15-8f70-79c8fbe19538"
- },
- {
- "Name": "hg-60",
- "RAM": 60000,
- "Ephemeral": 0,
- "VCPUs": 16,
- "Is Public": true,
- "Disk": 1600,
- "ID": "0297d7ac-fe6f-4ff1-b6e7-0b8b0908c94f"
- },
- {
- "Name": "win-sp-120-ssd-flex",
- "RAM": 120000,
- "Ephemeral": 0,
- "VCPUs": 8,
- "Is Public": true,
- "Disk": 50,
- "ID": "039e31f2-6541-46c8-85cf-7f47fab7ad78"
- },
- {
- "Name": "win-sp-60",
- "RAM": 60000,
- "Ephemeral": 0,
- "VCPUs": 4,
- "Is Public": true,
- "Disk": 400,
- "ID": "0417a0e6-f68a-4b8b-a642-ca5ecb9652f7"
- },
- {
- "Name": "hg-120-ssd",
- "RAM": 120000,
- "Ephemeral": 0,
- "VCPUs": 32,
- "Is Public": true,
- "Disk": 800,
- "ID": "042aefc6-b713-4a7e-ada5-3ff81daa1960"
- },
- {
- "Name": "win-sp-60-flex",
- "RAM": 60000,
- "Ephemeral": 0,
- "VCPUs": 4,
- "Is Public": true,
- "Disk": 50,
- "ID": "0609290c-ad2a-40f0-8c66-c755dd38fe3f"
- },
- {
- "Name": "win-eg-120",
- "RAM": 120000,
- "Ephemeral": 0,
- "VCPUs": 32,
- "Is Public": true,
- "Disk": 800,
- "ID": "0651080f-5d07-44b1-a759-7ea4594b669e"
- },
- {
- "Name": "win-sp-240",
- "RAM": 240000,
- "Ephemeral": 0,
- "VCPUs": 16,
- "Is Public": true,
- "Disk": 1600,
- "ID": "07885848-8831-486d-8525-91484c09cc7e"
- },
- {
- "Name": "win-hg-60-ssd",
- "RAM": 60000,
- "Ephemeral": 0,
- "VCPUs": 16,
- "Is Public": true,
- "Disk": 800,
- "ID": "079aa0a2-5e48-4e58-8205-719bc962736e"
- },
- {
- "Name": "eg-120",
- "RAM": 120000,
- "Ephemeral": 0,
- "VCPUs": 32,
- "Is Public": true,
- "Disk": 1600,
- "ID": "090f8b8c-673c-4ab8-9a07-6e54a8776e7b"
- },
- {
- "Name": "win-hg-15-ssd-flex",
- "RAM": 15000,
- "Ephemeral": 0,
- "VCPUs": 4,
- "Is Public": true,
- "Disk": 50,
- "ID": "10e10c58-d29f-4ff6-a1fd-085c35a3bd9b"
- },
- {
- "Name": "eg-15-ssd",
- "RAM": 15000,
- "Ephemeral": 0,
- "VCPUs": 4,
- "Is Public": true,
- "Disk": 200,
- "ID": "1340a920-0f2f-4c1b-8d74-e2502258da73"
- },
- {
- "Name": "win-eg-30-ssd-flex",
- "RAM": 30000,
- "Ephemeral": 0,
- "VCPUs": 8,
- "Is Public": true,
- "Disk": 50,
- "ID": "13e54752-fbd0-47a6-aa93-e5a67dfbc743"
- },
- {
- "Name": "eg-120-ssd-flex",
- "RAM": 120000,
- "Ephemeral": 0,
- "VCPUs": 32,
- "Is Public": true,
- "Disk": 50,
- "ID": "15c07a54-2dfb-41d9-aa73-6989fd8cafc2"
- },
- {
- "Name": "win-eg-120-ssd-flex",
- "RAM": 120000,
- "Ephemeral": 0,
- "VCPUs": 32,
- "Is Public": true,
- "Disk": 50,
- "ID": "15e0dfcc-10f4-4e70-8ac1-30bc323273e2"
- },
- {
- "Name": "vps-ssd-1",
- "RAM": 2000,
- "Ephemeral": 0,
- "VCPUs": 1,
- "Is Public": true,
- "Disk": 10,
- "ID": "164fcc7e-7771-414f-a607-b388cb7b7aa0"
- },
- {
- "Name": "win-sp-120-flex",
- "RAM": 120000,
- "Ephemeral": 0,
- "VCPUs": 8,
- "Is Public": true,
- "Disk": 50,
- "ID": "169415e1-0979-4527-94fb-638c885bbd8c"
- },
- {
- "Name": "win-hg-60-flex",
- "RAM": 60000,
- "Ephemeral": 0,
- "VCPUs": 16,
- "Is Public": true,
- "Disk": 50,
- "ID": "16f13d5b-be27-4b8b-88da-959d3904d3ba"
- },
- {
- "Name": "win-sp-30-ssd",
- "RAM": 30000,
- "Ephemeral": 0,
- "VCPUs": 2,
- "Is Public": true,
- "Disk": 100,
- "ID": "1788102b-ab80-4a0c-b819-541deaca7515"
- },
- {
- "Name": "win-sp-240-flex",
- "RAM": 240000,
- "Ephemeral": 0,
- "VCPUs": 16,
- "Is Public": true,
- "Disk": 50,
- "ID": "17bcfa14-135f-442f-9397-a4dc25265560"
- },
- {
- "Name": "win-eg-60-ssd-flex",
- "RAM": 60000,
- "Ephemeral": 0,
- "VCPUs": 16,
- "Is Public": true,
- "Disk": 50,
- "ID": "194ca9ba-04af-4d86-ba37-d7da883a7eab"
- },
- {
- "Name": "win-eg-60-flex",
- "RAM": 60000,
- "Ephemeral": 0,
- "VCPUs": 16,
- "Is Public": true,
- "Disk": 50,
- "ID": "19ff8837-4751-4f6c-a82b-290bc53c83c1"
- },
- {
- "Name": "win-eg-30-flex",
- "RAM": 30000,
- "Ephemeral": 0,
- "VCPUs": 8,
- "Is Public": true,
- "Disk": 50,
- "ID": "1aaef5e5-4df9-4462-80d3-701683ab9ff0"
- },
- {
- "Name": "eg-15",
- "RAM": 15000,
- "Ephemeral": 0,
- "VCPUs": 4,
- "Is Public": true,
- "Disk": 400,
- "ID": "1cd85b81-5e4d-477a-a127-eb496b1d75de"
- },
- {
- "Name": "hg-120",
- "RAM": 120000,
- "Ephemeral": 0,
- "VCPUs": 32,
- "Is Public": true,
- "Disk": 1600,
- "ID": "1f1efedf-ec91-4a42-acd7-f5cf64b02d3c"
- },
- {
- "Name": "hg-15-ssd-flex",
- "RAM": 15000,
- "Ephemeral": 0,
- "VCPUs": 4,
- "Is Public": true,
- "Disk": 50,
- "ID": "20347a07-a289-4c07-a645-93cb5e8e2d30"
- },
- {
- "Name": "win-eg-7-ssd",
- "RAM": 7000,
- "Ephemeral": 0,
- "VCPUs": 2,
- "Is Public": true,
- "Disk": 100,
- "ID": "20689394-bd77-4f4d-900e-52cc8a86aeb4"
- },
- {
- "Name": "win-sp-60-ssd-flex",
- "RAM": 60000,
- "Ephemeral": 0,
- "VCPUs": 4,
- "Is Public": true,
- "Disk": 50,
- "ID": "21104d99-ba7b-47a0-9133-7e884710089b"
- },
- {
- "Name": "win-sp-120-ssd",
- "RAM": 120000,
- "Ephemeral": 0,
- "VCPUs": 8,
- "Is Public": true,
- "Disk": 400,
- "ID": "23c21ecc-9ee8-4ad3-bd9f-aa17a3faf84e"
- },
- {
- "Name": "win-hg-15-ssd",
- "RAM": 15000,
- "Ephemeral": 0,
- "VCPUs": 4,
- "Is Public": true,
- "Disk": 200,
- "ID": "24e293ed-bc54-4f26-8fb7-7b9457d08e66"
- },
- {
- "Name": "eg-15-ssd-flex",
- "RAM": 15000,
- "Ephemeral": 0,
- "VCPUs": 4,
- "Is Public": true,
- "Disk": 50,
- "ID": "25f3534a-89e5-489d-aa8b-63f62e76875b"
- },
- {
- "Name": "win-eg-60",
- "RAM": 60000,
- "Ephemeral": 0,
- "VCPUs": 16,
- "Is Public": true,
- "Disk": 800,
- "ID": "291173f1-ea1d-410b-8045-667361a4addb"
- },
- {
- "Name": "sp-30-ssd-flex",
- "RAM": 30000,
- "Ephemeral": 0,
- "VCPUs": 2,
- "Is Public": true,
- "Disk": 50,
- "ID": "2b646463-2efa-428b-94ed-4059923c3636"
- },
- {
- "Name": "win-eg-120-flex",
- "RAM": 120000,
- "Ephemeral": 0,
- "VCPUs": 32,
- "Is Public": true,
- "Disk": 50,
- "ID": "2c74df82-29d2-4b1a-a32c-d5633e7359b4"
- },
- {
- "Name": "win-eg-15-ssd",
- "RAM": 15000,
- "Ephemeral": 0,
- "VCPUs": 4,
- "Is Public": true,
- "Disk": 200,
- "ID": "2fe4344f-d701-4bc4-8dcd-6d0b5d83fa13"
- },
- {
- "Name": "sp-30-flex",
- "RAM": 30000,
- "Ephemeral": 0,
- "VCPUs": 2,
- "Is Public": true,
- "Disk": 50,
- "ID": "31487b30-eeb6-472f-a9b6-38ace6587ebc"
- },
- {
- "Name": "win-sp-240-ssd",
- "RAM": 240000,
- "Ephemeral": 0,
- "VCPUs": 16,
- "Is Public": true,
- "Disk": 800,
- "ID": "325b602f-ecc4-4444-90bd-5a2cf4e0da53"
- },
- {
- "Name": "win-hg-7",
- "RAM": 7000,
- "Ephemeral": 0,
- "VCPUs": 2,
- "Is Public": true,
- "Disk": 200,
- "ID": "377ded36-491f-4ad7-9eb4-876798b2aea9"
- },
- {
- "Name": "sp-30-ssd",
- "RAM": 30000,
- "Ephemeral": 0,
- "VCPUs": 2,
- "Is Public": true,
- "Disk": 100,
- "ID": "382f2831-4dba-40c4-bb7a-6fadff71c4db"
- },
- {
- "Name": "hg-30",
- "RAM": 30000,
- "Ephemeral": 0,
- "VCPUs": 8,
- "Is Public": true,
- "Disk": 800,
- "ID": "3c1d6170-0097-4b5c-a3b3-adff1b7a86e0"
- },
- {
- "Name": "hg-60-flex",
- "RAM": 60000,
- "Ephemeral": 0,
- "VCPUs": 16,
- "Is Public": true,
- "Disk": 50,
- "ID": "3c669730-b5cd-4e44-8bd2-bc8d9f984ab2"
- },
- {
- "Name": "sp-240-ssd-flex",
- "RAM": 240000,
- "Ephemeral": 0,
- "VCPUs": 16,
- "Is Public": true,
- "Disk": 50,
- "ID": "3d66fea3-26f2-4195-97ab-fdea3b836099"
- },
- {
- "Name": "sp-240-flex",
- "RAM": 240000,
- "Ephemeral": 0,
- "VCPUs": 16,
- "Is Public": true,
- "Disk": 50,
- "ID": "40c781f7-d7a7-4b0d-bcca-5304aeabbcd9"
- },
- {
- "Name": "hg-7-flex",
- "RAM": 7000,
- "Ephemeral": 0,
- "VCPUs": 2,
- "Is Public": true,
- "Disk": 50,
- "ID": "42730e52-147d-46b8-9546-18e31e5ac8a9"
- },
- {
- "Name": "eg-30-ssd",
- "RAM": 30000,
- "Ephemeral": 0,
- "VCPUs": 8,
- "Is Public": true,
- "Disk": 400,
- "ID": "463f30e9-7d7a-4693-944f-142067cf553b"
- },
- {
- "Name": "hg-15-flex",
- "RAM": 15000,
- "Ephemeral": 0,
- "VCPUs": 4,
- "Is Public": true,
- "Disk": 50,
- "ID": "534f07c6-91af-44c8-9e62-156360fe8359"
- },
- {
- "Name": "win-sp-30-flex",
- "RAM": 30000,
- "Ephemeral": 0,
- "VCPUs": 2,
- "Is Public": true,
- "Disk": 50,
- "ID": "55533fdf-ad57-4aa7-a2c6-ee31bb94e77b"
- },
- {
- "Name": "win-hg-60-ssd-flex",
- "RAM": 60000,
- "Ephemeral": 0,
- "VCPUs": 16,
- "Is Public": true,
- "Disk": 50,
- "ID": "58b24234-3804-4c4f-9eb6-5406a3a13758"
- },
- {
- "Name": "hg-7-ssd-flex",
- "RAM": 7000,
- "Ephemeral": 0,
- "VCPUs": 2,
- "Is Public": true,
- "Disk": 50,
- "ID": "596c1276-8e53-40a0-b183-cdd9e9b1907d"
- },
- {
- "Name": "win-hg-30-ssd",
- "RAM": 30000,
- "Ephemeral": 0,
- "VCPUs": 8,
- "Is Public": true,
- "Disk": 400,
- "ID": "5c54dc08-28b9-4860-9f24-a2451b2a28ec"
- },
- {
- "Name": "eg-7",
- "RAM": 7000,
- "Ephemeral": 0,
- "VCPUs": 2,
- "Is Public": true,
- "Disk": 200,
- "ID": "5e409dbc-3f4b-46e8-a629-a418c8497922"
- },
- {
- "Name": "hg-30-flex",
- "RAM": 30000,
- "Ephemeral": 0,
- "VCPUs": 8,
- "Is Public": true,
- "Disk": 50,
- "ID": "656423ea-0551-48c6-9e0f-ec6e15952029"
- },
- {
- "Name": "hg-15",
- "RAM": 15000,
- "Ephemeral": 0,
- "VCPUs": 4,
- "Is Public": true,
- "Disk": 400,
- "ID": "675558ea-04fe-47a2-83de-40be9b2eacd4"
- },
- {
- "Name": "eg-60-flex",
- "RAM": 60000,
- "Ephemeral": 0,
- "VCPUs": 16,
- "Is Public": true,
- "Disk": 50,
- "ID": "68a8e4e1-d291-46e8-a724-fbb1c4b9b051"
- },
- {
- "Name": "hg-30-ssd",
- "RAM": 30000,
- "Ephemeral": 0,
- "VCPUs": 8,
- "Is Public": true,
- "Disk": 400,
- "ID": "6ab72807-e0a5-4e9f-bbb9-7cbbf0038b26"
- },
- {
- "Name": "win-hg-30",
- "RAM": 30000,
- "Ephemeral": 0,
- "VCPUs": 8,
- "Is Public": true,
- "Disk": 800,
- "ID": "6e12cae3-0492-483c-aa39-54a0dcaf86dd"
- },
- {
- "Name": "win-hg-7-ssd",
- "RAM": 7000,
- "Ephemeral": 0,
- "VCPUs": 2,
- "Is Public": true,
- "Disk": 100,
- "ID": "6ead771c-e8b9-424c-afa0-671280416422"
- },
- {
- "Name": "win-hg-30-flex",
- "RAM": 30000,
- "Ephemeral": 0,
- "VCPUs": 8,
- "Is Public": true,
- "Disk": 50,
- "ID": "70ded741-8f58-4bb9-8cfd-5e838b66b5f3"
- },
- {
- "Name": "win-sp-30-ssd-flex",
- "RAM": 30000,
- "Ephemeral": 0,
- "VCPUs": 2,
- "Is Public": true,
- "Disk": 50,
- "ID": "7284d104-a260-421d-8cee-6dc905107b25"
- },
- {
- "Name": "win-eg-120-ssd",
- "RAM": 120000,
- "Ephemeral": 0,
- "VCPUs": 32,
- "Is Public": true,
- "Disk": 800,
- "ID": "72c0b262-855d-40bb-a3e9-fd989a1bc466"
- },
- {
- "Name": "win-hg-7-flex",
- "RAM": 7000,
- "Ephemeral": 0,
- "VCPUs": 2,
- "Is Public": true,
- "Disk": 50,
- "ID": "73961591-c5f1-436f-b641-1a506eddaef4"
- },
- {
- "Name": "sp-240-ssd",
- "RAM": 240000,
- "Ephemeral": 0,
- "VCPUs": 16,
- "Is Public": true,
- "Disk": 800,
- "ID": "7568d834-3b16-42ce-a2c1-0654e0781160"
- },
- {
- "Name": "win-eg-60-ssd",
- "RAM": 60000,
- "Ephemeral": 0,
- "VCPUs": 16,
- "Is Public": true,
- "Disk": 800,
- "ID": "75f7fe5c-f87a-41d8-a961-a0169d02c268"
- },
- {
- "Name": "eg-7-ssd-flex",
- "RAM": 7000,
- "Ephemeral": 0,
- "VCPUs": 2,
- "Is Public": true,
- "Disk": 50,
- "ID": "77e1db73-0b36-4e37-8e47-32c2d2437ca9"
- },
- {
- "Name": "eg-60-ssd-flex",
- "RAM": 60000,
- "Ephemeral": 0,
- "VCPUs": 16,
- "Is Public": true,
- "Disk": 50,
- "ID": "78df4e30-98ca-4362-af68-037d958edaf0"
- },
- {
- "Name": "vps-ssd-2",
- "RAM": 4000,
- "Ephemeral": 0,
- "VCPUs": 1,
- "Is Public": true,
- "Disk": 20,
- "ID": "7939cc5c-79b1-45c0-be2d-aa935d92faa1"
- },
- {
- "Name": "sp-60",
- "RAM": 60000,
- "Ephemeral": 0,
- "VCPUs": 4,
- "Is Public": true,
- "Disk": 400,
- "ID": "80d8510a-79cc-4307-8db7-d1965c9e8ddb"
- },
- {
- "Name": "win-hg-120-ssd-flex",
- "RAM": 120000,
- "Ephemeral": 0,
- "VCPUs": 32,
- "Is Public": true,
- "Disk": 50,
- "ID": "835e734a-46b6-4cb2-be68-e8678fd71059"
- },
- {
- "Name": "win-eg-7",
- "RAM": 7000,
- "Ephemeral": 0,
- "VCPUs": 2,
- "Is Public": true,
- "Disk": 200,
- "ID": "84869b00-b43a-4523-babd-d47d206694e9"
- },
- {
- "Name": "win-eg-7-ssd-flex",
- "RAM": 7000,
- "Ephemeral": 0,
- "VCPUs": 2,
- "Is Public": true,
- "Disk": 50,
- "ID": "852308f8-b8bf-44a4-af41-cbc27437b275"
- },
- {
- "Name": "win-sp-30",
- "RAM": 30000,
- "Ephemeral": 0,
- "VCPUs": 2,
- "Is Public": true,
- "Disk": 200,
- "ID": "8be9dc29-3eca-499b-ae2d-e3c99699131a"
- },
- {
- "Name": "win-hg-7-ssd-flex",
- "RAM": 7000,
- "Ephemeral": 0,
- "VCPUs": 2,
- "Is Public": true,
- "Disk": 50,
- "ID": "8d704cfd-05b2-4d4a-add2-e2868bcc081f"
- },
- {
- "Name": "eg-30",
- "RAM": 30000,
- "Ephemeral": 0,
- "VCPUs": 8,
- "Is Public": true,
- "Disk": 800,
- "ID": "901f77c2-73f6-4fae-b28a-18b829b55a17"
- },
- {
- "Name": "sp-60-ssd-flex",
- "RAM": 60000,
- "Ephemeral": 0,
- "VCPUs": 4,
- "Is Public": true,
- "Disk": 50,
- "ID": "944b92fb-9a0c-406d-bb9f-a1d93cda9f01"
- },
- {
- "Name": "eg-30-flex",
- "RAM": 30000,
- "Ephemeral": 0,
- "VCPUs": 8,
- "Is Public": true,
- "Disk": 50,
- "ID": "965472c7-eb54-4d4d-bd6e-01ebb694a631"
- },
- {
- "Name": "sp-120-ssd",
- "RAM": 120000,
- "Ephemeral": 0,
- "VCPUs": 8,
- "Is Public": true,
- "Disk": 400,
- "ID": "97824a8c-e683-49a8-a70a-ead64240395c"
- },
- {
- "Name": "hg-60-ssd",
- "RAM": 60000,
- "Ephemeral": 0,
- "VCPUs": 16,
- "Is Public": true,
- "Disk": 800,
- "ID": "9831d7f1-3e79-483d-8958-88e3952c7ea2"
- },
- {
- "Name": "eg-60",
- "RAM": 60000,
- "Ephemeral": 0,
- "VCPUs": 16,
- "Is Public": true,
- "Disk": 1600,
- "ID": "9e1f13d0-4fcc-4abc-a9e6-9c76d662c92d"
- },
- {
- "Name": "win-eg-30-ssd",
- "RAM": 30000,
- "Ephemeral": 0,
- "VCPUs": 8,
- "Is Public": true,
- "Disk": 400,
- "ID": "9e6b85fa-6f37-45ce-a3d6-11ab40a28fad"
- },
- {
- "Name": "hg-120-flex",
- "RAM": 120000,
- "Ephemeral": 0,
- "VCPUs": 32,
- "Is Public": true,
- "Disk": 50,
- "ID": "9ed787cc-a0db-400b-8cc1-49b6384a1000"
- },
- {
- "Name": "sp-120-flex",
- "RAM": 120000,
- "Ephemeral": 0,
- "VCPUs": 8,
- "Is Public": true,
- "Disk": 50,
- "ID": "9f3cfdf7-b850-47cc-92be-33aefbfd2b05"
- },
- {
- "Name": "hg-60-ssd-flex",
- "RAM": 60000,
- "Ephemeral": 0,
- "VCPUs": 16,
- "Is Public": true,
- "Disk": 50,
- "ID": "a37bdf17-e1b1-41cc-a67f-ed665a120446"
- },
- {
- "Name": "win-hg-120-ssd",
- "RAM": 120000,
- "Ephemeral": 0,
- "VCPUs": 32,
- "Is Public": true,
- "Disk": 800,
- "ID": "aa753e73-dadb-4528-9c4a-24e36fc41bf4"
- },
- {
- "Name": "win-sp-240-ssd-flex",
- "RAM": 240000,
- "Ephemeral": 0,
- "VCPUs": 16,
- "Is Public": true,
- "Disk": 50,
- "ID": "abc007b8-cc44-4b6b-9606-fd647b03e101"
- },
- {
- "Name": "sp-120",
- "RAM": 120000,
- "Ephemeral": 0,
- "VCPUs": 8,
- "Is Public": true,
- "Disk": 800,
- "ID": "ac74cb45-d895-47dd-b9cf-c17778033d83"
- },
- {
- "Name": "win-hg-15",
- "RAM": 15000,
- "Ephemeral": 0,
- "VCPUs": 4,
- "Is Public": true,
- "Disk": 400,
- "ID": "ae900175-72bd-4fbc-8ab2-4673b468aa5b"
- },
- {
- "Name": "win-eg-15-ssd-flex",
- "RAM": 15000,
- "Ephemeral": 0,
- "VCPUs": 4,
- "Is Public": true,
- "Disk": 50,
- "ID": "aeb37dbf-d7c9-4fd7-93f1-f3818e488ede"
- },
- {
- "Name": "hg-7-ssd",
- "RAM": 7000,
- "Ephemeral": 0,
- "VCPUs": 2,
- "Is Public": true,
- "Disk": 100,
- "ID": "b1dc776c-b6e3-4a96-b230-850f570db3d5"
- },
- {
- "Name": "sp-60-ssd",
- "RAM": 60000,
- "Ephemeral": 0,
- "VCPUs": 4,
- "Is Public": true,
- "Disk": 200,
- "ID": "b24df495-10f3-466e-95ab-26f0f6839a2f"
- },
- {
- "Name": "win-hg-120",
- "RAM": 120000,
- "Ephemeral": 0,
- "VCPUs": 32,
- "Is Public": true,
- "Disk": 1600,
- "ID": "b798e44e-bf71-488c-9335-f20bf5976547"
- },
- {
- "Name": "eg-7-ssd",
- "RAM": 7000,
- "Ephemeral": 0,
- "VCPUs": 2,
- "Is Public": true,
- "Disk": 100,
- "ID": "b94e6623-913d-4147-b2a3-34ccf6fe7a5e"
- },
- {
- "Name": "eg-15-flex",
- "RAM": 15000,
- "Ephemeral": 0,
- "VCPUs": 4,
- "Is Public": true,
- "Disk": 50,
- "ID": "bb5fdda8-34ec-40c8-a4e3-308b9e2c9ee2"
- },
- {
- "Name": "win-eg-7-flex",
- "RAM": 7000,
- "Ephemeral": 0,
- "VCPUs": 2,
- "Is Public": true,
- "Disk": 50,
- "ID": "c65384f6-4665-461a-a292-2f3f5a016244"
- },
- {
- "Name": "eg-60-ssd",
- "RAM": 60000,
- "Ephemeral": 0,
- "VCPUs": 16,
- "Is Public": true,
- "Disk": 800,
- "ID": "c678f1a8-6542-4f9d-89af-ffc98715d674"
- },
- {
- "Name": "hg-30-ssd-flex",
- "RAM": 30000,
- "Ephemeral": 0,
- "VCPUs": 8,
- "Is Public": true,
- "Disk": 50,
- "ID": "d147a094-b653-41e7-9250-8d4da3044334"
- },
- {
- "Name": "sp-30",
- "RAM": 30000,
- "Ephemeral": 0,
- "VCPUs": 2,
- "Is Public": true,
- "Disk": 200,
- "ID": "d1acf88d-6f55-4c5c-a914-4ecbdbd50d6b"
- },
- {
- "Name": "sp-120-ssd-flex",
- "RAM": 120000,
- "Ephemeral": 0,
- "VCPUs": 8,
- "Is Public": true,
- "Disk": 50,
- "ID": "d2d33e8e-58b1-4661-8141-826c47f82166"
- },
- {
- "Name": "hg-120-ssd-flex",
- "RAM": 120000,
- "Ephemeral": 0,
- "VCPUs": 32,
- "Is Public": true,
- "Disk": 50,
- "ID": "d7322c37-9881-4a57-9b40-2499fe2e8f42"
- },
- {
- "Name": "win-hg-15-flex",
- "RAM": 15000,
- "Ephemeral": 0,
- "VCPUs": 4,
- "Is Public": true,
- "Disk": 50,
- "ID": "daf597ea-fbbc-4c71-a35e-5b41d33ccc6c"
- },
- {
- "Name": "win-hg-30-ssd-flex",
- "RAM": 30000,
- "Ephemeral": 0,
- "VCPUs": 8,
- "Is Public": true,
- "Disk": 50,
- "ID": "dcfd834c-3932-47a3-8b4b-cdfeecdfde2c"
- },
- {
- "Name": "win-hg-60",
- "RAM": 60000,
- "Ephemeral": 0,
- "VCPUs": 16,
- "Is Public": true,
- "Disk": 1600,
- "ID": "def75cbd-a4b1-4f82-9152-90c65df9587b"
- },
- {
- "Name": "eg-30-ssd-flex",
- "RAM": 30000,
- "Ephemeral": 0,
- "VCPUs": 8,
- "Is Public": true,
- "Disk": 50,
- "ID": "e04c7ad6-a5de-45f5-93c9-f3343bdfe8d1"
- },
- {
- "Name": "vps-ssd-3",
- "RAM": 8000,
- "Ephemeral": 0,
- "VCPUs": 2,
- "Is Public": true,
- "Disk": 40,
- "ID": "e43d7458-6b82-4a78-a712-3a4dc6748cf4"
- },
- {
- "Name": "win-eg-15-flex",
- "RAM": 15000,
- "Ephemeral": 0,
- "VCPUs": 4,
- "Is Public": true,
- "Disk": 50,
- "ID": "e8bd3402-7310-4a0f-8b99-d9212359c957"
- },
- {
- "Name": "win-eg-30",
- "RAM": 30000,
- "Ephemeral": 0,
- "VCPUs": 8,
- "Is Public": true,
- "Disk": 800,
- "ID": "ebf7a997-e2f8-42f4-84f7-33a3d53d1af9"
- },
- {
- "Name": "eg-120-flex",
- "RAM": 120000,
- "Ephemeral": 0,
- "VCPUs": 32,
- "Is Public": true,
- "Disk": 50,
- "ID": "ec852ed3-1e42-4c59-abc3-12bcd26abec8"
- },
- {
- "Name": "sp-240",
- "RAM": 240000,
- "Ephemeral": 0,
- "VCPUs": 16,
- "Is Public": true,
- "Disk": 1600,
- "ID": "ed286e2c-769f-4c47-ac52-b8de7a4891f6"
- },
- {
- "Name": "win-sp-60-ssd",
- "RAM": 60000,
- "Ephemeral": 0,
- "VCPUs": 4,
- "Is Public": true,
- "Disk": 200,
- "ID": "ed835a73-d9a0-43ee-bd89-999c51d8426d"
- },
- {
- "Name": "win-eg-15",
- "RAM": 15000,
- "Ephemeral": 0,
- "VCPUs": 4,
- "Is Public": true,
- "Disk": 400,
- "ID": "f06056c1-a2d4-40e7-a7d8-e5bfabada72e"
- },
- {
- "Name": "win-sp-120",
- "RAM": 120000,
- "Ephemeral": 0,
- "VCPUs": 8,
- "Is Public": true,
- "Disk": 800,
- "ID": "f247dc56-395b-49de-9a62-93ccc4fff4ed"
- },
- {
- "Name": "eg-7-flex",
- "RAM": 7000,
- "Ephemeral": 0,
- "VCPUs": 2,
- "Is Public": true,
- "Disk": 50,
- "ID": "f476f959-ffa6-46f2-94d8-72293570604d"
- },
- {
- "Name": "sp-60-flex",
- "RAM": 60000,
- "Ephemeral": 0,
- "VCPUs": 4,
- "Is Public": true,
- "Disk": 50,
- "ID": "f52db47a-315f-49d4-bc5c-67dd118e7ac0"
- },
- {
- "Name": "win-hg-120-flex",
- "RAM": 120000,
- "Ephemeral": 0,
- "VCPUs": 32,
- "Is Public": true,
- "Disk": 50,
- "ID": "f6cb8144-5d98-4057-b44f-46da342fb571"
- },
- {
- "Name": "hg-7",
- "RAM": 7000,
- "Ephemeral": 0,
- "VCPUs": 2,
- "Is Public": true,
- "Disk": 200,
- "ID": "fa3cc551-0358-4170-be64-56ea432b064c"
- },
- {
- "Name": "hg-15-ssd",
- "RAM": 15000,
- "Ephemeral": 0,
- "VCPUs": 4,
- "Is Public": true,
- "Disk": 200,
- "ID": "ff48c2cf-c17f-4682-aaf6-31d66786f808"
- }
- ]"""
-
- @classmethod
- def setup_class(self):
- if 'OS_AUTH_URL' not in os.environ:
- pytest.skip('no OS_AUTH_URL environment variable')
-
- @patch('teuthology.misc.sh')
- def test_sorted_flavors(self, m_sh):
- o = OpenStack()
- select = '^(vps|hg)-.*ssd'
- m_sh.return_value = TestOpenStack.flavors
- flavors = o.get_sorted_flavors('arch', select)
- assert [u'vps-ssd-1',
- u'vps-ssd-2',
- u'hg-7-ssd-flex',
- u'hg-7-ssd',
- u'vps-ssd-3',
- u'hg-15-ssd-flex',
- u'hg-15-ssd',
- u'hg-30-ssd-flex',
- u'hg-30-ssd',
- u'hg-60-ssd-flex',
- u'hg-60-ssd',
- u'hg-120-ssd-flex',
- u'hg-120-ssd',
- ] == [ f['Name'] for f in flavors ]
- m_sh.assert_called_with("openstack --quiet flavor list -f json")
-
- def test_flavor(self):
- def get_sorted_flavors(self, arch, select):
- return [
- {
- 'Name': 'too_small',
- 'RAM': 2048,
- 'Disk': 50,
- 'VCPUs': 1,
- },
- ]
- with patch.multiple(
- OpenStack,
- get_sorted_flavors=get_sorted_flavors,
- ):
- with pytest.raises(NoFlavorException):
- hint = { 'ram': 1000, 'disk': 40, 'cpus': 2 }
- OpenStack().flavor(hint, 'arch')
-
- flavor = 'good-flavor'
- def get_sorted_flavors(self, arch, select):
- return [
- {
- 'Name': flavor,
- 'RAM': 2048,
- 'Disk': 50,
- 'VCPUs': 2,
- },
- ]
- with patch.multiple(
- OpenStack,
- get_sorted_flavors=get_sorted_flavors,
- ):
- hint = { 'ram': 1000, 'disk': 40, 'cpus': 2 }
- assert flavor == OpenStack().flavor(hint, 'arch')
-
- def test_flavor_range(self):
- flavors = [
- {
- 'Name': 'too_small',
- 'RAM': 2048,
- 'Disk': 50,
- 'VCPUs': 1,
- },
- ]
- def get_sorted_flavors(self, arch, select):
- return flavors
-
- min = { 'ram': 1000, 'disk': 40, 'cpus': 2 }
- good = { 'ram': 4000, 'disk': 40, 'cpus': 2 }
-
- #
- # there are no flavors in the required range
- #
- with patch.multiple(
- OpenStack,
- get_sorted_flavors=get_sorted_flavors,
- ):
- with pytest.raises(NoFlavorException):
- OpenStack().flavor_range(min, good, 'arch')
-
- #
- # there is one flavor in the required range
- #
- flavors.append({
- 'Name': 'min',
- 'RAM': 2048,
- 'Disk': 40,
- 'VCPUs': 2,
- })
-
- with patch.multiple(
- OpenStack,
- get_sorted_flavors=get_sorted_flavors,
- ):
-
- assert 'min' == OpenStack().flavor_range(min, good, 'arch')
-
- #
- # out of the two flavors in the required range, get the bigger one
- #
- flavors.append({
- 'Name': 'good',
- 'RAM': 3000,
- 'Disk': 40,
- 'VCPUs': 2,
- })
-
- with patch.multiple(
- OpenStack,
- get_sorted_flavors=get_sorted_flavors,
- ):
-
- assert 'good' == OpenStack().flavor_range(min, good, 'arch')
-
- #
- # there is one flavor bigger or equal to good, get this one
- #
- flavors.append({
- 'Name': 'best',
- 'RAM': 4000,
- 'Disk': 40,
- 'VCPUs': 2,
- })
-
- with patch.multiple(
- OpenStack,
- get_sorted_flavors=get_sorted_flavors,
- ):
-
- assert 'best' == OpenStack().flavor_range(min, good, 'arch')
-
- #
- # there are two flavors bigger or equal to good, get the smallest one
- #
- flavors.append({
- 'Name': 'too_big',
- 'RAM': 30000,
- 'Disk': 400,
- 'VCPUs': 20,
- })
-
- with patch.multiple(
- OpenStack,
- get_sorted_flavors=get_sorted_flavors,
- ):
-
- assert 'best' == OpenStack().flavor_range(min, good, 'arch')
-
-
- def test_interpret_hints(self):
- defaults = {
- 'machine': {
- 'ram': 0,
- 'disk': 0,
- 'cpus': 0,
- },
- 'volumes': {
- 'count': 0,
- 'size': 0,
- },
- }
- expected_disk = 10 # first hint larger than the second
- expected_ram = 20 # second hint larger than the first
- expected_cpus = 0 # not set, hence zero by default
- expected_count = 30 # second hint larger than the first
- expected_size = 40 # does not exist in the first hint
- hints = [
- {
- 'machine': {
- 'ram': 2,
- 'disk': expected_disk,
- },
- 'volumes': {
- 'count': 9,
- 'size': expected_size,
- },
- },
- {
- 'machine': {
- 'ram': expected_ram,
- 'disk': 3,
- },
- 'volumes': {
- 'count': expected_count,
- },
- },
- ]
- hint = OpenStack().interpret_hints(defaults, hints)
- assert hint == {
- 'machine': {
- 'ram': expected_ram,
- 'disk': expected_disk,
- 'cpus': expected_cpus,
- },
- 'volumes': {
- 'count': expected_count,
- 'size': expected_size,
- }
- }
- assert defaults == OpenStack().interpret_hints(defaults, None)
-
- def test_get_provider(self):
- auth = os.environ.get('OS_AUTH_URL', None)
- os.environ['OS_AUTH_URL'] = 'cloud.ovh.net'
- assert OpenStack().get_provider() == 'ovh'
- if auth != None:
- os.environ['OS_AUTH_URL'] = auth
- else:
- del os.environ['OS_AUTH_URL']
-
- def test_get_os_url(self):
- o = OpenStack()
- #
- # Only for OVH
- #
- o.provider = 'something'
- assert "" == o.get_os_url("server ")
- o.provider = 'ovh'
- assert "" == o.get_os_url("unknown ")
- type2cmd = {
- 'compute': ('server', 'flavor'),
- 'network': ('ip', 'security', 'network'),
- 'image': ('image',),
- 'volume': ('volume',),
- }
- os.environ['OS_REGION_NAME'] = 'REGION'
- os.environ['OS_TENANT_ID'] = 'TENANT'
- for (type, cmds) in type2cmd.items():
- for cmd in cmds:
- assert ("//" + type) in o.get_os_url(cmd + " ")
- for type in type2cmd.keys():
- assert ("//" + type) in o.get_os_url("whatever ", type=type)
-
- @patch('teuthology.misc.sh')
- def test_cache_token(self, m_sh):
- token = 'TOKEN VALUE'
- m_sh.return_value = token
- OpenStack.token = None
- o = OpenStack()
- #
- # Only for OVH
- #
- o.provider = 'something'
- assert False == o.cache_token()
- o.provider = 'ovh'
- #
- # Set the environment with the token
- #
- assert 'OS_TOKEN_VALUE' not in os.environ
- assert 'OS_TOKEN_EXPIRES' not in os.environ
- assert True == o.cache_token()
- m_sh.assert_called_with('openstack -q token issue -c id -f value')
- assert token == os.environ['OS_TOKEN_VALUE']
- assert token == OpenStack.token
- assert time.time() < int(os.environ['OS_TOKEN_EXPIRES'])
- assert time.time() < OpenStack.token_expires
- #
- # Reset after it expires
- #
- token_expires = int(time.time()) - 2000
- OpenStack.token_expires = token_expires
- assert True == o.cache_token()
- assert time.time() < int(os.environ['OS_TOKEN_EXPIRES'])
- assert time.time() < OpenStack.token_expires
-
- @patch('teuthology.misc.sh')
- def test_cache_token_from_environment(self, m_sh):
- OpenStack.token = None
- o = OpenStack()
- o.provider = 'ovh'
- token = 'TOKEN VALUE'
- os.environ['OS_TOKEN_VALUE'] = token
- token_expires = int(time.time()) + OpenStack.token_cache_duration
- os.environ['OS_TOKEN_EXPIRES'] = str(token_expires)
- assert True == o.cache_token()
- assert token == OpenStack.token
- assert token_expires == OpenStack.token_expires
- m_sh.assert_not_called()
-
- @patch('teuthology.misc.sh')
- def test_cache_token_expired_environment(self, m_sh):
- token = 'TOKEN VALUE'
- m_sh.return_value = token
- OpenStack.token = None
- o = OpenStack()
- o.provider = 'ovh'
- os.environ['OS_TOKEN_VALUE'] = token
- token_expires = int(time.time()) - 2000
- os.environ['OS_TOKEN_EXPIRES'] = str(token_expires)
- assert True == o.cache_token()
- m_sh.assert_called_with('openstack -q token issue -c id -f value')
- assert token == os.environ['OS_TOKEN_VALUE']
- assert token == OpenStack.token
- assert time.time() < int(os.environ['OS_TOKEN_EXPIRES'])
- assert time.time() < OpenStack.token_expires
-
-class TestTeuthologyOpenStack(TestOpenStackBase):
-
- @classmethod
- def setup_class(self):
- if 'OS_AUTH_URL' not in os.environ:
- pytest.skip('no OS_AUTH_URL environment variable')
-
- teuthology.log.setLevel(logging.DEBUG)
- set_config_attr(argparse.Namespace())
-
- ip = TeuthologyOpenStack.create_floating_ip()
- if ip:
- ip_id = TeuthologyOpenStack.get_floating_ip_id(ip)
- OpenStack().run("ip floating delete " + ip_id)
- self.can_create_floating_ips = True
- else:
- self.can_create_floating_ips = False
-
- def setup_method(self):
- super(TestTeuthologyOpenStack, self).setup_method()
- self.key_filename = tempfile.mktemp()
- self.key_name = 'teuthology-test'
- self.name = 'teuthology-test'
- self.clobber()
- misc.sh("""
-openstack keypair create {key_name} > {key_filename}
-chmod 600 {key_filename}
- """.format(key_filename=self.key_filename,
- key_name=self.key_name))
- self.options = ['--key-name', self.key_name,
- '--key-filename', self.key_filename,
- '--name', self.name,
- '--verbose']
-
- def teardown_method(self):
- super(TestTeuthologyOpenStack, self).teardown_method()
- self.clobber()
- os.unlink(self.key_filename)
-
- def clobber(self):
- misc.sh("""
-openstack server delete {name} --wait || true
-openstack keypair delete {key_name} || true
- """.format(key_name=self.key_name,
- name=self.name))
-
- def test_create(self, caplog):
- teuthology_argv = [
- '--suite', 'upgrade/hammer',
- '--dry-run',
- '--ceph', 'main',
- '--kernel', 'distro',
- '--flavor', 'gcov',
- '--distro', 'ubuntu',
- '--suite-branch', 'hammer',
- '--email', 'loic@dachary.org',
- '--num', '10',
- '--limit', '23',
- '--subset', '1/2',
- '--priority', '101',
- '--timeout', '234',
- '--filter', 'trasher',
- '--filter-out', 'erasure-code',
- '--throttle', '3',
- ]
- archive_upload = 'user@archive:/tmp'
- argv = (self.options +
- ['--teuthology-git-url', 'TEUTHOLOGY_URL',
- '--teuthology-branch', 'TEUTHOLOGY_BRANCH',
- '--ceph-workbench-git-url', 'CEPH_WORKBENCH_URL',
- '--ceph-workbench-branch', 'CEPH_WORKBENCH_BRANCH',
- '--upload',
- '--archive-upload', archive_upload] +
- teuthology_argv)
- args = scripts.openstack.parse_args(argv)
- teuthology_argv.extend([
- '--archive-upload', archive_upload,
- '--archive-upload-url', args.archive_upload_url,
- ])
- teuthology = TeuthologyOpenStack(args, None, argv)
- teuthology.user_data = 'teuthology/openstack/test/user-data-test1.txt'
- teuthology.teuthology_suite = 'echo --'
-
- teuthology.main()
- assert 0 == teuthology.ssh("lsb_release -a")
- assert 0 == teuthology.ssh("grep 'substituded variables' /var/log/cloud-init.log")
- l = caplog.text
- assert 'Ubuntu 14.04' in l
- assert "nworkers=" + str(args.simultaneous_jobs) in l
- assert "username=" + teuthology.username in l
- assert "upload=--archive-upload user@archive:/tmp" in l
- assert ("ceph_workbench="
- " --ceph-workbench-branch CEPH_WORKBENCH_BRANCH"
- " --ceph-workbench-git-url CEPH_WORKBENCH_URL") in l
- assert "clone=git clone -b TEUTHOLOGY_BRANCH TEUTHOLOGY_URL" in l
- assert os.environ['OS_AUTH_URL'] in l
- assert " ".join(teuthology_argv) in l
-
- if self.can_create_floating_ips:
- ip = teuthology.get_floating_ip(self.name)
- teuthology.teardown()
- if self.can_create_floating_ips:
- assert teuthology.get_floating_ip_id(ip) == None
-
- def test_floating_ip(self):
- if not self.can_create_floating_ips:
- pytest.skip('unable to create floating ips')
-
- expected = TeuthologyOpenStack.create_floating_ip()
- ip = TeuthologyOpenStack.get_unassociated_floating_ip()
- assert expected == ip
- ip_id = TeuthologyOpenStack.get_floating_ip_id(ip)
- OpenStack().run("ip floating delete " + ip_id)
+++ /dev/null
-#cloud-config
-system_info:
- default_user:
- name: ubuntu
-final_message: "teuthology is up and running after $UPTIME seconds, substituded variables nworkers=NWORKERS openrc=OPENRC username=TEUTHOLOGY_USERNAME upload=UPLOAD ceph_workbench=CEPH_WORKBENCH clone=CLONE_OPENSTACK"
+++ /dev/null
-ceph 658 1 0 Jun08 ? 00:07:43 /usr/bin/ceph-mgr -f --cluster ceph --id host1 --setuser ceph --setgroup ceph
-ceph 1634 1 0 Jun08 ? 00:02:17 /usr/bin/ceph-mds -f --cluster ceph --id host1 --setuser ceph --setgroup ceph
-ceph 31555 1 0 Jun08 ? 01:13:50 /usr/bin/ceph-mon -f --cluster ceph --id host1 --setuser ceph --setgroup ceph
-ceph 31765 1 0 Jun08 ? 00:48:42 /usr/bin/radosgw -f --cluster ceph --name client.rgw.host1.rgw0 --setuser ceph --setgroup ceph
-ceph 97427 1 0 Jun17 ? 00:41:39 /usr/bin/ceph-osd -f --cluster ceph --id 0 --setuser ceph --setgroup ceph
\ No newline at end of file
+++ /dev/null
-from teuthology.orchestra import monkey
-monkey.patch_all()
-
-from io import StringIO
-
-import os
-from teuthology.orchestra import connection, remote, run
-from teuthology.orchestra.test.util import assert_raises
-from teuthology.exceptions import CommandCrashedError, ConnectionLostError
-
-from pytest import skip
-
-HOST = None
-
-
-class TestIntegration():
- def setup_method(self):
- try:
- host = os.environ['ORCHESTRA_TEST_HOST']
- except KeyError:
- skip('To run integration tests, set environment ' +
- 'variable ORCHESTRA_TEST_HOST to user@host to use.')
- global HOST
- HOST = host
-
- def test_crash(self):
- ssh = connection.connect(HOST)
- e = assert_raises(
- CommandCrashedError,
- run.run,
- client=ssh,
- args=['sh', '-c', 'kill -ABRT $$'],
- )
- assert e.command == "sh -c 'kill -ABRT $$'"
- assert str(e) == "Command crashed: \"sh -c 'kill -ABRT $$'\""
-
- def test_lost(self):
- ssh = connection.connect(HOST)
- e = assert_raises(
- ConnectionLostError,
- run.run,
- client=ssh,
- args=['sh', '-c', 'kill -ABRT $PPID'],
- name=HOST,
- )
- assert e.command == "sh -c 'kill -ABRT $PPID'"
- assert str(e) == \
- "SSH connection to {host} was lost: ".format(host=HOST) + \
- "\"sh -c 'kill -ABRT $PPID'\""
-
- def test_pipe(self):
- ssh = connection.connect(HOST)
- r = run.run(
- client=ssh,
- args=['cat'],
- stdin=run.PIPE,
- stdout=StringIO(),
- wait=False,
- )
- assert r.stdout.getvalue() == ''
- r.stdin.write('foo\n')
- r.stdin.write('bar\n')
- r.stdin.close()
-
- r.wait()
- got = r.exitstatus
- assert got == 0
- assert r.stdout.getvalue() == 'foo\nbar\n'
-
- def test_and(self):
- ssh = connection.connect(HOST)
- r = run.run(
- client=ssh,
- args=['true', run.Raw('&&'), 'echo', 'yup'],
- stdout=StringIO(),
- )
- assert r.stdout.getvalue() == 'yup\n'
-
- def test_os(self):
- rem = remote.Remote(HOST)
- assert rem.os.name
- assert rem.os.version
-
- def test_17102(self, caplog):
- # http://tracker.ceph.com/issues/17102
- rem = remote.Remote(HOST)
- interval = 3
- rem.run(args="echo before; sleep %s; echo after" % interval)
- for record in caplog.records:
- if record.msg == 'before':
- before_time = record.created
- elif record.msg == 'after':
- after_time = record.created
- assert int(round(after_time - before_time)) == interval
+++ /dev/null
-import pytest
-
-from mock import patch, Mock
-
-from teuthology.orchestra import cluster, remote, run
-
-
-class TestCluster(object):
- def test_init_empty(self):
- c = cluster.Cluster()
- assert c.remotes == {}
-
- def test_init(self):
- r1 = Mock()
- r2 = Mock()
- c = cluster.Cluster(
- remotes=[
- (r1, ['foo', 'bar']),
- (r2, ['baz']),
- ],
- )
- r3 = Mock()
- c.add(r3, ['xyzzy', 'thud', 'foo'])
- assert c.remotes == {
- r1: ['foo', 'bar'],
- r2: ['baz'],
- r3: ['xyzzy', 'thud', 'foo'],
- }
-
- def test_repr(self):
- r1 = remote.Remote('r1', ssh=Mock())
- r2 = remote.Remote('r2', ssh=Mock())
- c = cluster.Cluster(
- remotes=[
- (r1, ['foo', 'bar']),
- (r2, ['baz']),
- ],
- )
- assert repr(c) == \
- "Cluster(remotes=[[Remote(name='r1'), ['foo', 'bar']], " \
- "[Remote(name='r2'), ['baz']]])"
-
- def test_str(self):
- r1 = remote.Remote('r1', ssh=Mock())
- r2 = remote.Remote('r2', ssh=Mock())
- c = cluster.Cluster(
- remotes=[
- (r1, ['foo', 'bar']),
- (r2, ['baz']),
- ],
- )
- assert str(c) == "r1[foo,bar] r2[baz]"
-
- def test_run_all(self):
- r1 = Mock(spec=remote.Remote)
- r1.configure_mock(name='r1')
- ret1 = Mock(spec=run.RemoteProcess)
- r1.run.return_value = ret1
- r2 = Mock(spec=remote.Remote)
- r2.configure_mock(name='r2')
- ret2 = Mock(spec=run.RemoteProcess)
- r2.run.return_value = ret2
- c = cluster.Cluster(
- remotes=[
- (r1, ['foo', 'bar']),
- (r2, ['baz']),
- ],
- )
- got = c.run(args=['test'])
- r1.run.assert_called_once_with(args=['test'], wait=True)
- r2.run.assert_called_once_with(args=['test'], wait=True)
- assert len(got) == 2
- assert got, [ret1 == ret2]
- # check identity not equality
- assert got[0] is ret1
- assert got[1] is ret2
-
- def test_only_one(self):
- r1 = Mock()
- r2 = Mock()
- r3 = Mock()
- c = cluster.Cluster(
- remotes=[
- (r1, ['foo', 'bar']),
- (r2, ['bar']),
- (r3, ['foo']),
- ],
- )
- c_foo = c.only('foo')
- assert c_foo.remotes == {r1: ['foo', 'bar'], r3: ['foo']}
-
- def test_only_two(self):
- r1 = Mock()
- r2 = Mock()
- r3 = Mock()
- c = cluster.Cluster(
- remotes=[
- (r1, ['foo', 'bar']),
- (r2, ['bar']),
- (r3, ['foo']),
- ],
- )
- c_both = c.only('foo', 'bar')
- assert c_both.remotes, {r1: ['foo' == 'bar']}
-
- def test_only_none(self):
- r1 = Mock()
- r2 = Mock()
- r3 = Mock()
- c = cluster.Cluster(
- remotes=[
- (r1, ['foo', 'bar']),
- (r2, ['bar']),
- (r3, ['foo']),
- ],
- )
- c_none = c.only('impossible')
- assert c_none.remotes == {}
-
- def test_only_match(self):
- r1 = Mock()
- r2 = Mock()
- r3 = Mock()
- c = cluster.Cluster(
- remotes=[
- (r1, ['foo', 'bar']),
- (r2, ['bar']),
- (r3, ['foo']),
- ],
- )
- c_foo = c.only('foo', lambda role: role.startswith('b'))
- assert c_foo.remotes, {r1: ['foo' == 'bar']}
-
- def test_exclude_one(self):
- r1 = Mock()
- r2 = Mock()
- r3 = Mock()
- c = cluster.Cluster(
- remotes=[
- (r1, ['foo', 'bar']),
- (r2, ['bar']),
- (r3, ['foo']),
- ],
- )
- c_foo = c.exclude('foo')
- assert c_foo.remotes == {r2: ['bar']}
-
- def test_exclude_two(self):
- r1 = Mock()
- r2 = Mock()
- r3 = Mock()
- c = cluster.Cluster(
- remotes=[
- (r1, ['foo', 'bar']),
- (r2, ['bar']),
- (r3, ['foo']),
- ],
- )
- c_both = c.exclude('foo', 'bar')
- assert c_both.remotes == {r2: ['bar'], r3: ['foo']}
-
- def test_exclude_none(self):
- r1 = Mock()
- r2 = Mock()
- r3 = Mock()
- c = cluster.Cluster(
- remotes=[
- (r1, ['foo', 'bar']),
- (r2, ['bar']),
- (r3, ['foo']),
- ],
- )
- c_none = c.exclude('impossible')
- assert c_none.remotes == {r1: ['foo', 'bar'], r2: ['bar'], r3: ['foo']}
-
- def test_exclude_match(self):
- r1 = Mock()
- r2 = Mock()
- r3 = Mock()
- c = cluster.Cluster(
- remotes=[
- (r1, ['foo', 'bar']),
- (r2, ['bar']),
- (r3, ['foo']),
- ],
- )
- c_foo = c.exclude('foo', lambda role: role.startswith('b'))
- assert c_foo.remotes == {r2: ['bar'], r3: ['foo']}
-
- def test_filter(self):
- r1 = Mock(_name='r1')
- r2 = Mock(_name='r2')
- def func(r):
- return r._name == "r1"
- c = cluster.Cluster(remotes=[
- (r1, ['foo']),
- (r2, ['bar']),
- ])
- assert c.filter(func).remotes == {
- r1: ['foo']
- }
-
-
-class TestWriteFile(object):
- """ Tests for cluster.write_file """
- def setup_method(self):
- self.r1 = remote.Remote('r1', ssh=Mock())
- self.c = cluster.Cluster(
- remotes=[
- (self.r1, ['foo', 'bar']),
- ],
- )
-
- @patch("teuthology.orchestra.remote.RemoteShell.write_file")
- def test_write_file(self, m_write_file):
- self.c.write_file("filename", "content")
- m_write_file.assert_called_with("filename", "content")
-
- @patch("teuthology.orchestra.remote.RemoteShell.write_file")
- def test_fails_with_invalid_perms(self, m_write_file):
- with pytest.raises(ValueError):
- self.c.write_file("filename", "content", sudo=False, perms="invalid")
-
- @patch("teuthology.orchestra.remote.RemoteShell.write_file")
- def test_fails_with_invalid_owner(self, m_write_file):
- with pytest.raises(ValueError):
- self.c.write_file("filename", "content", sudo=False, owner="invalid")
-
- @patch("teuthology.orchestra.remote.RemoteShell.write_file")
- def test_with_sudo(self, m_write_file):
- self.c.write_file("filename", "content", sudo=True)
- m_write_file.assert_called_with("filename", "content", sudo=True, owner=None, mode=None)
+++ /dev/null
-from mock import patch, Mock
-
-from teuthology import config
-from teuthology.orchestra import connection
-from teuthology.orchestra.test.util import assert_raises
-
-
-class TestConnection(object):
- def setup_method(self):
- self.start_patchers()
-
- def teardown_method(self):
- self.stop_patchers()
-
- def start_patchers(self):
- self.patcher_sleep = patch(
- 'time.sleep',
- )
- self.patcher_sleep.start()
- self.m_ssh = Mock()
- self.patcher_ssh = patch(
- 'teuthology.orchestra.connection.paramiko.SSHClient',
- self.m_ssh,
- )
- self.patcher_ssh.start()
-
- def stop_patchers(self):
- self.patcher_ssh.stop()
- self.patcher_sleep.stop()
-
- def clear_config(self):
- config.config.teuthology_yaml = ''
- config.config.load()
- config.config.ssh_key = None
-
- def test_split_user_just_host(self):
- got = connection.split_user('somehost.example.com')
- assert got == (None, 'somehost.example.com')
-
- def test_split_user_both(self):
- got = connection.split_user('jdoe@somehost.example.com')
- assert got == ('jdoe', 'somehost.example.com')
-
- def test_split_user_empty_user(self):
- s = '@somehost.example.com'
- e = assert_raises(AssertionError, connection.split_user, s)
- assert str(e) == 'Bad input to split_user: {s!r}'.format(s=s)
-
- def test_connect(self):
- self.clear_config()
- config.config.verify_host_keys = True
- m_ssh_instance = self.m_ssh.return_value = Mock();
- m_transport = Mock()
- m_ssh_instance.get_transport.return_value = m_transport
- got = connection.connect(
- 'jdoe@orchestra.test.newdream.net.invalid',
- _SSHClient=self.m_ssh,
- )
- self.m_ssh.assert_called_once()
- m_ssh_instance.set_missing_host_key_policy.assert_called_once()
- m_ssh_instance.load_system_host_keys.assert_called_once_with()
- m_ssh_instance.connect.assert_called_once_with(
- hostname='orchestra.test.newdream.net.invalid',
- username='jdoe',
- timeout=60,
- )
- m_transport.set_keepalive.assert_called_once_with(False)
- assert got is m_ssh_instance
-
- def test_connect_no_verify_host_keys(self):
- self.clear_config()
- config.config.verify_host_keys = False
- m_ssh_instance = self.m_ssh.return_value = Mock();
- m_transport = Mock()
- m_ssh_instance.get_transport.return_value = m_transport
- got = connection.connect(
- 'jdoe@orchestra.test.newdream.net.invalid',
- _SSHClient=self.m_ssh,
- )
- self.m_ssh.assert_called_once()
- m_ssh_instance.set_missing_host_key_policy.assert_called_once()
- assert not m_ssh_instance.load_system_host_keys.called
- m_ssh_instance.connect.assert_called_once_with(
- hostname='orchestra.test.newdream.net.invalid',
- username='jdoe',
- timeout=60,
- )
- m_transport.set_keepalive.assert_called_once_with(False)
- assert got is m_ssh_instance
-
- def test_connect_override_hostkeys(self):
- self.clear_config()
- m_ssh_instance = self.m_ssh.return_value = Mock();
- m_transport = Mock()
- m_ssh_instance.get_transport.return_value = m_transport
- m_host_keys = Mock()
- m_ssh_instance.get_host_keys.return_value = m_host_keys
- m_create_key = Mock()
- m_create_key.return_value = "frobnitz"
- got = connection.connect(
- 'jdoe@orchestra.test.newdream.net.invalid',
- host_key='ssh-rsa testkey',
- _SSHClient=self.m_ssh,
- _create_key=m_create_key,
- )
- self.m_ssh.assert_called_once()
- m_ssh_instance.get_host_keys.assert_called_once()
- m_host_keys.add.assert_called_once_with(
- hostname='orchestra.test.newdream.net.invalid',
- keytype='ssh-rsa',
- key='frobnitz',
- )
- m_create_key.assert_called_once_with('ssh-rsa', 'testkey')
- m_ssh_instance.connect.assert_called_once_with(
- hostname='orchestra.test.newdream.net.invalid',
- username='jdoe',
- timeout=60,
- )
- m_transport.set_keepalive.assert_called_once_with(False)
- assert got is m_ssh_instance
+++ /dev/null
-from mock import patch
-
-from teuthology.config import config as teuth_config
-
-from teuthology.orchestra import console
-
-
-class TestConsole(object):
- pass
-
-
-class TestPhysicalConsole(TestConsole):
- klass = console.PhysicalConsole
- ipmi_cmd_templ = 'ipmitool -H {h}.{d} -I lanplus -U {u} -P {p} {c}'
- conserver_cmd_templ = 'console -M {m} -p {p} {mode} {h}'
-
- def setup_method(self):
- self.hostname = 'host'
- teuth_config.ipmi_domain = 'ipmi_domain'
- teuth_config.ipmi_user = 'ipmi_user'
- teuth_config.ipmi_password = 'ipmi_pass'
- teuth_config.conserver_master = 'conserver_master'
- teuth_config.conserver_port = 3109
- teuth_config.use_conserver = True
-
- def test_has_ipmi_creds(self):
- cons = self.klass(self.hostname)
- assert cons.has_ipmi_credentials is True
- teuth_config.ipmi_domain = None
- cons = self.klass(self.hostname)
- assert cons.has_ipmi_credentials is False
-
- def test_console_command_conserver(self):
- cons = self.klass(
- self.hostname,
- teuth_config.ipmi_user,
- teuth_config.ipmi_password,
- teuth_config.ipmi_domain,
- )
- cons.has_conserver = True
- console_cmd = cons._console_command()
- assert console_cmd == self.conserver_cmd_templ.format(
- m=teuth_config.conserver_master,
- p=teuth_config.conserver_port,
- mode='-s',
- h=self.hostname,
- )
- console_cmd = cons._console_command(readonly=False)
- assert console_cmd == self.conserver_cmd_templ.format(
- m=teuth_config.conserver_master,
- p=teuth_config.conserver_port,
- mode='-f',
- h=self.hostname,
- )
-
- def test_console_command_ipmi(self):
- teuth_config.conserver_master = None
- cons = self.klass(
- self.hostname,
- teuth_config.ipmi_user,
- teuth_config.ipmi_password,
- teuth_config.ipmi_domain,
- )
- sol_cmd = cons._console_command()
- assert sol_cmd == self.ipmi_cmd_templ.format(
- h=self.hostname,
- d=teuth_config.ipmi_domain,
- u=teuth_config.ipmi_user,
- p=teuth_config.ipmi_password,
- c='sol activate',
- )
-
- def test_ipmi_command_ipmi(self):
- cons = self.klass(
- self.hostname,
- teuth_config.ipmi_user,
- teuth_config.ipmi_password,
- teuth_config.ipmi_domain,
- )
- pc_cmd = cons._ipmi_command('power cycle')
- assert pc_cmd == self.ipmi_cmd_templ.format(
- h=self.hostname,
- d=teuth_config.ipmi_domain,
- u=teuth_config.ipmi_user,
- p=teuth_config.ipmi_password,
- c='power cycle',
- )
-
- def test_spawn_log_conserver(self):
- with patch(
- 'teuthology.orchestra.console.psutil.subprocess.Popen',
- autospec=True,
- ) as m_popen:
- m_popen.return_value.pid = 42
- m_popen.return_value.returncode = 0
- m_popen.return_value.wait.return_value = 0
- cons = self.klass(self.hostname)
- assert cons.has_conserver is True
- m_popen.reset_mock()
- m_popen.return_value.poll.return_value = None
- cons.spawn_sol_log('/fake/path')
- assert m_popen.call_count == 1
- call_args = m_popen.call_args_list[0][0][0]
- assert any(
- [teuth_config.conserver_master in arg for arg in call_args]
- )
-
- def test_spawn_log_ipmi(self):
- with patch(
- 'teuthology.orchestra.console.psutil.subprocess.Popen',
- autospec=True,
- ) as m_popen:
- m_popen.return_value.pid = 42
- m_popen.return_value.returncode = 1
- m_popen.return_value.wait.return_value = 1
- cons = self.klass(self.hostname)
- assert cons.has_conserver is False
- m_popen.reset_mock()
- m_popen.return_value.poll.return_value = 1
- cons.spawn_sol_log('/fake/path')
- assert m_popen.call_count == 1
- call_args = m_popen.call_args_list[0][0][0]
- assert any(
- ['ipmitool' in arg for arg in call_args]
- )
-
- def test_spawn_log_fallback(self):
- with patch(
- 'teuthology.orchestra.console.psutil.subprocess.Popen',
- autospec=True,
- ) as m_popen:
- m_popen.return_value.pid = 42
- m_popen.return_value.returncode = 0
- m_popen.return_value.wait.return_value = 0
- cons = self.klass(self.hostname)
- assert cons.has_conserver is True
- m_popen.reset_mock()
- m_popen.return_value.poll.return_value = 1
- cons.spawn_sol_log('/fake/path')
- assert cons.has_conserver is False
- assert m_popen.call_count == 2
- call_args = m_popen.call_args_list[1][0][0]
- assert any(
- ['ipmitool' in arg for arg in call_args]
- )
-
- def test_get_console_conserver(self):
- with patch(
- 'teuthology.orchestra.console.psutil.subprocess.Popen',
- autospec=True,
- ) as m_popen:
- m_popen.return_value.pid = 42
- m_popen.return_value.returncode = 0
- m_popen.return_value.wait.return_value = 0
- cons = self.klass(self.hostname)
- assert cons.has_conserver is True
- with patch(
- 'teuthology.orchestra.console.pexpect.spawn',
- autospec=True,
- ) as m_spawn:
- cons._get_console()
- assert m_spawn.call_count == 1
- assert teuth_config.conserver_master in \
- m_spawn.call_args_list[0][0][0]
-
- def test_get_console_ipmitool(self):
- with patch(
- 'teuthology.orchestra.console.psutil.subprocess.Popen',
- autospec=True,
- ) as m_popen:
- m_popen.return_value.pid = 42
- m_popen.return_value.returncode = 0
- m_popen.return_value.wait.return_value = 0
- cons = self.klass(self.hostname)
- assert cons.has_conserver is True
- with patch(
- 'teuthology.orchestra.console.pexpect.spawn',
- autospec=True,
- ) as m_spawn:
- cons.has_conserver = False
- cons._get_console()
- assert m_spawn.call_count == 1
- assert 'ipmitool' in m_spawn.call_args_list[0][0][0]
-
- def test_get_console_fallback(self):
- with patch(
- 'teuthology.orchestra.console.psutil.subprocess.Popen',
- autospec=True,
- ) as m_popen:
- m_popen.return_value.pid = 42
- m_popen.return_value.returncode = 0
- m_popen.return_value.wait.return_value = 0
- cons = self.klass(self.hostname)
- assert cons.has_conserver is True
- with patch(
- 'teuthology.orchestra.console.pexpect.spawn',
- autospec=True,
- ) as m_spawn:
- cons.has_conserver = True
- m_spawn.return_value.isalive.return_value = False
- cons._get_console()
- assert m_spawn.return_value.isalive.call_count == 1
- assert m_spawn.call_count == 2
- assert cons.has_conserver is False
- assert 'ipmitool' in m_spawn.call_args_list[1][0][0]
-
- def test_disable_conserver(self):
- with patch(
- 'teuthology.orchestra.console.psutil.subprocess.Popen',
- autospec=True,
- ) as m_popen:
- m_popen.return_value.pid = 42
- m_popen.return_value.returncode = 0
- m_popen.return_value.wait.return_value = 0
- teuth_config.use_conserver = False
- cons = self.klass(self.hostname)
- assert cons.has_conserver is False
+++ /dev/null
-from textwrap import dedent
-from teuthology.orchestra.opsys import OS
-import pytest
-
-
-class TestOS(object):
- str_centos_9_os_release = dedent("""
- NAME="CentOS Stream"
- VERSION="9"
- ID="centos"
- ID_LIKE="rhel fedora"
- VERSION_ID="9"
- PLATFORM_ID="platform:el9"
- PRETTY_NAME="CentOS Stream 9"
- ANSI_COLOR="0;31"
- LOGO="fedora-logo-icon"
- CPE_NAME="cpe:/o:centos:centos:9"
- HOME_URL="https://centos.org/"
- BUG_REPORT_URL="https://issues.redhat.com/"
- REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9"
- REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream"
- """)
-
- str_centos_7_os_release = dedent("""
- NAME="CentOS Linux"
- VERSION="7 (Core)"
- ID="centos"
- ID_LIKE="rhel fedora"
- VERSION_ID="7"
- PRETTY_NAME="CentOS Linux 7 (Core)"
- ANSI_COLOR="0;31"
- CPE_NAME="cpe:/o:centos:centos:7"
- HOME_URL="https://www.centos.org/"
- BUG_REPORT_URL="https://bugs.centos.org/"
- """)
-
- str_centos_7_os_release_newer = dedent("""
- NAME="CentOS Linux"
- VERSION="7 (Core)"
- ID="centos"
- ID_LIKE="rhel fedora"
- VERSION_ID="7"
- PRETTY_NAME="CentOS Linux 7 (Core)"
- ANSI_COLOR="0;31"
- CPE_NAME="cpe:/o:centos:centos:7"
- HOME_URL="https://www.centos.org/"
- BUG_REPORT_URL="https://bugs.centos.org/"
-
- CENTOS_MANTISBT_PROJECT="CentOS-7"
- CENTOS_MANTISBT_PROJECT_VERSION="7"
- REDHAT_SUPPORT_PRODUCT="centos"
- REDHAT_SUPPORT_PRODUCT_VERSION="7"
- """)
-
- str_debian_7_lsb_release = dedent("""
- Distributor ID: Debian
- Description: Debian GNU/Linux 7.1 (wheezy)
- Release: 7.1
- Codename: wheezy
- """)
-
- str_debian_7_os_release = dedent("""
- PRETTY_NAME="Debian GNU/Linux 7 (wheezy)"
- NAME="Debian GNU/Linux"
- VERSION_ID="7"
- VERSION="7 (wheezy)"
- ID=debian
- ANSI_COLOR="1;31"
- HOME_URL="http://www.debian.org/"
- SUPPORT_URL="http://www.debian.org/support/"
- BUG_REPORT_URL="http://bugs.debian.org/"
- """)
-
- str_debian_8_os_release = dedent("""
- PRETTY_NAME="Debian GNU/Linux 8 (jessie)"
- NAME="Debian GNU/Linux"
- VERSION_ID="8"
- VERSION="8 (jessie)"
- ID=debian
- HOME_URL="http://www.debian.org/"
- SUPPORT_URL="http://www.debian.org/support/"
- BUG_REPORT_URL="https://bugs.debian.org/"
- """)
-
- str_debian_9_os_release = dedent("""
- PRETTY_NAME="Debian GNU/Linux 9 (stretch)"
- NAME="Debian GNU/Linux"
- VERSION_ID="9"
- VERSION="9 (stretch)"
- ID=debian
- HOME_URL="https://www.debian.org/"
- SUPPORT_URL="https://www.debian.org/support"
- BUG_REPORT_URL="https://bugs.debian.org/"
- """)
-
- str_ubuntu_12_04_lsb_release = dedent("""
- Distributor ID: Ubuntu
- Description: Ubuntu 12.04.4 LTS
- Release: 12.04
- Codename: precise
- """)
-
- str_ubuntu_12_04_os_release = dedent("""
- NAME="Ubuntu"
- VERSION="12.04.4 LTS, Precise Pangolin"
- ID=ubuntu
- ID_LIKE=debian
- PRETTY_NAME="Ubuntu precise (12.04.4 LTS)"
- VERSION_ID="12.04"
- """)
-
- str_ubuntu_14_04_os_release = dedent("""
- NAME="Ubuntu"
- VERSION="14.04.4 LTS, Trusty Tahr"
- ID=ubuntu
- ID_LIKE=debian
- PRETTY_NAME="Ubuntu 14.04.4 LTS"
- VERSION_ID="14.04"
- HOME_URL="http://www.ubuntu.com/"
- SUPPORT_URL="http://help.ubuntu.com/"
- BUG_REPORT_URL="http://bugs.launchpad.net/ubuntu/"
- """)
-
- str_ubuntu_16_04_os_release = dedent("""
- NAME="Ubuntu"
- VERSION="16.04 LTS (Xenial Xerus)"
- ID=ubuntu
- ID_LIKE=debian
- PRETTY_NAME="Ubuntu 16.04 LTS"
- VERSION_ID="16.04"
- HOME_URL="http://www.ubuntu.com/"
- SUPPORT_URL="http://help.ubuntu.com/"
- BUG_REPORT_URL="http://bugs.launchpad.net/ubuntu/"
- UBUNTU_CODENAME=xenial
- """)
-
- str_ubuntu_18_04_os_release = dedent("""
- NAME="Ubuntu"
- VERSION="18.04 LTS (Bionic Beaver)"
- ID=ubuntu
- ID_LIKE=debian
- PRETTY_NAME="Ubuntu 18.04 LTS"
- VERSION_ID="18.04"
- HOME_URL="https://www.ubuntu.com/"
- SUPPORT_URL="https://help.ubuntu.com/"
- BUG_REPORT_URL="https://bugs.launchpad.net/ubuntu/"
- PRIVACY_POLICY_URL="https://www.ubuntu.com/legal/terms-and-policies/privacy-policy"
- VERSION_CODENAME=bionic
- UBUNTU_CODENAME=bionic
- """)
-
- str_rhel_6_4_lsb_release = dedent("""
- LSB Version: :base-4.0-amd64:base-4.0-noarch:core-4.0-amd64:core-4.0-noarch:graphics-4.0-amd64:graphics-4.0-noarch:printing-4.0-amd64:printing-4.0-noarch
- Distributor ID: RedHatEnterpriseServer
- Description: Red Hat Enterprise Linux Server release 6.4 (Santiago)
- Release: 6.4
- Codename: Santiago
- """)
-
- str_rhel_7_lsb_release = dedent("""
- LSB Version: :core-4.1-amd64:core-4.1-noarch:cxx-4.1-amd64:cxx-4.1-noarch:desktop-4.1-amd64:desktop-4.1-noarch:languages-4.1-amd64:languages-4.1-noarch:printing-4.1-amd64:printing-4.1-noarch
- Distributor ID: RedHatEnterpriseServer
- Description: Red Hat Enterprise Linux Server release 7.0 (Maipo)
- Release: 7.0
- Codename: Maipo
- """)
-
- str_rhel_7_os_release = dedent("""
- NAME="Red Hat Enterprise Linux Server"
- VERSION="7.0 (Maipo)"
- ID="rhel"
- ID_LIKE="fedora"
- VERSION_ID="7.0"
- PRETTY_NAME="Red Hat Enterprise Linux Server 7.0 (Maipo)"
- ANSI_COLOR="0;31"
- CPE_NAME="cpe:/o:redhat:enterprise_linux:7.0:GA:server"
- HOME_URL="https://www.redhat.com/"
- BUG_REPORT_URL="https://bugzilla.redhat.com/"
-
- REDHAT_BUGZILLA_PRODUCT="Red Hat Enterprise Linux 7"
- REDHAT_BUGZILLA_PRODUCT_VERSION=7.0
- REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux"
- REDHAT_SUPPORT_PRODUCT_VERSION=7.0
- """)
-
- str_fedora_26_os_release = dedent("""
- NAME=Fedora
- VERSION="26 (Twenty Six)"
- ID=fedora
- VERSION_ID=26
- PRETTY_NAME="Fedora 26 (Twenty Six)"
- ANSI_COLOR="0;34"
- CPE_NAME="cpe:/o:fedoraproject:fedora:26"
- HOME_URL="https://fedoraproject.org/"
- BUG_REPORT_URL="https://bugzilla.redhat.com/"
- REDHAT_BUGZILLA_PRODUCT="Fedora"
- REDHAT_BUGZILLA_PRODUCT_VERSION=26
- REDHAT_SUPPORT_PRODUCT="Fedora"
- REDHAT_SUPPORT_PRODUCT_VERSION=26
- PRIVACY_POLICY_URL=https://fedoraproject.org/wiki/Legal:PrivacyPolicy
- """)
-
- str_opensuse_42_3_os_release = dedent("""
- NAME="openSUSE Leap"
- VERSION="42.3"
- ID=opensuse
- ID_LIKE="suse"
- VERSION_ID="42.3"
- PRETTY_NAME="openSUSE Leap 42.3"
- ANSI_COLOR="0;32"
- CPE_NAME="cpe:/o:opensuse:leap:42.3"
- BUG_REPORT_URL="https://bugs.opensuse.org"
- HOME_URL="https://www.opensuse.org/"
- """)
-
- str_opensuse_15_0_os_release = dedent("""
- NAME="openSUSE Leap"
- VERSION="15.0"
- ID="opensuse-leap"
- ID_LIKE="suse opensuse"
- VERSION_ID="15.0"
- PRETTY_NAME="openSUSE Leap 15.0"
- ANSI_COLOR="0;32"
- CPE_NAME="cpe:/o:opensuse:leap:15.0"
- BUG_REPORT_URL="https://bugs.opensuse.org"
- HOME_URL="https://www.opensuse.org/"
- """)
-
- str_opensuse_15_1_os_release = dedent("""
- NAME="openSUSE Leap"
- VERSION="15.1"
- ID="opensuse-leap"
- ID_LIKE="suse opensuse"
- VERSION_ID="15.1"
- PRETTY_NAME="openSUSE Leap 15.1"
- ANSI_COLOR="0;32"
- CPE_NAME="cpe:/o:opensuse:leap:15.1"
- BUG_REPORT_URL="https://bugs.opensuse.org"
- HOME_URL="https://www.opensuse.org/"
- """)
-
- def test_centos_9_os_release(self):
- os = OS.from_os_release(self.str_centos_9_os_release)
- assert os.name == 'centos'
- assert os.version == '9.stream'
- assert os.codename == 'stream'
- assert os.package_type == 'rpm'
-
- def test_centos_7_os_release(self):
- os = OS.from_os_release(self.str_centos_7_os_release)
- assert os.name == 'centos'
- assert os.version == '7'
- assert os.codename == 'core'
- assert os.package_type == 'rpm'
-
- def test_centos_7_os_release_newer(self):
- os = OS.from_os_release(self.str_centos_7_os_release_newer)
- assert os.name == 'centos'
- assert os.version == '7'
- assert os.codename == 'core'
- assert os.package_type == 'rpm'
-
- def test_debian_7_lsb_release(self):
- os = OS.from_lsb_release(self.str_debian_7_lsb_release)
- assert os.name == 'debian'
- assert os.version == '7.1'
- assert os.codename == 'wheezy'
- assert os.package_type == 'deb'
-
- def test_debian_7_os_release(self):
- os = OS.from_os_release(self.str_debian_7_os_release)
- assert os.name == 'debian'
- assert os.version == '7'
- assert os.codename == 'wheezy'
- assert os.package_type == 'deb'
-
- def test_debian_8_os_release(self):
- os = OS.from_os_release(self.str_debian_8_os_release)
- assert os.name == 'debian'
- assert os.version == '8'
- assert os.codename == 'jessie'
- assert os.package_type == 'deb'
-
- def test_debian_9_os_release(self):
- os = OS.from_os_release(self.str_debian_9_os_release)
- assert os.name == 'debian'
- assert os.version == '9'
- assert os.codename == 'stretch'
- assert os.package_type == 'deb'
-
- def test_ubuntu_12_04_lsb_release(self):
- os = OS.from_lsb_release(self.str_ubuntu_12_04_lsb_release)
- assert os.name == 'ubuntu'
- assert os.version == '12.04'
- assert os.codename == 'precise'
- assert os.package_type == 'deb'
-
- def test_ubuntu_12_04_os_release(self):
- os = OS.from_os_release(self.str_ubuntu_12_04_os_release)
- assert os.name == 'ubuntu'
- assert os.version == '12.04'
- assert os.codename == 'precise'
- assert os.package_type == 'deb'
-
- def test_ubuntu_14_04_os_release(self):
- os = OS.from_os_release(self.str_ubuntu_14_04_os_release)
- assert os.name == 'ubuntu'
- assert os.version == '14.04'
- assert os.codename == 'trusty'
- assert os.package_type == 'deb'
-
- def test_ubuntu_16_04_os_release(self):
- os = OS.from_os_release(self.str_ubuntu_16_04_os_release)
- assert os.name == 'ubuntu'
- assert os.version == '16.04'
- assert os.codename == 'xenial'
- assert os.package_type == 'deb'
-
- def test_ubuntu_18_04_os_release(self):
- os = OS.from_os_release(self.str_ubuntu_18_04_os_release)
- assert os.name == 'ubuntu'
- assert os.version == '18.04'
- assert os.codename == 'bionic'
- assert os.package_type == 'deb'
-
- def test_rhel_6_4_lsb_release(self):
- os = OS.from_lsb_release(self.str_rhel_6_4_lsb_release)
- assert os.name == 'rhel'
- assert os.version == '6.4'
- assert os.codename == 'santiago'
- assert os.package_type == 'rpm'
-
- def test_rhel_7_lsb_release(self):
- os = OS.from_lsb_release(self.str_rhel_7_lsb_release)
- assert os.name == 'rhel'
- assert os.version == '7.0'
- assert os.codename == 'maipo'
- assert os.package_type == 'rpm'
-
- def test_rhel_7_os_release(self):
- os = OS.from_os_release(self.str_rhel_7_os_release)
- assert os.name == 'rhel'
- assert os.version == '7.0'
- assert os.codename == 'maipo'
- assert os.package_type == 'rpm'
-
- def test_fedora_26_os_release(self):
- os = OS.from_os_release(self.str_fedora_26_os_release)
- assert os.name == 'fedora'
- assert os.version == '26'
- assert os.codename == '26'
- assert os.package_type == 'rpm'
-
- def test_opensuse_42_3_os_release(self):
- os = OS.from_os_release(self.str_opensuse_42_3_os_release)
- assert os.name == 'opensuse'
- assert os.version == '42.3'
- assert os.codename == 'leap'
- assert os.package_type == 'rpm'
-
- def test_opensuse_15_0_os_release(self):
- os = OS.from_os_release(self.str_opensuse_15_0_os_release)
- assert os.name == 'opensuse'
- assert os.version == '15.0'
- assert os.codename == 'leap'
- assert os.package_type == 'rpm'
-
- def test_opensuse_15_1_os_release(self):
- os = OS.from_os_release(self.str_opensuse_15_1_os_release)
- assert os.name == 'opensuse'
- assert os.version == '15.1'
- assert os.codename == 'leap'
- assert os.package_type == 'rpm'
-
- def test_version_codename_success(self):
- assert OS.version_codename('ubuntu', '14.04') == ('14.04', 'trusty')
- assert OS.version_codename('ubuntu', 'trusty') == ('14.04', 'trusty')
-
- def test_version_codename_failure(self):
- with pytest.raises(KeyError) as excinfo:
- OS.version_codename('ubuntu', 'frog')
- assert excinfo.type == KeyError
- assert 'frog' in excinfo.value.args[0]
-
- def test_repr(self):
- os = OS(name='NAME', version='0.1.2', codename='code')
- assert repr(os) == "OS(name='NAME', version='0.1.2', codename='code')"
-
- def test_to_dict(self):
- os = OS(name='NAME', version='0.1.2', codename='code')
- ref_dict = dict(name='NAME', version='0.1.2', codename='code')
- assert os.to_dict() == ref_dict
-
- def test_version_no_codename(self):
- os = OS(name='ubuntu', version='16.04')
- assert os.codename == 'xenial'
-
- def test_codename_no_version(self):
- os = OS(name='ubuntu', codename='trusty')
- assert os.version == '14.04'
-
- def test_eq_equal(self):
- os = OS(name='ubuntu', codename='trusty', version='14.04')
- assert OS(name='ubuntu', codename='trusty', version='14.04') == os
-
- def test_eq_not_equal(self):
- os = OS(name='ubuntu', codename='trusty', version='16.04')
- assert OS(name='ubuntu', codename='trusty', version='14.04') != os
+++ /dev/null
-from mock import patch, Mock, MagicMock
-from pytest import raises
-
-from io import BytesIO
-
-from teuthology.orchestra import remote
-from teuthology.orchestra import opsys
-from teuthology.orchestra.run import RemoteProcess
-from teuthology.exceptions import CommandFailedError, UnitTestError
-
-
-class TestRemote(object):
-
- def setup_method(self):
- self.start_patchers()
-
- def teardown_method(self):
- self.stop_patchers()
-
- def start_patchers(self):
- self.m_ssh = MagicMock()
- self.patcher_ssh = patch(
- 'teuthology.orchestra.connection.paramiko.SSHClient',
- self.m_ssh,
- )
- self.patcher_ssh.start()
-
- def stop_patchers(self):
- self.patcher_ssh.stop()
-
- def test_shortname(self):
- r = remote.Remote(
- name='jdoe@xyzzy.example.com',
- shortname='xyz',
- ssh=self.m_ssh,
- )
- assert r.shortname == 'xyz'
- assert str(r) == 'jdoe@xyzzy.example.com'
-
- def test_shortname_default(self):
- r = remote.Remote(
- name='jdoe@xyzzy.example.com',
- ssh=self.m_ssh,
- )
- assert r.shortname == 'xyzzy'
- assert str(r) == 'jdoe@xyzzy.example.com'
-
- def test_run(self):
- m_transport = MagicMock()
- m_transport.getpeername.return_value = ('name', 22)
- self.m_ssh.get_transport.return_value = m_transport
- m_run = MagicMock()
- args = [
- 'something',
- 'more',
- ]
- proc = RemoteProcess(
- client=self.m_ssh,
- args=args,
- )
- m_run.return_value = proc
- rem = remote.Remote(name='jdoe@xyzzy.example.com', ssh=self.m_ssh)
- rem._runner = m_run
- result = rem.run(args=args)
- m_transport.getpeername.assert_called_once_with()
- m_run_call_kwargs = m_run.call_args_list[0][1]
- assert m_run_call_kwargs['args'] == args
- assert result is proc
- assert result.remote is rem
-
- @patch('teuthology.util.scanner.UnitTestScanner.scan_and_write')
- def test_run_unit_test(self, m_scan_and_write):
- m_transport = MagicMock()
- m_transport.getpeername.return_value = ('name', 22)
- self.m_ssh.get_transport.return_value = m_transport
- m_run = MagicMock(name="run", side_effect=CommandFailedError('mocked error', 1, 'smithi'))
- args = [
- 'something',
- 'more',
- ]
- rem = remote.Remote(name='jdoe@xyzzy.example.com', ssh=self.m_ssh)
- rem._runner = m_run
- m_scan_and_write.return_value = "Error Message"
- with raises(UnitTestError) as exc:
- rem.run_unit_test(args=args, xml_path_regex="xml_path", output_yaml="yaml_path")
- assert str(exc.value) == "Unit test failed on smithi with status 1: 'Error Message'"
-
- def test_hostname(self):
- m_transport = MagicMock()
- m_transport.getpeername.return_value = ('name', 22)
- self.m_ssh.get_transport.return_value = m_transport
- m_run = MagicMock()
- args = [
- 'hostname',
- '--fqdn',
- ]
- stdout = BytesIO(b'test_hostname')
- stdout.seek(0)
- proc = RemoteProcess(
- client=self.m_ssh,
- args=args,
- )
- proc.stdout = stdout
- proc._stdout_buf = Mock()
- proc._stdout_buf.channel.recv_exit_status.return_value = 0
- r = remote.Remote(name='xyzzy.example.com', ssh=self.m_ssh)
- m_run.return_value = proc
- r._runner = m_run
- assert r.hostname == 'test_hostname'
-
- def test_arch(self):
- m_transport = MagicMock()
- m_transport.getpeername.return_value = ('name', 22)
- self.m_ssh.get_transport.return_value = m_transport
- m_run = MagicMock()
- args = [
- 'uname',
- '-m',
- ]
- stdout = BytesIO(b'test_arch')
- stdout.seek(0)
- proc = RemoteProcess(
- client=self.m_ssh,
- args=args,
- )
- proc._stdout_buf = Mock()
- proc._stdout_buf.channel = Mock()
- proc._stdout_buf.channel.recv_exit_status.return_value = 0
- proc._stdout_buf.channel.expects('recv_exit_status').returns(0)
- proc.stdout = stdout
- m_run.return_value = proc
- r = remote.Remote(name='jdoe@xyzzy.example.com', ssh=self.m_ssh)
- r._runner = m_run
- assert r.arch == 'test_arch'
- assert len(m_run.call_args_list) == 1
- m_run_call_kwargs = m_run.call_args_list[0][1]
- assert m_run_call_kwargs['client'] == self.m_ssh
- assert m_run_call_kwargs['name'] == r.shortname
- assert m_run_call_kwargs['args'] == ' '.join(args)
-
- def test_host_key(self):
- m_key = MagicMock()
- m_key.get_name.return_value = 'key_type'
- m_key.get_base64.return_value = 'test ssh key'
- m_transport = MagicMock()
- m_transport.get_remote_server_key.return_value = m_key
- self.m_ssh.get_transport.return_value = m_transport
- r = remote.Remote(name='jdoe@xyzzy.example.com', ssh=self.m_ssh)
- assert r.host_key == 'key_type test ssh key'
- self.m_ssh.get_transport.assert_called_once_with()
- m_transport.get_remote_server_key.assert_called_once_with()
-
- def test_inventory_info(self):
- r = remote.Remote('user@host', host_key='host_key')
- r._arch = 'arch'
- r._os = opsys.OS(name='os_name', version='1.2.3', codename='code')
- inv_info = r.inventory_info
- assert inv_info == dict(
- name='host',
- user='user',
- arch='arch',
- os_type='os_name',
- os_version='1.2',
- ssh_pub_key='host_key',
- up=True,
- )
-
- def test_sftp_open_file(self):
- m_file_obj = MagicMock()
- m_stat = Mock()
- m_stat.st_size = 42
- m_file_obj.stat.return_value = m_stat
- m_open = MagicMock()
- m_open.return_value = m_file_obj
- m_open.return_value.__enter__.return_value = m_file_obj
- with patch.object(remote.Remote, '_sftp_open_file', new=m_open):
- rem = remote.Remote(name='jdoe@xyzzy.example.com', ssh=self.m_ssh)
- assert rem._sftp_open_file('x') is m_file_obj
- assert rem._sftp_open_file('x').stat() is m_stat
- assert rem._sftp_open_file('x').stat().st_size == 42
- with rem._sftp_open_file('x') as f:
- assert f == m_file_obj
-
- def test_sftp_get_size(self):
- m_file_obj = MagicMock()
- m_stat = Mock()
- m_stat.st_size = 42
- m_file_obj.stat.return_value = m_stat
- m_open = MagicMock()
- m_open.return_value = m_file_obj
- m_open.return_value.__enter__.return_value = m_file_obj
- with patch.object(remote.Remote, '_sftp_open_file', new=m_open):
- rem = remote.Remote(name='jdoe@xyzzy.example.com', ssh=self.m_ssh)
- assert rem._sftp_get_size('/fake/file') == 42
-
- def test_format_size(self):
- assert remote.Remote._format_size(1023).strip() == '1023B'
- assert remote.Remote._format_size(1024).strip() == '1KB'
- assert remote.Remote._format_size(1024**2).strip() == '1MB'
- assert remote.Remote._format_size(1024**5).strip() == '1TB'
- assert remote.Remote._format_size(1021112).strip() == '997KB'
- assert remote.Remote._format_size(1021112**2).strip() == '971GB'
-
- def test_is_container(self):
- m_transport = MagicMock()
- m_transport.getpeername.return_value = ('name', 22)
- self.m_ssh.get_transport.return_value = m_transport
- m_run = MagicMock()
- args = []
- proc = RemoteProcess(
- client=self.m_ssh,
- args=args,
- )
- proc.returncode = 0
- m_run.return_value = proc
- rem = remote.Remote(name='jdoe@xyzzy.example.com', ssh=self.m_ssh)
- rem._runner = m_run
- assert rem.is_container
- proc.returncode = 1
- rem2 = remote.Remote(name='jdoe@xyzzy.example.com', ssh=self.m_ssh)
- rem2._runner = m_run
- assert not rem2.is_container
-
- @patch("teuthology.orchestra.remote.Remote.sh")
- def test_resolve_ip(self, m_sh):
- r = remote.Remote(name="jdoe@xyzzy.example.com", ssh=self.m_ssh)
- m_sh.return_value = "smithi001.front.sepia.ceph.com has address 172.21.15.1\n"
- ip4 = remote.Remote.resolve_ip(r, 'smithi001')
- assert ip4 == "172.21.15.1"
- m_sh.return_value = "\n"
- try:
- ip4 = remote.Remote.resolve_ip(r, 'smithi001')
- except Exception as e:
- assert 'Cannot get IPv4 address' in str(e)
- try:
- ip4 = remote.Remote.resolve_ip(r, 'smithi001', 5)
- except Exception as e:
- assert 'Unknown IP version' in str(e)
-
- m_sh.return_value = ("google.com has address 142.251.37.14\n"
- "google.com has IPv6 address 2a00:1450:4016:80b::200e\n"
- "google.com mail is handled by 10 smtp.google.com.\n")
- ip4 = remote.Remote.resolve_ip(r, 'google.com')
- assert ip4 == "142.251.37.14"
- ip6 = remote.Remote.resolve_ip(r, 'google.com', '6')
- assert ip6 == "2a00:1450:4016:80b::200e"
-
-
- @patch("teuthology.orchestra.remote.Remote.run")
- def test_write_file(self, m_run):
- file = "fakefile"
- contents = "fakecontents"
- rem = remote.Remote(name='jdoe@xyzzy.example.com', ssh=self.m_ssh)
-
- remote.Remote.write_file(rem, file, contents, bs=1, offset=1024)
- m_run.assert_called_with(args=f"set -ex\ndd of={file} bs=1 seek=1024", stdin=contents, quiet=True)
-
- remote.Remote.write_file(rem, file, contents, sync=True)
- m_run.assert_called_with(args=f"set -ex\ndd of={file} conv=sync", stdin=contents, quiet=True)
+++ /dev/null
-from io import BytesIO
-
-import paramiko
-import socket
-
-from mock import MagicMock, patch
-from pytest import raises
-
-from teuthology.orchestra import run
-from teuthology.exceptions import (CommandCrashedError, CommandFailedError,
- ConnectionLostError)
-
-def set_buffer_contents(buf, contents):
- buf.seek(0)
- if isinstance(contents, bytes):
- buf.write(contents)
- elif isinstance(contents, (list, tuple)):
- buf.writelines(contents)
- elif isinstance(contents, str):
- buf.write(contents.encode())
- else:
- raise TypeError(
- "%s is a %s; should be a byte string, list or tuple" % (
- contents, type(contents)
- )
- )
- buf.seek(0)
-
-
-class TestRun(object):
- def setup_method(self):
- self.start_patchers()
-
- def teardown_method(self):
- self.stop_patchers()
-
- def start_patchers(self):
- self.m_remote_process = MagicMock(wraps=run.RemoteProcess)
- self.patcher_remote_proc = patch(
- 'teuthology.orchestra.run.RemoteProcess',
- self.m_remote_process,
- )
- self.m_channel = MagicMock(spec=paramiko.Channel)()
- """
- self.m_channelfile = MagicMock(wraps=paramiko.ChannelFile)
- self.m_stdin_buf = self.m_channelfile(self.m_channel())
- self.m_stdout_buf = self.m_channelfile(self.m_channel())
- self.m_stderr_buf = self.m_channelfile(self.m_channel())
- """
- class M_ChannelFile(BytesIO):
- channel = MagicMock(spec=paramiko.Channel)()
-
- self.m_channelfile = M_ChannelFile
- self.m_stdin_buf = self.m_channelfile()
- self.m_stdout_buf = self.m_channelfile()
- self.m_stderr_buf = self.m_channelfile()
- self.m_ssh = MagicMock()
- self.m_ssh.exec_command.return_value = (
- self.m_stdin_buf,
- self.m_stdout_buf,
- self.m_stderr_buf,
- )
- self.m_transport = MagicMock()
- self.m_transport.getpeername.return_value = ('name', 22)
- self.m_ssh.get_transport.return_value = self.m_transport
- self.patcher_ssh = patch(
- 'teuthology.orchestra.connection.paramiko.SSHClient',
- self.m_ssh,
- )
- self.patcher_ssh.start()
- # Tests must start this if they wish to use it
- # self.patcher_remote_proc.start()
-
- def stop_patchers(self):
- # If this patcher wasn't started, it's ok
- try:
- self.patcher_remote_proc.stop()
- except RuntimeError:
- pass
- self.patcher_ssh.stop()
-
- def test_exitstatus(self):
- self.m_stdout_buf.channel.recv_exit_status.return_value = 0
- proc = run.run(
- client=self.m_ssh,
- args=['foo', 'bar baz'],
- )
- assert proc.exitstatus == 0
-
- def test_run_cwd(self):
- self.m_stdout_buf.channel.recv_exit_status.return_value = 0
- run.run(
- client=self.m_ssh,
- args=['foo_bar_baz'],
- cwd='/cwd/test',
- )
- self.m_ssh.exec_command.assert_called_with('(cd /cwd/test && exec foo_bar_baz)')
-
- def test_capture_stdout(self):
- output = 'foo\nbar'
- set_buffer_contents(self.m_stdout_buf, output)
- self.m_stdout_buf.channel.recv_exit_status.return_value = 0
- stdout = BytesIO()
- proc = run.run(
- client=self.m_ssh,
- args=['foo', 'bar baz'],
- stdout=stdout,
- )
- assert proc.stdout is stdout
- assert proc.stdout.read().decode() == output
- assert proc.stdout.getvalue().decode() == output
-
- def test_capture_stderr_newline(self):
- output = 'foo\nbar\n'
- set_buffer_contents(self.m_stderr_buf, output)
- self.m_stderr_buf.channel.recv_exit_status.return_value = 0
- stderr = BytesIO()
- proc = run.run(
- client=self.m_ssh,
- args=['foo', 'bar baz'],
- stderr=stderr,
- )
- assert proc.stderr is stderr
- assert proc.stderr.read().decode() == output
- assert proc.stderr.getvalue().decode() == output
-
- def test_status_bad(self):
- self.m_stdout_buf.channel.recv_exit_status.return_value = 42
- with raises(CommandFailedError) as exc:
- run.run(
- client=self.m_ssh,
- args=['foo'],
- )
- assert str(exc.value) == "Command failed on name with status 42: 'foo'"
-
- def test_status_bad_nocheck(self):
- self.m_stdout_buf.channel.recv_exit_status.return_value = 42
- proc = run.run(
- client=self.m_ssh,
- args=['foo'],
- check_status=False,
- )
- assert proc.exitstatus == 42
-
- def test_status_crash(self):
- self.m_stdout_buf.channel.recv_exit_status.return_value = -1
- with raises(CommandCrashedError) as exc:
- run.run(
- client=self.m_ssh,
- args=['foo'],
- )
- assert str(exc.value) == "Command crashed: 'foo'"
-
- def test_status_crash_nocheck(self):
- self.m_stdout_buf.channel.recv_exit_status.return_value = -1
- proc = run.run(
- client=self.m_ssh,
- args=['foo'],
- check_status=False,
- )
- assert proc.exitstatus == -1
-
- def test_status_lost(self):
- m_transport = MagicMock()
- m_transport.getpeername.return_value = ('name', 22)
- m_transport.is_active.return_value = False
- self.m_stdout_buf.channel.recv_exit_status.return_value = -1
- self.m_ssh.get_transport.return_value = m_transport
- with raises(ConnectionLostError) as exc:
- run.run(
- client=self.m_ssh,
- args=['foo'],
- )
- assert str(exc.value) == "SSH connection to name was lost: 'foo'"
-
- def test_status_lost_socket(self):
- m_transport = MagicMock()
- m_transport.getpeername.side_effect = socket.error
- self.m_ssh.get_transport.return_value = m_transport
- with raises(ConnectionLostError) as exc:
- run.run(
- client=self.m_ssh,
- args=['foo'],
- )
- assert str(exc.value) == "SSH connection was lost: 'foo'"
-
- def test_status_lost_nocheck(self):
- m_transport = MagicMock()
- m_transport.getpeername.return_value = ('name', 22)
- m_transport.is_active.return_value = False
- self.m_stdout_buf.channel.recv_exit_status.return_value = -1
- self.m_ssh.get_transport.return_value = m_transport
- proc = run.run(
- client=self.m_ssh,
- args=['foo'],
- check_status=False,
- )
- assert proc.exitstatus == -1
-
- def test_status_bad_nowait(self):
- self.m_stdout_buf.channel.recv_exit_status.return_value = 42
- proc = run.run(
- client=self.m_ssh,
- args=['foo'],
- wait=False,
- )
- with raises(CommandFailedError) as exc:
- proc.wait()
- assert proc.returncode == 42
- assert str(exc.value) == "Command failed on name with status 42: 'foo'"
-
- def test_stdin_pipe(self):
- self.m_stdout_buf.channel.recv_exit_status.return_value = 0
- proc = run.run(
- client=self.m_ssh,
- args=['foo'],
- stdin=run.PIPE,
- wait=False
- )
- assert proc.poll() == 0
- code = proc.wait()
- assert code == 0
- assert proc.exitstatus == 0
-
- def test_stdout_pipe(self):
- self.m_stdout_buf.channel.recv_exit_status.return_value = 0
- lines = [b'one\n', b'two', b'']
- set_buffer_contents(self.m_stdout_buf, lines)
- proc = run.run(
- client=self.m_ssh,
- args=['foo'],
- stdout=run.PIPE,
- wait=False
- )
- assert proc.poll() == 0
- assert proc.stdout.readline() == lines[0]
- assert proc.stdout.readline() == lines[1]
- assert proc.stdout.readline() == lines[2]
- code = proc.wait()
- assert code == 0
- assert proc.exitstatus == 0
-
- def test_stderr_pipe(self):
- self.m_stdout_buf.channel.recv_exit_status.return_value = 0
- lines = [b'one\n', b'two', b'']
- set_buffer_contents(self.m_stderr_buf, lines)
- proc = run.run(
- client=self.m_ssh,
- args=['foo'],
- stderr=run.PIPE,
- wait=False
- )
- assert proc.poll() == 0
- assert proc.stderr.readline() == lines[0]
- assert proc.stderr.readline() == lines[1]
- assert proc.stderr.readline() == lines[2]
- code = proc.wait()
- assert code == 0
- assert proc.exitstatus == 0
-
- def test_copy_and_close(self):
- run.copy_and_close(None, MagicMock())
- run.copy_and_close('', MagicMock())
- run.copy_and_close(b'', MagicMock())
-
-
-class TestQuote(object):
- def test_quote_simple(self):
- got = run.quote(['a b', ' c', 'd e '])
- assert got == "'a b' ' c' 'd e '"
-
- def test_quote_and_quote(self):
- got = run.quote(['echo', 'this && is embedded', '&&',
- 'that was standalone'])
- assert got == "echo 'this && is embedded' '&&' 'that was standalone'"
-
- def test_quote_and_raw(self):
- got = run.quote(['true', run.Raw('&&'), 'echo', 'yay'])
- assert got == "true && echo yay"
-
-
-class TestRaw(object):
- def test_eq(self):
- str_ = "I am a raw something or other"
- raw = run.Raw(str_)
- assert raw == run.Raw(str_)
+++ /dev/null
-import argparse
-import os
-
-from logging import debug
-from teuthology import misc
-from teuthology.orchestra import cluster
-from teuthology.orchestra.run import quote
-from teuthology.orchestra.daemon.group import DaemonGroup
-import subprocess
-
-
-class FakeRemote(object):
- pass
-
-
-def test_pid():
- ctx = argparse.Namespace()
- ctx.daemons = DaemonGroup(use_systemd=True)
- remote = FakeRemote()
-
- ps_ef_output_path = os.path.join(
- os.path.dirname(__file__),
- "files/daemon-systemdstate-pid-ps-ef.output"
- )
-
- # patching ps -ef command output using a file
- def sh(args):
- args[0:2] = ["cat", ps_ef_output_path]
- debug(args)
- return subprocess.getoutput(quote(args))
-
- remote.sh = sh
- remote.init_system = 'systemd'
- remote.shortname = 'host1'
-
- ctx.cluster = cluster.Cluster(
- remotes=[
- (remote, ['rgw.0', 'mon.a', 'mgr.a', 'mds.a', 'osd.0'])
- ],
- )
-
- for remote, roles in ctx.cluster.remotes.items():
- for role in roles:
- _, rol, id_ = misc.split_role(role)
- if any(rol.startswith(x) for x in ['mon', 'mgr', 'mds']):
- ctx.daemons.register_daemon(remote, rol, remote.shortname)
- else:
- ctx.daemons.register_daemon(remote, rol, id_)
-
- for _, daemons in ctx.daemons.daemons.items():
- for daemon in daemons.values():
- pid = daemon.pid
- debug(pid)
- assert pid
+++ /dev/null
-def assert_raises(excClass, callableObj, *args, **kwargs):
- """
- Like unittest.TestCase.assertRaises, but returns the exception.
- """
- try:
- callableObj(*args, **kwargs)
- except excClass as e:
- return e
- else:
- if hasattr(excClass,'__name__'): excName = excClass.__name__
- else: excName = str(excClass)
- raise AssertionError("%s not raised" % excName)
+++ /dev/null
-from libcloud.compute.providers import get_driver
-from mock import patch
-
-from teuthology.config import config
-from teuthology.provision import cloud
-
-from test_cloud_init import dummy_config, dummy_drivers
-
-
-class TestBase(object):
- def setup_method(self):
- config.load()
- config.libcloud = dummy_config
- cloud.supported_drivers['dummy'] = dummy_drivers
-
- def teardown_method(self):
- del cloud.supported_drivers['dummy']
-
-
-class TestProvider(TestBase):
- def test_init(self):
- obj = cloud.get_provider('my_provider')
- assert obj.name == 'my_provider'
- assert obj.driver_name == 'dummy'
- assert obj.conf == dummy_config['providers']['my_provider']
-
- def test_driver(self):
- obj = cloud.get_provider('my_provider')
- assert isinstance(obj.driver, get_driver('dummy'))
-
-
-class TestProvisioner(TestBase):
- klass = cloud.base.Provisioner
-
- def get_obj(
- self, name='node_name', os_type='ubuntu', os_version='ubuntu'):
- return cloud.get_provisioner(
- 'my_provider',
- 'node_name',
- 'ubuntu',
- '16.04',
- )
-
- def test_init_provider_string(self):
- obj = self.klass('my_provider', 'ubuntu', '16.04')
- assert obj.provider.name == 'my_provider'
-
- def test_create(self):
- obj = self.get_obj()
- with patch.object(
- self.klass,
- '_create',
- ) as m_create:
- for val in [True, False]:
- m_create.return_value = val
- res = obj.create()
- assert res is val
- m_create.assert_called_once_with()
- m_create.reset_mock()
- m_create.side_effect = RuntimeError
- res = obj.create()
- assert res is False
- assert obj.create() is None
-
- def test_destroy(self):
- obj = self.get_obj()
- with patch.object(
- self.klass,
- '_destroy',
- ) as m_destroy:
- for val in [True, False]:
- m_destroy.return_value = val
- res = obj.destroy()
- assert res is val
- m_destroy.assert_called_once_with()
- m_destroy.reset_mock()
- m_destroy.side_effect = RuntimeError
- res = obj.destroy()
- assert res is False
- assert obj.destroy() is None
-
- def test_remote(self):
- obj = self.get_obj()
- assert obj.remote.shortname == 'node_name'
-
- def test_repr(self):
- obj = self.get_obj()
- assert repr(obj) == \
- "Provisioner(provider='my_provider', name='node_name', os_type='ubuntu', os_version='16.04')" # noqa
-
+++ /dev/null
-from teuthology.config import config
-from teuthology.provision import cloud
-
-dummy_config = dict(
- providers=dict(
- my_provider=dict(
- driver='dummy',
- driver_args=dict(
- creds=0,
- ),
- conf_1='1',
- conf_2='2',
- )
- )
-)
-
-
-class DummyProvider(cloud.base.Provider):
- # For libcloud's dummy driver
- _driver_posargs = ['creds']
-
-dummy_drivers = dict(
- provider=DummyProvider,
- provisioner=cloud.base.Provisioner,
-)
-
-
-class TestInit(object):
- def setup_method(self):
- config.load()
- config.libcloud = dummy_config
- cloud.supported_drivers['dummy'] = dummy_drivers
-
- def teardown_method(self):
- del cloud.supported_drivers['dummy']
-
- def test_get_types(self):
- assert list(cloud.get_types()) == ['my_provider']
-
- def test_get_provider_conf(self):
- expected = dummy_config['providers']['my_provider']
- assert cloud.get_provider_conf('my_provider') == expected
-
- def test_get_provider(self):
- obj = cloud.get_provider('my_provider')
- assert obj.name == 'my_provider'
- assert obj.driver_name == 'dummy'
-
- def test_get_provisioner(self):
- obj = cloud.get_provisioner(
- 'my_provider',
- 'node_name',
- 'ubuntu',
- '16.04',
- dict(foo='bar'),
- )
- assert obj.provider.name == 'my_provider'
- assert obj.name == 'node_name'
- assert obj.os_type == 'ubuntu'
- assert obj.os_version == '16.04'
+++ /dev/null
-import datetime
-import dateutil
-import json
-
-from mock import patch, mock_open
-from pytest import mark
-
-from teuthology.provision.cloud import util
-
-
-@mark.parametrize(
- 'path, exists',
- [
- ('/fake/path', True),
- ('/fake/path', False),
- ]
-)
-def test_get_user_ssh_pubkey(path, exists):
- with patch('os.path.exists') as m_exists:
- m_exists.return_value = exists
- with patch('teuthology.provision.cloud.util.open', mock_open(), create=True) as m_open:
- util.get_user_ssh_pubkey(path)
- if exists:
- m_open.assert_called_once_with(path)
-
-
-@mark.parametrize(
- 'input_, func, expected',
- [
- [
- [
- dict(sub0=dict(key0=0, key1=0)),
- dict(sub0=dict(key1=1, key2=2)),
- ],
- lambda x, y: x > y,
- dict(sub0=dict(key0=0, key1=1, key2=2))
- ],
- [
- [
- dict(),
- dict(sub0=dict(key1=1, key2=2)),
- ],
- lambda x, y: x > y,
- dict(sub0=dict(key1=1, key2=2))
- ],
- [
- [
- dict(sub0=dict(key1=1, key2=2)),
- dict(),
- ],
- lambda x, y: x > y,
- dict(sub0=dict(key1=1, key2=2))
- ],
- [
- [
- dict(sub0=dict(key0=0, key1=0, key2=0)),
- dict(sub0=dict(key0=1, key2=3), sub1=dict(key0=0)),
- dict(sub0=dict(key0=3, key1=2, key2=1)),
- dict(sub0=dict(key1=3),
- sub1=dict(key0=3, key1=0)),
- ],
- lambda x, y: x > y,
- dict(sub0=dict(key0=3, key1=3, key2=3),
- sub1=dict(key0=3, key1=0))
- ],
- ]
-)
-def test_combine_dicts(input_, func, expected):
- assert util.combine_dicts(input_, func) == expected
-
-
-def get_datetime(offset_hours=0):
- delta = datetime.timedelta(hours=offset_hours)
- return datetime.datetime.now(dateutil.tz.tzutc()) + delta
-
-
-def get_datetime_string(offset_hours=0):
- obj = get_datetime(offset_hours)
- return obj.strftime(util.AuthToken.time_format)
-
-
-class TestAuthToken(object):
- klass = util.AuthToken
-
- def setup_method(self):
- default_expires = get_datetime_string(0)
- self.test_data = dict(
- value='token_value',
- endpoint='endpoint',
- expires=default_expires,
- )
- self.patchers = dict()
- self.patchers['m_open'] = patch(
- 'teuthology.provision.cloud.util.open'
- )
- self.patchers['m_exists'] = patch(
- 'os.path.exists'
- )
- self.patchers['m_file_lock'] = patch(
- 'teuthology.provision.cloud.util.FileLock'
- )
- self.mocks = dict()
- for name, patcher in self.patchers.items():
- self.mocks[name] = patcher.start()
-
- def teardown_method(self):
- for patcher in self.patchers.values():
- patcher.stop()
-
- def get_obj(self, name='name', directory='/fake/directory'):
- return self.klass(
- name=name,
- directory=directory,
- )
-
- def test_no_token(self):
- obj = self.get_obj()
- self.mocks['m_exists'].return_value = False
- with obj:
- assert obj.value is None
- assert obj.expired is True
-
- @mark.parametrize(
- 'test_data, expired',
- [
- [
- dict(
- value='token_value',
- endpoint='endpoint',
- expires=get_datetime_string(-1),
- ),
- True
- ],
- [
- dict(
- value='token_value',
- endpoint='endpoint',
- expires=get_datetime_string(1),
- ),
- False
- ],
- ]
- )
- def test_token_read(self, test_data, expired):
- obj = self.get_obj()
- self.mocks['m_exists'].return_value = True
- self.mocks['m_open'].return_value.__enter__.return_value.read.return_value = \
- json.dumps(test_data)
- with obj:
- if expired:
- assert obj.value is None
- assert obj.expired is True
- else:
- assert obj.value == test_data['value']
-
- def test_token_write(self):
- obj = self.get_obj()
- datetime_obj = get_datetime(0)
- datetime_string = get_datetime_string(0)
- self.mocks['m_exists'].return_value = False
- with obj:
- obj.write('value', datetime_obj, 'endpoint')
- m_open = self.mocks['m_open']
- write_calls = m_open.return_value.__enter__.return_value.write\
- .call_args_list
- assert len(write_calls) == 1
- expected = json.dumps(dict(
- value='value',
- expires=datetime_string,
- endpoint='endpoint',
- ))
- assert write_calls[0][0][0] == expected
+++ /dev/null
-import socket
-import yaml
-import os
-
-from teuthology.util.compat import parse_qs
-
-from copy import deepcopy
-from libcloud.compute.providers import get_driver
-from mock import patch, Mock, DEFAULT
-from pytest import raises, mark
-
-from teuthology.config import config
-from teuthology.exceptions import MaxWhileTries
-from teuthology.provision import cloud
-
-test_config = dict(
- providers=dict(
- my_provider=dict(
- driver='openstack',
- driver_args=dict(
- username='user',
- password='password',
- ex_force_auth_url='http://127.0.0.1:9999/v2.0/tokens',
- ),
- ),
- image_exclude_provider=dict(
- driver='openstack',
- exclude_image=['.*-exclude1', '.*-exclude2'],
- driver_args=dict(
- username='user',
- password='password',
- ex_force_auth_url='http://127.0.0.1:9999/v2.0/tokens',
- ),
- )
- )
-)
-
-
-@patch('time.sleep')
-def test_retry(m_sleep):
- orig_exceptions = cloud.openstack.RETRY_EXCEPTIONS
- new_exceptions = orig_exceptions + (RuntimeError, )
-
- class test_cls(object):
- def __init__(self, min_val):
- self.min_val = min_val
- self.cur_val = 0
-
- def func(self):
- self.cur_val += 1
- if self.cur_val < self.min_val:
- raise RuntimeError
- return self.cur_val
-
- with patch.object(
- cloud.openstack,
- 'RETRY_EXCEPTIONS',
- new=new_exceptions,
- ):
- test_obj = test_cls(min_val=5)
- assert cloud.openstack.retry(test_obj.func) == 5
- test_obj = test_cls(min_val=1000)
- with raises(MaxWhileTries):
- cloud.openstack.retry(test_obj.func)
-
-
-def get_fake_obj(mock_args=None, attributes=None):
- if mock_args is None:
- mock_args = dict()
- if attributes is None:
- attributes = dict()
- obj = Mock(**mock_args)
- for name, value in attributes.items():
- setattr(obj, name, value)
- return obj
-
-
-class TestOpenStackBase(object):
- def setup_method(self):
- config.load(dict(libcloud=deepcopy(test_config)))
- self.start_patchers()
-
- def start_patchers(self):
- self.patchers = dict()
- self.patchers['m_list_images'] = patch(
- 'libcloud.compute.drivers.openstack'
- '.OpenStackNodeDriver.list_images'
- )
- self.patchers['m_list_sizes'] = patch(
- 'libcloud.compute.drivers.openstack'
- '.OpenStackNodeDriver.list_sizes'
- )
- self.patchers['m_ex_list_networks'] = patch(
- 'libcloud.compute.drivers.openstack'
- '.OpenStack_1_1_NodeDriver.ex_list_networks'
- )
- self.patchers['m_ex_list_security_groups'] = patch(
- 'libcloud.compute.drivers.openstack'
- '.OpenStack_1_1_NodeDriver.ex_list_security_groups'
- )
- self.patchers['m_get_user_ssh_pubkey'] = patch(
- 'teuthology.provision.cloud.util.get_user_ssh_pubkey'
- )
- self.patchers['m_list_nodes'] = patch(
- 'libcloud.compute.drivers.openstack'
- '.OpenStackNodeDriver.list_nodes'
- )
- self.patchers['m_create_node'] = patch(
- 'libcloud.compute.drivers.openstack'
- '.OpenStack_1_1_NodeDriver.create_node'
- )
- self.patchers['m_wait_until_running'] = patch(
- 'libcloud.compute.drivers.openstack'
- '.OpenStackNodeDriver.wait_until_running'
- )
- self.patchers['m_create_volume'] = patch(
- 'libcloud.compute.drivers.openstack'
- '.OpenStackNodeDriver.create_volume'
- )
- self.patchers['m_attach_volume'] = patch(
- 'libcloud.compute.drivers.openstack'
- '.OpenStackNodeDriver.attach_volume'
- )
- self.patchers['m_detach_volume'] = patch(
- 'libcloud.compute.drivers.openstack'
- '.OpenStackNodeDriver.detach_volume'
- )
- self.patchers['m_list_volumes'] = patch(
- 'libcloud.compute.drivers.openstack'
- '.OpenStackNodeDriver.list_volumes'
- )
- self.patchers['m_destroy_volume'] = patch(
- 'libcloud.compute.drivers.openstack'
- '.OpenStackNodeDriver.destroy_volume'
- )
- self.patchers['m_get_service_catalog'] = patch(
- 'libcloud.common.openstack'
- '.OpenStackBaseConnection.get_service_catalog'
- )
- self.patchers['m_auth_token'] = patch(
- 'teuthology.provision.cloud.util.AuthToken'
- )
- self.patchers['m_get_endpoint'] = patch(
- 'libcloud.common.openstack'
- '.OpenStackBaseConnection.get_endpoint',
- )
- self.patchers['m_connect'] = patch(
- 'libcloud.common.base'
- '.Connection.connect',
- )
- self.patchers['m_sleep'] = patch(
- 'time.sleep'
- )
- self.patchers['m_get'] = patch(
- 'requests.get'
- )
- self.mocks = dict()
- for name, patcher in self.patchers.items():
- self.mocks[name] = patcher.start()
- self.mocks['m_get_endpoint'].return_value = 'endpoint'
-
- def teardown_method(self):
- for patcher in self.patchers.values():
- patcher.stop()
-
-
-class TestOpenStackProvider(TestOpenStackBase):
- klass = cloud.openstack.OpenStackProvider
-
- def test_init(self):
- obj = cloud.get_provider('my_provider')
- assert obj.name == 'my_provider'
- assert obj.driver_name == 'openstack'
- assert obj.conf == test_config['providers']['my_provider']
-
- def test_driver(self):
- token = self.mocks['m_auth_token'].return_value
- self.mocks['m_auth_token'].return_value.__enter__.return_value = token
- token.value = None
- obj = cloud.get_provider('my_provider')
- assert isinstance(obj.driver, get_driver('openstack'))
- assert obj._auth_token.value is None
-
- def test_images(self):
- obj = cloud.get_provider('my_provider')
- self.mocks['m_list_images'].return_value = [
- get_fake_obj(attributes=dict(name=_))
- for _ in ['image0', 'image1']]
- assert not hasattr(obj, '_images')
- assert [_.name for _ in obj.images] == ['image0', 'image1']
- assert hasattr(obj, '_images')
-
- def test_exclude_image(self):
- obj = cloud.get_provider('image_exclude_provider')
- self.mocks['m_list_images'].return_value = [
- get_fake_obj(attributes=dict(name=_))
- for _ in ['image0', 'image1',
- 'image2-exclude1', 'image3-exclude2']]
- assert not hasattr(obj, '_images')
- assert [_.name for _ in obj.images] == ['image0', 'image1']
- assert hasattr(obj, '_images')
-
- def test_sizes(self):
- obj = cloud.get_provider('my_provider')
- fake_sizes = [get_fake_obj(attributes=dict(name='size%s' % i)) for
- i in range(2)]
- self.mocks['m_list_sizes'].return_value = fake_sizes
- assert not hasattr(obj, '_sizes')
- assert [s.name for s in obj.sizes] == ['size0', 'size1']
- assert hasattr(obj, '_sizes')
-
- def test_networks(self):
- obj = cloud.get_provider('my_provider')
- nets = [get_fake_obj(attributes=dict(name=i)) for i in ['net0', 'net1']]
- self.mocks['m_ex_list_networks'].return_value = nets
- assert not hasattr(obj, '_networks')
- assert [i.name for i in obj.networks] == [i.name for i in nets]
- assert hasattr(obj, '_networks')
- self.mocks['m_ex_list_networks'].side_effect = AttributeError
- obj = cloud.get_provider('my_provider')
- assert not hasattr(obj, '_networks')
- assert obj.networks == list()
- assert hasattr(obj, '_networks')
-
- def test_security_groups(self):
- obj = cloud.get_provider('my_provider')
- self.mocks['m_ex_list_security_groups'].return_value = ['sg0', 'sg1']
- assert not hasattr(obj, '_security_groups')
- assert obj.security_groups == ['sg0', 'sg1']
- assert hasattr(obj, '_security_groups')
- self.mocks['m_ex_list_security_groups'].side_effect = AttributeError
- obj = cloud.get_provider('my_provider')
- assert not hasattr(obj, '_security_groups')
- assert obj.security_groups == list()
- assert hasattr(obj, '_security_groups')
-
-
-class TestOpenStackCustomProvisioner(TestOpenStackBase):
- klass = cloud.openstack.OpenStackProvisioner
- def get_obj(
- self, name='node_name', os_type='ubuntu',
- os_version='16.04', conf=None, test_conf=None):
-
- if test_conf:
- yaml_file = os.path.dirname(__file__) + '/' + test_conf
- print("Reading conf: %s" % yaml_file)
- with open(yaml_file) as f:
- teuth_conf=yaml.safe_load(f)
- print(teuth_conf)
- config.libcloud = deepcopy(teuth_conf['libcloud'] or test_config)
- else:
- config.libcloud = deepcopy(test_config)
- return cloud.get_provisioner(
- node_type='my_provider',
- name=name,
- os_type=os_type,
- os_version=os_version,
- conf=conf,
- )
-
- @mark.parametrize(
- "conf",
- [
- dict(
- path='test_openstack_userdata_conf.yaml',
- runcmd_head=['uptime', 'date'],
- ssh_authorized_keys=['user_public_key1', 'user_public_key2'],
- user_ssh_pubkey='my_ssh_key',
- os_version='16.04',
- os_type='ubuntu',
- ),
- dict(
- path='test_openstack_userdata_conf.yaml',
- runcmd_head=['uptime', 'date'],
- ssh_authorized_keys=['user_public_key1', 'user_public_key2'],
- user_ssh_pubkey=None,
- os_version='16.04',
- os_type='ubuntu',
- ),
- dict(
- os_version='16.04',
- os_type='ubuntu',
- path=None,
- user_ssh_pubkey=None,
- ),
- ]
- )
- def test_userdata_conf(self, conf):
- self.mocks['m_get_user_ssh_pubkey'].return_value = conf['user_ssh_pubkey']
- obj = self.get_obj(os_version=conf['os_version'],
- os_type=conf['os_type'],
- test_conf=conf['path'])
- userdata = yaml.safe_load(obj.userdata)
- print(">>>> ", obj.conf)
- print(">>>> ", obj.provider.conf)
- print(">>>> ", obj.provider)
- print(obj.userdata)
- if conf and 'path' in conf and conf['path']:
- assert userdata['runcmd'][0:len(conf['runcmd_head'])] == conf['runcmd_head']
- assert userdata['bootcmd'] == [
- 'SuSEfirewall2 stop || true',
- 'service firewalld stop || true',
- ]
- assert 'packages' not in userdata
- else:
- assert 'bootcmd' not in userdata
- assert userdata['packages'] == ['git', 'wget', 'python', 'ntp']
- assert userdata['user'] == obj.user
- assert userdata['hostname'] == obj.hostname
- if 'user_ssh_pubkey' in conf and conf['user_ssh_pubkey']:
- assert userdata['ssh_authorized_keys'][-1] == conf['user_ssh_pubkey']
- if 'ssh_authorized_keys' in conf:
- keys = conf['ssh_authorized_keys']
- assert userdata['ssh_authorized_keys'][0:len(keys)] == keys
- else:
- if 'ssh_authorized_keys' in conf:
- keys = conf['ssh_authorized_keys']
- assert userdata['ssh_authorized_keys'][0:len(keys)] == keys
- else:
- assert 'ssh_authorized_keys' not in userdata
-
- @mark.parametrize(
- "conf",
- [
- dict(
- path='test_openstack_userdata_conf.yaml',
- runcmd_head=['uptime', 'date'],
- ),
- dict(
- path=None,
- ),
- ]
- )
- def test_userdata_conf_runcmd(self, conf):
- self.mocks['m_get_user_ssh_pubkey'].return_value = None
- obj = self.get_obj(test_conf=conf['path'])
- userdata = yaml.safe_load(obj.userdata)
- assert userdata['runcmd'][-2:] == [['passwd', '-d', 'ubuntu'], ['touch', '/.teuth_provisioned']]
-
- @mark.parametrize(
- "conf",
- [
- dict(
- path='test_openstack_userdata_conf.yaml',
- packages=None,
- ),
- dict(
- path=None,
- packages=['git', 'wget', 'python', 'ntp']
- ),
- ]
- )
- def test_userdata_conf_packages(self, conf):
- self.mocks['m_get_user_ssh_pubkey'].return_value = None
- obj = self.get_obj(test_conf=conf['path'])
- userdata = yaml.safe_load(obj.userdata)
- assert userdata.get('packages', None) == conf['packages']
-
-class TestOpenStackProvisioner(TestOpenStackBase):
- klass = cloud.openstack.OpenStackProvisioner
-
- def get_obj(
- self, name='node_name', os_type='ubuntu',
- os_version='16.04', conf=None):
- return cloud.get_provisioner(
- node_type='my_provider',
- name=name,
- os_type=os_type,
- os_version=os_version,
- conf=conf,
- )
-
- def test_init(self):
- with patch.object(
- self.klass,
- '_read_conf',
- ) as m_read_conf:
- self.get_obj()
- assert len(m_read_conf.call_args_list) == 1
-
- @mark.parametrize(
- 'input_conf',
- [
- dict(machine=dict(
- disk=42,
- ram=9001,
- cpus=3,
- )),
- dict(volumes=dict(
- count=3,
- size=100,
- )),
- dict(),
- dict(
- machine=dict(
- disk=1,
- ram=2,
- cpus=3,
- ),
- volumes=dict(
- count=4,
- size=5,
- )
- ),
- dict(
- machine=dict(
- disk=100,
- ),
- ),
- ]
- )
- def test_read_conf(self, input_conf):
- obj = self.get_obj(conf=input_conf)
- for topic in ['machine', 'volumes']:
- combined = cloud.util.combine_dicts(
- [input_conf, config.openstack],
- lambda x, y: x > y,
- )
- assert obj.conf[topic] == combined[topic]
-
- @mark.parametrize(
- 'input_conf, expected_machine, expected_vols',
- [
- [
- dict(openstack=[
- dict(machine=dict(disk=64, ram=10000, cpus=3)),
- dict(volumes=dict(count=1, size=1)),
- ]),
- dict(disk=64, ram=10000, cpus=3),
- dict(count=1, size=1),
- ],
- [
- dict(openstack=[
- dict(machine=dict(cpus=3)),
- dict(machine=dict(disk=1, ram=9000)),
- dict(machine=dict(disk=50, ram=2, cpus=1)),
- dict(machine=dict()),
- dict(volumes=dict()),
- dict(volumes=dict(count=0, size=0)),
- dict(volumes=dict(count=1, size=0)),
- dict(volumes=dict(size=1)),
- ]),
- dict(disk=50, ram=9000, cpus=3),
- dict(count=1, size=1),
- ],
- [
- dict(openstack=[
- dict(volumes=dict(count=3, size=30)),
- dict(volumes=dict(size=50)),
- ]),
- None,
- dict(count=3, size=50),
- ],
- [
- dict(openstack=[
- dict(machine=dict(disk=100)),
- dict(volumes=dict(count=3, size=30)),
- ]),
- dict(disk=100, ram=8000, cpus=1),
- dict(count=3, size=30),
- ],
- ]
- )
- def test_read_conf_legacy(
- self, input_conf, expected_machine, expected_vols):
- obj = self.get_obj(conf=input_conf)
- if expected_machine is not None:
- assert obj.conf['machine'] == expected_machine
- else:
- assert obj.conf['machine'] == config.openstack['machine']
- if expected_vols is not None:
- assert obj.conf['volumes'] == expected_vols
-
- @mark.parametrize(
- "os_type, os_version, should_find",
- [
- ('centos', '7', True),
- ('BeOS', '42', False),
- ]
- )
- def test_image(self, os_type, os_version, should_find):
- image_attrs = [
- dict(name='ubuntu-14.04'),
- dict(name='ubuntu-16.04'),
- dict(name='centos-7.0'),
- ]
- fake_images = list()
- for item in image_attrs:
- fake_images.append(
- get_fake_obj(attributes=item)
- )
- obj = self.get_obj(os_type=os_type, os_version=os_version)
- self.mocks['m_list_images'].return_value = fake_images
- if should_find:
- assert obj.os_version in obj.image.name
- assert obj.image in fake_images
- else:
- with raises(RuntimeError):
- obj.image
-
- @mark.parametrize(
- "input_attrs, func_or_exc",
- [
- (dict(ram=2**16),
- lambda s: s.ram == 2**16),
- (dict(disk=9999),
- lambda s: s.disk == 9999),
- (dict(cpus=99),
- lambda s: s.vcpus == 99),
- (dict(ram=2**16, disk=9999, cpus=99),
- IndexError),
- ]
- )
- def test_size(self, input_attrs, func_or_exc):
- size_attrs = [
- dict(ram=8000, disk=9999, vcpus=99, name='s0'),
- dict(ram=2**16, disk=20, vcpus=99, name='s1'),
- dict(ram=2**16, disk=9999, vcpus=1, name='s2'),
- ]
- fake_sizes = list()
- for item in size_attrs:
- fake_sizes.append(
- get_fake_obj(attributes=item)
- )
- base_spec = dict(machine=dict(
- ram=1,
- disk=1,
- cpus=1,
- ))
- spec = deepcopy(base_spec)
- spec['machine'].update(input_attrs)
- obj = self.get_obj(conf=spec)
- self.mocks['m_list_sizes'].return_value = fake_sizes
- if isinstance(func_or_exc, type):
- with raises(func_or_exc):
- obj.size
- else:
- assert obj.size in fake_sizes
- assert func_or_exc(obj.size) is True
-
- @mark.parametrize(
- "wanted_groups",
- [
- ['group1'],
- ['group0', 'group2'],
- [],
- ]
- )
- def test_security_groups(self, wanted_groups):
- group_names = ['group0', 'group1', 'group2']
- fake_groups = list()
- for name in group_names:
- fake_groups.append(
- get_fake_obj(attributes=dict(name=name))
- )
- self.mocks['m_ex_list_security_groups'].return_value = fake_groups
- obj = self.get_obj()
- assert obj.security_groups is None
- obj = self.get_obj()
- obj.provider.conf['security_groups'] = wanted_groups
- assert [g.name for g in obj.security_groups] == wanted_groups
-
- def test_security_groups_exc(self):
- fake_groups = [
- get_fake_obj(attributes=dict(name='sg')) for i in range(2)
- ]
- obj = self.get_obj()
- obj.provider.conf['security_groups'] = ['sg']
- with raises(RuntimeError):
- obj.security_groups
- self.mocks['m_ex_list_security_groups'].return_value = fake_groups
- obj = self.get_obj()
- obj.provider.conf['security_groups'] = ['sg']
- with raises(RuntimeError):
- obj.security_groups
-
- @mark.parametrize(
- "ssh_key",
- [
- 'my_ssh_key',
- None,
- ]
- )
- def test_userdata(self, ssh_key):
- self.mocks['m_get_user_ssh_pubkey'].return_value = ssh_key
- obj = self.get_obj()
- userdata = yaml.safe_load(obj.userdata)
- assert userdata['user'] == obj.user
- assert userdata['hostname'] == obj.hostname
- if ssh_key:
- assert userdata['ssh_authorized_keys'] == [ssh_key]
- else:
- assert 'ssh_authorized_keys' not in userdata
-
- @mark.parametrize(
- 'wanted_name, should_find, exception',
- [
- ('node0', True, None),
- ('node1', True, None),
- ('node2', False, RuntimeError),
- ('node3', False, None),
- ]
- )
- def test_node(self, wanted_name, should_find, exception):
- node_names = ['node0', 'node1', 'node2', 'node2']
- fake_nodes = list()
- for name in node_names:
- fake_nodes.append(
- get_fake_obj(attributes=dict(name=name))
- )
- self.mocks['m_list_nodes'].return_value = fake_nodes
- obj = self.get_obj(name=wanted_name)
- if should_find:
- assert obj.node.name == wanted_name
- elif exception:
- with raises(exception) as excinfo:
- obj.node
- assert excinfo.value.message
- else:
- assert obj.node is None
-
- @mark.parametrize(
- 'networks, security_groups',
- [
- ([], []),
- (['net0'], []),
- ([], ['sg0']),
- (['net0'], ['sg0']),
- ]
- )
- def test_create(self, networks, security_groups):
- node_name = 'node0'
- fake_sizes = [
- get_fake_obj(
- attributes=dict(ram=2**16, disk=9999, vcpus=99, name='s0')),
- ]
- fake_security_groups = [
- get_fake_obj(attributes=dict(name=name))
- for name in security_groups
- ]
- self.mocks['m_ex_list_networks'].return_value = networks
- self.mocks['m_ex_list_security_groups'].return_value = \
- fake_security_groups
- self.mocks['m_list_sizes'].return_value = fake_sizes
- fake_images = [
- get_fake_obj(attributes=dict(name='ubuntu-16.04')),
- ]
- self.mocks['m_list_images'].return_value = fake_images
- self.mocks['m_get_user_ssh_pubkey'].return_value = 'ssh_key'
- fake_node = get_fake_obj(attributes=dict(name=node_name))
- fake_ips = ['555.123.4.0']
- self.mocks['m_create_node'].return_value = fake_node
- self.mocks['m_wait_until_running'].return_value = \
- [(fake_node, fake_ips)]
- obj = self.get_obj(name=node_name)
- obj._networks = networks
- obj.provider.conf['security_groups'] = security_groups
- p_wait_for_ready = patch(
- 'teuthology.provision.cloud.openstack.OpenStackProvisioner'
- '._wait_for_ready'
- )
- with p_wait_for_ready:
- res = obj.create()
- assert res is obj.node
- # Test once again to ensure that if volume creation/attachment fails,
- # we destroy any remaining volumes and consider the node creation to
- # have failed as well.
- del obj._node
- with p_wait_for_ready:
- obj.conf['volumes']['count'] = 1
- obj.provider.driver.create_volume.side_effect = Exception
- with patch.object(obj, '_destroy_volumes'):
- assert obj.create() is False
- obj._destroy_volumes.assert_called_once_with()
-
- def test_update_dns(self):
- config.nsupdate_url = 'nsupdate_url'
- obj = self.get_obj()
- obj.name = 'x'
- obj.ips = ['y']
- obj._update_dns()
- call_args = self.mocks['m_get'].call_args_list
- assert len(call_args) == 1
- url_base, query_string = call_args[0][0][0].split('?')
- assert url_base == 'nsupdate_url'
- parsed_query = parse_qs(query_string)
- assert parsed_query == dict(name=['x'], ip=['y'])
-
- @mark.parametrize(
- 'nodes',
- [[], [Mock()], [Mock(), Mock()]]
- )
- def test_destroy(self, nodes):
- with patch(
- 'teuthology.provision.cloud.openstack.'
- 'OpenStackProvisioner._find_nodes'
- ) as m_find_nodes:
- m_find_nodes.return_value = nodes
- obj = self.get_obj()
- result = obj.destroy()
- if not all(nodes):
- assert result is True
- else:
- for node in nodes:
- node.destroy.assert_called_once_with()
-
- _volume_matrix = (
- 'count, size, should_succeed',
- [
- (1, 10, True),
- (0, 10, True),
- (10, 1, True),
- (1, 10, False),
- (10, 1, False),
- ]
- )
-
- @mark.parametrize(*_volume_matrix)
- def test_create_volumes(self, count, size, should_succeed):
- obj_conf = dict(volumes=dict(count=count, size=size))
- obj = self.get_obj(conf=obj_conf)
- node = get_fake_obj()
- if not should_succeed:
- obj.provider.driver.create_volume.side_effect = Exception
- obj._node = node
- result = obj._create_volumes()
- assert result is should_succeed
- if should_succeed:
- create_calls = obj.provider.driver.create_volume.call_args_list
- attach_calls = obj.provider.driver.attach_volume.call_args_list
- assert len(create_calls) == count
- assert len(attach_calls) == count
- for i in range(count):
- vol_size, vol_name = create_calls[i][0]
- assert vol_size == size
- assert vol_name == '%s_%s' % (obj.name, i)
- assert attach_calls[i][0][0] is obj._node
- assert attach_calls[i][1]['device'] is None
-
- @mark.parametrize(*_volume_matrix)
- def test_destroy_volumes(self, count, size, should_succeed):
- obj_conf = dict(volumes=dict(count=count, size=size))
- obj = self.get_obj(conf=obj_conf)
- fake_volumes = list()
- for i in range(count):
- vol_name = '%s_%s' % (obj.name, i)
- fake_volumes.append(
- get_fake_obj(attributes=dict(name=vol_name))
- )
- obj.provider.driver.list_volumes.return_value = fake_volumes
- obj._destroy_volumes()
- detach_calls = obj.provider.driver.detach_volume.call_args_list
- destroy_calls = obj.provider.driver.destroy_volume.call_args_list
- assert len(detach_calls) == count
- assert len(destroy_calls) == count
- assert len(obj.provider.driver.detach_volume.call_args_list) == count
- assert len(obj.provider.driver.destroy_volume.call_args_list) == count
- obj.provider.driver.detach_volume.reset_mock()
- obj.provider.driver.destroy_volume.reset_mock()
- obj.provider.driver.detach_volume.side_effect = Exception
- obj.provider.driver.destroy_volume.side_effect = Exception
- obj._destroy_volumes()
- assert len(obj.provider.driver.detach_volume.call_args_list) == count
- assert len(obj.provider.driver.destroy_volume.call_args_list) == count
-
- def test_destroy_volumes_exc(self):
- obj = self.get_obj()
- obj.provider.driver.detach_volume.side_effect = Exception
-
- def test_wait_for_ready(self):
- obj = self.get_obj()
- obj._node = get_fake_obj(attributes=dict(name='node_name'))
- with patch.multiple(
- 'teuthology.orchestra.remote.Remote',
- connect=DEFAULT,
- run=DEFAULT,
- ) as mocks:
- obj._wait_for_ready()
- mocks['connect'].side_effect = socket.error
- with raises(MaxWhileTries):
- obj._wait_for_ready()
+++ /dev/null
-libcloud:
- providers:
- my_provider:
- allow_networks:
- - sesci
- userdata:
- 'ubuntu-16.04':
- bootcmd:
- - 'SuSEfirewall2 stop || true'
- - 'service firewalld stop || true'
- runcmd:
- - 'uptime'
- - 'date'
- - 'zypper in -y lsb-release make gcc gcc-c++ chrony || true'
- - 'systemctl enable chronyd.service || true'
- - 'systemctl start chronyd.service || true'
- ssh_authorized_keys:
- - user_public_key1
- - user_public_key2
- driver: openstack
- driver_args:
- username: user
- password: password
- ex_force_auth_url: 'http://127.0.0.1:9999/v2.0/tokens'
+++ /dev/null
-from mock import Mock, MagicMock, patch
-
-from teuthology import provision
-
-
-class TestDownburst(object):
- def setup_method(self):
- self.ctx = Mock()
- self.ctx.os_type = 'rhel'
- self.ctx.os_version = '7.0'
- self.ctx.config = dict()
- self.name = 'vpm999'
- self.status = dict(
- vm_host=dict(name='host999'),
- is_vm=True,
- machine_type='mtype',
- locked_by='user@a',
- description="desc",
- )
-
- def test_create_if_vm_success(self):
- name = self.name
- ctx = self.ctx
- status = self.status
-
- dbrst = provision.downburst.Downburst(
- name, ctx.os_type, ctx.os_version, status)
- dbrst.executable = '/fake/path'
- dbrst.build_config = MagicMock(name='build_config')
- dbrst._run_create = MagicMock(name='_run_create')
- dbrst._run_create.return_value = (0, '', '')
- remove_config = MagicMock(name='remove_config')
- dbrst.remove_config = remove_config
-
- result = provision.create_if_vm(ctx, name, dbrst)
- assert result is True
-
- dbrst._run_create.assert_called_with()
- dbrst.build_config.assert_called_with()
- del dbrst
- remove_config.assert_called_with()
-
- def test_destroy_if_vm_success(self):
- name = self.name
- ctx = self.ctx
- status = self.status
-
- dbrst = provision.downburst.Downburst(
- name, ctx.os_type, ctx.os_version, status)
- dbrst.destroy = MagicMock(name='destroy')
- dbrst.destroy.return_value = True
-
- result = provision.destroy_if_vm(name, user="user@a", _downburst=dbrst)
- assert result is True
-
- dbrst.destroy.assert_called_with()
-
- def test_destroy_if_vm_wrong_owner(self):
- name = self.name
- ctx = self.ctx
- status = self.status
-
- dbrst = provision.downburst.Downburst(
- name, ctx.os_type, ctx.os_version, status)
- dbrst.destroy = MagicMock(name='destroy', side_effect=RuntimeError)
-
- result = provision.destroy_if_vm(name, user='user@b',
- _downburst=dbrst)
- assert result is False
-
- def test_destroy_if_vm_wrong_description(self):
- name = self.name
- ctx = self.ctx
- status = self.status
-
- dbrst = provision.downburst.Downburst(
- name, ctx.os_type, ctx.os_version, status)
- dbrst.destroy = MagicMock(name='destroy')
- dbrst.destroy = MagicMock(name='destroy', side_effect=RuntimeError)
-
- result = provision.destroy_if_vm(name, description='desc_b',
- _downburst=dbrst)
- assert result is False
-
- @patch('teuthology.provision.downburst.downburst_executable')
- def test_create_fails_without_executable(self, m_exec):
- name = self.name
- ctx = self.ctx
- status = self.status
- m_exec.return_value = ''
- dbrst = provision.downburst.Downburst(
- name, ctx.os_type, ctx.os_version, status)
- result = dbrst.create()
- assert result is False
-
- @patch('teuthology.provision.downburst.downburst_executable')
- def test_destroy_fails_without_executable(self, m_exec):
- name = self.name
- ctx = self.ctx
- status = self.status
- m_exec.return_value = ''
- dbrst = provision.downburst.Downburst(
- name, ctx.os_type, ctx.os_version, status)
- result = dbrst.destroy()
- assert result is False
+++ /dev/null
-import datetime
-
-from copy import deepcopy
-from mock import patch, DEFAULT, PropertyMock
-from pytest import raises, mark
-
-from teuthology.config import config
-from teuthology.exceptions import MaxWhileTries, CommandFailedError
-from teuthology.provision import fog
-
-
-test_config = dict(
- fog=dict(
- endpoint='http://fog.example.com/fog',
- api_token='API_TOKEN',
- user_token='USER_TOKEN',
- machine_types='type1,type2',
- ),
- fog_reimage_timeout=1800,
-)
-
-
-class TestFOG(object):
- klass = fog.FOG
-
- def setup_method(self):
- config.load()
- config.update(deepcopy(test_config))
- self.start_patchers()
-
- def start_patchers(self):
- self.patchers = dict()
- self.patchers['m_sleep'] = patch(
- 'time.sleep',
- )
- self.patchers['m_requests_Session_send'] = patch(
- 'requests.Session.send',
- )
- self.patchers['m_Remote_connect'] = patch(
- 'teuthology.orchestra.remote.Remote.connect'
- )
- self.patchers['m_Remote_run'] = patch(
- 'teuthology.orchestra.remote.Remote.run'
- )
- self.patchers['m_Remote_console'] = patch(
- 'teuthology.orchestra.remote.Remote.console',
- new_callable=PropertyMock,
- )
- self.patchers['m_Remote_hostname'] = patch(
- 'teuthology.orchestra.remote.Remote.hostname',
- new_callable=PropertyMock,
- )
- self.patchers['m_Remote_machine_type'] = patch(
- 'teuthology.orchestra.remote.Remote.machine_type',
- new_callable=PropertyMock,
- )
- self.mocks = dict()
- for name, patcher in self.patchers.items():
- self.mocks[name] = patcher.start()
-
- def teardown_method(self):
- for patcher in self.patchers.values():
- patcher.stop()
-
- @mark.parametrize('enabled', [True, False])
- def test_get_types(self, enabled):
- with patch('teuthology.provision.fog.enabled') as m_enabled:
- m_enabled.return_value = enabled
- types = fog.get_types()
- if enabled:
- assert types == test_config['fog']['machine_types'].split(',')
- else:
- assert types == []
-
- def test_disabled(self):
- config.fog['endpoint'] = None
- obj = self.klass('name.fqdn', 'type', '1.0')
- with raises(RuntimeError):
- obj.create()
-
- def test_init(self):
- self.mocks['m_Remote_hostname'].return_value = 'name.fqdn'
- obj = self.klass('name.fqdn', 'type', '1.0')
- assert obj.name == 'name.fqdn'
- assert obj.shortname == 'name'
- assert obj.os_type == 'type'
- assert obj.os_version == '1.0'
-
- @mark.parametrize('success', [True, False])
- def test_create(self, success):
- self.mocks['m_Remote_hostname'].return_value = 'name.fqdn'
- self.mocks['m_Remote_machine_type'].return_value = 'type1'
- obj = self.klass('name.fqdn', 'type', '1.0')
- host_id = 99
- task_id = 1234
- with patch.multiple(
- 'teuthology.provision.fog.FOG',
- get_host_data=DEFAULT,
- set_image=DEFAULT,
- schedule_deploy_task=DEFAULT,
- wait_for_deploy_task=DEFAULT,
- cancel_deploy_task=DEFAULT,
- _wait_for_ready=DEFAULT,
- _fix_hostname=DEFAULT,
- _verify_installed_os=DEFAULT,
- ) as local_mocks:
- local_mocks['get_host_data'].return_value = dict(id=host_id)
- local_mocks['schedule_deploy_task'].return_value = task_id
- if not success:
- local_mocks['wait_for_deploy_task'].side_effect = RuntimeError
- with raises(RuntimeError):
- obj.create()
- else:
- obj.create()
- local_mocks['get_host_data'].assert_called_once_with()
- local_mocks['set_image'].assert_called_once_with(host_id)
- local_mocks['schedule_deploy_task'].assert_called_once_with(host_id)
- local_mocks['wait_for_deploy_task'].assert_called_once_with(task_id)
- if success:
- local_mocks['_wait_for_ready'].assert_called_once_with()
- local_mocks['_fix_hostname'].assert_called_once_with()
- else:
- assert len(local_mocks['cancel_deploy_task'].call_args_list) == 1
- self.mocks['m_Remote_console'].return_value.power_off.assert_called_once_with()
- self.mocks['m_Remote_console'].return_value.power_on.assert_called_once_with()
-
- def test_do_request(self):
- obj = self.klass('name.fqdn', 'type', '1.0')
- obj.do_request('test_url', data='DATA', method='GET')
- assert len(self.mocks['m_requests_Session_send'].call_args_list) == 1
- req = self.mocks['m_requests_Session_send'].call_args_list[0][0][0]
- assert req.url == test_config['fog']['endpoint'] + 'test_url'
- assert req.method == 'GET'
- assert req.headers['fog-api-token'] == test_config['fog']['api_token']
- assert req.headers['fog-user-token'] == test_config['fog']['user_token']
- assert req.body == 'DATA'
-
- @mark.parametrize(
- 'count',
- [0, 1, 2],
- )
- def test_get_host_data(self, count):
- host_objs = [dict(id=i) for i in range(count)]
- resp_obj = dict(count=count, hosts=host_objs)
- self.mocks['m_requests_Session_send']\
- .return_value.json.return_value = resp_obj
- obj = self.klass('name.fqdn', 'type', '1.0')
- if count != 1:
- with raises(RuntimeError):
- result = obj.get_host_data()
- return
- result = obj.get_host_data()
- assert len(self.mocks['m_requests_Session_send'].call_args_list) == 1
- req = self.mocks['m_requests_Session_send'].call_args_list[0][0][0]
- assert req.url == test_config['fog']['endpoint'] + '/host'
- assert req.body == '{"name": "name"}'
- assert result == host_objs[0]
-
- @mark.parametrize(
- 'count',
- [0, 1, 2],
- )
- def test_get_image_data(self, count):
- img_objs = [dict(id=i) for i in range(count)]
- resp_obj = dict(count=count, images=img_objs)
- self.mocks['m_requests_Session_send']\
- .return_value.json.return_value = resp_obj
- self.mocks['m_Remote_machine_type'].return_value = 'type1'
- obj = self.klass('name.fqdn', 'windows', 'xp')
- if count < 1:
- with raises(RuntimeError):
- result = obj.get_image_data()
- return
- result = obj.get_image_data()
- assert len(self.mocks['m_requests_Session_send'].call_args_list) == 1
- req = self.mocks['m_requests_Session_send'].call_args_list[0][0][0]
- assert req.url == test_config['fog']['endpoint'] + '/image'
- assert req.body == '{"name": "type1_windows_xp"}'
- assert result == img_objs[0]
-
- def test_suggest_image_names(self):
- data = {'images': [
- {'name': 'mira_rhel_9.1'},
- {'name': 'mira_rhel_9.2'},
- ]}
- self.mocks['m_requests_Session_send']\
- .return_value.json.return_value = data
- self.mocks['m_Remote_machine_type'].return_value = 'mira'
- # Not sure what this klass() is for here:
- obj = self.klass('name.fqdn', 'mira', '1.0')
- result = obj.suggest_image_names()
- assert len(self.mocks['m_requests_Session_send'].call_args_list) == 1
- req = self.mocks['m_requests_Session_send'].call_args_list[0][0][0]
- assert req.url == test_config['fog']['endpoint'] + '/image/search/mira'
- assert result == ['mira_rhel_9.1', 'mira_rhel_9.2']
-
- def test_set_image(self):
- self.mocks['m_Remote_hostname'].return_value = 'name.fqdn'
- self.mocks['m_Remote_machine_type'].return_value = 'type1'
- host_id = 999
- obj = self.klass('name.fqdn', 'type', '1.0')
- with patch.multiple(
- 'teuthology.provision.fog.FOG',
- get_image_data=DEFAULT,
- do_request=DEFAULT,
- ) as local_mocks:
- local_mocks['get_image_data'].return_value = dict(id='13')
- obj.set_image(host_id)
- local_mocks['do_request'].assert_called_once_with(
- '/host/999', method='PUT', data='{"imageID": 13}',
- )
-
- def test_schedule_deploy_task(self):
- host_id = 12
- tasktype_id = 6
- task_id = 5
- tasktype_result = dict(tasktypes=[dict(name='deploy', id=tasktype_id)])
- schedule_result = dict()
- host_tasks = [dict(
- createdTime=datetime.datetime.strftime(
- datetime.datetime.now(datetime.timezone.utc),
- self.klass.timestamp_format
- ),
- id=task_id,
- )]
- self.mocks['m_requests_Session_send']\
- .return_value.json.side_effect = [
- tasktype_result, schedule_result,
- ]
- with patch.multiple(
- 'teuthology.provision.fog.FOG',
- get_deploy_tasks=DEFAULT,
- ) as local_mocks:
- local_mocks['get_deploy_tasks'].return_value = host_tasks
- obj = self.klass('name.fqdn', 'type', '1.0')
- result = obj.schedule_deploy_task(host_id)
- assert len(local_mocks['get_deploy_tasks'].call_args_list) == 2
- assert len(self.mocks['m_requests_Session_send'].call_args_list) == 3
- assert result == task_id
-
- def test_get_deploy_tasks(self):
- obj = self.klass('name.fqdn', 'type', '1.0')
- resp_obj = dict(
- count=2,
- tasks=[
- dict(host=dict(name='notme')),
- dict(host=dict(name='name')),
- ]
- )
- self.mocks['m_requests_Session_send']\
- .return_value.json.return_value = resp_obj
- result = obj.get_deploy_tasks()
- assert result[0]['host']['name'] == 'name'
-
- @mark.parametrize(
- 'active_ids',
- [
- [2, 4, 6, 8],
- [1],
- [],
- ]
- )
- def test_deploy_task_active(self, active_ids):
- our_task_id = 4
- result_objs = [dict(id=task_id) for task_id in active_ids]
- obj = self.klass('name.fqdn', 'type', '1.0')
- with patch.multiple(
- 'teuthology.provision.fog.FOG',
- get_deploy_tasks=DEFAULT,
- ) as local_mocks:
- local_mocks['get_deploy_tasks'].return_value = result_objs
- result = obj.deploy_task_active(our_task_id)
- assert result is (our_task_id in active_ids)
-
- @mark.parametrize(
- 'tries',
- [3, 121],
- )
- def test_wait_for_deploy_task(self, tries):
- wait_results = [True for i in range(tries)] + [False]
- obj = self.klass('name.fqdn', 'type', '1.0')
- with patch.multiple(
- 'teuthology.provision.fog.FOG',
- deploy_task_active=DEFAULT,
- ) as local_mocks:
- local_mocks['deploy_task_active'].side_effect = wait_results
- if tries >= 60:
- with raises(MaxWhileTries):
- obj.wait_for_deploy_task(9)
- return
- obj.wait_for_deploy_task(9)
- assert len(local_mocks['deploy_task_active'].call_args_list) == \
- tries + 1
-
- def test_cancel_deploy_task(self):
- obj = self.klass('name.fqdn', 'type', '1.0')
- with patch.multiple(
- 'teuthology.provision.fog.FOG',
- do_request=DEFAULT,
- ) as local_mocks:
- obj.cancel_deploy_task(10)
- local_mocks['do_request'].assert_called_once_with(
- '/task/cancel',
- method='DELETE',
- data='{"id": 10}',
- )
-
- @mark.parametrize(
- 'tries',
- [1, 101],
- )
- def test_wait_for_ready_tries(self, tries):
- connect_results = [MaxWhileTries for i in range(tries)] + [True]
- obj = self.klass('name.fqdn', 'type', '1.0')
- self.mocks['m_Remote_connect'].side_effect = connect_results
- if tries >= 100:
- with raises(MaxWhileTries):
- obj._wait_for_ready()
- return
- obj._wait_for_ready()
- assert len(self.mocks['m_Remote_connect'].call_args_list) == tries + 1
-
- @mark.parametrize(
- 'sentinel_present',
- ([False, True]),
- )
- def test_wait_for_ready_sentinel(self, sentinel_present):
- config.fog['sentinel_file'] = '/a_file'
- obj = self.klass('name.fqdn', 'type', '1.0')
- if not sentinel_present:
- self.mocks['m_Remote_run'].side_effect = [
- CommandFailedError(command='cmd', exitstatus=1)]
- with raises(CommandFailedError):
- obj._wait_for_ready()
- else:
- obj._wait_for_ready()
- assert len(self.mocks['m_Remote_run'].call_args_list) == 1
- assert "'/a_file'" in \
- self.mocks['m_Remote_run'].call_args_list[0][1]['args']
+++ /dev/null
-from copy import deepcopy
-from pytest import raises
-from teuthology.config import config
-
-import teuthology.provision
-
-test_config = dict(
- pelagos=dict(
- endpoint='http://pelagos.example:5000/',
- machine_types='ptype1,ptype2,common_type',
- ),
- fog=dict(
- endpoint='http://fog.example.com/fog',
- api_token='API_TOKEN',
- user_token='USER_TOKEN',
- machine_types='ftype1,ftype2,common_type',
- )
-)
-
-class TestInitProvision(object):
-
- def setup_method(self):
- config.load(deepcopy(test_config))
-
- def test_get_reimage_types(self):
- reimage_types = teuthology.provision.get_reimage_types()
- assert reimage_types == ["ptype1", "ptype2", "common_type",
- "ftype1", "ftype2", "common_type"]
-
- def test_reimage(self):
- class context:
- pass
- ctx = context()
- ctx.os_type = 'sle'
- ctx.os_version = '15.1'
- with raises(Exception) as e_info:
- teuthology.provision.reimage(ctx, 'f.q.d.n.org', 'not-defined-type')
- e_str = str(e_info)
- print("Caught exception: " + e_str)
- assert e_str.find(r"configured\sprovisioners") == -1
-
- with raises(Exception) as e_info:
- teuthology.provision.reimage(ctx, 'f.q.d.n.org', 'common_type')
- e_str = str(e_info)
- print("Caught exception: " + e_str)
- assert e_str.find(r"used\swith\sone\sprovisioner\sonly") == -1
+++ /dev/null
-from copy import deepcopy
-from mock import patch, DEFAULT, PropertyMock
-from pytest import raises, mark
-from requests.models import Response
-
-from teuthology.config import config
-from teuthology.orchestra.opsys import OS
-from teuthology.provision import maas
-
-
-test_config = dict(
- maas=dict(
- api_url="http://maas.example.com:5240/MAAS/api/2.0/",
- api_key="CONSUMER_KEY:ACCESS_TOKEN:SECRET",
- machine_types=["typeA", "typeB"],
- timeout=900,
- user_data="teuthology/maas/maas-{os_type}-{os_version}-user-data.txt"
- )
-)
-
-class TestMaas(object):
- klass = maas.MAAS
-
- def setup_method(self):
- config.load()
- config.update(deepcopy(test_config))
- self.start_patchers()
-
- def start_patchers(self):
- self.patchers = dict()
- self.patchers["m_Remote_hostname"] = patch(
- "teuthology.orchestra.remote.Remote.hostname",
- new_callable=PropertyMock,
- )
- self.patchers["m_Remote_machine_os"] = patch(
- "teuthology.orchestra.remote.Remote.os",
- new_callable=PropertyMock,
- )
- self.patchers["m_Remote_opsys_os"] = patch(
- "teuthology.orchestra.opsys.OS",
- new_callable=PropertyMock,
- )
-
- self.mocks = dict()
- for name, patcher in self.patchers.items():
- self.mocks[name] = patcher.start()
-
- def teardown_method(self):
- for patcher in self.patchers.values():
- patcher.stop()
-
- def _get_mock_response(self, status_code=200, content=None, headers=None):
- response = Response()
- response.status_code = status_code
- response._content = content
- if headers is not None:
- response.headers = headers
- return response
-
- def test_api_url_missing(self):
- config.maas["api_url"] = None
- with raises(RuntimeError):
- self.klass("name.fqdn")
-
- def test_api_key_missing(self):
- config.maas["api_key"] = None
- with raises(RuntimeError):
- self.klass("name.fqdn")
-
- @mark.parametrize("enabled", [True, False])
- def test_get_types(self, enabled):
- with patch("teuthology.provision.maas.enabled") as m_enabled:
- m_enabled.return_value = enabled
- types = maas.get_types()
-
- if enabled:
- assert types == test_config["maas"]["machine_types"]
- else:
- assert types == []
-
- @mark.parametrize("status_name, osystem, distro_series", [
- ("ready", "ubuntu", "jammy"),
- ("deployed", "ubuntu", "jammy"),
- ])
- def test_init(self, status_name, osystem, distro_series):
- self.mocks["m_Remote_hostname"].return_value = "name.fqdn"
- with patch(
- "teuthology.provision.maas.MAAS.get_machines_data"
- ) as maas_machine:
- maas_machine.return_value = {
- "status_name": status_name,
- "system_id": "abc123",
- "osystem": osystem,
- "distro_series": distro_series,
- "architecture": "amd64/generic"
- }
- if not (osystem and distro_series):
- with raises(RuntimeError):
- obj = self.klass(
- name="name.fqdn",
- os_type=osystem,
- os_version=distro_series
- )
- else:
- obj = self.klass(
- name="name.fqdn",
- os_type=osystem,
- os_version=distro_series
- )
- assert obj.name == "name.fqdn"
- assert obj.shortname == "name"
- assert obj.os_type == osystem
- assert obj.os_version == distro_series
- assert obj.system_id == "abc123"
-
- @mark.parametrize("os_name, os_version, codename", [
- ("ubuntu", "24.04", "noble"), ("centos", "9.stream", "centos9-stream"),
- ])
- def test_get_image_data(self, os_name, os_version, codename):
- with patch.multiple(
- "teuthology.provision.maas.MAAS",
- do_request=DEFAULT,
- get_machines_data=DEFAULT,
- ) as local_mocks:
- local_mocks["do_request"].return_value = self._get_mock_response(
- content=b'[{"name": "%s/%s", "architecture": "amd64/generic"}]' % (
- os_name.encode(), codename.encode()
- )
- )
- local_mocks["get_machines_data"].return_value = {
- "status_name": "ready",
- "architecture": "amd64/generic",
- }
- obj = self.klass(
- name="name.fqdn", os_type=os_name, os_version=os_version
- )
- assert obj.get_image_data() == {
- "name": f"{os_name}/{codename}",
- "architecture": "amd64/generic",
- }
-
- def test_lock_machine(self):
- with patch.multiple(
- "teuthology.provision.maas.MAAS",
- do_request=DEFAULT,
- get_machines_data=DEFAULT,
- ) as local_mocks:
- local_mocks["get_machines_data"].return_value = {
- "system_id": "1234abc",
- "architecture": "amd64/generic",
- }
- local_mocks["do_request"].return_value = self._get_mock_response(
- content=b'{"locked": "true"}'
- )
- assert self.klass(name="name.fqdn").lock_machine() is None
-
- def test_unlock_machine(self):
- self.mocks["m_Remote_hostname"].return_value = "name.fqdn"
- with patch.multiple(
- "teuthology.provision.maas.MAAS",
- do_request=DEFAULT,
- get_machines_data=DEFAULT,
- ) as local_mocks:
- local_mocks["get_machines_data"].return_value = {
- "system_id": "1234abc",
- "architecture": "amd64/generic",
- }
- local_mocks["do_request"].return_value = self._get_mock_response(
- content=b'{"locked": "true"}'
- )
- with raises(RuntimeError):
- self.klass(name="name.fqdn").unlock_machine()
-
- @mark.parametrize("type, status_name", [
- ("uploaded", ""), ("synced", "deploying"), ("synced", "ready"),
- ])
- def test_deploy_machine(self, type, status_name):
- with patch.multiple(
- "teuthology.provision.maas.MAAS",
- get_image_data=DEFAULT,
- _get_user_data=DEFAULT,
- get_machines_data=DEFAULT,
- do_request=DEFAULT,
- ) as local_mocks:
- local_mocks["get_machines_data"].return_value = {
- "system_id": "1234abc",
- "architecture": "amd64/generic",
- }
- local_mocks["get_image_data"].return_value = {
- "name": "ubuntu/noble", "type": type
- }
- local_mocks["_get_user_data"].return_value = "init-host-data"
- local_mocks["do_request"].return_value = self._get_mock_response(
- content=b'{"status_name": "%s"}' % status_name.encode()
- )
- obj = self.klass(name="name.fqdn")
- if (status_name != "deploying" or type != "synced"):
- with raises(RuntimeError):
- obj.deploy_machine()
- else:
- assert obj.deploy_machine() is None
-
- @mark.parametrize("response", [b'{}', ])
- def test_release_machine(self, response):
- with patch.multiple(
- "teuthology.provision.maas.MAAS",
- get_machines_data=DEFAULT,
- do_request=DEFAULT,
- ) as local_mocks:
- local_mocks["get_machines_data"].return_value = {
- "system_id": "1234abc",
- "architecture": "amd64/generic",
- "status_name": "ready"
- }
- local_mocks["do_request"].return_value = self._get_mock_response(
- content=b'{"status_name": "disk erasing"}'
- )
- assert self.klass(name="name.fqdn").release_machine() is None
-
- @mark.parametrize("status_name", ["deploying", "ready"])
- def test_abort_deploy(self, status_name):
- with patch.multiple(
- "teuthology.provision.maas.MAAS",
- get_machines_data=DEFAULT,
- do_request=DEFAULT,
- ) as local_mocks:
- local_mocks["get_machines_data"].return_value = {
- "system_id": "1234abc",
- "architecture": "amd64/generic",
- }
- if status_name == "deploying":
- local_mocks["do_request"].return_value = self._get_mock_response(
- content=b'{"status_name": "allocated"}'
- )
-
- assert self.klass(name="name.fqdn").abort_deploy() is None
-
- @mark.parametrize("lock_status, machine_status", [
- (False, "ready"), (False, "allocated"),
- (True, "deployed"), (False, "deployed"),
- (False, "releasing"),
- ])
- def test_release(self, lock_status, machine_status):
- with patch.multiple(
- "teuthology.provision.maas.MAAS",
- get_machines_data=DEFAULT,
- unlock_machine=DEFAULT,
- release_machine=DEFAULT,
- _wait_for_status=DEFAULT,
- ) as local_mocks:
- local_mocks["get_machines_data"].return_value = {
- "status_name": machine_status,
- "locked": lock_status,
- "system_id": "1234abc",
- "architecture": "amd64/generic",
- }
- obj = self.klass(name="name.fqdn")
- if machine_status in ["ready", "allocated"]:
- obj.release()
-
- elif machine_status == "deployed":
- local_mocks["unlock_machine"].return_value = None
- local_mocks["release_machine"].return_value = None
- local_mocks["_wait_for_status"].return_value = None
- obj.release()
-
- else:
- with raises(RuntimeError):
- obj.release()
-
- @mark.parametrize("user_data", [True, False])
- def test_get_user_data(self, user_data):
- with patch(
- "teuthology.provision.maas.MAAS.get_machines_data"
- ) as maas_machine:
- maas_machine.return_value = {
- "system_id": "1234abc",
- "architecture": "amd64/generic",
- }
- obj = self.klass(name="name.fqdn")
- if user_data:
- assert user_data is not None
- else:
- config.maas["user_data"] = None
- user_data = obj._get_user_data()
- assert user_data is None
-
- @mark.parametrize(
- "os_type, os_version, remote_os_type, remote_os_version", [
- ("centos", "8", "centos", "8"),
- ("", "", "ubuntu", "22.04")
- ])
- def test_verify_installed_os(
- self, os_type, os_version, remote_os_type, remote_os_version
- ):
- self.mocks["m_Remote_machine_os"].return_value = OS(
- name=remote_os_type, version=remote_os_version
- )
- with patch(
- "teuthology.provision.maas.MAAS.get_machines_data"
- ) as maas_machine:
- maas_machine.return_value = {
- "system_id": "1234abc",
- "architecture": "amd64/generic"
- }
- if os_type and os_version:
- self.klass(
- name="name.fqdn", os_type=os_type, os_version=os_version
- )._verify_installed_os()
- else:
- self.klass(name="name.fqdn")._verify_installed_os()
+++ /dev/null
-from copy import deepcopy
-from pytest import raises
-from teuthology.config import config
-from teuthology.provision import pelagos
-
-import teuthology.provision
-
-
-test_config = dict(
- pelagos=dict(
- endpoint='http://pelagos.example:5000/',
- machine_types='ptype1,ptype2',
- ),
-)
-
-class TestPelagos(object):
-
- def setup_method(self):
- config.load(deepcopy(test_config))
-
- def teardown_method(self):
- pass
-
- def test_get_types(self):
- #klass = pelagos.Pelagos
- types = pelagos.get_types()
- assert types == ["ptype1", "ptype2"]
-
- def test_disabled(self):
- config.pelagos['endpoint'] = None
- enabled = pelagos.enabled()
- assert enabled == False
-
- def test_pelagos(self):
- class context:
- pass
-
- ctx = context()
- ctx.os_type ='sle'
- ctx.os_version = '15.1'
- with raises(Exception) as e_info:
- teuthology.provision.reimage(ctx, 'f.q.d.n.org', 'ptype1')
- e_str = str(e_info)
- print("Caught exception: " + e_str)
- assert e_str.find(r"Name\sor\sservice\snot\sknown") == -1
-
+++ /dev/null
-from teuthology.config import config
-
-def pytest_runtest_setup():
- config.load({})
+++ /dev/null
-roles:
-- - mon.a
- - osd.0
-tasks:
-- exec:
- mon.a:
- - echo "Well done !"
+++ /dev/null
-import os
-import random
-
-from mock import patch, MagicMock
-
-from teuthology.suite import build_matrix
-from teuthology.test.fake_fs import make_fake_fstools
-
-
-class TestBuildMatrixSimple(object):
- def test_combine_path(self):
- result = build_matrix.combine_path("/path/to/left", "right/side")
- assert result == "/path/to/left/right/side"
-
- def test_combine_path_no_right(self):
- result = build_matrix.combine_path("/path/to/left", None)
- assert result == "/path/to/left"
-
-
-class TestBuildMatrix(object):
-
- patchpoints = [
- 'os.path.exists',
- 'os.listdir',
- 'os.path.isfile',
- 'os.path.isdir',
- 'builtins.open',
- ]
-
- def setup_method(self):
- self.mocks = dict()
- self.patchers = dict()
- for ppoint in self.__class__.patchpoints:
- self.mocks[ppoint] = MagicMock()
- self.patchers[ppoint] = patch(ppoint, self.mocks[ppoint])
-
- def start_patchers(self, fake_fs):
- fake_fns = make_fake_fstools(fake_fs)
- # N.B.: relies on fake_fns being in same order as patchpoints
- for ppoint, fn in zip(self.__class__.patchpoints, fake_fns):
- self.mocks[ppoint].side_effect = fn
- self.patchers[ppoint].start()
-
- def stop_patchers(self):
- for patcher in self.patchers.values():
- patcher.stop()
-
- def teardown_method(self):
- self.patchers.clear()
- self.mocks.clear()
-
- def fragment_occurences(self, jobs, fragment):
- # What fraction of jobs contain fragment?
- count = 0
- for (description, fragment_list) in jobs:
- for item in fragment_list:
- if item.endswith(fragment):
- count += 1
- return count / float(len(jobs))
-
- def test_concatenate_1x2x3(self):
- fake_fs = {
- 'd0_0': {
- '+': None,
- 'd1_0': {
- 'd1_0_0.yaml': None,
- },
- 'd1_1': {
- 'd1_1_0.yaml': None,
- 'd1_1_1.yaml': None,
- },
- 'd1_2': {
- 'd1_2_0.yaml': None,
- 'd1_2_1.yaml': None,
- 'd1_2_2.yaml': None,
- },
- },
- }
- self.start_patchers(fake_fs)
- try:
- result = build_matrix.build_matrix('d0_0')
- finally:
- self.stop_patchers()
- assert len(result) == 1
-
- def test_convolve_2x2(self):
- fake_fs = {
- 'd0_0': {
- '%': None,
- 'd1_0': {
- 'd1_0_0.yaml': None,
- 'd1_0_1.yaml': None,
- },
- 'd1_1': {
- 'd1_1_0.yaml': None,
- 'd1_1_1.yaml': None,
- },
- },
- }
- self.start_patchers(fake_fs)
- try:
- result = build_matrix.build_matrix('d0_0')
- finally:
- self.stop_patchers()
- assert len(result) == 4
- assert self.fragment_occurences(result, 'd1_1_1.yaml') == 0.5
-
- def test_convolve_2x2x2(self):
- fake_fs = {
- 'd0_0': {
- '%': None,
- 'd1_0': {
- 'd1_0_0.yaml': None,
- 'd1_0_1.yaml': None,
- },
- 'd1_1': {
- 'd1_1_0.yaml': None,
- 'd1_1_1.yaml': None,
- },
- 'd1_2': {
- 'd1_2_0.yaml': None,
- 'd1_2_1.yaml': None,
- },
- },
- }
- self.start_patchers(fake_fs)
- try:
- result = build_matrix.build_matrix('d0_0')
- finally:
- self.stop_patchers()
- assert len(result) == 8
- assert self.fragment_occurences(result, 'd1_2_0.yaml') == 0.5
-
- def test_convolve_1x2x4(self):
- fake_fs = {
- 'd0_0': {
- '%': None,
- 'd1_0': {
- 'd1_0_0.yaml': None,
- },
- 'd1_1': {
- 'd1_1_0.yaml': None,
- 'd1_1_1.yaml': None,
- },
- 'd1_2': {
- 'd1_2_0.yaml': None,
- 'd1_2_1.yaml': None,
- 'd1_2_2.yaml': None,
- 'd1_2_3.yaml': None,
- },
- },
- }
- self.start_patchers(fake_fs)
- try:
- result = build_matrix.build_matrix('d0_0')
- finally:
- self.stop_patchers()
- assert len(result) == 8
- assert self.fragment_occurences(result, 'd1_2_2.yaml') == 0.25
-
- def test_convolve_with_concat(self):
- fake_fs = {
- 'd0_0': {
- '%': None,
- 'd1_0': {
- 'd1_0_0.yaml': None,
- },
- 'd1_1': {
- 'd1_1_0.yaml': None,
- 'd1_1_1.yaml': None,
- },
- 'd1_2': {
- '+': None,
- 'd1_2_0.yaml': None,
- 'd1_2_1.yaml': None,
- 'd1_2_2.yaml': None,
- 'd1_2_3.yaml': None,
- },
- },
- }
- self.start_patchers(fake_fs)
- try:
- result = build_matrix.build_matrix('d0_0')
- finally:
- self.stop_patchers()
- assert len(result) == 2
- for i in result:
- assert 'd0_0/d1_2/d1_2_0.yaml' in i[1]
- assert 'd0_0/d1_2/d1_2_1.yaml' in i[1]
- assert 'd0_0/d1_2/d1_2_2.yaml' in i[1]
- assert 'd0_0/d1_2/d1_2_3.yaml' in i[1]
-
- def test_convolve_nested(self):
- fake_fs = {
- 'd0_0': {
- '%': None,
- 'd1_0': {
- 'd1_0_0.yaml': None,
- '%': '2',
- 'd1_0_1': {
- 'd1_0_1_0.yaml': None,
- 'd1_0_1_1.yaml': None,
- },
- 'd1_0_2': {
- 'd1_0_2_0.yaml': None,
- 'd1_0_2_1.yaml': None,
- },
- },
- 'd1_2': {
- 'd1_2_0.yaml': None,
- 'd1_2_1.yaml': None,
- 'd1_2_2.yaml': None,
- 'd1_2_3.yaml': None,
- },
- },
- }
- self.start_patchers(fake_fs)
- try:
- result = build_matrix.build_matrix('d0_0')
- finally:
- self.stop_patchers()
- assert len(result) == 8
- assert self.fragment_occurences(result, 'd1_0_0.yaml') == 1
- assert self.fragment_occurences(result, 'd1_0_1_0.yaml') == 0.5
- assert self.fragment_occurences(result, 'd1_0_1_1.yaml') == 0.5
- assert self.fragment_occurences(result, 'd1_0_2_0.yaml') == 0.5
- assert self.fragment_occurences(result, 'd1_0_2_1.yaml') == 0.5
- assert self.fragment_occurences(result, 'd1_2_0.yaml') == 0.25
- assert self.fragment_occurences(result, 'd1_2_1.yaml') == 0.25
- assert self.fragment_occurences(result, 'd1_2_2.yaml') == 0.25
- assert self.fragment_occurences(result, 'd1_2_3.yaml') == 0.25
-
-
- def test_random_dollar_sign_2x2x3(self):
- fake_fs = {
- 'd0_0': {
- '$': None,
- 'd1_0': {
- 'd1_0_0.yaml': None,
- 'd1_0_1.yaml': None,
- },
- 'd1_1': {
- 'd1_1_0.yaml': None,
- 'd1_1_1.yaml': None,
- },
- 'd1_2': {
- 'd1_2_0.yaml': None,
- 'd1_2_1.yaml': None,
- 'd1_2_2.yaml': None,
- },
- },
- }
- fake_fs1 = {
- 'd0_0$': {
- 'd1_0': {
- 'd1_0_0.yaml': None,
- 'd1_0_1.yaml': None,
- },
- 'd1_1': {
- 'd1_1_0.yaml': None,
- 'd1_1_1.yaml': None,
- },
- 'd1_2': {
- 'd1_2_0.yaml': None,
- 'd1_2_1.yaml': None,
- 'd1_2_2.yaml': None,
- },
- },
- }
- self.start_patchers(fake_fs)
- try:
- result = build_matrix.build_matrix('d0_0')
- finally:
- self.stop_patchers()
- assert len(result) == 1
- self.start_patchers(fake_fs1)
- try:
- result = build_matrix.build_matrix('d0_0$')
- finally:
- self.stop_patchers()
- assert len(result) == 1
-
- def test_random_dollar_sign_with_concat(self):
- fake_fs = {
- 'd0_0': {
- '$': None,
- 'd1_0': {
- 'd1_0_0.yaml': None,
- },
- 'd1_1': {
- 'd1_1_0.yaml': None,
- 'd1_1_1.yaml': None,
- },
- 'd1_2': {
- '+': None,
- 'd1_2_0.yaml': None,
- 'd1_2_1.yaml': None,
- 'd1_2_2.yaml': None,
- 'd1_2_3.yaml': None,
- },
- },
- }
- fake_fs1 = {
- 'd0_0$': {
- 'd1_0': {
- 'd1_0_0.yaml': None,
- },
- 'd1_1': {
- 'd1_1_0.yaml': None,
- 'd1_1_1.yaml': None,
- },
- 'd1_2': {
- '+': None,
- 'd1_2_0.yaml': None,
- 'd1_2_1.yaml': None,
- 'd1_2_2.yaml': None,
- 'd1_2_3.yaml': None,
- },
- },
- }
- for fs, root in [(fake_fs,'d0_0'), (fake_fs1,'d0_0$')]:
- self.start_patchers(fs)
- try:
- result = build_matrix.build_matrix(root)
- finally:
- self.stop_patchers()
- assert len(result) == 1
- if result[0][0][1:].startswith('d1_2'):
- for i in result:
- assert os.path.join(root, 'd1_2/d1_2_0.yaml') in i[1]
- assert os.path.join(root, 'd1_2/d1_2_1.yaml') in i[1]
- assert os.path.join(root, 'd1_2/d1_2_2.yaml') in i[1]
- assert os.path.join(root, 'd1_2/d1_2_3.yaml') in i[1]
-
- def test_random_dollar_sign_with_convolve(self):
- fake_fs = {
- 'd0_0': {
- '%': None,
- 'd1_0': {
- 'd1_0_0.yaml': None,
- 'd1_0_1.yaml': None,
- },
- 'd1_1': {
- 'd1_1_0.yaml': None,
- 'd1_1_1.yaml': None,
- },
- 'd1_2': {
- '$': None,
- 'd1_2_0.yaml': None,
- 'd1_2_1.yaml': None,
- 'd1_2_2.yaml': None,
- },
- },
- }
- self.start_patchers(fake_fs)
- try:
- result = build_matrix.build_matrix('d0_0')
- finally:
- self.stop_patchers()
- assert len(result) == 4
- fake_fs1 = {
- 'd0_0': {
- '%': None,
- 'd1_0': {
- 'd1_0_0.yaml': None,
- 'd1_0_1.yaml': None,
- },
- 'd1_1': {
- 'd1_1_0.yaml': None,
- 'd1_1_1.yaml': None,
- },
- 'd1_2$': {
- 'd1_2_0.yaml': None,
- 'd1_2_1.yaml': None,
- 'd1_2_2.yaml': None,
- },
- },
- }
- self.start_patchers(fake_fs1)
- try:
- result = build_matrix.build_matrix('d0_0')
- finally:
- self.stop_patchers()
- assert len(result) == 4
-
- def test_emulate_teuthology_noceph(self):
- fake_fs = {
- 'teuthology': {
- 'no-ceph': {
- '%': None,
- 'clusters': {
- 'single.yaml': None,
- },
- 'distros': {
- 'baremetal.yaml': None,
- 'rhel7.0.yaml': None,
- 'ubuntu12.04.yaml': None,
- 'ubuntu14.04.yaml': None,
- 'vps.yaml': None,
- 'vps_centos6.5.yaml': None,
- 'vps_debian7.yaml': None,
- 'vps_rhel6.4.yaml': None,
- 'vps_rhel6.5.yaml': None,
- 'vps_rhel7.0.yaml': None,
- 'vps_ubuntu14.04.yaml': None,
- },
- 'tasks': {
- 'teuthology.yaml': None,
- },
- },
- },
- }
- self.start_patchers(fake_fs)
- try:
- result = build_matrix.build_matrix('teuthology/no-ceph')
- finally:
- self.stop_patchers()
- assert len(result) == 11
- assert self.fragment_occurences(result, 'vps.yaml') == 1 / 11.0
-
- def test_empty_dirs(self):
- fake_fs = {
- 'teuthology': {
- 'no-ceph': {
- '%': None,
- 'clusters': {
- 'single.yaml': None,
- },
- 'distros': {
- 'baremetal.yaml': None,
- 'rhel7.0.yaml': None,
- 'ubuntu12.04.yaml': None,
- 'ubuntu14.04.yaml': None,
- 'vps.yaml': None,
- 'vps_centos6.5.yaml': None,
- 'vps_debian7.yaml': None,
- 'vps_rhel6.4.yaml': None,
- 'vps_rhel6.5.yaml': None,
- 'vps_rhel7.0.yaml': None,
- 'vps_ubuntu14.04.yaml': None,
- },
- 'tasks': {
- 'teuthology.yaml': None,
- },
- },
- },
- }
- self.start_patchers(fake_fs)
- try:
- result = build_matrix.build_matrix('teuthology/no-ceph')
- finally:
- self.stop_patchers()
-
- fake_fs2 = {
- 'teuthology': {
- 'no-ceph': {
- '%': None,
- 'clusters': {
- 'single.yaml': None,
- },
- 'distros': {
- 'empty': {},
- 'baremetal.yaml': None,
- 'rhel7.0.yaml': None,
- 'ubuntu12.04.yaml': None,
- 'ubuntu14.04.yaml': None,
- 'vps.yaml': None,
- 'vps_centos6.5.yaml': None,
- 'vps_debian7.yaml': None,
- 'vps_rhel6.4.yaml': None,
- 'vps_rhel6.5.yaml': None,
- 'vps_rhel7.0.yaml': None,
- 'vps_ubuntu14.04.yaml': None,
- },
- 'tasks': {
- 'teuthology.yaml': None,
- },
- 'empty': {},
- },
- },
- }
- self.start_patchers(fake_fs2)
- try:
- result2 = build_matrix.build_matrix('teuthology/no-ceph')
- finally:
- self.stop_patchers()
- assert len(result) == 11
- assert len(result2) == len(result)
-
- def test_hidden(self):
- fake_fs = {
- 'teuthology': {
- 'no-ceph': {
- '%': None,
- '.qa': None,
- 'clusters': {
- 'single.yaml': None,
- '.qa': None,
- },
- 'distros': {
- '.qa': None,
- 'baremetal.yaml': None,
- 'rhel7.0.yaml': None,
- 'ubuntu12.04.yaml': None,
- 'ubuntu14.04.yaml': None,
- 'vps.yaml': None,
- 'vps_centos6.5.yaml': None,
- 'vps_debian7.yaml': None,
- 'vps_rhel6.4.yaml': None,
- 'vps_rhel6.5.yaml': None,
- 'vps_rhel7.0.yaml': None,
- 'vps_ubuntu14.04.yaml': None,
- },
- 'tasks': {
- '.qa': None,
- 'teuthology.yaml': None,
- },
- '.foo': {
- '.qa': None,
- 'teuthology.yaml': None,
- },
- },
- },
- }
- self.start_patchers(fake_fs)
- try:
- result = build_matrix.build_matrix('teuthology/no-ceph')
- finally:
- self.stop_patchers()
-
- fake_fs2 = {
- 'teuthology': {
- 'no-ceph': {
- '%': None,
- 'clusters': {
- 'single.yaml': None,
- },
- 'distros': {
- 'baremetal.yaml': None,
- 'rhel7.0.yaml': None,
- 'ubuntu12.04.yaml': None,
- 'ubuntu14.04.yaml': None,
- 'vps.yaml': None,
- 'vps_centos6.5.yaml': None,
- 'vps_debian7.yaml': None,
- 'vps_rhel6.4.yaml': None,
- 'vps_rhel6.5.yaml': None,
- 'vps_rhel7.0.yaml': None,
- 'vps_ubuntu14.04.yaml': None,
- },
- 'tasks': {
- 'teuthology.yaml': None,
- },
- },
- },
- }
- self.start_patchers(fake_fs2)
- try:
- result2 = build_matrix.build_matrix('teuthology/no-ceph')
- finally:
- self.stop_patchers()
- assert len(result) == 11
- assert len(result2) == len(result)
-
- def test_disable_extension(self):
- fake_fs = {
- 'teuthology': {
- 'no-ceph': {
- '%': None,
- 'clusters': {
- 'single.yaml': None,
- },
- 'distros': {
- 'baremetal.yaml': None,
- 'rhel7.0.yaml': None,
- 'ubuntu12.04.yaml': None,
- 'ubuntu14.04.yaml': None,
- 'vps.yaml': None,
- 'vps_centos6.5.yaml': None,
- 'vps_debian7.yaml': None,
- 'vps_rhel6.4.yaml': None,
- 'vps_rhel6.5.yaml': None,
- 'vps_rhel7.0.yaml': None,
- 'vps_ubuntu14.04.yaml': None,
- },
- 'tasks': {
- 'teuthology.yaml': None,
- },
- },
- },
- }
- self.start_patchers(fake_fs)
- try:
- result = build_matrix.build_matrix('teuthology/no-ceph')
- finally:
- self.stop_patchers()
-
- fake_fs2 = {
- 'teuthology': {
- 'no-ceph': {
- '%': None,
- 'clusters': {
- 'single.yaml': None,
- },
- 'distros': {
- 'baremetal.yaml': None,
- 'rhel7.0.yaml': None,
- 'ubuntu12.04.yaml': None,
- 'ubuntu14.04.yaml': None,
- 'vps.yaml': None,
- 'vps_centos6.5.yaml': None,
- 'vps_debian7.yaml': None,
- 'vps_rhel6.4.yaml': None,
- 'vps_rhel6.5.yaml': None,
- 'vps_rhel7.0.yaml': None,
- 'vps_ubuntu14.04.yaml': None,
- 'forcefilevps_ubuntu14.04.yaml.disable': None,
- 'forcefilevps_ubuntu14.04.yaml.anotherextension': None,
- },
- 'tasks': {
- 'teuthology.yaml': None,
- 'forcefilevps_ubuntu14.04notyaml': None,
- },
- 'forcefilevps_ubuntu14.04notyaml': None,
- 'tasks.disable': {
- 'teuthology2.yaml': None,
- 'forcefilevps_ubuntu14.04notyaml': None,
- },
- },
- },
- }
- self.start_patchers(fake_fs2)
- try:
- result2 = build_matrix.build_matrix('teuthology/no-ceph')
- finally:
- self.stop_patchers()
- assert len(result) == 11
- assert len(result2) == len(result)
-
- def test_sort_order(self):
- # This test ensures that 'ceph' comes before 'ceph-thrash' when yaml
- # fragments are sorted.
- fake_fs = {
- 'thrash': {
- '%': None,
- 'ceph-thrash': {'default.yaml': None},
- 'ceph': {'base.yaml': None},
- 'clusters': {'mds-1active-1standby.yaml': None},
- 'debug': {'mds_client.yaml': None},
- 'fs': {'btrfs.yaml': None},
- 'msgr-failures': {'none.yaml': None},
- 'overrides': {'allowlist_wrongly_marked_down.yaml': None},
- 'tasks': {'cfuse_workunit_suites_fsstress.yaml': None},
- },
- }
- self.start_patchers(fake_fs)
- try:
- result = build_matrix.build_matrix('thrash')
- finally:
- self.stop_patchers()
- assert len(result) == 1
- assert self.fragment_occurences(result, 'base.yaml') == 1
- fragments = result[0][1]
- assert fragments[0] == 'thrash/ceph/base.yaml'
- assert fragments[1] == 'thrash/ceph-thrash/default.yaml'
-
-class TestSubset(object):
- patchpoints = [
- 'os.path.exists',
- 'os.listdir',
- 'os.path.isfile',
- 'os.path.isdir',
- 'builtins.open',
- ]
-
- def setup_method(self):
- self.mocks = dict()
- self.patchers = dict()
- for ppoint in self.__class__.patchpoints:
- self.mocks[ppoint] = MagicMock()
- self.patchers[ppoint] = patch(ppoint, self.mocks[ppoint])
-
- def start_patchers(self, fake_fs):
- fake_fns = make_fake_fstools(fake_fs)
- # N.B.: relies on fake_fns being in same order as patchpoints
- for ppoint, fn in zip(self.__class__.patchpoints, fake_fns):
- self.mocks[ppoint].side_effect = fn
- self.patchers[ppoint].start()
-
- def stop_patchers(self):
- for patcher in self.patchers.values():
- patcher.stop()
-
- def teardown_method(self):
- self.patchers.clear()
- self.mocks.clear()
-
- MAX_FACETS = 10
- MAX_FANOUT = 3
- MAX_DEPTH = 3
- MAX_SUBSET = 10
- @staticmethod
- def generate_fake_fs(max_facets, max_fanout, max_depth):
- def yamilify(name):
- return name + ".yaml"
- def name_generator():
- x = 0
- while True:
- yield(str(x))
- x += 1
- def generate_tree(
- max_facets, max_fanout, max_depth, namegen, top=True):
- if max_depth == 0:
- return None
- if max_facets == 0:
- return None
- items = random.choice(range(max_fanout))
- if items == 0 and top:
- items = 1
- if items == 0:
- return None
- sub_max_facets = max_facets / items
- tree = {}
- for i in range(items):
- subtree = generate_tree(
- sub_max_facets, max_fanout,
- max_depth - 1, namegen, top=False)
- if subtree is not None:
- tree['d' + next(namegen)] = subtree
- else:
- tree[yamilify('f' + next(namegen))] = None
- random.choice([
- lambda: tree.update({'%': None}),
- lambda: None])()
- return tree
- return {
- 'root': generate_tree(
- max_facets, max_fanout, max_depth, name_generator())
- }
-
- @staticmethod
- def generate_subset(maxsub):
- divisions = random.choice(range(maxsub-1))+1
- return (random.choice(range(divisions)), divisions)
-
- @staticmethod
- def generate_description_list(tree, subset):
- mat, first, matlimit = build_matrix._get_matrix(
- 'root', subset=subset)
- return [i[0] for i in build_matrix.generate_combinations(
- 'root', mat, first, matlimit)], mat, first, matlimit
-
- @staticmethod
- def verify_facets(tree, description_list, subset, mat, first, matlimit):
- def flatten(tree):
- for k,v in tree.items():
- if v is None and '.yaml' in k:
- yield k
- elif v is not None and '.disable' not in k:
- for x in flatten(v):
- yield x
-
- def pptree(tree, tabs=0):
- ret = ""
- for k, v in tree.items():
- if v is None:
- ret += ('\t'*tabs) + k.ljust(10) + "\n"
- else:
- ret += ('\t'*tabs) + (k + ':').ljust(10) + "\n"
- ret += pptree(v, tabs+1)
- return ret
- def deyamlify(name):
- if name.endswith('.yaml'):
- return name[:-5]
- else:
- return name
- for facet in (deyamlify(_) for _ in flatten(tree)):
- found = False
- for i in description_list:
- if facet in i:
- found = True
- break
- if not found:
- print("tree\n{tree}\ngenerated list\n{desc}\n\nfrom matrix\n\n{matrix}\nsubset {subset} without facet {fac}".format(
- tree=pptree(tree),
- desc='\n'.join(description_list),
- subset=subset,
- matrix=str(mat),
- fac=facet))
- all_desc = build_matrix.generate_combinations(
- 'root',
- mat,
- 0,
- mat.size())
- for i, desc in zip(range(mat.size()), all_desc):
- if i == first:
- print('==========')
- print("{} {}".format(i, desc))
- if i + 1 == matlimit:
- print('==========')
- assert found
-
- def test_random(self):
- for i in range(10000):
- tree = self.generate_fake_fs(
- self.MAX_FACETS,
- self.MAX_FANOUT,
- self.MAX_DEPTH)
- subset = self.generate_subset(self.MAX_SUBSET)
- self.start_patchers(tree)
- try:
- dlist, mat, first, matlimit = self.generate_description_list(tree, subset)
- finally:
- self.stop_patchers()
- self.verify_facets(tree, dlist, subset, mat, first, matlimit)
+++ /dev/null
-import os
-
-from copy import deepcopy
-
-from mock import patch, Mock, DEFAULT
-
-from teuthology import suite
-from scripts.suite import main
-from teuthology.config import config
-
-import pytest
-import time
-
-from teuthology.exceptions import ScheduleFailError
-
-def get_fake_time_and_sleep():
- # Below we set m_time.side_effect, but we also set m_time.return_value.
- # The reason for this is that we need to store a 'fake time' that
- # increments when m_sleep() is called; we could use any variable name we
- # wanted for the return value, but since 'return_value' is already a
- # standard term in mock, and since setting side_effect causes return_value
- # to be ignored, it's safe to just reuse the name here.
- m_time = Mock()
- m_time.return_value = time.time()
-
- def m_time_side_effect():
- # Fake the slow passage of time
- m_time.return_value += 0.1
- return m_time.return_value
- m_time.side_effect = m_time_side_effect
-
- def f_sleep(seconds):
- m_time.return_value += seconds
- m_sleep = Mock(wraps=f_sleep)
- return m_time, m_sleep
-
-
-def setup_module():
- global m_time
- global m_sleep
- m_time, m_sleep = get_fake_time_and_sleep()
- global patcher_time_sleep
- patcher_time_sleep = patch.multiple(
- 'teuthology.suite.time',
- time=m_time,
- sleep=m_sleep,
- )
- patcher_time_sleep.start()
-
-
-def teardown_module():
- patcher_time_sleep.stop()
-
-
-@patch.object(suite.ResultsReporter, 'get_jobs')
-def test_wait_success(m_get_jobs, caplog):
- results = [
- [{'status': 'queued', 'job_id': '2'}],
- [],
- ]
- final = [
- {'status': 'pass', 'job_id': '1',
- 'description': 'DESC1', 'log_href': 'http://URL1'},
- {'status': 'fail', 'job_id': '2',
- 'description': 'DESC2', 'log_href': 'http://URL2'},
- {'status': 'pass', 'job_id': '3',
- 'description': 'DESC3', 'log_href': 'http://URL3'},
- ]
-
- def get_jobs(name, **kwargs):
- if kwargs['fields'] == ['job_id', 'status']:
- return in_progress.pop(0)
- else:
- return final
- m_get_jobs.side_effect = get_jobs
- suite.Run.WAIT_PAUSE = 1
-
- in_progress = deepcopy(results)
- assert 0 == suite.wait('name', 1, 'http://UPLOAD_URL')
- m_get_jobs.assert_any_call('name', fields=['job_id', 'status'])
- assert 0 == len(in_progress)
- assert 'fail http://UPLOAD_URL/name/2' in caplog.text
-
- m_get_jobs.reset_mock()
- in_progress = deepcopy(results)
- assert 0 == suite.wait('name', 1, None)
- m_get_jobs.assert_any_call('name', fields=['job_id', 'status'])
- assert 0 == len(in_progress)
- assert 'fail http://URL2' in caplog.text
-
-
-@patch.object(suite.ResultsReporter, 'get_jobs')
-def test_wait_fails(m_get_jobs):
- results = []
- results.append([{'status': 'queued', 'job_id': '2'}])
- results.append([{'status': 'queued', 'job_id': '2'}])
- results.append([{'status': 'queued', 'job_id': '2'}])
-
- def get_jobs(name, **kwargs):
- return results.pop(0)
- m_get_jobs.side_effect = get_jobs
- suite.Run.WAIT_PAUSE = 1
- suite.Run.WAIT_MAX_JOB_TIME = 1
- with pytest.raises(suite.WaitException):
- suite.wait('name', 1, None)
-
-
-REPO_SHORTHAND = [
- ['https://github.com/dude/foo', 'bar',
- 'https://github.com/dude/bar.git'],
- ['https://github.com/dude/foo/', 'bar',
- 'https://github.com/dude/bar.git'],
- ['https://github.com/ceph/ceph', 'ceph',
- 'https://github.com/ceph/ceph.git'],
- ['https://github.com/ceph/ceph/', 'ceph',
- 'https://github.com/ceph/ceph.git'],
- ['https://github.com/ceph/ceph.git', 'ceph',
- 'https://github.com/ceph/ceph.git'],
- ['https://github.com/ceph/ceph', 'ceph-ci',
- 'https://github.com/ceph/ceph-ci.git'],
- ['https://github.com/ceph/ceph-ci', 'ceph',
- 'https://github.com/ceph/ceph.git'],
- ['git://git.ceph.com/ceph.git', 'ceph',
- 'git://git.ceph.com/ceph.git'],
- ['git://git.ceph.com/ceph.git', 'ceph-ci',
- 'git://git.ceph.com/ceph-ci.git'],
- ['git://git.ceph.com/ceph-ci.git', 'ceph',
- 'git://git.ceph.com/ceph.git'],
- ['https://github.com/ceph/ceph.git', 'ceph/ceph-ci',
- 'https://github.com/ceph/ceph-ci.git'],
- ['https://github.com/ceph/ceph.git', 'https://github.com/ceph/ceph-ci',
- 'https://github.com/ceph/ceph-ci'],
- ['https://github.com/ceph/ceph.git', 'https://github.com/ceph/ceph-ci/',
- 'https://github.com/ceph/ceph-ci/'],
- ['https://github.com/ceph/ceph.git', 'https://github.com/ceph/ceph-ci.git',
- 'https://github.com/ceph/ceph-ci.git'],
-]
-
-
-@pytest.mark.parametrize(['orig', 'shorthand', 'result'], REPO_SHORTHAND)
-def test_expand_short_repo_name(orig, shorthand, result):
- assert suite.expand_short_repo_name(shorthand, orig) == result
-
-
-class TestSuiteMain(object):
- def test_main(self):
- suite_name = 'SUITE'
- throttle = '3'
- machine_type = 'burnupi'
-
- def prepare_and_schedule(obj):
- assert obj.base_config.suite == suite_name
- assert obj.args.throttle == throttle
-
- def fake_str(*args, **kwargs):
- return 'fake'
-
- def fake_bool(*args, **kwargs):
- return True
-
- def fake_false(*args, **kwargs):
- return False
-
- with patch.multiple(
- 'teuthology.suite.run.util',
- fetch_repos=DEFAULT,
- package_version_for_hash=fake_str,
- git_branch_exists=fake_bool,
- git_ls_remote=fake_str,
- ):
- with patch.multiple(
- 'teuthology.suite.run.Run',
- prepare_and_schedule=prepare_and_schedule,
- ), patch.multiple(
- 'teuthology.suite.run.os.path',
- exists=fake_false,
- ):
- main([
- '--ceph', 'main',
- '--suite', suite_name,
- '--throttle', throttle,
- '--machine-type', machine_type,
- ])
-
- @patch('teuthology.suite.util.smtplib.SMTP')
- def test_machine_type_multi_error(self, m_smtp):
- config.results_email = "example@example.com"
- with pytest.raises(ScheduleFailError) as exc:
- main([
- '--ceph', 'main',
- '--suite', 'suite_name',
- '--throttle', '3',
- '--machine-type', 'multi',
- '--dry-run',
- ])
- assert str(exc.value) == "Scheduling failed: 'multi' is not a valid machine_type. \
-Maybe you want 'gibba,smithi,mira' or similar"
- m_smtp.assert_not_called()
-
- @patch('teuthology.suite.util.smtplib.SMTP')
- def test_machine_type_none_error(self, m_smtp):
- config.result_email = 'example@example.com'
- with pytest.raises(ScheduleFailError) as exc:
- main([
- '--ceph', 'main',
- '--suite', 'suite_name',
- '--throttle', '3',
- '--machine-type', 'None',
- '--dry-run',
- ])
- assert str(exc.value) == "Scheduling failed: Must specify a machine_type"
- m_smtp.assert_not_called()
-
- def test_schedule_suite_noverify(self):
- suite_name = 'noop'
- suite_dir = os.path.dirname(__file__)
- throttle = '3'
- machine_type = 'burnupi'
-
- with patch.multiple(
- 'teuthology.suite.util',
- fetch_repos=DEFAULT,
- teuthology_schedule=DEFAULT,
- get_arch=lambda x: 'x86_64',
- get_gitbuilder_hash=DEFAULT,
- git_ls_remote=lambda *args: '1234',
- package_version_for_hash=DEFAULT,
- ) as m:
- m['package_version_for_hash'].return_value = 'fake-9.5'
- config.suite_verify_ceph_hash = False
- main([
- '--ceph', 'main',
- '--suite', suite_name,
- '--suite-dir', suite_dir,
- '--suite-relpath', '',
- '--throttle', throttle,
- '--machine-type', machine_type
- ])
- m_sleep.assert_called_with(int(throttle))
- m['get_gitbuilder_hash'].assert_not_called()
-
- def test_schedule_suite(self):
- suite_name = 'noop'
- suite_dir = os.path.dirname(__file__)
- throttle = '3'
- machine_type = 'burnupi'
-
- with patch.multiple(
- 'teuthology.suite.util',
- fetch_repos=DEFAULT,
- teuthology_schedule=DEFAULT,
- get_arch=lambda x: 'x86_64',
- get_gitbuilder_hash=DEFAULT,
- git_ls_remote=lambda *args: '12345',
- package_version_for_hash=DEFAULT,
- ) as m:
- m['package_version_for_hash'].return_value = 'fake-9.5'
- config.suite_verify_ceph_hash = True
- main([
- '--ceph', 'main',
- '--suite', suite_name,
- '--suite-dir', suite_dir,
- '--suite-relpath', '',
- '--throttle', throttle,
- '--machine-type', machine_type
- ])
- m_sleep.assert_called_with(int(throttle))
+++ /dev/null
-from teuthology.suite import matrix
-
-
-def verify_matrix_output_diversity(res):
- """
- Verifies that the size of the matrix passed matches the number of unique
- outputs from res.index
- """
- sz = res.size()
- s = frozenset([matrix.generate_lists(res.index(i)) for i in range(sz)])
- for i in range(res.size()):
- assert sz == len(s)
-
-
-def mbs(num, l):
- return matrix.Sum(num*10, [matrix.Base(i + (100*num)) for i in l])
-
-
-class TestMatrix(object):
- def test_simple(self):
- verify_matrix_output_diversity(mbs(1, range(6)))
-
- def test_simple2(self):
- verify_matrix_output_diversity(mbs(1, range(5)))
-
- # The test_product* tests differ by the degree by which dimension
- # sizes share prime factors
- def test_product_simple(self):
- verify_matrix_output_diversity(
- matrix.Product(1, [mbs(1, range(6)), mbs(2, range(2))]))
-
- def test_product_3_facets_2_prime_factors(self):
- verify_matrix_output_diversity(matrix.Product(1, [
- mbs(1, range(6)),
- mbs(2, range(2)),
- mbs(3, range(3)),
- ]))
-
- def test_product_3_facets_2_prime_factors_one_larger(self):
- verify_matrix_output_diversity(matrix.Product(1, [
- mbs(1, range(2)),
- mbs(2, range(5)),
- mbs(4, range(4)),
- ]))
-
- def test_product_4_facets_2_prime_factors(self):
- verify_matrix_output_diversity(matrix.Sum(1, [
- mbs(1, range(6)),
- mbs(3, range(3)),
- mbs(2, range(2)),
- mbs(4, range(9)),
- ]))
-
- def test_product_2_facets_2_prime_factors(self):
- verify_matrix_output_diversity(matrix.Sum(1, [
- mbs(1, range(2)),
- mbs(2, range(5)),
- ]))
-
- def test_product_with_sum(self):
- verify_matrix_output_diversity(matrix.Sum(
- 9,
- [
- mbs(10, range(6)),
- matrix.Product(1, [
- mbs(1, range(2)),
- mbs(2, range(5)),
- mbs(4, range(4))]),
- matrix.Product(8, [
- mbs(7, range(2)),
- mbs(6, range(5)),
- mbs(5, range(4))])
- ]
- ))
-
- def test_product_with_pick_random(self):
- verify_matrix_output_diversity(matrix.PickRandom(1, [
- mbs(1, range(6)),
- mbs(3, range(3)),
- mbs(2, range(2)),
- mbs(4, range(9)),
- ]))
+++ /dev/null
-import logging
-from textwrap import dedent
-
-from mock import patch, MagicMock
-
-from teuthology.suite import build_matrix
-from teuthology.suite.merge import config_merge
-from teuthology.test.fake_fs import make_fake_fstools
-
-log = logging.getLogger(__name__)
-
-class TestMerge:
- patchpoints = [
- 'os.path.exists',
- 'os.listdir',
- 'os.path.isfile',
- 'os.path.isdir',
- 'builtins.open',
- ]
-
- def setup_method(self):
- self.mocks = dict()
- self.patchers = dict()
- for ppoint in self.__class__.patchpoints:
- self.mocks[ppoint] = MagicMock()
- self.patchers[ppoint] = patch(ppoint, self.mocks[ppoint])
-
- def start_patchers(self, fake_fs):
- fake_fns = make_fake_fstools(fake_fs)
- # N.B.: relies on fake_fns being in same order as patchpoints
- for ppoint, fn in zip(self.__class__.patchpoints, fake_fns):
- self.mocks[ppoint].side_effect = fn
- self.patchers[ppoint].start()
-
- def stop_patchers(self):
- for patcher in self.patchers.values():
- patcher.stop()
-
- def teardown_method(self):
- self.patchers.clear()
- self.mocks.clear()
-
- def test_premerge(self):
- fake_fs = {
- 'd0_0': {
- '%': None,
- 'd1_0': {
- 'a.yaml': dedent("""
- teuthology:
- premerge: reject()
- foo: bar
- """),
- },
- 'c.yaml': dedent("""
- top: pot
- """),
- },
- }
- self.start_patchers(fake_fs)
- try:
- result = build_matrix.build_matrix('d0_0')
- assert 1 == len(result)
- configs = list(config_merge(result))
- assert 1 == len(configs)
- desc, frags, yaml = configs[0]
- assert "top" in yaml
- assert "foo" not in yaml
- finally:
- self.stop_patchers()
-
- def test_postmerge(self):
- fake_fs = {
- 'd0_0': {
- '%': None,
- 'd1_0': {
- 'a.yaml': dedent("""
- teuthology:
- postmerge:
- - reject()
- foo: bar
- """),
- 'b.yaml': dedent("""
- baz: zab
- """),
- },
- 'c.yaml': dedent("""
- top: pot
- """),
- },
- }
- self.start_patchers(fake_fs)
- try:
- result = build_matrix.build_matrix('d0_0')
- assert 2 == len(result)
- configs = list(config_merge(result))
- assert 1 == len(configs)
- desc, frags, yaml = configs[0]
- assert "top" in yaml
- assert "baz" in yaml
- assert "foo" not in yaml
- finally:
- self.stop_patchers()
-
- def test_postmerge_concat(self):
- fake_fs = {
- 'd0_0': {
- '%': None,
- 'd1_0': {
- 'a.yaml': dedent("""
- teuthology:
- postmerge:
- - local a = 1
- foo: bar
- """),
- 'b.yaml': dedent("""
- teuthology:
- postmerge:
- - local a = 2
- baz: zab
- """),
- },
- 'z.yaml': dedent("""
- teuthology:
- postmerge:
- - if a == 1 then reject() end
- top: pot
- """),
- },
- }
- self.start_patchers(fake_fs)
- try:
- result = build_matrix.build_matrix('d0_0')
- assert 2 == len(result)
- configs = list(config_merge(result))
- assert 1 == len(configs)
- desc, frags, yaml = configs[0]
- assert "top" in yaml
- assert "baz" in yaml
- assert "foo" not in yaml
- finally:
- self.stop_patchers()
-
-
- def test_yaml_mutation(self):
- fake_fs = {
- 'd0_0': {
- '%': None,
- 'c.yaml': dedent("""
- teuthology:
- postmerge:
- - |
- yaml["test"] = py_dict()
- top: pot
- """),
- },
- }
- self.start_patchers(fake_fs)
- try:
- result = build_matrix.build_matrix('d0_0')
- assert 1 == len(result)
- configs = list(config_merge(result))
- assert 1 == len(configs)
- desc, frags, yaml = configs[0]
- assert "test" in yaml
- assert {} == yaml["test"]
- finally:
- self.stop_patchers()
-
- def test_sandbox(self):
- fake_fs = {
- 'd0_0': {
- '%': None,
- 'c.yaml': dedent("""
- teuthology:
- postmerge:
- - |
- log.debug(_ENV)
- log.debug("_ENV contains:")
- for k,v in pairs(_ENV) do
- log.debug("_ENV['%s'] = %s", tostring(k), tostring(v))
- end
- local check = {
- "assert",
- "error",
- "ipairs",
- "next",
- "pairs",
- "tonumber",
- "tostring",
- "py_attrgetter",
- "py_dict",
- "py_list",
- "py_tuple",
- "py_enumerate",
- "py_iterex",
- "py_itemgetter",
- "math",
- "reject",
- "accept",
- "deep_merge",
- "log",
- "reject",
- "yaml_load",
- }
- for _,v in ipairs(check) do
- log.debug("checking %s", tostring(v))
- assert(_ENV[v])
- end
- local block = {
- "coroutine",
- "debug",
- "io",
- "os",
- "package",
- }
- for _,v in ipairs(block) do
- log.debug("checking %s", tostring(v))
- assert(_ENV[v] == nil)
- end
- top: pot
- """),
- },
- }
- self.start_patchers(fake_fs)
- try:
- result = build_matrix.build_matrix('d0_0')
- assert 1 == len(result)
- configs = list(config_merge(result))
- assert 1 == len(configs)
- finally:
- self.stop_patchers()
+++ /dev/null
-from teuthology.suite.placeholder import (
- substitute_placeholders, dict_templ, Placeholder
-)
-
-
-class TestPlaceholder(object):
- def test_substitute_placeholders(self):
- suite_hash = 'suite_hash'
- input_dict = dict(
- suite='suite',
- suite_branch='suite_branch',
- suite_hash=suite_hash,
- ceph_branch='ceph_branch',
- ceph_hash='ceph_hash',
- teuthology_branch='teuthology_branch',
- teuthology_sha1='teuthology_sha1',
- machine_type='machine_type',
- distro='distro',
- distro_version='distro_version',
- archive_upload='archive_upload',
- archive_upload_key='archive_upload_key',
- suite_repo='https://example.com/ceph/suite.git',
- suite_relpath='',
- ceph_repo='https://example.com/ceph/ceph.git',
- flavor='default',
- expire='expire',
- )
- output_dict = substitute_placeholders(dict_templ, input_dict)
- assert output_dict['suite'] == 'suite'
- assert output_dict['suite_sha1'] == suite_hash
- assert isinstance(dict_templ['suite'], Placeholder)
- assert isinstance(
- dict_templ['overrides']['admin_socket']['branch'],
- Placeholder)
-
- def test_null_placeholders_dropped(self):
- input_dict = dict(
- suite='suite',
- suite_branch='suite_branch',
- suite_hash='suite_hash',
- ceph_branch='ceph_branch',
- ceph_hash='ceph_hash',
- teuthology_branch='teuthology_branch',
- teuthology_sha1='teuthology_sha1',
- machine_type='machine_type',
- archive_upload='archive_upload',
- archive_upload_key='archive_upload_key',
- distro=None,
- distro_version=None,
- suite_repo='https://example.com/ceph/suite.git',
- suite_relpath='',
- ceph_repo='https://example.com/ceph/ceph.git',
- flavor=None,
- expire='expire',
- )
- output_dict = substitute_placeholders(dict_templ, input_dict)
- assert 'os_type' not in output_dict
+++ /dev/null
-import logging
-import os
-import pytest
-import requests
-import contextlib
-import yaml
-
-from datetime import datetime, timedelta, timezone
-from mock import patch, call, ANY
-from io import StringIO
-from io import BytesIO
-
-from teuthology.config import config, YamlConfig
-from teuthology.exceptions import ScheduleFailError
-from teuthology.suite import run
-from teuthology.util.time import TIMESTAMP_FMT
-
-log = logging.getLogger(__name__)
-
-class TestRun(object):
- klass = run.Run
-
- def setup_method(self):
- self.args_dict = dict(
- suite='suite',
- suite_branch='suite_branch',
- suite_relpath='',
- ceph_branch='ceph_branch',
- ceph_sha1='ceph_sha1',
- email='address@example.com',
- teuthology_branch='teuthology_branch',
- kernel_branch=None,
- flavor='flavor',
- distro='ubuntu',
- machine_type='machine_type',
- base_yaml_paths=list(),
- )
- self.args = YamlConfig.from_dict(self.args_dict)
-
- @patch('teuthology.suite.run.util.fetch_repos')
- @patch('teuthology.suite.run.util.git_ls_remote')
- @patch('teuthology.suite.run.util.git_validate_sha1')
- def test_email_addr(self, m_git_validate_sha1,
- m_git_ls_remote, _):
- # neuter choose_X_branch
- m_git_validate_sha1.return_value = self.args_dict['ceph_sha1']
- self.args_dict['teuthology_branch'] = 'main'
- self.args_dict['suite_branch'] = 'main'
- m_git_ls_remote.return_value = 'suite_sha1'
-
- runobj = self.klass(self.args)
- assert runobj.base_config.email == self.args_dict['email']
-
- @patch('teuthology.suite.run.util.fetch_repos')
- def test_name(self, m_fetch_repos):
- stamp = datetime.now().strftime(TIMESTAMP_FMT)
- with patch.object(run.Run, 'create_initial_config',
- return_value=run.JobConfig()):
- name = run.Run(self.args).name
- assert str(stamp) in name
-
- @patch('teuthology.suite.run.util.fetch_repos')
- def test_name_owner(self, m_fetch_repos):
- self.args.owner = 'USER'
- with patch.object(run.Run, 'create_initial_config',
- return_value=run.JobConfig()):
- name = run.Run(self.args).name
- assert name.startswith('USER-')
-
- @patch('teuthology.suite.run.util.git_branch_exists')
- @patch('teuthology.suite.run.util.package_version_for_hash')
- @patch('teuthology.suite.run.util.git_ls_remote')
- def test_branch_nonexistent(
- self,
- m_git_ls_remote,
- m_package_version_for_hash,
- m_git_branch_exists,
- ):
- config.gitbuilder_host = 'example.com'
- m_git_ls_remote.side_effect = [
- # First call will be for the ceph hash
- None,
- # Second call will be for the suite hash
- 'suite_hash',
- ]
- m_package_version_for_hash.return_value = 'a_version'
- m_git_branch_exists.return_value = True
- self.args.ceph_branch = 'ceph_sha1'
- self.args.ceph_sha1 = None
- with pytest.raises(ScheduleFailError):
- self.klass(self.args)
-
- @pytest.mark.parametrize(
- ["expire", "delta", "result"],
- [
- [None, timedelta(), False],
- ["1m", timedelta(), True],
- ["1m", timedelta(minutes=-2), False],
- ["1m", timedelta(minutes=2), True],
- ["7d", timedelta(days=-14), False],
- ]
- )
- @patch('teuthology.repo_utils.fetch_repo')
- @patch('teuthology.suite.run.util.git_branch_exists')
- @patch('teuthology.suite.run.util.package_version_for_hash')
- @patch('teuthology.suite.run.util.git_ls_remote')
- def test_get_expiration(
- self,
- m_git_ls_remote,
- m_package_version_for_hash,
- m_git_branch_exists,
- m_fetch_repo,
- expire,
- delta,
- result,
- ):
- m_git_ls_remote.side_effect = 'hash'
- m_package_version_for_hash.return_value = 'a_version'
- m_git_branch_exists.return_value = True
- self.args.expire = expire
- obj = self.klass(self.args)
- now = datetime.now(timezone.utc)
- expires_result = obj.get_expiration(_base_time=now + delta)
- if expire is None:
- assert expires_result is None
- assert obj.base_config['expire'] is None
- else:
- assert expires_result is not None
- assert (now < expires_result) is result
- assert obj.base_config['expire']
-
- @patch('teuthology.suite.run.util.fetch_repos')
- @patch('requests.head')
- @patch('teuthology.suite.run.util.git_branch_exists')
- @patch('teuthology.suite.run.util.package_version_for_hash')
- @patch('teuthology.suite.run.util.git_ls_remote')
- def test_sha1_exists(
- self,
- m_git_ls_remote,
- m_package_version_for_hash,
- m_git_branch_exists,
- m_requests_head,
- m_fetch_repos,
- ):
- config.gitbuilder_host = 'example.com'
- m_package_version_for_hash.return_value = 'ceph_hash'
- m_git_branch_exists.return_value = True
- resp = requests.Response()
- resp.reason = 'OK'
- resp.status_code = 200
- m_requests_head.return_value = resp
- # only one call to git_ls_remote in this case
- m_git_ls_remote.return_value = "suite_branch"
- run = self.klass(self.args)
- assert run.base_config.sha1 == 'ceph_sha1'
- assert run.base_config.branch == 'ceph_branch'
-
- @patch('teuthology.suite.run.util.git_ls_remote')
- @patch('requests.head')
- @patch('teuthology.suite.util.git_branch_exists')
- @patch('teuthology.suite.util.package_version_for_hash')
- def test_sha1_nonexistent(
- self,
- m_git_ls_remote,
- m_package_version_for_hash,
- m_git_branch_exists,
- m_requests_head,
- ):
- config.gitbuilder_host = 'example.com'
- m_package_version_for_hash.return_value = 'ceph_hash'
- m_git_branch_exists.return_value = True
- resp = requests.Response()
- resp.reason = 'Not Found'
- resp.status_code = 404
- m_requests_head.return_value = resp
- self.args.ceph_sha1 = 'ceph_hash_dne'
- with pytest.raises(ScheduleFailError):
- self.klass(self.args)
-
- @patch('teuthology.suite.util.smtplib.SMTP')
- @patch('teuthology.suite.util.git_ls_remote')
- @patch('teuthology.suite.util.package_version_for_hash')
- def test_teuthology_branch_nonexistent(
- self,
- m_pvfh,
- m_git_ls_remote,
- m_smtp,
- ):
- m_git_ls_remote.return_value = None
- config.teuthology_path = None
- config.results_email = "example@example.com"
- self.args.dry_run = True
- self.args.teuthology_branch = 'no_branch'
- with pytest.raises(ScheduleFailError):
- self.klass(self.args)
- m_smtp.assert_not_called()
-
- @patch('teuthology.suite.run.util.fetch_repos')
- @patch('teuthology.suite.util.git_ls_remote')
- @patch('teuthology.suite.run.util.package_version_for_hash')
- def test_os_type(self, m_pvfh, m_git_ls_remote, m_fetch_repos):
- m_git_ls_remote.return_value = "sha1"
- del self.args['distro']
- run_ = run.Run(self.args)
- run_.base_args = run_.build_base_args()
- run_.base_config = run_.build_base_config()
- configs = [
- ["desc", [], {"os_type": "debian", "os_version": "8.0"}],
- ["desc", [], {"os_type": "ubuntu", "os_version": "24.0"}],
- ]
- missing, to_schedule = run_.collect_jobs('x86_64', configs, False, False)
- assert to_schedule[0]['yaml']['os_type'] == "debian"
- assert to_schedule[0]['yaml']['os_version'] == "8.0"
- assert to_schedule[1]['yaml']['os_type'] == "ubuntu"
- assert to_schedule[1]['yaml']['os_version'] == "24.0"
-
- @patch('teuthology.suite.run.util.fetch_repos')
- @patch('teuthology.suite.util.git_ls_remote')
- @patch('teuthology.suite.run.util.package_version_for_hash')
- def test_sha1(self, m_pvfh, m_git_ls_remote, m_fetch_repos):
- m_git_ls_remote.return_value = "sha1"
- del self.args['distro']
- run_ = run.Run(self.args)
- run_.base_args = run_.build_base_args()
- for i in range(5): # mock backtracking
- run_.config_input['ceph_hash'] = f"boo{i}"
- run_.config_input['suite_hash'] = f"bar{i}"
- run_.base_config = run_.build_base_config()
- configs = [
- ["desc", [], {"os_type": "debian", "os_version": "8.0",
- "sha1": "old_sha", "suite_sha1": "old_sha",
- "overrides": { "workunit": {"sha1": "old_sha"}, "ceph": {"sha1": "old_sha"} }
- }],
- ]
- missing, to_schedule = run_.collect_jobs('x86_64', configs, False, False)
- assert to_schedule[0]['yaml']['sha1'] == "boo4"
- assert to_schedule[0]['yaml']['suite_sha1'] == "bar4"
- assert to_schedule[0]['yaml']['overrides']['workunit']["sha1"] == "bar4"
- assert to_schedule[0]['yaml']['overrides']['ceph']["sha1"] == "boo4"
-
-class TestScheduleSuite(object):
- klass = run.Run
-
- def setup_method(self):
- self.args_dict = dict(
- suite='suite',
- suite_relpath='',
- suite_dir='suite_dir',
- suite_branch='main',
- suite_repo='ceph',
- ceph_repo='ceph',
- ceph_branch='main',
- ceph_sha1='ceph_sha1',
- teuthology_branch='main',
- kernel_branch=None,
- flavor='flavor',
- distro='ubuntu',
- distro_version='14.04',
- machine_type='machine_type',
- base_yaml_paths=list(),
- )
- self.args = YamlConfig.from_dict(self.args_dict)
-
- @patch('teuthology.suite.run.Run.schedule_jobs')
- @patch('teuthology.suite.run.Run.write_rerun_memo')
- @patch('teuthology.suite.util.get_install_task_flavor')
- @patch('teuthology.suite.merge.open')
- @patch('teuthology.suite.run.build_matrix')
- @patch('teuthology.suite.util.git_ls_remote')
- @patch('teuthology.suite.util.package_version_for_hash')
- @patch('teuthology.suite.util.git_validate_sha1')
- @patch('teuthology.suite.util.get_arch')
- def test_successful_schedule(
- self,
- m_get_arch,
- m_git_validate_sha1,
- m_package_version_for_hash,
- m_git_ls_remote,
- m_build_matrix,
- m_open,
- m_get_install_task_flavor,
- m_write_rerun_memo,
- m_schedule_jobs,
- ):
- m_get_arch.return_value = 'x86_64'
- m_git_validate_sha1.return_value = self.args.ceph_sha1
- m_package_version_for_hash.return_value = 'ceph_version'
- m_git_ls_remote.return_value = 'suite_hash'
- build_matrix_desc = 'desc'
- build_matrix_frags = ['frag1.yml', 'frag2.yml']
- build_matrix_output = [
- (build_matrix_desc, build_matrix_frags),
- ]
- m_build_matrix.return_value = build_matrix_output
- frag1_read_output = 'field1: val1'
- frag2_read_output = 'field2: val2'
- m_open.side_effect = [
- StringIO(frag1_read_output),
- StringIO(frag2_read_output),
- contextlib.closing(BytesIO())
- ]
- m_get_install_task_flavor.return_value = 'default'
- m_package_version_for_hash.return_value = "v1"
- # schedule_jobs() is just neutered; check calls below
-
- self.args.newest = 0
- self.args.num = 42
- runobj = self.klass(self.args)
- runobj.base_args = list()
- count = runobj.schedule_suite()
- assert(count == 1)
- assert runobj.base_config['suite_sha1'] == 'suite_hash'
- m_package_version_for_hash.assert_has_calls(
- [call('ceph_sha1', 'default', 'ubuntu', '14.04', 'machine_type')],
- )
- y = {
- 'field1': 'val1',
- 'field2': 'val2'
- }
- teuthology_keys = [
- 'branch',
- 'machine_type',
- 'name',
- 'os_type',
- 'os_version',
- 'overrides',
- 'priority',
- 'repo',
- 'seed',
- 'sha1',
- 'sleep_before_teardown',
- 'suite',
- 'suite_branch',
- 'suite_relpath',
- 'suite_repo',
- 'suite_sha1',
- 'tasks',
- 'teuthology_branch',
- 'teuthology_sha1',
- 'timestamp',
- 'user',
- 'teuthology',
- 'flavor',
- ]
- for t in teuthology_keys:
- y[t] = ANY
- expected_job = dict(
- yaml=y,
- sha1='ceph_sha1',
- args=[
- '--num',
- '42',
- '--description',
- os.path.join(self.args.suite, build_matrix_desc),
- '--',
- '-'
- ],
- stdin=ANY,
- desc=os.path.join(self.args.suite, build_matrix_desc),
- )
-
- m_schedule_jobs.assert_has_calls(
- [call([], [expected_job], runobj.name)],
- )
- args = m_schedule_jobs.call_args.args
- log.debug("args =\n%s", args)
- jobargs = args[1][0]
- stdin_yaml = yaml.safe_load(jobargs['stdin'])
- for k in y:
- assert y[k] == stdin_yaml[k]
- for k in teuthology_keys:
- assert k in stdin_yaml
- m_write_rerun_memo.assert_called_once_with()
-
- @patch('teuthology.suite.util.find_git_parents')
- @patch('teuthology.suite.run.Run.schedule_jobs')
- @patch('teuthology.suite.util.get_install_task_flavor')
- @patch('teuthology.suite.run.config_merge')
- @patch('teuthology.suite.run.build_matrix')
- @patch('teuthology.suite.util.git_ls_remote')
- @patch('teuthology.suite.util.package_version_for_hash')
- @patch('teuthology.suite.util.git_validate_sha1')
- @patch('teuthology.suite.util.get_arch')
- def test_newest_failure(
- self,
- m_get_arch,
- m_git_validate_sha1,
- m_package_version_for_hash,
- m_git_ls_remote,
- m_build_matrix,
- m_config_merge,
- m_get_install_task_flavor,
- m_schedule_jobs,
- m_find_git_parents,
- ):
- m_get_arch.return_value = 'x86_64'
- m_git_validate_sha1.return_value = self.args.ceph_sha1
- m_package_version_for_hash.return_value = None
- m_git_ls_remote.return_value = 'suite_hash'
- build_matrix_desc = 'desc'
- build_matrix_frags = ['frag.yml']
- build_matrix_output = [
- (build_matrix_desc, build_matrix_frags),
- ]
- m_build_matrix.return_value = build_matrix_output
- m_config_merge.return_value = [(a, b, {}) for a, b in build_matrix_output]
- m_get_install_task_flavor.return_value = 'default'
-
- m_find_git_parents.side_effect = lambda proj, sha1, count: [f"{sha1}_{i}" for i in range(11)]
-
- self.args.newest = 10
- runobj = self.klass(self.args)
- runobj.base_args = list()
- with pytest.raises(ScheduleFailError) as exc:
- runobj.schedule_suite()
- assert 'Exceeded 10 backtracks' in str(exc.value)
- m_find_git_parents.assert_has_calls(
- [call('ceph', 'ceph_sha1', 10)]
- )
-
- @patch('teuthology.suite.util.find_git_parents')
- @patch('teuthology.suite.run.Run.schedule_jobs')
- @patch('teuthology.suite.run.Run.write_rerun_memo')
- @patch('teuthology.suite.util.get_install_task_flavor')
- @patch('teuthology.suite.run.config_merge')
- @patch('teuthology.suite.run.build_matrix')
- @patch('teuthology.suite.util.git_ls_remote')
- @patch('teuthology.suite.util.package_version_for_hash')
- @patch('teuthology.suite.util.git_validate_sha1')
- @patch('teuthology.suite.util.get_arch')
- def test_newest_success_same_branch_same_repo(
- self,
- m_get_arch,
- m_git_validate_sha1,
- m_package_version_for_hash,
- m_git_ls_remote,
- m_build_matrix,
- m_config_merge,
- m_get_install_task_flavor,
- m_write_rerun_memo,
- m_schedule_jobs,
- m_find_git_parents,
- ):
- """
- Test that we can successfully schedule a job with newest
- backtracking when the ceph and suite branches are the same
- and the ceph_sha1 is not supplied. We should expect that the
- ceph_hash and suite_hash will be updated to the working sha1
- """
- m_get_arch.return_value = 'x86_64'
- # rig has_packages_for_distro to fail this many times, so
- # everything will run NUM_FAILS+1 times
- NUM_FAILS = 5
- # Here we just assume that even fi ceph_sha1 is not supplied,
- # in git_valid_sha1, util.git_ls_remote will give us ceph_sha1
- m_git_validate_sha1.return_value = self.args.ceph_sha1
- # Here we know that in create_initial_config, we call
- # git_ls_remote 3 times, choose_ceph_hash, choose_suite_hash,
- # and choose_teuthology_branch
- sha1_side_effect = [
- self.args.ceph_sha1, # ceph_sha1
- 'suite_sha1', # suite_sha1
- 'teuthology_sha1', # teuthology_sha1
- ]
- m_git_ls_remote.side_effect = sha1_side_effect
- build_matrix_desc = 'desc'
- build_matrix_frags = ['frag.yml']
- build_matrix_output = [
- (build_matrix_desc, build_matrix_frags),
- ]
- m_build_matrix.return_value = build_matrix_output
- m_config_merge.return_value = [(a, b, {}) for a, b in build_matrix_output]
- m_get_install_task_flavor.return_value = 'default'
-
- # Generate backtracked parent sha1s
- parent_sha1s = [f"ceph_sha1_{i}" for i in range(NUM_FAILS)]
- assert len(parent_sha1s)
- # Last sha1 will be the one that works!
- working_sha1 = parent_sha1s[-1]
-
- # NUM_FAILS attempts, then success on the last parent sha1
- m_package_version_for_hash.side_effect = \
- [None for i in range(NUM_FAILS)] + ["ceph_version"]
-
- m_find_git_parents.return_value = parent_sha1s
-
- self.args.newest = 10
- runobj = self.klass(self.args)
- runobj.base_args = list()
-
- # Call schedule_suite()
- count = runobj.schedule_suite()
- # Epect only 1 job to be scheduled
- assert count == 1
- # Expect that we called package_version_for_hash NUM_FAILS times + 1 for the working sha1
- m_package_version_for_hash.assert_has_calls(
- [call(self.args.ceph_sha1, 'default', 'ubuntu', '14.04', 'machine_type')] +
- [call(f"ceph_sha1_{i}", 'default', 'ubuntu', '14.04', 'machine_type')
- for i in range(0, NUM_FAILS)]
- )
- # (ceph, base_config.sha1, newest) called once to get grab the backtrace
- m_find_git_parents.assert_called_once_with('ceph', 'ceph_sha1', 10)
-
- # Verify that base_config was updated with the working SHA1
- assert runobj.base_config.sha1 == working_sha1
-
- # Verify that config_input's ceph_hash and suite_hash was updated
- assert runobj.config_input['ceph_hash'] == working_sha1
- assert runobj.config_input['suite_hash'] == working_sha1
-
- # Verify that config_input's ceph_hash and suite_hash are not the same as the original sha1s
- assert runobj.config_input['ceph_hash'] != sha1_side_effect[0] # ceph_sha1
- assert runobj.config_input['suite_hash'] != sha1_side_effect[1] # suite_sha1
-
- # Verify the sha1 in scheduled jobs
- args = m_schedule_jobs.call_args.args
- scheduled_jobs = args[1]
-
- # Check each job has the correct SHA1
- for job in scheduled_jobs:
- assert job['sha1'] == working_sha1
-
- # Parse YAML from stdin to check for sha1 and suite_hash
- if 'stdin' in job:
- job_yaml = yaml.safe_load(job['stdin'])
- assert job_yaml.get('sha1') == working_sha1
- assert job_yaml.get('suite_sha1') == working_sha1
-
- @patch('teuthology.suite.util.find_git_parents')
- @patch('teuthology.suite.run.Run.schedule_jobs')
- @patch('teuthology.suite.run.Run.write_rerun_memo')
- @patch('teuthology.suite.util.get_install_task_flavor')
- @patch('teuthology.suite.run.config_merge')
- @patch('teuthology.suite.run.build_matrix')
- @patch('teuthology.suite.util.git_ls_remote')
- @patch('teuthology.suite.util.package_version_for_hash')
- @patch('teuthology.suite.util.git_validate_sha1')
- @patch('teuthology.suite.util.get_arch')
- def test_newest_success_diff_branch_diff_repo(
- self,
- m_get_arch,
- m_git_validate_sha1,
- m_package_version_for_hash,
- m_git_ls_remote,
- m_build_matrix,
- m_config_merge,
- m_get_install_task_flavor,
- m_write_rerun_memo,
- m_schedule_jobs,
- m_find_git_parents,
- ):
- """
- Test that we can successfully schedule a job with newest
- backtracking when the ceph and suite branches are different
- and the ceph_sha1 is not supplied. We should expect that the
- ceph_hash will be updated to the working sha1,
- but the suite_hash will remain the original suite_sha1.
- """
- m_get_arch.return_value = 'x86_64'
- # Set different branches
- self.args.ceph_branch = 'ceph_different_branch'
- self.args.suite_branch = 'suite_different_branch'
-
- # rig has_packages_for_distro to fail this many times, so
- # everything will run NUM_FAILS+1 times
- NUM_FAILS = 5
- # Here we just assume that even fi ceph_sha1 is not supplied,
- # in git_valid_sha1, util.git_ls_remote will give us ceph_sha1
- m_git_validate_sha1.return_value = self.args.ceph_sha1
- # Here we know that in create_initial_config, we call
- # git_ls_remote 3 times, choose_ceph_hash, choose_suite_hash,
- # and choose_teuthology_branch
- sha1_side_effect = [
- self.args.ceph_sha1, # ceph_sha1
- 'suite_sha1', # suite_sha1
- 'teuthology_sha1', # teuthology_sha1
- ]
- m_git_ls_remote.side_effect = sha1_side_effect
- build_matrix_desc = 'desc'
- build_matrix_frags = ['frag.yml']
- build_matrix_output = [
- (build_matrix_desc, build_matrix_frags),
- ]
- m_build_matrix.return_value = build_matrix_output
- m_config_merge.return_value = [(a, b, {}) for a, b in build_matrix_output]
- m_get_install_task_flavor.return_value = 'default'
-
- # Generate backtracked parent sha1s
- parent_sha1s = [f"ceph_sha1_{i}" for i in range(NUM_FAILS)]
- assert len(parent_sha1s)
- # Last sha1 will be the one that works!
- working_sha1 = parent_sha1s[-1]
-
- # NUM_FAILS attempts, then success on the last parent sha1
- m_package_version_for_hash.side_effect = \
- [None for i in range(NUM_FAILS)] + ["ceph_version"]
-
- m_find_git_parents.return_value = parent_sha1s
-
- self.args.newest = 10
- runobj = self.klass(self.args)
- runobj.base_args = list()
-
- # Call schedule_suite()
- count = runobj.schedule_suite()
- # Epect only 1 job to be scheduled
- assert count == 1
- # Expect that we called package_version_for_hash NUM_FAILS times + 1 for the working sha1
- m_package_version_for_hash.assert_has_calls(
- [call(self.args.ceph_sha1, 'default', 'ubuntu', '14.04', 'machine_type')] +
- [call(f"ceph_sha1_{i}", 'default', 'ubuntu', '14.04', 'machine_type')
- for i in range(0, NUM_FAILS)]
- )
- # (ceph, base_config.sha1, newest) called once to get grab the backtrace
- m_find_git_parents.assert_called_once_with('ceph', 'ceph_sha1', 10)
-
- # Verify that base_config was updated with the working SHA1
- assert runobj.base_config.sha1 == working_sha1
-
- # Verify that config_input's ceph_hash was updated,
- # but suite_hash is still the original suite_sha1
- assert runobj.config_input['ceph_hash'] == working_sha1
- assert runobj.config_input['suite_hash'] != working_sha1
-
- # Verify that config_input's ceph_hash is not the same as the original sha1s
- # but suite_hash is still the original suite_sha1
- assert runobj.config_input['ceph_hash'] != sha1_side_effect[0] # ceph_sha1
- assert runobj.config_input['suite_hash'] == sha1_side_effect[1] # suite_sha1
-
- # Verify the sha1 in scheduled jobs
- args = m_schedule_jobs.call_args.args
- scheduled_jobs = args[1]
-
- # Check each job has the correct SHA1
- for job in scheduled_jobs:
- assert job['sha1'] == working_sha1
-
- # Parse YAML from stdin to check for sha1 and suite_hash
- if 'stdin' in job:
- job_yaml = yaml.safe_load(job['stdin'])
- assert job_yaml.get('sha1') == working_sha1
- assert job_yaml.get('suite_sha1') == sha1_side_effect[1]
+++ /dev/null
-import os
-import pytest
-import tempfile
-
-from mock import Mock, patch
-
-from teuthology.config import config
-from teuthology.orchestra.opsys import OS
-from teuthology.suite import util
-from teuthology.exceptions import BranchNotFoundError, ScheduleFailError
-
-
-REPO_PROJECTS_AND_URLS = [
- 'ceph',
- 'https://github.com/not_ceph/ceph.git',
-]
-
-
-@pytest.mark.parametrize('project_or_url', REPO_PROJECTS_AND_URLS)
-@patch('subprocess.check_output')
-def test_git_branch_exists(m_check_output, project_or_url):
- m_check_output.return_value = ''
- assert False == util.git_branch_exists(
- project_or_url, 'nobranchnowaycanthappen')
- m_check_output.return_value = b'HHH branch'
- assert True == util.git_branch_exists(project_or_url, 'main')
-
-
-@pytest.fixture
-def git_repository(request):
- d = tempfile.mkdtemp()
- os.system("""
- cd {d}
- git init
- touch A
- git config user.email 'you@example.com'
- git config user.name 'Your Name'
- git add A
- git commit -m 'A' A
- git rev-parse --abbrev-ref main || git checkout -b main
- """.format(d=d))
-
- def fin():
- os.system("rm -fr " + d)
- request.addfinalizer(fin)
- return d
-
-
-class TestUtil(object):
- @patch('teuthology.suite.util.smtplib.SMTP')
- def test_schedule_fail(self, m_smtp):
- config.results_email = "example@example.com"
- with pytest.raises(ScheduleFailError) as exc:
- util.schedule_fail(message="error msg", dry_run=False)
- assert str(exc.value) == "Scheduling failed: error msg"
- m_smtp.assert_called()
-
- @patch('teuthology.suite.util.smtplib.SMTP')
- def test_schedule_fail_dryrun(self, m_smtp):
- config.results_email = "example@example.com"
- with pytest.raises(ScheduleFailError) as exc:
- util.schedule_fail(message="error msg", dry_run=True)
- assert str(exc.value) == "Scheduling failed: error msg"
- m_smtp.assert_not_called()
-
- @patch('teuthology.suite.util.fetch_qa_suite')
- @patch('teuthology.suite.util.smtplib.SMTP')
- def test_fetch_repo_no_branch(self, m_smtp, m_fetch_qa_suite):
- m_fetch_qa_suite.side_effect = BranchNotFoundError(
- "no-branch", "https://github.com/ceph/ceph-ci.git")
- config.results_email = "example@example.com"
- with pytest.raises(ScheduleFailError) as exc:
- util.fetch_repos("no-branch", "test1", dry_run=False)
- assert str(exc.value) == "Scheduling test1 failed: \
-Branch 'no-branch' not found in repo: https://github.com/ceph/ceph-ci.git!"
- m_smtp.assert_called()
-
- @patch('teuthology.suite.util.fetch_qa_suite')
- @patch('teuthology.suite.util.smtplib.SMTP')
- def test_fetch_repo_no_branch_dryrun(self, m_smtp, m_fetch_qa_suite):
- m_fetch_qa_suite.side_effect = BranchNotFoundError(
- "no-branch", "https://github.com/ceph/ceph-ci.git")
- config.results_email = "example@example.com"
- with pytest.raises(ScheduleFailError) as exc:
- util.fetch_repos("no-branch", "test1", dry_run=True)
- assert str(exc.value) == "Scheduling test1 failed: \
-Branch 'no-branch' not found in repo: https://github.com/ceph/ceph-ci.git!"
- m_smtp.assert_not_called()
-
- @patch('requests.get')
- def test_get_branch_info(self, m_get):
- mock_resp = Mock()
- mock_resp.ok = True
- mock_resp.json.return_value = "some json"
- m_get.return_value = mock_resp
- result = util.get_branch_info("teuthology", "main")
- m_get.assert_called_with(
- "https://api.github.com/repos/ceph/teuthology/git/refs/heads/main"
- )
- assert result == "some json"
-
- @patch('teuthology.lock.query')
- def test_get_arch_fail(self, m_query):
- m_query.list_locks.return_value = False
- util.get_arch('magna')
- m_query.list_locks.assert_called_with(machine_type="magna", count=1, tries=1)
-
- @patch('teuthology.lock.query')
- def test_get_arch_success(self, m_query):
- m_query.list_locks.return_value = [{"arch": "arch"}]
- result = util.get_arch('magna')
- m_query.list_locks.assert_called_with(
- machine_type="magna",
- count=1, tries=1
- )
- assert result == "arch"
-
- def test_build_git_url_github(self):
- assert 'project' in util.build_git_url('project')
- owner = 'OWNER'
- git_url = util.build_git_url('project', project_owner=owner)
- assert owner in git_url
-
- @patch('teuthology.config.TeuthologyConfig.get_ceph_qa_suite_git_url')
- def test_build_git_url_ceph_qa_suite_custom(
- self,
- m_get_ceph_qa_suite_git_url):
- url = 'http://foo.com/some'
- m_get_ceph_qa_suite_git_url.return_value = url + '.git'
- assert url == util.build_git_url('ceph-qa-suite')
-
- @patch('teuthology.config.TeuthologyConfig.get_ceph_git_url')
- def test_build_git_url_ceph_custom(self, m_get_ceph_git_url):
- url = 'http://foo.com/some'
- m_get_ceph_git_url.return_value = url + '.git'
- assert url == util.build_git_url('ceph')
-
- @patch('teuthology.config.TeuthologyConfig.get_ceph_cm_ansible_git_url')
- def test_build_git_url_ceph_cm_ansible_custom(self, m_get_ceph_cm_ansible_git_url):
- url = 'http://foo.com/some'
- m_get_ceph_cm_ansible_git_url.return_value = url + '.git'
- assert url == util.build_git_url('ceph-cm-ansible')
-
- @patch('teuthology.config.TeuthologyConfig.get_ceph_git_url')
- def test_git_ls_remote(self, m_get_ceph_git_url, git_repository):
- m_get_ceph_git_url.return_value = git_repository
- assert util.git_ls_remote('ceph', 'nobranch') is None
- assert util.git_ls_remote('ceph', 'main') is not None
-
- @patch('teuthology.suite.util.requests.get')
- def test_find_git_parents(self, m_requests_get):
- history_resp = Mock(ok=True)
- history_resp.json.return_value = {'sha1s': ['sha1', 'sha1_p']}
- m_requests_get.return_value = history_resp
- parent_sha1s = util.find_git_parents('ceph', 'sha1')
- assert m_requests_get.call_count == 1
- assert parent_sha1s == ['sha1_p']
-
-
-class TestFlavor(object):
-
- def test_get_install_task_flavor_bare(self):
- config = dict(
- tasks=[
- dict(
- install=dict(),
- ),
- ],
- )
- assert util.get_install_task_flavor(config) == 'default'
-
- def test_get_install_task_flavor_simple(self):
- config = dict(
- tasks=[
- dict(
- install=dict(
- flavor='notcmalloc',
- ),
- ),
- ],
- )
- assert util.get_install_task_flavor(config) == 'notcmalloc'
-
- def test_get_install_task_flavor_override_simple(self):
- config = dict(
- tasks=[
- dict(install=dict()),
- ],
- overrides=dict(
- install=dict(
- flavor='notcmalloc',
- ),
- ),
- )
- assert util.get_install_task_flavor(config) == 'notcmalloc'
-
- def test_get_install_task_flavor_override_project(self):
- config = dict(
- tasks=[
- dict(install=dict()),
- ],
- overrides=dict(
- install=dict(
- ceph=dict(
- flavor='notcmalloc',
- ),
- ),
- ),
- )
- assert util.get_install_task_flavor(config) == 'notcmalloc'
-
-
-class TestMissingPackages(object):
- """
- Tests the functionality that checks to see if a
- scheduled job will have missing packages in shaman.
- """
- @patch("teuthology.packaging.ShamanProject._get_package_version")
- def test_distro_has_packages(self, m_gpv):
- m_gpv.return_value = "v1"
- result = util.package_version_for_hash(
- "sha1",
- "basic",
- "ubuntu",
- "14.04",
- "mtype",
- )
- assert result
-
- @patch("teuthology.packaging.ShamanProject._get_package_version")
- def test_distro_does_not_have_packages(self, m_gpv):
- m_gpv.return_value = None
- result = util.package_version_for_hash(
- "sha1",
- "basic",
- "rhel",
- "7.0",
- "mtype",
- )
- assert not result
-
-
-class TestDistroDefaults(object):
- def test_distro_defaults_plana(self):
- expected = ('x86_64', 'ubuntu/22.04',
- OS(name='ubuntu', version='22.04', codename='jammy'))
- assert util.get_distro_defaults('ubuntu', 'plana') == expected
-
- def test_distro_defaults_debian(self):
- expected = ('x86_64', 'debian/8.0',
- OS(name='debian', version='8.0', codename='jessie'))
- assert util.get_distro_defaults('debian', 'magna') == expected
-
- def test_distro_defaults_centos(self):
- expected = ('x86_64', 'centos/9',
- OS(name='centos', version='9.stream', codename='stream'))
- assert util.get_distro_defaults('centos', 'magna') == expected
-
- def test_distro_defaults_fedora(self):
- expected = ('x86_64', 'fedora/25',
- OS(name='fedora', version='25', codename='25'))
- assert util.get_distro_defaults('fedora', 'magna') == expected
-
- def test_distro_defaults_default(self):
- expected = ('x86_64', 'centos/9',
- OS(name='centos', version='9.stream', codename='stream'))
- assert util.get_distro_defaults('rhel', 'magna') == expected
super().end()
def run_cli(self):
- pytest_args = self.base_args + ['./teuthology/test', './scripts']
+ pytest_args = self.base_args + ['./tests/']
if len(self.cluster.remotes):
pytest_args.append('./teuthology/task/tests')
self.log.info(f"pytest args: {pytest_args}")
return status, []
def run_py(self):
- pytest_args = self.base_args + ['--pyargs', 'teuthology', 'scripts']
+ pytest_args = self.base_args + ['--pyargs', './tests/']
if len(self.cluster.remotes):
pytest_args.append(__name__)
self.log.info(f"pytest args: {pytest_args}")
+++ /dev/null
-import os
-import pytest
-import sys
-
-skipif_teuthology_process = pytest.mark.skipif(
- os.path.basename(sys.argv[0]) == "teuthology",
- reason="Skipped because this test cannot pass when run in a teuthology " \
- "process (as opposed to py.test)"
-)
\ No newline at end of file
+++ /dev/null
-import os
-import shutil
-import yaml
-import random
-
-
-class FakeArchive(object):
- def __init__(self, archive_base="./test_archive"):
- self.archive_base = archive_base
-
- def get_random_metadata(self, run_name, job_id=None, hung=False):
- """
- Generate a random info dict for a fake job. If 'hung' is not True, also
- generate a summary dict.
-
- :param run_name: Run name e.g. 'test_foo'
- :param job_id: Job ID e.g. '12345'
- :param hung: Simulate a hung job e.g. don't return a summary.yaml
- :return: A dict with keys 'job_id', 'info' and possibly
- 'summary', with corresponding values
- """
- rand = random.Random()
-
- description = 'description for job with id %s' % job_id
- owner = 'job@owner'
- duration = rand.randint(1, 36000)
- pid = rand.randint(1000, 99999)
- job_id = rand.randint(1, 99999)
-
- info = {
- 'description': description,
- 'job_id': job_id,
- 'run_name': run_name,
- 'owner': owner,
- 'pid': pid,
- }
-
- metadata = {
- 'info': info,
- 'job_id': job_id,
- }
-
- if not hung:
- success = True if rand.randint(0, 1) != 1 else False
-
- summary = {
- 'description': description,
- 'duration': duration,
- 'owner': owner,
- 'success': success,
- }
-
- if not success:
- summary['failure_reason'] = 'Failure reason!'
- metadata['summary'] = summary
-
- return metadata
-
- def setup(self):
- if os.path.exists(self.archive_base):
- shutil.rmtree(self.archive_base)
- os.mkdir(self.archive_base)
-
- def teardown(self):
- shutil.rmtree(self.archive_base)
-
- def populate_archive(self, run_name, jobs):
- run_archive_dir = os.path.join(self.archive_base, run_name)
- os.mkdir(run_archive_dir)
- for job in jobs:
- archive_dir = os.path.join(run_archive_dir, str(job['job_id']))
- os.mkdir(archive_dir)
-
- with open(os.path.join(archive_dir, 'info.yaml'), 'w') as yfile:
- yaml.safe_dump(job['info'], yfile)
-
- if 'summary' in job:
- summary_path = os.path.join(archive_dir, 'summary.yaml')
- with open(summary_path, 'w') as yfile:
- yaml.safe_dump(job['summary'], yfile)
-
- def create_fake_run(self, run_name, job_count, yaml_path, num_hung=0):
- """
- Creates a fake run using run_name. Uses the YAML specified for each
- job's config.yaml
-
- Returns a list of job_ids
- """
- assert os.path.exists(yaml_path)
- assert job_count > 0
- jobs = []
- made_hung = 0
- for i in range(job_count):
- if made_hung < num_hung:
- jobs.append(self.get_random_metadata(run_name, hung=True))
- made_hung += 1
- else:
- jobs.append(self.get_random_metadata(run_name, hung=False))
- #job_config = yaml.safe_load(yaml_path)
- self.populate_archive(run_name, jobs)
- for job in jobs:
- job_id = job['job_id']
- job_yaml_path = os.path.join(self.archive_base, run_name,
- str(job_id), 'config.yaml')
- shutil.copyfile(yaml_path, job_yaml_path)
- return jobs
-
+++ /dev/null
-from io import BytesIO
-from contextlib import closing
-
-
-try:
- FileNotFoundError, NotADirectoryError
-except NameError:
- FileNotFoundError = NotADirectoryError = OSError
-
-
-def make_fake_fstools(fake_filesystem):
- """
- Build fake versions of os.listdir(), os.isfile(), etc. for use in
- unit tests
-
- An example fake_filesystem value:
- >>> fake_fs = {\
- 'a_directory': {\
- 'another_directory': {\
- 'empty_file': None,\
- 'another_empty_file': None,\
- },\
- 'random_file': None,\
- 'yet_another_directory': {\
- 'empty_directory': {},\
- },\
- 'file_with_contents': 'data',\
- },\
- }
- >>> fake_listdir, fake_isfile, _, _ = \
- make_fake_fstools(fake_fs)
- >>> fake_listdir('a_directory/yet_another_directory')
- ['empty_directory']
- >>> fake_isfile('a_directory/yet_another_directory')
- False
-
- :param fake_filesystem: A dict representing a filesystem
- """
- assert isinstance(fake_filesystem, dict)
-
- def fake_listdir(path, fsdict=False):
- if fsdict is False:
- fsdict = fake_filesystem
-
- remainder = path.strip('/') + '/'
- subdict = fsdict
- while '/' in remainder:
- next_dir, remainder = remainder.split('/', 1)
- if next_dir not in subdict:
- raise FileNotFoundError(
- '[Errno 2] No such file or directory: %s' % next_dir)
- subdict = subdict.get(next_dir)
- if not isinstance(subdict, dict):
- raise NotADirectoryError('[Errno 20] Not a directory: %s' % next_dir)
- if subdict and not remainder:
- return list(subdict)
- return []
-
- def fake_isfile(path, fsdict=False):
- if fsdict is False:
- fsdict = fake_filesystem
-
- components = path.strip('/').split('/')
- subdict = fsdict
- for component in components:
- if component not in subdict:
- raise FileNotFoundError('[Errno 2] No such file or directory: %s' % component)
- subdict = subdict.get(component)
- return subdict is None or isinstance(subdict, str)
-
- def fake_isdir(path, fsdict=False):
- return not fake_isfile(path)
-
- def fake_exists(path, fsdict=False):
- return fake_isfile(path, fsdict) or fake_isdir(path, fsdict)
-
- def fake_open(path, mode=None, buffering=None):
- components = path.strip('/').split('/')
- subdict = fake_filesystem
- for component in components:
- if component not in subdict:
- raise FileNotFoundError('[Errno 2] No such file or directory: %s' % component)
- subdict = subdict.get(component)
- if isinstance(subdict, dict):
- raise IOError('[Errno 21] Is a directory: %s' % path)
- elif subdict is None:
- return closing(BytesIO(b''))
- return closing(BytesIO(subdict.encode()))
-
- return fake_exists, fake_listdir, fake_isfile, fake_isdir, fake_open
+++ /dev/null
-import os
-import requests
-from pytest import raises, skip
-
-from teuthology.config import config
-from teuthology import suite
-
-
-class TestSuiteOnline(object):
- def setup_method(self):
- if 'TEST_ONLINE' not in os.environ:
- skip("To run these sets, set the environment variable TEST_ONLINE")
-
- def test_ceph_hash_simple(self):
- resp = requests.get(
- 'https://api.github.com/repos/ceph/ceph/git/refs/heads/main')
- ref_hash = resp.json()['object']['sha']
- assert suite.get_hash('ceph') == ref_hash
-
- def test_kernel_hash_saya(self):
- # We don't currently have these packages.
- assert suite.get_hash('kernel', 'main', 'default', 'saya') is None
-
- def test_all_main_branches(self):
- # Don't attempt to send email
- config.results_email = None
- job_config = suite.create_initial_config('suite', 'main',
- 'main', 'main', 'testing',
- 'default', 'centos', 'plana')
- assert ((job_config.branch, job_config.teuthology_branch,
- job_config.suite_branch) == ('main', 'main', 'main'))
-
- def test_config_bogus_kernel_branch(self):
- # Don't attempt to send email
- config.results_email = None
- with raises(suite.ScheduleFailError):
- suite.create_initial_config('s', None, 'main', 't',
- 'bogus_kernel_branch', 'f', 'd', 'm')
-
- def test_config_bogus_flavor(self):
- # Don't attempt to send email
- config.results_email = None
- with raises(suite.ScheduleFailError):
- suite.create_initial_config('s', None, 'main', 't', 'k',
- 'bogus_flavor', 'd', 'm')
-
- def test_config_bogus_ceph_branch(self):
- # Don't attempt to send email
- config.results_email = None
- with raises(suite.ScheduleFailError):
- suite.create_initial_config('s', None, 'bogus_ceph_branch', 't',
- 'k', 'f', 'd', 'm')
-
- def test_config_bogus_suite_branch(self):
- # Don't attempt to send email
- config.results_email = None
- with raises(suite.ScheduleFailError):
- suite.create_initial_config('s', 'bogus_suite_branch', 'main',
- 't', 'k', 'f', 'd', 'm')
-
- def test_config_bogus_teuthology_branch(self):
- # Don't attempt to send email
- config.results_email = None
- with raises(suite.ScheduleFailError):
- suite.create_initial_config('s', None, 'main',
- 'bogus_teuth_branch', 'k', 'f', 'd',
- 'm')
-
- def test_config_substitution(self):
- # Don't attempt to send email
- config.results_email = None
- job_config = suite.create_initial_config('MY_SUITE', 'main',
- 'main', 'main', 'testing',
- 'default', 'centos', 'plana')
- assert job_config['suite'] == 'MY_SUITE'
-
- def test_config_kernel_section(self):
- # Don't attempt to send email
- config.results_email = None
- job_config = suite.create_initial_config('MY_SUITE', 'main',
- 'main', 'main', 'testing',
- 'default', 'centos', 'plana')
- assert job_config['kernel']['kdb'] is True
-
-
-# maybe use notario for the above?
+++ /dev/null
-from mock import patch, DEFAULT
-from pytest import raises
-
-from teuthology.config import FakeNamespace
-from teuthology.orchestra.cluster import Cluster
-from teuthology.orchestra.remote import Remote
-from teuthology.task import Task
-
-
-class TestTask(object):
- klass = Task
- task_name = 'task'
-
- def setup_method(self):
- self.ctx = FakeNamespace()
- self.ctx.config = dict()
- self.task_config = dict()
-
- def test_overrides(self):
- self.ctx.config['overrides'] = dict()
- self.ctx.config['overrides'][self.task_name] = dict(
- key_1='overridden',
- )
- self.task_config.update(dict(
- key_1='default',
- key_2='default',
- ))
- with patch.multiple(
- self.klass,
- begin=DEFAULT,
- end=DEFAULT,
- ):
- with self.klass(self.ctx, self.task_config) as task:
- assert task.config['key_1'] == 'overridden'
- assert task.config['key_2'] == 'default'
-
- def test_hosts_no_filter(self):
- self.ctx.cluster = Cluster()
- self.ctx.cluster.add(Remote('user@remote1'), ['role1'])
- self.ctx.cluster.add(Remote('user@remote2'), ['role2'])
- with patch.multiple(
- self.klass,
- begin=DEFAULT,
- end=DEFAULT,
- ):
- with self.klass(self.ctx, self.task_config) as task:
- task_hosts = list(task.cluster.remotes)
- assert len(task_hosts) == 2
- assert sorted(host.shortname for host in task_hosts) == \
- ['remote1', 'remote2']
-
- def test_hosts_no_results(self):
- self.ctx.cluster = Cluster()
- self.ctx.cluster.add(Remote('user@remote1'), ['role1'])
- self.task_config.update(dict(
- hosts=['role2'],
- ))
- with patch.multiple(
- self.klass,
- begin=DEFAULT,
- end=DEFAULT,
- ):
- with raises(RuntimeError):
- with self.klass(self.ctx, self.task_config):
- pass
-
- def test_hosts_one_role(self):
- self.ctx.cluster = Cluster()
- self.ctx.cluster.add(Remote('user@remote1'), ['role1'])
- self.ctx.cluster.add(Remote('user@remote2'), ['role2'])
- self.task_config.update(dict(
- hosts=['role1'],
- ))
- with patch.multiple(
- self.klass,
- begin=DEFAULT,
- end=DEFAULT,
- ):
- with self.klass(self.ctx, self.task_config) as task:
- task_hosts = list(task.cluster.remotes)
- assert len(task_hosts) == 1
- assert task_hosts[0].shortname == 'remote1'
-
- def test_hosts_two_roles(self):
- self.ctx.cluster = Cluster()
- self.ctx.cluster.add(Remote('user@remote1'), ['role1'])
- self.ctx.cluster.add(Remote('user@remote2'), ['role2'])
- self.ctx.cluster.add(Remote('user@remote3'), ['role3'])
- self.task_config.update(dict(
- hosts=['role1', 'role3'],
- ))
- with patch.multiple(
- self.klass,
- begin=DEFAULT,
- end=DEFAULT,
- ):
- with self.klass(self.ctx, self.task_config) as task:
- task_hosts = list(task.cluster.remotes)
- assert len(task_hosts) == 2
- hostnames = [host.shortname for host in task_hosts]
- assert sorted(hostnames) == ['remote1', 'remote3']
-
- def test_hosts_two_hostnames(self):
- self.ctx.cluster = Cluster()
- self.ctx.cluster.add(Remote('user@remote1.example.com'), ['role1'])
- self.ctx.cluster.add(Remote('user@remote2.example.com'), ['role2'])
- self.ctx.cluster.add(Remote('user@remote3.example.com'), ['role3'])
- self.task_config.update(dict(
- hosts=['remote1', 'remote2.example.com'],
- ))
- with patch.multiple(
- self.klass,
- begin=DEFAULT,
- end=DEFAULT,
- ):
- with self.klass(self.ctx, self.task_config) as task:
- task_hosts = list(task.cluster.remotes)
- assert len(task_hosts) == 2
- hostnames = [host.hostname for host in task_hosts]
- assert sorted(hostnames) == ['remote1.example.com',
- 'remote2.example.com']
-
- def test_hosts_one_role_one_hostname(self):
- self.ctx.cluster = Cluster()
- self.ctx.cluster.add(Remote('user@remote1.example.com'), ['role1'])
- self.ctx.cluster.add(Remote('user@remote2.example.com'), ['role2'])
- self.ctx.cluster.add(Remote('user@remote3.example.com'), ['role3'])
- self.task_config.update(dict(
- hosts=['role1', 'remote2.example.com'],
- ))
- with patch.multiple(
- self.klass,
- begin=DEFAULT,
- end=DEFAULT,
- ):
- with self.klass(self.ctx, self.task_config) as task:
- task_hosts = list(task.cluster.remotes)
- assert len(task_hosts) == 2
- hostnames = [host.hostname for host in task_hosts]
- assert sorted(hostnames) == ['remote1.example.com',
- 'remote2.example.com']
-
- def test_setup_called(self):
- with patch.multiple(
- self.klass,
- setup=DEFAULT,
- begin=DEFAULT,
- end=DEFAULT,
- teardown=DEFAULT,
- ):
- with self.klass(self.ctx, self.task_config) as task:
- task.setup.assert_called_once_with()
-
- def test_begin_called(self):
- with patch.multiple(
- self.klass,
- setup=DEFAULT,
- begin=DEFAULT,
- end=DEFAULT,
- teardown=DEFAULT,
- ):
- with self.klass(self.ctx, self.task_config) as task:
- task.begin.assert_called_once_with()
-
- def test_end_called(self):
- self.task_config.update(dict())
- with patch.multiple(
- self.klass,
- begin=DEFAULT,
- end=DEFAULT,
- ):
- with self.klass(self.ctx, self.task_config) as task:
- pass
- task.end.assert_called_once_with()
-
- def test_teardown_called(self):
- self.task_config.update(dict())
- with patch.multiple(
- self.klass,
- setup=DEFAULT,
- begin=DEFAULT,
- end=DEFAULT,
- teardown=DEFAULT,
- ):
- with self.klass(self.ctx, self.task_config) as task:
- pass
- task.teardown.assert_called_once_with()
-
- def test_skip_teardown(self):
- self.task_config.update(dict(
- skip_teardown=True,
- ))
-
- def fake_teardown(self):
- assert False
-
- with patch.multiple(
- self.klass,
- setup=DEFAULT,
- begin=DEFAULT,
- end=DEFAULT,
- teardown=fake_teardown,
- ):
- with self.klass(self.ctx, self.task_config):
- pass
+++ /dev/null
-import json
-import os
-import yaml
-
-from unittest.mock import patch, DEFAULT, Mock, mock_open
-from pytest import raises, mark
-from teuthology.util.compat import PY3
-if PY3:
- from io import StringIO as StringIO
-else:
- from io import BytesIO as StringIO
-
-from teuthology.config import config, FakeNamespace
-from teuthology.exceptions import CommandFailedError
-from teuthology.orchestra.cluster import Cluster
-from teuthology.orchestra.remote import Remote
-from teuthology.task import ansible
-from teuthology.task.ansible import Ansible, CephLab, FailureAnalyzer
-
-from teuthology.test.task import TestTask
-
-
-class TestFailureAnalyzer:
- klass = FailureAnalyzer
-
- @mark.parametrize(
- 'line,result',
- [
- [
- "W: --force-yes is deprecated, use one of the options starting with --allow instead.",
- "",
- ],
- [
- "E: Unable to fetch some archives, maybe run apt-get update or try with --fix-missing?",
- "",
- ],
- [
- "E: Failed to fetch http://security.ubuntu.com/ubuntu/pool/main/a/apache2/apache2-bin_2.4.41-4ubuntu3.14_amd64.deb Unable to connect to archive.ubuntu.com:http:",
- "Unable to connect to archive.ubuntu.com:http:"
- ],
- [
- "E: Failed to fetch http://archive.ubuntu.com/ubuntu/pool/main/libb/libb-hooks-op-check-perl/libb-hooks-op-check-perl_0.22-1build2_amd64.deb Temporary failure resolving 'archive.ubuntu.com'",
- "Temporary failure resolving 'archive.ubuntu.com'"
- ],
- [
- "Data could not be sent to remote host \"smithi068.front.sepia.ceph.com\".",
- "Data could not be sent to remote host \"smithi068.front.sepia.ceph.com\"."
- ],
- [
- "Permissions 0644 for '/root/.ssh/id_rsa' are too open.",
- "Permissions 0644 for '/root/.ssh/id_rsa' are too open."
- ],
- ]
- )
- def test_lines(self, line, result):
- obj = self.klass()
- assert obj.analyze_line(line) == result
-
-
-class TestAnsibleTask(TestTask):
- klass = Ansible
- task_name = 'ansible'
-
- def setup_method(self):
- self.ctx = FakeNamespace()
- self.ctx.cluster = Cluster()
- self.ctx.cluster.add(Remote('user@remote1'), ['role1'])
- self.ctx.cluster.add(Remote('user@remote2'), ['role2'])
- self.ctx.config = dict()
- self.ctx.summary = dict()
- self.ctx.archive = ""
- self.task_config = dict(playbook=[])
- self.start_patchers()
-
- def start_patchers(self):
- self.patchers = dict()
- self.mocks = dict()
- self.patchers['mkdtemp'] = patch(
- 'teuthology.task.ansible.mkdtemp', return_value='/tmp/'
- )
- m_NTF = Mock()
- m_file = Mock()
- m_file.name = 'file_name'
- m_NTF.return_value = m_file
- self.patchers['NTF'] = patch(
- 'teuthology.task.ansible.NamedTemporaryFile',
- m_NTF,
- )
- self.patchers['file'] = patch(
- 'teuthology.task.ansible.open', create=True)
- self.patchers['os_mkdir'] = patch(
- 'teuthology.task.ansible.os.mkdir',
- )
- self.patchers['os_remove'] = patch(
- 'teuthology.task.ansible.os.remove',
- )
- self.patchers['shutil_rmtree'] = patch(
- 'teuthology.task.ansible.shutil.rmtree',
- )
- for name in self.patchers.keys():
- self.start_patcher(name)
-
- def start_patcher(self, name):
- if name not in self.mocks.keys():
- self.mocks[name] = self.patchers[name].start()
-
- def teardown_method(self, method):
- self.stop_patchers()
-
- def stop_patchers(self):
- for name in list(self.mocks):
- self.stop_patcher(name)
-
- def stop_patcher(self, name):
- self.patchers[name].stop()
- del self.mocks[name]
-
- def test_setup(self):
- self.task_config.update(dict(
- playbook=[]
- ))
-
- def fake_get_playbook(self):
- self.playbook_file = 'fake'
-
- with patch.multiple(
- self.klass,
- find_repo=DEFAULT,
- get_playbook=fake_get_playbook,
- get_inventory=DEFAULT,
- generate_inventory=DEFAULT,
- generate_playbook=Mock(side_effect=Exception),
- ):
- task = self.klass(self.ctx, self.task_config)
- task.setup()
-
- def test_setup_generate_playbook(self):
- self.task_config.update(dict(
- playbook=[]
- ))
- with patch.multiple(
- self.klass,
- find_repo=DEFAULT,
- get_playbook=DEFAULT,
- get_inventory=DEFAULT,
- generate_inventory=DEFAULT,
- generate_playbook=DEFAULT,
- ):
- task = self.klass(self.ctx, self.task_config)
- task.setup()
- task.generate_playbook.assert_called_once_with()
-
- def test_find_repo_path(self):
- self.task_config.update(dict(
- repo='~/my/repo',
- ))
- task = self.klass(self.ctx, self.task_config)
- task.find_repo()
- assert task.repo_path == os.path.expanduser(self.task_config['repo'])
-
- @patch('teuthology.repo_utils.fetch_repo')
- def test_find_repo_path_remote(self, m_fetch_repo):
- self.task_config.update(dict(
- repo='git://fake_host/repo.git',
- ))
- m_fetch_repo.return_value = '/tmp/repo'
- task = self.klass(self.ctx, self.task_config)
- task.find_repo()
- assert task.repo_path == os.path.expanduser('/tmp/repo')
-
- @patch('teuthology.repo_utils.fetch_repo')
- def test_find_repo_http(self, m_fetch_repo):
- self.task_config.update(dict(
- repo='http://example.com/my/repo',
- ))
- task = self.klass(self.ctx, self.task_config)
- task.find_repo()
- m_fetch_repo.assert_called_once_with(self.task_config['repo'],
- 'main')
-
- @patch('teuthology.repo_utils.fetch_repo')
- def test_find_repo_git(self, m_fetch_repo):
- self.task_config.update(dict(
- repo='git@example.com/my/repo',
- ))
- task = self.klass(self.ctx, self.task_config)
- task.find_repo()
- m_fetch_repo.assert_called_once_with(self.task_config['repo'],
- 'main')
-
- def test_playbook_none(self):
- del self.task_config['playbook']
- task = self.klass(self.ctx, self.task_config)
- with raises(KeyError):
- task.get_playbook()
-
- def test_playbook_wrong_type(self):
- self.task_config.update(dict(
- playbook=dict(),
- ))
- task = self.klass(self.ctx, self.task_config)
- with raises(TypeError):
- task.get_playbook()
-
- def test_playbook_list(self):
- playbook = [
- dict(
- roles=['role1'],
- ),
- ]
- self.task_config.update(dict(
- playbook=playbook,
- ))
- task = self.klass(self.ctx, self.task_config)
- task.get_playbook()
- assert task.playbook == playbook
-
- @patch.object(ansible.requests, 'get')
- def test_playbook_http(self, m_get):
- m_get.return_value = Mock()
- m_get.return_value.text = 'fake playbook text'
- playbook = "http://example.com/my_playbook.yml"
- self.task_config.update(dict(
- playbook=playbook,
- ))
- task = self.klass(self.ctx, self.task_config)
- task.get_playbook()
- m_get.assert_called_once_with(playbook)
-
- def test_playbook_file(self):
- fake_playbook = [dict(fake_playbook=True)]
- fake_playbook_obj = StringIO(yaml.safe_dump(fake_playbook))
- self.task_config.update(dict(
- playbook='~/fake/playbook',
- ))
- task = self.klass(self.ctx, self.task_config)
- self.mocks['file'].return_value = fake_playbook_obj
- task.get_playbook()
- assert task.playbook == fake_playbook
-
- def test_playbook_file_missing(self):
- self.task_config.update(dict(
- playbook='~/fake/playbook',
- ))
- task = self.klass(self.ctx, self.task_config)
- self.mocks['file'].side_effect = IOError
- with raises(IOError):
- task.get_playbook()
-
- def test_inventory_none(self):
- self.task_config.update(dict(
- playbook=[]
- ))
- task = self.klass(self.ctx, self.task_config)
- with patch.object(ansible.os.path, 'exists') as m_exists:
- m_exists.return_value = False
- task.get_inventory()
- assert task.inventory is None
-
- def test_inventory_path(self):
- inventory = '/my/inventory'
- self.task_config.update(dict(
- playbook=[],
- inventory=inventory,
- ))
- task = self.klass(self.ctx, self.task_config)
- task.get_inventory()
- assert task.inventory == inventory
- assert task.generated_inventory is False
-
- def test_inventory_etc(self):
- self.task_config.update(dict(
- playbook=[]
- ))
- task = self.klass(self.ctx, self.task_config)
- with patch.object(ansible.os.path, 'exists') as m_exists:
- m_exists.return_value = True
- task.get_inventory()
- assert task.inventory == '/etc/ansible/hosts'
- assert task.generated_inventory is False
-
- @mark.parametrize(
- 'group_vars',
- [
- dict(),
- dict(all=dict(var0=0, var1=1)),
- dict(foo=dict(var0=0), bar=dict(var0=1)),
- ]
- )
- def test_generate_inventory(self, group_vars):
- self.task_config.update(dict(
- playbook=[]
- ))
- if group_vars:
- self.task_config.update(dict(group_vars=group_vars))
- task = self.klass(self.ctx, self.task_config)
- hosts_file_path = '/my/hosts/inventory'
- hosts_file_obj = StringIO()
- hosts_file_obj.name = hosts_file_path
- inventory_dir = os.path.dirname(hosts_file_path)
- gv_dir = os.path.join(inventory_dir, 'group_vars')
- self.mocks['mkdtemp'].return_value = inventory_dir
- m_file = self.mocks['file']
- fake_files = [hosts_file_obj]
- # Create StringIO object for each group_vars file
- if group_vars:
- fake_files += [StringIO() for i in sorted(group_vars)]
- m_file.side_effect = fake_files
- task.generate_inventory()
- file_calls = m_file.call_args_list
- # Verify the inventory file was created
- assert file_calls[0][0][0] == hosts_file_path
- # Verify each group_vars file was created
- for gv_name, call_obj in zip(sorted(group_vars), file_calls[1:]):
- gv_path = call_obj[0][0]
- assert gv_path == os.path.join(gv_dir, '%s.yml' % gv_name)
- # Verify the group_vars dir was created
- if group_vars:
- mkdir_call = self.mocks['os_mkdir'].call_args_list
- assert mkdir_call[0][0][0] == gv_dir
- assert task.generated_inventory is True
- assert task.inventory == inventory_dir
- # Verify the content of the inventory *file*
- hosts_file_obj.seek(0)
- assert hosts_file_obj.readlines() == [
- 'remote1\n',
- 'remote2\n',
- ]
- # Verify the contents of each group_vars file
- gv_names = sorted(group_vars)
- for i in range(len(gv_names)):
- gv_name = gv_names[i]
- in_val = group_vars[gv_name]
- gv_stringio = fake_files[1 + i]
- gv_stringio.seek(0)
- out_val = yaml.safe_load(gv_stringio)
- assert in_val == out_val
-
- def test_generate_playbook(self):
- playbook = [
- dict(
- roles=['role1', 'role2'],
- ),
- ]
- self.task_config.update(dict(
- playbook=playbook
- ))
- task = self.klass(self.ctx, self.task_config)
- playbook_file_path = '/my/playbook/file'
- playbook_file_obj = StringIO()
- playbook_file_obj.name = playbook_file_path
- with patch.object(ansible, 'NamedTemporaryFile') as m_NTF:
- m_NTF.return_value = playbook_file_obj
- task.find_repo()
- task.get_playbook()
- task.generate_playbook()
- m_NTF.assert_called_once_with(
- prefix="teuth_ansible_playbook_",
- dir=task.repo_path,
- delete=False,
- )
- assert task.generated_playbook is True
- assert task.playbook_file == playbook_file_obj
- playbook_file_obj.seek(0)
- playbook_result = yaml.safe_load(playbook_file_obj)
- assert playbook_result == playbook
-
- def test_execute_playbook(self):
- playbook = '/my/playbook'
- self.task_config.update(dict(
- playbook=playbook
- ))
- fake_playbook = [dict(fake_playbook=True)]
- fake_playbook_obj = StringIO(yaml.safe_dump(fake_playbook))
- fake_playbook_obj.name = playbook
- self.mocks['mkdtemp'].return_value = '/inventory/dir'
-
- task = self.klass(self.ctx, self.task_config)
- self.mocks['file'].return_value = fake_playbook_obj
- task.setup()
- args = task._build_args()
- logger = StringIO()
- with patch.object(ansible.pexpect, 'run') as m_run:
- m_run.return_value = ('', 0)
- with patch.object(Remote, 'reconnect') as m_reconnect:
- m_reconnect.return_value = True
- task.execute_playbook(_logfile=logger)
- m_run.assert_called_once_with(
- ' '.join(args),
- cwd=task.repo_path,
- logfile=logger,
- withexitstatus=True,
- timeout=None,
- )
-
- def test_execute_playbook_fail(self):
- self.task_config.update(dict(
- playbook=[],
- ))
- self.mocks['mkdtemp'].return_value = '/inventory/dir'
- task = self.klass(self.ctx, self.task_config)
- task.setup()
- with patch.object(ansible.pexpect, 'run') as m_run:
- with patch('teuthology.task.ansible.open', mock_open()):
- m_run.return_value = ('', 1)
- with raises(CommandFailedError):
- task.execute_playbook()
- assert task.ctx.summary.get('status') is None
-
- def test_build_args_no_tags(self):
- self.task_config.update(dict(
- playbook=[],
- ))
- task = self.klass(self.ctx, self.task_config)
- task.setup()
- args = task._build_args()
- assert '--tags' not in args
-
- def test_build_args_tags(self):
- self.task_config.update(dict(
- playbook=[],
- tags="user,pubkeys"
- ))
- task = self.klass(self.ctx, self.task_config)
- task.setup()
- args = task._build_args()
- assert args.count('--tags') == 1
- assert args[args.index('--tags') + 1] == 'user,pubkeys'
-
- def test_build_args_skip_tags(self):
- self.task_config.update(dict(
- playbook=[],
- skip_tags="user,pubkeys"
- ))
- task = self.klass(self.ctx, self.task_config)
- task.setup()
- args = task._build_args()
- assert args.count('--skip-tags') == 1
- assert args[args.index('--skip-tags') + 1] == 'user,pubkeys'
-
- def test_build_args_no_vars(self):
- self.task_config.update(dict(
- playbook=[],
- ))
- task = self.klass(self.ctx, self.task_config)
- task.setup()
- args = task._build_args()
- assert args.count('--extra-vars') == 1
- vars_str = args[args.index('--extra-vars') + 1].strip("'")
- extra_vars = json.loads(vars_str)
- assert list(extra_vars) == ['ansible_ssh_user']
-
- def test_build_args_vars(self):
- extra_vars = dict(
- string1='value1',
- list1=['item1'],
- dict1=dict(key='value'),
- )
-
- self.task_config.update(dict(
- playbook=[],
- vars=extra_vars,
- ))
- task = self.klass(self.ctx, self.task_config)
- task.setup()
- args = task._build_args()
- assert args.count('--extra-vars') == 1
- vars_str = args[args.index('--extra-vars') + 1].strip("'")
- got_extra_vars = json.loads(vars_str)
- assert 'ansible_ssh_user' in got_extra_vars
- assert got_extra_vars['string1'] == extra_vars['string1']
- assert got_extra_vars['list1'] == extra_vars['list1']
- assert got_extra_vars['dict1'] == extra_vars['dict1']
-
- def test_teardown_inventory(self):
- self.task_config.update(dict(
- playbook=[],
- ))
- task = self.klass(self.ctx, self.task_config)
- task.generated_inventory = True
- task.inventory = 'fake'
- with patch.object(ansible.shutil, 'rmtree') as m_rmtree:
- task.teardown()
- m_rmtree.assert_called_once_with('fake')
-
- def test_teardown_playbook(self):
- self.task_config.update(dict(
- playbook=[],
- ))
- task = self.klass(self.ctx, self.task_config)
- task.generated_playbook = True
- task.playbook_file = Mock()
- task.playbook_file.name = 'fake'
- with patch.object(ansible.os, 'remove') as m_remove:
- task.teardown()
- m_remove.assert_called_once_with('fake')
-
- def test_teardown_cleanup_with_vars(self):
- self.task_config.update(dict(
- playbook=[],
- cleanup=True,
- vars=dict(yum_repos="testing"),
- ))
- task = self.klass(self.ctx, self.task_config)
- task.inventory = "fake"
- task.generated_playbook = True
- task.playbook_file = Mock()
- task.playbook_file.name = 'fake'
- with patch.object(self.klass, 'execute_playbook') as m_execute:
- with patch.object(ansible.os, 'remove'):
- task.teardown()
- task._build_args()
- assert m_execute.called
- assert 'cleanup' in task.config['vars']
- assert 'yum_repos' in task.config['vars']
-
- def test_teardown_cleanup_with_no_vars(self):
- self.task_config.update(dict(
- playbook=[],
- cleanup=True,
- ))
- task = self.klass(self.ctx, self.task_config)
- task.inventory = "fake"
- task.generated_playbook = True
- task.playbook_file = Mock()
- task.playbook_file.name = 'fake'
- with patch.object(self.klass, 'execute_playbook') as m_execute:
- with patch.object(ansible.os, 'remove'):
- task.teardown()
- task._build_args()
- assert m_execute.called
- assert 'cleanup' in task.config['vars']
-
- def test_no_remotes(self):
- self.task_config.update(dict(
- playbook=[],
- ))
- self.ctx.cluster.remotes = dict()
- task = self.klass(self.ctx, self.task_config)
- with patch.object(ansible.pexpect, 'run') as m_run:
- task.setup()
- task.begin()
- assert not m_run.called
-
-
-class TestCephLabTask(TestAnsibleTask):
- klass = CephLab
- task_name = 'ansible.cephlab'
-
- def setup_method(self):
- super(TestCephLabTask, self).setup_method()
- self.task_config = dict()
-
- def start_patchers(self):
- super(TestCephLabTask, self).start_patchers()
- self.patchers['fetch_repo'] = patch(
- 'teuthology.repo_utils.fetch_repo',
- )
- self.patchers['fetch_repo'].return_value = 'PATH'
-
- def fake_get_playbook(self):
- self.playbook_file = Mock()
- self.playbook_file.name = 'cephlab.yml'
-
- self.patchers['get_playbook'] = patch(
- 'teuthology.task.ansible.CephLab.get_playbook',
- new=fake_get_playbook,
- )
- for name in self.patchers.keys():
- self.start_patcher(name)
-
- @patch('teuthology.repo_utils.fetch_repo')
- def test_find_repo_http(self, m_fetch_repo):
- repo = os.path.join(config.ceph_git_base_url,
- 'ceph-cm-ansible.git')
- task = self.klass(self.ctx, dict())
- task.find_repo()
- m_fetch_repo.assert_called_once_with(repo, 'main')
-
- def test_playbook_file(self):
- fake_playbook = [dict(fake_playbook=True)]
- fake_playbook_obj = StringIO(yaml.safe_dump(fake_playbook))
- playbook = 'cephlab.yml'
- fake_playbook_obj.name = playbook
- task = self.klass(self.ctx, dict())
- task.repo_path = '/tmp/fake/repo'
- self.mocks['file'].return_value = fake_playbook_obj
- task.get_playbook()
- assert task.playbook_file.name == playbook
-
- def test_generate_inventory(self):
- self.task_config.update(dict(
- playbook=[]
- ))
- task = self.klass(self.ctx, self.task_config)
- hosts_file_path = '/my/hosts/file'
- hosts_file_obj = StringIO()
- hosts_file_obj.name = hosts_file_path
- self.mocks['mkdtemp'].return_value = os.path.dirname(hosts_file_path)
- self.mocks['file'].return_value = hosts_file_obj
- task.generate_inventory()
- assert task.generated_inventory is True
- assert task.inventory == os.path.dirname(hosts_file_path)
- hosts_file_obj.seek(0)
- assert hosts_file_obj.readlines() == [
- '[testnodes]\n',
- 'remote1\n',
- 'remote2\n',
- ]
-
- def test_fail_status_dead(self):
- self.task_config.update(dict(
- playbook=[],
- ))
- self.mocks['mkdtemp'].return_value = '/inventory/dir'
- task = self.klass(self.ctx, self.task_config)
- task.ctx.summary = dict()
- task.setup()
- with patch.object(ansible.pexpect, 'run') as m_run:
- with patch('teuthology.task.ansible.open', mock_open()):
- m_run.return_value = ('', 1)
- with raises(CommandFailedError):
- task.execute_playbook()
- assert task.ctx.summary.get('status') == 'dead'
-
- def test_execute_playbook_fail(self):
- self.mocks['mkdtemp'].return_value = '/inventory/dir'
- task = self.klass(self.ctx, self.task_config)
- task.setup()
- with patch.object(ansible.pexpect, 'run') as m_run:
- with patch('teuthology.task.ansible.open', mock_open()):
- m_run.return_value = ('', 1)
- with raises(CommandFailedError):
- task.execute_playbook()
- assert task.ctx.summary.get('status') == 'dead'
-
- @mark.skip("Unsupported")
- def test_generate_playbook(self):
- pass
-
- @mark.skip("Unsupported")
- def test_playbook_http(self):
- pass
-
- @mark.skip("Unsupported")
- def test_playbook_none(self):
- pass
-
- @mark.skip("Unsupported")
- def test_playbook_wrong_type(self):
- pass
-
- @mark.skip("Unsupported")
- def test_playbook_list(self):
- pass
-
- @mark.skip("Test needs to be reimplemented for this class")
- def test_playbook_file_missing(self):
- pass
+++ /dev/null
-from mock import patch, MagicMock
-from pytest import skip
-from teuthology.util.compat import PY3
-if PY3:
- from io import StringIO as StringIO
-else:
- from io import BytesIO as StringIO
-
-from teuthology.config import FakeNamespace
-from teuthology.orchestra.cluster import Cluster
-from teuthology.orchestra.remote import Remote
-from teuthology.task import ceph_ansible
-from teuthology.task.ceph_ansible import CephAnsible
-
-from teuthology.test.task import TestTask
-
-SKIP_IRRELEVANT = "Not relevant to this subclass"
-
-
-class TestCephAnsibleTask(TestTask):
- klass = CephAnsible
- task_name = 'ceph_ansible'
-
- def setup_method(self):
- self.ctx = FakeNamespace()
- self.ctx.cluster = Cluster()
- self.ctx.cluster.add(Remote('user@remote1'), ['mon.0'])
- self.ctx.cluster.add(Remote('user@remote2'), ['mds.0'])
- self.ctx.cluster.add(Remote('user@remote3'), ['osd.0'])
- self.ctx.summary = dict()
- self.ctx.config = dict()
- self.ctx.archive = '../'
- self.task_config = dict()
- self.start_patchers()
-
- def start_patchers(self):
- m_fetch_repo = MagicMock()
- m_fetch_repo.return_value = 'PATH'
-
- def fake_get_scratch_devices(remote):
- return ['/dev/%s' % remote.shortname]
-
- self.patcher_get_scratch_devices = patch(
- 'teuthology.task.ceph_ansible.get_scratch_devices',
- fake_get_scratch_devices,
- )
- self.patcher_get_scratch_devices.start()
-
- self.patcher_teardown = patch(
- 'teuthology.task.ceph_ansible.CephAnsible.teardown',
- )
- self.patcher_teardown.start()
-
- def fake_set_iface_and_cidr(self):
- self._interface = 'eth0'
- self._cidr = '172.21.0.0/20'
-
- self.patcher_remote = patch.multiple(
- Remote,
- _set_iface_and_cidr=fake_set_iface_and_cidr,
- )
- self.patcher_remote.start()
-
- def stop_patchers(self):
- self.patcher_get_scratch_devices.stop()
- self.patcher_remote.stop()
- self.patcher_teardown.stop()
-
- def test_playbook_none(self):
- skip(SKIP_IRRELEVANT)
-
- def test_inventory_none(self):
- skip(SKIP_IRRELEVANT)
-
- def test_inventory_path(self):
- skip(SKIP_IRRELEVANT)
-
- def test_inventory_etc(self):
- skip(SKIP_IRRELEVANT)
-
- def test_generate_hosts_file(self):
- self.task_config.update(dict(
- playbook=[],
- vars=dict(
- osd_auto_discovery=True,
- monitor_interface='eth0',
- radosgw_interface='eth0',
- public_network='172.21.0.0/20',
- ),
- ))
- task = self.klass(self.ctx, self.task_config)
- hosts_file_path = '/my/hosts/file'
- hosts_file_obj = StringIO()
- hosts_file_obj.name = hosts_file_path
- with patch.object(ceph_ansible, 'NamedTemporaryFile') as m_NTF:
- m_NTF.return_value = hosts_file_obj
- task.generate_hosts_file()
- m_NTF.assert_called_once_with(prefix="teuth_ansible_hosts_",
- mode='w+',
- delete=False)
- assert task.generated_inventory is True
- assert task.inventory == hosts_file_path
- hosts_file_obj.seek(0)
- assert hosts_file_obj.read() == '\n'.join([
- '[mdss]',
- 'remote2',
- '',
- '[mons]',
- 'remote1',
- '',
- '[osds]',
- 'remote3',
- ])
-
- def test_generate_hosts_file_with_devices(self):
- self.task_config.update(dict(
- playbook=[],
- vars=dict(
- monitor_interface='eth0',
- radosgw_interface='eth0',
- public_network='172.21.0.0/20',
- ),
- ))
- task = self.klass(self.ctx, self.task_config)
- hosts_file_path = '/my/hosts/file'
- hosts_file_obj = StringIO()
- hosts_file_obj.name = hosts_file_path
- with patch.object(ceph_ansible, 'NamedTemporaryFile') as m_NTF:
- m_NTF.return_value = hosts_file_obj
- task.generate_hosts_file()
- m_NTF.assert_called_once_with(prefix="teuth_ansible_hosts_",
- mode='w+',
- delete=False)
- assert task.generated_inventory is True
- assert task.inventory == hosts_file_path
- hosts_file_obj.seek(0)
- assert hosts_file_obj.read() == '\n'.join([
- '[mdss]',
- 'remote2 devices=\'[]\'',
- '',
- '[mons]',
- 'remote1 devices=\'[]\'',
- '',
- '[osds]',
- 'remote3 devices=\'["/dev/remote3"]\'',
- ])
-
- def test_generate_hosts_file_with_network(self):
- self.task_config.update(dict(
- playbook=[],
- vars=dict(
- osd_auto_discovery=True,
- ),
- ))
- task = self.klass(self.ctx, self.task_config)
- hosts_file_path = '/my/hosts/file'
- hosts_file_obj = StringIO()
- hosts_file_obj.name = hosts_file_path
- with patch.object(ceph_ansible, 'NamedTemporaryFile') as m_NTF:
- m_NTF.return_value = hosts_file_obj
- task.generate_hosts_file()
- m_NTF.assert_called_once_with(prefix="teuth_ansible_hosts_",
- mode='w+',
- delete=False)
- assert task.generated_inventory is True
- assert task.inventory == hosts_file_path
- hosts_file_obj.seek(0)
- assert hosts_file_obj.read() == '\n'.join([
- '[mdss]',
- "remote2 monitor_interface='eth0' public_network='172.21.0.0/20' radosgw_interface='eth0'",
- '',
- '[mons]',
- "remote1 monitor_interface='eth0' public_network='172.21.0.0/20' radosgw_interface='eth0'",
- '',
- '[osds]',
- "remote3 monitor_interface='eth0' public_network='172.21.0.0/20' radosgw_interface='eth0'",
- ])
+++ /dev/null
-import os
-
-from mock import patch
-
-from teuthology.config import FakeNamespace
-from teuthology.config import config as teuth_config
-from teuthology.orchestra.cluster import Cluster
-from teuthology.orchestra.remote import Remote
-from teuthology.task.console_log import ConsoleLog
-
-from teuthology.test.task import TestTask
-
-
-class TestConsoleLog(TestTask):
- klass = ConsoleLog
- task_name = 'console_log'
-
- def setup_method(self):
- teuth_config.ipmi_domain = 'ipmi.domain'
- teuth_config.ipmi_user = 'ipmi_user'
- teuth_config.ipmi_password = 'ipmi_pass'
- self.ctx = FakeNamespace()
- self.ctx.cluster = Cluster()
- self.ctx.cluster.add(Remote('user@remote1'), ['role1'])
- self.ctx.cluster.add(Remote('user@remote2'), ['role2'])
- self.ctx.config = dict()
- self.ctx.archive = '/fake/path'
- self.task_config = dict()
- self.start_patchers()
-
- def start_patchers(self):
- self.patchers = dict()
- self.patchers['makedirs'] = patch(
- 'teuthology.task.console_log.os.makedirs',
- )
- self.patchers['is_vm'] = patch(
- 'teuthology.lock.query.is_vm',
- )
- self.patchers['is_vm'].return_value = False
- self.patchers['get_status'] = patch(
- 'teuthology.lock.query.get_status',
- )
- self.mocks = dict()
- for name, patcher in self.patchers.items():
- self.mocks[name] = patcher.start()
- self.mocks['is_vm'].return_value = False
-
- def teardown_method(self):
- for patcher in self.patchers.values():
- patcher.stop()
-
- def test_enabled(self):
- task = self.klass(self.ctx, self.task_config)
- assert task.enabled is True
-
- def test_disabled_noarchive(self):
- self.ctx.archive = None
- task = self.klass(self.ctx, self.task_config)
- assert task.enabled is False
-
- def test_has_ipmi_credentials(self):
- for remote in self.ctx.cluster.remotes.keys():
- remote.console.has_ipmi_credentials = False
- remote.console.has_conserver = False
- task = self.klass(self.ctx, self.task_config)
- assert len(task.cluster.remotes.keys()) == 0
-
- def test_remotes(self):
- with self.klass(self.ctx, self.task_config) as task:
- assert len(task.cluster.remotes) == len(self.ctx.cluster.remotes)
-
- @patch('teuthology.orchestra.console.PhysicalConsole')
- def test_begin(self, m_pconsole):
- with self.klass(self.ctx, self.task_config) as task:
- assert len(task.processes) == len(self.ctx.cluster.remotes)
- expected_log_paths = []
- for remote in task.cluster.remotes.keys():
- expected_log_paths.append(
- os.path.join(self.ctx.archive, 'console_logs', '%s.log' % remote.shortname)
- )
- assert len(m_pconsole().spawn_sol_log.call_args_list) == len(task.cluster.remotes)
- got_log_paths = [c[0][0] for c in m_pconsole().spawn_sol_log.call_args_list]
- assert got_log_paths == expected_log_paths
-
- @patch('teuthology.orchestra.console.PhysicalConsole')
- def test_end(self, m_pconsole):
- m_proc = m_pconsole().spawn_sol_log.return_value
- m_proc.poll.return_value = None
- with self.klass(self.ctx, self.task_config):
- pass
- assert len(m_proc.terminate.call_args_list) == len(self.ctx.cluster.remotes)
- assert len(m_proc.kill.call_args_list) == len(self.ctx.cluster.remotes)
+++ /dev/null
-import os
-import pytest
-import yaml
-
-from mock import patch, Mock
-from pytest import mark
-
-from teuthology.task import install
-
-
-class TestInstall(object):
-
- def _get_default_package_list(self, project='ceph', debug=False):
- path = os.path.join(
- os.path.dirname(__file__),
- '..', '..', 'task', 'install', 'packages.yaml',
- )
- pkgs = yaml.safe_load(open(path))[project]
- if not debug:
- pkgs['deb'] = [p for p in pkgs['deb']
- if not p.endswith('-dbg')]
- pkgs['rpm'] = [p for p in pkgs['rpm']
- if not p.endswith('-debuginfo')]
- return pkgs
-
- def test_get_package_list_debug(self):
- default_pkgs = self._get_default_package_list(debug=True)
- default_pkgs['rpm'].sort()
- default_pkgs['deb'].sort()
- config = dict(debuginfo=True)
- result = install.get_package_list(ctx=None, config=config)
- result['rpm'].sort()
- result['deb'].sort()
- assert result == default_pkgs
-
- def test_get_package_list_no_debug(self):
- default_pkgs = self._get_default_package_list(debug=False)
- default_pkgs['rpm'].sort()
- default_pkgs['deb'].sort()
- config = dict(debuginfo=False)
- result = install.get_package_list(ctx=None, config=config)
- result['rpm'].sort()
- result['deb'].sort()
- assert result == default_pkgs
-
- def test_get_package_list_custom_rpm(self):
- default_pkgs = self._get_default_package_list(debug=False)
- default_pkgs['rpm'].sort()
- default_pkgs['deb'].sort()
- rpms = ['rpm1', 'rpm2', 'rpm2-debuginfo']
- config = dict(packages=dict(rpm=rpms))
- result = install.get_package_list(ctx=None, config=config)
- result['rpm'].sort()
- result['deb'].sort()
- assert result['rpm'] == ['rpm1', 'rpm2']
- assert result['deb'] == default_pkgs['deb']
-
- @patch("teuthology.task.install._get_builder_project")
- @patch("teuthology.task.install.packaging.get_package_version")
- def test_get_upgrade_version(self, m_get_package_version,
- m_gitbuilder_project):
- gb = Mock()
- gb.version = "11.0.0"
- gb.project = "ceph"
- m_gitbuilder_project.return_value = gb
- m_get_package_version.return_value = "11.0.0"
- install.get_upgrade_version(Mock(), Mock(), Mock())
-
- @patch("teuthology.task.install._get_builder_project")
- @patch("teuthology.task.install.packaging.get_package_version")
- def test_verify_ceph_version_success(self, m_get_package_version,
- m_gitbuilder_project):
- gb = Mock()
- gb.version = "0.89.0"
- gb.project = "ceph"
- m_gitbuilder_project.return_value = gb
- m_get_package_version.return_value = "0.89.0"
- config = dict()
- install.verify_package_version(Mock(), config, Mock())
-
- @patch("teuthology.task.install._get_builder_project")
- @patch("teuthology.task.install.packaging.get_package_version")
- def test_verify_ceph_version_failed(self, m_get_package_version,
- m_gitbuilder_project):
- gb = Mock()
- gb.version = "0.89.0"
- gb.project = "ceph"
- m_gitbuilder_project.return_value = gb
- m_get_package_version.return_value = "0.89.1"
- config = dict()
- with pytest.raises(RuntimeError):
- install.verify_package_version(Mock(), config, Mock())
-
- def test_get_flavor_default(self):
- config = dict()
- assert install.get_flavor(config) == 'default'
-
- def test_get_flavor_simple(self):
- config = dict(
- flavor='notcmalloc'
- )
- assert install.get_flavor(config) == 'notcmalloc'
-
- def test_get_flavor_valgrind(self):
- config = dict(
- valgrind=True
- )
- assert install.get_flavor(config) == 'notcmalloc'
-
- def test_upgrade_is_downgrade(self):
- assert_ok_vals = [
- ('9.0.0', '10.0.0'),
- ('10.2.2-63-g8542898-1trusty', '10.2.2-64-gabcdef1-1trusty'),
- ('11.0.0-918.g13c13c7', '11.0.0-2165.gabcdef1')
- ]
- for t in assert_ok_vals:
- assert install._upgrade_is_downgrade(t[0], t[1]) == False
-
- @patch("teuthology.packaging.get_package_version")
- @patch("teuthology.misc.get_system_type")
- @patch("teuthology.task.install.verify_package_version")
- @patch("teuthology.task.install.get_upgrade_version")
- def test_upgrade_common(self,
- m_get_upgrade_version,
- m_verify_package_version,
- m_get_system_type,
- m_get_package_version):
- expected_system_type = 'deb'
- def make_remote():
- remote = Mock()
- remote.arch = 'x86_64'
- remote.os = Mock()
- remote.os.name = 'ubuntu'
- remote.os.version = '14.04'
- remote.os.codename = 'trusty'
- remote.system_type = expected_system_type
- return remote
- ctx = Mock()
- class cluster:
- remote1 = make_remote()
- remote2 = make_remote()
- remotes = {
- remote1: ['client.0'],
- remote2: ['mon.a','osd.0'],
- }
- def only(self, role):
- result = Mock()
- if role in ('client.0',):
- result.remotes = { cluster.remote1: None }
- if role in ('osd.0', 'mon.a'):
- result.remotes = { cluster.remote2: None }
- return result
- ctx.cluster = cluster()
- config = {
- 'client.0': {
- 'sha1': 'expectedsha1',
- },
- }
- ctx.config = {
- 'roles': [ ['client.0'], ['mon.a','osd.0'] ],
- 'tasks': [
- {
- 'install.upgrade': config,
- },
- ],
- }
- m_get_upgrade_version.return_value = "11.0.0"
- m_get_package_version.return_value = "10.2.4"
- m_get_system_type.return_value = "deb"
- def upgrade(ctx, node, remote, pkgs, system_type):
- assert system_type == expected_system_type
- assert install.upgrade_common(ctx, config, upgrade) == 1
- expected_config = {
- 'project': 'ceph',
- 'sha1': 'expectedsha1',
- }
- m_verify_package_version.assert_called_with(ctx,
- expected_config,
- cluster.remote1)
- def test_upgrade_remote_to_config(self):
- expected_system_type = 'deb'
- def make_remote():
- remote = Mock()
- remote.arch = 'x86_64'
- remote.os = Mock()
- remote.os.name = 'ubuntu'
- remote.os.version = '14.04'
- remote.os.codename = 'trusty'
- remote.system_type = expected_system_type
- return remote
- ctx = Mock()
- class cluster:
- remote1 = make_remote()
- remote2 = make_remote()
- remotes = {
- remote1: ['client.0'],
- remote2: ['mon.a','osd.0'],
- }
- def only(self, role):
- result = Mock()
- if role in ('client.0',):
- result.remotes = { cluster.remote1: None }
- elif role in ('osd.0', 'mon.a'):
- result.remotes = { cluster.remote2: None }
- else:
- result.remotes = None
- return result
- ctx.cluster = cluster()
- ctx.config = {
- 'roles': [ ['client.0'], ['mon.a','osd.0'] ],
- }
-
- # nothing -> nothing
- assert install.upgrade_remote_to_config(ctx, {}) == {}
-
- # select the remote for the osd.0 role
- # the 'ignored' role does not exist and is ignored
- # the remote for mon.a is the same as for osd.0 and
- # is silently ignored (actually it could be the other
- # way around, depending on how the keys are hashed)
- config = {
- 'osd.0': {
- 'sha1': 'expectedsha1',
- },
- 'ignored': None,
- 'mon.a': {
- 'sha1': 'expectedsha1',
- },
- }
- expected_config = {
- cluster.remote2: {
- 'project': 'ceph',
- 'sha1': 'expectedsha1',
- },
- }
- assert install.upgrade_remote_to_config(ctx, config) == expected_config
-
- # select all nodes, regardless
- config = {
- 'all': {
- 'sha1': 'expectedsha1',
- },
- }
- expected_config = {
- cluster.remote1: {
- 'project': 'ceph',
- 'sha1': 'expectedsha1',
- },
- cluster.remote2: {
- 'project': 'ceph',
- 'sha1': 'expectedsha1',
- },
- }
- assert install.upgrade_remote_to_config(ctx, config) == expected_config
-
- # verify that install overrides are used as default
- # values for the upgrade task, not as override
- ctx.config['overrides'] = {
- 'install': {
- 'ceph': {
- 'sha1': 'overridesha1',
- 'tag': 'overridetag',
- 'branch': 'overridebranch',
- },
- },
- }
- config = {
- 'client.0': {
- 'sha1': 'expectedsha1',
- },
- 'osd.0': {
- },
- }
- expected_config = {
- cluster.remote1: {
- 'project': 'ceph',
- 'sha1': 'expectedsha1',
- },
- cluster.remote2: {
- 'project': 'ceph',
- 'sha1': 'overridesha1',
- 'tag': 'overridetag',
- 'branch': 'overridebranch',
- },
- }
- assert install.upgrade_remote_to_config(ctx, config) == expected_config
-
-
- @patch("teuthology.task.install.packaging.get_package_version")
- @patch("teuthology.task.install.redhat.set_deb_repo")
- def test_rh_install_deb_pkgs(self, m_set_rh_deb_repo, m_get_pkg_version):
- ctx = Mock()
- remote = Mock()
- version = '1.3.2'
- rh_ds_yaml = dict()
- rh_ds_yaml = {
- 'versions': {'deb': {'mapped': {'1.3.2': '0.94.5'}}},
- 'pkgs': {'deb': ['pkg1', 'pkg2']},
- 'extra_system_packages': {'deb': ['es_pkg1', 'es_pkg2']},
- 'extra_packages': {'deb': ['e_pkg1', 'e_pkg2']},
- }
- m_get_pkg_version.return_value = "0.94.5"
- install.redhat.install_deb_pkgs(ctx, remote, version, rh_ds_yaml)
-
- @patch("teuthology.task.install.packaging.get_package_version")
- def test_rh_install_pkgs(self, m_get_pkg_version):
- ctx = Mock()
- remote = Mock()
- version = '1.3.2'
- rh_ds_yaml = dict()
- rh_ds_yaml = {
- 'versions': {'rpm': {'mapped': {'1.3.2': '0.94.5',
- '1.3.1': '0.94.3'}}},
- 'pkgs': {'rpm': ['pkg1', 'pkg2']},
- 'extra_system_packages': {'rpm': ['es_pkg1', 'es_pkg2']},
- 'extra_packages': {'rpm': ['e_pkg1', 'e_pkg2']},
- }
-
- m_get_pkg_version.return_value = "0.94.5"
- install.redhat.install_pkgs(ctx, remote, version, rh_ds_yaml)
- version = '1.3.1'
- with pytest.raises(RuntimeError) as e:
- install.redhat.install_pkgs(ctx, remote, version, rh_ds_yaml)
- assert "Version check failed" in str(e)
-
- @mark.parametrize(
- 'conf, expect',
- [
- [
- {
- 'tasks': [ { 'install': { 'clean': True, }, }, ],
- 'overrides': {
- 'install': {
- 'ceph': {
- 'extra_system_packages': ['alpha'],
- 'flavor': 'default',
- 'sha1': '0123456789abcdef0123456789abcdef01234567',
- },
- 'extra_system_packages': {
- 'deb': [],
- 'rpm': ['xerxes', 'yellow'],
- },
- },
- },
- },
- {
- 'deb': ['alpha'],
- 'rpm': ['alpha', 'xerxes', 'yellow'],
- }
- ],
- [
- {
- 'tasks': [ { 'install': { 'clean': True, }, }, ],
- 'overrides': {
- 'install': {
- 'ceph': {
- 'extra_system_packages': {
- 'deb': [],
- 'rpm': ['xerxes', 'yellow'],
- },
- 'flavor': 'default',
- 'sha1': '0123456789abcdef0123456789abcdef01234567',
- },
- 'extra_system_packages': ['alpha'],
- },
- },
- },
- {
- 'deb': ['alpha'],
- 'rpm': ['xerxes', 'yellow', 'alpha'],
- }
- ],
- [
- {
- 'tasks': [ { 'install': { 'clean': True, }, }, ],
- 'overrides': {
- 'install': {
- 'ceph': {
- 'flavor': 'default',
- 'sha1': '0123456789abcdef0123456789abcdef01234567',
- },
- 'extra_system_packages': {
- 'deb': [],
- 'rpm': ['xerxes', 'yellow'],
- },
- },
- },
- },
- {
- 'deb': [],
- 'rpm': ['xerxes', 'yellow'],
- }
- ],
- [
- {
- 'tasks': [ { 'install': { 'clean': True, }, }, ],
- 'overrides': {
- 'install': {
- 'ceph': {
- 'flavor': 'default',
- 'sha1': '0123456789abcdef0123456789abcdef01234567',
- },
- 'extra_system_packages': ['xerxes', 'yellow'],
- },
- },
- },
- {
- 'deb': ['xerxes', 'yellow'],
- 'rpm': ['xerxes', 'yellow'],
- }
- ],
- [
- {
- 'tasks': [ { 'install': { 'clean': True, }, }, ],
- 'overrides': { 'install': {
- 'ceph': { 'flavor': 'default', 'sha1': '012345', },
- 'extra_system_packages': { 'rpm': ['xerxes', 'yellow'], },
- },
- },
- },
- { 'deb': [], 'rpm': ['xerxes', 'yellow'], }
-
- ],
- [
- {
- 'tasks': [ { 'install': { 'clean': True,
- 'extra_system_packages': { 'deb': ['alpha'], 'rpm': ['bravo'] },
- }, }, ],
- 'overrides': { 'install': {
- 'ceph': { 'flavor': 'default', 'sha1': '012345', },
- 'extra_system_packages': { 'rpm': ['xerxes', 'yellow'], },
- },
- },
- },
- { 'deb': ['alpha'], 'rpm': ['bravo', 'xerxes', 'yellow'], }
-
- ],
-
- ]
- )
- def test_install_extra_system_packages(self, conf, expect):
- install_task = conf.get('tasks')[0].get('install')
- install_overrides = conf.get('overrides').get('install', {})
- install._override_extra_system_packages(install_task, 'ceph', install_overrides)
-
- assert install_task.get('extra_system_packages') == expect
+++ /dev/null
-from teuthology.config import FakeNamespace
-from teuthology.task import internal
-
-
-class TestInternal(object):
- def setup_method(self):
- self.ctx = FakeNamespace()
- self.ctx.config = dict()
-
- def test_buildpackages_prep(self):
- #
- # no buildpackages nor install tasks
- #
- self.ctx.config = { 'tasks': [] }
- assert internal.buildpackages_prep(self.ctx,
- self.ctx.config) == internal.BUILDPACKAGES_NOTHING
- #
- # make the buildpackages tasks the first to run
- #
- self.ctx.config = {
- 'tasks': [ { 'atask': None },
- { 'internal.buildpackages_prep': None },
- { 'btask': None },
- { 'install': None },
- { 'buildpackages': None } ],
- }
- assert internal.buildpackages_prep(self.ctx,
- self.ctx.config) == internal.BUILDPACKAGES_FIRST
- assert self.ctx.config == {
- 'tasks': [ { 'atask': None },
- { 'internal.buildpackages_prep': None },
- { 'buildpackages': None },
- { 'btask': None },
- { 'install': None } ],
- }
- #
- # the buildpackages task already the first task to run
- #
- assert internal.buildpackages_prep(self.ctx,
- self.ctx.config) == internal.BUILDPACKAGES_OK
- #
- # no buildpackages task
- #
- self.ctx.config = {
- 'tasks': [ { 'install': None } ],
- }
- assert internal.buildpackages_prep(self.ctx,
- self.ctx.config) == internal.BUILDPACKAGES_NOTHING
- #
- # no install task: the buildpackages task must be removed
- #
- self.ctx.config = {
- 'tasks': [ { 'buildpackages': None } ],
- }
- assert internal.buildpackages_prep(self.ctx,
- self.ctx.config) == internal.BUILDPACKAGES_REMOVED
- assert self.ctx.config == {'tasks': []}
+++ /dev/null
-from teuthology.config import FakeNamespace
-from teuthology.orchestra.cluster import Cluster
-from teuthology.orchestra.remote import Remote
-from teuthology.task.kernel import (
- normalize_and_apply_overrides,
- CONFIG_DEFAULT,
- TIMEOUT_DEFAULT,
-)
-
-class TestKernelNormalizeAndApplyOverrides(object):
-
- def setup_method(self):
- self.ctx = FakeNamespace()
- self.ctx.cluster = Cluster()
- self.ctx.cluster.add(Remote('remote1'), ['mon.a', 'client.0'])
- self.ctx.cluster.add(Remote('remote2'), ['osd.0', 'osd.1', 'osd.2'])
- self.ctx.cluster.add(Remote('remote3'), ['client.1'])
-
- def test_default(self):
- config = {}
- overrides = {}
- config, t = normalize_and_apply_overrides(self.ctx, config, overrides)
- assert config == {
- 'mon.a': CONFIG_DEFAULT,
- 'osd.0': CONFIG_DEFAULT,
- 'osd.1': CONFIG_DEFAULT,
- 'osd.2': CONFIG_DEFAULT,
- 'client.0': CONFIG_DEFAULT,
- 'client.1': CONFIG_DEFAULT,
- }
- assert t == TIMEOUT_DEFAULT
-
- def test_timeout_default(self):
- config = {
- 'client.0': {'branch': 'testing'},
- }
- overrides = {}
- config, t = normalize_and_apply_overrides(self.ctx, config, overrides)
- assert config == {
- 'client.0': {'branch': 'testing'},
- }
- assert t == TIMEOUT_DEFAULT
-
- def test_timeout(self):
- config = {
- 'client.0': {'branch': 'testing'},
- 'timeout': 100,
- }
- overrides = {}
- config, t = normalize_and_apply_overrides(self.ctx, config, overrides)
- assert config == {
- 'client.0': {'branch': 'testing'},
- }
- assert t == 100
-
- def test_override_timeout(self):
- config = {
- 'client.0': {'branch': 'testing'},
- 'timeout': 100,
- }
- overrides = {
- 'timeout': 200,
- }
- config, t = normalize_and_apply_overrides(self.ctx, config, overrides)
- assert config == {
- 'client.0': {'branch': 'testing'},
- }
- assert t == 200
-
- def test_override_same_version_key(self):
- config = {
- 'client.0': {'branch': 'testing'},
- }
- overrides = {
- 'client.0': {'branch': 'wip-foobar'},
- }
- config, t = normalize_and_apply_overrides(self.ctx, config, overrides)
- assert config == {
- 'client.0': {'branch': 'wip-foobar'},
- }
- assert t == TIMEOUT_DEFAULT
-
- def test_override_different_version_key(self):
- config = {
- 'client.0': {'branch': 'testing'},
- }
- overrides = {
- 'client.0': {'tag': 'v4.1'},
- }
- config, t = normalize_and_apply_overrides(self.ctx, config, overrides)
- assert config == {
- 'client.0': {'tag': 'v4.1'},
- }
- assert t == TIMEOUT_DEFAULT
-
- def test_override_actual(self):
- config = {
- 'osd.1': {'tag': 'v4.1'},
- 'client.0': {'branch': 'testing'},
- }
- overrides = {
- 'osd.1': {'koji': 1234, 'kdb': True},
- }
- config, t = normalize_and_apply_overrides(self.ctx, config, overrides)
- assert config == {
- 'osd.1': {'koji': 1234, 'kdb': True},
- 'client.0': {'branch': 'testing'},
- }
- assert t == TIMEOUT_DEFAULT
-
- def test_override_actual_with_generic(self):
- config = {
- 'osd.1': {'tag': 'v4.1', 'kdb': False},
- 'client.0': {'branch': 'testing'},
- }
- overrides = {
- 'osd': {'koji': 1234},
- }
- config, t = normalize_and_apply_overrides(self.ctx, config, overrides)
- assert config == {
- 'osd.0': {'koji': 1234},
- 'osd.1': {'koji': 1234, 'kdb': False},
- 'osd.2': {'koji': 1234},
- 'client.0': {'branch': 'testing'},
- }
- assert t == TIMEOUT_DEFAULT
-
- def test_override_actual_with_top_level(self):
- config = {
- 'osd.1': {'tag': 'v4.1'},
- 'client.0': {'branch': 'testing', 'kdb': False},
- }
- overrides = {'koji': 1234, 'kdb': True}
- config, t = normalize_and_apply_overrides(self.ctx, config, overrides)
- assert config == {
- 'mon.a': {'koji': 1234, 'kdb': True},
- 'osd.0': {'koji': 1234, 'kdb': True},
- 'osd.1': {'koji': 1234, 'kdb': True},
- 'osd.2': {'koji': 1234, 'kdb': True},
- 'client.0': {'koji': 1234, 'kdb': True},
- 'client.1': {'koji': 1234, 'kdb': True},
- }
- assert t == TIMEOUT_DEFAULT
-
- def test_override_generic(self):
- config = {
- 'osd': {'tag': 'v4.1'},
- 'client': {'branch': 'testing'},
- }
- overrides = {
- 'client': {'koji': 1234, 'kdb': True},
- }
- config, t = normalize_and_apply_overrides(self.ctx, config, overrides)
- assert config == {
- 'osd.0': {'tag': 'v4.1'},
- 'osd.1': {'tag': 'v4.1'},
- 'osd.2': {'tag': 'v4.1'},
- 'client.0': {'koji': 1234, 'kdb': True},
- 'client.1': {'koji': 1234, 'kdb': True},
- }
- assert t == TIMEOUT_DEFAULT
-
- def test_override_generic_with_top_level(self):
- config = {
- 'osd': {'tag': 'v4.1'},
- 'client': {'branch': 'testing', 'kdb': False},
- }
- overrides = {
- 'client': {'koji': 1234},
- }
- config, t = normalize_and_apply_overrides(self.ctx, config, overrides)
- assert config == {
- 'osd.0': {'tag': 'v4.1'},
- 'osd.1': {'tag': 'v4.1'},
- 'osd.2': {'tag': 'v4.1'},
- 'client.0': {'koji': 1234, 'kdb': False},
- 'client.1': {'koji': 1234, 'kdb': False},
- }
- assert t == TIMEOUT_DEFAULT
-
- def test_override_generic_with_actual(self):
- config = {
- 'osd': {'tag': 'v4.1', 'kdb': False},
- 'client': {'branch': 'testing'},
- }
- overrides = {
- 'osd.2': {'koji': 1234, 'kdb': True},
- }
- config, t = normalize_and_apply_overrides(self.ctx, config, overrides)
- assert config == {
- 'osd.0': {'tag': 'v4.1', 'kdb': False},
- 'osd.1': {'tag': 'v4.1', 'kdb': False},
- 'osd.2': {'koji': 1234, 'kdb': True},
- 'client.0': {'branch': 'testing'},
- 'client.1': {'branch': 'testing'},
- }
- assert t == TIMEOUT_DEFAULT
-
- def test_override_top_level(self):
- config = {'branch': 'testing'}
- overrides = {'koji': 1234, 'kdb': True}
- config, t = normalize_and_apply_overrides(self.ctx, config, overrides)
- assert config == {
- 'mon.a': {'koji': 1234, 'kdb': True},
- 'osd.0': {'koji': 1234, 'kdb': True},
- 'osd.1': {'koji': 1234, 'kdb': True},
- 'osd.2': {'koji': 1234, 'kdb': True},
- 'client.0': {'koji': 1234, 'kdb': True},
- 'client.1': {'koji': 1234, 'kdb': True},
- }
- assert t == TIMEOUT_DEFAULT
-
- def test_override_top_level_with_actual(self):
- config = {'branch': 'testing', 'kdb': False}
- overrides = {
- 'mon.a': {'koji': 1234},
- }
- config, t = normalize_and_apply_overrides(self.ctx, config, overrides)
- assert config == {
- 'mon.a': {'koji': 1234, 'kdb': False},
- 'osd.0': {'branch': 'testing', 'kdb': False},
- 'osd.1': {'branch': 'testing', 'kdb': False},
- 'osd.2': {'branch': 'testing', 'kdb': False},
- 'client.0': {'branch': 'testing', 'kdb': False},
- 'client.1': {'branch': 'testing', 'kdb': False},
- }
- assert t == TIMEOUT_DEFAULT
-
- def test_override_top_level_with_generic(self):
- config = {'branch': 'testing', 'kdb': False}
- overrides = {
- 'client': {'koji': 1234, 'kdb': True},
- }
- config, t = normalize_and_apply_overrides(self.ctx, config, overrides)
- assert config == {
- 'mon.a': {'branch': 'testing', 'kdb': False},
- 'osd.0': {'branch': 'testing', 'kdb': False},
- 'osd.1': {'branch': 'testing', 'kdb': False},
- 'osd.2': {'branch': 'testing', 'kdb': False},
- 'client.0': {'koji': 1234, 'kdb': True},
- 'client.1': {'koji': 1234, 'kdb': True},
- }
- assert t == TIMEOUT_DEFAULT
+++ /dev/null
-import os
-import requests
-
-from teuthology.util.compat import parse_qs, urljoin
-
-from mock import patch, DEFAULT, Mock, mock_open, call
-from pytest import raises
-
-from teuthology.config import config, FakeNamespace
-from teuthology.orchestra.cluster import Cluster
-from teuthology.orchestra.remote import Remote
-from teuthology.orchestra.run import Raw
-from teuthology.task.pcp import (PCPDataSource, PCPArchive, PCPGrapher,
- GrafanaGrapher, GraphiteGrapher, PCP)
-
-from teuthology.test.task import TestTask
-
-pcp_host = 'http://pcp.front.sepia.ceph.com:44323/'
-
-
-class TestPCPDataSource(object):
- klass = PCPDataSource
-
- def setup_method(self):
- config.pcp_host = pcp_host
-
- def test_init(self):
- hosts = ['host1', 'host2']
- time_from = 'now-2h'
- time_until = 'now'
- obj = self.klass(
- hosts=hosts,
- time_from=time_from,
- time_until=time_until,
- )
- assert obj.hosts == hosts
- assert obj.time_from == time_from
- assert obj.time_until == time_until
-
-
-class TestPCPArchive(TestPCPDataSource):
- klass = PCPArchive
-
- def test_get_archive_input_dir(self):
- hosts = ['host1', 'host2']
- time_from = 'now-1d'
- obj = self.klass(
- hosts=hosts,
- time_from=time_from,
- )
- assert obj.get_archive_input_dir('host1') == \
- '/var/log/pcp/pmlogger/host1'
-
- def test_get_pmlogextract_cmd(self):
- obj = self.klass(
- hosts=['host1'],
- time_from='now-3h',
- time_until='now-1h',
- )
- expected = [
- 'pmlogextract',
- '-S', 'now-3h',
- '-T', 'now-1h',
- Raw('/var/log/pcp/pmlogger/host1/*.0'),
- ]
- assert obj.get_pmlogextract_cmd('host1') == expected
-
- def test_format_time(self):
- assert self.klass._format_time(1462893484) == \
- '@ Tue May 10 15:18:04 2016'
-
- def test_format_time_now(self):
- assert self.klass._format_time('now-1h') == 'now-1h'
-
-
-class TestPCPGrapher(TestPCPDataSource):
- klass = PCPGrapher
-
- def test_init(self):
- hosts = ['host1', 'host2']
- time_from = 'now-2h'
- time_until = 'now'
- obj = self.klass(
- hosts=hosts,
- time_from=time_from,
- time_until=time_until,
- )
- assert obj.hosts == hosts
- assert obj.time_from == time_from
- assert obj.time_until == time_until
- expected_url = urljoin(config.pcp_host, self.klass._endpoint)
- assert obj.base_url == expected_url
-
-
-class TestGrafanaGrapher(TestPCPGrapher):
- klass = GrafanaGrapher
-
- def test_build_graph_url(self):
- hosts = ['host1']
- time_from = 'now-3h'
- time_until = 'now-1h'
- obj = self.klass(
- hosts=hosts,
- time_from=time_from,
- time_until=time_until,
- )
- base_url = urljoin(
- config.pcp_host,
- 'grafana/index.html#/dashboard/script/index.js',
- )
- assert obj.base_url == base_url
- got_url = obj.build_graph_url()
- parsed_query = parse_qs(got_url.split('?')[1])
- assert parsed_query['hosts'] == hosts
- assert len(parsed_query['time_from']) == 1
- assert parsed_query['time_from'][0] == time_from
- assert len(parsed_query['time_to']) == 1
- assert parsed_query['time_to'][0] == time_until
-
- def test_format_time(self):
- assert self.klass._format_time(1462893484) == \
- '2016-05-10T15:18:04'
-
- def test_format_time_now(self):
- assert self.klass._format_time('now-1h') == 'now-1h'
-
-
-class TestGraphiteGrapher(TestPCPGrapher):
- klass = GraphiteGrapher
-
- def test_build_graph_urls(self):
- obj = self.klass(
- hosts=['host1', 'host2'],
- time_from='now-3h',
- time_until='now-1h',
- )
- expected_urls = [obj.get_graph_url(m) for m in obj.metrics]
- obj.build_graph_urls()
- built_urls = []
- for metric in obj.graphs.keys():
- built_urls.append(obj.graphs[metric]['url'])
- assert len(built_urls) == len(expected_urls)
- assert sorted(built_urls) == sorted(expected_urls)
-
- def test_check_dest_dir(self):
- obj = self.klass(
- hosts=['host1'],
- time_from='now-3h',
- )
- assert obj.dest_dir is None
- with raises(RuntimeError):
- obj._check_dest_dir()
-
- def test_generate_html_dynamic(self):
- obj = self.klass(
- hosts=['host1'],
- time_from='now-3h',
- )
- html = obj.generate_html()
- assert config.pcp_host in html
-
- def test_download_graphs(self):
- dest_dir = '/fake/path'
- obj = self.klass(
- hosts=['host1'],
- time_from='now-3h',
- dest_dir=dest_dir,
- )
- _format = obj.graph_defaults.get('format')
- with patch('teuthology.task.pcp.requests.get', create=True) as m_get:
- m_resp = Mock()
- m_resp.ok = True
- m_get.return_value = m_resp
- with patch('teuthology.task.pcp.open', mock_open(), create=True):
- obj.download_graphs()
- expected_filenames = []
- for metric in obj.metrics:
- expected_filenames.append(
- "{}.{}".format(
- os.path.join(
- dest_dir,
- obj._sanitize_metric_name(metric),
- ),
- _format,
- )
- )
- graph_filenames = []
- for metric in obj.graphs.keys():
- graph_filenames.append(obj.graphs[metric]['file'])
- assert sorted(graph_filenames) == sorted(expected_filenames)
-
- def test_generate_html_static(self):
- obj = self.klass(
- hosts=['host1'],
- time_from='now-3h',
- dest_dir='/fake/path',
- )
- with patch('teuthology.task.pcp.requests.get', create=True) as m_get:
- m_resp = Mock()
- m_resp.ok = True
- m_get.return_value = m_resp
- with patch('teuthology.task.pcp.open', mock_open(), create=True):
- obj.download_graphs()
- html = obj.generate_html(mode='static')
- assert config.pcp_host not in html
-
- def test_sanitize_metric_name(self):
- sanitized_metrics = {
- 'foo.bar': 'foo.bar',
- 'foo.*': 'foo._all_',
- 'foo.bar baz': 'foo.bar_baz',
- 'foo.*.bar baz': 'foo._all_.bar_baz',
- }
- for in_, out in sanitized_metrics.items():
- assert self.klass._sanitize_metric_name(in_) == out
-
- def test_get_target_globs(self):
- obj = self.klass(
- hosts=['host1'],
- time_from='now-3h',
- )
- assert obj.get_target_globs() == ['*host1*']
- assert obj.get_target_globs('a.metric') == ['*host1*.a.metric']
- obj.hosts.append('host2')
- assert obj.get_target_globs() == ['*host1*', '*host2*']
- assert obj.get_target_globs('a.metric') == \
- ['*host1*.a.metric', '*host2*.a.metric']
-
-
-class TestPCPTask(TestTask):
- klass = PCP
- task_name = 'pcp'
-
- def setup_method(self):
- self.ctx = FakeNamespace()
- self.ctx.cluster = Cluster()
- self.ctx.cluster.add(Remote('user@remote1'), ['role1'])
- self.ctx.cluster.add(Remote('user@remote2'), ['role2'])
- self.ctx.config = dict()
- self.task_config = dict()
- config.pcp_host = pcp_host
-
- def test_init(self):
- task = self.klass(self.ctx, self.task_config)
- assert task.stop_time == 'now'
-
- def test_disabled(self):
- config.pcp_host = None
- with self.klass(self.ctx, self.task_config) as task:
- assert task.enabled is False
- assert not hasattr(task, 'grafana')
- assert not hasattr(task, 'graphite')
- assert not hasattr(task, 'archiver')
-
- def test_setup(self):
- with patch.multiple(
- self.klass,
- setup_collectors=DEFAULT,
- begin=DEFAULT,
- end=DEFAULT,
- ):
- with self.klass(self.ctx, self.task_config) as task:
- task.setup_collectors.assert_called_once_with()
- assert isinstance(task.start_time, int)
-
- def test_setup_collectors(self):
- with patch.multiple(
- self.klass,
- begin=DEFAULT,
- end=DEFAULT,
- ):
- with self.klass(self.ctx, self.task_config) as task:
- assert hasattr(task, 'grafana')
- assert not hasattr(task, 'graphite')
- assert not hasattr(task, 'archiver')
- self.task_config['grafana'] = False
- with self.klass(self.ctx, self.task_config) as task:
- assert not hasattr(task, 'grafana')
-
- @patch('os.makedirs')
- def test_setup_grafana(self, m_makedirs):
- with patch.multiple(
- self.klass,
- begin=DEFAULT,
- end=DEFAULT,
- ):
- self.ctx.archive = '/fake/path'
- with self.klass(self.ctx, self.task_config) as task:
- assert hasattr(task, 'grafana')
- self.task_config['grafana'] = False
- with self.klass(self.ctx, self.task_config) as task:
- assert not hasattr(task, 'grafana')
-
- @patch('os.makedirs')
- @patch('teuthology.task.pcp.GraphiteGrapher')
- def test_setup_graphite(self, m_graphite_grapher, m_makedirs):
- with patch.multiple(
- self.klass,
- begin=DEFAULT,
- end=DEFAULT,
- ):
- with self.klass(self.ctx, self.task_config) as task:
- assert not hasattr(task, 'graphite')
- self.task_config['graphite'] = False
- with self.klass(self.ctx, self.task_config) as task:
- assert not hasattr(task, 'graphite')
- self.ctx.archive = '/fake/path'
- self.task_config['graphite'] = True
- with self.klass(self.ctx, self.task_config) as task:
- assert hasattr(task, 'graphite')
- self.task_config['graphite'] = False
- with self.klass(self.ctx, self.task_config) as task:
- assert not hasattr(task, 'graphite')
-
- @patch('os.makedirs')
- @patch('teuthology.task.pcp.PCPArchive')
- def test_setup_archiver(self, m_archive, m_makedirs):
- with patch.multiple(
- self.klass,
- begin=DEFAULT,
- end=DEFAULT,
- ):
- self.task_config['fetch_archives'] = True
- with self.klass(self.ctx, self.task_config) as task:
- assert not hasattr(task, 'archiver')
- self.task_config['fetch_archives'] = False
- with self.klass(self.ctx, self.task_config) as task:
- assert not hasattr(task, 'archiver')
- self.ctx.archive = '/fake/path'
- self.task_config['fetch_archives'] = True
- with self.klass(self.ctx, self.task_config) as task:
- assert hasattr(task, 'archiver')
- self.task_config['fetch_archives'] = False
- with self.klass(self.ctx, self.task_config) as task:
- assert not hasattr(task, 'archiver')
-
- @patch('os.makedirs')
- @patch('teuthology.task.pcp.GrafanaGrapher')
- @patch('teuthology.task.pcp.GraphiteGrapher')
- def test_begin(self, m_grafana, m_graphite, m_makedirs):
- with patch.multiple(
- self.klass,
- end=DEFAULT,
- ):
- with self.klass(self.ctx, self.task_config) as task:
- task.grafana.build_graph_url.assert_called_once_with()
- self.task_config['graphite'] = True
- self.ctx.archive = '/fake/path'
- with self.klass(self.ctx, self.task_config) as task:
- task.graphite.write_html.assert_called_once_with()
-
- @patch('os.makedirs')
- @patch('teuthology.task.pcp.GrafanaGrapher')
- @patch('teuthology.task.pcp.GraphiteGrapher')
- def test_end(self, m_grafana, m_graphite, m_makedirs):
- self.ctx.archive = '/fake/path'
- with self.klass(self.ctx, self.task_config) as task:
- # begin() should have called write_html() once by now, with no args
- task.graphite.write_html.assert_called_once_with()
- # end() should have called write_html() a second time by now, with
- # mode=static
- second_call = task.graphite.write_html.call_args_list[1]
- assert second_call[1]['mode'] == 'static'
- assert isinstance(task.stop_time, int)
-
- @patch('os.makedirs')
- @patch('teuthology.task.pcp.GrafanaGrapher')
- @patch('teuthology.task.pcp.GraphiteGrapher')
- def test_end_16049(self, m_grafana, m_graphite, m_makedirs):
- # http://tracker.ceph.com/issues/16049
- # Jobs were failing if graph downloading failed. We don't want that.
- self.ctx.archive = '/fake/path'
- with self.klass(self.ctx, self.task_config) as task:
- task.graphite.download_graphs.side_effect = \
- requests.ConnectionError
- # Even though downloading graphs failed, we should have called
- # write_html() a second time, again with no args
- assert task.graphite.write_html.call_args_list == [call(), call()]
- assert isinstance(task.stop_time, int)
+++ /dev/null
-from mock import patch, Mock, DEFAULT
-
-from teuthology.config import FakeNamespace
-from teuthology.orchestra.cluster import Cluster
-from teuthology.orchestra.remote import Remote
-from teuthology.task.selinux import SELinux
-
-
-class TestSELinux(object):
- def setup_method(self):
- self.ctx = FakeNamespace()
- self.ctx.config = dict()
-
- def test_host_exclusion(self):
- with patch.multiple(
- Remote,
- os=DEFAULT,
- run=DEFAULT,
- ):
- self.ctx.cluster = Cluster()
- remote1 = Remote('remote1')
- remote1.os = Mock()
- remote1.os.package_type = 'rpm'
- remote1._is_vm = False
- self.ctx.cluster.add(remote1, ['role1'])
- remote2 = Remote('remote1')
- remote2.os = Mock()
- remote2.os.package_type = 'deb'
- remote2._is_vm = False
- self.ctx.cluster.add(remote2, ['role2'])
- task_config = dict()
- with SELinux(self.ctx, task_config) as task:
- remotes = list(task.cluster.remotes)
- assert remotes == [remote1]
-
+++ /dev/null
-import pytest
-
-from teuthology import config
-
-
-class TestYamlConfig(object):
- def setup_method(self):
- self.test_class = config.YamlConfig
-
- def test_set_multiple(self):
- conf_obj = self.test_class()
- conf_obj.foo = 'foo'
- conf_obj.bar = 'bar'
- assert conf_obj.foo == 'foo'
- assert conf_obj.bar == 'bar'
- assert conf_obj.to_dict()['foo'] == 'foo'
-
- def test_from_dict(self):
- in_dict = dict(foo='bar')
- conf_obj = self.test_class.from_dict(in_dict)
- assert conf_obj.foo == 'bar'
-
- def test_contains(self):
- in_dict = dict(foo='bar')
- conf_obj = self.test_class.from_dict(in_dict)
- conf_obj.bar = "foo"
- assert "bar" in conf_obj
- assert "foo" in conf_obj
- assert "baz" not in conf_obj
-
- def test_to_dict(self):
- in_dict = dict(foo='bar')
- conf_obj = self.test_class.from_dict(in_dict)
- assert conf_obj.to_dict() == in_dict
-
- def test_from_str(self):
- in_str = "foo: bar"
- conf_obj = self.test_class.from_str(in_str)
- assert conf_obj.foo == 'bar'
-
- def test_to_str(self):
- in_str = "foo: bar"
- conf_obj = self.test_class.from_str(in_str)
- assert conf_obj.to_str() == in_str
-
- def test_update(self):
- conf_obj = self.test_class(dict())
- conf_obj.foo = 'foo'
- conf_obj.bar = 'bar'
- conf_obj.update(dict(bar='baz'))
- assert conf_obj.foo == 'foo'
- assert conf_obj.bar == 'baz'
-
- def test_delattr(self):
- conf_obj = self.test_class()
- conf_obj.foo = 'bar'
- assert conf_obj.foo == 'bar'
- del conf_obj.foo
- assert conf_obj.foo is None
-
- def test_assignment(self):
- conf_obj = self.test_class()
- conf_obj["foo"] = "bar"
- assert conf_obj["foo"] == "bar"
- assert conf_obj.foo == "bar"
-
- def test_used_with_update(self):
- d = dict()
- conf_obj = self.test_class.from_dict({"foo": "bar"})
- d.update(conf_obj)
- assert d["foo"] == "bar"
-
- def test_get(self):
- conf_obj = self.test_class()
- assert conf_obj.get('foo') is None
- assert conf_obj.get('foo', 'bar') == 'bar'
- conf_obj.foo = 'baz'
- assert conf_obj.get('foo') == 'baz'
-
-
-class TestTeuthologyConfig(TestYamlConfig):
- def setup_method(self):
- self.test_class = config.TeuthologyConfig
-
- def test_get_ceph_git_base_default(self):
- conf_obj = self.test_class()
- conf_obj.yaml_path = ''
- conf_obj.load()
- assert conf_obj.ceph_git_base_url == "https://github.com/ceph/"
-
- def test_set_ceph_git_base_via_private(self):
- conf_obj = self.test_class()
- conf_obj._conf['ceph_git_base_url'] = \
- "git://git.ceph.com/"
- assert conf_obj.ceph_git_base_url == "git://git.ceph.com/"
-
- def test_get_reserve_machines_default(self):
- conf_obj = self.test_class()
- conf_obj.yaml_path = ''
- conf_obj.load()
- assert conf_obj.reserve_machines == 5
-
- def test_set_reserve_machines_via_private(self):
- conf_obj = self.test_class()
- conf_obj._conf['reserve_machines'] = 2
- assert conf_obj.reserve_machines == 2
-
- def test_set_nonstandard(self):
- conf_obj = self.test_class()
- conf_obj.something = 'something else'
- assert conf_obj.something == 'something else'
-
-
-class TestJobConfig(TestYamlConfig):
- def setup_method(self):
- self.test_class = config.JobConfig
-
-
-class TestFakeNamespace(TestYamlConfig):
- def setup_method(self):
- self.test_class = config.FakeNamespace
-
- def test_docopt_dict(self):
- """
- Tests if a dict in the format that docopt returns can
- be parsed correctly.
- """
- d = {
- "--verbose": True,
- "--an-option": "some_option",
- "<an_arg>": "the_arg",
- "something": "some_thing",
- }
- conf_obj = self.test_class(d)
- assert conf_obj.verbose
- assert conf_obj.an_option == "some_option"
- assert conf_obj.an_arg == "the_arg"
- assert conf_obj.something == "some_thing"
-
- def test_config(self):
- """
- Tests that a teuthology_config property is automatically added
- to the conf_obj
- """
- conf_obj = self.test_class(dict(foo="bar"))
- assert conf_obj["foo"] == "bar"
- assert conf_obj.foo == "bar"
- assert conf_obj.teuthology_config.get("fake key") is None
-
- def test_getattr(self):
- conf_obj = self.test_class.from_dict({"foo": "bar"})
- result = getattr(conf_obj, "not_there", "default")
- assert result == "default"
- result = getattr(conf_obj, "foo")
- assert result == "bar"
-
- def test_none(self):
- conf_obj = self.test_class.from_dict(dict(null=None))
- assert conf_obj.null is None
-
- def test_delattr(self):
- conf_obj = self.test_class()
- conf_obj.foo = 'bar'
- assert conf_obj.foo == 'bar'
- del conf_obj.foo
- with pytest.raises(AttributeError):
- conf_obj.foo
-
- def test_to_str(self):
- in_str = "foo: bar"
- conf_obj = self.test_class.from_str(in_str)
- assert conf_obj.to_str() == "{'foo': 'bar'}"
-
- def test_multiple_access(self):
- """
- Test that config.config and FakeNamespace.teuthology_config reflect
- each others' modifications
- """
- in_str = "foo: bar"
- conf_obj = self.test_class.from_str(in_str)
- assert config.config.get('test_key_1') is None
- assert conf_obj.teuthology_config.get('test_key_1') is None
- config.config.test_key_1 = 'test value'
- assert conf_obj.teuthology_config['test_key_1'] == 'test value'
-
- assert config.config.get('test_key_2') is None
- assert conf_obj.teuthology_config.get('test_key_2') is None
- conf_obj.teuthology_config['test_key_2'] = 'test value'
- assert config.config['test_key_2'] == 'test value'
+++ /dev/null
-from pytest import raises
-from teuthology import contextutil
-from logging import ERROR
-
-
-class TestSafeWhile(object):
-
- def setup_method(self):
- contextutil.log.setLevel(ERROR)
- self.fake_sleep = lambda s: True
- self.s_while = contextutil.safe_while
-
- def test_6_5_10_deal(self):
- with raises(contextutil.MaxWhileTries):
- with self.s_while(_sleeper=self.fake_sleep) as proceed:
- while proceed():
- pass
-
- def test_6_0_1_deal(self):
- with raises(contextutil.MaxWhileTries) as error:
- with self.s_while(
- tries=1,
- _sleeper=self.fake_sleep
- ) as proceed:
- while proceed():
- pass
-
- assert 'waiting for 6 seconds' in str(error)
-
- def test_1_0_10_deal(self):
- with raises(contextutil.MaxWhileTries) as error:
- with self.s_while(
- sleep=1,
- _sleeper=self.fake_sleep
- ) as proceed:
- while proceed():
- pass
-
- assert 'waiting for 10 seconds' in str(error)
-
- def test_6_1_10_deal(self):
- with raises(contextutil.MaxWhileTries) as error:
- with self.s_while(
- increment=1,
- _sleeper=self.fake_sleep
- ) as proceed:
- while proceed():
- pass
-
- assert 'waiting for 105 seconds' in str(error)
-
- def test_timeout(self):
- # series of sleep, increment, timeout params to test
- params = [(10, 0, 100),
- (1, 2, 30),
- (10, 0.5, 100),
- (2, 0, 5),
- (2, 3, 5),
- (10, 0, 15),
- (20, 10, 60)]
- for sleep, increment, timeout in params:
- print("trying ", sleep, increment, timeout)
- with raises(contextutil.MaxWhileTries) as error:
- with self.s_while(
- sleep=sleep,
- increment=increment,
- timeout=timeout,
- _sleeper=self.fake_sleep
- ) as proceed:
- while proceed():
- pass
-
- assert 'waiting for {timeout}'.format(timeout=timeout) in str(error)
-
- def test_action(self):
- with raises(contextutil.MaxWhileTries) as error:
- with self.s_while(
- action='doing the thing',
- _sleeper=self.fake_sleep
- ) as proceed:
- while proceed():
- pass
-
- assert "'doing the thing' reached maximum tries" in str(error)
-
- def test_no_raise(self):
- with self.s_while(_raise=False, _sleeper=self.fake_sleep) as proceed:
- while proceed():
- pass
-
- assert True
-
- def test_tries(self):
- attempts = 0
- with self.s_while(tries=-1, _sleeper=self.fake_sleep) as proceed:
- while attempts < 100 and proceed():
- attempts += 1
+++ /dev/null
-# -*- coding: utf-8 -*-
-import pytest
-
-from teuthology.test.fake_fs import make_fake_fstools
-from teuthology.describe_tests import (tree_with_info, extract_info,
- get_combinations)
-from teuthology.exceptions import ParseError
-from mock import MagicMock, patch
-
-realistic_fs = {
- 'basic': {
- '%': None,
- 'base': {
- 'install.yaml':
- """meta:
-- desc: install ceph
-install:
-"""
- },
- 'clusters': {
- 'fixed-1.yaml':
- """meta:
-- desc: single node cluster
-roles:
-- [osd.0, osd.1, osd.2, mon.a, mon.b, mon.c, client.0]
-""",
- 'fixed-2.yaml':
- """meta:
-- desc: couple node cluster
-roles:
-- [osd.0, osd.1, osd.2, mon.a, mon.b, mon.c]
-- [client.0]
-""",
- 'fixed-3.yaml':
- """meta:
-- desc: triple node cluster
-roles:
-- [osd.0, osd.1, osd.2, mon.a, mon.b, mon.c]
-- [client.0]
-- [client.1]
-"""
- },
- 'workloads': {
- 'rbd_api_tests_old_format.yaml':
- """meta:
-- desc: c/c++ librbd api tests with format 1 images
- rbd_features: none
-overrides:
- ceph:
- conf:
- client:
- rbd default format: 1
-tasks:
-- workunit:
- env:
- RBD_FEATURES: 0
- clients:
- client.0:
- - rbd/test_librbd.sh
-""",
- 'rbd_api_tests.yaml':
- """meta:
-- desc: c/c++ librbd api tests with default settings
- rbd_features: default
-tasks:
-- workunit:
- clients:
- client.0:
- - rbd/test_librbd.sh
-""",
- },
- },
-}
-
-
-expected_tree = """├── %
-├── base
-│ └── install.yaml
-├── clusters
-│ ├── fixed-1.yaml
-│ ├── fixed-2.yaml
-│ └── fixed-3.yaml
-└── workloads
- ├── rbd_api_tests.yaml
- └── rbd_api_tests_old_format.yaml""".split('\n')
-
-
-expected_facets = [
- '',
- '',
- 'base',
- '',
- 'clusters',
- 'clusters',
- 'clusters',
- '',
- 'workloads',
- 'workloads',
-]
-
-
-expected_desc = [
- '',
- '',
- 'install ceph',
- '',
- 'single node cluster',
- 'couple node cluster',
- 'triple node cluster',
- '',
- 'c/c++ librbd api tests with default settings',
- 'c/c++ librbd api tests with format 1 images',
-]
-
-
-expected_rbd_features = [
- '',
- '',
- '',
- '',
- '',
- '',
- '',
- '',
- 'default',
- 'none',
-]
-
-
-class TestDescribeTests(object):
-
- def setup_method(self):
- self.mocks = dict()
- self.patchers = dict()
- exists, listdir, isfile, isdir, open = make_fake_fstools(realistic_fs)
- for ppoint, fn in {
- 'os.listdir': listdir,
- 'os.path.isdir': isdir,
- 'teuthology.describe_tests.open': open,
- 'builtins.open': open,
- 'os.path.exists': exists,
- 'os.path.isfile': isfile,
- }.items():
- mockobj = MagicMock()
- patcher = patch(ppoint, mockobj)
- mockobj.side_effect = fn
- patcher.start()
- self.mocks[ppoint] = mockobj
- self.patchers[ppoint] = patcher
-
- def stop_patchers(self):
- for patcher in self.patchers.values():
- patcher.stop()
-
- def teardown_method(self):
- self.stop_patchers()
-
- @staticmethod
- def assert_expected_combo_headers(headers):
- assert headers == (['subsuite depth 0'] +
- sorted(set(filter(bool, expected_facets))))
-
- def test_no_filters(self):
- rows = tree_with_info('basic', [], False, '', [])
- assert rows == [[x] for x in expected_tree]
-
- def test_single_filter(self):
- rows = tree_with_info('basic', ['desc'], False, '', [])
- assert rows == [list(_) for _ in zip(expected_tree, expected_desc)]
-
- rows = tree_with_info('basic', ['rbd_features'], False, '', [])
- assert rows == [list(_) for _ in zip(expected_tree, expected_rbd_features)]
-
- def test_single_filter_with_facets(self):
- rows = tree_with_info('basic', ['desc'], True, '', [])
- assert rows == [list(_) for _ in zip(expected_tree, expected_facets,
- expected_desc)]
-
- rows = tree_with_info('basic', ['rbd_features'], True, '', [])
- assert rows == [list(_) for _ in zip(expected_tree, expected_facets,
- expected_rbd_features)]
-
- def test_no_matching(self):
- rows = tree_with_info('basic', ['extra'], False, '', [])
- assert rows == [list(_) for _ in zip(expected_tree, [''] * len(expected_tree))]
-
- rows = tree_with_info('basic', ['extra'], True, '', [])
- assert rows == [list(_) for _ in zip(expected_tree, expected_facets,
- [''] * len(expected_tree))]
-
- def test_multiple_filters(self):
- rows = tree_with_info('basic', ['desc', 'rbd_features'], False, '', [])
- assert rows == [list(_) for _ in zip(expected_tree,
- expected_desc,
- expected_rbd_features)]
-
- rows = tree_with_info('basic', ['rbd_features', 'desc'], False, '', [])
- assert rows == [list(_) for _ in zip(expected_tree,
- expected_rbd_features,
- expected_desc)]
-
- def test_multiple_filters_with_facets(self):
- rows = tree_with_info('basic', ['desc', 'rbd_features'], True, '', [])
- assert rows == [list(_) for _ in zip(expected_tree,
- expected_facets,
- expected_desc,
- expected_rbd_features)]
-
- rows = tree_with_info('basic', ['rbd_features', 'desc'], True, '', [])
- assert rows == [list(_) for _ in zip(expected_tree,
- expected_facets,
- expected_rbd_features,
- expected_desc)]
-
- def test_combinations_only_facets(self):
- headers, rows = get_combinations('basic',
- fields=[], subset=None, limit=1,
- filter_in=None, filter_out=None, filter_all=None,
- include_facet=True)
- self.assert_expected_combo_headers(headers)
- assert rows == [['basic', 'install', 'fixed-1', 'rbd_api_tests']]
-
- def test_combinations_desc_features(self):
- headers, rows = get_combinations('basic',
- fields=['desc', 'rbd_features'], subset=None, limit=1,
- filter_in=None, filter_out=None, filter_all=None,
- include_facet=False)
- assert headers == ['desc', 'rbd_features']
- descriptions = '\n'.join([
- 'install ceph',
- 'single node cluster',
- 'c/c++ librbd api tests with default settings',
- ])
- assert rows == [[descriptions, 'default']]
-
- def test_combinations_filter_in(self):
- headers, rows = get_combinations('basic',
- fields=[], subset=None, limit=0,
- filter_in=['old_format'], filter_out=None, filter_all=None,
- include_facet=True)
- self.assert_expected_combo_headers(headers)
- assert rows == [
- ['basic', 'install', 'fixed-1', 'rbd_api_tests_old_format'],
- ['basic', 'install', 'fixed-2', 'rbd_api_tests_old_format'],
- ['basic', 'install', 'fixed-3', 'rbd_api_tests_old_format'],
- ]
-
- def test_combinations_filter_out(self):
- headers, rows = get_combinations('basic',
- fields=[], subset=None, limit=0,
- filter_in=None, filter_out=['old_format'], filter_all=None,
- include_facet=True)
- self.assert_expected_combo_headers(headers)
- assert rows == [
- ['basic', 'install', 'fixed-1', 'rbd_api_tests'],
- ['basic', 'install', 'fixed-2', 'rbd_api_tests'],
- ['basic', 'install', 'fixed-3', 'rbd_api_tests'],
- ]
-
- def test_combinations_filter_all(self):
- headers, rows = get_combinations('basic',
- fields=[], subset=None, limit=0,
- filter_in=None, filter_out=None,
- filter_all=['fixed-2', 'old_format'],
- include_facet=True)
- self.assert_expected_combo_headers(headers)
- assert rows == [
- ['basic', 'install', 'fixed-2', 'rbd_api_tests_old_format']
- ]
-
-
-@patch('teuthology.describe_tests.open')
-@patch('os.path.isdir')
-def test_extract_info_dir(m_isdir, m_open):
- simple_fs = {'a': {'b.yaml': 'meta: [{foo: c}]'}}
- _, _, _, m_isdir.side_effect, m_open.side_effect = \
- make_fake_fstools(simple_fs)
- info = extract_info('a', [])
- assert info == {}
-
- info = extract_info('a', ['foo', 'bar'])
- assert info == {'foo': '', 'bar': ''}
-
- info = extract_info('a/b.yaml', ['foo', 'bar'])
- assert info == {'foo': 'c', 'bar': ''}
-
-
-@patch('teuthology.describe_tests.open')
-@patch('os.path.isdir')
-def check_parse_error(fs, m_isdir, m_open):
- _, _, _, m_isdir.side_effect, m_open.side_effect = make_fake_fstools(fs)
- with pytest.raises(ParseError):
- a = extract_info('a.yaml', ['a'])
- raise Exception(str(a))
-
-
-def test_extract_info_too_many_elements():
- check_parse_error({'a.yaml': 'meta: [{a: b}, {b: c}]'})
-
-
-def test_extract_info_not_a_list():
- check_parse_error({'a.yaml': 'meta: {a: b}'})
-
-
-def test_extract_info_not_a_dict():
- check_parse_error({'a.yaml': 'meta: [[a, b]]'})
-
-
-@patch('teuthology.describe_tests.open')
-@patch('os.path.isdir')
-def test_extract_info_empty_file(m_isdir, m_open):
- simple_fs = {'a.yaml': ''}
- _, _, _, m_isdir.side_effect, m_open.side_effect = \
- make_fake_fstools(simple_fs)
- info = extract_info('a.yaml', [])
- assert info == {}
+++ /dev/null
-from humanfriendly import format_timespan
-from mock import Mock, patch
-from pytest import mark
-from teuthology.config import config
-from teuthology.run_tasks import build_email_body as email_body
-from textwrap import dedent
-
-class TestSleepBeforeTeardownEmail(object):
- def setup_method(self):
- config.results_ui_server = "http://example.com/"
- config.archive_server = "http://qa-proxy.ceph.com/teuthology/"
-
- @mark.parametrize(
- ['status', 'owner', 'suite_name', 'run_name', 'job_id', 'dura'],
- [
- [
- 'pass',
- 'noreply@host',
- 'dummy',
- 'run-name',
- 123,
- 3600,
- ],
- [
- 'fail',
- 'noname',
- 'yummy',
- 'next-run',
- 1000,
- 99999,
- ],
- ]
- )
- @patch("teuthology.run_tasks.time.time")
- def test_sleep_before_teardown_email_body(self, m_time, status, owner,
- suite_name, run_name, job_id, dura):
- ctx = Mock()
- archive_path='archive/path'
- archive_dir='/archive/dir'
- date_sec=3661
- date_str='1970-01-01 01:01:01'
- m_time.return_value=float(date_sec)
- duration_sec=dura
- duration_str=format_timespan(duration_sec)
- ref_body=dedent("""
- Teuthology job {run}/{job} has fallen asleep at {date} for {duration_str}
-
- Owner: {owner}
- Suite Name: {suite}
- Sleep Date: {date}
- Sleep Time: {duration_sec} seconds ({duration_str})
- Job Info: http://example.com/{run}/
- Job Logs: http://qa-proxy.ceph.com/teuthology/path/{job}/
- Task Stack: a/b/c
- Current Status: {status}"""
- .format(duration_sec=duration_sec, duration_str=duration_str,
- owner=owner, suite=suite_name, run=run_name,
- job=job_id, status=status, date=date_str))
- print(ref_body)
- ctx.config = dict(
- archive_path=archive_path,
- job_id=job_id,
- suite=suite_name,
- )
- if status == 'pass':
- ctx.summary = dict(
- success=True,
- )
- elif status == 'fail':
- ctx.summary = dict(
- success=False,
- )
- else:
- ctx.summary = dict()
-
- ctx.owner = owner
- ctx.name = run_name
- ctx.archive_dir = archive_dir
- tasks = [('a', None), ('b', None), ('c', None)]
- (subj, body) = email_body(ctx, tasks, dura)
- assert body == ref_body.lstrip('\n')
+++ /dev/null
-import os
-import random
-
-from unittest.mock import patch, Mock
-
-from teuthology import exit
-
-
-class TestExiter(object):
- klass = exit.Exiter
-
- def setup_method(self):
- self.pid = os.getpid()
-
- # Below, we patch os.kill() in such a way that the first time it is
- # invoked it does actually send the signal. Any subsequent invocation
- # won't send any signal - this is so we don't kill the process running
- # our unit tests!
- self.patcher_kill = patch(
- 'teuthology.exit.os.kill',
- wraps=os.kill,
- )
-
- #Keep a copy of the unpatched kill and call this in place of os.kill
- #In the Exiter objects, the os.kill calls are patched.
- #So the call_count should be 1.
- self.kill_unpatched = os.kill
- self.m_kill = self.patcher_kill.start()
-
- def m_kill_unwrap(pid, sig):
- # Setting return_value of a mocked object disables the wrapping
- if self.m_kill.call_count > 1:
- self.m_kill.return_value = None
-
- self.m_kill.side_effect = m_kill_unwrap
-
- def teardown_method(self):
- self.patcher_kill.stop()
- del self.m_kill
-
- def test_basic(self):
- sig = 15
- obj = self.klass()
- m_func = Mock()
- obj.add_handler(sig, m_func)
- assert len(obj.handlers) == 1
- self.kill_unpatched(self.pid, sig)
- assert m_func.call_count == 1
- assert self.m_kill.call_count == 1
- for arg_list in self.m_kill.call_args_list:
- assert arg_list[0] == (self.pid, sig)
-
- def test_remove_handlers(self):
- sig = [1, 15]
- send_sig = random.choice(sig)
- n = 3
- obj = self.klass()
- handlers = list()
- for i in range(n):
- m_func = Mock(name="handler %s" % i)
- handlers.append(obj.add_handler(sig, m_func))
- assert obj.handlers == handlers
- for handler in handlers:
- handler.remove()
- assert obj.handlers == list()
- self.kill_unpatched(self.pid, send_sig)
- assert self.m_kill.call_count == 1
- for handler in handlers:
- assert handler.func.call_count == 0
-
- def test_n_handlers(self, n=10, sig=11):
- if isinstance(sig, int):
- send_sig = sig
- else:
- send_sig = random.choice(sig)
- obj = self.klass()
- handlers = list()
- for i in range(n):
- m_func = Mock(name="handler %s" % i)
- handlers.append(obj.add_handler(sig, m_func))
- assert obj.handlers == handlers
- self.kill_unpatched(self.pid, send_sig)
- for i in range(n):
- assert handlers[i].func.call_count == 1
- assert self.m_kill.call_count == 1
- for arg_list in self.m_kill.call_args_list:
- assert arg_list[0] == (self.pid, send_sig)
-
- def test_multiple_signals(self):
- self.test_n_handlers(n=3, sig=[1, 6, 11, 15])
+++ /dev/null
-from teuthology.misc import get_distro
-
-
-class Mock:
- pass
-
-
-class TestGetDistro(object):
-
- def setup_method(self):
- self.fake_ctx = Mock()
- self.fake_ctx.config = {}
- # os_type in ctx will always default to None
- self.fake_ctx.os_type = None
-
- def test_default_distro(self):
- distro = get_distro(self.fake_ctx)
- assert distro == 'ubuntu'
-
- def test_argument(self):
- # we don't want fake_ctx to have a config
- self.fake_ctx = Mock()
- self.fake_ctx.os_type = 'centos'
- distro = get_distro(self.fake_ctx)
- assert distro == 'centos'
-
- def test_teuth_config(self):
- self.fake_ctx.config = {'os_type': 'fedora'}
- distro = get_distro(self.fake_ctx)
- assert distro == 'fedora'
-
- def test_argument_takes_precedence(self):
- self.fake_ctx.config = {'os_type': 'fedora'}
- self.fake_ctx.os_type = "centos"
- distro = get_distro(self.fake_ctx)
- assert distro == 'centos'
-
- def test_no_config_or_os_type(self):
- self.fake_ctx = Mock()
- self.fake_ctx.os_type = None
- distro = get_distro(self.fake_ctx)
- assert distro == 'ubuntu'
-
- def test_config_os_type_is_none(self):
- self.fake_ctx.config["os_type"] = None
- distro = get_distro(self.fake_ctx)
- assert distro == 'ubuntu'
+++ /dev/null
-from teuthology.misc import get_distro_version
-
-
-class Mock:
- pass
-
-
-class TestGetDistroVersion(object):
-
- def setup_method(self):
- self.fake_ctx = Mock()
- self.fake_ctx.config = {}
- self.fake_ctx_noarg = Mock()
- self.fake_ctx_noarg.config = {}
- self.fake_ctx_noarg.os_version = None
- self.fake_ctx.os_type = None
- self.fake_ctx_noarg.os_type = None
-
- def test_default_distro_version(self):
- # Default distro is ubuntu, default version of ubuntu is 20.04
- self.fake_ctx.os_version = None
- distroversion = get_distro_version(self.fake_ctx)
- assert distroversion == '22.04'
-
- def test_argument_version(self):
- self.fake_ctx.os_version = '13.04'
- distroversion = get_distro_version(self.fake_ctx)
- assert distroversion == '13.04'
-
- def test_teuth_config_version(self):
- #Argument takes precidence.
- self.fake_ctx.os_version = '13.04'
- self.fake_ctx.config = {'os_version': '13.10'}
- distroversion = get_distro_version(self.fake_ctx)
- assert distroversion == '13.04'
-
- def test_teuth_config_noarg_version(self):
- self.fake_ctx_noarg.config = {'os_version': '13.04'}
- distroversion = get_distro_version(self.fake_ctx_noarg)
- assert distroversion == '13.04'
-
- def test_no_teuth_config(self):
- self.fake_ctx = Mock()
- self.fake_ctx.os_type = None
- self.fake_ctx.os_version = '13.04'
- distroversion = get_distro_version(self.fake_ctx)
- assert distroversion == '13.04'
+++ /dev/null
-from teuthology import misc as teuthology
-
-class Mock: pass
-
-class TestGetMultiMachineTypes(object):
-
- def test_space(self):
- give = 'burnupi plana vps'
- expect = ['burnupi','plana','vps']
- assert teuthology.get_multi_machine_types(give) == expect
-
- def test_tab(self):
- give = 'burnupi plana vps'
- expect = ['burnupi','plana','vps']
- assert teuthology.get_multi_machine_types(give) == expect
-
- def test_comma(self):
- give = 'burnupi,plana,vps'
- expect = ['burnupi','plana','vps']
- assert teuthology.get_multi_machine_types(give) == expect
-
- def test_single(self):
- give = 'burnupi'
- expect = ['burnupi']
- assert teuthology.get_multi_machine_types(give) == expect
-
-
+++ /dev/null
-import importlib
-import pytest
-import sys
-
-from pathlib import Path
-from typing import List
-
-root = Path("./teuthology")
-
-
-def find_modules() -> List[str]:
- modules = []
- for path in root.rglob("*.py"):
- if path.name.startswith("test_"):
- continue
- if "-" in path.name:
- continue
- if path.name == "__init__.py":
- path = path.parent
-
- path_name = str(path).replace("/", ".")
- if path_name.endswith(".py"):
- path_name = path_name[:-3]
- modules.append(path_name)
- return sorted(modules)
-
-
-@pytest.mark.parametrize("module", find_modules())
-def test_import_modules(module):
- importlib.import_module(module)
- assert module in sys.modules
+++ /dev/null
-from teuthology import job_status
-
-
-class TestJobStatus(object):
- def test_get_only_success_true(self):
- summary = dict(success=True)
- status = job_status.get_status(summary)
- assert status == 'pass'
-
- def test_get_only_success_false(self):
- summary = dict(success=False)
- status = job_status.get_status(summary)
- assert status == 'fail'
-
- def test_get_status_pass(self):
- summary = dict(status='pass')
- status = job_status.get_status(summary)
- assert status == 'pass'
-
- def test_get_status_fail(self):
- summary = dict(status='fail')
- status = job_status.get_status(summary)
- assert status == 'fail'
-
- def test_get_status_dead(self):
- summary = dict(status='dead')
- status = job_status.get_status(summary)
- assert status == 'dead'
-
- def test_get_status_none(self):
- summary = dict()
- status = job_status.get_status(summary)
- assert status is None
-
- def test_set_status_pass(self):
- summary = dict()
- job_status.set_status(summary, 'pass')
- assert summary == dict(status='pass', success=True)
-
- def test_set_status_dead(self):
- summary = dict()
- job_status.set_status(summary, 'dead')
- assert summary == dict(status='dead', success=False)
-
- def test_set_then_get_status_dead(self):
- summary = dict()
- job_status.set_status(summary, 'dead')
- status = job_status.get_status(summary)
- assert status == 'dead'
-
- def test_set_status_none(self):
- summary = dict()
- job_status.set_status(summary, None)
- assert summary == dict()
-
- def test_legacy_fail(self):
- summary = dict(success=True)
- summary['success'] = False
- status = job_status.get_status(summary)
- assert status == 'fail'
+++ /dev/null
-from unittest.mock import patch
-
-from teuthology.kill import find_targets
-
-
-class TestFindTargets(object):
- """ Tests for teuthology.kill.find_targets """
-
- @patch('teuthology.kill.report.ResultsReporter.get_jobs')
- def test_missing_run_find_targets(self, m_get_jobs):
- m_get_jobs.return_value = []
- run_targets = find_targets("run-name")
- assert run_targets == {}
-
- @patch('teuthology.kill.report.ResultsReporter.get_jobs')
- def test_missing_job_find_targets(self, m_get_jobs):
- m_get_jobs.return_value = {}
- job_targets = find_targets("run-name", "3")
- assert job_targets == {}
-
- @patch('teuthology.kill.report.ResultsReporter.get_jobs')
- def test_missing_run_targets_find_targets(self, m_get_jobs):
- m_get_jobs.return_value = [{"targets": None, "status": "waiting"}]
- run_targets = find_targets("run-name")
- assert run_targets == {}
-
- @patch('teuthology.kill.report.ResultsReporter.get_jobs')
- def test_missing_job_targets_find_targets(self, m_get_jobs):
- m_get_jobs.return_value = {"targets": None}
- job_targets = find_targets("run-name", "3")
- assert job_targets == {}
-
- @patch('teuthology.kill.report.ResultsReporter.get_jobs')
- def test_run_find_targets(self, m_get_jobs):
- m_get_jobs.return_value = [{"targets": {"node1": ""}, "status": "running"}]
- run_targets = find_targets("run-name")
- assert run_targets == {"node1": ""}
- m_get_jobs.return_value = [{"targets": {"node1": ""}}]
- run_targets = find_targets("run-name")
- assert run_targets == {}
-
- @patch('teuthology.kill.report.ResultsReporter.get_jobs')
- def test_job_find_targets(self, m_get_jobs):
- m_get_jobs.return_value = {"targets": {"node1": ""}}
- job_targets = find_targets("run-name", "3")
- assert job_targets == {"node1": ""}
+++ /dev/null
-import pytest
-
-from unittest.mock import patch, Mock
-
-from teuthology import ls
-
-
-class TestLs(object):
- """ Tests for teuthology.ls """
-
- @patch('os.path.isdir')
- @patch('os.listdir')
- def test_get_jobs(self, m_listdir, m_isdir):
- m_listdir.return_value = ["1", "a", "3"]
- m_isdir.return_value = True
- results = ls.get_jobs("some/archive/dir")
- assert results == ["1", "3"]
-
- @patch("yaml.safe_load_all")
- @patch("teuthology.ls.get_jobs")
- def test_ls(self, m_get_jobs, m_safe_load_all):
- m_get_jobs.return_value = ["1", "2"]
- m_safe_load_all.return_value = [{"failure_reason": "reasons"}]
- ls.ls("some/archive/div", True)
-
- @patch("teuthology.ls.open")
- @patch("teuthology.ls.get_jobs")
- def test_ls_ioerror(self, m_get_jobs, m_open):
- m_get_jobs.return_value = ["1", "2"]
- m_open.side_effect = IOError()
- with pytest.raises(IOError):
- ls.ls("some/archive/dir", True)
-
- @patch("teuthology.ls.open")
- @patch("os.popen")
- @patch("os.path.isdir")
- @patch("os.path.isfile")
- def test_print_debug_info(self, m_isfile, m_isdir, m_popen, m_open):
- m_isfile.return_value = True
- m_isdir.return_value = True
- m_popen.return_value = Mock()
- cmdline = Mock()
- cmdline.find = Mock(return_value=0)
- m1 = Mock()
- m2 = Mock()
- m2.read = Mock(return_value=cmdline)
- m_open.side_effect = [m1, m2]
- ls.print_debug_info("the_job", "job/dir", "some/archive/dir")
+++ /dev/null
-import argparse
-import pytest
-import subprocess
-
-from unittest.mock import Mock, patch
-
-from teuthology import misc
-from teuthology.config import config
-from teuthology.orchestra import cluster
-from teuthology.orchestra.remote import Remote
-
-
-class FakeRemote(object):
- pass
-
-
-def test_sh_normal(caplog):
- assert misc.sh("/bin/echo ABC") == "ABC\n"
- assert "truncated" not in caplog.text
-
-
-def test_sh_truncate(caplog):
- assert misc.sh("/bin/echo -n AB ; /bin/echo C", 2) == "ABC\n"
- assert "truncated" in caplog.text
- assert "ABC" not in caplog.text
-
-
-def test_sh_fail(caplog):
- with pytest.raises(subprocess.CalledProcessError) as excinfo:
- misc.sh("/bin/echo -n AB ; /bin/echo C ; exit 111", 2)
- assert excinfo.value.returncode == 111
- for record in caplog.records:
- if record.levelname == 'ERROR':
- assert ('replay full' in record.message or
- 'ABC\n' == record.message)
-
-def test_sh_progress(caplog):
- assert misc.sh("echo AB ; sleep 0.1 ; /bin/echo C", 2) == "AB\nC\n"
- records = caplog.records
- assert ':sh: ' in records[0].message
- assert 'AB' == records[1].message
- assert 'C' == records[2].message
- assert records[2].created > records[1].created
-
-
-def test_wait_until_osds_up():
- ctx = argparse.Namespace()
- ctx.daemons = Mock()
- ctx.daemons.iter_daemons_of_role.return_value = list()
- remote = Mock(spec=Remote)
- remote.sh.return_value = 'IGNORED\n{"osds":[{"state":["up"]}]}'
- ctx.cluster = cluster.Cluster(
- remotes=[
- (remote, ['osd.0', 'client.1'])
- ],
- )
- with patch.multiple(
- misc,
- get_testdir=lambda _: "TESTDIR",
- ):
- misc.wait_until_osds_up(ctx, ctx.cluster, remote)
-
-
-def test_get_clients_simple():
- ctx = argparse.Namespace()
- remote = FakeRemote()
- ctx.cluster = cluster.Cluster(
- remotes=[
- (remote, ['client.0', 'client.1'])
- ],
- )
- g = misc.get_clients(ctx=ctx, roles=['client.1'])
- got = next(g)
- assert len(got) == 2
- assert got[0] == ('1')
- assert got[1] is remote
- with pytest.raises(StopIteration):
- next(g)
-
-
-def test_get_mon_names():
- expected = [
- ([['mon.a', 'osd.0', 'mon.c']], 'ceph', ['mon.a', 'mon.c']),
- ([['ceph.mon.a', 'osd.0', 'ceph.mon.c']], 'ceph', ['ceph.mon.a', 'ceph.mon.c']),
- ([['mon.a', 'osd.0', 'mon.c'], ['ceph.mon.b']], 'ceph', ['mon.a', 'mon.c', 'ceph.mon.b']),
- ([['mon.a', 'osd.0', 'mon.c'], ['foo.mon.a']], 'ceph', ['mon.a', 'mon.c']),
- ([['mon.a', 'osd.0', 'mon.c'], ['foo.mon.a']], 'foo', ['foo.mon.a']),
- ]
- for remote_roles, cluster_name, expected_mons in expected:
- ctx = argparse.Namespace()
- ctx.cluster = Mock()
- ctx.cluster.remotes = {i: roles for i, roles in enumerate(remote_roles)}
- mons = misc.get_mon_names(ctx, cluster_name)
- assert expected_mons == mons
-
-
-def test_get_first_mon():
- expected = [
- ([['mon.a', 'osd.0', 'mon.c']], 'ceph', 'mon.a'),
- ([['ceph.mon.a', 'osd.0', 'ceph.mon.c']], 'ceph', 'ceph.mon.a'),
- ([['mon.a', 'osd.0', 'mon.c'], ['ceph.mon.b']], 'ceph', 'ceph.mon.b'),
- ([['mon.a', 'osd.0', 'mon.c'], ['foo.mon.a']], 'ceph', 'mon.a'),
- ([['foo.mon.b', 'osd.0', 'mon.c'], ['foo.mon.a']], 'foo', 'foo.mon.a'),
- ]
- for remote_roles, cluster_name, expected_mon in expected:
- ctx = argparse.Namespace()
- ctx.cluster = Mock()
- ctx.cluster.remotes = {i: roles for i, roles in enumerate(remote_roles)}
- mon = misc.get_first_mon(ctx, None, cluster_name)
- assert expected_mon == mon
-
-
-def test_roles_of_type():
- expected = [
- (['client.0', 'osd.0', 'ceph.osd.1'], 'osd', ['0', '1']),
- (['client.0', 'osd.0', 'ceph.osd.1'], 'client', ['0']),
- (['foo.client.1', 'bar.client.2.3', 'baz.osd.1'], 'mon', []),
- (['foo.client.1', 'bar.client.2.3', 'baz.osd.1'], 'client',
- ['1', '2.3']),
- ]
- for roles_for_host, type_, expected_ids in expected:
- ids = list(misc.roles_of_type(roles_for_host, type_))
- assert ids == expected_ids
-
-
-def test_cluster_roles_of_type():
- expected = [
- (['client.0', 'osd.0', 'ceph.osd.1'], 'osd', 'ceph',
- ['osd.0', 'ceph.osd.1']),
- (['client.0', 'osd.0', 'ceph.osd.1'], 'client', 'ceph',
- ['client.0']),
- (['foo.client.1', 'bar.client.2.3', 'baz.osd.1'], 'mon', None, []),
- (['foo.client.1', 'bar.client.2.3', 'baz.osd.1'], 'client', None,
- ['foo.client.1', 'bar.client.2.3']),
- (['foo.client.1', 'bar.client.2.3', 'baz.osd.1'], 'client', 'bar',
- ['bar.client.2.3']),
- ]
- for roles_for_host, type_, cluster_, expected_roles in expected:
- roles = list(misc.cluster_roles_of_type(roles_for_host, type_, cluster_))
- assert roles == expected_roles
-
-
-def test_all_roles_of_type():
- expected = [
- ([['client.0', 'osd.0', 'ceph.osd.1'], ['bar.osd.2']],
- 'osd', ['0', '1', '2']),
- ([['client.0', 'osd.0', 'ceph.osd.1'], ['bar.osd.2', 'baz.client.1']],
- 'client', ['0', '1']),
- ([['foo.client.1', 'bar.client.2.3'], ['baz.osd.1']], 'mon', []),
- ([['foo.client.1', 'bar.client.2.3'], ['baz.osd.1', 'ceph.client.bar']],
- 'client', ['1', '2.3', 'bar']),
- ]
- for host_roles, type_, expected_ids in expected:
- cluster_ = Mock()
- cluster_.remotes = dict(enumerate(host_roles))
- ids = list(misc.all_roles_of_type(cluster_, type_))
- assert ids == expected_ids
-
-
-def test_get_http_log_path():
- # Fake configuration
- archive_server = "http://example.com/server_root"
- config.archive_server = archive_server
- archive_dir = "/var/www/archives"
-
- path = misc.get_http_log_path(archive_dir)
- assert path == "http://example.com/server_root/archives/"
-
- job_id = '12345'
- path = misc.get_http_log_path(archive_dir, job_id)
- assert path == "http://example.com/server_root/archives/12345/"
-
- # Inktank configuration
- archive_server = "http://qa-proxy.ceph.com/teuthology/"
- config.archive_server = archive_server
- archive_dir = "/var/lib/teuthworker/archive/teuthology-2013-09-12_11:49:50-ceph-deploy-main-testing-basic-vps"
- job_id = 31087
- path = misc.get_http_log_path(archive_dir, job_id)
- assert path == "http://qa-proxy.ceph.com/teuthology/teuthology-2013-09-12_11:49:50-ceph-deploy-main-testing-basic-vps/31087/"
-
- path = misc.get_http_log_path(archive_dir)
- assert path == "http://qa-proxy.ceph.com/teuthology/teuthology-2013-09-12_11:49:50-ceph-deploy-main-testing-basic-vps/"
-
-
-def test_is_type():
- is_client = misc.is_type('client')
- assert is_client('client.0')
- assert is_client('ceph.client.0')
- assert is_client('foo.client.0')
- assert is_client('foo.client.bar.baz')
-
- with pytest.raises(ValueError):
- is_client('')
- is_client('client')
- assert not is_client('foo.bar.baz')
- assert not is_client('ceph.client')
- assert not is_client('hadoop.main.0')
-
-
-def test_is_type_in_cluster():
- is_c1_osd = misc.is_type('osd', 'c1')
- with pytest.raises(ValueError):
- is_c1_osd('')
- assert not is_c1_osd('osd.0')
- assert not is_c1_osd('ceph.osd.0')
- assert not is_c1_osd('ceph.osd.0')
- assert not is_c1_osd('c11.osd.0')
- assert is_c1_osd('c1.osd.0')
- assert is_c1_osd('c1.osd.999')
-
-
-def test_get_mons():
- ips = ['1.1.1.1', '2.2.2.2', '3.3.3.3']
- addrs = ['1.1.1.1:6789', '1.1.1.1:6790', '1.1.1.1:6791']
-
- mons = misc.get_mons([['mon.a']], ips)
- assert mons == {'mon.a': addrs[0]}
-
- mons = misc.get_mons([['cluster-a.mon.foo', 'client.b'], ['osd.0']], ips)
- assert mons == {'cluster-a.mon.foo': addrs[0]}
-
- mons = misc.get_mons([['mon.a', 'mon.b', 'ceph.mon.c']], ips)
- assert mons == {'mon.a': addrs[0],
- 'mon.b': addrs[1],
- 'ceph.mon.c': addrs[2]}
-
- mons = misc.get_mons([['mon.a'], ['mon.b'], ['ceph.mon.c']], ips)
- assert mons == {'mon.a': addrs[0],
- 'mon.b': ips[1] + ':6789',
- 'ceph.mon.c': ips[2] + ':6789'}
-
-
-def test_split_role():
- expected = {
- 'client.0': ('ceph', 'client', '0'),
- 'foo.client.0': ('foo', 'client', '0'),
- 'bar.baz.x.y.z': ('bar', 'baz', 'x.y.z'),
- 'mds.a-s-b': ('ceph', 'mds', 'a-s-b'),
- }
-
- for role, expected_split in expected.items():
- actual_split = misc.split_role(role)
- assert actual_split == expected_split
-
-def test_update_key():
- a = { "sha": "foo", "workunit": { "sha": "foo" }, "tasks": [{"task1": "ceph"}], "overrides": [{"sha": "foo"}] }
- b = { "sha": "blah", "workunit": { "sha": "bar" }, "tasks": [] }
-
- misc.update_key("sha", a, b)
- assert a == { "sha": "blah", "workunit": { "sha": "bar" }, "tasks": [{"task1": "ceph"}], "overrides": [{"sha": "foo"}] }
-
-class TestHostnames(object):
- def setup_method(self):
- config._conf = dict()
-
- def teardown_method(self):
- config.load()
-
- def test_canonicalize_hostname(self):
- host_base = 'box1'
- result = misc.canonicalize_hostname(host_base)
- assert result == 'ubuntu@box1.front.sepia.ceph.com'
-
- def test_decanonicalize_hostname(self):
- host = 'ubuntu@box1.front.sepia.ceph.com'
- result = misc.decanonicalize_hostname(host)
- assert result == 'box1'
-
- def test_canonicalize_hostname_nouser(self):
- host_base = 'box1'
- result = misc.canonicalize_hostname(host_base, user=None)
- assert result == 'box1.front.sepia.ceph.com'
-
- def test_decanonicalize_hostname_nouser(self):
- host = 'box1.front.sepia.ceph.com'
- result = misc.decanonicalize_hostname(host)
- assert result == 'box1'
-
- def test_canonicalize_hostname_otherlab(self):
- config.lab_domain = 'example.com'
- host_base = 'box1'
- result = misc.canonicalize_hostname(host_base)
- assert result == 'ubuntu@box1.example.com'
-
- def test_decanonicalize_hostname_otherlab(self):
- config.lab_domain = 'example.com'
- host = 'ubuntu@box1.example.com'
- result = misc.decanonicalize_hostname(host)
- assert result == 'box1'
-
- def test_canonicalize_hostname_nodomain(self):
- config.lab_domain = ''
- host = 'box2'
- result = misc.canonicalize_hostname(host)
- assert result == 'ubuntu@' + host
-
- def test_decanonicalize_hostname_nodomain(self):
- config.lab_domain = ''
- host = 'ubuntu@box2'
- result = misc.decanonicalize_hostname(host)
- assert result == 'box2'
-
- def test_canonicalize_hostname_full_other_user(self):
- config.lab_domain = 'example.com'
- host = 'user1@box1.example.come'
- result = misc.canonicalize_hostname(host)
- assert result == 'user1@box1.example.com'
-
- def test_decanonicalize_hostname_full_other_user(self):
- config.lab_domain = 'example.com'
- host = 'user1@box1.example.come'
- result = misc.decanonicalize_hostname(host)
- assert result == 'box1'
-
-class TestMergeConfigs(object):
- """ Tests merge_config and deep_merge in teuthology.misc """
-
- @patch("os.path.exists")
- @patch("yaml.safe_load")
- @patch("teuthology.misc.open")
- def test_merge_configs(self, m_open, m_safe_load, m_exists):
- """ Only tests with one yaml file being passed, mainly just to test
- the loop logic. The actual merge will be tested in subsequent
- tests.
- """
- expected = {"a": "b", "b": "c"}
- m_exists.return_value = True
- m_safe_load.return_value = expected
- result = misc.merge_configs(["path/to/config1"])
- assert result == expected
- m_open.assert_called_once_with("path/to/config1")
-
- def test_merge_configs_empty(self):
- assert misc.merge_configs([]) == {}
-
- def test_deep_merge(self):
- a = {"a": "b"}
- b = {"b": "c"}
- result = misc.deep_merge(a, b)
- assert result == {"a": "b", "b": "c"}
-
- def test_overwrite_deep_merge(self):
- a = {"a": "b"}
- b = {"a": "overwritten", "b": "c"}
- result = misc.deep_merge(a, b)
- assert result == {"a": "overwritten", "b": "c"}
-
- def test_list_deep_merge(self):
- a = [1, 2]
- b = [3, 4]
- result = misc.deep_merge(a, b)
- assert result == [1, 2, 3, 4]
-
- def test_missing_list_deep_merge(self):
- a = [1, 2]
- b = "not a list"
- with pytest.raises(AssertionError):
- misc.deep_merge(a, b)
-
- def test_missing_a_deep_merge(self):
- result = misc.deep_merge(None, [1, 2])
- assert result == [1, 2]
-
- def test_missing_b_deep_merge(self):
- result = misc.deep_merge([1, 2], None)
- assert result == [1, 2]
-
- def test_invalid_b_deep_merge(self):
- with pytest.raises(AssertionError):
- misc.deep_merge({"a": "b"}, "invalid")
-
-
-class TestIsInDict(object):
- def test_simple_membership(self):
- assert misc.is_in_dict('a', 'foo', {'a':'foo', 'b':'bar'})
-
- def test_dict_membership(self):
- assert misc.is_in_dict(
- 'a', {'sub1':'key1', 'sub2':'key2'},
- {'a':{'sub1':'key1', 'sub2':'key2', 'sub3':'key3'}}
- )
-
- def test_simple_nonmembership(self):
- assert not misc.is_in_dict('a', 'foo', {'a':'bar', 'b':'foo'})
-
- def test_nonmembership_with_presence_at_lower_level(self):
- assert not misc.is_in_dict('a', 'foo', {'a':{'a': 'foo'}})
+++ /dev/null
-import pytest
-
-from unittest.mock import patch, Mock, PropertyMock
-
-from teuthology import packaging
-from teuthology.exceptions import VersionNotFoundError
-
-KOJI_TASK_RPMS_MATRIX = [
- ('tasks/6745/9666745/kernel-4.1.0-0.rc2.git2.1.fc23.x86_64.rpm', 'kernel'),
- ('tasks/6745/9666745/kernel-modules-4.1.0-0.rc2.git2.1.fc23.x86_64.rpm', 'kernel-modules'),
- ('tasks/6745/9666745/kernel-tools-4.1.0-0.rc2.git2.1.fc23.x86_64.rpm', 'kernel-tools'),
- ('tasks/6745/9666745/kernel-tools-libs-devel-4.1.0-0.rc2.git2.1.fc23.x86_64.rpm', 'kernel-tools-libs-devel'),
- ('tasks/6745/9666745/kernel-headers-4.1.0-0.rc2.git2.1.fc23.x86_64.rpm', 'kernel-headers'),
- ('tasks/6745/9666745/kernel-tools-debuginfo-4.1.0-0.rc2.git2.1.fc23.x86_64.rpm', 'kernel-tools-debuginfo'),
- ('tasks/6745/9666745/kernel-debuginfo-common-x86_64-4.1.0-0.rc2.git2.1.fc23.x86_64.rpm', 'kernel-debuginfo-common-x86_64'),
- ('tasks/6745/9666745/perf-debuginfo-4.1.0-0.rc2.git2.1.fc23.x86_64.rpm', 'perf-debuginfo'),
- ('tasks/6745/9666745/kernel-modules-extra-4.1.0-0.rc2.git2.1.fc23.x86_64.rpm', 'kernel-modules-extra'),
- ('tasks/6745/9666745/kernel-tools-libs-4.1.0-0.rc2.git2.1.fc23.x86_64.rpm', 'kernel-tools-libs'),
- ('tasks/6745/9666745/kernel-core-4.1.0-0.rc2.git2.1.fc23.x86_64.rpm', 'kernel-core'),
- ('tasks/6745/9666745/kernel-debuginfo-4.1.0-0.rc2.git2.1.fc23.x86_64.rpm', 'kernel-debuginfo'),
- ('tasks/6745/9666745/python-perf-4.1.0-0.rc2.git2.1.fc23.x86_64.rpm', 'python-perf'),
- ('tasks/6745/9666745/kernel-devel-4.1.0-0.rc2.git2.1.fc23.x86_64.rpm', 'kernel-devel'),
- ('tasks/6745/9666745/python-perf-debuginfo-4.1.0-0.rc2.git2.1.fc23.x86_64.rpm', 'python-perf-debuginfo'),
- ('tasks/6745/9666745/perf-4.1.0-0.rc2.git2.1.fc23.x86_64.rpm', 'perf'),
-]
-
-KOJI_TASK_RPMS = [rpm[0] for rpm in KOJI_TASK_RPMS_MATRIX]
-
-
-class TestPackaging(object):
-
- def test_get_package_name_deb(self):
- remote = Mock()
- remote.os.package_type = "deb"
- assert packaging.get_package_name('sqlite', remote) == "sqlite3"
-
- def test_get_package_name_rpm(self):
- remote = Mock()
- remote.os.package_type = "rpm"
- assert packaging.get_package_name('sqlite', remote) is None
-
- def test_get_package_name_not_found(self):
- remote = Mock()
- remote.os.package_type = "rpm"
- assert packaging.get_package_name('notthere', remote) is None
-
- def test_get_service_name_deb(self):
- remote = Mock()
- remote.os.package_type = "deb"
- assert packaging.get_service_name('httpd', remote) == 'apache2'
-
- def test_get_service_name_rpm(self):
- remote = Mock()
- remote.os.package_type = "rpm"
- assert packaging.get_service_name('httpd', remote) == 'httpd'
-
- def test_get_service_name_not_found(self):
- remote = Mock()
- remote.os.package_type = "rpm"
- assert packaging.get_service_name('notthere', remote) is None
-
- def test_install_package_deb(self):
- m_remote = Mock()
- m_remote.os.package_type = "deb"
- expected = [
- 'DEBIAN_FRONTEND=noninteractive',
- 'sudo',
- '-E',
- 'apt-get',
- '-y',
- '--force-yes',
- 'install',
- 'apache2'
- ]
- packaging.install_package('apache2', m_remote)
- m_remote.run.assert_called_with(args=expected)
-
- def test_install_package_rpm(self):
- m_remote = Mock()
- m_remote.os.package_type = "rpm"
- expected = [
- 'sudo',
- 'yum',
- '-y',
- 'install',
- 'httpd'
- ]
- packaging.install_package('httpd', m_remote)
- m_remote.run.assert_called_with(args=expected)
-
- def test_remove_package_deb(self):
- m_remote = Mock()
- m_remote.os.package_type = "deb"
- expected = [
- 'DEBIAN_FRONTEND=noninteractive',
- 'sudo',
- '-E',
- 'apt-get',
- '-y',
- 'purge',
- 'apache2'
- ]
- packaging.remove_package('apache2', m_remote)
- m_remote.run.assert_called_with(args=expected)
-
- def test_remove_package_rpm(self):
- m_remote = Mock()
- m_remote.os.package_type = "rpm"
- expected = [
- 'sudo',
- 'yum',
- '-y',
- 'erase',
- 'httpd'
- ]
- packaging.remove_package('httpd', m_remote)
- m_remote.run.assert_called_with(args=expected)
-
- def test_get_koji_package_name(self):
- build_info = dict(version="3.10.0", release="123.20.1")
- result = packaging.get_koji_package_name("kernel", build_info)
- assert result == "kernel-3.10.0-123.20.1.x86_64.rpm"
-
- @patch("teuthology.packaging.config")
- def test_get_kojiroot_base_url(self, m_config):
- m_config.kojiroot_url = "http://kojiroot.com"
- build_info = dict(
- package_name="kernel",
- version="3.10.0",
- release="123.20.1",
- )
- result = packaging.get_kojiroot_base_url(build_info)
- expected = "http://kojiroot.com/kernel/3.10.0/123.20.1/x86_64/"
- assert result == expected
-
- @patch("teuthology.packaging.config")
- def test_get_koji_build_info_success(self, m_config):
- m_config.kojihub_url = "http://kojihub.com"
- m_proc = Mock()
- expected = dict(foo="bar")
- m_proc.exitstatus = 0
- m_proc.stdout.getvalue.return_value = str(expected)
- m_remote = Mock()
- m_remote.run.return_value = m_proc
- result = packaging.get_koji_build_info(1, m_remote, dict())
- assert result == expected
- args, kwargs = m_remote.run.call_args
- expected_args = [
- 'python', '-c',
- 'import koji; '
- 'hub = koji.ClientSession("http://kojihub.com"); '
- 'print(hub.getBuild(1))',
- ]
- assert expected_args == kwargs['args']
-
- @patch("teuthology.packaging.config")
- def test_get_koji_build_info_fail(self, m_config):
- m_config.kojihub_url = "http://kojihub.com"
- m_proc = Mock()
- m_proc.exitstatus = 1
- m_remote = Mock()
- m_remote.run.return_value = m_proc
- m_ctx = Mock()
- m_ctx.summary = dict()
- with pytest.raises(RuntimeError):
- packaging.get_koji_build_info(1, m_remote, m_ctx)
-
- @patch("teuthology.packaging.config")
- def test_get_koji_task_result_success(self, m_config):
- m_config.kojihub_url = "http://kojihub.com"
- m_proc = Mock()
- expected = dict(foo="bar")
- m_proc.exitstatus = 0
- m_proc.stdout.getvalue.return_value = str(expected)
- m_remote = Mock()
- m_remote.run.return_value = m_proc
- result = packaging.get_koji_task_result(1, m_remote, dict())
- assert result == expected
- args, kwargs = m_remote.run.call_args
- expected_args = [
- 'python', '-c',
- 'import koji; '
- 'hub = koji.ClientSession("http://kojihub.com"); '
- 'print(hub.getTaskResult(1))',
- ]
- assert expected_args == kwargs['args']
-
- @patch("teuthology.packaging.config")
- def test_get_koji_task_result_fail(self, m_config):
- m_config.kojihub_url = "http://kojihub.com"
- m_proc = Mock()
- m_proc.exitstatus = 1
- m_remote = Mock()
- m_remote.run.return_value = m_proc
- m_ctx = Mock()
- m_ctx.summary = dict()
- with pytest.raises(RuntimeError):
- packaging.get_koji_task_result(1, m_remote, m_ctx)
-
- @patch("teuthology.packaging.config")
- def test_get_koji_task_rpm_info_success(self, m_config):
- m_config.koji_task_url = "http://kojihub.com/work"
- expected = dict(
- base_url="http://kojihub.com/work/tasks/6745/9666745/",
- version="4.1.0-0.rc2.git2.1.fc23.x86_64",
- rpm_name="kernel-4.1.0-0.rc2.git2.1.fc23.x86_64.rpm",
- package_name="kernel",
- )
- result = packaging.get_koji_task_rpm_info('kernel', KOJI_TASK_RPMS)
- assert expected == result
-
- @patch("teuthology.packaging.config")
- def test_get_koji_task_rpm_info_fail(self, m_config):
- m_config.koji_task_url = "http://kojihub.com/work"
- with pytest.raises(RuntimeError):
- packaging.get_koji_task_rpm_info('ceph', KOJI_TASK_RPMS)
-
- def test_get_package_version_deb_found(self):
- remote = Mock()
- remote.os.package_type = "deb"
- proc = Mock()
- proc.exitstatus = 0
- proc.stdout.getvalue.return_value = "2.2"
- remote.run.return_value = proc
- result = packaging.get_package_version(remote, "apache2")
- assert result == "2.2"
-
- def test_get_package_version_deb_command(self):
- remote = Mock()
- remote.os.package_type = "deb"
- packaging.get_package_version(remote, "apache2")
- args, kwargs = remote.run.call_args
- expected_args = ['dpkg-query', '-W', '-f', '${Version}', 'apache2']
- assert expected_args == kwargs['args']
-
- def test_get_package_version_rpm_found(self):
- remote = Mock()
- remote.os.package_type = "rpm"
- proc = Mock()
- proc.exitstatus = 0
- proc.stdout.getvalue.return_value = "2.2"
- remote.run.return_value = proc
- result = packaging.get_package_version(remote, "httpd")
- assert result == "2.2"
-
- def test_get_package_version_rpm_command(self):
- remote = Mock()
- remote.os.package_type = "rpm"
- packaging.get_package_version(remote, "httpd")
- args, kwargs = remote.run.call_args
- expected_args = ['rpm', '-q', 'httpd', '--qf', '%{VERSION}-%{RELEASE}']
- assert expected_args == kwargs['args']
-
- def test_get_package_version_not_found(self):
- remote = Mock()
- remote.os.package_type = "rpm"
- proc = Mock()
- proc.exitstatus = 1
- proc.stdout.getvalue.return_value = "not installed"
- remote.run.return_value = proc
- result = packaging.get_package_version(remote, "httpd")
- assert result is None
-
- def test_get_package_version_invalid_version(self):
- # this tests the possibility that the package is not found
- # but the exitstatus is still 0. Not entirely sure we'll ever
- # hit this condition, but I want to test the codepath regardless
- remote = Mock()
- remote.os.package_type = "rpm"
- proc = Mock()
- proc.exitstatus = 0
- proc.stdout.getvalue.return_value = "not installed"
- remote.run.return_value = proc
- result = packaging.get_package_version(remote, "httpd")
- assert result is None
-
- @pytest.mark.parametrize("input, expected", KOJI_TASK_RPMS_MATRIX)
- def test_get_koji_task_result_package_name(self, input, expected):
- assert packaging._get_koji_task_result_package_name(input) == expected
-
- @patch("requests.get")
- def test_get_response_success(self, m_get):
- resp = Mock()
- resp.ok = True
- m_get.return_value = resp
- result = packaging._get_response("google.com")
- assert result == resp
-
- @patch("requests.get")
- def test_get_response_failed_wait(self, m_get):
- resp = Mock()
- resp.ok = False
- m_get.return_value = resp
- packaging._get_response("google.com", wait=True, sleep=1, tries=2)
- assert m_get.call_count == 2
-
- @patch("requests.get")
- def test_get_response_failed_no_wait(self, m_get):
- resp = Mock()
- resp.ok = False
- m_get.return_value = resp
- packaging._get_response("google.com", sleep=1, tries=2)
- assert m_get.call_count == 1
-
- @patch('teuthology.packaging.config')
- def test_get_builder_project_pulp(self, m_config):
- m_config.package_source = 'pulp'
- assert packaging.get_builder_project() is packaging.PulpProject
-
-
-class TestBuilderProject(object):
- klass = None
-
- def setup_method(self):
- if self.klass is None:
- pytest.skip()
-
- def _get_remote(self, arch="x86_64", system_type="deb", distro="ubuntu",
- codename="focal", version="20.04"):
- rem = Mock()
- rem.system_type = system_type
- rem.os.name = distro
- rem.os.codename = codename
- rem.os.version = version
- rem.arch = arch
-
- return rem
-
- def test_init_from_remote_base_url(self, expected=None):
- assert expected is not None
- rem = self._get_remote()
- ctx = dict(foo="bar")
- gp = self.klass("ceph", {}, ctx=ctx, remote=rem)
- result = gp.base_url
- assert result == expected
-
- def test_init_from_remote_base_url_debian(self, expected=None):
- assert expected is not None
- # remote.os.codename returns and empty string on debian
- rem = self._get_remote(distro="debian", codename='', version="7.1")
- ctx = dict(foo="bar")
- gp = self.klass("ceph", {}, ctx=ctx, remote=rem)
- result = gp.base_url
- assert result == expected
-
- def test_init_from_config_base_url(self, expected=None):
- assert expected is not None
- config = dict(
- os_type="ubuntu",
- os_version="20.04",
- sha1="sha1",
- )
- gp = self.klass("ceph", config)
- result = gp.base_url
- print(self.m_get.call_args_list)
- assert result == expected
-
- def test_init_from_config_branch_ref(self):
- config = dict(
- os_type="ubuntu",
- os_version="20.04",
- branch='jewel',
- )
- gp = self.klass("ceph", config)
- result = gp.uri_reference
- expected = 'ref/jewel'
- assert result == expected
-
- def test_init_from_config_tag_ref(self):
- config = dict(
- os_type="ubuntu",
- os_version="20.04",
- tag='v10.0.1',
- )
- gp = self.klass("ceph", config)
- result = gp.uri_reference
- expected = 'ref/v10.0.1'
- assert result == expected
-
- def test_init_from_config_tag_overrides_branch_ref(self, caplog):
- config = dict(
- os_type="ubuntu",
- os_version="20.04",
- branch='jewel',
- tag='v10.0.1',
- )
- gp = self.klass("ceph", config)
- result = gp.uri_reference
- expected = 'ref/v10.0.1'
- assert result == expected
- expected_log = 'More than one of ref, tag, branch, or sha1 supplied; using tag'
- assert expected_log in caplog.text
- return gp
-
- def test_init_from_config_branch_overrides_sha1(self, caplog):
- config = dict(
- os_type="ubuntu",
- os_version="20.04",
- branch='jewel',
- sha1='sha1',
- )
- gp = self.klass("ceph", config)
- result = gp.uri_reference
- expected = 'ref/jewel'
- assert result == expected
- expected_log = 'More than one of ref, tag, branch, or sha1 supplied; using branch'
- assert expected_log in caplog.text
- return gp
-
- REFERENCE_MATRIX = [
- ('the_ref', 'the_tag', 'the_branch', 'the_sha1', dict(ref='the_ref')),
- (None, 'the_tag', 'the_branch', 'the_sha1', dict(tag='the_tag')),
- (None, None, 'the_branch', 'the_sha1', dict(branch='the_branch')),
- (None, None, None, 'the_sha1', dict(sha1='the_sha1')),
- (None, None, 'the_branch', None, dict(branch='the_branch')),
- ]
-
- @pytest.mark.parametrize(
- "ref, tag, branch, sha1, expected",
- REFERENCE_MATRIX,
- )
- def test_choose_reference(self, ref, tag, branch, sha1, expected):
- config = dict(
- os_type='ubuntu',
- os_version='18.04',
- )
- if ref:
- config['ref'] = ref
- if tag:
- config['tag'] = tag
- if branch:
- config['branch'] = branch
- if sha1:
- config['sha1'] = sha1
- gp = self.klass("ceph", config)
- assert gp._choose_reference() == expected
-
- def test_get_package_version_found(self):
- rem = self._get_remote()
- ctx = dict(foo="bar")
- gp = self.klass("ceph", {}, ctx=ctx, remote=rem)
- assert gp.version == "0.90.0"
-
- @patch("teuthology.packaging._get_response")
- def test_get_package_version_not_found(self, m_get_response):
- rem = self._get_remote()
- ctx = dict(foo="bar")
- resp = Mock()
- resp.ok = False
- m_get_response.return_value = resp
- gp = self.klass("ceph", {}, ctx=ctx, remote=rem)
- with pytest.raises(VersionNotFoundError):
- gp.version
-
- def test_get_package_sha1_fetched_found(self):
- rem = self._get_remote()
- ctx = dict(foo="bar")
- gp = self.klass("ceph", {}, ctx=ctx, remote=rem)
- assert gp.sha1 == "the_sha1"
-
- def test_get_package_sha1_fetched_not_found(self):
- rem = self._get_remote()
- ctx = dict(foo="bar")
- gp = self.klass("ceph", {}, ctx=ctx, remote=rem)
- assert not gp.sha1
-
- DISTRO_MATRIX = [None] * 12
-
- @pytest.mark.parametrize(
- "matrix_index",
- range(len(DISTRO_MATRIX)),
- )
- def test_get_distro_remote(self, matrix_index):
- (distro, version, codename, expected) = \
- self.DISTRO_MATRIX[matrix_index]
- rem = self._get_remote(distro=distro, version=version,
- codename=codename)
- ctx = dict(foo="bar")
- gp = self.klass("ceph", {}, ctx=ctx, remote=rem)
- assert gp.distro == expected
-
- DISTRO_MATRIX_NOVER = [
- ('rhel', None, None, 'centos8'),
- ('centos', None, None, 'centos8'),
- ('fedora', None, None, 'fedora25'),
- ('ubuntu', None, None, 'focal'),
- ('debian', None, None, 'jessie'),
- ]
-
- @pytest.mark.parametrize(
- "matrix_index",
- range(len(DISTRO_MATRIX) + len(DISTRO_MATRIX_NOVER)),
- )
- def test_get_distro_config(self, matrix_index):
- (distro, version, codename, expected) = \
- (self.DISTRO_MATRIX + self.DISTRO_MATRIX_NOVER)[matrix_index]
- config = dict(
- os_type=distro,
- os_version=version
- )
- gp = self.klass("ceph", config)
- assert gp.distro == expected
-
- DIST_RELEASE_MATRIX = [
- ('rhel', '7.0', None, 'el7'),
- ('centos', '6.5', None, 'el6'),
- ('centos', '7.0', None, 'el7'),
- ('centos', '7.1', None, 'el7'),
- ('centos', '8.1', None, 'el8'),
- ('fedora', '20', None, 'fc20'),
- ('debian', '7.0', None, 'debian'),
- ('debian', '7', None, 'debian'),
- ('debian', '7.1', None, 'debian'),
- ('ubuntu', '12.04', None, 'ubuntu'),
- ('ubuntu', '14.04', None, 'ubuntu'),
- ('ubuntu', '16.04', None, 'ubuntu'),
- ('ubuntu', '18.04', None, 'ubuntu'),
- ('ubuntu', '20.04', None, 'ubuntu'),
- ]
-
- @pytest.mark.parametrize(
- "matrix_index",
- range(len(DIST_RELEASE_MATRIX)),
- )
- def test_get_dist_release(self, matrix_index):
- (distro, version, codename, expected) = \
- (self.DIST_RELEASE_MATRIX)[matrix_index]
- rem = self._get_remote(distro=distro, version=version,
- codename=codename)
- ctx = dict(foo="bar")
- gp = self.klass("ceph", {}, ctx=ctx, remote=rem)
- assert gp.dist_release == expected
-
-
-class TestShamanProject(TestBuilderProject):
- klass = packaging.ShamanProject
-
- def setup_method(self):
- self.p_config = patch('teuthology.packaging.config')
- self.m_config = self.p_config.start()
- self.m_config.package_source = 'shaman'
- self.m_config.shaman = {'endpoint': 'https://shaman.ceph.com/api/'}
- self.p_get_config_value = \
- patch('teuthology.packaging._get_config_value_for_remote')
- self.m_get_config_value = self.p_get_config_value.start()
- self.m_get_config_value.return_value = None
- self.p_get = patch('requests.get')
- self.m_get = self.p_get.start()
-
- def teardown_method(self):
- self.p_config.stop()
- self.p_get_config_value.stop()
- self.p_get.stop()
-
- def test_init_from_remote_base_url(self):
- # Here, we really just need to make sure ShamanProject._search()
- # queries the right URL. So let's make _get_base_url() just pass that
- # URL through and test that value.
- def m_get_base_url(obj):
- obj._search()
- return self.m_get.call_args_list[0][0][0]
- with patch(
- 'teuthology.packaging.ShamanProject._get_base_url',
- new=m_get_base_url,
- ):
- super(TestShamanProject, self)\
- .test_init_from_remote_base_url(
- "https://shaman.ceph.com/api/search?status=ready"
- "&project=ceph&flavor=default"
- "&distros=ubuntu%2F20.04%2Fx86_64&ref=main"
- )
-
- def test_init_from_remote_base_url_debian(self):
- # Here, we really just need to make sure ShamanProject._search()
- # queries the right URL. So let's make _get_base_url() just pass that
- # URL through and test that value.
- def m_get_base_url(obj):
- obj._search()
- return self.m_get.call_args_list[0][0][0]
- with patch(
- 'teuthology.packaging.ShamanProject._get_base_url',
- new=m_get_base_url,
- ):
- super(TestShamanProject, self)\
- .test_init_from_remote_base_url_debian(
- "https://shaman.ceph.com/api/search?status=ready"
- "&project=ceph&flavor=default"
- "&distros=debian%2F7.1%2Fx86_64&ref=main"
- )
-
- def test_init_from_config_base_url(self):
- # Here, we really just need to make sure ShamanProject._search()
- # queries the right URL. So let's make _get_base_url() just pass that
- # URL through and test that value.
- def m_get_base_url(obj):
- obj._search()
- return self.m_get.call_args_list[0][0][0]
- with patch(
- 'teuthology.packaging.ShamanProject._get_base_url',
- new=m_get_base_url,
- ):
- super(TestShamanProject, self).test_init_from_config_base_url(
- "https://shaman.ceph.com/api/search?status=ready&project=ceph" \
- "&flavor=default&distros=ubuntu%2F20.04%2Fx86_64&sha1=sha1"
- )
-
- @patch('teuthology.packaging.ShamanProject._get_package_sha1')
- def test_init_from_config_tag_ref(self, m_get_package_sha1):
- m_get_package_sha1.return_value = 'the_sha1'
- super(TestShamanProject, self).test_init_from_config_tag_ref()
-
- def test_init_from_config_tag_overrides_branch_ref(self, caplog):
- with patch(
- 'teuthology.packaging.repo_utils.ls_remote',
- ) as m_ls_remote:
- m_ls_remote.return_value = 'sha1_from_my_tag'
- obj = super(TestShamanProject, self)\
- .test_init_from_config_tag_overrides_branch_ref(caplog)
- search_uri = obj._search_uri
- assert 'sha1=sha1_from_my_tag' in search_uri
- assert 'jewel' not in search_uri
-
- def test_init_from_config_branch_overrides_sha1(self, caplog):
- obj = super(TestShamanProject, self)\
- .test_init_from_config_branch_overrides_sha1(caplog)
- search_uri = obj._search_uri
- assert 'jewel' in search_uri
- assert 'sha1' not in search_uri
-
- def test_get_package_version_found(self):
- resp = Mock()
- resp.ok = True
- resp.json.return_value = [
- dict(
- sha1='the_sha1',
- extra=dict(package_manager_version='0.90.0'),
- )
- ]
- self.m_get.return_value = resp
- super(TestShamanProject, self)\
- .test_get_package_version_found()
-
- def test_get_package_sha1_fetched_found(self):
- resp = Mock()
- resp.ok = True
- resp.json.return_value = [dict(sha1='the_sha1')]
- self.m_get.return_value = resp
- super(TestShamanProject, self)\
- .test_get_package_sha1_fetched_found()
-
- def test_get_package_sha1_fetched_not_found(self):
- resp = Mock()
- resp.json.return_value = []
- self.m_get.return_value = resp
- super(TestShamanProject, self)\
- .test_get_package_sha1_fetched_not_found()
-
- SHAMAN_SEARCH_RESPONSE = [
- {
- "status": "ready",
- "sha1": "534fc6d936bd506119f9e0921ff8cf8d47caa323",
- "extra": {
- "build_url": "https://jenkins.ceph.com/job/ceph-dev-build/ARCH=x86_64,AVAILABLE_ARCH=x86_64,AVAILABLE_DIST=centos8,DIST=centos8,MACHINE_SIZE=gigantic/48556/",
- "root_build_cause": "SCMTRIGGER",
- "version": "17.0.0-8856-g534fc6d9",
- "node_name": "172.21.2.7+braggi07",
- "job_name": "ceph-dev-build/ARCH=x86_64,AVAILABLE_ARCH=x86_64,AVAILABLE_DIST=centos8,DIST=centos8,MACHINE_SIZE=gigantic",
- "package_manager_version": "17.0.0-8856.g534fc6d9"
- },
- "url": "https://3.chacra.ceph.com/r/ceph/main/534fc6d936bd506119f9e0921ff8cf8d47caa323/centos/8/flavors/default/",
- "modified": "2021-11-06 21:40:40.669823",
- "distro_version": "8",
- "project": "ceph",
- "flavor": "default",
- "ref": "main",
- "chacra_url": "https://3.chacra.ceph.com/repos/ceph/main/534fc6d936bd506119f9e0921ff8cf8d47caa323/centos/8/flavors/default/",
- "archs": [
- "x86_64",
- "arm64",
- "source"
- ],
- "distro": "centos"
- }
- ]
-
- SHAMAN_BUILDS_RESPONSE = [
- {
- "status": "completed",
- "sha1": "534fc6d936bd506119f9e0921ff8cf8d47caa323",
- "distro_arch": "arm64",
- "started": "2021-11-06 20:20:15.121203",
- "completed": "2021-11-06 22:36:27.115950",
- "extra": {
- "node_name": "172.21.4.66+confusa04",
- "version": "17.0.0-8856-g534fc6d9",
- "build_user": "",
- "root_build_cause": "SCMTRIGGER",
-
- "job_name": "ceph-dev-build/ARCH=arm64,AVAILABLE_ARCH=arm64,AVAILABLE_DIST=centos8,DIST=centos8,MACHINE_SIZE=gigantic"
- },
- "modified": "2021-11-06 22:36:27.118043",
- "distro_version": "8",
- "project": "ceph",
- "url": "https://jenkins.ceph.com/job/ceph-dev-build/ARCH=arm64,AVAILABLE_ARCH=arm64,AVAILABLE_DIST=centos8,DIST=centos8,MACHINE_SIZE=gigantic/48556/",
- "log_url": "https://jenkins.ceph.com/job/ceph-dev-build/ARCH=arm64,AVAILABLE_ARCH=arm64,AVAILABLE_DIST=centos8,DIST=centos8,MACHINE_SIZE=gigantic/48556//consoleFull",
- "flavor": "default",
- "ref": "main",
- "distro": "centos"
- },
- {
- "status": "completed",
- "sha1": "534fc6d936bd506119f9e0921ff8cf8d47caa323",
- "distro_arch": "x86_64",
- "started": "2021-11-06 20:20:06.740692",
- "completed": "2021-11-06 21:43:51.711970",
- "extra": {
- "node_name": "172.21.2.7+braggi07",
- "version": "17.0.0-8856-g534fc6d9",
- "build_user": "",
- "root_build_cause": "SCMTRIGGER",
- "job_name": "ceph-dev-build/ARCH=x86_64,AVAILABLE_ARCH=x86_64,AVAILABLE_DIST=centos8,DIST=centos8,MACHINE_SIZE=gigantic"
- },
- "modified": "2021-11-06 21:43:51.713487",
- "distro_version": "8",
- "project": "ceph",
- "url": "https://jenkins.ceph.com/job/ceph-dev-build/ARCH=x86_64,AVAILABLE_ARCH=x86_64,AVAILABLE_DIST=centos8,DIST=centos8,MACHINE_SIZE=gigantic/48556/",
- "log_url": "https://jenkins.ceph.com/job/ceph-dev-build/ARCH=x86_64,AVAILABLE_ARCH=x86_64,AVAILABLE_DIST=centos8,DIST=centos8,MACHINE_SIZE=gigantic/48556//consoleFull",
- "flavor": "default",
- "ref": "main",
- "distro": "centos"
- }
- ]
-
- def test_build_complete_success(self):
- config = dict(
- os_type="centos",
- os_version="8",
- branch='main',
- arch='x86_64',
- flavor='default',
- )
- builder = self.klass("ceph", config)
-
- search_resp = Mock()
- search_resp.ok = True
- search_resp.json.return_value = self.SHAMAN_SEARCH_RESPONSE
- self.m_get.return_value = search_resp
- # cause builder to call requests.get and cache search_resp
- builder.assert_result()
-
- build_resp = Mock()
- build_resp.ok = True
- self.m_get.return_value = build_resp
-
- # both archs completed, so x86_64 build is complete
- builds = build_resp.json.return_value = self.SHAMAN_BUILDS_RESPONSE
- assert builder.build_complete
-
- # mark the arm64 build failed, x86_64 should still be complete
- builds[0]['status'] = "failed"
- build_resp.json.return_value = builds
- assert builder.build_complete
-
- # mark the x86_64 build failed, should show incomplete
- builds[1]['status'] = "failed"
- build_resp.json.return_value = builds
- assert not builder.build_complete
-
- # mark the arm64 build complete again, x86_64 still incomplete
- builds[0]['status'] = "completed"
- build_resp.json.return_value = builds
- assert not builder.build_complete
-
- DISTRO_MATRIX = [
- ('rhel', '7.0', None, 'centos/7'),
- ('alma', '9.7', None, 'alma/9'),
- ('rocky', '9.7', None, 'rocky/9'),
- ('rocky', '10.1', None, 'rocky/10'),
- ('centos', '6.5', None, 'centos/6'),
- ('centos', '7.0', None, 'centos/7'),
- ('centos', '7.1', None, 'centos/7'),
- ('centos', '8.1', None, 'centos/8'),
- ('fedora', '20', None, 'fedora/20'),
- ('ubuntu', '14.04', 'trusty', 'ubuntu/14.04'),
- ('ubuntu', '14.04', None, 'ubuntu/14.04'),
- ('debian', '7.0', None, 'debian/7.0'),
- ('debian', '7', None, 'debian/7'),
- ('debian', '7.1', None, 'debian/7.1'),
- ('ubuntu', '12.04', None, 'ubuntu/12.04'),
- ('ubuntu', '14.04', None, 'ubuntu/14.04'),
- ('ubuntu', '16.04', None, 'ubuntu/16.04'),
- ('ubuntu', '18.04', None, 'ubuntu/18.04'),
- ('ubuntu', '20.04', None, 'ubuntu/20.04'),
- ('opensuse', '15.6', None, 'opensuse/15.6'),
- ('sle', '15.5', None, 'sle/15.5'),
- ]
- @pytest.mark.parametrize(
- "matrix_index",
- range(len(DISTRO_MATRIX)),
- )
- def test_get_distro_remote(self, matrix_index):
- super().test_get_distro_remote(matrix_index)
-
- DISTRO_MATRIX_NOVER = [
- ('rhel', None, None, 'centos/8'),
- ('centos', None, None, 'centos/9'),
- ('fedora', None, None, 'fedora/25'),
- ('ubuntu', None, None, 'ubuntu/22.04'),
- ('alma', None, None, 'alma/9'),
- ('rocky', None, None, 'rocky/9'),
- ]
- @pytest.mark.parametrize(
- "matrix_index",
- range(len(DISTRO_MATRIX) + len(DISTRO_MATRIX_NOVER)),
- )
- def test_get_distro_config(self, matrix_index):
- super().test_get_distro_config(matrix_index)
-
-
-class TestPulpProject(TestBuilderProject):
- klass = packaging.PulpProject
-
- def setup_method(self):
- self.p_config = patch('teuthology.packaging.config')
- self.m_config = self.p_config.start()
- self.m_config.package_source = 'pulp'
- self.m_config.pulp = {
- 'endpoint': 'https://pulp.example.com/pulp/api/v3/',
- 'username': 'u',
- 'password': 'p',
- }
- self.p_get_config_value = \
- patch('teuthology.packaging._get_config_value_for_remote')
- self.m_get_config_value = self.p_get_config_value.start()
- self.m_get_config_value.return_value = None
- self.p_get = patch('requests.get')
- self.m_get = self.p_get.start()
-
- resp = Mock()
- resp.ok = True
- resp.json.return_value = {
- 'results': [{
- 'base_url': 'dist/ceph/main/sha/centos/8/flavors/default/',
- 'pulp_labels': {
- 'sha1': 'the_sha1',
- 'version': '0.90.0',
- },
- }],
- }
- self.m_get.return_value = resp
-
- def teardown_method(self):
- self.p_config.stop()
- self.p_get_config_value.stop()
- self.p_get.stop()
-
- def test_init_requires_pulp_credentials(self):
- self.m_config.pulp = {'username': '', 'password': 'p'}
- with pytest.raises(ValueError, match='Pulp username and password are required'):
- packaging.PulpProject('ceph', {}, ctx=dict(foo='bar'),
- remote=self._get_remote())
-
- self.m_config.pulp = {'username': 'u', 'password': ''}
- with pytest.raises(ValueError, match='Pulp username and password are required'):
- packaging.PulpProject('ceph', {}, ctx=dict(foo='bar'),
- remote=self._get_remote())
-
- def test_init_from_remote_base_url(self):
- def m_get_base_url(obj):
- obj._search()
- return self.m_get.call_args_list[0][0][0]
- with patch(
- 'teuthology.packaging.PulpProject._get_base_url',
- new=m_get_base_url,
- ):
- super(TestPulpProject, self).test_init_from_remote_base_url(
- 'https://pulp.example.com/pulp/api/v3/distributions/deb/apt',
- )
-
- def test_init_from_remote_base_url_debian(self):
- def m_get_base_url(obj):
- obj._search()
- return self.m_get.call_args_list[0][0][0]
- with patch(
- 'teuthology.packaging.PulpProject._get_base_url',
- new=m_get_base_url,
- ):
- super(TestPulpProject, self).test_init_from_remote_base_url_debian(
- 'https://pulp.example.com/pulp/api/v3/distributions/deb/apt',
- )
-
- def test_init_from_config_base_url(self):
- def m_get_base_url(obj):
- obj._search()
- return self.m_get.call_args_list[0][0][0]
- with patch(
- 'teuthology.packaging.PulpProject._get_base_url',
- new=m_get_base_url,
- ):
- super(TestPulpProject, self).test_init_from_config_base_url(
- 'https://pulp.example.com/pulp/api/v3/distributions/deb/apt',
- )
-
- @patch('teuthology.packaging.PulpProject._get_package_sha1')
- def test_init_from_config_tag_ref(self, m_get_package_sha1):
- m_get_package_sha1.return_value = 'the_sha1'
- super(TestPulpProject, self).test_init_from_config_tag_ref()
-
- def test_init_from_config_tag_overrides_branch_ref(self, caplog):
- obj = super(TestPulpProject, self)\
- .test_init_from_config_tag_overrides_branch_ref(caplog)
- obj._search()
- labels = self.m_get.call_args[1]['params']['pulp_label_select']
- assert 'tag=v10.0.1' in labels
- assert 'branch=jewel' not in labels
-
- def test_init_from_config_branch_overrides_sha1(self, caplog):
- obj = super(TestPulpProject, self)\
- .test_init_from_config_branch_overrides_sha1(caplog)
- obj._search()
- labels = self.m_get.call_args[1]['params']['pulp_label_select']
- assert 'branch=jewel' in labels
- assert 'sha1=sha1' not in labels
-
- def test_get_package_version_found(self):
- resp = Mock()
- resp.ok = True
- resp.json.return_value = {
- 'results': [{
- 'base_url': 'a/b/c/d',
- 'pulp_labels': {'version': '0.90.0'},
- }],
- }
- self.m_get.return_value = resp
- super(TestPulpProject, self).test_get_package_version_found()
-
- def test_get_package_version_not_found(self):
- rem = self._get_remote()
- ctx = dict(foo="bar")
- resp = Mock()
- resp.ok = False
- self.m_get.return_value = resp
- gp = self.klass("ceph", {}, ctx=ctx, remote=rem)
- with pytest.raises(VersionNotFoundError):
- gp.version
-
- def test_get_package_sha1_fetched_found(self):
- resp = Mock()
- resp.ok = True
- resp.json.return_value = {
- 'results': [{'base_url': 'a/b/c/d', 'pulp_labels': {'sha1': 'the_sha1'}}],
- }
- self.m_get.return_value = resp
- super(TestPulpProject, self).test_get_package_sha1_fetched_found()
-
- def test_get_package_sha1_fetched_not_found(self):
- resp = Mock()
- resp.ok = True
- resp.json.return_value = {
- 'results': [{'base_url': 'a/b/c/d', 'pulp_labels': {}}],
- }
- self.m_get.return_value = resp
- super(TestPulpProject, self).test_get_package_sha1_fetched_not_found()
-
- DISTRO_MATRIX = [
- ('rhel', '7.0', None, 'centos/7'),
- ('alma', '9.7', None, 'alma/9'),
- ('rocky', '10.1', None, 'rocky/10'),
- ('centos', '8.1', None, 'centos/8'),
- ('fedora', '20', None, 'fedora/20'),
- ('ubuntu', '20.04', 'focal', 'ubuntu/focal'),
- ]
-
- @pytest.mark.parametrize(
- "matrix_index",
- range(len(DISTRO_MATRIX)),
- )
- def test_get_distro_remote(self, matrix_index):
- super().test_get_distro_remote(matrix_index)
-
- DISTRO_MATRIX_NOVER = [
- ('rhel', None, None, 'centos/8'),
- ('centos', None, None, 'centos/9'),
- ('fedora', None, None, 'fedora/25'),
- ('ubuntu', None, None, 'ubuntu/jammy'),
- ('alma', None, None, 'alma/9'),
- ('rocky', None, None, 'rocky/9'),
- ]
-
- DISTRO_MATRIX_CONFIG = [
- ('rhel', '7.0', None, 'centos/7'),
- ('alma', '9.7', None, 'alma/9'),
- ('rocky', '10.1', None, 'rocky/10'),
- ('centos', '8.1', None, 'centos/8'),
- ('fedora', '20', None, 'fedora/20'),
- ('ubuntu', '20.04', None, 'ubuntu/focal'),
- ]
-
- @pytest.mark.parametrize(
- "matrix_index",
- range(len(DISTRO_MATRIX_CONFIG) + len(DISTRO_MATRIX_NOVER)),
- )
- def test_get_distro_config(self, matrix_index):
- (distro, version, _, expected) = (
- self.DISTRO_MATRIX_CONFIG + self.DISTRO_MATRIX_NOVER
- )[matrix_index]
- config = dict(
- os_type=distro,
- os_version=version
- )
- gp = self.klass("ceph", config)
- assert gp.distro == expected
-
- def test_get_rpm_repo_content_uses_pulp_base_url(self):
- pulp_base = (
- 'https://pulp.example.com/pulp/content/repos/ceph/tentacle/'
- 'abc123/rocky/10/flavors/default'
- )
- rem = self._get_remote()
- with patch(
- 'teuthology.packaging.PulpProject.base_url',
- new_callable=PropertyMock,
- return_value=pulp_base,
- ):
- gp = self.klass('ceph', {}, ctx=dict(foo='bar'), remote=rem)
- content = gp._get_rpm_repo_content()
- assert f'baseurl={pulp_base}/$basearch' in content
- assert f'baseurl={pulp_base}/noarch' in content
- assert f'baseurl={pulp_base}/SRPMS' in content
- assert 'chacra' not in content
-
- @patch('teuthology.packaging.sudo_write_file')
- def test_install_rpm_repo_writes_repo_file(self, m_sudo_write_file):
- pulp_base = (
- 'https://pulp.example.com/pulp/content/repos/ceph/tentacle/'
- 'abc123/rocky/10/flavors/default'
- )
- rem = self._get_remote()
- with patch(
- 'teuthology.packaging.PulpProject.base_url',
- new_callable=PropertyMock,
- return_value=pulp_base,
- ):
- gp = self.klass('ceph', {}, ctx=dict(foo='bar'), remote=rem)
- gp._install_rpm_repo()
- m_sudo_write_file.assert_called_once()
- assert m_sudo_write_file.call_args[0][1] == '/etc/yum.repos.d/ceph.repo'
- assert pulp_base in m_sudo_write_file.call_args[0][2]
-
- def test_get_deb_repo_content_uses_pulp_distribution_url(self):
- pulp_dist = (
- 'https://pulp.example.com/pulp/content/repos/ceph/tentacle/'
- 'ff309bba/ubuntu/jammy/flavors/default/x86_64/'
- )
- rem = self._get_remote(distro='ubuntu', version='22.04', codename='jammy')
- with patch(
- 'teuthology.packaging.PulpProject.repo_url',
- new_callable=PropertyMock,
- return_value=pulp_dist,
- ):
- gp = self.klass('ceph', {}, ctx=dict(foo='bar'), remote=rem)
- content = gp._get_deb_repo_content()
- assert content == (
- 'deb [trusted=yes] '
- 'https://pulp.example.com/pulp/content/repos/ceph/tentacle/'
- 'ff309bba/ubuntu/jammy/flavors/default/x86_64/ default all\n'
- )
-
- @patch('teuthology.packaging.sudo_write_file')
- def test_install_deb_repo_writes_sources_list(self, m_sudo_write_file):
- pulp_dist = (
- 'https://pulp.example.com/pulp/content/repos/ceph/tentacle/'
- 'ff309bba/ubuntu/jammy/flavors/default/x86_64/'
- )
- rem = self._get_remote(distro='ubuntu', version='22.04', codename='jammy')
- with patch(
- 'teuthology.packaging.PulpProject.repo_url',
- new_callable=PropertyMock,
- return_value=pulp_dist,
- ):
- gp = self.klass('ceph', {}, ctx=dict(foo='bar'), remote=rem)
- gp._install_deb_repo()
- m_sudo_write_file.assert_called_once()
- assert m_sudo_write_file.call_args[0][1] == '/etc/apt/sources.list.d/ceph.list'
- assert '[trusted=yes]' in m_sudo_write_file.call_args[0][2]
- assert '/x86_64/ default all' in m_sudo_write_file.call_args[0][2]
+++ /dev/null
-from teuthology.parallel import parallel
-
-
-def identity(item, input_set=None, remove=False):
- if input_set is not None:
- assert item in input_set
- if remove:
- input_set.remove(item)
- return item
-
-
-class TestParallel(object):
- def test_basic(self):
- in_set = set(range(10))
- with parallel() as para:
- for i in in_set:
- para.spawn(identity, i, in_set, remove=True)
- assert para.any_spawned is True
- assert para.count == len(in_set)
-
- def test_result(self):
- in_set = set(range(10))
- with parallel() as para:
- for i in in_set:
- para.spawn(identity, i, in_set)
- for result in para:
- in_set.remove(result)
-
+++ /dev/null
-import logging
-import unittest.mock as mock
-import os
-import os.path
-from pytest import raises, mark
-import re
-import shutil
-import subprocess
-import tempfile
-from packaging.version import parse
-
-from teuthology.exceptions import BranchNotFoundError, CommitNotFoundError
-from teuthology import repo_utils
-from teuthology import parallel
-repo_utils.log.setLevel(logging.WARNING)
-
-
-class TestRepoUtils(object):
-
- @classmethod
- def setup_class(cls):
- cls.temp_path = tempfile.mkdtemp(prefix='test_repo-')
- cls.dest_path = cls.temp_path + '/empty_dest'
- cls.src_path = cls.temp_path + '/empty_src'
-
- if 'TEST_ONLINE' in os.environ:
- cls.repo_url = 'https://github.com/ceph/empty.git'
- cls.commit = '71245d8e454a06a38a00bff09d8f19607c72e8bf'
- else:
- cls.repo_url = 'file://' + cls.src_path
- cls.commit = None
-
- cls.git_version = parse(cls.get_system_git_version())
-
- @classmethod
- def teardown_class(cls):
- shutil.rmtree(cls.temp_path)
-
- @classmethod
- def get_system_git_version(cls):
- # parsing following patterns
- # 1) git version 2.45.2
- # 2) git version 2.39.3 (Apple Git-146)
- git_version = subprocess.check_output(('git', 'version')).decode()
- m = re.match(r"git version (?P<ver>\d+.\d+.\d+) ?", git_version)
- return m['ver']
-
- def setup_method(self, method):
- # In git 2.28.0, the --initial-branch flag was added.
- if self.git_version >= parse("2.28.0"):
- subprocess.check_call(
- ('git', 'init', '--initial-branch', 'main', self.src_path)
- )
- else:
- subprocess.check_call(('git', 'init', self.src_path))
- subprocess.check_call(
- ('git', 'checkout', '-b', 'main'),
- cwd=self.src_path,
- )
- proc = subprocess.Popen(
- ('git', 'config', 'user.email', 'test@ceph.com'),
- cwd=self.src_path,
- stdout=subprocess.PIPE,
- )
- assert proc.wait() == 0
- proc = subprocess.Popen(
- ('git', 'config', 'user.name', 'Test User'),
- cwd=self.src_path,
- stdout=subprocess.PIPE,
- )
- assert proc.wait() == 0
- proc = subprocess.Popen(
- ('git', 'commit', '--allow-empty', '--allow-empty-message',
- '--no-edit'),
- cwd=self.src_path,
- stdout=subprocess.PIPE,
- )
- assert proc.wait() == 0
- if not self.commit:
- result = subprocess.check_output(
- 'git rev-parse HEAD',
- shell=True,
- cwd=self.src_path,
- ).split()
- assert result
- self.commit = result[0].decode()
-
- def teardown_method(self, method):
- shutil.rmtree(self.src_path, ignore_errors=True)
- shutil.rmtree(self.dest_path, ignore_errors=True)
-
- def test_clone_repo_existing_branch(self):
- repo_utils.clone_repo(self.repo_url, self.dest_path, 'main', self.commit)
- assert os.path.exists(self.dest_path)
-
- def test_clone_repo_non_existing_branch(self):
- with raises(BranchNotFoundError):
- repo_utils.clone_repo(self.repo_url, self.dest_path, 'nobranch', self.commit)
- assert not os.path.exists(self.dest_path)
-
- def test_fetch_no_repo(self):
- fake_dest_path = self.temp_path + '/not_a_repo'
- assert not os.path.exists(fake_dest_path)
- with raises(OSError):
- repo_utils.fetch(fake_dest_path)
- assert not os.path.exists(fake_dest_path)
-
- def test_fetch_noop(self):
- repo_utils.clone_repo(self.repo_url, self.dest_path, 'main', self.commit)
- repo_utils.fetch(self.dest_path)
- assert os.path.exists(self.dest_path)
-
- def test_fetch_branch_no_repo(self):
- fake_dest_path = self.temp_path + '/not_a_repo'
- assert not os.path.exists(fake_dest_path)
- with raises(OSError):
- repo_utils.fetch_branch(fake_dest_path, 'main')
- assert not os.path.exists(fake_dest_path)
-
- def test_fetch_branch_fake_branch(self):
- repo_utils.clone_repo(self.repo_url, self.dest_path, 'main', self.commit)
- with raises(BranchNotFoundError):
- repo_utils.fetch_branch(self.dest_path, 'nobranch')
-
- @mark.parametrize('git_str',
- ["fatal: couldn't find remote ref",
- "fatal: Couldn't find remote ref"])
- @mock.patch('subprocess.Popen')
- def test_fetch_branch_different_git_versions(self, mock_popen, git_str):
- """
- Newer git versions return a lower case string
- See: https://github.com/git/git/commit/0b9c3afdbfb629363
- """
- branch_name = 'nobranch'
- process_mock = mock.Mock()
- attrs = {
- 'wait.return_value': 1,
- 'stdout.read.return_value': f"{git_str} {branch_name}".encode(),
- }
- process_mock.configure_mock(**attrs)
- mock_popen.return_value = process_mock
- with raises(BranchNotFoundError):
- repo_utils.fetch_branch('', branch_name)
-
- def test_enforce_existing_branch(self):
- repo_utils.enforce_repo_state(self.repo_url, self.dest_path,
- 'main')
- assert os.path.exists(self.dest_path)
-
- def test_enforce_existing_commit(self):
- import logging
- logging.getLogger().info(subprocess.check_output("git branch", shell=True, cwd=self.src_path))
- repo_utils.enforce_repo_state(self.repo_url, self.dest_path,
- 'main', self.commit)
- assert os.path.exists(self.dest_path)
-
- def test_enforce_non_existing_branch(self):
- with raises(BranchNotFoundError):
- repo_utils.enforce_repo_state(self.repo_url, self.dest_path,
- 'blah', self.commit)
- assert not os.path.exists(self.dest_path)
-
- def test_enforce_non_existing_commit(self):
- with raises(CommitNotFoundError):
- repo_utils.enforce_repo_state(self.repo_url, self.dest_path,
- 'main', 'c69e90807d222c1719c45c8c758bf6fac3d985f1')
- assert not os.path.exists(self.dest_path)
-
- def test_enforce_multiple_calls_same_branch(self):
- repo_utils.enforce_repo_state(self.repo_url, self.dest_path,
- 'main', self.commit)
- assert os.path.exists(self.dest_path)
- repo_utils.enforce_repo_state(self.repo_url, self.dest_path,
- 'main', self.commit)
- assert os.path.exists(self.dest_path)
- repo_utils.enforce_repo_state(self.repo_url, self.dest_path,
- 'main', self.commit)
- assert os.path.exists(self.dest_path)
-
- def test_enforce_multiple_calls_different_branches(self):
- with raises(BranchNotFoundError):
- repo_utils.enforce_repo_state(self.repo_url, self.dest_path,
- 'blah1')
- assert not os.path.exists(self.dest_path)
- repo_utils.enforce_repo_state(self.repo_url, self.dest_path,
- 'main', self.commit)
- assert os.path.exists(self.dest_path)
- repo_utils.enforce_repo_state(self.repo_url, self.dest_path,
- 'main', self.commit)
- assert os.path.exists(self.dest_path)
- with raises(BranchNotFoundError):
- repo_utils.enforce_repo_state(self.repo_url, self.dest_path,
- 'blah2')
- assert not os.path.exists(self.dest_path)
- repo_utils.enforce_repo_state(self.repo_url, self.dest_path,
- 'main', self.commit)
- assert os.path.exists(self.dest_path)
-
- def test_enforce_invalid_branch(self):
- with raises(ValueError):
- repo_utils.enforce_repo_state(self.repo_url, self.dest_path, 'a b', self.commit)
-
- def test_simultaneous_access(self):
- count = 5
- with parallel.parallel() as p:
- for i in range(count):
- p.spawn(repo_utils.enforce_repo_state, self.repo_url,
- self.dest_path, 'main', self.commit)
- for result in p:
- assert result is None
-
- def test_simultaneous_access_different_branches(self):
- branches = [('main', self.commit), ('main', self.commit), ('nobranch', 'nocommit'),
- ('nobranch', 'nocommit'), ('main', self.commit), ('nobranch', 'nocommit')]
-
- with parallel.parallel() as p:
- for branch, commit in branches:
- if branch == 'main':
- p.spawn(repo_utils.enforce_repo_state, self.repo_url,
- self.dest_path, branch, commit)
- else:
- dest_path = self.dest_path + '_' + branch
-
- def func():
- repo_utils.enforce_repo_state(
- self.repo_url, dest_path,
- branch, commit)
- p.spawn(
- raises,
- BranchNotFoundError,
- func,
- )
- for result in p:
- pass
-
- URLS_AND_DIRNAMES = [
- ('git@git.ceph.com/ceph-qa-suite.git', 'git.ceph.com_ceph-qa-suite'),
- ('git://git.ceph.com/ceph-qa-suite.git', 'git.ceph.com_ceph-qa-suite'),
- ('https://github.com/ceph/ceph', 'github.com_ceph_ceph'),
- ('https://github.com/liewegas/ceph.git', 'github.com_liewegas_ceph'),
- ('file:///my/dir/has/ceph.git', 'my_dir_has_ceph'),
- ]
-
- @mark.parametrize("input_, expected", URLS_AND_DIRNAMES)
- def test_url_to_dirname(self, input_, expected):
- assert repo_utils.url_to_dirname(input_) == expected
-
- def test_current_branch(self):
- repo_utils.clone_repo(self.repo_url, self.dest_path, 'main', self.commit)
- assert repo_utils.current_branch(self.dest_path) == "main"
+++ /dev/null
-import json
-import pytest
-import yaml
-
-from teuthology.test import fake_archive
-from teuthology import report
-
-
-@pytest.fixture
-def archive(tmp_path):
- archive = fake_archive.FakeArchive(archive_base=str(tmp_path))
- yield archive
- archive.teardown()
-
-
-@pytest.fixture(autouse=True)
-def reporter(archive):
- archive.setup()
- return report.ResultsReporter(archive_base=archive.archive_base)
-
-
-def test_all_runs_one_run(archive, reporter):
- run_name = "test_all_runs"
- yaml_path = "examples/3node_ceph.yaml"
- job_count = 3
- archive.create_fake_run(run_name, job_count, yaml_path)
- assert [run_name] == reporter.serializer.all_runs
-
-
-def test_all_runs_three_runs(archive, reporter):
- run_count = 3
- runs = {}
- for i in range(run_count):
- run_name = "run #%s" % i
- yaml_path = "examples/3node_ceph.yaml"
- job_count = 3
- job_ids = archive.create_fake_run(
- run_name,
- job_count,
- yaml_path)
- runs[run_name] = job_ids
- assert sorted(runs.keys()) == sorted(reporter.serializer.all_runs)
-
-
-def test_jobs_for_run(archive, reporter):
- run_name = "test_jobs_for_run"
- yaml_path = "examples/3node_ceph.yaml"
- job_count = 3
- jobs = archive.create_fake_run(run_name, job_count, yaml_path)
- job_ids = [str(job['job_id']) for job in jobs]
-
- got_jobs = reporter.serializer.jobs_for_run(run_name)
- assert sorted(job_ids) == sorted(got_jobs.keys())
-
-
-def test_running_jobs_for_run(archive, reporter):
- run_name = "test_jobs_for_run"
- yaml_path = "examples/3node_ceph.yaml"
- job_count = 10
- num_hung = 3
- archive.create_fake_run(run_name, job_count, yaml_path,
- num_hung=num_hung)
-
- got_jobs = reporter.serializer.running_jobs_for_run(run_name)
- assert len(got_jobs) == num_hung
-
-
-def test_json_for_job(archive, reporter):
- run_name = "test_json_for_job"
- yaml_path = "examples/3node_ceph.yaml"
- job_count = 1
- jobs = archive.create_fake_run(run_name, job_count, yaml_path)
- job = jobs[0]
-
- with open(yaml_path) as yaml_file:
- obj_from_yaml = yaml.safe_load(yaml_file)
- full_obj = obj_from_yaml.copy()
- full_obj.update(job['info'])
- full_obj.update(job['summary'])
-
- out_json = reporter.serializer.json_for_job(
- run_name, str(job['job_id']))
- out_obj = json.loads(out_json)
- assert full_obj == out_obj
-
-
+++ /dev/null
-import textwrap
-from teuthology.config import config
-from teuthology import results
-from teuthology import report
-
-from unittest.mock import patch, DEFAULT
-
-
-class TestResultsEmail(object):
- reference = {
- 'run_name': 'test_name',
- 'jobs': [
- # Running
- {'description': 'description for job with name test_name',
- 'job_id': 30481,
- 'name': 'test_name',
- 'log_href': 'http://qa-proxy.ceph.com/teuthology/test_name/30481/teuthology.log', # noqa
- 'owner': 'job@owner',
- 'duration': None,
- 'status': 'running',
- },
- # Waiting
- {'description': 'description for job with name test_name',
- 'job_id': 62965,
- 'name': 'test_name',
- 'log_href': 'http://qa-proxy.ceph.com/teuthology/test_name/30481/teuthology.log', # noqa
- 'owner': 'job@owner',
- 'duration': None,
- 'status': 'waiting',
- },
- # Queued
- {'description': 'description for job with name test_name',
- 'job_id': 79063,
- 'name': 'test_name',
- 'log_href': 'http://qa-proxy.ceph.com/teuthology/test_name/30481/teuthology.log', # noqa
- 'owner': 'job@owner',
- 'duration': None,
- 'status': 'queued',
- },
- # Failed
- {'description': 'description for job with name test_name',
- 'job_id': 88979,
- 'name': 'test_name',
- 'log_href': 'http://qa-proxy.ceph.com/teuthology/test_name/88979/teuthology.log', # noqa
- 'owner': 'job@owner',
- 'duration': 35190,
- 'success': False,
- 'status': 'fail',
- 'failure_reason': 'Failure reason!',
- },
- # Dead
- {'description': 'description for job with name test_name',
- 'job_id': 69152,
- 'name': 'test_name',
- 'log_href': 'http://qa-proxy.ceph.com/teuthology/test_name/69152/teuthology.log', # noqa
- 'owner': 'job@owner',
- 'duration': 5225,
- 'success': False,
- 'status': 'dead',
- 'failure_reason': 'Dead reason!',
- },
- # Passed
- {'description': 'description for job with name test_name',
- 'job_id': 68369,
- 'name': 'test_name',
- 'log_href': 'http://qa-proxy.ceph.com/teuthology/test_name/68369/teuthology.log', # noqa
- 'owner': 'job@owner',
- 'duration': 33771,
- 'success': True,
- 'status': 'pass',
- },
- ],
- 'subject': '1 fail, 1 dead, 1 running, 1 waiting, 1 queued, 1 pass in test_name', # noqa
- 'body': textwrap.dedent("""
- Test Run: test_name
- =================================================================
- info: http://example.com/test_name/
- logs: http://qa-proxy.ceph.com/teuthology/test_name/
- failed: 1
- dead: 1
- running: 1
- waiting: 1
- queued: 1
- passed: 1
-
-
- Fail
- =================================================================
- [88979] description for job with name test_name
- -----------------------------------------------------------------
- time: 09:46:30
- info: http://example.com/test_name/88979/
- log: http://qa-proxy.ceph.com/teuthology/test_name/88979/
-
- Failure reason!
-
-
-
- Dead
- =================================================================
- [69152] description for job with name test_name
- -----------------------------------------------------------------
- time: 01:27:05
- info: http://example.com/test_name/69152/
- log: http://qa-proxy.ceph.com/teuthology/test_name/69152/
-
- Dead reason!
-
-
-
- Running
- =================================================================
- [30481] description for job with name test_name
- info: http://example.com/test_name/30481/
-
-
-
- Waiting
- =================================================================
- [62965] description for job with name test_name
- info: http://example.com/test_name/62965/
-
-
-
- Queued
- =================================================================
- [79063] description for job with name test_name
- info: http://example.com/test_name/79063/
-
-
-
- Pass
- =================================================================
- [68369] description for job with name test_name
- time: 09:22:51
- info: http://example.com/test_name/68369/
- """).strip(),
- }
-
- def setup_method(self):
- config.results_ui_server = "http://example.com/"
- config.archive_server = "http://qa-proxy.ceph.com/teuthology/"
-
- def test_build_email_body(self):
- run_name = self.reference['run_name']
- with patch.multiple(
- report,
- ResultsReporter=DEFAULT,
- ):
- reporter = report.ResultsReporter()
- reporter.get_jobs.return_value = self.reference['jobs']
- (subject, body) = results.build_email_body(
- run_name, _reporter=reporter)
- assert subject == self.reference['subject']
- assert body == self.reference['body']
+++ /dev/null
-import os
-import pytest
-import docopt
-
-from unittest.mock import patch, call
-
-from teuthology import run
-from scripts import run as scripts_run
-from teuthology.test import skipif_teuthology_process
-
-
-class TestRun(object):
- """ Tests for teuthology.run """
-
- @patch("teuthology.log.setLevel")
- @patch("teuthology.setup_log_file")
- @patch("os.mkdir")
- def test_set_up_logging(self, m_mkdir, m_setup_log_file, m_setLevel):
- run.set_up_logging(True, "path/to/archive")
- m_mkdir.assert_called_with("path/to/archive")
- m_setup_log_file.assert_called_with("path/to/archive/teuthology.log")
- assert m_setLevel.called
-
- # because of how we import things, mock merge_configs from run - where it's used
- # see: http://www.voidspace.org.uk/python/mock/patch.html#where-to-patch
- @patch("teuthology.run.merge_configs")
- def test_setup_config(self, m_merge_configs):
- config = {"job_id": 1, "foo": "bar"}
- m_merge_configs.return_value = config
- result = run.setup_config(["some/config.yaml"])
- assert m_merge_configs.called
- assert result["job_id"] == "1"
- assert result["foo"] == "bar"
-
- @patch("teuthology.run.merge_configs")
- def test_setup_config_targets_ok(self, m_merge_configs):
- config = {"targets": list(range(4)), "roles": list(range(2))}
- m_merge_configs.return_value = config
- result = run.setup_config(["some/config.yaml"])
- assert result["targets"] == [0, 1, 2, 3]
- assert result["roles"] == [0, 1]
-
- @patch("teuthology.run.merge_configs")
- def test_setup_config_targets_invalid(self, m_merge_configs):
- config = {"targets": range(2), "roles": range(4)}
- m_merge_configs.return_value = config
- with pytest.raises(AssertionError):
- run.setup_config(["some/config.yaml"])
-
- @patch("teuthology.run.open")
- def test_write_initial_metadata(self, m_open):
- config = {"job_id": "123", "foo": "bar"}
- run.write_initial_metadata(
- "some/archive/dir",
- config,
- "the_name",
- "the_description",
- "the_owner",
- )
- expected = [
- call('some/archive/dir/pid', 'w'),
- call('some/archive/dir/owner', 'w'),
- call('some/archive/dir/orig.config.yaml', 'w'),
- call('some/archive/dir/info.yaml', 'w')
- ]
- assert m_open.call_args_list == expected
-
- def test_get_machine_type(self):
- result = run.get_machine_type(None, {"machine-type": "the_machine_type"})
- assert result == "the_machine_type"
-
- def test_get_summary(self):
- result = run.get_summary("the_owner", "the_description")
- assert result == {"owner": "the_owner", "description": "the_description", "success": True}
- result = run.get_summary("the_owner", None)
- assert result == {"owner": "the_owner", "success": True}
-
- def test_validate_tasks_invalid(self):
- config = {"tasks": [{"kernel": "can't be here"}]}
- with pytest.raises(AssertionError) as excinfo:
- run.validate_tasks(config)
- assert excinfo.value.args[0].startswith("kernel installation")
-
- def test_validate_task_no_tasks(self):
- result = run.validate_tasks({})
- assert result == []
-
- def test_validate_tasks_valid(self):
- expected = [{"foo": "bar"}, {"bar": "foo"}]
- result = run.validate_tasks({"tasks": expected})
- assert result == expected
-
- def test_validate_tasks_is_list(self):
- with pytest.raises(AssertionError) as excinfo:
- run.validate_tasks({"tasks": {"foo": "bar"}})
- assert excinfo.value.args[0].startswith("Expected list")
-
- def test_get_initial_tasks_invalid(self):
- with pytest.raises(AssertionError) as excinfo:
- run.get_initial_tasks(True, {"targets": "can't be here",
- "roles": "roles" }, "machine_type")
- assert excinfo.value.args[0].startswith("You cannot")
-
- def test_get_inital_tasks(self):
- config = {"roles": range(2), "kernel": "the_kernel", "use_existing_cluster": False}
- result = run.get_initial_tasks(True, config, "machine_type")
- assert {"internal.lock_machines": (2, "machine_type")} in result
- assert {"kernel": "the_kernel"} in result
- # added because use_existing_cluster == False
- assert {'internal.vm_setup': None} in result
- assert {'internal.buildpackages_prep': None} in result
-
- # When tests are run in a teuthology process using the py.test
- # API, tasks will have already been imported. Patching sys.path
- # (and even calling sys.path_importer_cache.clear()) doesn't seem
- # to help "forget" where the tasks are, keeping this test from
- # passing. The test isn't critical to run in every single
- # environment, so skip.
- @skipif_teuthology_process
- @patch("teuthology.run.fetch_qa_suite")
- def test_fetch_tasks_if_needed(self, m_fetch_qa_suite):
- config = {"suite_path": "/some/suite/path", "suite_branch": "feature_branch",
- "suite_sha1": "commit"}
- m_fetch_qa_suite.return_value = "/some/other/suite/path"
- result = run.fetch_tasks_if_needed(config)
- m_fetch_qa_suite.assert_called_with("feature_branch", commit="commit")
- assert result == "/some/other/suite/path/qa"
-
- @patch("teuthology.run.get_status")
- @patch("yaml.safe_dump")
- @patch("teuthology.report.try_push_job_info")
- @patch("teuthology.run.email_results")
- @patch("teuthology.run.open")
- @patch("sys.exit")
- def test_report_outcome(self, m_sys_exit, m_open, m_email_results, m_try_push_job_info, m_safe_dump, m_get_status):
- m_get_status.return_value = "fail"
- summary = {"failure_reason": "reasons"}
- summary_dump = "failure_reason: reasons\n"
- config = {"email-on-error": True}
- config_dump = "email-on-error: true\n"
- m_safe_dump.side_effect = [None, summary_dump, config_dump]
- run.report_outcome(config, "the/archive/path", summary)
- m_try_push_job_info.assert_called_with(config, summary)
- m_open.assert_called_with("the/archive/path/summary.yaml", "w")
- assert m_email_results.called
- assert m_open.called
- assert m_sys_exit.called
-
- @patch("teuthology.run.set_up_logging")
- @patch("teuthology.run.setup_config")
- @patch("teuthology.run.get_user")
- @patch("teuthology.run.write_initial_metadata")
- @patch("teuthology.report.try_push_job_info")
- @patch("teuthology.run.get_machine_type")
- @patch("teuthology.run.get_summary")
- @patch("yaml.safe_dump")
- @patch("teuthology.run.validate_tasks")
- @patch("teuthology.run.get_initial_tasks")
- @patch("teuthology.run.fetch_tasks_if_needed")
- @patch("teuthology.run.run_tasks")
- @patch("teuthology.run.report_outcome")
- def test_main(self, m_report_outcome, m_run_tasks, m_fetch_tasks_if_needed, m_get_initial_tasks, m_validate_tasks,
- m_safe_dump, m_get_summary, m_get_machine_type, m_try_push_job_info, m_write_initial_metadata,
- m_get_user, m_setup_config, m_set_up_logging):
- """ This really should be an integration test of some sort. """
- config = {"job_id": 1}
- m_setup_config.return_value = config
- m_get_machine_type.return_value = "machine_type"
- doc = scripts_run.__doc__
- args = docopt.docopt(doc, [
- "--verbose",
- "--archive", "some/archive/dir",
- "--description", "the_description",
- "--lock",
- "--os-type", "os_type",
- "--os-version", "os_version",
- "--block",
- "--name", "the_name",
- "--suite-path", "some/suite/dir",
- "path/to/config.yml",
- ])
- m_get_user.return_value = "the_owner"
- m_get_summary.return_value = dict(success=True, owner="the_owner", description="the_description")
- m_validate_tasks.return_value = ['task3']
- m_get_initial_tasks.return_value = ['task1', 'task2']
- m_fetch_tasks_if_needed.return_value = "some/suite/dir"
- run.main(args)
- m_set_up_logging.assert_called_with(True, "some/archive/dir")
- m_setup_config.assert_called_with(["path/to/config.yml"])
- m_write_initial_metadata.assert_called_with(
- "some/archive/dir",
- config,
- "the_name",
- "the_description",
- "the_owner"
- )
- m_try_push_job_info.assert_called_with(config, dict(status='running', pid=os.getpid()))
- m_get_machine_type.assert_called_with(None, config)
- m_get_summary.assert_called_with("the_owner", "the_description")
- m_get_initial_tasks.assert_called_with(True, config, "machine_type")
- m_fetch_tasks_if_needed.assert_called_with(config)
- assert m_report_outcome.called
- args, kwargs = m_run_tasks.call_args
- fake_ctx = kwargs["ctx"]._conf
- # fields that must be in ctx for the tasks to behave
- expected_ctx = ["verbose", "archive", "description", "owner", "lock", "machine_type", "os_type", "os_version",
- "block", "name", "suite_path", "config", "summary"]
- for key in expected_ctx:
- assert key in fake_ctx
- assert isinstance(fake_ctx["config"], dict)
- assert isinstance(fake_ctx["summary"], dict)
- assert "tasks" in fake_ctx["config"]
- # ensures that values missing in args are added with the correct value
- assert fake_ctx["owner"] == "the_owner"
- assert fake_ctx["machine_type"] == "machine_type"
- # ensures os_type and os_version are property overwritten
- assert fake_ctx["config"]["os_type"] == "os_type"
- assert fake_ctx["config"]["os_version"] == "os_version"
-
- @patch("teuthology.run.set_up_logging")
- @patch("teuthology.run.setup_config")
- @patch("teuthology.run.get_user")
- @patch("teuthology.run.write_initial_metadata")
- @patch("teuthology.report.try_push_job_info")
- @patch("teuthology.run.get_machine_type")
- @patch("teuthology.run.get_summary")
- @patch("yaml.safe_dump")
- @patch("teuthology.run.validate_tasks")
- @patch("teuthology.run.get_initial_tasks")
- @patch("teuthology.run.fetch_tasks_if_needed")
- @patch("teuthology.run.run_tasks")
- @patch("teuthology.run.report_outcome")
- def test_main_interactive(
- self,
- m_report_outcome,
- m_run_tasks,
- m_fetch_tasks_if_needed,
- m_get_initial_tasks,
- m_validate_tasks,
- m_safe_dump,
- m_get_summary,
- m_get_machine_type,
- m_try_push_job_info,
- m_write_initial_metadata,
- m_get_user,
- m_setup_config,
- m_set_up_logging,
- ):
- config = {"job_id": 1}
- m_setup_config.return_value = config
- m_get_machine_type.return_value = "machine_type"
- doc = scripts_run.__doc__
- args = docopt.docopt(doc, [
- "--interactive-on-error",
- "path/to/config.yml",
- ])
- run.main(args)
- args, kwargs = m_run_tasks.call_args
- fake_ctx = kwargs["ctx"]._conf
- assert fake_ctx['interactive_on_error'] is True
-
- def test_get_teuthology_command(self):
- doc = scripts_run.__doc__
- args = docopt.docopt(doc, [
- "--archive", "some/archive/dir",
- "--description", "the_description",
- "--lock",
- "--block",
- "--name", "the_name",
- "--suite-path", "some/suite/dir",
- "path/to/config.yml", "path/to/config2.yaml",
- ])
- result = run.get_teuthology_command(args)
- result = result.split()
- expected = [
- "teuthology",
- "path/to/config.yml", "path/to/config2.yaml",
- "--suite-path", "some/suite/dir",
- "--lock",
- "--description", "the_description",
- "--name", "the_name",
- "--block",
- "--archive", "some/archive/dir",
- ]
- assert len(result) == len(expected)
- for arg in expected:
- assert arg in result
+++ /dev/null
-from teuthology import safepath
-
-class TestSafepath(object):
- def test_simple(self):
- got = safepath.munge('foo')
- assert got == 'foo'
-
- def test_empty(self):
- # really odd corner case
- got = safepath.munge('')
- assert got == '_'
-
- def test_slash(self):
- got = safepath.munge('/')
- assert got == '_'
-
- def test_slashslash(self):
- got = safepath.munge('//')
- assert got == '_'
-
- def test_absolute(self):
- got = safepath.munge('/evil')
- assert got == 'evil'
-
- def test_absolute_subdir(self):
- got = safepath.munge('/evil/here')
- assert got == 'evil/here'
-
- def test_dot_leading(self):
- got = safepath.munge('./foo')
- assert got == 'foo'
-
- def test_dot_middle(self):
- got = safepath.munge('evil/./foo')
- assert got == 'evil/foo'
-
- def test_dot_trailing(self):
- got = safepath.munge('evil/foo/.')
- assert got == 'evil/foo'
-
- def test_dotdot(self):
- got = safepath.munge('../evil/foo')
- assert got == '_./evil/foo'
-
- def test_dotdot_subdir(self):
- got = safepath.munge('evil/../foo')
- assert got == 'evil/_./foo'
-
- def test_hidden(self):
- got = safepath.munge('.evil')
- assert got == '_evil'
-
- def test_hidden_subdir(self):
- got = safepath.munge('foo/.evil')
- assert got == 'foo/_evil'
+++ /dev/null
-from teuthology.schedule import build_config
-from teuthology.misc import get_user
-
-
-class TestSchedule(object):
- basic_args = {
- '--verbose': False,
- '--owner': 'OWNER',
- '--description': 'DESC',
- '--email': 'EMAIL',
- '--first-in-suite': False,
- '--last-in-suite': True,
- '--name': 'NAME',
- '--worker': 'tala',
- '--timeout': '6',
- '--priority': '99',
- # TODO: make this work regardless of $PWD
- #'<conf_file>': ['../../examples/3node_ceph.yaml',
- # '../../examples/3node_rgw.yaml'],
- }
-
- def test_basic(self):
- expected = {
- 'description': 'DESC',
- 'email': 'EMAIL',
- 'first_in_suite': False,
- 'last_in_suite': True,
- 'machine_type': 'tala',
- 'name': 'NAME',
- 'owner': 'OWNER',
- 'priority': 99,
- 'results_timeout': '6',
- 'verbose': False,
- 'tube': 'tala',
- }
-
- job_dict = build_config(self.basic_args)
- assert job_dict == expected
-
- def test_owner(self):
- args = self.basic_args
- args['--owner'] = None
- job_dict = build_config(self.basic_args)
- assert job_dict['owner'] == 'scheduled_%s' % get_user()
-
+++ /dev/null
-from __future__ import with_statement
-
-import glob
-import gzip
-import os
-import shutil
-import tempfile
-import yaml
-from teuthology import scrape
-
-class FakeResultDir(object):
- """Mocks a Result Directory"""
-
- def __init__(self,
- failure_reason="Dummy reason",
- assertion="FAILED assert 1 == 2\n",
- blank_backtrace=False,
- assertion_osd=False,
- ):
- self.failure_reason = failure_reason
- self.assertion = assertion
- self.blank_backtrace = blank_backtrace
- self.path = tempfile.mkdtemp()
-
- with open(os.path.join(self.path, "config.yaml"), "w") as f:
- yaml.dump({"description": "Dummy test"}, f)
-
- with open(os.path.join(self.path, "summary.yaml"), "w") as f:
- yaml.dump({
- "success": "false",
- "failure_reason": self.failure_reason
- }, f)
-
- with open(os.path.join(self.path, "teuthology.log"), "w") as f:
- if not self.blank_backtrace:
- f.write(" ceph version 1000\n")
- f.write(".stderr: Dummy error\n")
- f.write(self.assertion)
- f.write(" NOTE: a copy of the executable dummy text\n")
-
- if assertion_osd:
- host = "host1"
- rem_log_dir = os.path.join(self.path, "remote", host, "log")
- os.makedirs(rem_log_dir, exist_ok=True)
- ceph_mon_log = os.path.join(rem_log_dir, "ceph-osd.0.log")
- with open(ceph_mon_log, "w") as f:
- f.write("ceph version 1000\n")
- f.write(self.assertion)
-
- def __enter__(self):
- return self
-
- def __exit__(self, exc_typ, exc_val, exc_tb):
- shutil.rmtree(self.path)
-
-class TestScrape(object):
- """Tests for teuthology.scrape"""
-
- def test_grep(self):
- with FakeResultDir() as d:
- filepath = os.path.join(d.path, "scrapetest.txt")
- with open(filepath, 'w') as f:
- f.write("Ceph is an open-source software storage platform\n\
- Teuthology is used for testing.")
-
- #System level grep is called
- value1 = scrape.grep(filepath, "software")
- value2 = scrape.grep(filepath, "device")
-
- assert value1 ==\
- ['Ceph is an open-source software storage platform', '']
- assert value2 == []
-
- def test_job(self):
- with FakeResultDir() as d:
- job = scrape.Job(d.path, 1)
- assert job.get_success() == "false"
- assert job.get_assertion() == "FAILED assert 1 == 2"
- assert job.get_last_tlog_line() ==\
- b"NOTE: a copy of the executable dummy text"
- assert job.get_failure_reason() == "Dummy reason"
-
- def test_timeoutreason(self):
- with FakeResultDir(failure_reason=\
- "status 124: timeout '123 /home/ubuntu/cephtest/workunit.client.0/cephtool/test.sh'") as d:
- job = scrape.Job(d.path, 1)
- assert scrape.TimeoutReason.could_be(job)
- assert scrape.TimeoutReason(job).match(job)
-
- def test_deadreason(self):
- with FakeResultDir() as d:
- job = scrape.Job(d.path, 1)
- #Summary is present
- #So this cannot be a DeadReason
- assert not scrape.DeadReason.could_be(job)
-
- def test_lockdepreason(self):
- lkReason = None
- with FakeResultDir(assertion=\
- "FAILED assert common/lockdep reason\n") as d:
- job = scrape.Job(d.path, 1)
- assert scrape.LockdepReason.could_be(job)
-
- lkReason = scrape.LockdepReason(job)
- #Backtraces of same jobs must match 100%
- assert lkReason.match(job)
- with FakeResultDir(blank_backtrace=True) as d:
- #Corresponding to 0% match
- assert not lkReason.match(scrape.Job(d.path, 2))
-
- def test_assertionreason(self):
- with FakeResultDir() as d:
- job = scrape.Job(d.path, 1)
- assert scrape.AssertionReason.could_be(job)
-
- def test_genericreason(self):
- d1 = FakeResultDir(blank_backtrace=True)
- d2 = FakeResultDir(failure_reason="Dummy dummy")
- d3 = FakeResultDir()
-
- job1 = scrape.Job(d1.path, 1)
- job2 = scrape.Job(d2.path, 2)
- job3 = scrape.Job(d3.path, 3)
-
- reason = scrape.GenericReason(job3)
-
- assert reason.match(job2)
- assert not reason.match(job1)
-
- shutil.rmtree(d1.path)
- shutil.rmtree(d2.path)
- shutil.rmtree(d3.path)
-
- def test_valgrindreason(self):
- vreason = None
- with FakeResultDir(
- failure_reason="saw valgrind issues",
- assertion="2014-08-22T20:07:18.668 ERROR:tasks.ceph:saw valgrind issue <kind>Leak_DefinitelyLost</kind> in /var/log/ceph/valgrind/osd.3.log.gz\n"
- ) as d:
- job = scrape.Job(d.path, 1)
- assert scrape.ValgrindReason.could_be(job)
-
- vreason = scrape.ValgrindReason(job)
- assert vreason.match(job)
-
- def test_give_me_a_reason(self):
- with FakeResultDir() as d:
- job = scrape.Job(d.path, 1)
-
- assert type(scrape.give_me_a_reason(job)) == scrape.AssertionReason
-
- #Test the lockdep ordering
- with FakeResultDir(assertion=\
- "FAILED assert common/lockdep reason\n") as d:
- job = scrape.Job(d.path, 1)
- assert type(scrape.give_me_a_reason(job)) == scrape.LockdepReason
-
- def test_scraper(self):
- d = FakeResultDir()
- os.mkdir(os.path.join(d.path, "test"))
- shutil.move(
- os.path.join(d.path, "config.yaml"),
- os.path.join(d.path, "test", "config.yaml")
- )
- shutil.move(
- os.path.join(d.path, "summary.yaml"),
- os.path.join(d.path, "test", "summary.yaml")
- )
- shutil.move(
- os.path.join(d.path, "teuthology.log"),
- os.path.join(d.path, "test", "teuthology.log")
- )
-
- scrape.Scraper(d.path).analyze()
-
- #scrape.log should be created
- assert os.path.exists(os.path.join(d.path, "scrape.log"))
-
- shutil.rmtree(d.path)
-
- def test_gzip_backtrace_decode(self):
- with FakeResultDir(assertion="FAILED assert dummy backtrace line",
- blank_backtrace=True,
- assertion_osd=True) as d:
-
- with open(os.path.join(d.path, "teuthology.log"), "a") as root_log:
- root_log.write(
- "command crashed with signal SIGSEGV tasks.ceph.osd.0.host1.stderr\n"
- )
-
- pattern = os.path.join(d.path, "**", "ceph-osd.0.log")
- raws = glob.glob(pattern, recursive=True)
- assert len(raws) == 1, f"expected one raw log, found: {raws}"
- raw_log = raws[0]
- gz_log = raw_log + ".gz"
-
- with gzip.open(gz_log, "wb") as out:
- out.write(open(raw_log, "rb").read())
- os.remove(raw_log)
-
- assert not os.path.exists(raw_log)
- assert os.path.exists(gz_log)
-
- job = scrape.Job(d.path, 1)
- assert job.get_assertion() == "FAILED assert dummy backtrace line"
\ No newline at end of file
+++ /dev/null
-from teuthology import timer
-
-from unittest.mock import MagicMock, patch, mock_open
-from time import time
-
-
-class TestTimer(object):
- def test_data_empty(self):
- self.timer = timer.Timer()
- assert self.timer.data == dict()
-
- def test_data_one_mark(self):
- self.timer = timer.Timer()
- # Avoid failing if ~1ms elapses between these two calls
- self.timer.precision = 2
- self.timer.mark()
- assert len(self.timer.data['marks']) == 1
- assert self.timer.data['marks'][0]['interval'] == 0
- assert self.timer.data['marks'][0]['message'] == ''
-
- def test_data_five_marks(self):
- self.timer = timer.Timer()
- for i in range(5):
- self.timer.mark(str(i))
- assert len(self.timer.data['marks']) == 5
- assert [m['message'] for m in self.timer.data['marks']] == \
- ['0', '1', '2', '3', '4']
-
- def test_intervals(self):
- fake_time = MagicMock()
- with patch('teuthology.timer.time.time', fake_time):
- self.timer = timer.Timer()
- now = start_time = fake_time.return_value = time()
- intervals = [0, 1, 1, 2, 3, 5, 8]
- for i in intervals:
- now += i
- fake_time.return_value = now
- self.timer.mark(str(i))
-
- summed_intervals = [sum(intervals[:x + 1]) for x in range(len(intervals))]
- result_intervals = [m['interval'] for m in self.timer.data['marks']]
- assert result_intervals == summed_intervals
- assert self.timer.data['start'] == \
- self.timer.get_datetime_string(start_time)
- assert self.timer.data['end'] == \
- self.timer.get_datetime_string(start_time + summed_intervals[-1])
- assert [m['message'] for m in self.timer.data['marks']] == \
- [str(i) for i in intervals]
- assert self.timer.data['elapsed'] == summed_intervals[-1]
-
- def test_write(self):
- _path = '/path'
- _safe_dump = MagicMock(name='safe_dump')
- with patch('teuthology.timer.yaml.safe_dump', _safe_dump):
- with patch('teuthology.timer.open', mock_open(), create=True) as _open:
- self.timer = timer.Timer(path=_path)
- assert self.timer.path == _path
- self.timer.write()
- _open.assert_called_once_with(_path, 'w')
- _safe_dump.assert_called_once_with(
- dict(),
- _open.return_value.__enter__.return_value,
- default_flow_style=False,
- )
-
- def test_sync(self):
- _path = '/path'
- _safe_dump = MagicMock(name='safe_dump')
- with patch('teuthology.timer.yaml.safe_dump', _safe_dump):
- with patch('teuthology.timer.open', mock_open(), create=True) as _open:
- self.timer = timer.Timer(path=_path, sync=True)
- assert self.timer.path == _path
- assert self.timer.sync is True
- self.timer.mark()
- _open.assert_called_once_with(_path, 'w')
- _safe_dump.assert_called_once_with(
- self.timer.data,
- _open.return_value.__enter__.return_value,
- default_flow_style=False,
- )
+++ /dev/null
-from unittest.mock import patch, Mock
-
-import teuthology.lock.util
-from teuthology import provision
-
-
-class TestVpsOsVersionParamCheck(object):
-
- def setup_method(self):
- self.fake_ctx = Mock()
- self.fake_ctx.machine_type = 'vps'
- self.fake_ctx.num_to_lock = 1
- self.fake_ctx.lock = False
-
- def fake_downburst_executable():
- return ''
-
- self.fake_downburst_executable = fake_downburst_executable
-
- def test_ubuntu_noble(self):
- self.fake_ctx.os_type = 'ubuntu'
- self.fake_ctx.os_version = 'noble'
- with patch.multiple(
- provision.downburst,
- downburst_executable=self.fake_downburst_executable,
- ):
- check_value = teuthology.lock.util.vps_version_or_type_valid(
- self.fake_ctx.machine_type,
- self.fake_ctx.os_type,
- self.fake_ctx.os_version)
-
- assert check_value
-
- def test_ubuntu_number(self):
- self.fake_ctx.os_type = 'ubuntu'
- self.fake_ctx.os_version = '24.04'
- with patch.multiple(
- provision.downburst,
- downburst_executable=self.fake_downburst_executable,
- ):
- check_value = teuthology.lock.util.vps_version_or_type_valid(
- self.fake_ctx.machine_type,
- self.fake_ctx.os_type,
- self.fake_ctx.os_version)
- assert check_value
-
- def test_mixup(self):
- self.fake_ctx.os_type = '6.5'
- self.fake_ctx.os_version = 'rhel'
- with patch.multiple(
- provision.downburst,
- downburst_executable=self.fake_downburst_executable,
- ):
- check_value = teuthology.lock.util.vps_version_or_type_valid(
- self.fake_ctx.machine_type,
- self.fake_ctx.os_type,
- self.fake_ctx.os_version)
- assert not check_value
-
- def test_bad_type(self):
- self.fake_ctx.os_type = 'aardvark'
- self.fake_ctx.os_version = '6.5'
- with patch.multiple(
- provision.downburst,
- downburst_executable=self.fake_downburst_executable,
- ):
- check_value = teuthology.lock.util.vps_version_or_type_valid(
- self.fake_ctx.machine_type,
- self.fake_ctx.os_type,
- self.fake_ctx.os_version)
- assert not check_value
-
- def test_bad_version(self):
- self.fake_ctx.os_type = 'ubuntu'
- self.fake_ctx.os_version = 'vampire_bat'
- with patch.multiple(
- provision.downburst,
- downburst_executable=self.fake_downburst_executable,
- ):
- check_value = teuthology.lock.util.vps_version_or_type_valid(
- self.fake_ctx.machine_type,
- self.fake_ctx.os_type,
- self.fake_ctx.os_version)
- assert not check_value
+++ /dev/null
-<?xml version="1.0" encoding="UTF-8"?>
-<testsuite name="nosetests" tests="644" errors="0" failures="1" skip="79">
-<testcase classname="s3tests_boto3.functional.test_s3" name="test_cors_origin_response" time="3.205"></testcase>
-<testcase classname="s3tests_boto3.functional.test_s3" name="test_cors_origin_wildcard" time="3.081"></testcase>
-<testcase classname="s3tests_boto3.functional.test_s3" name="test_cors_header_option" time="3.119"></testcase>
-<testcase classname="s3tests_boto3.functional.test_s3" name="test_set_bucket_tagging" time="0.059"><failure type="builtins.AssertionError" message="'NoSuchTagSetError' != 'NoSuchTagSet'"></failure></testcase>
-</testsuite>
\ No newline at end of file
+++ /dev/null
-<?xml version="1.0"?>
-<valgrindoutput>
-<error>
- <unique>0x870fc</unique>
- <tid>1</tid>
- <kind>Leak_DefinitelyLost</kind>
- <xwhat>
- <text>1,234 bytes in 1 blocks are definitely lost in loss record 198 of 201</text>
- <leakedbytes>1234</leakedbytes>
- <leakedblocks>1</leakedblocks>
- </xwhat>
- <stack>
- <frame>
- <ip>0x4C39B6F</ip>
- <obj>/usr/libexec/valgrind/vgpreload_memcheck-amd64-linux.so</obj>
- <fn>operator new[](unsigned long)</fn>
- <dir>/builddir/build/BUILD/valgrind-3.19.0/coregrind/m_replacemalloc</dir>
- <file>vg_replace_malloc.c</file>
- <line>640</line>
- </frame>
- <frame>
- <ip>0xF3F4B5</ip>
- <obj>/usr/bin/ceph-osd</obj>
- <fn>ceph::common::leak_some_memory()</fn>
- <dir>/usr/src/debug/ceph-18.0.0-5567.g64a4fc94.el8.x86_64/src/common</dir>
- <file>ceph_context.cc</file>
- <line>510</line>
- </frame>
- </stack>
-</error>
-</valgrindoutput>
+++ /dev/null
-from mock import patch, MagicMock
-
-from io import BytesIO
-import os, io
-
-from teuthology.orchestra import remote
-from teuthology.util.scanner import UnitTestScanner, ValgrindScanner
-
-
-class MockFile(io.StringIO):
- def close(self):
- pass
-
-
-class TestUnitTestScanner(object):
-
- def setup_method(self):
- self.remote = remote.Remote(
- name='jdoe@xyzzy.example.com', ssh=MagicMock())
- self.test_values = {
- "xml_path": os.path.dirname(__file__) + "/files/test_unit_test.xml",
- "error_msg": "FAILURE: Test `test_set_bucket_tagging` of `s3tests_boto3.functional.test_s3`. \
-Reason: 'NoSuchTagSetError' != 'NoSuchTagSet'.",
- "summary_data": [{'failed_testsuites': {'s3tests_boto3.functional.test_s3':
- [{'kind': 'failure', 'testcase': 'test_set_bucket_tagging',
- 'message': "'NoSuchTagSetError' != 'NoSuchTagSet'"}]},
- 'num_of_failures': 1,
- 'file_path': f'{os.path.dirname(__file__)}/files/test_unit_test.xml'}],
- "yaml_data": r"""- failed_testsuites:
- s3tests_boto3.functional.test_s3:
- - kind: failure
- message: '''NoSuchTagSetError'' != ''NoSuchTagSet'''
- testcase: test_set_bucket_tagging
- file_path: {file_dir}/files/test_unit_test.xml
- num_of_failures: 1
-""".format(file_dir=os.path.dirname(__file__))
- }
-
- @patch('teuthology.util.scanner.UnitTestScanner.write_summary')
- def test_scan_and_write(self, m_write_summary):
- xml_path = self.test_values["xml_path"]
- self.remote.ssh.exec_command.return_value = (None, BytesIO(xml_path.encode('utf-8')), None)
- m_open = MagicMock()
- m_open.return_value = open(xml_path, "rb")
- self.remote._sftp_open_file = m_open
- result = UnitTestScanner(remote=self.remote).scan_and_write(xml_path, "test_summary.yaml")
- assert result == "(total 1 failed) " + self.test_values["error_msg"]
-
- def test_parse(self):
- xml_content = b'<?xml version="1.0" encoding="UTF-8"?>\n<testsuite name="xyz" tests="1" \
-errors="0" failures="1">\n<testcase classname="xyz" name="abc" time="0.059"><failure \
-type="builtins.AssertionError" message="error_msg"></failure></testcase>\n</testsuite>'
- scanner = UnitTestScanner(self.remote)
- result = scanner._parse(xml_content)
- assert result == (
- 'FAILURE: Test `abc` of `xyz`. Reason: error_msg.',
- {'failed_testsuites': {'xyz':
- [{'kind': 'failure','message': 'error_msg','testcase': 'abc'}]},
- 'num_of_failures': 1
- }
- )
-
- def test_scan_file(self):
- xml_path = self.test_values["xml_path"]
- m_open = MagicMock()
- m_open.return_value = open(xml_path, "rb")
- self.remote._sftp_open_file = m_open
- scanner = UnitTestScanner(remote=self.remote)
- result = scanner.scan_file(xml_path)
- assert result == self.test_values["error_msg"]
- assert scanner.summary_data == self.test_values["summary_data"]
-
- def test_scan_all_files(self):
- xml_path = self.test_values["xml_path"]
- self.remote.ssh.exec_command.return_value = (None, BytesIO(xml_path.encode('utf-8')), None)
- m_open = MagicMock()
- m_open.return_value = open(xml_path, "rb")
- self.remote._sftp_open_file = m_open
- scanner = UnitTestScanner(remote=self.remote)
- result = scanner.scan_all_files(xml_path)
- assert result == [self.test_values["error_msg"]]
-
- @patch('builtins.open')
- def test_write_summary(self, m_open):
- scanner = UnitTestScanner(self.remote)
- mock_yaml_file = MockFile()
- scanner.summary_data = self.test_values["summary_data"]
- m_open.return_value = mock_yaml_file
- scanner.write_summary("path/file.yaml")
- written_content = mock_yaml_file.getvalue()
- assert written_content == self.test_values["yaml_data"]
-
-
-class TestValgrindScanner(object):
-
- def setup_method(self):
- self.remote = remote.Remote(
- name='jdoe@xyzzy.example.com', ssh=MagicMock())
- self.test_values = {
- "xml_path": os.path.dirname(__file__) + "/files/test_valgrind.xml",
- "error_msg": "valgrind error: Leak_DefinitelyLost\noperator new[]\
-(unsigned long)\nceph::common::leak_some_memory()",
- "summary_data": [{'kind': 'Leak_DefinitelyLost', 'traceback': [{'file':
- '/builddir/build/BUILD/valgrind-3.19.0/coregrind/m_replacemalloc/vg_replace_malloc.c',
- 'line': '640', 'function': 'operator new[](unsigned long)'},
- {'file': '/usr/src/debug/ceph-18.0.0-5567.g64a4fc94.el8.x86_64/src/common/ceph_context.cc',
- 'line': '510', 'function': 'ceph::common::leak_some_memory()'}], 'file_path':
- f'{os.path.dirname(__file__)}/files/test_valgrind.xml'}],
- "yaml_data": r"""- file_path: {file_dir}/files/test_valgrind.xml
- kind: Leak_DefinitelyLost
- traceback:
- - file: /builddir/build/BUILD/valgrind-3.19.0/coregrind/m_replacemalloc/vg_replace_malloc.c
- function: operator new[](unsigned long)
- line: '640'
- - file: /usr/src/debug/ceph-18.0.0-5567.g64a4fc94.el8.x86_64/src/common/ceph_context.cc
- function: ceph::common::leak_some_memory()
- line: '510'
-""".format(file_dir=os.path.dirname(__file__))
- }
-
- def test_parse_with_traceback(self):
- xml_content = b'''<?xml version="1.0"?>
-<valgrindoutput>
-<error>
- <kind>Leak_DefinitelyLost</kind>
- <stack>
- <frame>
- <fn>func()</fn>
- <dir>/dir</dir>
- <file>file1.ext</file>
- <line>640</line>
- </frame>
- </stack>
-</error>
-</valgrindoutput>
-'''
- scanner = ValgrindScanner(self.remote)
- result = scanner._parse(xml_content)
- assert result == (
- 'valgrind error: Leak_DefinitelyLost\nfunc()',
- {'kind': 'Leak_DefinitelyLost', 'traceback':
- [{'file': '/dir/file1.ext', 'line': '640', 'function': 'func()'}]
- }
- )
-
- def test_parse_without_trackback(self):
- xml_content = b'''<?xml version="1.0"?>
-<valgrindoutput>
-<error>
- <kind>Leak_DefinitelyLost</kind>
- <stack>
- </stack>
-</error>
-</valgrindoutput>
-'''
- scanner = ValgrindScanner(self.remote)
- result = scanner._parse(xml_content)
- assert result == (
- 'valgrind error: Leak_DefinitelyLost\n',
- {'kind': 'Leak_DefinitelyLost', 'traceback': []}
- )
-
- def test_scan_file(self):
- xml_path = self.test_values["xml_path"]
- m_open = MagicMock()
- m_open.return_value = open(xml_path, "rb")
- self.remote._sftp_open_file = m_open
- scanner = ValgrindScanner(remote=self.remote)
- result = scanner.scan_file(xml_path)
- assert result == self.test_values["error_msg"]
- assert scanner.summary_data == self.test_values["summary_data"]
-
- def test_scan_all_files(self):
- xml_path = self.test_values["xml_path"]
- self.remote.ssh.exec_command.return_value = (None, BytesIO(xml_path.encode('utf-8')), None)
- m_open = MagicMock()
- m_open.return_value = open(xml_path, "rb")
- self.remote._sftp_open_file = m_open
- scanner = ValgrindScanner(remote=self.remote)
- result = scanner.scan_all_files(xml_path)
- assert result == [self.test_values["error_msg"]]
-
- @patch('builtins.open')
- def test_write_summary(self, m_open):
- scanner = ValgrindScanner(self.remote)
- mock_yaml_file = MockFile()
- scanner.summary_data = self.test_values["summary_data"]
- m_open.return_value = mock_yaml_file
- scanner.write_summary("path/file.yaml")
- written_content = mock_yaml_file.getvalue()
- assert written_content == self.test_values["yaml_data"]
\ No newline at end of file
+++ /dev/null
-import pytest
-
-from datetime import datetime, timedelta, timezone
-from typing import Type
-
-from teuthology.util import time
-
-
-@pytest.mark.parametrize(
- ["timestamp", "result"],
- [
- ["1999-12-31_23:59:59", datetime(1999, 12, 31, 23, 59, 59, tzinfo=timezone.utc)],
- ["1999-12-31_23:59", datetime(1999, 12, 31, 23, 59, 0, tzinfo=timezone.utc)],
- ["1999-12-31T23:59:59", datetime(1999, 12, 31, 23, 59, 59, tzinfo=timezone.utc)],
- ["1999-12-31T23:59:59+00:00", datetime(1999, 12, 31, 23, 59, 59, tzinfo=timezone.utc)],
- ["1999-12-31T17:59:59-06:00", datetime(1999, 12, 31, 23, 59, 59, tzinfo=timezone.utc)],
- ["2024-01-01", datetime(2024, 1, 1, 0, 0, tzinfo=timezone.utc)],
- ["tomorrow", ValueError],
- ["1d", ValueError],
- ["", ValueError],
- ["2024", ValueError],
-
- ]
-)
-def test_parse_timestamp(timestamp: str, result: datetime | Type[Exception]):
- if isinstance(result, datetime):
- assert time.parse_timestamp(timestamp) == result
- else:
- with pytest.raises(result):
- time.parse_timestamp(timestamp)
-
-
-@pytest.mark.parametrize(
- ["offset", "result"],
- [
- ["1s", timedelta(seconds=1)],
- ["1m", timedelta(minutes=1)],
- ["1h", timedelta(hours=1)],
- ["1d", timedelta(days=1)],
- ["1w", timedelta(weeks=1)],
- ["365d", timedelta(days=365)],
- ["1x", ValueError],
- ["-1m", ValueError],
- ["0xde", ValueError],
- ["frog", ValueError],
- ["7dwarfs", ValueError],
- ]
-)
-def test_parse_offset(offset: str, result: timedelta | Type[Exception]):
- if isinstance(result, timedelta):
- assert time.parse_offset(offset) == result
- else:
- with pytest.raises(result):
- time.parse_offset(offset)
extras = test
log_format = %(asctime)s %(levelname)s %(message)s
commands =
- python -m pytest --cov=teuthology --cov-report=term -v {posargs:teuthology scripts}
+ python -m pytest --cov=teuthology --cov-report=term -v {posargs:./tests}
[testenv:docs]
changedir = docs
projects / teuthology.git / commitdiff