From 9f0a61db0d9c86dc0d67a03f952e2c00814dab55 Mon Sep 17 00:00:00 2001 From: Aishwarya Mathuria Date: Mon, 7 Jun 2021 19:12:23 +0530 Subject: [PATCH] Modify teuthology worker unit test and fix flake8 errors Previously the teuthology worker unit test used beanstalk, the test will be using Paddles now. Signed-off-by: Aishwarya Mathuria --- teuthology/dispatcher/__init__.py | 1 - teuthology/kill.py | 2 - teuthology/paddles_queue.py | 4 -- teuthology/schedule.py | 3 - teuthology/test/test_worker.py | 114 +++++++++++------------------- 5 files changed, 40 insertions(+), 84 deletions(-) diff --git a/teuthology/dispatcher/__init__.py b/teuthology/dispatcher/__init__.py index dd2930f1e..e079f6bed 100644 --- a/teuthology/dispatcher/__init__.py +++ b/teuthology/dispatcher/__init__.py @@ -3,7 +3,6 @@ import os import subprocess import sys import yaml -import json from datetime import datetime diff --git a/teuthology/kill.py b/teuthology/kill.py index f9080f018..63a63ab4a 100755 --- a/teuthology/kill.py +++ b/teuthology/kill.py @@ -10,7 +10,6 @@ import getpass from teuthology import report -from teuthology.config import config from teuthology import misc log = logging.getLogger(__name__) @@ -102,7 +101,6 @@ def find_run_info(serializer, run_name): job_info = {} job_num = 0 jobs = serializer.jobs_for_run(run_name) - job_total = len(jobs) for (job_id, job_dir) in jobs.items(): if not os.path.isdir(job_dir): continue diff --git a/teuthology/paddles_queue.py b/teuthology/paddles_queue.py index 5287ea46f..87c96d8cb 100644 --- a/teuthology/paddles_queue.py +++ b/teuthology/paddles_queue.py @@ -3,7 +3,6 @@ import pprint import sys from collections import OrderedDict -from teuthology.config import config from teuthology import report @@ -88,14 +87,11 @@ def walk_jobs(machine_type, processor, user): log.info('No jobs in queue') return - # Try to figure out a sane timeout based on how many jobs are in the queue - timeout = job_count / 2000.0 * 60 for i in range(1, job_count + 1): print_progress(i, job_count, "Loading") job = jobs[i-1] if job is None: continue - job_name = job['name'] job_id = job['job_id'] processor.add_job(job_id, job) end_progress() diff --git a/teuthology/schedule.py b/teuthology/schedule.py index 4836884a9..4ccf78084 100644 --- a/teuthology/schedule.py +++ b/teuthology/schedule.py @@ -3,7 +3,6 @@ import yaml from teuthology.misc import get_user, merge_configs from teuthology import report -from teuthology.config import config def main(args): @@ -95,8 +94,6 @@ def schedule_job(job_config, num=1, report_status=True): :param num: The number of times to schedule the job """ num = int(num) - job = yaml.safe_dump(job_config) - ''' Add 'machine_type' queue to DB here. ''' diff --git a/teuthology/test/test_worker.py b/teuthology/test/test_worker.py index 87d3af88c..c5ed88799 100644 --- a/teuthology/test/test_worker.py +++ b/teuthology/test/test_worker.py @@ -1,4 +1,3 @@ -import beanstalkc import os from unittest.mock import patch, Mock, MagicMock @@ -212,96 +211,63 @@ class TestWorker(object): assert m_fetch_qa_suite.called_once_with_args(branch='main') assert got_config['suite_path'] == '/suite/path' - def build_fake_jobs(self, m_connection, m_job, job_bodies): + def build_fake_jobs(self, job_bodies): """ - Given patched copies of: - beanstalkc.Connection - beanstalkc.Job And a list of basic job bodies, return a list of mocked Job objects """ - # Make sure instantiating m_job returns a new object each time - m_job.side_effect = lambda **kwargs: Mock(spec=beanstalkc.Job) jobs = [] job_id = 0 for job_body in job_bodies: job_id += 1 - job = m_job(conn=m_connection, jid=job_id, body=job_body) - job.jid = job_id - job.body = job_body + job = {} + job['job_id'] = job_id + job['body'] = job_body jobs.append(job) return jobs - @patch("teuthology.worker.run_job") - @patch("teuthology.worker.prep_job") - @patch("beanstalkc.Job", autospec=True) - @patch("teuthology.worker.fetch_qa_suite") - @patch("teuthology.worker.fetch_teuthology") - @patch("teuthology.worker.beanstalk.watch_tube") - @patch("teuthology.worker.beanstalk.connect") - @patch("os.path.isdir", return_value=True) - @patch("teuthology.worker.setup_log_file") - def test_main_loop( - self, m_setup_log_file, m_isdir, m_connect, m_watch_tube, - m_fetch_teuthology, m_fetch_qa_suite, m_job, m_prep_job, m_run_job, - ): - m_connection = Mock() - jobs = self.build_fake_jobs( - m_connection, - m_job, - [ - 'foo: bar', - 'stop_worker: true', - ], - ) - m_connection.reserve.side_effect = jobs - m_connect.return_value = m_connection - m_prep_job.return_value = (dict(), '/bin/path') - worker.main(self.ctx) - # There should be one reserve call per item in the jobs list - expected_reserve_calls = [ - dict(timeout=60) for i in range(len(jobs)) - ] - got_reserve_calls = [ - call[1] for call in m_connection.reserve.call_args_list - ] - assert got_reserve_calls == expected_reserve_calls - for job in jobs: - job.bury.assert_called_once_with() - job.delete.assert_called_once_with() - @patch("teuthology.worker.report.try_push_job_info") - @patch("teuthology.worker.run_job") - @patch("beanstalkc.Job", autospec=True) - @patch("teuthology.worker.fetch_qa_suite") - @patch("teuthology.worker.fetch_teuthology") - @patch("teuthology.worker.beanstalk.watch_tube") - @patch("teuthology.worker.beanstalk.connect") - @patch("os.path.isdir", return_value=True) @patch("teuthology.worker.setup_log_file") + @patch("os.path.isdir", return_value=True) + @patch("teuthology.worker.fetch_teuthology") + @patch("teuthology.worker.fetch_qa_suite") + @patch("teuthology.worker.run_job") + @patch("teuthology.worker.report.try_push_job_info") + @patch("teuthology.worker.report.get_queued_job") + @patch("teuthology.worker.clean_config") def test_main_loop_13925( - self, m_setup_log_file, m_isdir, m_connect, m_watch_tube, - m_fetch_teuthology, m_fetch_qa_suite, m_job, m_run_job, - m_try_push_job_info, + self, m_setup_log_file, m_isdir, + m_fetch_teuthology, m_fetch_qa_suite, m_run_job, + m_try_push_job_info, m_get_queued_job, m_clean_config ): - m_connection = Mock() - jobs = self.build_fake_jobs( - m_connection, - m_job, - [ - 'name: name', - 'name: name\nstop_worker: true', - ], - ) - m_connection.reserve.side_effect = jobs - m_connect.return_value = m_connection m_fetch_qa_suite.side_effect = [ '/suite/path', MaxWhileTries(), MaxWhileTries(), ] + job = { + 'job_id': '1', + 'description': 'DESC', + 'email': 'EMAIL', + 'first_in_suite': False, + 'last_in_suite': True, + 'machine_type': 'test_queue', + 'name': 'NAME', + 'owner': 'OWNER', + 'priority': 99, + 'results_timeout': '6', + 'verbose': False, + 'stop_worker': True + } + m_get_queued_job.return_value = job + m_clean_config.return_value = job + + mock_prep_job_patcher = patch('teuthology.worker.prep_job') + mock_prep_job = mock_prep_job_patcher.start() + mock_prep_job.return_value = (dict(), '/teuth/bin/path') + worker.main(self.ctx) - assert len(m_run_job.call_args_list) == 0 - assert len(m_try_push_job_info.call_args_list) == len(jobs) - for i in range(len(jobs)): - push_call = m_try_push_job_info.call_args_list[i] - assert push_call[0][1]['status'] == 'dead' + mock_prep_job_patcher.stop() + assert len(m_run_job.call_args_list) == 1 + assert len(m_try_push_job_info.call_args_list) == 1 + assert m_try_push_job_info.called_once_with(job, dict(status='running')) + -- 2.47.3