From f6cfc90d3a5af66e0c07fa85e71b59b533dc8710 Mon Sep 17 00:00:00 2001 From: Neha Ojha Date: Thu, 1 Jul 2021 18:28:05 +0000 Subject: [PATCH] teuthology/suite/run.py, scripts/suite.py: disallow scheduling too many jobs Add check_num_jobs() to prevent users from accidentally scheduling too many jobs, like in rfriedma-2021-06-26_19:32:15-rados-wip-ronenf-scrubs-config-distro-basic-smithi. JOBS_TO_SCHEDULE_THRESHOLD, set to 500 (most runs have fewer jobs than this), will disallow users from scheduling more than 500 jobs. Users can schedule more than 500 jobs by disabling this check using the --disable-num-jobs-check flag. Signed-off-by: Neha Ojha --- scripts/suite.py | 5 +++++ teuthology/suite/run.py | 12 ++++++++++++ 2 files changed, 17 insertions(+) diff --git a/scripts/suite.py b/scripts/suite.py index 8097b1454..f8a0f8f84 100644 --- a/scripts/suite.py +++ b/scripts/suite.py @@ -172,6 +172,11 @@ Scheduler arguments: in the output of teuthology-suite command. -1 for a random seed [default: -1]. --force-priority Skip the priority check. + --disable-num-jobs-check Skip the number of jobs check. By default, + teuthology will not allow you to schedule more + than JOBS_TO_SCHEDULE_THRESHOLD=500 jobs, because + it is too high. Use this if you need to schedule + more than 500 jobs. """.format( default_machine_type=config.default_machine_type, diff --git a/teuthology/suite/run.py b/teuthology/suite/run.py index c980bc690..aedaec800 100644 --- a/teuthology/suite/run.py +++ b/teuthology/suite/run.py @@ -29,6 +29,7 @@ log = logging.getLogger(__name__) class Run(object): WAIT_MAX_JOB_TIME = 30 * 60 WAIT_PAUSE = 5 * 60 + JOBS_TO_SCHEDULE_THRESHOLD = 500 __slots__ = ( 'args', 'name', 'base_config', 'suite_repo_path', 'base_yaml_paths', 'base_args', 'package_versions', 'kernel_dict', 'config_input', @@ -525,6 +526,13 @@ Note: To force run, use --force-priority''' elif priority < 150 and jobs_to_schedule > 100: util.schedule_fail(msg) + def check_num_jobs(self, jobs_to_schedule): + msg=f'''Unable to schedule {jobs_to_schedule} jobs, too many jobs. + +Note: If you still want to go ahead, use --disable-num-jobs-check''' + if jobs_to_schedule > Run.JOBS_TO_SCHEDULE_THRESHOLD: + util.schedule_fail(msg) + def schedule_suite(self): """ Schedule the suite-run. Returns the number of jobs scheduled. @@ -641,6 +649,10 @@ Note: To force run, use --force-priority''' if self.args.priority and jobs_to_schedule and not self.args.force_priority: self.check_priority(len(jobs_to_schedule)) + # Before scheduling jobs, check number of jobs to avoid scheduling 1000s + if jobs_to_schedule and not self.args.disable_num_jobs_check: + self.check_num_jobs(len(jobs_to_schedule)) + self.schedule_jobs(jobs_missing_packages, jobs_to_schedule, name) os.remove(base_yaml_path) -- 2.47.3