teuthology/suite/run.py, scripts/suite.py: disallow scheduling too many jobs

Add check_num_jobs() to prevent users from accidentally scheduling too many jobs, like
in rfriedma-2021-06-26_19:32:15-rados-wip-ronenf-scrubs-config-distro-basic-smithi.
JOBS_TO_SCHEDULE_THRESHOLD, set to 500 (most runs have fewer jobs than this),
will disallow users from scheduling more than 500 jobs. Users can schedule
more than 500 jobs by disabling this check using the --disable-num-jobs-check flag.

Signed-off-by: Neha Ojha <nojha@redhat.com>

diff --git a/scripts/suite.py b/scripts/suite.py
index 8097b14545fc65ba897f29843294ec604b38c11f..f8a0f8f84f08e8e51bf925ff4ab41cbb29cf07e4 100644 (file)
--- a/scripts/suite.py
+++ b/scripts/suite.py
@@ -172,6 +172,11 @@ Scheduler arguments:
                               in the output of teuthology-suite command. -1
                               for a random seed [default: -1].
  --force-priority             Skip the priority check.
+ --disable-num-jobs-check     Skip the number of jobs check. By default,
+                              teuthology will not allow you to schedule more
+                              than JOBS_TO_SCHEDULE_THRESHOLD=500 jobs, because
+                              it is too high. Use this if you need to schedule
+                              more than 500 jobs.
 
 """.format(
     default_machine_type=config.default_machine_type,

diff --git a/teuthology/suite/run.py b/teuthology/suite/run.py
index c980bc6909553650426723b2bd445b9093226179..aedaec8004fe07dc301a133fd31dea81870b9418 100644 (file)
--- a/teuthology/suite/run.py
+++ b/teuthology/suite/run.py
@@ -29,6 +29,7 @@ log = logging.getLogger(__name__)
 class Run(object):
     WAIT_MAX_JOB_TIME = 30 * 60
     WAIT_PAUSE = 5 * 60
+    JOBS_TO_SCHEDULE_THRESHOLD = 500
     __slots__ = (
         'args', 'name', 'base_config', 'suite_repo_path', 'base_yaml_paths',
         'base_args', 'package_versions', 'kernel_dict', 'config_input',
@@ -525,6 +526,13 @@ Note: To force run, use --force-priority'''
         elif priority < 150 and jobs_to_schedule > 100:
             util.schedule_fail(msg)
 
+    def check_num_jobs(self, jobs_to_schedule):
+        msg=f'''Unable to schedule {jobs_to_schedule} jobs, too many jobs.
+
+Note: If you still want to go ahead, use --disable-num-jobs-check'''
+        if jobs_to_schedule > Run.JOBS_TO_SCHEDULE_THRESHOLD:
+            util.schedule_fail(msg)
+
     def schedule_suite(self):
         """
         Schedule the suite-run. Returns the number of jobs scheduled.
@@ -641,6 +649,10 @@ Note: To force run, use --force-priority'''
         if self.args.priority and jobs_to_schedule and not self.args.force_priority:
             self.check_priority(len(jobs_to_schedule))
 
+        # Before scheduling jobs, check number of jobs to avoid scheduling 1000s
+        if jobs_to_schedule and not self.args.disable_num_jobs_check:
+            self.check_num_jobs(len(jobs_to_schedule))
+
         self.schedule_jobs(jobs_missing_packages, jobs_to_schedule, name)
 
         os.remove(base_yaml_path)