suite: add --throttle to pause between jobs

When scheduling, wait SLEEP seconds between jobs.  Useful to avoid
bursts that may be too hard on the underlying infrastructure or exceed
OpenStack API limits (server creation per minute for instance).

http://tracker.ceph.com/issues/12977 Refs: #12977

Signed-off-by: Loic Dachary <loic@dachary.org>

diff --git a/scripts/suite.py b/scripts/suite.py
index 5bcf9fdf2498fe4caa53d25230f7e9db7c92430b..4b9b71270d7e28a09f7382205b9652f85ae2ca78 100644 (file)
--- a/scripts/suite.py
+++ b/scripts/suite.py
@@ -81,6 +81,10 @@ Scheduler arguments:
                               the keywords in the comma separated keyword
                               string specified.
   --archive-upload RSYNC_DEST Rsync destination to upload archives.
+  --throttle SLEEP            When scheduling, wait SLEEP seconds between jobs.
+                              Useful to avoid bursts that may be too hard on
+                              the underlying infrastructure or exceed OpenStack API
+                              limits (server creation per minute for instance).
 """.format(default_machine_type=config.default_machine_type,
            default_results_timeout=config.results_timeout)
 

diff --git a/teuthology/suite.py b/teuthology/suite.py
index 80d1dfc4e5455dc9ae31adaeb8ba371c6145bb1c..8e3b5e13f4634a10419d0c1feaa6bff22d77153c 100644 (file)
--- a/teuthology/suite.py
+++ b/teuthology/suite.py
@@ -12,6 +12,7 @@ import subprocess
 import smtplib
 import socket
 import sys
+from time import sleep
 import yaml
 import math
 from email.mime.text import MIMEText
@@ -64,6 +65,7 @@ def main(args):
     timeout = args['--timeout']
     filter_in = args['--filter']
     filter_out = args['--filter-out']
+    throttle = args['--throttle']
 
     subset = None
     if args['--subset']:
@@ -115,6 +117,7 @@ def main(args):
                          filter_in=filter_in,
                          filter_out=filter_out,
                          subset=subset,
+                         throttle=throttle,
                          )
     os.remove(base_yaml_path)
 
@@ -260,7 +263,8 @@ def prepare_and_schedule(job_config, suite_repo_path, base_yaml_paths, limit,
                          num, timeout, dry_run, verbose,
                          filter_in,
                          filter_out,
-                         subset):
+                         subset,
+                         throttle):
     """
     Puts together some "base arguments" with which to execute
     teuthology-schedule for each job, then passes them and other parameters to
@@ -303,7 +307,8 @@ def prepare_and_schedule(job_config, suite_repo_path, base_yaml_paths, limit,
         verbose=verbose,
         filter_in=filter_in,
         filter_out=filter_out,
-        subset=subset
+        subset=subset,
+        throttle=throttle
     )
 
     if job_config.email and num_jobs:
@@ -504,7 +509,8 @@ def schedule_suite(job_config,
                    verbose=1,
                    filter_in=None,
                    filter_out=None,
-                   subset=None
+                   subset=None,
+                   throttle=None,
                    ):
     """
     schedule one suite.
@@ -618,6 +624,9 @@ def schedule_suite(job_config,
             verbose=verbose,
             log_prefix=log_prefix,
         )
+        if not dry_run and throttle:
+            log.info("pause between jobs : --throttle " + str(throttle))
+            sleep(int(throttle))
 
     count = len(jobs_to_schedule)
     missing_count = len(jobs_missing_packages)

diff --git a/teuthology/test/suites/noop/noop.yaml b/teuthology/test/suites/noop/noop.yaml
new file mode 100644 (file)
index 0000000..fb674b1
--- /dev/null
+++ b/teuthology/test/suites/noop/noop.yaml
@@ -0,0 +1,7 @@
+roles:
+- - mon.a
+  - osd.0
+tasks:
+- exec:
+    mon.a:
+      - echo "Well done !"

diff --git a/teuthology/test/test_suite.py b/teuthology/test/test_suite.py
index 0b25b598bd4bdc6cf2e72611a4b1b45fdf8e4e1a..b39ded8129f2bb1b3a03f448ad521cc5d31017ce 100644 (file)
--- a/teuthology/test/test_suite.py
+++ b/teuthology/test/test_suite.py
@@ -1,9 +1,10 @@
 from copy import deepcopy
 from datetime import datetime
 
-from mock import patch, Mock
+from mock import patch, Mock, DEFAULT
 
 from teuthology import suite
+from scripts.suite import main
 from teuthology.config import config
 
 
@@ -703,3 +704,39 @@ class TestBuildMatrix(object):
         assert fragments[0] == 'thrash/ceph/base.yaml'
         assert fragments[1] == 'thrash/ceph-thrash/default.yaml'
 
+
+class TestSuiteMain(object):
+
+    def test_main(self):
+        suite_name = 'SUITE'
+        throttle = '3'
+        machine_type = 'burnupi'
+        def prepare_and_schedule(**kwargs):
+            assert kwargs['job_config']['suite'] == suite_name
+            assert kwargs['throttle'] == throttle
+
+        with patch.multiple(
+                suite,
+                fetch_repos=DEFAULT,
+                prepare_and_schedule=prepare_and_schedule,
+                ):
+            main(['--suite', suite_name,
+                  '--throttle', throttle,
+                  '--machine-type', machine_type])
+
+    def test_schedule_suite(self):
+        suite_name = 'noop'
+        throttle = '3'
+        machine_type = 'burnupi'
+
+        with patch.multiple(
+                suite,
+                fetch_repos=DEFAULT,
+                teuthology_schedule=DEFAULT,
+                sleep=DEFAULT,
+                ) as m:
+            main(['--suite', suite_name,
+                  '--suite-dir', 'teuthology/test',
+                  '--throttle', throttle,
+                  '--machine-type', machine_type])
+            m['sleep'].assert_called_with(int(throttle))