From: Loic Dachary <ldachary@redhat.com>
Date: Mon, 7 Sep 2015 11:40:47 +0000 (+0200)
Subject: suite: add --throttle to pause between jobs
X-Git-Tag: 1.1.0~824^2~1
X-Git-Url: http://git.apps.os.sepia.ceph.com/?a=commitdiff_plain;h=5172d19830db58d5d553b0e06380117dce3058a6;p=teuthology.git

suite: add --throttle to pause between jobs

When scheduling, wait SLEEP seconds between jobs.  Useful to avoid
bursts that may be too hard on the underlying infrastructure or exceed
OpenStack API limits (server creation per minute for instance).

http://tracker.ceph.com/issues/12977 Refs: #12977

Signed-off-by: Loic Dachary <loic@dachary.org>
---

diff --git a/scripts/suite.py b/scripts/suite.py
index 5bcf9fdf24..4b9b71270d 100644
--- a/scripts/suite.py
+++ b/scripts/suite.py
@@ -81,6 +81,10 @@ Scheduler arguments:
                               the keywords in the comma separated keyword
                               string specified.
   --archive-upload RSYNC_DEST Rsync destination to upload archives.
+  --throttle SLEEP            When scheduling, wait SLEEP seconds between jobs.
+                              Useful to avoid bursts that may be too hard on
+                              the underlying infrastructure or exceed OpenStack API
+                              limits (server creation per minute for instance).
 """.format(default_machine_type=config.default_machine_type,
            default_results_timeout=config.results_timeout)
 
diff --git a/teuthology/suite.py b/teuthology/suite.py
index 80d1dfc4e5..8e3b5e13f4 100644
--- a/teuthology/suite.py
+++ b/teuthology/suite.py
@@ -12,6 +12,7 @@ import subprocess
 import smtplib
 import socket
 import sys
+from time import sleep
 import yaml
 import math
 from email.mime.text import MIMEText
@@ -64,6 +65,7 @@ def main(args):
     timeout = args['--timeout']
     filter_in = args['--filter']
     filter_out = args['--filter-out']
+    throttle = args['--throttle']
 
     subset = None
     if args['--subset']:
@@ -115,6 +117,7 @@ def main(args):
                          filter_in=filter_in,
                          filter_out=filter_out,
                          subset=subset,
+                         throttle=throttle,
                          )
     os.remove(base_yaml_path)
 
@@ -260,7 +263,8 @@ def prepare_and_schedule(job_config, suite_repo_path, base_yaml_paths, limit,
                          num, timeout, dry_run, verbose,
                          filter_in,
                          filter_out,
-                         subset):
+                         subset,
+                         throttle):
     """
     Puts together some "base arguments" with which to execute
     teuthology-schedule for each job, then passes them and other parameters to
@@ -303,7 +307,8 @@ def prepare_and_schedule(job_config, suite_repo_path, base_yaml_paths, limit,
         verbose=verbose,
         filter_in=filter_in,
         filter_out=filter_out,
-        subset=subset
+        subset=subset,
+        throttle=throttle
     )
 
     if job_config.email and num_jobs:
@@ -504,7 +509,8 @@ def schedule_suite(job_config,
                    verbose=1,
                    filter_in=None,
                    filter_out=None,
-                   subset=None
+                   subset=None,
+                   throttle=None,
                    ):
     """
     schedule one suite.
@@ -618,6 +624,9 @@ def schedule_suite(job_config,
             verbose=verbose,
             log_prefix=log_prefix,
         )
+        if not dry_run and throttle:
+            log.info("pause between jobs : --throttle " + str(throttle))
+            sleep(int(throttle))
 
     count = len(jobs_to_schedule)
     missing_count = len(jobs_missing_packages)
diff --git a/teuthology/test/suites/noop/noop.yaml b/teuthology/test/suites/noop/noop.yaml
new file mode 100644
index 0000000000..fb674b1b13
--- /dev/null
+++ b/teuthology/test/suites/noop/noop.yaml
@@ -0,0 +1,7 @@
+roles:
+- - mon.a
+  - osd.0
+tasks:
+- exec:
+    mon.a:
+      - echo "Well done !"
diff --git a/teuthology/test/test_suite.py b/teuthology/test/test_suite.py
index 0b25b598bd..b39ded8129 100644
--- a/teuthology/test/test_suite.py
+++ b/teuthology/test/test_suite.py
@@ -1,9 +1,10 @@
 from copy import deepcopy
 from datetime import datetime
 
-from mock import patch, Mock
+from mock import patch, Mock, DEFAULT
 
 from teuthology import suite
+from scripts.suite import main
 from teuthology.config import config
 
 
@@ -703,3 +704,39 @@ class TestBuildMatrix(object):
         assert fragments[0] == 'thrash/ceph/base.yaml'
         assert fragments[1] == 'thrash/ceph-thrash/default.yaml'
 
+
+class TestSuiteMain(object):
+
+    def test_main(self):
+        suite_name = 'SUITE'
+        throttle = '3'
+        machine_type = 'burnupi'
+        def prepare_and_schedule(**kwargs):
+            assert kwargs['job_config']['suite'] == suite_name
+            assert kwargs['throttle'] == throttle
+
+        with patch.multiple(
+                suite,
+                fetch_repos=DEFAULT,
+                prepare_and_schedule=prepare_and_schedule,
+                ):
+            main(['--suite', suite_name,
+                  '--throttle', throttle,
+                  '--machine-type', machine_type])
+
+    def test_schedule_suite(self):
+        suite_name = 'noop'
+        throttle = '3'
+        machine_type = 'burnupi'
+
+        with patch.multiple(
+                suite,
+                fetch_repos=DEFAULT,
+                teuthology_schedule=DEFAULT,
+                sleep=DEFAULT,
+                ) as m:
+            main(['--suite', suite_name,
+                  '--suite-dir', 'teuthology/test',
+                  '--throttle', throttle,
+                  '--machine-type', machine_type])
+            m['sleep'].assert_called_with(int(throttle))