From: Kyr Shatskyy <kyrylo.shatskyy@suse.com>
Date: Sun, 10 May 2020 15:01:21 +0000 (+0200)
Subject: schedule: add more than one queue backend
X-Git-Tag: 1.1.0~60^2~1
X-Git-Url: http://git.apps.os.sepia.ceph.com/?a=commitdiff_plain;h=4988fdce5b91d66ab468e9978193387d2e37e87d;p=teuthology.git

schedule: add more than one queue backend

This allows to have more than one beanstalk queue for jobs.
For example, use paddles or local sqlite for managing queue.

At the moment it gives a possibility to omit 'beanstalk'
queue in order to get job configurations letting a user
to run teuthology job autonomously.

Signed-off-by: Kyr Shatskyy <kyrylo.shatskyy@suse.com>
---

diff --git a/scripts/schedule.py b/scripts/schedule.py
index feeafd770e..ffcb112b45 100644
--- a/scripts/schedule.py
+++ b/scripts/schedule.py
@@ -16,6 +16,11 @@ positional arguments:
 optional arguments:
   -h, --help                           Show this help message and exit
   -v, --verbose                        Be more verbose
+  -b <backend>, --queue-backend <backend>
+                                       Queue backend name, use prefix '@'
+                                       to append job config to the given
+                                       file path as yaml.
+                                       [default: beanstalk]
   -n <name>, --name <name>             Name of suite run the job is part of
   -d <desc>, --description <desc>      Job description
   -o <owner>, --owner <owner>          Job owner
diff --git a/scripts/suite.py b/scripts/suite.py
index 59036f5992..0de7be36e8 100644
--- a/scripts/suite.py
+++ b/scripts/suite.py
@@ -103,6 +103,8 @@ Standard arguments:
 
 Scheduler arguments:
   --owner <owner>             Job owner
+  -b <backend>, --queue-backend <backend>
+                              Scheduler queue backend name
   -e <email>, --email <email>
                               When tests finish or time out, send an email
                               here. May also be specified in ~/.teuthology.yaml
diff --git a/teuthology/schedule.py b/teuthology/schedule.py
index f8206cf017..01c998367e 100644
--- a/teuthology/schedule.py
+++ b/teuthology/schedule.py
@@ -1,3 +1,4 @@
+import os
 import yaml
 
 import teuthology.beanstalk
@@ -31,10 +32,16 @@ def main(args):
     if not name or name.isdigit():
         raise ValueError("Please use a more descriptive value for --name")
     job_config = build_config(args)
+    backend = args['--queue-backend']
     if args['--dry-run']:
         print('---\n' + yaml.safe_dump(job_config))
-    else:
+    elif backend == 'beanstalk':
         schedule_job(job_config, args['--num'], report_status)
+    elif backend.startswith('@'):
+        dump_job_to_file(backend.lstrip('@'), job_config, args['--num'])
+    else:
+        raise ValueError("Provided schedule backend '%s' is not supported. "
+                         "Try 'beanstalk' or '@path-to-a-file" % backend)
 
 
 def build_config(args):
@@ -103,3 +110,33 @@ def schedule_job(job_config, num=1, report_status=True):
         if report_status:
             report.try_push_job_info(job_config, dict(status='queued'))
         num -= 1
+
+
+def dump_job_to_file(path, job_config, num=1):
+    """
+    Schedule a job.
+
+    :param job_config: The complete job dict
+    :param num:      The number of times to schedule the job
+    :param path:     The file path where the job config to append
+    """
+    num = int(num)
+    count_file_path = path + '.count'
+
+    jid = 0
+    if os.path.exists(count_file_path):
+        with open(count_file_path, 'r') as f:
+            jid=int(f.read() or '0')
+
+    with open(path, 'a') as f:
+        while num > 0:
+            jid += 1
+            job_config['job_id'] = str(jid)
+            job = yaml.safe_dump(job_config)
+            print('Job scheduled with name {name} and ID {jid}'.format(
+                name=job_config['name'], jid=jid))
+            f.write('---\n' + job)
+            num -= 1
+    with open(count_file_path, 'w') as f:
+        f.write(str(jid))
+
diff --git a/teuthology/suite/run.py b/teuthology/suite/run.py
index 1a71404b13..63169910bf 100644
--- a/teuthology/suite/run.py
+++ b/teuthology/suite/run.py
@@ -331,6 +331,8 @@ class Run(object):
             base_args.append('-v')
         if self.args.owner:
             base_args.extend(['--owner', self.args.owner])
+        if self.args.queue_backend:
+            base_args.extend(['--queue-backend', self.args.queue_backend])
         return base_args