Use pytest to auto discover and run tests in task.tests

Signed-off-by: Andrew Schoen <aschoen@redhat.com>

diff --git a/.gitignore b/.gitignore
index 4f6054c7eb377d812ecc43fc44d881a851dcf73f..2a67bdafd6889bc9d2efd4ad83ef5c7f4f9d86fb 100644 (file)
--- a/.gitignore
+++ b/.gitignore
@@ -23,3 +23,4 @@ docs/modules.rst
 docs/teuthology.rst
 docs/teuthology.task.rst
 docs/teuthology.orchestra.rst
+docs/teuthology.task.tests.rst

diff --git a/pytest.ini b/pytest.ini
index b50b1e6084e792883e250b4c0f123f118fde3cdc..5c4ad5f196631ce0b97775c7de13b50f8abb7219 100644 (file)
--- a/pytest.ini
+++ b/pytest.ini
@@ -1,2 +1,2 @@
 [pytest]
-norecursedirs = .git build virtualenv teuthology.egg-info .tox */integration
+norecursedirs = .git build virtualenv teuthology.egg-info .tox */integration task/tests

diff --git a/requirements.txt b/requirements.txt
index daff4290369b78fe2cf8b24c8a7a4cbbba8ec19a..ceed08602c9f41bc447cf583f8e952cf079f243f 100644 (file)
--- a/requirements.txt
+++ b/requirements.txt
@@ -19,6 +19,7 @@ web.py
 docopt
 psutil >= 2.1.0
 configparser
+pytest
 
 # Test Dependencies
 # nose >=1.0.0

diff --git a/teuthology/task/tests.py b/teuthology/task/tests.py
index 5bead718aaf167d71bbedc0a0c2b28c9e2a7f6ed..9d9e6000f7602422e4110f063471fb6470ff38e9 100644 (file)
--- a/teuthology/task/tests.py
+++ b/teuthology/task/tests.py
@@ -3,6 +3,7 @@ A place to put various testing functions for teuthology. Maybe at some point we
 can turn this into a proper test suite of sorts.
 """
 import logging
+import pytest
 
 from functools import wraps
 
@@ -11,25 +12,37 @@ from teuthology.exceptions import CommandFailedError
 log = logging.getLogger(__name__)
 
 
-def test(f):
+def log_test_results(f):
+    """
+    Use this to decorate test functions to log the result of
+    the test in the teuthology.log.
+    """
+    # TODO: it'd be nice to have the output from pytest use our logger
+    # I tried a few different ways, but couldn't get it to work so I settled
+    # on using this decorator instead.
     @wraps(f)
-    def func(*args, **kwargs):
+    def func(ctx, config):
+        name = f.func_name
         try:
-            log.info("running {name}...".format(name=f.func_name))
-            f(*args, **kwargs)
+            log.info("running {name}...".format(name=name))
+            f(ctx, config)
         except AssertionError:
-            log.error("***** FAILED")
-            args[0].summary["status"] = "fail"
+            log.error("***** FAILED {name}".format(name=name))
+            ctx.summary["status"] = "fail"
+            ctx.summary["failure_reason"] = "failed: {0}".format(name)
+            raise
         except Exception:
-            log.error("***** ERROR")
-            args[0].summary["status"] = "fail"
+            log.error("***** ERROR {name}".format(name=name))
+            ctx.summary["status"] = "fail"
+            ctx.summary["failure_reason"] = "error: {0}".format(name)
+            raise
         else:
-            log.info("***** PASSED")
+            log.info("***** PASSED {name}".format(name=name))
 
     return func
 
 
-@test
+@log_test_results
 def test_command_failed_label(ctx, config):
     result = ""
     try:
@@ -41,13 +54,46 @@ def test_command_failed_label(ctx, config):
 
 
 def force_command_failure(ctx, config):
+    log.info("forcing a command failure...")
     ctx.cluster.run(
         args=["python", "-c", "assert False"],
         label="working as expected, nothing to see here"
     )
 
 
+@pytest.fixture
+def ctx():
+    return {}
+
+
+@pytest.fixture
+def config():
+    return []
+
+
+class TeuthologyContextPlugin(object):
+    def __init__(self, ctx, config):
+        self.ctx = ctx
+        self.config = config
+
+    # this is pytest hook for generating tests with custom parameters
+    def pytest_generate_tests(self, metafunc):
+        # pass the teuthology ctx and config to each test method
+        metafunc.parametrize(["ctx", "config"], [(self.ctx, self.config),])
+
+
 def task(ctx, config):
-    # TODO: find a way to auto discover test functions in this file
-    # and execute them
-    test_command_failed_label(ctx, config)
+    # use pytest to find any test_* methods in this file
+    # and execute them with the teuthology ctx and config args
+    status = pytest.main(
+        args=[
+            '-s', '-q',
+            '--pyargs', __name__
+        ],
+        plugins=[TeuthologyContextPlugin(ctx, config)]
+    )
+    if status == 0:
+        log.info("OK. All tests passed!")
+    else:
+        log.error("FAIL. Saw test failures...")
+        ctx.summary["status"] = "fail"

diff --git a/tox.ini b/tox.ini
index 0c98db7f78ef8e45d2ee35fc9fcbc4281c42cd86..a34866a88414a7697974ba14e5a9a2bc2b355b67 100644 (file)
--- a/tox.ini
+++ b/tox.ini
@@ -5,7 +5,6 @@ envlist = docs, py27, py27-integration, flake8
 sitepackages=True
 deps=
   -r{toxinidir}/requirements.txt
-  pytest
   mock
   fudge
   nose
@@ -18,7 +17,6 @@ commands=py.test --cov=teuthology --cov-report=term -v {posargs:teuthology scrip
 sitepackages=True
 deps=
   -r{toxinidir}/requirements.txt
-  pytest
   mock
   fudge
   nose