task/mpi: Explicit check for version mismatch

Instead of proceeding to have mpiexec fail or hang,
do an up-front check that the MPI version is the same
on all of the nodes in the test.

Signed-off-by: John Spray <john.spray@redhat.com>

diff --git a/teuthology/task/mpi.py b/teuthology/task/mpi.py
index 6d2381ee58e09a28b5425dd061db36314403f07a..e92d9390337184eb198205aa83fe8ac68b6377f0 100644 (file)
--- a/teuthology/task/mpi.py
+++ b/teuthology/task/mpi.py
@@ -1,12 +1,36 @@
 """
 Start mpi processes (and allow commands to be run inside process)
 """
+from StringIO import StringIO
 import logging
+import re
 
 from teuthology import misc as teuthology
 
 log = logging.getLogger(__name__)
 
+
+def _check_mpi_version(remotes):
+    """
+    Retrieve the MPI version from each of `remotes` and raise an exception
+    if they are not all the same version.
+    """
+    versions = set()
+    for remote in remotes:
+        version_str = remote.run(args=["mpiexec", "--version"], stdout=StringIO()).stdout.getvalue()
+        try:
+            version = re.search("^\s+Version:\s+(.+)$", version_str, re.MULTILINE).group(1)
+        except AttributeError:
+            raise RuntimeError("Malformed MPI version output: {0}".format(version_str))
+        else:
+            versions.add(version)
+
+    if len(versions) != 1:
+        raise RuntimeError("MPI version mismatch.  Versions are: {0}".format(", ".join(versions)))
+    else:
+        log.info("MPI version {0}".format(list(versions)[0]))
+
+
 def task(ctx, config):
     """
     Setup MPI and execute commands
@@ -91,6 +115,9 @@ def task(ctx, config):
             hosts.append(ip)
             remotes.append(remote)
 
+    # mpich is sensitive to different versions on different nodes
+    _check_mpi_version(remotes)
+
     workdir = []
     if 'workdir' in config:
         workdir = ['-wdir', config['workdir'].replace('$TESTDIR', testdir) ]