From: Patrick Donnelly Date: Sun, 30 Sep 2018 00:20:03 +0000 (-0700) Subject: run: do not block on greenlets after command exits X-Git-Tag: 1.1.0~304^2 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=refs%2Fpull%2F1216%2Fhead;p=teuthology.git run: do not block on greenlets after command exits The stdout/stderr greenlets will not necessarily exit when the command does if child processes are stuck in an uninterruptible sleep. For example, the fsx.sh workunit spawns fsx processes that may be left behind in the D state after /bin/timeout kills fsx.sh. These are connected to the stdout/stderr pipes which prevent the greenlets from exiting normally. Signed-off-by: Patrick Donnelly --- diff --git a/teuthology/orchestra/run.py b/teuthology/orchestra/run.py index 50d01ae74..7a0fe1606 100644 --- a/teuthology/orchestra/run.py +++ b/teuthology/orchestra/run.py @@ -139,10 +139,16 @@ class RemoteProcess(object): :returns: self.returncode """ - for greenlet in self.greenlets: - greenlet.get() status = self._get_exitstatus() + if status != 0: + log.debug("got remote process result: {}".format(status)) + for greenlet in self.greenlets: + try: + greenlet.get(block=True,timeout=60) + except gevent.Timeout: + log.debug("timed out waiting; will kill: {}".format(greenlet)) + greenlet.kill(block=False) for stream in ('stdout', 'stderr'): if hasattr(self, stream): stream_obj = getattr(self, stream)