From f3906070151993345168187341a4cf60972b66f0 Mon Sep 17 00:00:00 2001 From: Patrick Donnelly Date: Sat, 29 Sep 2018 17:20:03 -0700 Subject: [PATCH] run: do not block on greenlets after command exits The stdout/stderr greenlets will not necessarily exit when the command does if child processes are stuck in an uninterruptible sleep. For example, the fsx.sh workunit spawns fsx processes that may be left behind in the D state after /bin/timeout kills fsx.sh. These are connected to the stdout/stderr pipes which prevent the greenlets from exiting normally. Signed-off-by: Patrick Donnelly --- teuthology/orchestra/run.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/teuthology/orchestra/run.py b/teuthology/orchestra/run.py index 50d01ae746..7a0fe16065 100644 --- a/teuthology/orchestra/run.py +++ b/teuthology/orchestra/run.py @@ -139,10 +139,16 @@ class RemoteProcess(object): :returns: self.returncode """ - for greenlet in self.greenlets: - greenlet.get() status = self._get_exitstatus() + if status != 0: + log.debug("got remote process result: {}".format(status)) + for greenlet in self.greenlets: + try: + greenlet.get(block=True,timeout=60) + except gevent.Timeout: + log.debug("timed out waiting; will kill: {}".format(greenlet)) + greenlet.kill(block=False) for stream in ('stdout', 'stderr'): if hasattr(self, stream): stream_obj = getattr(self, stream) -- 2.39.5