From: Adam King <adking@redhat.com>
Date: Wed, 7 Jun 2023 14:33:13 +0000 (-0400)
Subject: mgr/cephadm: Also catch concurrent.futures.TimeoutError for timeouts
X-Git-Tag: testing/wip-batrick-testing-20240411.154038~667^2~2
X-Git-Url: http://git.apps.os.sepia.ceph.com/?a=commitdiff_plain;h=4df348c17ba0c72d2ecf08d3607b70b3ae253176;p=ceph-ci.git

mgr/cephadm: Also catch concurrent.futures.TimeoutError for timeouts

On python 3.6 which Ceph currently uses for its
container builds (which are based on centos 8 stream builds
hence the python version) the exception raised by a timeout
from a concurrent.futures.Future is successfully caught by
looking for asyncio.TimeoutError. However, in builds with
later python versions, e.g. 3.9.16, the timeout is no
longer caught. This results in situations like

Traceback (most recent call last):
  File "/usr/share/ceph/mgr/cephadm/utils.py", line 79, in do_work
    return f(*arg)
  File "/usr/share/ceph/mgr/cephadm/serve.py", line 241, in refresh
    r = self._refresh_host_devices(host)
  File "/usr/share/ceph/mgr/cephadm/serve.py", line 352, in _refresh_host_devices
    devices = self.mgr.wait_async(self._run_cephadm_json(
  File "/usr/share/ceph/mgr/cephadm/module.py", line 635, in wait_async
    return self.event_loop.get_result(coro, timeout)
  File "/usr/share/ceph/mgr/cephadm/ssh.py", line 63, in get_result
    return future.result(timeout)
  File "/lib64/python3.9/concurrent/futures/_base.py", line 448, in result
    raise TimeoutError()
concurrent.futures._base.TimeoutError

which causes the cephadm module to crash whenever one of these
command timeouts happen. This patch is to also catch the
newer exception type so it works on later python versions as well

Signed-off-by: Adam King <adking@redhat.com>
---

diff --git a/src/pybind/mgr/cephadm/module.py b/src/pybind/mgr/cephadm/module.py
index de0f49fb2b4..46848add150 100644
--- a/src/pybind/mgr/cephadm/module.py
+++ b/src/pybind/mgr/cephadm/module.py
@@ -1,4 +1,5 @@
 import asyncio
+import concurrent
 import json
 import errno
 import ipaddress
@@ -724,7 +725,7 @@ class CephadmOrchestrator(orchestrator.Orchestrator, MgrModule,
         # are provided, that will be included in the OrchestratorError's message
         try:
             yield
-        except asyncio.TimeoutError:
+        except (asyncio.TimeoutError, concurrent.futures.TimeoutError):
             err_str: str = ''
             if cmd:
                 err_str = f'Command "{cmd}" timed out '
diff --git a/src/pybind/mgr/cephadm/ssh.py b/src/pybind/mgr/cephadm/ssh.py
index d17cc0fcc19..7460fc159d7 100644
--- a/src/pybind/mgr/cephadm/ssh.py
+++ b/src/pybind/mgr/cephadm/ssh.py
@@ -1,6 +1,7 @@
 import logging
 import os
 import asyncio
+import concurrent
 from tempfile import NamedTemporaryFile
 from threading import Thread
 from contextlib import contextmanager
@@ -61,7 +62,7 @@ class EventLoopThread(Thread):
         future = asyncio.run_coroutine_threadsafe(coro, self._loop)
         try:
             return future.result(timeout)
-        except asyncio.TimeoutError:
+        except (asyncio.TimeoutError, concurrent.futures.TimeoutError):
             # try to cancel the task before raising the exception further up
             future.cancel()
             raise