From 23128391986bf89b29fad14e0f26e9cb6ecf5cc3 Mon Sep 17 00:00:00 2001 From: Volker Theile Date: Tue, 24 Jul 2018 11:52:29 +0200 Subject: [PATCH] mgr/dashboard: Set timeout in RestClient calls Set a default timeout of 45 seconds to all REST client calls. This can be customized via 'ceph dashboard set-rest-requests-timeout '. Currently the REST client is only used by the RGW controller. Signed-off-by: Volker Theile --- doc/mgr/dashboard.rst | 9 +++- src/pybind/mgr/dashboard/rest_client.py | 23 +++++++++- src/pybind/mgr/dashboard/settings.py | 1 + .../mgr/dashboard/tests/test_rest_client.py | 44 +++++++++++++++++++ 4 files changed, 74 insertions(+), 3 deletions(-) create mode 100644 src/pybind/mgr/dashboard/tests/test_rest_client.py diff --git a/doc/mgr/dashboard.rst b/doc/mgr/dashboard.rst index 7950758ab9060..c521a24a6442a 100644 --- a/doc/mgr/dashboard.rst +++ b/doc/mgr/dashboard.rst @@ -190,7 +190,7 @@ commands:: Enabling the Object Gateway management frontend -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ To use the Object Gateway management functionality of the dashboard, you will need to provide the login credentials of a user with the ``system`` flag @@ -232,6 +232,13 @@ exist and you may find yourself in the situation that you have to use them:: $ ceph dashboard set-rgw-api-admin-resource $ ceph dashboard set-rgw-api-user-id +If the Object Gateway takes too long to process requests and the dashboard runs +into timeouts, then you can set the timeout value to your needs:: + + $ ceph dashboard set-rest-requests-timeout + +The default value is 45 seconds. + Enabling the Embedding of Grafana Dashboards ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/src/pybind/mgr/dashboard/rest_client.py b/src/pybind/mgr/dashboard/rest_client.py index 3105545fc3beb..6099fe945254f 100644 --- a/src/pybind/mgr/dashboard/rest_client.py +++ b/src/pybind/mgr/dashboard/rest_client.py @@ -13,11 +13,12 @@ """ from __future__ import absolute_import +from .settings import Settings from .tools import build_url import inspect import re import requests -from requests.exceptions import ConnectionError, InvalidURL +from requests.exceptions import ConnectionError, InvalidURL, Timeout from . import logger try: @@ -26,6 +27,18 @@ except ImportError: from urllib3.exceptions import SSLError +class TimeoutRequestsSession(requests.Session): + """ + Set timeout argument for all requests if this is not already done. + """ + def request(self, *args, **kwargs): + if ((args[8] if len(args) > 8 else None) is None) \ + and kwargs.get('timeout') is None: + if Settings.REST_REQUESTS_TIMEOUT > 0: + kwargs['timeout'] = Settings.REST_REQUESTS_TIMEOUT + return super(TimeoutRequestsSession, self).request(*args, **kwargs) + + class RequestException(Exception): def __init__(self, message, @@ -315,7 +328,7 @@ class RestClient(object): logger.debug("REST service base URL: %s", self.base_url) self.headers = {'Accept': 'application/json'} self.auth = auth - self.session = requests.Session() + self.session = TimeoutRequestsSession() def _login(self, request=None): pass @@ -465,6 +478,12 @@ class RestClient(object): logger.exception("%s REST API failed %s: %s", self.client_name, method.upper(), str(ex)) raise RequestException(str(ex)) + except Timeout as ex: + msg = "{} REST API {} timed out after {} seconds (url={}).".format( + self.client_name, ex.request.method, Settings.REST_REQUESTS_TIMEOUT, + ex.request.url) + logger.exception(msg) + raise RequestException(msg) @staticmethod def api(path, **api_kwargs): diff --git a/src/pybind/mgr/dashboard/settings.py b/src/pybind/mgr/dashboard/settings.py index 4d1153e9e283b..16426f48bc23a 100644 --- a/src/pybind/mgr/dashboard/settings.py +++ b/src/pybind/mgr/dashboard/settings.py @@ -19,6 +19,7 @@ class Options(object): GRAFANA_API_PORT = (3000, int) """ ENABLE_BROWSABLE_API = (True, bool) + REST_REQUESTS_TIMEOUT = (45, int) # RGW settings RGW_API_HOST = ('', str) diff --git a/src/pybind/mgr/dashboard/tests/test_rest_client.py b/src/pybind/mgr/dashboard/tests/test_rest_client.py new file mode 100644 index 0000000000000..7a15319ff9ece --- /dev/null +++ b/src/pybind/mgr/dashboard/tests/test_rest_client.py @@ -0,0 +1,44 @@ +# -*- coding: utf-8 -*- +import unittest + +from mock import patch +from .. import mgr +from ..rest_client import RestClient + + +class RestClientTest(unittest.TestCase): + def setUp(self): + settings = {'REST_REQUESTS_TIMEOUT': 45} + mgr.get_config.side_effect = settings.get + + def test_timeout_auto_set(self): + with patch('requests.Session.request') as mock_request: + rest_client = RestClient('localhost', 8000) + rest_client.session.request('GET', '/test') + mock_request.assert_called_with('GET', '/test', timeout=45) + + def test_timeout_auto_set_arg(self): + with patch('requests.Session.request') as mock_request: + rest_client = RestClient('localhost', 8000) + rest_client.session.request( + 'GET', '/test', None, None, None, None, + None, None, None) + mock_request.assert_called_with( + 'GET', '/test', None, None, None, None, + None, None, None, timeout=45) + + def test_timeout_no_auto_set_kwarg(self): + with patch('requests.Session.request') as mock_request: + rest_client = RestClient('localhost', 8000) + rest_client.session.request('GET', '/test', timeout=20) + mock_request.assert_called_with('GET', '/test', timeout=20) + + def test_timeout_no_auto_set_arg(self): + with patch('requests.Session.request') as mock_request: + rest_client = RestClient('localhost', 8000) + rest_client.session.request( + 'GET', '/test', None, None, None, None, + None, None, 40) + mock_request.assert_called_with( + 'GET', '/test', None, None, None, None, + None, None, 40) -- 2.39.5