self._session = requests.Session()
self._resp = None
- def _request(self, url, method, data=None):
+ def _request(self, url, method, data=None, params=None):
url = "{}{}".format(self.base_uri, url)
+
log.info("request %s to %s", method, url)
if method == 'GET':
- self._resp = self._session.get(url)
+ self._resp = self._session.get(url, params=params)
try:
return self._resp.json()
except ValueError as ex:
log.exception("Failed to decode response: %s", self._resp.text)
raise ex
elif method == 'POST':
- self._resp = self._session.post(url, json=data)
+ self._resp = self._session.post(url, json=data, params=params)
elif method == 'DELETE':
- self._resp = self._session.delete(url, json=data)
+ self._resp = self._session.delete(url, json=data, params=params)
elif method == 'PUT':
- self._resp = self._session.put(url, json=data)
+ self._resp = self._session.put(url, json=data, params=params)
else:
assert False
+ return None
- def _get(self, url):
- return self._request(url, 'GET')
+ def _get(self, url, params=None):
+ return self._request(url, 'GET', params=params)
- def _post(self, url, data=None):
- self._request(url, 'POST', data)
+ def _post(self, url, data=None, params=None):
+ self._request(url, 'POST', data, params=params)
- def _delete(self, url, data=None):
- self._request(url, 'DELETE', data)
+ def _delete(self, url, data=None, params=None):
+ self._request(url, 'DELETE', data, params=params)
- def _put(self, url, data=None):
- self._request(url, 'PUT', data)
+ def _put(self, url, data=None, params=None):
+ self._request(url, 'PUT', data, params=params)
def cookies(self):
return self._resp.cookies
args.extend(cmd)
cls.mgr_cluster.admin_remote.run(args=args)
+ @classmethod
+ def _radosgw_admin_cmd(cls, cmd):
+ args = [
+ 'radosgw-admin'
+ ]
+ args.extend(cmd)
+ cls.mgr_cluster.admin_remote.run(args=args)
+
@classmethod
def mons(cls):
out = cls.ceph_cluster.mon_manager.raw_cluster_cmd('mon_status')
# -*- coding: utf-8 -*-
from __future__ import absolute_import
+import urllib
+import logging
+logger = logging.getLogger(__name__)
from .helper import DashboardTestCase, authenticate
class RgwControllerTest(DashboardTestCase):
-
@authenticate
def test_rgw_daemon_list(self):
data = self._get('/api/rgw/daemon')
self.assertIn('rgw_id', data)
self.assertIn('rgw_status', data)
self.assertTrue(data['rgw_metadata'])
+
+class RgwProxyExceptionsTest(DashboardTestCase):
+
+ @classmethod
+ def setUpClass(cls):
+ super(RgwProxyExceptionsTest, cls).setUpClass()
+
+ cls._ceph_cmd(['dashboard', 'set-rgw-api-secret-key', ''])
+ cls._ceph_cmd(['dashboard', 'set-rgw-api-access-key', ''])
+
+ @authenticate
+ def test_no_credentials_exception(self):
+ resp = self._get('/api/rgw/proxy/status')
+ self.assertStatus(401)
+ self.assertIn('message', resp)
+
+
+class RgwProxyTest(DashboardTestCase):
+ @classmethod
+ def setUpClass(cls):
+ super(RgwProxyTest, cls).setUpClass()
+ cls._radosgw_admin_cmd([
+ 'user', 'create', '--uid=admin', '--display-name=admin',
+ '--system', '--access-key=admin', '--secret=admin'
+ ])
+ cls._ceph_cmd(['dashboard', 'set-rgw-api-secret-key', 'admin'])
+ cls._ceph_cmd(['dashboard', 'set-rgw-api-access-key', 'admin'])
+
+ def _assert_user_data(self, data):
+ self.assertIn('caps', data)
+ self.assertIn('display_name', data)
+ self.assertIn('email', data)
+ self.assertIn('keys', data)
+ self.assertGreaterEqual(len(data['keys']), 1)
+ self.assertIn('max_buckets', data)
+ self.assertIn('subusers', data)
+ self.assertIn('suspended', data)
+ self.assertIn('swift_keys', data)
+ self.assertIn('tenant', data)
+ self.assertIn('user_id', data)
+
+ def _test_put(self):
+ self._put(
+ '/api/rgw/proxy/user',
+ params={
+ 'uid': 'teuth-test-user',
+ 'display-name': 'display name',
+ })
+ data = self._resp.json()
+
+ self._assert_user_data(data)
+ self.assertStatus(200)
+
+ data = self._get(
+ '/api/rgw/proxy/user', params={'uid': 'teuth-test-user'})
+
+ self.assertStatus(200)
+ self.assertEqual(data['user_id'], 'teuth-test-user')
+
+ def _test_get(self):
+ data = self._get(
+ '/api/rgw/proxy/user', params={'uid': 'teuth-test-user'})
+
+ self._assert_user_data(data)
+ self.assertStatus(200)
+ self.assertEquals(data['user_id'], 'teuth-test-user')
+
+ def _test_post(self):
+ """Updates the user"""
+ self._post(
+ '/api/rgw/proxy/user',
+ params={
+ 'uid': 'teuth-test-user',
+ 'display-name': 'new name'
+ })
+
+ self.assertStatus(200)
+ self._assert_user_data(self._resp.json())
+ self.assertEqual(self._resp.json()['display_name'], 'new name')
+
+ def _test_delete(self):
+ self._delete('/api/rgw/proxy/user', params={'uid': 'teuth-test-user'})
+ self.assertStatus(200)
+
+ self._delete('/api/rgw/proxy/user', params={'uid': 'teuth-test-user'})
+ self.assertStatus(404)
+ resp = self._resp.json()
+ self.assertIn('Code', resp)
+ self.assertIn('HostId', resp)
+ self.assertIn('RequestId', resp)
+ self.assertEqual(resp['Code'], 'NoSuchUser')
+
+ @authenticate
+ def test_rgw_proxy(self):
+ """Test basic request types"""
+ self.maxDiff = None
+
+ # PUT - Create a user
+ self._test_put()
+
+ # GET - Get the user details
+ self._test_get()
+
+ # POST - Update the user details
+ self._test_post()
+
+ # DELETE - Delete the user
+ self._test_delete()
The Dashboard's WebUI should then be reachable on TCP port 8080.
+Enabling the Object Gateway management frontend
+-----------------------------------------------
+
+If you want to use the Object Gateway management functionality of the
+dashboard, you will need to provide credentials. If you do not have a user
+which shall be used for providing those credentials, you will also need to
+create one::
+
+ $ radosgw-admin user create --uuid=<user> --display-name=<display-name> \
+ --system
+
+The credentials of a user can also be obtained by using `radosgw-admin`::
+
+ $ radosgw-admin user info --uid=<user>
+
+Finally, set the credentials to the dashboard module::
+
+ $ ceph dashboard set-rgw-api-secret-key <secret_key>
+ $ ceph dashboard set-rgw-api-access-key <access_key>
+
+This is all you have to do to get the Object Gateway management functionality
+working. The host and port of the Object Gateway are determined automatically.
+If multiple zones are used, it will automatically determine the host within the
+master zone group and master zone. This should be sufficient for most setups,
+but in some circumstances you might want to set the host and port manually::
+
+ $ ceph dashboard set-rgw-api-host <host>
+ $ ceph dashboard set-rgw-api-port <port>
+
+In addition to the settings mentioned so far, the following settings do also
+exist and you may find yourself in the situation that you have to use them::
+
+ $ ceph dashboard set-rgw-api-scheme <scheme> # http or https
+ $ ceph dashboard set-rgw-api-admin-resource <admin-resource>
+ $ ceph dashboard set-rgw-api-user-id <user-id>
+
Working on the Dashboard Code
-----------------------------
# -*- coding: utf-8 -*-
-<<<<<<< HEAD
# pylint: disable-all
-=======
->>>>>>> 60417c2dcc... dashboard/mgr: RGW proxy: Include `python-requests-aws`
#
# Copyright (c) 2012-2013 Paul Tax <paultax@gmail.com> All rights reserved.
#
import json
-from . import ApiController, RESTController, AuthRequired
+import cherrypy
+
from .. import logger
from ..services.ceph_service import CephService
+from ..tools import ApiController, RESTController, AuthRequired
+from ..services.rgw_client import RgwClient
+from ..rest_client import RequestException
+from ..exceptions import NoCredentialsException
@ApiController('rgw')
@ApiController('rgw/daemon')
@AuthRequired()
class RgwDaemon(RESTController):
-
def list(self):
daemons = []
for hostname, server in CephService.get_service_map('rgw').items():
for service in server['services']:
metadata = service['metadata']
+ status = service['status']
+ if 'json' in status:
+ try:
+ status = json.loads(status['json'])
+ except ValueError:
+ logger.warning("%s had invalid status json", service['id'])
+ status = {}
+ else:
+ logger.warning('%s has no key "json" in status', service['id'])
# extract per-daemon service data and health
daemon = {
daemon['rgw_metadata'] = metadata
daemon['rgw_status'] = status
+
return daemon
+
+
+@ApiController('rgw/proxy')
+@AuthRequired()
+class RgwProxy(RESTController):
+ @cherrypy.expose
+ def default(self, *vpath, **params):
+ try:
+ rgw_client = RgwClient.admin_instance()
+
+ except NoCredentialsException as e:
+ cherrypy.response.headers['Content-Type'] = 'application/json'
+ cherrypy.response.status = 401
+ return json.dumps({'message': e.message})
+
+ method = cherrypy.request.method
+ path = '/'.join(vpath)
+ data = None
+
+ if cherrypy.request.body.length:
+ data = cherrypy.request.body.read()
+
+ try:
+ return rgw_client.proxy(method, path, params, data)
+ except RequestException as e:
+ cherrypy.response.status = e.status_code
+ return e.content
--- /dev/null
+# -*- coding: utf-8 -*-
+from __future__ import absolute_import
+
+
+class NoCredentialsException(Exception):
+ def __init__(self):
+ super(Exception, self).__init__(
+ 'No RGW credentials found, '
+ 'please consult the documentation on how to enable RGW for '
+ 'the dashboard.')
--- /dev/null
+# -*- coding: utf-8 -*-
+"""
+ * Copyright (c) 2017 SUSE LLC
+ *
+ * openATTIC is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; version 2.
+ *
+ * This package is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+"""
+from __future__ import absolute_import
+
+from .tools import build_url
+import inspect
+import itertools
+import re
+import requests
+from requests.exceptions import ConnectionError, InvalidURL
+from . import logger
+
+try:
+ from requests.packages.urllib3.exceptions import SSLError
+except ImportError:
+ from urllib3.exceptions import SSLError
+
+
+class RequestException(Exception):
+ def __init__(self,
+ message,
+ status_code=None,
+ content=None,
+ conn_errno=None,
+ conn_strerror=None):
+ super(RequestException, self).__init__(message)
+ self.status_code = status_code
+ self.content = content
+ self.conn_errno = conn_errno
+ self.conn_strerror = conn_strerror
+
+
+class BadResponseFormatException(RequestException):
+ def __init__(self, message):
+ super(BadResponseFormatException, self).__init__(
+ "Bad response format" if message is None else message, None)
+
+
+class _ResponseValidator(object):
+ """Simple JSON schema validator
+
+ This class implements a very simple validator for the JSON formatted
+ messages received by request responses from a RestClient instance.
+
+ The validator validates the JSON response against a "structure" string that
+ specifies the structure that the JSON response must comply. The validation
+ procedure raises a BadResponseFormatException in case of a validation
+ failure.
+
+ The structure syntax is given by the following grammar:
+
+ Structure ::= Level
+ Level ::= Path | Path '&' Level
+ Path ::= Step | Step '>'+ Path
+ Step ::= Key | '?' Key | '*' | '(' Level ')'
+ Key ::= <string> | Array+
+ Array ::= '[' <int> ']' | '[' '*' ']' | '[' '+' ']'
+
+ The symbols enclosed in ' ' are tokens of the language, and the + symbol
+ denotes repetition of of the preceding token at least once.
+
+ Examples of usage:
+
+ Example 1:
+ Validator args:
+ structure = "return > *"
+ response = { 'return': { ... } }
+
+ In the above example the structure will validate against any response
+ that contains a key named "return" in the root of the response
+ dictionary and its value is also a dictionary.
+
+ Example 2:
+ Validator args:
+ structure = "[*]"
+ response = [...]
+
+ In the above example the structure will validate against any response
+ that is an array of any size.
+
+ Example 3:
+ Validator args:
+ structure = "return[*]"
+ response = { 'return': [....] }
+
+ In the above example the structure will validate against any response
+ that contains a key named "return" in the root of the response
+ dictionary and its value is an array.
+
+ Example 4:
+ Validator args:
+ structure = "return[0] > token"
+ response = { 'return': [ { 'token': .... } ] }
+
+ In the above example the structure will validate against any response
+ that contains a key named "return" in the root of the response
+ dictionary and its value is an array, and the first element of the
+ array is a dictionary that contains the key 'token'.
+
+ Example 5:
+ Validator args:
+ structure = "return[0][*] > key1"
+ response = { 'return': [ [ { 'key1': ... } ], ...] }
+
+ In the above example the structure will validate against any response
+ that contains a key named "return" in the root of the response
+ dictionary where its value is an array, and the first value of this
+ array is also an array where all it's values must be a dictionary
+ containing a key named "key1".
+
+ Example 6:
+ Validator args:
+ structure = "return > (key1[*] & key2 & ?key3 > subkey)"
+ response = { 'return': { 'key1': [...], 'key2: .... } ] }
+
+ In the above example the structure will validate against any response
+ that contains a key named "return" in the root of the response
+ dictionary and its value is a dictionary that must contain a key named
+ "key1" that is an array, a key named "key2", and optionaly a key named
+ "key3" that is a dictionary that contains a key named "subkey".
+
+ Example 7:
+ Validator args:
+ structure = "return >> roles[*]"
+ response = { 'return': { 'key1': { 'roles': [...] }, 'key2': { 'roles': [...] } } }
+
+ In the above example the structure will validate against any response
+ that contains a key named "return" in the root of the response
+ dictionary, and its value is a dictionary that for any key present in
+ the dictionary their value is also a dictionary that must contain a key
+ named 'roles' that is an array. Please note that you can use any
+ number of successive '>' to denote the level in the JSON tree that you
+ want to match next step in the path.
+
+ """
+
+ @staticmethod
+ def validate(structure, response):
+ if structure is None:
+ return
+
+ _ResponseValidator._validate_level(structure, response)
+
+ @staticmethod
+ def _validate_level(level, resp):
+ if not isinstance(resp, dict) and not isinstance(resp, list):
+ raise BadResponseFormatException(
+ "{} is neither a dict nor a list".format(resp))
+
+ paths = _ResponseValidator._parse_level_paths(level)
+ for path in paths:
+ path_sep = path.find('>')
+ if path_sep != -1:
+ level_next = path[path_sep + 1:].strip()
+ else:
+ path_sep = len(path)
+ level_next = None
+ key = path[:path_sep].strip()
+
+ if key == '*':
+ continue
+ elif key == '': # check all keys
+ for k in resp.keys():
+ _ResponseValidator._validate_key(k, level_next, resp)
+ else:
+ _ResponseValidator._validate_key(key, level_next, resp)
+
+ @staticmethod
+ def _validate_array(array_seq, level_next, resp):
+ if array_seq:
+ if not isinstance(resp, list):
+ raise BadResponseFormatException(
+ "{} is not an array".format(resp))
+ if array_seq[0].isdigit():
+ idx = int(array_seq[0])
+ if len(resp) <= idx:
+ raise BadResponseFormatException(
+ "length of array {} is lower than the index {}".format(
+ resp, idx))
+ _ResponseValidator._validate_array(array_seq[1:], level_next,
+ resp[idx])
+ elif array_seq[0] == '*':
+ for r in resp:
+ _ResponseValidator._validate_array(array_seq[1:],
+ level_next, r)
+ elif array_seq[0] == '+':
+ if len(resp) < 1:
+ raise BadResponseFormatException(
+ "array should not be empty")
+ for r in resp:
+ _ResponseValidator._validate_array(array_seq[1:],
+ level_next, r)
+ else:
+ raise Exception(
+ "Response structure is invalid: only <int> | '*' are "
+ "allowed as array index arguments")
+ else:
+ if level_next:
+ _ResponseValidator._validate_level(level_next, resp)
+
+ @staticmethod
+ def _validate_key(key, level_next, resp):
+ array_access = [a.strip() for a in key.split("[")]
+ key = array_access[0]
+ if key:
+ optional = key[0] == '?'
+ if optional:
+ key = key[1:]
+ if key not in resp:
+ if optional:
+ return
+ raise BadResponseFormatException(
+ "key {} is not in dict {}".format(key, resp))
+ resp_next = resp[key]
+ else:
+ resp_next = resp
+ if len(array_access) > 1:
+ _ResponseValidator._validate_array(
+ [a[0:-1] for a in array_access[1:]], level_next, resp_next)
+ else:
+ if level_next:
+ _ResponseValidator._validate_level(level_next, resp_next)
+
+ @staticmethod
+ def _parse_level_paths(level):
+ level = level.strip()
+ if level[0] == '(':
+ level = level[1:]
+ if level[-1] == ')':
+ level = level[:-1]
+
+ paths = []
+ lp = 0
+ nested = 0
+ for i, c in enumerate(level):
+ if c == '&' and nested == 0:
+ paths.append(level[lp:i].strip())
+ lp = i + 1
+ elif c == '(':
+ nested += 1
+ elif c == ')':
+ nested -= 1
+ paths.append(level[lp:].strip())
+ return paths
+
+
+class _Request(object):
+ def __init__(self, method, path, path_params, rest_client, resp_structure):
+ self.method = method
+ self.path = path
+ self.path_params = path_params
+ self.rest_client = rest_client
+ self.resp_structure = resp_structure
+
+ def _gen_path(self):
+ new_path = self.path
+ matches = re.finditer(r'\{(\w+?)\}', self.path)
+ for match in matches:
+ if match:
+ param_key = match.group(1)
+ if param_key in self.path_params:
+ new_path = new_path.replace(
+ match.group(0), self.path_params[param_key])
+ else:
+ raise RequestException(
+ 'Invalid path. Param "{}" was not specified'
+ .format(param_key), None)
+ return new_path
+
+ def __call__(self,
+ req_data=None,
+ method=None,
+ params=None,
+ data=None,
+ raw_content=False):
+ method = method if method else self.method
+ if not method:
+ raise Exception('No HTTP request method specified')
+ if req_data:
+ if method == 'get':
+ if params:
+ raise Exception('Ambiguous source of GET params')
+ params = req_data
+ else:
+ if data:
+ raise Exception('Ambiguous source of {} data'.format(
+ method.upper()))
+ data = req_data
+ resp = self.rest_client.do_request(method, self._gen_path(), params,
+ data, raw_content)
+ if raw_content and self.resp_structure:
+ raise Exception("Cannot validate reponse in raw format")
+ _ResponseValidator.validate(self.resp_structure, resp)
+ return resp
+
+
+class RestClient(object):
+ def __init__(self, host, port, client_name=None, ssl=False, auth=None):
+ super(RestClient, self).__init__()
+ self.client_name = client_name if client_name else ''
+ self.host = host
+ self.port = port
+ self.base_url = build_url(
+ scheme='https' if ssl else 'http', host=host, port=port)
+ logger.debug("REST service base URL: %s", self.base_url)
+ self.headers = {'Accept': 'application/json'}
+ self.auth = auth
+ self.session = requests.Session()
+
+ def _login(self, request=None):
+ pass
+
+ def _is_logged_in(self):
+ pass
+
+ def _reset_login(self):
+ pass
+
+ def is_service_online(self, request=None):
+ pass
+
+ @staticmethod
+ def requires_login(func):
+ def func_wrapper(self, *args, **kwargs):
+ retries = 2
+ while True:
+ try:
+ if not self._is_logged_in():
+ self._login()
+ resp = func(self, *args, **kwargs)
+ return resp
+ except RequestException as e:
+ if isinstance(e, BadResponseFormatException):
+ raise e
+ retries -= 1
+ if e.status_code not in [401, 403] or retries == 0:
+ raise e
+ self._reset_login()
+
+ return func_wrapper
+
+ def do_request(self,
+ method,
+ path,
+ params=None,
+ data=None,
+ raw_content=False):
+ url = '{}{}'.format(self.base_url, path)
+ logger.debug('%s REST API %s req: %s data: %s', self.client_name,
+ method.upper(), path, data)
+ try:
+ if method.lower() == 'get':
+ resp = self.session.get(
+ url, headers=self.headers, params=params, auth=self.auth)
+ elif method.lower() == 'post':
+ resp = self.session.post(
+ url,
+ headers=self.headers,
+ params=params,
+ data=data,
+ auth=self.auth)
+ elif method.lower() == 'put':
+ resp = self.session.put(
+ url,
+ headers=self.headers,
+ params=params,
+ data=data,
+ auth=self.auth)
+ elif method.lower() == 'delete':
+ resp = self.session.delete(
+ url,
+ headers=self.headers,
+ params=params,
+ data=data,
+ auth=self.auth)
+ else:
+ raise RequestException('Method "{}" not supported'.format(
+ method.upper()), None)
+ if resp.ok:
+ logger.debug("%s REST API %s res status: %s content: %s",
+ self.client_name, method.upper(),
+ resp.status_code, resp.text)
+ if raw_content:
+ return resp.content
+ try:
+ return resp.json() if resp.text else None
+ except ValueError:
+ logger.error(
+ "%s REST API failed %s req while decoding JSON "
+ "response : %s",
+ self.client_name, method.upper(), resp.text)
+ raise RequestException(
+ "{} REST API failed request while decoding JSON "
+ "response: {}".format(self.client_name, resp.text),
+ resp.status_code, resp.text)
+ else:
+ logger.error(
+ "%s REST API failed %s req status: %s", self.client_name,
+ method.upper(), resp.status_code)
+ from pprint import pprint as pp
+ from pprint import pformat as pf
+
+ raise RequestException(
+ "{} REST API failed request with status code {}\n"
+ "{}" # TODO remove
+ .format(self.client_name, resp.status_code, pf(
+ resp.content)),
+ resp.status_code,
+ resp.content)
+ except ConnectionError as ex:
+ if ex.args:
+ if isinstance(ex.args[0], SSLError):
+ errno = "n/a"
+ strerror = "SSL error. Probably trying to access a non " \
+ "SSL connection."
+ logger.error("%s REST API failed %s, SSL error.",
+ self.client_name, method.upper())
+ else:
+ match = re.match(r'.*: \[Errno (-?\d+)\] (.+)',
+ ex.args[0].reason.args[0])
+ if match:
+ errno = match.group(1)
+ strerror = match.group(2)
+ logger.error(
+ "%s REST API failed %s, connection error: "
+ "[errno: %s] %s",
+ self.client_name, method.upper(), errno, strerror)
+ else:
+ errno = "n/a"
+ strerror = "n/a"
+ logger.error(
+ "%s REST API failed %s, connection error.",
+ self.client_name, method.upper())
+ else:
+ errno = "n/a"
+ strerror = "n/a"
+ logger.error("%s REST API failed %s, connection error.",
+ self.client_name, method.upper())
+
+ if errno != "n/a":
+ ex_msg = (
+ "{} REST API cannot be reached: {} [errno {}]. "
+ "Please check your configuration and that the API endpoint"
+ " is accessible"
+ .format(self.client_name, strerror, errno))
+ else:
+ ex_msg = (
+ "{} REST API cannot be reached. Please check "
+ "your configuration and that the API endpoint is"
+ " accessible"
+ .format(self.client_name))
+ raise RequestException(
+ ex_msg, conn_errno=errno, conn_strerror=strerror)
+ except InvalidURL as ex:
+ logger.exception("%s REST API failed %s: %s", self.client_name,
+ method.upper(), str(ex))
+ raise RequestException(str(ex))
+
+ @staticmethod
+ def api(path, **api_kwargs):
+ def call_decorator(func):
+ def func_wrapper(self, *args, **kwargs):
+ method = api_kwargs.get('method', None)
+ resp_structure = api_kwargs.get('resp_structure', None)
+ args_name = inspect.getargspec(func).args
+ args_dict = dict(itertools.izip(args_name[1:], args))
+ for key, val in kwargs:
+ args_dict[key] = val
+ return func(
+ self,
+ *args,
+ request=_Request(method, path, args_dict, self,
+ resp_structure),
+ **kwargs)
+
+ return func_wrapper
+
+ return call_decorator
+
+ @staticmethod
+ def api_get(path, resp_structure=None):
+ return RestClient.api(
+ path, method='get', resp_structure=resp_structure)
+
+ @staticmethod
+ def api_post(path, resp_structure=None):
+ return RestClient.api(
+ path, method='post', resp_structure=resp_structure)
+
+ @staticmethod
+ def api_put(path, resp_structure=None):
+ return RestClient.api(
+ path, method='put', resp_structure=resp_structure)
+
+ @staticmethod
+ def api_delete(path, resp_structure=None):
+ return RestClient.api(
+ path, method='delete', resp_structure=resp_structure)
--- /dev/null
+# -*- coding: utf-8 -*-
+from __future__ import absolute_import
+
+import re
+from ..awsauth import S3Auth
+from ..settings import Settings, Options
+from ..rest_client import RestClient, RequestException
+from ..tools import build_url, dict_contains_path
+from ..exceptions import NoCredentialsException
+from .. import mgr, logger
+
+
+def _determine_rgw_addr():
+ service_map = mgr.get('service_map')
+
+ if not dict_contains_path(service_map, ['services', 'rgw', 'daemons', 'rgw']):
+ msg = 'No RGW found.'
+ raise LookupError(msg)
+
+ daemon = service_map['services']['rgw']['daemons']['rgw']
+ addr = daemon['addr'].split(':')[0]
+ match = re.search(r'port=(\d+)',
+ daemon['metadata']['frontend_config#0'])
+ if match:
+ port = int(match.group(1))
+ else:
+ msg = 'Failed to determine RGW port'
+ raise LookupError(msg)
+
+ return addr, port
+
+
+class RgwClient(RestClient):
+ _SYSTEM_USERID = None
+ _ADMIN_PATH = None
+ _host = None
+ _port = None
+ _ssl = None
+ _user_instances = {}
+
+ @staticmethod
+ def _load_settings():
+ if Settings.RGW_API_SCHEME and Settings.RGW_API_ACCESS_KEY and \
+ Settings.RGW_API_SECRET_KEY:
+ if Options.has_default_value('RGW_API_HOST') and \
+ Options.has_default_value('RGW_API_PORT'):
+ host, port = _determine_rgw_addr()
+ else:
+ host, port = Settings.RGW_API_HOST, Settings.RGW_API_PORT
+ else:
+ logger.warning('No credentials found, please consult the '
+ 'documentation about how to enable RGW for the '
+ 'dashboard.')
+ raise NoCredentialsException()
+
+ RgwClient._host = host
+ RgwClient._port = port
+ RgwClient._ssl = Settings.RGW_API_SCHEME == 'https'
+ RgwClient._ADMIN_PATH = Settings.RGW_API_ADMIN_RESOURCE
+ RgwClient._SYSTEM_USERID = Settings.RGW_API_USER_ID
+
+ logger.info("Creating new connection for user: %s",
+ Settings.RGW_API_USER_ID)
+ RgwClient._user_instances[RgwClient._SYSTEM_USERID] = \
+ RgwClient(Settings.RGW_API_USER_ID, Settings.RGW_API_ACCESS_KEY,
+ Settings.RGW_API_SECRET_KEY)
+
+ @staticmethod
+ def instance(userid):
+ if not RgwClient._user_instances:
+ RgwClient._load_settings()
+ if not userid:
+ userid = RgwClient._SYSTEM_USERID
+ if userid not in RgwClient._user_instances:
+ logger.info("Creating new connection for user: %s", userid)
+ keys = RgwClient.admin_instance().get_user_keys(userid)
+ if not keys:
+ raise Exception(
+ "User '{}' does not have any keys configured.".format(
+ userid))
+
+ RgwClient._user_instances[userid] = RgwClient(
+ userid, keys['access_key'], keys['secret_key'])
+ return RgwClient._user_instances[userid]
+
+ @staticmethod
+ def admin_instance():
+ return RgwClient.instance(RgwClient._SYSTEM_USERID)
+
+ def _reset_login(self):
+ if self.userid != RgwClient._SYSTEM_USERID:
+ logger.info("Fetching new keys for user: %s", self.userid)
+ keys = RgwClient.admin_instance().get_user_keys(self.userid)
+ self.auth = S3Auth(keys['access_key'], keys['secret_key'],
+ service_url=self.service_url)
+ else:
+ raise RequestException('Authentication failed for the "{}" user: wrong credentials'
+ .format(self.userid), status_code=401)
+
+ def __init__(self, # pylint: disable-msg=R0913
+ userid,
+ access_key,
+ secret_key,
+ host=None,
+ port=None,
+ admin_path='admin',
+ ssl=False):
+
+ if not host and not RgwClient._host:
+ RgwClient._load_settings()
+ host = host if host else RgwClient._host
+ port = port if port else RgwClient._port
+ admin_path = admin_path if admin_path else RgwClient._ADMIN_PATH
+ ssl = ssl if ssl else RgwClient._ssl
+
+ self.userid = userid
+ self.service_url = build_url(host=host, port=port)
+ self.admin_path = admin_path
+
+ s3auth = S3Auth(access_key, secret_key, service_url=self.service_url)
+ super(RgwClient, self).__init__(host, port, 'RGW', ssl, s3auth)
+
+ logger.info("Creating new connection")
+
+ @RestClient.api_get('/', resp_structure='[0] > ID')
+ def is_service_online(self, request=None):
+ response = request({'format': 'json'})
+ return response[0]['ID'] == 'online'
+
+ @RestClient.api_get('/{admin_path}/metadata/user', resp_structure='[+]')
+ def _is_system_user(self, admin_path, request=None):
+ # pylint: disable=unused-argument
+ response = request()
+ return self.userid in response
+
+ def is_system_user(self):
+ return self._is_system_user(self.admin_path)
+
+ @RestClient.api_get(
+ '/{admin_path}/user',
+ resp_structure='tenant & user_id & email & keys[*] > '
+ ' (user & access_key & secret_key)')
+ def _admin_get_user_keys(self, admin_path, userid, request=None):
+ # pylint: disable=unused-argument
+ colon_idx = userid.find(':')
+ user = userid if colon_idx == -1 else userid[:colon_idx]
+ response = request({'uid': user})
+ for keys in response['keys']:
+ if keys['user'] == userid:
+ return {
+ 'access_key': keys['access_key'],
+ 'secret_key': keys['secret_key']
+ }
+ return None
+
+ def get_user_keys(self, userid):
+ return self._admin_get_user_keys(self.admin_path, userid)
+
+ @RestClient.api('/{admin_path}/{path}')
+ def _proxy_request(self, # pylint: disable=too-many-arguments
+ admin_path,
+ path,
+ method,
+ params,
+ data,
+ request=None):
+ # pylint: disable=unused-argument
+ return request(
+ method=method, params=params, data=data, raw_content=True)
+
+ def proxy(self, method, path, params, data):
+ logger.debug("proxying method=%s path=%s params=%s data=%s", method,
+ path, params, data)
+ return self._proxy_request(self.admin_path, path, method, params, data)
+
+ @RestClient.api_get('/', resp_structure='[1][*] > Name')
+ def get_buckets(self, request=None):
+ """
+ Get a list of names from all existing buckets of this user.
+ :return: Returns a list of bucket names.
+ """
+ response = request({'format': 'json'})
+ return [bucket['Name'] for bucket in response[1]]
+
+ @RestClient.api_get('/{bucket_name}')
+ def bucket_exists(self, bucket_name, userid, request=None):
+ """
+ Check if the specified bucket exists for this user.
+ :param bucket_name: The name of the bucket.
+ :return: Returns True if the bucket exists, otherwise False.
+ """
+ # pylint: disable=unused-argument
+ try:
+ request()
+ my_buckets = self.get_buckets()
+ if bucket_name not in my_buckets:
+ raise RequestException(
+ 'Bucket "{}" belongs to other user'.format(bucket_name),
+ 403)
+ return True
+ except RequestException as e:
+ if e.status_code == 404:
+ return False
+ else:
+ raise e
+
+ @RestClient.api_put('/{bucket_name}')
+ def create_bucket(self, bucket_name, request=None):
+ logger.info("Creating bucket: %s", bucket_name)
+ return request()
"""
ENABLE_BROWSABLE_API = (True, bool)
+ # RGW settings
+ RGW_API_HOST = ('', str)
+ RGW_API_PORT = (80, int)
+ RGW_API_ACCESS_KEY = ('', str)
+ RGW_API_SECRET_KEY = ('', str)
+ RGW_API_ADMIN_RESOURCE = ('admin', str)
+ RGW_API_SCHEME = ('http', str)
+ RGW_API_USER_ID = ('', str)
+
+ @staticmethod
+ def has_default_value(name):
+ return getattr(Settings, name, None) is None or \
+ getattr(Settings, name) == getattr(Options, name)[0]
+
class SettingsMeta(type):
def __getattr__(cls, attr):
# -*- coding: utf-8 -*-
from __future__ import absolute_import
+import unittest
+
from cherrypy.lib.sessions import RamSession
from mock import patch
from .helper import ControllerTestCase
from ..controllers import RESTController, ApiController
+from ..tools import is_valid_ipv6_address, dict_contains_path
# pylint: disable=W0613
headers=[('Accept', 'text/html'), ('Content-Length', '0')],
method='put')
self.assertStatus(404)
+
+
+class TestFunctions(unittest.TestCase):
+
+ def test_is_valid_ipv6_address(self):
+ self.assertTrue(is_valid_ipv6_address('::'))
+ self.assertTrue(is_valid_ipv6_address('::1'))
+ self.assertFalse(is_valid_ipv6_address('127.0.0.1'))
+ self.assertFalse(is_valid_ipv6_address('localhost'))
+ self.assertTrue(is_valid_ipv6_address('1200:0000:AB00:1234:0000:2552:7777:1313'))
+ self.assertFalse(is_valid_ipv6_address('1200::AB00:1234::2552:7777:1313'))
+
+ def test_dict_contains_path(self):
+ x = {'a': {'b': {'c': 'foo'}}}
+ self.assertTrue(dict_contains_path(x, ['a', 'b', 'c']))
+ self.assertTrue(dict_contains_path(x, ['a', 'b', 'c']))
+ self.assertTrue(dict_contains_path(x, ['a']))
+ self.assertFalse(dict_contains_path(x, ['a', 'c']))
+
+ self.assertTrue(dict_contains_path(x, []))
import fnmatch
import time
import threading
-
+import socket
import cherrypy
from . import logger
+from six.moves import urllib
class RequestLoggingTool(cherrypy.Tool):
self.progress = percentage
if not in_lock:
self.lock.release()
+
+
+def is_valid_ipv6_address(addr):
+ try:
+ socket.inet_pton(socket.AF_INET6, addr)
+ return True
+ except socket.error:
+ return False
+
+
+def build_url(host, scheme=None, port=None):
+ """
+ Build a valid URL. IPv6 addresses specified in host will be enclosed in brackets
+ automatically.
+
+ >>> build_url('example.com', 'https', 443)
+ 'https://example.com:443'
+
+ >>> build_url(host='example.com', port=443)
+ '//example.com:443'
+
+ >>> build_url('fce:9af7:a667:7286:4917:b8d3:34df:8373', port=80, scheme='http')
+ 'http://[fce:9af7:a667:7286:4917:b8d3:34df:8373]:80'
+
+ :param scheme: The scheme, e.g. http, https or ftp.
+ :type scheme: str
+ :param host: Consisting of either a registered name (including but not limited to
+ a hostname) or an IP address.
+ :type host: str
+ :type port: int
+ :rtype: str
+ """
+ netloc = host if not is_valid_ipv6_address(host) else '[{}]'.format(host)
+ if port:
+ netloc += ':{}'.format(port)
+ pr = urllib.parse.ParseResult(
+ scheme=scheme if scheme else '',
+ netloc=netloc,
+ path='',
+ params='',
+ query='',
+ fragment='')
+ return pr.geturl()
+
+
+def dict_contains_path(dct, keys):
+ """
+ Tests wheter the keys exist recursively in `dictionary`.
+
+ :type dct: dict
+ :type keys: list
+ :rtype: bool
+ """
+ if keys:
+ key = keys.pop(0)
+ if key in dct:
+ dct = dct[key]
+ return dict_contains_path(dct, keys)
+ return False
+ return True
LD_LIBRARY_PATH = {toxinidir}/../../../../build/lib
PATH = {toxinidir}/../../../../build/bin:$PATH
commands=
- {envbindir}/py.test --cov=. --cov-report= --junitxml=junit.{envname}.xml --doctest-modules controllers/rbd.py services/ tests/
+ {envbindir}/py.test --cov=. --cov-report= --junitxml=junit.{envname}.xml --doctest-modules controllers/rbd.py services/ tools.py tests/
[testenv:cov-init]
setenv =
projects / ceph-ci.git / commitdiff