From 77c676ca6fa1187b4d6bb1d34aa8d0133bdd687d Mon Sep 17 00:00:00 2001 From: Ricardo Dias Date: Fri, 16 Mar 2018 13:13:26 +0000 Subject: [PATCH] qa/tasks/mgr/dashboard: rbd: new image list format Signed-off-by: Ricardo Dias --- qa/tasks/mgr/dashboard/helper.py | 90 +++++++----- qa/tasks/mgr/dashboard/test_rbd.py | 222 ++++++++++++++++++++++------- 2 files changed, 225 insertions(+), 87 deletions(-) diff --git a/qa/tasks/mgr/dashboard/helper.py b/qa/tasks/mgr/dashboard/helper.py index 6f93d42705dcb..ce1cbe69a302e 100644 --- a/qa/tasks/mgr/dashboard/helper.py +++ b/qa/tasks/mgr/dashboard/helper.py @@ -4,9 +4,6 @@ from __future__ import absolute_import import json import logging -import os -import subprocess -import sys from collections import namedtuple import requests @@ -34,6 +31,9 @@ class DashboardTestCase(MgrTestCase): CLIENTS_REQUIRED = 1 CEPHFS = False + _session = None + _resp = None + @classmethod def setUpClass(cls): super(DashboardTestCase, cls).setUpClass() @@ -67,56 +67,80 @@ class DashboardTestCase(MgrTestCase): # wait for mds restart to complete... cls.fs.wait_for_daemons() + cls._session = requests.Session() + cls._resp = None + @classmethod def tearDownClass(cls): super(DashboardTestCase, cls).tearDownClass() - def __init__(self, *args, **kwargs): - super(DashboardTestCase, self).__init__(*args, **kwargs) - self._session = requests.Session() - self._resp = None - - def _request(self, url, method, data=None, params=None): - url = "{}{}".format(self.base_uri, url) - + # pylint: disable=inconsistent-return-statements + @classmethod + def _request(cls, url, method, data=None, params=None): + url = "{}{}".format(cls.base_uri, url) log.info("request %s to %s", method, url) if method == 'GET': - self._resp = self._session.get(url, params=params) + cls._resp = cls._session.get(url, params=params) try: - return self._resp.json() + return cls._resp.json() except ValueError as ex: - log.exception("Failed to decode response: %s", self._resp.text) + log.exception("Failed to decode response: %s", cls._resp.text) raise ex elif method == 'POST': - self._resp = self._session.post(url, json=data, params=params) + cls._resp = cls._session.post(url, json=data, params=params) elif method == 'DELETE': - self._resp = self._session.delete(url, json=data, params=params) + cls._resp = cls._session.delete(url, json=data, params=params) elif method == 'PUT': - self._resp = self._session.put(url, json=data, params=params) + cls._resp = cls._session.put(url, json=data, params=params) else: assert False return None - def _get(self, url, params=None): - return self._request(url, 'GET', params=params) + @classmethod + def _get(cls, url, params=None): + return cls._request(url, 'GET', params=params) + + @classmethod + def _get_view_cache(cls, url, retries=5): + retry = True + while retry and retries > 0: + retry = False + res = cls._get(url) + if isinstance(res, dict): + res = [res] + for view in res: + assert 'value' in view + if not view['value']: + retry = True + retries -= 1 + if retries == 0: + raise Exception("{} view cache exceeded number of retries={}" + .format(url, retries)) + return res - def _post(self, url, data=None, params=None): - self._request(url, 'POST', data, params=params) + @classmethod + def _post(cls, url, data=None, params=None): + cls._request(url, 'POST', data, params) - def _delete(self, url, data=None, params=None): - self._request(url, 'DELETE', data, params=params) + @classmethod + def _delete(cls, url, data=None, params=None): + cls._request(url, 'DELETE', data, params) - def _put(self, url, data=None, params=None): - self._request(url, 'PUT', data, params=params) + @classmethod + def _put(cls, url, data=None, params=None): + cls._request(url, 'PUT', data, params) - def cookies(self): - return self._resp.cookies + @classmethod + def cookies(cls): + return cls._resp.cookies - def jsonBody(self): - return self._resp.json() + @classmethod + def jsonBody(cls): + return cls._resp.json() - def reset_session(self): - self._session = requests.Session() + @classmethod + def reset_session(cls): + cls._session = requests.Session() def assertJsonBody(self, data): body = self._resp.json() @@ -200,6 +224,7 @@ class _ValError(Exception): super(_ValError, self).__init__('In `input{}`: {}'.format(path_str, msg)) +# pylint: disable=dangerous-default-value,inconsistent-return-statements def _validate_json(val, schema, path=[]): """ >>> d = {'a': 1, 'b': 'x', 'c': range(10)} @@ -235,6 +260,3 @@ def _validate_json(val, schema, path=[]): ) assert False, str(path) - - - diff --git a/qa/tasks/mgr/dashboard/test_rbd.py b/qa/tasks/mgr/dashboard/test_rbd.py index 84a04d80935a3..957fb3955daee 100644 --- a/qa/tasks/mgr/dashboard/test_rbd.py +++ b/qa/tasks/mgr/dashboard/test_rbd.py @@ -2,20 +2,37 @@ from __future__ import absolute_import -import unittest - -from .helper import DashboardTestCase, authenticate +from .helper import DashboardTestCase class RbdTest(DashboardTestCase): + @classmethod + def authenticate(cls): + cls._ceph_cmd(['dashboard', 'set-login-credentials', 'admin', 'admin']) + cls._post('/api/auth', {'username': 'admin', 'password': 'admin'}) + + @classmethod + def create_pool(cls, name, pg_num, pool_type, application='rbd'): + cls._post("/api/pool", {'pool': name, 'pg_num': pg_num, + 'pool_type': pool_type, + 'application_metadata': application}) + + @classmethod + def create_image(cls, name, pool, size): + cls._post('/api/rbd', {'name': name, 'pool_name': pool, 'size': size}) + @classmethod def setUpClass(cls): super(RbdTest, cls).setUpClass() - cls._ceph_cmd(['osd', 'pool', 'create', 'rbd', '100', '100']) - cls._ceph_cmd(['osd', 'pool', 'application', 'enable', 'rbd', 'rbd']) - cls._rbd_cmd(['create', '--size=1G', 'img1']) - cls._rbd_cmd(['create', '--size=2G', 'img2']) + cls.authenticate() + cls.create_pool('rbd', 10, 'replicated') + cls.create_pool('rbd_iscsi', 10, 'replicated') + + cls.create_image('img1', 'rbd', 2**30) + cls.create_image('img2', 'rbd', 2*2**30) + cls.create_image('img1', 'rbd_iscsi', 2**30) + cls.create_image('img2', 'rbd_iscsi', 2*2**30) osd_metadata = cls.ceph_cluster.mon_manager.get_osd_metadata() cls.bluestore_support = True @@ -28,31 +45,100 @@ class RbdTest(DashboardTestCase): def tearDownClass(cls): super(RbdTest, cls).tearDownClass() cls._ceph_cmd(['osd', 'pool', 'delete', 'rbd', 'rbd', '--yes-i-really-really-mean-it']) + cls._ceph_cmd(['osd', 'pool', 'delete', 'rbd_iscsi', 'rbd_iscsi', + '--yes-i-really-really-mean-it']) + cls._ceph_cmd(['osd', 'pool', 'delete', 'rbd_data', 'rbd_data', + '--yes-i-really-really-mean-it']) + + def _validate_image(self, img, **kwargs): + """ + Example of an RBD image json: + + { + "size": 1073741824, + "obj_size": 4194304, + "num_objs": 256, + "order": 22, + "block_name_prefix": "rbd_data.10ae2ae8944a", + "name": "img1", + "pool_name": "rbd", + "features": 61, + "features_name": ["deep-flatten", "exclusive-lock", "fast-diff", "layering", + "object-map"] + } + """ + self.assertIn('size', img) + self.assertIn('obj_size', img) + self.assertIn('num_objs', img) + self.assertIn('order', img) + self.assertIn('block_name_prefix', img) + self.assertIn('name', img) + self.assertIn('id', img) + self.assertIn('pool_name', img) + self.assertIn('features', img) + self.assertIn('features_name', img) + self.assertIn('stripe_count', img) + self.assertIn('stripe_unit', img) + self.assertIn('parent', img) + self.assertIn('data_pool', img) + self.assertIn('snapshots', img) + + for k, v in kwargs.items(): + if isinstance(v, list): + self.assertSetEqual(set(img[k]), set(v)) + else: + self.assertEqual(img[k], v) + + def _validate_snapshot(self, snap, **kwargs): + self.assertIn('id', snap) + self.assertIn('name', snap) + self.assertIn('is_protected', snap) + self.assertIn('timestamp', snap) + self.assertIn('size', snap) + self.assertIn('children', snap) + + for k, v in kwargs.items(): + if isinstance(v, list): + self.assertSetEqual(set(snap[k]), set(v)) + else: + self.assertEqual(snap[k], v) - @authenticate def test_list(self): - data = self._get('/api/rbd/rbd') + data = self._get_view_cache('/api/rbd') self.assertStatus(200) + self.assertEqual(len(data), 2) + + for pool_view in data: + self.assertEqual(pool_view['status'], 0) + self.assertIsNotNone(pool_view['value']) + self.assertIn('pool_name', pool_view) + self.assertIn(pool_view['pool_name'], ['rbd', 'rbd_iscsi']) + image_list = pool_view['value'] + self.assertEqual(len(image_list), 2) + + for img in image_list: + self.assertIn('name', img) + self.assertIn('pool_name', img) + self.assertIn(img['pool_name'], ['rbd', 'rbd_iscsi']) + if img['name'] == 'img1': + self._validate_image(img, size=1073741824, + num_objs=256, obj_size=4194304, + features_name=['deep-flatten', + 'exclusive-lock', + 'fast-diff', + 'layering', + 'object-map']) + elif img['name'] == 'img2': + self._validate_image(img, size=2147483648, + num_objs=512, obj_size=4194304, + features_name=['deep-flatten', + 'exclusive-lock', + 'fast-diff', + 'layering', + 'object-map']) + else: + assert False, "Unexcepted image '{}' in result list".format(img['name']) - img1 = data['value'][0] - self.assertEqual(img1['name'], 'img1') - self.assertEqual(img1['size'], 1073741824) - self.assertEqual(img1['num_objs'], 256) - self.assertEqual(img1['obj_size'], 4194304) - self.assertEqual(img1['features_name'], - ['deep-flatten', 'exclusive-lock', 'fast-diff', 'layering', 'object-map']) - self.assertIn('id', img1) - - img2 = data['value'][1] - self.assertEqual(img2['name'], 'img2') - self.assertEqual(img2['size'], 2147483648) - self.assertEqual(img2['num_objs'], 512) - self.assertEqual(img2['obj_size'], 4194304) - self.assertEqual(img2['features_name'], - ['deep-flatten', 'exclusive-lock', 'fast-diff', 'layering', 'object-map']) - self.assertIn('id', img2) - - @authenticate def test_create(self): rbd_name = 'test_rbd' data = {'pool_name': 'rbd', @@ -62,21 +148,18 @@ class RbdTest(DashboardTestCase): self.assertStatus(201) self.assertJsonBody({"success": True}) - # TODO: change to GET the specific RBD instead of the list as soon as it is available? - get_res = self._get('/api/rbd/rbd') + img = self._get('/api/rbd/rbd/test_rbd') self.assertStatus(200) - for rbd in get_res['value']: - if rbd['name'] == rbd_name: - self.assertEqual(rbd['size'], 10240) - self.assertEqual(rbd['num_objs'], 1) - self.assertEqual(rbd['obj_size'], 4194304) - self.assertEqual(rbd['features_name'], - ['deep-flatten', 'exclusive-lock', 'fast-diff', 'layering', - 'object-map']) - break + self._validate_image(img, name=rbd_name, size=10240, + num_objs=1, obj_size=4194304, + features_name=['deep-flatten', + 'exclusive-lock', + 'fast-diff', 'layering', + 'object-map']) + + self._rbd_cmd(['rm', 'rbd/{}'.format(rbd_name)]) - @authenticate def test_create_rbd_in_data_pool(self): if not self.bluestore_support: self.skipTest('requires bluestore cluster') @@ -94,31 +177,64 @@ class RbdTest(DashboardTestCase): self.assertStatus(201) self.assertJsonBody({"success": True}) - # TODO: possibly change to GET the specific RBD (see above) - get_res = self._get('/api/rbd/rbd') + img = self._get('/api/rbd/rbd/test_rbd_in_data_pool') self.assertStatus(200) - for rbd in get_res['value']: - if rbd['name'] == rbd_name: - self.assertEqual(rbd['size'], 10240) - self.assertEqual(rbd['num_objs'], 1) - self.assertEqual(rbd['obj_size'], 4194304) - self.assertEqual(rbd['features_name'], - ['data-pool', 'deep-flatten', 'exclusive-lock', 'fast-diff', - 'layering', 'object-map']) - break + self._validate_image(img, name=rbd_name, size=10240, + num_objs=1, obj_size=4194304, + data_pool='data_pool', + features_name=['data-pool', 'deep-flatten', + 'exclusive-lock', + 'fast-diff', 'layering', + 'object-map']) + self._rbd_cmd(['rm', 'rbd/{}'.format(rbd_name)]) self._ceph_cmd(['osd', 'pool', 'delete', 'data_pool', 'data_pool', - '--yes-i-really-really-mean-it']) + '--yes-i-really-really-mean-it']) - @authenticate def test_create_rbd_twice(self): data = {'pool_name': 'rbd', 'name': 'test_rbd_twice', 'size': 10240} self._post('/api/rbd', data) + self.assertStatus(201) self._post('/api/rbd', data) self.assertStatus(400) self.assertJsonBody({"success": False, "errno": 17, "detail": "[errno 17] error creating image"}) + self._rbd_cmd(['rm', 'rbd/test_rbd_twice']) + + def test_snapshots_and_clone_info(self): + self._rbd_cmd(['snap', 'create', 'rbd/img1@snap1']) + self._rbd_cmd(['snap', 'create', 'rbd/img1@snap2']) + self._rbd_cmd(['snap', 'protect', 'rbd/img1@snap1']) + self._rbd_cmd(['clone', 'rbd/img1@snap1', 'rbd_iscsi/img1_clone']) + + img = self._get('/api/rbd/rbd/img1') + self.assertStatus(200) + self._validate_image(img, name='img1', size=1073741824, + num_objs=256, obj_size=4194304, parent=None, + features_name=['deep-flatten', 'exclusive-lock', + 'fast-diff', 'layering', + 'object-map']) + for snap in img['snapshots']: + if snap['name'] == 'snap1': + self._validate_snapshot(snap, is_protected=True) + self.assertEqual(len(snap['children']), 1) + self.assertDictEqual(snap['children'][0], + {'pool_name': 'rbd_iscsi', + 'image_name': 'img1_clone'}) + elif snap['name'] == 'snap2': + self._validate_snapshot(snap, is_protected=False) + + img = self._get('/api/rbd/rbd_iscsi/img1_clone') + self.assertStatus(200) + self._validate_image(img, name='img1_clone', size=1073741824, + num_objs=256, obj_size=4194304, + parent={'pool_name': 'rbd', 'image_name': 'img1', + 'snap_name': 'snap1'}, + features_name=['deep-flatten', 'exclusive-lock', + 'fast-diff', 'layering', + 'object-map']) + self._rbd_cmd(['rm', 'rbd_iscsi/img1_clone']) -- 2.47.3