From 812c105a4336cad56a768add5be2d697edc3b6f9 Mon Sep 17 00:00:00 2001 From: Kiefer Chang Date: Tue, 24 Nov 2020 12:41:13 +0800 Subject: [PATCH] mgr/dashboard: refactor /api/orchestrator/* endpoints - API changes: - Move `/api/orchestrator/identify_device` to `/api/host//identify_device`. - Move `/api/orchestartor/inventory` to `/ui-api/host/inventory`. This UI API provides a shortcut to get all inventories. - Add `/api/host//inventory` for getting a host's inventory. - Add inventory schema to improve OpenAPI doc. - Backend unittests: - Refactor: Remove duplicated orchestrator patch calls. - Add unittest for identify device. Fixes: https://tracker.ceph.com/issues/43165 Signed-off-by: Kiefer Chang --- qa/tasks/mgr/dashboard/test_host.py | 46 ++ qa/tasks/mgr/dashboard/test_orchestrator.py | 59 +-- src/pybind/mgr/dashboard/controllers/host.py | 184 +++++++- .../mgr/dashboard/controllers/orchestrator.py | 97 +--- .../inventory-devices.component.ts | 6 +- .../inventory/inventory.component.spec.ts | 13 +- .../cluster/inventory/inventory.component.ts | 9 +- .../osd/osd-form/osd-form.component.spec.ts | 9 +- .../osd/osd-form/osd-form.component.ts | 4 +- .../src/app/shared/api/host.service.spec.ts | 20 + .../src/app/shared/api/host.service.ts | 77 +++- .../shared/api/orchestrator.service.spec.ts | 28 -- .../app/shared/api/orchestrator.service.ts | 41 +- .../shared/services/task-message.service.ts | 9 +- src/pybind/mgr/dashboard/openapi.yaml | 430 ++++++++++++++---- src/pybind/mgr/dashboard/tests/test_host.py | 356 +++++++++++---- .../mgr/dashboard/tests/test_orchestrator.py | 133 +----- 17 files changed, 976 insertions(+), 545 deletions(-) diff --git a/qa/tasks/mgr/dashboard/test_host.py b/qa/tasks/mgr/dashboard/test_host.py index 069f729579e65..49bb33533cd71 100644 --- a/qa/tasks/mgr/dashboard/test_host.py +++ b/qa/tasks/mgr/dashboard/test_host.py @@ -9,6 +9,7 @@ class HostControllerTest(DashboardTestCase): AUTH_ROLES = ['read-only'] URL_HOST = '/api/host' + URL_UI_HOST = '/ui-api/host' ORCHESTRATOR = True @@ -21,6 +22,14 @@ class HostControllerTest(DashboardTestCase): cmd = ['test_orchestrator', 'load_data', '-i', '-'] cls.mgr_cluster.mon_manager.raw_cluster_cmd_result(*cmd, stdin='{}') + @property + def test_data_inventory(self): + return self.ORCHESTRATOR_TEST_DATA['inventory'] + + @property + def test_data_daemons(self): + return self.ORCHESTRATOR_TEST_DATA['daemons'] + @DashboardTestCase.RunAs('test', 'test', ['block-manager']) def test_access_permissions(self): self._get(self.URL_HOST) @@ -97,6 +106,43 @@ class HostControllerTest(DashboardTestCase): self._get('{}/smart'.format('{}/{}'.format(self.URL_HOST, hosts[0]))) self.assertStatus(200) + def _validate_inventory(self, data, resp_data): + self.assertEqual(data['name'], resp_data['name']) + self.assertEqual(len(data['devices']), len(resp_data['devices'])) + + if not data['devices']: + return + test_devices = sorted(data['devices'], key=lambda d: d['path']) + resp_devices = sorted(resp_data['devices'], key=lambda d: d['path']) + + for test, resp in zip(test_devices, resp_devices): + self._validate_device(test, resp) + + def _validate_device(self, data, resp_data): + for key, value in data.items(): + self.assertEqual(value, resp_data[key]) + + def test_inventory_get(self): + # get a inventory + node = self.test_data_inventory[0] + resp = self._get('{}/{}/inventory'.format(self.URL_HOST, node['name'])) + self.assertStatus(200) + self._validate_inventory(node, resp) + + def test_inventory_list(self): + # get all inventory + data = self._get('{}/inventory'.format(self.URL_UI_HOST)) + self.assertStatus(200) + + def sorting_key(node): + return node['name'] + + test_inventory = sorted(self.test_data_inventory, key=sorting_key) + resp_inventory = sorted(data, key=sorting_key) + self.assertEqual(len(test_inventory), len(resp_inventory)) + for test, resp in zip(test_inventory, resp_inventory): + self._validate_inventory(test, resp) + class HostControllerNoOrchestratorTest(DashboardTestCase): def test_host_create(self): diff --git a/qa/tasks/mgr/dashboard/test_orchestrator.py b/qa/tasks/mgr/dashboard/test_orchestrator.py index 4fc4574408480..8395853e3d673 100644 --- a/qa/tasks/mgr/dashboard/test_orchestrator.py +++ b/qa/tasks/mgr/dashboard/test_orchestrator.py @@ -9,19 +9,9 @@ class OrchestratorControllerTest(DashboardTestCase): AUTH_ROLES = ['cluster-manager'] URL_STATUS = '/api/orchestrator/status' - URL_INVENTORY = '/api/orchestrator/inventory' - URL_OSD = '/api/orchestrator/osd' ORCHESTRATOR = True - @property - def test_data_inventory(self): - return self.ORCHESTRATOR_TEST_DATA['inventory'] - - @property - def test_data_daemons(self): - return self.ORCHESTRATOR_TEST_DATA['daemons'] - @classmethod def setUpClass(cls): super(OrchestratorControllerTest, cls).setUpClass() @@ -31,54 +21,7 @@ class OrchestratorControllerTest(DashboardTestCase): cmd = ['test_orchestrator', 'load_data', '-i', '-'] cls.mgr_cluster.mon_manager.raw_cluster_cmd_result(*cmd, stdin='{}') - def _validate_inventory(self, data, resp_data): - self.assertEqual(data['name'], resp_data['name']) - self.assertEqual(len(data['devices']), len(resp_data['devices'])) - - if not data['devices']: - return - test_devices = sorted(data['devices'], key=lambda d: d['path']) - resp_devices = sorted(resp_data['devices'], key=lambda d: d['path']) - - for test, resp in zip(test_devices, resp_devices): - self._validate_device(test, resp) - - def _validate_device(self, data, resp_data): - for key, value in data.items(): - self.assertEqual(value, resp_data[key]) - - def _validate_daemon(self, data, resp_data): - for key, value in data.items(): - self.assertEqual(value, resp_data[key]) - - @DashboardTestCase.RunAs('test', 'test', ['block-manager']) - def test_access_permissions(self): - self._get(self.URL_STATUS) - self.assertStatus(200) - self._get(self.URL_INVENTORY) - self.assertStatus(403) - def test_status_get(self): data = self._get(self.URL_STATUS) - self.assertTrue(data['available']) - - def test_inventory_list(self): - # get all inventory - data = self._get(self.URL_INVENTORY) - self.assertStatus(200) - - def sorting_key(node): - return node['name'] - - test_inventory = sorted(self.test_data_inventory, key=sorting_key) - resp_inventory = sorted(data, key=sorting_key) - self.assertEqual(len(test_inventory), len(resp_inventory)) - for test, resp in zip(test_inventory, resp_inventory): - self._validate_inventory(test, resp) - - # get inventory by hostname - node = self.test_data_inventory[-1] - data = self._get('{}?hostname={}'.format(self.URL_INVENTORY, node['name'])) self.assertStatus(200) - self.assertEqual(len(data), 1) - self._validate_inventory(node, data[0]) + self.assertTrue(data['available']) diff --git a/src/pybind/mgr/dashboard/controllers/host.py b/src/pybind/mgr/dashboard/controllers/host.py index 8394fc578e3ab..cdce5c895a4c9 100644 --- a/src/pybind/mgr/dashboard/controllers/host.py +++ b/src/pybind/mgr/dashboard/controllers/host.py @@ -2,7 +2,9 @@ from __future__ import absolute_import import copy -from typing import Dict, List +import os +import time +from typing import Dict, List, Optional import cherrypy from mgr_util import merge_dicts @@ -14,9 +16,10 @@ from ..security import Scope from ..services.ceph_service import CephService from ..services.exception import handle_orchestrator_error from ..services.orchestrator import OrchClient, OrchFeature +from ..tools import TaskManager, str_to_bool from . import ApiController, BaseController, ControllerDoc, Endpoint, \ EndpointDoc, ReadPermission, RESTController, Task, UiApiController, \ - allow_empty_body + UpdatePermission, allow_empty_body from .orchestrator import raise_if_no_orchestrator LIST_HOST_SCHEMA = { @@ -36,6 +39,74 @@ LIST_HOST_SCHEMA = { "status": (str, "") } +INVENTORY_SCHEMA = { + "name": (str, "Hostname"), + "addr": (str, "Host address"), + "devices": ([{ + "rejected_reasons": ([str], ""), + "available": (bool, "If the device can be provisioned to an OSD"), + "path": (str, "Device path"), + "sys_api": ({ + "removable": (str, ""), + "ro": (str, ""), + "vendor": (str, ""), + "model": (str, ""), + "rev": (str, ""), + "sas_address": (str, ""), + "sas_device_handle": (str, ""), + "support_discard": (str, ""), + "rotational": (str, ""), + "nr_requests": (str, ""), + "scheduler_mode": (str, ""), + "partitions": ({ + "partition_name": ({ + "start": (str, ""), + "sectors": (str, ""), + "sectorsize": (int, ""), + "size": (int, ""), + "human_readable_size": (str, ""), + "holders": ([str], "") + }, "") + }, ""), + "sectors": (int, ""), + "sectorsize": (str, ""), + "size": (int, ""), + "human_readable_size": (str, ""), + "path": (str, ""), + "locked": (int, "") + }, ""), + "lvs": ([{ + "name": (str, ""), + "osd_id": (str, ""), + "cluster_name": (str, ""), + "type": (str, ""), + "osd_fsid": (str, ""), + "cluster_fsid": (str, ""), + "osdspec_affinity": (str, ""), + "block_uuid": (str, ""), + }], ""), + "human_readable_type": (str, "Device type. ssd or hdd"), + "device_id": (str, "Device's udev ID"), + "lsm_data": ({ + "serialNum": (str, ""), + "transport": (str, ""), + "mediaType": (str, ""), + "rpm": (str, ""), + "linkSpeed": (str, ""), + "health": (str, ""), + "ledSupport": ({ + "IDENTsupport": (str, ""), + "IDENTstatus": (str, ""), + "FAILsupport": (str, ""), + "FAILstatus": (str, ""), + }, ""), + "errors": ([str], "") + }, ""), + "osd_ids": ([int], "Device OSD IDs") + }], "Host devices"), + "labels": ([str], "Host labels") +} + def host_task(name, metadata, wait_for=10.0): return Task("host/{}".format(name), metadata, wait_for) @@ -121,6 +192,69 @@ def get_host(hostname: str) -> Dict: raise cherrypy.HTTPError(404) +def get_device_osd_map(): + """Get mappings from inventory devices to OSD IDs. + + :return: Returns a dictionary containing mappings. Note one device might + shared between multiple OSDs. + e.g. { + 'node1': { + 'nvme0n1': [0, 1], + 'vdc': [0], + 'vdb': [1] + }, + 'node2': { + 'vdc': [2] + } + } + :rtype: dict + """ + result: dict = {} + for osd_id, osd_metadata in mgr.get('osd_metadata').items(): + hostname = osd_metadata.get('hostname') + devices = osd_metadata.get('devices') + if not hostname or not devices: + continue + if hostname not in result: + result[hostname] = {} + # for OSD contains multiple devices, devices is in `sda,sdb` + for device in devices.split(','): + if device not in result[hostname]: + result[hostname][device] = [int(osd_id)] + else: + result[hostname][device].append(int(osd_id)) + return result + + +def get_inventories(hosts: Optional[List[str]] = None, + refresh: Optional[bool] = None) -> List[dict]: + """Get inventories from the Orchestrator and link devices with OSD IDs. + + :param hosts: Hostnames to query. + :param refresh: Ask the Orchestrator to refresh the inventories. Note the this is an + asynchronous operation, the updated version of inventories need to + be re-qeuried later. + :return: Returns list of inventory. + :rtype: list + """ + do_refresh = False + if refresh is not None: + do_refresh = str_to_bool(refresh) + orch = OrchClient.instance() + inventory_hosts = [host.to_json() + for host in orch.inventory.list(hosts=hosts, refresh=do_refresh)] + device_osd_map = get_device_osd_map() + for inventory_host in inventory_hosts: + host_osds = device_osd_map.get(inventory_host['name']) + for device in inventory_host['devices']: + if host_osds: # pragma: no cover + dev_name = os.path.basename(device['path']) + device['osd_ids'] = sorted(host_osds.get(dev_name, [])) + else: + device['osd_ids'] = [] + return inventory_hosts + + @ApiController('/host', Scope.HOSTS) @ControllerDoc("Get Host Details", "Host") class Host(RESTController): @@ -185,6 +319,45 @@ class Host(RESTController): # type: (str) -> dict return CephService.get_smart_data_by_host(hostname) + @RESTController.Resource('GET') + @raise_if_no_orchestrator([OrchFeature.DEVICE_LIST]) + @handle_orchestrator_error('host') + @EndpointDoc('Get inventory of a host', + parameters={ + 'hostname': (str, 'Hostname'), + 'refresh': (str, 'Trigger asynchronous refresh'), + }, + responses={200: INVENTORY_SCHEMA}) + def inventory(self, hostname, refresh=None): + inventory = get_inventories([hostname], refresh) + if inventory: + return inventory[0] + return {} + + @RESTController.Resource('POST') + @UpdatePermission + @raise_if_no_orchestrator([OrchFeature.DEVICE_BLINK_LIGHT]) + @handle_orchestrator_error('host') + @host_task('identify_device', ['{hostname}', '{device}'], wait_for=2.0) + def identify_device(self, hostname, device, duration): + # type: (str, str, int) -> None + """ + Identify a device by switching on the device light for N seconds. + :param hostname: The hostname of the device to process. + :param device: The device identifier to process, e.g. ``/dev/dm-0`` or + ``ABC1234DEF567-1R1234_ABC8DE0Q``. + :param duration: The duration in seconds how long the LED should flash. + """ + orch = OrchClient.instance() + TaskManager.current_task().set_progress(0) + orch.blink_device_light(hostname, device, 'ident', True) + for i in range(int(duration)): + percentage = int(round(i / float(duration) * 100)) + TaskManager.current_task().set_progress(percentage) + time.sleep(1) + orch.blink_device_light(hostname, device, 'ident', False) + TaskManager.current_task().set_progress(100) + @RESTController.Resource('GET') @raise_if_no_orchestrator([OrchFeature.DAEMON_LIST]) def daemons(self, hostname: str) -> List[dict]: @@ -241,3 +414,10 @@ class HostUi(BaseController): labels.extend(host.labels) labels.sort() return list(set(labels)) # Filter duplicate labels. + + @Endpoint('GET') + @ReadPermission + @raise_if_no_orchestrator([OrchFeature.DEVICE_LIST]) + @handle_orchestrator_error('host') + def inventory(self, refresh=None): + return get_inventories(None, refresh) diff --git a/src/pybind/mgr/dashboard/controllers/orchestrator.py b/src/pybind/mgr/dashboard/controllers/orchestrator.py index 13b4a171a8956..085870a0f4aac 100644 --- a/src/pybind/mgr/dashboard/controllers/orchestrator.py +++ b/src/pybind/mgr/dashboard/controllers/orchestrator.py @@ -1,18 +1,11 @@ # -*- coding: utf-8 -*- from __future__ import absolute_import -import os.path -import time from functools import wraps -from .. import mgr from ..exceptions import DashboardException -from ..security import Scope -from ..services.exception import handle_orchestrator_error -from ..services.orchestrator import OrchClient, OrchFeature -from ..tools import TaskManager, str_to_bool -from . import ApiController, ControllerDoc, Endpoint, EndpointDoc, \ - ReadPermission, RESTController, Task, UpdatePermission +from ..services.orchestrator import OrchClient +from . import ApiController, ControllerDoc, Endpoint, EndpointDoc, ReadPermission, RESTController STATUS_SCHEMA = { "available": (bool, "Orchestrator status"), @@ -20,44 +13,6 @@ STATUS_SCHEMA = { } -def get_device_osd_map(): - """Get mappings from inventory devices to OSD IDs. - - :return: Returns a dictionary containing mappings. Note one device might - shared between multiple OSDs. - e.g. { - 'node1': { - 'nvme0n1': [0, 1], - 'vdc': [0], - 'vdb': [1] - }, - 'node2': { - 'vdc': [2] - } - } - :rtype: dict - """ - result: dict = {} - for osd_id, osd_metadata in mgr.get('osd_metadata').items(): - hostname = osd_metadata.get('hostname') - devices = osd_metadata.get('devices') - if not hostname or not devices: - continue - if hostname not in result: - result[hostname] = {} - # for OSD contains multiple devices, devices is in `sda,sdb` - for device in devices.split(','): - if device not in result[hostname]: - result[hostname][device] = [int(osd_id)] - else: - result[hostname][device].append(int(osd_id)) - return result - - -def orchestrator_task(name, metadata, wait_for=2.0): - return Task("orchestrator/{}".format(name), metadata, wait_for) - - def raise_if_no_orchestrator(features=None): def inner(method): @wraps(method) @@ -91,51 +46,3 @@ class Orchestrator(RESTController): responses={200: STATUS_SCHEMA}) def status(self): return OrchClient.instance().status() - - @Endpoint(method='POST') - @UpdatePermission - @raise_if_no_orchestrator([OrchFeature.DEVICE_BLINK_LIGHT]) - @handle_orchestrator_error('osd') - @orchestrator_task('identify_device', ['{hostname}', '{device}']) - def identify_device(self, hostname, device, duration): # pragma: no cover - # type: (str, str, int) -> None - """ - Identify a device by switching on the device light for N seconds. - :param hostname: The hostname of the device to process. - :param device: The device identifier to process, e.g. ``/dev/dm-0`` or - ``ABC1234DEF567-1R1234_ABC8DE0Q``. - :param duration: The duration in seconds how long the LED should flash. - """ - orch = OrchClient.instance() - TaskManager.current_task().set_progress(0) - orch.blink_device_light(hostname, device, 'ident', True) - for i in range(int(duration)): - percentage = int(round(i / float(duration) * 100)) - TaskManager.current_task().set_progress(percentage) - time.sleep(1) - orch.blink_device_light(hostname, device, 'ident', False) - TaskManager.current_task().set_progress(100) - - -@ApiController('/orchestrator/inventory', Scope.HOSTS) -@ControllerDoc("Get Orchestrator Inventory Details", "OrchestratorInventory") -class OrchestratorInventory(RESTController): - - @raise_if_no_orchestrator([OrchFeature.DEVICE_LIST]) - def list(self, hostname=None, refresh=None): - orch = OrchClient.instance() - hosts = [hostname] if hostname else None - do_refresh = False - if refresh is not None: - do_refresh = str_to_bool(refresh) - inventory_hosts = [host.to_json() for host in orch.inventory.list(hosts, do_refresh)] - device_osd_map = get_device_osd_map() - for inventory_host in inventory_hosts: - host_osds = device_osd_map.get(inventory_host['name']) - for device in inventory_host['devices']: - if host_osds: # pragma: no cover - dev_name = os.path.basename(device['path']) - device['osd_ids'] = sorted(host_osds.get(dev_name, [])) - else: - device['osd_ids'] = [] - return inventory_hosts diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/inventory/inventory-devices/inventory-devices.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/inventory/inventory-devices/inventory-devices.component.ts index 669a9bc89e2c5..fa778d5b4f29b 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/inventory/inventory-devices/inventory-devices.component.ts +++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/inventory/inventory-devices/inventory-devices.component.ts @@ -11,6 +11,7 @@ import { import _ from 'lodash'; import { Subscription } from 'rxjs'; +import { HostService } from '~/app/shared/api/host.service'; import { OrchestratorService } from '~/app/shared/api/orchestrator.service'; import { FormModalComponent } from '~/app/shared/components/form-modal/form-modal.component'; import { TableComponent } from '~/app/shared/datatable/table/table.component'; @@ -80,7 +81,8 @@ export class InventoryDevicesComponent implements OnInit, OnDestroy { private dimlessBinary: DimlessBinaryPipe, private modalService: ModalService, private notificationService: NotificationService, - private orchService: OrchestratorService + private orchService: OrchestratorService, + private hostService: HostService ) {} ngOnInit() { @@ -223,7 +225,7 @@ export class InventoryDevicesComponent implements OnInit, OnDestroy { ], submitButtonText: $localize`Execute`, onSubmit: (values: any) => { - this.orchService.identifyDevice(hostname, device, values.duration).subscribe(() => { + this.hostService.identifyDevice(hostname, device, values.duration).subscribe(() => { this.notificationService.show( NotificationType.success, $localize`Identifying '${device}' started on host '${hostname}'` diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/inventory/inventory.component.spec.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/inventory/inventory.component.spec.ts index 48edbaec598a3..70c24ecba9d5f 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/inventory/inventory.component.spec.ts +++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/inventory/inventory.component.spec.ts @@ -7,6 +7,7 @@ import { RouterTestingModule } from '@angular/router/testing'; import { ToastrModule } from 'ngx-toastr'; import { of } from 'rxjs'; +import { HostService } from '~/app/shared/api/host.service'; import { OrchestratorService } from '~/app/shared/api/orchestrator.service'; import { SharedModule } from '~/app/shared/shared.module'; import { configureTestBed } from '~/testing/unit-test-helper'; @@ -17,6 +18,7 @@ describe('InventoryComponent', () => { let component: InventoryComponent; let fixture: ComponentFixture; let orchService: OrchestratorService; + let hostService: HostService; configureTestBed({ imports: [ @@ -34,8 +36,9 @@ describe('InventoryComponent', () => { fixture = TestBed.createComponent(InventoryComponent); component = fixture.componentInstance; orchService = TestBed.inject(OrchestratorService); + hostService = TestBed.inject(HostService); spyOn(orchService, 'status').and.returnValue(of({ available: true })); - spyOn(orchService, 'inventoryDeviceList').and.callThrough(); + spyOn(hostService, 'inventoryDeviceList').and.callThrough(); }); it('should create', () => { @@ -45,17 +48,17 @@ describe('InventoryComponent', () => { describe('after ngOnInit', () => { it('should load devices', () => { fixture.detectChanges(); - expect(orchService.inventoryDeviceList).toHaveBeenNthCalledWith(1, undefined, false); + expect(hostService.inventoryDeviceList).toHaveBeenNthCalledWith(1, undefined, false); component.refresh(); // click refresh button - expect(orchService.inventoryDeviceList).toHaveBeenNthCalledWith(2, undefined, true); + expect(hostService.inventoryDeviceList).toHaveBeenNthCalledWith(2, undefined, true); const newHost = 'host0'; component.hostname = newHost; fixture.detectChanges(); component.ngOnChanges(); - expect(orchService.inventoryDeviceList).toHaveBeenNthCalledWith(3, newHost, false); + expect(hostService.inventoryDeviceList).toHaveBeenNthCalledWith(3, newHost, false); component.refresh(); // click refresh button - expect(orchService.inventoryDeviceList).toHaveBeenNthCalledWith(4, newHost, true); + expect(hostService.inventoryDeviceList).toHaveBeenNthCalledWith(4, newHost, true); }); }); }); diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/inventory/inventory.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/inventory/inventory.component.ts index edf7f61e10768..28a47396ecdf3 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/inventory/inventory.component.ts +++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/inventory/inventory.component.ts @@ -2,6 +2,7 @@ import { Component, Input, NgZone, OnChanges, OnDestroy, OnInit } from '@angular import { Subscription, timer as observableTimer } from 'rxjs'; +import { HostService } from '~/app/shared/api/host.service'; import { OrchestratorService } from '~/app/shared/api/orchestrator.service'; import { Icons } from '~/app/shared/enum/icons.enum'; import { OrchestratorStatus } from '~/app/shared/models/orchestrator.interface'; @@ -26,7 +27,11 @@ export class InventoryComponent implements OnChanges, OnInit, OnDestroy { devices: Array = []; - constructor(private orchService: OrchestratorService, private ngZone: NgZone) {} + constructor( + private orchService: OrchestratorService, + private hostService: HostService, + private ngZone: NgZone + ) {} ngOnInit() { this.orchService.status().subscribe((status) => { @@ -64,7 +69,7 @@ export class InventoryComponent implements OnChanges, OnInit, OnDestroy { if (this.hostname === '') { return; } - this.orchService.inventoryDeviceList(this.hostname, refresh).subscribe( + this.hostService.inventoryDeviceList(this.hostname, refresh).subscribe( (devices: InventoryDevice[]) => { this.devices = devices; }, diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/osd/osd-form/osd-form.component.spec.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/osd/osd-form/osd-form.component.spec.ts index b58b9c0a6f7be..0919df2ae00e8 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/osd/osd-form/osd-form.component.spec.ts +++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/osd/osd-form/osd-form.component.spec.ts @@ -9,6 +9,7 @@ import { BehaviorSubject, of } from 'rxjs'; import { InventoryDevice } from '~/app/ceph/cluster/inventory/inventory-devices/inventory-device.model'; import { InventoryDevicesComponent } from '~/app/ceph/cluster/inventory/inventory-devices/inventory-devices.component'; +import { HostService } from '~/app/shared/api/host.service'; import { OrchestratorService } from '~/app/shared/api/orchestrator.service'; import { CdFormGroup } from '~/app/shared/forms/cd-form-group'; import { SummaryService } from '~/app/shared/services/summary.service'; @@ -26,6 +27,7 @@ describe('OsdFormComponent', () => { let fixture: ComponentFixture; let fixtureHelper: FixtureHelper; let orchService: OrchestratorService; + let hostService: HostService; let summaryService: SummaryService; const devices: InventoryDevice[] = [ { @@ -109,6 +111,7 @@ describe('OsdFormComponent', () => { form = component.form; formHelper = new FormHelper(form); orchService = TestBed.inject(OrchestratorService); + hostService = TestBed.inject(HostService); summaryService = TestBed.inject(SummaryService); summaryService['summaryDataSource'] = new BehaviorSubject(null); summaryService['summaryData$'] = summaryService['summaryDataSource'].asObservable(); @@ -122,7 +125,7 @@ describe('OsdFormComponent', () => { describe('without orchestrator', () => { beforeEach(() => { spyOn(orchService, 'status').and.returnValue(of({ available: false })); - spyOn(orchService, 'inventoryDeviceList').and.callThrough(); + spyOn(hostService, 'inventoryDeviceList').and.callThrough(); fixture.detectChanges(); }); @@ -132,14 +135,14 @@ describe('OsdFormComponent', () => { }); it('should not call inventoryDeviceList', () => { - expect(orchService.inventoryDeviceList).not.toHaveBeenCalled(); + expect(hostService.inventoryDeviceList).not.toHaveBeenCalled(); }); }); describe('with orchestrator', () => { beforeEach(() => { spyOn(orchService, 'status').and.returnValue(of({ available: true })); - spyOn(orchService, 'inventoryDeviceList').and.returnValue(of([])); + spyOn(hostService, 'inventoryDeviceList').and.returnValue(of([])); fixture.detectChanges(); }); diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/osd/osd-form/osd-form.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/osd/osd-form/osd-form.component.ts index 71bec7adfc1ea..4ddf454c64087 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/osd/osd-form/osd-form.component.ts +++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/osd/osd-form/osd-form.component.ts @@ -5,6 +5,7 @@ import { Router } from '@angular/router'; import _ from 'lodash'; import { InventoryDevice } from '~/app/ceph/cluster/inventory/inventory-devices/inventory-device.model'; +import { HostService } from '~/app/shared/api/host.service'; import { OrchestratorService } from '~/app/shared/api/orchestrator.service'; import { SubmitButtonComponent } from '~/app/shared/components/submit-button/submit-button.component'; import { ActionLabelsI18n } from '~/app/shared/constants/app.constants'; @@ -65,6 +66,7 @@ export class OsdFormComponent extends CdForm implements OnInit { public actionLabels: ActionLabelsI18n, private authStorageService: AuthStorageService, private orchService: OrchestratorService, + private hostService: HostService, private router: Router, private modalService: ModalService ) { @@ -120,7 +122,7 @@ export class OsdFormComponent extends CdForm implements OnInit { } getDataDevices() { - this.orchService.inventoryDeviceList().subscribe( + this.hostService.inventoryDeviceList().subscribe( (devices: InventoryDevice[]) => { this.allDevices = _.filter(devices, 'available'); this.availDevices = [...this.allDevices]; diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/api/host.service.spec.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/host.service.spec.ts index aea5f045ffc64..babddca3e6199 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/shared/api/host.service.spec.ts +++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/host.service.spec.ts @@ -49,4 +49,24 @@ describe('HostService', () => { expect(req.request.method).toBe('PUT'); expect(req.request.body).toEqual({ labels: ['foo', 'bar'] }); })); + + it('should call getInventory', () => { + service.getInventory('host-0').subscribe(); + let req = httpTesting.expectOne('api/host/host-0/inventory'); + expect(req.request.method).toBe('GET'); + + service.getInventory('host-0', true).subscribe(); + req = httpTesting.expectOne('api/host/host-0/inventory?refresh=true'); + expect(req.request.method).toBe('GET'); + }); + + it('should call inventoryList', () => { + service.inventoryList().subscribe(); + let req = httpTesting.expectOne('ui-api/host/inventory'); + expect(req.request.method).toBe('GET'); + + service.inventoryList(true).subscribe(); + req = httpTesting.expectOne('ui-api/host/inventory?refresh=true'); + expect(req.request.method).toBe('GET'); + }); }); diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/api/host.service.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/host.service.ts index 3b9e7068e7768..5f34d96af4d64 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/shared/api/host.service.ts +++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/host.service.ts @@ -1,9 +1,12 @@ -import { HttpClient } from '@angular/common/http'; +import { HttpClient, HttpParams } from '@angular/common/http'; import { Injectable } from '@angular/core'; -import { Observable } from 'rxjs'; -import { map } from 'rxjs/operators'; +import _ from 'lodash'; +import { Observable, of as observableOf } from 'rxjs'; +import { map, mergeMap, toArray } from 'rxjs/operators'; +import { InventoryDevice } from '~/app/ceph/cluster/inventory/inventory-devices/inventory-device.model'; +import { InventoryHost } from '~/app/ceph/cluster/inventory/inventory-host.model'; import { Daemon } from '../models/daemon.interface'; import { CdDevice } from '../models/devices'; import { SmartDataResponseV1 } from '../models/smart'; @@ -14,6 +17,7 @@ import { DeviceService } from '../services/device.service'; }) export class HostService { baseURL = 'api/host'; + baseUIURL = 'ui-api/host'; constructor(private http: HttpClient, private deviceService: DeviceService) {} @@ -44,10 +48,75 @@ export class HostService { } getLabels(): Observable { - return this.http.get('ui-api/host/labels'); + return this.http.get(`${this.baseUIURL}/labels`); } update(hostname: string, labels: string[]) { return this.http.put(`${this.baseURL}/${hostname}`, { labels: labels }); } + + identifyDevice(hostname: string, device: string, duration: number) { + return this.http.post(`${this.baseURL}/${hostname}/identify_device`, { + device, + duration + }); + } + + private getInventoryParams(refresh?: boolean): HttpParams { + let params = new HttpParams(); + if (refresh) { + params = params.append('refresh', _.toString(refresh)); + } + return params; + } + + /** + * Get inventory of a host. + * + * @param hostname the host query. + * @param refresh true to ask the Orchestrator to refresh inventory. + */ + getInventory(hostname: string, refresh?: boolean): Observable { + const params = this.getInventoryParams(refresh); + return this.http.get(`${this.baseURL}/${hostname}/inventory`, { + params: params + }); + } + + /** + * Get inventories of all hosts. + * + * @param refresh true to ask the Orchestrator to refresh inventory. + */ + inventoryList(refresh?: boolean): Observable { + const params = this.getInventoryParams(refresh); + return this.http.get(`${this.baseUIURL}/inventory`, { params: params }); + } + + /** + * Get device list via host inventories. + * + * @param hostname the host to query. undefined for all hosts. + * @param refresh true to ask the Orchestrator to refresh inventory. + */ + inventoryDeviceList(hostname?: string, refresh?: boolean): Observable { + let observable; + if (hostname) { + observable = this.getInventory(hostname, refresh).pipe(toArray()); + } else { + observable = this.inventoryList(refresh); + } + return observable.pipe( + mergeMap((hosts: InventoryHost[]) => { + const devices = _.flatMap(hosts, (host) => { + return host.devices.map((device) => { + device.hostname = host.name; + device.uid = device.device_id ? device.device_id : `${device.hostname}-${device.path}`; + return device; + }); + }); + return observableOf(devices); + }) + ); + } } diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/api/orchestrator.service.spec.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/orchestrator.service.spec.ts index 4dfc595bfee67..f4c7e4390ca18 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/shared/api/orchestrator.service.spec.ts +++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/orchestrator.service.spec.ts @@ -32,32 +32,4 @@ describe('OrchestratorService', () => { const req = httpTesting.expectOne(`${apiPath}/status`); expect(req.request.method).toBe('GET'); }); - - it('should call inventoryList with arguments', () => { - const inventoryPath = `${apiPath}/inventory`; - const tests: { args: any[]; expectedUrl: any }[] = [ - { - args: [], - expectedUrl: inventoryPath - }, - { - args: ['host0'], - expectedUrl: `${inventoryPath}?hostname=host0` - }, - { - args: [undefined, true], - expectedUrl: `${inventoryPath}?refresh=true` - }, - { - args: ['host0', true], - expectedUrl: `${inventoryPath}?hostname=host0&refresh=true` - } - ]; - - for (const test of tests) { - service.inventoryList(...test.args).subscribe(); - const req = httpTesting.expectOne(test.expectedUrl); - expect(req.request.method).toBe('GET'); - } - }); }); diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/api/orchestrator.service.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/orchestrator.service.ts index ddd0f75bb4de5..2011715821512 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/shared/api/orchestrator.service.ts +++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/orchestrator.service.ts @@ -1,12 +1,9 @@ -import { HttpClient, HttpParams } from '@angular/common/http'; +import { HttpClient } from '@angular/common/http'; import { Injectable } from '@angular/core'; import _ from 'lodash'; -import { Observable, of as observableOf } from 'rxjs'; -import { mergeMap } from 'rxjs/operators'; +import { Observable } from 'rxjs'; -import { InventoryDevice } from '~/app/ceph/cluster/inventory/inventory-devices/inventory-device.model'; -import { InventoryHost } from '~/app/ceph/cluster/inventory/inventory-host.model'; import { OrchestratorFeature } from '../models/orchestrator.enum'; import { OrchestratorStatus } from '../models/orchestrator.interface'; @@ -46,38 +43,4 @@ export class OrchestratorService { } return false; } - - identifyDevice(hostname: string, device: string, duration: number) { - return this.http.post(`${this.url}/identify_device`, { - hostname, - device, - duration - }); - } - - inventoryList(hostname?: string, refresh?: boolean): Observable { - let params = new HttpParams(); - if (hostname) { - params = params.append('hostname', hostname); - } - if (refresh) { - params = params.append('refresh', _.toString(refresh)); - } - return this.http.get(`${this.url}/inventory`, { params: params }); - } - - inventoryDeviceList(hostname?: string, refresh?: boolean): Observable { - return this.inventoryList(hostname, refresh).pipe( - mergeMap((hosts: InventoryHost[]) => { - const devices = _.flatMap(hosts, (host) => { - return host.devices.map((device) => { - device.hostname = host.name; - device.uid = device.device_id ? device.device_id : `${device.hostname}-${device.path}`; - return device; - }); - }); - return observableOf(devices); - }) - ); - } } diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/services/task-message.service.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/services/task-message.service.ts index 528ad82c6b024..c39bb0c26b695 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/shared/services/task-message.service.ts +++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/services/task-message.service.ts @@ -120,6 +120,10 @@ export class TaskMessageService { 'host/delete': this.newTaskMessage(this.commonOperations.delete, (metadata) => this.host(metadata) ), + 'host/identify_device': this.newTaskMessage( + new TaskMessageOperation($localize`Identifying`, $localize`identify`, $localize`Identified`), + (metadata) => $localize`device '${metadata.device}' on host '${metadata.hostname}'` + ), // OSD tasks 'osd/create': this.newTaskMessage( this.commonOperations.create, @@ -328,11 +332,6 @@ export class TaskMessageService { this.grafana.update_dashboards, () => ({}) ), - // Orchestrator tasks - 'orchestrator/identify_device': this.newTaskMessage( - new TaskMessageOperation($localize`Identifying`, $localize`identify`, $localize`Identified`), - (metadata) => $localize`device '${metadata.device}' on host '${metadata.hostname}'` - ), // Service tasks 'service/create': this.newTaskMessage(this.commonOperations.create, (metadata) => this.service(metadata) diff --git a/src/pybind/mgr/dashboard/openapi.yaml b/src/pybind/mgr/dashboard/openapi.yaml index 0019cfb2d7080..cd2a570a8be2e 100644 --- a/src/pybind/mgr/dashboard/openapi.yaml +++ b/src/pybind/mgr/dashboard/openapi.yaml @@ -3427,6 +3427,354 @@ paths: - jwt: [] tags: - Host + /api/host/{hostname}/identify_device: + post: + description: "\n Identify a device by switching on the device light for\ + \ N seconds.\n :param hostname: The hostname of the device to process.\n\ + \ :param device: The device identifier to process, e.g. ``/dev/dm-0``\ + \ or\n ``ABC1234DEF567-1R1234_ABC8DE0Q``.\n :param duration:\ + \ The duration in seconds how long the LED should flash.\n " + parameters: + - in: path + name: hostname + required: true + schema: + type: string + requestBody: + content: + application/json: + schema: + properties: + device: + type: string + duration: + type: string + required: + - device + - duration + type: object + responses: + '201': + content: + application/vnd.ceph.api.v1.0+json: + type: object + description: Resource created. + '202': + content: + application/vnd.ceph.api.v1.0+json: + type: object + description: Operation is still executing. Please check the task queue. + '400': + description: Operation exception. Please check the response body for details. + '401': + description: Unauthenticated access. Please login first. + '403': + description: Unauthorized access. Please check your permissions. + '500': + description: Unexpected error. Please check the response body for the stack + trace. + security: + - jwt: [] + tags: + - Host + /api/host/{hostname}/inventory: + get: + parameters: + - description: Hostname + in: path + name: hostname + required: true + schema: + type: string + - allowEmptyValue: true + description: Trigger asynchronous refresh + in: query + name: refresh + schema: + type: string + responses: + '200': + content: + application/vnd.ceph.api.v1.0+json: + schema: + properties: + addr: + description: Host address + type: string + devices: + description: Host devices + items: + properties: + available: + description: If the device can be provisioned to an OSD + type: boolean + device_id: + description: Device's udev ID + type: string + human_readable_type: + description: Device type. ssd or hdd + type: string + lsm_data: + description: '' + properties: + errors: + description: '' + items: + type: string + type: array + health: + description: '' + type: string + ledSupport: + description: '' + properties: + FAILstatus: + description: '' + type: string + FAILsupport: + description: '' + type: string + IDENTstatus: + description: '' + type: string + IDENTsupport: + description: '' + type: string + required: + - IDENTsupport + - IDENTstatus + - FAILsupport + - FAILstatus + type: object + linkSpeed: + description: '' + type: string + mediaType: + description: '' + type: string + rpm: + description: '' + type: string + serialNum: + description: '' + type: string + transport: + description: '' + type: string + required: + - serialNum + - transport + - mediaType + - rpm + - linkSpeed + - health + - ledSupport + - errors + type: object + lvs: + description: '' + items: + properties: + block_uuid: + description: '' + type: string + cluster_fsid: + description: '' + type: string + cluster_name: + description: '' + type: string + name: + description: '' + type: string + osd_fsid: + description: '' + type: string + osd_id: + description: '' + type: string + osdspec_affinity: + description: '' + type: string + type: + description: '' + type: string + required: + - name + - osd_id + - cluster_name + - type + - osd_fsid + - cluster_fsid + - osdspec_affinity + - block_uuid + type: object + type: array + osd_ids: + description: Device OSD IDs + items: + type: integer + type: array + path: + description: Device path + type: string + rejected_reasons: + description: '' + items: + type: string + type: array + sys_api: + description: '' + properties: + human_readable_size: + description: '' + type: string + locked: + description: '' + type: integer + model: + description: '' + type: string + nr_requests: + description: '' + type: string + partitions: + description: '' + properties: + partition_name: + description: '' + properties: + holders: + description: '' + items: + type: string + type: array + human_readable_size: + description: '' + type: string + sectors: + description: '' + type: string + sectorsize: + description: '' + type: integer + size: + description: '' + type: integer + start: + description: '' + type: string + required: + - start + - sectors + - sectorsize + - size + - human_readable_size + - holders + type: object + required: + - partition_name + type: object + path: + description: '' + type: string + removable: + description: '' + type: string + rev: + description: '' + type: string + ro: + description: '' + type: string + rotational: + description: '' + type: string + sas_address: + description: '' + type: string + sas_device_handle: + description: '' + type: string + scheduler_mode: + description: '' + type: string + sectors: + description: '' + type: integer + sectorsize: + description: '' + type: string + size: + description: '' + type: integer + support_discard: + description: '' + type: string + vendor: + description: '' + type: string + required: + - removable + - ro + - vendor + - model + - rev + - sas_address + - sas_device_handle + - support_discard + - rotational + - nr_requests + - scheduler_mode + - partitions + - sectors + - sectorsize + - size + - human_readable_size + - path + - locked + type: object + required: + - rejected_reasons + - available + - path + - sys_api + - lvs + - human_readable_type + - device_id + - lsm_data + - osd_ids + type: object + type: array + labels: + description: Host labels + items: + type: string + type: array + name: + description: Hostname + type: string + required: + - name + - addr + - devices + - labels + type: object + description: OK + '400': + description: Operation exception. Please check the response body for details. + '401': + description: Unauthenticated access. Please login first. + '403': + description: Unauthorized access. Please check your permissions. + '500': + description: Unexpected error. Please check the response body for the stack + trace. + security: + - jwt: [] + summary: Get inventory of a host + tags: + - Host /api/host/{hostname}/smart: get: parameters: @@ -5352,86 +5700,6 @@ paths: summary: Status of NFS-Ganesha management feature tags: - NFS-Ganesha - /api/orchestrator/identify_device: - post: - description: "\n Identify a device by switching on the device light for\ - \ N seconds.\n :param hostname: The hostname of the device to process.\n\ - \ :param device: The device identifier to process, e.g. ``/dev/dm-0``\ - \ or\n ``ABC1234DEF567-1R1234_ABC8DE0Q``.\n :param duration:\ - \ The duration in seconds how long the LED should flash.\n " - parameters: [] - requestBody: - content: - application/json: - schema: - properties: - device: - type: string - duration: - type: string - hostname: - type: string - required: - - hostname - - device - - duration - type: object - responses: - '201': - content: - application/vnd.ceph.api.v1.0+json: - type: object - description: Resource created. - '202': - content: - application/vnd.ceph.api.v1.0+json: - type: object - description: Operation is still executing. Please check the task queue. - '400': - description: Operation exception. Please check the response body for details. - '401': - description: Unauthenticated access. Please login first. - '403': - description: Unauthorized access. Please check your permissions. - '500': - description: Unexpected error. Please check the response body for the stack - trace. - security: - - jwt: [] - tags: - - Orchestrator - /api/orchestrator/inventory: - get: - parameters: - - allowEmptyValue: true - in: query - name: hostname - schema: - type: string - - allowEmptyValue: true - in: query - name: refresh - schema: - type: string - responses: - '200': - content: - application/vnd.ceph.api.v1.0+json: - type: object - description: OK - '400': - description: Operation exception. Please check the response body for details. - '401': - description: Unauthenticated access. Please login first. - '403': - description: Unauthorized access. Please check your permissions. - '500': - description: Unexpected error. Please check the response body for the stack - trace. - security: - - jwt: [] - tags: - - OrchestratorInventory /api/orchestrator/status: get: parameters: [] @@ -9958,8 +10226,6 @@ tags: name: OSD - description: Orchestrator Management API name: Orchestrator -- description: Get Orchestrator Inventory Details - name: OrchestratorInventory - description: OSD Perf Counters Management API name: OsdPerfCounter - description: Perf Counters Management API diff --git a/src/pybind/mgr/dashboard/tests/test_host.py b/src/pybind/mgr/dashboard/tests/test_host.py index 9c2500cd0787a..050e28e881076 100644 --- a/src/pybind/mgr/dashboard/tests/test_host.py +++ b/src/pybind/mgr/dashboard/tests/test_host.py @@ -1,26 +1,55 @@ +import contextlib import unittest +from typing import List, Optional +from unittest import mock -try: - import mock -except ImportError: - from unittest import mock - -from orchestrator import HostSpec +from orchestrator import HostSpec, InventoryHost from .. import mgr -from ..controllers.host import Host, HostUi, get_hosts +from ..controllers.host import Host, HostUi, get_device_osd_map, get_hosts, get_inventories +from ..tools import NotificationQueue, TaskManager from . import ControllerTestCase # pylint: disable=no-name-in-module +@contextlib.contextmanager +def patch_orch(available: bool, hosts: Optional[List[HostSpec]] = None, + inventory: Optional[List[dict]] = None): + with mock.patch('dashboard.controllers.orchestrator.OrchClient.instance') as instance: + fake_client = mock.Mock() + fake_client.available.return_value = available + fake_client.get_missing_features.return_value = [] + + if hosts is not None: + fake_client.hosts.list.return_value = hosts + + if inventory is not None: + def _list_inventory(hosts=None, refresh=False): # pylint: disable=unused-argument + inv_hosts = [] + for inv_host in inventory: + if hosts is None or inv_host['name'] in hosts: + inv_hosts.append(InventoryHost.from_json(inv_host)) + return inv_hosts + fake_client.inventory.list.side_effect = _list_inventory + + instance.return_value = fake_client + yield fake_client + + class HostControllerTest(ControllerTestCase): URL_HOST = '/api/host' @classmethod def setup_server(cls): + NotificationQueue.start_queue() + TaskManager.init() # pylint: disable=protected-access Host._cp_config['tools.authenticate.on'] = False cls.setup_controllers([Host]) + @classmethod + def tearDownClass(cls): + NotificationQueue.stop() + @mock.patch('dashboard.controllers.host.get_hosts') def test_host_list(self, mock_get_hosts): hosts = [{ @@ -70,60 +99,96 @@ class HostControllerTest(ControllerTestCase): self.assertStatus(200) self.assertJsonBody(hosts) - @mock.patch('dashboard.controllers.orchestrator.OrchClient.instance') - def test_get_1(self, instance): + def test_get_1(self): mgr.list_servers.return_value = [] - fake_client = mock.Mock() - fake_client.available.return_value = False - instance.return_value = fake_client - - self._get('{}/node1'.format(self.URL_HOST)) - self.assertStatus(404) + with patch_orch(False): + self._get('{}/node1'.format(self.URL_HOST)) + self.assertStatus(404) - @mock.patch('dashboard.controllers.orchestrator.OrchClient.instance') - def test_get_2(self, instance): + def test_get_2(self): mgr.list_servers.return_value = [{'hostname': 'node1'}] - fake_client = mock.Mock() - fake_client.available.return_value = False - instance.return_value = fake_client - - self._get('{}/node1'.format(self.URL_HOST)) - self.assertStatus(200) - self.assertIn('labels', self.json_body()) + with patch_orch(False): + self._get('{}/node1'.format(self.URL_HOST)) + self.assertStatus(200) + self.assertIn('labels', self.json_body()) - @mock.patch('dashboard.controllers.orchestrator.OrchClient.instance') - def test_get_3(self, instance): + def test_get_3(self): mgr.list_servers.return_value = [] - fake_client = mock.Mock() - fake_client.available.return_value = True - fake_client.hosts.list.return_value = [HostSpec('node1')] - instance.return_value = fake_client + with patch_orch(True, hosts=[HostSpec('node1')]): + self._get('{}/node1'.format(self.URL_HOST)) + self.assertStatus(200) + self.assertIn('labels', self.json_body()) - self._get('{}/node1'.format(self.URL_HOST)) - self.assertStatus(200) - self.assertIn('labels', self.json_body()) - - @mock.patch('dashboard.controllers.orchestrator.OrchClient.instance') - def test_set_labels(self, instance): + def test_set_labels(self): mgr.list_servers.return_value = [] - - fake_client = mock.Mock() - fake_client.available.return_value = True - fake_client.get_missing_features.return_value = [] - fake_client.hosts.list.return_value = [ + orch_hosts = [ HostSpec('node0', labels=['aaa', 'bbb']) ] - fake_client.hosts.remove_label = mock.Mock() - fake_client.hosts.add_label = mock.Mock() - instance.return_value = fake_client + with patch_orch(True, hosts=orch_hosts) as fake_client: + fake_client.hosts.remove_label = mock.Mock() + fake_client.hosts.add_label = mock.Mock() + + self._put('{}/node0'.format(self.URL_HOST), {'labels': ['bbb', 'ccc']}) + self.assertStatus(200) + fake_client.hosts.remove_label.assert_called_once_with('node0', 'aaa') + fake_client.hosts.add_label.assert_called_once_with('node0', 'ccc') + + @mock.patch('dashboard.controllers.host.time') + def test_identify_device(self, mock_time): + url = '{}/host-0/identify_device'.format(self.URL_HOST) + with patch_orch(True) as fake_client: + payload = { + 'device': '/dev/sdz', + 'duration': '1' + } + self._task_post(url, payload) + self.assertStatus(200) + mock_time.sleep.assert_called() + calls = [ + mock.call('host-0', '/dev/sdz', 'ident', True), + mock.call('host-0', '/dev/sdz', 'ident', False), + ] + fake_client.blink_device_light.assert_has_calls(calls) + + @mock.patch('dashboard.controllers.host.get_inventories') + def test_inventory(self, mock_get_inventories): + inventory_url = '{}/host-0/inventory'.format(self.URL_HOST) + with patch_orch(True): + tests = [ + { + 'url': inventory_url, + 'inventories': [{'a': 'b'}], + 'refresh': None, + 'resp': {'a': 'b'} + }, + { + 'url': '{}?refresh=true'.format(inventory_url), + 'inventories': [{'a': 'b'}], + 'refresh': "true", + 'resp': {'a': 'b'} + }, + { + 'url': inventory_url, + 'inventories': [], + 'refresh': None, + 'resp': {} + }, + ] + for test in tests: + mock_get_inventories.reset_mock() + mock_get_inventories.return_value = test['inventories'] + self._get(test['url']) + mock_get_inventories.assert_called_once_with(['host-0'], test['refresh']) + self.assertEqual(self.json_body(), test['resp']) + self.assertStatus(200) - self._put('{}/node0'.format(self.URL_HOST), {'labels': ['bbb', 'ccc']}) - self.assertStatus(200) - fake_client.hosts.remove_label.assert_called_once_with('node0', 'aaa') - fake_client.hosts.add_label.assert_called_once_with('node0', 'ccc') + # list without orchestrator service + with patch_orch(False): + self._get(inventory_url) + self.assertStatus(503) class HostUiControllerTest(ControllerTestCase): @@ -135,66 +200,179 @@ class HostUiControllerTest(ControllerTestCase): HostUi._cp_config['tools.authenticate.on'] = False cls.setup_controllers([HostUi]) - @mock.patch('dashboard.controllers.orchestrator.OrchClient.instance') - def test_labels(self, instance): - fake_client = mock.Mock() - fake_client.available.return_value = True - fake_client.hosts.list.return_value = [ + def test_labels(self): + orch_hosts = [ HostSpec('node1', labels=['foo']), HostSpec('node2', labels=['foo', 'bar']) ] - instance.return_value = fake_client - self._get('{}/labels'.format(self.URL_HOST)) - self.assertStatus(200) - labels = self.json_body() - labels.sort() - self.assertListEqual(labels, ['bar', 'foo']) + with patch_orch(True, hosts=orch_hosts): + self._get('{}/labels'.format(self.URL_HOST)) + self.assertStatus(200) + labels = self.json_body() + labels.sort() + self.assertListEqual(labels, ['bar', 'foo']) + + @mock.patch('dashboard.controllers.host.get_inventories') + def test_inventory(self, mock_get_inventories): + inventory_url = '{}/inventory'.format(self.URL_HOST) + with patch_orch(True): + tests = [ + { + 'url': inventory_url, + 'refresh': None + }, + { + 'url': '{}?refresh=true'.format(inventory_url), + 'refresh': "true" + }, + ] + for test in tests: + mock_get_inventories.reset_mock() + mock_get_inventories.return_value = [{'a': 'b'}] + self._get(test['url']) + mock_get_inventories.assert_called_once_with(None, test['refresh']) + self.assertEqual(self.json_body(), [{'a': 'b'}]) + self.assertStatus(200) + + # list without orchestrator service + with patch_orch(False): + self._get(inventory_url) + self.assertStatus(503) class TestHosts(unittest.TestCase): - @mock.patch('dashboard.controllers.orchestrator.OrchClient.instance') - def test_get_hosts(self, instance): + def test_get_hosts(self): mgr.list_servers.return_value = [{ 'hostname': 'node1' }, { 'hostname': 'localhost' }] - - fake_client = mock.Mock() - fake_client.available.return_value = True - fake_client.hosts.list.return_value = [ + orch_hosts = [ HostSpec('node1', labels=['foo', 'bar']), HostSpec('node2', labels=['bar']) ] - instance.return_value = fake_client - hosts = get_hosts() - self.assertEqual(len(hosts), 3) - checks = { - 'localhost': { - 'sources': { - 'ceph': True, - 'orchestrator': False + with patch_orch(True, hosts=orch_hosts): + hosts = get_hosts() + self.assertEqual(len(hosts), 3) + checks = { + 'localhost': { + 'sources': { + 'ceph': True, + 'orchestrator': False + }, + 'labels': [] }, - 'labels': [] + 'node1': { + 'sources': { + 'ceph': True, + 'orchestrator': True + }, + 'labels': ['bar', 'foo'] + }, + 'node2': { + 'sources': { + 'ceph': False, + 'orchestrator': True + }, + 'labels': ['bar'] + } + } + for host in hosts: + hostname = host['hostname'] + self.assertDictEqual(host['sources'], checks[hostname]['sources']) + self.assertListEqual(host['labels'], checks[hostname]['labels']) + + @mock.patch('dashboard.controllers.host.mgr.get') + def test_get_device_osd_map(self, mgr_get): + mgr_get.side_effect = lambda key: { + 'osd_metadata': { + '0': { + 'hostname': 'node0', + 'devices': 'nvme0n1,sdb', + }, + '1': { + 'hostname': 'node0', + 'devices': 'nvme0n1,sdc', + }, + '2': { + 'hostname': 'node1', + 'devices': 'sda', + }, + '3': { + 'hostname': 'node2', + 'devices': '', + } + } + }[key] + + device_osd_map = get_device_osd_map() + mgr.get.assert_called_with('osd_metadata') + # sort OSD IDs to make assertDictEqual work + for devices in device_osd_map.values(): + for host in devices.keys(): + devices[host] = sorted(devices[host]) + self.assertDictEqual(device_osd_map, { + 'node0': { + 'nvme0n1': [0, 1], + 'sdb': [0], + 'sdc': [1], }, 'node1': { - 'sources': { - 'ceph': True, - 'orchestrator': True - }, - 'labels': ['bar', 'foo'] + 'sda': [2] + } + }) + + @mock.patch('dashboard.controllers.host.str_to_bool') + @mock.patch('dashboard.controllers.host.get_device_osd_map') + def test_get_inventories(self, mock_get_device_osd_map, mock_str_to_bool): + mock_get_device_osd_map.return_value = { + 'host-0': { + 'nvme0n1': [1, 2], + 'sdb': [1], + 'sdc': [2] }, - 'node2': { - 'sources': { - 'ceph': False, - 'orchestrator': True - }, - 'labels': ['bar'] + 'host-1': { + 'sdb': [3] } } - for host in hosts: - hostname = host['hostname'] - self.assertDictEqual(host['sources'], checks[hostname]['sources']) - self.assertListEqual(host['labels'], checks[hostname]['labels']) + inventory = [ + { + 'name': 'host-0', + 'addr': '1.2.3.4', + 'devices': [ + {'path': 'nvme0n1'}, + {'path': '/dev/sdb'}, + {'path': '/dev/sdc'}, + ] + }, + { + 'name': 'host-1', + 'addr': '1.2.3.5', + 'devices': [ + {'path': '/dev/sda'}, + {'path': 'sdb'}, + ] + } + ] + + with patch_orch(True, inventory=inventory) as orch_client: + mock_str_to_bool.return_value = True + + hosts = ['host-0', 'host-1'] + inventories = get_inventories(hosts, 'true') + mock_str_to_bool.assert_called_with('true') + orch_client.inventory.list.assert_called_once_with(hosts=hosts, refresh=True) + self.assertEqual(len(inventories), 2) + host0 = inventories[0] + self.assertEqual(host0['name'], 'host-0') + self.assertEqual(host0['addr'], '1.2.3.4') + self.assertEqual(host0['devices'][0]['osd_ids'], [1, 2]) + self.assertEqual(host0['devices'][1]['osd_ids'], [1]) + self.assertEqual(host0['devices'][2]['osd_ids'], [2]) + host1 = inventories[1] + self.assertEqual(host1['name'], 'host-1') + self.assertEqual(host1['addr'], '1.2.3.5') + self.assertEqual(host1['devices'][0]['osd_ids'], []) + self.assertEqual(host1['devices'][1]['osd_ids'], [3]) diff --git a/src/pybind/mgr/dashboard/tests/test_orchestrator.py b/src/pybind/mgr/dashboard/tests/test_orchestrator.py index 00102f36a5875..d9ee85cf3053f 100644 --- a/src/pybind/mgr/dashboard/tests/test_orchestrator.py +++ b/src/pybind/mgr/dashboard/tests/test_orchestrator.py @@ -1,16 +1,10 @@ import inspect import unittest +from unittest import mock -try: - import mock -except ImportError: - from unittest import mock - -from orchestrator import InventoryHost from orchestrator import Orchestrator as OrchestratorBase -from .. import mgr -from ..controllers.orchestrator import Orchestrator, OrchestratorInventory, get_device_osd_map +from ..controllers.orchestrator import Orchestrator from ..services.orchestrator import OrchFeature from . import ControllerTestCase # pylint: disable=no-name-in-module @@ -23,9 +17,7 @@ class OrchestratorControllerTest(ControllerTestCase): def setup_server(cls): # pylint: disable=protected-access Orchestrator._cp_config['tools.authenticate.on'] = False - OrchestratorInventory._cp_config['tools.authenticate.on'] = False - cls.setup_controllers([Orchestrator, - OrchestratorInventory]) + cls.setup_controllers([Orchestrator]) @mock.patch('dashboard.controllers.orchestrator.OrchClient.instance') def test_status_get(self, instance): @@ -39,127 +31,8 @@ class OrchestratorControllerTest(ControllerTestCase): self.assertStatus(200) self.assertJsonBody(status) - def _set_inventory(self, mock_instance, inventory): - # pylint: disable=unused-argument - def _list_inventory(hosts=None, refresh=False): - inv_hosts = [] - for inv_host in inventory: - if hosts is None or inv_host['name'] in hosts: - inv_hosts.append(InventoryHost.from_json(inv_host)) - return inv_hosts - mock_instance.inventory.list.side_effect = _list_inventory - - @mock.patch('dashboard.controllers.orchestrator.get_device_osd_map') - @mock.patch('dashboard.controllers.orchestrator.OrchClient.instance') - def test_inventory_list(self, instance, get_dev_osd_map): - get_dev_osd_map.return_value = { - 'host-0': { - 'nvme0n1': [1, 2], - 'sdb': [1], - 'sdc': [2] - }, - 'host-1': { - 'sdb': [3] - } - } - inventory = [ - { - 'name': 'host-0', - 'addr': '1.2.3.4', - 'devices': [ - {'path': 'nvme0n1'}, - {'path': '/dev/sdb'}, - {'path': '/dev/sdc'}, - ] - }, - { - 'name': 'host-1', - 'addr': '1.2.3.5', - 'devices': [ - {'path': '/dev/sda'}, - {'path': 'sdb'}, - ] - } - ] - fake_client = mock.Mock() - fake_client.available.return_value = True - fake_client.get_missing_features.return_value = [] - self._set_inventory(fake_client, inventory) - instance.return_value = fake_client - - # list - self._get(self.URL_INVENTORY) - self.assertStatus(200) - resp = self.json_body() - self.assertEqual(len(resp), 2) - host0 = resp[0] - self.assertEqual(host0['name'], 'host-0') - self.assertEqual(host0['addr'], '1.2.3.4') - self.assertEqual(host0['devices'][0]['osd_ids'], [1, 2]) - self.assertEqual(host0['devices'][1]['osd_ids'], [1]) - self.assertEqual(host0['devices'][2]['osd_ids'], [2]) - host1 = resp[1] - self.assertEqual(host1['name'], 'host-1') - self.assertEqual(host1['addr'], '1.2.3.5') - self.assertEqual(host1['devices'][0]['osd_ids'], []) - self.assertEqual(host1['devices'][1]['osd_ids'], [3]) - - # list with existent hostname - self._get('{}?hostname=host-0'.format(self.URL_INVENTORY)) - self.assertStatus(200) - self.assertEqual(self.json_body()[0]['name'], 'host-0') - - # list with non-existent inventory - self._get('{}?hostname=host-10'.format(self.URL_INVENTORY)) - self.assertStatus(200) - self.assertJsonBody([]) - - # list without orchestrator service - fake_client.available.return_value = False - self._get(self.URL_INVENTORY) - self.assertStatus(503) - class TestOrchestrator(unittest.TestCase): - def test_get_device_osd_map(self): - mgr.get.side_effect = lambda key: { - 'osd_metadata': { - '0': { - 'hostname': 'node0', - 'devices': 'nvme0n1,sdb', - }, - '1': { - 'hostname': 'node0', - 'devices': 'nvme0n1,sdc', - }, - '2': { - 'hostname': 'node1', - 'devices': 'sda', - }, - '3': { - 'hostname': 'node2', - 'devices': '', - } - } - }[key] - - device_osd_map = get_device_osd_map() - mgr.get.assert_called_with('osd_metadata') - # sort OSD IDs to make assertDictEqual work - for devices in device_osd_map.values(): - for host in devices.keys(): - devices[host] = sorted(devices[host]) - self.assertDictEqual(device_osd_map, { - 'node0': { - 'nvme0n1': [0, 1], - 'sdb': [0], - 'sdc': [1], - }, - 'node1': { - 'sda': [2] - } - }) - def test_features_has_corresponding_methods(self): defined_methods = [v for k, v in inspect.getmembers( OrchFeature, lambda m: not inspect.isroutine(m)) if not k.startswith('_')] -- 2.39.5