From: Avan Thakkar Date: Tue, 3 Aug 2021 09:01:57 +0000 (+0530) Subject: mgr/dashboard: introduce gather facts in host list X-Git-Tag: v17.1.0~699^2~1 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=6644a00a2c6d6a7fa709ee82f5ada0d73a0929a8;p=ceph.git mgr/dashboard: introduce gather facts in host list Fixes: https://tracker.ceph.com/issues/52017 Signed-off-by: Avan Thakkar --- diff --git a/qa/tasks/mgr/dashboard/test_auth.py b/qa/tasks/mgr/dashboard/test_auth.py index 98566344444f..a2266229bef7 100644 --- a/qa/tasks/mgr/dashboard/test_auth.py +++ b/qa/tasks/mgr/dashboard/test_auth.py @@ -154,7 +154,7 @@ class AuthTest(DashboardTestCase): self.assertJsonBody({ "redirect_url": "#/login" }) - self._get("/api/host") + self._get("/api/host", version='1.1') self.assertStatus(401) self.set_jwt_token(None) @@ -169,7 +169,7 @@ class AuthTest(DashboardTestCase): self.assertJsonBody({ "redirect_url": "#/login" }) - self._get("/api/host", set_cookies=True) + self._get("/api/host", set_cookies=True, version='1.1') self.assertStatus(401) self.set_jwt_token(None) @@ -179,10 +179,10 @@ class AuthTest(DashboardTestCase): self._post("/api/auth", {'username': 'admin', 'password': 'admin'}) self.assertStatus(201) self.set_jwt_token(self.jsonBody()['token']) - self._get("/api/host") + self._get("/api/host", version='1.1') self.assertStatus(200) time.sleep(6) - self._get("/api/host") + self._get("/api/host", version='1.1') self.assertStatus(401) self._ceph_cmd(['dashboard', 'set-jwt-token-ttl', '28800']) self.set_jwt_token(None) @@ -192,10 +192,10 @@ class AuthTest(DashboardTestCase): self._post("/api/auth", {'username': 'admin', 'password': 'admin'}, set_cookies=True) self.assertStatus(201) self.set_jwt_token(self.jsonBody()['token']) - self._get("/api/host", set_cookies=True) + self._get("/api/host", set_cookies=True, version='1.1') self.assertStatus(200) time.sleep(6) - self._get("/api/host", set_cookies=True) + self._get("/api/host", set_cookies=True, version='1.1') self.assertStatus(401) self._ceph_cmd(['dashboard', 'set-jwt-token-ttl', '28800']) self.set_jwt_token(None) @@ -209,7 +209,7 @@ class AuthTest(DashboardTestCase): # the following call adds the token to the blocklist self._post("/api/auth/logout") self.assertStatus(200) - self._get("/api/host") + self._get("/api/host", version='1.1') self.assertStatus(401) time.sleep(6) self._ceph_cmd(['dashboard', 'set-jwt-token-ttl', '28800']) @@ -229,7 +229,7 @@ class AuthTest(DashboardTestCase): # the following call adds the token to the blocklist self._post("/api/auth/logout", set_cookies=True) self.assertStatus(200) - self._get("/api/host", set_cookies=True) + self._get("/api/host", set_cookies=True, version='1.1') self.assertStatus(401) time.sleep(6) self._ceph_cmd(['dashboard', 'set-jwt-token-ttl', '28800']) @@ -243,61 +243,61 @@ class AuthTest(DashboardTestCase): def test_unauthorized(self): # test with Authorization header - self._get("/api/host") + self._get("/api/host", version='1.1') self.assertStatus(401) # test with Cookies set - self._get("/api/host", set_cookies=True) + self._get("/api/host", set_cookies=True, version='1.1') self.assertStatus(401) def test_invalidate_token_by_admin(self): # test with Authorization header - self._get("/api/host") + self._get("/api/host", version='1.1') self.assertStatus(401) self.create_user('user', 'user', ['read-only']) time.sleep(1) self._post("/api/auth", {'username': 'user', 'password': 'user'}) self.assertStatus(201) self.set_jwt_token(self.jsonBody()['token']) - self._get("/api/host") + self._get("/api/host", version='1.1') self.assertStatus(200) time.sleep(1) self._ceph_cmd_with_secret(['dashboard', 'ac-user-set-password', '--force-password', 'user'], 'user2') time.sleep(1) - self._get("/api/host") + self._get("/api/host", version='1.1') self.assertStatus(401) self.set_jwt_token(None) self._post("/api/auth", {'username': 'user', 'password': 'user2'}) self.assertStatus(201) self.set_jwt_token(self.jsonBody()['token']) - self._get("/api/host") + self._get("/api/host", version='1.1') self.assertStatus(200) self.delete_user("user") # test with Cookies set - self._get("/api/host", set_cookies=True) + self._get("/api/host", set_cookies=True, version='1.1') self.assertStatus(401) self.create_user('user', 'user', ['read-only']) time.sleep(1) self._post("/api/auth", {'username': 'user', 'password': 'user'}, set_cookies=True) self.assertStatus(201) self.set_jwt_token(self.jsonBody()['token']) - self._get("/api/host", set_cookies=True) + self._get("/api/host", set_cookies=True, version='1.1') self.assertStatus(200) time.sleep(1) self._ceph_cmd_with_secret(['dashboard', 'ac-user-set-password', '--force-password', 'user'], 'user2') time.sleep(1) - self._get("/api/host", set_cookies=True) + self._get("/api/host", set_cookies=True, version='1.1') self.assertStatus(401) self.set_jwt_token(None) self._post("/api/auth", {'username': 'user', 'password': 'user2'}, set_cookies=True) self.assertStatus(201) self.set_jwt_token(self.jsonBody()['token']) - self._get("/api/host", set_cookies=True) + self._get("/api/host", set_cookies=True, version='1.1') self.assertStatus(200) self.delete_user("user") diff --git a/qa/tasks/mgr/dashboard/test_host.py b/qa/tasks/mgr/dashboard/test_host.py index 124fff8d1544..78d784473f3c 100644 --- a/qa/tasks/mgr/dashboard/test_host.py +++ b/qa/tasks/mgr/dashboard/test_host.py @@ -32,11 +32,11 @@ class HostControllerTest(DashboardTestCase): @DashboardTestCase.RunAs('test', 'test', ['block-manager']) def test_access_permissions(self): - self._get(self.URL_HOST) + self._get(self.URL_HOST, version='1.1') self.assertStatus(403) def test_host_list(self): - data = self._get(self.URL_HOST) + data = self._get(self.URL_HOST, version='1.1') self.assertStatus(200) orch_hostnames = {inventory_node['name'] for inventory_node in @@ -65,14 +65,14 @@ class HostControllerTest(DashboardTestCase): self.assertIn(server['hostname'], orch_hostnames) def test_host_list_with_sources(self): - data = self._get('{}?sources=orchestrator'.format(self.URL_HOST)) + data = self._get('{}?sources=orchestrator'.format(self.URL_HOST), version='1.1') self.assertStatus(200) test_hostnames = {inventory_node['name'] for inventory_node in self.ORCHESTRATOR_TEST_DATA['inventory']} resp_hostnames = {host['hostname'] for host in data} self.assertEqual(test_hostnames, resp_hostnames) - data = self._get('{}?sources=ceph'.format(self.URL_HOST)) + data = self._get('{}?sources=ceph'.format(self.URL_HOST), version='1.1') self.assertStatus(200) test_hostnames = {inventory_node['name'] for inventory_node in self.ORCHESTRATOR_TEST_DATA['inventory']} @@ -80,7 +80,7 @@ class HostControllerTest(DashboardTestCase): self.assertEqual(len(test_hostnames.intersection(resp_hostnames)), 0) def test_host_devices(self): - hosts = self._get('{}'.format(self.URL_HOST)) + hosts = self._get('{}'.format(self.URL_HOST), version='1.1') hosts = [host['hostname'] for host in hosts if host['hostname'] != ''] assert hosts[0] data = self._get('{}/devices'.format('{}/{}'.format(self.URL_HOST, hosts[0]))) @@ -88,7 +88,7 @@ class HostControllerTest(DashboardTestCase): self.assertSchema(data, devices_schema) def test_host_daemons(self): - hosts = self._get('{}'.format(self.URL_HOST)) + hosts = self._get('{}'.format(self.URL_HOST), version='1.1') hosts = [host['hostname'] for host in hosts if host['hostname'] != ''] assert hosts[0] data = self._get('{}/daemons'.format('{}/{}'.format(self.URL_HOST, hosts[0]))) @@ -100,7 +100,7 @@ class HostControllerTest(DashboardTestCase): }))) def test_host_smart(self): - hosts = self._get('{}'.format(self.URL_HOST)) + hosts = self._get('{}'.format(self.URL_HOST), version='1.1') hosts = [host['hostname'] for host in hosts if host['hostname'] != ''] assert hosts[0] self._get('{}/smart'.format('{}/{}'.format(self.URL_HOST, hosts[0]))) diff --git a/src/pybind/mgr/dashboard/controllers/cluster.py b/src/pybind/mgr/dashboard/controllers/cluster.py index 5ec49e39b1c2..d8170e672e99 100644 --- a/src/pybind/mgr/dashboard/controllers/cluster.py +++ b/src/pybind/mgr/dashboard/controllers/cluster.py @@ -2,18 +2,19 @@ from ..security import Scope from ..services.cluster import ClusterModel -from . import ApiController, ControllerDoc, EndpointDoc, RESTController +from . import APIDoc, APIRouter, EndpointDoc, RESTController +from ._version import APIVersion -@ApiController('/cluster', Scope.CONFIG_OPT) -@ControllerDoc("Get Cluster Details", "Cluster") +@APIRouter('/cluster', Scope.CONFIG_OPT) +@APIDoc("Get Cluster Details", "Cluster") class Cluster(RESTController): - @RESTController.MethodMap(version='0.1') + @RESTController.MethodMap(version=APIVersion.EXPERIMENTAL) @EndpointDoc("Get the cluster status") def list(self): return ClusterModel.from_db().dict() - @RESTController.MethodMap(version='0.1') + @RESTController.MethodMap(version=APIVersion.EXPERIMENTAL) @EndpointDoc("Update the cluster status", parameters={'status': (str, 'Cluster Status')}) def singleton_set(self, status: str): diff --git a/src/pybind/mgr/dashboard/controllers/host.py b/src/pybind/mgr/dashboard/controllers/host.py index c246de8f9f12..0a897002a4a1 100644 --- a/src/pybind/mgr/dashboard/controllers/host.py +++ b/src/pybind/mgr/dashboard/controllers/host.py @@ -15,7 +15,7 @@ from ..security import Scope from ..services.ceph_service import CephService from ..services.exception import handle_orchestrator_error from ..services.orchestrator import OrchClient, OrchFeature -from ..tools import TaskManager, str_to_bool +from ..tools import TaskManager, merge_list_of_dicts_by_key, str_to_bool from . import APIDoc, APIRouter, BaseController, Endpoint, EndpointDoc, \ ReadPermission, RESTController, Task, UIRouter, UpdatePermission, \ allow_empty_body @@ -154,10 +154,17 @@ def merge_hosts_by_hostname(ceph_hosts, orch_hosts): return hosts -def get_hosts(from_ceph=True, from_orchestrator=True): +def get_hosts(sources=None): """ Get hosts from various sources. """ + from_ceph = True + from_orchestrator = True + if sources: + _sources = sources.split(',') + from_ceph = 'ceph' in _sources + from_orchestrator = 'orchestrator' in _sources + ceph_hosts = [] if from_ceph: ceph_hosts = [ @@ -165,7 +172,6 @@ def get_hosts(from_ceph=True, from_orchestrator=True): server, { 'addr': '', 'labels': [], - 'service_type': '', 'sources': { 'ceph': True, 'orchestrator': False @@ -273,15 +279,24 @@ class Host(RESTController): @EndpointDoc("List Host Specifications", parameters={ 'sources': (str, 'Host Sources'), + 'facts': (bool, 'Host Facts') }, responses={200: LIST_HOST_SCHEMA}) - def list(self, sources=None): - if sources is None: - return get_hosts() - _sources = sources.split(',') - from_ceph = 'ceph' in _sources - from_orchestrator = 'orchestrator' in _sources - return get_hosts(from_ceph, from_orchestrator) + @RESTController.MethodMap(version=APIVersion(1, 1)) + def list(self, sources=None, facts=False): + hosts = get_hosts(sources) + orch = OrchClient.instance() + if str_to_bool(facts): + if orch.available(): + hosts_facts = orch.hosts.get_facts() + return merge_list_of_dicts_by_key(hosts, hosts_facts, 'hostname') + + raise DashboardException(code='orchestrator_status_unavailable', # pragma: no cover + msg="Please configure and enable the orchestrator if you " + "really want to gather facts from hosts", + component='orchestrator', + http_status_code=400) + return hosts @raise_if_no_orchestrator([OrchFeature.HOST_LIST, OrchFeature.HOST_ADD]) @handle_orchestrator_error('host') diff --git a/src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/workflow/05-create-cluster-review.e2e-spec.ts b/src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/workflow/05-create-cluster-review.e2e-spec.ts index 9097034e1a88..29fd3a27f8ed 100644 --- a/src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/workflow/05-create-cluster-review.e2e-spec.ts +++ b/src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/workflow/05-create-cluster-review.e2e-spec.ts @@ -14,8 +14,8 @@ describe('Create Cluster Review page', () => { cy.get('button[aria-label="Next"]').click(); }); - describe('navigation link and title test', () => { - it('should check if nav-link and title contains Review', () => { + describe('navigation link test', () => { + it('should check if nav-link contains Review', () => { cy.get('.nav-link').should('contain.text', 'Review'); }); }); @@ -28,6 +28,8 @@ describe('Create Cluster Review page', () => { // check for fields in table createCluster.getStatusTables().should('contain.text', 'Hosts'); createCluster.getStatusTables().should('contain.text', 'Storage Capacity'); + createCluster.getStatusTables().should('contain.text', 'CPUs'); + createCluster.getStatusTables().should('contain.text', 'Memory'); }); it('should check Hosts by Services and Host Details tables are present', () => { @@ -46,14 +48,26 @@ describe('Create Cluster Review page', () => { createCluster.getDataTableHeaders(0).contains('Number of Hosts'); // verify correct columns on Host Details table - createCluster.getDataTableHeaders(1).contains('Host Name'); + createCluster.getDataTableHeaders(1).contains('Hostname'); createCluster.getDataTableHeaders(1).contains('Labels'); - }); - it('should check hosts count and default host name are present', () => { - createCluster.getStatusTables().contains(2); + createCluster.getDataTableHeaders(1).contains('CPUs'); + + createCluster.getDataTableHeaders(1).contains('Cores'); + + createCluster.getDataTableHeaders(1).contains('Total Memory'); + + createCluster.getDataTableHeaders(1).contains('Raw Capacity'); + + createCluster.getDataTableHeaders(1).contains('HDDs'); + + createCluster.getDataTableHeaders(1).contains('Flash'); + + createCluster.getDataTableHeaders(1).contains('NICs'); + }); + it('should check default host name is present', () => { createCluster.check_for_host(); }); }); diff --git a/src/pybind/mgr/dashboard/frontend/cypress/integration/page-helper.po.ts b/src/pybind/mgr/dashboard/frontend/cypress/integration/page-helper.po.ts index 4e15088a2c9c..5405da94f012 100644 --- a/src/pybind/mgr/dashboard/frontend/cypress/integration/page-helper.po.ts +++ b/src/pybind/mgr/dashboard/frontend/cypress/integration/page-helper.po.ts @@ -206,7 +206,7 @@ export abstract class PageHelper { getDataTableHeaders(index = 0) { this.waitDataTableToLoad(); - return cy.get('.datatable-header').its(index).find('.datatable-header-cell-label'); + return cy.get('.datatable-header').its(index).find('.datatable-header-cell'); } /** diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/create-cluster/create-cluster-review.component.html b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/create-cluster/create-cluster-review.component.html index e6f31dc9e74f..f59cf6baceb5 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/create-cluster/create-cluster-review.component.html +++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/create-cluster/create-cluster-review.component.html @@ -15,6 +15,16 @@ Number of devices: {{ totalDevices }}. Raw capacity: {{ totalCapacity | dimlessBinary }}. + + CPUs + {{ totalCPUs }} + + + Memory + {{ totalMemory }} + @@ -26,12 +36,14 @@ [columns]="hostsByService['columns']" [toolHeader]="false"> - - Host Details - - + +Host Details + + diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/create-cluster/create-cluster-review.component.scss b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/create-cluster/create-cluster-review.component.scss index e69de29bb2d1..beecca096716 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/create-cluster/create-cluster-review.component.scss +++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/create-cluster/create-cluster-review.component.scss @@ -0,0 +1,5 @@ +cd-hosts { + ::ng-deep .nav { + display: none; + } +} diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/create-cluster/create-cluster-review.component.spec.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/create-cluster/create-cluster-review.component.spec.ts index 8122cb682f0b..3d9652be434e 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/create-cluster/create-cluster-review.component.spec.ts +++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/create-cluster/create-cluster-review.component.spec.ts @@ -2,6 +2,7 @@ import { HttpClientTestingModule } from '@angular/common/http/testing'; import { ComponentFixture, TestBed } from '@angular/core/testing'; import _ from 'lodash'; +import { ToastrModule } from 'ngx-toastr'; import { of } from 'rxjs'; import { CephModule } from '~/app/ceph/ceph.module'; @@ -18,7 +19,7 @@ describe('CreateClusterReviewComponent', () => { let serviceListSpy: jasmine.Spy; configureTestBed({ - imports: [HttpClientTestingModule, SharedModule, CoreModule, CephModule] + imports: [HttpClientTestingModule, SharedModule, ToastrModule.forRoot(), CephModule, CoreModule] }); beforeEach(() => { diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/create-cluster/create-cluster-review.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/create-cluster/create-cluster-review.component.ts index 90e25d1162dc..3e0f44f958a8 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/create-cluster/create-cluster-review.component.ts +++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/create-cluster/create-cluster-review.component.ts @@ -7,6 +7,7 @@ import { HostService } from '~/app/shared/api/host.service'; import { OsdService } from '~/app/shared/api/osd.service'; import { CellTemplate } from '~/app/shared/enum/cell-template.enum'; import { CephServiceSpec } from '~/app/shared/models/service.interface'; +import { DimlessBinaryPipe } from '~/app/shared/pipes/dimless-binary.pipe'; import { WizardStepsService } from '~/app/shared/services/wizard-steps.service'; @Component({ @@ -16,7 +17,6 @@ import { WizardStepsService } from '~/app/shared/services/wizard-steps.service'; }) export class CreateClusterReviewComponent implements OnInit { hosts: object[] = []; - hostsDetails: object; hostsByService: object; hostsCount: number; serviceCount: number; @@ -26,10 +26,13 @@ export class CreateClusterReviewComponent implements OnInit { totalDevices: number; totalCapacity = 0; services: Array = []; + totalCPUs = 0; + totalMemory = 0; constructor( public wizardStepsService: WizardStepsService, public cephServiceService: CephServiceService, + private dimlessBinary: DimlessBinaryPipe, public hostService: HostService, private osdService: OsdService ) {} @@ -41,24 +44,6 @@ export class CreateClusterReviewComponent implements OnInit { let walDeviceCapacity = 0; let dbDevices = 0; let dbDeviceCapacity = 0; - this.hostsDetails = { - columns: [ - { - prop: 'hostname', - name: $localize`Host Name`, - flexGrow: 2 - }, - { - name: $localize`Labels`, - prop: 'labels', - flexGrow: 1, - cellTransformation: CellTemplate.badge, - customTemplateConfig: { - class: 'badge-dark' - } - } - ] - }; this.hostsByService = { columns: [ @@ -88,6 +73,7 @@ export class CreateClusterReviewComponent implements OnInit { (this.serviceOccurrences[serviceKey['service_type']] || 0) + 1; this.uniqueServices.add(serviceKey['service_type']); }); + this.totalMemory = this.dimlessBinary.transform(this.totalMemory); this.uniqueServices.forEach((serviceType) => { this.hostsCountPerService.push({ @@ -99,10 +85,15 @@ export class CreateClusterReviewComponent implements OnInit { this.hostsByService['data'] = [...this.hostsCountPerService]; }); - this.hostService.list().subscribe((resp: object[]) => { + this.hostService.list('true').subscribe((resp: object[]) => { this.hosts = resp; this.hostsCount = this.hosts.length; - this.hostsDetails['data'] = [...this.hosts]; + _.forEach(this.hosts, (hostKey) => { + this.totalCPUs = this.totalCPUs + hostKey['cpu_count']; + // convert to bytes + this.totalMemory = this.totalMemory + hostKey['memory_total_kb'] * 1024; + }); + this.totalMemory = this.dimlessBinary.transform(this.totalMemory); }); if (this.osdService.osdDevices['data']) { diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/create-cluster/create-cluster.component.html b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/create-cluster/create-cluster.component.html index 9ae80f4c0218..5394dc1d33b2 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/create-cluster/create-cluster.component.html +++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/create-cluster/create-cluster.component.html @@ -40,7 +40,7 @@

Add Hosts


- { + this.hostService.list('false').subscribe((hosts) => { hosts.forEach((host) => { const index = host['labels'].indexOf('_no_schedule', 0); if (index > -1) { diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/hosts/host-form/host-form.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/hosts/host-form/host-form.component.ts index 99313a5923aa..704b659127f2 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/hosts/host-form/host-form.component.ts +++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/hosts/host-form/host-form.component.ts @@ -51,7 +51,7 @@ export class HostFormComponent extends CdForm implements OnInit { this.pageURL = 'hosts'; } this.createForm(); - this.hostService.list().subscribe((resp: any[]) => { + this.hostService.list('false').subscribe((resp: any[]) => { this.hostnames = resp.map((host) => { return host['hostname']; }); diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/hosts/hosts.component.html b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/hosts/hosts.component.html index cb0ebca6bcd6..a4d30918cc56 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/hosts/hosts.component.html +++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/hosts/hosts.component.html @@ -67,4 +67,16 @@ Are you sure you want to continue? + + + Unavailable + + + + Flash + diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/hosts/hosts.component.spec.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/hosts/hosts.component.spec.ts index 288f62e7a3d2..2c6a4db6983a 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/hosts/hosts.component.spec.ts +++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/hosts/hosts.component.spec.ts @@ -12,6 +12,7 @@ import { CoreModule } from '~/app/core/core.module'; import { HostService } from '~/app/shared/api/host.service'; import { OrchestratorService } from '~/app/shared/api/orchestrator.service'; import { TableActionsComponent } from '~/app/shared/datatable/table-actions/table-actions.component'; +import { CdTableFetchDataContext } from '~/app/shared/models/cd-table-fetch-data-context'; import { CdTableSelection } from '~/app/shared/models/cd-table-selection'; import { OrchestratorFeature } from '~/app/shared/models/orchestrator.enum'; import { Permissions } from '~/app/shared/models/permissions'; @@ -99,7 +100,6 @@ describe('HostsComponent', () => { } ], hostname: hostname, - ceph_version: 'ceph version Development', labels: ['foo', 'bar'] } ]; @@ -108,14 +108,73 @@ describe('HostsComponent', () => { hostListSpy.and.callFake(() => of(payload)); fixture.detectChanges(); - return fixture.whenStable().then(() => { - fixture.detectChanges(); + component.getHosts(new CdTableFetchDataContext(() => undefined)); + fixture.detectChanges(); - const spans = fixture.debugElement.nativeElement.querySelectorAll( - '.datatable-body-cell-label span' - ); - expect(spans[0].textContent).toBe(hostname); - }); + const spans = fixture.debugElement.nativeElement.querySelectorAll( + '.datatable-body-cell-label span' + ); + expect(spans[0].textContent).toBe(hostname); + }); + + it('should test if host facts are tranformed correctly if orch available', () => { + const payload = [ + { + hostname: 'host_test', + services: [ + { + type: 'osd', + id: '0' + } + ], + cpu_count: 2, + cpu_cores: 1, + memory_total_kb: 1024, + hdd_count: 4, + hdd_capacity_bytes: 1024, + flash_count: 4, + flash_capacity_bytes: 1024, + nic_count: 1 + } + ]; + OrchestratorHelper.mockStatus(true); + hostListSpy.and.callFake(() => of(payload)); + fixture.detectChanges(); + + component.getHosts(new CdTableFetchDataContext(() => undefined)); + expect(hostListSpy).toHaveBeenCalled(); + expect(component.hosts[0]['cpu_count']).toEqual(2); + expect(component.hosts[0]['memory_total_bytes']).toEqual(1048576); + expect(component.hosts[0]['raw_capacity']).toEqual(2048); + expect(component.hosts[0]['hdd_count']).toEqual(4); + expect(component.hosts[0]['flash_count']).toEqual(4); + expect(component.hosts[0]['cpu_cores']).toEqual(1); + expect(component.hosts[0]['nic_count']).toEqual(1); + }); + + it('should test if host facts are unavailable if no orch available', () => { + const payload = [ + { + hostname: 'host_test', + services: [ + { + type: 'osd', + id: '0' + } + ] + } + ]; + OrchestratorHelper.mockStatus(false); + hostListSpy.and.callFake(() => of(payload)); + fixture.detectChanges(); + + component.getHosts(new CdTableFetchDataContext(() => undefined)); + fixture.detectChanges(); + + const spans = fixture.debugElement.nativeElement.querySelectorAll( + '.datatable-body-cell-label span' + ); + expect(spans[7].textContent).toBe('Unavailable'); }); it('should show force maintenance modal when it is safe to stop host', () => { diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/hosts/hosts.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/hosts/hosts.component.ts index 6bd8dbbdc252..fe0b797bd1d7 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/hosts/hosts.component.ts +++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/hosts/hosts.component.ts @@ -1,8 +1,10 @@ -import { Component, Input, OnInit, TemplateRef, ViewChild } from '@angular/core'; +import { Component, Input, OnDestroy, OnInit, TemplateRef, ViewChild } from '@angular/core'; import { Router } from '@angular/router'; import { NgbModalRef } from '@ng-bootstrap/ng-bootstrap'; import _ from 'lodash'; +import { Subscription } from 'rxjs'; +import { map, mergeMap } from 'rxjs/operators'; import { HostService } from '~/app/shared/api/host.service'; import { OrchestratorService } from '~/app/shared/api/orchestrator.service'; @@ -24,7 +26,7 @@ import { FinishedTask } from '~/app/shared/models/finished-task'; import { OrchestratorFeature } from '~/app/shared/models/orchestrator.enum'; import { OrchestratorStatus } from '~/app/shared/models/orchestrator.interface'; import { Permissions } from '~/app/shared/models/permissions'; -import { CephShortVersionPipe } from '~/app/shared/pipes/ceph-short-version.pipe'; +import { DimlessBinaryPipe } from '~/app/shared/pipes/dimless-binary.pipe'; import { AuthStorageService } from '~/app/shared/services/auth-storage.service'; import { ModalService } from '~/app/shared/services/modal.service'; import { NotificationService } from '~/app/shared/services/notification.service'; @@ -40,13 +42,19 @@ const BASE_URL = 'hosts'; styleUrls: ['./hosts.component.scss'], providers: [{ provide: URLBuilderService, useValue: new URLBuilderService(BASE_URL) }] }) -export class HostsComponent extends ListWithDetails implements OnInit { +export class HostsComponent extends ListWithDetails implements OnDestroy, OnInit { + private sub = new Subscription(); + @ViewChild(TableComponent) table: TableComponent; @ViewChild('servicesTpl', { static: true }) public servicesTpl: TemplateRef; @ViewChild('maintenanceConfirmTpl', { static: true }) maintenanceConfirmTpl: TemplateRef; + @ViewChild('orchTmpl', { static: true }) + orchTmpl: TemplateRef; + @ViewChild('flashTmpl', { static: true }) + flashTmpl: TemplateRef; @Input() hiddenColumns: string[] = []; @@ -95,8 +103,8 @@ export class HostsComponent extends ListWithDetails implements OnInit { constructor( private authStorageService: AuthStorageService, + private dimlessBinary: DimlessBinaryPipe, private hostService: HostService, - private cephShortVersionPipe: CephShortVersionPipe, private actionLabels: ActionLabelsI18n, private modalService: ModalService, private taskWrapper: TaskWrapperService, @@ -162,7 +170,7 @@ export class HostsComponent extends ListWithDetails implements OnInit { { name: $localize`Services`, prop: 'services', - flexGrow: 3, + flexGrow: 2, cellTemplate: this.servicesTpl }, { @@ -186,21 +194,59 @@ export class HostsComponent extends ListWithDetails implements OnInit { } }, { - name: $localize`Version`, - prop: 'ceph_version', - flexGrow: 1, - pipe: this.cephShortVersionPipe + name: $localize`Model`, + prop: 'model', + flexGrow: 1 + }, + { + name: $localize`CPUs`, + prop: 'cpu_count', + flexGrow: 0.3 + }, + { + name: $localize`Cores`, + prop: 'cpu_cores', + flexGrow: 0.3 + }, + { + name: $localize`Total Memory`, + prop: 'memory_total_bytes', + pipe: this.dimlessBinary, + flexGrow: 0.4 + }, + { + name: $localize`Raw Capacity`, + prop: 'raw_capacity', + pipe: this.dimlessBinary, + flexGrow: 0.5 + }, + { + name: $localize`HDDs`, + prop: 'hdd_count', + flexGrow: 0.3 + }, + { + name: $localize`Flash`, + prop: 'flash_count', + headerTemplate: this.flashTmpl, + flexGrow: 0.3 + }, + { + name: $localize`NICs`, + prop: 'nic_count', + flexGrow: 0.3 } ]; - this.orchService.status().subscribe((status: OrchestratorStatus) => { - this.orchStatus = status; - }); this.columns = this.columns.filter((col: any) => { return !this.hiddenColumns.includes(col.prop); }); } + ngOnDestroy() { + this.sub.unsubscribe(); + } + updateSelection(selection: CdTableSelection) { this.selection = selection; this.enableButton = false; @@ -343,6 +389,21 @@ export class HostsComponent extends ListWithDetails implements OnInit { }); } + transformHostsData() { + if (this.orchStatus?.available) { + _.forEach(this.hosts, (hostKey) => { + hostKey['memory_total_bytes'] = hostKey['memory_total_kb'] * 1024; + hostKey['raw_capacity'] = hostKey['hdd_capacity_bytes'] + hostKey['flash_capacity_bytes']; + }); + } else { + // mark host facts columns unavailable + for (let column = 4; column < this.columns.length; column++) { + this.columns[column]['prop'] = ''; + this.columns[column]['cellTemplate'] = this.orchTmpl; + } + } + } + getHosts(context: CdTableFetchDataContext) { if (this.isLoadingHosts) { return; @@ -357,24 +418,36 @@ export class HostsComponent extends ListWithDetails implements OnInit { 'tcmu-runner': 'iscsi' }; this.isLoadingHosts = true; - this.hostService.list().subscribe( - (resp: any[]) => { - resp.map((host) => { - host.services.map((service: any) => { - service.cdLink = `/perf_counters/${service.type}/${encodeURIComponent(service.id)}`; - const permission = this.permissions[typeToPermissionKey[service.type]]; - service.canRead = permission ? permission.read : false; - return service; - }); - return host; - }); - this.hosts = resp; - this.isLoadingHosts = false; - }, - () => { - this.isLoadingHosts = false; - context.error(); - } - ); + this.sub = this.orchService + .status() + .pipe( + mergeMap((orchStatus) => { + this.orchStatus = orchStatus; + + return this.hostService.list(`${this.orchStatus?.available}`); + }), + map((hostList: object[]) => + hostList.map((host) => { + host['services'].map((service: any) => { + service.cdLink = `/perf_counters/${service.type}/${encodeURIComponent(service.id)}`; + const permission = this.permissions[typeToPermissionKey[service.type]]; + service.canRead = permission ? permission.read : false; + return service; + }); + return host; + }) + ) + ) + .subscribe( + (hostList) => { + this.hosts = hostList; + this.transformHostsData(); + this.isLoadingHosts = false; + }, + () => { + this.isLoadingHosts = false; + context.error(); + } + ); } } diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/services/service-form/service-form.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/services/service-form/service-form.component.ts index 941eabfb67c6..2b424d7f26a3 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/services/service-form/service-form.component.ts +++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/services/service-form/service-form.component.ts @@ -231,7 +231,7 @@ export class ServiceFormComponent extends CdForm implements OnInit { this.serviceTypes = _.difference(resp, this.hiddenServices).sort(); }); - this.hostService.list().subscribe((resp: object[]) => { + this.hostService.list('false').subscribe((resp: object[]) => { const options: SelectOption[] = []; _.forEach(resp, (host: object) => { if (_.get(host, 'sources.orchestrator', false)) { diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/api/host.service.spec.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/host.service.spec.ts index 8a7de5e25d63..797f94455a25 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/shared/api/host.service.spec.ts +++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/host.service.spec.ts @@ -28,8 +28,8 @@ describe('HostService', () => { it('should call list', fakeAsync(() => { let result; - service.list().subscribe((resp) => (result = resp)); - const req = httpTesting.expectOne('api/host'); + service.list('true').subscribe((resp) => (result = resp)); + const req = httpTesting.expectOne('api/host?facts=true'); expect(req.request.method).toBe('GET'); req.flush(['foo', 'bar']); tick(); diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/api/host.service.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/host.service.ts index 984ee88fff11..a1f2497b5a70 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/shared/api/host.service.ts +++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/host.service.ts @@ -22,8 +22,11 @@ export class HostService { constructor(private http: HttpClient, private deviceService: DeviceService) {} - list(): Observable { - return this.http.get(this.baseURL); + list(facts: string): Observable { + return this.http.get(this.baseURL, { + headers: { Accept: 'application/vnd.ceph.api.v1.1+json' }, + params: { facts: facts } + }); } create(hostname: string, addr: string, labels: string[], status: string) { diff --git a/src/pybind/mgr/dashboard/openapi.yaml b/src/pybind/mgr/dashboard/openapi.yaml index f03102599b0d..ecf34a2bf4ee 100644 --- a/src/pybind/mgr/dashboard/openapi.yaml +++ b/src/pybind/mgr/dashboard/openapi.yaml @@ -3293,10 +3293,16 @@ paths: name: sources schema: type: string + - default: false + description: Host Facts + in: query + name: facts + schema: + type: boolean responses: '200': content: - application/vnd.ceph.api.v1.0+json: + application/vnd.ceph.api.v1.1+json: schema: properties: addr: diff --git a/src/pybind/mgr/dashboard/tests/test_host.py b/src/pybind/mgr/dashboard/tests/test_host.py index 8b1e03b0ce31..a74d7d8e0aa0 100644 --- a/src/pybind/mgr/dashboard/tests/test_host.py +++ b/src/pybind/mgr/dashboard/tests/test_host.py @@ -52,7 +52,7 @@ class HostControllerTest(ControllerTestCase): NotificationQueue.stop() @mock.patch('dashboard.controllers.host.get_hosts') - def test_host_list(self, mock_get_hosts): + def test_host_list_with_sources(self, mock_get_hosts): hosts = [{ 'hostname': 'host-0', 'sources': { @@ -73,33 +73,109 @@ class HostControllerTest(ControllerTestCase): } }] - def _get_hosts(from_ceph=True, from_orchestrator=True): - _hosts = [] - if from_ceph: - _hosts.append(hosts[0]) - if from_orchestrator: - _hosts.append(hosts[1]) - _hosts.append(hosts[2]) - return _hosts + def _get_hosts(sources=None): + if sources == 'ceph': + return hosts[0] + if sources == 'orchestrator': + return hosts[1:] + if sources == 'ceph, orchestrator': + return hosts[2] + return hosts mock_get_hosts.side_effect = _get_hosts - self._get(self.URL_HOST) + self._get(self.URL_HOST, version=APIVersion(1, 1)) self.assertStatus(200) self.assertJsonBody(hosts) - self._get('{}?sources=ceph'.format(self.URL_HOST)) + self._get('{}?sources=ceph'.format(self.URL_HOST), version=APIVersion(1, 1)) self.assertStatus(200) - self.assertJsonBody([hosts[0]]) + self.assertJsonBody(hosts[0]) - self._get('{}?sources=orchestrator'.format(self.URL_HOST)) + self._get('{}?sources=orchestrator'.format(self.URL_HOST), version=APIVersion(1, 1)) self.assertStatus(200) self.assertJsonBody(hosts[1:]) - self._get('{}?sources=ceph,orchestrator'.format(self.URL_HOST)) + self._get('{}?sources=ceph,orchestrator'.format(self.URL_HOST), version=APIVersion(1, 1)) self.assertStatus(200) self.assertJsonBody(hosts) + @mock.patch('dashboard.controllers.host.get_hosts') + def test_host_list_with_facts(self, mock_get_hosts): + hosts_without_facts = [{ + 'hostname': 'host-0', + 'sources': { + 'ceph': True, + 'orchestrator': False + } + }, { + 'hostname': 'host-1', + 'sources': { + 'ceph': False, + 'orchestrator': True + } + }] + + hosts_facts = [{ + 'hostname': 'host-0', + 'cpu_count': 1, + 'memory_total_kb': 1024 + }, { + 'hostname': 'host-1', + 'cpu_count': 2, + 'memory_total_kb': 1024 + }] + + hosts_with_facts = [{ + 'hostname': 'host-0', + 'sources': { + 'ceph': True, + 'orchestrator': False + }, + 'cpu_count': 1, + 'memory_total_kb': 1024 + }, { + 'hostname': 'host-1', + 'sources': { + 'ceph': False, + 'orchestrator': True + }, + 'cpu_count': 2, + 'memory_total_kb': 1024 + }] + # test with orchestrator available + with patch_orch(True, hosts=hosts_without_facts) as fake_client: + mock_get_hosts.return_value = hosts_without_facts + fake_client.hosts.get_facts.return_value = hosts_facts + # test with ?facts=true + self._get('{}?facts=true'.format(self.URL_HOST), version=APIVersion(1, 1)) + self.assertStatus(200) + self.assertHeader('Content-Type', + 'application/vnd.ceph.api.v1.1+json') + self.assertJsonBody(hosts_with_facts) + + # test with ?facts=false + self._get('{}?facts=false'.format(self.URL_HOST), version=APIVersion(1, 1)) + self.assertStatus(200) + self.assertHeader('Content-Type', + 'application/vnd.ceph.api.v1.1+json') + self.assertJsonBody(hosts_without_facts) + + # test with no orchestrator available + with patch_orch(False): + mock_get_hosts.return_value = hosts_without_facts + + # test with ?facts=true + self._get('{}?facts=true'.format(self.URL_HOST), version=APIVersion(1, 1)) + self.assertStatus(400) + + # test with ?facts=false + self._get('{}?facts=false'.format(self.URL_HOST), version=APIVersion(1, 1)) + self.assertStatus(200) + self.assertHeader('Content-Type', + 'application/vnd.ceph.api.v1.1+json') + self.assertJsonBody(hosts_without_facts) + def test_get_1(self): mgr.list_servers.return_value = [] diff --git a/src/pybind/mgr/dashboard/tests/test_tools.py b/src/pybind/mgr/dashboard/tests/test_tools.py index 51f9216a19c4..fa14195e71e3 100644 --- a/src/pybind/mgr/dashboard/tests/test_tools.py +++ b/src/pybind/mgr/dashboard/tests/test_tools.py @@ -13,7 +13,8 @@ except ImportError: from ..controllers import APIRouter, BaseController, Proxy, RESTController, Router from ..controllers._version import APIVersion from ..services.exception import handle_rados_error -from ..tools import dict_contains_path, dict_get, json_str_to_object, partial_dict +from ..tools import dict_contains_path, dict_get, json_str_to_object, \ + merge_list_of_dicts_by_key, partial_dict from . import ControllerTestCase # pylint: disable=no-name-in-module @@ -197,3 +198,13 @@ class TestFunctions(unittest.TestCase): self.assertFalse(dict_get({'foo': {'bar': False}}, 'foo.bar')) self.assertIsNone(dict_get({'foo': {'bar': False}}, 'foo.bar.baz')) self.assertEqual(dict_get({'foo': {'bar': False}, 'baz': 'xyz'}, 'baz'), 'xyz') + + def test_merge_list_of_dicts_by_key(self): + expected_result = [{'a': 1, 'b': 2, 'c': 3}, {'a': 4, 'b': 5, 'c': 6}] + self.assertEqual(expected_result, merge_list_of_dicts_by_key( + [{'a': 1, 'b': 2}, {'a': 4, 'b': 5}], [{'a': 1, 'c': 3}, {'a': 4, 'c': 6}], 'a')) + + expected_result = [{'a': 1, 'b': 2}, {'a': 4, 'b': 5, 'c': 6}] + self.assertEqual(expected_result, merge_list_of_dicts_by_key( + [{'a': 1, 'b': 2}, {'a': 4, 'b': 5}], [{}, {'a': 4, 'c': 6}], 'a')) + self.assertRaises(TypeError, merge_list_of_dicts_by_key, None) diff --git a/src/pybind/mgr/dashboard/tools.py b/src/pybind/mgr/dashboard/tools.py index 1092534ea186..4e4837d9323e 100644 --- a/src/pybind/mgr/dashboard/tools.py +++ b/src/pybind/mgr/dashboard/tools.py @@ -828,3 +828,13 @@ def find_object_in_list(key, value, iterable): if key in obj and obj[key] == value: return obj return None + + +def merge_list_of_dicts_by_key(target_list: list, source_list: list, key: str): + target_list = {d[key]: d for d in target_list} + for sdict in source_list: + if bool(sdict): + if sdict[key] in target_list: + target_list[sdict[key]].update(sdict) + target_list = [value for value in target_list.values()] + return target_list