# -*- coding: utf-8 -*-
-import copy
import os
import time
from collections import Counter
import cherrypy
from mgr_util import merge_dicts
-from orchestrator import HostSpec
from .. import mgr
from ..exceptions import DashboardException
+from ..plugins.ttl_cache import ttl_cache, ttl_cache_invalidator
from ..security import Scope
+from ..services._paginate import ListPaginator
from ..services.ceph_service import CephService
from ..services.exception import handle_orchestrator_error
from ..services.orchestrator import OrchClient, OrchFeature
return Task("host/{}".format(name), metadata, wait_for)
-def merge_hosts_by_hostname(ceph_hosts, orch_hosts):
- # type: (List[dict], List[HostSpec]) -> List[dict]
- """
- Merge Ceph hosts with orchestrator hosts by hostnames.
-
- :param ceph_hosts: hosts returned from mgr
- :type ceph_hosts: list of dict
- :param orch_hosts: hosts returned from ochestrator
- :type orch_hosts: list of HostSpec
- :return list of dict
- """
- hosts = copy.deepcopy(ceph_hosts)
- orch_hosts_map = {host.hostname: host.to_json() for host in orch_hosts}
-
- # Sort labels.
- for hostname in orch_hosts_map:
- orch_hosts_map[hostname]['labels'].sort()
-
- # Hosts in both Ceph and Orchestrator.
- for host in hosts:
- hostname = host['hostname']
- if hostname in orch_hosts_map:
- host.update(orch_hosts_map[hostname])
- host['sources']['orchestrator'] = True
- orch_hosts_map.pop(hostname)
-
- # Hosts only in Orchestrator.
- orch_hosts_only = [
- merge_dicts(
- {
- 'ceph_version': '',
- 'services': [],
- 'sources': {
- 'ceph': False,
- 'orchestrator': True
- }
- }, orch_hosts_map[hostname]) for hostname in orch_hosts_map
- ]
- hosts.extend(orch_hosts_only)
- for host in hosts:
- host['service_instances'] = populate_service_instances(
- host['hostname'], host['services'])
- return hosts
-
-
def populate_service_instances(hostname, services):
orch = OrchClient.instance()
if orch.available():
return [{'type': k, 'count': v} for k, v in Counter(services).items()]
+@ttl_cache(60, label='get_hosts')
def get_hosts(sources=None):
"""
Get hosts from various sources.
from_ceph = 'ceph' in _sources
from_orchestrator = 'orchestrator' in _sources
+ if from_orchestrator:
+ orch = OrchClient.instance()
+ if orch.available():
+ hosts = [
+ merge_dicts(
+ {
+ 'ceph_version': '',
+ 'services': [],
+ 'sources': {
+ 'ceph': False,
+ 'orchestrator': True
+ }
+ }, host.to_json()) for host in orch.hosts.list()
+ ]
+ return hosts
+
ceph_hosts = []
if from_ceph:
ceph_hosts = [
'status': ''
}) for server in mgr.list_servers()
]
- if from_orchestrator:
- orch = OrchClient.instance()
- if orch.available():
- return merge_hosts_by_hostname(ceph_hosts, orch.hosts.list())
- for host in ceph_hosts:
- host['service_instances'] = populate_service_instances(host['hostname'], host['services'])
return ceph_hosts
'facts': (bool, 'Host Facts')
},
responses={200: LIST_HOST_SCHEMA})
- @RESTController.MethodMap(version=APIVersion(1, 2))
- def list(self, sources=None, facts=False):
+ @RESTController.MethodMap(version=APIVersion(1, 3))
+ def list(self, sources=None, facts=False, offset: int = 0,
+ limit: int = 5, search: str = '', sort: str = ''):
hosts = get_hosts(sources)
+ params = ['hostname']
+ paginator = ListPaginator(int(offset), int(limit), sort, search, hosts,
+ searchable_params=params, sortable_params=params,
+ default_sort='+hostname')
+ # pylint: disable=unnecessary-comprehension
+ hosts = [host for host in paginator.list()]
orch = OrchClient.instance()
+ cherrypy.response.headers['X-Total-Count'] = paginator.get_count()
+ for host in hosts:
+ if 'services' not in host:
+ host['services'] = []
+ host['service_instances'] = populate_service_instances(
+ host['hostname'], host['services'])
if str_to_bool(facts):
if orch.available():
if not orch.get_missing_features(['get_facts']):
- hosts_facts = orch.hosts.get_facts()
+ hosts_facts = []
+ for host in hosts:
+ facts = orch.hosts.get_facts(host['hostname'])[0]
+ hosts_facts.append(facts)
return merge_list_of_dicts_by_key(hosts, hosts_facts, 'hostname')
raise DashboardException(
return [d.to_dict() for d in daemons]
@handle_orchestrator_error('host')
+ @RESTController.MethodMap(version=APIVersion(1, 2))
def get(self, hostname: str) -> Dict:
"""
Get the specified host.
:raises: cherrypy.HTTPError: If host not found.
"""
- return get_host(hostname)
+ host = get_host(hostname)
+ host['service_instances'] = populate_service_instances(
+ host['hostname'], host['services'])
+ return host
+ @ttl_cache_invalidator('get_hosts')
@raise_if_no_orchestrator([OrchFeature.HOST_LABEL_ADD,
OrchFeature.HOST_LABEL_REMOVE,
OrchFeature.HOST_MAINTENANCE_ENTER,
drain(hostname: string) {
this.getTableCell(this.columnIndex.hostname, hostname, true).click();
this.clickActionButton('start-drain');
+ cy.wait(1000);
this.checkLabelExists(hostname, ['_no_schedule'], true);
this.clickTab('cd-host-details', hostname, 'Daemons');
import { FinishedTask } from '~/app/shared/models/finished-task';
import { Permission } from '~/app/shared/models/permissions';
import { TaskWrapperService } from '~/app/shared/services/task-wrapper.service';
+import { CdTableFetchDataContext } from '~/app/shared/models/cd-table-fetch-data-context';
@Component({
selector: 'cd-cephfs-form',
}
ngOnInit() {
- this.hostService.list('false').subscribe((resp: object[]) => {
+ const hostContext = new CdTableFetchDataContext(() => undefined);
+ this.hostService.list(hostContext.toParams(), 'false').subscribe((resp: object[]) => {
const options: SelectOption[] = [];
_.forEach(resp, (host: object) => {
if (_.get(host, 'sources.orchestrator', false)) {
import { CephServiceService } from '~/app/shared/api/ceph-service.service';
import { HostService } from '~/app/shared/api/host.service';
import { OsdService } from '~/app/shared/api/osd.service';
+import { CdTableFetchDataContext } from '~/app/shared/models/cd-table-fetch-data-context';
import { CephServiceSpec } from '~/app/shared/models/service.interface';
import { DimlessBinaryPipe } from '~/app/shared/pipes/dimless-binary.pipe';
import { WizardStepsService } from '~/app/shared/services/wizard-steps.service';
let dbDevices = 0;
let dbDeviceCapacity = 0;
- this.hostService.list('true').subscribe((resp: object[]) => {
+ const hostContext = new CdTableFetchDataContext(() => undefined);
+ this.hostService.list(hostContext.toParams(), 'true').subscribe((resp: object[]) => {
this.hosts = resp;
this.hostsCount = this.hosts.length;
_.forEach(this.hosts, (hostKey) => {
import { ConfirmationModalComponent } from '~/app/shared/components/confirmation-modal/confirmation-modal.component';
import { ActionLabelsI18n, AppConstants, URLVerbs } from '~/app/shared/constants/app.constants';
import { NotificationType } from '~/app/shared/enum/notification-type.enum';
+import { CdTableFetchDataContext } from '~/app/shared/models/cd-table-fetch-data-context';
import { FinishedTask } from '~/app/shared/models/finished-task';
import { DeploymentOptions } from '~/app/shared/models/osd-deployment-options';
import { Permissions } from '~/app/shared/models/permissions';
onSubmit() {
if (!this.stepsToSkip['Add Hosts']) {
- this.hostService.list('false').subscribe((hosts) => {
+ const hostContext = new CdTableFetchDataContext(() => undefined);
+ this.hostService.list(hostContext.toParams(), 'false').subscribe((hosts) => {
hosts.forEach((host) => {
const index = host['labels'].indexOf('_no_schedule', 0);
if (index > -1) {
import { CdForm } from '~/app/shared/forms/cd-form';
import { CdFormGroup } from '~/app/shared/forms/cd-form-group';
import { CdValidators } from '~/app/shared/forms/cd-validators';
+import { CdTableFetchDataContext } from '~/app/shared/models/cd-table-fetch-data-context';
import { FinishedTask } from '~/app/shared/models/finished-task';
import { TaskWrapperService } from '~/app/shared/services/task-wrapper.service';
this.pageURL = 'hosts';
}
this.createForm();
- this.hostService.list('false').subscribe((resp: any[]) => {
+ const hostContext = new CdTableFetchDataContext(() => undefined);
+ this.hostService.list(hostContext.toParams(), 'false').subscribe((resp: any[]) => {
this.hostnames = resp.map((host) => {
return host['hostname'];
});
selectionType="single"
[searchableObjects]="true"
[hasDetails]="hasTableDetails"
+ [serverSide]="true"
+ [count]="count"
+ [maxLimit]="25"
(setExpandedRow)="setExpandedRow($event)"
(updateSelection)="updateSelection($event)"
[toolHeader]="!hideToolHeader">
+import { HttpHeaders } from '@angular/common/http';
import { HttpClientTestingModule } from '@angular/common/http/testing';
import { ComponentFixture, TestBed } from '@angular/core/testing';
import { BrowserAnimationsModule } from '@angular/platform-browser/animations';
let hostListSpy: jasmine.Spy;
let orchService: OrchestratorService;
let showForceMaintenanceModal: MockShowForceMaintenanceModal;
+ let headers: HttpHeaders;
const fakeAuthStorageService = {
getPermissions: () => {
component = fixture.componentInstance;
hostListSpy = spyOn(TestBed.inject(HostService), 'list');
orchService = TestBed.inject(OrchestratorService);
+ headers = new HttpHeaders().set('x-total-count', '10');
});
it('should create', () => {
}
],
hostname: hostname,
- labels: ['foo', 'bar']
+ labels: ['foo', 'bar'],
+ headers: headers
}
];
OrchestratorHelper.mockStatus(false);
+ fixture.detectChanges();
hostListSpy.and.callFake(() => of(payload));
fixture.detectChanges();
}
],
hostname: hostname,
- labels: ['foo', 'bar']
+ labels: ['foo', 'bar'],
+ headers: headers
}
];
OrchestratorHelper.mockStatus(false);
+ fixture.detectChanges();
hostListSpy.and.callFake(() => of(payload));
fixture.detectChanges();
hdd_capacity_bytes: 1024,
flash_count: 4,
flash_capacity_bytes: 1024,
- nic_count: 1
+ nic_count: 1,
+ headers: headers
}
];
OrchestratorHelper.mockStatus(true, features);
+ fixture.detectChanges();
hostListSpy.and.callFake(() => of(payload));
fixture.detectChanges();
type: 'osd',
id: '0'
}
- ]
+ ],
+ headers: headers
}
];
OrchestratorHelper.mockStatus(false);
+ fixture.detectChanges();
hostListSpy.and.callFake(() => of(payload));
fixture.detectChanges();
type: 'osd',
id: '0'
}
- ]
+ ],
+ headers: headers
}
];
OrchestratorHelper.mockStatus(true);
+ fixture.detectChanges();
hostListSpy.and.callFake(() => of(payload));
fixture.detectChanges();
hdd_capacity_bytes: undefined,
flash_count: 4,
flash_capacity_bytes: undefined,
- nic_count: 1
+ nic_count: 1,
+ headers: headers
}
];
OrchestratorHelper.mockStatus(true, features);
+ fixture.detectChanges();
hostListSpy.and.callFake(() => of(hostPayload));
fixture.detectChanges();
const fakeHosts = require('./fixtures/host_list_response.json');
beforeEach(() => {
- hostListSpy.and.callFake(() => of(fakeHosts));
+ let headers = new HttpHeaders().set('x-total-count', '10');
+ headers = headers.set('x-total-count', '10');
+ fakeHosts[0].headers = headers;
+ fakeHosts[1].headers = headers;
});
const testTableActions = async (
fixture.detectChanges();
await fixture.whenStable();
+ component.getHosts(new CdTableFetchDataContext(() => undefined));
+ hostListSpy.and.callFake(() => of(fakeHosts));
+ fixture.detectChanges();
for (const test of tests) {
if (test.selectRow) {
component.selection = new CdTableSelection();
import { DimlessBinaryPipe } from '~/app/shared/pipes/dimless-binary.pipe';
import { EmptyPipe } from '~/app/shared/pipes/empty.pipe';
import { AuthStorageService } from '~/app/shared/services/auth-storage.service';
+import { CdTableServerSideService } from '~/app/shared/services/cd-table-server-side.service';
import { ModalService } from '~/app/shared/services/modal.service';
import { NotificationService } from '~/app/shared/services/notification.service';
import { TaskWrapperService } from '~/app/shared/services/task-wrapper.service';
bsModalRef: NgbModalRef;
icons = Icons;
+ private tableContext: CdTableFetchDataContext = null;
+ count = 5;
messages = {
nonOrchHost: $localize`The feature is disabled because the selected host is not managed by Orchestrator.`
}
getHosts(context: CdTableFetchDataContext) {
+ if (context !== null) {
+ this.tableContext = context;
+ }
+ if (this.tableContext == null) {
+ this.tableContext = new CdTableFetchDataContext(() => undefined);
+ }
if (this.isLoadingHosts) {
return;
}
mergeMap((orchStatus) => {
this.orchStatus = orchStatus;
const factsAvailable = this.checkHostsFactsAvailable();
- return this.hostService.list(`${factsAvailable}`);
+ return this.hostService.list(this.tableContext?.toParams(), factsAvailable.toString());
})
)
.subscribe(
- (hostList) => {
+ (hostList: any[]) => {
this.hosts = hostList;
this.hosts.forEach((host: object) => {
if (host['status'] === '') {
});
this.transformHostsData();
this.isLoadingHosts = false;
+ if (this.hosts.length > 0) {
+ this.count = CdTableServerSideService.getCount(hostList[0]);
+ } else {
+ this.count = 0;
+ }
},
() => {
this.isLoadingHosts = false;
import { CdFormBuilder } from '~/app/shared/forms/cd-form-builder';
import { CdFormGroup } from '~/app/shared/forms/cd-form-group';
import { CdValidators } from '~/app/shared/forms/cd-validators';
+import { CdTableFetchDataContext } from '~/app/shared/models/cd-table-fetch-data-context';
import { FinishedTask } from '~/app/shared/models/finished-task';
import { CephServiceSpec } from '~/app/shared/models/service.interface';
import { ModalService } from '~/app/shared/services/modal.service';
this.serviceTypes = _.difference(resp, this.hiddenServices).sort();
});
- this.hostService.list('false').subscribe((resp: object[]) => {
+ const hostContext = new CdTableFetchDataContext(() => undefined);
+ this.hostService.list(hostContext.toParams(), 'false').subscribe((resp: object[]) => {
const options: SelectOption[] = [];
_.forEach(resp, (host: object) => {
if (_.get(host, 'sources.orchestrator', false)) {
import { fakeAsync, TestBed, tick } from '@angular/core/testing';
import { configureTestBed } from '~/testing/unit-test-helper';
+import { CdTableFetchDataContext } from '../models/cd-table-fetch-data-context';
import { HostService } from './host.service';
describe('HostService', () => {
});
it('should call list', fakeAsync(() => {
- let result;
- service.list('true').subscribe((resp) => (result = resp));
- const req = httpTesting.expectOne('api/host?facts=true');
+ let result: any[] = [{}, {}];
+ const hostContext = new CdTableFetchDataContext(() => undefined);
+ service.list(hostContext.toParams(), 'true').subscribe((resp) => (result = resp));
+ const req = httpTesting.expectOne('api/host?offset=0&limit=10&search=&sort=+name&facts=true');
expect(req.request.method).toBe('GET');
- req.flush(['foo', 'bar']);
+ req.flush([{ foo: 1 }, { bar: 2 }]);
tick();
- expect(result).toEqual(['foo', 'bar']);
+ expect(result[0].foo).toEqual(1);
+ expect(result[1].bar).toEqual(2);
}));
it('should make a GET request on the devices endpoint when requesting devices', () => {
super();
}
- list(facts: string): Observable<object[]> {
- return this.http.get<object[]>(this.baseURL, {
- headers: { Accept: this.getVersionHeaderValue(1, 2) },
- params: { facts: facts }
- });
+ list(params: any, facts: string): Observable<object[]> {
+ params = params.set('facts', facts);
+ return this.http
+ .get<object[]>(this.baseURL, {
+ headers: { Accept: this.getVersionHeaderValue(1, 2) },
+ params: params,
+ observe: 'response'
+ })
+ .pipe(
+ map((response: any) => {
+ return response['body'].map((host: any) => {
+ host['headers'] = response.headers;
+ return host;
+ });
+ })
+ );
}
create(hostname: string, addr: string, labels: string[], status: string) {
</span>
<ng-template #serverSideTpl>
- {{ data?.length || 0 }} <ng-container i18n="X found">found</ng-container> /
- {{ rowCount }} <ng-container i18n="X total">total</ng-container>
+ <span>
+ {{ data?.length || 0 }} <ng-container i18n="X found">found</ng-container> /
+ {{ rowCount }} <ng-container i18n="X total">total</ng-container>
+ </span>
</ng-template>
</div>
<cd-table-pagination [page]="curPage"
name: facts
schema:
type: boolean
+ - default: 0
+ in: query
+ name: offset
+ schema:
+ type: integer
+ - default: 5
+ in: query
+ name: limit
+ schema:
+ type: integer
+ - default: ''
+ in: query
+ name: search
+ schema:
+ type: string
+ - default: ''
+ in: query
+ name: sort
+ schema:
+ type: string
responses:
'200':
content:
- application/vnd.ceph.api.v1.2+json:
+ application/vnd.ceph.api.v1.3+json:
schema:
properties:
addr:
responses:
'200':
content:
- application/vnd.ceph.api.v1.0+json:
+ application/vnd.ceph.api.v1.2+json:
type: object
description: OK
'400':
Based on Python 3 functools and backports.functools_lru_cache.
"""
+import os
from collections import OrderedDict
from functools import wraps
from threading import RLock
if typed is not False:
raise NotImplementedError("typed caching not supported")
+ # disable caching while running unit tests
+ if 'UNITTEST' in os.environ:
+ ttl = 0
+
def decorating_function(function):
cache_name = label
if not cache_name:
def _get_hosts(sources=None):
if sources == 'ceph':
- return hosts[0]
+ return [hosts[0]]
if sources == 'orchestrator':
return hosts[1:]
if sources == 'ceph, orchestrator':
- return hosts[2]
+ return [hosts[2]]
return hosts
- mock_get_hosts.side_effect = _get_hosts
-
- self._get(self.URL_HOST, version=APIVersion(1, 1))
- self.assertStatus(200)
- self.assertJsonBody(hosts)
+ with patch_orch(True, hosts=hosts):
+ mock_get_hosts.side_effect = _get_hosts
+ self._get(self.URL_HOST, version=APIVersion(1, 1))
+ self.assertStatus(200)
+ self.assertJsonBody(hosts)
- self._get('{}?sources=ceph'.format(self.URL_HOST), version=APIVersion(1, 1))
- self.assertStatus(200)
- self.assertJsonBody(hosts[0])
+ self._get('{}?sources=ceph'.format(self.URL_HOST), version=APIVersion(1, 1))
+ self.assertStatus(200)
+ self.assertJsonBody([hosts[0]])
- self._get('{}?sources=orchestrator'.format(self.URL_HOST), version=APIVersion(1, 1))
- self.assertStatus(200)
- self.assertJsonBody(hosts[1:])
+ self._get('{}?sources=orchestrator'.format(self.URL_HOST), version=APIVersion(1, 1))
+ self.assertStatus(200)
+ self.assertJsonBody(hosts[1:])
- self._get('{}?sources=ceph,orchestrator'.format(self.URL_HOST), version=APIVersion(1, 1))
- self.assertStatus(200)
- self.assertJsonBody(hosts)
+ self._get('{}?sources=ceph,orchestrator'.format(self.URL_HOST),
+ version=APIVersion(1, 1))
+ self.assertStatus(200)
+ self.assertJsonBody(hosts)
@mock.patch('dashboard.controllers.host.get_hosts')
def test_host_list_with_facts(self, mock_get_hosts):
'orchestrator': False
},
'cpu_count': 1,
- 'memory_total_kb': 1024
+ 'memory_total_kb': 1024,
+ 'services': [],
+ 'service_instances': [{'type': 'mon', 'count': 1}]
}, {
'hostname': 'host-1',
'sources': {
'orchestrator': True
},
'cpu_count': 2,
- 'memory_total_kb': 1024
+ 'memory_total_kb': 1024,
+ 'services': [],
+ 'service_instances': [{'type': 'mon', 'count': 1}]
}]
# test with orchestrator available
with patch_orch(True, hosts=hosts_without_facts) as fake_client:
mock_get_hosts.return_value = hosts_without_facts
- fake_client.hosts.get_facts.return_value = hosts_facts
+
+ def get_facts_mock(hostname: str):
+ if hostname == 'host-0':
+ return [hosts_facts[0]]
+ return [hosts_facts[1]]
+ fake_client.hosts.get_facts.side_effect = get_facts_mock
# test with ?facts=true
- self._get('{}?facts=true'.format(self.URL_HOST), version=APIVersion(1, 1))
+ self._get('{}?facts=true'.format(self.URL_HOST), version=APIVersion(1, 3))
self.assertStatus(200)
self.assertHeader('Content-Type',
- APIVersion(1, 2).to_mime_type())
+ APIVersion(1, 3).to_mime_type())
self.assertJsonBody(hosts_with_facts)
# test with ?facts=false
- self._get('{}?facts=false'.format(self.URL_HOST), version=APIVersion(1, 1))
+ self._get('{}?facts=false'.format(self.URL_HOST), version=APIVersion(1, 3))
self.assertStatus(200)
self.assertHeader('Content-Type',
- APIVersion(1, 2).to_mime_type())
+ APIVersion(1, 3).to_mime_type())
self.assertJsonBody(hosts_without_facts)
# test with orchestrator available but orch backend!=cephadm
with patch_orch(True, missing_features=['get_facts']) as fake_client:
mock_get_hosts.return_value = hosts_without_facts
# test with ?facts=true
- self._get('{}?facts=true'.format(self.URL_HOST), version=APIVersion(1, 1))
+ self._get('{}?facts=true'.format(self.URL_HOST), version=APIVersion(1, 3))
self.assertStatus(400)
# test with no orchestrator available
mock_get_hosts.return_value = hosts_without_facts
# test with ?facts=true
- self._get('{}?facts=true'.format(self.URL_HOST), version=APIVersion(1, 1))
+ self._get('{}?facts=true'.format(self.URL_HOST), version=APIVersion(1, 3))
self.assertStatus(400)
# test with ?facts=false
- self._get('{}?facts=false'.format(self.URL_HOST), version=APIVersion(1, 1))
+ self._get('{}?facts=false'.format(self.URL_HOST), version=APIVersion(1, 3))
self.assertStatus(200)
self.assertHeader('Content-Type',
- APIVersion(1, 2).to_mime_type())
+ APIVersion(1, 3).to_mime_type())
self.assertJsonBody(hosts_without_facts)
def test_get_1(self):
with patch_orch(True, hosts=orch_hosts):
hosts = get_hosts()
- self.assertEqual(len(hosts), 3)
+ self.assertEqual(len(hosts), 2)
checks = {
- 'localhost': {
- 'sources': {
- 'ceph': True,
- 'orchestrator': False
- },
- 'labels': []
- },
'node1': {
'sources': {
- 'ceph': True,
+ 'ceph': False,
'orchestrator': True
},
- 'labels': ['bar', 'foo']
+ 'labels': ['foo', 'bar']
},
'node2': {
'sources': {