- Adapted code to changes introduced in: https://github.com/ceph/ceph/pull/40220
- Improved error handling.
- Increased test coverage.
- Some refactoring.
- Simplified documentation about setting default daemon host and port.
Fixes: https://tracker.ceph.com/issues/49655
Signed-off-by: Alfonso Martínez <almartin@redhat.com>
dashboard will try to automatically determine the host and port
from the Ceph Manager's service map.
-If multiple zones are used, it will automatically determine the host within the
-master zone group and master zone. This should be sufficient for most setups,
-but in some circumstances you might want to set the host and port manually::
+In case of having several Object Gateways, you might want to set
+the default one by setting its host and port manually::
$ ceph dashboard set-rgw-api-host <host>
$ ceph dashboard set-rgw-api-port <port>
# -*- coding: utf-8 -*-
from __future__ import absolute_import
+from typing import Any, Dict
+
import cherrypy
from .. import mgr
class RgwPerfCounter(PerfCounter):
service_type = 'rgw'
+ def get(self, service_id: str) -> Dict[str, Any]:
+ svc_data = CephService.get_service_data_by_metadata_id(self.service_type, service_id)
+ service_map_id = svc_data['service_map_id']
+ schema_dict = mgr.get_perf_schema(self.service_type, service_map_id)
+ try:
+ schema = schema_dict["{}.{}".format(self.service_type, service_map_id)]
+ except KeyError as e:
+ raise cherrypy.HTTPError(404, "{0} not found".format(e))
+ counters = []
+
+ for key, value in sorted(schema.items()):
+ counter = dict()
+ counter['name'] = str(key)
+ counter['description'] = value['description']
+ # pylint: disable=W0212
+ if mgr._stattype_to_str(value['type']) == 'counter':
+ counter['value'] = CephService.get_rate(
+ self.service_type, service_map_id, key)
+ counter['unit'] = mgr._unit_to_str(value['units'])
+ else:
+ counter['value'] = mgr.get_latest(
+ self.service_type, service_map_id, key)
+ counter['unit'] = ''
+ counters.append(counter)
+
+ return {
+ 'service': {
+ 'type': self.service_type,
+ 'id': svc_data['id']
+ },
+ 'counters': counters
+ }
+
@ApiController('perf_counters/rbd-mirror', Scope.RBD_MIRRORING)
@ControllerDoc("Rgw Mirroring Perf Counters Management API", "RgwMirrorPerfCounter")
from ..security import Permission, Scope
from ..services.auth import AuthManager, JwtManager
from ..services.ceph_service import CephService
-from ..services.rgw_client import RgwClient
+from ..services.rgw_client import NoRgwDaemonsException, RgwClient
from ..tools import json_str_to_object, str_to_bool
from . import ApiController, BaseController, ControllerDoc, Endpoint, \
EndpointDoc, ReadPermission, RESTController, allow_empty_body
@ReadPermission
@EndpointDoc("Display RGW Status",
responses={200: RGW_SCHEMA})
- def status(self):
+ def status(self) -> dict:
status = {'available': False, 'message': None}
try:
- if not CephService.get_service_list('rgw'):
- raise LookupError('No RGW service is running.')
instance = RgwClient.admin_instance()
# Check if the service is online.
try:
instance.userid)
raise RequestException(msg)
status['available'] = True
- except (RequestException, LookupError) as ex:
+ except (DashboardException, RequestException, NoRgwDaemonsException) as ex:
status['message'] = str(ex) # type: ignore
return status
class RgwDaemon(RESTController):
@EndpointDoc("Display RGW Daemons",
responses={200: [RGW_DAEMON_SCHEMA]})
- def list(self):
- # type: () -> List[dict]
- daemons = []
- instance = RgwClient.admin_instance()
+ def list(self) -> List[dict]:
+ daemons: List[dict] = []
+ try:
+ instance = RgwClient.admin_instance()
+ except NoRgwDaemonsException:
+ return daemons
+
for hostname, server in CephService.get_service_map('rgw').items():
for service in server['services']:
metadata = service['metadata']
# extract per-daemon service data and health
daemon = {
- 'id': service['id'],
+ 'id': metadata['id'],
'version': metadata['ceph_version'],
'server_hostname': hostname,
'zonegroup_name': metadata['zonegroup_name'],
'zone_name': metadata['zone_name'],
- 'default': instance.daemon.name == service['id']
+ 'default': instance.daemon.name == metadata['id']
}
daemons.append(daemon)
result = json_str_to_object(result)
return result
except (DashboardException, RequestException) as e:
- raise DashboardException(e, http_status_code=500, component='rgw')
+ http_status_code = e.status if isinstance(e, DashboardException) else 500
+ raise DashboardException(e, http_status_code=http_status_code, component='rgw')
@ApiController('/rgw/site', Scope.RGW)
component = fixture.componentInstance;
httpTesting = TestBed.inject(HttpTestingController);
activatedRoute = <ActivatedRouteStub>TestBed.inject(ActivatedRoute);
+ RgwHelper.selectDaemon();
fixture.detectChanges();
httpTesting.expectOne('api/nfs-ganesha/daemon').flush([
httpTesting.expectOne('ui-api/nfs-ganesha/fsals').flush(['CEPH', 'RGW']);
httpTesting.expectOne('ui-api/nfs-ganesha/cephx/clients').flush(['admin', 'fs', 'rgw']);
httpTesting.expectOne('ui-api/nfs-ganesha/cephfs/filesystems').flush([{ id: 1, name: 'a' }]);
- RgwHelper.getCurrentDaemon();
httpTesting.expectOne(`api/rgw/user?${RgwHelper.DAEMON_QUERY_PARAM}`).flush(['test', 'dev']);
const user_dev = {
suspended: 0,
import { of } from 'rxjs';
-import { RgwDaemon } from '~/app/ceph/rgw/models/rgw-daemon';
import { Permissions } from '~/app/shared/models/permissions';
import { AuthStorageService } from '~/app/shared/services/auth-storage.service';
import {
FeatureTogglesMap,
FeatureTogglesService
} from '~/app/shared/services/feature-toggles.service';
-import { configureTestBed } from '~/testing/unit-test-helper';
+import { configureTestBed, RgwHelper } from '~/testing/unit-test-helper';
import { ContextComponent } from './context.component';
describe('ContextComponent', () => {
let ftMap: FeatureTogglesMap;
let httpTesting: HttpTestingController;
- const getDaemonList = () => {
- const daemonList: RgwDaemon[] = [];
- for (let daemonIndex = 1; daemonIndex <= 3; daemonIndex++) {
- const rgwDaemon = new RgwDaemon();
- rgwDaemon.id = `daemon${daemonIndex}`;
- rgwDaemon.default = daemonIndex === 2;
- rgwDaemon.zonegroup_name = `zonegroup${daemonIndex}`;
- daemonList.push(rgwDaemon);
- }
- return daemonList;
- };
+ const daemonList = RgwHelper.getDaemonList();
configureTestBed({
declarations: [ContextComponent],
getFeatureTogglesSpy.and.returnValue(of(ftMap));
fixture = TestBed.createComponent(ContextComponent);
component = fixture.componentInstance;
- fixture.detectChanges();
- const req = httpTesting.expectOne('api/rgw/daemon');
- req.flush(getDaemonList());
});
it('should create', () => {
it('should select the default daemon', fakeAsync(() => {
component.isRgwRoute = true;
+ fixture.detectChanges();
tick();
+ const req = httpTesting.expectOne('api/rgw/daemon');
+ req.flush(daemonList);
fixture.detectChanges();
const selectedDaemon = fixture.debugElement.nativeElement.querySelector(
'.ctx-bar-selected-rgw-daemon'
const availableDaemons = fixture.debugElement.nativeElement.querySelectorAll(
'.ctx-bar-available-rgw-daemon'
);
- expect(availableDaemons.length).toEqual(getDaemonList().length);
+ expect(availableDaemons.length).toEqual(daemonList.length);
expect(availableDaemons[0].textContent).toEqual(' daemon1 ( zonegroup1 ) ');
+ component.ngOnDestroy();
}));
it('should select the chosen daemon', fakeAsync(() => {
component.isRgwRoute = true;
- component.onDaemonSelection(getDaemonList()[2]);
+ fixture.detectChanges();
tick();
+ const req = httpTesting.expectOne('api/rgw/daemon');
+ req.flush(daemonList);
fixture.detectChanges();
-
+ component.onDaemonSelection(daemonList[2]);
expect(routerNavigateByUrlSpy).toHaveBeenCalledTimes(1);
+ fixture.detectChanges();
+ tick();
expect(routerNavigateSpy).toHaveBeenCalledTimes(1);
-
const selectedDaemon = fixture.debugElement.nativeElement.querySelector(
'.ctx-bar-selected-rgw-daemon'
);
expect(selectedDaemon.textContent).toEqual(' daemon3 ( zonegroup3 ) ');
+ component.ngOnDestroy();
}));
});
.pipe(filter((event: Event) => event instanceof NavigationEnd))
.subscribe(() => (this.isRgwRoute = this.router.url.includes(this.rgwUrlPrefix)))
);
- // Select default daemon:
- this.rgwDaemonService.list().subscribe((daemons: RgwDaemon[]) => {
- this.rgwDaemonService.selectDefaultDaemon(daemons);
- });
// Set daemon list polling only when in RGW route:
this.subs.add(
this.timerService
beforeEach(() => {
service = TestBed.inject(RgwBucketService);
httpTesting = TestBed.inject(HttpTestingController);
+ RgwHelper.selectDaemon();
});
afterEach(() => {
it('should call list', () => {
service.list().subscribe();
- RgwHelper.getCurrentDaemon();
const req = httpTesting.expectOne(`api/rgw/bucket?${RgwHelper.DAEMON_QUERY_PARAM}&stats=true`);
expect(req.request.method).toBe('GET');
});
it('should call get', () => {
service.get('foo').subscribe();
- RgwHelper.getCurrentDaemon();
const req = httpTesting.expectOne(`api/rgw/bucket/foo?${RgwHelper.DAEMON_QUERY_PARAM}`);
expect(req.request.method).toBe('GET');
});
service
.create('foo', 'bar', 'default', 'default-placement', false, 'COMPLIANCE', '10', '0')
.subscribe();
- RgwHelper.getCurrentDaemon();
const req = httpTesting.expectOne(
`api/rgw/bucket?bucket=foo&uid=bar&zonegroup=default&placement_target=default-placement&lock_enabled=false&lock_mode=COMPLIANCE&lock_retention_period_days=10&lock_retention_period_years=0&${RgwHelper.DAEMON_QUERY_PARAM}`
);
service
.update('foo', 'bar', 'baz', 'Enabled', 'Enabled', '1', '223344', 'GOVERNANCE', '0', '1')
.subscribe();
- RgwHelper.getCurrentDaemon();
const req = httpTesting.expectOne(
`api/rgw/bucket/foo?${RgwHelper.DAEMON_QUERY_PARAM}&bucket_id=bar&uid=baz&versioning_state=Enabled&mfa_delete=Enabled&mfa_token_serial=1&mfa_token_pin=223344&lock_mode=GOVERNANCE&lock_retention_period_days=0&lock_retention_period_years=1`
);
it('should call delete, with purgeObjects = true', () => {
service.delete('foo').subscribe();
- RgwHelper.getCurrentDaemon();
const req = httpTesting.expectOne(
`api/rgw/bucket/foo?${RgwHelper.DAEMON_QUERY_PARAM}&purge_objects=true`
);
it('should call delete, with purgeObjects = false', () => {
service.delete('foo', false).subscribe();
- RgwHelper.getCurrentDaemon();
const req = httpTesting.expectOne(
`api/rgw/bucket/foo?${RgwHelper.DAEMON_QUERY_PARAM}&purge_objects=false`
);
service.exists('foo').subscribe((resp) => {
result = resp;
});
- RgwHelper.getCurrentDaemon();
const req = httpTesting.expectOne(`api/rgw/bucket?${RgwHelper.DAEMON_QUERY_PARAM}`);
expect(req.request.method).toBe('GET');
req.flush(['foo', 'bar']);
import { HttpClientTestingModule, HttpTestingController } from '@angular/common/http/testing';
-import { TestBed } from '@angular/core/testing';
+import { fakeAsync, TestBed, tick } from '@angular/core/testing';
-import { configureTestBed } from '~/testing/unit-test-helper';
+import { of } from 'rxjs';
+
+import { RgwDaemon } from '~/app/ceph/rgw/models/rgw-daemon';
+import { configureTestBed, RgwHelper } from '~/testing/unit-test-helper';
import { RgwDaemonService } from './rgw-daemon.service';
describe('RgwDaemonService', () => {
let service: RgwDaemonService;
let httpTesting: HttpTestingController;
+ let selectDaemonSpy: jasmine.Spy;
+
+ const daemonList = RgwHelper.getDaemonList();
+ const retrieveDaemonList = (reqDaemonList: RgwDaemon[], daemon: RgwDaemon) => {
+ service
+ .request((params) => of(params))
+ .subscribe((params) => expect(params.get('daemon_name')).toBe(daemon.id));
+ const listReq = httpTesting.expectOne('api/rgw/daemon');
+ listReq.flush(reqDaemonList);
+ tick();
+ expect(service['selectedDaemon'].getValue()).toEqual(daemon);
+ };
configureTestBed({
providers: [RgwDaemonService],
beforeEach(() => {
service = TestBed.inject(RgwDaemonService);
+ selectDaemonSpy = spyOn(service, 'selectDaemon').and.callThrough();
httpTesting = TestBed.inject(HttpTestingController);
});
expect(service).toBeTruthy();
});
- it('should call list', () => {
+ it('should get daemon list', () => {
service.list().subscribe();
const req = httpTesting.expectOne('api/rgw/daemon');
+ req.flush(daemonList);
expect(req.request.method).toBe('GET');
+ expect(service['daemons'].getValue()).toEqual(daemonList);
});
- it('should call get', () => {
+ it('should get daemon ', () => {
service.get('foo').subscribe();
const req = httpTesting.expectOne('api/rgw/daemon/foo');
expect(req.request.method).toBe('GET');
});
+
+ it('should call request and not select any daemon from empty daemon list', fakeAsync(() => {
+ expect(() => retrieveDaemonList([], null)).toThrowError('No RGW daemons found!');
+ expect(selectDaemonSpy).toHaveBeenCalledTimes(0);
+ }));
+
+ it('should call request and select default daemon from daemon list', fakeAsync(() => {
+ retrieveDaemonList(daemonList, daemonList[1]);
+ expect(selectDaemonSpy).toHaveBeenCalledTimes(1);
+ expect(selectDaemonSpy).toHaveBeenCalledWith(daemonList[1]);
+ }));
+
+ it('should call request and select first daemon from daemon list that has no default', fakeAsync(() => {
+ const noDefaultDaemonList = daemonList.map((daemon) => {
+ daemon.default = false;
+ return daemon;
+ });
+ retrieveDaemonList(noDefaultDaemonList, noDefaultDaemonList[0]);
+ expect(selectDaemonSpy).toHaveBeenCalledTimes(1);
+ expect(selectDaemonSpy).toHaveBeenCalledWith(noDefaultDaemonList[0]);
+ }));
});
import { HttpClient, HttpParams } from '@angular/common/http';
import { Injectable } from '@angular/core';
-import { Observable, ReplaySubject } from 'rxjs';
-import { mergeMap, take, tap } from 'rxjs/operators';
+import _ from 'lodash';
+import { BehaviorSubject, Observable, of, throwError } from 'rxjs';
+import { mergeMap, retryWhen, take, tap } from 'rxjs/operators';
import { RgwDaemon } from '~/app/ceph/rgw/models/rgw-daemon';
import { cdEncode } from '~/app/shared/decorators/cd-encode';
})
export class RgwDaemonService {
private url = 'api/rgw/daemon';
- private daemons = new ReplaySubject<RgwDaemon[]>(1);
+ private daemons = new BehaviorSubject<RgwDaemon[]>([]);
daemons$ = this.daemons.asObservable();
- private selectedDaemon = new ReplaySubject<RgwDaemon>(1);
+ private selectedDaemon = new BehaviorSubject<RgwDaemon>(null);
selectedDaemon$ = this.selectedDaemon.asObservable();
constructor(private http: HttpClient) {}
return this.http.get<RgwDaemon[]>(this.url).pipe(
tap((daemons: RgwDaemon[]) => {
this.daemons.next(daemons);
+ if (_.isEmpty(this.selectedDaemon.getValue())) {
+ this.selectDefaultDaemon(daemons);
+ }
})
);
}
this.selectedDaemon.next(daemon);
}
- selectDefaultDaemon(daemons: RgwDaemon[]): RgwDaemon {
+ private selectDefaultDaemon(daemons: RgwDaemon[]): RgwDaemon {
+ if (daemons.length === 0) {
+ return null;
+ }
+
for (const daemon of daemons) {
if (daemon.default) {
this.selectDaemon(daemon);
}
}
- throw new Error('No default RGW daemon found.');
+ this.selectDaemon(daemons[0]);
+ return daemons[0];
}
request(next: (params: HttpParams) => Observable<any>) {
return this.selectedDaemon.pipe(
+ mergeMap((daemon: RgwDaemon) =>
+ // If there is no selected daemon, retrieve daemon list (default daemon will be selected)
+ // and try again if daemon list is not empty.
+ _.isEmpty(daemon)
+ ? this.list().pipe(mergeMap((daemons) => throwError(!_.isEmpty(daemons))))
+ : of(daemon)
+ ),
+ retryWhen((error) =>
+ error.pipe(
+ mergeMap((hasToRetry) => (hasToRetry ? error : throwError('No RGW daemons found!')))
+ )
+ ),
take(1),
mergeMap((daemon: RgwDaemon) => {
let params = new HttpParams();
beforeEach(() => {
service = TestBed.inject(RgwSiteService);
httpTesting = TestBed.inject(HttpTestingController);
+ RgwHelper.selectDaemon();
});
afterEach(() => {
it('should contain site endpoint in GET request', () => {
service.get().subscribe();
- RgwHelper.getCurrentDaemon();
const req = httpTesting.expectOne(`${service['url']}?${RgwHelper.DAEMON_QUERY_PARAM}`);
expect(req.request.method).toBe('GET');
});
it('should add query param in GET request', () => {
const query = 'placement-targets';
service.get(query).subscribe();
- RgwHelper.getCurrentDaemon();
httpTesting.expectOne(
`${service['url']}?${RgwHelper.DAEMON_QUERY_PARAM}&query=placement-targets`
);
beforeEach(() => {
service = TestBed.inject(RgwUserService);
httpTesting = TestBed.inject(HttpTestingController);
+ RgwHelper.selectDaemon();
});
afterEach(() => {
service.list().subscribe((resp) => {
result = resp;
});
- RgwHelper.getCurrentDaemon();
const req = httpTesting.expectOne(`api/rgw/user?${RgwHelper.DAEMON_QUERY_PARAM}`);
expect(req.request.method).toBe('GET');
req.flush([]);
service.list().subscribe((resp) => {
result = resp;
});
- RgwHelper.getCurrentDaemon();
let req = httpTesting.expectOne(`api/rgw/user?${RgwHelper.DAEMON_QUERY_PARAM}`);
expect(req.request.method).toBe('GET');
req.flush(['foo', 'bar']);
it('should call enumerate', () => {
service.enumerate().subscribe();
- RgwHelper.getCurrentDaemon();
const req = httpTesting.expectOne(`api/rgw/user?${RgwHelper.DAEMON_QUERY_PARAM}`);
expect(req.request.method).toBe('GET');
});
it('should call get', () => {
service.get('foo').subscribe();
- RgwHelper.getCurrentDaemon();
const req = httpTesting.expectOne(`api/rgw/user/foo?${RgwHelper.DAEMON_QUERY_PARAM}`);
expect(req.request.method).toBe('GET');
});
it('should call getQuota', () => {
service.getQuota('foo').subscribe();
- RgwHelper.getCurrentDaemon();
const req = httpTesting.expectOne(`api/rgw/user/foo/quota?${RgwHelper.DAEMON_QUERY_PARAM}`);
expect(req.request.method).toBe('GET');
});
it('should call update', () => {
service.update('foo', { xxx: 'yyy' }).subscribe();
- RgwHelper.getCurrentDaemon();
const req = httpTesting.expectOne(`api/rgw/user/foo?${RgwHelper.DAEMON_QUERY_PARAM}&xxx=yyy`);
expect(req.request.method).toBe('PUT');
});
it('should call updateQuota', () => {
service.updateQuota('foo', { xxx: 'yyy' }).subscribe();
- RgwHelper.getCurrentDaemon();
const req = httpTesting.expectOne(
`api/rgw/user/foo/quota?${RgwHelper.DAEMON_QUERY_PARAM}&xxx=yyy`
);
it('should call create', () => {
service.create({ foo: 'bar' }).subscribe();
- RgwHelper.getCurrentDaemon();
const req = httpTesting.expectOne(`api/rgw/user?${RgwHelper.DAEMON_QUERY_PARAM}&foo=bar`);
expect(req.request.method).toBe('POST');
});
it('should call delete', () => {
service.delete('foo').subscribe();
- RgwHelper.getCurrentDaemon();
const req = httpTesting.expectOne(`api/rgw/user/foo?${RgwHelper.DAEMON_QUERY_PARAM}`);
expect(req.request.method).toBe('DELETE');
});
it('should call createSubuser', () => {
service.createSubuser('foo', { xxx: 'yyy' }).subscribe();
- RgwHelper.getCurrentDaemon();
const req = httpTesting.expectOne(
`api/rgw/user/foo/subuser?${RgwHelper.DAEMON_QUERY_PARAM}&xxx=yyy`
);
it('should call deleteSubuser', () => {
service.deleteSubuser('foo', 'bar').subscribe();
- RgwHelper.getCurrentDaemon();
const req = httpTesting.expectOne(
`api/rgw/user/foo/subuser/bar?${RgwHelper.DAEMON_QUERY_PARAM}`
);
it('should call addCapability', () => {
service.addCapability('foo', 'bar', 'baz').subscribe();
- RgwHelper.getCurrentDaemon();
const req = httpTesting.expectOne(
`api/rgw/user/foo/capability?${RgwHelper.DAEMON_QUERY_PARAM}&type=bar&perm=baz`
);
it('should call deleteCapability', () => {
service.deleteCapability('foo', 'bar', 'baz').subscribe();
- RgwHelper.getCurrentDaemon();
const req = httpTesting.expectOne(
`api/rgw/user/foo/capability?${RgwHelper.DAEMON_QUERY_PARAM}&type=bar&perm=baz`
);
it('should call addS3Key', () => {
service.addS3Key('foo', { xxx: 'yyy' }).subscribe();
- RgwHelper.getCurrentDaemon();
const req = httpTesting.expectOne(
`api/rgw/user/foo/key?${RgwHelper.DAEMON_QUERY_PARAM}&key_type=s3&xxx=yyy`
);
it('should call deleteS3Key', () => {
service.deleteS3Key('foo', 'bar').subscribe();
- RgwHelper.getCurrentDaemon();
const req = httpTesting.expectOne(
`api/rgw/user/foo/key?${RgwHelper.DAEMON_QUERY_PARAM}&key_type=s3&access_key=bar`
);
}
export class RgwHelper {
- static readonly DAEMON_NAME = 'daemon1';
- static readonly DAEMON_QUERY_PARAM = `daemon_name=${RgwHelper.DAEMON_NAME}`;
+ static readonly daemons = RgwHelper.getDaemonList();
+ static readonly DAEMON_QUERY_PARAM = `daemon_name=${RgwHelper.daemons[0].id}`;
+
+ static getDaemonList() {
+ const daemonList: RgwDaemon[] = [];
+ for (let daemonIndex = 1; daemonIndex <= 3; daemonIndex++) {
+ const rgwDaemon = new RgwDaemon();
+ rgwDaemon.id = `daemon${daemonIndex}`;
+ rgwDaemon.default = daemonIndex === 2;
+ rgwDaemon.zonegroup_name = `zonegroup${daemonIndex}`;
+ daemonList.push(rgwDaemon);
+ }
+ return daemonList;
+ }
- static getCurrentDaemon() {
- const rgwDaemon = new RgwDaemon();
- rgwDaemon.id = this.DAEMON_NAME;
- rgwDaemon.default = true;
+ static selectDaemon() {
const service = TestBed.inject(RgwDaemonService);
- service.selectDaemon(rgwDaemon);
+ service.selectDaemon(this.daemons[0]);
}
}
"{}" # TODO remove
.format(self.client_name, resp.status_code, pf(
resp.content)),
- resp.status_code,
+ self._handle_response_status_code(resp.status_code),
resp.content)
except ConnectionError as ex:
if ex.args:
logger.exception(msg)
raise RequestException(msg)
+ @staticmethod
+ def _handle_response_status_code(status_code: int) -> int:
+ """
+ Method to be overridden by subclasses that need specific handling.
+ """
+ return status_code
+
@staticmethod
def api(path, **api_kwargs):
def call_decorator(func):
def func_wrapper(self, *args, **kwargs):
method = api_kwargs.get('method', None)
resp_structure = api_kwargs.get('resp_structure', None)
- args_name = inspect.getargspec(func).args
+ args_name = inspect.getfullargspec(func).args
args_dict = dict(zip(args_name[1:], args))
for key, val in kwargs.items():
args_dict[key] = val
return [svc for _, svcs in service_map.items() for svc in svcs['services']]
@classmethod
- def get_service(cls, service_name, service_id):
+ def get_service_data_by_metadata_id(cls,
+ service_type: str,
+ metadata_id: str) -> Optional[Dict[str, Any]]:
for server in mgr.list_servers():
for service in server['services']:
- if service['type'] == service_name:
- inst_id = service['id']
- if inst_id == service_id:
- metadata = mgr.get_metadata(service_name, inst_id)
- status = mgr.get_daemon_status(service_name, inst_id)
+ if service['type'] == service_type:
+ metadata = mgr.get_metadata(service_type, service['id'])
+ if metadata_id == metadata['id']:
return {
- 'id': inst_id,
- 'type': service_name,
+ 'id': metadata['id'],
+ 'service_map_id': str(service['id']),
+ 'type': service_type,
'hostname': server['hostname'],
- 'metadata': metadata,
- 'status': status
+ 'metadata': metadata
}
return None
+ @classmethod
+ def get_service(cls, service_type: str, metadata_id: str) -> Optional[Dict[str, Any]]:
+ svc_data = cls.get_service_data_by_metadata_id(service_type, metadata_id)
+ if svc_data:
+ svc_data['status'] = mgr.get_daemon_status(svc_data['type'], svc_data['service_map_id'])
+ return svc_data
+
@classmethod
def get_pool_list(cls, application=None):
osd_map = mgr.get('osd_map')
logger = logging.getLogger('rgw_client')
+class NoRgwDaemonsException(Exception):
+ def __init__(self):
+ super().__init__('No RGW service is running.')
+
+
class NoCredentialsException(RequestException):
def __init__(self):
super(NoCredentialsException, self).__init__(
def _get_daemons() -> Dict[str, RgwDaemon]:
"""
Retrieve RGW daemon info from MGR.
- Note, the service id of the RGW daemons may differ depending on the setup.
- Example 1:
- {
- ...
- 'services': {
- 'rgw': {
- 'daemons': {
- 'summary': '',
- '0': {
- ...
- 'addr': '[2001:db8:85a3::8a2e:370:7334]:49774/1534999298',
- 'metadata': {
- 'frontend_config#0': 'civetweb port=7280',
- }
- ...
- }
- }
- }
- }
- }
- Example 2:
- {
- ...
- 'services': {
- 'rgw': {
- 'daemons': {
- 'summary': '',
- 'rgw': {
- ...
- 'addr': '192.168.178.3:49774/1534999298',
- 'metadata': {
- 'frontend_config#0': 'civetweb port=8000',
- }
- ...
- }
- }
- }
- }
- }
"""
service_map = mgr.get('service_map')
if not dict_contains_path(service_map, ['services', 'rgw', 'daemons']):
- raise LookupError('No RGW found')
+ raise NoRgwDaemonsException
+
daemons = {}
daemon_map = service_map['services']['rgw']['daemons']
for key in daemon_map.keys():
if dict_contains_path(daemon_map[key], ['metadata', 'frontend_config#0']):
daemon = _determine_rgw_addr(daemon_map[key])
- daemon.name = key
+ daemon.name = daemon_map[key]['metadata']['id']
daemon.zonegroup_name = daemon_map[key]['metadata']['zonegroup_name']
daemons[daemon.name] = daemon
logger.info('Found RGW daemon with configuration: host=%s, port=%d, ssl=%s',
daemon.host, daemon.port, str(daemon.ssl))
if not daemons:
- raise LookupError('No RGW daemon found')
+ raise NoRgwDaemonsException
return daemons
got_keys_from_config: bool
userid: str
+ @staticmethod
+ def _handle_response_status_code(status_code: int) -> int:
+ # Do not return auth error codes (so they are not handled as ceph API user auth errors).
+ return 404 if status_code in [401, 403] else status_code
+
@staticmethod
def _get_daemon_connection_info(daemon_name: str) -> dict:
try:
# Legacy string values.
access_key = Settings.RGW_API_ACCESS_KEY
secret_key = Settings.RGW_API_SECRET_KEY
+ except KeyError as error:
+ raise DashboardException(msg='Credentials not found for RGW Daemon: {}'.format(error),
+ http_status_code=404,
+ component='rgw')
return {'access_key': access_key, 'secret_key': secret_key}
def instance(userid: Optional[str] = None,
daemon_name: Optional[str] = None) -> 'RgwClient':
# pylint: disable=too-many-branches
+
+ RgwClient._daemons = _get_daemons()
+
# The API access key and secret key are mandatory for a minimal configuration.
if not (Settings.RGW_API_ACCESS_KEY and Settings.RGW_API_SECRET_KEY):
logger.warning('No credentials found, please consult the '
'dashboard.')
raise NoCredentialsException()
- if not RgwClient._daemons:
- RgwClient._daemons = _get_daemons()
-
if not daemon_name:
# Select default daemon if configured in settings:
if Settings.RGW_API_HOST and Settings.RGW_API_PORT:
daemon_name = daemon.name
break
if not daemon_name:
- raise LookupError('No RGW daemon found with host: {}, port: {}'.format(
- Settings.RGW_API_HOST,
- Settings.RGW_API_PORT))
+ raise DashboardException(
+ msg='No RGW daemon found with user-defined host: {}, port: {}'.format(
+ Settings.RGW_API_HOST,
+ Settings.RGW_API_PORT),
+ http_status_code=404,
+ component='rgw')
# Select 1st daemon:
else:
daemon_name = next(iter(RgwClient._daemons.keys()))
secret_key,
daemon_name,
user_id=None):
- daemon = RgwClient._daemons[daemon_name]
+ try:
+ daemon = RgwClient._daemons[daemon_name]
+ except KeyError as error:
+ raise DashboardException(msg='RGW Daemon not found: {}'.format(error),
+ http_status_code=404,
+ component='rgw')
ssl_verify = Settings.RGW_API_SSL_VERIFY
self.admin_path = Settings.RGW_API_ADMIN_RESOURCE
self.service_url = build_url(host=daemon.host, port=daemon.port)
self.userid = self._get_user_id(self.admin_path) if self.got_keys_from_config \
else user_id
except RequestException as error:
- # Avoid dashboard GUI redirections caused by status code (403, ...):
- http_status_code = 400 if 400 <= error.status_code < 500 else error.status_code
- raise DashboardException(msg='Error connecting to Object Gateway.',
- http_status_code=http_status_code,
+ msg = 'Error connecting to Object Gateway'
+ if error.status_code == 404:
+ msg = '{}: {}'.format(msg, str(error))
+ raise DashboardException(msg=msg,
+ http_status_code=error.status_code,
component='rgw')
self.daemon = daemon
daemon.name, daemon.host, daemon.port, daemon.ssl, ssl_verify)
@RestClient.api_get('/', resp_structure='[0] > (ID & DisplayName)')
- def is_service_online(self, request=None):
+ def is_service_online(self, request=None) -> bool:
"""
Consider the service as online if the response contains the
specified keys. Nothing more is checked here.
@RestClient.api_get('/{admin_path}/metadata/user?key={userid}',
resp_structure='data > system')
- def _is_system_user(self, admin_path, userid, request=None):
+ def _is_system_user(self, admin_path, userid, request=None) -> bool:
# pylint: disable=unused-argument
response = request()
return strtobool(response['data']['system'])
- def is_system_user(self):
+ def is_system_user(self) -> bool:
return self._is_system_user(self.admin_path, self.userid)
@RestClient.api_get(
request(data=data, headers=headers)
except RequestException as error:
msg = str(error)
- if error.status_code == 403:
+ if mfa_delete and mfa_token_serial and mfa_token_pin \
+ and 'AccessDenied' in error.content.decode():
msg = 'Bad MFA credentials: {}'.format(msg)
- # Avoid dashboard GUI redirections caused by status code (403, ...):
- http_status_code = 400 if 400 <= error.status_code < 500 else error.status_code
raise DashboardException(msg=msg,
- http_status_code=http_status_code,
+ http_status_code=error.status_code,
component='rgw')
@RestClient.api_get('/{bucket_name}?object-lock')
import logging
import threading
import time
+from unittest.mock import Mock
import cherrypy
from cherrypy._cptools import HandlerWrapperTool
if msg is None:
msg = 'expected %r to be in %r' % (data, json_body)
self._handlewebError(msg)
+
+
+class Stub:
+ """Test class for returning predefined values"""
+
+ @classmethod
+ def get_mgr_no_services(cls):
+ mgr.get = Mock(return_value={})
+
+
+class RgwStub(Stub):
+
+ @classmethod
+ def get_daemons(cls):
+ mgr.get = Mock(return_value={'services': {'rgw': {'daemons': {
+ '5297': {
+ 'addr': '192.168.178.3:49774/1534999298',
+ 'metadata': {
+ 'frontend_config#0': 'beast port=8000',
+ 'id': 'daemon1',
+ 'zonegroup_name': 'zonegroup1',
+ 'zone_name': 'zone1'
+ }
+ },
+ '5398': {
+ 'addr': '[2001:db8:85a3::8a2e:370:7334]:49774/1534999298',
+ 'metadata': {
+ 'frontend_config#0': 'civetweb port=8002',
+ 'id': 'daemon2',
+ 'zonegroup_name': 'zonegroup2',
+ 'zone_name': 'zone2'
+ }
+ }
+ }}}})
+
+ @classmethod
+ def get_settings(cls):
+ settings = {
+ 'RGW_API_HOST': '',
+ 'RGW_API_PORT': 0,
+ 'RGW_API_ACCESS_KEY': 'fake-access-key',
+ 'RGW_API_SECRET_KEY': 'fake-secret-key',
+ }
+ mgr.get_module_option = Mock(side_effect=settings.get)
from ..rest_client import RequestException, RestClient
+class RestClientTestClass(RestClient):
+ """RestClient subclass for testing purposes."""
+ @RestClient.api_get('/')
+ def fake_endpoint_method_with_annotation(self, request=None) -> bool:
+ pass
+
+
class RestClientTest(unittest.TestCase):
def setUp(self):
settings = {'REST_REQUESTS_TIMEOUT': 45}
@classmethod
def setUpClass(cls):
cls.mock_requests = patch('requests.Session').start()
- cls.rest_client = RestClient('localhost', 8000, 'UnitTest')
+ cls.rest_client = RestClientTestClass('localhost', 8000, 'UnitTest')
+
+ def test_endpoint_method_with_annotation(self):
+ self.assertEqual(self.rest_client.fake_endpoint_method_with_annotation(), None)
def test_do_request_exception_no_args(self):
self.mock_requests().get.side_effect = requests.exceptions.ConnectionError()
-try:
- import mock
-except ImportError:
- import unittest.mock as mock
+from unittest.mock import Mock, call, patch
from .. import mgr
-from ..controllers.rgw import Rgw, RgwUser
-from . import ControllerTestCase # pylint: disable=no-name-in-module
+from ..controllers.rgw import Rgw, RgwDaemon, RgwUser
+from ..rest_client import RequestException
+from ..services.rgw_client import RgwClient
+from . import ControllerTestCase, RgwStub # pylint: disable=no-name-in-module
class RgwControllerTestCase(ControllerTestCase):
Rgw._cp_config['tools.authenticate.on'] = False # pylint: disable=protected-access
cls.setup_controllers([Rgw], '/test')
+ def setUp(self) -> None:
+ RgwStub.get_daemons()
+ RgwStub.get_settings()
+
+ @patch.object(RgwClient, '_get_user_id', Mock(return_value='fake-user'))
+ @patch.object(RgwClient, 'is_service_online', Mock(return_value=True))
+ @patch.object(RgwClient, '_is_system_user', Mock(return_value=True))
+ def test_status_available(self):
+ self._get('/test/api/rgw/status')
+ self.assertStatus(200)
+ self.assertJsonBody({'available': True, 'message': None})
+
+ @patch.object(RgwClient, '_get_user_id', Mock(return_value='fake-user'))
+ @patch.object(RgwClient, 'is_service_online', Mock(
+ side_effect=RequestException('My test error')))
+ def test_status_online_check_error(self):
+ self._get('/test/api/rgw/status')
+ self.assertStatus(200)
+ self.assertJsonBody({'available': False,
+ 'message': 'My test error'})
+
+ @patch.object(RgwClient, '_get_user_id', Mock(return_value='fake-user'))
+ @patch.object(RgwClient, 'is_service_online', Mock(return_value=False))
+ def test_status_not_online(self):
+ self._get('/test/api/rgw/status')
+ self.assertStatus(200)
+ self.assertJsonBody({'available': False,
+ 'message': "Failed to connect to the Object Gateway's Admin Ops API."})
+
+ @patch.object(RgwClient, '_get_user_id', Mock(return_value='fake-user'))
+ @patch.object(RgwClient, 'is_service_online', Mock(return_value=True))
+ @patch.object(RgwClient, '_is_system_user', Mock(return_value=False))
+ def test_status_not_system_user(self):
+ self._get('/test/api/rgw/status')
+ self.assertStatus(200)
+ self.assertJsonBody({'available': False,
+ 'message': 'The system flag is not set for user "fake-user".'})
+
def test_status_no_service(self):
- mgr.list_servers.return_value = []
+ RgwStub.get_mgr_no_services()
self._get('/test/api/rgw/status')
self.assertStatus(200)
self.assertJsonBody({'available': False, 'message': 'No RGW service is running.'})
+class RgwDaemonControllerTestCase(ControllerTestCase):
+ @classmethod
+ def setup_server(cls):
+ RgwDaemon._cp_config['tools.authenticate.on'] = False # pylint: disable=protected-access
+ cls.setup_controllers([RgwDaemon], '/test')
+
+ @patch('dashboard.services.rgw_client.RgwClient._get_user_id', Mock(
+ return_value='dummy_admin'))
+ def test_list(self):
+ RgwStub.get_daemons()
+ RgwStub.get_settings()
+ mgr.list_servers.return_value = [{
+ 'hostname': 'host1',
+ 'services': [{'id': 'daemon1', 'type': 'rgw'}, {'id': 'daemon2', 'type': 'rgw'}]
+ }]
+ mgr.get_metadata.side_effect = [
+ {
+ 'ceph_version': 'ceph version master (dev)',
+ 'id': 'daemon1',
+ 'zonegroup_name': 'zg1',
+ 'zone_name': 'zone1'
+ },
+ {
+ 'ceph_version': 'ceph version master (dev)',
+ 'id': 'daemon2',
+ 'zonegroup_name': 'zg2',
+ 'zone_name': 'zone2'
+ }]
+ self._get('/test/api/rgw/daemon')
+ self.assertStatus(200)
+ self.assertJsonBody([{
+ 'id': 'daemon1',
+ 'version': 'ceph version master (dev)',
+ 'server_hostname': 'host1',
+ 'zonegroup_name': 'zg1',
+ 'zone_name': 'zone1', 'default': True
+ },
+ {
+ 'id': 'daemon2',
+ 'version': 'ceph version master (dev)',
+ 'server_hostname': 'host1',
+ 'zonegroup_name': 'zg2',
+ 'zone_name': 'zone2',
+ 'default': False
+ }])
+
+ def test_list_empty(self):
+ RgwStub.get_mgr_no_services()
+ self._get('/test/api/rgw/daemon')
+ self.assertStatus(200)
+ self.assertJsonBody([])
+
+
class RgwUserControllerTestCase(ControllerTestCase):
@classmethod
def setup_server(cls):
RgwUser._cp_config['tools.authenticate.on'] = False # pylint: disable=protected-access
cls.setup_controllers([RgwUser], '/test')
- @mock.patch('dashboard.controllers.rgw.RgwRESTController.proxy')
+ @patch('dashboard.controllers.rgw.RgwRESTController.proxy')
def test_user_list(self, mock_proxy):
mock_proxy.side_effect = [{
'count': 3,
self._get('/test/api/rgw/user?daemon_name=dummy-daemon')
self.assertStatus(200)
mock_proxy.assert_has_calls([
- mock.call('dummy-daemon', 'GET', 'user?list', {})
+ call('dummy-daemon', 'GET', 'user?list', {})
])
self.assertJsonBody(['test1', 'test2', 'test3'])
- @mock.patch('dashboard.controllers.rgw.RgwRESTController.proxy')
+ @patch('dashboard.controllers.rgw.RgwRESTController.proxy')
def test_user_list_marker(self, mock_proxy):
mock_proxy.side_effect = [{
'count': 3,
self._get('/test/api/rgw/user')
self.assertStatus(200)
mock_proxy.assert_has_calls([
- mock.call(None, 'GET', 'user?list', {}),
- mock.call(None, 'GET', 'user?list', {'marker': 'foo:bar'})
+ call(None, 'GET', 'user?list', {}),
+ call(None, 'GET', 'user?list', {'marker': 'foo:bar'})
])
self.assertJsonBody(['test1', 'test2', 'test3', 'admin'])
- @mock.patch('dashboard.controllers.rgw.RgwRESTController.proxy')
+ @patch('dashboard.controllers.rgw.RgwRESTController.proxy')
def test_user_list_duplicate_marker(self, mock_proxy):
mock_proxy.side_effect = [{
'count': 3,
self._get('/test/api/rgw/user')
self.assertStatus(500)
- @mock.patch('dashboard.controllers.rgw.RgwRESTController.proxy')
+ @patch('dashboard.controllers.rgw.RgwRESTController.proxy')
def test_user_list_invalid_marker(self, mock_proxy):
mock_proxy.side_effect = [{
'count': 3,
self._get('/test/api/rgw/user')
self.assertStatus(500)
- @mock.patch('dashboard.controllers.rgw.RgwRESTController.proxy')
- @mock.patch.object(RgwUser, '_keys_allowed')
+ @patch('dashboard.controllers.rgw.RgwRESTController.proxy')
+ @patch.object(RgwUser, '_keys_allowed')
def test_user_get_with_keys(self, keys_allowed, mock_proxy):
keys_allowed.return_value = True
mock_proxy.return_value = {
self.assertInJsonBody('keys')
self.assertInJsonBody('swift_keys')
- @mock.patch('dashboard.controllers.rgw.RgwRESTController.proxy')
- @mock.patch.object(RgwUser, '_keys_allowed')
+ @patch('dashboard.controllers.rgw.RgwRESTController.proxy')
+ @patch.object(RgwUser, '_keys_allowed')
def test_user_get_without_keys(self, keys_allowed, mock_proxy):
keys_allowed.return_value = False
mock_proxy.return_value = {
# -*- coding: utf-8 -*-
# pylint: disable=too-many-public-methods
-import unittest
+from unittest import TestCase
+from unittest.mock import Mock, patch
-try:
- from unittest.mock import MagicMock, patch
-except ImportError:
- from mock import MagicMock, patch # type: ignore
-
-from ..services.rgw_client import NoCredentialsException, RgwClient, \
- RgwDaemon, _parse_frontend_config
+from ..exceptions import DashboardException
+from ..services.rgw_client import NoCredentialsException, \
+ NoRgwDaemonsException, RgwClient, _parse_frontend_config
from ..settings import Settings
-from . import KVStoreMockMixin # pylint: disable=no-name-in-module
-
-
-def _get_daemons_stub():
- daemon = RgwDaemon()
- daemon.host = 'rgw.1.myorg.com'
- daemon.port = 8000
- daemon.ssl = True
- daemon.name = 'rgw.1.myorg.com'
- daemon.zonegroup_name = 'zonegroup2-realm1'
- return {daemon.name: daemon}
+from . import KVStoreMockMixin, RgwStub # pylint: disable=no-name-in-module
-@patch('dashboard.services.rgw_client._get_daemons', _get_daemons_stub)
-@patch('dashboard.services.rgw_client.RgwClient._get_user_id', MagicMock(
+@patch('dashboard.services.rgw_client.RgwClient._get_user_id', Mock(
return_value='dummy_admin'))
-class RgwClientTest(unittest.TestCase, KVStoreMockMixin):
+class RgwClientTest(TestCase, KVStoreMockMixin):
def setUp(self):
+ RgwStub.get_daemons()
self.mock_kv_store()
self.CONFIG_KEY_DICT.update({
'RGW_API_ACCESS_KEY': 'klausmustermann',
instance = RgwClient.admin_instance()
self.assertFalse(instance.session.verify)
+ def test_no_daemons(self):
+ RgwStub.get_mgr_no_services()
+ with self.assertRaises(NoRgwDaemonsException) as cm:
+ RgwClient.admin_instance()
+ self.assertIn('No RGW service is running.', str(cm.exception))
+
def test_no_credentials(self):
self.CONFIG_KEY_DICT.update({
'RGW_API_ACCESS_KEY': '',
def test_default_daemon_wrong_settings(self):
self.CONFIG_KEY_DICT.update({
- 'RGW_API_HOST': 'localhost',
+ 'RGW_API_HOST': '172.20.0.2',
'RGW_API_PORT': '7990',
})
- with self.assertRaises(LookupError) as cm:
+ with self.assertRaises(DashboardException) as cm:
RgwClient.admin_instance()
- self.assertIn('No RGW daemon found with host:', str(cm.exception))
+ self.assertIn('No RGW daemon found with user-defined host:', str(cm.exception))
@patch.object(RgwClient, '_get_daemon_zone_info')
def test_get_placement_targets_from_zone(self, zone_info):
instance = RgwClient.admin_instance()
expected_result = {
- 'zonegroup': 'zonegroup2-realm1',
+ 'zonegroup': 'zonegroup1',
'placement_targets': [
{
'name': 'default-placement',
self.assertEqual([], instance.get_realms())
-class RgwClientHelperTest(unittest.TestCase):
+class RgwClientHelperTest(TestCase):
def test_parse_frontend_config_1(self):
self.assertEqual(_parse_frontend_config('beast port=8000'), (8000, False))
projects / ceph.git / commitdiff