From 9d1700cbafd761e7b16a0f6687f01025edd2fa6e Mon Sep 17 00:00:00 2001 From: alfonsomthd Date: Fri, 19 Jul 2019 16:02:44 +0200 Subject: [PATCH] mgr/dashboard: select placement target on RGW bucket creation * Select a placement target from the zone that the RGW daemon is running on. Fixes: https://tracker.ceph.com/issues/40567 Signed-off-by: alfonsomthd --- qa/tasks/mgr/dashboard/test_rgw.py | 8 +- .../mgr/dashboard/controllers/health.py | 25 +++-- src/pybind/mgr/dashboard/controllers/rgw.py | 24 ++++- .../frontend/e2e/rgw/buckets.e2e-spec.ts | 28 +++--- .../dashboard/frontend/e2e/rgw/buckets.po.ts | 27 +++++- .../rgw-bucket-form.component.html | 40 +++++++- .../rgw-bucket-form.component.spec.ts | 42 +++++++-- .../rgw-bucket-form.component.ts | 55 ++++++++--- .../app/shared/api/rgw-bucket.service.spec.ts | 6 +- .../src/app/shared/api/rgw-bucket.service.ts | 5 +- .../app/shared/api/rgw-site.service.spec.ts | 34 +++++++ .../src/app/shared/api/rgw-site.service.ts | 22 +++++ .../mgr/dashboard/services/rgw_client.py | 55 ++++++++++- .../mgr/dashboard/tests/test_rgw_client.py | 92 +++++++++++++++++++ src/pybind/mgr/dashboard/tests/test_tools.py | 18 +++- src/pybind/mgr/dashboard/tools.py | 35 +++++++ 16 files changed, 451 insertions(+), 65 deletions(-) create mode 100644 src/pybind/mgr/dashboard/frontend/src/app/shared/api/rgw-site.service.spec.ts create mode 100644 src/pybind/mgr/dashboard/frontend/src/app/shared/api/rgw-site.service.ts diff --git a/qa/tasks/mgr/dashboard/test_rgw.py b/qa/tasks/mgr/dashboard/test_rgw.py index bbb9df987122f..77f6c47985a1a 100644 --- a/qa/tasks/mgr/dashboard/test_rgw.py +++ b/qa/tasks/mgr/dashboard/test_rgw.py @@ -131,7 +131,9 @@ class RgwBucketTest(RgwTestCase): '/api/rgw/bucket', params={ 'bucket': 'teuth-test-bucket', - 'uid': 'admin' + 'uid': 'admin', + 'zonegroup': 'default', + 'placement_target': 'default-placement' }) self.assertStatus(201) data = self.jsonBody() @@ -201,7 +203,9 @@ class RgwBucketTest(RgwTestCase): '/api/rgw/bucket', params={ 'bucket': 'teuth-test-bucket', - 'uid': 'testx$teuth-test-user' + 'uid': 'testx$teuth-test-user', + 'zonegroup': 'default', + 'placement_target': 'default-placement' }) self.assertStatus(201) # It's not possible to validate the result because there diff --git a/src/pybind/mgr/dashboard/controllers/health.py b/src/pybind/mgr/dashboard/controllers/health.py index 30d1e5285299a..ecb771cd01f36 100644 --- a/src/pybind/mgr/dashboard/controllers/health.py +++ b/src/pybind/mgr/dashboard/controllers/health.py @@ -11,6 +11,7 @@ from ..security import Permission, Scope from ..services.ceph_service import CephService from ..services.iscsi_cli import IscsiGatewaysConfig from ..services.iscsi_client import IscsiClient +from ..tools import partial_dict class HealthData(object): @@ -26,10 +27,6 @@ class HealthData(object): self._has_permissions = auth_callback self._minimal = minimal - @staticmethod - def _partial_dict(orig, keys): - return {k: orig[k] for k in keys} - def all_health(self): result = { "health": self.basic_health(), @@ -83,7 +80,7 @@ class HealthData(object): def client_perf(self): result = CephService.get_client_perf() if self._minimal: - result = self._partial_dict( + result = partial_dict( result, ['read_bytes_sec', 'read_op_per_sec', 'recovering_bytes_per_sec', 'write_bytes_sec', @@ -97,7 +94,7 @@ class HealthData(object): del df['stats_by_class'] if self._minimal: - df = dict(stats=self._partial_dict( + df = dict(stats=partial_dict( df['stats'], ['total_avail_bytes', 'total_bytes', 'total_used_raw_bytes'] @@ -107,15 +104,15 @@ class HealthData(object): def fs_map(self): fs_map = mgr.get('fs_map') if self._minimal: - fs_map = self._partial_dict(fs_map, ['filesystems', 'standbys']) + fs_map = partial_dict(fs_map, ['filesystems', 'standbys']) fs_map['standbys'] = [{}] * len(fs_map['standbys']) - fs_map['filesystems'] = [self._partial_dict(item, ['mdsmap']) for + fs_map['filesystems'] = [partial_dict(item, ['mdsmap']) for item in fs_map['filesystems']] for fs in fs_map['filesystems']: mdsmap_info = fs['mdsmap']['info'] min_mdsmap_info = dict() for k, v in mdsmap_info.items(): - min_mdsmap_info[k] = self._partial_dict(v, ['state']) + min_mdsmap_info[k] = partial_dict(v, ['state']) fs['mdsmap'] = dict(info=min_mdsmap_info) return fs_map @@ -136,15 +133,15 @@ class HealthData(object): def mgr_map(self): mgr_map = mgr.get('mgr_map') if self._minimal: - mgr_map = self._partial_dict(mgr_map, ['active_name', 'standbys']) + mgr_map = partial_dict(mgr_map, ['active_name', 'standbys']) mgr_map['standbys'] = [{}] * len(mgr_map['standbys']) return mgr_map def mon_status(self): mon_status = json.loads(mgr.get('mon_status')['json']) if self._minimal: - mon_status = self._partial_dict(mon_status, ['monmap', 'quorum']) - mon_status['monmap'] = self._partial_dict( + mon_status = partial_dict(mon_status, ['monmap', 'quorum']) + mon_status['monmap'] = partial_dict( mon_status['monmap'], ['mons'] ) mon_status['monmap']['mons'] = [{}] * \ @@ -157,9 +154,9 @@ class HealthData(object): # Not needed, skip the effort of transmitting this to UI del osd_map['pg_temp'] if self._minimal: - osd_map = self._partial_dict(osd_map, ['osds']) + osd_map = partial_dict(osd_map, ['osds']) osd_map['osds'] = [ - self._partial_dict(item, ['in', 'up']) + partial_dict(item, ['in', 'up']) for item in osd_map['osds'] ] else: diff --git a/src/pybind/mgr/dashboard/controllers/rgw.py b/src/pybind/mgr/dashboard/controllers/rgw.py index 871fa00fdf4ee..485159c646d09 100644 --- a/src/pybind/mgr/dashboard/controllers/rgw.py +++ b/src/pybind/mgr/dashboard/controllers/rgw.py @@ -13,6 +13,7 @@ from ..rest_client import RequestException from ..security import Scope from ..services.ceph_service import CephService from ..services.rgw_client import RgwClient +from ..tools import json_str_to_object @ApiController('/rgw', Scope.RGW) @@ -96,13 +97,28 @@ class RgwRESTController(RESTController): try: instance = RgwClient.admin_instance() result = instance.proxy(method, path, params, None) - if json_response and result != '': - result = json.loads(result.decode('utf-8')) + if json_response: + result = json_str_to_object(result) return result except (DashboardException, RequestException) as e: raise DashboardException(e, http_status_code=500, component='rgw') +@ApiController('/rgw/site', Scope.RGW) +class RgwSite(RgwRESTController): + + def list(self, query=None): + if query == 'placement-targets': + instance = RgwClient.admin_instance() + result = instance.get_placement_targets() + else: + # @TODO: (it'll be required for multisite workflows): + # by default, retrieve cluster realms/zonegroups map. + raise DashboardException(http_status_code=501, component='rgw', msg='Not Implemented') + + return result + + @ApiController('/rgw/bucket', Scope.RGW) class RgwBucket(RgwRESTController): @@ -128,10 +144,10 @@ class RgwBucket(RgwRESTController): result = self.proxy('GET', 'bucket', {'bucket': bucket}) return self._append_bid(result) - def create(self, bucket, uid): + def create(self, bucket, uid, zonegroup, placement_target): try: rgw_client = RgwClient.instance(uid) - return rgw_client.create_bucket(bucket) + return rgw_client.create_bucket(bucket, zonegroup, placement_target) except RequestException as e: raise DashboardException(e, http_status_code=500, component='rgw') diff --git a/src/pybind/mgr/dashboard/frontend/e2e/rgw/buckets.e2e-spec.ts b/src/pybind/mgr/dashboard/frontend/e2e/rgw/buckets.e2e-spec.ts index d6685f4541c71..0f5e6b707f38f 100644 --- a/src/pybind/mgr/dashboard/frontend/e2e/rgw/buckets.e2e-spec.ts +++ b/src/pybind/mgr/dashboard/frontend/e2e/rgw/buckets.e2e-spec.ts @@ -1,11 +1,13 @@ import { Helper } from '../helper.po'; import { PageHelper } from '../page-helper.po'; +import { BucketsPageHelper } from './buckets.po'; describe('RGW buckets page', () => { - let buckets; + let buckets: BucketsPageHelper; beforeAll(() => { buckets = new Helper().buckets; + buckets.navigateTo(); }); afterEach(() => { @@ -13,22 +15,18 @@ describe('RGW buckets page', () => { }); describe('breadcrumb test', () => { - beforeAll(() => { - buckets.navigateTo(); - }); - it('should open and show breadcrumb', () => { expect(PageHelper.getBreadcrumbText()).toEqual('Buckets'); }); }); describe('create, edit & delete bucket test', () => { - beforeAll(() => { - buckets.navigateTo(); - }); - it('should create bucket', () => { - buckets.create('000test', '0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef'); + buckets.create( + '000test', + '0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef', + 'default-placement' + ); expect(PageHelper.getTableCell('000test').isPresent()).toBe(true); }); @@ -44,16 +42,16 @@ describe('RGW buckets page', () => { }); describe('Invalid Input in Create and Edit tests', () => { - beforeAll(() => { - buckets.navigateTo(); - }); - it('should test invalid inputs in create fields', () => { buckets.invalidCreate(); }); it('should test invalid input in edit owner field', () => { - buckets.create('000rq', '0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef'); + buckets.create( + '000rq', + '0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef', + 'default-placement' + ); buckets.invalidEdit('000rq'); buckets.delete('000rq'); }); diff --git a/src/pybind/mgr/dashboard/frontend/e2e/rgw/buckets.po.ts b/src/pybind/mgr/dashboard/frontend/e2e/rgw/buckets.po.ts index 3506a019075af..834c501f792e9 100644 --- a/src/pybind/mgr/dashboard/frontend/e2e/rgw/buckets.po.ts +++ b/src/pybind/mgr/dashboard/frontend/e2e/rgw/buckets.po.ts @@ -8,7 +8,7 @@ export class BucketsPageHelper extends PageHelper { create: '/#/rgw/bucket/create' }; - create(name, owner) { + create(name, owner, placementTarget) { this.navigateTo('create'); // Enter in bucket name @@ -19,6 +19,11 @@ export class BucketsPageHelper extends PageHelper { element(by.cssContainingText('select[name=owner] option', owner)).click(); expect(element(by.id('owner')).getAttribute('class')).toContain('ng-valid'); + // Select bucket placement target: + element(by.id('owner')).click(); + element(by.cssContainingText('select[name=placement-target] option', placementTarget)).click(); + expect(element(by.id('placement-target')).getAttribute('class')).toContain('ng-valid'); + // Click the create button and wait for bucket to be made const createButton = element(by.cssContainingText('button', 'Create Bucket')); createButton.click().then(() => { @@ -39,6 +44,10 @@ export class BucketsPageHelper extends PageHelper { expect(PageHelper.getBreadcrumbText()).toEqual('Edit'); + expect(element(by.css('input[name=placement-target]')).getAttribute('value')).toBe( + 'default-placement' + ); + const ownerDropDown = element(by.id('owner')); ownerDropDown.click(); // click owner dropdown menu @@ -134,6 +143,22 @@ export class BucketsPageHelper extends PageHelper { 'This field is required.' ); + // Check invalid placement target input + PageHelper.moveClick(ownerDropDown); + element(by.cssContainingText('select[name=owner] option', 'dev')).click(); + // The drop down error message will not appear unless a valid option is previsously selected. + element( + by.cssContainingText('select[name=placement-target] option', 'default-placement') + ).click(); + element( + by.cssContainingText('select[name=placement-target] option', 'Select a placement target') + ).click(); + PageHelper.moveClick(nameInputField); // To trigger a validation + expect(element(by.id('placement-target')).getAttribute('class')).toContain('ng-invalid'); + expect(element(by.css('#placement-target + .invalid-feedback')).getText()).toMatch( + 'This field is required.' + ); + // Clicks the Create Bucket button but the page doesn't move. Done by testing // for the breadcrumb PageHelper.moveClick(element(by.cssContainingText('button', 'Create Bucket'))); // Clicks Create Bucket button diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-bucket-form/rgw-bucket-form.component.html b/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-bucket-form/rgw-bucket-form.component.html index 9eb6e67fed0d3..43d0f467986f8 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-bucket-form/rgw-bucket-form.component.html +++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-bucket-form/rgw-bucket-form.component.html @@ -19,7 +19,7 @@
+ +
+ +
+ + + This field is required. + + + + +
+
+
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-bucket-form/rgw-bucket-form.component.spec.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-bucket-form/rgw-bucket-form.component.spec.ts index dfebff4b6ea81..3264e62dd06cf 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-bucket-form/rgw-bucket-form.component.spec.ts +++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-bucket-form/rgw-bucket-form.component.spec.ts @@ -9,6 +9,7 @@ import { of as observableOf } from 'rxjs'; import { configureTestBed, i18nProviders } from '../../../../testing/unit-test-helper'; import { RgwBucketService } from '../../../shared/api/rgw-bucket.service'; +import { RgwSiteService } from '../../../shared/api/rgw-site.service'; import { NotificationType } from '../../../shared/enum/notification-type.enum'; import { NotificationService } from '../../../shared/services/notification.service'; import { SharedModule } from '../../../shared/shared.module'; @@ -17,7 +18,8 @@ import { RgwBucketFormComponent } from './rgw-bucket-form.component'; describe('RgwBucketFormComponent', () => { let component: RgwBucketFormComponent; let fixture: ComponentFixture; - let rwgBucketService: RgwBucketService; + let rgwBucketService: RgwBucketService; + let getPlacementTargetsSpy; configureTestBed({ declarations: [RgwBucketFormComponent], @@ -34,8 +36,8 @@ describe('RgwBucketFormComponent', () => { beforeEach(() => { fixture = TestBed.createComponent(RgwBucketFormComponent); component = fixture.componentInstance; - fixture.detectChanges(); - rwgBucketService = TestBed.get(RgwBucketService); + rgwBucketService = TestBed.get(RgwBucketService); + getPlacementTargetsSpy = spyOn(TestBed.get(RgwSiteService), 'getPlacementTargets'); }); it('should create', () => { @@ -82,7 +84,7 @@ describe('RgwBucketFormComponent', () => { }); it('should validate name (4/4)', () => { - spyOn(rwgBucketService, 'enumerate').and.returnValue(observableOf(['abcd'])); + spyOn(rgwBucketService, 'enumerate').and.returnValue(observableOf(['abcd'])); const validatorFn = component.bucketNameValidator(); const ctrl = new FormControl('abcd'); ctrl.markAsDirty(); @@ -95,6 +97,34 @@ describe('RgwBucketFormComponent', () => { }); } }); + + it('should get zonegroup and placement targets', () => { + const payload = { + zonegroup: 'default', + placement_targets: [ + { + name: 'default-placement', + data_pool: 'default.rgw.buckets.data' + }, + { + name: 'placement-target2', + data_pool: 'placement-target2.rgw.buckets.data' + } + ] + }; + getPlacementTargetsSpy.and.returnValue(observableOf(payload)); + fixture.detectChanges(); + + expect(component.zonegroup).toBe(payload.zonegroup); + const placementTargets = []; + for (const placementTarget of payload['placement_targets']) { + placementTarget['description'] = `${placementTarget['name']} (pool: ${ + placementTarget['data_pool'] + })`; + placementTargets.push(placementTarget); + } + expect(component.placementTargets).toEqual(placementTargets); + }); }); describe('submit form', () => { @@ -107,7 +137,7 @@ describe('RgwBucketFormComponent', () => { }); it('tests create success notification', () => { - spyOn(rwgBucketService, 'create').and.returnValue(observableOf([])); + spyOn(rgwBucketService, 'create').and.returnValue(observableOf([])); component.editing = false; component.bucketForm.markAsDirty(); component.submit(); @@ -118,7 +148,7 @@ describe('RgwBucketFormComponent', () => { }); it('tests update success notification', () => { - spyOn(rwgBucketService, 'update').and.returnValue(observableOf([])); + spyOn(rgwBucketService, 'update').and.returnValue(observableOf([])); component.editing = true; component.bucketForm.markAsDirty(); component.submit(); diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-bucket-form/rgw-bucket-form.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-bucket-form/rgw-bucket-form.component.ts index 8a3391f8ab296..09d7a099e5bee 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-bucket-form/rgw-bucket-form.component.ts +++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-bucket-form/rgw-bucket-form.component.ts @@ -6,6 +6,7 @@ import { I18n } from '@ngx-translate/i18n-polyfill'; import * as _ from 'lodash'; import { RgwBucketService } from '../../../shared/api/rgw-bucket.service'; +import { RgwSiteService } from '../../../shared/api/rgw-site.service'; import { RgwUserService } from '../../../shared/api/rgw-user.service'; import { ActionLabelsI18n, URLVerbs } from '../../../shared/constants/app.constants'; import { NotificationType } from '../../../shared/enum/notification-type.enum'; @@ -26,12 +27,15 @@ export class RgwBucketFormComponent implements OnInit { owners = null; action: string; resource: string; + zonegroup: string; + placementTargets: Object[] = []; constructor( private route: ActivatedRoute, private router: Router, private formBuilder: CdFormBuilder, private rgwBucketService: RgwBucketService, + private rgwSiteService: RgwSiteService, private rgwUserService: RgwUserService, private notificationService: NotificationService, private i18n: I18n, @@ -47,7 +51,8 @@ export class RgwBucketFormComponent implements OnInit { this.bucketForm = this.formBuilder.group({ id: [null], bid: [null, [Validators.required], [this.bucketNameValidator()]], - owner: [null, [Validators.required]] + owner: [null, [Validators.required]], + 'placement-target': [null, this.editing ? [] : [Validators.required]] }); } @@ -57,6 +62,24 @@ export class RgwBucketFormComponent implements OnInit { this.owners = resp.sort(); }); + if (!this.editing) { + // Get placement targets: + this.rgwSiteService.getPlacementTargets().subscribe((placementTargets) => { + this.zonegroup = placementTargets['zonegroup']; + _.forEach(placementTargets['placement_targets'], (placementTarget) => { + placementTarget['description'] = `${placementTarget['name']} (${this.i18n('pool')}: ${ + placementTarget['data_pool'] + })`; + this.placementTargets.push(placementTarget); + }); + + // If there is only 1 placement target, select it by default: + if (this.placementTargets.length === 1) { + this.bucketForm.get('placement-target').setValue(this.placementTargets[0]['name']); + } + }); + } + // Process route parameters. this.route.params.subscribe( (params: { bid: string }) => { @@ -72,6 +95,7 @@ export class RgwBucketFormComponent implements OnInit { const defaults = _.clone(this.bucketForm.value); // Extract the values displayed in the form. let value = _.pick(resp, _.keys(this.bucketForm.value)); + value['placement-target'] = resp['placement_rule']; // Append default values. value = _.merge(defaults, value); // Update the form. @@ -96,6 +120,7 @@ export class RgwBucketFormComponent implements OnInit { } const bidCtl = this.bucketForm.get('bid'); const ownerCtl = this.bucketForm.get('owner'); + const placementTargetCtl = this.bucketForm.get('placement-target'); if (this.editing) { // Edit const idCtl = this.bucketForm.get('id'); @@ -114,19 +139,21 @@ export class RgwBucketFormComponent implements OnInit { ); } else { // Add - this.rgwBucketService.create(bidCtl.value, ownerCtl.value).subscribe( - () => { - this.notificationService.show( - NotificationType.success, - this.i18n('Created Object Gateway bucket "{{bid}}"', { bid: bidCtl.value }) - ); - this.goToListView(); - }, - () => { - // Reset the 'Submit' button. - this.bucketForm.setErrors({ cdSubmitButton: true }); - } - ); + this.rgwBucketService + .create(bidCtl.value, ownerCtl.value, this.zonegroup, placementTargetCtl.value) + .subscribe( + () => { + this.notificationService.show( + NotificationType.success, + this.i18n('Created Object Gateway bucket "{{bid}}"', { bid: bidCtl.value }) + ); + this.goToListView(); + }, + () => { + // Reset the 'Submit' button. + this.bucketForm.setErrors({ cdSubmitButton: true }); + } + ); } } diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/api/rgw-bucket.service.spec.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/rgw-bucket.service.spec.ts index 0aed3268137cd..b6c6f73154848 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/shared/api/rgw-bucket.service.spec.ts +++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/rgw-bucket.service.spec.ts @@ -62,8 +62,10 @@ describe('RgwBucketService', () => { }); it('should call create', () => { - service.create('foo', 'bar').subscribe(); - const req = httpTesting.expectOne('api/rgw/bucket?bucket=foo&uid=bar'); + service.create('foo', 'bar', 'default', 'default-placement').subscribe(); + const req = httpTesting.expectOne( + 'api/rgw/bucket?bucket=foo&uid=bar&zonegroup=default&placement_target=default-placement' + ); expect(req.request.method).toBe('POST'); }); diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/api/rgw-bucket.service.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/rgw-bucket.service.ts index fdd94672dfa95..c60fcfba9f6f7 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/shared/api/rgw-bucket.service.ts +++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/rgw-bucket.service.ts @@ -48,10 +48,13 @@ export class RgwBucketService { return this.http.get(`${this.url}/${bucket}`); } - create(bucket: string, uid: string) { + create(bucket: string, uid: string, zonegroup: string, placementTarget: string) { let params = new HttpParams(); params = params.append('bucket', bucket); params = params.append('uid', uid); + params = params.append('zonegroup', zonegroup); + params = params.append('placement_target', placementTarget); + return this.http.post(this.url, null, { params: params }); } diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/api/rgw-site.service.spec.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/rgw-site.service.spec.ts new file mode 100644 index 0000000000000..8f4df0bc00032 --- /dev/null +++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/rgw-site.service.spec.ts @@ -0,0 +1,34 @@ +import { HttpClientTestingModule, HttpTestingController } from '@angular/common/http/testing'; +import { TestBed } from '@angular/core/testing'; + +import { configureTestBed } from '../../../testing/unit-test-helper'; +import { RgwSiteService } from './rgw-site.service'; + +describe('RgwSiteService', () => { + let service: RgwSiteService; + let httpTesting: HttpTestingController; + + configureTestBed({ + providers: [RgwSiteService], + imports: [HttpClientTestingModule] + }); + + beforeEach(() => { + service = TestBed.get(RgwSiteService); + httpTesting = TestBed.get(HttpTestingController); + }); + + afterEach(() => { + httpTesting.verify(); + }); + + it('should be created', () => { + expect(service).toBeTruthy(); + }); + + it('should call getPlacementTargets', () => { + service.getPlacementTargets().subscribe(); + const req = httpTesting.expectOne('api/rgw/site?query=placement-targets'); + expect(req.request.method).toBe('GET'); + }); +}); diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/api/rgw-site.service.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/rgw-site.service.ts new file mode 100644 index 0000000000000..3d76b3f8d494c --- /dev/null +++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/rgw-site.service.ts @@ -0,0 +1,22 @@ +import { HttpClient, HttpParams } from '@angular/common/http'; +import { Injectable } from '@angular/core'; + +import { cdEncode } from '../decorators/cd-encode'; +import { ApiModule } from './api.module'; + +@cdEncode +@Injectable({ + providedIn: ApiModule +}) +export class RgwSiteService { + private url = 'api/rgw/site'; + + constructor(private http: HttpClient) {} + + getPlacementTargets() { + let params = new HttpParams(); + params = params.append('query', 'placement-targets'); + + return this.http.get(this.url, { params: params }); + } +} diff --git a/src/pybind/mgr/dashboard/services/rgw_client.py b/src/pybind/mgr/dashboard/services/rgw_client.py index 4d7ed2c77ec64..51c7649e96dfe 100644 --- a/src/pybind/mgr/dashboard/services/rgw_client.py +++ b/src/pybind/mgr/dashboard/services/rgw_client.py @@ -4,13 +4,19 @@ from __future__ import absolute_import import re import ipaddress from distutils.util import strtobool +import xml.etree.ElementTree as ET import six from ..awsauth import S3Auth from ..settings import Settings, Options from ..rest_client import RestClient, RequestException -from ..tools import build_url, dict_contains_path +from ..tools import build_url, dict_contains_path, json_str_to_object, partial_dict from .. import mgr, logger +try: + from typing import Any, Dict, List # pylint: disable=unused-import +except ImportError: + pass # For typing only + class NoCredentialsException(RequestException): def __init__(self): @@ -235,6 +241,19 @@ class RgwClient(RestClient): # Append the instance to the internal map. RgwClient._user_instances[RgwClient._SYSTEM_USERID] = instance + def _get_daemon_zone_info(self): # type: () -> Dict[str, Any] + return json_str_to_object(self.proxy('GET', 'config?type=zone', None, None)) + + def _get_daemon_zonegroup_map(self): # type: () -> List[Dict[str, Any]] + zonegroups = json_str_to_object( + self.proxy('GET', 'config?type=zonegroup-map', None, None) + ) + + return [partial_dict( + zonegroup['val'], + ['api_name', 'zones'] + ) for zonegroup in zonegroups['zonegroups']] + @staticmethod def _rgw_settings(): return (Settings.RGW_API_HOST, @@ -429,6 +448,34 @@ class RgwClient(RestClient): raise e @RestClient.api_put('/{bucket_name}') - def create_bucket(self, bucket_name, request=None): - logger.info("Creating bucket: %s", bucket_name) - return request() + def create_bucket(self, bucket_name, zonegroup, placement_target, request=None): + logger.info("Creating bucket: %s, zonegroup: %s, placement_target: %s", + bucket_name, zonegroup, placement_target) + create_bucket_configuration = ET.Element('CreateBucketConfiguration') + location_constraint = ET.SubElement(create_bucket_configuration, 'LocationConstraint') + location_constraint.text = '{}:{}'.format(zonegroup, placement_target) + + return request(data=ET.tostring(create_bucket_configuration, encoding='utf-8')) + + def get_placement_targets(self): # type: () -> Dict[str, Any] + zone = self._get_daemon_zone_info() + # A zone without realm id can only belong to default zonegroup. + zonegroup_name = 'default' + if zone['realm_id']: + zonegroup_map = self._get_daemon_zonegroup_map() + for zonegroup in zonegroup_map: + for realm_zone in zonegroup['zones']: + if realm_zone['id'] == zone['id']: + zonegroup_name = zonegroup['api_name'] + break + + placement_targets = [] # type: List[Dict] + for placement_pool in zone['placement_pools']: + placement_targets.append( + { + 'name': placement_pool['key'], + 'data_pool': placement_pool['val']['storage_classes']['STANDARD']['data_pool'] + } + ) + + return {'zonegroup': zonegroup_name, 'placement_targets': placement_targets} diff --git a/src/pybind/mgr/dashboard/tests/test_rgw_client.py b/src/pybind/mgr/dashboard/tests/test_rgw_client.py index 8bdb01a1b5c37..7d476f72f98f9 100644 --- a/src/pybind/mgr/dashboard/tests/test_rgw_client.py +++ b/src/pybind/mgr/dashboard/tests/test_rgw_client.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- import unittest +from mock import patch from .. import mgr from ..services.rgw_client import RgwClient @@ -38,3 +39,94 @@ class RgwClientTest(unittest.TestCase): mgr.set_module_option('RGW_API_SSL_VERIFY', False) instance = RgwClient.admin_instance() self.assertFalse(instance.session.verify) + + @patch.object(RgwClient, '_get_daemon_zone_info') + def test_get_placement_targets_from_default_zone(self, zone_info): + zone_info.return_value = { + 'placement_pools': [ + { + 'key': 'default-placement', + 'val': { + 'index_pool': 'default.rgw.buckets.index', + 'storage_classes': { + 'STANDARD': { + 'data_pool': 'default.rgw.buckets.data' + } + }, + 'data_extra_pool': 'default.rgw.buckets.non-ec', + 'index_type': 0 + } + } + ], + 'realm_id': '' + } + + instance = RgwClient.admin_instance() + expected_result = { + 'zonegroup': 'default', + 'placement_targets': [ + { + 'name': 'default-placement', + 'data_pool': 'default.rgw.buckets.data' + } + ] + } + self.assertEqual(expected_result, instance.get_placement_targets()) + + @patch.object(RgwClient, '_get_daemon_zone_info') + @patch.object(RgwClient, '_get_daemon_zonegroup_map') + def test_get_placement_targets_from_realm_zone(self, zonegroup_map, zone_info): + zone_info.return_value = { + 'id': 'a0df30ea-4b5b-4830-b143-2bedf684663d', + 'placement_pools': [ + { + 'key': 'default-placement', + 'val': { + 'index_pool': 'default.rgw.buckets.index', + 'storage_classes': { + 'STANDARD': { + 'data_pool': 'default.rgw.buckets.data' + } + } + } + } + ], + 'realm_id': 'b5a25d1b-e7ed-4fe5-b461-74f24b8e759b' + } + + zonegroup_map.return_value = [ + { + 'api_name': 'zonegroup1-realm1', + 'zones': [ + { + 'id': '2ef7d0ef-7616-4e9c-8553-b732ebf0592b' + }, + { + 'id': 'b1d15925-6c8e-408e-8485-5a62cbccfe1f' + } + ] + }, + { + 'api_name': 'zonegroup2-realm1', + 'zones': [ + { + 'id': '645f0f59-8fcc-4e11-95d5-24f289ee8e25' + }, + { + 'id': 'a0df30ea-4b5b-4830-b143-2bedf684663d' + } + ] + } + ] + + instance = RgwClient.admin_instance() + expected_result = { + 'zonegroup': 'zonegroup2-realm1', + 'placement_targets': [ + { + 'name': 'default-placement', + 'data_pool': 'default.rgw.buckets.data' + } + ] + } + self.assertEqual(expected_result, instance.get_placement_targets()) diff --git a/src/pybind/mgr/dashboard/tests/test_tools.py b/src/pybind/mgr/dashboard/tests/test_tools.py index 7095b7598ab42..3506e176cd584 100644 --- a/src/pybind/mgr/dashboard/tests/test_tools.py +++ b/src/pybind/mgr/dashboard/tests/test_tools.py @@ -11,7 +11,7 @@ from . import ControllerTestCase from ..services.exception import handle_rados_error from ..controllers import RESTController, ApiController, Controller, \ BaseController, Proxy -from ..tools import dict_contains_path, RequestLoggingTool +from ..tools import dict_contains_path, json_str_to_object, partial_dict, RequestLoggingTool # pylint: disable=W0613 @@ -178,3 +178,19 @@ class TestFunctions(unittest.TestCase): self.assertTrue(dict_contains_path(x, ['a'])) self.assertFalse(dict_contains_path(x, ['a', 'c'])) self.assertTrue(dict_contains_path(x, [])) + + def test_json_str_to_object(self): + expected_result = {'a': 1, 'b': 'bbb'} + self.assertEqual(expected_result, json_str_to_object('{"a": 1, "b": "bbb"}')) + self.assertEqual(expected_result, json_str_to_object(b'{"a": 1, "b": "bbb"}')) + self.assertEqual('', json_str_to_object('')) + self.assertRaises(TypeError, json_str_to_object, None) + + def test_partial_dict(self): + expected_result = {'a': 1, 'c': 3} + self.assertEqual(expected_result, partial_dict({'a': 1, 'b': 2, 'c': 3}, ['a', 'c'])) + self.assertEqual({}, partial_dict({'a': 1, 'b': 2, 'c': 3}, [])) + self.assertEqual({}, partial_dict({}, [])) + self.assertRaises(KeyError, partial_dict, {'a': 1, 'b': 2, 'c': 3}, ['d']) + self.assertRaises(TypeError, partial_dict, None, ['a']) + self.assertRaises(TypeError, partial_dict, {'a': 1, 'b': 2, 'c': 3}, None) diff --git a/src/pybind/mgr/dashboard/tools.py b/src/pybind/mgr/dashboard/tools.py index defe2d352dca9..9d01154d871ac 100644 --- a/src/pybind/mgr/dashboard/tools.py +++ b/src/pybind/mgr/dashboard/tools.py @@ -27,6 +27,11 @@ from .exceptions import ViewCacheNoDataException from .settings import Settings from .services.auth import JwtManager +try: + from typing import Any, AnyStr, Dict, List # pylint: disable=unused-import +except ImportError: + pass # For typing only + class RequestLoggingTool(cherrypy.Tool): def __init__(self): @@ -766,6 +771,36 @@ def str_to_bool(val): return bool(strtobool(val)) +def json_str_to_object(value): # type: (AnyStr) -> Any + """ + It converts a JSON valid string representation to object. + + >>> result = json_str_to_object('{"a": 1}') + >>> result == {'a': 1} + True + """ + if value == '': + return value + + try: + # json.loads accepts binary input from version >=3.6 + value = value.decode('utf-8') + except AttributeError: + pass + + return json.loads(value) + + +def partial_dict(orig, keys): # type: (Dict, List[str]) -> Dict + """ + It returns Dict containing only the selected keys of original Dict. + + >>> partial_dict({'a': 1, 'b': 2}, ['b']) + {'b': 2} + """ + return {k: orig[k] for k in keys} + + def get_request_body_params(request): """ Helper function to get parameters from the request body. -- 2.39.5