From a09de47e3b488fab5ff115be4580bbaf0d9db4e3 Mon Sep 17 00:00:00 2001 From: =?utf8?q?Alfonso=20Mart=C3=ADnez?= Date: Fri, 22 May 2020 13:36:10 +0200 Subject: [PATCH] mgr/dashboard: grafana panels for rgw multisite sync performance MIME-Version: 1.0 Content-Type: text/plain; charset=utf8 Content-Transfer-Encoding: 8bit * RGW sync perf. counters are now exposed through grafana panels. * Sync Performance tab is only shown if rgw realm is detected. * Prometheus module: added metrics suitable for prometheus consumption (from existing ones, not replacing for backward compatibility). Fixes: https://tracker.ceph.com/issues/45310 Signed-off-by: Alfonso Martínez (cherry picked from commit cf4ff7d2f03bc285a3fae3f27577333f11dab58a) Conflicts: - Solved conflicts from cherry-pick: src/pybind/mgr/dashboard/controllers/rgw.py src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-bucket-form/rgw-bucket-form.component.spec.ts src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-bucket-form/rgw-bucket-form.component.ts src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-daemon-list/rgw-daemon-list.component.ts src/pybind/mgr/dashboard/frontend/src/app/shared/api/rgw-site.service.spec.ts src/pybind/mgr/dashboard/frontend/src/app/shared/api/rgw-site.service.ts src/pybind/mgr/dashboard/services/rgw_client.py src/pybind/mgr/dashboard/tests/test_rgw_client.py - src/pybind/mgr/dashboard/tools.py: added method included in a feature not to be backported. - src/pybind/mgr/dashboard/module.py: fixed linting issue. --- .../dashboards/radosgw-sync-overview.json | 440 ++++++++++++++++++ qa/tasks/mgr/dashboard/test_rgw.py | 10 + src/pybind/mgr/dashboard/controllers/rgw.py | 12 + .../rgw-daemon-list.component.html | 9 + .../rgw-daemon-list.component.spec.ts | 40 +- .../rgw-daemon-list.component.ts | 20 +- .../app/shared/api/rgw-site.service.spec.ts | 40 ++ .../src/app/shared/api/rgw-site.service.ts | 24 + src/pybind/mgr/dashboard/module.py | 2 +- .../mgr/dashboard/services/rgw_client.py | 12 +- .../mgr/dashboard/tests/test_rgw_client.py | 22 + src/pybind/mgr/dashboard/tools.py | 20 + src/pybind/mgr/prometheus/module.py | 28 ++ 13 files changed, 670 insertions(+), 9 deletions(-) create mode 100644 monitoring/grafana/dashboards/radosgw-sync-overview.json create mode 100644 src/pybind/mgr/dashboard/frontend/src/app/shared/api/rgw-site.service.spec.ts create mode 100644 src/pybind/mgr/dashboard/frontend/src/app/shared/api/rgw-site.service.ts diff --git a/monitoring/grafana/dashboards/radosgw-sync-overview.json b/monitoring/grafana/dashboards/radosgw-sync-overview.json new file mode 100644 index 0000000000000..e9136d78e3b7c --- /dev/null +++ b/monitoring/grafana/dashboards/radosgw-sync-overview.json @@ -0,0 +1,440 @@ +{ + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "5.0.0" + }, + { + "type": "panel", + "id": "graph", + "name": "Graph", + "version": "5.0.0" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 0, + "id": null, + "iteration": 1534386107523, + "links": [], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 0 + }, + "id": 1, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (source_zone) (rate(ceph_data_sync_from_zone_fetch_bytes_sum[30s]))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{source_zone}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Replication (throughput) from Source Zone", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "unit": "bytes", + "format": "Bps", + "decimals": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + "h": 7, + "w": 7.4, + "x": 8.3, + "y": 0 + }, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (source_zone) (rate(ceph_data_sync_from_zone_fetch_bytes_count[30s]))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{source_zone}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Replication (objects) from Source Zone", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "decimals": null, + "label": "Objects/s", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 0 + }, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (source_zone) (rate(ceph_data_sync_from_zone_poll_latency_sum[30s]) * 1000)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{source_zone}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Polling Request Latency from Source Zone", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "unit": "s", + "format": "ms", + "decimals": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 7 + }, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (source_zone) (rate(ceph_data_sync_from_zone_fetch_errors[30s]))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{source_zone}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Unsuccessful Object Replications from Source Zone", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "decimals": null, + "label": "Count/s", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "refresh": "15s", + "schemaVersion": 16, + "style": "dark", + "tags": [ + "overview" + ], + "templating": { + "list": [ + { + "allValue": null, + "current": {}, + "datasource": "$datasource", + "hide": 2, + "includeAll": true, + "label": null, + "multi": false, + "name": "rgw_servers", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "current": { + "tags": [], + "text": "default", + "value": "default" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "15s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "RGW Sync Overview", + "uid": "rgw-sync-overview", + "version": 2 +} diff --git a/qa/tasks/mgr/dashboard/test_rgw.py b/qa/tasks/mgr/dashboard/test_rgw.py index 9e781142069bd..8baaf0565b58d 100644 --- a/qa/tasks/mgr/dashboard/test_rgw.py +++ b/qa/tasks/mgr/dashboard/test_rgw.py @@ -107,6 +107,16 @@ class RgwApiCredentialsTest(RgwTestCase): data['message']) +class RgwSiteTest(RgwTestCase): + + AUTH_ROLES = ['rgw-manager'] + + def test_get_realms(self): + data = self._get('/api/rgw/site?query=realms') + self.assertStatus(200) + self.assertSchema(data, JList(str)) + + class RgwBucketTest(RgwTestCase): AUTH_ROLES = ['rgw-manager'] diff --git a/src/pybind/mgr/dashboard/controllers/rgw.py b/src/pybind/mgr/dashboard/controllers/rgw.py index 085155aa893a7..0cb819849a8dc 100644 --- a/src/pybind/mgr/dashboard/controllers/rgw.py +++ b/src/pybind/mgr/dashboard/controllers/rgw.py @@ -118,6 +118,18 @@ class RgwRESTController(RESTController): raise DashboardException(e, http_status_code=500, component='rgw') +@ApiController('/rgw/site', Scope.RGW) +class RgwSite(RgwRESTController): + def list(self, query=None): + if query == 'realms': + result = RgwClient.admin_instance().get_realms() + else: + # @TODO: for multisite: by default, retrieve cluster topology/map. + raise DashboardException(http_status_code=501, component='rgw', msg='Not Implemented') + + return result + + @ApiController('/rgw/bucket', Scope.RGW) class RgwBucket(RgwRESTController): diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-daemon-list/rgw-daemon-list.component.html b/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-daemon-list/rgw-daemon-list.component.html index 7efc3c11594f8..2a99284e0e84f 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-daemon-list/rgw-daemon-list.component.html +++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-daemon-list/rgw-daemon-list.component.html @@ -19,4 +19,13 @@ grafanaStyle="two"> + + + + + diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-daemon-list/rgw-daemon-list.component.spec.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-daemon-list/rgw-daemon-list.component.spec.ts index 65b7102c8698f..e143a706543a0 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-daemon-list/rgw-daemon-list.component.spec.ts +++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-daemon-list/rgw-daemon-list.component.spec.ts @@ -3,8 +3,12 @@ import { ComponentFixture, TestBed } from '@angular/core/testing'; import { RouterTestingModule } from '@angular/router/testing'; import { TabsModule } from 'ngx-bootstrap/tabs'; +import { of } from 'rxjs'; import { configureTestBed, i18nProviders } from '../../../../testing/unit-test-helper'; +import { RgwSiteService } from '../../../shared/api/rgw-site.service'; +import { Permissions } from '../../../shared/models/permissions'; +import { AuthStorageService } from '../../../shared/services/auth-storage.service'; import { SharedModule } from '../../../shared/shared.module'; import { PerformanceCounterModule } from '../../performance-counter/performance-counter.module'; import { RgwDaemonDetailsComponent } from '../rgw-daemon-details/rgw-daemon-details.component'; @@ -13,6 +17,15 @@ import { RgwDaemonListComponent } from './rgw-daemon-list.component'; describe('RgwDaemonListComponent', () => { let component: RgwDaemonListComponent; let fixture: ComponentFixture; + let getPermissionsSpy: jasmine.Spy; + let getRealmsSpy: jasmine.Spy; + const permissions = new Permissions({ grafana: ['read'] }); + const expectTabsAndHeading = (length: number, heading: string) => { + const tabs = fixture.debugElement.nativeElement.querySelectorAll('tab'); + + expect(tabs.length).toEqual(length); + expect(tabs[length - 1].getAttribute('heading')).toEqual(heading); + }; configureTestBed({ declarations: [RgwDaemonListComponent, RgwDaemonDetailsComponent], @@ -27,12 +40,37 @@ describe('RgwDaemonListComponent', () => { }); beforeEach(() => { + getPermissionsSpy = spyOn(TestBed.get(AuthStorageService), 'getPermissions'); + getPermissionsSpy.and.returnValue(new Permissions({})); + getRealmsSpy = spyOn(TestBed.get(RgwSiteService), 'get'); + getRealmsSpy.and.returnValue(of([])); fixture = TestBed.createComponent(RgwDaemonListComponent); component = fixture.componentInstance; - fixture.detectChanges(); }); it('should create', () => { + fixture.detectChanges(); expect(component).toBeTruthy(); }); + + it('should only show Daemons List tab', () => { + fixture.detectChanges(); + + expectTabsAndHeading(1, 'Daemons List'); + }); + + it('should show Overall Performance tab', () => { + getPermissionsSpy.and.returnValue(permissions); + fixture.detectChanges(); + + expectTabsAndHeading(2, 'Overall Performance'); + }); + + it('should show Sync Performance tab', () => { + getPermissionsSpy.and.returnValue(permissions); + getRealmsSpy.and.returnValue(of(['realm1'])); + fixture.detectChanges(); + + expectTabsAndHeading(3, 'Sync Performance'); + }); }); diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-daemon-list/rgw-daemon-list.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-daemon-list/rgw-daemon-list.component.ts index 7f68961552782..fe2b215b338b4 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-daemon-list/rgw-daemon-list.component.ts +++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-daemon-list/rgw-daemon-list.component.ts @@ -1,8 +1,9 @@ -import { Component } from '@angular/core'; +import { Component, OnInit } from '@angular/core'; import { I18n } from '@ngx-translate/i18n-polyfill'; import { RgwDaemonService } from '../../../shared/api/rgw-daemon.service'; +import { RgwSiteService } from '../../../shared/api/rgw-site.service'; import { CdTableColumn } from '../../../shared/models/cd-table-column'; import { CdTableFetchDataContext } from '../../../shared/models/cd-table-fetch-data-context'; import { CdTableSelection } from '../../../shared/models/cd-table-selection'; @@ -15,18 +16,22 @@ import { AuthStorageService } from '../../../shared/services/auth-storage.servic templateUrl: './rgw-daemon-list.component.html', styleUrls: ['./rgw-daemon-list.component.scss'] }) -export class RgwDaemonListComponent { +export class RgwDaemonListComponent implements OnInit { columns: CdTableColumn[] = []; daemons: object[] = []; selection: CdTableSelection = new CdTableSelection(); grafanaPermission: Permission; + isMultiSite: boolean; constructor( private rgwDaemonService: RgwDaemonService, private authStorageService: AuthStorageService, - cephShortVersionPipe: CephShortVersionPipe, - private i18n: I18n - ) { + private cephShortVersionPipe: CephShortVersionPipe, + private i18n: I18n, + private rgwSiteService: RgwSiteService + ) {} + + ngOnInit(): void { this.grafanaPermission = this.authStorageService.getPermissions().grafana; this.columns = [ { @@ -43,9 +48,12 @@ export class RgwDaemonListComponent { name: this.i18n('Version'), prop: 'version', flexGrow: 1, - pipe: cephShortVersionPipe + pipe: this.cephShortVersionPipe } ]; + this.rgwSiteService + .get('realms') + .subscribe((realms: string[]) => (this.isMultiSite = realms.length > 0)); } getDaemonList(context: CdTableFetchDataContext) { diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/api/rgw-site.service.spec.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/rgw-site.service.spec.ts new file mode 100644 index 0000000000000..a059c2f918840 --- /dev/null +++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/rgw-site.service.spec.ts @@ -0,0 +1,40 @@ +import { HttpClientTestingModule, HttpTestingController } from '@angular/common/http/testing'; +import { TestBed } from '@angular/core/testing'; + +import { configureTestBed } from '../../../testing/unit-test-helper'; +import { RgwSiteService } from './rgw-site.service'; + +describe('RgwSiteService', () => { + let service: RgwSiteService; + let httpTesting: HttpTestingController; + + configureTestBed({ + providers: [RgwSiteService], + imports: [HttpClientTestingModule] + }); + + beforeEach(() => { + service = TestBed.get(RgwSiteService); + httpTesting = TestBed.get(HttpTestingController); + }); + + afterEach(() => { + httpTesting.verify(); + }); + + it('should be created', () => { + expect(service).toBeTruthy(); + }); + + it('should contain site endpoint in GET request', () => { + service.get().subscribe(); + const req = httpTesting.expectOne(service['url']); + expect(req.request.method).toBe('GET'); + }); + + it('should add query param in GET request', () => { + const query = 'placement-targets'; + service.get(query).subscribe(); + httpTesting.expectOne(`${service['url']}?query=placement-targets`); + }); +}); diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/api/rgw-site.service.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/rgw-site.service.ts new file mode 100644 index 0000000000000..a0bd328c70443 --- /dev/null +++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/rgw-site.service.ts @@ -0,0 +1,24 @@ +import { HttpClient, HttpParams } from '@angular/common/http'; +import { Injectable } from '@angular/core'; + +import { cdEncode } from '../decorators/cd-encode'; +import { ApiModule } from './api.module'; + +@cdEncode +@Injectable({ + providedIn: ApiModule +}) +export class RgwSiteService { + private url = 'api/rgw/site'; + + constructor(private http: HttpClient) {} + + get(query?: string) { + let params = new HttpParams(); + if (query) { + params = params.append('query', query); + } + + return this.http.get(this.url, { params: params }); + } +} diff --git a/src/pybind/mgr/dashboard/module.py b/src/pybind/mgr/dashboard/module.py index fa4fe3cdc1361..a0e51a2765109 100644 --- a/src/pybind/mgr/dashboard/module.py +++ b/src/pybind/mgr/dashboard/module.py @@ -20,7 +20,7 @@ from OpenSSL import crypto from mgr_module import MgrModule, MgrStandbyModule, Option, CLIWriteCommand from mgr_util import get_default_addr, ServerConfigException, verify_tls_files -import _strptime # pylint: disable=unused-import +import _strptime # pylint: disable=unused-import,wrong-import-order try: import cherrypy diff --git a/src/pybind/mgr/dashboard/services/rgw_client.py b/src/pybind/mgr/dashboard/services/rgw_client.py index eef11794091b3..d3e00868fed7a 100644 --- a/src/pybind/mgr/dashboard/services/rgw_client.py +++ b/src/pybind/mgr/dashboard/services/rgw_client.py @@ -6,7 +6,7 @@ from distutils.util import strtobool from ..awsauth import S3Auth from ..settings import Settings, Options from ..rest_client import RestClient, RequestException -from ..tools import build_url, dict_contains_path, is_valid_ip_address +from ..tools import build_url, dict_contains_path, is_valid_ip_address, json_str_to_object from .. import mgr, logger @@ -227,6 +227,9 @@ class RgwClient(RestClient): # Append the instance to the internal map. RgwClient._user_instances[RgwClient._SYSTEM_USERID] = instance + def _get_realms_info(self): # type: () -> dict + return json_str_to_object(self.proxy('GET', 'realm?list', None, None)) + @staticmethod def _rgw_settings(): return (Settings.RGW_API_HOST, @@ -435,3 +438,10 @@ class RgwClient(RestClient): def create_bucket(self, bucket_name, request=None): logger.info("Creating bucket: %s", bucket_name) return request() + + def get_realms(self): # type: () -> List + realms_info = self._get_realms_info() + if 'realms' in realms_info and realms_info['realms']: + return realms_info['realms'] + + return [] diff --git a/src/pybind/mgr/dashboard/tests/test_rgw_client.py b/src/pybind/mgr/dashboard/tests/test_rgw_client.py index 0824665f9d343..0b27a7eab4481 100644 --- a/src/pybind/mgr/dashboard/tests/test_rgw_client.py +++ b/src/pybind/mgr/dashboard/tests/test_rgw_client.py @@ -2,6 +2,11 @@ # pylint: disable=too-many-public-methods import unittest +try: + from unittest.mock import patch +except ImportError: + from mock import patch # type: ignore + from ..services.rgw_client import RgwClient, _parse_frontend_config from ..settings import Settings from . import KVStoreMockMixin @@ -28,6 +33,23 @@ class RgwClientTest(unittest.TestCase, KVStoreMockMixin): instance = RgwClient.admin_instance() self.assertFalse(instance.session.verify) + @patch.object(RgwClient, '_get_realms_info') + def test_get_realms(self, realms_info): + realms_info.side_effect = [ + { + 'default_info': '51de8373-bc24-4f74-a9b7-8e9ef4cb71f7', + 'realms': [ + 'realm1', + 'realm2' + ] + }, + {} + ] + instance = RgwClient.admin_instance() + + self.assertEqual(['realm1', 'realm2'], instance.get_realms()) + self.assertEqual([], instance.get_realms()) + class RgwClientHelperTest(unittest.TestCase): def test_parse_frontend_config_1(self): diff --git a/src/pybind/mgr/dashboard/tools.py b/src/pybind/mgr/dashboard/tools.py index 82a82194aa0ed..56b1425da4f06 100644 --- a/src/pybind/mgr/dashboard/tools.py +++ b/src/pybind/mgr/dashboard/tools.py @@ -880,6 +880,26 @@ def str_to_bool(val): return bool(strtobool(val)) +def json_str_to_object(value): # type: (AnyStr) -> Any + """ + It converts a JSON valid string representation to object. + + >>> result = json_str_to_object('{"a": 1}') + >>> result == {'a': 1} + True + """ + if value == '': + return value + + try: + # json.loads accepts binary input from version >=3.6 + value = value.decode('utf-8') # type: ignore + except AttributeError: + pass + + return json.loads(value) + + def get_request_body_params(request): """ Helper function to get parameters from the request body. diff --git a/src/pybind/mgr/prometheus/module.py b/src/pybind/mgr/prometheus/module.py index 16fd6f7fdc781..142c88847eef5 100644 --- a/src/pybind/mgr/prometheus/module.py +++ b/src/pybind/mgr/prometheus/module.py @@ -1033,6 +1033,33 @@ class Module(MgrModule): del self.rbd_stats['query'] self.rbd_stats['pools'].clear() + def add_fixed_name_metrics(self): + """ + Add fixed name metrics from existing ones that have details in their names + that should be in labels (not in name). + For backward compatibility, a new fixed name metric is created (instead of replacing) + and details are put in new labels. + Intended for RGW sync perf. counters but extendable as required. + See: https://tracker.ceph.com/issues/45311 + """ + new_metrics = {} + for metric_path in self.metrics.keys(): + # Address RGW sync perf. counters. + match = re.search('^data-sync-from-(.*)\.', metric_path) + if match: + new_path = re.sub('from-([^.]*)', 'from-zone', metric_path) + if new_path not in new_metrics: + new_metrics[new_path] = Metric( + self.metrics[metric_path].mtype, + new_path, + self.metrics[metric_path].desc, + self.metrics[metric_path].labelnames + ('source_zone',) + ) + for label_values, value in self.metrics[metric_path].value.items(): + new_metrics[new_path].set(value, label_values + (match.group(1),)) + + self.metrics.update(new_metrics) + def collect(self): # Clear the metrics before scraping for k in self.metrics.keys(): @@ -1097,6 +1124,7 @@ class Module(MgrModule): ) self.metrics[path].set(value, labels) + self.add_fixed_name_metrics() self.get_rbd_stats() _end_time = time.time() -- 2.39.5