--- /dev/null
+export interface Bucket {
+ bucket: string;
+ tenant: string;
+ versioning: string;
+ zonegroup: string;
+ placement_rule: string;
+ explicit_placement: {
+ data_pool: string;
+ data_extra_pool: string;
+ index_pool: string;
+ };
+ id: string;
+ marker: string;
+ index_type: string;
+ index_generation: number;
+ num_shards: number;
+ reshard_status: string;
+ judge_reshard_lock_time: string;
+ object_lock_enabled: boolean;
+ mfa_enabled: boolean;
+ owner: string;
+ ver: string;
+ master_ver: string;
+ mtime: string;
+ creation_time: string;
+ max_marker: string;
+ usage: Record<string, any>;
+ bucket_quota: {
+ enabled: boolean;
+ check_on_raw: boolean;
+ max_size: number;
+ max_size_kb: number;
+ max_objects: number;
+ };
+ read_tracker: number;
+ bid: string;
+}
-import { Component, NgZone, OnInit, TemplateRef, ViewChild } from '@angular/core';
+import { Component, NgZone, OnDestroy, OnInit, TemplateRef, ViewChild } from '@angular/core';
import _ from 'lodash';
-import { forkJoin as observableForkJoin, Observable, Subscriber } from 'rxjs';
+import { forkJoin as observableForkJoin, Observable, Subscriber, Subscription } from 'rxjs';
+import { switchMap } from 'rxjs/operators';
import { RgwBucketService } from '~/app/shared/api/rgw-bucket.service';
import { ListWithDetails } from '~/app/shared/classes/list-with-details.class';
import { ModalCdsService } from '~/app/shared/services/modal-cds.service';
import { TaskWrapperService } from '~/app/shared/services/task-wrapper.service';
import { URLBuilderService } from '~/app/shared/services/url-builder.service';
+import { Bucket } from '../models/rgw-bucket';
const BASE_URL = 'rgw/bucket';
styleUrls: ['./rgw-bucket-list.component.scss'],
providers: [{ provide: URLBuilderService, useValue: new URLBuilderService(BASE_URL) }]
})
-export class RgwBucketListComponent extends ListWithDetails implements OnInit {
+export class RgwBucketListComponent extends ListWithDetails implements OnInit, OnDestroy {
@ViewChild(TableComponent, { static: true })
table: TableComponent;
@ViewChild('bucketSizeTpl', { static: true })
permission: Permission;
tableActions: CdTableAction[];
columns: CdTableColumn[] = [];
- buckets: object[] = [];
+ buckets: Bucket[] = [];
selection: CdTableSelection = new CdTableSelection();
declare staleTimeout: number;
+ private subs: Subscription = new Subscription();
constructor(
private authStorageService: AuthStorageService,
this.setTableRefreshTimeout();
}
- transformBucketData() {
- _.forEach(this.buckets, (bucketKey) => {
- const maxBucketSize = bucketKey['bucket_quota']['max_size'];
- const maxBucketObjects = bucketKey['bucket_quota']['max_objects'];
- bucketKey['bucket_size'] = 0;
- bucketKey['num_objects'] = 0;
- if (!_.isEmpty(bucketKey['usage'])) {
- bucketKey['bucket_size'] = bucketKey['usage']['rgw.main']['size_actual'];
- bucketKey['num_objects'] = bucketKey['usage']['rgw.main']['num_objects'];
- }
- bucketKey['size_usage'] =
- maxBucketSize > 0 ? bucketKey['bucket_size'] / maxBucketSize : undefined;
- bucketKey['object_usage'] =
- maxBucketObjects > 0 ? bucketKey['num_objects'] / maxBucketObjects : undefined;
- });
- }
-
getBucketList(context: CdTableFetchDataContext) {
this.setTableRefreshTimeout();
- this.rgwBucketService.list(true).subscribe(
- (resp: object[]) => {
- this.buckets = resp;
- this.transformBucketData();
- },
- () => {
- context.error();
- }
+ this.subs.add(
+ this.rgwBucketService
+ .fetchAndTransformBuckets()
+ .pipe(switchMap(() => this.rgwBucketService.buckets$))
+ .subscribe({
+ next: (buckets) => {
+ this.buckets = buckets;
+ },
+ error: () => context.error()
+ })
);
}
}
});
}
+
+ ngOnDestroy() {
+ this.subs.unsubscribe();
+ }
}
-import { ComponentFixture, TestBed } from '@angular/core/testing';
-
+import { ComponentFixture, TestBed, fakeAsync, tick } from '@angular/core/testing';
+import { of, BehaviorSubject, combineLatest } from 'rxjs';
import { RgwOverviewDashboardComponent } from './rgw-overview-dashboard.component';
-import { of } from 'rxjs';
+import { HttpClientTestingModule } from '@angular/common/http/testing';
+import { RgwBucketService } from '~/app/shared/api/rgw-bucket.service';
import { RgwDaemonService } from '~/app/shared/api/rgw-daemon.service';
import { RgwDaemon } from '../models/rgw-daemon';
-import { HttpClientTestingModule } from '@angular/common/http/testing';
+import { CardComponent } from '~/app/shared/components/card/card.component';
+import { CardRowComponent } from '~/app/shared/components/card-row/card-row.component';
import { DimlessBinaryPipe } from '~/app/shared/pipes/dimless-binary.pipe';
+import { NO_ERRORS_SCHEMA } from '@angular/core';
import { RgwRealmService } from '~/app/shared/api/rgw-realm.service';
-import { RgwZonegroupService } from '~/app/shared/api/rgw-zonegroup.service';
import { RgwZoneService } from '~/app/shared/api/rgw-zone.service';
-import { RgwBucketService } from '~/app/shared/api/rgw-bucket.service';
-import { HealthService } from '~/app/shared/api/health.service';
-import { CardRowComponent } from '~/app/shared/components/card-row/card-row.component';
-import { CardComponent } from '~/app/shared/components/card/card.component';
-import { NO_ERRORS_SCHEMA } from '@angular/core';
-import { configureTestBed } from '~/testing/unit-test-helper';
+import { RgwZonegroupService } from '~/app/shared/api/rgw-zonegroup.service';
describe('RgwOverviewDashboardComponent', () => {
let component: RgwOverviewDashboardComponent;
let fixture: ComponentFixture<RgwOverviewDashboardComponent>;
+ let listDaemonsSpy: jest.SpyInstance;
+ let listRealmsSpy: jest.SpyInstance;
+ let listZonegroupsSpy: jest.SpyInstance;
+ let listZonesSpy: jest.SpyInstance;
+ let fetchAndTransformBucketsSpy: jest.SpyInstance;
+ let totalBucketsAndUsersSpy: jest.SpyInstance;
+
+ const totalNumObjectsSubject = new BehaviorSubject<number>(290);
+ const totalUsedCapacitySubject = new BehaviorSubject<number>(9338880);
+ const averageObjectSizeSubject = new BehaviorSubject<number>(1280);
+ const bucketsCount = 2;
+ const usersCount = 5;
const daemon: RgwDaemon = {
id: '8000',
service_map_id: '4803',
zones: ['zone4', 'zone5', 'zone6', 'zone7']
};
- const bucketAndUserList = {
- buckets_count: 2,
- users_count: 2
- };
-
- const healthData = {
- total_objects: '290',
- total_pool_bytes_used: 9338880
- };
-
- let listDaemonsSpy: jest.SpyInstance;
- let listZonesSpy: jest.SpyInstance;
- let listZonegroupsSpy: jest.SpyInstance;
- let listRealmsSpy: jest.SpyInstance;
- let listBucketsSpy: jest.SpyInstance;
- let healthDataSpy: jest.SpyInstance;
-
- configureTestBed({
- declarations: [
- RgwOverviewDashboardComponent,
- CardComponent,
- CardRowComponent,
- DimlessBinaryPipe
- ],
- schemas: [NO_ERRORS_SCHEMA],
- imports: [HttpClientTestingModule]
- });
-
beforeEach(() => {
+ TestBed.configureTestingModule({
+ declarations: [
+ RgwOverviewDashboardComponent,
+ CardComponent,
+ CardRowComponent,
+ DimlessBinaryPipe
+ ],
+ schemas: [NO_ERRORS_SCHEMA],
+ providers: [
+ { provide: RgwDaemonService, useValue: { list: jest.fn() } },
+ { provide: RgwRealmService, useValue: { list: jest.fn() } },
+ { provide: RgwZonegroupService, useValue: { list: jest.fn() } },
+ { provide: RgwZoneService, useValue: { list: jest.fn() } },
+ {
+ provide: RgwBucketService,
+ useValue: {
+ fetchAndTransformBuckets: jest.fn(),
+ totalNumObjects$: totalNumObjectsSubject.asObservable(),
+ totalUsedCapacity$: totalUsedCapacitySubject.asObservable(),
+ averageObjectSize$: averageObjectSizeSubject.asObservable(),
+ getTotalBucketsAndUsersLength: jest.fn()
+ }
+ }
+ ],
+ imports: [HttpClientTestingModule]
+ }).compileComponents();
+ fixture = TestBed.createComponent(RgwOverviewDashboardComponent);
+ component = fixture.componentInstance;
listDaemonsSpy = jest
.spyOn(TestBed.inject(RgwDaemonService), 'list')
.mockReturnValue(of([daemon]));
+ fetchAndTransformBucketsSpy = jest
+ .spyOn(TestBed.inject(RgwBucketService), 'fetchAndTransformBuckets')
+ .mockReturnValue(of(null));
+ totalBucketsAndUsersSpy = jest
+ .spyOn(TestBed.inject(RgwBucketService), 'getTotalBucketsAndUsersLength')
+ .mockReturnValue(of({ buckets_count: bucketsCount, users_count: usersCount }));
listRealmsSpy = jest
.spyOn(TestBed.inject(RgwRealmService), 'list')
.mockReturnValue(of(realmList));
.spyOn(TestBed.inject(RgwZonegroupService), 'list')
.mockReturnValue(of(zonegroupList));
listZonesSpy = jest.spyOn(TestBed.inject(RgwZoneService), 'list').mockReturnValue(of(zoneList));
- listBucketsSpy = jest
- .spyOn(TestBed.inject(RgwBucketService), 'getTotalBucketsAndUsersLength')
- .mockReturnValue(of(bucketAndUserList));
- healthDataSpy = jest
- .spyOn(TestBed.inject(HealthService), 'getClusterCapacity')
- .mockReturnValue(of(healthData));
- fixture = TestBed.createComponent(RgwOverviewDashboardComponent);
- component = fixture.componentInstance;
fixture.detectChanges();
});
- it('should create', () => {
+ it('should create the component', () => {
expect(component).toBeTruthy();
});
it('should render all cards', () => {
- fixture.detectChanges();
const dashboardCards = fixture.debugElement.nativeElement.querySelectorAll('cd-card');
expect(dashboardCards.length).toBe(5);
});
- it('should get corresponding data into Daemons', () => {
- expect(listDaemonsSpy).toHaveBeenCalled();
- expect(component.rgwDaemonCount).toEqual(1);
- });
-
- it('should get corresponding data into Realms', () => {
+ it('should get data for Realms', () => {
expect(listRealmsSpy).toHaveBeenCalled();
expect(component.rgwRealmCount).toEqual(2);
});
- it('should get corresponding data into Zonegroups', () => {
+ it('should get data for Zonegroups', () => {
expect(listZonegroupsSpy).toHaveBeenCalled();
expect(component.rgwZonegroupCount).toEqual(3);
});
- it('should get corresponding data into Zones', () => {
+ it('should get data for Zones', () => {
expect(listZonesSpy).toHaveBeenCalled();
expect(component.rgwZoneCount).toEqual(4);
});
- it('should get corresponding data into Buckets', () => {
- expect(listBucketsSpy).toHaveBeenCalled();
- expect(component.rgwBucketCount).toEqual(2);
- expect(component.UserCount).toEqual(2);
- });
-
- it('should get corresponding data into Objects and capacity', () => {
- expect(healthDataSpy).toHaveBeenCalled();
- expect(component.objectCount).toEqual('290');
+ it('should set component properties from services using combineLatest', fakeAsync(() => {
+ component.interval = of(null).subscribe(() => {
+ component.fetchDataSub = combineLatest([
+ TestBed.inject(RgwDaemonService).list(),
+ TestBed.inject(RgwBucketService).fetchAndTransformBuckets(),
+ totalNumObjectsSubject.asObservable(),
+ totalUsedCapacitySubject.asObservable(),
+ averageObjectSizeSubject.asObservable(),
+ TestBed.inject(RgwBucketService).getTotalBucketsAndUsersLength()
+ ]).subscribe(([daemonData, _, objectCount, usedCapacity, averageSize, bucketData]) => {
+ component.rgwDaemonCount = daemonData.length;
+ component.objectCount = objectCount;
+ component.totalPoolUsedBytes = usedCapacity;
+ component.averageObjectSize = averageSize;
+ component.rgwBucketCount = bucketData.buckets_count;
+ component.UserCount = bucketData.users_count;
+ });
+ });
+ tick();
+ expect(listDaemonsSpy).toHaveBeenCalled();
+ expect(fetchAndTransformBucketsSpy).toHaveBeenCalled();
+ expect(totalBucketsAndUsersSpy).toHaveBeenCalled();
+ expect(component.rgwDaemonCount).toEqual(1);
+ expect(component.objectCount).toEqual(290);
expect(component.totalPoolUsedBytes).toEqual(9338880);
- });
+ expect(component.averageObjectSize).toEqual(1280);
+ expect(component.rgwBucketCount).toEqual(bucketsCount);
+ expect(component.UserCount).toEqual(usersCount);
+ }));
});
import { Component, OnDestroy, OnInit } from '@angular/core';
import _ from 'lodash';
-import { Observable, ReplaySubject, Subscription, of } from 'rxjs';
+import { Observable, ReplaySubject, Subscription, combineLatest, of } from 'rxjs';
import { Permissions } from '~/app/shared/models/permissions';
import { AuthStorageService } from '~/app/shared/services/auth-storage.service';
import { PrometheusService } from '~/app/shared/api/prometheus.service';
import { RgwPromqls as queries } from '~/app/shared/enum/dashboard-promqls.enum';
-import { HealthService } from '~/app/shared/api/health.service';
import { Icons } from '~/app/shared/enum/icons.enum';
import { RgwMultisiteService } from '~/app/shared/api/rgw-multisite.service';
import { catchError, shareReplay, switchMap, tap } from 'rxjs/operators';
totalPoolUsedBytes = 0;
averageObjectSize = 0;
realmData: any;
- daemonSub: Subscription;
realmSub: Subscription;
multisiteInfo: object[] = [];
ZonegroupSub: Subscription;
ZoneSUb: Subscription;
- HealthSub: Subscription;
- BucketSub: Subscription;
queriesResults: { [key: string]: [] } = {
RGW_REQUEST_PER_SECOND: [],
BANDWIDTH: [],
multisiteSyncStatus$: Observable<any>;
subject = new ReplaySubject<any>();
syncCardLoading = true;
+ fetchDataSub: Subscription;
constructor(
private authStorageService: AuthStorageService,
- private healthService: HealthService,
private refreshIntervalService: RefreshIntervalService,
private rgwDaemonService: RgwDaemonService,
private rgwRealmService: RgwRealmService,
ngOnInit() {
this.interval = this.refreshIntervalService.intervalData$.subscribe(() => {
- this.daemonSub = this.rgwDaemonService.list().subscribe((data: any) => {
- this.rgwDaemonCount = data.length;
- });
- this.HealthSub = this.healthService.getClusterCapacity().subscribe((data: any) => {
- this.objectCount = data['total_objects'];
- this.totalPoolUsedBytes = data['total_pool_bytes_used'];
- this.averageObjectSize = data['average_object_size'];
- });
- setTimeout(() => {
+ this.fetchDataSub = combineLatest([
+ this.rgwDaemonService.list(),
+ this.rgwBucketService.fetchAndTransformBuckets(),
+ this.rgwBucketService.totalNumObjects$,
+ this.rgwBucketService.totalUsedCapacity$,
+ this.rgwBucketService.averageObjectSize$,
+ this.rgwBucketService.getTotalBucketsAndUsersLength()
+ ]).subscribe(([daemonData, _, objectCount, usedCapacity, averageSize, bucketData]) => {
+ this.rgwDaemonCount = daemonData.length;
+ this.objectCount = objectCount;
+ this.totalPoolUsedBytes = usedCapacity;
+ this.averageObjectSize = averageSize;
+ this.rgwBucketCount = bucketData.buckets_count;
+ this.UserCount = bucketData.users_count;
this.getSyncStatus();
});
});
- this.BucketSub = this.rgwBucketService
- .getTotalBucketsAndUsersLength()
- .subscribe((data: any) => {
- this.rgwBucketCount = data['buckets_count'];
- this.UserCount = data['users_count'];
- });
this.realmSub = this.rgwRealmService.list().subscribe((data: any) => {
this.rgwRealmCount = data['realms'].length;
});
}
ngOnDestroy() {
- this.interval.unsubscribe();
- this.daemonSub.unsubscribe();
- this.realmSub.unsubscribe();
- this.ZonegroupSub.unsubscribe();
- this.ZoneSUb.unsubscribe();
- this.BucketSub.unsubscribe();
- this.HealthSub.unsubscribe();
- this.prometheusService.unsubscribe();
+ this.interval?.unsubscribe();
+ this.realmSub?.unsubscribe();
+ this.ZonegroupSub?.unsubscribe();
+ this.ZoneSUb?.unsubscribe();
+ this.fetchDataSub?.unsubscribe();
+ this.prometheusService?.unsubscribe();
}
getPrometheusData(selectedTime: any) {
import { Injectable } from '@angular/core';
import _ from 'lodash';
-import { of as observableOf } from 'rxjs';
-import { catchError, mapTo } from 'rxjs/operators';
+import { BehaviorSubject, of as observableOf } from 'rxjs';
+import { catchError, map, mapTo } from 'rxjs/operators';
+import { Bucket } from '~/app/ceph/rgw/models/rgw-bucket';
import { ApiClient } from '~/app/shared/api/api-client';
import { RgwDaemonService } from '~/app/shared/api/rgw-daemon.service';
})
export class RgwBucketService extends ApiClient {
private url = 'api/rgw/bucket';
+ private bucketsSubject = new BehaviorSubject<Bucket[]>([]);
+ private totalNumObjectsSubject = new BehaviorSubject<number>(0);
+ private totalUsedCapacitySubject = new BehaviorSubject<number>(0);
+ private averageObjectSizeSubject = new BehaviorSubject<number>(0);
+ buckets$ = this.bucketsSubject.asObservable();
+ totalNumObjects$ = this.totalNumObjectsSubject.asObservable();
+ totalUsedCapacity$ = this.totalUsedCapacitySubject.asObservable();
+ averageObjectSize$ = this.averageObjectSizeSubject.asObservable();
constructor(private http: HttpClient, private rgwDaemonService: RgwDaemonService) {
super();
}
+ fetchAndTransformBuckets() {
+ return this.list(true).pipe(
+ map((buckets: Bucket[]) => {
+ let totalNumObjects = 0;
+ let totalUsedCapacity = 0;
+ let averageObjectSize = 0;
+ const transformedBuckets = buckets.map((bucket) => this.transformBucket(bucket));
+ transformedBuckets.forEach((bucket) => {
+ totalNumObjects += bucket?.num_objects || 0;
+ totalUsedCapacity += bucket?.bucket_size || 0;
+ });
+ averageObjectSize = this.calculateAverageObjectSize(totalNumObjects, totalUsedCapacity);
+ this.bucketsSubject.next(transformedBuckets);
+ this.totalNumObjectsSubject.next(totalNumObjects);
+ this.totalUsedCapacitySubject.next(totalUsedCapacity);
+ this.averageObjectSizeSubject.next(averageObjectSize);
+ })
+ );
+ }
+
+ transformBucket(bucket: Bucket) {
+ const maxBucketSize = bucket?.bucket_quota?.max_size ?? 0;
+ const maxBucketObjects = bucket?.bucket_quota?.max_objects ?? 0;
+ const bucket_size = bucket['usage']?.['rgw.main']?.['size_actual'] || 0;
+ const num_objects = bucket['usage']?.['rgw.main']?.['num_objects'] || 0;
+ return {
+ ...bucket,
+ bucket_size,
+ num_objects,
+ size_usage: this.calculateSizeUsage(bucket_size, maxBucketSize),
+ object_usage: this.calculateObjectUsage(num_objects, maxBucketObjects)
+ };
+ }
+
+ calculateSizeUsage(bucket_size: number, maxBucketSize: number) {
+ return maxBucketSize > 0 ? bucket_size / maxBucketSize : undefined;
+ }
+
+ calculateObjectUsage(num_objects: number, maxBucketObjects: number) {
+ return maxBucketObjects > 0 ? num_objects / maxBucketObjects : undefined;
+ }
+
+ calculateAverageObjectSize(totalNumObjects: number, totalUsedCapacity: number) {
+ return totalNumObjects > 0 ? totalUsedCapacity / totalNumObjects : 0;
+ }
+
/**
* Get the list of buckets.
* @return Observable<Object[]>
total_avail_bytes: int
total_bytes: int
total_used_raw_bytes: int
- total_objects: int
- total_pool_bytes_used: int
- average_object_size: int
class ClusterModel:
@classmethod
def get_capacity(cls) -> ClusterCapacity:
df = mgr.get('df')
- total_pool_bytes_used = 0
- average_object_size = 0
- total_data_pool_objects = 0
- total_data_pool_bytes_used = 0
- rgw_pools_data = cls.get_rgw_pools()
-
- for pool in df['pools']:
- pool_name = str(pool['name'])
- if pool_name in rgw_pools_data:
- if pool_name.endswith('.data'):
- objects = pool['stats']['objects']
- pool_bytes_used = pool['stats']['bytes_used']
- total_pool_bytes_used += pool_bytes_used
- total_data_pool_objects += objects
- replica = rgw_pools_data[pool_name]
- total_data_pool_bytes_used += pool_bytes_used / replica
-
- average_object_size = total_data_pool_bytes_used / total_data_pool_objects if total_data_pool_objects != 0 else 0 # noqa E501 #pylint: disable=line-too-long
-
- return ClusterCapacity(
- total_avail_bytes=df['stats']['total_avail_bytes'],
- total_bytes=df['stats']['total_bytes'],
- total_used_raw_bytes=df['stats']['total_used_raw_bytes'],
- total_objects=total_data_pool_objects,
- total_pool_bytes_used=total_pool_bytes_used,
- average_object_size=average_object_size
- )._asdict()
+ return ClusterCapacity(total_avail_bytes=df['stats']['total_avail_bytes'],
+ total_bytes=df['stats']['total_bytes'],
+ total_used_raw_bytes=df['stats']['total_used_raw_bytes'])._asdict()
@classmethod
def get_rgw_pools(cls):
projects / ceph-ci.git / commitdiff