mgr/dashboard: fix performance issue when listing large amounts of buckets

Fixes: https://tracker.ceph.com/issues/47543
Signed-off-by: Alfonso Martínez <almartin@redhat.com>

diff --git a/qa/tasks/mgr/dashboard/test_rgw.py b/qa/tasks/mgr/dashboard/test_rgw.py
index 78a4d56ba00e416eb2b87e326d047cd0b9435bd4..8b4fe2a6755d8d43f8571f9d79e0f305aee71d35 100644 (file)
--- a/qa/tasks/mgr/dashboard/test_rgw.py
+++ b/qa/tasks/mgr/dashboard/test_rgw.py
@@ -210,6 +210,20 @@ class RgwBucketTest(RgwTestCase):
         self.assertEqual(len(data), 1)
         self.assertIn('teuth-test-bucket', data)
 
+        # List all buckets with stats.
+        data = self._get('/api/rgw/bucket?stats=true')
+        self.assertStatus(200)
+        self.assertEqual(len(data), 1)
+        self.assertSchema(data[0], JObj(sub_elems={
+            'bid': JLeaf(str),
+            'bucket': JLeaf(str),
+            'bucket_quota': JObj(sub_elems={}, allow_unknown=True),
+            'id': JLeaf(str),
+            'owner': JLeaf(str),
+            'usage': JObj(sub_elems={}, allow_unknown=True),
+            'tenant': JLeaf(str),
+        }, allow_unknown=True))
+
         # Get the bucket.
         data = self._get('/api/rgw/bucket/teuth-test-bucket')
         self.assertStatus(200)
@@ -219,7 +233,10 @@ class RgwBucketTest(RgwTestCase):
             'tenant': JLeaf(str),
             'bucket': JLeaf(str),
             'bucket_quota': JObj(sub_elems={}, allow_unknown=True),
-            'owner': JLeaf(str)
+            'owner': JLeaf(str),
+            'mfa_delete': JLeaf(str),
+            'usage': JObj(sub_elems={}, allow_unknown=True),
+            'versioning': JLeaf(str)
         }, allow_unknown=True))
         self.assertEqual(data['bucket'], 'teuth-test-bucket')
         self.assertEqual(data['owner'], 'admin')

diff --git a/src/pybind/mgr/dashboard/controllers/rgw.py b/src/pybind/mgr/dashboard/controllers/rgw.py
index 8b8eaba2faa424f6a43f4257e04a91f6471b542a..d3fcb9163c1c31e756a221e33fb0cc724fa3cad0 100644 (file)
--- a/src/pybind/mgr/dashboard/controllers/rgw.py
+++ b/src/pybind/mgr/dashboard/controllers/rgw.py
@@ -16,7 +16,7 @@ from ..services.rgw_client import RgwClient
 from ..tools import json_str_to_object, str_to_bool
 
 try:
-    from typing import List
+    from typing import Any, List
 except ImportError:  # pragma: no cover
     pass  # Just for type checking
 
@@ -215,9 +215,15 @@ class RgwBucket(RgwRESTController):
             bucket_name = '{}:{}'.format(tenant, bucket_name)
         return bucket_name
 
-    def list(self):
-        # type: () -> List[str]
-        return self.proxy('GET', 'bucket')
+    def list(self, stats=False):
+        # type: (bool) -> List[Any]
+        query_params = '?stats' if stats else ''
+        result = self.proxy('GET', 'bucket{}'.format(query_params))
+
+        if stats:
+            result = [self._append_bid(bucket) for bucket in result]
+
+        return result
 
     def get(self, bucket):
         # type: (str) -> dict

diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-bucket-details/rgw-bucket-details.component.spec.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-bucket-details/rgw-bucket-details.component.spec.ts
index 60f2c67fa90bebe0e81e4b82a4592daeff67fc55..dd6ccf4cd6d42232782f3cb46ea95225ba131dc2 100644 (file)
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-bucket-details/rgw-bucket-details.component.spec.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-bucket-details/rgw-bucket-details.component.spec.ts
@@ -1,6 +1,10 @@
+import { HttpClientTestingModule } from '@angular/common/http/testing';
 import { ComponentFixture, TestBed } from '@angular/core/testing';
 
+import { of } from 'rxjs';
+
 import { configureTestBed } from '../../../../testing/unit-test-helper';
+import { RgwBucketService } from '../../../shared/api/rgw-bucket.service';
 import { CdTableSelection } from '../../../shared/models/cd-table-selection';
 import { SharedModule } from '../../../shared/shared.module';
 import { RgwBucketDetailsComponent } from './rgw-bucket-details.component';
@@ -8,13 +12,18 @@ import { RgwBucketDetailsComponent } from './rgw-bucket-details.component';
 describe('RgwBucketDetailsComponent', () => {
   let component: RgwBucketDetailsComponent;
   let fixture: ComponentFixture<RgwBucketDetailsComponent>;
+  let rgwBucketService: RgwBucketService;
+  let rgwBucketServiceGetSpy: jasmine.Spy;
 
   configureTestBed({
     declarations: [RgwBucketDetailsComponent],
-    imports: [SharedModule]
+    imports: [SharedModule, HttpClientTestingModule]
   });
 
   beforeEach(() => {
+    rgwBucketService = TestBed.inject(RgwBucketService);
+    rgwBucketServiceGetSpy = spyOn(rgwBucketService, 'get');
+    rgwBucketServiceGetSpy.and.returnValue(of(null));
     fixture = TestBed.createComponent(RgwBucketDetailsComponent);
     component = fixture.componentInstance;
     component.selection = new CdTableSelection();
@@ -24,4 +33,10 @@ describe('RgwBucketDetailsComponent', () => {
   it('should create', () => {
     expect(component).toBeTruthy();
   });
+
+  it('should retrieve bucket full info', () => {
+    component.selection = { bid: 'bucket' };
+    component.ngOnChanges();
+    expect(rgwBucketServiceGetSpy).toHaveBeenCalled();
+  });
 });

diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-bucket-details/rgw-bucket-details.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-bucket-details/rgw-bucket-details.component.ts
index 63c93492008463c991eff3a8466e5de8b51057cf..adf0b13d310da9661d006e0e28a4236c7ad27dd5 100644 (file)
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-bucket-details/rgw-bucket-details.component.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-bucket-details/rgw-bucket-details.component.ts
@@ -1,11 +1,23 @@
-import { Component, Input } from '@angular/core';
+import { Component, Input, OnChanges } from '@angular/core';
+
+import { RgwBucketService } from '../../../shared/api/rgw-bucket.service';
 
 @Component({
   selector: 'cd-rgw-bucket-details',
   templateUrl: './rgw-bucket-details.component.html',
   styleUrls: ['./rgw-bucket-details.component.scss']
 })
-export class RgwBucketDetailsComponent {
+export class RgwBucketDetailsComponent implements OnChanges {
   @Input()
   selection: any;
+
+  constructor(private rgwBucketService: RgwBucketService) {}
+
+  ngOnChanges() {
+    if (this.selection) {
+      this.rgwBucketService.get(this.selection.bid).subscribe((bucket: object) => {
+        this.selection = bucket;
+      });
+    }
+  }
 }

diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/api/rgw-bucket.service.spec.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/rgw-bucket.service.spec.ts
index c7666da4c2ed6f113b7e3f7df68785af908670b3..ff460e65e9cf3f304a2872e055003c30e6292af5 100644 (file)
--- a/src/pybind/mgr/dashboard/frontend/src/app/shared/api/rgw-bucket.service.spec.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/rgw-bucket.service.spec.ts
@@ -26,33 +26,10 @@ describe('RgwBucketService', () => {
     expect(service).toBeTruthy();
   });
 
-  it('should call list, with enumerate returning empty', () => {
-    let result;
-    service.list().subscribe((resp) => {
-      result = resp;
-    });
-    const req = httpTesting.expectOne('api/rgw/bucket');
-    req.flush([]);
-    expect(req.request.method).toBe('GET');
-    expect(result).toEqual([]);
-  });
-
-  it('should call list, with enumerate returning 2 elements', () => {
-    let result;
-    service.list().subscribe((resp) => {
-      result = resp;
-    });
-    let req = httpTesting.expectOne('api/rgw/bucket');
-    req.flush(['foo', 'bar']);
-
-    req = httpTesting.expectOne('api/rgw/bucket/foo');
-    req.flush({ name: 'foo' });
-
-    req = httpTesting.expectOne('api/rgw/bucket/bar');
-    req.flush({ name: 'bar' });
-
+  it('should call list', () => {
+    service.list().subscribe();
+    const req = httpTesting.expectOne('api/rgw/bucket?stats=true');
     expect(req.request.method).toBe('GET');
-    expect(result).toEqual([{ name: 'foo' }, { name: 'bar' }]);
   });
 
   it('should call get', () => {

diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/api/rgw-bucket.service.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/rgw-bucket.service.ts
index 4c8654d4cb954353a6b3de64f65c60a54d6dce90..baeaa3dde4837cfdbf68c5cdf70b575495e0e7c6 100644 (file)
--- a/src/pybind/mgr/dashboard/frontend/src/app/shared/api/rgw-bucket.service.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/rgw-bucket.service.ts
@@ -2,7 +2,7 @@ import { HttpClient, HttpParams } from '@angular/common/http';
 import { Injectable } from '@angular/core';
 
 import _ from 'lodash';
-import { forkJoin as observableForkJoin, of as observableOf } from 'rxjs';
+import { of as observableOf } from 'rxjs';
 import { mergeMap } from 'rxjs/operators';
 
 import { cdEncode } from '../decorators/cd-encode';
@@ -21,18 +21,9 @@ export class RgwBucketService {
    * @return {Observable<Object[]>}
    */
   list() {
-    return this.enumerate().pipe(
-      mergeMap((buckets: string[]) => {
-        if (buckets.length > 0) {
-          return observableForkJoin(
-            buckets.map((bucket: string) => {
-              return this.get(bucket);
-            })
-          );
-        }
-        return observableOf([]);
-      })
-    );
+    let params = new HttpParams();
+    params = params.append('stats', 'true');
+    return this.http.get(this.url, { params: params });
   }
 
   /**