From 8142ceebf147432246b103119a719c810af66c66 Mon Sep 17 00:00:00 2001 From: =?utf8?q?Alfonso=20Mart=C3=ADnez?= Date: Mon, 21 Sep 2020 15:59:38 +0200 Subject: [PATCH] nautilus: mgr/dashboard: fix perf. issue when listing large amounts of buckets MIME-Version: 1.0 Content-Type: text/plain; charset=utf8 Content-Transfer-Encoding: 8bit NOTE: Due to base code divergence between master (pacific) & nautilus, This is a dedicated fix for nautilus. Fixes: https://tracker.ceph.com/issues/47618 Signed-off-by: Alfonso Martínez --- qa/tasks/mgr/dashboard/test_rgw.py | 17 ++++++++++- src/pybind/mgr/dashboard/controllers/rgw.py | 10 +++++-- .../app/shared/api/rgw-bucket.service.spec.ts | 29 ++----------------- .../src/app/shared/api/rgw-bucket.service.ts | 17 +++-------- 4 files changed, 31 insertions(+), 42 deletions(-) diff --git a/qa/tasks/mgr/dashboard/test_rgw.py b/qa/tasks/mgr/dashboard/test_rgw.py index 5c41b207e11f0..dadec1091634e 100644 --- a/qa/tasks/mgr/dashboard/test_rgw.py +++ b/qa/tasks/mgr/dashboard/test_rgw.py @@ -164,6 +164,20 @@ class RgwBucketTest(RgwTestCase): self.assertEqual(len(data), 1) self.assertIn('teuth-test-bucket', data) + # List all buckets with stats. + data = self._get('/api/rgw/bucket?stats=true') + self.assertStatus(200) + self.assertEqual(len(data), 1) + self.assertSchema(data[0], JObj(sub_elems={ + 'bid': JLeaf(str), + 'bucket': JLeaf(str), + 'bucket_quota': JObj(sub_elems={}, allow_unknown=True), + 'id': JLeaf(str), + 'owner': JLeaf(str), + 'usage': JObj(sub_elems={}, allow_unknown=True), + 'tenant': JLeaf(str), + }, allow_unknown=True)) + # Get the bucket. data = self._get('/api/rgw/bucket/teuth-test-bucket') self.assertStatus(200) @@ -173,7 +187,8 @@ class RgwBucketTest(RgwTestCase): 'tenant': JLeaf(str), 'bucket': JLeaf(str), 'bucket_quota': JObj(sub_elems={}, allow_unknown=True), - 'owner': JLeaf(str) + 'owner': JLeaf(str), + 'usage': JObj(sub_elems={}, allow_unknown=True), }, allow_unknown=True)) self.assertEqual(data['bucket'], 'teuth-test-bucket') self.assertEqual(data['owner'], 'admin') diff --git a/src/pybind/mgr/dashboard/controllers/rgw.py b/src/pybind/mgr/dashboard/controllers/rgw.py index 2a53f368352c6..2205277c26400 100644 --- a/src/pybind/mgr/dashboard/controllers/rgw.py +++ b/src/pybind/mgr/dashboard/controllers/rgw.py @@ -142,8 +142,14 @@ class RgwBucket(RgwRESTController): return bucket_name - def list(self): - return self.proxy('GET', 'bucket') + def list(self, stats=False): + query_params = '?stats' if stats else '' + result = self.proxy('GET', 'bucket{}'.format(query_params)) + + if stats: + result = [self._append_bid(bucket) for bucket in result] + + return result def get(self, bucket): result = self.proxy('GET', 'bucket', {'bucket': bucket}) diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/api/rgw-bucket.service.spec.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/rgw-bucket.service.spec.ts index 0aed3268137cd..9951dfeb93e7b 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/shared/api/rgw-bucket.service.spec.ts +++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/rgw-bucket.service.spec.ts @@ -26,33 +26,10 @@ describe('RgwBucketService', () => { expect(service).toBeTruthy(); }); - it('should call list, with enumerate returning empty', () => { - let result; - service.list().subscribe((resp) => { - result = resp; - }); - const req = httpTesting.expectOne('api/rgw/bucket'); - req.flush([]); - expect(req.request.method).toBe('GET'); - expect(result).toEqual([]); - }); - - it('should call list, with enumerate returning 2 elements', () => { - let result; - service.list().subscribe((resp) => { - result = resp; - }); - let req = httpTesting.expectOne('api/rgw/bucket'); - req.flush(['foo', 'bar']); - - req = httpTesting.expectOne('api/rgw/bucket/foo'); - req.flush({ name: 'foo' }); - - req = httpTesting.expectOne('api/rgw/bucket/bar'); - req.flush({ name: 'bar' }); - + it('should call list', () => { + service.list().subscribe(); + const req = httpTesting.expectOne('api/rgw/bucket?stats=true'); expect(req.request.method).toBe('GET'); - expect(result).toEqual([{ name: 'foo' }, { name: 'bar' }]); }); it('should call get', () => { diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/api/rgw-bucket.service.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/rgw-bucket.service.ts index fdd94672dfa95..b23b1999fdc73 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/shared/api/rgw-bucket.service.ts +++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/rgw-bucket.service.ts @@ -2,7 +2,7 @@ import { HttpClient, HttpParams } from '@angular/common/http'; import { Injectable } from '@angular/core'; import * as _ from 'lodash'; -import { forkJoin as observableForkJoin, of as observableOf } from 'rxjs'; +import { of as observableOf } from 'rxjs'; import { mergeMap } from 'rxjs/operators'; import { cdEncode } from '../decorators/cd-encode'; @@ -22,18 +22,9 @@ export class RgwBucketService { * @return {Observable} */ list() { - return this.enumerate().pipe( - mergeMap((buckets: string[]) => { - if (buckets.length > 0) { - return observableForkJoin( - buckets.map((bucket: string) => { - return this.get(bucket); - }) - ); - } - return observableOf([]); - }) - ); + let params = new HttpParams(); + params = params.append('stats', 'true'); + return this.http.get(this.url, { params: params }); } /** -- 2.39.5