From eeee28c0eb58311625729e1360c2c77fa8bdaec2 Mon Sep 17 00:00:00 2001 From: Pere Diaz Bou Date: Mon, 13 Jun 2022 17:15:45 +0200 Subject: [PATCH] mgr/dashboard: rbd pagination poc Signed-off-by: Pere Diaz Bou (cherry picked from commit efd9a8ba5c9194e89b95aae021f62f3e164f3581) (cherry picked from commit a1fbe19b0aae53bead3d7d4feebc71f4af2eb5a7) Conflicts: src/pybind/mgr/dashboard/frontend/src/app/shared/datatable/table/table.component.ts Add pageinfo import Resolves: rhbz#2125432 --- src/pybind/mgr/dashboard/controllers/rbd.py | 33 ++++---- .../iscsi-target-form.component.ts | 2 +- .../block/rbd-list/rbd-list.component.html | 6 +- .../ceph/block/rbd-list/rbd-list.component.ts | 77 ++++++++++++++++--- .../frontend/src/app/shared/api/rbd.model.ts | 1 + .../src/app/shared/api/rbd.service.ts | 23 +++--- .../datatable/table/table.component.html | 3 + .../shared/datatable/table/table.component.ts | 35 ++++++++- .../models/cd-table-fetch-data-context.ts | 17 ++++ .../src/app/shared/models/cd-table-paging.ts | 20 +++++ .../src/app/shared/models/cd-user-config.ts | 1 + .../cd-table-server-side.service.spec.ts | 16 ++++ .../services/cd-table-server-side.service.ts | 14 ++++ src/pybind/mgr/dashboard/services/rbd.py | 74 ++++++++++++------ 14 files changed, 255 insertions(+), 67 deletions(-) create mode 100644 src/pybind/mgr/dashboard/frontend/src/app/shared/models/cd-table-paging.ts create mode 100644 src/pybind/mgr/dashboard/frontend/src/app/shared/services/cd-table-server-side.service.spec.ts create mode 100644 src/pybind/mgr/dashboard/frontend/src/app/shared/services/cd-table-server-side.service.ts diff --git a/src/pybind/mgr/dashboard/controllers/rbd.py b/src/pybind/mgr/dashboard/controllers/rbd.py index ffdc957456e81..0cfb2752b4d09 100644 --- a/src/pybind/mgr/dashboard/controllers/rbd.py +++ b/src/pybind/mgr/dashboard/controllers/rbd.py @@ -4,6 +4,7 @@ import logging import math +import cherrypy from datetime import datetime from functools import partial @@ -78,31 +79,37 @@ class Rbd(RESTController): ALLOW_DISABLE_FEATURES = {"exclusive-lock", "object-map", "fast-diff", "deep-flatten", "journaling"} - def _rbd_list(self, pool_name=None): + def _rbd_list(self, pool_name=None, offset=0, limit=5): if pool_name: pools = [pool_name] else: pools = [p['pool_name'] for p in CephService.get_pool_list('rbd')] result = [] - for pool in pools: - # pylint: disable=unbalanced-tuple-unpacking - status, value = RbdService.rbd_pool_list(pool) - for i, image in enumerate(value): - value[i]['configuration'] = RbdConfiguration( - pool, image['namespace'], image['name']).list() - result.append({'status': status, 'value': value, 'pool_name': pool}) - return result + images, num_total_images = RbdService.rbd_pool_list(pools, offset=offset, limit=limit) + cherrypy.response.headers['X-Total-Count'] = num_total_images + pool_result = {} + for i, image in enumerate(images): + pool = image['pool'] + if pool not in pool_result: + pool_result[pool] = {'status': 1, 'value': [], 'pool_name': image['pool']} + pool_result[pool]['value'].append(image) + + images[i]['configuration'] = RbdConfiguration( + pool, image['namespace'], image['name']).list() + return list(pool_result.values()) @handle_rbd_error() @handle_rados_error('pool') @EndpointDoc("Display Rbd Images", parameters={ 'pool_name': (str, 'Pool Name'), + 'limit': (int, 'limit'), + 'offset': (int, 'offset'), }, responses={200: RBD_SCHEMA}) - def list(self, pool_name=None): - return self._rbd_list(pool_name) + def list(self, pool_name=None, offset: int = 0, limit: int = 5): + return self._rbd_list(pool_name, offset=offset, limit=limit) @handle_rbd_error() @handle_rados_error('pool') @@ -527,7 +534,7 @@ class RbdNamespace(RESTController): def delete(self, pool_name, namespace): with mgr.rados.open_ioctx(pool_name) as ioctx: # pylint: disable=unbalanced-tuple-unpacking - _, images = RbdService.rbd_pool_list(pool_name, namespace) + images, _ = RbdService.rbd_pool_list([pool_name], namespace=namespace) if images: raise DashboardException( msg='Namespace contains images which must be deleted first', @@ -541,7 +548,7 @@ class RbdNamespace(RESTController): namespaces = self.rbd_inst.namespace_list(ioctx) for namespace in namespaces: # pylint: disable=unbalanced-tuple-unpacking - _, images = RbdService.rbd_pool_list(pool_name, namespace) + images, _ = RbdService.rbd_pool_list([pool_name], namespace=namespace) result.append({ 'namespace': namespace, 'num_images': len(images) if images else 0 diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/block/iscsi-target-form/iscsi-target-form.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/block/iscsi-target-form/iscsi-target-form.component.ts index d84647e91dd62..8b97634f13972 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/block/iscsi-target-form/iscsi-target-form.component.ts +++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/block/iscsi-target-form/iscsi-target-form.component.ts @@ -88,7 +88,7 @@ export class IscsiTargetFormComponent extends CdForm implements OnInit { ngOnInit() { const promises: any[] = [ this.iscsiService.listTargets(), - this.rbdService.list(), + this.rbdService.list({'offset': 0, 'limit': 5}), this.iscsiService.portals(), this.iscsiService.settings(), this.iscsiService.version() diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/block/rbd-list/rbd-list.component.html b/src/pybind/mgr/dashboard/frontend/src/app/ceph/block/rbd-list/rbd-list.component.html index 0f68f2df252cf..7946a7191cead 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/block/rbd-list/rbd-list.component.html +++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/block/rbd-list/rbd-list.component.html @@ -6,12 +6,14 @@ [columns]="columns" identifier="unique_id" [searchableObjects]="true" + [serverSide]="true" + [count]="count" forceIdentifier="true" selectionType="single" [hasDetails]="true" [status]="tableStatus" - [autoReload]="-1" - (fetchData)="taskListService.fetch()" + [autoReload]="5000" + (fetchData)="getRbdImages($event)" (setExpandedRow)="setExpandedRow($event)" (updateSelection)="updateSelection($event)"> @@ -344,15 +348,18 @@ export class RbdListComponent extends ListWithDetails implements OnInit { ].includes(task.name); }; - this.taskListService.init( - () => this.rbdService.list(), - (resp) => this.prepareResponse(resp), - (images) => (this.images = images), - () => this.onFetchError(), - taskFilter, - itemFilter, - this.builders - ); + console.log(taskFilter); + console.log(itemFilter); + console.log(this.builders); + // this.taskListService.init( + // (context: CdTableFetchDataContext) => this.rbdService.list(context.toParams()), + // (resp) => this.prepareResponse(resp), + // (images) => (this.images = images), + // () => this.onFetchError(), + // taskFilter, + // itemFilter, + // this.builders + // ); } onFetchError() { @@ -360,10 +367,22 @@ export class RbdListComponent extends ListWithDetails implements OnInit { this.tableStatus = new TableStatusViewCache(ViewCacheStatus.ValueException); } - prepareResponse(resp: any[]): any[] { - let images: any[] = []; + getRbdImages(context: CdTableFetchDataContext = null){ + if(context !== null) { + this.tableContext = context; + } + console.log(context); + this.rbdService.list(this.tableContext.toParams()). + subscribe((resp: any) => { + console.log(resp); + this.prepareResponse(resp); + }); + } + prepareResponse(resp: any[]): any[] { + let images: any[] = []; const viewCacheStatusMap = {}; + console.log(resp); resp.forEach((pool) => { if (_.isUndefined(viewCacheStatusMap[pool.status])) { viewCacheStatusMap[pool.status] = []; @@ -404,6 +423,13 @@ export class RbdListComponent extends ListWithDetails implements OnInit { } }); + console.group('reponse'); + console.log(resp); + console.log(resp[0].headers); + this.count = CdTableServerSideService.getCount(resp[0]); + console.log(this.count); + this.images = images; + console.groupEnd(); return images; } @@ -626,3 +652,30 @@ export class RbdListComponent extends ListWithDetails implements OnInit { return false; } } + + +/* + for pool in pools + for namespace in namespaces + refs = get_image_refs + for ref in refs: + get_data(ref) + +@ttl_cache(5) +def get_refs(); + joint_refs = [] + for pool in pools + for namespace in namespaces + refs = get_image_refs + for ref in refs: + joint_refs.append(ref) + return joint_refs + +sort(joint_refs) + for ref in joint_refs[offset:offset+limit]: +get_data(ref) + + + + +*/ diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/api/rbd.model.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/rbd.model.ts index af646d8ac61d3..37eb0ee3c7737 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/shared/api/rbd.model.ts +++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/rbd.model.ts @@ -4,6 +4,7 @@ export interface RbdPool { pool_name: string; status: number; value: RbdImage[]; + headers: any; } export interface RbdImage { diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/api/rbd.service.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/rbd.service.ts index 5482f09312286..dbf396b1f5915 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/shared/api/rbd.service.ts +++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/rbd.service.ts @@ -36,27 +36,30 @@ export class RbdService { }); } - get(imageSpec: ImageSpec) { - return this.http.get(`api/block/image/${imageSpec.toStringEncoded()}`); + get(imageSpec: ImageSpec) { + return this.http.get(`api/block/image/${imageSpec.toStringEncoded()}`); } - list() { - return this.http.get('api/block/image').pipe( - map((pools) => - pools.map((pool) => { - pool.value.map((image) => { + list(params: any) { + return this.http.get('api/block/image', {params: params, observe: 'response'}).pipe( + map((response: any) => { + console.log(response); + return response['body'].map((pool: any) => { + pool.value.map((image: any) => { if (!image.configuration) { return image; } - image.configuration.map((option) => + image.configuration.map((option: any) => Object.assign(option, this.rbdConfigurationService.getOptionByName(option.name)) ); return image; }); + pool['headers'] = response.headers; return pool; }) - ) - ); + } + ) + ); } copy(imageSpec: ImageSpec, rbd: any) { diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/datatable/table/table.component.html b/src/pybind/mgr/dashboard/frontend/src/app/shared/datatable/table/table.component.html index b38477c813733..1ba4db6e2c8a8 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/shared/datatable/table/table.component.html +++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/datatable/table/table.component.html @@ -172,7 +172,10 @@ [rowClass]="getRowClass()" [headerHeight]="header ? 'auto' : 0" [footerHeight]="footer ? 'auto' : 0" + [count]="count" [limit]="userConfig.limit > 0 ? userConfig.limit : undefined" + [offset]="userConfig.offset >= 0 ? userConfig.offset : 0" + (page)="changePage($event)" [loadingIndicator]="loadingIndicator" [rowIdentity]="rowIdentity()" [rowHeight]="'auto'"> diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/datatable/table/table.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/datatable/table/table.component.ts index 19ad5bc4e4e46..bcc6f306e6ec5 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/shared/datatable/table/table.component.ts +++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/datatable/table/table.component.ts @@ -35,6 +35,7 @@ import { CdTableColumnFiltersChange } from '~/app/shared/models/cd-table-column- import { CdTableFetchDataContext } from '~/app/shared/models/cd-table-fetch-data-context'; import { CdTableSelection } from '~/app/shared/models/cd-table-selection'; import { CdUserConfig } from '~/app/shared/models/cd-user-config'; +import { PageInfo } from '../../models/cd-table-paging'; @Component({ selector: 'cd-table', @@ -112,7 +113,7 @@ export class TableComponent implements AfterContentChecked, OnInit, OnChanges, O * prevent triggering fetchData when initializing the table. */ @Input() - autoReload: any = 5000; + autoReload: number = 5000; // Which row property is unique for a row. If the identifier is not specified in any // column, then the property name of the first column is used. Defaults to 'id'. @@ -151,6 +152,17 @@ export class TableComponent implements AfterContentChecked, OnInit, OnChanges, O @Input() status = new TableStatus(); + // Support server-side pagination/sorting/etc. + @Input() + serverSide: boolean = false; + + /* + Only required when serverSide is enabled. + It should be provided by the server via "X-Total-Count" HTTP Header + */ + @Input() + count: number = 0; + /** * Should be a function to update the input data if undefined nothing will be triggered * @@ -161,7 +173,7 @@ export class TableComponent implements AfterContentChecked, OnInit, OnChanges, O * The function is triggered through one table and all tables will update */ @Output() - fetchData = new EventEmitter(); + fetchData = new EventEmitter(); /** * This should be defined if you need access to the selection object. @@ -325,6 +337,9 @@ export class TableComponent implements AfterContentChecked, OnInit, OnChanges, O if (!this.userConfig.limit) { this.userConfig.limit = this.limit; } + if (!(this.userConfig.offset >= 0)) { + this.userConfig.offset = this.table.offset; + } if (!this.userConfig.sorts) { this.userConfig.sorts = this.sorts; } @@ -603,11 +618,14 @@ export class TableComponent implements AfterContentChecked, OnInit, OnChanges, O } setLimit(e: any) { - const value = parseInt(e.target.value, 10); + const value = Number(e.target.value); if (value > 0) { this.userConfig.limit = value; } - } + if (this.serverSide) { + this.reloadData(); + } + } reloadData() { if (!this.updating) { @@ -625,6 +643,8 @@ export class TableComponent implements AfterContentChecked, OnInit, OnChanges, O // to the correct state. this.useData(); }); + context.pageInfo.offset = this.userConfig.offset; + context.pageInfo.limit = this.userConfig.limit; this.fetchData.emit(context); this.updating = true; } @@ -635,6 +655,13 @@ export class TableComponent implements AfterContentChecked, OnInit, OnChanges, O this.reloadData(); } + changePage(pageInfo: PageInfo) { + this.userConfig.offset = pageInfo.offset; + this.userConfig.limit = pageInfo.limit; + if (this.serverSide) { + this.reloadData(); + } + } rowIdentity() { return (row: any) => { const id = row[this.identifier]; diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/models/cd-table-fetch-data-context.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/models/cd-table-fetch-data-context.ts index b7c6e672d7d71..dc7a46d1ff927 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/shared/models/cd-table-fetch-data-context.ts +++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/models/cd-table-fetch-data-context.ts @@ -1,3 +1,6 @@ +import { HttpParams } from "@angular/common/http"; +import { PageInfo } from "./cd-table-paging"; + export class CdTableFetchDataContext { errorConfig = { resetData: true, // Force data table to show no data @@ -10,8 +13,22 @@ export class CdTableFetchDataContext { * reset the data table to the correct state. */ error: Function; + pageInfo: PageInfo = new PageInfo; constructor(error: () => void) { this.error = error; } + + toParams(): HttpParams { + if (this.pageInfo.limit === null) { + this.pageInfo.limit = 0; + } + return new HttpParams({ + fromObject: { + offset: String(this.pageInfo.offset * this.pageInfo.limit), + limit: String(this.pageInfo.limit) + } + }); + + } } diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/models/cd-table-paging.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/models/cd-table-paging.ts new file mode 100644 index 0000000000000..c0d5842907a44 --- /dev/null +++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/models/cd-table-paging.ts @@ -0,0 +1,20 @@ +export const PAGE_LIMIT = 10; + +export class PageInfo { + // Total number of rows in a table + count: number; + + // Current page (current row = offset x limit or pageSize) + offset: number = 0; + + // Max. number of rows fetched from the server + limit: number = PAGE_LIMIT; + + /* + pageSize and limit can be decoupled if hybrid server-side and client-side + are used. A use-case would be to reduce the amount of queries: that is, + the pageSize (client-side paging) might be 10, but the back-end queries + could have a limit of 100. That would avoid triggering requests + */ + pageSize: number = PAGE_LIMIT; +} diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/models/cd-user-config.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/models/cd-user-config.ts index 2e1a9e1970a47..3bb426b8dc1f7 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/shared/models/cd-user-config.ts +++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/models/cd-user-config.ts @@ -4,6 +4,7 @@ import { CdTableColumn } from './cd-table-column'; export interface CdUserConfig { limit?: number; + offset?: number; sorts?: SortPropDir[]; columns?: CdTableColumn[]; } diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/services/cd-table-server-side.service.spec.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/services/cd-table-server-side.service.spec.ts new file mode 100644 index 0000000000000..dbe7bb4526e7b --- /dev/null +++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/services/cd-table-server-side.service.spec.ts @@ -0,0 +1,16 @@ +import { TestBed } from '@angular/core/testing'; + +import { CdTableServerSideService } from './cd-table-server-side.service'; + +describe('CdTableServerSideService', () => { + let service: CdTableServerSideService; + + beforeEach(() => { + TestBed.configureTestingModule({}); + service = TestBed.inject(CdTableServerSideService); + }); + + it('should be created', () => { + expect(service).toBeTruthy(); + }); +}); diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/services/cd-table-server-side.service.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/services/cd-table-server-side.service.ts new file mode 100644 index 0000000000000..8e68a25b6cdc2 --- /dev/null +++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/services/cd-table-server-side.service.ts @@ -0,0 +1,14 @@ +import { HttpResponse } from '@angular/common/http'; +import { Injectable } from '@angular/core'; + +@Injectable({ + providedIn: 'root' +}) +export class CdTableServerSideService { + + constructor() { } + + static getCount(resp: HttpResponse): number { + return Number(resp.headers.get('X-Total-Count')) + } +} diff --git a/src/pybind/mgr/dashboard/services/rbd.py b/src/pybind/mgr/dashboard/services/rbd.py index 922da3fbd9106..d64ae52368096 100644 --- a/src/pybind/mgr/dashboard/services/rbd.py +++ b/src/pybind/mgr/dashboard/services/rbd.py @@ -402,34 +402,58 @@ class RbdService(object): errno=errno.ENOENT) @classmethod - @ViewCache() - def rbd_pool_list(cls, pool_name, namespace=None): + def _rbd_pool_image_refs(cls, pool_names: List[str], namespace=None): + joint_refs = [] rbd_inst = rbd.RBD() - with mgr.rados.open_ioctx(pool_name) as ioctx: - result = [] - if namespace: - namespaces = [namespace] - else: - namespaces = rbd_inst.namespace_list(ioctx) - # images without namespace - namespaces.append('') - for current_namespace in namespaces: - ioctx.set_namespace(current_namespace) - image_refs = cls._rbd_image_refs(ioctx) - for image_ref in image_refs: + for pool in pool_names: + with mgr.rados.open_ioctx(pool) as ioctx: + result = [] + if namespace: + namespaces = [namespace] + else: + namespaces = rbd_inst.namespace_list(ioctx) + # images without namespace + namespaces.append('') + for current_namespace in namespaces: + ioctx.set_namespace(current_namespace) + image_refs = cls._rbd_image_refs(ioctx) + for image in image_refs: + image['namespace'] = current_namespace + image['pool'] = pool + joint_refs.append(image) + return joint_refs + + @classmethod + def rbd_pool_list(cls, pool_names: List[str], namespace=None, offset=0, limit=0): + offset = int(offset) + limit = int(limit) + if limit < 0: + return [] + + refs = cls._rbd_pool_image_refs(pool_names, namespace) + image_refs = [] + # transform to list so that we can count + for i in refs: + image_refs.append(i) + + result = [] + for image_ref in sorted(image_refs, key=lambda v: v['name'])[offset:offset+limit]: + with mgr.rados.open_ioctx(image_ref['pool']) as ioctx: + ioctx.set_namespace(image_ref['namespace']) + try: + stat = cls._rbd_image_stat( + ioctx, image_ref['pool'], image_ref['namespace'], image_ref['name']) + except rbd.ImageNotFound: + # Check if the RBD has been deleted partially. This happens for example if + # the deletion process of the RBD has been started and was interrupted. try: - stat = cls._rbd_image_stat( - ioctx, pool_name, current_namespace, image_ref['name']) + stat = cls._rbd_image_stat_removing( + ioctx, image_ref['pool'], image_ref['namespace'], image_ref['id']) except rbd.ImageNotFound: - # Check if the RBD has been deleted partially. This happens for example if - # the deletion process of the RBD has been started and was interrupted. - try: - stat = cls._rbd_image_stat_removing( - ioctx, pool_name, current_namespace, image_ref['id']) - except rbd.ImageNotFound: - continue - result.append(stat) - return result + continue + stat['pool'] = image_ref['pool'] + result.append(stat) + return result, len(image_refs) @classmethod def get_image(cls, image_spec): -- 2.39.5