From: Stephan Müller Date: Wed, 8 Jan 2020 17:02:29 +0000 (+0100) Subject: mgr/dashboard: Reload all CephFS directories X-Git-Tag: v15.1.1~322^2 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=e753d987e5a1a5ccd9b4989801a00c4b4ad75fc2;p=ceph.git mgr/dashboard: Reload all CephFS directories Now angular tree is used instead of ng2-tree, as it provides a better way to dynamically load children and it provides a way to update all children without losing all track of everything. The loading icon will rotate now on any fetch. The tree will detect new directories and removed directories. It's also now possible to select the root directory of a CephFS in order to create snapshots of the whole FS. Fixes: https://tracker.ceph.com/issues/42617 Signed-off-by: Stephan Müller --- diff --git a/qa/tasks/mgr/dashboard/test_cephfs.py b/qa/tasks/mgr/dashboard/test_cephfs.py index bfd274025196b..e6594450f7141 100644 --- a/qa/tasks/mgr/dashboard/test_cephfs.py +++ b/qa/tasks/mgr/dashboard/test_cephfs.py @@ -31,11 +31,23 @@ class CephfsTest(DashboardTestCase): params={'path': path}) self.assertStatus(expectedStatus) + def get_root_directory(self, expectedStatus=200): + data = self._get("/api/cephfs/{}/get_root_directory".format(self.get_fs_id())) + self.assertStatus(expectedStatus) + self.assertIsInstance(data, dict) + return data + def ls_dir(self, path, expectedLength, depth = None): + return self._ls_dir(path, expectedLength, depth, "api") + + def ui_ls_dir(self, path, expectedLength, depth = None): + return self._ls_dir(path, expectedLength, depth, "ui-api") + + def _ls_dir(self, path, expectedLength, depth, baseApiPath): params = {'path': path} if depth is not None: params['depth'] = depth - data = self._get("/api/cephfs/{}/ls_dir".format(self.get_fs_id()), + data = self._get("/{}/cephfs/{}/ls_dir".format(baseApiPath, self.get_fs_id()), params=params) self.assertStatus(200) self.assertIsInstance(data, list) @@ -247,3 +259,16 @@ class CephfsTest(DashboardTestCase): self.setQuotas(0, 0) self.assertQuotas(0, 0) + def test_listing_of_root_dir(self): + self.ls_dir('/', 0) # Should not list root + ui_root = self.ui_ls_dir('/', 1)[0] # Should list root by default + root = self.get_root_directory() + self.assertEqual(ui_root, root) + + def test_listing_of_ui_api_ls_on_deeper_levels(self): + # The UI-API and API ls_dir methods should behave the same way on deeper levels + self.mk_dirs('/pictures') + api_ls = self.ls_dir('/pictures', 0) + ui_api_ls = self.ui_ls_dir('/pictures', 0) + self.assertEqual(api_ls, ui_api_ls) + self.rm_dir('/pictures') diff --git a/src/pybind/mgr/dashboard/controllers/cephfs.py b/src/pybind/mgr/dashboard/controllers/cephfs.py index 7678e45a50cd3..f4c024d619e9b 100644 --- a/src/pybind/mgr/dashboard/controllers/cephfs.py +++ b/src/pybind/mgr/dashboard/controllers/cephfs.py @@ -324,6 +324,30 @@ class CephFS(RESTController): raise cherrypy.HTTPError(404, "CephFS id {} not found".format(fs_id)) return CephFS_(fs_name) + @RESTController.Resource('GET') + def get_root_directory(self, fs_id): + """ + The root directory that can't be fetched using ls_dir (api). + :param fs_id: The filesystem identifier. + :return: The root directory + :rtype: dict + """ + try: + return self._get_root_directory(self._cephfs_instance(fs_id)) + except (cephfs.PermissionError, cephfs.ObjectNotFound): + return None + + def _get_root_directory(self, cfs): + """ + The root directory that can't be fetched using ls_dir (api). + It's used in ls_dir (ui-api) and in get_root_directory (api). + :param cfs: CephFS service instance + :type cfs: CephFS + :return: The root directory + :rtype: dict + """ + return cfs.get_directory(os.sep.encode()) + @RESTController.Resource('GET') def ls_dir(self, fs_id, path=None, depth=1): """ @@ -331,29 +355,35 @@ class CephFS(RESTController): :param fs_id: The filesystem identifier. :param path: The path where to start listing the directory content. Defaults to '/' if not set. + :type path: str | bytes + :param depth: The number of steps to go down the directory tree. + :type depth: int | str :return: The names of the directories below the specified path. :rtype: list """ - if path is None: - path = os.sep - else: - path = os.path.normpath(path) + path = self._set_ls_dir_path(path) try: cfs = self._cephfs_instance(fs_id) - paths = cfs.ls_dir(path, int(depth)) - # Convert (bytes => string), prettify paths (strip slashes) - # and append additional information. - paths = [{ - 'name': os.path.basename(p.decode()), - 'path': p.decode(), - 'parent': os.path.dirname(p.decode()), - 'snapshots': cfs.ls_snapshots(p.decode()), - 'quotas': cfs.get_quotas(p.decode()) - } for p in paths if p != path.encode()] + paths = cfs.ls_dir(path, depth) except (cephfs.PermissionError, cephfs.ObjectNotFound): paths = [] return paths + def _set_ls_dir_path(self, path): + """ + Transforms input path parameter of ls_dir methods (api and ui-api). + :param path: The path where to start listing the directory content. + Defaults to '/' if not set. + :type path: str | bytes + :return: Normalized path or root path + :return: str + """ + if path is None: + path = os.sep + else: + path = os.path.normpath(path) + return path + @RESTController.Resource('POST') def mk_dirs(self, fs_id, path): """ @@ -462,3 +492,29 @@ class CephFsUi(CephFS): data['clients'] = self._clients(fs_id) return data + + @RESTController.Resource('GET') + def ls_dir(self, fs_id, path=None, depth=1): + """ + The difference to the API version is that the root directory will be send when listing + the root directory. + To only do one request this endpoint was created. + :param fs_id: The filesystem identifier. + :type fs_id: int | str + :param path: The path where to start listing the directory content. + Defaults to '/' if not set. + :type path: str | bytes + :param depth: The number of steps to go down the directory tree. + :type depth: int | str + :return: The names of the directories below the specified path. + :rtype: list + """ + path = self._set_ls_dir_path(path) + try: + cfs = self._cephfs_instance(fs_id) + paths = cfs.ls_dir(path, depth) + if path == os.sep: + paths = [self._get_root_directory(cfs)] + paths + except (cephfs.PermissionError, cephfs.ObjectNotFound): + paths = [] + return paths diff --git a/src/pybind/mgr/dashboard/frontend/package-lock.json b/src/pybind/mgr/dashboard/frontend/package-lock.json index 9071fce742a7e..2a6ee4934587c 100644 --- a/src/pybind/mgr/dashboard/frontend/package-lock.json +++ b/src/pybind/mgr/dashboard/frontend/package-lock.json @@ -3236,6 +3236,17 @@ "integrity": "sha1-SlKCrBZHKek2Gbz9OtFR+BfOkfU=", "dev": true }, + "angular-tree-component": { + "version": "8.5.2", + "resolved": "https://registry.npmjs.org/angular-tree-component/-/angular-tree-component-8.5.2.tgz", + "integrity": "sha512-3NwMB+vLq1+WHz2UVgsZA73E1LmIIWJlrrasCKXbLJ3S7NmY9O/wKcolji3Vp2W//5KQ33RXu1jiPXCOQdRzVA==", + "requires": { + "lodash": "^4.17.11", + "mobx": "^5.14.2", + "mobx-angular": "3.0.3", + "opencollective-postinstall": "^2.0.2" + } + }, "ansi-colors": { "version": "3.2.4", "resolved": "https://registry.npmjs.org/ansi-colors/-/ansi-colors-3.2.4.tgz", @@ -11225,6 +11236,16 @@ } } }, + "mobx": { + "version": "5.15.2", + "resolved": "https://registry.npmjs.org/mobx/-/mobx-5.15.2.tgz", + "integrity": "sha512-eVmHGuSYd0ZU6x8gYMdgLEnCC9kfBJaf7/qJt+/yIxczVVUiVzHvTBjZH3xEa5FD5VJJSh1/Ba4SThE4ErEGjw==" + }, + "mobx-angular": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/mobx-angular/-/mobx-angular-3.0.3.tgz", + "integrity": "sha512-mZuuose70V+Sd0hMWDElpRe3mA6GhYjSQN3mHzqk2XWXRJ+eWQa/f3Lqhw+Me/Xd2etWsGR1hnRa1BfQ2ZDtpw==" + }, "moment": { "version": "2.24.0", "resolved": "https://registry.npmjs.org/moment/-/moment-2.24.0.tgz", diff --git a/src/pybind/mgr/dashboard/frontend/package.json b/src/pybind/mgr/dashboard/frontend/package.json index 3afa944a65a5c..e6cec0bef30c1 100644 --- a/src/pybind/mgr/dashboard/frontend/package.json +++ b/src/pybind/mgr/dashboard/frontend/package.json @@ -90,6 +90,7 @@ "@auth0/angular-jwt": "2.1.1", "@ngx-translate/i18n-polyfill": "1.0.0", "@swimlane/ngx-datatable": "15.0.2", + "angular-tree-component": "8.5.2", "async-mutex": "0.1.4", "bootstrap": "4.3.1", "chart.js": "2.8.0", diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-directories/cephfs-directories.component.html b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-directories/cephfs-directories.component.html index 8a6e0f176fe71..7753d599bb6e0 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-directories/cephfs-directories.component.html +++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-directories/cephfs-directories.component.html @@ -1,12 +1,33 @@
-
- - - - - +
+
+
+ +
+
+ + + + + + +
+
- Quotas - - - - + + Quotas + + + + + Snapshots { let component: CephfsDirectoriesComponent; let fixture: ComponentFixture; let cephfsService: CephfsService; + let noAsyncUpdate: boolean; let lsDirSpy: jasmine.Spy; let modalShowSpy: jasmine.Spy; let notificationShowSpy: jasmine.Spy; @@ -53,11 +55,12 @@ describe('CephfsDirectoriesComponent', () => { // Object contains mock data that will be reset before each test. let mockData: { - nodes: TreeModel[]; - parent: Tree; + nodes: any; + parent: any; createdSnaps: CephfsSnapshot[] | any[]; deletedSnaps: CephfsSnapshot[] | any[]; updatedQuotas: { [path: string]: CephfsQuotas }; + createdDirs: CephfsDir[]; }; // Object contains mock functions @@ -101,16 +104,18 @@ describe('CephfsDirectoriesComponent', () => { }, // Only used inside other mocks lsSingleDir: (path = ''): CephfsDir[] => { - if (path.includes('b')) { + const customDirs = mockData.createdDirs.filter((d) => d.parent === path); + const isCustomDir = mockData.createdDirs.some((d) => d.path === path); + if (isCustomDir || path.includes('b')) { // 'b' has no sub directories - return []; + return customDirs; } - return [ + return customDirs.concat([ // Directories are not sorted! mockLib.dir(path, 'c', 3), mockLib.dir(path, 'a', 1), mockLib.dir(path, 'b', 2) - ]; + ]); }, lsDir: (_id: number, path = ''): Observable => { // will return 2 levels deep @@ -119,6 +124,14 @@ describe('CephfsDirectoriesComponent', () => { paths.forEach((pathL2) => { data = data.concat(mockLib.lsSingleDir(pathL2)); }); + if (path === '' || path === '/') { + // Adds root directory on ls of '/' to the directories list. + const root = mockLib.dir(path, '/', 1); + root.path = '/'; + root.parent = undefined; + root.quotas = undefined; + data = [root].concat(data); + } return of(data); }, mkSnapshot: (_id: any, path: string, name: string): Observable => { @@ -145,46 +158,55 @@ describe('CephfsDirectoriesComponent', () => { modal = modalServiceShow(comp, init); return modal.ref; }, - getControllerByPath: (path: string) => { - return { - expand: () => mockLib.expand(path), - select: () => component.onNodeSelected(mockLib.getNodeEvent(path)) - }; + getNodeById: (path: string) => { + return mockLib.useNode(path); + }, + updateNodes: (path: string) => { + const p: Promise = component.treeOptions.getChildren({ id: path }); + return noAsyncUpdate ? () => p : mockLib.asyncNodeUpdate(p); }, - // Only used inside other mocks to mock "tree.expand" of every node - expand: (path: string) => { - component.updateDirectory(path, (nodes) => { + asyncNodeUpdate: fakeAsync((p: Promise) => { + p.then((nodes) => { mockData.nodes = mockData.nodes.concat(nodes); }); - }, - getNodeEvent: (path: string): NodeEvent => { - const tree = mockData.nodes.find((n) => n.id === path) as Tree; - if (mockData.parent) { - tree.parent = mockData.parent; - } else { - const dir = get.nodeIds()[path]; - const parentNode = mockData.nodes.find((n) => n.id === dir.parent); - tree.parent = parentNode as Tree; - } - return { node: tree } as NodeEvent; - }, + tick(); + }), changeId: (id: number) => { + // For some reason this spy has to be renewed after usage + spyOn(global, 'setTimeout').and.callFake((fn) => fn()); component.id = id; component.ngOnChanges(); - mockData.nodes = [component.tree].concat(component.tree.children); + mockData.nodes = component.nodes.concat(mockData.nodes); }, selectNode: (path: string) => { - mockLib.getControllerByPath(path).select(); + component.treeOptions.actionMapping.mouse.click(undefined, mockLib.useNode(path), undefined); + }, + // Creates TreeNode with parents until root + useNode: (path: string): { id: string; parent: any; data: any; loadNodeChildren: Function } => { + const parentPath = path.split('/'); + parentPath.pop(); + const parentIsRoot = parentPath.length === 1; + const parent = parentIsRoot ? { id: '/' } : mockLib.useNode(parentPath.join('/')); + return { + id: path, + parent, + data: {}, + loadNodeChildren: () => mockLib.updateNodes(path) + }; + }, + treeActions: { + toggleActive: (_a: any, node: any, _b: any) => { + return mockLib.updateNodes(node.id); + } }, mkDir: (path: string, name: string, maxFiles: number, maxBytes: number) => { const dir = mockLib.dir(path, name, 3); dir.quotas.max_bytes = maxBytes * 1024; dir.quotas.max_files = maxFiles; + mockData.createdDirs.push(dir); + // Below is needed for quota tests only where 4 dirs are mocked get.nodeIds()[dir.path] = dir; - mockData.nodes.push({ - id: dir.path, - value: name - }); + mockData.nodes.push({ id: dir.path }); }, createSnapshotThroughModal: (name: string) => { component.createSnapshot(); @@ -214,7 +236,7 @@ describe('CephfsDirectoriesComponent', () => { let path = ''; quotas.forEach((quota, index) => { index += 1; - mockLib.mkDir(path, index.toString(), quota[0], quota[1]); + mockLib.mkDir(path === '' ? '/' : path, index.toString(), quota[0], quota[1]); path += '/' + index; }); mockData.parent = { @@ -229,7 +251,7 @@ describe('CephfsDirectoriesComponent', () => { parent: { value: '/', id: '/' } } } - } as Tree; + }; mockLib.selectNode('/1/2/3/4'); } }; @@ -239,6 +261,10 @@ describe('CephfsDirectoriesComponent', () => { dirLength: (n: number) => expect(get.dirs().length).toBe(n), nodeLength: (n: number) => expect(mockData.nodes.length).toBe(n), lsDirCalledTimes: (n: number) => expect(lsDirSpy).toHaveBeenCalledTimes(n), + lsDirHasBeenCalledWith: (id: number, paths: string[]) => { + paths.forEach((path) => expect(lsDirSpy).toHaveBeenCalledWith(id, path)); + assert.lsDirCalledTimes(paths.length); + }, requestedPaths: (expected: string[]) => expect(get.requestedPaths()).toEqual(expected), snapshotsByName: (snaps: string[]) => expect(component.selectedDir.snapshots.map((s) => s.name)).toEqual(snaps), @@ -337,7 +363,7 @@ describe('CephfsDirectoriesComponent', () => { HttpClientTestingModule, SharedModule, RouterTestingModule, - TreeModule, + TreeModule.forRoot(), NgBootstrapFormValidationModule.forRoot(), ToastrModule.forRoot(), ModalModule.forRoot() @@ -347,11 +373,13 @@ describe('CephfsDirectoriesComponent', () => { }); beforeEach(() => { + noAsyncUpdate = false; mockData = { - nodes: undefined, + nodes: [], parent: undefined, createdSnaps: [], deletedSnaps: [], + createdDirs: [], updatedQuotas: {} }; @@ -368,9 +396,12 @@ describe('CephfsDirectoriesComponent', () => { component = fixture.componentInstance; fixture.detectChanges(); - spyOn(component.treeComponent, 'getControllerByNodeId').and.callFake((id) => - mockLib.getControllerByPath(id) - ); + spyOn(TREE_ACTIONS, 'TOGGLE_ACTIVE').and.callFake(mockLib.treeActions.toggleActive); + + component.treeComponent = { + sizeChanged: () => null, + treeModel: { getNodeById: mockLib.getNodeById, update: () => null } + } as TreeComponent; }); it('should create', () => { @@ -474,7 +505,6 @@ describe('CephfsDirectoriesComponent', () => { }); it('calls lsDir only if an id exits', () => { - component.ngOnChanges(); assert.lsDirCalledTimes(0); mockLib.changeId(1); @@ -501,7 +531,9 @@ describe('CephfsDirectoriesComponent', () => { it('expands first level', () => { // Tree will only show '*' if nor 'loadChildren' or 'children' are defined expect( - mockData.nodes.map((node) => ({ [node.id]: Boolean(node.loadChildren || node.children) })) + mockData.nodes.map((node: any) => ({ + [node.id]: node.hasChildren || node.isExpanded || Boolean(node.children) + })) ).toEqual([{ '/': true }, { '/a': true }, { '/b': false }, { '/c': true }]); }); @@ -519,7 +551,7 @@ describe('CephfsDirectoriesComponent', () => { * */ assert.requestedPaths(['/', '/a']); assert.nodeLength(7); - assert.dirLength(15); + assert.dirLength(16); expect(component.selectedDir).toBeDefined(); mockLib.changeId(undefined); @@ -536,7 +568,7 @@ describe('CephfsDirectoriesComponent', () => { assert.quotaIsNotInherited('bytes', '1 KiB', 0); }); - it('should extend the list by subdirectories when expanding and omit already called path', () => { + it('should extend the list by subdirectories when expanding', () => { mockLib.selectNode('/a'); mockLib.selectNode('/a/c'); /** @@ -554,10 +586,19 @@ describe('CephfsDirectoriesComponent', () => { * */ assert.lsDirCalledTimes(3); assert.requestedPaths(['/', '/a', '/a/c']); - assert.dirLength(21); + assert.dirLength(22); assert.nodeLength(10); }); + it('should update the tree after each selection', () => { + const spy = spyOn(component.treeComponent, 'sizeChanged').and.callThrough(); + expect(spy).toHaveBeenCalledTimes(0); + mockLib.selectNode('/a'); + expect(spy).toHaveBeenCalledTimes(1); + mockLib.selectNode('/a/c'); + expect(spy).toHaveBeenCalledTimes(2); + }); + it('should select parent by path', () => { mockLib.selectNode('/a'); mockLib.selectNode('/a/c'); @@ -566,7 +607,7 @@ describe('CephfsDirectoriesComponent', () => { expect(component.selectedDir.path).toBe('/a'); }); - it('should omit call for directories that have no sub directories', () => { + it('should refresh directories with no sub directories as they could have some now', () => { mockLib.selectNode('/b'); /** * Tree looks like this: @@ -575,8 +616,8 @@ describe('CephfsDirectoriesComponent', () => { * * b <- Selected * > c * */ - assert.lsDirCalledTimes(1); - assert.requestedPaths(['/']); + assert.lsDirCalledTimes(2); + assert.requestedPaths(['/', '/b']); assert.nodeLength(4); }); @@ -909,4 +950,98 @@ describe('CephfsDirectoriesComponent', () => { }); }); }); + + describe('reload all', () => { + const calledPaths = ['/', '/a', '/a/c', '/a/c/a', '/a/c/a/b']; + + const dirsByPath = (): string[] => get.dirs().map((d) => d.path); + + beforeEach(() => { + mockLib.changeId(1); + mockLib.selectNode('/a'); + mockLib.selectNode('/a/c'); + mockLib.selectNode('/a/c/a'); + mockLib.selectNode('/a/c/a/b'); + }); + + it('should reload all requested paths', () => { + assert.lsDirHasBeenCalledWith(1, calledPaths); + lsDirSpy.calls.reset(); + assert.lsDirHasBeenCalledWith(1, []); + component.refreshAllDirectories(); + assert.lsDirHasBeenCalledWith(1, calledPaths); + }); + + it('should reload all requested paths if not selected anything', () => { + lsDirSpy.calls.reset(); + mockLib.changeId(2); + assert.lsDirHasBeenCalledWith(2, ['/']); + lsDirSpy.calls.reset(); + component.refreshAllDirectories(); + assert.lsDirHasBeenCalledWith(2, ['/']); + }); + + it('should add new directories', () => { + // Create two new directories in preparation + const dirsBeforeRefresh = dirsByPath(); + expect(dirsBeforeRefresh.includes('/a/c/has_dir_now')).toBe(false); + mockLib.mkDir('/a/c', 'has_dir_now', 0, 0); + mockLib.mkDir('/a/c/a/b', 'has_dir_now_too', 0, 0); + // Now the new directories will be fetched + component.refreshAllDirectories(); + const dirsAfterRefresh = dirsByPath(); + expect(dirsAfterRefresh.length - dirsBeforeRefresh.length).toBe(2); + expect(dirsAfterRefresh.includes('/a/c/has_dir_now')).toBe(true); + expect(dirsAfterRefresh.includes('/a/c/a/b/has_dir_now_too')).toBe(true); + }); + + it('should remove deleted directories', () => { + // Create one new directory and refresh in order to have it added to the directories list + mockLib.mkDir('/a/c', 'will_be_removed_shortly', 0, 0); + component.refreshAllDirectories(); + const dirsBeforeRefresh = dirsByPath(); + expect(dirsBeforeRefresh.includes('/a/c/will_be_removed_shortly')).toBe(true); + mockData.createdDirs = []; // Mocks the deletion of the directory + // Now the deleted directory will be missing on refresh + component.refreshAllDirectories(); + const dirsAfterRefresh = dirsByPath(); + expect(dirsAfterRefresh.length - dirsBeforeRefresh.length).toBe(-1); + expect(dirsAfterRefresh.includes('/a/c/will_be_removed_shortly')).toBe(false); + }); + + describe('loading indicator', () => { + beforeEach(() => { + noAsyncUpdate = true; + }); + + it('should have set loading indicator to false after refreshing all dirs', fakeAsync(() => { + component.refreshAllDirectories(); + expect(component.loadingIndicator).toBe(true); + tick(3000); // To resolve all promises + expect(component.loadingIndicator).toBe(false); + })); + + it('should only update the tree once and not on every call', fakeAsync(() => { + const spy = spyOn(component.treeComponent, 'sizeChanged').and.callThrough(); + component.refreshAllDirectories(); + expect(spy).toHaveBeenCalledTimes(0); + tick(3000); // To resolve all promises + // Called during the interval and at the end of timeout + expect(spy).toHaveBeenCalledTimes(2); + })); + + it('should have set all loaded dirs as attribute names of "indicators"', () => { + noAsyncUpdate = false; + component.refreshAllDirectories(); + expect(Object.keys(component.loading).sort()).toEqual(calledPaths); + }); + + it('should set an indicator to true during load', () => { + lsDirSpy.and.callFake(() => Observable.create((): null => null)); + component.refreshAllDirectories(); + expect(Object.values(component.loading).every((b) => b)).toBe(true); + expect(component.loadingIndicator).toBe(true); + }); + }); + }); }); diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-directories/cephfs-directories.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-directories/cephfs-directories.component.ts index d07a945911fd3..7f3b073aeab06 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-directories/cephfs-directories.component.ts +++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-directories/cephfs-directories.component.ts @@ -2,9 +2,15 @@ import { Component, Input, OnChanges, OnInit, TemplateRef, ViewChild } from '@an import { Validators } from '@angular/forms'; import { I18n } from '@ngx-translate/i18n-polyfill'; +import { + ITreeOptions, + TREE_ACTIONS, + TreeComponent, + TreeModel, + TreeNode +} from 'angular-tree-component'; import * as _ from 'lodash'; import * as moment from 'moment'; -import { NodeEvent, Tree, TreeComponent, TreeModel } from 'ng2-tree'; import { BsModalRef, BsModalService } from 'ngx-bootstrap/modal'; import { CephfsService } from '../../../shared/api/cephfs.service'; @@ -51,7 +57,7 @@ class QuotaSetting { styleUrls: ['./cephfs-directories.component.scss'] }) export class CephfsDirectoriesComponent implements OnInit, OnChanges { - @ViewChild(TreeComponent, { static: true }) + @ViewChild(TreeComponent, { static: false }) treeComponent: TreeComponent; @ViewChild('origin', { static: true }) originTmpl: TemplateRef; @@ -63,7 +69,23 @@ export class CephfsDirectoriesComponent implements OnInit, OnChanges { private dirs: CephfsDir[]; private nodeIds: { [path: string]: CephfsDir }; private requestedPaths: string[]; - private selectedNode: Tree; + private loadingTimeout: any; + + icons = Icons; + loadingIndicator = false; + loading = {}; + treeOptions: ITreeOptions = { + useVirtualScroll: true, + getChildren: (node: TreeNode): Promise => { + return this.updateDirectory(node.id); + }, + actionMapping: { + mouse: { + click: this.selectAndShowNode.bind(this), + expanderClick: this.selectAndShowNode.bind(this) + } + } + }; permission: Permission; selectedDir: CephfsDir; @@ -80,7 +102,7 @@ export class CephfsDirectoriesComponent implements OnInit, OnChanges { tableActions: CdTableAction[]; updateSelection: Function; }; - tree: TreeModel; + nodes: any[]; constructor( private authStorageService: AuthStorageService, @@ -93,6 +115,20 @@ export class CephfsDirectoriesComponent implements OnInit, OnChanges { private dimlessBinaryPipe: DimlessBinaryPipe ) {} + private selectAndShowNode(tree: TreeModel, node: TreeNode, $event: any) { + TREE_ACTIONS.TOGGLE_EXPANDED(tree, node, $event); + this.selectNode(node); + } + + private selectNode(node: TreeNode) { + TREE_ACTIONS.TOGGLE_ACTIVE(undefined, node, undefined); + this.selectedDir = this.getDirectory(node); + if (node.id === '/') { + return; + } + this.setSettings(node); + } + ngOnInit() { this.permission = this.authStorageService.getPermissions().cephfs; this.setUpQuotaTable(); @@ -203,96 +239,82 @@ export class CephfsDirectoriesComponent implements OnInit, OnChanges { this.dirs = []; this.requestedPaths = []; this.nodeIds = {}; - if (_.isUndefined(this.id)) { - this.setRootNode([]); - } else { + if (this.id) { + this.setRootNode(); this.firstCall(); } } - private setRootNode(nodes: TreeModel[]) { - const tree: TreeModel = { - value: '/', - id: '/', - settings: { - selectionAllowed: false, - static: true + private setRootNode() { + this.nodes = [ + { + name: '/', + id: '/', + isExpanded: true } - }; - if (nodes.length > 0) { - tree.children = nodes; - } - this.tree = tree; + ]; } private firstCall() { - this.updateDirectory('/', (nodes) => this.setRootNode(nodes)); + const path = '/'; + setTimeout(() => { + this.getNode(path).loadNodeChildren(); + }, 10); } - updateDirectory(path: string, callback: (x: any[]) => void) { - if ( - !this.requestedPaths.includes(path) && - (path === '/' || this.getSubDirectories(path).length > 0) - ) { + updateDirectory(path: string): Promise { + this.unsetLoadingIndicator(); + if (!this.requestedPaths.includes(path)) { this.requestedPaths.push(path); - this.cephfsService - .lsDir(this.id, path) - .subscribe((data) => this.loadDirectory(data, path, callback)); - } else { - this.getChildren(path, callback); + } else if (this.loading[path] === true) { + return undefined; // Path is currently fetched. } + return new Promise((resolve) => { + this.setLoadingIndicator(path, true); + this.cephfsService.lsDir(this.id, path).subscribe((dirs) => { + this.updateTreeStructure(dirs); + this.updateQuotaTable(); + this.updateTree(); + resolve(this.getChildren(path)); + this.setLoadingIndicator(path, false); + }); + }); } - private getSubDirectories(path: string, tree: CephfsDir[] = this.dirs): CephfsDir[] { - return tree.filter((d) => d.parent === path); + private setLoadingIndicator(path: string, loading: boolean) { + this.loading[path] = loading; + this.unsetLoadingIndicator(); } - private loadDirectory(data: CephfsDir[], path: string, callback: (x: any[]) => void) { - if (path !== '/') { - // As always to levels are loaded all sub-directories of the current called path are - // already loaded, that's why they are filtered out. - data = data.filter((dir) => dir.parent !== path); - } - this.dirs = this.dirs.concat(data); - this.getChildren(path, callback); + private getSubDirectories(path: string, tree: CephfsDir[] = this.dirs): CephfsDir[] { + return tree.filter((d) => d.parent === path); } - private getChildren(path: string, callback: (x: any[]) => void) { + private getChildren(path: string): any[] { const subTree = this.getSubTree(path); - const nodes = _.sortBy(this.getSubDirectories(path), 'path').map((d) => { - this.nodeIds[d.path] = d; - const newNode: TreeModel = { - value: d.name, - id: d.path, - settings: { static: true } - }; - if (this.getSubDirectories(d.path, subTree).length > 0) { - // LoadChildren will be triggered if a node is expanded - newNode.loadChildren = (treeCallback) => this.updateDirectory(d.path, treeCallback); - } - return newNode; - }); - callback(nodes); - } - - private getSubTree(path: string): CephfsDir[] { - return this.dirs.filter((d) => d.parent.startsWith(path)); + return _.sortBy(this.getSubDirectories(path), 'path').map((dir) => + this.createNode(dir, subTree) + ); } - selectOrigin(path: string) { - this.treeComponent.getControllerByNodeId(path).select(); + private createNode(dir: CephfsDir, subTree?: CephfsDir[]): any { + this.nodeIds[dir.path] = dir; + if (!subTree) { + this.getSubTree(dir.parent); + } + return { + name: dir.name, + id: dir.path, + hasChildren: this.getSubDirectories(dir.path, subTree).length > 0 + }; } - onNodeSelected(e: NodeEvent) { - const node = e.node; - this.treeComponent.getControllerByNodeId(node.id).expand(); - this.setSettings(node); - this.selectedDir = this.getDirectory(node); - this.selectedNode = node; + private getSubTree(path: string): CephfsDir[] { + return this.dirs.filter((d) => d.parent && d.parent.startsWith(path)); } - private setSettings(node: Tree) { - const readable = (value: number, fn?: (number: number) => number | string): number | string => + private setSettings(node: TreeNode) { + const readable = (value: number, fn?: (arg0: number) => number | string): number | string => value ? (fn ? fn(value) : value) : ''; this.settings = [ @@ -304,7 +326,7 @@ export class CephfsDirectoriesComponent implements OnInit, OnChanges { } private getQuota( - tree: Tree, + tree: TreeNode, quotaKey: string, valueConvertFn: (number: number) => number | string ): QuotaSetting { @@ -318,7 +340,7 @@ export class CephfsDirectoriesComponent implements OnInit, OnChanges { let nextMaxValue = value; let nextMaxPath = dir.path; if (tree.id === currentPath) { - if (tree.parent.value === '/') { + if (tree.parent.id === '/') { // The value will never inherit any other value, so it has no maximum. nextMaxValue = 0; } else { @@ -355,8 +377,8 @@ export class CephfsDirectoriesComponent implements OnInit, OnChanges { * | /a (10) | 4th | 10 => true | /a | * */ - private getOrigin(tree: Tree, quotaSetting: string): Tree { - if (tree.parent.value !== '/') { + private getOrigin(tree: TreeNode, quotaSetting: string): TreeNode { + if (tree.parent && tree.parent.id !== '/') { const current = this.getQuotaFromTree(tree, quotaSetting); // Get the next used quota and node above the current one (until it hits the root directory) @@ -370,15 +392,23 @@ export class CephfsDirectoriesComponent implements OnInit, OnChanges { return tree; } - private getQuotaFromTree(tree: Tree, quotaSetting: string): number { + private getQuotaFromTree(tree: TreeNode, quotaSetting: string): number { return this.getDirectory(tree).quotas[quotaSetting]; } - private getDirectory(node: Tree): CephfsDir { + private getDirectory(node: TreeNode): CephfsDir { const path = node.id as string; return this.nodeIds[path]; } + selectOrigin(path: string) { + this.selectNode(this.getNode(path)); + } + + private getNode(path: string): TreeNode { + return this.treeComponent.treeModel.getNodeById(path); + } + updateQuotaModal() { const path = this.selectedDir.path; const selection: QuotaSetting = this.quota.selection.first(); @@ -544,17 +574,103 @@ export class CephfsDirectoriesComponent implements OnInit, OnChanges { * As all nodes point by their path on an directory object, the easiest way is to update * the objects by merge with their latest change. */ - private forceDirRefresh() { - const path = this.selectedNode.parent.id as string; - this.cephfsService.lsDir(this.id, path).subscribe((data) => { - data.forEach((d) => { - Object.assign(this.dirs.find((sub) => sub.path === d.path), d); + private forceDirRefresh(path?: string) { + if (!path) { + const dir = this.selectedDir; + if (!dir) { + throw new Error('This function can only be called without path if an selection was made'); + } + // Parent has to be called in order to update the object referring + // to the current selected directory + path = dir.parent ? dir.parent : dir.path; + } + const node = this.getNode(path); + node.loadNodeChildren(); + } + + private updateTreeStructure(dirs: CephfsDir[]) { + const getChildrenAndPaths = ( + directories: CephfsDir[], + parent: string + ): { children: CephfsDir[]; paths: string[] } => { + const children = directories.filter((d) => d.parent === parent); + const paths = children.map((d) => d.path); + return { children, paths }; + }; + + const parents = _.uniq(dirs.map((d) => d.parent).sort()); + parents.forEach((p) => { + const received = getChildrenAndPaths(dirs, p); + const cached = getChildrenAndPaths(this.dirs, p); + + cached.children.forEach((d) => { + if (!received.paths.includes(d.path)) { + this.removeOldDirectory(d); + } + }); + received.children.forEach((d) => { + if (cached.paths.includes(d.path)) { + this.updateExistingDirectory(cached.children, d); + } else { + this.addNewDirectory(d); + } }); - // Now update quotas - this.setSettings(this.selectedNode); }); } + private removeOldDirectory(rmDir: CephfsDir) { + const path = rmDir.path; + // Remove directory from local variables + _.remove(this.dirs, (d) => d.path === path); + delete this.nodeIds[path]; + this.updateDirectoriesParentNode(rmDir); + } + + private updateDirectoriesParentNode(dir: CephfsDir) { + const parent = dir.parent; + if (!parent) { + return; + } + const node = this.getNode(parent); + if (!node) { + // Node will not be found for new sub sub directories - this is the intended behaviour + return; + } + const children = this.getChildren(parent); + node.data.children = children; + node.data.hasChildren = children.length > 0; + this.treeComponent.treeModel.update(); + } + + private addNewDirectory(newDir: CephfsDir) { + this.dirs.push(newDir); + this.nodeIds[newDir.path] = newDir; + this.updateDirectoriesParentNode(newDir); + } + + private updateExistingDirectory(source: CephfsDir[], updatedDir: CephfsDir) { + const currentDirObject = source.find((sub) => sub.path === updatedDir.path); + Object.assign(currentDirObject, updatedDir); + } + + private updateQuotaTable() { + const node = this.selectedDir ? this.getNode(this.selectedDir.path) : undefined; + if (node && node.id !== '/') { + this.setSettings(node); + } + } + + private updateTree(force: boolean = false) { + if (this.loadingIndicator && !force) { + // In order to make the page scrollable during load, the render cycle for each node + // is omitted and only be called if all updates were loaded. + return; + } + this.treeComponent.treeModel.update(); + this.nodes = [...this.nodes]; + this.treeComponent.sizeChanged(); + } + deleteSnapshotModal() { this.modalRef = this.modalService.show(CriticalConfirmationModalComponent, { initialState: { @@ -584,4 +700,34 @@ export class CephfsDirectoriesComponent implements OnInit, OnChanges { this.modalRef.hide(); this.forceDirRefresh(); } + + refreshAllDirectories() { + // In order to make the page scrollable during load, the render cycle for each node + // is omitted and only be called if all updates were loaded. + this.loadingIndicator = true; + this.requestedPaths.map((path) => this.forceDirRefresh(path)); + const interval = setInterval(() => { + this.updateTree(true); + if (!this.loadingIndicator) { + clearInterval(interval); + } + }, 3000); + } + + unsetLoadingIndicator() { + if (!this.loadingIndicator) { + return; + } + clearTimeout(this.loadingTimeout); + this.loadingTimeout = setTimeout(() => { + const loading = Object.values(this.loading).some((l) => l); + if (loading) { + return this.unsetLoadingIndicator(); + } + this.loadingIndicator = false; + this.updateTree(); + // The problem is that we can't subscribe to an useful updated tree event and the time + // between fetching all calls and rebuilding the tree can take some time + }, 3000); + } } diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-tabs/cephfs-tabs.component.html b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-tabs/cephfs-tabs.component.html index 126b30c76df90..bef5caf7ee9d4 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-tabs/cephfs-tabs.component.html +++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-tabs/cephfs-tabs.component.html @@ -16,8 +16,10 @@ - + { it('should call lsDir', () => { service.lsDir(1).subscribe(); - const req = httpTesting.expectOne('api/cephfs/1/ls_dir?depth=2'); + const req = httpTesting.expectOne('ui-api/cephfs/1/ls_dir?depth=2'); expect(req.request.method).toBe('GET'); service.lsDir(2, '/some/path').subscribe(); - httpTesting.expectOne('api/cephfs/2/ls_dir?depth=2&path=%252Fsome%252Fpath'); + httpTesting.expectOne('ui-api/cephfs/2/ls_dir?depth=2&path=%252Fsome%252Fpath'); }); it('should call mkSnapshot', () => { diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/api/cephfs.service.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/cephfs.service.ts index d109ba32bc492..246b5213fe510 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/shared/api/cephfs.service.ts +++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/cephfs.service.ts @@ -14,6 +14,7 @@ import { ApiModule } from './api.module'; }) export class CephfsService { baseURL = 'api/cephfs'; + baseUiURL = 'ui-api/cephfs'; constructor(private http: HttpClient) {} @@ -22,7 +23,7 @@ export class CephfsService { } lsDir(id: number, path?: string): Observable { - let apiPath = `${this.baseURL}/${id}/ls_dir?depth=2`; + let apiPath = `${this.baseUiURL}/${id}/ls_dir?depth=2`; if (path) { apiPath += `&path=${encodeURIComponent(path)}`; } diff --git a/src/pybind/mgr/dashboard/frontend/src/styles.scss b/src/pybind/mgr/dashboard/frontend/src/styles.scss index 90c44e92f4c32..a4cda0c8b45d1 100644 --- a/src/pybind/mgr/dashboard/frontend/src/styles.scss +++ b/src/pybind/mgr/dashboard/frontend/src/styles.scss @@ -1,6 +1,9 @@ /* You can add global styles to this file, and also import other style files */ @import 'defaults'; +// Angular2-Tree Component +@import '~angular-tree-component/dist/angular-tree-component.css'; + // Fork-Awesome $fa-font-path: '~fork-awesome/fonts'; $font-family-icon: 'ForkAwesome'; diff --git a/src/pybind/mgr/dashboard/services/cephfs.py b/src/pybind/mgr/dashboard/services/cephfs.py index 75ac8ba3dc732..a40a370c8cf14 100644 --- a/src/pybind/mgr/dashboard/services/cephfs.py +++ b/src/pybind/mgr/dashboard/services/cephfs.py @@ -62,26 +62,41 @@ class CephFS(object): if d: self.cfs.closedir(d) - def ls_dir(self, path, level): + def ls_dir(self, path, depth): + """ + List directories of specified path with additional information. + :param path: The root directory path. + :type path: str | bytes + :param depth: The number of steps to go down the directory tree. + :type depth: int | str + :return: A list of directory dicts which consist of name, path, + parent, snapshots and quotas. + :rtype: list + """ + paths = self._ls_dir(path, int(depth)) + # Convert (bytes => string), prettify paths (strip slashes) + # and append additional information. + return [self.get_directory(p) for p in paths if p != path.encode()] + + def _ls_dir(self, path, depth): """ List directories of specified path. :param path: The root directory path. :type path: str | bytes - :param level: The number of steps to go down the directory tree. - :type level: int - :return: A list of directory paths (bytes encoded). The specified - root directory is also included. + :param depth: The number of steps to go down the directory tree. + :type depth: int + :return: A list of directory paths (bytes encoded). Example: ls_dir('/photos', 1) => [ - b'/photos', b'/photos/flowers', b'/photos/cars' + b'/photos/flowers', b'/photos/cars' ] :rtype: list """ if isinstance(path, six.string_types): path = path.encode() - logger.debug("get_dir_list dirpath=%s level=%s", path, - level) - if level == 0: + logger.debug("get_dir_list dirpath=%s depth=%s", path, + depth) + if depth == 0: return [path] logger.debug("opening dirpath=%s", path) with self.opendir(path) as d: @@ -95,10 +110,28 @@ class CephFS(object): if dent.is_dir(): logger.debug("found dir=%s", dent.d_name) subdir_path = os.path.join(path, dent.d_name) - paths.extend(self.ls_dir(subdir_path, level - 1)) + paths.extend(self._ls_dir(subdir_path, depth - 1)) dent = self.cfs.readdir(d) return paths + def get_directory(self, path): + """ + Transforms path of directory into a meaningful dictionary. + :param path: The root directory path. + :type path: str | bytes + :return: Dict consists of name, path, parent, snapshots and quotas. + :rtype: dict + """ + path = path.decode() + not_root = path != os.sep + return { + 'name': os.path.basename(path) if not_root else path, + 'path': path, + 'parent': os.path.dirname(path) if not_root else None, + 'snapshots': self.ls_snapshots(path), + 'quotas': self.get_quotas(path) if not_root else None + } + def dir_exists(self, path): try: with self.opendir(path):