From 9700d190e81efa50e1d59206833dd6e3a6d5107f Mon Sep 17 00:00:00 2001 From: Ramana Raja Date: Thu, 20 Jun 2019 17:45:14 +0530 Subject: [PATCH] mgr/volumes: allow setting data pool layout ... of fs subvolumes and subvolume groups during their creation. Fixes: https://tracker.ceph.com/issues/40431 Signed-off-by: Ramana Raja (cherry picked from commit 22ed8915281033c39df5e6ecceb5eebdc4e1c594) --- qa/tasks/cephfs/test_volumes.py | 67 ++++++++++++++++++++++++++ src/pybind/mgr/volumes/fs/subvolume.py | 20 ++++---- src/pybind/mgr/volumes/fs/volume.py | 8 +-- src/pybind/mgr/volumes/module.py | 19 +++++--- 4 files changed, 94 insertions(+), 20 deletions(-) diff --git a/qa/tasks/cephfs/test_volumes.py b/qa/tasks/cephfs/test_volumes.py index 50217bd9b23a6..19dce198aa740 100644 --- a/qa/tasks/cephfs/test_volumes.py +++ b/qa/tasks/cephfs/test_volumes.py @@ -38,6 +38,15 @@ class TestVolumes(CephFSTestCase): else: self.volname = result[0]['name'] + def _get_subvolume_path(self, vol_name, subvol_name, group_name=None): + args = ["subvolume", "getpath", vol_name, subvol_name] + if group_name: + args.append(group_name) + args = tuple(args) + path = self._fs_cmd(*args) + # remove the leading '/', and trailing whitespaces + return path[1:].rstrip() + def _delete_test_volume(self): self._fs_cmd("volume", "rm", self.volname) @@ -127,6 +136,64 @@ class TestVolumes(CephFSTestCase): # remove group self._fs_cmd("subvolumegroup", "rm", self.volname, group) + def test_subvolume_group_create_with_desired_data_pool_layout(self): + group1 = self._generate_random_group_name() + group2 = self._generate_random_group_name() + + # create group + self._fs_cmd("subvolumegroup", "create", self.volname, group1) + group1_path = os.path.join('volumes', group1) + + default_pool = self.mount_a.getfattr(group1_path, "ceph.dir.layout.pool") + new_pool = "new_pool" + self.assertNotEqual(default_pool, new_pool) + + # add data pool + self.fs.add_data_pool(new_pool) + + # create group specifying the new data pool as its pool layout + self._fs_cmd("subvolumegroup", "create", self.volname, group2, + "--pool_layout", new_pool) + group2_path = os.path.join('volumes', group2) + + desired_pool = self.mount_a.getfattr(group2_path, "ceph.dir.layout.pool") + self.assertEqual(desired_pool, new_pool) + + self._fs_cmd("subvolumegroup", "rm", self.volname, group1) + self._fs_cmd("subvolumegroup", "rm", self.volname, group2) + + def test_subvolume_create_with_desired_data_pool_layout_in_group(self): + subvol1 = self._generate_random_subvolume_name() + subvol2 = self._generate_random_subvolume_name() + group = self._generate_random_group_name() + + # create group. this also helps set default pool layout for subvolumes + # created within the group. + self._fs_cmd("subvolumegroup", "create", self.volname, group) + + # create subvolume in group. + self._fs_cmd("subvolume", "create", self.volname, subvol1, "--group_name", group) + subvol1_path = self._get_subvolume_path(self.volname, subvol1, group_name=group) + + default_pool = self.mount_a.getfattr(subvol1_path, "ceph.dir.layout.pool") + new_pool = "new_pool" + self.assertNotEqual(default_pool, new_pool) + + # add data pool + self.fs.add_data_pool(new_pool) + + # create subvolume specifying the new data pool as its pool layout + self._fs_cmd("subvolume", "create", self.volname, subvol2, "--group_name", group, + "--pool_layout", new_pool) + subvol2_path = self._get_subvolume_path(self.volname, subvol2, group_name=group) + + desired_pool = self.mount_a.getfattr(subvol2_path, "ceph.dir.layout.pool") + self.assertEqual(desired_pool, new_pool) + + self._fs_cmd("subvolume", "rm", self.volname, subvol2, group) + self._fs_cmd("subvolume", "rm", self.volname, subvol1, group) + self._fs_cmd("subvolumegroup", "rm", self.volname, group) + def test_nonexistent_subvolme_group_rm(self): group = "non_existent_group" diff --git a/src/pybind/mgr/volumes/fs/subvolume.py b/src/pybind/mgr/volumes/fs/subvolume.py index 6849278ff9785..ee49edbcf2e58 100644 --- a/src/pybind/mgr/volumes/fs/subvolume.py +++ b/src/pybind/mgr/volumes/fs/subvolume.py @@ -61,7 +61,7 @@ class SubVolume(object): ### basic subvolume operations - def create_subvolume(self, spec, size=None, namespace_isolated=True, mode=0o755): + def create_subvolume(self, spec, size=None, namespace_isolated=True, mode=0o755, pool=None): """ Set up metadata, pools and auth for a subvolume. @@ -71,6 +71,7 @@ class SubVolume(object): :param spec: subvolume path specification :param size: In bytes, or None for no size limit :param namespace_isolated: If true, use separate RADOS namespace for this subvolume + :param pool: the RADOS pool where the data objects of the subvolumes will be stored :return: None """ subvolpath = spec.subvolume_path @@ -81,12 +82,15 @@ class SubVolume(object): if size is not None: self.fs.setxattr(subvolpath, 'ceph.quota.max_bytes', str(size).encode('utf-8'), 0) + if pool: + self.fs.setxattr(subvolpath, 'ceph.dir.layout.pool', pool.encode('utf-8'), 0) + xattr_key = xattr_val = None if namespace_isolated: # enforce security isolation, use separate namespace for this subvolume xattr_key = 'ceph.dir.layout.pool_namespace' xattr_val = spec.fs_namespace - else: + elif not pool: # If subvolume's namespace layout is not set, then the subvolume's pool # layout remains unset and will undesirably change with ancestor's # pool layout changes. @@ -169,9 +173,12 @@ class SubVolume(object): ### group operations - def create_group(self, spec, mode=0o755): + def create_group(self, spec, mode=0o755, pool=None): path = spec.group_path self._mkdir_p(path, mode) + if not pool: + pool = self._get_ancestor_xattr(path, "ceph.dir.layout.pool") + self.fs.setxattr(path, 'ceph.dir.layout.pool', pool.encode('utf-8'), 0) def remove_group(self, spec, force): path = spec.group_path @@ -197,12 +204,7 @@ class SubVolume(object): on the requested path, keep checking parents until we find it. """ try: - result = self.fs.getxattr(path, attr).decode('utf-8') - if result == "": - # Annoying! cephfs gives us empty instead of an error when attr not found - raise cephfs.NoData() - else: - return result + return self.fs.getxattr(path, attr).decode('utf-8') except cephfs.NoData: if path == "/": raise diff --git a/src/pybind/mgr/volumes/fs/volume.py b/src/pybind/mgr/volumes/fs/volume.py index 7b936b71b2e40..63ff2f834ddfb 100644 --- a/src/pybind/mgr/volumes/fs/volume.py +++ b/src/pybind/mgr/volumes/fs/volume.py @@ -171,7 +171,7 @@ class VolumeClient(object): ### subvolume operations - def create_subvolume(self, volname, subvolname, groupname, size): + def create_subvolume(self, volname, subvolname, groupname, size, pool=None): ret = 0, "", "" try: if not self.volume_exists(volname): @@ -184,7 +184,7 @@ class VolumeClient(object): raise VolumeException( -errno.ENOENT, "Subvolume group '{0}' not found, create it with " \ "`ceph fs subvolumegroup create` before creating subvolumes".format(groupname)) - sv.create_subvolume(spec, size) + sv.create_subvolume(spec, size, pool=pool) except VolumeException as ve: ret = self.volume_exception_to_retval(ve) return ret @@ -284,7 +284,7 @@ class VolumeClient(object): ### group operations - def create_subvolume_group(self, volname, groupname): + def create_subvolume_group(self, volname, groupname, pool=None): ret = 0, "", "" try: if not self.volume_exists(volname): @@ -295,7 +295,7 @@ class VolumeClient(object): # TODO: validate that subvol size fits in volume size with SubVolume(self.mgr, fs_name=volname) as sv: spec = SubvolumeSpec("", groupname) - sv.create_group(spec) + sv.create_group(spec, pool=pool) except VolumeException as ve: ret = self.volume_exception_to_retval(ve) return ret diff --git a/src/pybind/mgr/volumes/module.py b/src/pybind/mgr/volumes/module.py index 214b63fb4de25..98b18c67eba19 100644 --- a/src/pybind/mgr/volumes/module.py +++ b/src/pybind/mgr/volumes/module.py @@ -46,8 +46,10 @@ class Module(orchestrator.OrchestratorClientMixin, MgrModule): { 'cmd': 'fs subvolumegroup create ' 'name=vol_name,type=CephString ' - 'name=group_name,type=CephString ', - 'desc': "Create a CephFS subvolume group in a volume", + 'name=group_name,type=CephString ' + 'name=pool_layout,type=CephString,req=false ', + 'desc': "Create a CephFS subvolume group in a volume, and optionally, " + "with a specific data pool layout", 'perm': 'rw' }, { @@ -63,10 +65,11 @@ class Module(orchestrator.OrchestratorClientMixin, MgrModule): 'name=vol_name,type=CephString ' 'name=sub_name,type=CephString ' 'name=size,type=CephInt,req=false ' - 'name=group_name,type=CephString,req=false ', + 'name=group_name,type=CephString,req=false ' + 'name=pool_layout,type=CephString,req=false ', 'desc': "Create a CephFS subvolume in a volume, and optionally, " - "with a specific size (in bytes) and in a specific " - "subvolume group", + "with a specific size (in bytes), a specific data pool layout " + "and in a specific subvolume group", 'perm': 'rw' }, { @@ -197,8 +200,9 @@ class Module(orchestrator.OrchestratorClientMixin, MgrModule): """ vol_name = cmd['vol_name'] group_name = cmd['group_name'] + pool_layout = cmd.get('pool_layout', None) - return self.vc.create_subvolume_group(vol_name, group_name) + return self.vc.create_subvolume_group(vol_name, group_name, pool=pool_layout) def _cmd_fs_subvolumegroup_rm(self, inbuf, cmd): """ @@ -218,8 +222,9 @@ class Module(orchestrator.OrchestratorClientMixin, MgrModule): sub_name = cmd['sub_name'] size = cmd.get('size', None) group_name = cmd.get('group_name', None) + pool_layout = cmd.get('pool_layout', None) - return self.vc.create_subvolume(vol_name, sub_name, group_name, size) + return self.vc.create_subvolume(vol_name, sub_name, group_name, size, pool=pool_layout) def _cmd_fs_subvolume_rm(self, inbuf, cmd): """ -- 2.39.5