From 982041e89d6db6aa764e287a05f1c2937efa719a Mon Sep 17 00:00:00 2001 From: Patrick Donnelly Date: Tue, 9 Jun 2020 21:14:23 -0700 Subject: [PATCH] pybind/mgr/volumes: wire up pinning subvolumes/subvolumegroups Fixes: https://tracker.ceph.com/issues/41541 Signed-off-by: Patrick Donnelly --- qa/tasks/cephfs/test_volumes.py | 39 +++++++++++++++++++ src/pybind/mgr/volumes/fs/operations/group.py | 4 ++ .../mgr/volumes/fs/operations/pin_util.py | 34 ++++++++++++++++ .../mgr/volumes/fs/operations/template.py | 10 +++++ .../fs/operations/versions/subvolume_base.py | 4 ++ src/pybind/mgr/volumes/fs/volume.py | 34 ++++++++++++++++ src/pybind/mgr/volumes/module.py | 30 ++++++++++++++ 7 files changed, 155 insertions(+) create mode 100644 src/pybind/mgr/volumes/fs/operations/pin_util.py diff --git a/qa/tasks/cephfs/test_volumes.py b/qa/tasks/cephfs/test_volumes.py index 6f895f9cd52..97b8c9b2664 100644 --- a/qa/tasks/cephfs/test_volumes.py +++ b/qa/tasks/cephfs/test_volumes.py @@ -21,6 +21,7 @@ class TestVolumes(CephFSTestCase): # for filling subvolume with data CLIENTS_REQUIRED = 1 + MDSS_REQUIRED = 2 # io defaults DEFAULT_FILE_SIZE = 1 # MB @@ -590,6 +591,44 @@ class TestVolumes(CephFSTestCase): # verify trash dir is clean self._wait_for_trash_empty() + def test_subvolume_pin_export(self): + self.fs.set_max_mds(2) + status = self.fs.wait_for_daemons() + + subvolume = self._generate_random_subvolume_name() + self._fs_cmd("subvolume", "create", self.volname, subvolume) + self._fs_cmd("subvolume", "pin", self.volname, subvolume, "export", "1") + path = self._fs_cmd("subvolume", "getpath", self.volname, subvolume) + path = os.path.dirname(path) # get subvolume path + + subtrees = self._get_subtrees(status=status, rank=1) + self._wait_subtrees([(path, 1)], status=status) + + def test_subvolumegroup_pin_distributed(self): + self.fs.set_max_mds(2) + status = self.fs.wait_for_daemons() + self.config_set('mds', 'mds_export_ephemeral_distributed', True) + + group = "pinme" + self._fs_cmd("subvolumegroup", "create", self.volname, group) + self._fs_cmd("subvolumegroup", "pin", self.volname, group, "distributed", "True") + # (no effect on distribution) pin the group directory to 0 so rank 0 has all subtree bounds visible + self._fs_cmd("subvolumegroup", "pin", self.volname, group, "export", "0") + for i in range(10): + subvolume = self._generate_random_subvolume_name() + self._fs_cmd("subvolume", "create", self.volname, subvolume, "--group_name", group) + self._wait_distributed_subtrees(10, status=status) + + def test_subvolume_pin_random(self): + self.fs.set_max_mds(2) + status = self.fs.wait_for_daemons() + self.config_set('mds', 'mds_export_ephemeral_random', True) + + subvolume = self._generate_random_subvolume_name() + self._fs_cmd("subvolume", "create", self.volname, subvolume) + self._fs_cmd("subvolume", "pin", self.volname, subvolume, "random", ".01") + # no verification + def test_subvolume_create_isolated_namespace(self): """ Create subvolume in separate rados namespace diff --git a/src/pybind/mgr/volumes/fs/operations/group.py b/src/pybind/mgr/volumes/fs/operations/group.py index 29bc8b73c52..8e6eaf936bb 100644 --- a/src/pybind/mgr/volumes/fs/operations/group.py +++ b/src/pybind/mgr/volumes/fs/operations/group.py @@ -6,6 +6,7 @@ from contextlib import contextmanager import cephfs from .snapshot_util import mksnap, rmsnap +from .pin_util import pin from .template import GroupTemplate from ..fs_util import listdir, get_ancestor_xattr from ..exception import VolumeException @@ -61,6 +62,9 @@ class Group(GroupTemplate): return [] raise + def pin(self, pin_type, pin_setting): + return pin(self.fs, self.path, pin_type, pin_setting) + def create_snapshot(self, snapname): snappath = os.path.join(self.path, self.vol_spec.snapshot_dir_prefix.encode('utf-8'), diff --git a/src/pybind/mgr/volumes/fs/operations/pin_util.py b/src/pybind/mgr/volumes/fs/operations/pin_util.py new file mode 100644 index 00000000000..9ea79e546e2 --- /dev/null +++ b/src/pybind/mgr/volumes/fs/operations/pin_util.py @@ -0,0 +1,34 @@ +import os +import errno + +import cephfs + +from ..exception import VolumeException +from distutils.util import strtobool + +_pin_value = { + "export": lambda x: int(x), + "distributed": lambda x: int(strtobool(x)), + "random": lambda x: float(x), +} +_pin_xattr = { + "export": "ceph.dir.pin", + "distributed": "ceph.dir.pin.distributed", + "random": "ceph.dir.pin.random", +} + +def pin(fs, path, pin_type, pin_setting): + """ + Set a pin on a directory. + """ + assert pin_type in _pin_xattr + + try: + pin_setting = _pin_value[pin_type](pin_setting) + except ValueError as e: + raise VolumeException(-errno.EINVAL, f"pin value wrong type: {pin_setting}") + + try: + fs.setxattr(path, _pin_xattr[pin_type], str(pin_setting).encode('utf-8'), 0) + except cephfs.Error as e: + raise VolumeException(-e.args[0], e.args[1]) diff --git a/src/pybind/mgr/volumes/fs/operations/template.py b/src/pybind/mgr/volumes/fs/operations/template.py index 362d62819e8..2b95bb80696 100644 --- a/src/pybind/mgr/volumes/fs/operations/template.py +++ b/src/pybind/mgr/volumes/fs/operations/template.py @@ -95,6 +95,16 @@ class SubvolumeTemplate(object): """ raise VolumeException(-errno.ENOTSUP, "operation not supported.") + def pin(self, pin_type, pin_setting): + """ + pin a subvolume + + :param pin_type: type of pin + :param pin_setting: setting for pin + :return: None + """ + raise VolumeException(-errno.ENOTSUP, "operation not supported.") + def create_snapshot(self, snapname): """ snapshot a subvolume. diff --git a/src/pybind/mgr/volumes/fs/operations/versions/subvolume_base.py b/src/pybind/mgr/volumes/fs/operations/versions/subvolume_base.py index 30c9235a871..3801a6c67e0 100644 --- a/src/pybind/mgr/volumes/fs/operations/versions/subvolume_base.py +++ b/src/pybind/mgr/volumes/fs/operations/versions/subvolume_base.py @@ -6,6 +6,7 @@ from hashlib import md5 import cephfs +from ..pin_util import pin from .metadata_manager import MetadataManager from ..trash import create_trashcan, open_trashcan from ...fs_util import get_ancestor_xattr @@ -195,6 +196,9 @@ class SubvolumeBase(object): raise VolumeException(-e.args[0], "Cannot set new size for the subvolume. '{0}'".format(e.args[1])) return newsize, subvolstat.st_size + def pin(self, pin_type, pin_setting): + return pin(self.fs, self.base_path, pin_type, pin_setting) + def init_config(self, version, subvolume_type, subvolume_path, subvolume_state): self.metadata_mgr.init(version, subvolume_type, subvolume_path, subvolume_state) self.metadata_mgr.flush() diff --git a/src/pybind/mgr/volumes/fs/volume.py b/src/pybind/mgr/volumes/fs/volume.py index 836cce0b866..c56960766d5 100644 --- a/src/pybind/mgr/volumes/fs/volume.py +++ b/src/pybind/mgr/volumes/fs/volume.py @@ -203,6 +203,24 @@ class VolumeClient(CephfsClient): ret = self.volume_exception_to_retval(ve) return ret + def subvolume_pin(self, **kwargs): + ret = 0, "", "" + volname = kwargs['vol_name'] + subvolname = kwargs['sub_name'] + pin_type = kwargs['pin_type'] + pin_setting = kwargs['pin_setting'] + groupname = kwargs['group_name'] + + try: + with open_volume(self, volname) as fs_handle: + with open_group(fs_handle, self.volspec, groupname) as group: + with open_subvol(fs_handle, self.volspec, group, subvolname) as subvolume: + subvolume.pin(pin_type, pin_setting) + ret = 0, json.dumps({}), "" + except VolumeException as ve: + ret = self.volume_exception_to_retval(ve) + return ret + def subvolume_getpath(self, **kwargs): ret = None volname = kwargs['vol_name'] @@ -502,6 +520,22 @@ class VolumeClient(CephfsClient): ret = self.volume_exception_to_retval(ve) return ret + def pin_subvolume_group(self, **kwargs): + ret = 0, "", "" + volname = kwargs['vol_name'] + groupname = kwargs['group_name'] + pin_type = kwargs['pin_type'] + pin_setting = kwargs['pin_setting'] + + try: + with open_volume(self, volname) as fs_handle: + with open_group(fs_handle, self.volspec, groupname) as group: + group.pin(pin_type, pin_setting) + ret = 0, json.dumps({}), "" + except VolumeException as ve: + ret = self.volume_exception_to_retval(ve) + return ret + ### group snapshot def create_subvolume_group_snapshot(self, **kwargs): diff --git a/src/pybind/mgr/volumes/module.py b/src/pybind/mgr/volumes/module.py index a947b948c95..fd860e4f798 100644 --- a/src/pybind/mgr/volumes/module.py +++ b/src/pybind/mgr/volumes/module.py @@ -113,6 +113,15 @@ class Module(orchestrator.OrchestratorClientMixin, MgrModule): "and optionally, in a specific subvolume group", 'perm': 'r' }, + { + 'cmd': 'fs subvolumegroup pin' + ' name=vol_name,type=CephString' + ' name=group_name,type=CephString,req=true' + ' name=pin_type,type=CephChoices,strings=export|distributed|random' + ' name=pin_setting,type=CephString,req=true', + 'desc': "Set MDS pinning policy for subvolumegroup", + 'perm': 'rw' + }, { 'cmd': 'fs subvolumegroup snapshot ls ' 'name=vol_name,type=CephString ' @@ -186,6 +195,16 @@ class Module(orchestrator.OrchestratorClientMixin, MgrModule): 'desc': "Resize a CephFS subvolume", 'perm': 'rw' }, + { + 'cmd': 'fs subvolume pin' + ' name=vol_name,type=CephString' + ' name=sub_name,type=CephString' + ' name=pin_type,type=CephChoices,strings=export|distributed|random' + ' name=pin_setting,type=CephString,req=true' + ' name=group_name,type=CephString,req=false', + 'desc': "Set MDS pinning policy for subvolume", + 'perm': 'rw' + }, { 'cmd': 'fs subvolume snapshot protect ' 'name=vol_name,type=CephString ' @@ -384,6 +403,11 @@ class Module(orchestrator.OrchestratorClientMixin, MgrModule): sub_name=cmd['sub_name'], group_name=cmd.get('group_name', None)) + def _cmd_fs_subvolumegroup_pin(self, inbuf, cmd): + return self.vc.pin_subvolume_group(vol_name=cmd['vol_name'], + group_name=cmd['group_name'], pin_type=cmd['pin_type'], + pin_setting=cmd['pin_setting']) + def _cmd_fs_subvolumegroup_snapshot_create(self, inbuf, cmd): return self.vc.create_subvolume_group_snapshot(vol_name=cmd['vol_name'], group_name=cmd['group_name'], @@ -428,6 +452,12 @@ class Module(orchestrator.OrchestratorClientMixin, MgrModule): new_size=cmd['new_size'], group_name=cmd.get('group_name', None), no_shrink=cmd.get('no_shrink', False)) + def _cmd_fs_subvolume_pin(self, inbuf, cmd): + return self.vc.subvolume_pin(vol_name=cmd['vol_name'], + sub_name=cmd['sub_name'], pin_type=cmd['pin_type'], + pin_setting=cmd['pin_setting'], + group_name=cmd.get('group_name', None)) + def _cmd_fs_subvolume_snapshot_protect(self, inbuf, cmd): return self.vc.protect_subvolume_snapshot(vol_name=cmd['vol_name'], sub_name=cmd['sub_name'], snap_name=cmd['snap_name'], group_name=cmd.get('group_name', None)) -- 2.39.5