pybind/mgr/volumes: wire up pinning subvolumes/subvolumegroups

Fixes: https://tracker.ceph.com/issues/41541
Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
(cherry picked from commit 982041e89d6db6aa764e287a05f1c2937efa719a)

diff --git a/qa/tasks/cephfs/test_volumes.py b/qa/tasks/cephfs/test_volumes.py
index 11c23605ae8c99ad6875214ae76f841030bac9f0..dec08bf64de2c3473095d320a923ea2510494c41 100644 (file)
--- a/qa/tasks/cephfs/test_volumes.py
+++ b/qa/tasks/cephfs/test_volumes.py
@@ -21,6 +21,7 @@ class TestVolumes(CephFSTestCase):
 
     # for filling subvolume with data
     CLIENTS_REQUIRED = 1
+    MDSS_REQUIRED = 2
 
     # io defaults
     DEFAULT_FILE_SIZE = 1 # MB
@@ -592,6 +593,44 @@ class TestVolumes(CephFSTestCase):
         # verify trash dir is clean
         self._wait_for_trash_empty()
 
+    def test_subvolume_pin_export(self):
+        self.fs.set_max_mds(2)
+        status = self.fs.wait_for_daemons()
+
+        subvolume = self._generate_random_subvolume_name()
+        self._fs_cmd("subvolume", "create", self.volname, subvolume)
+        self._fs_cmd("subvolume", "pin", self.volname, subvolume, "export", "1")
+        path = self._fs_cmd("subvolume", "getpath", self.volname, subvolume)
+        path = os.path.dirname(path) # get subvolume path
+
+        subtrees = self._get_subtrees(status=status, rank=1)
+        self._wait_subtrees([(path, 1)], status=status)
+
+    def test_subvolumegroup_pin_distributed(self):
+        self.fs.set_max_mds(2)
+        status = self.fs.wait_for_daemons()
+        self.config_set('mds', 'mds_export_ephemeral_distributed', True)
+
+        group = "pinme"
+        self._fs_cmd("subvolumegroup", "create", self.volname, group)
+        self._fs_cmd("subvolumegroup", "pin", self.volname, group, "distributed", "True")
+        # (no effect on distribution) pin the group directory to 0 so rank 0 has all subtree bounds visible
+        self._fs_cmd("subvolumegroup", "pin", self.volname, group, "export", "0")
+        for i in range(10):
+            subvolume = self._generate_random_subvolume_name()
+            self._fs_cmd("subvolume", "create", self.volname, subvolume, "--group_name", group)
+        self._wait_distributed_subtrees(10, status=status)
+
+    def test_subvolume_pin_random(self):
+        self.fs.set_max_mds(2)
+        status = self.fs.wait_for_daemons()
+        self.config_set('mds', 'mds_export_ephemeral_random', True)
+
+        subvolume = self._generate_random_subvolume_name()
+        self._fs_cmd("subvolume", "create", self.volname, subvolume)
+        self._fs_cmd("subvolume", "pin", self.volname, subvolume, "random", ".01")
+        # no verification
+
     def test_subvolume_create_isolated_namespace(self):
         """
         Create subvolume in separate rados namespace

diff --git a/src/pybind/mgr/volumes/fs/operations/group.py b/src/pybind/mgr/volumes/fs/operations/group.py
index 29bc8b73c52fc1e7d712fbca5a3a75fd6ffef14c..8e6eaf936bb1c20134e326c03ea7851409e70486 100644 (file)
--- a/src/pybind/mgr/volumes/fs/operations/group.py
+++ b/src/pybind/mgr/volumes/fs/operations/group.py
@@ -6,6 +6,7 @@ from contextlib import contextmanager
 import cephfs
 
 from .snapshot_util import mksnap, rmsnap
+from .pin_util import pin
 from .template import GroupTemplate
 from ..fs_util import listdir, get_ancestor_xattr
 from ..exception import VolumeException
@@ -61,6 +62,9 @@ class Group(GroupTemplate):
                 return []
             raise
 
+    def pin(self, pin_type, pin_setting):
+        return pin(self.fs, self.path, pin_type, pin_setting)
+
     def create_snapshot(self, snapname):
         snappath = os.path.join(self.path,
                                 self.vol_spec.snapshot_dir_prefix.encode('utf-8'),

diff --git a/src/pybind/mgr/volumes/fs/operations/pin_util.py b/src/pybind/mgr/volumes/fs/operations/pin_util.py
new file mode 100644 (file)
index 0000000..9ea79e5
--- /dev/null
+++ b/src/pybind/mgr/volumes/fs/operations/pin_util.py
@@ -0,0 +1,34 @@
+import os
+import errno
+
+import cephfs
+
+from ..exception import VolumeException
+from distutils.util import strtobool
+
+_pin_value = {
+    "export": lambda x: int(x),
+    "distributed": lambda x: int(strtobool(x)),
+    "random": lambda x: float(x),
+}
+_pin_xattr = {
+    "export": "ceph.dir.pin",
+    "distributed": "ceph.dir.pin.distributed",
+    "random": "ceph.dir.pin.random",
+}
+
+def pin(fs, path, pin_type, pin_setting):
+    """
+    Set a pin on a directory.
+    """
+    assert pin_type in _pin_xattr
+
+    try:
+        pin_setting = _pin_value[pin_type](pin_setting)
+    except ValueError as e:
+        raise VolumeException(-errno.EINVAL, f"pin value wrong type: {pin_setting}")
+
+    try:
+        fs.setxattr(path, _pin_xattr[pin_type], str(pin_setting).encode('utf-8'), 0)
+    except cephfs.Error as e:
+        raise VolumeException(-e.args[0], e.args[1])

diff --git a/src/pybind/mgr/volumes/fs/operations/template.py b/src/pybind/mgr/volumes/fs/operations/template.py
index 362d62819e872db2b87abca5eab106d297e75854..2b95bb80696ae66042d3b77a328d0a6118ddc4a1 100644 (file)
--- a/src/pybind/mgr/volumes/fs/operations/template.py
+++ b/src/pybind/mgr/volumes/fs/operations/template.py
@@ -95,6 +95,16 @@ class SubvolumeTemplate(object):
         """
         raise VolumeException(-errno.ENOTSUP, "operation not supported.")
 
+    def pin(self, pin_type, pin_setting):
+        """
+        pin a subvolume
+
+        :param pin_type: type of pin
+        :param pin_setting: setting for pin
+        :return: None
+        """
+        raise VolumeException(-errno.ENOTSUP, "operation not supported.")
+
     def create_snapshot(self, snapname):
         """
         snapshot a subvolume.

diff --git a/src/pybind/mgr/volumes/fs/operations/versions/subvolume_base.py b/src/pybind/mgr/volumes/fs/operations/versions/subvolume_base.py
index 30c9235a87183c2971372b9339e7571eee2a95be..3801a6c67e0e7b1b172727d8d76d83331fcfcf45 100644 (file)
--- a/src/pybind/mgr/volumes/fs/operations/versions/subvolume_base.py
+++ b/src/pybind/mgr/volumes/fs/operations/versions/subvolume_base.py
@@ -6,6 +6,7 @@ from hashlib import md5
 
 import cephfs
 
+from ..pin_util import pin
 from .metadata_manager import MetadataManager
 from ..trash import create_trashcan, open_trashcan
 from ...fs_util import get_ancestor_xattr
@@ -195,6 +196,9 @@ class SubvolumeBase(object):
                 raise VolumeException(-e.args[0], "Cannot set new size for the subvolume. '{0}'".format(e.args[1]))
         return newsize, subvolstat.st_size
 
+    def pin(self, pin_type, pin_setting):
+        return pin(self.fs, self.base_path, pin_type, pin_setting)
+
     def init_config(self, version, subvolume_type, subvolume_path, subvolume_state):
         self.metadata_mgr.init(version, subvolume_type, subvolume_path, subvolume_state)
         self.metadata_mgr.flush()

diff --git a/src/pybind/mgr/volumes/fs/volume.py b/src/pybind/mgr/volumes/fs/volume.py
index a6ff65d4295fd51cd070a033be2e776babce8c67..0a8347a146df454726b8fad2b3adb1e3ae8c89a5 100644 (file)
--- a/src/pybind/mgr/volumes/fs/volume.py
+++ b/src/pybind/mgr/volumes/fs/volume.py
@@ -203,6 +203,24 @@ class VolumeClient(object):
             ret = self.volume_exception_to_retval(ve)
         return ret
 
+    def subvolume_pin(self, **kwargs):
+        ret         = 0, "", ""
+        volname     = kwargs['vol_name']
+        subvolname  = kwargs['sub_name']
+        pin_type    = kwargs['pin_type']
+        pin_setting = kwargs['pin_setting']
+        groupname   = kwargs['group_name']
+
+        try:
+            with open_volume(self, volname) as fs_handle:
+                with open_group(fs_handle, self.volspec, groupname) as group:
+                    with open_subvol(fs_handle, self.volspec, group, subvolname) as subvolume:
+                        subvolume.pin(pin_type, pin_setting)
+                        ret = 0, json.dumps({}), ""
+        except VolumeException as ve:
+            ret = self.volume_exception_to_retval(ve)
+        return ret
+
     def subvolume_getpath(self, **kwargs):
         ret        = None
         volname    = kwargs['vol_name']
@@ -502,6 +520,22 @@ class VolumeClient(object):
                 ret = self.volume_exception_to_retval(ve)
         return ret
 
+    def pin_subvolume_group(self, **kwargs):
+        ret           = 0, "", ""
+        volname       = kwargs['vol_name']
+        groupname     = kwargs['group_name']
+        pin_type      = kwargs['pin_type']
+        pin_setting   = kwargs['pin_setting']
+
+        try:
+            with open_volume(self, volname) as fs_handle:
+                with open_group(fs_handle, self.volspec, groupname) as group:
+                    group.pin(pin_type, pin_setting)
+                    ret = 0, json.dumps({}), ""
+        except VolumeException as ve:
+            ret = self.volume_exception_to_retval(ve)
+        return ret
+
     ### group snapshot
 
     def create_subvolume_group_snapshot(self, **kwargs):

diff --git a/src/pybind/mgr/volumes/module.py b/src/pybind/mgr/volumes/module.py
index 86ee8a0e338871ccdbd961fff180230c14d4bd84..671fe3274aa2086c21e8eb2b67dc9afb23b6f93b 100644 (file)
--- a/src/pybind/mgr/volumes/module.py
+++ b/src/pybind/mgr/volumes/module.py
@@ -112,6 +112,15 @@ class Module(orchestrator.OrchestratorClientMixin, MgrModule):
                     "and optionally, in a specific subvolume group",
             'perm': 'r'
         },
+        {
+            'cmd': 'fs subvolumegroup pin'
+                   ' name=vol_name,type=CephString'
+                   ' name=group_name,type=CephString,req=true'
+                   ' name=pin_type,type=CephChoices,strings=export|distributed|random'
+                   ' name=pin_setting,type=CephString,req=true',
+            'desc': "Set MDS pinning policy for subvolumegroup",
+            'perm': 'rw'
+        },
         {
             'cmd': 'fs subvolumegroup snapshot ls '
                    'name=vol_name,type=CephString '
@@ -185,6 +194,16 @@ class Module(orchestrator.OrchestratorClientMixin, MgrModule):
             'desc': "Resize a CephFS subvolume",
             'perm': 'rw'
         },
+        {
+            'cmd': 'fs subvolume pin'
+                   ' name=vol_name,type=CephString'
+                   ' name=sub_name,type=CephString'
+                   ' name=pin_type,type=CephChoices,strings=export|distributed|random'
+                   ' name=pin_setting,type=CephString,req=true'
+                   ' name=group_name,type=CephString,req=false',
+            'desc': "Set MDS pinning policy for subvolume",
+            'perm': 'rw'
+        },
         {
             'cmd': 'fs subvolume snapshot protect '
                    'name=vol_name,type=CephString '
@@ -344,6 +363,11 @@ class Module(orchestrator.OrchestratorClientMixin, MgrModule):
                                       sub_name=cmd['sub_name'],
                                       group_name=cmd.get('group_name', None))
 
+    def _cmd_fs_subvolumegroup_pin(self, inbuf, cmd):
+        return self.vc.pin_subvolume_group(vol_name=cmd['vol_name'],
+            group_name=cmd['group_name'], pin_type=cmd['pin_type'],
+            pin_setting=cmd['pin_setting'])
+
     def _cmd_fs_subvolumegroup_snapshot_create(self, inbuf, cmd):
         return self.vc.create_subvolume_group_snapshot(vol_name=cmd['vol_name'],
                                                        group_name=cmd['group_name'],
@@ -388,6 +412,12 @@ class Module(orchestrator.OrchestratorClientMixin, MgrModule):
                                         new_size=cmd['new_size'], group_name=cmd.get('group_name', None),
                                         no_shrink=cmd.get('no_shrink', False))
 
+    def _cmd_fs_subvolume_pin(self, inbuf, cmd):
+        return self.vc.subvolume_pin(vol_name=cmd['vol_name'],
+            sub_name=cmd['sub_name'], pin_type=cmd['pin_type'],
+            pin_setting=cmd['pin_setting'],
+            group_name=cmd.get('group_name', None))
+
     def _cmd_fs_subvolume_snapshot_protect(self, inbuf, cmd):
         return self.vc.protect_subvolume_snapshot(vol_name=cmd['vol_name'], sub_name=cmd['sub_name'],
                                                   snap_name=cmd['snap_name'], group_name=cmd.get('group_name', None))