mgr/volumes: allow setting data pool layout

... of fs subvolumes and subvolume groups during their creation.

Fixes: https://tracker.ceph.com/issues/40431
Signed-off-by: Ramana Raja <rraja@redhat.com>
(cherry picked from commit 22ed8915281033c39df5e6ecceb5eebdc4e1c594)

diff --git a/qa/tasks/cephfs/test_volumes.py b/qa/tasks/cephfs/test_volumes.py
index 50217bd9b23a69af65aadd1f1786f35be32f386c..19dce198aa740c0b638a4a1fd95efe1fb69a60c2 100644 (file)
--- a/qa/tasks/cephfs/test_volumes.py
+++ b/qa/tasks/cephfs/test_volumes.py
@@ -38,6 +38,15 @@ class TestVolumes(CephFSTestCase):
         else:
             self.volname = result[0]['name']
 
+    def  _get_subvolume_path(self, vol_name, subvol_name, group_name=None):
+        args = ["subvolume", "getpath", vol_name, subvol_name]
+        if group_name:
+            args.append(group_name)
+        args = tuple(args)
+        path = self._fs_cmd(*args)
+        # remove the leading '/', and trailing whitespaces
+        return path[1:].rstrip()
+
     def _delete_test_volume(self):
         self._fs_cmd("volume", "rm", self.volname)
 
@@ -127,6 +136,64 @@ class TestVolumes(CephFSTestCase):
         # remove group
         self._fs_cmd("subvolumegroup", "rm", self.volname, group)
 
+    def test_subvolume_group_create_with_desired_data_pool_layout(self):
+        group1 = self._generate_random_group_name()
+        group2 = self._generate_random_group_name()
+
+        # create group
+        self._fs_cmd("subvolumegroup", "create", self.volname, group1)
+        group1_path = os.path.join('volumes', group1)
+
+        default_pool = self.mount_a.getfattr(group1_path, "ceph.dir.layout.pool")
+        new_pool = "new_pool"
+        self.assertNotEqual(default_pool, new_pool)
+
+        # add data pool
+        self.fs.add_data_pool(new_pool)
+
+        # create group specifying the new data pool as its pool layout
+        self._fs_cmd("subvolumegroup", "create", self.volname, group2,
+                     "--pool_layout", new_pool)
+        group2_path = os.path.join('volumes', group2)
+
+        desired_pool = self.mount_a.getfattr(group2_path, "ceph.dir.layout.pool")
+        self.assertEqual(desired_pool, new_pool)
+
+        self._fs_cmd("subvolumegroup", "rm", self.volname, group1)
+        self._fs_cmd("subvolumegroup", "rm", self.volname, group2)
+
+    def test_subvolume_create_with_desired_data_pool_layout_in_group(self):
+        subvol1 = self._generate_random_subvolume_name()
+        subvol2 = self._generate_random_subvolume_name()
+        group = self._generate_random_group_name()
+
+        # create group. this also helps set default pool layout for subvolumes
+        # created within the group.
+        self._fs_cmd("subvolumegroup", "create", self.volname, group)
+
+        # create subvolume in group.
+        self._fs_cmd("subvolume", "create", self.volname, subvol1, "--group_name", group)
+        subvol1_path = self._get_subvolume_path(self.volname, subvol1, group_name=group)
+
+        default_pool = self.mount_a.getfattr(subvol1_path, "ceph.dir.layout.pool")
+        new_pool = "new_pool"
+        self.assertNotEqual(default_pool, new_pool)
+
+        # add data pool
+        self.fs.add_data_pool(new_pool)
+
+        # create subvolume specifying the new data pool as its pool layout
+        self._fs_cmd("subvolume", "create", self.volname, subvol2, "--group_name", group,
+                     "--pool_layout", new_pool)
+        subvol2_path = self._get_subvolume_path(self.volname, subvol2, group_name=group)
+
+        desired_pool = self.mount_a.getfattr(subvol2_path, "ceph.dir.layout.pool")
+        self.assertEqual(desired_pool, new_pool)
+
+        self._fs_cmd("subvolume", "rm", self.volname, subvol2, group)
+        self._fs_cmd("subvolume", "rm", self.volname, subvol1, group)
+        self._fs_cmd("subvolumegroup", "rm", self.volname, group)
+
     def test_nonexistent_subvolme_group_rm(self):
         group = "non_existent_group"
 

diff --git a/src/pybind/mgr/volumes/fs/subvolume.py b/src/pybind/mgr/volumes/fs/subvolume.py
index 6849278ff97850dafd63e4513e5801e90ad3b208..ee49edbcf2e58010f4dfa6b78eca7043383c36a7 100644 (file)
--- a/src/pybind/mgr/volumes/fs/subvolume.py
+++ b/src/pybind/mgr/volumes/fs/subvolume.py
@@ -61,7 +61,7 @@ class SubVolume(object):
 
     ### basic subvolume operations
 
-    def create_subvolume(self, spec, size=None, namespace_isolated=True, mode=0o755):
+    def create_subvolume(self, spec, size=None, namespace_isolated=True, mode=0o755, pool=None):
         """
         Set up metadata, pools and auth for a subvolume.
 
@@ -71,6 +71,7 @@ class SubVolume(object):
         :param spec: subvolume path specification
         :param size: In bytes, or None for no size limit
         :param namespace_isolated: If true, use separate RADOS namespace for this subvolume
+        :param pool: the RADOS pool where the data objects of the subvolumes will be stored
         :return: None
         """
         subvolpath = spec.subvolume_path
@@ -81,12 +82,15 @@ class SubVolume(object):
         if size is not None:
             self.fs.setxattr(subvolpath, 'ceph.quota.max_bytes', str(size).encode('utf-8'), 0)
 
+        if pool:
+            self.fs.setxattr(subvolpath, 'ceph.dir.layout.pool', pool.encode('utf-8'), 0)
+
         xattr_key = xattr_val = None
         if namespace_isolated:
             # enforce security isolation, use separate namespace for this subvolume
             xattr_key = 'ceph.dir.layout.pool_namespace'
             xattr_val = spec.fs_namespace
-        else:
+        elif not pool:
             # If subvolume's namespace layout is not set, then the subvolume's pool
             # layout remains unset and will undesirably change with ancestor's
             # pool layout changes.
@@ -169,9 +173,12 @@ class SubVolume(object):
 
     ### group operations
 
-    def create_group(self, spec, mode=0o755):
+    def create_group(self, spec, mode=0o755, pool=None):
         path = spec.group_path
         self._mkdir_p(path, mode)
+        if not pool:
+            pool = self._get_ancestor_xattr(path, "ceph.dir.layout.pool")
+        self.fs.setxattr(path, 'ceph.dir.layout.pool', pool.encode('utf-8'), 0)
 
     def remove_group(self, spec, force):
         path = spec.group_path
@@ -197,12 +204,7 @@ class SubVolume(object):
         on the requested path, keep checking parents until we find it.
         """
         try:
-            result = self.fs.getxattr(path, attr).decode('utf-8')
-            if result == "":
-                # Annoying!  cephfs gives us empty instead of an error when attr not found
-                raise cephfs.NoData()
-            else:
-                return result
+            return self.fs.getxattr(path, attr).decode('utf-8')
         except cephfs.NoData:
             if path == "/":
                 raise

diff --git a/src/pybind/mgr/volumes/fs/volume.py b/src/pybind/mgr/volumes/fs/volume.py
index 7b936b71b2e400a30cc1c17ca198f48b4468d937..63ff2f834ddfb269886a851ffc2c505488b4a56d 100644 (file)
--- a/src/pybind/mgr/volumes/fs/volume.py
+++ b/src/pybind/mgr/volumes/fs/volume.py
@@ -171,7 +171,7 @@ class VolumeClient(object):
 
     ### subvolume operations
 
-    def create_subvolume(self, volname, subvolname, groupname, size):
+    def create_subvolume(self, volname, subvolname, groupname, size, pool=None):
         ret = 0, "", ""
         try:
             if not self.volume_exists(volname):
@@ -184,7 +184,7 @@ class VolumeClient(object):
                     raise VolumeException(
                         -errno.ENOENT, "Subvolume group '{0}' not found, create it with " \
                         "`ceph fs subvolumegroup create` before creating subvolumes".format(groupname))
-                sv.create_subvolume(spec, size)
+                sv.create_subvolume(spec, size, pool=pool)
         except VolumeException as ve:
             ret = self.volume_exception_to_retval(ve)
         return ret
@@ -284,7 +284,7 @@ class VolumeClient(object):
 
     ### group operations
 
-    def create_subvolume_group(self, volname, groupname):
+    def create_subvolume_group(self, volname, groupname, pool=None):
         ret = 0, "", ""
         try:
             if not self.volume_exists(volname):
@@ -295,7 +295,7 @@ class VolumeClient(object):
             # TODO: validate that subvol size fits in volume size
             with SubVolume(self.mgr, fs_name=volname) as sv:
                 spec = SubvolumeSpec("", groupname)
-                sv.create_group(spec)
+                sv.create_group(spec, pool=pool)
         except VolumeException as ve:
             ret = self.volume_exception_to_retval(ve)
         return ret

diff --git a/src/pybind/mgr/volumes/module.py b/src/pybind/mgr/volumes/module.py
index 214b63fb4de25a22c240cffffc1b00ad9ea30bc5..98b18c67eba19fbd070fe079465f89af44c1bcb5 100644 (file)
--- a/src/pybind/mgr/volumes/module.py
+++ b/src/pybind/mgr/volumes/module.py
@@ -46,8 +46,10 @@ class Module(orchestrator.OrchestratorClientMixin, MgrModule):
         {
             'cmd': 'fs subvolumegroup create '
                    'name=vol_name,type=CephString '
-                   'name=group_name,type=CephString ',
-            'desc': "Create a CephFS subvolume group in a volume",
+                   'name=group_name,type=CephString '
+                   'name=pool_layout,type=CephString,req=false ',
+            'desc': "Create a CephFS subvolume group in a volume, and optionally, "
+                    "with a specific data pool layout",
             'perm': 'rw'
         },
         {
@@ -63,10 +65,11 @@ class Module(orchestrator.OrchestratorClientMixin, MgrModule):
                    'name=vol_name,type=CephString '
                    'name=sub_name,type=CephString '
                    'name=size,type=CephInt,req=false '
-                   'name=group_name,type=CephString,req=false ',
+                   'name=group_name,type=CephString,req=false '
+                   'name=pool_layout,type=CephString,req=false ',
             'desc': "Create a CephFS subvolume in a volume, and optionally, "
-                    "with a specific size (in bytes) and in a specific "
-                    "subvolume group",
+                    "with a specific size (in bytes), a specific data pool layout "
+                    "and in a specific subvolume group",
             'perm': 'rw'
         },
         {
@@ -197,8 +200,9 @@ class Module(orchestrator.OrchestratorClientMixin, MgrModule):
         """
         vol_name = cmd['vol_name']
         group_name = cmd['group_name']
+        pool_layout = cmd.get('pool_layout', None)
 
-        return self.vc.create_subvolume_group(vol_name, group_name)
+        return self.vc.create_subvolume_group(vol_name, group_name, pool=pool_layout)
 
     def _cmd_fs_subvolumegroup_rm(self, inbuf, cmd):
         """
@@ -218,8 +222,9 @@ class Module(orchestrator.OrchestratorClientMixin, MgrModule):
         sub_name = cmd['sub_name']
         size = cmd.get('size', None)
         group_name = cmd.get('group_name', None)
+        pool_layout = cmd.get('pool_layout', None)
 
-        return self.vc.create_subvolume(vol_name, sub_name, group_name, size)
+        return self.vc.create_subvolume(vol_name, sub_name, group_name, size, pool=pool_layout)
 
     def _cmd_fs_subvolume_rm(self, inbuf, cmd):
         """