From 24ae3e0563e951dbf3fa89fdc0583c88a5784e07 Mon Sep 17 00:00:00 2001 From: Christopher Hoffman Date: Mon, 24 Feb 2025 16:08:12 +0000 Subject: [PATCH] mgr/volumes: Add enctag to subvol Add functionality to support enctag for subvols. This will be useful for app or administrator to know which master key to use. Fixes: https://tracker.ceph.com/issues/69693 Signed-off-by: Christopher Hoffman --- .../mgr/volumes/fs/operations/subvolume.py | 5 +- .../mgr/volumes/fs/operations/template.py | 3 + .../fs/operations/versions/subvolume_base.py | 24 +++++++ .../fs/operations/versions/subvolume_v1.py | 3 +- .../fs/operations/versions/subvolume_v2.py | 3 +- src/pybind/mgr/volumes/fs/volume.py | 68 +++++++++++++++++- src/pybind/mgr/volumes/module.py | 50 ++++++++++++- src/python-common/ceph/fs/enctag.py | 72 +++++++++++++++++++ 8 files changed, 221 insertions(+), 7 deletions(-) create mode 100644 src/python-common/ceph/fs/enctag.py diff --git a/src/pybind/mgr/volumes/fs/operations/subvolume.py b/src/pybind/mgr/volumes/fs/operations/subvolume.py index 3c8d2241afb..4182e4f7571 100644 --- a/src/pybind/mgr/volumes/fs/operations/subvolume.py +++ b/src/pybind/mgr/volumes/fs/operations/subvolume.py @@ -5,7 +5,7 @@ from .group import open_group from .template import SubvolumeOpType from .versions import loaded_subvolumes -def create_subvol(mgr, fs, vol_spec, group, subvolname, size, isolate_nspace, pool, mode, uid, gid, earmark, normalization, casesensitive): +def create_subvol(mgr, fs, vol_spec, group, subvolname, size, isolate_nspace, pool, mode, uid, gid, earmark, normalization, casesensitive, enctag): """ create a subvolume (create a subvolume with the max known version). @@ -21,10 +21,11 @@ def create_subvol(mgr, fs, vol_spec, group, subvolname, size, isolate_nspace, po :param earmark: metadata string to identify if subvolume is associated with nfs/smb :param normalization: the unicode normalization form to use (nfd, nfc, nfkd or nfkc) :param casesensitive: whether to make the subvolume case insensitive or not + :param enctag: metadata string to associate subvolume with an encryption tag :return: None """ subvolume = loaded_subvolumes.get_subvolume_object_max(mgr, fs, vol_spec, group, subvolname) - subvolume.create(size, isolate_nspace, pool, mode, uid, gid, earmark, normalization, casesensitive) + subvolume.create(size, isolate_nspace, pool, mode, uid, gid, earmark, normalization, casesensitive, enctag) def create_clone(mgr, fs, vol_spec, group, subvolname, pool, source_volume, source_subvolume, snapname): diff --git a/src/pybind/mgr/volumes/fs/operations/template.py b/src/pybind/mgr/volumes/fs/operations/template.py index 7aa953045a1..428a484786f 100644 --- a/src/pybind/mgr/volumes/fs/operations/template.py +++ b/src/pybind/mgr/volumes/fs/operations/template.py @@ -73,6 +73,9 @@ class SubvolumeOpType(Enum): EARMARK_GET = 'earmark-get' EARMARK_SET = 'earmark-set' EARMARK_CLEAR = 'earmark-clear' + ENCTAG_GET = 'enctag-get' + ENCTAG_SET = 'enctag-set' + ENCTAG_CLEAR = 'enctag-clear' class SubvolumeTemplate(object): VERSION = None # type: int diff --git a/src/pybind/mgr/volumes/fs/operations/versions/subvolume_base.py b/src/pybind/mgr/volumes/fs/operations/versions/subvolume_base.py index 5dc6d14b734..99906fbae1d 100644 --- a/src/pybind/mgr/volumes/fs/operations/versions/subvolume_base.py +++ b/src/pybind/mgr/volumes/fs/operations/versions/subvolume_base.py @@ -20,6 +20,7 @@ from .auth_metadata import AuthMetadataManager from .subvolume_attrs import SubvolumeStates from ceph.fs.earmarking import CephFSVolumeEarmarking, EarmarkException +from ceph.fs.enctag import CephFSVolumeEncryptionTag, EncryptionTagException log = logging.getLogger(__name__) @@ -216,6 +217,14 @@ class SubvolumeBase(object): except cephfs.NoData: attrs["casesensitive"] = True + try: + fs_enctag = CephFSVolumeEncryptionTag(self.fs, pathname) + attrs["enctag"] = fs_enctag.get_tag() + except cephfs.NoData: + attrs["enctag"] = '' + except EncryptionTagException: + attrs["enctag"] = '' + return attrs def set_attrs(self, path, attrs): @@ -321,6 +330,12 @@ class SubvolumeBase(object): except cephfs.Error as e: raise VolumeException(-e.args[0], e.args[1]) + # set encryption tag string identifier + enctag = attrs.get("enctag", None) + if enctag is not None: + fs_enctag = CephFSVolumeEncryptionTag(self.fs, path) + fs_enctag.set_tag(enctag) + def _resize(self, path, newsize, noshrink): try: newsize = int(newsize) @@ -522,6 +537,14 @@ class SubvolumeBase(object): except cephfs.NoData: casesensitive = True + try: + fs_enctag = CephFSVolumeEncryptionTag(self.fs, subvolpath) + enctag = fs_enctag.get_tag() + except cephfs.NoData: + enctag = '' + except EncryptionTagException: + enctag = '' + subvol_info = { 'path': subvolpath, 'type': etype.value, @@ -544,6 +567,7 @@ class SubvolumeBase(object): 'earmark': earmark, 'normalization': normalization, 'casesensitive': casesensitive, + 'enctag': enctag, } subvol_src_info = self._get_clone_source() diff --git a/src/pybind/mgr/volumes/fs/operations/versions/subvolume_v1.py b/src/pybind/mgr/volumes/fs/operations/versions/subvolume_v1.py index 7b7c0751f96..506796a583f 100644 --- a/src/pybind/mgr/volumes/fs/operations/versions/subvolume_v1.py +++ b/src/pybind/mgr/volumes/fs/operations/versions/subvolume_v1.py @@ -85,7 +85,7 @@ class SubvolumeV1(SubvolumeBase, SubvolumeTemplate): """ Path to user data directory within a subvolume snapshot named 'snapname' """ return self.snapshot_path(snapname) - def create(self, size, isolate_nspace, pool, mode, uid, gid, earmark, normalization, casesensitive): + def create(self, size, isolate_nspace, pool, mode, uid, gid, earmark, normalization, casesensitive, enctag): subvolume_type = SubvolumeTypes.TYPE_NORMAL try: initial_state = SubvolumeOpSm.get_init_state(subvolume_type) @@ -107,6 +107,7 @@ class SubvolumeV1(SubvolumeBase, SubvolumeTemplate): 'earmark': earmark, 'normalization': normalization, 'casesensitive': casesensitive, + 'enctag': enctag, } self.set_attrs(subvol_path, attrs) diff --git a/src/pybind/mgr/volumes/fs/operations/versions/subvolume_v2.py b/src/pybind/mgr/volumes/fs/operations/versions/subvolume_v2.py index a67c971b8db..0467e00086b 100644 --- a/src/pybind/mgr/volumes/fs/operations/versions/subvolume_v2.py +++ b/src/pybind/mgr/volumes/fs/operations/versions/subvolume_v2.py @@ -154,7 +154,7 @@ class SubvolumeV2(SubvolumeV1): self.metadata_mgr.update_global_section(MetadataManager.GLOBAL_META_KEY_PATH, qpath) self.metadata_mgr.update_global_section(MetadataManager.GLOBAL_META_KEY_STATE, initial_state.value) - def create(self, size, isolate_nspace, pool, mode, uid, gid, earmark, normalization, casesensitive): + def create(self, size, isolate_nspace, pool, mode, uid, gid, earmark, normalization, casesensitive, enctag): subvolume_type = SubvolumeTypes.TYPE_NORMAL try: initial_state = SubvolumeOpSm.get_init_state(subvolume_type) @@ -179,6 +179,7 @@ class SubvolumeV2(SubvolumeV1): 'earmark': earmark, 'normalization': normalization, 'casesensitive': casesensitive, + 'enctag': enctag, } self.set_attrs(subvol_path, attrs) diff --git a/src/pybind/mgr/volumes/fs/volume.py b/src/pybind/mgr/volumes/fs/volume.py index 6b796f878c2..bcf8717839a 100644 --- a/src/pybind/mgr/volumes/fs/volume.py +++ b/src/pybind/mgr/volumes/fs/volume.py @@ -10,6 +10,7 @@ from urllib.parse import urlsplit, urlunsplit import cephfs from ceph.fs.earmarking import CephFSVolumeEarmarking, EarmarkException +from ceph.fs.enctag import CephFSVolumeEncryptionTag, EncryptionTagException from mgr_util import CephfsClient @@ -234,12 +235,13 @@ class VolumeClient(CephfsClient["Module"]): earmark = kwargs['earmark'] or '' # if not set, default to empty string --> no earmark normalization = kwargs['normalization'] casesensitive = kwargs['casesensitive'] + enctag = kwargs['enctag'] or '' # if not set, default to empty string oct_mode = octal_str_to_decimal_int(mode) try: create_subvol( - self.mgr, fs_handle, self.volspec, group, subvolname, size, isolate_nspace, pool, oct_mode, uid, gid, earmark, normalization, casesensitive) + self.mgr, fs_handle, self.volspec, group, subvolname, size, isolate_nspace, pool, oct_mode, uid, gid, earmark, normalization, casesensitive, enctag) except VolumeException as ve: # kick the purge threads for async removal -- note that this # assumes that the subvolume is moved to trashcan for cleanup on error. @@ -260,6 +262,7 @@ class VolumeClient(CephfsClient["Module"]): earmark = kwargs['earmark'] or '' # if not set, default to empty string --> no earmark normalization = kwargs['normalization'] casesensitive = kwargs['casesensitive'] + enctag = kwargs['enctag'] or '' # if not set, default to empty string --> no encryption tag try: with open_volume(self, volname) as fs_handle: @@ -277,6 +280,7 @@ class VolumeClient(CephfsClient["Module"]): 'earmark': earmark, 'normalization': normalization, 'casesensitive': casesensitive, + 'enctag': enctag, } subvolume.set_attrs(subvolume.path, attrs) except VolumeException as ve: @@ -732,6 +736,68 @@ class VolumeClient(CephfsClient["Module"]): ret = ee.to_tuple() # type: ignore return ret + def get_enctag(self, **kwargs) -> Tuple[int, Optional[str], str]: + ret: Tuple[int, Optional[str], str] = 0, "", "" + volname = kwargs['vol_name'] + subvolname = kwargs['sub_name'] + groupname = kwargs['group_name'] + + try: + with open_volume(self, volname) as fs_handle: + with open_group(fs_handle, self.volspec, groupname) as group: + with open_subvol(self.mgr, fs_handle, self.volspec, group, subvolname, SubvolumeOpType.ENCTAG_GET) as subvolume: + log.info("Getting enctag for subvolume %s", subvolume.path) + fs_enctag = CephFSVolumeEncryptionTag(fs_handle, subvolume.path) + enctag = fs_enctag.get_tag() + ret = 0, enctag, "" + except VolumeException as ve: + ret = self.volume_exception_to_retval(ve) + except EncryptionTagException as ee: + log.error(f"EncryptionTag error occurred: {ee}") + ret = ee.to_tuple() + return ret + + def set_enctag(self, **kwargs): # type: ignore + ret = 0, "", "" + volname = kwargs['vol_name'] + subvolname = kwargs['sub_name'] + groupname = kwargs['group_name'] + enctag = kwargs['enctag'] + + try: + with open_volume(self, volname) as fs_handle: + with open_group(fs_handle, self.volspec, groupname) as group: + with open_subvol(self.mgr, fs_handle, self.volspec, group, subvolname, SubvolumeOpType.ENCTAG_SET) as subvolume: + log.info("Setting enctag %s for subvolume %s", enctag, subvolume.path) + fs_enctag = CephFSVolumeEncryptionTag(fs_handle, subvolume.path) + fs_enctag.set_tag(enctag) + except VolumeException as ve: + ret = self.volume_exception_to_retval(ve) + except EncryptionTagException as ee: + log.error(f"EncryptionTag error occurred: {ee}") + ret = ee.to_tuple() # type: ignore + return ret + + def clear_enctag(self, **kwargs): # type: ignore + ret = 0, "", "" + volname = kwargs['vol_name'] + subvolname = kwargs['sub_name'] + groupname = kwargs['group_name'] + + try: + with open_volume(self, volname) as fs_handle: + with open_group(fs_handle, self.volspec, groupname) as group: + with open_subvol(self.mgr, fs_handle, self.volspec, group, subvolname, SubvolumeOpType.ENCTAG_CLEAR) as subvolume: + log.info("Removing enctag for subvolume %s", subvolume.path) + fs_enctag = CephFSVolumeEncryptionTag(fs_handle, subvolume.path) + fs_enctag.clear_tag() + except VolumeException as ve: + ret = self.volume_exception_to_retval(ve) + except EncryptionTagException as ee: + log.error(f"EncryptionTag error occurred: {ee}") + ret = ee.to_tuple() # type: ignore + return ret + ### subvolume snapshot def create_subvolume_snapshot(self, **kwargs): diff --git a/src/pybind/mgr/volumes/module.py b/src/pybind/mgr/volumes/module.py index e91c6bb504c..d69733e821c 100644 --- a/src/pybind/mgr/volumes/module.py +++ b/src/pybind/mgr/volumes/module.py @@ -146,7 +146,8 @@ class Module(orchestrator.OrchestratorClientMixin, MgrModule): 'name=namespace_isolated,type=CephBool,req=false ' 'name=earmark,type=CephString,req=false ' 'name=normalization,type=CephChoices,strings=nfd|nfc|nfkd|nfkc,req=false ' - 'name=casesensitive,type=CephBool,req=false ', + 'name=casesensitive,type=CephBool,req=false ' + 'name=enctag,type=CephString,req=false ', 'desc': "Create a CephFS subvolume in a volume, and optionally, " "with a specific size (in bytes), a specific data pool layout, " "a specific mode, in a specific subvolume group and in separate " @@ -302,6 +303,31 @@ class Module(orchestrator.OrchestratorClientMixin, MgrModule): 'desc': "Remove earmark from a subvolume", 'perm': 'rw' }, + { + 'cmd': 'fs subvolume enctag get ' + 'name=vol_name,type=CephString ' + 'name=sub_name,type=CephString ' + 'name=group_name,type=CephString,req=false ', + 'desc': "Get encryption tag for a subvolume", + 'perm': 'r' + }, + { + 'cmd': 'fs subvolume enctag set ' + 'name=vol_name,type=CephString ' + 'name=sub_name,type=CephString ' + 'name=group_name,type=CephString,req=false ' + 'name=enctag,type=CephString ', + 'desc': "Set encryption tag for a subvolume", + 'perm': 'rw' + }, + { + 'cmd': 'fs subvolume enctag rm ' + 'name=vol_name,type=CephString ' + 'name=sub_name,type=CephString ' + 'name=group_name,type=CephString,req=false ', + 'desc': "Remove encryption tag from a subvolume", + 'perm': 'rw' + }, { 'cmd': 'fs quiesce ' 'name=vol_name,type=CephString ' @@ -774,7 +800,8 @@ class Module(orchestrator.OrchestratorClientMixin, MgrModule): namespace_isolated=cmd.get('namespace_isolated', False), earmark=cmd.get('earmark', None), normalization=cmd.get('normalization', None), - casesensitive=cmd.get('casesensitive', None)) + casesensitive=cmd.get('casesensitive', None), + enctag=cmd.get('enctag', None)) @mgr_cmd_wrap def _cmd_fs_subvolume_rm(self, inbuf, cmd): @@ -904,6 +931,25 @@ class Module(orchestrator.OrchestratorClientMixin, MgrModule): sub_name=cmd['sub_name'], group_name=cmd.get('group_name', None)) + @mgr_cmd_wrap + def _cmd_fs_subvolume_enctag_get(self, inbuf, cmd): + return self.vc.get_enctag(vol_name=cmd['vol_name'], + sub_name=cmd['sub_name'], + group_name=cmd.get('group_name', None)) + + @mgr_cmd_wrap + def _cmd_fs_subvolume_enctag_set(self, inbuf, cmd): + return self.vc.set_enctag(vol_name=cmd['vol_name'], + sub_name=cmd['sub_name'], + group_name=cmd.get('group_name', None), + enctag=cmd['enctag']) + + @mgr_cmd_wrap + def _cmd_fs_subvolume_enctag_rm(self, inbuf, cmd): + return self.vc.clear_enctag(vol_name=cmd['vol_name'], + sub_name=cmd['sub_name'], + group_name=cmd.get('group_name', None)) + @mgr_cmd_wrap def _cmd_fs_quiesce(self, inbuf, cmd): return self.vc.quiesce(cmd) diff --git a/src/python-common/ceph/fs/enctag.py b/src/python-common/ceph/fs/enctag.py new file mode 100644 index 00000000000..14bdbac6560 --- /dev/null +++ b/src/python-common/ceph/fs/enctag.py @@ -0,0 +1,72 @@ +""" +Module: CephFS Volume Encryption Tag + +This module provides the `CephFSVolumeEncryptionTag` class, which is designed to manage encryption tags +of subvolumes within a CephFS filesystem. The encryption tag mechanism allows +administrators to tag specific subvolumes with identifiers that indicate encryption information, +such as a keyid or other itentifier tags. + +Key Features: +- **Set Encryption Tag**: Assigns an tag to a subvolume. +- **Get Encryption Tag**: Retrieves the existing tag of a subvolume, if any. +- **Remove Tag**: Removes the tag from a subvolume, making it available for reallocation. +supported top-level scopes. +""" + +import errno +import enum +import logging +from typing import Optional, Tuple + +log = logging.getLogger(__name__) + +XATTR_SUBVOLUME_ENCTAG_NAME = 'user.ceph.subvolume.enctag' + + +class EncryptionTagException(Exception): + def __init__(self, error_code: int, error_message: str) -> None: + self.errno = error_code + self.error_str = error_message + + def to_tuple(self) -> Tuple[int, Optional[str], str]: + return self.errno, "", self.error_str + + def __str__(self) -> str: + return f"{self.errno} ({self.error_str})" + + +class CephFSVolumeEncryptionTag: + def __init__(self, fs, path: str) -> None: + self.fs = fs + self.path = path + + def _handle_cephfs_error(self, e: Exception, action: str) -> None: + if isinstance(e, ValueError): + raise EncryptionTagException(errno.EINVAL, f"Invalid encryption tag specified: {e}") from e + elif isinstance(e, OSError): + log.error(f"Error {action} encryption tag: {e}") + raise EncryptionTagException(-e.errno, e.strerror) from e + else: + log.error(f"Unexpected error {action} encryption tag: {e}") + raise EncryptionTagException(errno.EIO, "Unexpected error") from e + + def get_tag(self) -> Optional[str]: + try: + enc_tag_value = ( + self.fs.getxattr(self.path, XATTR_SUBVOLUME_ENCTAG_NAME) + .decode('utf-8') + ) + return enc_tag_value + except Exception as e: + self._handle_cephfs_error(e, "getting") + return None + + def set_tag(self, enc_tag: str): + try: + self.fs.setxattr(self.path, XATTR_SUBVOLUME_ENCTAG_NAME, enc_tag.encode('utf-8'), 0) + log.info(f"Encryption Tag '{enc_tag}' set on {self.path}.") + except Exception as e: + self._handle_cephfs_error(e, "setting") + + def clear_tag(self) -> None: + self.set_tag("") -- 2.39.5