From: Shweta Bhosale Date: Wed, 12 Feb 2025 14:08:07 +0000 (+0530) Subject: mgr/nfs: NFS cluster and export commands to enable and disable ops/s control X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=8db7c50e5cb902be9240b3cf5bdf7f1fa8f050f1;p=ceph.git mgr/nfs: NFS cluster and export commands to enable and disable ops/s control Fixes: https://tracker.ceph.com/issues/69861 Signed-off-by: Shweta Bhosale --- diff --git a/src/pybind/mgr/nfs/cluster.py b/src/pybind/mgr/nfs/cluster.py index 97138792ac35..6541517894f4 100644 --- a/src/pybind/mgr/nfs/cluster.py +++ b/src/pybind/mgr/nfs/cluster.py @@ -26,7 +26,8 @@ from .ganesha_conf import format_block, GaneshaConfParser from .qos_conf import ( QOS, QOSType, - QOSBandwidthControl) + QOSBandwidthControl, + QOSOpsControl) if TYPE_CHECKING: from nfs.module import Module @@ -376,23 +377,27 @@ class NFSCluster: return qos_obj return None - def update_cluster_qos_bw(self, - cluster_id: str, - enable_qos: bool, - bw_obj: QOSBandwidthControl, - qos_type: Optional[QOSType] = None) -> None: + def update_cluster_qos_obj(self, + cluster_id: str, + qos_obj: Optional[QOS], + enable_qos: bool, + qos_type: Optional[QOSType] = None, + bw_obj: Optional[QOSBandwidthControl] = None, + ops_obj: Optional[QOSOpsControl] = None) -> None: """Update cluster QOS config""" qos_obj_exists = False - qos_obj = self.get_cluster_qos_config(cluster_id) if not qos_obj: log.debug(f"Creating new QOS block for cluster {cluster_id}") - qos_obj = QOS(True, enable_qos, qos_type, bw_obj) + qos_obj = QOS(True, enable_qos, qos_type, bw_obj, ops_obj) else: log.debug(f"Updating existing QOS block for cluster {cluster_id}") qos_obj_exists = True qos_obj.enable_qos = enable_qos qos_obj.qos_type = qos_type - qos_obj.bw_obj = bw_obj + if bw_obj: + qos_obj.bw_obj = bw_obj + if ops_obj: + qos_obj.ops_obj = ops_obj qos_config = format_block(qos_obj.to_qos_block()) rados_obj = self._rados(cluster_id) @@ -404,6 +409,53 @@ class NFSCluster: conf_obj_name(cluster_id), should_notify=False) log.debug(f"Successfully saved {cluster_id}s QOS bandwidth control config: \n {qos_config}") + def update_cluster_qos(self, + cluster_id: str, + qos_obj: Optional[QOS], + enable_qos: bool, + qos_type: Optional[QOSType] = None, + bw_obj: Optional[QOSBandwidthControl] = None, + ops_obj: Optional[QOSOpsControl] = None) -> None: + try: + if cluster_id in available_clusters(self.mgr): + self.update_cluster_qos_obj(cluster_id, qos_obj, enable_qos, qos_type, bw_obj, ops_obj) + restart_nfs_service(self.mgr, cluster_id) + return + raise ClusterNotFound() + except NotImplementedError: + raise ManualRestartRequired(f"NFS-Ganesha QOS config added successfully for {cluster_id}") + + def validate_qos_type(self, + qos_obj: QOS, + qos_type: QOSType, + bw_obj: Optional[QOSBandwidthControl] = None, + ops_obj: Optional[QOSOpsControl] = None) -> None: + if not qos_obj or not (bw_obj or ops_obj): + return + # if qos is not enabled then we can set new directly + if not (qos_obj.enable_qos and qos_obj.qos_type): + return + + other_qos_obj: Any = None + if bw_obj: + other_qos_obj = qos_obj.ops_obj + is_other_enable = qos_obj.ops_obj.enable_iops_ctrl if qos_obj.ops_obj else False + is_this_enable = qos_obj.bw_obj.enable_bw_ctrl if qos_obj.bw_obj else False + ctrl_type = "IOPS" + else: + other_qos_obj = qos_obj.bw_obj + is_other_enable = qos_obj.bw_obj.enable_bw_ctrl if qos_obj.bw_obj else False + is_this_enable = qos_obj.ops_obj.enable_iops_ctrl if qos_obj.ops_obj else False + ctrl_type = "Bandwidth" + + if other_qos_obj and is_other_enable: + # if earlier only other qos control is enabled + if not is_this_enable and qos_obj.qos_type != qos_type: + raise Exception(f"{ctrl_type} control is using {qos_obj.qos_type.name} qos type, please update that qos type for {ctrl_type} first.") + # if both qos control are enabled, the user will need to disable one first to change qos type + elif is_this_enable and qos_obj.qos_type != qos_type: + raise Exception(f"{ctrl_type} control is using {qos_obj.qos_type.name} qos type, please disable {ctrl_type} control to update qos type and then enable {ctrl_type} control again with new qos type") + def enable_cluster_qos_bw(self, cluster_id: str, qos_type: QOSType, @@ -422,17 +474,15 @@ class NFSCluster: c. If qos_type is pershare_perclient, then export_rw_bw and client_rw_bw parameters are compulsory """ try: + qos_obj = self.get_cluster_qos_config(cluster_id) + if qos_obj: + self.validate_qos_type(qos_obj, qos_type, bw_obj=bw_obj) bw_obj.qos_bandwidth_checks(qos_type) - if cluster_id in available_clusters(self.mgr): - self.update_cluster_qos_bw(cluster_id, True, bw_obj, qos_type) - restart_nfs_service(self.mgr, cluster_id) - log.info(f"QOS bandwidth control has been successfully enabled for cluster {cluster_id}. " - "If the qos_type is changed during this process, ensure that the bandwidth " - "values for all exports are updated accordingly.") - return - raise ClusterNotFound() - except NotImplementedError: - raise ManualRestartRequired(f"NFS-Ganesha QOS bandwidth control config added Successfully for {cluster_id}") + self.update_cluster_qos(cluster_id, qos_obj, True, qos_type=qos_type, bw_obj=bw_obj) + log.info(f"QOS bandwidth control has been successfully enabled for cluster {cluster_id}. " + "If the qos_type is changed during this process, ensure that the bandwidth " + "values for all exports are updated accordingly.") + return except Exception as e: log.exception(f"Setting NFS-Ganesha QOS bandwidth control config failed for {cluster_id}") raise ErrorResponse.wrap(e) @@ -449,16 +499,51 @@ class NFSCluster: def disable_cluster_qos_bw(self, cluster_id: str) -> None: try: - if cluster_id in available_clusters(self.mgr): - self.update_cluster_qos_bw(cluster_id, False, QOSBandwidthControl()) - restart_nfs_service(self.mgr, cluster_id) - log.info("Cluster-level QoS bandwidth control has been successfully disabled for " - f"cluster {cluster_id}. As a result, export-level bandwidth control will " - "no longer have any effect, even if it's enabled.") - return - raise ClusterNotFound() - except NotImplementedError: - raise ManualRestartRequired(f"NFS-Ganesha QOS bandwidth control config added successfully for {cluster_id}") + qos_obj = self.get_cluster_qos_config(cluster_id) + status = False + qos_type = None + if qos_obj: + status = qos_obj.get_enable_qos_val(disable_bw=True) + if status: + qos_type = qos_obj.qos_type + self.update_cluster_qos(cluster_id, qos_obj, status, qos_type, bw_obj=QOSBandwidthControl()) + log.info("Cluster-level QoS bandwidth control has been successfully disabled for " + f"cluster {cluster_id}. As a result, export-level bandwidth control will " + "no longer have any effect, even if it's enabled.") + return except Exception as e: log.exception(f"Setting NFS-Ganesha QOS bandwidth control config failed for {cluster_id}") raise ErrorResponse.wrap(e) + + def enable_cluster_qos_ops(self, cluster_id: str, qos_type: QOSType, ops_obj: QOSOpsControl) -> None: + try: + qos_obj = self.get_cluster_qos_config(cluster_id) + if qos_obj: + self.validate_qos_type(qos_obj, qos_type, ops_obj=ops_obj) + ops_obj.qos_ops_checks(qos_type) + self.update_cluster_qos(cluster_id, qos_obj, True, qos_type=qos_type, ops_obj=ops_obj) + log.info(f"QOS IOPS control has been successfully enabled for cluster {cluster_id}. " + "If the qos_type is changed during this process, ensure that ops count " + "values for all exports are updated accordingly.") + return + except Exception as e: + log.exception(f"Setting NFS-Ganesha QOS IOPS control config failed for {cluster_id}") + raise ErrorResponse.wrap(e) + + def disable_cluster_qos_ops(self, cluster_id: str) -> None: + try: + qos_obj = self.get_cluster_qos_config(cluster_id) + status = False + qos_type = None + if qos_obj: + status = qos_obj.get_enable_qos_val(disable_ops=True) + if status: + qos_type = qos_obj.qos_type + self.update_cluster_qos(cluster_id, qos_obj, status, qos_type, ops_obj=QOSOpsControl()) + log.info("Cluster-level QoS IOPS control has been successfully disabled for " + f"cluster {cluster_id}. As a result, export-level ops control will " + "no longer have any effect, even if it's enabled.") + return + except Exception as e: + log.exception(f"Setting NFS-Ganesha QOS IOPS control config failed for {cluster_id}") + raise ErrorResponse.wrap(e) diff --git a/src/pybind/mgr/nfs/export.py b/src/pybind/mgr/nfs/export.py index 724ea969d91e..b8a54a5bfbe2 100644 --- a/src/pybind/mgr/nfs/export.py +++ b/src/pybind/mgr/nfs/export.py @@ -28,8 +28,8 @@ from .ganesha_conf import ( GaneshaConfParser, RGWFSAL, format_block) -from .qos_conf import QOS, QOSBandwidthControl -from .export_utils import export_qos_bw_checks, export_dict_qos_checks +from .qos_conf import QOS, QOSBandwidthControl, QOSOpsControl +from .export_utils import export_qos_bw_checks, export_dict_qos_bw_ops_checks, export_qos_ops_checks from .exception import NFSException, NFSInvalidOperation, FSNotFound, NFSObjectNotFound from .utils import ( EXPORT_PREFIX, @@ -860,7 +860,7 @@ class ExportMgr: # check QOS if new_export_dict.get('qos_block'): - export_dict_qos_checks(cluster_id, self.mgr, dict(new_export_dict.get('qos_block', {}))) + export_dict_qos_bw_ops_checks(cluster_id, self.mgr, dict(new_export_dict.get('qos_block', {}))) self.exports[cluster_id].remove(old_export) @@ -880,27 +880,36 @@ class ExportMgr: exports_count += 1 return exports_count - def update_export_qos_bw(self, - cluster_id: str, - pseudo_path: str, - enable_qos: bool, - bw_obj: QOSBandwidthControl) -> None: + def update_export_qos(self, + cluster_id: str, + pseudo_path: str, + export_obj: Export, + enable_qos: bool, + bw_obj: Optional[QOSBandwidthControl] = None, + ops_obj: Optional[QOSOpsControl] = None) -> None: """Update Export QOS block""" - export = self._fetch_export(cluster_id, pseudo_path) - if not export: - raise NFSObjectNotFound(f"Export {pseudo_path} not found in NFS cluster {cluster_id}") # if qos_block does not exists in export create one else update existing block - if not export.qos_block: + if not export_obj.qos_block: log.debug(f"Creating new QOS block for export {pseudo_path} of cluster {cluster_id}") - export.qos_block = QOS(enable_qos=enable_qos, bw_obj=bw_obj) + export_obj.qos_block = QOS(enable_qos=enable_qos, bw_obj=bw_obj, ops_obj=ops_obj) else: log.debug(f"Updating existing QOS block of export {pseudo_path} of cluster {cluster_id}") - export.qos_block.enable_qos = enable_qos - export.qos_block.bw_obj = bw_obj + export_obj.qos_block.enable_qos = enable_qos + if bw_obj: + export_obj.qos_block.bw_obj = bw_obj + if ops_obj: + export_obj.qos_block.ops_obj = ops_obj - self.exports[cluster_id].remove(export) - self._update_export(cluster_id, export, False) - log.debug(f"Successfully updated QOS bandwidth control config for export {pseudo_path} of cluster {cluster_id}") + self.exports[cluster_id].remove(export_obj) + self._update_export(cluster_id, export_obj, False) + log.debug(f"Successfully updated QOS control config for export {pseudo_path} of cluster {cluster_id}") + + def get_export_obj(self, cluster_id: str, pseudo_path: str) -> Export: + self._validate_cluster_id(cluster_id) + export = self._fetch_export(cluster_id, pseudo_path) + if not export: + raise NFSObjectNotFound(f"Export {pseudo_path} not found in NFS cluster {cluster_id}") + return export def enable_export_qos_bw(self, cluster_id: str, @@ -920,32 +929,54 @@ class ExportMgr: c. If qos_type is pershare_perclient, then export_rw_bw and client_rw_bw parameters are compulsory """ try: - self._validate_cluster_id(cluster_id) - assert pseudo_path + export_obj = self.get_export_obj(cluster_id, pseudo_path) export_qos_bw_checks(cluster_id, self.mgr, bw_obj=bw_obj) - self.update_export_qos_bw(cluster_id, pseudo_path, True, bw_obj) + self.update_export_qos(cluster_id, pseudo_path, export_obj, True, bw_obj=bw_obj) + log.debug(f"Successfully enabled QOS bandwidth control for export {pseudo_path} of cluster {cluster_id}") except Exception as e: - log.exception(f"Setting NFS-Ganesha QOS bandwidth control config failed for {pseudo_path} of {cluster_id}") + log.exception(f"Setting NFS-Ganesha QOS bandwidth control failed for {pseudo_path} of {cluster_id}") raise ErrorResponse.wrap(e) def get_export_qos(self, cluster_id: str, pseudo_path: str) -> Dict[str, int]: try: - self._validate_cluster_id(cluster_id) - export = self._fetch_export(cluster_id, pseudo_path) - if not export: - raise NFSObjectNotFound(f"Export {pseudo_path} not found in NFS cluster {cluster_id}") - return export.qos_block.to_dict() if export.qos_block else {} + export_obj = self.get_export_obj(cluster_id, pseudo_path) + return export_obj.qos_block.to_dict() if export_obj.qos_block else {} except Exception as e: log.exception(f"Failed to get QOS configuration for {pseudo_path} of {cluster_id}") raise ErrorResponse.wrap(e) def disable_export_qos_bw(self, cluster_id: str, pseudo_path: str) -> None: try: - self._validate_cluster_id(cluster_id) - assert pseudo_path - self.update_export_qos_bw(cluster_id, pseudo_path, False, QOSBandwidthControl()) + export_obj = self.get_export_obj(cluster_id, pseudo_path) + if export_obj.qos_block: + status = export_obj.qos_block.get_enable_qos_val(disable_bw=True) + self.update_export_qos(cluster_id, pseudo_path, export_obj, status, bw_obj=QOSBandwidthControl()) + log.debug(f"Successfully disabled QOS bandwidth control for export {pseudo_path} of cluster {cluster_id}") + except Exception as e: + log.exception(f"Setting NFS-Ganesha QOS bandwidth control failed for {pseudo_path} of {cluster_id}") + raise ErrorResponse.wrap(e) + + def enable_export_qos_ops(self, cluster_id: str, pseudo_path: str, ops_obj: QOSOpsControl) -> None: + try: + export_obj = self.get_export_obj(cluster_id, pseudo_path) + export_qos_ops_checks(cluster_id, self.mgr, ops_obj=ops_obj) + self.update_export_qos(cluster_id, pseudo_path, export_obj, True, ops_obj=ops_obj) + log.debug(f"Successfully enabled QOS IOPS control for export {pseudo_path} of cluster {cluster_id}") + except Exception as e: + log.exception(f"Setting NFS-Ganesha QOS IOPS control failed for {pseudo_path} of {cluster_id}") + raise ErrorResponse.wrap(e) + + def disable_export_qos_ops(self, cluster_id: str, pseudo_path: str) -> None: + try: + export_obj = self.get_export_obj(cluster_id, pseudo_path) + if not export_obj: + raise NFSObjectNotFound(f"Export {pseudo_path} not found in NFS cluster {cluster_id}") + if export_obj.qos_block: + status = export_obj.qos_block.get_enable_qos_val(disable_ops=True) + self.update_export_qos(cluster_id, pseudo_path, export_obj, status, ops_obj=QOSOpsControl()) + log.debug(f"Successfully updated QOS IOPS control for export {pseudo_path} of cluster {cluster_id}") except Exception as e: - log.exception(f"Setting NFS-Ganesha QOS bandwidth control Config failed for {pseudo_path} of {cluster_id}") + log.exception(f"Setting NFS-Ganesha QOS IOPS control failed for {pseudo_path} of {cluster_id}") raise ErrorResponse.wrap(e) diff --git a/src/pybind/mgr/nfs/export_utils.py b/src/pybind/mgr/nfs/export_utils.py index 86f560b84f45..f5c45c3f4433 100644 --- a/src/pybind/mgr/nfs/export_utils.py +++ b/src/pybind/mgr/nfs/export_utils.py @@ -1,7 +1,7 @@ from typing import Any from .cluster import NFSCluster -from .qos_conf import QOSType, QOSParams, QOSBandwidthControl +from .qos_conf import QOSType, QOSParams, QOSBandwidthControl, QOSOpsControl def export_dict_bw_checks(cluster_id: str, @@ -18,10 +18,10 @@ def export_dict_bw_checks(cluster_id: str, if combined_bw_ctrl is None: combined_bw_ctrl = False if not qos_enable and enable_bw_ctrl: - raise Exception('To enable bandwidth control, qos_enable should be true.') + raise Exception('To enable bandwidth control, qos_enable and enable_bw_control should be true.') if not (isinstance(enable_bw_ctrl, bool) and isinstance(combined_bw_ctrl, bool)): raise Exception('Invalid values for the enable_bw_ctrl and combined_bw_ctrl parameters.') - # if qos is disabled, then bandwidths should not be set and no need to bandwidth checks + # if qos bandwidth control is disabled, then bandwidths should not be set and no need to bandwidth checks if not enable_bw_ctrl: if bandwith_param_exists: raise Exception('Bandwidths should not be passed when enable_bw_control is false.') @@ -39,9 +39,31 @@ def export_dict_bw_checks(cluster_id: str, export_qos_bw_checks(cluster_id, mgr_obj, bw_obj) -def export_dict_qos_checks(cluster_id: str, +def export_dict_ops_checks(cluster_id: str, mgr_obj: Any, + qos_enable: bool, qos_dict: dict) -> None: + enable_iops_ctrl = qos_dict.get(QOSParams.enable_iops_ctrl.value) + if enable_iops_ctrl is None: + return + if not isinstance(enable_iops_ctrl, bool): + raise Exception(f'Invalid values for the {QOSParams.enable_iops_ctrl.value} parameter') + ops_param_exists = any(key.endswith('iops') for key in qos_dict) + if not enable_iops_ctrl: + if ops_param_exists: + raise Exception(f'IOPS count parameters should not be passed when {QOSParams.enable_iops_ctrl.value} is false.') + return + if enable_iops_ctrl and not ops_param_exists: + raise Exception(f'IOPS count parameters should be set when {QOSParams.enable_iops_ctrl.value} is true.') + ops_obj = QOSOpsControl(enable_iops_ctrl, + max_export_iops=qos_dict.get(QOSParams.max_export_iops.value, 0), + max_client_iops=qos_dict.get(QOSParams.max_client_iops.value, 0)) + export_qos_ops_checks(cluster_id, mgr_obj, ops_obj) + + +def export_dict_qos_bw_ops_checks(cluster_id: str, + mgr_obj: Any, + qos_dict: dict) -> None: """Validate the qos block of dict passed to apply_export method""" qos_enable = qos_dict.get('enable_qos') if qos_enable is None: @@ -49,19 +71,41 @@ def export_dict_qos_checks(cluster_id: str, if not isinstance(qos_enable, bool): raise Exception('Invalid value for the enable_qos parameter') export_dict_bw_checks(cluster_id, mgr_obj, qos_enable, qos_dict) + export_dict_ops_checks(cluster_id, mgr_obj, qos_enable, qos_dict) def export_qos_bw_checks(cluster_id: str, mgr_obj: Any, bw_obj: QOSBandwidthControl, nfs_clust_obj: Any = None) -> None: - """check cluster level qos is enabled to enable export level qos and validate bandwidths""" + """check cluster level qos bandwidth control is enabled to enable export level qos + bandwidth control and validate bandwidths""" if not nfs_clust_obj: nfs_clust_obj = NFSCluster(mgr_obj) clust_qos_obj = nfs_clust_obj.get_cluster_qos_config(cluster_id) - if not clust_qos_obj or (clust_qos_obj and not (clust_qos_obj.enable_qos)): - raise Exception('To configure bandwidth control for export, you must first enable bandwidth control at the cluster level.') + if not clust_qos_obj or (clust_qos_obj and not (clust_qos_obj.enable_qos + and clust_qos_obj.bw_obj + and clust_qos_obj.bw_obj.enable_bw_ctrl)): + raise Exception(f'To configure bandwidth control for export, you must first enable bandwidth control at the cluster level for {cluster_id}.') if clust_qos_obj.qos_type: if clust_qos_obj.qos_type == QOSType.PerClient: - raise Exception('Export-level QoS bandwidth control cannot be enabled if the QoS type at the cluster level is set to PerClient.') + raise Exception(f'Export-level QoS bandwidth control cannot be enabled if the QoS type at the cluster {cluster_id} level is set to PerClient.') bw_obj.qos_bandwidth_checks(clust_qos_obj.qos_type) + + +def export_qos_ops_checks(cluster_id: str, + mgr_obj: Any, + ops_obj: QOSOpsControl, + nfs_clust_obj: Any = None) -> None: + """check cluster level qos IOPS is enabled to enable export level qos IOPS and validate IOPS count""" + if not nfs_clust_obj: + nfs_clust_obj = NFSCluster(mgr_obj) + clust_qos_obj = nfs_clust_obj.get_cluster_qos_config(cluster_id) + if not clust_qos_obj or (clust_qos_obj and not (clust_qos_obj.enable_qos + and clust_qos_obj.ops_obj + and clust_qos_obj.ops_obj.enable_iops_ctrl)): + raise Exception(f'To configure IOPS control for export, you must first enable IOPS control at the cluster level {cluster_id}.') + if clust_qos_obj.qos_type: + if clust_qos_obj.qos_type == QOSType.PerClient: + raise Exception(f'Export-level QoS IOPS control cannot be enabled if the QoS type at the cluster {cluster_id} level is set to PerClient.') + ops_obj.qos_ops_checks(clust_qos_obj.qos_type) diff --git a/src/pybind/mgr/nfs/module.py b/src/pybind/mgr/nfs/module.py index 747c198b4d6a..8ba5bc175201 100644 --- a/src/pybind/mgr/nfs/module.py +++ b/src/pybind/mgr/nfs/module.py @@ -14,7 +14,7 @@ from mgr_util import CephFSEarmarkResolver from .export import ExportMgr, AppliedExportResults from .cluster import NFSCluster from .utils import available_clusters -from .qos_conf import QOSType, QOSBandwidthControl, UserQoSType +from .qos_conf import QOSType, QOSBandwidthControl, UserQoSType, QOSOpsControl log = logging.getLogger(__name__) @@ -258,7 +258,7 @@ class Module(orchestrator.OrchestratorClientMixin, MgrModule): max_export_combined_bw: str = '0', max_client_combined_bw: str = '0' ) -> None: - """enable QOS config for NFS export and set different bandwidth""" + """enable QOS bandwidth control for NFS export and set different bandwidth""" try: bw_obj = QOSBandwidthControl(enable_bw_ctrl=True, combined_bw_ctrl=combined_rw_bw_ctrl, @@ -283,7 +283,7 @@ class Module(orchestrator.OrchestratorClientMixin, MgrModule): @CLICommand('nfs export qos disable bandwidth_control', perm='rw') @object_format.EmptyResponder() def _cmd_export_qos_bw_disable(self, cluster_id: str, pseudo_path: str) -> None: - """Disable NFS export QOS config""" + """Disable NFS export QOS bandwidth control""" return self.export_mgr.disable_export_qos_bw(cluster_id, pseudo_path) @CLICommand('nfs cluster qos enable bandwidth_control', perm='rw') @@ -298,7 +298,7 @@ class Module(orchestrator.OrchestratorClientMixin, MgrModule): max_client_read_bw: str = '0', max_export_combined_bw: str = '0', max_client_combined_bw: str = '0') -> None: - """Enable QOS ratelimiting for NFS cluster and set default export and client max bandwidth""" + """Enable QOS bandwidth control for NFS cluster and set default export and client max bandwidth""" try: bw_obj = QOSBandwidthControl(enable_bw_ctrl=True, combined_bw_ctrl=combined_rw_bw_ctrl, @@ -317,7 +317,7 @@ class Module(orchestrator.OrchestratorClientMixin, MgrModule): @CLICommand('nfs cluster qos disable bandwidth_control', perm='rw') @object_format.EmptyResponder() def _cmd_cluster_qos_bw_disable(self, cluster_id: str) -> None: - """Disable QOS for NFS cluster""" + """Disable QOS bandwidth control for NFS cluster""" return self.nfs.disable_cluster_qos_bw(cluster_id) @CLICommand('nfs cluster qos get', perm='r') @@ -325,3 +325,53 @@ class Module(orchestrator.OrchestratorClientMixin, MgrModule): def _cmd_cluster_qos_get(self, cluster_id: str) -> Dict[str, Any]: """Get QOS configuration of NFS cluster""" return self.nfs.get_cluster_qos(cluster_id) + + @CLICommand('nfs export qos enable ops_control', perm='rw') + @object_format.EmptyResponder() + def _cmd_export_qos_ops_enable(self, + cluster_id: str, + pseudo_path: str, + max_export_iops: int = 0, + max_client_iops: int = 0, + ) -> None: + """enable QOS IOPS control for NFS export""" + try: + ops_obj = QOSOpsControl(enable_iops_ctrl=True, + max_export_iops=max_export_iops, + max_client_iops=max_client_iops) + except Exception as e: + raise object_format.ErrorResponse.wrap(e) + return self.export_mgr.enable_export_qos_ops(cluster_id=cluster_id, + pseudo_path=pseudo_path, + ops_obj=ops_obj) + + @CLICommand('nfs export qos disable ops_control', perm='rw') + @object_format.EmptyResponder() + def _cmd_export_qos_ops_disable(self, cluster_id: str, pseudo_path: str) -> None: + """Disable NFS export QOS IOPS control""" + return self.export_mgr.disable_export_qos_ops(cluster_id, pseudo_path) + + @CLICommand('nfs cluster qos enable ops_control', perm='rw') + @object_format.EmptyResponder() + def _cmd_cluster_qos_ops_enable(self, + cluster_id: str, + qos_type: UserQoSType, + max_export_iops: int = 0, + max_client_iops: int = 0, + ) -> None: + """enable QOS IOPS control for NFS cluster""" + try: + ops_obj = QOSOpsControl(enable_iops_ctrl=True, + max_export_iops=max_export_iops, + max_client_iops=max_client_iops) + except Exception as e: + raise object_format.ErrorResponse.wrap(e) + return self.nfs.enable_cluster_qos_ops(cluster_id=cluster_id, + qos_type=QOSType[qos_type.value], + ops_obj=ops_obj) + + @CLICommand('nfs cluster qos disable ops_control', perm='rw') + @object_format.EmptyResponder() + def _cmd_cluster_qos_ops_disable(self, cluster_id: str) -> None: + """Disable NFS cluster QOS IOPS control""" + return self.nfs.disable_cluster_qos_ops(cluster_id) diff --git a/src/pybind/mgr/nfs/qos_conf.py b/src/pybind/mgr/nfs/qos_conf.py index 91c1ca35e701..b76ec54b0606 100644 --- a/src/pybind/mgr/nfs/qos_conf.py +++ b/src/pybind/mgr/nfs/qos_conf.py @@ -29,15 +29,20 @@ class QOSParams(Enum): clust_block = "QOS_DEFAULT_CONFIG" export_block = "QOS_BLOCK" enable_qos = "enable_qos" + qos_type = "qos_type" + # bandwidth control enable_bw_ctrl = "enable_bw_control" combined_bw_ctrl = "combined_rw_bw_control" - qos_type = "qos_type" export_writebw = "max_export_write_bw" export_readbw = "max_export_read_bw" client_writebw = "max_client_write_bw" client_readbw = "max_client_read_bw" export_rw_bw = "max_export_combined_bw" client_rw_bw = "max_client_combined_bw" + # ops control + enable_iops_ctrl = "enable_iops_control" + max_export_iops = "max_export_iops" + max_client_iops = "max_client_iops" class UserQoSType(Enum): @@ -61,7 +66,15 @@ def _validate_qos_bw(bandwidth: str) -> int: return bw_bytes -QOS_REQ_PARAMS = { +def _validate_qos_ops(count: int) -> int: + min_cnt = 1000 # 1K + max_cnt = 500000 # 5L + if count != 0 and (count < min_cnt or count > max_cnt): + raise Exception(f"Provided IOS count value is not in range, Please enter a value between {min_cnt} (1K) and {max_cnt} (5L) bytes") + return count + + +QOS_REQ_BW_PARAMS = { 'combined_bw_disabled': { 'PerShare': ['max_export_write_bw', 'max_export_read_bw'], 'PerClient': ['max_client_write_bw', 'max_client_read_bw'], @@ -74,6 +87,12 @@ QOS_REQ_PARAMS = { } } +QOS_REQ_OPS_PARAMS = { + 'PerShare': ['max_export_iops'], + 'PerClient': ['max_client_iops'], + 'PerShare_PerClient': ['max_export_iops', 'max_client_iops'] +} + class QOSBandwidthControl(object): def __init__(self, @@ -100,7 +119,7 @@ class QOSBandwidthControl(object): @classmethod def from_dict(cls, qos_dict: Dict[str, Any]) -> 'QOSBandwidthControl': - # json has bandwidths in human readable format(str) + # qos dict has bandwidths in human readable format(str) bw_kwargs = { 'enable_bw_ctrl': qos_dict.get(QOSParams.enable_bw_ctrl.value, False), 'combined_bw_ctrl': qos_dict.get(QOSParams.combined_bw_ctrl.value, False), @@ -115,7 +134,7 @@ class QOSBandwidthControl(object): @classmethod def from_qos_block(cls, qos_block: RawBlock) -> 'QOSBandwidthControl': - # block has bandwidths in bytes(int) + # qos block has bandwidths in bytes(int) bw_kwargs = { 'enable_bw_ctrl': qos_block.values.get(QOSParams.enable_bw_ctrl.value, False), 'combined_bw_ctrl': qos_block.values.get(QOSParams.combined_bw_ctrl.value, False), @@ -165,16 +184,16 @@ class QOSBandwidthControl(object): return r def qos_bandwidth_checks(self, qos_type: QOSType) -> None: - """Checks for enabling qos""" + """Checks for enabling qos bandwidth control""" params = {} d = vars(self) for key in d: if key.endswith('bw'): params[QOSParams[key].value] = d[key] if not self.combined_bw_ctrl: - req_params = QOS_REQ_PARAMS['combined_bw_disabled'][qos_type.name] + req_params = QOS_REQ_BW_PARAMS['combined_bw_disabled'][qos_type.name] else: - req_params = QOS_REQ_PARAMS['combined_bw_enabled'][qos_type.name] + req_params = QOS_REQ_BW_PARAMS['combined_bw_enabled'][qos_type.name] allowed_params = [] not_allowed_params = [] for key in params: @@ -185,8 +204,73 @@ class QOSBandwidthControl(object): if allowed_params or not_allowed_params: raise Exception(f"When combined_rw_bw is {'enabled' if self.combined_bw_ctrl else 'disabled'} " f"and qos_type is {qos_type.name}, " - f"{'attributes ' + ', '.join(allowed_params) + ' required' if allowed_params else ''} " - f"{'attributes ' + ', '.join(not_allowed_params) + ' are not allowed' if not_allowed_params else ''}.") + f"{'attribute ' + ', '.join(allowed_params) + ' required' if allowed_params else ''} " + f"{'attribute ' + ', '.join(not_allowed_params) + ' are not allowed' if not_allowed_params else ''}.") + + +class QOSOpsControl(object): + def __init__(self, + enable_iops_ctrl: bool = False, + max_export_iops: int = 0, + max_client_iops: int = 0 + ) -> None: + self.enable_iops_ctrl = enable_iops_ctrl + self.max_export_iops = _validate_qos_ops(max_export_iops) + self.max_client_iops = _validate_qos_ops(max_client_iops) + + @classmethod + def from_dict(cls, qos_dict: Dict[str, Any]) -> 'QOSOpsControl': + kwargs: dict[str, Any] = {} + kwargs['enable_iops_ctrl'] = qos_dict.get(QOSParams.enable_iops_ctrl.value, False) + kwargs['max_export_iops'] = qos_dict.get(QOSParams.max_export_iops.value, 0) + kwargs['max_client_iops'] = qos_dict.get(QOSParams.max_client_iops.value, 0) + return cls(**kwargs) + + @classmethod + def from_qos_block(cls, qos_block: RawBlock) -> 'QOSOpsControl': + kwargs: dict[str, Any] = {} + kwargs['enable_iops_ctrl'] = qos_block.values.get(QOSParams.enable_iops_ctrl.value, False) + kwargs['max_export_iops'] = qos_block.values.get(QOSParams.max_export_iops.value, 0) + kwargs['max_client_iops'] = qos_block.values.get(QOSParams.max_client_iops.value, 0) + return cls(**kwargs) + + def to_qos_block(self) -> RawBlock: + result = RawBlock('qos_ops_control') + result.values[QOSParams.enable_iops_ctrl.value] = self.enable_iops_ctrl + if self.max_export_iops: + result.values[QOSParams.max_export_iops.value] = self.max_export_iops + if self.max_client_iops: + result.values[QOSParams.max_client_iops.value] = self.max_client_iops + return result + + def to_dict(self) -> Dict[str, Any]: + r: dict[str, Any] = {} + r[QOSParams.enable_iops_ctrl.value] = self.enable_iops_ctrl + if self.max_export_iops: + r[QOSParams.max_export_iops.value] = self.max_export_iops + if self.max_client_iops: + r[QOSParams.max_client_iops.value] = self.max_client_iops + return r + + def qos_ops_checks(self, qos_type: QOSType) -> None: + """Checks for enabling qos IOPS control""" + params = {} + d = vars(self) + for key in d: + if key.endswith('iops'): + params[QOSParams[key].value] = d[key] + req_params = QOS_REQ_OPS_PARAMS[qos_type.name] + allowed_params = [] + not_allowed_params = [] + for key in params: + if key in req_params and params[key] == 0: + allowed_params.append(key) + elif key not in req_params and params[key] != 0: + not_allowed_params.append(key) + if allowed_params or not_allowed_params: + raise Exception(f"When qos_type is {qos_type.name}, " + f"{'attribute ' + ', '.join(allowed_params) + ' required' if allowed_params else ''} " + f"{'attribute ' + ', '.join(not_allowed_params) + ' are not allowed' if not_allowed_params else ''}.") class QOS(object): @@ -194,12 +278,14 @@ class QOS(object): cluster_op: bool = False, enable_qos: bool = False, qos_type: Optional[QOSType] = None, - bw_obj: Optional[QOSBandwidthControl] = None + bw_obj: Optional[QOSBandwidthControl] = None, + ops_obj: Optional[QOSOpsControl] = None ) -> None: self.cluster_op = cluster_op self.enable_qos = enable_qos self.qos_type = qos_type self.bw_obj = bw_obj + self.ops_obj = ops_obj @classmethod def from_dict(cls, qos_dict: Dict[str, Any], cluster_op: bool = False) -> 'QOS': @@ -211,6 +297,7 @@ class QOS(object): kwargs['qos_type'] = QOSType[qos_type] kwargs['enable_qos'] = qos_dict.get(QOSParams.enable_qos.value) kwargs['bw_obj'] = QOSBandwidthControl.from_dict(qos_dict) + kwargs['ops_obj'] = QOSOpsControl.from_dict(qos_dict) return cls(cluster_op, **kwargs) @classmethod @@ -223,6 +310,7 @@ class QOS(object): kwargs['qos_type'] = QOSType(qos_type) kwargs['enable_qos'] = qos_block.values.get(QOSParams.enable_qos.value) kwargs['bw_obj'] = QOSBandwidthControl.from_qos_block(qos_block) + kwargs['ops_obj'] = QOSOpsControl.from_qos_block(qos_block) return cls(cluster_op, **kwargs) def to_qos_block(self) -> RawBlock: @@ -235,6 +323,8 @@ class QOS(object): result.values[QOSParams.qos_type.value] = self.qos_type.value if self.bw_obj and (res := self.bw_obj.to_qos_block()): result.values.update(res.values) + if self.ops_obj and (res := self.ops_obj.to_qos_block()): + result.values.update(res.values) return result def to_dict(self) -> Dict[str, Any]: @@ -244,4 +334,17 @@ class QOS(object): r[QOSParams.qos_type.value] = self.qos_type.name if self.bw_obj and (res := self.bw_obj.to_dict()): r.update(res) + if self.ops_obj and (res := self.ops_obj.to_dict()): + r.update(res) return r + + def get_enable_qos_val(self, disable_bw: bool = False, disable_ops: bool = False) -> bool: + if not (self.enable_qos) or not (disable_bw or disable_ops): + return False + # check if ops control is enabled + if disable_bw and self.ops_obj and self.ops_obj.enable_iops_ctrl: + return True + # check if bandwidth control is enabled + if disable_ops and self.bw_obj and self.bw_obj.enable_bw_ctrl: + return True + return False diff --git a/src/pybind/mgr/nfs/tests/test_nfs.py b/src/pybind/mgr/nfs/tests/test_nfs.py index e3a3f7e6f4c1..39c18c786d0f 100644 --- a/src/pybind/mgr/nfs/tests/test_nfs.py +++ b/src/pybind/mgr/nfs/tests/test_nfs.py @@ -15,7 +15,15 @@ from ceph.utils import with_units_to_int, bytes_to_human from nfs import Module from nfs.export import ExportMgr, normalize_path from nfs.ganesha_conf import GaneshaConfParser, Export -from nfs.qos_conf import RawBlock, QOS, QOSType, QOSParams, QOS_REQ_PARAMS, QOSBandwidthControl +from nfs.qos_conf import ( + RawBlock, + QOS, + QOSType, + QOSParams, + QOS_REQ_BW_PARAMS, + QOSBandwidthControl, + QOSOpsControl, + QOS_REQ_OPS_PARAMS) from nfs.cluster import NFSCluster from orchestrator import ServiceDescription, DaemonDescription, OrchResult @@ -174,7 +182,8 @@ QOS_BLOCK { "max_client_write_bw": "3.0MB", "max_export_read_bw": "2.0MB", "max_export_write_bw": "1.0MB", - "qos_type": "PerShare_PerClient" + "qos_type": "PerShare_PerClient", + "enable_iops_control": False } qos_export_dict = { @@ -184,7 +193,8 @@ QOS_BLOCK { "max_client_read_bw": "4.0MB", "max_client_write_bw": "3.0MB", "max_export_read_bw": "2.0MB", - "max_export_write_bw": "1.0MB" + "max_export_write_bw": "1.0MB", + "enable_iops_control": False } class RObject(object): @@ -1453,7 +1463,7 @@ EXPORT { qos = QOS.from_qos_block(blocks[0], True) assert qos.to_dict() == qos_dict - def _do_test_cluster_qos(self, qos_type, combined_bw_ctrl, params, positive_tc): + def _do_test_cluster_qos_bw(self, qos_type, combined_bw_ctrl, params, positive_tc): nfs_mod = Module('nfs', '', '') cluster = NFSCluster(nfs_mod) try: @@ -1462,14 +1472,16 @@ EXPORT { except Exception: if not positive_tc: return + if not positive_tc: + raise Exception("This TC was supposed to fail") out = cluster.get_cluster_qos(self.cluster_id) - expected_out = {"enable_bw_control": True, "enable_qos": True, "combined_rw_bw_control": combined_bw_ctrl, "qos_type": qos_type.name} + expected_out = {"enable_bw_control": True, "enable_qos": True, "combined_rw_bw_control": combined_bw_ctrl, "qos_type": qos_type.name, "enable_iops_control": False} for key in params: expected_out[QOSParams[key].value] = bytes_to_human(with_units_to_int(params[key])) assert out == expected_out cluster.disable_cluster_qos_bw(self.cluster_id) out = cluster.get_cluster_qos(self.cluster_id) - assert out == {"enable_bw_control": False, "enable_qos": False, "combined_rw_bw_control": False} + assert out == {"enable_bw_control": False, "enable_qos": False, "combined_rw_bw_control": False, "enable_iops_control": False} @pytest.mark.parametrize("qos_type, combined_bw_ctrl, params, positive_tc", [ (QOSType['PerShare'], False, {'export_writebw': '100MB', 'export_readbw': '200MB'}, True), @@ -1490,10 +1502,10 @@ EXPORT { (QOSType['PerClient'], True, {'client_rw_bw': '200MB', 'export_rw_bw': '100MB'}, False), (QOSType['PerShare_PerClient'], True, {'export_rw_bw': '100MB'}, False) ]) - def test_cluster_qos(self, qos_type, combined_bw_ctrl, params, positive_tc): - self._do_mock_test(self._do_test_cluster_qos, qos_type, combined_bw_ctrl, params, positive_tc) + def test_cluster_qos_bw(self, qos_type, combined_bw_ctrl, params, positive_tc): + self._do_mock_test(self._do_test_cluster_qos_bw, qos_type, combined_bw_ctrl, params, positive_tc) - def _do_test_export_qos(self, qos_type, clust_combined_bw_ctrl, clust_params, export_combined_bw_ctrl, export_params): + def _do_test_export_qos_bw(self, qos_type, clust_combined_bw_ctrl, clust_params, export_combined_bw_ctrl, export_params): nfs_mod = Module('nfs', '', '') cluster = NFSCluster(nfs_mod) export_mgr = ExportMgr(nfs_mod) @@ -1502,7 +1514,7 @@ EXPORT { bw_obj = QOSBandwidthControl(True, export_combined_bw_ctrl, **export_params) export_mgr.enable_export_qos_bw(self.cluster_id, '/cephfs_a/', bw_obj) except Exception as e: - assert str(e) == 'To configure bandwidth control for export, you must first enable bandwidth control at the cluster level.' + assert str(e) == 'To configure bandwidth control for export, you must first enable bandwidth control at the cluster level for foo.' bw_obj = QOSBandwidthControl(True, clust_combined_bw_ctrl, **clust_params) cluster.enable_cluster_qos_bw(self.cluster_id, qos_type, bw_obj) @@ -1512,9 +1524,9 @@ EXPORT { export_mgr.enable_export_qos_bw(self.cluster_id, '/cephfs_a/', bw_obj) except Exception: if export_combined_bw_ctrl: - req = QOS_REQ_PARAMS['combined_bw_enabled'][qos_type.name] + req = QOS_REQ_BW_PARAMS['combined_bw_enabled'][qos_type.name] else: - req = QOS_REQ_PARAMS['combined_bw_enabled'][qos_type.name] + req = QOS_REQ_BW_PARAMS['combined_bw_disabled'][qos_type.name] if sorted(export_params.keys()) != sorted(req): return if qos_type.name == 'PerClient': @@ -1545,11 +1557,216 @@ EXPORT { (True, {'client_rw_bw': '200MB'}), (True, {'export_rw_bw': '100MB', 'client_rw_bw': '200MB'}) ]) - def test_export_qos(self, qos_type, clust_combined_bw_ctrl, clust_params, + def test_export_qos_bw(self, qos_type, clust_combined_bw_ctrl, clust_params, export_combined_bw_ctrl, export_params): - self._do_mock_test(self._do_test_export_qos, qos_type, clust_combined_bw_ctrl, + self._do_mock_test(self._do_test_export_qos_bw, qos_type, clust_combined_bw_ctrl, clust_params, export_combined_bw_ctrl, export_params) + def _do_test_cluster_qos_ops(self, qos_type, params, positive_tc): + nfs_mod = Module('nfs', '', '') + cluster = NFSCluster(nfs_mod) + try: + ops_obj = QOSOpsControl(True, **params) + cluster.enable_cluster_qos_ops(self.cluster_id, qos_type, ops_obj) + except Exception: + if not positive_tc: + return + if not positive_tc: + raise Exception("This TC was supposed to fail") + out = cluster.get_cluster_qos(self.cluster_id) + expected_out = {"enable_bw_control": False, "enable_qos": True, "combined_rw_bw_control": False, "qos_type": qos_type.name, "enable_iops_control": True} + for key in params: + expected_out[QOSParams[key].value] = params[key] + assert out == expected_out + cluster.disable_cluster_qos_ops(self.cluster_id) + out = cluster.get_cluster_qos(self.cluster_id) + assert out == {"enable_bw_control": False, "enable_qos": False, "combined_rw_bw_control": False, "enable_iops_control": False} + + @pytest.mark.parametrize("qos_type, params, positive_tc", [ + (QOSType['PerShare'], {'max_export_iops': 10000}, True), + (QOSType['PerClient'], {'max_client_iops': 20000}, True), + (QOSType['PerShare_PerClient'], {'max_export_iops': 3000, 'max_client_iops': 40000}, True), + # negative testing + (QOSType['PerShare_PerClient'], {'max_export_iops': 1000}, False), + (QOSType['PerShare'], {'max_client_iops': 10000}, False), + (QOSType['PerShare'], {}, False), + (QOSType['PerClient'], {'max_export_iops': 2000, 'max_client_iops': 1000}, False), + (QOSType['PerShare_PerClient'], {'max_export_iops': 10}, False) + ]) + def test_cluster_qos_ops(self, qos_type, params, positive_tc): + self._do_mock_test(self._do_test_cluster_qos_ops, qos_type, params, positive_tc) + + def _do_test_export_qos_ops(self, qos_type, clust_params, export_params): + nfs_mod = Module('nfs', '', '') + cluster = NFSCluster(nfs_mod) + export_mgr = ExportMgr(nfs_mod) + # try enabling export level qos before enabling cluster level qos + try: + ops_obj = QOSOpsControl(True, **export_params) + export_mgr.enable_export_qos_ops(self.cluster_id, '/cephfs_a/', ops_obj) + except Exception as e: + assert str(e) == 'To configure IOPS control for export, you must first enable IOPS control at the cluster level foo.' + ops_obj = QOSOpsControl(True, **clust_params) + cluster.enable_cluster_qos_ops(self.cluster_id, qos_type, ops_obj) + + # set export qos + try: + ops_obj = QOSOpsControl(True, **export_params) + export_mgr.enable_export_qos_ops(self.cluster_id, '/cephfs_a/', ops_obj) + except Exception: + req = QOS_REQ_OPS_PARAMS[qos_type.name] + if sorted(export_params.keys()) != sorted(req): + return + if qos_type.name == 'PerClient': + return + out = export_mgr.get_export_qos(self.cluster_id, '/cephfs_a/') + expected_out = {"enable_iops_control": True, "enable_qos": True} + for key in export_params: + expected_out[QOSParams[key].value] = export_params[key] + assert out == expected_out + export_mgr.disable_export_qos_ops(self.cluster_id, '/cephfs_a/') + out = export_mgr.get_export_qos(self.cluster_id, '/cephfs_a/') + assert out == {"enable_iops_control": False, "enable_qos": False} + + @pytest.mark.parametrize("qos_type, clust_params", [ + (QOSType['PerShare'], {'max_export_iops': 10000}), + (QOSType['PerClient'], {'max_client_iops': 20000}), + (QOSType['PerShare_PerClient'], {'max_export_iops': 3000, 'max_client_iops': 40000}) + ]) + @pytest.mark.parametrize("export_params", [ + ({'max_export_iops': 10000}), + ({'max_client_iops': 20000}), + ({'max_export_iops': 3000, 'max_client_iops': 40000}) + ]) + def test_export_qos_ops(self, qos_type, clust_params, export_params): + self._do_mock_test(self._do_test_export_qos_ops, qos_type, clust_params, export_params) + + def _do_test_cluster_qos_bw_ops(self, bw_qos_type, bw_params, ops_qos_type, ops_params, positive_tc): + nfs_mod = Module('nfs', '', '') + cluster = NFSCluster(nfs_mod) + try: + bw_obj = QOSBandwidthControl(True, combined_bw_ctrl=False, **bw_params) + cluster.enable_cluster_qos_bw(self.cluster_id, bw_qos_type, bw_obj) + ops_obj = QOSOpsControl(True, **ops_params) + cluster.enable_cluster_qos_ops(self.cluster_id, ops_qos_type, ops_obj) + except Exception: + if not positive_tc: + return + if not positive_tc: + raise Exception("This TC passed but it was supposed to fail") + out = cluster.get_cluster_qos(self.cluster_id) + expected_out = {"enable_bw_control": True, "enable_qos": True, "combined_rw_bw_control": False, "qos_type": ops_qos_type.name, "enable_iops_control": True} + bw_out = {} + ops_out = {} + for key in bw_params: + bw_out[QOSParams[key].value] = bytes_to_human(with_units_to_int(bw_params[key])) + for key in ops_params: + ops_out[QOSParams[key].value] = ops_params[key] + expected_out.update(bw_out) + expected_out.update(ops_out) + assert out == expected_out + # disable bandwidth control + cluster.disable_cluster_qos_bw(self.cluster_id) + out = cluster.get_cluster_qos(self.cluster_id) + ops_out.update({"enable_bw_control": False, "enable_qos": True, "combined_rw_bw_control": False, "enable_iops_control": True, "qos_type": ops_qos_type.name}) + assert out == ops_out + # disable ops control + cluster.disable_cluster_qos_ops(self.cluster_id) + out = cluster.get_cluster_qos(self.cluster_id) + assert out == {"enable_bw_control": False, "enable_qos": False, "combined_rw_bw_control": False, "enable_iops_control": False} + + @pytest.mark.parametrize("bw_qos_type, bw_params, ops_qos_type, ops_params, positive_tc", [ + # positive TCs + (QOSType['PerShare'], {'export_writebw': '100MB', 'export_readbw': '200MB'}, + QOSType['PerShare'], {'max_export_iops': 10000}, True), + (QOSType['PerClient'], {'client_writebw': '300MB', 'client_readbw': '400MB'}, + QOSType['PerClient'], {'max_client_iops': 20000}, True), + (QOSType['PerShare_PerClient'], + {'export_writebw': '100MB', 'export_readbw': '200MB', 'client_writebw': '300MB', 'client_readbw': '400MB'}, + QOSType['PerShare_PerClient'], {'max_export_iops': 3000, 'max_client_iops': 40000}, True), + # negative TCs + (QOSType['PerShare'], {'export_writebw': '100MB', 'export_readbw': '200MB'}, QOSType['PerClient'], {}, False), + (QOSType['PerClient'], {'client_writebw': '300MB', 'client_readbw': '400MB'}, QOSType['PerShare'], {}, False), + (QOSType['PerShare_PerClient'], {'export_writebw': '100MB', 'export_readbw': '200MB', 'client_writebw': '300MB', 'client_readbw': '400MB'}, QOSType['PerClient'], {'max_client_iops': 20000}, False), + ]) + def test_cluster_qos_bw_ops(self, bw_qos_type, bw_params, ops_qos_type, ops_params, positive_tc): + self._do_mock_test(self._do_test_cluster_qos_bw_ops, bw_qos_type, bw_params, ops_qos_type, ops_params, positive_tc) + + def _do_test_export_qos_bw_ops(self, qos_type, clust_bw_params, clust_ops_params, export_bw_params, export_ops_params): + nfs_mod = Module('nfs', '', '') + cluster = NFSCluster(nfs_mod) + export_mgr = ExportMgr(nfs_mod) + # enable cluster level bandwidth conrtol and try to enable ops control for export + bw_obj = QOSBandwidthControl(True, combined_bw_ctrl=False, **clust_bw_params) + cluster.enable_cluster_qos_bw(self.cluster_id, qos_type, bw_obj) + try: + ops_obj = QOSOpsControl(True, **export_ops_params) + export_mgr.enable_export_qos_ops(self.cluster_id, '/cephfs_a/', ops_obj) + except Exception: + pass + cluster.disable_cluster_qos_bw(self.cluster_id) + # enable ops control for cluster and try to enable bw control for export + ops_obj = QOSOpsControl(True, **clust_ops_params) + cluster.enable_cluster_qos_ops(self.cluster_id, qos_type, ops_obj) + try: + bw_obj = QOSBandwidthControl(True, combined_bw_ctrl=False, **export_bw_params) + export_mgr.enable_export_qos_bw(self.cluster_id, '/cephfs_a/', bw_obj) + except Exception: + pass + # enbale both and verify export get + bw_obj = QOSBandwidthControl(True, combined_bw_ctrl=False, **clust_bw_params) + cluster.enable_cluster_qos_bw(self.cluster_id, qos_type, bw_obj) + try: + bw_obj = QOSBandwidthControl(True, combined_bw_ctrl=False, **export_bw_params) + export_mgr.enable_export_qos_bw(self.cluster_id, '/cephfs_a/', bw_obj) + ops_obj = QOSOpsControl(True, **export_ops_params) + export_mgr.enable_export_qos_ops(self.cluster_id, '/cephfs_a/', ops_obj) + except Exception: + req = QOS_REQ_BW_PARAMS['combined_bw_disabled'][qos_type.name] + if sorted(export_bw_params.keys()) != sorted(req): + return + if qos_type.name == 'PerClient': + return + req = QOS_REQ_OPS_PARAMS[qos_type.name] + if sorted(export_ops_params.keys()) != sorted(req): + return + out = export_mgr.get_export_qos(self.cluster_id, '/cephfs_a/') + expected_out = {"enable_bw_control": True, "enable_qos": True, "combined_rw_bw_control": False, "enable_iops_control": True} + bw_out = {} + ops_out = {} + for key in export_bw_params: + bw_out[QOSParams[key].value] = bytes_to_human(with_units_to_int(export_bw_params[key])) + for key in export_ops_params: + ops_out[QOSParams[key].value] = export_ops_params[key] + expected_out.update(bw_out) + expected_out.update(ops_out) + assert out == expected_out + # disable bandwidth control of export + export_mgr.disable_export_qos_bw(self.cluster_id, '/cephfs_a/') + out = export_mgr.get_export_qos(self.cluster_id, '/cephfs_a/') + ops_out.update({"enable_bw_control": False, "enable_qos": True, "combined_rw_bw_control": False, "enable_iops_control": True}) + assert out == ops_out + # disable ops control of export + export_mgr.disable_export_qos_ops(self.cluster_id, '/cephfs_a/') + out = export_mgr.get_export_qos(self.cluster_id, '/cephfs_a/') + assert out == {"enable_bw_control": False, "enable_qos": False, "combined_rw_bw_control": False, "enable_iops_control": False} + + @pytest.mark.parametrize("qos_type, clust_bw_params, clust_ops_params", [ + # positive TCs + (QOSType['PerShare'], {'export_writebw': '100MB', 'export_readbw': '200MB'}, {'max_export_iops': 10000}), + (QOSType['PerClient'], {'client_writebw': '300MB', 'client_readbw': '400MB'}, {'max_client_iops': 20000}), + (QOSType['PerShare_PerClient'], + {'export_writebw': '100MB', 'export_readbw': '200MB', 'client_writebw': '300MB', 'client_readbw': '400MB'}, {'max_export_iops': 3000, 'max_client_iops': 40000}) + ]) + @pytest.mark.parametrize("export_bw_params, export_ops_params", [ + ({'export_writebw': '100MB', 'export_readbw': '200MB'}, {'max_export_iops': 10000}), + ({'client_writebw': '300MB', 'client_readbw': '400MB'}, {'max_client_iops': 20000}), + ({'export_writebw': '100MB', 'export_readbw': '200MB', 'client_writebw': '300MB', 'client_readbw': '400MB'}, {'max_export_iops': 3000, 'max_client_iops': 40000}) + ]) + def test_export_qos_bw_ops(self, qos_type, clust_bw_params, clust_ops_params, export_bw_params, export_ops_params): + self._do_mock_test(self._do_test_export_qos_bw_ops, qos_type, clust_bw_params, clust_ops_params, export_bw_params, export_ops_params) + + @pytest.mark.parametrize( "path,expected", [