From d5fef517e1e3c7ffd8aa49c597477609e01460d1 Mon Sep 17 00:00:00 2001 From: Adam King Date: Wed, 17 Jul 2024 16:29:02 -0400 Subject: [PATCH] mgr/rgw: add ability for rgw realm bootstrap to create pools Specifically by setting the data_pool_attributes field in the RGW spec. For example service_type: rgw service_id: my_realm.my_zone placement: count: 2 spec: data_pool_attributes: type: ec k: 6 m: 2 rgw_realm: my_realm rgw_zone: my_zone rgw_zonegroup: my_zonegroup would create the my_zone.rgw.buckets.data pool as an ec pool with a new ec porfile with k=8 and m=2. Currently the ec profile fields this spec will make use of are only "k", "m", "pg_num" and "crush-device-class". If other attributes are set, or if the pool type is "replicated" the key value pairs are assumed to be ones that can be passed to the "ceph osd pool create" command. The other pools for the rgw zone (e.g. buckets index pool) are all made as replicated pool with defaults settings Signed-off-by: Adam King --- src/pybind/mgr/rgw/module.py | 95 ++++++++++++++++++- .../ceph/deployment/service_spec.py | 15 +++ 2 files changed, 109 insertions(+), 1 deletion(-) diff --git a/src/pybind/mgr/rgw/module.py b/src/pybind/mgr/rgw/module.py index d32ded6c52d..f42c9cc64ec 100644 --- a/src/pybind/mgr/rgw/module.py +++ b/src/pybind/mgr/rgw/module.py @@ -6,7 +6,13 @@ import base64 import functools import sys -from mgr_module import MgrModule, CLICommand, HandleCommandResult, Option +from mgr_module import ( + MgrModule, + CLICommand, + HandleCommandResult, + Option, + MonCommandFailed, +) import orchestrator from ceph.deployment.service_spec import RGWSpec, PlacementSpec, SpecValidationError @@ -49,6 +55,10 @@ class RGWSpecParsingError(Exception): pass +class PoolCreationError(Exception): + pass + + class OrchestratorAPI(OrchestratorClientMixin): def __init__(self, mgr: MgrModule): super(OrchestratorAPI, self).__init__() @@ -196,10 +206,14 @@ class Module(orchestrator.OrchestratorClientMixin, MgrModule): try: for spec in rgw_specs: + self.create_pools(spec) RGWAM(self.env).realm_bootstrap(spec, start_radosgw) except RGWAMException as e: self.log.error('cmd run exception: (%d) %s' % (e.retcode, e.message)) return HandleCommandResult(retval=e.retcode, stdout=e.stdout, stderr=e.stderr) + except PoolCreationError as e: + self.log.error(f'Pool creation failure: {str(e)}') + return HandleCommandResult(retval=-errno.EINVAL, stderr=str(e)) return HandleCommandResult(retval=0, stdout="Realm(s) created correctly. Please, use 'ceph rgw realm tokens' to get the token.", stderr='') @@ -229,6 +243,85 @@ class Module(orchestrator.OrchestratorClientMixin, MgrModule): return rgw_specs + def create_pools(self, spec: RGWSpec) -> None: + def _pool_create_command( + pool_name: str, + pool_type: str, + pool_attrs: Optional[Dict[str, Union[str, List[str]]]] = None + ) -> None: + try: + cmd_dict: Dict[str, Union[str, List[str]]] = { + 'prefix': 'osd pool create', + 'pool': pool_name, + 'pool_type': pool_type, + } + if pool_attrs: + for k, v in pool_attrs.items(): + cmd_dict[k] = v + self.check_mon_command(cmd_dict) + except MonCommandFailed as e: + raise PoolCreationError(f'RGW module failed to create pool {pool_name} ' + f'of type {pool_type} with attrs [{pool_attrs}]: {str(e)}') + # enable the rgw application on the pool + try: + self.check_mon_command({ + 'prefix': 'osd pool application enable', + 'pool': pool_name, + 'app': 'rgw', + }) + except MonCommandFailed as e: + raise PoolCreationError(f'Failed enabling application "rgw" on pool {pool_name}: {str(e)}') + + zone_name = spec.rgw_zone + for pool_suffix in [ + 'buckets.index', + 'meta', + 'log', + 'control' + ]: + # TODO: add size? + non_data_pool_attrs: Dict[str, Union[str, List[str]]] = { + 'pg-num': '16' if 'index' in pool_suffix else '8', + } + _pool_create_command(f'{zone_name}.rgw.{pool_suffix}', 'replicated', non_data_pool_attrs) + + if spec.data_pool_attributes: + if spec.data_pool_attributes.get('type', 'ec') == 'ec': + # we need to create ec profile + assert zone_name is not None + profile_name = self.create_zone_ec_profile(zone_name, spec.data_pool_attributes) + # now we can pass the ec profile into the pool create command + data_pool_attrs: Dict[str, Union[str, List[str]]] = { + 'erasure_code_profile': profile_name + } + if 'pg_num' in spec.data_pool_attributes: + data_pool_attrs['pg_num'] = spec.data_pool_attributes['pg_num'] + _pool_create_command(f'{zone_name}.rgw.buckets.data', 'erasure', data_pool_attrs) + else: + # replicated pool + data_pool_attrs = {k: v for k, v in spec.data_pool_attributes.items() if k != 'type'} + _pool_create_command(f'{zone_name}.rgw.buckets.data', 'replicated', data_pool_attrs) + + def create_zone_ec_profile(self, zone_name: str, pool_attributes: Optional[Dict[str, str]]) -> str: + # creates ec profile and returns profile name + ec_pool_kv_pairs = {} + if pool_attributes is not None: + ec_pool_kv_pairs = {k: v for k, v in pool_attributes.items() if k not in ['type', 'pg_num']} + profile_name = f'{zone_name}_zone_data_pool_ec_profile' + profile_attrs = [f'{k}={v}' for k, v in ec_pool_kv_pairs.items()] + cmd_dict: Dict[str, Union[str, List[str]]] = { + 'prefix': 'osd erasure-code-profile set', + 'name': profile_name, + } + if profile_attrs: + cmd_dict['profile'] = profile_attrs + try: + self.check_mon_command(cmd_dict) + except MonCommandFailed as e: + raise PoolCreationError(f'RGW module failed to create ec profile {profile_name} ' + f'with attrs {profile_attrs}: {str(e)}') + return profile_name + @CLICommand('rgw realm zone-creds create', perm='rw') def _cmd_rgw_realm_new_zone_creds(self, realm_name: Optional[str] = None, diff --git a/src/python-common/ceph/deployment/service_spec.py b/src/python-common/ceph/deployment/service_spec.py index 93ea0511de1..1c1b1825f15 100644 --- a/src/python-common/ceph/deployment/service_spec.py +++ b/src/python-common/ceph/deployment/service_spec.py @@ -1228,6 +1228,7 @@ class RGWSpec(ServiceSpec): update_endpoints: Optional[bool] = False, zone_endpoints: Optional[str] = None, # comma separated endpoints list zonegroup_hostnames: Optional[List[str]] = None, + data_pool_attributes: Optional[Dict[str, str]] = None, rgw_user_counters_cache: Optional[bool] = False, rgw_user_counters_cache_size: Optional[int] = None, rgw_bucket_counters_cache: Optional[bool] = False, @@ -1290,6 +1291,8 @@ class RGWSpec(ServiceSpec): #: Used to make RGW not do multisite replication so it can dedicate to IO self.disable_multisite_sync_traffic = disable_multisite_sync_traffic self.wildcard_enabled = wildcard_enabled + #: Attributes for .rgw.buckets.data pool created in rgw realm bootstrap command + self.data_pool_attributes = data_pool_attributes def get_port_start(self) -> List[int]: ports = self.get_port() @@ -1340,6 +1343,18 @@ class RGWSpec(ServiceSpec): raise SpecValidationError('"generate_cert" field and "rgw_frontend_ssl_certificate" ' 'field are mutually exclusive') + if self.data_pool_attributes: + if self.data_pool_attributes.get('type', 'ec') == 'ec': + if any(attr not in self.data_pool_attributes.keys() for attr in ['k', 'm']): + raise SpecValidationError( + '"k" and "m" are required parameters for ec pool (defaults to ec pool)' + ) + if 'erasure_code_profile' in self.data_pool_attributes.keys(): + raise SpecValidationError( + 'invalid option in data_pool_attribues "erasure_code_profile"' + 'ec profile will be generated automatically based on provided attributes' + ) + yaml.add_representer(RGWSpec, ServiceSpec.yaml_representer) -- 2.39.5