mgr/rgw: add ability for rgw realm bootstrap to create pools

Specifically by setting the data_pool_attributes field in
the RGW spec. For example

service_type: rgw
service_id: my_realm.my_zone
placement:
  count: 2
spec:
  data_pool_attributes:
    type: ec
    k: 6
    m: 2
  rgw_realm: my_realm
  rgw_zone: my_zone
  rgw_zonegroup: my_zonegroup

would create the my_zone.rgw.buckets.data pool as an
ec pool with a new ec porfile with k=8 and m=2. Currently
the ec profile fields this spec will make use of
are only "k", "m", "pg_num" and "crush-device-class".
If other attributes are set, or if the pool type is
"replicated" the key value pairs are assumed to
be ones that can be passed to the "ceph osd pool create"
command. The other pools for the rgw zone (e.g. buckets
index pool) are all made as replicated pool with
defaults settings

Signed-off-by: Adam King <adking@redhat.com>

diff --git a/src/pybind/mgr/rgw/module.py b/src/pybind/mgr/rgw/module.py
index d32ded6c52dc1e7e584c04b6008f21766dda4dde..f42c9cc64ec3a205f111ac83d24a51e77504170f 100644 (file)
--- a/src/pybind/mgr/rgw/module.py
+++ b/src/pybind/mgr/rgw/module.py
@@ -6,7 +6,13 @@ import base64
 import functools
 import sys
 
-from mgr_module import MgrModule, CLICommand, HandleCommandResult, Option
+from mgr_module import (
+    MgrModule,
+    CLICommand,
+    HandleCommandResult,
+    Option,
+    MonCommandFailed,
+)
 import orchestrator
 
 from ceph.deployment.service_spec import RGWSpec, PlacementSpec, SpecValidationError
@@ -49,6 +55,10 @@ class RGWSpecParsingError(Exception):
     pass
 
 
+class PoolCreationError(Exception):
+    pass
+
+
 class OrchestratorAPI(OrchestratorClientMixin):
     def __init__(self, mgr: MgrModule):
         super(OrchestratorAPI, self).__init__()
@@ -196,10 +206,14 @@ class Module(orchestrator.OrchestratorClientMixin, MgrModule):
 
         try:
             for spec in rgw_specs:
+                self.create_pools(spec)
                 RGWAM(self.env).realm_bootstrap(spec, start_radosgw)
         except RGWAMException as e:
             self.log.error('cmd run exception: (%d) %s' % (e.retcode, e.message))
             return HandleCommandResult(retval=e.retcode, stdout=e.stdout, stderr=e.stderr)
+        except PoolCreationError as e:
+            self.log.error(f'Pool creation failure: {str(e)}')
+            return HandleCommandResult(retval=-errno.EINVAL, stderr=str(e))
 
         return HandleCommandResult(retval=0, stdout="Realm(s) created correctly. Please, use 'ceph rgw realm tokens' to get the token.", stderr='')
 
@@ -229,6 +243,85 @@ class Module(orchestrator.OrchestratorClientMixin, MgrModule):
 
         return rgw_specs
 
+    def create_pools(self, spec: RGWSpec) -> None:
+        def _pool_create_command(
+            pool_name: str,
+            pool_type: str,
+            pool_attrs: Optional[Dict[str, Union[str, List[str]]]] = None
+        ) -> None:
+            try:
+                cmd_dict: Dict[str, Union[str, List[str]]] = {
+                    'prefix': 'osd pool create',
+                    'pool': pool_name,
+                    'pool_type': pool_type,
+                }
+                if pool_attrs:
+                    for k, v in pool_attrs.items():
+                        cmd_dict[k] = v
+                self.check_mon_command(cmd_dict)
+            except MonCommandFailed as e:
+                raise PoolCreationError(f'RGW module failed to create pool {pool_name} '
+                                        f'of type {pool_type} with attrs [{pool_attrs}]: {str(e)}')
+            # enable the rgw application on the pool
+            try:
+                self.check_mon_command({
+                    'prefix': 'osd pool application enable',
+                    'pool': pool_name,
+                    'app': 'rgw',
+                })
+            except MonCommandFailed as e:
+                raise PoolCreationError(f'Failed enabling application "rgw" on pool {pool_name}: {str(e)}')
+
+        zone_name = spec.rgw_zone
+        for pool_suffix in [
+            'buckets.index',
+            'meta',
+            'log',
+            'control'
+        ]:
+            # TODO: add size?
+            non_data_pool_attrs: Dict[str, Union[str, List[str]]] = {
+                'pg-num': '16' if 'index' in pool_suffix else '8',
+            }
+            _pool_create_command(f'{zone_name}.rgw.{pool_suffix}', 'replicated', non_data_pool_attrs)
+
+        if spec.data_pool_attributes:
+            if spec.data_pool_attributes.get('type', 'ec') == 'ec':
+                # we need to create ec profile
+                assert zone_name is not None
+                profile_name = self.create_zone_ec_profile(zone_name, spec.data_pool_attributes)
+                # now we can pass the ec profile into the pool create command
+                data_pool_attrs: Dict[str, Union[str, List[str]]] = {
+                    'erasure_code_profile': profile_name
+                }
+                if 'pg_num' in spec.data_pool_attributes:
+                    data_pool_attrs['pg_num'] = spec.data_pool_attributes['pg_num']
+                _pool_create_command(f'{zone_name}.rgw.buckets.data', 'erasure', data_pool_attrs)
+            else:
+                # replicated pool
+                data_pool_attrs = {k: v for k, v in spec.data_pool_attributes.items() if k != 'type'}
+                _pool_create_command(f'{zone_name}.rgw.buckets.data', 'replicated', data_pool_attrs)
+
+    def create_zone_ec_profile(self, zone_name: str, pool_attributes: Optional[Dict[str, str]]) -> str:
+        # creates ec profile and returns profile name
+        ec_pool_kv_pairs = {}
+        if pool_attributes is not None:
+            ec_pool_kv_pairs = {k: v for k, v in pool_attributes.items() if k not in ['type', 'pg_num']}
+        profile_name = f'{zone_name}_zone_data_pool_ec_profile'
+        profile_attrs = [f'{k}={v}' for k, v in ec_pool_kv_pairs.items()]
+        cmd_dict: Dict[str, Union[str, List[str]]] = {
+            'prefix': 'osd erasure-code-profile set',
+            'name': profile_name,
+        }
+        if profile_attrs:
+            cmd_dict['profile'] = profile_attrs
+        try:
+            self.check_mon_command(cmd_dict)
+        except MonCommandFailed as e:
+            raise PoolCreationError(f'RGW module failed to create ec profile {profile_name} '
+                                    f'with attrs {profile_attrs}: {str(e)}')
+        return profile_name
+
     @CLICommand('rgw realm zone-creds create', perm='rw')
     def _cmd_rgw_realm_new_zone_creds(self,
                                       realm_name: Optional[str] = None,

diff --git a/src/python-common/ceph/deployment/service_spec.py b/src/python-common/ceph/deployment/service_spec.py
index 93ea0511de1f2c943c549f069e50d51ca2a2d5bf..1c1b1825f155c4d3601ce7defcdbde105fb6f021 100644 (file)
--- a/src/python-common/ceph/deployment/service_spec.py
+++ b/src/python-common/ceph/deployment/service_spec.py
@@ -1228,6 +1228,7 @@ class RGWSpec(ServiceSpec):
                  update_endpoints: Optional[bool] = False,
                  zone_endpoints: Optional[str] = None,  # comma separated endpoints list
                  zonegroup_hostnames: Optional[List[str]] = None,
+                 data_pool_attributes: Optional[Dict[str, str]] = None,
                  rgw_user_counters_cache: Optional[bool] = False,
                  rgw_user_counters_cache_size: Optional[int] = None,
                  rgw_bucket_counters_cache: Optional[bool] = False,
@@ -1290,6 +1291,8 @@ class RGWSpec(ServiceSpec):
         #: Used to make RGW not do multisite replication so it can dedicate to IO
         self.disable_multisite_sync_traffic = disable_multisite_sync_traffic
         self.wildcard_enabled = wildcard_enabled
+        #: Attributes for <zone-name>.rgw.buckets.data pool created in rgw realm bootstrap command
+        self.data_pool_attributes = data_pool_attributes
 
     def get_port_start(self) -> List[int]:
         ports = self.get_port()
@@ -1340,6 +1343,18 @@ class RGWSpec(ServiceSpec):
             raise SpecValidationError('"generate_cert" field and "rgw_frontend_ssl_certificate" '
                                       'field are mutually exclusive')
 
+        if self.data_pool_attributes:
+            if self.data_pool_attributes.get('type', 'ec') == 'ec':
+                if any(attr not in self.data_pool_attributes.keys() for attr in ['k', 'm']):
+                    raise SpecValidationError(
+                        '"k" and "m" are required parameters for ec pool (defaults to ec pool)'
+                    )
+                if 'erasure_code_profile' in self.data_pool_attributes.keys():
+                    raise SpecValidationError(
+                        'invalid option in data_pool_attribues "erasure_code_profile"'
+                        'ec profile will be generated automatically based on provided attributes'
+                    )
+
 
 yaml.add_representer(RGWSpec, ServiceSpec.yaml_representer)