From: Redouane Kachach Date: Thu, 18 Aug 2022 13:33:21 +0000 (+0200) Subject: mgr/rgw: Adding rgw multisite support X-Git-Tag: v17.2.7~485^2~18 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=52c5cff6e18cff841219ad02e40c5d8698ea4ad4;p=ceph.git mgr/rgw: Adding rgw multisite support Fixes: https://tracker.ceph.com/issues/57160 Signed-off-by: Redouane Kachach (cherry picked from commit d15a5dcfe2d05240059b9d4a72c581d867d5e7a4) Conflicts: doc/mgr/rgw.rst --- diff --git a/doc/cephadm/services/rgw.rst b/doc/cephadm/services/rgw.rst index d0a58000130c..be813a0820dc 100644 --- a/doc/cephadm/services/rgw.rst +++ b/doc/cephadm/services/rgw.rst @@ -83,16 +83,16 @@ To deploy RGWs serving the multisite *myorg* realm and the *us-east-1* zone on .. prompt:: bash # - ceph orch apply rgw east --realm=myorg --zone=us-east-1 --placement="2 myhost1 myhost2" + ceph orch apply rgw east --realm=myorg --zonegroup=us-east-zg-1 --zone=us-east-1 --placement="2 myhost1 myhost2" Note that in a multisite situation, cephadm only deploys the daemons. It does not create -or update the realm or zone configurations. To create a new realm and zone, you need to do -something like: +or update the realm or zone configurations. To create a new realm and zone, you can use +:ref:`mgr-rgw-module` or manually using something like: .. prompt:: bash # radosgw-admin realm create --rgw-realm= --default - + .. prompt:: bash # radosgw-admin zonegroup create --rgw-zonegroup= --master --default diff --git a/doc/mgr/rgw.rst b/doc/mgr/rgw.rst index 82778e5775ab..f2a6de1161d5 100644 --- a/doc/mgr/rgw.rst +++ b/doc/mgr/rgw.rst @@ -2,8 +2,9 @@ RGW Module ============ -The rgw module helps with bootstraping and configuring RGW realm -and the different related entities. +The rgw module provides a simple interface to deploy RGW multisite. +It helps with bootstrapping and configuring RGW realm, zonegroup and +the different related entities. Enabling -------- @@ -18,36 +19,111 @@ RGW Realm Operations Bootstrapping RGW realm creates a new RGW realm entity, a new zonegroup, and a new zone. It configures a new system user that can be used for -multisite sync operations, and returns a corresponding token. It sets -up new RGW instances via the orchestrator. +multisite sync operations. Under the hood this module instructs the +orchestrator to create and deploy the corresponding RGW daemons. The module +supports both passing the arguments in the cmd line as in the form of a spec +file: -It is also possible to create a new zone that connects to the master -zone and synchronizes data to/from it. +.. prompt:: bash # + + rgw realm bootstrap [--realm-name] [--zonegroup-name] [--zone-name] [--port] [--placement] [--start-radosgw] + +The command supports providing the configuration through a spec file (`-i option`): + +.. prompt:: bash # + + ceph rgw realm bootstrap -i myrgw.yaml + +Following is an example of RGW mutlisite spec file: + +.. code-block:: yaml + + rgw_realm: myrealm + rgw_zonegroup: myzonegroup + rgw_zone: myzone + placement: + hosts: + - ceph-node-1 + - ceph-node-2 + spec: + rgw_frontend_port: 5500 + +.. note:: The spec file used by RGW has the same format as the one used by cephadm. Thus, + the user can provide any cephadm rgw supported parameter as any other advanced + configuration items such as SSL certificates etc. Realm Credentials Token ----------------------- -A new token is created when bootstrapping a new realm, and also -when creating one explicitly. The token encapsulates -the master zone endpoint, and a set of credentials that are associated -with a system user. -Removal of this token would remove the credentials, and if the corresponding -system user has no more access keys, it is removed. +User can list the available tokens for the created (or already existing) realms. +The token is a base64 string that encapsulates the realm information and its +master zone endpoint authentication data. Following is an example of +the `ceph rgw realm tokens` output: + +.. prompt:: bash # + + ceph rgw realm tokens | jq + +.. code-block:: json + + [ + { + "realm": "myrealm1", + "token": "ewogICAgInJlYWxtX25hbWUiOiAibXlyZWFs....NHlBTFhoIgp9" + }, + { + "realm": "myrealm2", + "token": "ewogICAgInJlYWxtX25hbWUiOiAibXlyZWFs....RUU12ZDB0Igp9" + } + ] + +User can use the token to create and synchronize a secondary zones +on another cluster with the master zone by using `ceph rgw zone create` +command and proving the corresponding token. + +Following is an example of zone spec file: + +.. code-block:: yaml + + rgw_realm: myrealm + rgw_zonegroup: myzonegroup + rgw_zone: my-secondary-zone + rgw_realm_token: + placement: + hosts: + - ceph-node-1 + - ceph-node-2 + spec: + rgw_frontend_port: 5500 + + +.. prompt:: bash # + + ceph rgw zone create -i zone-spec.yaml + +.. note:: The spec file used by RGW has the same format as the one used by cephadm. Thus, + the user can provide any cephadm rgw supported parameter as any other advanced + configuration items such as SSL certificates etc. Commands -------- :: - ceph rgw realm bootstrap + ceph rgw realm bootstrap -i spec.yaml Create a new realm + zonegroup + zone and deploy rgw daemons via the -orchestrator. Command returns a realm token that allows new zones to easily -join this realm +orchestrator using the information specified in the YAML file. + +:: + + ceph rgw realm tokens + +List the tokens of all the available realms :: - ceph rgw zone create + ceph rgw zone create -i spec.yaml Create a new zone and join existing realm (using the realm token) @@ -60,7 +136,7 @@ Create new credentials and return a token for new zone connection :: ceph rgw zone-creds remove - + Remove credentials and/or user that are associated with the specified token diff --git a/src/pybind/mgr/cephadm/serve.py b/src/pybind/mgr/cephadm/serve.py index c93e4147dc14..1cf7ffaa4553 100644 --- a/src/pybind/mgr/cephadm/serve.py +++ b/src/pybind/mgr/cephadm/serve.py @@ -10,7 +10,7 @@ from typing import TYPE_CHECKING, Optional, List, cast, Dict, Any, Union, Tuple, from ceph.deployment import inventory from ceph.deployment.drive_group import DriveGroupSpec -from ceph.deployment.service_spec import ServiceSpec, CustomContainerSpec, PlacementSpec +from ceph.deployment.service_spec import ServiceSpec, CustomContainerSpec, PlacementSpec, RGWSpec from ceph.utils import datetime_now import orchestrator @@ -821,6 +821,28 @@ class CephadmServe: # This can be accomplished by scheduling a restart of the active mgr. self.mgr._schedule_daemon_action(active_mgr.name(), 'restart') + if service_type == 'rgw': + rgw_spec = cast(RGWSpec, spec) + if rgw_spec.update_endpoints and rgw_spec.rgw_realm_token is not None: + ep = [] + for s in self.mgr.cache.get_daemons_by_service(rgw_spec.service_name()): + if s.ports: + for p in s.ports: + ep.append(f'http://{s.hostname}:{p}') + zone_update_cmd = { + 'prefix': 'rgw zone update', + 'realm_name': rgw_spec.rgw_realm, + 'zonegroup_name': rgw_spec.rgw_zonegroup, + 'zone_name': rgw_spec.rgw_zone, + 'realm_token': rgw_spec.rgw_realm_token, + 'endpoints': ep, + } + self.log.debug(f'rgw cmd: {zone_update_cmd}') + rc, out, err = self.mgr.mon_command(zone_update_cmd) + rgw_spec.update_endpoints = (rc != 0) # keep trying on failure + if rc != 0: + self.log.error(f'Error when trying to update rgw zone {err}.. keep trying') + # remove any? def _ok_to_stop(remove_daemons: List[orchestrator.DaemonDescription]) -> bool: daemon_ids = [d.daemon_id for d in remove_daemons] diff --git a/src/pybind/mgr/cephadm/services/cephadmservice.py b/src/pybind/mgr/cephadm/services/cephadmservice.py index e7c15132d79c..cde7a021062f 100644 --- a/src/pybind/mgr/cephadm/services/cephadmservice.py +++ b/src/pybind/mgr/cephadm/services/cephadmservice.py @@ -827,7 +827,7 @@ class RgwService(CephService): def config(self, spec: RGWSpec) -> None: # type: ignore assert self.TYPE == spec.service_type - # set rgw_realm and rgw_zone, if present + # set rgw_realm rgw_zonegroup and rgw_zone, if present if spec.rgw_realm: ret, out, err = self.mgr.check_mon_command({ 'prefix': 'config set', @@ -835,6 +835,13 @@ class RgwService(CephService): 'name': 'rgw_realm', 'value': spec.rgw_realm, }) + if spec.rgw_zonegroup: + ret, out, err = self.mgr.check_mon_command({ + 'prefix': 'config set', + 'who': f"{utils.name_to_config_section('rgw')}.{spec.service_id}", + 'name': 'rgw_zonegroup', + 'value': spec.rgw_zonegroup, + }) if spec.rgw_zone: ret, out, err = self.mgr.check_mon_command({ 'prefix': 'config set', diff --git a/src/pybind/mgr/orchestrator/module.py b/src/pybind/mgr/orchestrator/module.py index 73bd137484db..8821666e4362 100644 --- a/src/pybind/mgr/orchestrator/module.py +++ b/src/pybind/mgr/orchestrator/module.py @@ -1131,6 +1131,7 @@ Usage: placement: Optional[str] = None, _end_positional_: int = 0, realm: Optional[str] = None, + zonegroup: Optional[str] = None, zone: Optional[str] = None, port: Optional[int] = None, ssl: bool = False, @@ -1153,6 +1154,7 @@ Usage: spec = RGWSpec( service_id=svc_id, rgw_realm=realm, + rgw_zonegroup=zonegroup, rgw_zone=zone, rgw_frontend_port=port, ssl=ssl, diff --git a/src/pybind/mgr/rgw/__init__.py b/src/pybind/mgr/rgw/__init__.py index b14787608bc6..081f4c7d85dd 100644 --- a/src/pybind/mgr/rgw/__init__.py +++ b/src/pybind/mgr/rgw/__init__.py @@ -1,3 +1,4 @@ +# flake8: noqa try: from .module import Module except ImportError: diff --git a/src/pybind/mgr/rgw/module.py b/src/pybind/mgr/rgw/module.py index c8b7a59bbf4a..fd7246c28a27 100644 --- a/src/pybind/mgr/rgw/module.py +++ b/src/pybind/mgr/rgw/module.py @@ -1,17 +1,16 @@ -import logging +import json import threading -import os -import subprocess +import yaml +import errno +import base64 -from mgr_module import MgrModule, CLICommand, HandleCommandResult, Option +from mgr_module import MgrModule, CLICommand, HandleCommandResult import orchestrator -from ceph.deployment.service_spec import RGWSpec +from ceph.deployment.service_spec import RGWSpec, PlacementSpec +from typing import Any, Optional, Sequence, Iterator, List -from typing import cast, Any, Optional, Sequence - -from . import * -from ceph.rgw.types import RGWAMException, RGWAMEnvMgr +from ceph.rgw.types import RGWAMException, RGWAMEnvMgr, RealmToken from ceph.rgw.rgwam_core import EnvArgs, RGWAM @@ -20,19 +19,15 @@ class RGWAMOrchMgr(RGWAMEnvMgr): self.mgr = mgr def tool_exec(self, prog, args): - cmd = [ prog ] + args - rc, stdout, stderr = self.mgr.tool_exec(args = cmd) + cmd = [prog] + args + rc, stdout, stderr = self.mgr.tool_exec(args=cmd) return cmd, rc, stdout, stderr - def apply_rgw(self, svc_id, realm_name, zone_name, port = None): - spec = RGWSpec(service_id = svc_id, - rgw_realm = realm_name, - rgw_zone = zone_name, - rgw_frontend_port = port) + def apply_rgw(self, spec): completion = self.mgr.apply_rgw(spec) orchestrator.raise_if_exception(completion) - def list_daemons(self, service_name, daemon_type = None, daemon_id = None, host = None, refresh = True): + def list_daemons(self, service_name, daemon_type=None, daemon_id=None, host=None, refresh=True): completion = self.mgr.list_daemons(service_name, daemon_type, daemon_id=daemon_id, @@ -63,8 +58,6 @@ class Module(orchestrator.OrchestratorClientMixin, MgrModule): self.run = True self.event = threading.Event() - - def config_notify(self) -> None: """ This method is called whenever one of our config options is changed. @@ -85,7 +78,6 @@ class Module(orchestrator.OrchestratorClientMixin, MgrModule): self.get_ceph_option(opt)) self.log.debug(' native option %s = %s', opt, getattr(self, opt)) - @CLICommand('rgw admin', perm='rw') def _cmd_rgw_admin(self, params: Sequence[str]): """rgw admin""" @@ -99,30 +91,55 @@ class Module(orchestrator.OrchestratorClientMixin, MgrModule): @CLICommand('rgw realm bootstrap', perm='rw') def _cmd_rgw_realm_bootstrap(self, - realm_name : Optional[str] = None, + realm_name: Optional[str] = None, zonegroup_name: Optional[str] = None, zone_name: Optional[str] = None, - endpoints: Optional[str] = None, - sys_uid: Optional[str] = None, - uid: Optional[str] = None, - start_radosgw: Optional[bool] = True): + port: Optional[int] = None, + placement: Optional[str] = None, + start_radosgw: Optional[bool] = True, + inbuf: Optional[str] = None): """Bootstrap new rgw realm, zonegroup, and zone""" - - try: - retval, out, err = RGWAM(self.env).realm_bootstrap(realm_name, zonegroup_name, - zone_name, endpoints, sys_uid, uid, start_radosgw) + if inbuf: + rgw_specs = self._parse_rgw_specs(inbuf) + elif (realm_name and zonegroup_name and zone_name): + placement_spec = PlacementSpec.from_string(placement) if placement else None + rgw_specs = [RGWSpec(rgw_realm=realm_name, + rgw_zonegroup=zonegroup_name, + rgw_zone=zone_name, + rgw_frontend_port=port, + placement=placement_spec)] + else: + return HandleCommandResult(retval=-errno.EINVAL, stdout='', stderr='Invalid arguments: -h or --help for usage') + + for spec in rgw_specs: + RGWAM(self.env).realm_bootstrap(spec, start_radosgw) + except RGWAMException as e: self.log.error('cmd run exception: (%d) %s' % (e.retcode, e.message)) return (e.retcode, e.message, e.stderr) - return HandleCommandResult(retval=retval, stdout=out, stderr=err) + return HandleCommandResult(retval=0, stdout="Realm(s) created correctly. Please, use 'ceph rgw realm tokens' to get the token.", stderr='') + + def _parse_rgw_specs(self, inbuf: Optional[str] = None): + """Parse RGW specs from a YAML file.""" + # YAML '---' document separator with no content generates + # None entries in the output. Let's skip them silently. + yaml_objs: Iterator = yaml.safe_load_all(inbuf) + specs = [o for o in yaml_objs if o is not None] + rgw_specs = [] + for spec in specs: + # TODO(rkachach): should we use a new spec instead of RGWSpec here! + rgw_spec = RGWSpec.from_json(spec) + rgw_spec.validate() + rgw_specs.append(rgw_spec) + return rgw_specs @CLICommand('rgw realm zone-creds create', perm='rw') def _cmd_rgw_realm_new_zone_creds(self, - realm_name: Optional[str] = None, - endpoints: Optional[str] = None, - sys_uid: Optional[str] = None): + realm_name: Optional[str] = None, + endpoints: Optional[str] = None, + sys_uid: Optional[str] = None): """Create credentials for new zone creation""" try: @@ -134,8 +151,7 @@ class Module(orchestrator.OrchestratorClientMixin, MgrModule): return HandleCommandResult(retval=retval, stdout=out, stderr=err) @CLICommand('rgw realm zone-creds remove', perm='rw') - def _cmd_rgw_realm_rm_zone_creds(self, - realm_token : Optional[str] = None): + def _cmd_rgw_realm_rm_zone_creds(self, realm_token: Optional[str] = None): """Create credentials for new zone creation""" try: @@ -146,18 +162,83 @@ class Module(orchestrator.OrchestratorClientMixin, MgrModule): return HandleCommandResult(retval=retval, stdout=out, stderr=err) + @CLICommand('rgw realm tokens', perm='r') + def list_realm_tokens(self): + realms_info = [] + for realm_info in RGWAM(self.env).get_realms_info(): + if not realm_info['master_zone_id']: + realms_info.append({'realm': realm_info['realm_name'], 'token': 'realm has no master zone'}) + elif not realm_info['endpoint']: + realms_info.append({'realm': realm_info['realm_name'], 'token': 'master zone has no endpoint'}) + elif not (realm_info['access_key'] and realm_info['secret']): + realms_info.append({'realm': realm_info['realm_name'], 'token': 'master zone has no access/secret keys'}) + else: + keys = ['realm_name', 'realm_id', 'is_primary', 'endpoint', 'access_key', 'secret'] + realm_token = RealmToken(**{k: realm_info[k] for k in keys}) + realm_token_b = realm_token.to_json().encode('utf-8') + realm_token_s = base64.b64encode(realm_token_b).decode('utf-8') + realms_info.append({'realm': realm_info['realm_name'], 'token': realm_token_s}) + + return HandleCommandResult(retval=0, stdout=json.dumps(realms_info), stderr='') + + @CLICommand('rgw zone update', perm='rw') + def update_zone_info(self, realm_name: str, zonegroup_name: str, zone_name: str, realm_token: str, endpoints: List[str]): + try: + retval, out, err = RGWAM(self.env).zone_modify(realm_name, + zonegroup_name, + zone_name, + endpoints, + realm_token) + return (retval, 'Zone updated successfully', '') + except RGWAMException as e: + self.log.error('cmd run exception: (%d) %s' % (e.retcode, e.message)) + return (e.retcode, e.message, e.stderr) + @CLICommand('rgw zone create', perm='rw') def _cmd_rgw_zone_create(self, - realm_token : Optional[str] = None, - zonegroup_name: Optional[str] = None, zone_name: Optional[str] = None, - endpoints: Optional[str] = None, - start_radosgw: Optional[bool] = True): + realm_token: Optional[str] = None, + port: Optional[int] = None, + placement: Optional[str] = None, + start_radosgw: Optional[bool] = True, + inbuf: Optional[str] = None): """Bootstrap new rgw zone that syncs with existing zone""" + try: + if inbuf: + rgw_specs = self._parse_rgw_specs(inbuf) + elif (zone_name and realm_token): + placement_spec = PlacementSpec.from_string(placement) if placement else None + rgw_specs = [RGWSpec(rgw_realm_token=realm_token, + rgw_zone=zone_name, + rgw_frontend_port=port, + placement=placement_spec)] + else: + return HandleCommandResult(retval=-errno.EINVAL, stdout='', stderr='Invalid arguments: -h or --help for usage') + + for rgw_spec in rgw_specs: + retval, out, err = RGWAM(self.env).zone_create(rgw_spec, start_radosgw) + if retval != 0: + break + + except RGWAMException as e: + self.log.error('cmd run exception: (%d) %s' % (e.retcode, e.message)) + return (e.retcode, e.message, e.stderr) + + return HandleCommandResult(retval=retval, stdout=out, stderr=err) + + @CLICommand('rgw zonegroup create', perm='rw') + def _cmd_rgw_zonegroup_create(self, + realm_token: Optional[str] = None, + zonegroup_name: Optional[str] = None, + endpoints: Optional[str] = None, + zonegroup_is_master: Optional[bool] = True): + """Bootstrap new rgw zonegroup""" try: - retval, out, err = RGWAM(self.env).zone_create(realm_token, zonegroup_name, - zone_name, endpoints, start_radosgw) + retval, out, err = RGWAM(self.env).zonegroup_create(realm_token, + zonegroup_name, + endpoints, + zonegroup_is_master) except RGWAMException as e: self.log.error('cmd run exception: (%d) %s' % (e.retcode, e.message)) return (e.retcode, e.message, e.stderr) @@ -166,10 +247,10 @@ class Module(orchestrator.OrchestratorClientMixin, MgrModule): @CLICommand('rgw realm reconcile', perm='rw') def _cmd_rgw_realm_reconcile(self, - realm_name : Optional[str] = None, - zonegroup_name: Optional[str] = None, - zone_name: Optional[str] = None, - update: Optional[bool] = False): + realm_name: Optional[str] = None, + zonegroup_name: Optional[str] = None, + zone_name: Optional[str] = None, + update: Optional[bool] = False): """Bootstrap new rgw zone that syncs with existing zone""" try: diff --git a/src/python-common/ceph/deployment/service_spec.py b/src/python-common/ceph/deployment/service_spec.py index 70757304ad9e..33fff1c881f1 100644 --- a/src/python-common/ceph/deployment/service_spec.py +++ b/src/python-common/ceph/deployment/service_spec.py @@ -839,6 +839,7 @@ class RGWSpec(ServiceSpec): service_id: myrealm.myzone spec: rgw_realm: myrealm + rgw_zonegroup: myzonegroup rgw_zone: myzone ssl: true rgw_frontend_port: 1234 @@ -851,6 +852,7 @@ class RGWSpec(ServiceSpec): MANAGED_CONFIG_OPTIONS = ServiceSpec.MANAGED_CONFIG_OPTIONS + [ 'rgw_zone', 'rgw_realm', + 'rgw_zonegroup', 'rgw_frontends', ] @@ -859,6 +861,7 @@ class RGWSpec(ServiceSpec): service_id: Optional[str] = None, placement: Optional[PlacementSpec] = None, rgw_realm: Optional[str] = None, + rgw_zonegroup: Optional[str] = None, rgw_zone: Optional[str] = None, rgw_frontend_port: Optional[int] = None, rgw_frontend_ssl_certificate: Optional[List[str]] = None, @@ -872,6 +875,8 @@ class RGWSpec(ServiceSpec): extra_container_args: Optional[List[str]] = None, extra_entrypoint_args: Optional[List[str]] = None, custom_configs: Optional[List[CustomConfig]] = None, + rgw_realm_token: Optional[str] = None, + update_endpoints: Optional[bool] = False, ): assert service_type == 'rgw', service_type @@ -888,6 +893,8 @@ class RGWSpec(ServiceSpec): #: The RGW realm associated with this service. Needs to be manually created self.rgw_realm: Optional[str] = rgw_realm + #: The RGW zonegroup associated with this service. Needs to be manually created + self.rgw_zonegroup: Optional[str] = rgw_zonegroup #: The RGW zone associated with this service. Needs to be manually created self.rgw_zone: Optional[str] = rgw_zone #: Port of the RGW daemons @@ -898,6 +905,8 @@ class RGWSpec(ServiceSpec): self.rgw_frontend_type: Optional[str] = rgw_frontend_type #: enable SSL self.ssl = ssl + self.rgw_realm_token = rgw_realm_token + self.update_endpoints = update_endpoints def get_port_start(self) -> List[int]: return [self.get_port()] diff --git a/src/python-common/ceph/rgw/rgwam_core.py b/src/python-common/ceph/rgw/rgwam_core.py index 7f073bf54b9c..af7197ddb8ee 100644 --- a/src/python-common/ceph/rgw/rgwam_core.py +++ b/src/python-common/ceph/rgw/rgwam_core.py @@ -13,12 +13,9 @@ import base64 import logging import errno -from urllib.parse import urlparse - from .types import RGWAMException, RGWAMCmdRunException, RGWPeriod, RGWUser, RealmToken from .diff import RealmsEPs - DEFAULT_PORT = 8000 log = logging.getLogger(__name__) @@ -193,39 +190,32 @@ class RealmOp: self.env = env def list(self): - ze = ZoneEnv(self.env) - - params = ['realm', - 'list'] - - return RGWAdminJSONCmd(ze).run(params) + try: + ze = ZoneEnv(self.env) + params = ['realm', 'list'] + output = RGWAdminJSONCmd(ze).run(params) + return output.get('realms') or [] + except RGWAMException: + # in case the realm list is empty an exception is raised + return [] def get(self, realm: EntityKey = None): - ze = ZoneEnv(self.env, realm=realm) - - params = ['realm', - 'get'] - + params = ['realm', 'get'] return RGWAdminJSONCmd(ze).run(params) def create(self, realm: EntityKey = None): ze = ZoneEnv(self.env).init_realm(realm=realm, gen=True) - - params = ['realm', - 'create'] - + params = ['realm', 'create'] return RGWAdminJSONCmd(ze).run(params) - def pull(self, url, access_key, secret, set_default=False): + def pull(self, realm, url, access_key, secret, set_default=False): params = ['realm', 'pull', '--url', url, '--access-key', access_key, '--secret', secret] - - ze = ZoneEnv(self.env) - + ze = ZoneEnv(self.env, realm=realm) return RGWAdminJSONCmd(ze).run(params) @@ -233,6 +223,21 @@ class ZonegroupOp: def __init__(self, env: EnvArgs): self.env = env + def list(self): + try: + ze = ZoneEnv(self.env) + params = ['zonegroup', 'list'] + output = RGWAdminJSONCmd(ze).run(params) + return output.get('zonegroups') or [] + except RGWAMException: + return [] + + def get(self, zonegroup: EntityKey = None): + ze = ZoneEnv(self.env) + params = ['zonegroup', 'get'] + opt_arg(params, '--rgw-zonegroup', zonegroup) + return RGWAdminJSONCmd(ze).run(params) + def create(self, realm: EntityKey, zg: EntityKey = None, endpoints=None, is_master=True): ze = ZoneEnv(self.env, realm=realm).init_zg(zg, gen=True) @@ -251,6 +256,15 @@ class ZoneOp: def __init__(self, env: EnvArgs): self.env = env + def list(self): + try: + ze = ZoneEnv(self.env) + params = ['zone', 'list'] + output = RGWAdminJSONCmd(ze).run(params) + return output.get('zones') or [] + except RGWAMException: + return [] + def get(self, zone: EntityKey): ze = ZoneEnv(self.env, zone=zone) @@ -295,20 +309,50 @@ class PeriodOp: self.env = env def update(self, realm: EntityKey, zonegroup: EntityKey, zone: EntityKey, commit=True): - ze = ZoneEnv(self.env, realm=realm, zg=zonegroup, zone=zone) - - params = ['period', - 'update'] - + master_zone_info = self.get_master_zone(realm, zonegroup) + master_zone = EntityName(master_zone_info['name']) if master_zone_info else zone + master_zonegroup_info = self.get_master_zonegroup(realm) + master_zonegroup = EntityName(master_zonegroup_info['name']) \ + if master_zonegroup_info else zonegroup + ze = ZoneEnv(self.env, realm=realm, zg=master_zonegroup, zone=master_zone) + params = ['period', 'update'] opt_arg_bool(params, '--commit', commit) - return RGWAdminJSONCmd(ze).run(params) + def get_master_zone(self, realm, zonegroup=None): + try: + ze = ZoneEnv(self.env, realm=realm, zg=zonegroup) + params = ['zone', 'get'] + return RGWAdminJSONCmd(ze).run(params) + except RGWAMCmdRunException: + return None + + def get_master_zone_ep(self, realm, zonegroup=None): + try: + ze = ZoneEnv(self.env, realm=realm, zg=zonegroup) + params = ['period', 'get'] + output = RGWAdminJSONCmd(ze).run(params) + for zg in output['period_map']['zonegroups']: + if not bool(zg['is_master']): + continue + for zone in zg['zones']: + if zone['id'] == zg['master_zone']: + return zone['endpoints'] + return None + except RGWAMCmdRunException: + return None + + def get_master_zonegroup(self, realm): + try: + ze = ZoneEnv(self.env, realm=realm) + params = ['zonegroup', 'get'] + return RGWAdminJSONCmd(ze).run(params) + except RGWAMCmdRunException: + return None + def get(self, realm=None): ze = ZoneEnv(self.env, realm=realm) - params = ['period', - 'get'] - + params = ['period', 'get'] return RGWAdminJSONCmd(ze).run(params) @@ -386,101 +430,125 @@ class RGWAM: def user_op(self): return UserOp(self.env) - def realm_bootstrap(self, realm_name, zonegroup_name, zone_name, endpoints, sys_uid, uid, - start_radosgw): - endpoints = get_endpoints(endpoints) + def get_realm(self, realm_name): + try: + realm_info = self.realm_op().get(EntityName(realm_name)) + realm = EntityKey(realm_info['name'], realm_info['id']) + return realm + except RGWAMException: + raise None + def create_realm(self, realm_name): try: realm_info = self.realm_op().create(EntityName(realm_name)) + realm = EntityKey(realm_info['name'], realm_info['id']) + logging.info(f'Created realm name={realm.name} id={realm.id}') + return realm except RGWAMException as e: raise RGWAMException('failed to create realm', e) - realm_name = realm_info['name'] - realm_id = realm_info['id'] - - realm = EntityID(realm_id) - - logging.info('Created realm %s (%s)' % (realm_name, realm_id)) - + def create_zonegroup(self, realm, zonegroup_name, zonegroup_is_master, endpoints=None): try: - zg_info = self.zonegroup_op().create(realm, EntityName(zonegroup_name), endpoints, - is_master=True) + zg_info = self.zonegroup_op().create(realm, + EntityName(zonegroup_name), + endpoints, + is_master=zonegroup_is_master) + zonegroup = EntityKey(zg_info['name'], zg_info['id']) + logging.info(f'Created zonegroup name={zonegroup.name} id={zonegroup.id}') + return zonegroup except RGWAMException as e: raise RGWAMException('failed to create zonegroup', e) - zg_name = zg_info['name'] - zg_id = zg_info['id'] - logging.info('Created zonegroup %s (%s)' % (zg_name, zg_id)) - - zg = EntityName(zg_name) - + def create_zone(self, realm, zg, zone_name, zone_is_master, access_key=None, + secret=None, endpoints=None): try: - zone_info = self.zone_op().create(realm, zg, EntityName(zone_name), endpoints, - is_master=True) + zone_info = self.zone_op().create(realm, zg, + EntityName(zone_name), + endpoints, + is_master=zone_is_master, + access_key=access_key, + secret=secret) + + zone = EntityKey(zone_info['name'], zone_info['id']) + logging.info(f'Created zone name={zone.name} id={zone.id}') + return zone except RGWAMException as e: raise RGWAMException('failed to create zone', e) - zone_name = zone_info['name'] - zone_id = zone_info['id'] - logging.info('Created zone %s (%s)' % (zone_name, zone_id)) - - zone = EntityName(zone_name) - + def create_system_user(self, realm, zonegroup, zone): try: - period_info = self.period_op().update(realm, EntityName(zg_name), zone, commit=True) - except RGWAMCmdRunException as e: - raise RGWAMException('failed to update period', e) - - period = RGWPeriod(period_info) - - logging.info('Period: ' + period.id) - - try: - sys_user_info = self.user_op().create(zone, zg, uid=sys_uid, uid_prefix='user-sys', + sys_user_info = self.user_op().create(zone, + zonegroup, + uid=f'sysuser-{realm.name}', + uid_prefix='user-sys', is_system=True) + sys_user = RGWUser(sys_user_info) + logging.info('Created system user: %s' % sys_user.uid) + access_key = sys_user.keys[0].access_key if sys_user and sys_user.keys else '' + secret_key = sys_user.keys[0].secret_key if sys_user and sys_user.keys else '' + sys_user.add_key(access_key, secret_key) + return sys_user except RGWAMException as e: raise RGWAMException('failed to create system user', e) - sys_user = RGWUser(sys_user_info) - - logging.info('Created system user: %s' % sys_user.uid) - - sys_access_key = '' - sys_secret = '' - - if len(sys_user.keys) > 0: - sys_access_key = sys_user.keys[0].access_key - sys_secret = sys_user.keys[0].secret_key - - try: - zone_info = self.zone_op().modify(zone, zg, endpoints, None, sys_access_key, sys_secret) - except RGWAMException as e: - raise RGWAMException('failed to modify zone info', e) - + def create_normal_user(self, zg, zone, uid=None): try: user_info = self.user_op().create(zone, zg, uid=uid, is_system=False) + user = RGWUser(user_info) + logging.info('Created regular user: %s' % user.uid) + return user except RGWAMException as e: raise RGWAMException('failed to create user', e) - user = RGWUser(user_info) - - logging.info('Created regular user: %s' % user.uid) - - eps = endpoints.split(',') - ep = '' - if len(eps) > 0: - ep = eps[0] - if start_radosgw: - o = urlparse(ep) - svc_id = realm_name + '.' + zone_name - self.env.mgr.apply_rgw(svc_id, realm_name, zone_name, o.port) - - realm_token = RealmToken(realm_id, ep, sys_user.uid, sys_access_key, sys_secret) - - logging.info(realm_token.to_json()) + def update_period(self, realm, zg, zone=None): + try: + period_info = self.period_op().update(realm, zg, zone, commit=True) + period = RGWPeriod(period_info) + logging.info('Period: ' + period.id) + except RGWAMCmdRunException as e: + raise RGWAMException('failed to update period', e) - realm_token_b = realm_token.to_json().encode('utf-8') - return (0, 'Realm Token: %s' % base64.b64encode(realm_token_b).decode('utf-8'), '') + def realm_bootstrap(self, rgw_spec, start_radosgw=True): + + realm_name = rgw_spec.rgw_realm + zonegroup_name = rgw_spec.rgw_zonegroup + zone_name = rgw_spec.rgw_zone + + # Some sanity checks + if realm_name in self.realm_op().list(): + raise RGWAMException(f'Realm {realm_name} already exists') + if zonegroup_name in self.zonegroup_op().list(): + raise RGWAMException(f'ZonegroupOp {zonegroup_name} already exists') + if zone_name in self.zone_op().list(): + raise RGWAMException(f'Zone {zone_name} already exists') + + # Create RGW multisite entities and update the period + realm = self.create_realm(realm_name) + zonegroup = self.create_zonegroup(realm, zonegroup_name, zonegroup_is_master=True) + zone = self.create_zone(realm, zonegroup, zone_name, zone_is_master=True) + self.update_period(realm, zonegroup) + + # Create system user, normal user and update the master zone + sys_user = self.create_system_user(realm, zonegroup, zone) + access_key = sys_user.keys[0].access_key + secret = sys_user.keys[0].secret_key + self.zone_op().modify(zone, zonegroup, None, None, access_key, secret) + self.update_period(realm, zonegroup) + self.create_normal_user(zonegroup, zone) + + if start_radosgw: + # Instruct the orchestrator to start RGW daemons, asynchronically, this will + # call back the rgw module to update the master zone with the corresponding endpoints + realm_token = RealmToken(realm_name, + realm.id, + True, # primary cluster + None, # no endpoint + access_key, secret) + realm_token_b = realm_token.to_json().encode('utf-8') + realm_token_s = base64.b64encode(realm_token_b).decode('utf-8') + rgw_spec.rgw_realm_token = realm_token_s + rgw_spec.update_endpoints = True + self.env.mgr.apply_rgw(rgw_spec) def realm_new_zone_creds(self, realm_name, endpoints, sys_uid): try: @@ -532,7 +600,7 @@ class RGWAM: sys_access_key = sys_user.keys[0].access_key sys_secret = sys_user.keys[0].secret_key - realm_token = RealmToken(period.realm_id, ep, sys_user.uid, sys_access_key, sys_secret) + realm_token = RealmToken(realm_name, period.realm_id, ep, sys_access_key, sys_secret) logging.info(realm_token.to_json()) @@ -544,37 +612,26 @@ class RGWAM: print('missing realm token') return False - realm_token_b = base64.b64decode(realm_token_b64) - realm_token_s = realm_token_b.decode('utf-8') - - realm_token = json.loads(realm_token_s) - - access_key = realm_token['access_key'] - realm_id = realm_token['realm_id'] - + realm_token = RealmToken.from_base64_str(realm_token_b64) try: - period_info = self.period_op().get(EntityID(realm_id)) + period_info = self.period_op().get(EntityID(realm_token.realm_id)) except RGWAMException as e: raise RGWAMException('failed to fetch period info', e) period = RGWPeriod(period_info) - master_zg = EntityID(period.master_zonegroup) master_zone = EntityID(period.master_zone) - logging.info('Period: ' + period.id) logging.info('Master zone: ' + period.master_zone) - try: zone_info = self.zone_op().get(zone=master_zone) except RGWAMException as e: raise RGWAMException('failed to access master zone', e) - zone_id = zone_info['id'] - - if period.master_zone != zone_id: + if period.master_zone != zone_info['id']: return (-errno.EINVAL, '', 'Command needs to run on master zone') + access_key = realm_token.access_key try: user_info = self.user_op().info(master_zone, master_zg, access_key=access_key) except RGWAMException as e: @@ -612,99 +669,159 @@ class RGWAM: return (0, success_message, '') - def zone_create(self, realm_token_b64, zonegroup_name=None, zone_name=None, - endpoints=None, start_radosgw=True): - if not realm_token_b64: - print('missing realm access config') - return False - - realm_token_b = base64.b64decode(realm_token_b64) - realm_token_s = realm_token_b.decode('utf-8') - - realm_token = json.loads(realm_token_s) + def zone_modify(self, realm_name, zonegroup_name, zone_name, endpoints, realm_token_b64): - access_key = realm_token['access_key'] - secret = realm_token['secret'] + if not realm_token_b64: + raise RGWAMException('missing realm access config') + if zone_name is None: + raise RGWAMException('Zone name is a mandatory parameter') + realm_token = RealmToken.from_base64_str(realm_token_b64) + access_key = realm_token.access_key + secret = realm_token.secret try: - realm_info = self.realm_op().pull( - realm_token['endpoint'], access_key, secret, set_default=True) + # We only pull the realm if we are on a secondary cluster + if not realm_token.is_primary and realm_token.endpoint is not None: + self.realm_op().pull(EntityName(realm_name), + realm_token.endpoint, access_key, secret) except RGWAMException as e: raise RGWAMException('failed to pull realm', e) - realm_name = realm_info['name'] - realm_id = realm_info['id'] - logging.info('Pulled realm %s (%s)' % (realm_name, realm_id)) + realm_name = realm_token.realm_name + realm_id = realm_token.realm_id + logging.info(f'Using realm {realm_name} {realm_id}') realm = EntityID(realm_id) - period_info = self.period_op().get(realm) - period = RGWPeriod(period_info) - logging.info('Period: ' + period.id) - zonegroup = period.find_zonegroup_by_name(zonegroup_name) if not zonegroup: - raise RGWAMException('zonegroup %s not found' % (zonegroup or '')) + raise RGWAMException(f'zonegroup {zonegroup_name} not found') zg = EntityName(zonegroup.name) - - try: - zone_info = self.zone_op().create(realm, zg, EntityName(zone_name), endpoints, False, - access_key, secret) - except RGWAMException as e: - raise RGWAMException('failed to create zone', e) - - zone_name = zone_info['name'] - zone_id = zone_info['id'] - zone = EntityName(zone_name) - - success_message = 'Created zone %s (%s)' % (zone_name, zone_id) + success_message = f'Modified zone {realm_name} {zonegroup_name} {zone_name}' logging.info(success_message) + try: + self.zone_op().modify(zone, zg, endpoints=','.join(endpoints), + access_key=access_key, secret=secret) + except RGWAMException as e: + raise RGWAMException('failed to modify zone', e) + # done, let's update the period try: period_info = self.period_op().update(realm, zg, zone, True) except RGWAMException as e: raise RGWAMException('failed to update period', e) period = RGWPeriod(period_info) - logging.debug(period.to_json()) - svc_id = realm_name + '.' + zone_name + return (0, success_message, '') + + def get_realms_info(self): + realms_info = [] + for realm_name in self.realm_op().list(): + realm = self.get_realm(realm_name) + master_zone_inf = self.period_op().get_master_zone(realm) + zone_ep = self.period_op().get_master_zone_ep(realm) + if master_zone_inf and 'system_key' in master_zone_inf: + access_key = master_zone_inf['system_key']['access_key'] + secret = master_zone_inf['system_key']['secret_key'] + else: + access_key = '' + secret = '' + realms_info.append({"realm_name": realm_name, + "realm_id": realm.id, + "is_primary": False, + "master_zone_id": master_zone_inf['id'] if master_zone_inf else '', + "endpoint": zone_ep[0] if zone_ep else None, + "access_key": access_key, + "secret": secret}) + return realms_info + + def zonegroup_create(self, realm_token_b64, zonegroup_name=None, + endpoints=None, zonegroup_is_master=True): + if not realm_token_b64: + print('missing realm access config') + return False + + realm_token = RealmToken.from_base64_str(realm_token_b64) + access_key = realm_token.access_key + secret = realm_token.secret + try: + realm_info = self.realm_op().pull(EntityName(realm_token.realm_name), + realm_token.endpoint, access_key, + secret, set_default=True) + except RGWAMException as e: + raise RGWAMException('failed to pull realm', e) + + realm_name = realm_info['name'] + realm_id = realm_info['id'] + logging.info(f"Pulled realm {realm_name} ({realm_id})") - # if endpoints: - # eps = endpoints.split(',') - # ep = '' - # if len(eps) > 0: - # ep = eps[0] - # o = urlparse(ep) - # port = o.port - # spec = RGWSpec(service_id = svc_id, - # rgw_realm = realm_name, - # rgw_zone = zone_name, - # rgw_frontend_port = o.port) - # self.env.mgr.apply_rgw(spec) + realm = EntityID(realm_id) + zonegroup = self.create_zonegroup(realm, zonegroup_name, zonegroup_is_master, endpoints) + self.update_period(realm, zonegroup) - self.env.mgr.apply_rgw(svc_id, realm_name, zone_name) + return (0, f'Created zonegroup {zonegroup_name} on realm {realm.name}', '') - daemons = self.env.mgr.list_daemons(svc_id, 'rgw', refresh=True) + def zone_create(self, rgw_spec, start_radosgw): - ep = [] - for s in daemons: - for p in s.ports: - ep.append('http://%s:%d' % (s.hostname, p)) + if not rgw_spec.rgw_realm_token: + raise RGWAMException('missing realm token') + if rgw_spec.rgw_zone is None: + raise RGWAMException('Zone name is a mandatory parameter') + if rgw_spec.rgw_zone in self.zone_op().list(): + raise RGWAMException(f'Zone {rgw_spec.rgw_zone} already exists') - log.error('ERROR: ep=%s' % ','.join(ep)) + realm_token = RealmToken.from_base64_str(rgw_spec.rgw_realm_token) + if realm_token.endpoint is None: + raise RGWAMException('Provided realm token has no endpoint') + access_key = realm_token.access_key + secret = realm_token.secret try: - zone_info = self.zone_op().modify(zone, zg, endpoints=','.join(ep)) + realm_info = self.realm_op().pull(EntityName(realm_token.realm_name), + realm_token.endpoint, access_key, secret) except RGWAMException as e: - raise RGWAMException('failed to modify zone', e) + raise RGWAMException('failed to pull realm', e) - return (0, success_message, '') + logging.info(f"Pulled realm {realm_info['name']} ({realm_info['id']})") + realm_name = realm_info['name'] + realm_id = realm_info['id'] + logging.info(f"Pulled realm {realm_name} ({realm_id})") + + realm = EntityID(realm_id) + period_info = self.period_op().get(realm) + period = RGWPeriod(period_info) + logging.info('Period: ' + period.id) + + zonegroup = period.find_zonegroup_by_name(rgw_spec.rgw_zonegroup) + if not zonegroup: + raise RGWAMException(f'zonegroup {rgw_spec.rgw_zonegroup}') + + zone = self.create_zone(realm, zonegroup, rgw_spec.rgw_zone, False, access_key, secret) + self.update_period(realm, zonegroup, zone) + + period = RGWPeriod(period_info) + logging.debug(period.to_json()) + + if start_radosgw: + secondary_realm_token = RealmToken(realm_name, + realm_id, + False, # is_primary + None, # no endpoint + realm_token.access_key, + realm_token.secret) + realm_token_b = secondary_realm_token.to_json().encode('utf-8') + realm_token_s = base64.b64encode(realm_token_b).decode('utf-8') + rgw_spec.update_endpoints = True + rgw_spec.rgw_token = realm_token_s + self.env.mgr.apply_rgw(rgw_spec) + + return (0, f'Created zone {zone.name} {zone.id}', '') def _get_daemon_eps(self, realm_name=None, zonegroup_name=None, zone_name=None): # get running daemons info @@ -754,14 +871,11 @@ class RGWAM: rep = RealmsEPs() try: - realm_list_ret = self.realm_op().list() + realms = self.realm_op().list() except RGWAMException as e: raise RGWAMException('failed to list realms', e) - realms = realm_list_ret.get('realms') or [] - zones_map = {} - for realm in realms: if realm_name and realm != realm_name: log.debug('skipping realm %s' % realm) diff --git a/src/python-common/ceph/rgw/types.py b/src/python-common/ceph/rgw/types.py index 7f58cd7f6066..4b894f29755a 100644 --- a/src/python-common/ceph/rgw/types.py +++ b/src/python-common/ceph/rgw/types.py @@ -1,4 +1,5 @@ import json +import base64 from abc import abstractmethod @@ -47,13 +48,21 @@ class JSONObj: class RealmToken(JSONObj): - def __init__(self, realm_id, endpoint, uid, access_key, secret): + def __init__(self, realm_name, realm_id, is_primary, endpoint, access_key, secret): + self.realm_name = realm_name self.realm_id = realm_id + self.is_primary = is_primary self.endpoint = endpoint - self.uid = uid self.access_key = access_key self.secret = secret + @classmethod + def from_base64_str(cls, realm_token_b64): + realm_token_b = base64.b64decode(realm_token_b64) + realm_token_s = realm_token_b.decode('utf-8') + realm_token = json.loads(realm_token_s) + return cls(**realm_token) + class RGWZone(JSONObj): def __init__(self, zone_dict): @@ -160,3 +169,8 @@ class RGWUser(JSONObj): is_system = d.get('system') or 'false' self.system = (is_system == 'true') + + def add_key(self, access_key, secret): + self.keys.append(RGWAccessKey({'user': self.uid, + 'access_key': access_key, + 'secret_key': secret}))