From: Yehuda Sadeh Date: Thu, 10 Jun 2021 22:25:17 +0000 (-0700) Subject: mgr/rgw: apply rgw through orchestrator interface X-Git-Tag: v17.1.0~340^2~15 X-Git-Url: http://git.apps.os.sepia.ceph.com/?a=commitdiff_plain;h=8116ac9f2f4c94e848d69675ec68423ea2ce409b;p=ceph-ci.git mgr/rgw: apply rgw through orchestrator interface Signed-off-by: Yehuda Sadeh --- diff --git a/src/pybind/mgr/rgw/module.py b/src/pybind/mgr/rgw/module.py index 4b31e4ea140..f06d7339594 100644 --- a/src/pybind/mgr/rgw/module.py +++ b/src/pybind/mgr/rgw/module.py @@ -4,33 +4,40 @@ import os import subprocess from mgr_module import MgrModule, CLICommand, HandleCommandResult, Option +import orchestrator from typing import cast, Any, Optional, Sequence from . import * from .types import RGWAMException -from .rgwam import CephCommonArgs, RGWAM +from .rgwam import EnvArgs, RGWAM -class Module(MgrModule): +class Module(orchestrator.OrchestratorClientMixin, MgrModule): MODULE_OPTIONS = [] # These are "native" Ceph options that this module cares about. NATIVE_OPTIONS = [] def __init__(self, *args: Any, **kwargs: Any): - super().__init__(*args, **kwargs) + self.inited = False + self.lock = threading.Lock() + super(Module, self).__init__(*args, **kwargs) + + # ensure config options members are initialized; see config_notify() + self.config_notify() + + with self.lock: + self.inited = True + self.env = EnvArgs(self, + str(self.get_ceph_conf_path()), + f'mgr.{self.get_mgr_id()}', + str(self.get_ceph_option('keyring'))) # set up some members to enable the serve() method and shutdown() self.run = True self.event = threading.Event() - # ensure config options members are initialized; see config_notify() - self.config_notify() - - self.ceph_common_args = CephCommonArgs(str(self.get_ceph_conf_path()), - f'mgr.{self.get_mgr_id()}', - str(self.get_ceph_option('keyring'))) def config_notify(self) -> None: @@ -86,7 +93,7 @@ class Module(MgrModule): try: - retval, out, err = RGWAM(self.ceph_common_args).realm_bootstrap(realm_name, zonegroup_name, + retval, out, err = RGWAM(self.env).realm_bootstrap(realm_name, zonegroup_name, zone_name, endpoints, sys_uid, uid, start_radosgw) except RGWAMException as e: self.log.error('cmd run exception: (%d) %s' % (e.retcode, e.message)) @@ -101,7 +108,7 @@ class Module(MgrModule): """Create credentials for new zone creation""" try: - retval, out, err = RGWAM(self.ceph_common_args).realm_new_zone_creds(endpoints, sys_uid) + retval, out, err = RGWAM(self.env).realm_new_zone_creds(endpoints, sys_uid) except RGWAMException as e: self.log.error('cmd run exception: (%d) %s' % (e.retcode, e.message)) return (e.retcode, e.message, e.stderr) @@ -118,7 +125,7 @@ class Module(MgrModule): """Bootstrap new rgw zone that syncs with existing zone""" try: - retval, out, err = RGWAM(self.ceph_common_args).zone_create(realm_token, zonegroup_name, + retval, out, err = RGWAM(self.env).zone_create(realm_token, zonegroup_name, zone_name, endpoints, start_radosgw) except RGWAMException as e: self.log.error('cmd run exception: (%d) %s' % (e.retcode, e.message)) diff --git a/src/pybind/mgr/rgw/rgwam.py b/src/pybind/mgr/rgw/rgwam.py index d814a6b3279..ffde6643549 100644 --- a/src/pybind/mgr/rgw/rgwam.py +++ b/src/pybind/mgr/rgw/rgwam.py @@ -21,6 +21,9 @@ from urllib.parse import urlparse from .types import RGWAMException, RGWAMCmdRunException, RGWPeriod, RGWUser, RealmToken +from ceph.deployment.service_spec import RGWSpec + + DEFAULT_PORT = 8000 log = logging.getLogger(__name__) @@ -47,21 +50,22 @@ def get_endpoints(endpoints, period = None): port += 1 -class CephCommonArgs: - def __init__(self, ceph_conf, ceph_name, ceph_keyring): +class EnvArgs: + def __init__(self, mgr, ceph_conf, ceph_name, ceph_keyring): + self.mgr = mgr self.ceph_conf = ceph_conf self.ceph_name = ceph_name self.ceph_keyring = ceph_keyring class RGWCmdBase: - def __init__(self, prog, common_args): + def __init__(self, prog, env): self.cmd_prefix = [ prog ] - if common_args.ceph_conf: - self.cmd_prefix += [ '-c', common_args.ceph_conf ] - if common_args.ceph_name: - self.cmd_prefix += [ '-n', common_args.ceph_name ] - if common_args.ceph_keyring: - self.cmd_prefix += [ '-k', common_args.ceph_keyring ] + if env.ceph_conf: + self.cmd_prefix += [ '-c', env.ceph_conf ] + if env.ceph_name: + self.cmd_prefix += [ '-n', env.ceph_name ] + if env.ceph_keyring: + self.cmd_prefix += [ '-k', env.ceph_keyring ] def run(self, cmd): run_cmd = self.cmd_prefix + cmd @@ -82,12 +86,12 @@ class RGWCmdBase: return (stdout, stderr) class RGWAdminCmd(RGWCmdBase): - def __init__(self, common_args): - super().__init__('radosgw-admin', common_args) + def __init__(self, env): + super().__init__('radosgw-admin', env) class RGWAdminJSONCmd(RGWAdminCmd): - def __init__(self, common_args): - super().__init__(common_args) + def __init__(self, env): + super().__init__(env) def run(self, cmd): stdout, _ = RGWAdminCmd.run(self, cmd) @@ -96,12 +100,12 @@ class RGWAdminJSONCmd(RGWAdminCmd): class RGWCmd(RGWCmdBase): - def __init__(self, common_args): - super().__init__('radosgw', common_args) + def __init__(self, env): + super().__init__('radosgw', env) class RealmOp(RGWAdminCmd): - def __init__(self, common_args): - super().__init__(common_args) + def __init__(self, env): + super().__init__(env) def get(self): params = [ 'realm', @@ -136,8 +140,8 @@ class RealmOp(RGWAdminCmd): return RGWAdminJSONCmd.run(self, params) class ZonegroupOp(RGWAdminCmd): - def __init__(self, common_args): - super().__init__(common_args) + def __init__(self, env): + super().__init__(env) def create(self, realm, name = None, endpoints = None, is_master = True, is_default = True): self.name = name @@ -163,8 +167,8 @@ class ZonegroupOp(RGWAdminCmd): return self.info class ZoneOp(RGWAdminCmd): - def __init__(self, common_args): - super().__init__(common_args) + def __init__(self, env): + super().__init__(env) def get(self): params = [ 'zone', @@ -221,8 +225,8 @@ class ZoneOp(RGWAdminCmd): return RGWAdminJSONCmd.run(self, params) class PeriodOp(RGWAdminCmd): - def __init__(self, common_args): - super().__init__(common_args) + def __init__(self, env): + super().__init__(env) def update(self, realm, commit = True): @@ -245,8 +249,8 @@ class PeriodOp(RGWAdminCmd): return RGWAdminJSONCmd.run(self, params) class UserOp(RGWAdminCmd): - def __init__(self, common_args): - super().__init__(common_args) + def __init__(self, env): + super().__init__(env) def create(self, uid = None, uid_prefix = None, display_name = None, email = None, is_system = False): self.uid = uid @@ -272,23 +276,23 @@ class UserOp(RGWAdminCmd): return RGWAdminJSONCmd.run(self, params) class RGWAM: - def __init__(self, common_args): - self.common_args = common_args + def __init__(self, env): + self.env = env def realm_op(self): - return RealmOp(self.common_args) + return RealmOp(self.env) def period_op(self): - return PeriodOp(self.common_args) + return PeriodOp(self.env) def zonegroup_op(self): - return ZonegroupOp(self.common_args) + return ZonegroupOp(self.env) def zone_op(self): - return ZoneOp(self.common_args) + return ZoneOp(self.env) def user_op(self): - return UserOp(self.common_args) + return UserOp(self.env) def realm_bootstrap(self, realm, zonegroup, zone, endpoints, sys_uid, uid, start_radosgw): endpoints = get_endpoints(endpoints) @@ -365,7 +369,12 @@ class RGWAM: ep = eps[0] if start_radosgw: o = urlparse(ep) - self.run_radosgw(port = o.port) + svc_id = realm_name + '.' + zone_name + spec = RGWSpec(service_id = svc_id, + rgw_realm = realm_name, + rgw_zone = zone_name, + rgw_frontend_port = o.port) + self.env.mgr.apply_rgw(spec) realm_token = RealmToken(ep, sys_user.uid, sys_access_key, sys_secret) @@ -483,15 +492,43 @@ class RGWAM: logging.debug(period.to_json()) - if start_radosgw: - eps = endpoints.split(',') - ep = '' - if len(eps) > 0: - ep = eps[0] - o = urlparse(ep) - ret = self.run_radosgw(port = o.port) - if not ret: - logging.warning('failed to start radosgw') + svc_id = realm_name + '.' + zone_name + + #if endpoints: + # eps = endpoints.split(',') + # ep = '' + # if len(eps) > 0: + # ep = eps[0] + # o = urlparse(ep) + # port = o.port + # spec = RGWSpec(service_id = svc_id, + # rgw_realm = realm_name, + # rgw_zone = zone_name, + # rgw_frontend_port = o.port) + # self.env.mgr.apply_rgw(spec) + + spec = RGWSpec(service_id = svc_id, + rgw_realm = realm_name, + rgw_zone = zone_name) + + completion = self.env.mgr.apply_rgw(spec) + orchestrator.raise_if_exception(completion) + + completion = self.env.mgr.list_daemons(svc_id, 'rgw', refresh=True) + + daemons = orchestrator.raise_if_exception(completion) + + ep = [] + for s in daemons: + for p in s.ports: + ep.append('http://%s:%d' % (s.hostname, p)) + + log.error('ERROR: ep=%s' % ','.join(ep)) + + try: + zone_info = self.zone_op().modify(zone_name, endpoints = ','.join(ep)) + except RGWAMException as e: + raise RGWAMException('failed to modify zone', e) return (0, success_message, '') @@ -513,7 +550,7 @@ class RGWAM: if debug_rgw: params += [ '--debug-rgw', debug_rgw ] - (retcode, stdout, stderr) = RGWCmd(self.common_args).run(params) + (retcode, stdout, stderr) = RGWCmd(self.env).run(params) return (retcode, stdout, stderr)