From de43dab44012fc239e4c8c09be264bf5a5a7981c Mon Sep 17 00:00:00 2001 From: Adam King Date: Mon, 27 Jun 2022 16:52:14 -0400 Subject: [PATCH] mgr/cephadm: allow mounting custom conf files Fixes: https://tracker.ceph.com/issues/56394 Signed-off-by: Adam King --- src/cephadm/cephadm | 90 ++++++++++++++----- src/pybind/mgr/cephadm/serve.py | 6 ++ .../mgr/cephadm/services/cephadmservice.py | 10 +-- 3 files changed, 80 insertions(+), 26 deletions(-) diff --git a/src/cephadm/cephadm b/src/cephadm/cephadm index 5c3bdc2bbcf52..c00c46fdcc4ef 100755 --- a/src/cephadm/cephadm +++ b/src/cephadm/cephadm @@ -2728,10 +2728,46 @@ def create_daemon_dirs(ctx, fsid, daemon_type, daemon_id, uid, gid, sg = SNMPGateway.init(ctx, fsid, daemon_id) sg.create_daemon_conf() + _write_custom_conf_files(ctx, daemon_type, str(daemon_id), fsid, uid, gid) -def get_parm(option): - # type: (str) -> Dict[str, str] +def _write_custom_conf_files(ctx: CephadmContext, daemon_type: str, daemon_id: str, fsid: str, uid: int, gid: int) -> None: + # mostly making this its own function to make unit testing easier + if 'config_json' not in ctx or not ctx.config_json: + return + config_json = get_custom_config_files(ctx.config_json) + custom_config_dir = os.path.join(ctx.data_dir, fsid, 'custom_config_files', f'{daemon_type}.{daemon_id}') + if not os.path.exists(custom_config_dir): + makedirs(custom_config_dir, uid, gid, 0o755) + mandatory_keys = ['mount_path', 'content'] + for ccf in config_json['custom_config_files']: + if all(k in ccf for k in mandatory_keys): + file_path = os.path.join(custom_config_dir, os.path.basename(ccf['mount_path'])) + with open(file_path, 'w+', encoding='utf-8') as f: + os.fchown(f.fileno(), uid, gid) + os.fchmod(f.fileno(), 0o600) + f.write(ccf['content']) + + +def get_parm(option: str) -> Dict[str, str]: + js = _get_config_json(option) + # custom_config_files is a special field that may be in the config + # dict. It is used for mounting custom config files into daemon's containers + # and should be accessed through the "get_custom_config_files" function. + # For get_parm we need to discard it. + js.pop('custom_config_files', None) + return js + + +def get_custom_config_files(option: str) -> Dict[str, List[Dict[str, str]]]: + js = _get_config_json(option) + res: Dict[str, List[Dict[str, str]]] = {'custom_config_files': []} + if 'custom_config_files' in js: + res['custom_config_files'] = js['custom_config_files'] + return res + + +def _get_config_json(option: str) -> Dict[str, Any]: if not option: return dict() @@ -5749,16 +5785,30 @@ def extract_uid_gid_monitoring(ctx, daemon_type): return uid, gid -def get_container_with_extra_args(ctx: CephadmContext, - fsid: str, daemon_type: str, daemon_id: Union[int, str], - privileged: bool = False, - ptrace: bool = False, - container_args: Optional[List[str]] = None) -> 'CephContainer': - # wrapper for get_container that additionally adds extra_container_args if present - # used for deploying daemons with additional podman/docker container arguments +def get_deployment_container(ctx: CephadmContext, + fsid: str, daemon_type: str, daemon_id: Union[int, str], + privileged: bool = False, + ptrace: bool = False, + container_args: Optional[List[str]] = None) -> 'CephContainer': + # wrapper for get_container specifically for containers made during the `cephadm deploy` + # command. Adds some extra things such as extra container args and custom config files c = get_container(ctx, fsid, daemon_type, daemon_id, privileged, ptrace, container_args) if 'extra_container_args' in ctx and ctx.extra_container_args: c.container_args.extend(ctx.extra_container_args) + if 'config_json' in ctx and ctx.config_json: + conf_files = get_custom_config_files(ctx.config_json) + mandatory_keys = ['mount_path', 'content'] + for conf in conf_files['custom_config_files']: + if all(k in conf for k in mandatory_keys): + mount_path = conf['mount_path'] + file_path = os.path.join( + ctx.data_dir, + fsid, + 'custom_config_files', + f'{daemon_type}.{daemon_id}', + os.path.basename(mount_path) + ) + c.volume_mounts[file_path] = mount_path return c @@ -5803,8 +5853,8 @@ def command_deploy(ctx): uid, gid = extract_uid_gid(ctx) make_var_run(ctx, ctx.fsid, uid, gid) - c = get_container_with_extra_args(ctx, ctx.fsid, daemon_type, daemon_id, - ptrace=ctx.allow_ptrace) + c = get_deployment_container(ctx, ctx.fsid, daemon_type, daemon_id, + ptrace=ctx.allow_ptrace) deploy_daemon(ctx, ctx.fsid, daemon_type, daemon_id, c, uid, gid, config=config, keyring=keyring, osd_fsid=ctx.osd_fsid, @@ -5828,7 +5878,7 @@ def command_deploy(ctx): 'contain arg for {}'.format(daemon_type.capitalize(), ', '.join(required_args))) uid, gid = extract_uid_gid_monitoring(ctx, daemon_type) - c = get_container_with_extra_args(ctx, ctx.fsid, daemon_type, daemon_id) + c = get_deployment_container(ctx, ctx.fsid, daemon_type, daemon_id) deploy_daemon(ctx, ctx.fsid, daemon_type, daemon_id, c, uid, gid, reconfig=ctx.reconfig, ports=daemon_ports) @@ -5840,7 +5890,7 @@ def command_deploy(ctx): config, keyring = get_config_and_keyring(ctx) # TODO: extract ganesha uid/gid (997, 994) ? uid, gid = extract_uid_gid(ctx) - c = get_container_with_extra_args(ctx, ctx.fsid, daemon_type, daemon_id) + c = get_deployment_container(ctx, ctx.fsid, daemon_type, daemon_id) deploy_daemon(ctx, ctx.fsid, daemon_type, daemon_id, c, uid, gid, config=config, keyring=keyring, reconfig=ctx.reconfig, @@ -5849,7 +5899,7 @@ def command_deploy(ctx): elif daemon_type == CephIscsi.daemon_type: config, keyring = get_config_and_keyring(ctx) uid, gid = extract_uid_gid(ctx) - c = get_container_with_extra_args(ctx, ctx.fsid, daemon_type, daemon_id) + c = get_deployment_container(ctx, ctx.fsid, daemon_type, daemon_id) deploy_daemon(ctx, ctx.fsid, daemon_type, daemon_id, c, uid, gid, config=config, keyring=keyring, reconfig=ctx.reconfig, @@ -5863,7 +5913,7 @@ def command_deploy(ctx): elif daemon_type == HAproxy.daemon_type: haproxy = HAproxy.init(ctx, ctx.fsid, daemon_id) uid, gid = haproxy.extract_uid_gid_haproxy() - c = get_container_with_extra_args(ctx, ctx.fsid, daemon_type, daemon_id) + c = get_deployment_container(ctx, ctx.fsid, daemon_type, daemon_id) deploy_daemon(ctx, ctx.fsid, daemon_type, daemon_id, c, uid, gid, reconfig=ctx.reconfig, ports=daemon_ports) @@ -5871,7 +5921,7 @@ def command_deploy(ctx): elif daemon_type == Keepalived.daemon_type: keepalived = Keepalived.init(ctx, ctx.fsid, daemon_id) uid, gid = keepalived.extract_uid_gid_keepalived() - c = get_container_with_extra_args(ctx, ctx.fsid, daemon_type, daemon_id) + c = get_deployment_container(ctx, ctx.fsid, daemon_type, daemon_id) deploy_daemon(ctx, ctx.fsid, daemon_type, daemon_id, c, uid, gid, reconfig=ctx.reconfig, ports=daemon_ports) @@ -5880,9 +5930,9 @@ def command_deploy(ctx): cc = CustomContainer.init(ctx, ctx.fsid, daemon_id) if not ctx.reconfig and not redeploy: daemon_ports.extend(cc.ports) - c = get_container_with_extra_args(ctx, ctx.fsid, daemon_type, daemon_id, - privileged=cc.privileged, - ptrace=ctx.allow_ptrace) + c = get_deployment_container(ctx, ctx.fsid, daemon_type, daemon_id, + privileged=cc.privileged, + ptrace=ctx.allow_ptrace) deploy_daemon(ctx, ctx.fsid, daemon_type, daemon_id, c, uid=cc.uid, gid=cc.gid, config=None, keyring=None, reconfig=ctx.reconfig, @@ -5897,7 +5947,7 @@ def command_deploy(ctx): elif daemon_type == SNMPGateway.daemon_type: sc = SNMPGateway.init(ctx, ctx.fsid, daemon_id) - c = get_container_with_extra_args(ctx, ctx.fsid, daemon_type, daemon_id) + c = get_deployment_container(ctx, ctx.fsid, daemon_type, daemon_id) deploy_daemon(ctx, ctx.fsid, daemon_type, daemon_id, c, sc.uid, sc.gid, ports=daemon_ports) diff --git a/src/pybind/mgr/cephadm/serve.py b/src/pybind/mgr/cephadm/serve.py index a1ee9cfccbce1..2f26ca70900f7 100644 --- a/src/pybind/mgr/cephadm/serve.py +++ b/src/pybind/mgr/cephadm/serve.py @@ -1132,6 +1132,12 @@ class CephadmServe: except AttributeError: eca = None + if daemon_spec.service_name in self.mgr.spec_store: + configs = self.mgr.spec_store[daemon_spec.service_name].spec.custom_configs + if configs is not None: + daemon_spec.final_config.update( + {'custom_config_files': [c.to_json() for c in configs]}) + if self.mgr.cache.host_needs_registry_login(daemon_spec.host) and self.mgr.registry_url: await self._registry_login(daemon_spec.host, json.loads(str(self.mgr.get_store('registry_credentials')))) diff --git a/src/pybind/mgr/cephadm/services/cephadmservice.py b/src/pybind/mgr/cephadm/services/cephadmservice.py index 8abb0e63a2c10..8028b27c661ca 100644 --- a/src/pybind/mgr/cephadm/services/cephadmservice.py +++ b/src/pybind/mgr/cephadm/services/cephadmservice.py @@ -40,7 +40,8 @@ class CephadmDaemonDeploySpec: ports: Optional[List[int]] = None, rank: Optional[int] = None, rank_generation: Optional[int] = None, - extra_container_args: Optional[List[str]] = None): + extra_container_args: Optional[List[str]] = None, + ): """ A data struction to encapsulate `cephadm deploy ... """ @@ -178,10 +179,6 @@ class CephadmService(metaclass=ABCMeta): rank: Optional[int] = None, rank_generation: Optional[int] = None, ) -> CephadmDaemonDeploySpec: - try: - eca = spec.extra_container_args - except AttributeError: - eca = None return CephadmDaemonDeploySpec( host=host, daemon_id=daemon_id, @@ -192,7 +189,8 @@ class CephadmService(metaclass=ABCMeta): ip=ip, rank=rank, rank_generation=rank_generation, - extra_container_args=eca, + extra_container_args=spec.extra_container_args if hasattr( + spec, 'extra_container_args') else None, ) def prepare_create(self, daemon_spec: CephadmDaemonDeploySpec) -> CephadmDaemonDeploySpec: -- 2.39.5