From a22ee365985baaf0d16d3f0ce10c8d24621703bb Mon Sep 17 00:00:00 2001 From: Shweta Bhosale Date: Sat, 13 Dec 2025 12:54:25 +0530 Subject: [PATCH] mgr/cephadm: Cephadm to call invoker if ssh hardening is enabled Fixes: https://tracker.ceph.com/issues/74045 Signed-off-by: Shweta Bhosale --- src/pybind/mgr/cephadm/module.py | 13 ++++ src/pybind/mgr/cephadm/serve.py | 67 ++++++++++++------- src/pybind/mgr/cephadm/ssh.py | 64 +++++++++++++++++- .../cephadm/tests/test_remote_executables.py | 4 +- src/pybind/mgr/cephadm/tests/test_ssh.py | 11 ++- 5 files changed, 125 insertions(+), 34 deletions(-) diff --git a/src/pybind/mgr/cephadm/module.py b/src/pybind/mgr/cephadm/module.py index 457695084b99..a9df95ff056e 100644 --- a/src/pybind/mgr/cephadm/module.py +++ b/src/pybind/mgr/cephadm/module.py @@ -515,6 +515,14 @@ class CephadmOrchestrator(orchestrator.Orchestrator, MgrModule): default='169.254.1.1', desc="Default IP address for RedFish API (OOB management)." ), + Option( + 'ssh_hardening', + type='bool', + default=False, + desc='Enable SSH hardening by routing all command execution through invoker.py. ' + 'When enabled, cephadm and bash commands are validated and executed via ' + 'the secure invoker wrapper.' + ), ] for image in DefaultImages: MODULE_OPTIONS.append(Option(image.key, default=image.image_ref, desc=image.desc)) @@ -542,6 +550,9 @@ class CephadmOrchestrator(orchestrator.Orchestrator, MgrModule): else: self.paused = False + self.ssh_hardening = False + self.invoker_path = '/usr/libexec/cephadm_invoker.py' + # for mypy which does not run the code if TYPE_CHECKING: self.ssh_config_file = None # type: Optional[str] @@ -621,6 +632,8 @@ class CephadmOrchestrator(orchestrator.Orchestrator, MgrModule): self.oob_default_addr = '' self.ssh_keepalive_interval = 0 self.ssh_keepalive_count_max = 0 + self.ssh_hardening = False + self.invoker_path = '/usr/libexec/cephadm_invoker.py' self.certificate_duration_days = 0 self.certificate_renewal_threshold_days = 0 self.certificate_automated_rotation_enabled = False diff --git a/src/pybind/mgr/cephadm/serve.py b/src/pybind/mgr/cephadm/serve.py index 54985d2641f4..4e60a7028023 100644 --- a/src/pybind/mgr/cephadm/serve.py +++ b/src/pybind/mgr/cephadm/serve.py @@ -46,7 +46,6 @@ logger = logging.getLogger(__name__) REQUIRES_POST_ACTIONS = ['grafana', 'iscsi', 'prometheus', 'alertmanager', 'rgw', 'nvmeof', 'mgmt-gateway'] -WHICH = ssh.RemoteExecutable('which') CEPHADM_EXE = ssh.RemoteExecutable('/usr/bin/cephadm') @@ -1785,30 +1784,36 @@ class CephadmServe: if stdin and 'agent' not in str(entity): self.log.debug('stdin: %s' % stdin) - cmd = ssh.RemoteCommand(WHICH, ['python3']) - try: - # when connection was broken/closed, retrying resets the connection - python = await self.mgr.ssh._check_execute_command(host, cmd, addr=addr) - except ssh.HostConnectionError: - python = await self.mgr.ssh._check_execute_command(host, cmd, addr=addr) - - # N.B. because the python3 executable is based on the results of the - # which command we can not know it ahead of time and must be converted - # into a RemoteExecutable. - cmd = ssh.RemoteCommand( - ssh.RemoteExecutable(python), - [self.mgr.cephadm_binary_path] + final_args - ) + # If SSH hardening is enabled, call invoker directly without which python + if self.mgr.ssh_hardening and self.mgr.invoker_path: + # For invoker, pass all args as a single string + cmd = ssh.RemoteCommand( + ssh.Executables.INVOKER, + ['run', self.mgr.cephadm_binary_path, '--'] + final_args + ) + else: + # cephadm_binary_path must be converted into a RemoteExecutable. + cmd = ssh.RemoteCommand( + ssh.RemoteExecutable(self.mgr.cephadm_binary_path), + final_args + ) try: out, err, code = await self.mgr.ssh._execute_command( host, cmd, stdin=stdin, addr=addr) - if code == 2: - ls_cmd = ssh.RemoteCommand( - ssh.Executables.LS, - [self.mgr.cephadm_binary_path] - ) - out_ls, err_ls, code_ls = await self.mgr.ssh._execute_command(host, ls_cmd, addr=addr, + if code == 2 or code == 127: + # Use invoker to check file existence when SSH hardening is enabled + if self.mgr.ssh_hardening and self.mgr.invoker_path: + check_cmd = ssh.RemoteCommand( + ssh.Executables.INVOKER, + ['check_existence', self.mgr.cephadm_binary_path] + ) + else: + check_cmd = ssh.RemoteCommand( + ssh.Executables.LS, + [self.mgr.cephadm_binary_path] + ) + out_ls, err_ls, code_ls = await self.mgr.ssh._execute_command(host, check_cmd, addr=addr, log_command=log_output) if code_ls == 2: await self._deploy_cephadm_binary(host, addr) @@ -1828,7 +1833,14 @@ class CephadmServe: elif self.mgr.mode == 'cephadm-package': try: - cmd = ssh.RemoteCommand(CEPHADM_EXE, final_args) + # Wrap with invoker if SSH hardening is enabled + if self.mgr.ssh_hardening and self.mgr.invoker_path: + cmd = ssh.RemoteCommand( + ssh.Executables.INVOKER, + ['run', str(CEPHADM_EXE), '--'] + final_args + ) + else: + cmd = ssh.RemoteCommand(CEPHADM_EXE, final_args) out, err, code = await self.mgr.ssh._execute_command( host, cmd, stdin=stdin, addr=addr) except Exception as e: @@ -1900,8 +1912,15 @@ class CephadmServe: async def _deploy_cephadm_binary(self, host: str, addr: Optional[str] = None) -> None: # Use tee (from coreutils) to create a copy of cephadm on the target machine self.log.info(f"Deploying cephadm binary to {host}") - await self.mgr.ssh._write_remote_file(host, self.mgr.cephadm_binary_path, - self.mgr._cephadm, addr=addr, bypass_cephadm_exec=True) + if self.mgr.ssh_hardening and self.mgr.invoker_path: + # Use invoker for secure deployment when SSH hardening is enabled + await self.mgr.ssh._deploy_cephadm_binary_via_invoker( + host, self.mgr.cephadm_binary_path, self.mgr._cephadm, addr=addr) + else: + await self.mgr.ssh._write_remote_file(host, self.mgr.cephadm_binary_path, + self.mgr._cephadm, addr=addr, mode=0o744, + bypass_cephadm_exec=True) + async def run_cephadm_exec(self, host: str, cmd: List[str], diff --git a/src/pybind/mgr/cephadm/ssh.py b/src/pybind/mgr/cephadm/ssh.py index ab70b6d902af..53c0b2209df7 100644 --- a/src/pybind/mgr/cephadm/ssh.py +++ b/src/pybind/mgr/cephadm/ssh.py @@ -111,6 +111,7 @@ class Executables(RemoteExecutable, enum.Enum): SYSCTL = RemoteExecutable('sysctl') TOUCH = RemoteExecutable('touch') TRUE = RemoteExecutable('true') + INVOKER = RemoteExecutable('/usr/libexec/cephadm_invoker.py') def __str__(self) -> str: return self.value @@ -263,6 +264,27 @@ class SSHManager: with self.mgr.async_timeout_handler(host, f'ssh {host} (addr {addr})'): return self.mgr.wait_async(self._remote_connection(host, addr)) + def _enforce_ssh_hardening(self, + host: str, + cmd_components: RemoteCommand) -> None: + """ + Enforce that commands are wrapped with invoker when SSH hardening + is enabled. + """ + if not self.mgr.ssh_hardening: + return + + is_wrapped = ( + isinstance(cmd_components.exe, RemoteExecutable) + and str(cmd_components.exe) == str(Executables.INVOKER) + ) + if not is_wrapped: + msg = (f'SSH hardening is enabled but command is not ' + f'wrapped with invoker for host {host}. ' + f'Command: {cmd_components}') + logger.error(msg) + raise OrchestratorError(msg) + async def _execute_command(self, host: str, cmd_components: RemoteCommand, @@ -279,6 +301,9 @@ class SSHManager: if is_host_being_added: logger.debug(f'Host {host} is being added, using root user without sudo') + # Enforce invoker usage if SSH hardening is enabled + self._enforce_ssh_hardening(host, cmd_components) + rcmd = RemoteSudoCommand.wrap(cmd_components, use_sudo=use_sudo) try: address = addr or self.mgr.inventory.get_addr(host) @@ -518,13 +543,14 @@ class SSHManager: conn = await self._remote_connection(host, addr) async with conn.start_sftp_client() as sftp: await sftp.put(f.name, tmp_path) - if uid is not None and gid is not None and mode is not None: + if uid is not None and gid is not None: # shlex quote takes str or byte object, not int chown = RemoteCommand( Executables.CHOWN, ['-R', str(uid) + ':' + str(gid), tmp_path] ) await execute_method(host, chown, addr=addr) + if mode is not None: chmod = RemoteCommand(Executables.CHMOD, [oct(mode)[2:], tmp_path]) await execute_method(host, chmod, addr=addr) mv = RemoteCommand(Executables.MV, ['-Z', tmp_path, path]) @@ -534,6 +560,42 @@ class SSHManager: logger.exception(msg) raise OrchestratorError(msg) + async def _deploy_cephadm_binary_via_invoker( + self, + host: str, + cephadm_path: str, + cephadm_content: bytes, + addr: Optional[str] = None + ) -> None: + """ + Deploy cephadm binary using the invoker for secure operations. + This creates a temp file locally, copies it to remote host, then + calls the invoker to perform all deployment operations. + """ + with NamedTemporaryFile(prefix='cephadm-deploy-', delete=False) as local_tmp: + local_tmp.write(cephadm_content) + local_tmp.flush() + local_tmp_path = local_tmp.name + + try: + remote_tmp_path = f'/tmp/cephadm-{self.mgr._cluster_fsid}.new' + + conn = await self._remote_connection(host, addr) + async with conn.start_sftp_client() as sftp: + await sftp.put(local_tmp_path, remote_tmp_path) + + invoker_cmd = RemoteCommand( + Executables.INVOKER, + ['deploy_cephadm_binary', remote_tmp_path, cephadm_path] + ) + await self._execute_command(host, invoker_cmd, addr=addr) + + finally: + try: + os.unlink(local_tmp_path) + except OSError: + pass + def write_remote_file(self, host: str, path: str, diff --git a/src/pybind/mgr/cephadm/tests/test_remote_executables.py b/src/pybind/mgr/cephadm/tests/test_remote_executables.py index 26b02e9de96b..47fa6ee8463f 100644 --- a/src/pybind/mgr/cephadm/tests/test_remote_executables.py +++ b/src/pybind/mgr/cephadm/tests/test_remote_executables.py @@ -54,9 +54,9 @@ EXPECTED = [ ('sysctl', True, ssh_py), ('touch', True, ssh_py), ('true', True, ssh_py), - ('which', True, serve_py), # variable executables - ('python', False, serve_py), + ('self.mgr.cephadm_binary_path', False, serve_py), + ('/usr/libexec/cephadm_invoker.py', True, ssh_py), ] diff --git a/src/pybind/mgr/cephadm/tests/test_ssh.py b/src/pybind/mgr/cephadm/tests/test_ssh.py index 95cd87716f23..937e31567602 100644 --- a/src/pybind/mgr/cephadm/tests/test_ssh.py +++ b/src/pybind/mgr/cephadm/tests/test_ssh.py @@ -79,15 +79,12 @@ class TestWithSSH: with with_host(cephadm_module, host): CephadmServe(cephadm_module)._check_host(host) - # Test case 1: command failure - run_test('test1', FakeConn(returncode=1), "Command .+ failed") + # Test case 1: connection error + run_test('test1', FakeConn(exception=asyncssh.ChannelOpenError(1, "", "")), "Unable to reach remote host test1") - # Test case 2: connection error - run_test('test2', FakeConn(exception=asyncssh.ChannelOpenError(1, "", "")), "Unable to reach remote host test2.") - - # Test case 3: asyncssh ProcessError + # Test case 2: asyncssh ProcessError stderr = "my-process-stderr" - run_test('test3', FakeConn(exception=asyncssh.ProcessError(returncode=3, + run_test('test2', FakeConn(exception=asyncssh.ProcessError(returncode=3, env="", command="", subsystem="", -- 2.47.3