From: Shweta Bhosale Date: Sat, 13 Dec 2025 07:24:25 +0000 (+0530) Subject: mgr/cephadm: Cephadm to call invoker if ssh hardening is enabled X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=067bfe24cc0d214fb7cdc67d46050f8b557f25fa;p=ceph-ci.git mgr/cephadm: Cephadm to call invoker if ssh hardening is enabled Fixes: https://tracker.ceph.com/issues/74045 Signed-off-by: Shweta Bhosale --- diff --git a/src/pybind/mgr/cephadm/module.py b/src/pybind/mgr/cephadm/module.py index c114e00ed0e..777f7b4368b 100644 --- a/src/pybind/mgr/cephadm/module.py +++ b/src/pybind/mgr/cephadm/module.py @@ -505,6 +505,14 @@ class CephadmOrchestrator(orchestrator.Orchestrator, MgrModule): default='169.254.1.1', desc="Default IP address for RedFish API (OOB management)." ), + Option( + 'ssh_hardening', + type='bool', + default=False, + desc='Enable SSH hardening by routing all command execution through invoker.py. ' + 'When enabled, cephadm and bash commands are validated and executed via ' + 'the secure invoker wrapper.' + ), ] for image in DefaultImages: MODULE_OPTIONS.append(Option(image.key, default=image.image_ref, desc=image.desc)) @@ -532,6 +540,9 @@ class CephadmOrchestrator(orchestrator.Orchestrator, MgrModule): else: self.paused = False + self.ssh_hardening = False + self.invoker_path = '/usr/libexec/cephadm_invoker.py' + # for mypy which does not run the code if TYPE_CHECKING: self.ssh_config_file = None # type: Optional[str] @@ -610,6 +621,8 @@ class CephadmOrchestrator(orchestrator.Orchestrator, MgrModule): self.oob_default_addr = '' self.ssh_keepalive_interval = 0 self.ssh_keepalive_count_max = 0 + self.ssh_hardening = False + self.invoker_path = '/usr/libexec/cephadm_invoker.py' self.certificate_duration_days = 0 self.certificate_renewal_threshold_days = 0 self.certificate_automated_rotation_enabled = False diff --git a/src/pybind/mgr/cephadm/serve.py b/src/pybind/mgr/cephadm/serve.py index c2de2c6a755..f01f9c3d3ff 100644 --- a/src/pybind/mgr/cephadm/serve.py +++ b/src/pybind/mgr/cephadm/serve.py @@ -45,7 +45,6 @@ logger = logging.getLogger(__name__) REQUIRES_POST_ACTIONS = ['grafana', 'iscsi', 'prometheus', 'alertmanager', 'rgw', 'nvmeof', 'mgmt-gateway'] -WHICH = ssh.RemoteExecutable('which') CEPHADM_EXE = ssh.RemoteExecutable('/usr/bin/cephadm') @@ -1766,30 +1765,36 @@ class CephadmServe: if stdin and 'agent' not in str(entity): self.log.debug('stdin: %s' % stdin) - cmd = ssh.RemoteCommand(WHICH, ['python3']) - try: - # when connection was broken/closed, retrying resets the connection - python = await self.mgr.ssh._check_execute_command(host, cmd, addr=addr) - except ssh.HostConnectionError: - python = await self.mgr.ssh._check_execute_command(host, cmd, addr=addr) - - # N.B. because the python3 executable is based on the results of the - # which command we can not know it ahead of time and must be converted - # into a RemoteExecutable. - cmd = ssh.RemoteCommand( - ssh.RemoteExecutable(python), - [self.mgr.cephadm_binary_path] + final_args - ) + # If SSH hardening is enabled, call invoker directly without which python + if self.mgr.ssh_hardening and self.mgr.invoker_path: + # For invoker, pass all args as a single string + cmd = ssh.RemoteCommand( + ssh.Executables.INVOKER, + ['run', self.mgr.cephadm_binary_path, '--'] + final_args + ) + else: + # cephadm_binary_path must be converted into a RemoteExecutable. + cmd = ssh.RemoteCommand( + ssh.RemoteExecutable(self.mgr.cephadm_binary_path), + final_args + ) try: out, err, code = await self.mgr.ssh._execute_command( host, cmd, stdin=stdin, addr=addr) - if code == 2: - ls_cmd = ssh.RemoteCommand( - ssh.Executables.LS, - [self.mgr.cephadm_binary_path] - ) - out_ls, err_ls, code_ls = await self.mgr.ssh._execute_command(host, ls_cmd, addr=addr, + if code == 2 or code == 127: + # Use invoker to check file existence when SSH hardening is enabled + if self.mgr.ssh_hardening and self.mgr.invoker_path: + check_cmd = ssh.RemoteCommand( + ssh.Executables.INVOKER, + ['check_existence', self.mgr.cephadm_binary_path] + ) + else: + check_cmd = ssh.RemoteCommand( + ssh.Executables.LS, + [self.mgr.cephadm_binary_path] + ) + out_ls, err_ls, code_ls = await self.mgr.ssh._execute_command(host, check_cmd, addr=addr, log_command=log_output) if code_ls == 2: await self._deploy_cephadm_binary(host, addr) @@ -1809,7 +1814,14 @@ class CephadmServe: elif self.mgr.mode == 'cephadm-package': try: - cmd = ssh.RemoteCommand(CEPHADM_EXE, final_args) + # Wrap with invoker if SSH hardening is enabled + if self.mgr.ssh_hardening and self.mgr.invoker_path: + cmd = ssh.RemoteCommand( + ssh.Executables.INVOKER, + ['run', str(CEPHADM_EXE), '--'] + final_args + ) + else: + cmd = ssh.RemoteCommand(CEPHADM_EXE, final_args) out, err, code = await self.mgr.ssh._execute_command( host, cmd, stdin=stdin, addr=addr) except Exception as e: @@ -1881,8 +1893,15 @@ class CephadmServe: async def _deploy_cephadm_binary(self, host: str, addr: Optional[str] = None) -> None: # Use tee (from coreutils) to create a copy of cephadm on the target machine self.log.info(f"Deploying cephadm binary to {host}") - await self.mgr.ssh._write_remote_file(host, self.mgr.cephadm_binary_path, - self.mgr._cephadm, addr=addr, bypass_cephadm_exec=True) + if self.mgr.ssh_hardening and self.mgr.invoker_path: + # Use invoker for secure deployment when SSH hardening is enabled + await self.mgr.ssh._deploy_cephadm_binary_via_invoker( + host, self.mgr.cephadm_binary_path, self.mgr._cephadm, addr=addr) + else: + await self.mgr.ssh._write_remote_file(host, self.mgr.cephadm_binary_path, + self.mgr._cephadm, addr=addr, mode=0o744, + bypass_cephadm_exec=True) + async def run_cephadm_exec(self, host: str, cmd: List[str], diff --git a/src/pybind/mgr/cephadm/ssh.py b/src/pybind/mgr/cephadm/ssh.py index 63f26b511b8..8a90b86c922 100644 --- a/src/pybind/mgr/cephadm/ssh.py +++ b/src/pybind/mgr/cephadm/ssh.py @@ -111,6 +111,7 @@ class Executables(RemoteExecutable, enum.Enum): SYSCTL = RemoteExecutable('sysctl') TOUCH = RemoteExecutable('touch') TRUE = RemoteExecutable('true') + INVOKER = RemoteExecutable('/usr/libexec/cephadm_invoker.py') def __str__(self) -> str: return self.value @@ -229,6 +230,27 @@ class SSHManager: with self.mgr.async_timeout_handler(host, f'ssh {host} (addr {addr})'): return self.mgr.wait_async(self._remote_connection(host, addr)) + def _enforce_ssh_hardening(self, + host: str, + cmd_components: RemoteCommand) -> None: + """ + Enforce that commands are wrapped with invoker when SSH hardening + is enabled. + """ + if not self.mgr.ssh_hardening: + return + + is_wrapped = ( + isinstance(cmd_components.exe, RemoteExecutable) + and str(cmd_components.exe) == str(Executables.INVOKER) + ) + if not is_wrapped: + msg = (f'SSH hardening is enabled but command is not ' + f'wrapped with invoker for host {host}. ' + f'Command: {cmd_components}') + logger.error(msg) + raise OrchestratorError(msg) + async def _execute_command(self, host: str, cmd_components: RemoteCommand, @@ -245,6 +267,9 @@ class SSHManager: if is_host_being_added: logger.debug(f'Host {host} is being added, using root user without sudo') + # Enforce invoker usage if SSH hardening is enabled + self._enforce_ssh_hardening(host, cmd_components) + rcmd = RemoteSudoCommand.wrap(cmd_components, use_sudo=use_sudo) try: address = addr or self.mgr.inventory.get_addr(host) @@ -429,13 +454,14 @@ class SSHManager: conn = await self._remote_connection(host, addr) async with conn.start_sftp_client() as sftp: await sftp.put(f.name, tmp_path) - if uid is not None and gid is not None and mode is not None: + if uid is not None and gid is not None: # shlex quote takes str or byte object, not int chown = RemoteCommand( Executables.CHOWN, ['-R', str(uid) + ':' + str(gid), tmp_path] ) await execute_method(host, chown, addr=addr) + if mode is not None: chmod = RemoteCommand(Executables.CHMOD, [oct(mode)[2:], tmp_path]) await execute_method(host, chmod, addr=addr) mv = RemoteCommand(Executables.MV, ['-Z', tmp_path, path]) @@ -445,6 +471,42 @@ class SSHManager: logger.exception(msg) raise OrchestratorError(msg) + async def _deploy_cephadm_binary_via_invoker( + self, + host: str, + cephadm_path: str, + cephadm_content: bytes, + addr: Optional[str] = None + ) -> None: + """ + Deploy cephadm binary using the invoker for secure operations. + This creates a temp file locally, copies it to remote host, then + calls the invoker to perform all deployment operations. + """ + with NamedTemporaryFile(prefix='cephadm-deploy-', delete=False) as local_tmp: + local_tmp.write(cephadm_content) + local_tmp.flush() + local_tmp_path = local_tmp.name + + try: + remote_tmp_path = f'/tmp/cephadm-{self.mgr._cluster_fsid}.new' + + conn = await self._remote_connection(host, addr) + async with conn.start_sftp_client() as sftp: + await sftp.put(local_tmp_path, remote_tmp_path) + + invoker_cmd = RemoteCommand( + Executables.INVOKER, + ['deploy_cephadm_binary', remote_tmp_path, cephadm_path] + ) + await self._execute_command(host, invoker_cmd, addr=addr) + + finally: + try: + os.unlink(local_tmp_path) + except OSError: + pass + def write_remote_file(self, host: str, path: str, diff --git a/src/pybind/mgr/cephadm/tests/test_remote_executables.py b/src/pybind/mgr/cephadm/tests/test_remote_executables.py index 9d5bd458254..64335600e27 100644 --- a/src/pybind/mgr/cephadm/tests/test_remote_executables.py +++ b/src/pybind/mgr/cephadm/tests/test_remote_executables.py @@ -54,9 +54,9 @@ EXPECTED = [ ('sysctl', True, ssh_py), ('touch', True, ssh_py), ('true', True, ssh_py), - ('which', True, serve_py), # variable executables - ('python', False, serve_py), + ('self.mgr.cephadm_binary_path', False, serve_py), + ('/usr/libexec/cephadm_invoker.py', True, ssh_py), ] diff --git a/src/pybind/mgr/cephadm/tests/test_ssh.py b/src/pybind/mgr/cephadm/tests/test_ssh.py index 44ef3d429b7..d32f6432311 100644 --- a/src/pybind/mgr/cephadm/tests/test_ssh.py +++ b/src/pybind/mgr/cephadm/tests/test_ssh.py @@ -79,15 +79,12 @@ class TestWithSSH: with with_host(cephadm_module, host): CephadmServe(cephadm_module)._check_host(host) - # Test case 1: command failure - run_test('test1', FakeConn(returncode=1), "Command .+ failed") + # Test case 1: connection error + run_test('test1', FakeConn(exception=asyncssh.ChannelOpenError(1, "", "")), "Unable to reach remote host test1") - # Test case 2: connection error - run_test('test2', FakeConn(exception=asyncssh.ChannelOpenError(1, "", "")), "Unable to reach remote host test2.") - - # Test case 3: asyncssh ProcessError + # Test case 2: asyncssh ProcessError stderr = "my-process-stderr" - run_test('test3', FakeConn(exception=asyncssh.ProcessError(returncode=3, + run_test('test2', FakeConn(exception=asyncssh.ProcessError(returncode=3, env="", command="", subsystem="", @@ -95,8 +92,8 @@ class TestWithSSH: exit_signal="", stderr=stderr, stdout="")), f"Cannot execute the command.+{stderr}") - # Test case 4: generic error - run_test('test4', FakeConn(exception=Exception), "Generic error while executing command.+") + # Test case 3: generic error + run_test('test3', FakeConn(exception=Exception), "Generic error while executing command.+") @pytest.mark.skipif(ConnectionLost is not None, reason='asyncssh')