From 794652e268b8301ff97b895680a9e71070d7477b Mon Sep 17 00:00:00 2001 From: John Mulligan Date: Fri, 17 Nov 2023 15:41:17 -0500 Subject: [PATCH] cephadm: convert iscsi daemon form to use sidecars Convert the iscsi container daemon form class to produce a sidecar container for the tcmu runner and reduce the amount of special casing for iscsi. Signed-off-by: John Mulligan --- src/cephadm/cephadm.py | 33 ++++-------------- src/cephadm/cephadmlib/daemons/iscsi.py | 45 +++++++++++-------------- src/cephadm/tests/test_cephadm.py | 40 ++++++++++++++-------- 3 files changed, 51 insertions(+), 67 deletions(-) diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py index b50a03854f8ff..800484b4195fc 100755 --- a/src/cephadm/cephadm.py +++ b/src/cephadm/cephadm.py @@ -1112,8 +1112,12 @@ def deploy_daemon_units( _osd_unit_poststop_commands(ctx, ident, osd_fsid) ) if ident.daemon_type == CephIscsi.daemon_type: - pre_start_commands.extend(_iscsi_unit_run_commands(ctx, ident, data_dir)) - post_stop_commands.extend(_iscsi_unit_poststop_commands(ctx, ident, data_dir)) + pre_start_commands.append( + CephIscsi.configfs_mount_umount(data_dir, mount=True) + ) + post_stop_commands.append( + CephIscsi.configfs_mount_umount(data_dir, mount=False) + ) runscripts.write_service_scripts( ctx, @@ -1214,17 +1218,6 @@ def _osd_unit_run_commands( return cmds -def _iscsi_unit_run_commands( - ctx: CephadmContext, ident: 'DaemonIdentity', data_dir: str -) -> List[runscripts.Command]: - cmds: List[runscripts.Command] = [] - cmds.append(' '.join(CephIscsi.configfs_mount_umount(data_dir, mount=True)) + '\n') - ceph_iscsi = CephIscsi.init(ctx, ident.fsid, ident.daemon_id) - tcmu_container = ceph_iscsi.get_tcmu_runner_container() - cmds.append(runscripts.ContainerCommand(tcmu_container, comment='iscsi tcmu-runner container', background=True)) - return cmds - - def _osd_unit_poststop_commands( ctx: CephadmContext, ident: 'DaemonIdentity', osd_fsid: str ) -> List[runscripts.Command]: @@ -1241,20 +1234,6 @@ def _osd_unit_poststop_commands( return [runscripts.ContainerCommand(poststop, comment='deactivate osd')] -def _iscsi_unit_poststop_commands( - ctx: CephadmContext, ident: 'DaemonIdentity', data_dir: str -) -> List[runscripts.Command]: - # make sure we also stop the tcmu container - cmds: List[runscripts.Command] = [] - runtime_dir = '/run' - ceph_iscsi = CephIscsi.init(ctx, ident.fsid, ident.daemon_id) - tcmu_container = ceph_iscsi.get_tcmu_runner_container() - cmds.append('! ' + ' '.join(tcmu_container.stop_cmd()) + '\n') - cmds.append('! ' + 'rm ' + runtime_dir + '/ceph-%s@%s.%s.service-pid' % (ident.fsid, ident.daemon_type, ident.daemon_id + '.tcmu') + '\n') - cmds.append('! ' + 'rm ' + runtime_dir + '/ceph-%s@%s.%s.service-cid' % (ident.fsid, ident.daemon_type, ident.daemon_id + '.tcmu') + '\n') - cmds.append(' '.join(CephIscsi.configfs_mount_umount(data_dir, mount=False)) + '\n') - return cmds - ################################## diff --git a/src/cephadm/cephadmlib/daemons/iscsi.py b/src/cephadm/cephadmlib/daemons/iscsi.py index 1845a37bf4efb..ade88a90af0ff 100644 --- a/src/cephadm/cephadmlib/daemons/iscsi.py +++ b/src/cephadm/cephadmlib/daemons/iscsi.py @@ -5,10 +5,10 @@ import re from typing import Dict, List, Optional, Tuple from ..container_daemon_form import ContainerDaemonForm, daemon_to_container -from ..container_types import CephContainer, extract_uid_gid +from ..container_types import CephContainer, SidecarContainer, extract_uid_gid from ..context_getters import fetch_configs, get_config_and_keyring from ..daemon_form import register as register_daemon_form -from ..daemon_identity import DaemonIdentity, DaemonSubIdentity +from ..daemon_identity import DaemonIdentity from ..constants import DEFAULT_IMAGE from ..context import CephadmContext from ..data_utils import dict_get, is_fsid @@ -190,8 +190,7 @@ class CephIscsi(ContainerDaemonForm): os.chmod(os.path.join(data_dir, 'tcmu-runner-entrypoint.sh'), 0o700) @staticmethod - def configfs_mount_umount(data_dir, mount=True): - # type: (str, bool) -> List[str] + def configfs_mount_umount(data_dir: str, mount: bool = True) -> str: mount_path = os.path.join(data_dir, 'configfs') if mount: cmd = ( @@ -203,7 +202,7 @@ class CephIscsi(ContainerDaemonForm): 'if grep -qs {0} /proc/mounts; then ' 'umount {0}; fi'.format(mount_path) ) - return cmd.split() + return cmd @staticmethod def tcmu_runner_entrypoint_script() -> str: @@ -242,27 +241,6 @@ do done """ - def get_tcmu_runner_container(self): - # type: () -> CephContainer - # daemon_id, is used to generated the cid and pid files used by podman but as both tcmu-runner - # and rbd-target-api have the same daemon_id, it conflits and prevent the second container from - # starting. .tcmu runner is appended to the daemon_id to fix that. - subident = DaemonSubIdentity( - self.fsid, self.daemon_type, self.daemon_id, 'tcmu' - ) - tcmu_dmn = self.create(self.ctx, subident) - tcmu_container = to_deployment_container( - self.ctx, daemon_to_container(self.ctx, tcmu_dmn, privileged=True) - ) - # TODO: Eventually we don't want to run tcmu-runner through this script. - # This is intended to be a workaround backported to older releases - # and should eventually be removed in at least squid onward - tcmu_container.entrypoint = ( - '/usr/local/scripts/tcmu-runner-entrypoint.sh' - ) - tcmu_container.cname = self.get_container_name(desc='tcmu') - return tcmu_container - def container(self, ctx: CephadmContext) -> CephContainer: # So the container can modprobe iscsi_target_mod and have write perms # to configfs we need to make this a privileged container. @@ -284,3 +262,18 @@ done self, ctx: CephadmContext, args: List[str] ) -> None: args.append(ctx.container_engine.unlimited_pids_option) + + def sidecar_containers( + self, ctx: CephadmContext + ) -> List[SidecarContainer]: + tcmu_sidecar = SidecarContainer.from_primary_and_values( + ctx, + self.container(ctx), + 'tcmu', + # TODO: Eventually we don't want to run tcmu-runner through this + # script. This is intended to be a workaround backported to older + # releases and should eventually be removed in at least squid + # onward + entrypoint='/usr/local/scripts/tcmu-runner-entrypoint.sh', + ) + return [tcmu_sidecar] diff --git a/src/cephadm/tests/test_cephadm.py b/src/cephadm/tests/test_cephadm.py index 82850ab597d6b..7bca8b9cbf3ca 100644 --- a/src/cephadm/tests/test_cephadm.py +++ b/src/cephadm/tests/test_cephadm.py @@ -1760,7 +1760,11 @@ class TestCephVolume(object): class TestIscsi: - def test_unit_run(self, cephadm_fs): + def test_unit_run(self, cephadm_fs, funkypatch): + funkypatch.patch( + 'cephadmlib.daemons.iscsi.extract_uid_gid' + ).return_value = (123, 123) + fsid = '9b9d7609-f4d5-4aba-94c8-effa764d96c9' config_json = { 'files': {'iscsi-gateway.cfg': ''} @@ -1773,27 +1777,35 @@ class TestIscsi: _cephadm.get_parm.return_value = config_json ident = _cephadm.DaemonIdentity(fsid, 'iscsi', 'daemon_id') - c = _cephadm.get_container(ctx, ident) - _cephadm.make_data_dir(ctx, ident) - _cephadm.deploy_daemon_units( - ctx, - ident, - 0, 0, - c, - True, True + + _cephadm._deploy_daemon_container( + ctx, ident, [], _cephadm.DeploymentType.DEFAULT ) with open('/var/lib/ceph/9b9d7609-f4d5-4aba-94c8-effa764d96c9/iscsi.daemon_id/unit.run') as f: - assert f.read() == """set -e + contents = f.read() + assert contents == """set -e if ! grep -qs /var/lib/ceph/9b9d7609-f4d5-4aba-94c8-effa764d96c9/iscsi.daemon_id/configfs /proc/mounts; then mount -t configfs none /var/lib/ceph/9b9d7609-f4d5-4aba-94c8-effa764d96c9/iscsi.daemon_id/configfs; fi -# iscsi tcmu-runner container -! /usr/bin/docker rm -f ceph-9b9d7609-f4d5-4aba-94c8-effa764d96c9-iscsi.daemon_id-tcmu 2> /dev/null -! /usr/bin/docker rm -f ceph-9b9d7609-f4d5-4aba-94c8-effa764d96c9-iscsi-daemon_id-tcmu 2> /dev/null -/usr/bin/docker run --rm --ipc=host --stop-signal=SIGTERM --ulimit nofile=1048576 --net=host --entrypoint /usr/local/scripts/tcmu-runner-entrypoint.sh --privileged --group-add=disk --init --name ceph-9b9d7609-f4d5-4aba-94c8-effa764d96c9-iscsi-daemon_id-tcmu --pids-limit=0 -e CONTAINER_IMAGE=ceph/ceph -e NODE_NAME=host1 -v /var/lib/ceph/9b9d7609-f4d5-4aba-94c8-effa764d96c9/iscsi.daemon_id/config:/etc/ceph/ceph.conf:z -v /var/lib/ceph/9b9d7609-f4d5-4aba-94c8-effa764d96c9/iscsi.daemon_id/keyring:/etc/ceph/keyring:z -v /var/lib/ceph/9b9d7609-f4d5-4aba-94c8-effa764d96c9/iscsi.daemon_id/iscsi-gateway.cfg:/etc/ceph/iscsi-gateway.cfg:z -v /var/lib/ceph/9b9d7609-f4d5-4aba-94c8-effa764d96c9/iscsi.daemon_id/configfs:/sys/kernel/config -v /var/lib/ceph/9b9d7609-f4d5-4aba-94c8-effa764d96c9/iscsi.daemon_id/tcmu-runner-entrypoint.sh:/usr/local/scripts/tcmu-runner-entrypoint.sh -v /var/log/ceph/9b9d7609-f4d5-4aba-94c8-effa764d96c9:/var/log:z -v /dev:/dev --mount type=bind,source=/lib/modules,destination=/lib/modules,ro=true ceph/ceph & # iscsi.daemon_id ! /usr/bin/docker rm -f ceph-9b9d7609-f4d5-4aba-94c8-effa764d96c9-iscsi.daemon_id 2> /dev/null ! /usr/bin/docker rm -f ceph-9b9d7609-f4d5-4aba-94c8-effa764d96c9-iscsi-daemon_id 2> /dev/null /usr/bin/docker run --rm --ipc=host --stop-signal=SIGTERM --ulimit nofile=1048576 --net=host --entrypoint /usr/bin/rbd-target-api --privileged --group-add=disk --init --name ceph-9b9d7609-f4d5-4aba-94c8-effa764d96c9-iscsi-daemon_id --pids-limit=0 -e CONTAINER_IMAGE=ceph/ceph -e NODE_NAME=host1 -v /var/lib/ceph/9b9d7609-f4d5-4aba-94c8-effa764d96c9/iscsi.daemon_id/config:/etc/ceph/ceph.conf:z -v /var/lib/ceph/9b9d7609-f4d5-4aba-94c8-effa764d96c9/iscsi.daemon_id/keyring:/etc/ceph/keyring:z -v /var/lib/ceph/9b9d7609-f4d5-4aba-94c8-effa764d96c9/iscsi.daemon_id/iscsi-gateway.cfg:/etc/ceph/iscsi-gateway.cfg:z -v /var/lib/ceph/9b9d7609-f4d5-4aba-94c8-effa764d96c9/iscsi.daemon_id/configfs:/sys/kernel/config -v /var/lib/ceph/9b9d7609-f4d5-4aba-94c8-effa764d96c9/iscsi.daemon_id/tcmu-runner-entrypoint.sh:/usr/local/scripts/tcmu-runner-entrypoint.sh -v /var/log/ceph/9b9d7609-f4d5-4aba-94c8-effa764d96c9:/var/log:z -v /dev:/dev --mount type=bind,source=/lib/modules,destination=/lib/modules,ro=true ceph/ceph +""" + with open('/var/lib/ceph/9b9d7609-f4d5-4aba-94c8-effa764d96c9/iscsi.daemon_id/sidecar-tcmu.run') as f: + contents = f.read() + assert contents == """#!/bin/sh +# sidecar: tcmu + +set -e +if [ "$1" = stop ] || [ "$1" = poststop ]; then + ! /usr/bin/docker stop ceph-9b9d7609-f4d5-4aba-94c8-effa764d96c9-iscsi-daemon_id-tcmu + ! /usr/bin/docker inspect ceph-9b9d7609-f4d5-4aba-94c8-effa764d96c9-iscsi-daemon_id-tcmu &>/dev/null + exit $? +fi + +! /usr/bin/docker rm -f ceph-9b9d7609-f4d5-4aba-94c8-effa764d96c9-iscsi-daemon_id-tcmu 2> /dev/null + +exec /usr/bin/docker run --rm --ipc=host --stop-signal=SIGTERM --ulimit nofile=1048576 --net=host --entrypoint /usr/local/scripts/tcmu-runner-entrypoint.sh --privileged --group-add=disk --init --name ceph-9b9d7609-f4d5-4aba-94c8-effa764d96c9-iscsi-daemon_id-tcmu --pids-limit=0 -e CONTAINER_IMAGE=ceph/ceph -e NODE_NAME=host1 -v /var/lib/ceph/9b9d7609-f4d5-4aba-94c8-effa764d96c9/iscsi.daemon_id/config:/etc/ceph/ceph.conf:z -v /var/lib/ceph/9b9d7609-f4d5-4aba-94c8-effa764d96c9/iscsi.daemon_id/keyring:/etc/ceph/keyring:z -v /var/lib/ceph/9b9d7609-f4d5-4aba-94c8-effa764d96c9/iscsi.daemon_id/iscsi-gateway.cfg:/etc/ceph/iscsi-gateway.cfg:z -v /var/lib/ceph/9b9d7609-f4d5-4aba-94c8-effa764d96c9/iscsi.daemon_id/configfs:/sys/kernel/config -v /var/lib/ceph/9b9d7609-f4d5-4aba-94c8-effa764d96c9/iscsi.daemon_id/tcmu-runner-entrypoint.sh:/usr/local/scripts/tcmu-runner-entrypoint.sh -v /var/log/ceph/9b9d7609-f4d5-4aba-94c8-effa764d96c9:/var/log:z -v /dev:/dev --mount type=bind,source=/lib/modules,destination=/lib/modules,ro=true ceph/ceph """ def test_get_container(self): -- 2.39.5