cephadm: convert iscsi daemon form to use sidecars

Convert the iscsi container daemon form class to produce a sidecar
container for the tcmu runner and reduce the amount of special casing
for iscsi.

Signed-off-by: John Mulligan <jmulligan@redhat.com>

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index b50a03854f8ff094c775adbe90d1dc10046e9c4c..800484b4195fcdfe5702a6cb4d08ed55e56a5349 100755 (executable)
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -1112,8 +1112,12 @@ def deploy_daemon_units(
             _osd_unit_poststop_commands(ctx, ident, osd_fsid)
         )
     if ident.daemon_type == CephIscsi.daemon_type:
-        pre_start_commands.extend(_iscsi_unit_run_commands(ctx, ident, data_dir))
-        post_stop_commands.extend(_iscsi_unit_poststop_commands(ctx, ident, data_dir))
+        pre_start_commands.append(
+            CephIscsi.configfs_mount_umount(data_dir, mount=True)
+        )
+        post_stop_commands.append(
+            CephIscsi.configfs_mount_umount(data_dir, mount=False)
+        )
 
     runscripts.write_service_scripts(
         ctx,
@@ -1214,17 +1218,6 @@ def _osd_unit_run_commands(
     return cmds
 
 
-def _iscsi_unit_run_commands(
-    ctx: CephadmContext, ident: 'DaemonIdentity', data_dir: str
-) -> List[runscripts.Command]:
-    cmds: List[runscripts.Command] = []
-    cmds.append(' '.join(CephIscsi.configfs_mount_umount(data_dir, mount=True)) + '\n')
-    ceph_iscsi = CephIscsi.init(ctx, ident.fsid, ident.daemon_id)
-    tcmu_container = ceph_iscsi.get_tcmu_runner_container()
-    cmds.append(runscripts.ContainerCommand(tcmu_container, comment='iscsi tcmu-runner container', background=True))
-    return cmds
-
-
 def _osd_unit_poststop_commands(
     ctx: CephadmContext, ident: 'DaemonIdentity', osd_fsid: str
 ) -> List[runscripts.Command]:
@@ -1241,20 +1234,6 @@ def _osd_unit_poststop_commands(
     return [runscripts.ContainerCommand(poststop, comment='deactivate osd')]
 
 
-def _iscsi_unit_poststop_commands(
-    ctx: CephadmContext, ident: 'DaemonIdentity', data_dir: str
-) -> List[runscripts.Command]:
-    # make sure we also stop the tcmu container
-    cmds: List[runscripts.Command] = []
-    runtime_dir = '/run'
-    ceph_iscsi = CephIscsi.init(ctx, ident.fsid, ident.daemon_id)
-    tcmu_container = ceph_iscsi.get_tcmu_runner_container()
-    cmds.append('! ' + ' '.join(tcmu_container.stop_cmd()) + '\n')
-    cmds.append('! ' + 'rm ' + runtime_dir + '/ceph-%s@%s.%s.service-pid' % (ident.fsid, ident.daemon_type, ident.daemon_id + '.tcmu') + '\n')
-    cmds.append('! ' + 'rm ' + runtime_dir + '/ceph-%s@%s.%s.service-cid' % (ident.fsid, ident.daemon_type, ident.daemon_id + '.tcmu') + '\n')
-    cmds.append(' '.join(CephIscsi.configfs_mount_umount(data_dir, mount=False)) + '\n')
-    return cmds
-
 ##################################
 
 

diff --git a/src/cephadm/cephadmlib/daemons/iscsi.py b/src/cephadm/cephadmlib/daemons/iscsi.py
index 1845a37bf4efbceb72df705c88cfe9becc8e33b4..ade88a90af0ffe7f380877d52f272775158cb37c 100644 (file)
--- a/src/cephadm/cephadmlib/daemons/iscsi.py
+++ b/src/cephadm/cephadmlib/daemons/iscsi.py
@@ -5,10 +5,10 @@ import re
 from typing import Dict, List, Optional, Tuple
 
 from ..container_daemon_form import ContainerDaemonForm, daemon_to_container
-from ..container_types import CephContainer, extract_uid_gid
+from ..container_types import CephContainer, SidecarContainer, extract_uid_gid
 from ..context_getters import fetch_configs, get_config_and_keyring
 from ..daemon_form import register as register_daemon_form
-from ..daemon_identity import DaemonIdentity, DaemonSubIdentity
+from ..daemon_identity import DaemonIdentity
 from ..constants import DEFAULT_IMAGE
 from ..context import CephadmContext
 from ..data_utils import dict_get, is_fsid
@@ -190,8 +190,7 @@ class CephIscsi(ContainerDaemonForm):
         os.chmod(os.path.join(data_dir, 'tcmu-runner-entrypoint.sh'), 0o700)
 
     @staticmethod
-    def configfs_mount_umount(data_dir, mount=True):
-        # type: (str, bool) -> List[str]
+    def configfs_mount_umount(data_dir: str, mount: bool = True) -> str:
         mount_path = os.path.join(data_dir, 'configfs')
         if mount:
             cmd = (
@@ -203,7 +202,7 @@ class CephIscsi(ContainerDaemonForm):
                 'if grep -qs {0} /proc/mounts; then '
                 'umount {0}; fi'.format(mount_path)
             )
-        return cmd.split()
+        return cmd
 
     @staticmethod
     def tcmu_runner_entrypoint_script() -> str:
@@ -242,27 +241,6 @@ do
 done
 """
 
-    def get_tcmu_runner_container(self):
-        # type: () -> CephContainer
-        # daemon_id, is used to generated the cid and pid files used by podman but as both tcmu-runner
-        # and rbd-target-api have the same daemon_id, it conflits and prevent the second container from
-        # starting. .tcmu runner is appended to the daemon_id to fix that.
-        subident = DaemonSubIdentity(
-            self.fsid, self.daemon_type, self.daemon_id, 'tcmu'
-        )
-        tcmu_dmn = self.create(self.ctx, subident)
-        tcmu_container = to_deployment_container(
-            self.ctx, daemon_to_container(self.ctx, tcmu_dmn, privileged=True)
-        )
-        # TODO: Eventually we don't want to run tcmu-runner through this script.
-        # This is intended to be a workaround backported to older releases
-        # and should eventually be removed in at least squid onward
-        tcmu_container.entrypoint = (
-            '/usr/local/scripts/tcmu-runner-entrypoint.sh'
-        )
-        tcmu_container.cname = self.get_container_name(desc='tcmu')
-        return tcmu_container
-
     def container(self, ctx: CephadmContext) -> CephContainer:
         # So the container can modprobe iscsi_target_mod and have write perms
         # to configfs we need to make this a privileged container.
@@ -284,3 +262,18 @@ done
         self, ctx: CephadmContext, args: List[str]
     ) -> None:
         args.append(ctx.container_engine.unlimited_pids_option)
+
+    def sidecar_containers(
+        self, ctx: CephadmContext
+    ) -> List[SidecarContainer]:
+        tcmu_sidecar = SidecarContainer.from_primary_and_values(
+            ctx,
+            self.container(ctx),
+            'tcmu',
+            # TODO: Eventually we don't want to run tcmu-runner through this
+            # script.  This is intended to be a workaround backported to older
+            # releases and should eventually be removed in at least squid
+            # onward
+            entrypoint='/usr/local/scripts/tcmu-runner-entrypoint.sh',
+        )
+        return [tcmu_sidecar]

diff --git a/src/cephadm/tests/test_cephadm.py b/src/cephadm/tests/test_cephadm.py
index 82850ab597d6bb349a47d7346627817f5988b23e..7bca8b9cbf3ca99159c2ea9881e35eff40a12b62 100644 (file)
--- a/src/cephadm/tests/test_cephadm.py
+++ b/src/cephadm/tests/test_cephadm.py
@@ -1760,7 +1760,11 @@ class TestCephVolume(object):
 
 
 class TestIscsi:
-    def test_unit_run(self, cephadm_fs):
+    def test_unit_run(self, cephadm_fs, funkypatch):
+        funkypatch.patch(
+            'cephadmlib.daemons.iscsi.extract_uid_gid'
+        ).return_value = (123, 123)
+
         fsid = '9b9d7609-f4d5-4aba-94c8-effa764d96c9'
         config_json = {
                 'files': {'iscsi-gateway.cfg': ''}
@@ -1773,27 +1777,35 @@ class TestIscsi:
             _cephadm.get_parm.return_value = config_json
 
             ident = _cephadm.DaemonIdentity(fsid, 'iscsi', 'daemon_id')
-            c = _cephadm.get_container(ctx, ident)
-            _cephadm.make_data_dir(ctx, ident)
-            _cephadm.deploy_daemon_units(
-                ctx,
-                ident,
-                0, 0,
-                c,
-                True, True
+
+            _cephadm._deploy_daemon_container(
+                ctx, ident, [], _cephadm.DeploymentType.DEFAULT
             )
 
             with open('/var/lib/ceph/9b9d7609-f4d5-4aba-94c8-effa764d96c9/iscsi.daemon_id/unit.run') as f:
-                assert f.read() == """set -e
+                contents = f.read()
+                assert contents == """set -e
 if ! grep -qs /var/lib/ceph/9b9d7609-f4d5-4aba-94c8-effa764d96c9/iscsi.daemon_id/configfs /proc/mounts; then mount -t configfs none /var/lib/ceph/9b9d7609-f4d5-4aba-94c8-effa764d96c9/iscsi.daemon_id/configfs; fi
-# iscsi tcmu-runner container
-! /usr/bin/docker rm -f ceph-9b9d7609-f4d5-4aba-94c8-effa764d96c9-iscsi.daemon_id-tcmu 2> /dev/null
-! /usr/bin/docker rm -f ceph-9b9d7609-f4d5-4aba-94c8-effa764d96c9-iscsi-daemon_id-tcmu 2> /dev/null
-/usr/bin/docker run --rm --ipc=host --stop-signal=SIGTERM --ulimit nofile=1048576 --net=host --entrypoint /usr/local/scripts/tcmu-runner-entrypoint.sh --privileged --group-add=disk --init --name ceph-9b9d7609-f4d5-4aba-94c8-effa764d96c9-iscsi-daemon_id-tcmu --pids-limit=0 -e CONTAINER_IMAGE=ceph/ceph -e NODE_NAME=host1 -v /var/lib/ceph/9b9d7609-f4d5-4aba-94c8-effa764d96c9/iscsi.daemon_id/config:/etc/ceph/ceph.conf:z -v /var/lib/ceph/9b9d7609-f4d5-4aba-94c8-effa764d96c9/iscsi.daemon_id/keyring:/etc/ceph/keyring:z -v /var/lib/ceph/9b9d7609-f4d5-4aba-94c8-effa764d96c9/iscsi.daemon_id/iscsi-gateway.cfg:/etc/ceph/iscsi-gateway.cfg:z -v /var/lib/ceph/9b9d7609-f4d5-4aba-94c8-effa764d96c9/iscsi.daemon_id/configfs:/sys/kernel/config -v /var/lib/ceph/9b9d7609-f4d5-4aba-94c8-effa764d96c9/iscsi.daemon_id/tcmu-runner-entrypoint.sh:/usr/local/scripts/tcmu-runner-entrypoint.sh -v /var/log/ceph/9b9d7609-f4d5-4aba-94c8-effa764d96c9:/var/log:z -v /dev:/dev --mount type=bind,source=/lib/modules,destination=/lib/modules,ro=true ceph/ceph &
 # iscsi.daemon_id
 ! /usr/bin/docker rm -f ceph-9b9d7609-f4d5-4aba-94c8-effa764d96c9-iscsi.daemon_id 2> /dev/null
 ! /usr/bin/docker rm -f ceph-9b9d7609-f4d5-4aba-94c8-effa764d96c9-iscsi-daemon_id 2> /dev/null
 /usr/bin/docker run --rm --ipc=host --stop-signal=SIGTERM --ulimit nofile=1048576 --net=host --entrypoint /usr/bin/rbd-target-api --privileged --group-add=disk --init --name ceph-9b9d7609-f4d5-4aba-94c8-effa764d96c9-iscsi-daemon_id --pids-limit=0 -e CONTAINER_IMAGE=ceph/ceph -e NODE_NAME=host1 -v /var/lib/ceph/9b9d7609-f4d5-4aba-94c8-effa764d96c9/iscsi.daemon_id/config:/etc/ceph/ceph.conf:z -v /var/lib/ceph/9b9d7609-f4d5-4aba-94c8-effa764d96c9/iscsi.daemon_id/keyring:/etc/ceph/keyring:z -v /var/lib/ceph/9b9d7609-f4d5-4aba-94c8-effa764d96c9/iscsi.daemon_id/iscsi-gateway.cfg:/etc/ceph/iscsi-gateway.cfg:z -v /var/lib/ceph/9b9d7609-f4d5-4aba-94c8-effa764d96c9/iscsi.daemon_id/configfs:/sys/kernel/config -v /var/lib/ceph/9b9d7609-f4d5-4aba-94c8-effa764d96c9/iscsi.daemon_id/tcmu-runner-entrypoint.sh:/usr/local/scripts/tcmu-runner-entrypoint.sh -v /var/log/ceph/9b9d7609-f4d5-4aba-94c8-effa764d96c9:/var/log:z -v /dev:/dev --mount type=bind,source=/lib/modules,destination=/lib/modules,ro=true ceph/ceph
+"""
+            with open('/var/lib/ceph/9b9d7609-f4d5-4aba-94c8-effa764d96c9/iscsi.daemon_id/sidecar-tcmu.run') as f:
+                contents = f.read()
+            assert contents == """#!/bin/sh
+# sidecar: tcmu
+
+set -e
+if [ "$1" = stop ] || [ "$1" = poststop ]; then
+    ! /usr/bin/docker stop ceph-9b9d7609-f4d5-4aba-94c8-effa764d96c9-iscsi-daemon_id-tcmu
+    ! /usr/bin/docker inspect ceph-9b9d7609-f4d5-4aba-94c8-effa764d96c9-iscsi-daemon_id-tcmu &>/dev/null
+    exit $?
+fi
+
+! /usr/bin/docker rm -f ceph-9b9d7609-f4d5-4aba-94c8-effa764d96c9-iscsi-daemon_id-tcmu 2> /dev/null
+
+exec /usr/bin/docker run --rm --ipc=host --stop-signal=SIGTERM --ulimit nofile=1048576 --net=host --entrypoint /usr/local/scripts/tcmu-runner-entrypoint.sh --privileged --group-add=disk --init --name ceph-9b9d7609-f4d5-4aba-94c8-effa764d96c9-iscsi-daemon_id-tcmu --pids-limit=0 -e CONTAINER_IMAGE=ceph/ceph -e NODE_NAME=host1 -v /var/lib/ceph/9b9d7609-f4d5-4aba-94c8-effa764d96c9/iscsi.daemon_id/config:/etc/ceph/ceph.conf:z -v /var/lib/ceph/9b9d7609-f4d5-4aba-94c8-effa764d96c9/iscsi.daemon_id/keyring:/etc/ceph/keyring:z -v /var/lib/ceph/9b9d7609-f4d5-4aba-94c8-effa764d96c9/iscsi.daemon_id/iscsi-gateway.cfg:/etc/ceph/iscsi-gateway.cfg:z -v /var/lib/ceph/9b9d7609-f4d5-4aba-94c8-effa764d96c9/iscsi.daemon_id/configfs:/sys/kernel/config -v /var/lib/ceph/9b9d7609-f4d5-4aba-94c8-effa764d96c9/iscsi.daemon_id/tcmu-runner-entrypoint.sh:/usr/local/scripts/tcmu-runner-entrypoint.sh -v /var/log/ceph/9b9d7609-f4d5-4aba-94c8-effa764d96c9:/var/log:z -v /dev:/dev --mount type=bind,source=/lib/modules,destination=/lib/modules,ro=true ceph/ceph
 """
 
     def test_get_container(self):