From f59fd86b687fd14a191528658e20d9f7a5ec0cba Mon Sep 17 00:00:00 2001 From: Teoman ONAY Date: Thu, 11 Nov 2021 15:05:49 +0000 Subject: [PATCH] cephadm: remove containers pids-limit The default pids-limit (docker 4096/podman 2048) prevent some customization from working (http threads on RGW) or limits the number of luns per iscsi target. Fixes: https://tracker.ceph.com/issues/52898 Signed-off-by: Teoman ONAY (cherry picked from commit de8b3c2676e65eb61df54c65cfd3b3af1e68da56) --- .../workunits/task/test_iscsi_pids_limit.yaml | 20 ++++++++++++++++ qa/workunits/cephadm/test_iscsi_pids_limit.sh | 24 +++++++++++++++++++ src/cephadm/cephadm | 13 ++++++++++ src/cephadm/tests/test_cephadm.py | 4 ++-- 4 files changed, 59 insertions(+), 2 deletions(-) create mode 100644 qa/suites/orch/cephadm/workunits/task/test_iscsi_pids_limit.yaml create mode 100755 qa/workunits/cephadm/test_iscsi_pids_limit.sh diff --git a/qa/suites/orch/cephadm/workunits/task/test_iscsi_pids_limit.yaml b/qa/suites/orch/cephadm/workunits/task/test_iscsi_pids_limit.yaml new file mode 100644 index 0000000000000..b271777e1874a --- /dev/null +++ b/qa/suites/orch/cephadm/workunits/task/test_iscsi_pids_limit.yaml @@ -0,0 +1,20 @@ +roles: +- - host.a + - osd.0 + - osd.1 + - osd.2 + - mon.a + - mgr.a + - client.0 +tasks: +- install: +- cephadm: +- cephadm.shell: + host.a: + - ceph osd pool create foo + - rbd pool init foo + - ceph orch apply iscsi foo u p +- workunit: + clients: + client.0: + - cephadm/test_iscsi_pids_limit.sh diff --git a/qa/workunits/cephadm/test_iscsi_pids_limit.sh b/qa/workunits/cephadm/test_iscsi_pids_limit.sh new file mode 100755 index 0000000000000..32326f140f2f6 --- /dev/null +++ b/qa/workunits/cephadm/test_iscsi_pids_limit.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +# checks if the containers default pids-limit (4096) is removed and Iscsi +# containers continue to run +# exits 1 if fails + +set -ex + +ISCSI_CONT_IDS=$(sudo podman ps -qa --filter='name=iscsi') +CONT_COUNT=$(echo ${ISCSI_CONT_IDS} | wc -w) +test ${CONT_COUNT} -eq 2 + +for i in ${ISCSI_CONT_IDS} +do + sudo podman exec ${i} /bin/sh -c 'for j in {0..20000}; do sleep 30 & done' +done + +for i in ${ISCSI_CONT_IDS} +do + SLEEP_COUNT=$(sudo podman exec ${i} /bin/sh -c 'ps -ef | grep -c sleep') + test ${SLEEP_COUNT} -gt 20000 +done + +echo OK diff --git a/src/cephadm/cephadm b/src/cephadm/cephadm index a3a7ca6eb0039..322e5e1d678be 100755 --- a/src/cephadm/cephadm +++ b/src/cephadm/cephadm @@ -827,6 +827,10 @@ class CephIscsi(object): # remove extra container args for tcmu container. # extra args could cause issue with forking service type tcmu_container.container_args = [] + # set container limits to unlimited as defaults (Docker 4096 / Podman 2048) + # prevents the creation of max lun (default 255) + pids_unlimited = '-1' if isinstance(self.ctx.container_engine, Podman) else '0' + tcmu_container.container_args.extend(['--pids-limit=%s' % pids_unlimited]) return tcmu_container ################################## @@ -2769,6 +2773,8 @@ def get_container(ctx: CephadmContext, envs.extend(Keepalived.get_container_envs()) container_args.extend(['--cap-add=NET_ADMIN', '--cap-add=NET_RAW']) elif daemon_type == CephIscsi.daemon_type: + # Applies only on rbd-target-api as get_tcmu_runner_container() + # removes all tcmu-runner arguments entrypoint = CephIscsi.entrypoint name = '%s.%s' % (daemon_type, daemon_id) # So the container can modprobe iscsi_target_mod and have write perms @@ -2807,6 +2813,8 @@ def get_container(ctx: CephadmContext, # if using podman, set -d, --conmon-pidfile & --cidfile flags # so service can have Type=Forking + # set containers limits to unlimited as defaults (Docker 4096 / Podman 2048) + # prevents some app customizations from running if isinstance(ctx.container_engine, Podman): runtime_dir = '/run' container_args.extend([ @@ -2815,9 +2823,14 @@ def get_container(ctx: CephadmContext, runtime_dir + '/ceph-%s@%s.%s.service-pid' % (fsid, daemon_type, daemon_id), '--cidfile', runtime_dir + '/ceph-%s@%s.%s.service-cid' % (fsid, daemon_type, daemon_id), + '--pids-limit=-1', ]) if ctx.container_engine.version >= CGROUPS_SPLIT_PODMAN_VERSION: container_args.append('--cgroups=split') + else: + container_args.extend([ + '--pids-limit=0', + ]) return CephContainer.for_daemon( ctx, diff --git a/src/cephadm/tests/test_cephadm.py b/src/cephadm/tests/test_cephadm.py index 7383ac0ee1861..17c934fa7624f 100644 --- a/src/cephadm/tests/test_cephadm.py +++ b/src/cephadm/tests/test_cephadm.py @@ -1635,11 +1635,11 @@ if ! grep -qs /var/lib/ceph/9b9d7609-f4d5-4aba-94c8-effa764d96c9/iscsi.daemon_id # iscsi tcmu-runner container ! /usr/bin/podman rm -f ceph-9b9d7609-f4d5-4aba-94c8-effa764d96c9-iscsi.daemon_id-tcmu 2> /dev/null ! /usr/bin/podman rm -f ceph-9b9d7609-f4d5-4aba-94c8-effa764d96c9-iscsi-daemon_id-tcmu 2> /dev/null -/usr/bin/podman run --rm --ipc=host --stop-signal=SIGTERM --net=host --entrypoint /usr/bin/tcmu-runner --privileged --group-add=disk --init --name ceph-9b9d7609-f4d5-4aba-94c8-effa764d96c9-iscsi-daemon_id-tcmu -e CONTAINER_IMAGE=ceph/ceph -e NODE_NAME=host1 -e CEPH_USE_RANDOM_NONCE=1 -v /var/lib/ceph/9b9d7609-f4d5-4aba-94c8-effa764d96c9/iscsi.daemon_id/config:/etc/ceph/ceph.conf:z -v /var/lib/ceph/9b9d7609-f4d5-4aba-94c8-effa764d96c9/iscsi.daemon_id/keyring:/etc/ceph/keyring:z -v /var/lib/ceph/9b9d7609-f4d5-4aba-94c8-effa764d96c9/iscsi.daemon_id/iscsi-gateway.cfg:/etc/ceph/iscsi-gateway.cfg:z -v /var/lib/ceph/9b9d7609-f4d5-4aba-94c8-effa764d96c9/iscsi.daemon_id/configfs:/sys/kernel/config -v /var/log/ceph/9b9d7609-f4d5-4aba-94c8-effa764d96c9:/var/log:z -v /dev:/dev --mount type=bind,source=/lib/modules,destination=/lib/modules,ro=true ceph/ceph & +/usr/bin/podman run --rm --ipc=host --stop-signal=SIGTERM --net=host --entrypoint /usr/bin/tcmu-runner --privileged --group-add=disk --init --name ceph-9b9d7609-f4d5-4aba-94c8-effa764d96c9-iscsi-daemon_id-tcmu --pids-limit=0 -e CONTAINER_IMAGE=ceph/ceph -e NODE_NAME=host1 -e CEPH_USE_RANDOM_NONCE=1 -v /var/lib/ceph/9b9d7609-f4d5-4aba-94c8-effa764d96c9/iscsi.daemon_id/config:/etc/ceph/ceph.conf:z -v /var/lib/ceph/9b9d7609-f4d5-4aba-94c8-effa764d96c9/iscsi.daemon_id/keyring:/etc/ceph/keyring:z -v /var/lib/ceph/9b9d7609-f4d5-4aba-94c8-effa764d96c9/iscsi.daemon_id/iscsi-gateway.cfg:/etc/ceph/iscsi-gateway.cfg:z -v /var/lib/ceph/9b9d7609-f4d5-4aba-94c8-effa764d96c9/iscsi.daemon_id/configfs:/sys/kernel/config -v /var/log/ceph/9b9d7609-f4d5-4aba-94c8-effa764d96c9:/var/log:z -v /dev:/dev --mount type=bind,source=/lib/modules,destination=/lib/modules,ro=true ceph/ceph & # iscsi.daemon_id ! /usr/bin/podman rm -f ceph-9b9d7609-f4d5-4aba-94c8-effa764d96c9-iscsi.daemon_id 2> /dev/null ! /usr/bin/podman rm -f ceph-9b9d7609-f4d5-4aba-94c8-effa764d96c9-iscsi-daemon_id 2> /dev/null -/usr/bin/podman run --rm --ipc=host --stop-signal=SIGTERM --net=host --entrypoint /usr/bin/rbd-target-api --privileged --group-add=disk --init --name ceph-9b9d7609-f4d5-4aba-94c8-effa764d96c9-iscsi-daemon_id -e CONTAINER_IMAGE=ceph/ceph -e NODE_NAME=host1 -e CEPH_USE_RANDOM_NONCE=1 -v /var/lib/ceph/9b9d7609-f4d5-4aba-94c8-effa764d96c9/iscsi.daemon_id/config:/etc/ceph/ceph.conf:z -v /var/lib/ceph/9b9d7609-f4d5-4aba-94c8-effa764d96c9/iscsi.daemon_id/keyring:/etc/ceph/keyring:z -v /var/lib/ceph/9b9d7609-f4d5-4aba-94c8-effa764d96c9/iscsi.daemon_id/iscsi-gateway.cfg:/etc/ceph/iscsi-gateway.cfg:z -v /var/lib/ceph/9b9d7609-f4d5-4aba-94c8-effa764d96c9/iscsi.daemon_id/configfs:/sys/kernel/config -v /var/log/ceph/9b9d7609-f4d5-4aba-94c8-effa764d96c9:/var/log:z -v /dev:/dev --mount type=bind,source=/lib/modules,destination=/lib/modules,ro=true ceph/ceph +/usr/bin/podman run --rm --ipc=host --stop-signal=SIGTERM --net=host --entrypoint /usr/bin/rbd-target-api --privileged --group-add=disk --init --name ceph-9b9d7609-f4d5-4aba-94c8-effa764d96c9-iscsi-daemon_id --pids-limit=0 -e CONTAINER_IMAGE=ceph/ceph -e NODE_NAME=host1 -e CEPH_USE_RANDOM_NONCE=1 -v /var/lib/ceph/9b9d7609-f4d5-4aba-94c8-effa764d96c9/iscsi.daemon_id/config:/etc/ceph/ceph.conf:z -v /var/lib/ceph/9b9d7609-f4d5-4aba-94c8-effa764d96c9/iscsi.daemon_id/keyring:/etc/ceph/keyring:z -v /var/lib/ceph/9b9d7609-f4d5-4aba-94c8-effa764d96c9/iscsi.daemon_id/iscsi-gateway.cfg:/etc/ceph/iscsi-gateway.cfg:z -v /var/lib/ceph/9b9d7609-f4d5-4aba-94c8-effa764d96c9/iscsi.daemon_id/configfs:/sys/kernel/config -v /var/log/ceph/9b9d7609-f4d5-4aba-94c8-effa764d96c9:/var/log:z -v /dev:/dev --mount type=bind,source=/lib/modules,destination=/lib/modules,ro=true ceph/ceph """ def test_get_container(self): -- 2.39.5