From 9a4ac46d1977726c6e9b0ce8e4f051f15ce2ca12 Mon Sep 17 00:00:00 2001 From: Dimitri Savineau Date: Tue, 6 Aug 2019 11:52:59 -0400 Subject: [PATCH] ceph-osd: Add ulimit nofile on container start On containerized deployment, the OSD entrypoint runs some ceph-volume commands (lvm/simple scan and/or activate) which perform badly without the ulimit option. This option was added for all previous ceph-volume commands but not on the ceph-osd container startup. Also updating hard limit value to 4096 to reflect default baremetal value. Signed-off-by: Dimitri Savineau --- library/ceph_volume.py | 2 +- library/test_ceph_volume.py | 14 +++++++------- roles/ceph-osd/tasks/start_osds.yml | 2 +- roles/ceph-osd/templates/ceph-osd-run.sh.j2 | 1 + 4 files changed, 10 insertions(+), 9 deletions(-) diff --git a/library/ceph_volume.py b/library/ceph_volume.py index 252f8419a..48f3fef54 100644 --- a/library/ceph_volume.py +++ b/library/ceph_volume.py @@ -189,7 +189,7 @@ def container_exec(binary, container_image): container_binary = os.getenv('CEPH_CONTAINER_BINARY') command_exec = [container_binary, 'run', '--rm', '--privileged', '--net=host', '--ipc=host', - '--ulimit', 'nofile=1024:1024', + '--ulimit', 'nofile=1024:4096', '-v', '/run/lock/lvm:/run/lock/lvm:z', '-v', '/var/run/udev/:/var/run/udev/:z', '-v', '/dev:/dev', '-v', '/etc/ceph:/etc/ceph:z', diff --git a/library/test_ceph_volume.py b/library/test_ceph_volume.py index adebd6577..cf0d9ff64 100644 --- a/library/test_ceph_volume.py +++ b/library/test_ceph_volume.py @@ -53,7 +53,7 @@ class TestCephVolumeModule(object): fake_binary = "ceph-volume" fake_container_image = "docker.io/ceph/daemon:latest-luminous" expected_command_list = ['docker', 'run', '--rm', '--privileged', '--net=host', '--ipc=host', # noqa E501 - '--ulimit', 'nofile=1024:1024', + '--ulimit', 'nofile=1024:4096', '-v', '/run/lock/lvm:/run/lock/lvm:z', '-v', '/var/run/udev/:/var/run/udev/:z', '-v', '/dev:/dev', '-v', '/etc/ceph:/etc/ceph:z', # noqa E501 @@ -70,7 +70,7 @@ class TestCephVolumeModule(object): fake_module.params = {'data': '/dev/sda'} fake_container_image = "docker.io/ceph/daemon:latest-luminous" expected_command_list = ['docker', 'run', '--rm', '--privileged', '--net=host', '--ipc=host', # noqa E501 - '--ulimit', 'nofile=1024:1024', + '--ulimit', 'nofile=1024:4096', '-v', '/run/lock/lvm:/run/lock/lvm:z', '-v', '/var/run/udev/:/var/run/udev/:z', '-v', '/dev:/dev', '-v', '/etc/ceph:/etc/ceph:z', # noqa E501 @@ -139,7 +139,7 @@ class TestCephVolumeModule(object): fake_module.params = {'cluster': 'ceph', 'data': '/dev/sda'} fake_container_image = "docker.io/ceph/daemon:latest-luminous" expected_command_list = ['docker', 'run', '--rm', '--privileged', '--net=host', '--ipc=host', # noqa E501 - '--ulimit', 'nofile=1024:1024', + '--ulimit', 'nofile=1024:4096', '-v', '/run/lock/lvm:/run/lock/lvm:z', '-v', '/var/run/udev/:/var/run/udev/:z', '-v', '/dev:/dev', '-v', '/etc/ceph:/etc/ceph:z', # noqa E501 @@ -172,7 +172,7 @@ class TestCephVolumeModule(object): fake_module = MagicMock() fake_container_image = "docker.io/ceph/daemon:latest-luminous" expected_command_list = ['docker', 'run', '--rm', '--privileged', '--net=host', '--ipc=host', # noqa E501 - '--ulimit', 'nofile=1024:1024', + '--ulimit', 'nofile=1024:4096', '-v', '/run/lock/lvm:/run/lock/lvm:z', '-v', '/var/run/udev/:/var/run/udev/:z', '-v', '/dev:/dev', '-v', '/etc/ceph:/etc/ceph:z', # noqa E501 @@ -196,7 +196,7 @@ class TestCephVolumeModule(object): fake_action = "create" fake_container_image = "docker.io/ceph/daemon:latest-luminous" expected_command_list = ['docker', 'run', '--rm', '--privileged', '--net=host', '--ipc=host', # noqa E501 - '--ulimit', 'nofile=1024:1024', + '--ulimit', 'nofile=1024:4096', '-v', '/run/lock/lvm:/run/lock/lvm:z', '-v', '/var/run/udev/:/var/run/udev/:z', '-v', '/dev:/dev', '-v', '/etc/ceph:/etc/ceph:z', # noqa E501 @@ -245,7 +245,7 @@ class TestCephVolumeModule(object): fake_action = "prepare" fake_container_image = "docker.io/ceph/daemon:latest-luminous" expected_command_list = ['docker', 'run', '--rm', '--privileged', '--net=host', '--ipc=host', # noqa E501 - '--ulimit', 'nofile=1024:1024', + '--ulimit', 'nofile=1024:4096', '-v', '/run/lock/lvm:/run/lock/lvm:z', '-v', '/var/run/udev/:/var/run/udev/:z', '-v', '/dev:/dev', '-v', '/etc/ceph:/etc/ceph:z', # noqa E501 @@ -295,7 +295,7 @@ class TestCephVolumeModule(object): fake_container_image = "docker.io/ceph/daemon:latest-luminous" expected_command_list = ['docker', 'run', '--rm', '--privileged', '--net=host', '--ipc=host', # noqa E501 - '--ulimit', 'nofile=1024:1024', + '--ulimit', 'nofile=1024:4096', '-v', '/run/lock/lvm:/run/lock/lvm:z', '-v', '/var/run/udev/:/var/run/udev/:z', '-v', '/dev:/dev', '-v', '/etc/ceph:/etc/ceph:z', # noqa E501 diff --git a/roles/ceph-osd/tasks/start_osds.yml b/roles/ceph-osd/tasks/start_osds.yml index 809e6c68c..7e7ce5022 100644 --- a/roles/ceph-osd/tasks/start_osds.yml +++ b/roles/ceph-osd/tasks/start_osds.yml @@ -29,7 +29,7 @@ - name: set_fact docker_exec_start_osd set_fact: - docker_exec_start_osd: "{{ '{{ container_binary }} run --rm --ulimit nofile=1024:1024 --net=host --privileged=true -v /var/run/udev/:/var/run/udev/:z -v /run/lvm/:/run/lvm/ -v /etc/ceph:/etc/ceph:z -v /dev:/dev --entrypoint=ceph-volume ' + ceph_docker_registry + '/' + ceph_docker_image + ':' + ceph_docker_image_tag if containerized_deployment else 'ceph-volume' }}" + docker_exec_start_osd: "{{ '{{ container_binary }} run --rm --ulimit nofile=1024:4096 --net=host --privileged=true -v /var/run/udev/:/var/run/udev/:z -v /run/lvm/:/run/lvm/ -v /etc/ceph:/etc/ceph:z -v /dev:/dev --entrypoint=ceph-volume ' + ceph_docker_registry + '/' + ceph_docker_image + ':' + ceph_docker_image_tag if containerized_deployment else 'ceph-volume' }}" - name: collect osd ids command: "{{ docker_exec_start_osd }} lvm list --format json" diff --git a/roles/ceph-osd/templates/ceph-osd-run.sh.j2 b/roles/ceph-osd/templates/ceph-osd-run.sh.j2 index 0c2cab348..55e2b946c 100644 --- a/roles/ceph-osd/templates/ceph-osd-run.sh.j2 +++ b/roles/ceph-osd/templates/ceph-osd-run.sh.j2 @@ -17,6 +17,7 @@ numactl \ --privileged=true \ --pid=host \ --ipc=host \ + --ulimit nofile=1024:4096 \ {% if osd_objectstore == 'filestore' -%} --memory={{ ceph_osd_docker_memory_limit }} \ {% endif -%} -- 2.39.5