From f4212b20e5dce289533f5a229435c3b91604d8e3 Mon Sep 17 00:00:00 2001 From: Dimitri Savineau Date: Thu, 20 Jun 2019 10:28:44 -0400 Subject: [PATCH] ceph-volume: Set max open files limit on container The ceph-volume lvm list command takes ages to complete when having a lot of LV devices on containerized deployment. For instance, with 25 OSDs on a node it takes 3 mins 44s to list the OSD. Adding the max open files limit to the container engine cli when executing the ceph-volume command seems to improve a lot thee execution time ~30s. This was impacting the OSDs creation with ceph-volume (both filestore and bluestore) when using multiple LV devices. Closes: https://bugzilla.redhat.com/show_bug.cgi?id=1702285 Signed-off-by: Dimitri Savineau (cherry picked from commit b98753488110b04cd2071c2b103493235dfc0c80) --- library/ceph_volume.py | 1 + library/test_ceph_volume.py | 6 ++++++ roles/ceph-osd/tasks/start_osds.yml | 2 +- 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/library/ceph_volume.py b/library/ceph_volume.py index 4d45551c3..2600c20fe 100644 --- a/library/ceph_volume.py +++ b/library/ceph_volume.py @@ -183,6 +183,7 @@ def container_exec(binary, container_image): Build the docker CLI to run a command inside a container ''' command_exec = ['docker', 'run', '--rm', '--privileged', '--net=host', '--ipc=host', + '--ulimit', 'nofile=1024:1024', '-v', '/run/lock/lvm:/run/lock/lvm:z', '-v', '/var/run/udev/:/var/run/udev/:z', '-v', '/dev:/dev', '-v', '/etc/ceph:/etc/ceph:z', diff --git a/library/test_ceph_volume.py b/library/test_ceph_volume.py index df020e479..11eb9bb66 100644 --- a/library/test_ceph_volume.py +++ b/library/test_ceph_volume.py @@ -40,6 +40,7 @@ class TestCephVolumeModule(object): fake_binary = "ceph-volume" fake_container_image = "docker.io/ceph/daemon:latest-luminous" expected_command_list = ['docker', 'run', '--rm', '--privileged', '--net=host', '--ipc=host', # noqa E501 + '--ulimit', 'nofile=1024:1024', '-v', '/run/lock/lvm:/run/lock/lvm:z', '-v', '/var/run/udev/:/var/run/udev/:z', '-v', '/dev:/dev', '-v', '/etc/ceph:/etc/ceph:z', # noqa E501 @@ -56,6 +57,7 @@ class TestCephVolumeModule(object): fake_module.params = {'data': '/dev/sda'} fake_container_image = "docker.io/ceph/daemon:latest-luminous" expected_command_list = ['docker', 'run', '--rm', '--privileged', '--net=host', '--ipc=host', # noqa E501 + '--ulimit', 'nofile=1024:1024', '-v', '/run/lock/lvm:/run/lock/lvm:z', '-v', '/var/run/udev/:/var/run/udev/:z', '-v', '/dev:/dev', '-v', '/etc/ceph:/etc/ceph:z', # noqa E501 @@ -124,6 +126,7 @@ class TestCephVolumeModule(object): fake_module.params = {'cluster': 'ceph', 'data': '/dev/sda'} fake_container_image = "docker.io/ceph/daemon:latest-luminous" expected_command_list = ['docker', 'run', '--rm', '--privileged', '--net=host', '--ipc=host', # noqa E501 + '--ulimit', 'nofile=1024:1024', '-v', '/run/lock/lvm:/run/lock/lvm:z', '-v', '/var/run/udev/:/var/run/udev/:z', '-v', '/dev:/dev', '-v', '/etc/ceph:/etc/ceph:z', # noqa E501 @@ -151,6 +154,7 @@ class TestCephVolumeModule(object): fake_action = "create" fake_container_image = "docker.io/ceph/daemon:latest-luminous" expected_command_list = ['docker', 'run', '--rm', '--privileged', '--net=host', '--ipc=host', # noqa E501 + '--ulimit', 'nofile=1024:1024', '-v', '/run/lock/lvm:/run/lock/lvm:z', '-v', '/var/run/udev/:/var/run/udev/:z', '-v', '/dev:/dev', '-v', '/etc/ceph:/etc/ceph:z', # noqa E501 @@ -199,6 +203,7 @@ class TestCephVolumeModule(object): fake_action = "prepare" fake_container_image = "docker.io/ceph/daemon:latest-luminous" expected_command_list = ['docker', 'run', '--rm', '--privileged', '--net=host', '--ipc=host', # noqa E501 + '--ulimit', 'nofile=1024:1024', '-v', '/run/lock/lvm:/run/lock/lvm:z', '-v', '/var/run/udev/:/var/run/udev/:z', '-v', '/dev:/dev', '-v', '/etc/ceph:/etc/ceph:z', # noqa E501 @@ -248,6 +253,7 @@ class TestCephVolumeModule(object): fake_container_image = "docker.io/ceph/daemon:latest-luminous" expected_command_list = ['docker', 'run', '--rm', '--privileged', '--net=host', '--ipc=host', # noqa E501 + '--ulimit', 'nofile=1024:1024', '-v', '/run/lock/lvm:/run/lock/lvm:z', '-v', '/var/run/udev/:/var/run/udev/:z', '-v', '/dev:/dev', '-v', '/etc/ceph:/etc/ceph:z', # noqa E501 diff --git a/roles/ceph-osd/tasks/start_osds.yml b/roles/ceph-osd/tasks/start_osds.yml index fb4cc05d1..d3dab5f53 100644 --- a/roles/ceph-osd/tasks/start_osds.yml +++ b/roles/ceph-osd/tasks/start_osds.yml @@ -48,7 +48,7 @@ - name: set_fact docker_exec_start_osd set_fact: - docker_exec_start_osd: "{{ 'docker run --rm --privileged=true -v /run/lvm/lvmetad.socket:/run/lvm/lvmetad.socket -v /var/run/udev/:/var/run/udev/:z -v /etc/ceph:/etc/ceph:z -v /dev:/dev --entrypoint=ceph-volume ' + ceph_docker_registry + '/' + ceph_docker_image + ':' + ceph_docker_image_tag if containerized_deployment else 'ceph-volume' }}" + docker_exec_start_osd: "{{ 'docker run --rm --ulimit nofile=1024:1024 --privileged=true -v /run/lvm/lvmetad.socket:/run/lvm/lvmetad.socket -v /var/run/udev/:/var/run/udev/:z -v /etc/ceph:/etc/ceph:z -v /dev:/dev --entrypoint=ceph-volume ' + ceph_docker_registry + '/' + ceph_docker_image + ':' + ceph_docker_image_tag if containerized_deployment else 'ceph-volume' }}" - name: collect osd ids shell: > -- 2.47.3