ceph-osd: Add ulimit nofile on container start

On containerized deployment, the OSD entrypoint runs some ceph-volume
commands (lvm/simple scan and/or activate) which perform badly without
the ulimit option.
This option was added for all previous ceph-volume commands but not on
the ceph-osd container startup.
Also updating hard limit value to 4096 to reflect default baremetal
value.

Signed-off-by: Dimitri Savineau <dsavinea@redhat.com>
(cherry picked from commit 9a4ac46d1977726c6e9b0ce8e4f051f15ce2ca12)

diff --git a/library/ceph_volume.py b/library/ceph_volume.py
index 252f8419aaeeac27e0438dbe6d97234597947281..48f3fef54a442ad65ba63647689636412fb8714e 100644 (file)
--- a/library/ceph_volume.py
+++ b/library/ceph_volume.py
@@ -189,7 +189,7 @@ def container_exec(binary, container_image):
     container_binary = os.getenv('CEPH_CONTAINER_BINARY')
     command_exec = [container_binary, 'run',
                     '--rm', '--privileged', '--net=host', '--ipc=host',
-                    '--ulimit', 'nofile=1024:1024',
+                    '--ulimit', 'nofile=1024:4096',
                     '-v', '/run/lock/lvm:/run/lock/lvm:z',
                     '-v', '/var/run/udev/:/var/run/udev/:z',
                     '-v', '/dev:/dev', '-v', '/etc/ceph:/etc/ceph:z',

diff --git a/library/test_ceph_volume.py b/library/test_ceph_volume.py
index adebd6577c4202e883140ca698d6464ceb3585cb..cf0d9ff64bcec5b29bb619805f002c9beb3a356f 100644 (file)
--- a/library/test_ceph_volume.py
+++ b/library/test_ceph_volume.py
@@ -53,7 +53,7 @@ class TestCephVolumeModule(object):
         fake_binary = "ceph-volume"
         fake_container_image = "docker.io/ceph/daemon:latest-luminous"
         expected_command_list = ['docker', 'run', '--rm', '--privileged', '--net=host', '--ipc=host',  # noqa E501
-                                 '--ulimit', 'nofile=1024:1024',
+                                 '--ulimit', 'nofile=1024:4096',
                                  '-v', '/run/lock/lvm:/run/lock/lvm:z',
                                  '-v', '/var/run/udev/:/var/run/udev/:z',
                                  '-v', '/dev:/dev', '-v', '/etc/ceph:/etc/ceph:z',  # noqa E501
@@ -70,7 +70,7 @@ class TestCephVolumeModule(object):
         fake_module.params = {'data': '/dev/sda'}
         fake_container_image = "docker.io/ceph/daemon:latest-luminous"
         expected_command_list = ['docker', 'run', '--rm', '--privileged', '--net=host', '--ipc=host',  # noqa E501
-                                 '--ulimit', 'nofile=1024:1024',
+                                 '--ulimit', 'nofile=1024:4096',
                                  '-v', '/run/lock/lvm:/run/lock/lvm:z',
                                  '-v', '/var/run/udev/:/var/run/udev/:z',
                                  '-v', '/dev:/dev', '-v', '/etc/ceph:/etc/ceph:z',  # noqa E501
@@ -139,7 +139,7 @@ class TestCephVolumeModule(object):
         fake_module.params = {'cluster': 'ceph', 'data': '/dev/sda'}
         fake_container_image = "docker.io/ceph/daemon:latest-luminous"
         expected_command_list = ['docker', 'run', '--rm', '--privileged', '--net=host', '--ipc=host',  # noqa E501
-                                 '--ulimit', 'nofile=1024:1024',
+                                 '--ulimit', 'nofile=1024:4096',
                                  '-v', '/run/lock/lvm:/run/lock/lvm:z',
                                  '-v', '/var/run/udev/:/var/run/udev/:z',
                                  '-v', '/dev:/dev', '-v', '/etc/ceph:/etc/ceph:z',  # noqa E501
@@ -172,7 +172,7 @@ class TestCephVolumeModule(object):
         fake_module = MagicMock()
         fake_container_image = "docker.io/ceph/daemon:latest-luminous"
         expected_command_list = ['docker', 'run', '--rm', '--privileged', '--net=host', '--ipc=host',  # noqa E501
-                                 '--ulimit', 'nofile=1024:1024',
+                                 '--ulimit', 'nofile=1024:4096',
                                  '-v', '/run/lock/lvm:/run/lock/lvm:z',
                                  '-v', '/var/run/udev/:/var/run/udev/:z',
                                  '-v', '/dev:/dev', '-v', '/etc/ceph:/etc/ceph:z',  # noqa E501
@@ -196,7 +196,7 @@ class TestCephVolumeModule(object):
         fake_action = "create"
         fake_container_image = "docker.io/ceph/daemon:latest-luminous"
         expected_command_list = ['docker', 'run', '--rm', '--privileged', '--net=host', '--ipc=host',  # noqa E501
-                                 '--ulimit', 'nofile=1024:1024',
+                                 '--ulimit', 'nofile=1024:4096',
                                  '-v', '/run/lock/lvm:/run/lock/lvm:z',
                                  '-v', '/var/run/udev/:/var/run/udev/:z',
                                  '-v', '/dev:/dev', '-v', '/etc/ceph:/etc/ceph:z',  # noqa E501
@@ -245,7 +245,7 @@ class TestCephVolumeModule(object):
         fake_action = "prepare"
         fake_container_image = "docker.io/ceph/daemon:latest-luminous"
         expected_command_list = ['docker', 'run', '--rm', '--privileged', '--net=host', '--ipc=host',  # noqa E501
-                                 '--ulimit', 'nofile=1024:1024',
+                                 '--ulimit', 'nofile=1024:4096',
                                  '-v', '/run/lock/lvm:/run/lock/lvm:z',
                                  '-v', '/var/run/udev/:/var/run/udev/:z',
                                  '-v', '/dev:/dev', '-v', '/etc/ceph:/etc/ceph:z',  # noqa E501
@@ -295,7 +295,7 @@ class TestCephVolumeModule(object):
 
         fake_container_image = "docker.io/ceph/daemon:latest-luminous"
         expected_command_list = ['docker', 'run', '--rm', '--privileged', '--net=host', '--ipc=host',  # noqa E501
-                                 '--ulimit', 'nofile=1024:1024',
+                                 '--ulimit', 'nofile=1024:4096',
                                  '-v', '/run/lock/lvm:/run/lock/lvm:z',
                                  '-v', '/var/run/udev/:/var/run/udev/:z',
                                  '-v', '/dev:/dev', '-v', '/etc/ceph:/etc/ceph:z',  # noqa E501

diff --git a/roles/ceph-osd/tasks/start_osds.yml b/roles/ceph-osd/tasks/start_osds.yml
index 809e6c68c5e9c081bbe14a7c8ddf1dca4e09f875..7e7ce5022bd5d4c618241b83725a7ebc18be264a 100644 (file)
--- a/roles/ceph-osd/tasks/start_osds.yml
+++ b/roles/ceph-osd/tasks/start_osds.yml
@@ -29,7 +29,7 @@
 
 - name: set_fact docker_exec_start_osd
   set_fact:
-    docker_exec_start_osd: "{{ '{{ container_binary }} run --rm --ulimit nofile=1024:1024 --net=host --privileged=true -v /var/run/udev/:/var/run/udev/:z -v /run/lvm/:/run/lvm/ -v /etc/ceph:/etc/ceph:z -v /dev:/dev --entrypoint=ceph-volume ' + ceph_docker_registry + '/' + ceph_docker_image + ':' + ceph_docker_image_tag if containerized_deployment else 'ceph-volume' }}"
+    docker_exec_start_osd: "{{ '{{ container_binary }} run --rm --ulimit nofile=1024:4096 --net=host --privileged=true -v /var/run/udev/:/var/run/udev/:z -v /run/lvm/:/run/lvm/ -v /etc/ceph:/etc/ceph:z -v /dev:/dev --entrypoint=ceph-volume ' + ceph_docker_registry + '/' + ceph_docker_image + ':' + ceph_docker_image_tag if containerized_deployment else 'ceph-volume' }}"
 
 - name: collect osd ids
   command: "{{ docker_exec_start_osd }} lvm list --format json"

diff --git a/roles/ceph-osd/templates/ceph-osd-run.sh.j2 b/roles/ceph-osd/templates/ceph-osd-run.sh.j2
index 0c2cab34864c6022e39d94a2cd628b33720c1a7e..55e2b946c9a612c22feb25b46127fc7c58eff972 100644 (file)
--- a/roles/ceph-osd/templates/ceph-osd-run.sh.j2
+++ b/roles/ceph-osd/templates/ceph-osd-run.sh.j2
@@ -17,6 +17,7 @@ numactl \
   --privileged=true \
   --pid=host \
   --ipc=host \
+  --ulimit nofile=1024:4096 \
   {% if osd_objectstore == 'filestore' -%}
   --memory={{ ceph_osd_docker_memory_limit }} \
   {% endif -%}