From 2fa151b9e8a7ab91e2be74832bebcd7b7e925830 Mon Sep 17 00:00:00 2001 From: =?utf8?q?S=C3=A9bastien=20Han?= Date: Wed, 6 Sep 2017 10:53:30 +0200 Subject: [PATCH] container: introduce resource limitation for containers MIME-Version: 1.0 Content-Type: text/plain; charset=utf8 Content-Transfer-Encoding: 8bit This can be controlled via 2 options: * ceph_$DAEMON_docker_memory_limit * ceph_$DAEMON_docker_cpu_limit All daemons default to 1GB for memory and 1 CPU by default. Recommendations from: https://access.redhat.com/documentation/en-us/red_hat_ceph_storage/2/html/red_hat_ceph_storage_hardware_guide/minimum_recommendations Signed-off-by: Sébastien Han --- group_vars/mdss.yml.sample | 7 ++ group_vars/mgrs.yml.sample | 7 ++ group_vars/mons.yml.sample | 9 ++- group_vars/osds.yml.sample | 7 ++ group_vars/rbd-mirrors.yml.sample | 10 +++ group_vars/rgws.yml.sample | 7 ++ roles/ceph-docker-common/tasks/main.yml | 9 +++ roles/ceph-mds/defaults/main.yml | 7 ++ roles/ceph-mds/templates/ceph-mds.service.j2 | 34 ++++++---- roles/ceph-mgr/defaults/main.yml | 7 ++ roles/ceph-mgr/templates/ceph-mgr.service.j2 | 32 +++++---- roles/ceph-mon/defaults/main.yml | 9 ++- roles/ceph-mon/templates/ceph-mon.service.j2 | 68 ++++++++++--------- roles/ceph-osd/defaults/main.yml | 7 ++ roles/ceph-osd/templates/ceph-osd-run.sh.j2 | 6 ++ roles/ceph-rbd-mirror/defaults/main.yml | 10 +++ .../templates/ceph-rbd-mirror.service.j2 | 29 +++++--- roles/ceph-rgw/defaults/main.yml | 7 ++ .../templates/ceph-radosgw.service.j2 | 6 ++ 19 files changed, 207 insertions(+), 71 deletions(-) diff --git a/group_vars/mdss.yml.sample b/group_vars/mdss.yml.sample index aced2742d..4dbf4e795 100644 --- a/group_vars/mdss.yml.sample +++ b/group_vars/mdss.yml.sample @@ -25,6 +25,13 @@ dummy: # DOCKER # ########## +# Resource limitation +# For the whole list of limits you can apply see: docs.docker.com/engine/admin/resource_constraints +# Default values are based from: https://access.redhat.com/documentation/en-us/red_hat_ceph_storage/2/html/red_hat_ceph_storage_hardware_guide/minimum_recommendations +# These options can be passed using the 'ceph_mds_docker_extra_env' variable. +#ceph_mds_docker_memory_limit: 1g +#ceph_mds_docker_cpu_limit: 1 + #ceph_mds_docker_extra_env: -e CLUSTER={{ cluster }} -e MDS_NAME={{ ansible_hostname }} #ceph_config_keys: [] # DON'T TOUCH ME diff --git a/group_vars/mgrs.yml.sample b/group_vars/mgrs.yml.sample index 0422383dc..d82d3b067 100644 --- a/group_vars/mgrs.yml.sample +++ b/group_vars/mgrs.yml.sample @@ -11,6 +11,13 @@ dummy: # DOCKER # ########## +# Resource limitation +# For the whole list of limits you can apply see: docs.docker.com/engine/admin/resource_constraints +# Default values are based from: https://access.redhat.com/documentation/en-us/red_hat_ceph_storage/2/html/red_hat_ceph_storage_hardware_guide/minimum_recommendations +# These options can be passed using the 'ceph_mgr_docker_extra_env' variable. +#ceph_mgr_docker_memory_limit: 1g +#ceph_mgr_docker_cpu_limit: 1 + #ceph_mgr_docker_extra_env: -e CLUSTER={{ cluster }} -e MGR_NAME={{ ansible_hostname }} #ceph_config_keys: [] # DON'T TOUCH ME diff --git a/group_vars/mons.yml.sample b/group_vars/mons.yml.sample index 7afc6417d..ccdaa177d 100644 --- a/group_vars/mons.yml.sample +++ b/group_vars/mons.yml.sample @@ -100,7 +100,7 @@ dummy: # The value for 'key' can be a pre-generated key, # e.g key: "AQDC2UxZH4yeLhAAgTaZb+4wDUlYOsr1OfZSpQ==" # By default, keys will be auto-generated. -# +# # To have have ansible setfacl the generated key, set the acls var like so: # acls: ["u:nova:r--", "u:cinder:r--", "u:glance:r--", "u:gnocchi:r--"] #openstack_keys: @@ -115,6 +115,13 @@ dummy: ########## #ceph_mon_docker_subnet: "{{ public_network }}"# subnet of the monitor_interface +# Resource limitation +# For the whole list of limits you can apply see: docs.docker.com/engine/admin/resource_constraints +# Default values are based from: https://access.redhat.com/documentation/en-us/red_hat_ceph_storage/2/html/red_hat_ceph_storage_hardware_guide/minimum_recommendations +# These options can be passed using the 'ceph_mon_docker_extra_env' variable. +#ceph_mon_docker_memory_limit: 1g +#ceph_mon_docker_cpu_limit: 1 + # ceph_mon_docker_extra_env: # # Use this variable to add extra env configuration to run your mon container. diff --git a/group_vars/osds.yml.sample b/group_vars/osds.yml.sample index 777a5615b..985fd12a8 100644 --- a/group_vars/osds.yml.sample +++ b/group_vars/osds.yml.sample @@ -226,6 +226,13 @@ dummy: #ceph_config_keys: [] # DON'T TOUCH ME +# Resource limitation +# For the whole list of limits you can apply see: docs.docker.com/engine/admin/resource_constraints +# Default values are based from: https://access.redhat.com/documentation/en-us/red_hat_ceph_storage/2/html/red_hat_ceph_storage_hardware_guide/minimum_recommendations +# These options can be passed using the 'ceph_osd_docker_extra_env' variable. +#ceph_osd_docker_memory_limit: 1g +#ceph_osd_docker_cpu_limit: 1 + # PREPARE DEVICE # # WARNING /!\ DMCRYPT scenario ONLY works with Docker version 1.12.5 and above diff --git a/group_vars/rbd-mirrors.yml.sample b/group_vars/rbd-mirrors.yml.sample index 994eb98b4..b680b054c 100644 --- a/group_vars/rbd-mirrors.yml.sample +++ b/group_vars/rbd-mirrors.yml.sample @@ -23,6 +23,7 @@ dummy: # NOTE: deprecated generic local user id for pre-Luminous releases #ceph_rbd_mirror_local_user: "admin" + ################# # CONFIGURATION # ################# @@ -41,10 +42,19 @@ dummy: # /etc/ceph/.client..keyring #ceph_rbd_mirror_remote_user: "" + ########## # DOCKER # ########## +# Resource limitation +# For the whole list of limits you can apply see: docs.docker.com/engine/admin/resource_constraints +# Default values are based from: https://access.redhat.com/documentation/en-us/red_hat_ceph_storage/2/html/red_hat_ceph_storage_hardware_guide/minimum_recommendations +# These options can be passed using the 'ceph_rbd_mirror_docker_extra_env' variable. +#ceph_rbd_mirror_docker_memory_limit: 1g +#ceph_rbd_mirror_docker_cpu_limit: 1 + +#ceph_rbd_mirror_docker_extra_env: #ceph_config_keys: [] # DON'T TOUCH ME diff --git a/group_vars/rgws.yml.sample b/group_vars/rgws.yml.sample index 2f9945349..164e51016 100644 --- a/group_vars/rgws.yml.sample +++ b/group_vars/rgws.yml.sample @@ -57,6 +57,13 @@ dummy: # DOCKER # ########## +# Resource limitation +# For the whole list of limits you can apply see: docs.docker.com/engine/admin/resource_constraints +# Default values are based from: https://access.redhat.com/documentation/en-us/red_hat_ceph_storage/2/html/red_hat_ceph_storage_hardware_guide/minimum_recommendations +# These options can be passed using the 'ceph_rgw_docker_extra_env' variable. +#ceph_rgw_docker_memory_limit: 1g +#ceph_rgw_docker_cpu_limit: 1 + #ceph_rgw_docker_extra_env: #ceph_config_keys: [] # DON'T TOUCH ME #rgw_config_keys: "/" # DON'T TOUCH ME diff --git a/roles/ceph-docker-common/tasks/main.yml b/roles/ceph-docker-common/tasks/main.yml index 9c16f5d62..73003ed87 100644 --- a/roles/ceph-docker-common/tasks/main.yml +++ b/roles/ceph-docker-common/tasks/main.yml @@ -22,6 +22,15 @@ monitor_name: "{{ ansible_fqdn }}" when: mon_use_fqdn +- name: get docker version + command: docker --version + changed_when: false + always_run: yes + register: docker_version + +- set_fact: + docker_version: "{{ docker_version.stdout.split(' ')[2] }}" + - name: check if a cluster is already running command: "docker ps -q -a --filter='ancestor={{ ceph_docker_image }}:{{ ceph_docker_image_tag }}'" register: ceph_health diff --git a/roles/ceph-mds/defaults/main.yml b/roles/ceph-mds/defaults/main.yml index bd87d1c6f..9d8fbbd43 100644 --- a/roles/ceph-mds/defaults/main.yml +++ b/roles/ceph-mds/defaults/main.yml @@ -17,6 +17,13 @@ copy_admin_key: false # DOCKER # ########## +# Resource limitation +# For the whole list of limits you can apply see: docs.docker.com/engine/admin/resource_constraints +# Default values are based from: https://access.redhat.com/documentation/en-us/red_hat_ceph_storage/2/html/red_hat_ceph_storage_hardware_guide/minimum_recommendations +# These options can be passed using the 'ceph_mds_docker_extra_env' variable. +ceph_mds_docker_memory_limit: 1g +ceph_mds_docker_cpu_limit: 1 + ceph_mds_docker_extra_env: -e CLUSTER={{ cluster }} -e MDS_NAME={{ ansible_hostname }} ceph_config_keys: [] # DON'T TOUCH ME diff --git a/roles/ceph-mds/templates/ceph-mds.service.j2 b/roles/ceph-mds/templates/ceph-mds.service.j2 index 6c901a718..0d99e5314 100644 --- a/roles/ceph-mds/templates/ceph-mds.service.j2 +++ b/roles/ceph-mds/templates/ceph-mds.service.j2 @@ -7,20 +7,26 @@ EnvironmentFile=-/etc/environment ExecStartPre=-/usr/bin/docker stop ceph-mds-{{ ansible_hostname }} ExecStartPre=-/usr/bin/docker rm ceph-mds-{{ ansible_hostname }} ExecStart=/usr/bin/docker run --rm --net=host \ - {% if not containerized_deployment_with_kv -%} - -v /var/lib/ceph:/var/lib/ceph \ - -v /etc/ceph:/etc/ceph \ - {% else -%} - -e KV_TYPE={{kv_type}} \ - -e KV_IP={{kv_endpoint}} \ - -e KV_PORT={{kv_port}} \ - {% endif -%} - -v /etc/localtime:/etc/localtime:ro \ - -e CEPH_DAEMON=MDS \ - -e CEPHFS_CREATE=1 \ - {{ ceph_mds_docker_extra_env }} \ - --name=ceph-mds-{{ ansible_hostname }} \ - {{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }} + --memory={{ ceph_mds_docker_memory_limit }} \ + {% if docker_version.split('.')[0] | version_compare('13', '>=') -%} + --cpus={{ ceph_mds_docker_cpu_limit }} \ + {% else -%} + --cpu-quota={{ ceph_mds_docker_cpu_limit * 100000 }} \ + {% endif -%} + {% if not containerized_deployment_with_kv -%} + -v /var/lib/ceph:/var/lib/ceph \ + -v /etc/ceph:/etc/ceph \ + {% else -%} + -e KV_TYPE={{kv_type}} \ + -e KV_IP={{kv_endpoint}} \ + -e KV_PORT={{kv_port}} \ + {% endif -%} + -v /etc/localtime:/etc/localtime:ro \ + -e CEPH_DAEMON=MDS \ + -e CEPHFS_CREATE=1 \ + {{ ceph_mds_docker_extra_env }} \ + --name=ceph-mds-{{ ansible_hostname }} \ + {{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }} ExecStopPost=-/usr/bin/docker stop ceph-mds-{{ ansible_hostname }} Restart=always RestartSec=10s diff --git a/roles/ceph-mgr/defaults/main.yml b/roles/ceph-mgr/defaults/main.yml index a0166404e..78a83fd35 100644 --- a/roles/ceph-mgr/defaults/main.yml +++ b/roles/ceph-mgr/defaults/main.yml @@ -3,6 +3,13 @@ # DOCKER # ########## +# Resource limitation +# For the whole list of limits you can apply see: docs.docker.com/engine/admin/resource_constraints +# Default values are based from: https://access.redhat.com/documentation/en-us/red_hat_ceph_storage/2/html/red_hat_ceph_storage_hardware_guide/minimum_recommendations +# These options can be passed using the 'ceph_mgr_docker_extra_env' variable. +ceph_mgr_docker_memory_limit: 1g +ceph_mgr_docker_cpu_limit: 1 + ceph_mgr_docker_extra_env: -e CLUSTER={{ cluster }} -e MGR_NAME={{ ansible_hostname }} ceph_config_keys: [] # DON'T TOUCH ME diff --git a/roles/ceph-mgr/templates/ceph-mgr.service.j2 b/roles/ceph-mgr/templates/ceph-mgr.service.j2 index 02cbd96eb..5aecf4ce3 100644 --- a/roles/ceph-mgr/templates/ceph-mgr.service.j2 +++ b/roles/ceph-mgr/templates/ceph-mgr.service.j2 @@ -7,19 +7,25 @@ EnvironmentFile=-/etc/environment ExecStartPre=-/usr/bin/docker stop ceph-mgr-{{ ansible_hostname }} ExecStartPre=-/usr/bin/docker rm ceph-mgr-{{ ansible_hostname }} ExecStart=/usr/bin/docker run --rm --net=host \ - {% if not containerized_deployment_with_kv -%} - -v /var/lib/ceph:/var/lib/ceph \ - -v /etc/ceph:/etc/ceph \ - {% else -%} - -e KV_TYPE={{kv_type}} \ - -e KV_IP={{kv_endpoint}} \ - -e KV_PORT={{kv_port}} \ - {% endif -%} - -v /etc/localtime:/etc/localtime:ro \ - -e CEPH_DAEMON=MGR \ - {{ ceph_mgr_docker_extra_env }} \ - --name=ceph-mgr-{{ ansible_hostname }} \ - {{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }} + --memory={{ ceph_mgr_docker_memory_limit }} \ + {% if docker_version.split('.')[0] | version_compare('13', '>=') -%} + --cpus={{ ceph_mgr_docker_cpu_limit }} \ + {% else -%} + --cpu-quota={{ ceph_mgr_docker_cpu_limit * 100000 }} \ + {% endif -%} + {% if not containerized_deployment_with_kv -%} + -v /var/lib/ceph:/var/lib/ceph \ + -v /etc/ceph:/etc/ceph \ + {% else -%} + -e KV_TYPE={{kv_type}} \ + -e KV_IP={{kv_endpoint}} \ + -e KV_PORT={{kv_port}} \ + {% endif -%} + -v /etc/localtime:/etc/localtime:ro \ + -e CEPH_DAEMON=MGR \ + {{ ceph_mgr_docker_extra_env }} \ + --name=ceph-mgr-{{ ansible_hostname }} \ + {{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }} ExecStopPost=-/usr/bin/docker stop ceph-mgr-{{ ansible_hostname }} Restart=always RestartSec=10s diff --git a/roles/ceph-mon/defaults/main.yml b/roles/ceph-mon/defaults/main.yml index 398ec6519..88e0d80fb 100644 --- a/roles/ceph-mon/defaults/main.yml +++ b/roles/ceph-mon/defaults/main.yml @@ -92,7 +92,7 @@ openstack_pools: # The value for 'key' can be a pre-generated key, # e.g key: "AQDC2UxZH4yeLhAAgTaZb+4wDUlYOsr1OfZSpQ==" # By default, keys will be auto-generated. -# +# # To have have ansible setfacl the generated key, set the acls var like so: # acls: ["u:nova:r--", "u:cinder:r--", "u:glance:r--", "u:gnocchi:r--"] openstack_keys: @@ -107,6 +107,13 @@ openstack_keys: ########## ceph_mon_docker_subnet: "{{ public_network }}"# subnet of the monitor_interface +# Resource limitation +# For the whole list of limits you can apply see: docs.docker.com/engine/admin/resource_constraints +# Default values are based from: https://access.redhat.com/documentation/en-us/red_hat_ceph_storage/2/html/red_hat_ceph_storage_hardware_guide/minimum_recommendations +# These options can be passed using the 'ceph_mon_docker_extra_env' variable. +ceph_mon_docker_memory_limit: 1g +ceph_mon_docker_cpu_limit: 1 + # ceph_mon_docker_extra_env: # # Use this variable to add extra env configuration to run your mon container. diff --git a/roles/ceph-mon/templates/ceph-mon.service.j2 b/roles/ceph-mon/templates/ceph-mon.service.j2 index a5f790f27..07c776a2c 100644 --- a/roles/ceph-mon/templates/ceph-mon.service.j2 +++ b/roles/ceph-mon/templates/ceph-mon.service.j2 @@ -7,37 +7,43 @@ EnvironmentFile=-/etc/environment ExecStartPre=-/usr/bin/docker rm ceph-mon-%i ExecStartPre=$(command -v mkdir) -p /etc/ceph /var/lib/ceph/mon ExecStart=/usr/bin/docker run --rm --name ceph-mon-%i --net=host \ - {% if not containerized_deployment_with_kv -%} - -v /var/lib/ceph:/var/lib/ceph \ - -v /etc/ceph:/etc/ceph \ - {% else -%} - -e KV_TYPE={{kv_type}} \ - -e KV_IP={{kv_endpoint}}\ - -e KV_PORT={{kv_port}} \ - {% endif -%} - -v /etc/localtime:/etc/localtime:ro \ - {% if mon_docker_privileged -%} - --privileged \ - {% endif -%} - {% if mon_docker_net_host -%} - --net=host \ - {% endif -%} - -e CEPH_DAEMON=MON \ - -e CEPH_PUBLIC_NETWORK={{ ceph_mon_docker_subnet }} \ - -e IP_VERSION={{ ip_version[-1:] }} \ - {% if monitor_address is defined and monitor_address != '0.0.0.0' %} - {% if ip_version == 'ipv4' -%} - -e MON_IP={{ monitor_address }} \ - {% elif ip_version == 'ipv6' -%} - -e MON_IP=[{{ monitor_address }}] \ - {% endif -%} - {% elif ip_version == 'ipv4' -%} - -e MON_IP={{ hostvars[inventory_hostname]['ansible_' + monitor_interface][ip_version]['address'] }} \ - {% elif ip_version =='ipv6' -%} - -e MON_IP=[{{ hostvars[inventory_hostname]['ansible_' + monitor_interface][ip_version][0]['address'] }}] \ - {% endif -%} - {{ ceph_mon_docker_extra_env }} \ - {{ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }} + --memory={{ ceph_mon_docker_memory_limit }} \ + {% if docker_version.split('.')[0] | version_compare('13', '>=') -%} + --cpus={{ ceph_mon_docker_cpu_limit }} \ + {% else -%} + --cpu-quota={{ ceph_mon_docker_cpu_limit * 100000 }} \ + {% endif -%} + {% if not containerized_deployment_with_kv -%} + -v /var/lib/ceph:/var/lib/ceph \ + -v /etc/ceph:/etc/ceph \ + {% else -%} + -e KV_TYPE={{kv_type}} \ + -e KV_IP={{kv_endpoint}}\ + -e KV_PORT={{kv_port}} \ + {% endif -%} + -v /etc/localtime:/etc/localtime:ro \ + {% if mon_docker_privileged -%} + --privileged \ + {% endif -%} + {% if mon_docker_net_host -%} + --net=host \ + {% endif -%} + -e CEPH_DAEMON=MON \ + -e CEPH_PUBLIC_NETWORK={{ ceph_mon_docker_subnet }} \ + -e IP_VERSION={{ ip_version[-1:] }} \ + {% if monitor_address is defined and monitor_address != '0.0.0.0' %} + {% if ip_version == 'ipv4' -%} + -e MON_IP={{ monitor_address }} \ + {% elif ip_version == 'ipv6' -%} + -e MON_IP=[{{ monitor_address }}] \ + {% endif -%} + {% elif ip_version == 'ipv4' -%} + -e MON_IP={{ hostvars[inventory_hostname]['ansible_' + monitor_interface][ip_version]['address'] }} \ + {% elif ip_version =='ipv6' -%} + -e MON_IP=[{{ hostvars[inventory_hostname]['ansible_' + monitor_interface][ip_version][0]['address'] }}] \ + {% endif -%} + {{ ceph_mon_docker_extra_env }} \ + {{ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }} ExecStopPost=-/usr/bin/docker stop ceph-mon-%i Restart=always RestartSec=10s diff --git a/roles/ceph-osd/defaults/main.yml b/roles/ceph-osd/defaults/main.yml index 2c6588541..f771dd7a4 100644 --- a/roles/ceph-osd/defaults/main.yml +++ b/roles/ceph-osd/defaults/main.yml @@ -218,6 +218,13 @@ lvm_volumes: [] ceph_config_keys: [] # DON'T TOUCH ME +# Resource limitation +# For the whole list of limits you can apply see: docs.docker.com/engine/admin/resource_constraints +# Default values are based from: https://access.redhat.com/documentation/en-us/red_hat_ceph_storage/2/html/red_hat_ceph_storage_hardware_guide/minimum_recommendations +# These options can be passed using the 'ceph_osd_docker_extra_env' variable. +ceph_osd_docker_memory_limit: 1g +ceph_osd_docker_cpu_limit: 1 + # PREPARE DEVICE # # WARNING /!\ DMCRYPT scenario ONLY works with Docker version 1.12.5 and above diff --git a/roles/ceph-osd/templates/ceph-osd-run.sh.j2 b/roles/ceph-osd/templates/ceph-osd-run.sh.j2 index 1ac606f53..acc852ff3 100644 --- a/roles/ceph-osd/templates/ceph-osd-run.sh.j2 +++ b/roles/ceph-osd/templates/ceph-osd-run.sh.j2 @@ -54,6 +54,12 @@ expose_partitions $1 --net=host \ --privileged=true \ --pid=host \ + --memory={{ ceph_osd_docker_memory_limit }} \ + {% if docker_version.split('.')[0] | version_compare('13', '>=') -%} + --cpus={{ ceph_osd_docker_cpu_limit }} \ + {% else -%} + --cpu-quota={{ ceph_osd_docker_cpu_limit * 100000 }} \ + {% endif -%} -v /dev:/dev \ -v /etc/localtime:/etc/localtime:ro \ -v /var/lib/ceph:/var/lib/ceph \ diff --git a/roles/ceph-rbd-mirror/defaults/main.yml b/roles/ceph-rbd-mirror/defaults/main.yml index b343a2058..4c1ecb7d5 100644 --- a/roles/ceph-rbd-mirror/defaults/main.yml +++ b/roles/ceph-rbd-mirror/defaults/main.yml @@ -15,6 +15,7 @@ copy_admin_key: false # NOTE: deprecated generic local user id for pre-Luminous releases ceph_rbd_mirror_local_user: "admin" + ################# # CONFIGURATION # ################# @@ -33,10 +34,19 @@ ceph_rbd_mirror_remote_cluster: "" # /etc/ceph/.client..keyring ceph_rbd_mirror_remote_user: "" + ########## # DOCKER # ########## +# Resource limitation +# For the whole list of limits you can apply see: docs.docker.com/engine/admin/resource_constraints +# Default values are based from: https://access.redhat.com/documentation/en-us/red_hat_ceph_storage/2/html/red_hat_ceph_storage_hardware_guide/minimum_recommendations +# These options can be passed using the 'ceph_rbd_mirror_docker_extra_env' variable. +ceph_rbd_mirror_docker_memory_limit: 1g +ceph_rbd_mirror_docker_cpu_limit: 1 + +ceph_rbd_mirror_docker_extra_env: ceph_config_keys: [] # DON'T TOUCH ME diff --git a/roles/ceph-rbd-mirror/templates/ceph-rbd-mirror.service.j2 b/roles/ceph-rbd-mirror/templates/ceph-rbd-mirror.service.j2 index 4b6537ba1..446685fcc 100644 --- a/roles/ceph-rbd-mirror/templates/ceph-rbd-mirror.service.j2 +++ b/roles/ceph-rbd-mirror/templates/ceph-rbd-mirror.service.j2 @@ -7,17 +7,24 @@ EnvironmentFile=-/etc/environment ExecStartPre=-/usr/bin/docker stop ceph-rbd-mirror-{{ ansible_hostname }} ExecStartPre=-/usr/bin/docker rm ceph-rbd-mirror-{{ ansible_hostname }} ExecStart=/usr/bin/docker run --rm --net=host \ - {% if not containerized_deployment_with_kv -%} - -v /etc/ceph:/etc/ceph \ - {% else -%} - -e KV_TYPE={{kv_type}} \ - -e KV_IP={{kv_endpoint}} \ - -e KV_PORT={{kv_port}} \ - {% endif -%} - -v /etc/localtime:/etc/localtime:ro \ - -e CEPH_DAEMON=RBD_MIRROR \ - --name=ceph-rbd-mirror-{{ ansible_hostname }} \ - {{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }} + --memory={{ ceph_rbd_mirorr_docker_memory_limit }} \ + {% if docker_version.split('.')[0] | version_compare('13', '>=') -%} + --cpus={{ ceph_rbd_mirror_docker_cpu_limit }} \ + {% else -%} + --cpu-quota={{ ceph_rbd_mirorr_docker_cpu_limit * 100000 }} \ + {% endif -%} + {% if not containerized_deployment_with_kv -%} + -v /etc/ceph:/etc/ceph \ + {% else -%} + -e KV_TYPE={{kv_type}} \ + -e KV_IP={{kv_endpoint}} \ + -e KV_PORT={{kv_port}} \ + {% endif -%} + -v /etc/localtime:/etc/localtime:ro \ + -e CEPH_DAEMON=RBD_MIRROR \ + --name=ceph-rbd-mirror-{{ ansible_hostname }} \ + {{ ceph_rbd_mirror_docker_extra_env }} \ + {{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }} ExecStopPost=-/usr/bin/docker stop ceph-rbd-mirror-{{ ansible_hostname }} Restart=always RestartSec=10s diff --git a/roles/ceph-rgw/defaults/main.yml b/roles/ceph-rgw/defaults/main.yml index 4487865dc..7f0a8aa4c 100644 --- a/roles/ceph-rgw/defaults/main.yml +++ b/roles/ceph-rgw/defaults/main.yml @@ -49,6 +49,13 @@ rgw_pull_proto: "http" # DOCKER # ########## +# Resource limitation +# For the whole list of limits you can apply see: docs.docker.com/engine/admin/resource_constraints +# Default values are based from: https://access.redhat.com/documentation/en-us/red_hat_ceph_storage/2/html/red_hat_ceph_storage_hardware_guide/minimum_recommendations +# These options can be passed using the 'ceph_rgw_docker_extra_env' variable. +ceph_rgw_docker_memory_limit: 1g +ceph_rgw_docker_cpu_limit: 1 + ceph_rgw_docker_extra_env: ceph_config_keys: [] # DON'T TOUCH ME rgw_config_keys: "/" # DON'T TOUCH ME diff --git a/roles/ceph-rgw/templates/ceph-radosgw.service.j2 b/roles/ceph-rgw/templates/ceph-radosgw.service.j2 index afe7c72c6..25f8b5b0b 100644 --- a/roles/ceph-rgw/templates/ceph-radosgw.service.j2 +++ b/roles/ceph-rgw/templates/ceph-radosgw.service.j2 @@ -7,6 +7,12 @@ EnvironmentFile=-/etc/environment ExecStartPre=-/usr/bin/docker stop ceph-rgw-{{ ansible_hostname }} ExecStartPre=-/usr/bin/docker rm ceph-rgw-{{ ansible_hostname }} ExecStart=/usr/bin/docker run --rm --net=host \ + --memory={{ ceph_rgw_docker_memory_limit }} \ + {% if docker_version.split('.')[0] | version_compare('13', '>=') -%} + --cpus={{ ceph_rgw_docker_cpu_limit }} \ + {% else -%} + --cpu-quota={{ ceph_rgw_docker_cpu_limit * 100000 }} \ + {% endif -%} {% if not containerized_deployment_with_kv -%} -v /var/lib/ceph:/var/lib/ceph \ -v /etc/ceph:/etc/ceph \ -- 2.39.5