rgw: add cpuset support

1/ The OSD already supports cpuset to be used for containerized deployments
through the use of the ceph_osd_docker_cpuset_cpus variable. This adds similar
support to the RGW service for containerized deployments by setting a new
variable named ceph_rgw_docker_cpuset_cpus. Like the OSD, there are times where
using distinct cores has advantages over using the CFS in kernel scheduler.

ceph_rgw_docker_cpuset_cpus accepts a comma delimited set of CPU ids

2/ Add support for specifying --cpuset-mem variable to restrict the cgroup's memory
allocations to a particular numa node, which should typically correspond with
the cpu ids of that numa node that were provided with --cpuset-cpus. To ensure
the correct cpu ids are used one can run `numactl --hardware`  to list the nodes
and which cpu ids correspond to each.

Signed-off-by: Kyle Bader <kbader@redhat.com>
Co-authored-by: Guillaume Abrioux <gabrioux@redhat.com>
(cherry picked from commit 0bee90b20195f0764c3d9001fb06d0999b5fc6cf)

diff --git a/group_vars/rgws.yml.sample b/group_vars/rgws.yml.sample
index 1200c0593c3f88c87b732bffb263f4c974810d64..166f19f345773a5ba6fc86a67707e8c02fb278c8 100644 (file)
--- a/group_vars/rgws.yml.sample
+++ b/group_vars/rgws.yml.sample
@@ -59,6 +59,8 @@ dummy:
 # These options can be passed using the 'ceph_rgw_docker_extra_env' variable.
 #ceph_rgw_docker_memory_limit: "{{ ansible_memtotal_mb }}m"
 #ceph_rgw_docker_cpu_limit: 8
+#ceph_rgw_docker_cpuset_cpus: "0,2,4,6,8,10,12,14,16"
+#ceph_rgw_docker_cpuset_mems: "0"
 
 #ceph_rgw_docker_extra_env:
 #ceph_config_keys: [] # DON'T TOUCH ME

diff --git a/roles/ceph-rgw/defaults/main.yml b/roles/ceph-rgw/defaults/main.yml
index f77c1f6ab8159ac1b0142205065002056cac4d65..aa69905755b0c261d77b1ca0896cc5cfd39dd550 100644 (file)
--- a/roles/ceph-rgw/defaults/main.yml
+++ b/roles/ceph-rgw/defaults/main.yml
@@ -51,6 +51,8 @@ copy_admin_key: false
 # These options can be passed using the 'ceph_rgw_docker_extra_env' variable.
 ceph_rgw_docker_memory_limit: "{{ ansible_memtotal_mb }}m"
 ceph_rgw_docker_cpu_limit: 8
+#ceph_rgw_docker_cpuset_cpus: "0,2,4,6,8,10,12,14,16"
+#ceph_rgw_docker_cpuset_mems: "0"
 
 ceph_rgw_docker_extra_env:
 ceph_config_keys: [] # DON'T TOUCH ME

diff --git a/roles/ceph-rgw/templates/ceph-radosgw.service.j2 b/roles/ceph-rgw/templates/ceph-radosgw.service.j2
index a5ac9497389a5da159c3cb1da9b2c85ff90bee8a..3ba88a536cd2a1fa5334ec73617b9cb14f4ca928 100644 (file)
--- a/roles/ceph-rgw/templates/ceph-radosgw.service.j2
+++ b/roles/ceph-rgw/templates/ceph-radosgw.service.j2
@@ -16,6 +16,12 @@ ExecStart=/usr/bin/{{ container_binary }} run --rm --net=host \
   {% else -%}
   --cpu-quota={{ cpu_limit * 100000 }} \
   {% endif -%}
+  {% if ceph_rgw_docker_cpuset_cpus is defined -%}
+  --cpuset-cpus="{{ ceph_rgw_docker_cpuset_cpus }}" \
+  {% endif -%}
+  {% if ceph_rgw_docker_cpuset_mems is defined -%}
+  --cpuset-mems="{{ ceph_rgw_docker_cpuset_mems }}" \
+  {% endif -%}
   -v /var/lib/ceph:/var/lib/ceph:z \
   -v /etc/ceph:/etc/ceph:z \
   -v /var/run/ceph:/var/run/ceph:z \