Add flags for OSD 'docker run --cpuset-{cpus,mems}'

Add the variables ceph_osd_docker_cpuset_cpus and
ceph_osd_docker_cpuset_mems, so that a user may specify
the CPUs and memory nodes of NUMA systems on which OSD
containers are run.

Provides a example in osds.yaml.sample to guide user
based on sample `lscpu` output since cpuset-mems refers
to the memory by NUMA node only while cpuset-cpus can
refer to individual vCPUs within a NUMA node.

diff --git a/group_vars/osds.yml.sample b/group_vars/osds.yml.sample
index eabdb51ecef5bb1ef65b1825d1375cdf2e161a03..8ae7b606552c6ffc6b1ddd948b642de1abca9b71 100644 (file)
--- a/group_vars/osds.yml.sample
+++ b/group_vars/osds.yml.sample
@@ -263,6 +263,14 @@ dummy:
 #ceph_osd_docker_memory_limit: 1g
 #ceph_osd_docker_cpu_limit: 1
 
+# The next two variables are undefined, and thus, unused by default.
+# If `lscpu | grep NUMA` returned the following:
+#  NUMA node0 CPU(s):     0,2,4,6,8,10,12,14,16
+#  NUMA node1 CPU(s):     1,3,5,7,9,11,13,15,17
+# then, the following would run the OSD on the first NUMA node only.
+#ceph_osd_docker_cpuset_cpus: "0,2,4,6,8,10,12,14,16"
+#ceph_osd_docker_cpuset_mems: "0"
+
 # PREPARE DEVICE
 #
 # WARNING /!\ DMCRYPT scenario ONLY works with Docker version 1.12.5 and above

diff --git a/roles/ceph-osd/defaults/main.yml b/roles/ceph-osd/defaults/main.yml
index 7d3c81dee3f88e79c2024854b510918b80a8b94a..8b43b438fc3dd8af85f446f5e13dd61380e91254 100644 (file)
--- a/roles/ceph-osd/defaults/main.yml
+++ b/roles/ceph-osd/defaults/main.yml
@@ -255,6 +255,14 @@ ceph_config_keys: [] # DON'T TOUCH ME
 ceph_osd_docker_memory_limit: 1g
 ceph_osd_docker_cpu_limit: 1
 
+# The next two variables are undefined, and thus, unused by default.
+# If `lscpu | grep NUMA` returned the following:
+#  NUMA node0 CPU(s):     0,2,4,6,8,10,12,14,16
+#  NUMA node1 CPU(s):     1,3,5,7,9,11,13,15,17
+# then, the following would run the OSD on the first NUMA node only.
+#ceph_osd_docker_cpuset_cpus: "0,2,4,6,8,10,12,14,16"
+#ceph_osd_docker_cpuset_mems: "0"
+
 # PREPARE DEVICE
 #
 # WARNING /!\ DMCRYPT scenario ONLY works with Docker version 1.12.5 and above

diff --git a/roles/ceph-osd/templates/ceph-osd-run.sh.j2 b/roles/ceph-osd/templates/ceph-osd-run.sh.j2
index 11f8df85268a5ca228bb8450a8737e2dff351adb..eea22a8b4c7b4073eb83c521ea43ced76e4b1c0d 100644 (file)
--- a/roles/ceph-osd/templates/ceph-osd-run.sh.j2
+++ b/roles/ceph-osd/templates/ceph-osd-run.sh.j2
@@ -54,6 +54,12 @@ expose_partitions "$1"
   {% else -%}
   --cpu-quota={{ ceph_osd_docker_cpu_limit * 100000 }} \
   {% endif -%}
+  {% if ceph_osd_docker_cpuset_cpus is defined -%}
+  --cpuset-cpus='{{ ceph_osd_docker_cpuset_cpus }}' \
+  {% endif -%}
+  {% if ceph_osd_docker_cpuset_mems is defined -%}
+  --cpuset-mems='{{ ceph_osd_docker_cpuset_mems }}' \
+  {% endif -%}
   -v /dev:/dev \
   -v /etc/localtime:/etc/localtime:ro \
   -v /var/lib/ceph:/var/lib/ceph \