Option to set TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES

Use "ceph_tcmalloc_max_total_thread_cache" to set the
TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES value inside /etc/default/ceph for
Debian installs, or /etc/sysconfig/ceph for Red Hat/CentOS installs.

By default this is set to 0, so the default package value will be used,
if specified this value will be changed to match the variable, and ceph
osd services will be restarted.

(cherry picked from commit 7f6c39102d4fc1c9ec987fe3dd06693a94ad56fb)
Signed-off-by: Sébastien Han <seb@redhat.com>

diff --git a/group_vars/all.yml.sample b/group_vars/all.yml.sample
index 462ce1a92c532232f6a7b75ecd7ce61e89e22d07..7e18182515c517263d92edd787f5a048a94d79fe 100644 (file)
--- a/group_vars/all.yml.sample
+++ b/group_vars/all.yml.sample
@@ -460,6 +460,11 @@ dummy:
 #  - { name: vm.swappiness, value: 10 }
 #  - { name: vm.min_free_kbytes, value: "{{ vm_min_free_kbytes }}" }
 
+# For Debian & Red Hat/CentOS installs set TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES
+# Set this to a byte value (e.g. 134217728)
+# A value of 0 will leave the package default.
+#ceph_tcmalloc_max_total_thread_cache: 0
+
 
 ##########
 # DOCKER #

diff --git a/group_vars/rhcs.yml.sample b/group_vars/rhcs.yml.sample
index 6753a6501a25a8c30d13d8d9af5dbafddee215c8..f457ff0de6be92ab4510408b1b228fa2a947128b 100644 (file)
--- a/group_vars/rhcs.yml.sample
+++ b/group_vars/rhcs.yml.sample
@@ -460,6 +460,11 @@ ceph_repository: rhcs
 #  - { name: vm.swappiness, value: 10 }
 #  - { name: vm.min_free_kbytes, value: "{{ vm_min_free_kbytes }}" }
 
+# For Debian & Red Hat/CentOS installs set TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES
+# Set this to a byte value (e.g. 134217728)
+# A value of 0 will leave the package default.
+#ceph_tcmalloc_max_total_thread_cache: 0
+
 
 ##########
 # DOCKER #

diff --git a/roles/ceph-common/tasks/configure_memory_allocator.yml b/roles/ceph-common/tasks/configure_memory_allocator.yml
new file mode 100644 (file)
index 0000000..7810e1b
--- /dev/null
+++ b/roles/ceph-common/tasks/configure_memory_allocator.yml
@@ -0,0 +1,25 @@
+---
+- name: configure TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES for debian
+  lineinfile:
+    dest: "{{ etc_default_ceph.stat.isdir | ternary('/etc/default/ceph/ceph', '/etc/default/ceph') }}"
+    insertafter: EOF
+    create: yes
+    regexp: "^TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES="
+    line: "TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES={{ ceph_tcmalloc_max_total_thread_cache }}"
+  when:
+    - ansible_os_family == 'Debian'
+    - etc_default_ceph.stat.exists
+  notify:
+    - restart ceph osds
+
+- name: configure TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES for redhat
+  lineinfile:
+    dest: "/etc/sysconfig/ceph"
+    insertafter: EOF
+    create: yes
+    regexp: "^TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES="
+    line: "TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES={{ ceph_tcmalloc_max_total_thread_cache }}"
+  when:
+    - ansible_os_family == 'RedHat'
+  notify:
+    - restart ceph osds

diff --git a/roles/ceph-common/tasks/main.yml b/roles/ceph-common/tasks/main.yml
index 7658530471e3c64ee131b2a49ddde46214613606..6f3c29f7c1ca697e0696acdc41185ac206fb800e 100644 (file)
--- a/roles/ceph-common/tasks/main.yml
+++ b/roles/ceph-common/tasks/main.yml
@@ -97,3 +97,10 @@
 
 - name: include configure_cluster_name.yml
   include: configure_cluster_name.yml
+
+- name: include configure_memory_allocator.yml
+  include: configure_memory_allocator.yml
+  when:
+    - (ceph_tcmalloc_max_total_thread_cache | int) > 0
+    - osd_objectstore == 'filestore'
+    - (ceph_origin == 'repository' or ceph_origin == 'distro')

diff --git a/roles/ceph-defaults/defaults/main.yml b/roles/ceph-defaults/defaults/main.yml
index 2c9a126471ca42c67c982db7810ceabbb22a0339..f47dd67c7964dfd9bcfcb5965f87d247182cb539 100644 (file)
--- a/roles/ceph-defaults/defaults/main.yml
+++ b/roles/ceph-defaults/defaults/main.yml
@@ -452,6 +452,11 @@ os_tuning_params:
   - { name: vm.swappiness, value: 10 }
   - { name: vm.min_free_kbytes, value: "{{ vm_min_free_kbytes }}" }
 
+# For Debian & Red Hat/CentOS installs set TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES
+# Set this to a byte value (e.g. 134217728)
+# A value of 0 will leave the package default.
+ceph_tcmalloc_max_total_thread_cache: 0
+
 
 ##########
 # DOCKER #

diff --git a/roles/ceph-osd/templates/ceph-osd-run.sh.j2 b/roles/ceph-osd/templates/ceph-osd-run.sh.j2
index bd2f5efea171a34c272c757837680475adf75bb8..867234c6d56374df19ff2880471a3bab2df19664 100644 (file)
--- a/roles/ceph-osd/templates/ceph-osd-run.sh.j2
+++ b/roles/ceph-osd/templates/ceph-osd-run.sh.j2
@@ -73,6 +73,9 @@ expose_partitions "$1"
   {% endif -%}
   -e CLUSTER={{ cluster }} \
   -e OSD_DEVICE=/dev/${1} \
+  {% if (ceph_tcmalloc_max_total_thread_cache | int) > 0 and osd_objectstore == 'filestore' -%}
+  -e TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES={{ ceph_tcmalloc_max_total_thread_cache }} \
+  {% endif -%}
   -e CEPH_DAEMON=OSD_CEPH_DISK_ACTIVATE \
   {{ ceph_osd_docker_extra_env }} \
   --name=ceph-osd-{{ ansible_hostname }}-${1} \