From: Andy McCrae <andy.mccrae@gmail.com>
Date: Mon, 23 Oct 2017 13:57:24 +0000 (+0100)
Subject: Option to set TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES
X-Git-Tag: v3.0.5^2
X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=refs%2Fpull%2F2108%2Fhead;p=ceph-ansible.git

Option to set TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES

Use "ceph_tcmalloc_max_total_thread_cache" to set the
TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES value inside /etc/default/ceph for
Debian installs, or /etc/sysconfig/ceph for Red Hat/CentOS installs.

By default this is set to 0, so the default package value will be used,
if specified this value will be changed to match the variable, and ceph
osd services will be restarted.

(cherry picked from commit 7f6c39102d4fc1c9ec987fe3dd06693a94ad56fb)
Signed-off-by: Sébastien Han <seb@redhat.com>
---

diff --git a/group_vars/all.yml.sample b/group_vars/all.yml.sample
index 462ce1a92..7e1818251 100644
--- a/group_vars/all.yml.sample
+++ b/group_vars/all.yml.sample
@@ -460,6 +460,11 @@ dummy:
 #  - { name: vm.swappiness, value: 10 }
 #  - { name: vm.min_free_kbytes, value: "{{ vm_min_free_kbytes }}" }
 
+# For Debian & Red Hat/CentOS installs set TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES
+# Set this to a byte value (e.g. 134217728)
+# A value of 0 will leave the package default.
+#ceph_tcmalloc_max_total_thread_cache: 0
+
 
 ##########
 # DOCKER #
diff --git a/group_vars/rhcs.yml.sample b/group_vars/rhcs.yml.sample
index 6753a6501..f457ff0de 100644
--- a/group_vars/rhcs.yml.sample
+++ b/group_vars/rhcs.yml.sample
@@ -460,6 +460,11 @@ ceph_repository: rhcs
 #  - { name: vm.swappiness, value: 10 }
 #  - { name: vm.min_free_kbytes, value: "{{ vm_min_free_kbytes }}" }
 
+# For Debian & Red Hat/CentOS installs set TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES
+# Set this to a byte value (e.g. 134217728)
+# A value of 0 will leave the package default.
+#ceph_tcmalloc_max_total_thread_cache: 0
+
 
 ##########
 # DOCKER #
diff --git a/roles/ceph-common/tasks/configure_memory_allocator.yml b/roles/ceph-common/tasks/configure_memory_allocator.yml
new file mode 100644
index 000000000..7810e1b34
--- /dev/null
+++ b/roles/ceph-common/tasks/configure_memory_allocator.yml
@@ -0,0 +1,25 @@
+---
+- name: configure TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES for debian
+  lineinfile:
+    dest: "{{ etc_default_ceph.stat.isdir | ternary('/etc/default/ceph/ceph', '/etc/default/ceph') }}"
+    insertafter: EOF
+    create: yes
+    regexp: "^TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES="
+    line: "TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES={{ ceph_tcmalloc_max_total_thread_cache }}"
+  when:
+    - ansible_os_family == 'Debian'
+    - etc_default_ceph.stat.exists
+  notify:
+    - restart ceph osds
+
+- name: configure TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES for redhat
+  lineinfile:
+    dest: "/etc/sysconfig/ceph"
+    insertafter: EOF
+    create: yes
+    regexp: "^TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES="
+    line: "TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES={{ ceph_tcmalloc_max_total_thread_cache }}"
+  when:
+    - ansible_os_family == 'RedHat'
+  notify:
+    - restart ceph osds
diff --git a/roles/ceph-common/tasks/main.yml b/roles/ceph-common/tasks/main.yml
index 765853047..6f3c29f7c 100644
--- a/roles/ceph-common/tasks/main.yml
+++ b/roles/ceph-common/tasks/main.yml
@@ -97,3 +97,10 @@
 
 - name: include configure_cluster_name.yml
   include: configure_cluster_name.yml
+
+- name: include configure_memory_allocator.yml
+  include: configure_memory_allocator.yml
+  when:
+    - (ceph_tcmalloc_max_total_thread_cache | int) > 0
+    - osd_objectstore == 'filestore'
+    - (ceph_origin == 'repository' or ceph_origin == 'distro')
diff --git a/roles/ceph-defaults/defaults/main.yml b/roles/ceph-defaults/defaults/main.yml
index 2c9a12647..f47dd67c7 100644
--- a/roles/ceph-defaults/defaults/main.yml
+++ b/roles/ceph-defaults/defaults/main.yml
@@ -452,6 +452,11 @@ os_tuning_params:
   - { name: vm.swappiness, value: 10 }
   - { name: vm.min_free_kbytes, value: "{{ vm_min_free_kbytes }}" }
 
+# For Debian & Red Hat/CentOS installs set TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES
+# Set this to a byte value (e.g. 134217728)
+# A value of 0 will leave the package default.
+ceph_tcmalloc_max_total_thread_cache: 0
+
 
 ##########
 # DOCKER #
diff --git a/roles/ceph-osd/templates/ceph-osd-run.sh.j2 b/roles/ceph-osd/templates/ceph-osd-run.sh.j2
index bd2f5efea..867234c6d 100644
--- a/roles/ceph-osd/templates/ceph-osd-run.sh.j2
+++ b/roles/ceph-osd/templates/ceph-osd-run.sh.j2
@@ -73,6 +73,9 @@ expose_partitions "$1"
   {% endif -%}
   -e CLUSTER={{ cluster }} \
   -e OSD_DEVICE=/dev/${1} \
+  {% if (ceph_tcmalloc_max_total_thread_cache | int) > 0 and osd_objectstore == 'filestore' -%}
+  -e TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES={{ ceph_tcmalloc_max_total_thread_cache }} \
+  {% endif -%}
   -e CEPH_DAEMON=OSD_CEPH_DISK_ACTIVATE \
   {{ ceph_osd_docker_extra_env }} \
   --name=ceph-osd-{{ ansible_hostname }}-${1} \