Option to set TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES

Use "ceph_tcmalloc_max_total_thread_cache" to set the
TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES value inside /etc/default/ceph for
Debian installs, or /etc/sysconfig/ceph for Red Hat/CentOS installs.

By default this is set to 0, so the default package value will be used,
if specified this value will be changed to match the variable, and ceph
osd services will be restarted.

diff --git a/group_vars/all.yml.sample b/group_vars/all.yml.sample
index 462ce1a92c532232f6a7b75ecd7ce61e89e22d07..7e18182515c517263d92edd787f5a048a94d79fe 100644 (file)
--- a/group_vars/all.yml.sample
+++ b/group_vars/all.yml.sample
@@ -460,6 +460,11 @@ dummy:
 #  - { name: vm.swappiness, value: 10 }
 #  - { name: vm.min_free_kbytes, value: "{{ vm_min_free_kbytes }}" }
 
+# For Debian & Red Hat/CentOS installs set TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES
+# Set this to a byte value (e.g. 134217728)
+# A value of 0 will leave the package default.
+#ceph_tcmalloc_max_total_thread_cache: 0
+
 
 ##########
 # DOCKER #

diff --git a/group_vars/rhcs.yml.sample b/group_vars/rhcs.yml.sample
index 6753a6501a25a8c30d13d8d9af5dbafddee215c8..f457ff0de6be92ab4510408b1b228fa2a947128b 100644 (file)
--- a/group_vars/rhcs.yml.sample
+++ b/group_vars/rhcs.yml.sample
@@ -460,6 +460,11 @@ ceph_repository: rhcs
 #  - { name: vm.swappiness, value: 10 }
 #  - { name: vm.min_free_kbytes, value: "{{ vm_min_free_kbytes }}" }
 
+# For Debian & Red Hat/CentOS installs set TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES
+# Set this to a byte value (e.g. 134217728)
+# A value of 0 will leave the package default.
+#ceph_tcmalloc_max_total_thread_cache: 0
+
 
 ##########
 # DOCKER #

diff --git a/roles/ceph-common/tasks/configure_memory_allocator.yml b/roles/ceph-common/tasks/configure_memory_allocator.yml
new file mode 100644 (file)
index 0000000..7810e1b
--- /dev/null
+++ b/roles/ceph-common/tasks/configure_memory_allocator.yml
@@ -0,0 +1,25 @@
+---
+- name: configure TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES for debian
+  lineinfile:
+    dest: "{{ etc_default_ceph.stat.isdir | ternary('/etc/default/ceph/ceph', '/etc/default/ceph') }}"
+    insertafter: EOF
+    create: yes
+    regexp: "^TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES="
+    line: "TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES={{ ceph_tcmalloc_max_total_thread_cache }}"
+  when:
+    - ansible_os_family == 'Debian'
+    - etc_default_ceph.stat.exists
+  notify:
+    - restart ceph osds
+
+- name: configure TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES for redhat
+  lineinfile:
+    dest: "/etc/sysconfig/ceph"
+    insertafter: EOF
+    create: yes
+    regexp: "^TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES="
+    line: "TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES={{ ceph_tcmalloc_max_total_thread_cache }}"
+  when:
+    - ansible_os_family == 'RedHat'
+  notify:
+    - restart ceph osds

diff --git a/roles/ceph-common/tasks/main.yml b/roles/ceph-common/tasks/main.yml
index 20fa452c0c45725d5ee33dcb661a06d503e0fa67..4a2be0a7e67a62452300cd2df38e05006f78eafb 100644 (file)
--- a/roles/ceph-common/tasks/main.yml
+++ b/roles/ceph-common/tasks/main.yml
@@ -90,3 +90,10 @@
 
 - name: include configure_cluster_name.yml
   include: configure_cluster_name.yml
+
+- name: include configure_memory_allocator.yml
+  include: configure_memory_allocator.yml
+  when:
+    - (ceph_tcmalloc_max_total_thread_cache | int) > 0
+    - osd_objectstore == 'filestore'
+    - (ceph_origin == 'repository' or ceph_origin == 'distro')

diff --git a/roles/ceph-defaults/defaults/main.yml b/roles/ceph-defaults/defaults/main.yml
index 2c9a126471ca42c67c982db7810ceabbb22a0339..f47dd67c7964dfd9bcfcb5965f87d247182cb539 100644 (file)
--- a/roles/ceph-defaults/defaults/main.yml
+++ b/roles/ceph-defaults/defaults/main.yml
@@ -452,6 +452,11 @@ os_tuning_params:
   - { name: vm.swappiness, value: 10 }
   - { name: vm.min_free_kbytes, value: "{{ vm_min_free_kbytes }}" }
 
+# For Debian & Red Hat/CentOS installs set TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES
+# Set this to a byte value (e.g. 134217728)
+# A value of 0 will leave the package default.
+ceph_tcmalloc_max_total_thread_cache: 0
+
 
 ##########
 # DOCKER #

diff --git a/roles/ceph-osd/templates/ceph-osd-run.sh.j2 b/roles/ceph-osd/templates/ceph-osd-run.sh.j2
index 0ff07ede8628962e6ef44a342c1629a858701074..f9aae5b363da85bc0ed7b17a6b16f1ddd020be59 100644 (file)
--- a/roles/ceph-osd/templates/ceph-osd-run.sh.j2
+++ b/roles/ceph-osd/templates/ceph-osd-run.sh.j2
@@ -62,6 +62,9 @@ expose_partitions "$1"
   {% endif -%}
   -e CLUSTER={{ cluster }} \
   -e OSD_DEVICE=/dev/${1} \
+  {% if (ceph_tcmalloc_max_total_thread_cache | int) > 0 and osd_objectstore == 'filestore' -%}
+  -e TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES={{ ceph_tcmalloc_max_total_thread_cache }} \
+  {% endif -%}
   -e CEPH_DAEMON=OSD_CEPH_DISK_ACTIVATE \
   {{ ceph_osd_docker_extra_env }} \
   --name=ceph-osd-{{ ansible_hostname }}-${1} \