From 7f6c39102d4fc1c9ec987fe3dd06693a94ad56fb Mon Sep 17 00:00:00 2001 From: Andy McCrae Date: Mon, 23 Oct 2017 14:57:24 +0100 Subject: [PATCH] Option to set TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES Use "ceph_tcmalloc_max_total_thread_cache" to set the TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES value inside /etc/default/ceph for Debian installs, or /etc/sysconfig/ceph for Red Hat/CentOS installs. By default this is set to 0, so the default package value will be used, if specified this value will be changed to match the variable, and ceph osd services will be restarted. --- group_vars/all.yml.sample | 5 ++++ group_vars/rhcs.yml.sample | 5 ++++ .../tasks/configure_memory_allocator.yml | 25 +++++++++++++++++++ roles/ceph-common/tasks/main.yml | 7 ++++++ roles/ceph-defaults/defaults/main.yml | 5 ++++ roles/ceph-osd/templates/ceph-osd-run.sh.j2 | 3 +++ 6 files changed, 50 insertions(+) create mode 100644 roles/ceph-common/tasks/configure_memory_allocator.yml diff --git a/group_vars/all.yml.sample b/group_vars/all.yml.sample index 462ce1a92..7e1818251 100644 --- a/group_vars/all.yml.sample +++ b/group_vars/all.yml.sample @@ -460,6 +460,11 @@ dummy: # - { name: vm.swappiness, value: 10 } # - { name: vm.min_free_kbytes, value: "{{ vm_min_free_kbytes }}" } +# For Debian & Red Hat/CentOS installs set TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES +# Set this to a byte value (e.g. 134217728) +# A value of 0 will leave the package default. +#ceph_tcmalloc_max_total_thread_cache: 0 + ########## # DOCKER # diff --git a/group_vars/rhcs.yml.sample b/group_vars/rhcs.yml.sample index 6753a6501..f457ff0de 100644 --- a/group_vars/rhcs.yml.sample +++ b/group_vars/rhcs.yml.sample @@ -460,6 +460,11 @@ ceph_repository: rhcs # - { name: vm.swappiness, value: 10 } # - { name: vm.min_free_kbytes, value: "{{ vm_min_free_kbytes }}" } +# For Debian & Red Hat/CentOS installs set TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES +# Set this to a byte value (e.g. 134217728) +# A value of 0 will leave the package default. +#ceph_tcmalloc_max_total_thread_cache: 0 + ########## # DOCKER # diff --git a/roles/ceph-common/tasks/configure_memory_allocator.yml b/roles/ceph-common/tasks/configure_memory_allocator.yml new file mode 100644 index 000000000..7810e1b34 --- /dev/null +++ b/roles/ceph-common/tasks/configure_memory_allocator.yml @@ -0,0 +1,25 @@ +--- +- name: configure TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES for debian + lineinfile: + dest: "{{ etc_default_ceph.stat.isdir | ternary('/etc/default/ceph/ceph', '/etc/default/ceph') }}" + insertafter: EOF + create: yes + regexp: "^TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES=" + line: "TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES={{ ceph_tcmalloc_max_total_thread_cache }}" + when: + - ansible_os_family == 'Debian' + - etc_default_ceph.stat.exists + notify: + - restart ceph osds + +- name: configure TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES for redhat + lineinfile: + dest: "/etc/sysconfig/ceph" + insertafter: EOF + create: yes + regexp: "^TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES=" + line: "TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES={{ ceph_tcmalloc_max_total_thread_cache }}" + when: + - ansible_os_family == 'RedHat' + notify: + - restart ceph osds diff --git a/roles/ceph-common/tasks/main.yml b/roles/ceph-common/tasks/main.yml index 20fa452c0..4a2be0a7e 100644 --- a/roles/ceph-common/tasks/main.yml +++ b/roles/ceph-common/tasks/main.yml @@ -90,3 +90,10 @@ - name: include configure_cluster_name.yml include: configure_cluster_name.yml + +- name: include configure_memory_allocator.yml + include: configure_memory_allocator.yml + when: + - (ceph_tcmalloc_max_total_thread_cache | int) > 0 + - osd_objectstore == 'filestore' + - (ceph_origin == 'repository' or ceph_origin == 'distro') diff --git a/roles/ceph-defaults/defaults/main.yml b/roles/ceph-defaults/defaults/main.yml index 2c9a12647..f47dd67c7 100644 --- a/roles/ceph-defaults/defaults/main.yml +++ b/roles/ceph-defaults/defaults/main.yml @@ -452,6 +452,11 @@ os_tuning_params: - { name: vm.swappiness, value: 10 } - { name: vm.min_free_kbytes, value: "{{ vm_min_free_kbytes }}" } +# For Debian & Red Hat/CentOS installs set TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES +# Set this to a byte value (e.g. 134217728) +# A value of 0 will leave the package default. +ceph_tcmalloc_max_total_thread_cache: 0 + ########## # DOCKER # diff --git a/roles/ceph-osd/templates/ceph-osd-run.sh.j2 b/roles/ceph-osd/templates/ceph-osd-run.sh.j2 index 0ff07ede8..f9aae5b36 100644 --- a/roles/ceph-osd/templates/ceph-osd-run.sh.j2 +++ b/roles/ceph-osd/templates/ceph-osd-run.sh.j2 @@ -62,6 +62,9 @@ expose_partitions "$1" {% endif -%} -e CLUSTER={{ cluster }} \ -e OSD_DEVICE=/dev/${1} \ + {% if (ceph_tcmalloc_max_total_thread_cache | int) > 0 and osd_objectstore == 'filestore' -%} + -e TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES={{ ceph_tcmalloc_max_total_thread_cache }} \ + {% endif -%} -e CEPH_DAEMON=OSD_CEPH_DISK_ACTIVATE \ {{ ceph_osd_docker_extra_env }} \ --name=ceph-osd-{{ ansible_hostname }}-${1} \ -- 2.39.5