keep the ceph.conf very simple.
manage the common options such as `public_network` with `ceph_config`
module.
Signed-off-by: Guillaume Abrioux <gabrioux@ibm.com>
# TUNING #
##########
-# To support buckets with a very large number of objects it's
-# important to split them into shards. We suggest about 100K
-# objects per shard as a conservative maximum.
-#rgw_override_bucket_index_max_shards: 16
-
-# Consider setting a quota on buckets so that exceeding this
-# limit will require admin intervention.
-#rgw_bucket_default_quota_max_objects: 1638400 # i.e., 100K * 16
-
# Declaring rgw_create_pools will create pools with the given number of pgs,
# size, and type. The following are some important notes on this automatic
# pool creation:
- "{{ ceph_conf_overrides.get('osd', {}).get('osd_memory_target', '') }}"
when: item
- - name: drop osd_memory_target from conf override
- set_fact:
- ceph_conf_overrides: "{{ ceph_conf_overrides | combine({'osd': {item: omit}}, recursive=true) }}"
- loop:
- - osd memory target
- - osd_memory_target
-
- name: set_fact _osd_memory_target
set_fact:
_osd_memory_target: "{{ ((ansible_facts['memtotal_mb'] * 1048576 * safety_factor | float) / num_osds | float) | int }}"
owner: "{{ ceph_uid if containerized_deployment | bool else 'ceph' }}"
group: "{{ ceph_uid if containerized_deployment | bool else 'ceph' }}"
mode: "0644"
- config_overrides: "{{ ceph_conf_overrides }}"
config_type: ini
notify:
- restart ceph mons
# {{ ansible_managed }}
[global]
-{% if not cephx | bool %}
-auth cluster required = none
-auth service required = none
-auth client required = none
-{% endif %}
-{% if ip_version == 'ipv6' %}
-ms bind ipv6 = true
-ms bind ipv4 = false
-{% endif %}
-{% if common_single_host_mode is defined and common_single_host_mode %}
-osd crush chooseleaf type = 0
-{% endif %}
+#{% if not cephx | bool %}
+#auth cluster required = none
+#auth service required = none
+#auth client required = none
+#{% endif %}
{# NOTE (leseb): the blank lines in-between are needed otherwise we won't get any line break #}
{% set nb_mon = groups.get(mon_group_name, []) | length | int %}
-{% set nb_client = groups.get(client_group_name, []) | length | int %}
-{% set nb_osd = groups.get(osd_group_name, []) | length | int %}
-{% if inventory_hostname in groups.get(client_group_name, []) and not inventory_hostname == groups.get(client_group_name, []) | first %}
-{% endif %}
-
-{% if nb_mon > 0 and inventory_hostname in groups.get(mon_group_name, []) %}
-mon initial members = {% for host in groups[mon_group_name] %}
- {% if hostvars[host]['ansible_facts']['hostname'] is defined -%}
- {{ hostvars[host]['ansible_facts']['hostname'] }}
- {%- endif %}
- {%- if not loop.last %},{% endif %}
- {% endfor %}
-
-osd pool default crush rule = {{ osd_pool_default_crush_rule }}
-{% endif %}
fsid = {{ fsid }}
mon host = {% if nb_mon > 0 %}
{{ external_cluster_mon_ips }}
{% endif %}
-{% if public_network is defined %}
-public network = {{ public_network | regex_replace(' ', '') }}
-{% endif %}
-{% if cluster_network is defined %}
-cluster network = {{ cluster_network | regex_replace(' ', '') }}
-{% endif %}
-{% if rgw_override_bucket_index_max_shards is defined %}
-rgw override bucket index max shards = {{ rgw_override_bucket_index_max_shards }}
-{% endif %}
-{% if rgw_bucket_default_quota_max_objects is defined %}
-rgw bucket default quota max objects = {{ rgw_bucket_default_quota_max_objects }}
-{% endif %}
-
{% if inventory_hostname in groups.get(client_group_name, []) %}
[client.libvirt]
admin socket = {{ rbd_client_admin_socket_path }}/$cluster-$type.$id.$pid.$cctid.asok # must be writable by QEMU and allowed by SELinux or AppArmor
log file = {{ rbd_client_log_file }} # must be writable by QEMU and allowed by SELinux or AppArmor
{% endif %}
-
-{% if inventory_hostname in groups.get(osd_group_name, []) %}
-[osd]
-osd memory target = {{ _osd_memory_target | default(osd_memory_target) }}
-{% endif %}
-
-{% if inventory_hostname in groups.get(rgw_group_name, []) %}
-{% set _rgw_hostname = hostvars[inventory_hostname]['rgw_hostname'] | default(hostvars[inventory_hostname]['ansible_facts']['hostname']) %}
-{# {{ hostvars[host]['rgw_hostname'] }} for backward compatibility, fqdn issues. See bz1580408 #}
-{% if hostvars[inventory_hostname]['rgw_instances'] is defined %}
-{% for instance in hostvars[inventory_hostname]['rgw_instances'] %}
-[client.rgw.{{ _rgw_hostname + '.' + instance['instance_name'] }}]
-host = {{ _rgw_hostname }}
-keyring = /var/lib/ceph/radosgw/{{ cluster }}-rgw.{{ _rgw_hostname + '.' + instance['instance_name'] }}/keyring
-log file = /var/log/ceph/{{ cluster }}-rgw-{{ hostvars[inventory_hostname]['ansible_facts']['hostname'] + '.' + instance['instance_name'] }}.log
-{% set _rgw_binding_socket = instance['radosgw_address'] | default(_radosgw_address) | string + ':' + instance['radosgw_frontend_port'] | default(radosgw_frontend_port) | string %}
-{%- macro frontend_line(frontend_type) -%}
-{%- if frontend_type == 'civetweb' -%}
-{{ radosgw_frontend_type }} port={{ _rgw_binding_socket }}{{ 's ssl_certificate='+radosgw_frontend_ssl_certificate if radosgw_frontend_ssl_certificate else '' }}
-{%- elif frontend_type == 'beast' -%}
-{{ radosgw_frontend_type }} {{ 'ssl_' if radosgw_frontend_ssl_certificate else '' }}endpoint={{ _rgw_binding_socket }}{{ ' ssl_certificate='+radosgw_frontend_ssl_certificate if radosgw_frontend_ssl_certificate else '' }}
-{%- endif -%}
-{%- endmacro -%}
-rgw frontends = {{ frontend_line(radosgw_frontend_type) }} {{ radosgw_frontend_options }}
-{% if 'num_threads' not in radosgw_frontend_options %}
-rgw thread pool size = {{ radosgw_thread_pool_size }}
-{% endif %}
-{% if rgw_multisite | bool %}
-{% if ((instance['rgw_zonemaster'] | default(rgw_zonemaster) | bool) or (deploy_secondary_zones | default(True) | bool)) %}
-rgw_realm = {{ instance['rgw_realm'] }}
-rgw_zonegroup = {{ instance['rgw_zonegroup'] }}
-rgw_zone = {{ instance['rgw_zone'] }}
-{% endif %}
-{% endif %}
-{% endfor %}
-{% endif %}
-{% endif %}
-
-{% if inventory_hostname in groups.get(nfs_group_name, []) and inventory_hostname not in groups.get(rgw_group_name, []) %}
-{% for host in groups[nfs_group_name] %}
-{% set _rgw_hostname = hostvars[host]['rgw_hostname'] | default(hostvars[host]['ansible_facts']['hostname']) %}
-{% if nfs_obj_gw | bool %}
-[client.rgw.{{ _rgw_hostname }}]
-host = {{ _rgw_hostname }}
-keyring = /var/lib/ceph/radosgw/{{ cluster }}-rgw.{{ _rgw_hostname }}/keyring
-log file = /var/log/ceph/{{ cluster }}-rgw-{{ hostvars[host]['ansible_facts']['hostname'] }}.log
-{% endif %}
-{% endfor %}
-{% endif %}
include_tasks: pre_requisite_container.yml
when: containerized_deployment | bool
+- name: set_fact _rgw_hostname
+ set_fact:
+ _rgw_hostname: "{{ hostvars[inventory_hostname]['rgw_hostname'] | default(hostvars[inventory_hostname]['ansible_facts']['hostname']) }}"
+
+- name: set rgw parameter (log file)
+ ceph_config:
+ action: set
+ who: "client.rgw.{{ _rgw_hostname }}"
+ option: "log file"
+ value: "/var/log/ceph/{{ cluster }}-rgw-{{ hostvars[inventory_hostname]['ansible_facts']['hostname'] }}.log"
+ environment:
+ CEPH_CONTAINER_IMAGE: "{{ ceph_docker_registry + '/' + ceph_docker_image + ':' + ceph_docker_image_tag if containerized_deployment | bool else None }}"
+ CEPH_CONTAINER_BINARY: "{{ container_binary }}"
+ loop: "{{ groups.get('nfss', []) }}"
+
- name: include create_rgw_nfs_user.yml
import_tasks: create_rgw_nfs_user.yml
when: groups.get(mon_group_name, []) | length > 0
# TUNING #
##########
-# To support buckets with a very large number of objects it's
-# important to split them into shards. We suggest about 100K
-# objects per shard as a conservative maximum.
-#rgw_override_bucket_index_max_shards: 16
-
-# Consider setting a quota on buckets so that exceeding this
-# limit will require admin intervention.
-#rgw_bucket_default_quota_max_objects: 1638400 # i.e., 100K * 16
-
# Declaring rgw_create_pools will create pools with the given number of pgs,
# size, and type. The following are some important notes on this automatic
# pool creation:
---
+- name: set global config
+ ceph_config:
+ action: set
+ who: "client.rgw.{{ _rgw_hostname + '.' + item.0.instance_name }}"
+ option: "{{ item.1 }}"
+ value: "{{ item.0[item.1] }}"
+ environment:
+ CEPH_CONTAINER_IMAGE: "{{ ceph_docker_registry + '/' + ceph_docker_image + ':' + ceph_docker_image_tag if containerized_deployment | bool else None }}"
+ CEPH_CONTAINER_BINARY: "{{ container_binary }}"
+ delegate_to: "{{ groups[mon_group_name][0] }}"
+ run_once: true
+ with_nested:
+ - "{{ rgw_instances }}"
+ - [ 'rgw_realm', 'rgw_zonegroup', 'rgw_zone']
+
- name: set_fact realms
set_fact:
realms: '{{ realms | default([]) | union([item.rgw_realm]) }}'
---
+- name: set_fact _rgw_hostname
+ set_fact:
+ _rgw_hostname: "{{ hostvars[inventory_hostname]['rgw_hostname'] | default(hostvars[inventory_hostname]['ansible_facts']['hostname']) }}"
+
+- name: set rgw parameter (log file)
+ ceph_config:
+ action: set
+ who: "client.rgw.{{ _rgw_hostname + '.' + item.instance_name }}"
+ option: "log file"
+ value: "/var/log/ceph/{{ cluster }}-rgw-{{ hostvars[inventory_hostname]['ansible_facts']['hostname'] + '.' + item.instance_name }}.log"
+ environment:
+ CEPH_CONTAINER_IMAGE: "{{ ceph_docker_registry + '/' + ceph_docker_image + ':' + ceph_docker_image_tag if containerized_deployment | bool else None }}"
+ CEPH_CONTAINER_BINARY: "{{ container_binary }}"
+ loop: "{{ hostvars[inventory_hostname]['rgw_instances'] }}"
+
+- name: set rgw parameter (rgw_frontends)
+ ceph_config:
+ action: set
+ who: "client.rgw.{{ _rgw_hostname + '.' + item.instance_name }}"
+ option: "rgw_frontends"
+ value: "beast port={{ item.radosgw_frontend_port | string }}"
+ environment:
+ CEPH_CONTAINER_IMAGE: "{{ ceph_docker_registry + '/' + ceph_docker_image + ':' + ceph_docker_image_tag if containerized_deployment | bool else None }}"
+ CEPH_CONTAINER_BINARY: "{{ container_binary }}"
+ loop: "{{ hostvars[inventory_hostname]['rgw_instances'] }}"
+ notify: restart ceph rgws
+
+# rgw_frontends
+# {{ 'ssl_' if radosgw_frontend_ssl_certificate else '' }}endpoint={{ _rgw_binding_socket }}{{ ' ssl_certificate='+radosgw_frontend_ssl_certificate if radosgw_frontend_ssl_certificate else '' }}
+
- name: create rados gateway directories
file:
path: "/var/lib/ceph/radosgw/{{ cluster }}-rgw.{{ ansible_facts['hostname'] }}.{{ item.instance_name }}"
status: "Complete"
end: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
+- hosts: mons[0]
+ become: True
+ gather_facts: false
+ any_errors_fatal: true
+ tasks:
+ - import_role:
+ name: ceph-defaults
+
+ - name: set global config
+ ceph_config:
+ action: set
+ who: "global"
+ option: "{{ item.key }}"
+ value: "{{ item.value }}"
+ environment:
+ CEPH_CONTAINER_IMAGE: "{{ ceph_docker_registry + '/' + ceph_docker_image + ':' + ceph_docker_image_tag if containerized_deployment | bool else None }}"
+ CEPH_CONTAINER_BINARY: "{{ container_binary }}"
+ with_dict:
+ "{{ {
+ 'public_network': public_network | default(False),
+ 'cluster_network': cluster_network | default(False),
+ 'osd pool default crush rule': osd_pool_default_crush_rule,
+ 'ms bind ipv6': 'true' if ip_version == 'ipv6' else 'false',
+ 'ms bind ipv4': 'false' if ip_version == 'ipv6' else 'true',
+ 'osd crush chooseleaf type': '0' if common_single_host_mode | default(False) | bool else False,
+ } }}"
+ when:
+ - inventory_hostname == ansible_play_hosts_all | last
+ - item.value
+
+ - name: set global config overrides
+ ceph_config:
+ action: set
+ who: "global"
+ option: "{{ item.key }}"
+ value: "{{ item.value }}"
+ environment:
+ CEPH_CONTAINER_IMAGE: "{{ ceph_docker_registry + '/' + ceph_docker_image + ':' + ceph_docker_image_tag if containerized_deployment | bool else None }}"
+ CEPH_CONTAINER_BINARY: "{{ container_binary }}"
+ when: inventory_hostname == ansible_play_hosts_all | last
+ with_dict: "{{ ceph_conf_overrides['global'] }}"
+
+ - name: set osd_memory_target
+ ceph_config:
+ action: set
+ who: "osd.*/{{ item }}:host"
+ option: "osd_memory_target"
+ value: "{{ _osd_memory_target | default(osd_memory_target) }}"
+ environment:
+ CEPH_CONTAINER_IMAGE: "{{ ceph_docker_registry + '/' + ceph_docker_image + ':' + ceph_docker_image_tag if containerized_deployment | bool else None }}"
+ CEPH_CONTAINER_BINARY: "{{ container_binary }}"
+ when: inventory_hostname == ansible_play_hosts_all | last
+ loop: "{{ groups[osd_group_name] | default([]) }}"
+
- hosts: osds
become: True
gather_facts: false
status: "Complete"
end: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
+- hosts: mons[0]
+ become: True
+ gather_facts: false
+ any_errors_fatal: true
+ tasks:
+ - import_role:
+ name: ceph-defaults
+
+ - name: set global config
+ ceph_config:
+ action: set
+ who: "global"
+ option: "{{ item.key }}"
+ value: "{{ item.value }}"
+ with_dict:
+ "{{ {
+ 'public_network': public_network | default(False),
+ 'cluster_network': cluster_network | default(False),
+ 'osd pool default crush rule': osd_pool_default_crush_rule,
+ 'ms bind ipv6': 'true' if ip_version == 'ipv6' else 'false',
+ 'ms bind ipv4': 'false' if ip_version == 'ipv6' else 'true',
+ 'osd crush chooseleaf type': '0' if common_single_host_mode | default(False) | bool else False,
+ } }}"
+ when:
+ - inventory_hostname == ansible_play_hosts_all | last
+ - item.value
+
+ - name: set global config overrides
+ ceph_config:
+ action: set
+ who: "global"
+ option: "{{ item.key }}"
+ value: "{{ item.value }}"
+ when: inventory_hostname == ansible_play_hosts_all | last
+ with_dict: "{{ ceph_conf_overrides['global'] }}"
+
+ - name: set osd_memory_target
+ ceph_config:
+ action: set
+ who: "osd.*/{{ item }}:host"
+ option: "osd_memory_target"
+ value: "{{ _osd_memory_target | default(osd_memory_target) }}"
+ when: inventory_hostname == ansible_play_hosts_all | last
+ loop: "{{ groups[osd_group_name] | default([]) }}"
+
- hosts: osds
gather_facts: false
become: True
output = host.check_output(cmd)
assert output.strip().startswith("cluster")
- def test_ceph_config_has_inital_members_line(self, node, host, setup):
- assert host.file(setup["conf_path"]).contains("^mon initial members = .*$")
-
- def test_initial_members_line_has_correct_value(self, node, host, setup):
- mon_initial_members_line = host.check_output("grep 'mon initial members = ' /etc/ceph/{cluster}.conf".format(cluster=setup['cluster_name'])) # noqa E501
- result = True
- for host in node["vars"]["groups"]["mons"]:
- pattern = re.compile(host)
- if pattern.search(mon_initial_members_line) == None: # noqa E501
- result = False
- assert result
+++ /dev/null
-import pytest
-import json
-
-
-class TestRGWs(object):
-
- @pytest.mark.no_docker
- def test_rgw_bucket_default_quota_is_set(self, node, host, setup):
- assert host.file(setup["conf_path"]).contains(
- "rgw override bucket index max shards")
- assert host.file(setup["conf_path"]).contains(
- "rgw bucket default quota max objects")
-
- @pytest.mark.no_docker
- def test_rgw_bucket_default_quota_is_applied(self, node, host, setup):
- radosgw_admin_cmd = "timeout --foreground -s KILL 5 sudo radosgw-admin --cluster={cluster} -n client.rgw.{hostname}.rgw0 --keyring /var/lib/ceph/radosgw/{cluster}-rgw.{hostname}.rgw0/keyring user info --uid=test --format=json".format( # noqa E501
- hostname=node["vars"]["inventory_hostname"],
- cluster=setup['cluster_name']
- )
- radosgw_admin_output = host.run(radosgw_admin_cmd)
- if radosgw_admin_output.rc == 22:
- radosgw_admin_cmd = "timeout --foreground -s KILL 5 sudo radosgw-admin --cluster={cluster} -n client.rgw.{hostname}.rgw0 --keyring /var/lib/ceph/radosgw/{cluster}-rgw.{hostname}.rgw0/keyring user create --uid=test --display-name Test".format( # noqa E501
- hostname=node["vars"]["inventory_hostname"],
- cluster=setup['cluster_name']
- )
- radosgw_admin_output = host.run(radosgw_admin_cmd)
- radosgw_admin_output_json = json.loads(radosgw_admin_output.stdout)
- assert radosgw_admin_output_json["bucket_quota"]["enabled"] == True # noqa E501
- assert radosgw_admin_output_json["bucket_quota"]["max_objects"] == 1638400 # noqa E501
-
- @pytest.mark.no_docker
- def test_rgw_tuning_pools_are_set(self, node, host, setup):
- pools = node["vars"]["rgw_create_pools"]
- if pools is None:
- pytest.skip('rgw_create_pools not defined, nothing to test')
- for pool_name in pools.keys():
- cmd = host.run("sudo ceph --cluster={cluster} --connect-timeout 5 -n client.rgw.{hostname}.rgw0 --keyring /var/lib/ceph/radosgw/{cluster}-rgw.{hostname}.rgw0/keyring osd pool get {pool_name} size".format( # noqa E501
- hostname=node["vars"]["inventory_hostname"],
- cluster=setup['cluster_name'],
- pool_name=pool_name
- ))
- assert cmd.rc == 0
-
- @pytest.mark.docker
- def test_docker_rgw_tuning_pools_are_set(self, node, host, setup):
- hostname = node["vars"]["inventory_hostname"]
- cluster = setup['cluster_name']
- container_binary = setup["container_binary"]
- pools = node["vars"].get("rgw_create_pools")
- if pools is None:
- pytest.skip('rgw_create_pools not defined, nothing to test')
- for pool_name in pools.keys():
- cmd = host.run("sudo {container_binary} exec ceph-rgw-{hostname}-rgw0 ceph --cluster={cluster} -n client.rgw.{hostname}.rgw0 --connect-timeout 5 --keyring /var/lib/ceph/radosgw/{cluster}-rgw.{hostname}.rgw0/keyring osd pool get {pool_name} size".format( # noqa E501
- hostname=hostname,
- cluster=cluster,
- pool_name=pool_name,
- container_binary=container_binary
- ))
- assert cmd.rc == 0
projects / ceph-ansible.git / commitdiff