From 14b4abf7c002c1db4c80f85116651bb33b9645e4 Mon Sep 17 00:00:00 2001 From: Guillaume Abrioux Date: Fri, 4 Aug 2023 09:45:28 +0200 Subject: [PATCH] migrate from ceph.conf to ceph config keep the ceph.conf very simple. manage the common options such as `public_network` with `ceph_config` module. Signed-off-by: Guillaume Abrioux --- group_vars/rgws.yml.sample | 9 -- roles/ceph-config/tasks/main.yml | 8 -- roles/ceph-config/templates/ceph.conf.j2 | 94 +------------------ roles/ceph-nfs/tasks/main.yml | 15 +++ roles/ceph-rgw/defaults/main.yml | 9 -- roles/ceph-rgw/tasks/multisite/main.yml | 15 +++ roles/ceph-rgw/tasks/pre_requisite.yml | 30 ++++++ site-container.yml.sample | 54 +++++++++++ site.yml.sample | 45 +++++++++ tests/functional/tests/mon/test_mons.py | 11 --- tests/functional/tests/rgw/test_rgw_tuning.py | 59 ------------ 11 files changed, 164 insertions(+), 185 deletions(-) delete mode 100644 tests/functional/tests/rgw/test_rgw_tuning.py diff --git a/group_vars/rgws.yml.sample b/group_vars/rgws.yml.sample index 69b1fc0e8..15c3bf8a5 100644 --- a/group_vars/rgws.yml.sample +++ b/group_vars/rgws.yml.sample @@ -23,15 +23,6 @@ dummy: # TUNING # ########## -# To support buckets with a very large number of objects it's -# important to split them into shards. We suggest about 100K -# objects per shard as a conservative maximum. -#rgw_override_bucket_index_max_shards: 16 - -# Consider setting a quota on buckets so that exceeding this -# limit will require admin intervention. -#rgw_bucket_default_quota_max_objects: 1638400 # i.e., 100K * 16 - # Declaring rgw_create_pools will create pools with the given number of pgs, # size, and type. The following are some important notes on this automatic # pool creation: diff --git a/roles/ceph-config/tasks/main.yml b/roles/ceph-config/tasks/main.yml index 1e31f86a9..36b46f759 100644 --- a/roles/ceph-config/tasks/main.yml +++ b/roles/ceph-config/tasks/main.yml @@ -104,13 +104,6 @@ - "{{ ceph_conf_overrides.get('osd', {}).get('osd_memory_target', '') }}" when: item - - name: drop osd_memory_target from conf override - set_fact: - ceph_conf_overrides: "{{ ceph_conf_overrides | combine({'osd': {item: omit}}, recursive=true) }}" - loop: - - osd memory target - - osd_memory_target - - name: set_fact _osd_memory_target set_fact: _osd_memory_target: "{{ ((ansible_facts['memtotal_mb'] * 1048576 * safety_factor | float) / num_osds | float) | int }}" @@ -141,7 +134,6 @@ owner: "{{ ceph_uid if containerized_deployment | bool else 'ceph' }}" group: "{{ ceph_uid if containerized_deployment | bool else 'ceph' }}" mode: "0644" - config_overrides: "{{ ceph_conf_overrides }}" config_type: ini notify: - restart ceph mons diff --git a/roles/ceph-config/templates/ceph.conf.j2 b/roles/ceph-config/templates/ceph.conf.j2 index 9208a2205..69850ed08 100644 --- a/roles/ceph-config/templates/ceph.conf.j2 +++ b/roles/ceph-config/templates/ceph.conf.j2 @@ -2,36 +2,14 @@ # {{ ansible_managed }} [global] -{% if not cephx | bool %} -auth cluster required = none -auth service required = none -auth client required = none -{% endif %} -{% if ip_version == 'ipv6' %} -ms bind ipv6 = true -ms bind ipv4 = false -{% endif %} -{% if common_single_host_mode is defined and common_single_host_mode %} -osd crush chooseleaf type = 0 -{% endif %} +#{% if not cephx | bool %} +#auth cluster required = none +#auth service required = none +#auth client required = none +#{% endif %} {# NOTE (leseb): the blank lines in-between are needed otherwise we won't get any line break #} {% set nb_mon = groups.get(mon_group_name, []) | length | int %} -{% set nb_client = groups.get(client_group_name, []) | length | int %} -{% set nb_osd = groups.get(osd_group_name, []) | length | int %} -{% if inventory_hostname in groups.get(client_group_name, []) and not inventory_hostname == groups.get(client_group_name, []) | first %} -{% endif %} - -{% if nb_mon > 0 and inventory_hostname in groups.get(mon_group_name, []) %} -mon initial members = {% for host in groups[mon_group_name] %} - {% if hostvars[host]['ansible_facts']['hostname'] is defined -%} - {{ hostvars[host]['ansible_facts']['hostname'] }} - {%- endif %} - {%- if not loop.last %},{% endif %} - {% endfor %} - -osd pool default crush rule = {{ osd_pool_default_crush_rule }} -{% endif %} fsid = {{ fsid }} mon host = {% if nb_mon > 0 %} @@ -46,70 +24,8 @@ mon host = {% if nb_mon > 0 %} {{ external_cluster_mon_ips }} {% endif %} -{% if public_network is defined %} -public network = {{ public_network | regex_replace(' ', '') }} -{% endif %} -{% if cluster_network is defined %} -cluster network = {{ cluster_network | regex_replace(' ', '') }} -{% endif %} -{% if rgw_override_bucket_index_max_shards is defined %} -rgw override bucket index max shards = {{ rgw_override_bucket_index_max_shards }} -{% endif %} -{% if rgw_bucket_default_quota_max_objects is defined %} -rgw bucket default quota max objects = {{ rgw_bucket_default_quota_max_objects }} -{% endif %} - {% if inventory_hostname in groups.get(client_group_name, []) %} [client.libvirt] admin socket = {{ rbd_client_admin_socket_path }}/$cluster-$type.$id.$pid.$cctid.asok # must be writable by QEMU and allowed by SELinux or AppArmor log file = {{ rbd_client_log_file }} # must be writable by QEMU and allowed by SELinux or AppArmor {% endif %} - -{% if inventory_hostname in groups.get(osd_group_name, []) %} -[osd] -osd memory target = {{ _osd_memory_target | default(osd_memory_target) }} -{% endif %} - -{% if inventory_hostname in groups.get(rgw_group_name, []) %} -{% set _rgw_hostname = hostvars[inventory_hostname]['rgw_hostname'] | default(hostvars[inventory_hostname]['ansible_facts']['hostname']) %} -{# {{ hostvars[host]['rgw_hostname'] }} for backward compatibility, fqdn issues. See bz1580408 #} -{% if hostvars[inventory_hostname]['rgw_instances'] is defined %} -{% for instance in hostvars[inventory_hostname]['rgw_instances'] %} -[client.rgw.{{ _rgw_hostname + '.' + instance['instance_name'] }}] -host = {{ _rgw_hostname }} -keyring = /var/lib/ceph/radosgw/{{ cluster }}-rgw.{{ _rgw_hostname + '.' + instance['instance_name'] }}/keyring -log file = /var/log/ceph/{{ cluster }}-rgw-{{ hostvars[inventory_hostname]['ansible_facts']['hostname'] + '.' + instance['instance_name'] }}.log -{% set _rgw_binding_socket = instance['radosgw_address'] | default(_radosgw_address) | string + ':' + instance['radosgw_frontend_port'] | default(radosgw_frontend_port) | string %} -{%- macro frontend_line(frontend_type) -%} -{%- if frontend_type == 'civetweb' -%} -{{ radosgw_frontend_type }} port={{ _rgw_binding_socket }}{{ 's ssl_certificate='+radosgw_frontend_ssl_certificate if radosgw_frontend_ssl_certificate else '' }} -{%- elif frontend_type == 'beast' -%} -{{ radosgw_frontend_type }} {{ 'ssl_' if radosgw_frontend_ssl_certificate else '' }}endpoint={{ _rgw_binding_socket }}{{ ' ssl_certificate='+radosgw_frontend_ssl_certificate if radosgw_frontend_ssl_certificate else '' }} -{%- endif -%} -{%- endmacro -%} -rgw frontends = {{ frontend_line(radosgw_frontend_type) }} {{ radosgw_frontend_options }} -{% if 'num_threads' not in radosgw_frontend_options %} -rgw thread pool size = {{ radosgw_thread_pool_size }} -{% endif %} -{% if rgw_multisite | bool %} -{% if ((instance['rgw_zonemaster'] | default(rgw_zonemaster) | bool) or (deploy_secondary_zones | default(True) | bool)) %} -rgw_realm = {{ instance['rgw_realm'] }} -rgw_zonegroup = {{ instance['rgw_zonegroup'] }} -rgw_zone = {{ instance['rgw_zone'] }} -{% endif %} -{% endif %} -{% endfor %} -{% endif %} -{% endif %} - -{% if inventory_hostname in groups.get(nfs_group_name, []) and inventory_hostname not in groups.get(rgw_group_name, []) %} -{% for host in groups[nfs_group_name] %} -{% set _rgw_hostname = hostvars[host]['rgw_hostname'] | default(hostvars[host]['ansible_facts']['hostname']) %} -{% if nfs_obj_gw | bool %} -[client.rgw.{{ _rgw_hostname }}] -host = {{ _rgw_hostname }} -keyring = /var/lib/ceph/radosgw/{{ cluster }}-rgw.{{ _rgw_hostname }}/keyring -log file = /var/log/ceph/{{ cluster }}-rgw-{{ hostvars[host]['ansible_facts']['hostname'] }}.log -{% endif %} -{% endfor %} -{% endif %} diff --git a/roles/ceph-nfs/tasks/main.yml b/roles/ceph-nfs/tasks/main.yml index aa609d8c8..624d97644 100644 --- a/roles/ceph-nfs/tasks/main.yml +++ b/roles/ceph-nfs/tasks/main.yml @@ -15,6 +15,21 @@ include_tasks: pre_requisite_container.yml when: containerized_deployment | bool +- name: set_fact _rgw_hostname + set_fact: + _rgw_hostname: "{{ hostvars[inventory_hostname]['rgw_hostname'] | default(hostvars[inventory_hostname]['ansible_facts']['hostname']) }}" + +- name: set rgw parameter (log file) + ceph_config: + action: set + who: "client.rgw.{{ _rgw_hostname }}" + option: "log file" + value: "/var/log/ceph/{{ cluster }}-rgw-{{ hostvars[inventory_hostname]['ansible_facts']['hostname'] }}.log" + environment: + CEPH_CONTAINER_IMAGE: "{{ ceph_docker_registry + '/' + ceph_docker_image + ':' + ceph_docker_image_tag if containerized_deployment | bool else None }}" + CEPH_CONTAINER_BINARY: "{{ container_binary }}" + loop: "{{ groups.get('nfss', []) }}" + - name: include create_rgw_nfs_user.yml import_tasks: create_rgw_nfs_user.yml when: groups.get(mon_group_name, []) | length > 0 diff --git a/roles/ceph-rgw/defaults/main.yml b/roles/ceph-rgw/defaults/main.yml index a25468b8d..e86f80ef2 100644 --- a/roles/ceph-rgw/defaults/main.yml +++ b/roles/ceph-rgw/defaults/main.yml @@ -15,15 +15,6 @@ copy_admin_key: false # TUNING # ########## -# To support buckets with a very large number of objects it's -# important to split them into shards. We suggest about 100K -# objects per shard as a conservative maximum. -#rgw_override_bucket_index_max_shards: 16 - -# Consider setting a quota on buckets so that exceeding this -# limit will require admin intervention. -#rgw_bucket_default_quota_max_objects: 1638400 # i.e., 100K * 16 - # Declaring rgw_create_pools will create pools with the given number of pgs, # size, and type. The following are some important notes on this automatic # pool creation: diff --git a/roles/ceph-rgw/tasks/multisite/main.yml b/roles/ceph-rgw/tasks/multisite/main.yml index 17aff9aa7..ade0bf760 100644 --- a/roles/ceph-rgw/tasks/multisite/main.yml +++ b/roles/ceph-rgw/tasks/multisite/main.yml @@ -1,4 +1,19 @@ --- +- name: set global config + ceph_config: + action: set + who: "client.rgw.{{ _rgw_hostname + '.' + item.0.instance_name }}" + option: "{{ item.1 }}" + value: "{{ item.0[item.1] }}" + environment: + CEPH_CONTAINER_IMAGE: "{{ ceph_docker_registry + '/' + ceph_docker_image + ':' + ceph_docker_image_tag if containerized_deployment | bool else None }}" + CEPH_CONTAINER_BINARY: "{{ container_binary }}" + delegate_to: "{{ groups[mon_group_name][0] }}" + run_once: true + with_nested: + - "{{ rgw_instances }}" + - [ 'rgw_realm', 'rgw_zonegroup', 'rgw_zone'] + - name: set_fact realms set_fact: realms: '{{ realms | default([]) | union([item.rgw_realm]) }}' diff --git a/roles/ceph-rgw/tasks/pre_requisite.yml b/roles/ceph-rgw/tasks/pre_requisite.yml index 2e2ddaf6c..374ab854a 100644 --- a/roles/ceph-rgw/tasks/pre_requisite.yml +++ b/roles/ceph-rgw/tasks/pre_requisite.yml @@ -1,4 +1,34 @@ --- +- name: set_fact _rgw_hostname + set_fact: + _rgw_hostname: "{{ hostvars[inventory_hostname]['rgw_hostname'] | default(hostvars[inventory_hostname]['ansible_facts']['hostname']) }}" + +- name: set rgw parameter (log file) + ceph_config: + action: set + who: "client.rgw.{{ _rgw_hostname + '.' + item.instance_name }}" + option: "log file" + value: "/var/log/ceph/{{ cluster }}-rgw-{{ hostvars[inventory_hostname]['ansible_facts']['hostname'] + '.' + item.instance_name }}.log" + environment: + CEPH_CONTAINER_IMAGE: "{{ ceph_docker_registry + '/' + ceph_docker_image + ':' + ceph_docker_image_tag if containerized_deployment | bool else None }}" + CEPH_CONTAINER_BINARY: "{{ container_binary }}" + loop: "{{ hostvars[inventory_hostname]['rgw_instances'] }}" + +- name: set rgw parameter (rgw_frontends) + ceph_config: + action: set + who: "client.rgw.{{ _rgw_hostname + '.' + item.instance_name }}" + option: "rgw_frontends" + value: "beast port={{ item.radosgw_frontend_port | string }}" + environment: + CEPH_CONTAINER_IMAGE: "{{ ceph_docker_registry + '/' + ceph_docker_image + ':' + ceph_docker_image_tag if containerized_deployment | bool else None }}" + CEPH_CONTAINER_BINARY: "{{ container_binary }}" + loop: "{{ hostvars[inventory_hostname]['rgw_instances'] }}" + notify: restart ceph rgws + +# rgw_frontends +# {{ 'ssl_' if radosgw_frontend_ssl_certificate else '' }}endpoint={{ _rgw_binding_socket }}{{ ' ssl_certificate='+radosgw_frontend_ssl_certificate if radosgw_frontend_ssl_certificate else '' }} + - name: create rados gateway directories file: path: "/var/lib/ceph/radosgw/{{ cluster }}-rgw.{{ ansible_facts['hostname'] }}.{{ item.instance_name }}" diff --git a/site-container.yml.sample b/site-container.yml.sample index b3b48fa39..4f67ff3cd 100644 --- a/site-container.yml.sample +++ b/site-container.yml.sample @@ -174,6 +174,60 @@ status: "Complete" end: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}" +- hosts: mons[0] + become: True + gather_facts: false + any_errors_fatal: true + tasks: + - import_role: + name: ceph-defaults + + - name: set global config + ceph_config: + action: set + who: "global" + option: "{{ item.key }}" + value: "{{ item.value }}" + environment: + CEPH_CONTAINER_IMAGE: "{{ ceph_docker_registry + '/' + ceph_docker_image + ':' + ceph_docker_image_tag if containerized_deployment | bool else None }}" + CEPH_CONTAINER_BINARY: "{{ container_binary }}" + with_dict: + "{{ { + 'public_network': public_network | default(False), + 'cluster_network': cluster_network | default(False), + 'osd pool default crush rule': osd_pool_default_crush_rule, + 'ms bind ipv6': 'true' if ip_version == 'ipv6' else 'false', + 'ms bind ipv4': 'false' if ip_version == 'ipv6' else 'true', + 'osd crush chooseleaf type': '0' if common_single_host_mode | default(False) | bool else False, + } }}" + when: + - inventory_hostname == ansible_play_hosts_all | last + - item.value + + - name: set global config overrides + ceph_config: + action: set + who: "global" + option: "{{ item.key }}" + value: "{{ item.value }}" + environment: + CEPH_CONTAINER_IMAGE: "{{ ceph_docker_registry + '/' + ceph_docker_image + ':' + ceph_docker_image_tag if containerized_deployment | bool else None }}" + CEPH_CONTAINER_BINARY: "{{ container_binary }}" + when: inventory_hostname == ansible_play_hosts_all | last + with_dict: "{{ ceph_conf_overrides['global'] }}" + + - name: set osd_memory_target + ceph_config: + action: set + who: "osd.*/{{ item }}:host" + option: "osd_memory_target" + value: "{{ _osd_memory_target | default(osd_memory_target) }}" + environment: + CEPH_CONTAINER_IMAGE: "{{ ceph_docker_registry + '/' + ceph_docker_image + ':' + ceph_docker_image_tag if containerized_deployment | bool else None }}" + CEPH_CONTAINER_BINARY: "{{ container_binary }}" + when: inventory_hostname == ansible_play_hosts_all | last + loop: "{{ groups[osd_group_name] | default([]) }}" + - hosts: osds become: True gather_facts: false diff --git a/site.yml.sample b/site.yml.sample index 26a3aa3fe..a5c2fdd22 100644 --- a/site.yml.sample +++ b/site.yml.sample @@ -166,6 +166,51 @@ status: "Complete" end: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}" +- hosts: mons[0] + become: True + gather_facts: false + any_errors_fatal: true + tasks: + - import_role: + name: ceph-defaults + + - name: set global config + ceph_config: + action: set + who: "global" + option: "{{ item.key }}" + value: "{{ item.value }}" + with_dict: + "{{ { + 'public_network': public_network | default(False), + 'cluster_network': cluster_network | default(False), + 'osd pool default crush rule': osd_pool_default_crush_rule, + 'ms bind ipv6': 'true' if ip_version == 'ipv6' else 'false', + 'ms bind ipv4': 'false' if ip_version == 'ipv6' else 'true', + 'osd crush chooseleaf type': '0' if common_single_host_mode | default(False) | bool else False, + } }}" + when: + - inventory_hostname == ansible_play_hosts_all | last + - item.value + + - name: set global config overrides + ceph_config: + action: set + who: "global" + option: "{{ item.key }}" + value: "{{ item.value }}" + when: inventory_hostname == ansible_play_hosts_all | last + with_dict: "{{ ceph_conf_overrides['global'] }}" + + - name: set osd_memory_target + ceph_config: + action: set + who: "osd.*/{{ item }}:host" + option: "osd_memory_target" + value: "{{ _osd_memory_target | default(osd_memory_target) }}" + when: inventory_hostname == ansible_play_hosts_all | last + loop: "{{ groups[osd_group_name] | default([]) }}" + - hosts: osds gather_facts: false become: True diff --git a/tests/functional/tests/mon/test_mons.py b/tests/functional/tests/mon/test_mons.py index 791eb8d0b..fd09c0d69 100644 --- a/tests/functional/tests/mon/test_mons.py +++ b/tests/functional/tests/mon/test_mons.py @@ -29,14 +29,3 @@ class TestMons(object): output = host.check_output(cmd) assert output.strip().startswith("cluster") - def test_ceph_config_has_inital_members_line(self, node, host, setup): - assert host.file(setup["conf_path"]).contains("^mon initial members = .*$") - - def test_initial_members_line_has_correct_value(self, node, host, setup): - mon_initial_members_line = host.check_output("grep 'mon initial members = ' /etc/ceph/{cluster}.conf".format(cluster=setup['cluster_name'])) # noqa E501 - result = True - for host in node["vars"]["groups"]["mons"]: - pattern = re.compile(host) - if pattern.search(mon_initial_members_line) == None: # noqa E501 - result = False - assert result diff --git a/tests/functional/tests/rgw/test_rgw_tuning.py b/tests/functional/tests/rgw/test_rgw_tuning.py deleted file mode 100644 index 15fa2c0f0..000000000 --- a/tests/functional/tests/rgw/test_rgw_tuning.py +++ /dev/null @@ -1,59 +0,0 @@ -import pytest -import json - - -class TestRGWs(object): - - @pytest.mark.no_docker - def test_rgw_bucket_default_quota_is_set(self, node, host, setup): - assert host.file(setup["conf_path"]).contains( - "rgw override bucket index max shards") - assert host.file(setup["conf_path"]).contains( - "rgw bucket default quota max objects") - - @pytest.mark.no_docker - def test_rgw_bucket_default_quota_is_applied(self, node, host, setup): - radosgw_admin_cmd = "timeout --foreground -s KILL 5 sudo radosgw-admin --cluster={cluster} -n client.rgw.{hostname}.rgw0 --keyring /var/lib/ceph/radosgw/{cluster}-rgw.{hostname}.rgw0/keyring user info --uid=test --format=json".format( # noqa E501 - hostname=node["vars"]["inventory_hostname"], - cluster=setup['cluster_name'] - ) - radosgw_admin_output = host.run(radosgw_admin_cmd) - if radosgw_admin_output.rc == 22: - radosgw_admin_cmd = "timeout --foreground -s KILL 5 sudo radosgw-admin --cluster={cluster} -n client.rgw.{hostname}.rgw0 --keyring /var/lib/ceph/radosgw/{cluster}-rgw.{hostname}.rgw0/keyring user create --uid=test --display-name Test".format( # noqa E501 - hostname=node["vars"]["inventory_hostname"], - cluster=setup['cluster_name'] - ) - radosgw_admin_output = host.run(radosgw_admin_cmd) - radosgw_admin_output_json = json.loads(radosgw_admin_output.stdout) - assert radosgw_admin_output_json["bucket_quota"]["enabled"] == True # noqa E501 - assert radosgw_admin_output_json["bucket_quota"]["max_objects"] == 1638400 # noqa E501 - - @pytest.mark.no_docker - def test_rgw_tuning_pools_are_set(self, node, host, setup): - pools = node["vars"]["rgw_create_pools"] - if pools is None: - pytest.skip('rgw_create_pools not defined, nothing to test') - for pool_name in pools.keys(): - cmd = host.run("sudo ceph --cluster={cluster} --connect-timeout 5 -n client.rgw.{hostname}.rgw0 --keyring /var/lib/ceph/radosgw/{cluster}-rgw.{hostname}.rgw0/keyring osd pool get {pool_name} size".format( # noqa E501 - hostname=node["vars"]["inventory_hostname"], - cluster=setup['cluster_name'], - pool_name=pool_name - )) - assert cmd.rc == 0 - - @pytest.mark.docker - def test_docker_rgw_tuning_pools_are_set(self, node, host, setup): - hostname = node["vars"]["inventory_hostname"] - cluster = setup['cluster_name'] - container_binary = setup["container_binary"] - pools = node["vars"].get("rgw_create_pools") - if pools is None: - pytest.skip('rgw_create_pools not defined, nothing to test') - for pool_name in pools.keys(): - cmd = host.run("sudo {container_binary} exec ceph-rgw-{hostname}-rgw0 ceph --cluster={cluster} -n client.rgw.{hostname}.rgw0 --connect-timeout 5 --keyring /var/lib/ceph/radosgw/{cluster}-rgw.{hostname}.rgw0/keyring osd pool get {pool_name} size".format( # noqa E501 - hostname=hostname, - cluster=cluster, - pool_name=pool_name, - container_binary=container_binary - )) - assert cmd.rc == 0 -- 2.39.5