From c26e80fdbf49c7d63a2121d44883586b2ec3b777 Mon Sep 17 00:00:00 2001 From: Guillaume Abrioux Date: Mon, 9 Mar 2020 11:05:01 +0100 Subject: [PATCH] rgw: add multi-instances support when deploying multisite This commit adds the multi-instances when deploying rgw multisite Signed-off-by: Guillaume Abrioux Co-authored-by: Dimitri Savineau (cherry picked from commit 60a2e28189758c239e543d360fef8e66e70e7bf7) --- roles/ceph-config/templates/ceph.conf.j2 | 2 +- .../ceph-facts/tasks/set_radosgw_address.yml | 35 +++++--- roles/ceph-rgw/tasks/multisite/checks.yml | 15 +++- .../create_realm_zonegroup_zone_lists.yml | 86 ++++--------------- .../tasks/multisite/create_zone_user.yml | 39 ++++----- roles/ceph-rgw/tasks/multisite/main.yml | 6 +- roles/ceph-rgw/tasks/multisite/master.yml | 82 ++++-------------- roles/ceph-rgw/tasks/multisite/secondary.yml | 31 +++---- .../tasks/check_rgw_multisite.yml | 48 ++++------- .../rgw-multisite/container/group_vars/rgws | 15 +--- .../rgw-multisite/container/host_vars/osd0 | 24 ++++++ .../functional/rgw-multisite/container/hosts | 2 +- .../container/secondary/group_vars/rgws | 6 ++ .../container/secondary/host_vars/osd0 | 26 ++++++ .../rgw-multisite/container/secondary/hosts | 2 +- .../functional/rgw-multisite/group_vars/rgws | 15 +--- tests/functional/rgw-multisite/host_vars/osd0 | 24 ++++++ tests/functional/rgw-multisite/hosts | 2 +- .../rgw-multisite/secondary/group_vars/rgws | 6 ++ .../rgw-multisite/secondary/host_vars/osd0 | 26 ++++++ .../functional/rgw-multisite/secondary/hosts | 2 +- tests/functional/rgw_multisite.yml | 2 +- 22 files changed, 246 insertions(+), 250 deletions(-) create mode 100644 tests/functional/rgw-multisite/container/host_vars/osd0 create mode 100644 tests/functional/rgw-multisite/container/secondary/host_vars/osd0 create mode 100644 tests/functional/rgw-multisite/host_vars/osd0 create mode 100644 tests/functional/rgw-multisite/secondary/host_vars/osd0 diff --git a/roles/ceph-config/templates/ceph.conf.j2 b/roles/ceph-config/templates/ceph.conf.j2 index 879d96dd8..a4da2372e 100644 --- a/roles/ceph-config/templates/ceph.conf.j2 +++ b/roles/ceph-config/templates/ceph.conf.j2 @@ -117,7 +117,7 @@ osd memory target = {{ _osd_memory_target | default(osd_memory_target) }} host = {{ _rgw_hostname }} keyring = /var/lib/ceph/radosgw/{{ cluster }}-rgw.{{ _rgw_hostname + '.' + instance['instance_name'] }}/keyring log file = /var/log/ceph/{{ cluster }}-rgw-{{ hostvars[host]['ansible_hostname'] + '.' + instance['instance_name'] }}.log -{% set _rgw_binding_socket = instance['radosgw_address']|string + ':' + instance['radosgw_frontend_port']|string %} +{% set _rgw_binding_socket = instance['radosgw_address'] | default(_radosgw_address) | string + ':' + instance['radosgw_frontend_port'] | default(radosgw_frontend_port) | string %} {%- macro frontend_line(frontend_type) -%} {%- if frontend_type == 'civetweb' -%} {{ radosgw_frontend_type }} port={{ _rgw_binding_socket }}{{ 's ssl_certificate='+radosgw_frontend_ssl_certificate if radosgw_frontend_ssl_certificate else '' }} diff --git a/roles/ceph-facts/tasks/set_radosgw_address.yml b/roles/ceph-facts/tasks/set_radosgw_address.yml index f510f15f3..8f6ed2d0d 100644 --- a/roles/ceph-facts/tasks/set_radosgw_address.yml +++ b/roles/ceph-facts/tasks/set_radosgw_address.yml @@ -28,19 +28,19 @@ - radosgw_address == 'x.x.x.x' - radosgw_interface != 'interface' block: - - name: set_fact _interface - set_fact: - _interface: "{{ 'ansible_' + (radosgw_interface | replace('-', '_')) }}" + - name: set_fact _interface + set_fact: + _interface: "{{ 'ansible_' + (radosgw_interface | replace('-', '_')) }}" - - name: set_fact _radosgw_address to radosgw_interface - ipv4 - set_fact: - _radosgw_address: "{{ hostvars[inventory_hostname][_interface][ip_version]['address'] }}" - when: ip_version == 'ipv4' + - name: set_fact _radosgw_address to radosgw_interface - ipv4 + set_fact: + _radosgw_address: "{{ hostvars[inventory_hostname][_interface][ip_version]['address'] }}" + when: ip_version == 'ipv4' - - name: set_fact _radosgw_address to radosgw_interface - ipv6 - set_fact: - _radosgw_address: "{{ hostvars[inventory_hostname][_interface][ip_version][0]['address'] }}" - when: ip_version == 'ipv6' + - name: set_fact _radosgw_address to radosgw_interface - ipv6 + set_fact: + _radosgw_address: "{{ hostvars[inventory_hostname][_interface][ip_version][0]['address'] }}" + when: ip_version == 'ipv6' - name: set_fact rgw_instances without rgw multisite set_fact: @@ -52,8 +52,19 @@ - name: set_fact rgw_instances with rgw multisite set_fact: - rgw_instances: "{{ rgw_instances|default([]) | union([{'instance_name': 'rgw' + item|string, 'radosgw_address': _radosgw_address, 'radosgw_frontend_port': radosgw_frontend_port|int, 'rgw_realm': rgw_realm|string, 'rgw_zonegroup': rgw_zonegroup|string, 'rgw_zone': rgw_zone|string}]) }}" + rgw_instances: "{{ rgw_instances|default([]) | union([{'instance_name': 'rgw' + item | string, 'radosgw_address': _radosgw_address, 'radosgw_frontend_port': radosgw_frontend_port | int, 'rgw_realm': rgw_realm | string, 'rgw_zonegroup': rgw_zonegroup | string, 'rgw_zone': rgw_zone | string, 'system_access_key': system_access_key, 'system_secret_key': 'system_secret_key', 'rgw_zone_user': rgw_zone_user, 'rgw_zone_user_display_name': rgw_zone_user_display_name, 'endpoint': (rgw_pull_proto + '://' + rgw_pullhost + ':' + rgw_pull_port | string) if not rgw_zonemaster | bool and rgw_zonesecondary | bool else omit }]) }}" with_sequence: start=0 end={{ radosgw_num_instances|int - 1 }} + when: + - inventory_hostname in groups.get(rgw_group_name, []) + - rgw_instances is undefined + - rgw_multisite | bool + +- name: set_fact rgw_instances_all + set_fact: + rgw_instances_all: '{{ rgw_instances_all | default([]) | union([item.1 | combine({"host": item.0})]) }}' + with_nested: + - "{{ groups.get(rgw_group_name, []) }}" + - "{{ rgw_instances }}" when: - inventory_hostname in groups.get(rgw_group_name, []) - rgw_multisite | bool diff --git a/roles/ceph-rgw/tasks/multisite/checks.yml b/roles/ceph-rgw/tasks/multisite/checks.yml index d41fb4b11..a9fc55f63 100644 --- a/roles/ceph-rgw/tasks/multisite/checks.yml +++ b/roles/ceph-rgw/tasks/multisite/checks.yml @@ -1,24 +1,33 @@ --- - name: check if the realm already exists - command: "{{ container_exec_cmd }} radosgw-admin realm get --cluster={{ cluster }} --rgw-realm={{ rgw_realm }}" + command: "{{ container_exec_cmd }} radosgw-admin realm get --cluster={{ cluster }} --rgw-realm={{ item }}" delegate_to: "{{ groups[mon_group_name][0] }}" register: realmcheck failed_when: False changed_when: False check_mode: no + run_once: True + loop: "{{ realms }}" + when: realms is defined - name: check if the zonegroup already exists - command: "{{ container_exec_cmd }} radosgw-admin zonegroup get --cluster={{ cluster }} --rgw-realm={{ rgw_realm }} --rgw-zonegroup={{ rgw_zonegroup }}" + command: "{{ container_exec_cmd }} radosgw-admin zonegroup get --cluster={{ cluster }} --rgw-realm={{ item.realm }} --rgw-zonegroup={{ item.zonegroup }}" delegate_to: "{{ groups[mon_group_name][0] }}" register: zonegroupcheck failed_when: False changed_when: False check_mode: no + run_once: True + loop: "{{ zonegroups }}" + when: zonegroups is defined - name: check if the zone already exists - command: "{{ container_exec_cmd }} radosgw-admin zone get --rgw-realm={{ rgw_realm }} --cluster={{ cluster }} --rgw-zonegroup={{ rgw_zonegroup }} --rgw-zone={{ rgw_zone }}" + command: "{{ container_exec_cmd }} radosgw-admin zone get --rgw-realm={{ item.realm }} --cluster={{ cluster }} --rgw-zonegroup={{ item.zonegroup }} --rgw-zone={{ item.zone }}" delegate_to: "{{ groups[mon_group_name][0] }}" register: zonecheck failed_when: False changed_when: False check_mode: no + run_once: True + loop: "{{ zones }}" + when: zones is defined diff --git a/roles/ceph-rgw/tasks/multisite/create_realm_zonegroup_zone_lists.yml b/roles/ceph-rgw/tasks/multisite/create_realm_zonegroup_zone_lists.yml index 1ee8f649d..03e37b307 100644 --- a/roles/ceph-rgw/tasks/multisite/create_realm_zonegroup_zone_lists.yml +++ b/roles/ceph-rgw/tasks/multisite/create_realm_zonegroup_zone_lists.yml @@ -1,93 +1,39 @@ -- name: create list realms +--- +- name: set_fact realms set_fact: - realms: "{{ realms | default([]) + [{ 'realm': hostvars[item]['rgw_realm'] }] }}" - with_items: "{{ groups.get(rgw_group_name, []) }}" + realms: '{{ realms | default([]) | union([item.rgw_realm]) }}' + loop: "{{ rgw_instances_all }}" run_once: true - when: - - hostvars[item]['rgw_zonemaster'] | bool - - "'No such file or directory' in hostvars[item]['realmcheck'].stderr" - -- name: make all items in realms unique - set_fact: - realms: "{{ realms | unique }}" - run_once: true - when: - - realms is defined - name: create list secondary_realms set_fact: - secondary_realms: "{{ secondary_realms | default([]) + [{ 'realm': hostvars[item]['rgw_realm'], 'is_master': hostvars[item]['rgw_zonemaster'], 'endpoint': hostvars[item]['rgw_pull_proto'] + '://' + hostvars[item]['rgw_pullhost'] + ':' + hostvars[item]['rgw_pull_port']|string, 'access_key': hostvars[item]['system_access_key'], 'secret_key': hostvars[item]['system_secret_key'] }] }}" - with_items: "{{ groups.get(rgw_group_name, []) }}" - run_once: true - when: - - not hostvars[item]['rgw_zonemaster'] | bool - -- name: make all items in secondary_realms unique - set_fact: - secondary_realms: "{{ secondary_realms | unique }}" + secondary_realms: "{{ secondary_realms | default([]) | union([{ 'realm': item.rgw_realm, 'endpoint': item.endpoint, 'system_access_key': item.system_access_key, 'system_secret_key': item.system_secret_key, 'is_master': hostvars[item.host]['rgw_zonemaster'] }]) }}" + loop: "{{ rgw_instances_all }}" run_once: true - when: - - secondary_realms is defined + when: not hostvars[item.host]['rgw_zonemaster'] | bool - name: create list zonegroups set_fact: - zonegroups: "{{ zonegroups | default([]) + [{ 'realm': hostvars[item]['rgw_realm'], 'zonegroup': hostvars[item]['rgw_zonegroup'], 'is_master': hostvars[item]['rgw_zonegroupmaster'] }] }}" - with_items: "{{ groups.get(rgw_group_name, []) }}" - run_once: true - when: - - hostvars[item]['rgw_zonemaster'] | bool - - "'No such file or directory' in hostvars[item]['zonegroupcheck'].stderr" - -- name: make all items in zonegroups unique - set_fact: - zonegroups: "{{ zonegroups | unique }}" + zonegroups: "{{ zonegroups | default([]) | union([{ 'realm': item.rgw_realm, 'zonegroup': item.rgw_zonegroup, 'is_master': hostvars[item.host]['rgw_zonegroupmaster'] }]) }}" + loop: "{{ rgw_instances_all }}" run_once: true when: - - zonegroups is defined + - hostvars[item.host]['rgw_zonemaster'] | bool - name: create list zones set_fact: - zones: "{{ zones | default([]) + [{ 'realm': hostvars[item]['rgw_realm'], 'zonegroup': hostvars[item]['rgw_zonegroup'], 'zone': hostvars[item]['rgw_zone'], 'is_master': hostvars[item]['rgw_zonemaster'], 'access_key': hostvars[item]['system_access_key'], 'secret_key': hostvars[item]['system_secret_key'] }] }}" - with_items: "{{ groups.get(rgw_group_name, []) }}" + zones: "{{ zones | default([]) | union([{ 'realm': item.rgw_realm, 'zonegroup': item.rgw_zonegroup, 'zone': item.rgw_zone, 'is_master': hostvars[item.host]['rgw_zonemaster'], 'system_access_key': item.system_access_key, 'system_secret_key': item.system_secret_key }]) }}" + loop: "{{ rgw_instances_all }}" run_once: true - when: "'No such file or directory' in hostvars[item]['zonecheck'].stderr" - -- name: make all items in zones unique - set_fact: - zones: "{{ zones | unique }}" - run_once: true - when: - - zones is defined - name: create a list of dicts with each rgw endpoint and it's zone set_fact: - zone_endpoint_pairs: "{{ zone_endpoint_pairs | default([]) + [{ 'endpoint': hostvars[item]['rgw_multisite_proto'] + '://' + hostvars[item]['_radosgw_address'] + ':' + radosgw_frontend_port|string, 'rgw_zone': hostvars[item]['rgw_zone'] }] }}" - with_items: "{{ groups.get(rgw_group_name, []) }}" + zone_endpoint_pairs: "{{ zone_endpoint_pairs | default([]) | union([{ 'endpoint': hostvars[item.host]['rgw_multisite_proto'] + '://' + item.radosgw_address + ':' + item.radosgw_frontend_port | string, 'rgw_zone': item.rgw_zone, 'rgw_realm': item.rgw_realm, 'rgw_zonegroup': item.rgw_zonegroup, 'rgw_zonemaster': hostvars[item.host]['rgw_zonemaster']}]) }}" + loop: "{{ rgw_instances_all }}" run_once: true -- name: create string of all the endpoints in the same rgw_zone - set_fact: - zone_endpoints_string: "{{ zone_endpoints_string | default('') + item.endpoint + ',' }}" - with_items: "{{ zone_endpoint_pairs }}" - when: item.rgw_zone == rgw_zone - -- name: remove ',' after last endpoint in a endpoints string - set_fact: - zone_endpoints_string: "{{ zone_endpoints_string[:-1] }}" - when: - - endpoints_string is defined - - endpoints_string[-1] == ',' - - name: create a list of zones and all their endpoints set_fact: - zone_endpoints_list: "{{ zone_endpoints_list | default([]) + [{ 'endpoints': hostvars[item]['zone_endpoints_string'], 'zone': hostvars[item]['rgw_zone'], 'zonegroup': hostvars[item]['rgw_zonegroup'], 'realm': hostvars[item]['rgw_realm'], 'is_master': hostvars[item]['rgw_zonemaster'] }] }}" - with_items: "{{ groups.get(rgw_group_name, []) }}" - run_once: true - when: hostvars[item]['zone_endpoints_string'] is defined - -- name: make all items in zone_endpoints_list unique - set_fact: - zone_endpoints_list: "{{ zone_endpoints_list | unique }}" + zone_endpoints_list: "{{ zone_endpoints_list | default([]) | union([{'zone': item.rgw_zone, 'zonegroup': item.rgw_zonegroup, 'realm': item.rgw_realm, 'is_master': item.rgw_zonemaster, 'endpoints': ','.join(zone_endpoint_pairs | selectattr('rgw_zone','==',item.rgw_zone) | selectattr('rgw_realm','==',item.rgw_realm) | selectattr('rgw_zonegroup', '==', item.rgw_zonegroup) | map(attribute='endpoint'))}]) }}" + loop: "{{ zone_endpoint_pairs }}" run_once: true - when: - - zone_endpoints_list is defined diff --git a/roles/ceph-rgw/tasks/multisite/create_zone_user.yml b/roles/ceph-rgw/tasks/multisite/create_zone_user.yml index 3f474d55c..601f431ce 100644 --- a/roles/ceph-rgw/tasks/multisite/create_zone_user.yml +++ b/roles/ceph-rgw/tasks/multisite/create_zone_user.yml @@ -1,31 +1,28 @@ +--- +- name: create list zone_users + set_fact: + zone_users: "{{ zone_users | default([]) | union([{ 'realm': item.rgw_realm, 'zonegroup': item.rgw_zonegroup, 'zone': item.rgw_zone, 'system_access_key': item.system_access_key, 'system_secret_key': item.system_secret_key, 'user': item.rgw_zone_user, 'display_name': item.rgw_zone_user_display_name }]) }}" + loop: "{{ rgw_instances_all }}" + run_once: true + when: + - hostvars[item.host]['rgw_zonemaster'] | bool + - hostvars[item.host]['rgw_zonegroupmaster'] | bool + - name: check if the realm system user already exists - command: "{{ container_exec_cmd }} radosgw-admin user info --cluster={{ cluster }} --rgw-realm={{ rgw_realm }} --rgw-zonegroup={{ rgw_zonegroup }} --rgw-zone={{ rgw_zone }} --uid={{ rgw_zone_user }}" + command: "{{ container_exec_cmd }} radosgw-admin user info --cluster={{ cluster }} --rgw-realm={{ item.realm }} --rgw-zonegroup={{ item.zonegroup }} --rgw-zone={{ item.zone }} --uid={{ item.user }}" delegate_to: "{{ groups[mon_group_name][0] }}" register: usercheck failed_when: False changed_when: False check_mode: no - -- name: create list zone_users - set_fact: - zone_users: "{{ zone_users | default([]) + [{ 'realm': hostvars[item]['rgw_realm'], 'zonegroup': hostvars[item]['rgw_zonegroup'], 'zone': hostvars[item]['rgw_zone'], 'access_key': hostvars[item]['system_access_key'], 'secret_key': hostvars[item]['system_secret_key'], 'user': hostvars[item]['rgw_zone_user'], 'display_name': hostvars[item]['rgw_zone_user_display_name'] }] }}" - with_items: "{{ groups.get(rgw_group_name, []) }}" - run_once: true - when: - - hostvars[item]['rgw_zonemaster'] | bool - - hostvars[item]['rgw_zonegroupmaster'] | bool - - "'could not fetch user info: no user info saved' in hostvars[item]['usercheck'].stderr" - -- name: make all items in zone_users unique - set_fact: - zone_users: "{{ zone_users | unique }}" - run_once: true - when: - - zone_users is defined + run_once: True + loop: "{{ zone_users }}" - name: create the zone user(s) - command: "{{ container_exec_cmd }} radosgw-admin user create --cluster={{ cluster }} --rgw-realm={{ item.realm }} --rgw-zonegroup={{ item.zonegroup }} --rgw-zone={{ item.zone }} --uid={{ item.user }} --display-name='{{ item.display_name }}' --access-key={{ item.access_key }} --secret={{ item.secret_key }} --system" + command: "{{ container_exec_cmd }} radosgw-admin user create --cluster={{ cluster }} --rgw-realm={{ item.item.realm }} --rgw-zonegroup={{ item.item.zonegroup }} --rgw-zone={{ item.item.zone }} --uid={{ item.item.user }} --display-name='{{ item.item.display_name }}' --access-key={{ item.item.system_access_key }} --secret={{ item.item.system_secret_key }} --system" delegate_to: "{{ groups[mon_group_name][0] }}" run_once: true - with_items: "{{ zone_users }}" - when: zone_users is defined + loop: "{{ usercheck.results }}" + when: + - zone_users is defined + - "'could not fetch user info: no user info saved' in item.stderr" diff --git a/roles/ceph-rgw/tasks/multisite/main.yml b/roles/ceph-rgw/tasks/multisite/main.yml index d350c1f8b..93299b3ad 100644 --- a/roles/ceph-rgw/tasks/multisite/main.yml +++ b/roles/ceph-rgw/tasks/multisite/main.yml @@ -1,10 +1,10 @@ --- -- name: include multisite checks - include_tasks: checks.yml - - name: include_tasks create_realm_zonegroup_zone_lists.yml include_tasks: create_realm_zonegroup_zone_lists.yml +- name: include multisite checks + include_tasks: checks.yml + # Include the tasks depending on the zone type - name: include_tasks master.yml include_tasks: master.yml diff --git a/roles/ceph-rgw/tasks/multisite/master.yml b/roles/ceph-rgw/tasks/multisite/master.yml index aea2b4ee1..b8c29b96d 100644 --- a/roles/ceph-rgw/tasks/multisite/master.yml +++ b/roles/ceph-rgw/tasks/multisite/master.yml @@ -1,85 +1,39 @@ --- -- name: create default realm - command: "{{ container_exec_cmd }} radosgw-admin realm create --cluster={{ cluster }} --rgw-realm={{ item.realm }} --default" - delegate_to: "{{ groups[mon_group_name][0] }}" - run_once: true - with_items: "{{ realms }}" - when: - - realms is defined - - realms | length == 1 - - name: create the realm(s) - command: "{{ container_exec_cmd }} radosgw-admin realm create --cluster={{ cluster }} --rgw-realm={{ item.realm }}" + command: "{{ container_exec_cmd }} radosgw-admin realm create --cluster={{ cluster }} --rgw-realm={{ item.item }} {{ '--default' if realms | length == 1 else '' }}" delegate_to: "{{ groups[mon_group_name][0] }}" run_once: true - with_items: "{{ realms }}" + loop: "{{ realmcheck.results }}" when: - realms is defined - - realms | length > 1 + - realms | length > 0 + - "'No such file or directory' in item.stderr" -- name: create default master zonegroup(s) - command: "{{ container_exec_cmd }} radosgw-admin zonegroup create --cluster={{ cluster }} --rgw-realm={{ item.realm }} --rgw-zonegroup={{ item.zonegroup }} --default --master" +- name: create zonegroup(s) + command: "{{ container_exec_cmd }} radosgw-admin zonegroup create --cluster={{ cluster }} --rgw-realm={{ item.item.realm }} --rgw-zonegroup={{ item.item.zonegroup }} {{ '--default' if zonegroups | length == 1 else '' }} {{ '--master' if item.item.is_master | bool else '' }}" delegate_to: "{{ groups[mon_group_name][0] }}" run_once: true - with_items: "{{ zonegroups }}" + loop: "{{ zonegroupcheck.results }}" when: - zonegroups is defined - - zonegroups | length == 1 - - item.is_master | bool + - zonegroups | length > 0 + - item.item.is_master | bool + - "'No such file or directory' in item.stderr" -- name: create default zonegroup(s) - command: "{{ container_exec_cmd }} radosgw-admin zonegroup create --cluster={{ cluster }} --rgw-realm={{ item.realm }} --rgw-zonegroup={{ item.zonegroup }} --default" +- name: create the master zone + command: "{{ container_exec_cmd }} radosgw-admin zone create --cluster={{ cluster }} --rgw-realm={{ item.item.realm }} --rgw-zonegroup={{ item.item.zonegroup }} --rgw-zone={{ item.item.zone }} --access-key={{ item.item.system_access_key }} --secret={{ item.item.system_secret_key }} {{ '--default' if zones | length == 1 else '' }} --master" delegate_to: "{{ groups[mon_group_name][0] }}" run_once: true - with_items: "{{ zonegroups }}" - when: - - zonegroups is defined - - zonegroups | length == 1 - - not item.is_master | bool - -- name: create master zonegroup(s) - command: "{{ container_exec_cmd }} radosgw-admin zonegroup create --cluster={{ cluster }} --rgw-realm={{ item.realm }} --rgw-zonegroup={{ item.zonegroup }} --master" - delegate_to: "{{ groups[mon_group_name][0] }}" - run_once: true - with_items: "{{ zonegroups }}" - when: - - zonegroups is defined - - zonegroups | length > 1 - - item.is_master | bool - -- name: create non-master non-default zonegroup(s) - command: "{{ container_exec_cmd }} radosgw-admin zonegroup create --cluster={{ cluster }} --rgw-realm={{ item.realm }} --rgw-zonegroup={{ item.zonegroup }}" - delegate_to: "{{ groups[mon_group_name][0] }}" - run_once: true - with_items: "{{ zonegroups }}" - when: - - zonegroups is defined - - zonegroups | length > 1 - - not item.is_master | bool - -- name: create the default master zone - command: "{{ container_exec_cmd }} radosgw-admin zone create --cluster={{ cluster }} --rgw-realm={{ item.realm }} --rgw-zonegroup={{ item.zonegroup }} --rgw-zone={{ item.zone }} --access-key={{ item.access_key }} --secret={{ item.secret_key }} --master --default" - delegate_to: "{{ groups[mon_group_name][0] }}" - run_once: true - with_items: "{{ zones }}" + loop: "{{ zonecheck.results }}" when: - zones is defined - - zones | length == 1 - - item.is_master | bool - -- name: create the master zone(s) - command: "{{ container_exec_cmd }} radosgw-admin zone create --cluster={{ cluster }} --rgw-realm={{ item.realm }} --rgw-zonegroup={{ item.zonegroup }} --rgw-zone={{ item.zone }} --access-key={{ item.access_key }} --secret={{ item.secret_key }} --master" - delegate_to: "{{ groups[mon_group_name][0] }}" - run_once: true - with_items: "{{ zones }}" - when: - - zones is defined - - zones | length > 1 - - item.is_master | bool + - zones | length > 0 + - item.item.is_master | bool + - "'No such file or directory' in item.stderr" - name: add endpoints to their zone(s) command: "{{ container_exec_cmd }} radosgw-admin zone modify --cluster={{ cluster }} --rgw-realm={{ item.realm }} --rgw-zonegroup={{ item.zonegroup }} --rgw-zone={{ item.zone }} --endpoints {{ item.endpoints }}" - with_items: "{{ zone_endpoints_list }}" + loop: "{{ zone_endpoints_list }}" delegate_to: "{{ groups[mon_group_name][0] }}" run_once: true when: @@ -90,7 +44,7 @@ command: "{{ container_exec_cmd }} radosgw-admin --cluster={{ cluster }} --rgw-realm={{ item.realm }} --rgw-zonegroup={{ item.zonegroup }} --rgw-zone={{ item.zone }} period update --commit" delegate_to: "{{ groups[mon_group_name][0] }}" run_once: true - with_items: "{{ zone_endpoints_list }}" + loop: "{{ zone_endpoints_list }}" when: - zone_endpoints_list is defined - item.is_master | bool diff --git a/roles/ceph-rgw/tasks/multisite/secondary.yml b/roles/ceph-rgw/tasks/multisite/secondary.yml index 42b0aef93..920418955 100644 --- a/roles/ceph-rgw/tasks/multisite/secondary.yml +++ b/roles/ceph-rgw/tasks/multisite/secondary.yml @@ -1,41 +1,32 @@ --- - name: fetch the realm(s) - command: "{{ container_exec_cmd }} radosgw-admin realm pull --cluster={{ cluster }} --rgw-realm={{ item.realm }} --url={{ item.endpoint }} --access-key={{ item.access_key }} --secret={{ item.secret_key }}" + command: "{{ container_exec_cmd }} radosgw-admin realm pull --cluster={{ cluster }} --rgw-realm={{ item.realm }} --url={{ item.endpoint }} --access-key={{ item.system_access_key }} --secret={{ item.system_secret_key }}" delegate_to: "{{ groups[mon_group_name][0] }}" run_once: true - with_items: "{{ secondary_realms }}" + loop: "{{ secondary_realms }}" when: secondary_realms is defined - name: get the period(s) command: "{{ container_exec_cmd }} radosgw-admin period get --cluster={{ cluster }} --rgw-realm={{ item.realm }}" delegate_to: "{{ groups[mon_group_name][0] }}" run_once: true - with_items: "{{ secondary_realms }}" + loop: "{{ secondary_realms }}" when: secondary_realms is defined -- name: create the default zone - command: "{{ container_exec_cmd }} radosgw-admin zone create --cluster={{ cluster }} --rgw-realm={{ item.realm }} --rgw-zonegroup={{ item.zonegroup }} --rgw-zone={{ item.zone }} --access-key={{ item.access_key }} --secret={{ item.secret_key }} --default" +- name: create the zone + command: "{{ container_exec_cmd }} radosgw-admin zone create --cluster={{ cluster }} --rgw-realm={{ item.item.realm }} --rgw-zonegroup={{ item.item.zonegroup }} --rgw-zone={{ item.item.zone }} --access-key={{ item.item.system_access_key }} --secret={{ item.item.system_secret_key }} {{ '--default' if zones | length == 1 else '' }}" delegate_to: "{{ groups[mon_group_name][0] }}" run_once: true - with_items: "{{ zones }}" + loop: "{{ zonecheck.results }}" when: - zones is defined - - zones | length == 1 - - not item.is_master | bool - -- name: create the non-master non-default zone(s) - command: "{{ container_exec_cmd }} radosgw-admin zone create --cluster={{ cluster }} --rgw-realm={{ item.realm }} --rgw-zonegroup={{ item.zonegroup }} --rgw-zone={{ item.zone }} --access-key={{ item.access_key }} --secret={{ item.secret_key }}" - delegate_to: "{{ groups[mon_group_name][0] }}" - run_once: true - with_items: "{{ zones }}" - when: - - zones is defined - - zones | length > 1 - - not item.is_master | bool + - zones | length > 0 + - not item.item.is_master | bool + - "'No such file or directory' in item.stderr" - name: add endpoints to their zone(s) command: "{{ container_exec_cmd }} radosgw-admin zone modify --cluster={{ cluster }} --rgw-realm={{ item.realm }} --rgw-zonegroup={{ item.zonegroup }} --rgw-zone={{ item.zone }} --endpoints {{ item.endpoints }}" - with_items: "{{ zone_endpoints_list }}" + loop: "{{ zone_endpoints_list }}" delegate_to: "{{ groups[mon_group_name][0] }}" run_once: true when: @@ -46,7 +37,7 @@ command: "{{ container_exec_cmd }} radosgw-admin --cluster={{ cluster }} --rgw-realm={{ item.realm }} --rgw-zonegroup={{ item.zonegroup }} --rgw-zone={{ item.zone }} period update --commit" delegate_to: "{{ groups[mon_group_name][0] }}" run_once: true - with_items: "{{ zone_endpoints_list }}" + loop: "{{ zone_endpoints_list }}" when: - zone_endpoints_list is defined - not item.is_master | bool diff --git a/roles/ceph-validate/tasks/check_rgw_multisite.yml b/roles/ceph-validate/tasks/check_rgw_multisite.yml index b324db7ac..84ffcd45e 100644 --- a/roles/ceph-validate/tasks/check_rgw_multisite.yml +++ b/roles/ceph-validate/tasks/check_rgw_multisite.yml @@ -2,7 +2,8 @@ - name: fail if rgw_zone is default fail: msg: "rgw_zone cannot be named 'default'" - when: rgw_zone == 'default' or rgw_zone is undefined + loop: "{{ rgw_instances }}" + when: item.rgw_zone is undefined or item.rgw_zone == 'default' - name: fail if either rgw_zonemaster or rgw_zonesecondary is undefined fail: @@ -19,55 +20,44 @@ - name: fail if rgw_zonegroup is not set fail: msg: "rgw_zonegroup has not been set by the user" - when: rgw_zonegroup is undefined + loop: "{{ rgw_instances }}" + when: item.rgw_zonegroup is undefined - name: fail if rgw_zone_user is not set fail: msg: "rgw_zone_user has not been set by the user" - when: rgw_zone_user is undefined + loop: "{{ rgw_instances }}" + when: item.rgw_zone_user is undefined - name: fail if rgw_zone_user_display_name is not set fail: msg: "rgw_zone_user_display_name has not been set by the user" - when: rgw_zone_user_display_name is undefined + loop: "{{ rgw_instances }}" + when: item.rgw_zone_user_display_name is undefined - name: fail if rgw_realm is not set fail: msg: "rgw_realm has not been set by the user" - when: rgw_realm is undefined + loop: "{{ rgw_instances }}" + when: item.rgw_realm is undefined - name: fail if system_access_key is not set fail: msg: "system_access_key has not been set by the user" - when: system_access_key is undefined + loop: "{{ rgw_instances }}" + when: item.system_access_key is undefined - name: fail if system_secret_key is not set fail: msg: "system_secret_key has not been set by the user" - when: system_secret_key is undefined + loop: "{{ rgw_instances }}" + when: item.system_secret_key is undefined -- name: fail if rgw_pull_port is not set +- name: fail if endpoint is not set fail: - msg: "rgw_pull_port has not been set by the user" + msg: "endpoint has not been set by the user" + loop: "{{ rgw_instances }}" when: - rgw_zonesecondary | bool - - rgw_pull_port is undefined - -- name: fail if rgw_pull_proto is not set - fail: - msg: "rgw_pull_proto has not been set by the user" - when: - - rgw_zonesecondary | bool - - rgw_pull_proto is undefined - -- name: fail if rgw_pullhost is not set - fail: - msg: "rgw_pullhost has not been set by the user" - when: - - rgw_zonesecondary | bool - - rgw_pullhost is undefined - -- name: fail if radosgw_num_instances is not 1 - fail: - msg: "radosgw_num_instances cannot be more than 1" - when: radosgw_num_instances|int > 1 + - rgw_pull_port is undefined and rgw_pullhost is undefined and rgw_pull_proto is undefined + - item.endpoint is undefined \ No newline at end of file diff --git a/tests/functional/rgw-multisite/container/group_vars/rgws b/tests/functional/rgw-multisite/container/group_vars/rgws index bb77a4eef..949999d52 100644 --- a/tests/functional/rgw-multisite/container/group_vars/rgws +++ b/tests/functional/rgw-multisite/container/group_vars/rgws @@ -2,17 +2,6 @@ copy_admin_key: true # Enable Multisite support rgw_multisite: true -rgw_zone: jupiter -rgw_zonemaster: true -rgw_zonesecondary: false -rgw_zonegroup: solarsystem -rgw_zonegroupmaster: True -rgw_zone_user: zone.user -rgw_zone_user_display_name: "Zone User" -rgw_multisite_proto: http -rgw_realm: milkyway -system_access_key: 6kWkikvapSnHyE22P7nO -system_secret_key: MGecsMrWtKZgngOHZdrd6d3JxGO5CPWgT2lcnpSt rgw_create_pools: foo: pg_num: 16 @@ -21,3 +10,7 @@ rgw_create_pools: pg_num: 16 rgw_override_bucket_index_max_shards: 16 rgw_bucket_default_quota_max_objects: 1638400 +rgw_zonemaster: True +rgw_zonesecondary: False +rgw_zonegroupmaster: True +rgw_multisite_proto: http diff --git a/tests/functional/rgw-multisite/container/host_vars/osd0 b/tests/functional/rgw-multisite/container/host_vars/osd0 new file mode 100644 index 000000000..a2b979a96 --- /dev/null +++ b/tests/functional/rgw-multisite/container/host_vars/osd0 @@ -0,0 +1,24 @@ +rgw_instances: + - instance_name: 'rgw0' + rgw_realm: 'france' + rgw_zonegroup: 'idf' + rgw_zone: 'paris' + radosgw_address: "{{ _radosgw_address }}" + radosgw_frontend_port: 8080 + rgw_zone_user: jacques.chirac + rgw_zone_user_display_name: "Jacques Chirac" + system_access_key: P9Eb6S8XNyo4dtZZUUMy + system_secret_key: qqHCUtfdNnpHq3PZRHW5un9l0bEBM812Uhow0XfB + - instance_name: 'rgw1' + rgw_realm: 'usa' + rgw_zonegroup: 'alaska' + rgw_zone: 'juneau' + radosgw_address: "{{ _radosgw_address }}" + radosgw_frontend_port: 8081 + rgw_zone_user: edward.lewis + rgw_zone_user_display_name: "Edward Lewis" + system_access_key: yu17wkvAx3B8Wyn08XoF + system_secret_key: 5YZfaSUPqxSNIkZQQA3lBZ495hnIV6k2HAz710BY +# functional testing +rgw_multisite_endpoint_addr: 192.168.105.11 +radosgw_num_instances: 2 \ No newline at end of file diff --git a/tests/functional/rgw-multisite/container/hosts b/tests/functional/rgw-multisite/container/hosts index df1764b57..6a04f9610 100644 --- a/tests/functional/rgw-multisite/container/hosts +++ b/tests/functional/rgw-multisite/container/hosts @@ -5,4 +5,4 @@ mon0 osd0 [rgws] -osd0 rgw_multisite_proto=http rgw_multisite_endpoint_addr=192.168.105.11 +osd0 diff --git a/tests/functional/rgw-multisite/container/secondary/group_vars/rgws b/tests/functional/rgw-multisite/container/secondary/group_vars/rgws index 2b3f09b5b..a902ad633 100644 --- a/tests/functional/rgw-multisite/container/secondary/group_vars/rgws +++ b/tests/functional/rgw-multisite/container/secondary/group_vars/rgws @@ -1,4 +1,6 @@ --- +# Enable Multisite support +rgw_multisite: true rgw_create_pools: foo: pg_num: 16 @@ -7,3 +9,7 @@ rgw_create_pools: pg_num: 16 rgw_override_bucket_index_max_shards: 16 rgw_bucket_default_quota_max_objects: 1638400 +rgw_zonemaster: False +rgw_zonesecondary: True +rgw_zonegroupmaster: True +rgw_multisite_proto: http diff --git a/tests/functional/rgw-multisite/container/secondary/host_vars/osd0 b/tests/functional/rgw-multisite/container/secondary/host_vars/osd0 new file mode 100644 index 000000000..0f63a3e56 --- /dev/null +++ b/tests/functional/rgw-multisite/container/secondary/host_vars/osd0 @@ -0,0 +1,26 @@ +rgw_instances: + - instance_name: 'rgw0' + rgw_realm: 'france' + rgw_zonegroup: 'idf' + rgw_zone: 'versailles' + radosgw_address: "{{ _radosgw_address }}" + radosgw_frontend_port: 8080 + rgw_zone_user: jacques.chirac + rgw_zone_user_display_name: "Jacques Chirac" + system_access_key: P9Eb6S8XNyo4dtZZUUMy + system_secret_key: qqHCUtfdNnpHq3PZRHW5un9l0bEBM812Uhow0XfB + endpoint: http://192.168.105.11:8080 + - instance_name: 'rgw1' + rgw_realm: 'usa' + rgw_zonegroup: 'alaska' + rgw_zone: 'anchorage' + radosgw_address: "{{ _radosgw_address }}" + radosgw_frontend_port: 8081 + rgw_zone_user: edward.lewis + rgw_zone_user_display_name: "Edward Lewis" + system_access_key: yu17wkvAx3B8Wyn08XoF + system_secret_key: 5YZfaSUPqxSNIkZQQA3lBZ495hnIV6k2HAz710BY + endpoint: http://192.168.105.11:8081 +# functional testing +rgw_multisite_endpoint_addr: 192.168.107.11 +radosgw_num_instances: 2 \ No newline at end of file diff --git a/tests/functional/rgw-multisite/container/secondary/hosts b/tests/functional/rgw-multisite/container/secondary/hosts index d87993451..6a04f9610 100644 --- a/tests/functional/rgw-multisite/container/secondary/hosts +++ b/tests/functional/rgw-multisite/container/secondary/hosts @@ -5,4 +5,4 @@ mon0 osd0 [rgws] -osd0 copy_admin_key=True rgw_multisite=True rgw_zone=mars rgw_zonemaster=False rgw_zonesecondary=True rgw_zonegroup=solarsystem rgw_zonegroupmaster=True rgw_zone_user=zone.user rgw_zone_user_display_name="Zone User" rgw_multisite_proto=http rgw_realm=milkyway system_access_key=6kWkikvapSnHyE22P7nO system_secret_key=MGecsMrWtKZgngOHZdrd6d3JxGO5CPWgT2lcnpSt rgw_pull_proto=http rgw_pull_port=8080 rgw_pullhost=192.168.105.11 rgw_multisite_endpoint_addr=192.168.107.11 +osd0 diff --git a/tests/functional/rgw-multisite/group_vars/rgws b/tests/functional/rgw-multisite/group_vars/rgws index bb77a4eef..949999d52 100644 --- a/tests/functional/rgw-multisite/group_vars/rgws +++ b/tests/functional/rgw-multisite/group_vars/rgws @@ -2,17 +2,6 @@ copy_admin_key: true # Enable Multisite support rgw_multisite: true -rgw_zone: jupiter -rgw_zonemaster: true -rgw_zonesecondary: false -rgw_zonegroup: solarsystem -rgw_zonegroupmaster: True -rgw_zone_user: zone.user -rgw_zone_user_display_name: "Zone User" -rgw_multisite_proto: http -rgw_realm: milkyway -system_access_key: 6kWkikvapSnHyE22P7nO -system_secret_key: MGecsMrWtKZgngOHZdrd6d3JxGO5CPWgT2lcnpSt rgw_create_pools: foo: pg_num: 16 @@ -21,3 +10,7 @@ rgw_create_pools: pg_num: 16 rgw_override_bucket_index_max_shards: 16 rgw_bucket_default_quota_max_objects: 1638400 +rgw_zonemaster: True +rgw_zonesecondary: False +rgw_zonegroupmaster: True +rgw_multisite_proto: http diff --git a/tests/functional/rgw-multisite/host_vars/osd0 b/tests/functional/rgw-multisite/host_vars/osd0 new file mode 100644 index 000000000..131d64483 --- /dev/null +++ b/tests/functional/rgw-multisite/host_vars/osd0 @@ -0,0 +1,24 @@ +rgw_instances: + - instance_name: 'rgw0' + rgw_realm: 'france' + rgw_zonegroup: 'idf' + rgw_zone: 'paris' + radosgw_address: "{{ _radosgw_address }}" + radosgw_frontend_port: 8080 + rgw_zone_user: jacques.chirac + rgw_zone_user_display_name: "Jacques Chirac" + system_access_key: P9Eb6S8XNyo4dtZZUUMy + system_secret_key: qqHCUtfdNnpHq3PZRHW5un9l0bEBM812Uhow0XfB + - instance_name: 'rgw1' + rgw_realm: 'usa' + rgw_zonegroup: 'alaska' + rgw_zone: 'juneau' + radosgw_address: "{{ _radosgw_address }}" + radosgw_frontend_port: 8081 + rgw_zone_user: edward.lewis + rgw_zone_user_display_name: "Edward Lewis" + system_access_key: yu17wkvAx3B8Wyn08XoF + system_secret_key: 5YZfaSUPqxSNIkZQQA3lBZ495hnIV6k2HAz710BY +# functional testing +rgw_multisite_endpoint_addr: 192.168.101.11 +radosgw_num_instances: 2 \ No newline at end of file diff --git a/tests/functional/rgw-multisite/hosts b/tests/functional/rgw-multisite/hosts index abf3b0deb..6a04f9610 100644 --- a/tests/functional/rgw-multisite/hosts +++ b/tests/functional/rgw-multisite/hosts @@ -5,4 +5,4 @@ mon0 osd0 [rgws] -osd0 rgw_multisite_proto=http rgw_multisite_endpoint_addr=192.168.101.11 +osd0 diff --git a/tests/functional/rgw-multisite/secondary/group_vars/rgws b/tests/functional/rgw-multisite/secondary/group_vars/rgws index 2b3f09b5b..a902ad633 100644 --- a/tests/functional/rgw-multisite/secondary/group_vars/rgws +++ b/tests/functional/rgw-multisite/secondary/group_vars/rgws @@ -1,4 +1,6 @@ --- +# Enable Multisite support +rgw_multisite: true rgw_create_pools: foo: pg_num: 16 @@ -7,3 +9,7 @@ rgw_create_pools: pg_num: 16 rgw_override_bucket_index_max_shards: 16 rgw_bucket_default_quota_max_objects: 1638400 +rgw_zonemaster: False +rgw_zonesecondary: True +rgw_zonegroupmaster: True +rgw_multisite_proto: http diff --git a/tests/functional/rgw-multisite/secondary/host_vars/osd0 b/tests/functional/rgw-multisite/secondary/host_vars/osd0 new file mode 100644 index 000000000..5b7d5306b --- /dev/null +++ b/tests/functional/rgw-multisite/secondary/host_vars/osd0 @@ -0,0 +1,26 @@ +rgw_instances: + - instance_name: 'rgw0' + rgw_realm: 'france' + rgw_zonegroup: 'idf' + rgw_zone: 'versailles' + radosgw_address: "{{ _radosgw_address }}" + radosgw_frontend_port: 8080 + rgw_zone_user: jacques.chirac + rgw_zone_user_display_name: "Jacques Chirac" + system_access_key: P9Eb6S8XNyo4dtZZUUMy + system_secret_key: qqHCUtfdNnpHq3PZRHW5un9l0bEBM812Uhow0XfB + endpoint: http://192.168.101.11:8080 + - instance_name: 'rgw1' + rgw_realm: 'usa' + rgw_zonegroup: 'alaska' + rgw_zone: 'anchorage' + radosgw_address: "{{ _radosgw_address }}" + radosgw_frontend_port: 8081 + rgw_zone_user: edward.lewis + rgw_zone_user_display_name: "Edward Lewis" + system_access_key: yu17wkvAx3B8Wyn08XoF + system_secret_key: 5YZfaSUPqxSNIkZQQA3lBZ495hnIV6k2HAz710BY + endpoint: http://192.168.101.11:8081 +# functional testing +rgw_multisite_endpoint_addr: 192.168.103.11 +radosgw_num_instances: 2 \ No newline at end of file diff --git a/tests/functional/rgw-multisite/secondary/hosts b/tests/functional/rgw-multisite/secondary/hosts index 5b7d1a5cb..6a04f9610 100644 --- a/tests/functional/rgw-multisite/secondary/hosts +++ b/tests/functional/rgw-multisite/secondary/hosts @@ -5,4 +5,4 @@ mon0 osd0 [rgws] -osd0 rgw_multisite=True rgw_zone=mars rgw_zonemaster=False rgw_zonesecondary=True rgw_zonegroup=solarsystem rgw_zonegroupmaster=True rgw_zone_user=zone.user rgw_zone_user_display_name="Zone User" rgw_realm=milkyway rgw_multisite_proto=http system_access_key=6kWkikvapSnHyE22P7nO system_secret_key=MGecsMrWtKZgngOHZdrd6d3JxGO5CPWgT2lcnpSt rgw_pull_proto=http rgw_pull_port=8080 rgw_pullhost=192.168.101.11 rgw_multisite_endpoint_addr=192.168.103.11 +osd0 diff --git a/tests/functional/rgw_multisite.yml b/tests/functional/rgw_multisite.yml index 222c3801e..425578c1c 100644 --- a/tests/functional/rgw_multisite.yml +++ b/tests/functional/rgw_multisite.yml @@ -3,7 +3,7 @@ gather_facts: True become: True vars: - s3cmd_cmd: "s3cmd --no-ssl --access_key={{ system_access_key }} --secret_key={{ system_secret_key }} --host={{ rgw_multisite_endpoint_addr }}:8080 --host-bucket={{ rgw_multisite_endpoint_addr }}:8080" + s3cmd_cmd: "s3cmd --no-ssl --access_key=P9Eb6S8XNyo4dtZZUUMy --secret_key=qqHCUtfdNnpHq3PZRHW5un9l0bEBM812Uhow0XfB --host={{ rgw_multisite_endpoint_addr }}:8080 --host-bucket={{ rgw_multisite_endpoint_addr }}:8080" tasks: - name: check if it is Atomic host -- 2.39.5