From c962d87defb0646b06997adefe81c4ce263cc9a3 Mon Sep 17 00:00:00 2001 From: Guillaume Abrioux Date: Thu, 10 Oct 2019 19:09:49 +0200 Subject: [PATCH] update: follow new recommandation to upgrade mds cluster Refact the mds cluster upgrade code in order to follow the documented recommandation. See: https://github.com/ceph/ceph/blob/master/doc/cephfs/upgrading.rst Closes: https://bugzilla.redhat.com/show_bug.cgi?id=1569689 Signed-off-by: Guillaume Abrioux (cherry picked from commit 71cebf80a623388c64dfcb190133eb5f54a524f9) --- infrastructure-playbooks/rolling_update.yml | 129 ++++++++++++++++-- .../ceph-mds/tasks/create_mds_filesystems.yml | 4 +- 2 files changed, 120 insertions(+), 13 deletions(-) diff --git a/infrastructure-playbooks/rolling_update.yml b/infrastructure-playbooks/rolling_update.yml index be1df0486..b18aa248d 100644 --- a/infrastructure-playbooks/rolling_update.yml +++ b/infrastructure-playbooks/rolling_update.yml @@ -536,25 +536,91 @@ - (ceph_versions.get('stdout', '{}')|from_json).get('osd', {}) | length == 1 - ceph_versions_osd | string is search("ceph version 12") -- name: upgrade ceph mdss cluster - vars: - upgrade_ceph_packages: True - hosts: "{{ mds_group_name|default('mdss') }}" - serial: 1 - become: True + +- name: upgrade ceph mdss cluster, deactivate all rank > 0 + hosts: "{{ groups[mon_group_name|default('mons')][0] }}" + become: true tasks: - - name: stop ceph mds + - import_role: + name: ceph-defaults + - import_role: + name: ceph-facts + + - name: set max_mds 1 on ceph fs + command: "{{ container_exec_cmd | default('') }} ceph --cluster {{ cluster }} fs set {{ cephfs }} max_mds 1" + changed_when: false + + - name: wait until only rank 0 is up + command: "{{ container_exec_cmd | default('') }} ceph --cluster {{ cluster }} fs get {{ cephfs }} -f json" + changed_when: false + register: wait_rank_zero + retries: 720 + delay: 5 + until: (wait_rank_zero.stdout | from_json).mdsmap.in | length == 1 and (wait_rank_zero.stdout | from_json).mdsmap.in[0] == 0 + + - name: get name of remaining active mds + command: "{{ container_exec_cmd | default('') }} ceph --cluster {{ cluster }} fs dump -f json" + changed_when: false + register: _mds_active_name + + - name: set_fact mds_active_name + set_fact: + mds_active_name: "{{ [(_mds_active_name.stdout | from_json)['filesystems'][0]['mdsmap']['info'][item.key]['name']] }}" + with_dict: "{{ (_mds_active_name.stdout | from_json).filesystems[0]['mdsmap']['info'] }}" + + - name: create active_mdss group + add_host: + name: "{{ mds_active_name[0] }}" + groups: active_mdss + ansible_host: "{{ hostvars[mds_active_name]['ansible_host'] | default(omit) }}" + ansible_port: "{{ hostvars[mds_active_name]['ansible_port'] | default(omit) }}" + + - name: create standby_mdss group + add_host: + name: "{{ standby_mdss }}" + groups: standby_mdss + ansible_host: "{{ hostvars[item]['ansible_host'] | default(omit) }}" + ansible_port: "{{ hostvars[item]['ansible_port'] | default(omit) }}" + with_items: "{{ groups[mds_group_name] | difference(mds_active_name) }}" + + - name: stop standby ceph mds systemd: - name: ceph-mds@{{ ansible_hostname }} + name: ceph-mds@{{ hostvars[item]['ansible_hostname'] }} state: stopped enabled: no masked: yes - when: not containerized_deployment | bool + delegate_to: "{{ item }}" + with_items: "{{ groups['standby_mdss'] }}" + when: groups['standby_mdss'] | default([]) | length > 0 + + - name: wait until all standbys mds are stopped + command: "{{ container_exec_cmd | default('') }} ceph --cluster {{ cluster }} fs dump -f json" + changed_when: false + register: wait_standbys_down + retries: 300 + delay: 5 + until: (wait_standbys_down.stdout | from_json).standbys | length == 0 + +- name: upgrade active mds + vars: + upgrade_ceph_packages: True + hosts: active_mdss + become: true + tasks: - import_role: name: ceph-defaults + - import_role: name: ceph-facts + + - name: prevent restart from the packaging + systemd: + name: ceph-mds@{{ ansible_hostname }} + enabled: no + masked: yes + when: not containerized_deployment | bool + - import_role: name: ceph-handler - import_role: @@ -568,14 +634,48 @@ - import_role: name: ceph-mds - - name: start ceph mds + - name: restart ceph mds systemd: name: ceph-mds@{{ ansible_hostname }} - state: started + state: restarted enabled: yes masked: no + daemon_reload: yes + + +- name: upgrade standbys ceph mdss cluster + vars: + upgrade_ceph_packages: True + hosts: standby_mdss | default([]) + become: True + + tasks: + - import_role: + name: ceph-defaults + + - import_role: + name: ceph-facts + + - name: prevent restarts from the packaging + systemd: + name: ceph-mds@{{ ansible_hostname }} + enabled: no + masked: yes when: not containerized_deployment | bool + - import_role: + name: ceph-handler + - import_role: + name: ceph-common + when: not containerized_deployment | bool + - import_role: + name: ceph-container-common + when: containerized_deployment | bool + - import_role: + name: ceph-config + - import_role: + name: ceph-mds + - name: restart ceph mds systemd: name: ceph-mds@{{ ansible_hostname }} @@ -583,7 +683,12 @@ enabled: yes masked: no daemon_reload: yes - when: containerized_deployment | bool + + - name: set max_mds + command: "{{ container_exec_cmd | default('') }} ceph --cluster {{ cluster }} fs set {{ cephfs }} max_mds {{ mds_max_mds }}" + changed_when: false + delegate_to: "{{ groups[mon_group_name][0] }}" + when: inventory_hostname == groups['standby_mdss'] | last - name: upgrade ceph rgws cluster diff --git a/roles/ceph-mds/tasks/create_mds_filesystems.yml b/roles/ceph-mds/tasks/create_mds_filesystems.yml index cda1f508c..0055b4488 100644 --- a/roles/ceph-mds/tasks/create_mds_filesystems.yml +++ b/roles/ceph-mds/tasks/create_mds_filesystems.yml @@ -74,4 +74,6 @@ command: "{{ container_exec_cmd | default('') }} ceph --cluster {{ cluster }} fs set {{ cephfs }} max_mds {{ mds_max_mds }}" changed_when: false delegate_to: "{{ groups[mon_group_name][0] }}" - when: mds_max_mds > 1 + when: + - mds_max_mds > 1 + - not rolling_update -- 2.39.5