rolling_update: check quorum state before upgrade

If one a the monitor is out of the quorum then nothing prevents the upgrade
playbook to run.
We only check if we have at least three monitor nodes but we should also
check if those monitor nodes are correctly present in the quorum.

Closes: https://bugzilla.redhat.com/show_bug.cgi?id=1952571
Signed-off-by: Dimitri Savineau <dsavinea@redhat.com>
(cherry picked from commit 97148dd58c77a84aff1235dc9be3cb8c9d73cc09)

diff --git a/infrastructure-playbooks/rolling_update.yml b/infrastructure-playbooks/rolling_update.yml
index be5f27566d09af0e04c597cef8594da0c0937072..05465541655dc1c1b6d0ce99d3db4b7541454473 100644 (file)
--- a/infrastructure-playbooks/rolling_update.yml
+++ b/infrastructure-playbooks/rolling_update.yml
@@ -162,6 +162,16 @@
                       fail:
                         msg: "cluster is not in an acceptable state!"
                   when: (check_cluster_health.stdout | from_json).status == 'HEALTH_ERR'
+
+                - name: get the ceph quorum status
+                  command: "{{ container_exec_cmd }} ceph --cluster {{ cluster }} quorum_status -f json"
+                  register: check_quorum_status
+                  delegate_to: "{{ mon_host }}"
+
+                - name: fail if the cluster quorum isn't in an acceptable state
+                  fail:
+                    msg: "cluster quorum is not in an acceptable state!"
+                  when: (check_quorum_status.stdout | from_json).quorum | length != groups[mon_group_name] | length
               when: inventory_hostname == groups[mon_group_name] | first
 
         - name: ensure /var/lib/ceph/bootstrap-rbd-mirror is present