rolling_update: check quorum state before upgrade

If one a the monitor is out of the quorum then nothing prevents the upgrade
playbook to run.
We only check if we have at least three monitor nodes but we should also
check if those monitor nodes are correctly present in the quorum.

Closes: https://bugzilla.redhat.com/show_bug.cgi?id=1952571
Signed-off-by: Dimitri Savineau <dsavinea@redhat.com>
(cherry picked from commit 97148dd58c77a84aff1235dc9be3cb8c9d73cc09)

diff --git a/infrastructure-playbooks/rolling_update.yml b/infrastructure-playbooks/rolling_update.yml
index b65d5b935fd9820f0f195da80e19f1a58ce441dc..48969460b496bcb3a98ded123afbec19cde8f259 100644 (file)
--- a/infrastructure-playbooks/rolling_update.yml
+++ b/infrastructure-playbooks/rolling_update.yml
@@ -186,6 +186,16 @@
                   fail:
                     msg: "cluster is not in an acceptable state!"
               when: (check_cluster_health.stdout | from_json).status == 'HEALTH_ERR'
+
+            - name: get the ceph quorum status
+              command: "{{ container_exec_cmd }} ceph --cluster {{ cluster }} quorum_status -f json"
+              register: check_quorum_status
+              delegate_to: "{{ mon_host }}"
+
+            - name: fail if the cluster quorum isn't in an acceptable state
+              fail:
+                msg: "cluster quorum is not in an acceptable state!"
+              when: (check_quorum_status.stdout | from_json).quorum | length != groups[mon_group_name] | length
           when: inventory_hostname == groups[mon_group_name] | first
 
         - name: ensure /var/lib/ceph/bootstrap-rbd-mirror is present