update: reset flags before and after each osd node upgrade

It might be possible at some point even with osd flags `noout` and
`norebalance` set the PGs states can change depending on the amount of data
written meantime. It means the check for PGs state will fail.

This commit changes the way we set those flags:
we set them before an OSD node upgrade and unset them before the PGs
state check so they can recover.

Fixes: #3961
Signed-off-by: Guillaume Abrioux <gabrioux@redhat.com>

diff --git a/infrastructure-playbooks/rolling_update.yml b/infrastructure-playbooks/rolling_update.yml
index 5c0c610449ce9f5d4ce2bca86d97add5d8554256..7bb4540a17f41b1f35a10aa6719681ea523a5979 100644 (file)
--- a/infrastructure-playbooks/rolling_update.yml
+++ b/infrastructure-playbooks/rolling_update.yml
@@ -187,27 +187,6 @@
       when: inventory_hostname in groups[mgr_group_name] | default([])
             or groups[mgr_group_name] | default([]) | length == 0
 
-    - name: set osd flags
-      command: ceph --cluster {{ cluster }} osd set {{ item }}
-      with_items:
-        - noout
-        - norebalance
-      delegate_to: "{{ mon_host }}"
-      when:
-        - inventory_hostname == groups[mon_group_name][0]
-        - not containerized_deployment | bool
-
-    - name: set containerized osd flags
-      command: >
-        {{ container_binary }} exec ceph-mon-{{ hostvars[mon_host]['ansible_hostname'] }} ceph --cluster {{ cluster }} osd set {{ item }}
-      with_items:
-        - noout
-        - norebalance
-      delegate_to: "{{ mon_host }}"
-      when:
-        - inventory_hostname == groups[mon_group_name][0]
-        - containerized_deployment | bool
-
     - import_role:
         name: ceph-handler
     - import_role:
@@ -362,6 +341,11 @@
   serial: 1
   become: True
   tasks:
+    - import_role:
+        name: ceph-defaults
+    - import_role:
+        name: ceph-facts
+
     - name: get osd numbers - non container
       shell: "if [ -d /var/lib/ceph/osd ] ; then ls /var/lib/ceph/osd | sed 's/.*-//' ; fi"
       register: osd_ids
@@ -379,6 +363,20 @@
         num_osds: "{{ osd_names.stdout_lines|default([])|length }}"
       when: containerized_deployment | bool
 
+    - name: set_fact container_exec_cmd_osd
+      set_fact:
+        container_exec_cmd_update_osd: "{{ container_binary }} exec ceph-mon-{{ hostvars[groups[mon_group_name][0]]['ansible_hostname'] }}"
+      when: containerized_deployment | bool
+
+    - name: set osd flags
+      command: "{{ container_exec_cmd_update_osd | default('') }} ceph --cluster {{ cluster }} osd set {{ item }}"
+      with_items:
+        - noout
+        - norebalance
+        - norecover
+        - nobackfill
+      delegate_to: "{{ groups[mon_group_name][0] }}"
+
     - name: stop ceph osd
       systemd:
         name: ceph-osd@{{ item }}
@@ -393,10 +391,6 @@
         num_osds: "{{ osd_ids.stdout_lines|default([])|length }}"
       when: not containerized_deployment | bool
 
-    - import_role:
-        name: ceph-defaults
-    - import_role:
-        name: ceph-facts
     - import_role:
         name: ceph-handler
     - import_role:
@@ -451,29 +445,14 @@
         - ceph_release in ["nautilus", "octopus"]
         - not containerized_deployment | bool
 
-    - name: set_fact container_exec_cmd_osd
-      set_fact:
-        container_exec_cmd_update_osd: "{{ container_binary }} exec ceph-mon-{{ hostvars[groups[mon_group_name][0]]['ansible_hostname'] }}"
-      when: containerized_deployment | bool
-
-    - name: get osd versions
-      command: "{{ container_exec_cmd_update_osd|default('') }} ceph --cluster {{ cluster }} versions"
-      register: ceph_versions
-      delegate_to: "{{ groups[mon_group_name][0] }}"
-
-    - name: set_fact ceph_versions_osd
-      set_fact:
-        ceph_versions_osd: "{{ (ceph_versions.stdout|from_json).osd }}"
-      delegate_to: "{{ groups[mon_group_name][0] }}"
-
-    # length == 1 means there is a single osds versions entry
-    # thus all the osds are running the same version
-    - name: osd set sortbitwise
-      command: "{{ container_exec_cmd_update_osd|default('') }} ceph --cluster {{ cluster }} osd set sortbitwise"
+    - name: unset osd flags
+      command: "{{ container_exec_cmd_update_osd | default('') }} ceph osd unset {{ item }} --cluster {{ cluster }}"
+      with_items:
+        - noout
+        - norebalance
+        - norecover
+        - nobackfill
       delegate_to: "{{ groups[mon_group_name][0] }}"
-      when:
-        - (ceph_versions.get('stdout', '{}')|from_json).get('osd', {}) | length == 1
-        - ceph_versions_osd | string is search("ceph version 10")
 
     - name: get num_pgs - non container
       command: "{{ container_exec_cmd_update_osd|default('') }} ceph --cluster {{ cluster }} -s --format json"
@@ -493,12 +472,9 @@
       when: (ceph_pgs.stdout | from_json).pgmap.num_pgs != 0
 
 
-- name: unset osd flags
-
+- name: complete osd upgrade
   hosts: "{{ mon_group_name|default('mons') }}"
-
   become: True
-
   tasks:
     - import_role:
         name: ceph-defaults
@@ -510,13 +486,6 @@
         container_exec_cmd_update_osd: "{{ container_binary }} exec ceph-mon-{{ hostvars[groups[mon_group_name][0]]['ansible_hostname'] }}"
       when: containerized_deployment | bool
 
-    - name: unset osd flags
-      command: "{{ container_exec_cmd_update_osd|default('') }} ceph osd unset {{ item }} --cluster {{ cluster }}"
-      with_items:
-        - noout
-        - norebalance
-      delegate_to: "{{ groups[mon_group_name][0] }}"
-
     - name: get osd versions
       command: "{{ container_exec_cmd_update_osd|default('') }} ceph --cluster {{ cluster }} versions"
       register: ceph_versions