ceph-volume/tests: retry when destroying osd

Sometimes, it can happen that the osds being destroyed in those tests
are not yet marked as 'down' for some reason. Let's add some retries on
those tasks to avoid CI failures.

Fixes: https://tracker.ceph.com/issues/51903
Signed-off-by: Guillaume Abrioux <gabrioux@redhat.com>
(cherry picked from commit 38882161cc77c913e3f64c786734fd713a951f93)

diff --git a/src/ceph-volume/ceph_volume/tests/functional/lvm/centos8/bluestore/dmcrypt/test.yml b/src/ceph-volume/ceph_volume/tests/functional/lvm/centos8/bluestore/dmcrypt/test.yml
index a05eef6eddd0b3e0576e380bd61fdefdd9624140..0a47b5eb851e5f57a7ef3a2bff53ff6dde7f246e 100644 (file)
--- a/src/ceph-volume/ceph_volume/tests/functional/lvm/centos8/bluestore/dmcrypt/test.yml
+++ b/src/ceph-volume/ceph_volume/tests/functional/lvm/centos8/bluestore/dmcrypt/test.yml
@@ -15,12 +15,25 @@
 - hosts: mons
   become: yes
   tasks:
+    - name: mark osds down
+      command: "ceph --cluster {{ cluster }} osd down osd.{{ item }}"
+      with_items:
+        - 0
+        - 2
 
     - name: destroy osd.2
       command: "ceph --cluster {{ cluster }} osd destroy osd.2 --yes-i-really-mean-it"
+      register: result
+      retries: 30
+      delay: 1
+      until: result is succeeded
 
     - name: destroy osd.0
       command: "ceph --cluster {{ cluster }} osd destroy osd.0 --yes-i-really-mean-it"
+      register: result
+      retries: 30
+      delay: 1
+      until: result is succeeded
 
 - hosts: osds
   become: yes
@@ -68,9 +81,15 @@
 - hosts: mons
   become: yes
   tasks:
+    - name: mark osds down
+      command: "ceph --cluster {{ cluster }} osd down osd.0"
 
     - name: destroy osd.0
       command: "ceph --cluster {{ cluster }} osd destroy osd.0 --yes-i-really-mean-it"
+      register: result
+      retries: 30
+      delay: 1
+      until: result is succeeded
 
 
 - hosts: osds

diff --git a/src/ceph-volume/ceph_volume/tests/functional/lvm/centos8/filestore/dmcrypt/test.yml b/src/ceph-volume/ceph_volume/tests/functional/lvm/centos8/filestore/dmcrypt/test.yml
index f0408736ebfa8e5cc643eeae14c4c0889c420c5d..21eff00fa84659a2a83194db76198a3138ca9713 100644 (file)
--- a/src/ceph-volume/ceph_volume/tests/functional/lvm/centos8/filestore/dmcrypt/test.yml
+++ b/src/ceph-volume/ceph_volume/tests/functional/lvm/centos8/filestore/dmcrypt/test.yml
@@ -17,13 +17,25 @@
 - hosts: mons
   become: yes
   tasks:
+    - name: mark osds down
+      command: "ceph --cluster {{ cluster }} osd down osd.{{ item }}"
+      with_items:
+        - 0
+        - 2
 
     - name: destroy osd.2
       command: "ceph --cluster {{ cluster }} osd destroy osd.2 --yes-i-really-mean-it"
+      register: result
+      retries: 30
+      delay: 1
+      until: result is succeeded
 
     - name: destroy osd.0
       command: "ceph --cluster {{ cluster }} osd destroy osd.0 --yes-i-really-mean-it"
-
+      register: result
+      retries: 30
+      delay: 1
+      until: result is succeeded
 
 - hosts: osds
   become: yes

diff --git a/src/ceph-volume/ceph_volume/tests/functional/lvm/playbooks/test_bluestore.yml b/src/ceph-volume/ceph_volume/tests/functional/lvm/playbooks/test_bluestore.yml
index 2cf83e477fbd6d0533a61252e4e193b27f2c2419..97d77a7f4601aedf9a05d56bb5f252ad10289e93 100644 (file)
--- a/src/ceph-volume/ceph_volume/tests/functional/lvm/playbooks/test_bluestore.yml
+++ b/src/ceph-volume/ceph_volume/tests/functional/lvm/playbooks/test_bluestore.yml
@@ -17,12 +17,25 @@
 - hosts: mons
   become: yes
   tasks:
+    - name: mark osds down
+      command: "ceph --cluster {{ cluster }} osd down osd.{{ item }}"
+      with_items:
+        - 0
+        - 2
 
     - name: destroy osd.2
       command: "ceph --cluster {{ cluster }} osd destroy osd.2 --yes-i-really-mean-it"
+      register: result
+      retries: 30
+      delay: 1
+      until: result is succeeded
 
     - name: destroy osd.0
       command: "ceph --cluster {{ cluster }} osd destroy osd.0 --yes-i-really-mean-it"
+      register: result
+      retries: 30
+      delay: 1
+      until: result is succeeded
 
 
 - hosts: osds

diff --git a/src/ceph-volume/ceph_volume/tests/functional/lvm/playbooks/test_filestore.yml b/src/ceph-volume/ceph_volume/tests/functional/lvm/playbooks/test_filestore.yml
index 42ee40a1baa65f49e95f9e47df8f2cb4f0ddb84c..aca1f40a652c6b3dd8d5dd1c9b2599d2b554693c 100644 (file)
--- a/src/ceph-volume/ceph_volume/tests/functional/lvm/playbooks/test_filestore.yml
+++ b/src/ceph-volume/ceph_volume/tests/functional/lvm/playbooks/test_filestore.yml
@@ -17,13 +17,25 @@
 - hosts: mons
   become: yes
   tasks:
+    - name: mark osds down
+      command: "ceph --cluster {{ cluster }} osd down osd.{{ item }}"
+      with_items:
+        - 0
+        - 2
 
     - name: destroy osd.2
       command: "ceph --cluster {{ cluster }} osd destroy osd.2 --yes-i-really-mean-it"
+      register: result
+      retries: 30
+      delay: 1
+      until: result is succeeded
 
     - name: destroy osd.0
       command: "ceph --cluster {{ cluster }} osd destroy osd.0 --yes-i-really-mean-it"
-
+      register: result
+      retries: 30
+      delay: 1
+      until: result is succeeded
 
 - hosts: osds
   become: yes