From: Guillaume Abrioux <gabrioux@redhat.com>
Date: Wed, 28 Jul 2021 14:22:09 +0000 (+0200)
Subject: ceph-volume/tests: retry when destroying osd
X-Git-Tag: v16.2.6~80^2
X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=refs%2Fpull%2F42546%2Fhead;p=ceph.git

ceph-volume/tests: retry when destroying osd

Sometimes, it can happen that the osds being destroyed in those tests
are not yet marked as 'down' for some reason. Let's add some retries on
those tasks to avoid CI failures.

Fixes: https://tracker.ceph.com/issues/51903

Signed-off-by: Guillaume Abrioux <gabrioux@redhat.com>
(cherry picked from commit 38882161cc77c913e3f64c786734fd713a951f93)
---

diff --git a/src/ceph-volume/ceph_volume/tests/functional/lvm/centos8/bluestore/dmcrypt/test.yml b/src/ceph-volume/ceph_volume/tests/functional/lvm/centos8/bluestore/dmcrypt/test.yml
index a05eef6eddd..0a47b5eb851 100644
--- a/src/ceph-volume/ceph_volume/tests/functional/lvm/centos8/bluestore/dmcrypt/test.yml
+++ b/src/ceph-volume/ceph_volume/tests/functional/lvm/centos8/bluestore/dmcrypt/test.yml
@@ -15,12 +15,25 @@
 - hosts: mons
   become: yes
   tasks:
+    - name: mark osds down
+      command: "ceph --cluster {{ cluster }} osd down osd.{{ item }}"
+      with_items:
+        - 0
+        - 2
 
     - name: destroy osd.2
       command: "ceph --cluster {{ cluster }} osd destroy osd.2 --yes-i-really-mean-it"
+      register: result
+      retries: 30
+      delay: 1
+      until: result is succeeded
 
     - name: destroy osd.0
       command: "ceph --cluster {{ cluster }} osd destroy osd.0 --yes-i-really-mean-it"
+      register: result
+      retries: 30
+      delay: 1
+      until: result is succeeded
 
 - hosts: osds
   become: yes
@@ -68,9 +81,15 @@
 - hosts: mons
   become: yes
   tasks:
+    - name: mark osds down
+      command: "ceph --cluster {{ cluster }} osd down osd.0"
 
     - name: destroy osd.0
       command: "ceph --cluster {{ cluster }} osd destroy osd.0 --yes-i-really-mean-it"
+      register: result
+      retries: 30
+      delay: 1
+      until: result is succeeded
 
 
 - hosts: osds
diff --git a/src/ceph-volume/ceph_volume/tests/functional/lvm/centos8/filestore/dmcrypt/test.yml b/src/ceph-volume/ceph_volume/tests/functional/lvm/centos8/filestore/dmcrypt/test.yml
index f0408736ebf..21eff00fa84 100644
--- a/src/ceph-volume/ceph_volume/tests/functional/lvm/centos8/filestore/dmcrypt/test.yml
+++ b/src/ceph-volume/ceph_volume/tests/functional/lvm/centos8/filestore/dmcrypt/test.yml
@@ -17,13 +17,25 @@
 - hosts: mons
   become: yes
   tasks:
+    - name: mark osds down
+      command: "ceph --cluster {{ cluster }} osd down osd.{{ item }}"
+      with_items:
+        - 0
+        - 2
 
     - name: destroy osd.2
       command: "ceph --cluster {{ cluster }} osd destroy osd.2 --yes-i-really-mean-it"
+      register: result
+      retries: 30
+      delay: 1
+      until: result is succeeded
 
     - name: destroy osd.0
       command: "ceph --cluster {{ cluster }} osd destroy osd.0 --yes-i-really-mean-it"
-
+      register: result
+      retries: 30
+      delay: 1
+      until: result is succeeded
 
 - hosts: osds
   become: yes
diff --git a/src/ceph-volume/ceph_volume/tests/functional/lvm/playbooks/test_bluestore.yml b/src/ceph-volume/ceph_volume/tests/functional/lvm/playbooks/test_bluestore.yml
index 2cf83e477fb..97d77a7f460 100644
--- a/src/ceph-volume/ceph_volume/tests/functional/lvm/playbooks/test_bluestore.yml
+++ b/src/ceph-volume/ceph_volume/tests/functional/lvm/playbooks/test_bluestore.yml
@@ -17,12 +17,25 @@
 - hosts: mons
   become: yes
   tasks:
+    - name: mark osds down
+      command: "ceph --cluster {{ cluster }} osd down osd.{{ item }}"
+      with_items:
+        - 0
+        - 2
 
     - name: destroy osd.2
       command: "ceph --cluster {{ cluster }} osd destroy osd.2 --yes-i-really-mean-it"
+      register: result
+      retries: 30
+      delay: 1
+      until: result is succeeded
 
     - name: destroy osd.0
       command: "ceph --cluster {{ cluster }} osd destroy osd.0 --yes-i-really-mean-it"
+      register: result
+      retries: 30
+      delay: 1
+      until: result is succeeded
 
 
 - hosts: osds
diff --git a/src/ceph-volume/ceph_volume/tests/functional/lvm/playbooks/test_filestore.yml b/src/ceph-volume/ceph_volume/tests/functional/lvm/playbooks/test_filestore.yml
index 42ee40a1baa..aca1f40a652 100644
--- a/src/ceph-volume/ceph_volume/tests/functional/lvm/playbooks/test_filestore.yml
+++ b/src/ceph-volume/ceph_volume/tests/functional/lvm/playbooks/test_filestore.yml
@@ -17,13 +17,25 @@
 - hosts: mons
   become: yes
   tasks:
+    - name: mark osds down
+      command: "ceph --cluster {{ cluster }} osd down osd.{{ item }}"
+      with_items:
+        - 0
+        - 2
 
     - name: destroy osd.2
       command: "ceph --cluster {{ cluster }} osd destroy osd.2 --yes-i-really-mean-it"
+      register: result
+      retries: 30
+      delay: 1
+      until: result is succeeded
 
     - name: destroy osd.0
       command: "ceph --cluster {{ cluster }} osd destroy osd.0 --yes-i-really-mean-it"
-
+      register: result
+      retries: 30
+      delay: 1
+      until: result is succeeded
 
 - hosts: osds
   become: yes