From: Guillaume Abrioux <gabrioux@redhat.com>
Date: Wed, 28 Jul 2021 14:22:09 +0000 (+0200)
Subject: ceph-volume/tests: retry when destroying osd
X-Git-Tag: v17.1.0~1257^2
X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=refs%2Fpull%2F42524%2Fhead;p=ceph.git

ceph-volume/tests: retry when destroying osd

Sometimes, it can happen that the osds being destroyed in those tests
are not yet marked as 'down' for some reason. Let's add some retries on
those tasks to avoid CI failures.

Fixes: https://tracker.ceph.com/issues/51903

Signed-off-by: Guillaume Abrioux <gabrioux@redhat.com>
---

diff --git a/src/ceph-volume/ceph_volume/tests/functional/lvm/centos8/bluestore/dmcrypt/test.yml b/src/ceph-volume/ceph_volume/tests/functional/lvm/centos8/bluestore/dmcrypt/test.yml
index a05eef6eddd0..0a47b5eb851e 100644
--- a/src/ceph-volume/ceph_volume/tests/functional/lvm/centos8/bluestore/dmcrypt/test.yml
+++ b/src/ceph-volume/ceph_volume/tests/functional/lvm/centos8/bluestore/dmcrypt/test.yml
@@ -15,12 +15,25 @@
 - hosts: mons
   become: yes
   tasks:
+    - name: mark osds down
+      command: "ceph --cluster {{ cluster }} osd down osd.{{ item }}"
+      with_items:
+        - 0
+        - 2
 
     - name: destroy osd.2
       command: "ceph --cluster {{ cluster }} osd destroy osd.2 --yes-i-really-mean-it"
+      register: result
+      retries: 30
+      delay: 1
+      until: result is succeeded
 
     - name: destroy osd.0
       command: "ceph --cluster {{ cluster }} osd destroy osd.0 --yes-i-really-mean-it"
+      register: result
+      retries: 30
+      delay: 1
+      until: result is succeeded
 
 - hosts: osds
   become: yes
@@ -68,9 +81,15 @@
 - hosts: mons
   become: yes
   tasks:
+    - name: mark osds down
+      command: "ceph --cluster {{ cluster }} osd down osd.0"
 
     - name: destroy osd.0
       command: "ceph --cluster {{ cluster }} osd destroy osd.0 --yes-i-really-mean-it"
+      register: result
+      retries: 30
+      delay: 1
+      until: result is succeeded
 
 
 - hosts: osds
diff --git a/src/ceph-volume/ceph_volume/tests/functional/lvm/centos8/filestore/dmcrypt/test.yml b/src/ceph-volume/ceph_volume/tests/functional/lvm/centos8/filestore/dmcrypt/test.yml
index f0408736ebfa..21eff00fa846 100644
--- a/src/ceph-volume/ceph_volume/tests/functional/lvm/centos8/filestore/dmcrypt/test.yml
+++ b/src/ceph-volume/ceph_volume/tests/functional/lvm/centos8/filestore/dmcrypt/test.yml
@@ -17,13 +17,25 @@
 - hosts: mons
   become: yes
   tasks:
+    - name: mark osds down
+      command: "ceph --cluster {{ cluster }} osd down osd.{{ item }}"
+      with_items:
+        - 0
+        - 2
 
     - name: destroy osd.2
       command: "ceph --cluster {{ cluster }} osd destroy osd.2 --yes-i-really-mean-it"
+      register: result
+      retries: 30
+      delay: 1
+      until: result is succeeded
 
     - name: destroy osd.0
       command: "ceph --cluster {{ cluster }} osd destroy osd.0 --yes-i-really-mean-it"
-
+      register: result
+      retries: 30
+      delay: 1
+      until: result is succeeded
 
 - hosts: osds
   become: yes
diff --git a/src/ceph-volume/ceph_volume/tests/functional/lvm/playbooks/test_bluestore.yml b/src/ceph-volume/ceph_volume/tests/functional/lvm/playbooks/test_bluestore.yml
index 2cf83e477fbd..97d77a7f4601 100644
--- a/src/ceph-volume/ceph_volume/tests/functional/lvm/playbooks/test_bluestore.yml
+++ b/src/ceph-volume/ceph_volume/tests/functional/lvm/playbooks/test_bluestore.yml
@@ -17,12 +17,25 @@
 - hosts: mons
   become: yes
   tasks:
+    - name: mark osds down
+      command: "ceph --cluster {{ cluster }} osd down osd.{{ item }}"
+      with_items:
+        - 0
+        - 2
 
     - name: destroy osd.2
       command: "ceph --cluster {{ cluster }} osd destroy osd.2 --yes-i-really-mean-it"
+      register: result
+      retries: 30
+      delay: 1
+      until: result is succeeded
 
     - name: destroy osd.0
       command: "ceph --cluster {{ cluster }} osd destroy osd.0 --yes-i-really-mean-it"
+      register: result
+      retries: 30
+      delay: 1
+      until: result is succeeded
 
 
 - hosts: osds
diff --git a/src/ceph-volume/ceph_volume/tests/functional/lvm/playbooks/test_filestore.yml b/src/ceph-volume/ceph_volume/tests/functional/lvm/playbooks/test_filestore.yml
index 42ee40a1baa6..aca1f40a652c 100644
--- a/src/ceph-volume/ceph_volume/tests/functional/lvm/playbooks/test_filestore.yml
+++ b/src/ceph-volume/ceph_volume/tests/functional/lvm/playbooks/test_filestore.yml
@@ -17,13 +17,25 @@
 - hosts: mons
   become: yes
   tasks:
+    - name: mark osds down
+      command: "ceph --cluster {{ cluster }} osd down osd.{{ item }}"
+      with_items:
+        - 0
+        - 2
 
     - name: destroy osd.2
       command: "ceph --cluster {{ cluster }} osd destroy osd.2 --yes-i-really-mean-it"
+      register: result
+      retries: 30
+      delay: 1
+      until: result is succeeded
 
     - name: destroy osd.0
       command: "ceph --cluster {{ cluster }} osd destroy osd.0 --yes-i-really-mean-it"
-
+      register: result
+      retries: 30
+      delay: 1
+      until: result is succeeded
 
 - hosts: osds
   become: yes