From: Guillaume Abrioux Date: Wed, 28 Jul 2021 14:22:09 +0000 (+0200) Subject: ceph-volume/tests: retry when destroying osd X-Git-Tag: v15.2.15~55^2 X-Git-Url: http://git.apps.os.sepia.ceph.com/?a=commitdiff_plain;h=e97448fbbb2d30a2425adc9150146f7c6d17757c;p=ceph.git ceph-volume/tests: retry when destroying osd Sometimes, it can happen that the osds being destroyed in those tests are not yet marked as 'down' for some reason. Let's add some retries on those tasks to avoid CI failures. Fixes: https://tracker.ceph.com/issues/51903 Signed-off-by: Guillaume Abrioux (cherry picked from commit 38882161cc77c913e3f64c786734fd713a951f93) --- diff --git a/src/ceph-volume/ceph_volume/tests/functional/lvm/centos8/bluestore/dmcrypt/test.yml b/src/ceph-volume/ceph_volume/tests/functional/lvm/centos8/bluestore/dmcrypt/test.yml index a05eef6eddd0b..0a47b5eb851e5 100644 --- a/src/ceph-volume/ceph_volume/tests/functional/lvm/centos8/bluestore/dmcrypt/test.yml +++ b/src/ceph-volume/ceph_volume/tests/functional/lvm/centos8/bluestore/dmcrypt/test.yml @@ -15,12 +15,25 @@ - hosts: mons become: yes tasks: + - name: mark osds down + command: "ceph --cluster {{ cluster }} osd down osd.{{ item }}" + with_items: + - 0 + - 2 - name: destroy osd.2 command: "ceph --cluster {{ cluster }} osd destroy osd.2 --yes-i-really-mean-it" + register: result + retries: 30 + delay: 1 + until: result is succeeded - name: destroy osd.0 command: "ceph --cluster {{ cluster }} osd destroy osd.0 --yes-i-really-mean-it" + register: result + retries: 30 + delay: 1 + until: result is succeeded - hosts: osds become: yes @@ -68,9 +81,15 @@ - hosts: mons become: yes tasks: + - name: mark osds down + command: "ceph --cluster {{ cluster }} osd down osd.0" - name: destroy osd.0 command: "ceph --cluster {{ cluster }} osd destroy osd.0 --yes-i-really-mean-it" + register: result + retries: 30 + delay: 1 + until: result is succeeded - hosts: osds diff --git a/src/ceph-volume/ceph_volume/tests/functional/lvm/centos8/filestore/dmcrypt/test.yml b/src/ceph-volume/ceph_volume/tests/functional/lvm/centos8/filestore/dmcrypt/test.yml index f0408736ebfa8..21eff00fa8465 100644 --- a/src/ceph-volume/ceph_volume/tests/functional/lvm/centos8/filestore/dmcrypt/test.yml +++ b/src/ceph-volume/ceph_volume/tests/functional/lvm/centos8/filestore/dmcrypt/test.yml @@ -17,13 +17,25 @@ - hosts: mons become: yes tasks: + - name: mark osds down + command: "ceph --cluster {{ cluster }} osd down osd.{{ item }}" + with_items: + - 0 + - 2 - name: destroy osd.2 command: "ceph --cluster {{ cluster }} osd destroy osd.2 --yes-i-really-mean-it" + register: result + retries: 30 + delay: 1 + until: result is succeeded - name: destroy osd.0 command: "ceph --cluster {{ cluster }} osd destroy osd.0 --yes-i-really-mean-it" - + register: result + retries: 30 + delay: 1 + until: result is succeeded - hosts: osds become: yes diff --git a/src/ceph-volume/ceph_volume/tests/functional/lvm/playbooks/test_bluestore.yml b/src/ceph-volume/ceph_volume/tests/functional/lvm/playbooks/test_bluestore.yml index 2cf83e477fbd6..97d77a7f4601a 100644 --- a/src/ceph-volume/ceph_volume/tests/functional/lvm/playbooks/test_bluestore.yml +++ b/src/ceph-volume/ceph_volume/tests/functional/lvm/playbooks/test_bluestore.yml @@ -17,12 +17,25 @@ - hosts: mons become: yes tasks: + - name: mark osds down + command: "ceph --cluster {{ cluster }} osd down osd.{{ item }}" + with_items: + - 0 + - 2 - name: destroy osd.2 command: "ceph --cluster {{ cluster }} osd destroy osd.2 --yes-i-really-mean-it" + register: result + retries: 30 + delay: 1 + until: result is succeeded - name: destroy osd.0 command: "ceph --cluster {{ cluster }} osd destroy osd.0 --yes-i-really-mean-it" + register: result + retries: 30 + delay: 1 + until: result is succeeded - hosts: osds diff --git a/src/ceph-volume/ceph_volume/tests/functional/lvm/playbooks/test_filestore.yml b/src/ceph-volume/ceph_volume/tests/functional/lvm/playbooks/test_filestore.yml index 42ee40a1baa65..aca1f40a652c6 100644 --- a/src/ceph-volume/ceph_volume/tests/functional/lvm/playbooks/test_filestore.yml +++ b/src/ceph-volume/ceph_volume/tests/functional/lvm/playbooks/test_filestore.yml @@ -17,13 +17,25 @@ - hosts: mons become: yes tasks: + - name: mark osds down + command: "ceph --cluster {{ cluster }} osd down osd.{{ item }}" + with_items: + - 0 + - 2 - name: destroy osd.2 command: "ceph --cluster {{ cluster }} osd destroy osd.2 --yes-i-really-mean-it" + register: result + retries: 30 + delay: 1 + until: result is succeeded - name: destroy osd.0 command: "ceph --cluster {{ cluster }} osd destroy osd.0 --yes-i-really-mean-it" - + register: result + retries: 30 + delay: 1 + until: result is succeeded - hosts: osds become: yes