From 38882161cc77c913e3f64c786734fd713a951f93 Mon Sep 17 00:00:00 2001 From: Guillaume Abrioux Date: Wed, 28 Jul 2021 16:22:09 +0200 Subject: [PATCH] ceph-volume/tests: retry when destroying osd Sometimes, it can happen that the osds being destroyed in those tests are not yet marked as 'down' for some reason. Let's add some retries on those tasks to avoid CI failures. Fixes: https://tracker.ceph.com/issues/51903 Signed-off-by: Guillaume Abrioux --- .../lvm/centos8/bluestore/dmcrypt/test.yml | 19 +++++++++++++++++++ .../lvm/centos8/filestore/dmcrypt/test.yml | 14 +++++++++++++- .../lvm/playbooks/test_bluestore.yml | 13 +++++++++++++ .../lvm/playbooks/test_filestore.yml | 14 +++++++++++++- 4 files changed, 58 insertions(+), 2 deletions(-) diff --git a/src/ceph-volume/ceph_volume/tests/functional/lvm/centos8/bluestore/dmcrypt/test.yml b/src/ceph-volume/ceph_volume/tests/functional/lvm/centos8/bluestore/dmcrypt/test.yml index a05eef6eddd0..0a47b5eb851e 100644 --- a/src/ceph-volume/ceph_volume/tests/functional/lvm/centos8/bluestore/dmcrypt/test.yml +++ b/src/ceph-volume/ceph_volume/tests/functional/lvm/centos8/bluestore/dmcrypt/test.yml @@ -15,12 +15,25 @@ - hosts: mons become: yes tasks: + - name: mark osds down + command: "ceph --cluster {{ cluster }} osd down osd.{{ item }}" + with_items: + - 0 + - 2 - name: destroy osd.2 command: "ceph --cluster {{ cluster }} osd destroy osd.2 --yes-i-really-mean-it" + register: result + retries: 30 + delay: 1 + until: result is succeeded - name: destroy osd.0 command: "ceph --cluster {{ cluster }} osd destroy osd.0 --yes-i-really-mean-it" + register: result + retries: 30 + delay: 1 + until: result is succeeded - hosts: osds become: yes @@ -68,9 +81,15 @@ - hosts: mons become: yes tasks: + - name: mark osds down + command: "ceph --cluster {{ cluster }} osd down osd.0" - name: destroy osd.0 command: "ceph --cluster {{ cluster }} osd destroy osd.0 --yes-i-really-mean-it" + register: result + retries: 30 + delay: 1 + until: result is succeeded - hosts: osds diff --git a/src/ceph-volume/ceph_volume/tests/functional/lvm/centos8/filestore/dmcrypt/test.yml b/src/ceph-volume/ceph_volume/tests/functional/lvm/centos8/filestore/dmcrypt/test.yml index f0408736ebfa..21eff00fa846 100644 --- a/src/ceph-volume/ceph_volume/tests/functional/lvm/centos8/filestore/dmcrypt/test.yml +++ b/src/ceph-volume/ceph_volume/tests/functional/lvm/centos8/filestore/dmcrypt/test.yml @@ -17,13 +17,25 @@ - hosts: mons become: yes tasks: + - name: mark osds down + command: "ceph --cluster {{ cluster }} osd down osd.{{ item }}" + with_items: + - 0 + - 2 - name: destroy osd.2 command: "ceph --cluster {{ cluster }} osd destroy osd.2 --yes-i-really-mean-it" + register: result + retries: 30 + delay: 1 + until: result is succeeded - name: destroy osd.0 command: "ceph --cluster {{ cluster }} osd destroy osd.0 --yes-i-really-mean-it" - + register: result + retries: 30 + delay: 1 + until: result is succeeded - hosts: osds become: yes diff --git a/src/ceph-volume/ceph_volume/tests/functional/lvm/playbooks/test_bluestore.yml b/src/ceph-volume/ceph_volume/tests/functional/lvm/playbooks/test_bluestore.yml index 2cf83e477fbd..97d77a7f4601 100644 --- a/src/ceph-volume/ceph_volume/tests/functional/lvm/playbooks/test_bluestore.yml +++ b/src/ceph-volume/ceph_volume/tests/functional/lvm/playbooks/test_bluestore.yml @@ -17,12 +17,25 @@ - hosts: mons become: yes tasks: + - name: mark osds down + command: "ceph --cluster {{ cluster }} osd down osd.{{ item }}" + with_items: + - 0 + - 2 - name: destroy osd.2 command: "ceph --cluster {{ cluster }} osd destroy osd.2 --yes-i-really-mean-it" + register: result + retries: 30 + delay: 1 + until: result is succeeded - name: destroy osd.0 command: "ceph --cluster {{ cluster }} osd destroy osd.0 --yes-i-really-mean-it" + register: result + retries: 30 + delay: 1 + until: result is succeeded - hosts: osds diff --git a/src/ceph-volume/ceph_volume/tests/functional/lvm/playbooks/test_filestore.yml b/src/ceph-volume/ceph_volume/tests/functional/lvm/playbooks/test_filestore.yml index 42ee40a1baa6..aca1f40a652c 100644 --- a/src/ceph-volume/ceph_volume/tests/functional/lvm/playbooks/test_filestore.yml +++ b/src/ceph-volume/ceph_volume/tests/functional/lvm/playbooks/test_filestore.yml @@ -17,13 +17,25 @@ - hosts: mons become: yes tasks: + - name: mark osds down + command: "ceph --cluster {{ cluster }} osd down osd.{{ item }}" + with_items: + - 0 + - 2 - name: destroy osd.2 command: "ceph --cluster {{ cluster }} osd destroy osd.2 --yes-i-really-mean-it" + register: result + retries: 30 + delay: 1 + until: result is succeeded - name: destroy osd.0 command: "ceph --cluster {{ cluster }} osd destroy osd.0 --yes-i-really-mean-it" - + register: result + retries: 30 + delay: 1 + until: result is succeeded - hosts: osds become: yes -- 2.47.3