tests: add retries on failing tests in testinfra

This commit adds `pytest-rerunfailures` in requirements.txt so we can
retry failing test in testinfra to avoid false positive. (eg: sometimes it
can happen for some reason a service takes too much time to start)

Signed-off-by: Guillaume Abrioux <gabrioux@redhat.com>

diff --git a/tests/requirements.txt b/tests/requirements.txt
index bd472792442546cf8618646c987ab8621dd2a7b0..ab75cb11a92e00101fc6e7b8bc7d094b29326f7e 100644 (file)
--- a/tests/requirements.txt
+++ b/tests/requirements.txt
@@ -9,3 +9,4 @@ netaddr
 mock
 jmespath
 paramiko
+pytest-rerunfailures
\ No newline at end of file

diff --git a/tox-dashboard.ini b/tox-dashboard.ini
index def8690a3a0a4ca15ff536b0ed99c6a2785653a0..29cb7b51fee754dffe7bd9b346a6b1ffe2f25b2d 100644 (file)
--- a/tox-dashboard.ini
+++ b/tox-dashboard.ini
@@ -64,6 +64,6 @@ commands=
       ceph_dev_sha1={env:UPDATE_CEPH_DEV_SHA1:latest} \
   "
 
-  bash -c "CEPH_STABLE_RELEASE={env:CEPH_STABLE_RELEASE:nautilus} py.test -n 8 --durations=0 --sudo -v --connection=ansible --ansible-inventory={changedir}/{env:INVENTORY} --ssh-config={changedir}/vagrant_ssh_config {toxinidir}/tests/functional/tests"
+  bash -c "CEPH_STABLE_RELEASE={env:CEPH_STABLE_RELEASE:nautilus} py.test --reruns 5 --reruns-delay 1 -n 8 --durations=0 --sudo -v --connection=ansible --ansible-inventory={changedir}/{env:INVENTORY} --ssh-config={changedir}/vagrant_ssh_config {toxinidir}/tests/functional/tests"
 
   vagrant destroy --force

diff --git a/tox-update.ini b/tox-update.ini
index 0ef2f3c07bbadd4ab77bea04ec14d219dad80fc3..b3bf7e76263fcb5bf2a8a4fedc3598c89e132cae 100644 (file)
--- a/tox-update.ini
+++ b/tox-update.ini
@@ -80,6 +80,6 @@ commands=
       ceph_dev_sha1={env:UPDATE_CEPH_DEV_SHA1:latest} \
   "
 
-  bash -c "CEPH_STABLE_RELEASE={env:UPDATE_CEPH_STABLE_RELEASE:nautilus} py.test -n 8 --durations=0 --sudo -v --connection=ansible --ansible-inventory={changedir}/{env:INVENTORY} --ssh-config={changedir}/vagrant_ssh_config {toxinidir}/tests/functional/tests"
+  bash -c "CEPH_STABLE_RELEASE={env:UPDATE_CEPH_STABLE_RELEASE:nautilus} py.test --reruns 5 --reruns-delay 1 -n 8 --durations=0 --sudo -v --connection=ansible --ansible-inventory={changedir}/{env:INVENTORY} --ssh-config={changedir}/vagrant_ssh_config {toxinidir}/tests/functional/tests"
 
   vagrant destroy --force

diff --git a/tox.ini b/tox.ini
index a716ad5bccd18e82eebf30394576384572dd268e..9d40657d0c09445697ec91aeaf29c52d76384760 100644 (file)
--- a/tox.ini
+++ b/tox.ini
@@ -51,7 +51,7 @@ commands=
   # wait 30sec for services to be ready
   sleep 30
   # test cluster state using ceph-ansible tests
-  py.test -n 8 --durations=0 --sudo -v --connection=ansible --ansible-inventory={changedir}/hosts --ssh-config={changedir}/vagrant_ssh_config {envdir}/tmp/ceph-ansible/tests/functional/tests
+  py.test --reruns 5 --reruns-delay 1 -n 8 --durations=0 --sudo -v --connection=ansible --ansible-inventory={changedir}/hosts --ssh-config={changedir}/vagrant_ssh_config {envdir}/tmp/ceph-ansible/tests/functional/tests
 
   # install ceph-ansible@master requirements
   pip install -r {toxinidir}/tests/requirements.txt
@@ -68,13 +68,13 @@ commands=
   "
 
   # test cluster state again using ceph-ansible tests
-  bash -c "CEPH_STABLE_RELEASE=nautilus py.test -n 8 --durations=0 --sudo -v --connection=ansible --ansible-inventory={changedir}/hosts --ssh-config={changedir}/vagrant_ssh_config {toxinidir}/tests/functional/tests"
+  bash -c "CEPH_STABLE_RELEASE=nautilus py.test --reruns 5 --reruns-delay 1 -n 8 --durations=0 --sudo -v --connection=ansible --ansible-inventory={changedir}/hosts --ssh-config={changedir}/vagrant_ssh_config {toxinidir}/tests/functional/tests"
 
   # reboot all vms
   ansible-playbook -vv -i {changedir}/hosts {toxinidir}/tests/functional/reboot.yml
 
   # retest to ensure cluster came back up correctly after rebooting
-  bash -c "CEPH_STABLE_RELEASE=nautilus py.test -n 8 --durations=0 --sudo -v --connection=ansible --ansible-inventory={changedir}/hosts --ssh-config={changedir}/vagrant_ssh_config {toxinidir}/tests/functional/tests"
+  bash -c "CEPH_STABLE_RELEASE=nautilus py.test --reruns 5 --reruns-delay 1 -n 8 --durations=0 --sudo -v --connection=ansible --ansible-inventory={changedir}/hosts --ssh-config={changedir}/vagrant_ssh_config {toxinidir}/tests/functional/tests"
 
   vagrant destroy --force
 
@@ -138,7 +138,7 @@ commands=
       ceph_dev_sha1={env:CEPH_DEV_SHA1:latest} \
   "
   # test that the cluster can be redeployed in a healthy state
-  py.test -n 8 --durations=0 --sudo -v --connection=ansible --ansible-inventory={changedir}/{env:INVENTORY} --ssh-config={changedir}/vagrant_ssh_config {toxinidir}/tests/functional/tests
+  py.test --reruns 5 --reruns-delay 1 -n 8 --durations=0 --sudo -v --connection=ansible --ansible-inventory={changedir}/{env:INVENTORY} --ssh-config={changedir}/vagrant_ssh_config {toxinidir}/tests/functional/tests
 
 [purge-lvm]
 commands=
@@ -165,7 +165,7 @@ commands=
       ceph_dev_sha1={env:CEPH_DEV_SHA1:latest} \
   "
   # test that the cluster can be redeployed in a healthy state
-  py.test -n 8 --durations=0 --sudo -v --connection=ansible --ansible-inventory={changedir}/{env:INVENTORY} --ssh-config={changedir}/vagrant_ssh_config {toxinidir}/tests/functional/tests
+  py.test --reruns 5 --reruns-delay 1 -n 8 --durations=0 --sudo -v --connection=ansible --ansible-inventory={changedir}/{env:INVENTORY} --ssh-config={changedir}/vagrant_ssh_config {toxinidir}/tests/functional/tests
 
 [shrink-mon]
 commands=
@@ -196,7 +196,7 @@ commands=
       ceph_dev_sha1={env:CEPH_DEV_SHA1:latest} \
   "
 
-  py.test -n 8 --durations=0 --sudo -v --connection=ansible --ansible-inventory={changedir}/hosts-switch-to-containers --ssh-config={changedir}/vagrant_ssh_config {toxinidir}/tests/functional/tests
+  py.test --reruns 5 --reruns-delay 1 -n 8 --durations=0 --sudo -v --connection=ansible --ansible-inventory={changedir}/hosts-switch-to-containers --ssh-config={changedir}/vagrant_ssh_config {toxinidir}/tests/functional/tests
 
 [add-mons]
 commands=
@@ -211,7 +211,7 @@ commands=
       ceph_dev_branch={env:CEPH_DEV_BRANCH:master} \
       ceph_dev_sha1={env:CEPH_DEV_SHA1:latest} \
       "
-  py.test -n 8 --durations=0 --sudo -v --connection=ansible --ansible-inventory={changedir}/hosts-2 --ssh-config={changedir}/vagrant_ssh_config {toxinidir}/tests/functional/tests
+  py.test --reruns 5 --reruns-delay 1 -n 8 --durations=0 --sudo -v --connection=ansible --ansible-inventory={changedir}/hosts-2 --ssh-config={changedir}/vagrant_ssh_config {toxinidir}/tests/functional/tests
 
 [add-osds]
 commands=
@@ -227,7 +227,7 @@ commands=
       ceph_dev_branch={env:CEPH_DEV_BRANCH:master} \
       ceph_dev_sha1={env:CEPH_DEV_SHA1:latest} \
       "
-  py.test -n 8 --durations=0 --sudo -v --connection=ansible --ansible-inventory={changedir}/hosts-2 --ssh-config={changedir}/vagrant_ssh_config {toxinidir}/tests/functional/tests
+  py.test --reruns 5 --reruns-delay 1 -n 8 --durations=0 --sudo -v --connection=ansible --ansible-inventory={changedir}/hosts-2 --ssh-config={changedir}/vagrant_ssh_config {toxinidir}/tests/functional/tests
 
 [add-mgrs]
 commands=
@@ -257,7 +257,7 @@ commands=
       ceph_dev_branch={env:CEPH_DEV_BRANCH:master} \
       ceph_dev_sha1={env:CEPH_DEV_SHA1:latest} \
       "
-  py.test -n 8 --sudo -v --connection=ansible --ansible-inventory={changedir}/hosts-2 --ssh-config={changedir}/vagrant_ssh_config {toxinidir}/tests/functional/tests
+  py.test --reruns 5 --reruns-delay 1 -n 8 --sudo -v --connection=ansible --ansible-inventory={changedir}/hosts-2 --ssh-config={changedir}/vagrant_ssh_config {toxinidir}/tests/functional/tests
 
 [add-rbdmirrors]
 commands=
@@ -272,7 +272,7 @@ commands=
       ceph_dev_branch={env:CEPH_DEV_BRANCH:master} \
       ceph_dev_sha1={env:CEPH_DEV_SHA1:latest} \
       "
-  py.test -n 8 --sudo -v --connection=ansible --ansible-inventory={changedir}/hosts-2 --ssh-config={changedir}/vagrant_ssh_config {toxinidir}/tests/functional/tests
+  py.test --reruns 5 --reruns-delay 1 -n 8 --sudo -v --connection=ansible --ansible-inventory={changedir}/hosts-2 --ssh-config={changedir}/vagrant_ssh_config {toxinidir}/tests/functional/tests
 
 [add-rgws]
 commands=
@@ -287,7 +287,7 @@ commands=
       ceph_dev_branch={env:CEPH_DEV_BRANCH:master} \
       ceph_dev_sha1={env:CEPH_DEV_SHA1:latest} \
       "
-  py.test -n 8 --sudo -v --connection=ansible --ansible-inventory={changedir}/hosts-2 --ssh-config={changedir}/vagrant_ssh_config {toxinidir}/tests/functional/tests
+  py.test --reruns 5 --reruns-delay 1 -n 8 --sudo -v --connection=ansible --ansible-inventory={changedir}/hosts-2 --ssh-config={changedir}/vagrant_ssh_config {toxinidir}/tests/functional/tests
 
 [rgw-multisite]
 commands=
@@ -434,14 +434,14 @@ commands=
   # wait 30sec for services to be ready
   sleep 30
   # test cluster state using ceph-ansible tests
-  py.test -n 8 --durations=0 --sudo -v --connection=ansible --ansible-inventory={changedir}/{env:INVENTORY} --ssh-config={changedir}/vagrant_ssh_config {toxinidir}/tests/functional/tests
+  py.test --reruns 5 --reruns-delay 1 -n 8 --durations=0 --sudo -v --connection=ansible --ansible-inventory={changedir}/{env:INVENTORY} --ssh-config={changedir}/vagrant_ssh_config {toxinidir}/tests/functional/tests
 
   # reboot all vms
   all_daemons: ansible-playbook -vv -i {changedir}/{env:INVENTORY} {toxinidir}/tests/functional/reboot.yml
 
   # wait 30sec for services to be ready
   # retest to ensure cluster came back up correctly after rebooting
-  all_daemons: py.test -n 8 --durations=0 --sudo -v --connection=ansible --ansible-inventory={changedir}/{env:INVENTORY} --ssh-config={changedir}/vagrant_ssh_config {toxinidir}/tests/functional/tests
+  all_daemons: py.test --reruns 5 --reruns-delay 1 -n 8 --durations=0 --sudo -v --connection=ansible --ansible-inventory={changedir}/{env:INVENTORY} --ssh-config={changedir}/vagrant_ssh_config {toxinidir}/tests/functional/tests
 
   # handlers/idempotency test
   all_daemons: ansible-playbook -vv -i {changedir}/{env:INVENTORY} {toxinidir}/{env:PLAYBOOK:site.yml.sample} --extra-vars "delegate_facts_host={env:DELEGATE_FACTS_HOST:True} fetch_directory={env:FETCH_DIRECTORY:{changedir}/fetch} ceph_stable_release={env:CEPH_STABLE_RELEASE:nautilus} ceph_docker_registry={env:CEPH_DOCKER_REGISTRY:docker.io} ceph_docker_image={env:CEPH_DOCKER_IMAGE:ceph/daemon} ceph_docker_image_tag={env:CEPH_DOCKER_IMAGE_TAG_BIS:latest-bis-master} ceph_dev_branch={env:CEPH_DEV_BRANCH:master} ceph_dev_sha1={env:CEPH_DEV_SHA1:latest} copy_admin_key={env:COPY_ADMIN_KEY:False}" --extra-vars @ceph-override.json