From d7e77012ef4fe523762928c5d0081ff773be7068 Mon Sep 17 00:00:00 2001 From: Guillaume Abrioux Date: Wed, 19 Dec 2018 14:55:01 +0100 Subject: [PATCH] retry on packages and repositories failures add register/until on all packaging related tasks to avoid non valid CI failure. Signed-off-by: Guillaume Abrioux --- infrastructure-playbooks/lv-create.yml | 2 ++ infrastructure-playbooks/lv-teardown.yml | 2 ++ .../untested-by-ci/make-osd-partitions.yml | 2 ++ .../untested-by-ci/migrate-journal-to-ssd.yml | 2 ++ .../recover-osds-after-ssd-journal-failure.yml | 2 ++ roles/ceph-agent/tasks/pre_requisite.yml | 2 ++ .../installs/configure_redhat_local_installation.yml | 2 ++ .../tasks/installs/debian_uca_repository.yml | 2 ++ .../tasks/installs/install_debian_packages.yml | 2 ++ .../tasks/installs/install_debian_rhcs_packages.yml | 2 ++ .../ceph-common/tasks/installs/install_on_debian.yml | 2 ++ .../tasks/installs/install_redhat_packages.yml | 6 ++++++ .../tasks/installs/install_suse_packages.yml | 4 ++++ .../tasks/pre_requisites/debian_prerequisites.yml | 9 +++++++++ .../tasks/pre_requisites/prerequisites.yml | 6 ++++++ roles/ceph-infra/tasks/ntp_debian.yml | 4 ++++ roles/ceph-infra/tasks/ntp_rpm.yml | 4 ++++ .../tasks/non-container/prerequisites.yml | 2 ++ roles/ceph-mds/tasks/non_containerized.yml | 2 ++ roles/ceph-mgr/tasks/pre_requisite.yml | 4 ++++ roles/ceph-nfs/tasks/ganesha_selinux_fix.yml | 2 ++ .../tasks/pre_requisite_non_container_debian.yml | 12 ++++++++++++ .../tasks/pre_requisite_non_container_red_hat.yml | 4 ++++ roles/ceph-osd/tasks/main.yml | 2 ++ roles/ceph-rbd-mirror/tasks/pre_requisite.yml | 2 ++ roles/ceph-rgw/tasks/openstack-keystone.yml | 4 ++++ tests/functional/lvm_setup.yml | 2 ++ tests/functional/rgw_multisite.yml | 2 ++ tests/functional/rhcs_setup.yml | 2 ++ tests/functional/setup.yml | 4 ++++ 30 files changed, 99 insertions(+) diff --git a/infrastructure-playbooks/lv-create.yml b/infrastructure-playbooks/lv-create.yml index 2bd75984d..a05ce15a0 100644 --- a/infrastructure-playbooks/lv-create.yml +++ b/infrastructure-playbooks/lv-create.yml @@ -38,6 +38,8 @@ package: name: lvm2 state: present + register: result + until: result is succeeded # Make entire nvme device a VG - name: add nvme device as lvm pv diff --git a/infrastructure-playbooks/lv-teardown.yml b/infrastructure-playbooks/lv-teardown.yml index 4be713475..a717f9f7e 100644 --- a/infrastructure-playbooks/lv-teardown.yml +++ b/infrastructure-playbooks/lv-teardown.yml @@ -29,6 +29,8 @@ package: name: lvm2 state: present + register: result + until: result is succeeded # BEGIN TEARDOWN - name: find any existing osd filesystems diff --git a/infrastructure-playbooks/untested-by-ci/make-osd-partitions.yml b/infrastructure-playbooks/untested-by-ci/make-osd-partitions.yml index 0fc6892d2..eb7f9f878 100644 --- a/infrastructure-playbooks/untested-by-ci/make-osd-partitions.yml +++ b/infrastructure-playbooks/untested-by-ci/make-osd-partitions.yml @@ -49,6 +49,8 @@ package: name: gdisk state: present + register: result + until: result is succeeded - name: erase all previous partitions(dangerous!!!) shell: sgdisk --zap-all -- /dev/{{item.device_name}} diff --git a/infrastructure-playbooks/untested-by-ci/migrate-journal-to-ssd.yml b/infrastructure-playbooks/untested-by-ci/migrate-journal-to-ssd.yml index 44a75e01b..e999759ed 100644 --- a/infrastructure-playbooks/untested-by-ci/migrate-journal-to-ssd.yml +++ b/infrastructure-playbooks/untested-by-ci/migrate-journal-to-ssd.yml @@ -58,6 +58,8 @@ package: name: gdisk state: present + register: result + until: result is succeeded when: osds_journal_devices is defined - name: generate uuid for osds journal diff --git a/infrastructure-playbooks/untested-by-ci/recover-osds-after-ssd-journal-failure.yml b/infrastructure-playbooks/untested-by-ci/recover-osds-after-ssd-journal-failure.yml index de3b6e86e..d19fe129c 100644 --- a/infrastructure-playbooks/untested-by-ci/recover-osds-after-ssd-journal-failure.yml +++ b/infrastructure-playbooks/untested-by-ci/recover-osds-after-ssd-journal-failure.yml @@ -80,6 +80,8 @@ package: name: gdisk state: present + register: result + until: result is succeeded - name: get osd(s) journal uuid command: cat "/var/lib/ceph/osd/{{ cluster }}-{{ item.1.osd_id }}/journal_uuid" diff --git a/roles/ceph-agent/tasks/pre_requisite.yml b/roles/ceph-agent/tasks/pre_requisite.yml index 67dab66ab..3b1ed5d98 100644 --- a/roles/ceph-agent/tasks/pre_requisite.yml +++ b/roles/ceph-agent/tasks/pre_requisite.yml @@ -5,6 +5,8 @@ package: name: rhscon-agent state: present + register: result + until: result is succeeded tags: - package-install diff --git a/roles/ceph-common/tasks/installs/configure_redhat_local_installation.yml b/roles/ceph-common/tasks/installs/configure_redhat_local_installation.yml index 5edd179dc..941f23929 100644 --- a/roles/ceph-common/tasks/installs/configure_redhat_local_installation.yml +++ b/roles/ceph-common/tasks/installs/configure_redhat_local_installation.yml @@ -31,6 +31,8 @@ package: name: rsync state: present + register: result + until: result is succeeded - name: synchronize ceph install synchronize: diff --git a/roles/ceph-common/tasks/installs/debian_uca_repository.yml b/roles/ceph-common/tasks/installs/debian_uca_repository.yml index 76aea933b..92ff0c6d4 100644 --- a/roles/ceph-common/tasks/installs/debian_uca_repository.yml +++ b/roles/ceph-common/tasks/installs/debian_uca_repository.yml @@ -2,6 +2,8 @@ - name: add ubuntu cloud archive key package package: name: ubuntu-cloud-keyring + register: result + until: result is succeeded - name: add ubuntu cloud archive repository apt_repository: diff --git a/roles/ceph-common/tasks/installs/install_debian_packages.yml b/roles/ceph-common/tasks/installs/install_debian_packages.yml index fc14e01ac..1af787006 100644 --- a/roles/ceph-common/tasks/installs/install_debian_packages.yml +++ b/roles/ceph-common/tasks/installs/install_debian_packages.yml @@ -5,3 +5,5 @@ update_cache: no state: "{{ (upgrade_ceph_packages|bool) | ternary('latest','present') }}" default_release: "{{ ceph_stable_release_uca | default(omit) }}{{ ansible_distribution_release ~ '-backports' if ceph_origin == 'distro' and ceph_use_distro_backports else '' }}" + register: result + until: result is succeeded \ No newline at end of file diff --git a/roles/ceph-common/tasks/installs/install_debian_rhcs_packages.yml b/roles/ceph-common/tasks/installs/install_debian_rhcs_packages.yml index 257f18d69..f1a25197f 100644 --- a/roles/ceph-common/tasks/installs/install_debian_rhcs_packages.yml +++ b/roles/ceph-common/tasks/installs/install_debian_rhcs_packages.yml @@ -3,3 +3,5 @@ apt: pkg: "{{ debian_ceph_pkgs | unique }}" state: "{{ (upgrade_ceph_packages|bool) | ternary('latest','present') }}" + register: result + until: result is succeeded \ No newline at end of file diff --git a/roles/ceph-common/tasks/installs/install_on_debian.yml b/roles/ceph-common/tasks/installs/install_on_debian.yml index 30b70a97b..5c1730bdc 100644 --- a/roles/ceph-common/tasks/installs/install_on_debian.yml +++ b/roles/ceph-common/tasks/installs/install_on_debian.yml @@ -15,6 +15,8 @@ state: present update_cache: yes cache_valid_time: 3600 + register: result + until: result is succeeded - name: include install_debian_packages.yml include_tasks: install_debian_packages.yml diff --git a/roles/ceph-common/tasks/installs/install_redhat_packages.yml b/roles/ceph-common/tasks/installs/install_redhat_packages.yml index 4446465a9..d692c1758 100644 --- a/roles/ceph-common/tasks/installs/install_redhat_packages.yml +++ b/roles/ceph-common/tasks/installs/install_redhat_packages.yml @@ -3,6 +3,8 @@ package: name: "{{ redhat_package_dependencies }}" state: present + register: result + until: result is succeeded when: - ansible_distribution == 'RedHat' @@ -10,6 +12,8 @@ yum: name: "{{ centos_package_dependencies }}" state: present + register: result + until: result is succeeded when: - ansible_distribution == 'CentOS' @@ -17,3 +21,5 @@ package: name: "{{ redhat_ceph_pkgs | unique }}" state: "{{ (upgrade_ceph_packages|bool) | ternary('latest','present') }}" + register: result + until: result is succeeded \ No newline at end of file diff --git a/roles/ceph-common/tasks/installs/install_suse_packages.yml b/roles/ceph-common/tasks/installs/install_suse_packages.yml index c202c5ece..f7a4593a8 100644 --- a/roles/ceph-common/tasks/installs/install_suse_packages.yml +++ b/roles/ceph-common/tasks/installs/install_suse_packages.yml @@ -3,8 +3,12 @@ package: name: "{{ suse_package_dependencies }}" state: present + register: result + until: result is succeeded - name: install suse ceph packages package: name: "{{ suse_ceph_pkgs | unique }}" state: "{{ (upgrade_ceph_packages|bool) | ternary('latest','present') }}" + register: result + until: result is succeeded \ No newline at end of file diff --git a/roles/ceph-container-common/tasks/pre_requisites/debian_prerequisites.yml b/roles/ceph-container-common/tasks/pre_requisites/debian_prerequisites.yml index 3958a5188..c1011f2fc 100644 --- a/roles/ceph-container-common/tasks/pre_requisites/debian_prerequisites.yml +++ b/roles/ceph-container-common/tasks/pre_requisites/debian_prerequisites.yml @@ -4,6 +4,8 @@ name: ['apt-transport-https', 'ca-certificates', 'curl', 'software-properties-common'] state: present update_cache: yes + register: result + until: result is succeeded - name: add docker's gpg key apt_key: @@ -35,19 +37,26 @@ state: present default_release: testing update_cache: yes + register: result + until: result is succeeded - name: install docker-py via pip for debian pip: name: docker-py state: latest + register: result + until: result is succeeded tags: - skip_ansible_lint + - name: install docker on debian package: name: "{{ 'docker-ce' if ansible_architecture == 'aarch64' else 'docker-engine' }}" state: present update_cache: yes + register: result + until: result is succeeded # NOTE (jimcurtis): need at least version 1.9.0 of six or we get: # re:NameError: global name 'DEFAULT_DOCKER_API_VERSION' is not defined diff --git a/roles/ceph-container-common/tasks/pre_requisites/prerequisites.yml b/roles/ceph-container-common/tasks/pre_requisites/prerequisites.yml index 30d74f248..9e1dac16c 100644 --- a/roles/ceph-container-common/tasks/pre_requisites/prerequisites.yml +++ b/roles/ceph-container-common/tasks/pre_requisites/prerequisites.yml @@ -13,6 +13,8 @@ name: python-six state: present update_cache: yes + register: result + until: result is succeeded tags: with_pkg when: @@ -25,6 +27,8 @@ name: ['docker.io', 'python-docker'] state: present update_cache: yes + register: result + until: result is succeeded tags: with_pkg when: @@ -48,6 +52,8 @@ package: name: ['docker', 'python-docker-py'] state: present + register: result + until: result is succeeded tags: with_pkg when: diff --git a/roles/ceph-infra/tasks/ntp_debian.yml b/roles/ceph-infra/tasks/ntp_debian.yml index 80c8fac82..db3edc322 100644 --- a/roles/ceph-infra/tasks/ntp_debian.yml +++ b/roles/ceph-infra/tasks/ntp_debian.yml @@ -7,6 +7,8 @@ package: name: ntp state: present + register: result + until: result is succeeded - name: start ntp service: name: ntp @@ -22,6 +24,8 @@ package: name: chrony state: present + register: result + until: result is succeeded - name: start chronyd service: name: chronyd diff --git a/roles/ceph-infra/tasks/ntp_rpm.yml b/roles/ceph-infra/tasks/ntp_rpm.yml index 9e4d38cd2..d8d41e2b0 100644 --- a/roles/ceph-infra/tasks/ntp_rpm.yml +++ b/roles/ceph-infra/tasks/ntp_rpm.yml @@ -7,6 +7,8 @@ package: name: ntp state: present + register: result + until: result is succeeded - name: start ntp service: name: ntpd @@ -22,6 +24,8 @@ package: name: chrony state: present + register: result + until: result is succeeded - name: start chrony service: name: chronyd diff --git a/roles/ceph-iscsi-gw/tasks/non-container/prerequisites.yml b/roles/ceph-iscsi-gw/tasks/non-container/prerequisites.yml index 0e1d60dfb..3fa2dbbdd 100644 --- a/roles/ceph-iscsi-gw/tasks/non-container/prerequisites.yml +++ b/roles/ceph-iscsi-gw/tasks/non-container/prerequisites.yml @@ -37,6 +37,8 @@ package: name: "{{ item }}" state: "{{ (upgrade_ceph_packages|bool) | ternary('latest','present') }}" + register: result + until: result is succeeded with_items: - tcmu-runner - ceph-iscsi-config diff --git a/roles/ceph-mds/tasks/non_containerized.yml b/roles/ceph-mds/tasks/non_containerized.yml index 3546dd646..fe5c68c8e 100644 --- a/roles/ceph-mds/tasks/non_containerized.yml +++ b/roles/ceph-mds/tasks/non_containerized.yml @@ -12,6 +12,8 @@ package: name: "ceph-mds" state: "{{ (upgrade_ceph_packages|bool) | ternary('latest','present') }}" + register: result + until: result is succeeded when: - mds_group_name in group_names - ansible_os_family == 'RedHat' diff --git a/roles/ceph-mgr/tasks/pre_requisite.yml b/roles/ceph-mgr/tasks/pre_requisite.yml index 401e5fd6d..b3a1d587b 100644 --- a/roles/ceph-mgr/tasks/pre_requisite.yml +++ b/roles/ceph-mgr/tasks/pre_requisite.yml @@ -3,6 +3,8 @@ package: name: ceph-mgr state: "{{ (upgrade_ceph_packages|bool) | ternary('latest','present') }}" + register: result + until: result is succeeded when: - ansible_os_family in ['RedHat', 'Suse'] @@ -11,5 +13,7 @@ name: ceph-mgr state: "{{ (upgrade_ceph_packages|bool) | ternary('latest','present') }}" default_release: "{{ ceph_stable_release_uca | default(omit) }}{{ ansible_distribution_release ~ '-backports' if ceph_origin == 'distro' and ceph_use_distro_backports else '' }}" + register: result + until: result is succeeded when: - ansible_os_family == 'Debian' diff --git a/roles/ceph-nfs/tasks/ganesha_selinux_fix.yml b/roles/ceph-nfs/tasks/ganesha_selinux_fix.yml index 3546c8280..1c3c0ba7d 100644 --- a/roles/ceph-nfs/tasks/ganesha_selinux_fix.yml +++ b/roles/ceph-nfs/tasks/ganesha_selinux_fix.yml @@ -10,6 +10,8 @@ package: name: policycoreutils-python state: present + register: result + until: result is succeeded when: - selinuxstatus.stdout != 'Disabled' diff --git a/roles/ceph-nfs/tasks/pre_requisite_non_container_debian.yml b/roles/ceph-nfs/tasks/pre_requisite_non_container_debian.yml index 21324116d..cc392c86b 100644 --- a/roles/ceph-nfs/tasks/pre_requisite_non_container_debian.yml +++ b/roles/ceph-nfs/tasks/pre_requisite_non_container_debian.yml @@ -53,15 +53,21 @@ name: libjemalloc1 state: "{{ (upgrade_ceph_packages|bool) | ternary('latest','present') }}" update_cache: yes + register: result + until: result is succeeded - name: install nfs rgw/cephfs gateway - debian apt: name: ['nfs-ganesha-rgw', 'radosgw'] allow_unauthenticated: yes + register: result + until: result is succeeded when: nfs_obj_gw - name: install nfs rgw/cephfs gateway - debian apt: name: nfs-ganesha-ceph allow_unauthenticated: yes + register: result + until: result is succeeded when: nfs_file_gw when: - (ceph_origin == 'repository' or ceph_origin == 'distro') @@ -73,16 +79,22 @@ apt: name: nfs-ganesha state: "{{ (upgrade_ceph_packages|bool) | ternary('latest','present') }}" + register: result + until: result is succeeded - name: install red hat storage nfs file gateway apt: name: nfs-ganesha-ceph state: "{{ (upgrade_ceph_packages|bool) | ternary('latest','present') }}" + register: result + until: result is succeeded when: - nfs_file_gw - name: install red hat storage nfs obj gateway apt: name: nfs-ganesha-rgw state: "{{ (upgrade_ceph_packages|bool) | ternary('latest','present') }}" + register: result + until: result is succeeded when: - nfs_obj_gw when: diff --git a/roles/ceph-nfs/tasks/pre_requisite_non_container_red_hat.yml b/roles/ceph-nfs/tasks/pre_requisite_non_container_red_hat.yml index d7ebcc389..9878f16bc 100644 --- a/roles/ceph-nfs/tasks/pre_requisite_non_container_red_hat.yml +++ b/roles/ceph-nfs/tasks/pre_requisite_non_container_red_hat.yml @@ -40,6 +40,8 @@ package: name: nfs-ganesha-ceph state: "{{ (upgrade_ceph_packages|bool) | ternary('latest','present') }}" + register: result + until: result is succeeded when: - nfs_file_gw @@ -47,5 +49,7 @@ package: name: ['nfs-ganesha-rgw', 'ceph-radosgw'] state: "{{ (upgrade_ceph_packages|bool) | ternary('latest','present') }}" + register: result + until: result is succeeded when: - nfs_obj_gw diff --git a/roles/ceph-osd/tasks/main.yml b/roles/ceph-osd/tasks/main.yml index 48f109b01..082109d10 100644 --- a/roles/ceph-osd/tasks/main.yml +++ b/roles/ceph-osd/tasks/main.yml @@ -6,6 +6,8 @@ package: name: parted state: present + register: result + until: result is succeeded when: - not containerized_deployment - ansible_os_family != 'ClearLinux' diff --git a/roles/ceph-rbd-mirror/tasks/pre_requisite.yml b/roles/ceph-rbd-mirror/tasks/pre_requisite.yml index ba2a1c6f2..1adba11ca 100644 --- a/roles/ceph-rbd-mirror/tasks/pre_requisite.yml +++ b/roles/ceph-rbd-mirror/tasks/pre_requisite.yml @@ -5,6 +5,8 @@ package: name: rbd-mirror state: present + register: result + until: result is succeeded tags: - package-install diff --git a/roles/ceph-rgw/tasks/openstack-keystone.yml b/roles/ceph-rgw/tasks/openstack-keystone.yml index 5514c6f4b..2579f1a19 100644 --- a/roles/ceph-rgw/tasks/openstack-keystone.yml +++ b/roles/ceph-rgw/tasks/openstack-keystone.yml @@ -3,6 +3,8 @@ package: name: nss-tools state: present + register: result + until: result is succeeded when: - ansible_pkg_mgr == 'yum' or ansible_pkg_mgr == 'dnf' @@ -10,6 +12,8 @@ package: name: libnss3-tools state: present + register: result + until: result is succeeded when: - ansible_pkg_mgr == 'apt' diff --git a/tests/functional/lvm_setup.yml b/tests/functional/lvm_setup.yml index c6908a4be..c87b73914 100644 --- a/tests/functional/lvm_setup.yml +++ b/tests/functional/lvm_setup.yml @@ -23,6 +23,8 @@ package: name: lvm2 state: present + register: result + until: result is succeeded when: - not is_atomic diff --git a/tests/functional/rgw_multisite.yml b/tests/functional/rgw_multisite.yml index 27e55b375..45502c560 100644 --- a/tests/functional/rgw_multisite.yml +++ b/tests/functional/rgw_multisite.yml @@ -19,6 +19,8 @@ package: name: s3cmd state: present + register: result + until: result is succeeded when: - not is_atomic diff --git a/tests/functional/rhcs_setup.yml b/tests/functional/rhcs_setup.yml index df45c1d19..abbb4030f 100644 --- a/tests/functional/rhcs_setup.yml +++ b/tests/functional/rhcs_setup.yml @@ -80,6 +80,8 @@ package: name: docker state: present + register: result + until: result is succeeded when: ansible_os_family == 'RedHat' - name: allow insecure docker registries diff --git a/tests/functional/setup.yml b/tests/functional/setup.yml index 989ae5078..752b0aa30 100644 --- a/tests/functional/setup.yml +++ b/tests/functional/setup.yml @@ -20,6 +20,8 @@ package: name: net-tools state: present + register: result + until: result is succeeded when: - not is_atomic @@ -35,6 +37,8 @@ package: name: epel-release state: present + register: result + until: result is succeeded - name: enable local epel repository ini_file: path: /etc/yum.repos.d/epel.repo -- 2.39.5