From a6976700cb5298fb0946338e8bbc5262c002ebff Mon Sep 17 00:00:00 2001 From: Xiubo Li Date: Thu, 27 Aug 2020 01:40:52 -0400 Subject: [PATCH] rpm: retry installing the package if the mirror server is busy When installing some packages, if the mirror server failed with 503 code, which means the mirror server temporarily not available, we should retry it later. But the yum tool just skips it and retries other mirrors, which may not contain them. For the cephfs suites, there maybe will fire the many test cases at the same time, and for each test case it may fire several nodes to install tens of packages at the same time. This may cause the mirror server overloaded. We need one safe method to retry it. Fixes: https://tracker.ceph.com/issues/47166 Signed-off-by: Xiubo Li --- teuthology/task/install/rpm.py | 29 ++++++++++++++++++++++------- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/teuthology/task/install/rpm.py b/teuthology/task/install/rpm.py index f603ed1ced..7f513056fd 100644 --- a/teuthology/task/install/rpm.py +++ b/teuthology/task/install/rpm.py @@ -1,9 +1,11 @@ import logging import os.path +from io import StringIO from distutils.version import LooseVersion from teuthology.config import config as teuth_config +from teuthology.contextutil import safe_while from teuthology.orchestra import run from teuthology import packaging @@ -143,6 +145,18 @@ def _downgrade_packages(ctx, remote, pkgs, pkg_version, config): remote.run(args='sudo yum -y downgrade {}'.format(' '.join(pkgs_opt))) return [pkg for pkg in pkgs if pkg not in downgrade_pkgs] +def _retry_if_503_in_output(remote, args): + # wait at most 5 minutes + with safe_while(sleep=10, tries=30) as proceed: + while proceed(): + stdout = StringIO() + try: + return remote.run(args=args, stdout=stdout) + except run.CommandFailedError: + if "status code: 503" in stdout.getvalue().lower(): + continue + else: + raise def _update_package_list_and_install(ctx, remote, rpm, config): """ @@ -236,26 +250,27 @@ def _update_package_list_and_install(ctx, remote, rpm, config): rpm = _downgrade_packages(ctx, remote, rpm, pkg_version, config) if system_pkglist: - remote.run( + _retry_if_503_in_output(remote, args='{install_cmd} {rpms}' .format(install_cmd=install_cmd, rpms=' '.join(rpm)) ) else: for cpack in rpm: if ldir: - remote.run(args=''' + _retry_if_503_in_output(remote, + args=''' if test -e {pkg} ; then {remove_cmd} {pkg} ; {install_cmd} {pkg} ; else {install_cmd} {cpack} ; fi - '''.format(remove_cmd=remove_cmd, - install_cmd=install_cmd, - pkg=os.path.join(ldir, cpack), - cpack=cpack)) + '''.format(remove_cmd=remove_cmd, + install_cmd=install_cmd, + pkg=os.path.join(ldir, cpack), + cpack=cpack)) else: - remote.run( + _retry_if_503_in_output(remote, args='{install_cmd} {cpack}' .format(install_cmd=install_cmd, cpack=cpack) ) -- 2.39.5