rpm: retry installing the package if the mirror server is busy

When installing some packages, if the mirror server failed with
503 code, which means the mirror server temporarily not available,
we should retry it later. But the yum tool just skips it and
retries other mirrors, which may not contain them.

For the cephfs suites, there maybe will fire the many test cases
at the same time, and for each test case it may fire several nodes
to install tens of packages at the same time. This may cause the
mirror server overloaded.

We need one safe method to retry it.

Fixes: https://tracker.ceph.com/issues/47166
Signed-off-by: Xiubo Li <xiubli@redhat.com>

diff --git a/teuthology/task/install/rpm.py b/teuthology/task/install/rpm.py
index f603ed1ced21a0193e36d763c75ed479937c3149..7f513056fd0a5a6c71df9ca1c5c91792b2d5a658 100644 (file)
--- a/teuthology/task/install/rpm.py
+++ b/teuthology/task/install/rpm.py
@@ -1,9 +1,11 @@
 import logging
 import os.path
+from io import StringIO
 
 from distutils.version import LooseVersion
 
 from teuthology.config import config as teuth_config
+from teuthology.contextutil import safe_while
 from teuthology.orchestra import run
 from teuthology import packaging
 
@@ -143,6 +145,18 @@ def _downgrade_packages(ctx, remote, pkgs, pkg_version, config):
     remote.run(args='sudo yum -y downgrade {}'.format(' '.join(pkgs_opt)))
     return [pkg for pkg in pkgs if pkg not in downgrade_pkgs]
 
+def _retry_if_503_in_output(remote, args):
+    # wait at most 5 minutes
+    with safe_while(sleep=10, tries=30) as proceed:
+        while proceed():
+            stdout = StringIO()
+            try:
+                return remote.run(args=args, stdout=stdout)
+            except run.CommandFailedError:
+                if "status code: 503" in stdout.getvalue().lower():
+                    continue
+                else:
+                    raise
 
 def _update_package_list_and_install(ctx, remote, rpm, config):
     """
@@ -236,26 +250,27 @@ def _update_package_list_and_install(ctx, remote, rpm, config):
         rpm = _downgrade_packages(ctx, remote, rpm, pkg_version, config)
 
     if system_pkglist:
-        remote.run(
+        _retry_if_503_in_output(remote,
             args='{install_cmd} {rpms}'
                  .format(install_cmd=install_cmd, rpms=' '.join(rpm))
             )
     else:
         for cpack in rpm:
             if ldir:
-                remote.run(args='''
+                _retry_if_503_in_output(remote,
+                  args='''
                   if test -e {pkg} ; then
                     {remove_cmd} {pkg} ;
                     {install_cmd} {pkg} ;
                   else
                     {install_cmd} {cpack} ;
                   fi
-                '''.format(remove_cmd=remove_cmd,
-                           install_cmd=install_cmd,
-                           pkg=os.path.join(ldir, cpack),
-                           cpack=cpack))
+                  '''.format(remove_cmd=remove_cmd,
+                             install_cmd=install_cmd,
+                             pkg=os.path.join(ldir, cpack),
+                             cpack=cpack))
             else:
-                remote.run(
+                _retry_if_503_in_output(remote,
                     args='{install_cmd} {cpack}'
                          .format(install_cmd=install_cmd, cpack=cpack)
                     )