deb: retry installing the package if resource temporarily unavailable

Fixes: https://tracker.ceph.com/issues/46878
Signed-off-by: Xiubo Li <xiubli@redhat.com>

diff --git a/teuthology/task/install/deb.py b/teuthology/task/install/deb.py
index 0a31b0d11edbf88bace99a090a45b882492c8c38..92ad7b2d62321d3980e5fbbe49a57a9093be574d 100644 (file)
--- a/teuthology/task/install/deb.py
+++ b/teuthology/task/install/deb.py
@@ -4,12 +4,32 @@ import os
 from io import StringIO
 
 from teuthology.orchestra import run
+from teuthology.contextutil import safe_while
 
 from teuthology.task.install.util import _get_builder_project, _get_local_dir
 
 
 log = logging.getLogger(__name__)
 
+def _retry_if_eagain_in_output(remote, args):
+    # wait at most 5 minutes
+    with safe_while(sleep=10, tries=30) as proceed:
+        while proceed():
+            stderr = StringIO()
+            try:
+                return remote.run(args=args, stderr=stderr)
+            except run.CommandFailedError:
+                if "could not get lock" in stderr.getvalue().lower():
+                    stdout = StringIO()
+                    args = ['sudo', 'fuser', '-v', '/var/lib/dpkg/lock-frontend']
+                    remote.run(args=args, stdout=stdout)
+                    log.info("The processes holding 'lock-frontend':\n{}".format(stdout.getvalue()))
+                    continue
+                else:
+                    raise
+
+def install_dep_packages(remote, args):
+    _retry_if_eagain_in_output(remote, args)
 
 def _update_package_list_and_install(ctx, remote, debs, config):
     """
@@ -71,11 +91,11 @@ def _update_package_list_and_install(ctx, remote, debs, config):
                 'Dpkg::Options::="--force-confold"'),
             'install',
         ]
-    remote.run(
+    install_dep_packages(remote,
         args=install_cmd + ['%s=%s' % (d, version) for d in debs],
     )
     if system_pkglist:
-        remote.run(
+        install_dep_packages(remote,
             args=install_cmd + system_pkglist,
         )
     ldir = _get_local_dir(config, remote)
@@ -195,7 +215,7 @@ def _upgrade_packages(ctx, config, remote, debs):
     builder.install_repo()
 
     remote.run(args=['sudo', 'apt-get', 'update'], check_status=False)
-    remote.run(
+    install_dep_packages(remote,
         args=[
             'sudo',
             'DEBIAN_FRONTEND=noninteractive', 'apt-get', '-y', '--force-yes',

diff --git a/teuthology/task/kernel.py b/teuthology/task/kernel.py
index 73f8071a0f579d7678a7303bbf03411e8f1e648b..159d05e437f499d19e40fc4b31362e1953392830 100644 (file)
--- a/teuthology/task/kernel.py
+++ b/teuthology/task/kernel.py
@@ -28,6 +28,7 @@ from teuthology.packaging import (
     get_koji_task_result,
     get_builder_project,
 )
+from teuthology.task.install.deb import install_dep_packages
 
 log = logging.getLogger(__name__)
 
@@ -1078,8 +1079,8 @@ def get_latest_image_version_deb(remote, ostype):
     # Note that a dependency list may have multiple comma-separated entries,
     # but also each entry may be an alternative (pkg1 | pkg2)
     if 'debian' in ostype:
-        remote.run(args=['sudo', 'apt-get', '-y', 'install',
-                         'linux-image-amd64'], stdout=output)
+        args=['sudo', 'apt-get', '-y', 'install', 'linux-image-amd64']
+        install_dep_packages(remote, args)
         remote.run(args=['dpkg', '-s', 'linux-image-amd64'], stdout=output)
         for line in output.getvalue().split('\n'):
             if 'Depends:' in line:
@@ -1089,23 +1090,23 @@ def get_latest_image_version_deb(remote, ostype):
     # Ubuntu is a depend in a depend.
     if 'ubuntu' in ostype:
         try:
-            remote.run(args=['sudo', 'DEBIAN_FRONTEND=noninteractive',
-                             'apt-get', '-y', 'install',
-                             'linux-image-current-generic'])
+            args=['sudo', 'DEBIAN_FRONTEND=noninteractive',
+                  'apt-get', '-y', 'install', 'linux-image-current-generic']
+            install_dep_packages(remote, args)
             remote.run(args=['dpkg', '-s', 'linux-image-current-generic'],
                        stdout=output)
             for line in output.getvalue().split('\n'):
                 if 'Depends:' in line:
                     depends = line.split('Depends: ')[1]
-                    remote.run(args=['sudo', 'apt-get', '-y', 'install',
-                                     depends])
+                    install_dep_packages(remote, args=['sudo', 'apt-get', '-y',
+                                                       'install', depends])
             remote.run(args=['dpkg', '-s', depends], stdout=output)
         except run.CommandFailedError:
             # Non precise ubuntu machines (like trusty) don't have
             # linux-image-current-generic so use linux-image-generic instead.
-            remote.run(args=['sudo', 'DEBIAN_FRONTEND=noninteractive',
-                             'apt-get', '-y', 'install',
-                             'linux-image-generic'], stdout=output)
+            args=['sudo', 'DEBIAN_FRONTEND=noninteractive',
+                  'apt-get', '-y', 'install', 'linux-image-generic']
+            install_dep_packages(remote, args)
             remote.run(args=['dpkg', '-s', 'linux-image-generic'],
                        stdout=output)
         for line in output.getvalue().split('\n'):