From: Zack Cerza <zack@redhat.com>
Date: Tue, 8 Nov 2016 00:30:05 +0000 (-0700)
Subject: Create a task.install.deb submodule
X-Git-Tag: 1.1.0~503^2~4
X-Git-Url: http://git.apps.os.sepia.ceph.com/?a=commitdiff_plain;h=d8a58af7aaf51f7cb43546a6c0a03615288dbb8d;p=teuthology.git

Create a task.install.deb submodule

This will hold most of the code specific to Debian-based systems

Signed-off-by: Zack Cerza <zack@redhat.com>
---

diff --git a/teuthology/task/install/__init__.py b/teuthology/task/install/__init__.py
index 5e5fdabc33..df33470a2e 100644
--- a/teuthology/task/install/__init__.py
+++ b/teuthology/task/install/__init__.py
@@ -15,83 +15,14 @@ from teuthology.orchestra import run
 from teuthology.task import ansible
 
 from .util import (
-    _get_builder_project, get_flavor, ship_utilities, _get_local_dir
+    _get_builder_project, get_flavor, ship_utilities,
 )
 
-from . import rpm
+from . import rpm, deb
 
 log = logging.getLogger(__name__)
 
 
-def _update_deb_package_list_and_install(ctx, remote, debs, config):
-    """
-    Runs ``apt-get update`` first, then runs ``apt-get install``, installing
-    the requested packages on the remote system.
-
-    TODO: split this into at least two functions.
-
-    :param ctx: the argparse.Namespace object
-    :param remote: the teuthology.orchestra.remote.Remote object
-    :param debs: list of packages names to install
-    :param config: the config dict
-    """
-
-    # check for ceph release key
-    r = remote.run(
-        args=[
-            'sudo', 'apt-key', 'list', run.Raw('|'), 'grep', 'Ceph',
-        ],
-        stdout=StringIO(),
-        check_status=False,
-    )
-    if r.stdout.getvalue().find('Ceph automated package') == -1:
-        # if it doesn't exist, add it
-        remote.run(
-            args=[
-                'wget', '-q', '-O-',
-                'http://git.ceph.com/?p=ceph.git;a=blob_plain;f=keys/autobuild.asc',
-                run.Raw('|'),
-                'sudo', 'apt-key', 'add', '-',
-            ],
-            stdout=StringIO(),
-        )
-
-    builder = util._get_builder_project(ctx, remote, config)
-    log.info("Installing packages: {pkglist} on remote deb {arch}".format(
-        pkglist=", ".join(debs), arch=builder.arch)
-    )
-    # get baseurl
-    log.info('Pulling from %s', builder.base_url)
-
-    version = builder.version
-    log.info('Package version is %s', version)
-
-    remote.run(
-        args=[
-            'echo', 'deb', builder.base_url, builder.codename, 'main',
-            run.Raw('|'),
-            'sudo', 'tee', '/etc/apt/sources.list.d/{proj}.list'.format(
-                proj=config.get('project', 'ceph')),
-        ],
-        stdout=StringIO(),
-    )
-    remote.run(args=['sudo', 'apt-get', 'update'], check_status=False)
-    remote.run(
-        args=[
-            'sudo', 'DEBIAN_FRONTEND=noninteractive', 'apt-get', '-y',
-            '--force-yes',
-            '-o', run.Raw('Dpkg::Options::="--force-confdef"'), '-o', run.Raw(
-                'Dpkg::Options::="--force-confold"'),
-            'install',
-        ] + ['%s=%s' % (d, version) for d in debs],
-    )
-    ldir = util._get_local_dir(config, remote)
-    if ldir:
-        for fyle in os.listdir(ldir):
-            fname = "%s/%s" % (ldir, fyle)
-            remote.run(args=['sudo', 'dpkg', '-i', fname],)
-
-
 def verify_package_version(ctx, config, remote):
     """
     Ensures that the version of package installed is what
@@ -175,7 +106,7 @@ def install_packages(ctx, pkgs, config):
     :param config: the config dict
     """
     install_pkgs = {
-        "deb": _update_deb_package_list_and_install,
+        "deb": deb._update_package_list_and_install,
         "rpm": rpm._update_package_list_and_install,
     }
     with parallel() as p:
@@ -190,61 +121,6 @@ def install_packages(ctx, pkgs, config):
         verify_package_version(ctx, config, remote)
 
 
-def _remove_deb(ctx, config, remote, debs):
-    """
-    Removes Debian packages from remote, rudely
-
-    TODO: be less rude (e.g. using --force-yes)
-
-    :param ctx: the argparse.Namespace object
-    :param config: the config dict
-    :param remote: the teuthology.orchestra.remote.Remote object
-    :param debs: list of packages names to install
-    """
-    log.info("Removing packages: {pkglist} on Debian system.".format(
-        pkglist=", ".join(debs)))
-    # first ask nicely
-    remote.run(
-        args=[
-            'for', 'd', 'in',
-        ] + debs + [
-            run.Raw(';'),
-            'do',
-            'sudo', 'DEBIAN_FRONTEND=noninteractive', 'apt-get', '-y', '--force-yes',
-            '-o', run.Raw('Dpkg::Options::="--force-confdef"'), '-o', run.Raw(
-                'Dpkg::Options::="--force-confold"'), 'purge',
-            run.Raw('$d'),
-            run.Raw('||'),
-            'true',
-            run.Raw(';'),
-            'done',
-        ])
-    # mop up anything that is broken
-    remote.run(
-        args=[
-            'dpkg', '-l',
-            run.Raw('|'),
-            # Any package that is unpacked or half-installed and also requires
-            # reinstallation
-            'grep', '^.\(U\|H\)R',
-            run.Raw('|'),
-            'awk', '{print $2}',
-            run.Raw('|'),
-            'sudo',
-            'xargs', '--no-run-if-empty',
-            'dpkg', '-P', '--force-remove-reinstreq',
-        ])
-    # then let apt clean up
-    remote.run(
-        args=[
-            'sudo', 'DEBIAN_FRONTEND=noninteractive', 'apt-get', '-y', '--force-yes',
-            '-o', run.Raw('Dpkg::Options::="--force-confdef"'), '-o', run.Raw(
-                'Dpkg::Options::="--force-confold"'),
-            'autoremove',
-        ],
-    )
-
-
 def remove_packages(ctx, config, pkgs):
     """
     Removes packages from each remote in ctx.
@@ -254,7 +130,7 @@ def remove_packages(ctx, config, pkgs):
     :param pkgs: list of packages names to remove
     """
     remove_pkgs = {
-        "deb": _remove_deb,
+        "deb": deb._remove,
         "rpm": rpm._remove,
     }
     with parallel() as p:
@@ -264,25 +140,6 @@ def remove_packages(ctx, config, pkgs):
                     system_type], ctx, config, remote, pkgs[system_type])
 
 
-def _remove_sources_list_deb(remote, proj):
-    """
-    Removes /etc/apt/sources.list.d/{proj}.list and then runs ``apt-get
-    update``.
-
-    :param remote: the teuthology.orchestra.remote.Remote object
-    :param proj: the project whose sources.list needs removing
-    """
-    remote.run(
-        args=[
-            'sudo', 'rm', '-f', '/etc/apt/sources.list.d/{proj}.list'.format(
-                proj=proj),
-            run.Raw('&&'),
-            'sudo', 'apt-get', 'update',
-        ],
-        check_status=False,
-    )
-
-
 def remove_sources(ctx, config):
     """
     Removes repo source files from each remote in ctx.
@@ -291,7 +148,7 @@ def remove_sources(ctx, config):
     :param config: the config dict
     """
     remove_sources_pkgs = {
-        'deb': _remove_sources_list_deb,
+        'deb': deb._remove_sources_list,
         'rpm': rpm._remove_sources_list,
     }
     with parallel() as p:
@@ -402,65 +259,6 @@ def install(ctx, config):
             purge_data(ctx)
 
 
-def _upgrade_deb_packages(ctx, config, remote, debs):
-    """
-    Upgrade project's packages on remote Debian host
-    Before doing so, installs the project's GPG key, writes a sources.list
-    file, and runs ``apt-get update``.
-
-    :param ctx: the argparse.Namespace object
-    :param config: the config dict
-    :param remote: the teuthology.orchestra.remote.Remote object
-    :param debs: the Debian packages to be installed
-    :param branch: the branch of the project to be used
-    """
-    # check for ceph release key
-    r = remote.run(
-        args=[
-            'sudo', 'apt-key', 'list', run.Raw('|'), 'grep', 'Ceph',
-        ],
-        stdout=StringIO(),
-        check_status=False,
-    )
-    if r.stdout.getvalue().find('Ceph automated package') == -1:
-        # if it doesn't exist, add it
-        remote.run(
-            args=[
-                'wget', '-q', '-O-',
-                'http://git.ceph.com/?p=ceph.git;a=blob_plain;f=keys/autobuild.asc',
-                run.Raw('|'),
-                'sudo', 'apt-key', 'add', '-',
-            ],
-            stdout=StringIO(),
-        )
-
-    builder = util._get_builder_project(ctx, remote, config)
-    base_url = builder.base_url
-    log.info('Pulling from %s', base_url)
-
-    version = builder.version
-    log.info('Package version is %s', version)
-
-    remote.run(
-        args=[
-            'echo', 'deb', base_url, builder.codename, 'main',
-            run.Raw('|'),
-            'sudo', 'tee', '/etc/apt/sources.list.d/{proj}.list'.format(
-                proj=config.get('project', 'ceph')),
-        ],
-        stdout=StringIO(),
-    )
-    remote.run(args=['sudo', 'apt-get', 'update'], check_status=False)
-    remote.run(
-        args=[
-            'sudo', 'DEBIAN_FRONTEND=noninteractive', 'apt-get', '-y', '--force-yes',
-            '-o', run.Raw('Dpkg::Options::="--force-confdef"'), '-o', run.Raw(
-                'Dpkg::Options::="--force-confold"'),
-            'install',
-        ] + ['%s=%s' % (d, version) for d in debs],
-    )
-
-
 @contextlib.contextmanager
 def rh_install(ctx, config):
     """
@@ -612,10 +410,11 @@ def upgrade_old_style(ctx, node, remote, pkgs, system_type):
     Handle the upgrade using methods in use prior to ceph-deploy.
     """
     if system_type == 'deb':
-        _upgrade_deb_packages(ctx, node, remote, pkgs)
+        deb._upgrade_packages(ctx, node, remote, pkgs)
     elif system_type == 'rpm':
         rpm._upgrade_packages(ctx, node, remote, pkgs)
 
+
 def upgrade_with_ceph_deploy(ctx, node, remote, pkgs, sys_type):
     """
     Upgrade using ceph-deploy
diff --git a/teuthology/task/install/deb.py b/teuthology/task/install/deb.py
new file mode 100644
index 0000000000..6135979e13
--- /dev/null
+++ b/teuthology/task/install/deb.py
@@ -0,0 +1,216 @@
+import logging
+import os
+
+from cStringIO import StringIO
+
+from teuthology.orchestra import run
+
+from .util import _get_builder_project, _get_local_dir
+
+
+log = logging.getLogger(__name__)
+
+
+def _update_package_list_and_install(ctx, remote, debs, config):
+    """
+    Runs ``apt-get update`` first, then runs ``apt-get install``, installing
+    the requested packages on the remote system.
+
+    TODO: split this into at least two functions.
+
+    :param ctx: the argparse.Namespace object
+    :param remote: the teuthology.orchestra.remote.Remote object
+    :param debs: list of packages names to install
+    :param config: the config dict
+    """
+
+    # check for ceph release key
+    r = remote.run(
+        args=[
+            'sudo', 'apt-key', 'list', run.Raw('|'), 'grep', 'Ceph',
+        ],
+        stdout=StringIO(),
+        check_status=False,
+    )
+    if r.stdout.getvalue().find('Ceph automated package') == -1:
+        # if it doesn't exist, add it
+        remote.run(
+            args=[
+                'wget', '-q', '-O-',
+                'http://git.ceph.com/?p=ceph.git;a=blob_plain;f=keys/autobuild.asc',  # noqa
+                run.Raw('|'),
+                'sudo', 'apt-key', 'add', '-',
+            ],
+            stdout=StringIO(),
+        )
+
+    builder = _get_builder_project(ctx, remote, config)
+    log.info("Installing packages: {pkglist} on remote deb {arch}".format(
+        pkglist=", ".join(debs), arch=builder.arch)
+    )
+    # get baseurl
+    log.info('Pulling from %s', builder.base_url)
+
+    version = builder.version
+    log.info('Package version is %s', version)
+
+    remote.run(
+        args=[
+            'echo', 'deb', builder.base_url, builder.codename, 'main',
+            run.Raw('|'),
+            'sudo', 'tee', '/etc/apt/sources.list.d/{proj}.list'.format(
+                proj=config.get('project', 'ceph')),
+        ],
+        stdout=StringIO(),
+    )
+    remote.run(args=['sudo', 'apt-get', 'update'], check_status=False)
+    remote.run(
+        args=[
+            'sudo', 'DEBIAN_FRONTEND=noninteractive', 'apt-get', '-y',
+            '--force-yes',
+            '-o', run.Raw('Dpkg::Options::="--force-confdef"'), '-o', run.Raw(
+                'Dpkg::Options::="--force-confold"'),
+            'install',
+        ] + ['%s=%s' % (d, version) for d in debs],
+    )
+    ldir = _get_local_dir(config, remote)
+    if ldir:
+        for fyle in os.listdir(ldir):
+            fname = "%s/%s" % (ldir, fyle)
+            remote.run(args=['sudo', 'dpkg', '-i', fname],)
+
+
+def _remove(ctx, config, remote, debs):
+    """
+    Removes Debian packages from remote, rudely
+
+    TODO: be less rude (e.g. using --force-yes)
+
+    :param ctx: the argparse.Namespace object
+    :param config: the config dict
+    :param remote: the teuthology.orchestra.remote.Remote object
+    :param debs: list of packages names to install
+    """
+    log.info("Removing packages: {pkglist} on Debian system.".format(
+        pkglist=", ".join(debs)))
+    # first ask nicely
+    remote.run(
+        args=[
+            'for', 'd', 'in',
+        ] + debs + [
+            run.Raw(';'),
+            'do',
+            'sudo',
+            'DEBIAN_FRONTEND=noninteractive', 'apt-get', '-y', '--force-yes',
+            '-o', run.Raw('Dpkg::Options::="--force-confdef"'), '-o', run.Raw(
+                'Dpkg::Options::="--force-confold"'), 'purge',
+            run.Raw('$d'),
+            run.Raw('||'),
+            'true',
+            run.Raw(';'),
+            'done',
+        ])
+    # mop up anything that is broken
+    remote.run(
+        args=[
+            'dpkg', '-l',
+            run.Raw('|'),
+            # Any package that is unpacked or half-installed and also requires
+            # reinstallation
+            'grep', '^.\(U\|H\)R',
+            run.Raw('|'),
+            'awk', '{print $2}',
+            run.Raw('|'),
+            'sudo',
+            'xargs', '--no-run-if-empty',
+            'dpkg', '-P', '--force-remove-reinstreq',
+        ])
+    # then let apt clean up
+    remote.run(
+        args=[
+            'sudo',
+            'DEBIAN_FRONTEND=noninteractive', 'apt-get', '-y', '--force-yes',
+            '-o', run.Raw('Dpkg::Options::="--force-confdef"'), '-o', run.Raw(
+                'Dpkg::Options::="--force-confold"'),
+            'autoremove',
+        ],
+    )
+
+
+def _remove_sources_list(remote, proj):
+    """
+    Removes /etc/apt/sources.list.d/{proj}.list and then runs ``apt-get
+    update``.
+
+    :param remote: the teuthology.orchestra.remote.Remote object
+    :param proj: the project whose sources.list needs removing
+    """
+    remote.run(
+        args=[
+            'sudo', 'rm', '-f', '/etc/apt/sources.list.d/{proj}.list'.format(
+                proj=proj),
+            run.Raw('&&'),
+            'sudo', 'apt-get', 'update',
+        ],
+        check_status=False,
+    )
+
+
+def _upgrade_packages(ctx, config, remote, debs):
+    """
+    Upgrade project's packages on remote Debian host
+    Before doing so, installs the project's GPG key, writes a sources.list
+    file, and runs ``apt-get update``.
+
+    :param ctx: the argparse.Namespace object
+    :param config: the config dict
+    :param remote: the teuthology.orchestra.remote.Remote object
+    :param debs: the Debian packages to be installed
+    :param branch: the branch of the project to be used
+    """
+    # check for ceph release key
+    r = remote.run(
+        args=[
+            'sudo', 'apt-key', 'list', run.Raw('|'), 'grep', 'Ceph',
+        ],
+        stdout=StringIO(),
+        check_status=False,
+    )
+    if r.stdout.getvalue().find('Ceph automated package') == -1:
+        # if it doesn't exist, add it
+        remote.run(
+            args=[
+                'wget', '-q', '-O-',
+                'http://git.ceph.com/?p=ceph.git;a=blob_plain;f=keys/autobuild.asc',  # noqa
+                run.Raw('|'),
+                'sudo', 'apt-key', 'add', '-',
+            ],
+            stdout=StringIO(),
+        )
+
+    builder = _get_builder_project(ctx, remote, config)
+    base_url = builder.base_url
+    log.info('Pulling from %s', base_url)
+
+    version = builder.version
+    log.info('Package version is %s', version)
+
+    remote.run(
+        args=[
+            'echo', 'deb', base_url, builder.codename, 'main',
+            run.Raw('|'),
+            'sudo', 'tee', '/etc/apt/sources.list.d/{proj}.list'.format(
+                proj=config.get('project', 'ceph')),
+        ],
+        stdout=StringIO(),
+    )
+    remote.run(args=['sudo', 'apt-get', 'update'], check_status=False)
+    remote.run(
+        args=[
+            'sudo',
+            'DEBIAN_FRONTEND=noninteractive', 'apt-get', '-y', '--force-yes',
+            '-o', run.Raw('Dpkg::Options::="--force-confdef"'), '-o', run.Raw(
+                'Dpkg::Options::="--force-confold"'),
+            'install',
+        ] + ['%s=%s' % (d, version) for d in debs],
+    )