Add ShamanProject

Signed-off-by: Zack Cerza <zack@redhat.com>

diff --git a/teuthology/config.py b/teuthology/config.py
index e3f537229ce9585501d843b5c92f6537fdc8f356..505666d3b5d2833b6b7d5a6f0341e36e6616a80c 100644 (file)
--- a/teuthology/config.py
+++ b/teuthology/config.py
@@ -157,6 +157,8 @@ class TeuthologyConfig(YamlConfig):
         'kojiroot_url': 'http://kojipkgs.fedoraproject.org/packages',
         'koji_task_url': 'https://kojipkgs.fedoraproject.org/work/',
         'baseurl_template': 'http://{host}/{proj}-{pkg_type}-{dist}-{arch}-{flavor}/{uri}',
+        'use_shaman': False,
+        'shaman_host': 'shaman.ceph.com',
         'teuthology_path': None,
         'suite_verify_ceph_hash': True,
         'suite_allow_missing_packages': False,

diff --git a/teuthology/packaging.py b/teuthology/packaging.py
index 8828999b31c2184864a024cf0722c25aa11ed3fd..d779420f87c0799d5b8709ac8a73ab67636677e6 100644 (file)
--- a/teuthology/packaging.py
+++ b/teuthology/packaging.py
@@ -2,7 +2,10 @@ import logging
 import ast
 import re
 import requests
+import urllib
+import urlparse
 
+from collections import OrderedDict
 from cStringIO import StringIO
 
 from .config import config
@@ -749,3 +752,68 @@ class GitbuilderProject(object):
             log.info("Found sha1: {0}".format(sha1))
 
         return sha1
+
+
+class ShamanProject(GitbuilderProject):
+    def __init__(self, project, job_config, ctx=None, remote=None):
+        super(ShamanProject, self).__init__(project, job_config, ctx, remote)
+        self.query_url = 'https://%s/api/' % config.shaman_host
+
+    def _get_base_url(self):
+        self.assert_result()
+        return self._result.json()[0]['url']
+
+    @property
+    def _result(self):
+        if getattr(self, '_result_obj', None) is None:
+            self._result_obj = self._search()
+        return self._result_obj
+
+    def _search(self):
+        flavor = self.flavor
+        if flavor == 'basic':
+            flavor = 'default'
+        req_obj = OrderedDict()
+        req_obj['status'] = 'ready'
+        req_obj['project'] = self.project
+        req_obj['flavor'] = flavor
+        req_obj['distros'] = '%s/%s' % (self.distro, self.arch)
+        if getattr(self, 'branch', None):
+            req_obj['ref'] = self.branch
+        if self.sha1:
+            req_obj['sha1'] = self.sha1
+        req_str = urllib.urlencode(req_obj)
+        uri = urlparse.urljoin(
+            self.query_url,
+            'search',
+        ) + '?%s' % req_str
+        log.debug("Querying %s", uri)
+        resp = requests.get(
+            uri,
+            headers={'content-type': 'application/json'},
+        )
+        resp.raise_for_status()
+        return resp
+
+    def assert_result(self):
+        if len(self._result.json()) == 0:
+            raise VersionNotFoundError(self._result.url)
+
+    @classmethod
+    def _get_distro(cls, distro=None, version=None, codename=None):
+        if distro in ('centos', 'rhel'):
+            distro = 'centos'
+            version = cls._parse_version(version)
+        return "%s/%s" % (distro, version)
+
+    def _get_package_sha1(self):
+        # This doesn't raise because GitbuilderProject._get_package_sha1()
+        # doesn't either.
+        if not len(self._result.json()):
+            log.error("sha1 not found: %s", self._result.url)
+        else:
+            return self._result.json()[0]['sha1']
+
+    def _get_package_version(self):
+        self.assert_result()
+        return self._result.json()[0]['extra']['package_manager_version']

diff --git a/teuthology/test/test_packaging.py b/teuthology/test/test_packaging.py
index 8ea118ea162c7494541c07f9370aa2192112af4e..94656e8ac717c7a62b88b19fe21fde946ab753b3 100644 (file)
--- a/teuthology/test/test_packaging.py
+++ b/teuthology/test/test_packaging.py
@@ -558,3 +558,127 @@ class TestGitbuilderProject(TestBuilderProject):
         ('ubuntu', '12.04', None, 'precise'),
         ('ubuntu', '14.04', None, 'trusty'),
     ]
+
+
+class TestShamanProject(TestBuilderProject):
+    klass = packaging.ShamanProject
+
+    def setup(self):
+        self.p_config = patch('teuthology.packaging.config')
+        self.m_config = self.p_config.start()
+        self.m_config.use_shaman = True
+        self.m_config.shaman_host = 'shaman.ceph.com'
+        self.p_get_config_value = \
+            patch('teuthology.packaging._get_config_value_for_remote')
+        self.m_get_config_value = self.p_get_config_value.start()
+        self.m_get_config_value.return_value = None
+        self.p_get = patch('requests.get')
+        self.m_get = self.p_get.start()
+
+    def teardown(self):
+        self.p_config.stop()
+        self.p_get_config_value.stop()
+        self.p_get.stop()
+
+    def test_init_from_remote_base_url(self):
+        # Here, we really just need to make sure ShamanProject._search()
+        # queries the right URL. So let's make _get_base_url() just pass that
+        # URL through and test that value.
+        def m_get_base_url(obj):
+            obj._search()
+            return self.m_get.call_args_list[0][0][0]
+        with patch(
+            'teuthology.packaging.ShamanProject._get_base_url',
+            new=m_get_base_url,
+        ):
+            super(TestShamanProject, self)\
+                .test_init_from_remote_base_url(
+                    "https://shaman.ceph.com/api/search?status=ready"
+                    "&project=ceph&flavor=default"
+                    "&distros=ubuntu%2F14.04%2Fx86_64"
+                )
+
+    def test_init_from_remote_base_url_debian(self):
+        # Here, we really just need to make sure ShamanProject._search()
+        # queries the right URL. So let's make _get_base_url() just pass that
+        # URL through and test that value.
+        def m_get_base_url(obj):
+            obj._search()
+            return self.m_get.call_args_list[0][0][0]
+        with patch(
+            'teuthology.packaging.ShamanProject._get_base_url',
+            new=m_get_base_url,
+        ):
+            super(TestShamanProject, self)\
+                .test_init_from_remote_base_url_debian(
+                    "https://shaman.ceph.com/api/search?status=ready"
+                    "&project=ceph&flavor=default"
+                    "&distros=debian%2F7.1%2Fx86_64"
+                )
+
+    def test_init_from_config_base_url(self):
+        # Here, we really just need to make sure ShamanProject._search()
+        # queries the right URL. So let's make _get_base_url() just pass that
+        # URL through and test that value.
+        def m_get_base_url(obj):
+            obj._search()
+            return self.m_get.call_args_list[0][0][0]
+        with patch(
+            'teuthology.packaging.ShamanProject._get_base_url',
+            new=m_get_base_url,
+        ):
+            super(TestShamanProject, self).test_init_from_config_base_url(
+                "https://shaman.ceph.com/api/search?status=ready&project=ceph" \
+                "&flavor=default&distros=ubuntu%2F14.04%2Fx86_64&sha1=sha1"
+            )
+
+    def test_get_package_version_found(self):
+        resp = Mock()
+        resp.ok = True
+        resp.json.return_value = [
+            dict(
+                sha1='the_sha1',
+                extra=dict(package_manager_version='0.90.0'),
+            )
+        ]
+        self.m_get.return_value = resp
+        super(TestShamanProject, self)\
+            .test_get_package_version_found()
+
+    def test_get_package_sha1_fetched_found(self):
+        resp = Mock()
+        resp.ok = True
+        resp.json.return_value = [dict(sha1='the_sha1')]
+        self.m_get.return_value = resp
+        super(TestShamanProject, self)\
+            .test_get_package_sha1_fetched_found()
+
+    def test_get_package_sha1_fetched_not_found(self):
+        resp = Mock()
+        resp.json.return_value = []
+        self.m_get.return_value = resp
+        super(TestShamanProject, self)\
+            .test_get_package_sha1_fetched_not_found()
+
+    DISTRO_MATRIX = [
+        ('rhel', '7.0', None, 'centos/7'),
+        ('centos', '6.5', None, 'centos/6'),
+        ('centos', '7.0', None, 'centos/7'),
+        ('centos', '7.1', None, 'centos/7'),
+        ('fedora', '20', None, 'fedora/20'),
+        ('ubuntu', '14.04', 'trusty', 'ubuntu/14.04'),
+        ('ubuntu', '14.04', None, 'ubuntu/14.04'),
+        ('debian', '7.0', None, 'debian/7.0'),
+        ('debian', '7', None, 'debian/7'),
+        ('debian', '7.1', None, 'debian/7.1'),
+        ('ubuntu', '12.04', None, 'ubuntu/12.04'),
+        ('ubuntu', '14.04', None, 'ubuntu/14.04'),
+    ]
+
+    DISTRO_MATRIX_NOVER = [
+        ('rhel', None, None, 'centos/7'),
+        ('centos', None, None, 'centos/7'),
+        ('fedora', None, None, 'fedora/20'),
+        ('ubuntu', None, None, 'ubuntu/14.04'),
+        ('debian', None, None, 'debian/7.0'),
+    ]