From: Andrew Schoen Date: Wed, 6 Jan 2016 16:41:54 +0000 (-0600) Subject: packaging.GitbuilderProject will retry now when fetching package versions X-Git-Tag: 1.1.0~706^2~2 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=de3773dc602b41b519c12c1f92b60b845ec55562;p=teuthology.git packaging.GitbuilderProject will retry now when fetching package versions If we don't get a 200, retry to account for gitbuilder finishing a build. Signed-off-by: Andrew Schoen --- diff --git a/teuthology/packaging.py b/teuthology/packaging.py index 007de0633..b0789632b 100644 --- a/teuthology/packaging.py +++ b/teuthology/packaging.py @@ -6,6 +6,8 @@ import requests from cStringIO import StringIO from .config import config +from .contextutil import safe_while +from .exceptions import VersionNotFoundError log = logging.getLogger(__name__) @@ -422,6 +424,21 @@ def _get_config_value_for_remote(ctx, remote, config, key): return config.get(key) +def _get_response(url, wait=False, sleep=15, tries=10): + with safe_while(sleep=sleep, tries=tries, _raise=False) as proceed: + while proceed(): + resp = requests.get(url) + if resp.ok or not wait: + break + + log.info( + 'Package not there yet (got HTTP code %s), waiting...', + resp.status_code, + ) + + return resp + + class GitbuilderProject(object): """ Represents a project that is built by gitbuilder. @@ -687,29 +704,27 @@ class GitbuilderProject(object): """ url = "{0}/version".format(self.base_url) log.info("Looking for package version: {0}".format(url)) - resp = requests.get(url) - version = None + # will loop and retry until a 200 is returned or the retry + # limits are reached + resp = _get_response(url, wait=self.job_config.get("wait_for_package", False)) + if not resp.ok: - log.info( - 'Package not there yet (got HTTP code %s), waiting...', - resp.status_code, - ) - else: - version = resp.text.strip() - if self.pkg_type == "rpm" and self.project == "ceph": - # TODO: move this parsing into a different function for - # easier testing - # FIXME: 'version' as retreived from the repo is actually the - # RPM version PLUS *part* of the release. Example: - # Right now, ceph master is given the following version in the - # repo file: v0.67-rc3.164.gd5aa3a9 - whereas in reality the RPM - # version is 0.61.7 and the release is 37.g1243c97.el6 (centos6). - # Point being, I have to mangle a little here. - if version[0] == 'v': - version = version[1:] - if '-' in version: - version = version.split('-')[0] - log.info("Found version: {0}".format(version)) + raise VersionNotFoundError(url) + version = resp.text.strip() + if self.pkg_type == "rpm" and self.project == "ceph": + # TODO: move this parsing into a different function for + # easier testing + # FIXME: 'version' as retreived from the repo is actually the + # RPM version PLUS *part* of the release. Example: + # Right now, ceph master is given the following version in the + # repo file: v0.67-rc3.164.gd5aa3a9 - whereas in reality the RPM + # version is 0.61.7 and the release is 37.g1243c97.el6 (centos6). + # Point being, I have to mangle a little here. + if version[0] == 'v': + version = version[1:] + if '-' in version: + version = version.split('-')[0] + log.info("Found version: {0}".format(version)) return version def _get_package_sha1(self): diff --git a/teuthology/test/test_packaging.py b/teuthology/test/test_packaging.py index 780dbafca..b69e47636 100644 --- a/teuthology/test/test_packaging.py +++ b/teuthology/test/test_packaging.py @@ -3,6 +3,7 @@ import pytest from mock import patch, Mock from teuthology import packaging +from teuthology.exceptions import VersionNotFoundError KOJI_TASK_RPMS_MATRIX = [ ('tasks/6745/9666745/kernel-4.1.0-0.rc2.git2.1.fc23.x86_64.rpm', 'kernel'), @@ -276,6 +277,30 @@ class TestPackaging(object): def test_get_koji_task_result_package_name(self, input, expected): assert packaging._get_koji_task_result_package_name(input) == expected + @patch("requests.get") + def test_get_response_success(self, m_get): + resp = Mock() + resp.ok = True + m_get.return_value = resp + result = packaging._get_response("google.com") + assert result == resp + + @patch("requests.get") + def test_get_response_failed_wait(self, m_get): + resp = Mock() + resp.ok = False + m_get.return_value = resp + packaging._get_response("google.com", wait=True, sleep=1, tries=2) + assert m_get.call_count == 2 + + @patch("requests.get") + def test_get_response_failed_no_wait(self, m_get): + resp = Mock() + resp.ok = False + m_get.return_value = resp + packaging._get_response("google.com", sleep=1, tries=2) + assert m_get.call_count == 1 + class TestGitbuilderProject(object): @@ -333,8 +358,8 @@ class TestGitbuilderProject(object): @patch("teuthology.packaging.config") @patch("teuthology.packaging._get_config_value_for_remote") - @patch("requests.get") - def test_get_package_version_found(self, m_get, m_get_config_value, + @patch("teuthology.packaging._get_response") + def test_get_package_version_found(self, m_get_response, m_get_config_value, m_config): m_config.baseurl_template = 'http://{host}/{proj}-{pkg_type}-{dist}-{arch}-{flavor}/{uri}' m_config.gitbuilder_host = "gitbuilder.ceph.com" @@ -342,27 +367,28 @@ class TestGitbuilderProject(object): resp = Mock() resp.ok = True resp.text = "0.90.0" - m_get.return_value = resp + m_get_response.return_value = resp rem = self._get_remote() ctx = dict(foo="bar") gp = packaging.GitbuilderProject("ceph", {}, ctx=ctx, remote=rem) assert gp.version == "0.90.0" + @patch("teuthology.packaging._get_response") @patch("teuthology.packaging.config") @patch("teuthology.packaging._get_config_value_for_remote") - @patch("requests.get") - def test_get_package_version_not_found(self, m_get, m_get_config_value, - m_config): + def test_get_package_version_not_found(self, m_get_config_value, + m_config, m_get_response): m_config.baseurl_template = 'http://{host}/{proj}-{pkg_type}-{dist}-{arch}-{flavor}/{uri}' m_config.gitbuilder_host = "gitbuilder.ceph.com" m_get_config_value.return_value = None - resp = Mock() - resp.ok = False - m_get.return_value = resp rem = self._get_remote() ctx = dict(foo="bar") + resp = Mock() + resp.ok = False + m_get_response.return_value = resp gp = packaging.GitbuilderProject("ceph", {}, ctx=ctx, remote=rem) - assert not gp.version + with pytest.raises(VersionNotFoundError): + gp.version @patch("teuthology.packaging.config") @patch("teuthology.packaging._get_config_value_for_remote")