From 5a61f449d1b63061159614d456542dba0dcde0e7 Mon Sep 17 00:00:00 2001 From: Zack Cerza Date: Mon, 30 Jun 2014 10:10:31 -0600 Subject: [PATCH] Split up repo helper into separate functions Signed-off-by: Zack Cerza --- teuthology/repo_utils.py | 113 +++++++++++++++++++++-------- teuthology/suite.py | 6 +- teuthology/test/test_repo_utils.py | 30 +++++--- teuthology/worker.py | 4 +- 4 files changed, 108 insertions(+), 45 deletions(-) diff --git a/teuthology/repo_utils.py b/teuthology/repo_utils.py index 1cebd695ff..7c5bd43cb8 100644 --- a/teuthology/repo_utils.py +++ b/teuthology/repo_utils.py @@ -7,7 +7,7 @@ import time log = logging.getLogger(__name__) -def checkout_repo(repo_url, dest_path, branch): +def enforce_repo_state(repo_url, dest_path, branch): """ Use git to either clone or update a given repo, forcing it to switch to the specified branch. @@ -18,34 +18,84 @@ def checkout_repo(repo_url, dest_path, branch): :raises: BranchNotFoundError if the branch is not found; RuntimeError for other errors """ - if not os.path.isdir(dest_path): - log.info("Cloning %s %s from upstream", repo_url, branch) - proc = subprocess.Popen( - ('git', 'clone', '--branch', branch, repo_url, dest_path), - cwd=os.path.dirname(dest_path), - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT) + try: + if not os.path.isdir(dest_path): + clone_repo(repo_url, dest_path, branch) + elif time.time() - os.stat('/etc/passwd').st_mtime > 60: + # only do this at most once per minute + fetch_branch(dest_path, branch) + out = subprocess.check_output(('touch', dest_path)) + if out: + log.info(out) + else: + log.info("%s was just updated; assuming it is current", branch) + + reset_repo(repo_url, dest_path, branch) + except BranchNotFoundError: + shutil.rmtree(dest_path, ignore_errors=True) + raise + + +def clone_repo(repo_url, dest_path, branch): + """ + Clone a repo into a path + + :param repo_url: The full URL to the repo (not including the branch) + :param dest_path: The full path to the destination directory + :param branch: The branch. + :raises: BranchNotFoundError if the branch is not found; + RuntimeError for other errors + """ + log.info("Cloning %s %s from upstream", repo_url, branch) + proc = subprocess.Popen( + ('git', 'clone', '--branch', branch, repo_url, dest_path), + cwd=os.path.dirname(dest_path), + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) + if proc.wait() != 0: not_found_str = "Remote branch %s not found" % branch out = proc.stdout.read() - if proc.wait() != 0: - log.error(out) - if not_found_str in out: - raise BranchNotFoundError(branch, repo_url) - else: - raise RuntimeError("git clone failed!") - elif time.time() - os.stat('/etc/passwd').st_mtime > 60: - # only do this at most once per minute - log.info("Fetching %s from upstream", branch) - out = subprocess.check_output(('git', 'fetch', '-p', 'origin'), - cwd=dest_path) - if out: - log.info(out) - out = subprocess.check_output(('touch', dest_path)) - if out: - log.info(out) - else: - log.info("%s was just updated; assuming it is current", branch) + log.error(out) + if not_found_str in out: + raise BranchNotFoundError(branch, repo_url) + else: + raise RuntimeError("git clone failed!") + +def fetch_branch(dest_path, branch): + """ + Call "git fetch -p origin " + + :param dest_path: The full path to the destination directory + :param branch: The branch. + :raises: BranchNotFoundError if the branch is not found; + RuntimeError for other errors + """ + log.info("Fetching %s from upstream", branch) + proc = subprocess.Popen( + ('git', 'fetch', '-p', 'origin', branch), + cwd=dest_path, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) + if proc.wait() != 0: + not_found_str = "fatal: Couldn't find remote ref %s" % branch + out = proc.stdout.read() + log.error(out) + if not_found_str in out: + raise BranchNotFoundError(branch) + else: + raise RuntimeError("git fetch failed!") + + +def reset_repo(repo_url, dest_path, branch): + """ + + :param repo_url: The full URL to the repo (not including the branch) + :param dest_path: The full path to the destination directory + :param branch: The branch. + :raises: BranchNotFoundError if the branch is not found; + RuntimeError for other errors + """ # This try/except block will notice if the requested branch doesn't # exist, whether it was cloned or fetched. try: @@ -54,15 +104,18 @@ def checkout_repo(repo_url, dest_path, branch): cwd=dest_path, ) except subprocess.CalledProcessError: - shutil.rmtree(dest_path) raise BranchNotFoundError(branch, repo_url) class BranchNotFoundError(ValueError): - def __init__(self, branch, repo): + def __init__(self, branch, repo=None): self.branch = branch self.repo = repo def __str__(self): - return "Branch {branch} not found in repo: {repo}".format( - branch=self.branch, repo=self.repo) + if self.repo: + repo_str = " in repo: %s" % self.repo + else: + repo_str = "" + return "Branch {branch} not found{repo_str}!".format( + branch=self.branch, repo_str=repo_str) diff --git a/teuthology/suite.py b/teuthology/suite.py index 9d5f83f722..5fd22be12b 100644 --- a/teuthology/suite.py +++ b/teuthology/suite.py @@ -19,7 +19,7 @@ from tempfile import NamedTemporaryFile import teuthology from . import lock from .config import config -from .repo_utils import checkout_repo, BranchNotFoundError +from .repo_utils import enforce_repo_state, BranchNotFoundError log = logging.getLogger(__name__) @@ -119,11 +119,11 @@ def fetch_suite_repo(branch, test_name): suite_repo_path = os.path.join(src_base_path, 'ceph-qa-suite_' + branch) try: - checkout_repo( + enforce_repo_state( repo_url=os.path.join(config.ceph_git_base_url, 'teuthology.git'), dest_path=os.path.join(src_base_path, 'teuthology'), branch='master') - checkout_repo( + enforce_repo_state( repo_url=os.path.join(config.ceph_git_base_url, 'ceph-qa-suite.git'), dest_path=suite_repo_path, diff --git a/teuthology/test/test_repo_utils.py b/teuthology/test/test_repo_utils.py index fdb8227834..9152b125b6 100644 --- a/teuthology/test/test_repo_utils.py +++ b/teuthology/test/test_repo_utils.py @@ -18,32 +18,42 @@ class TestRepoUtils(object): shutil.rmtree(self.local_dir, ignore_errors=True) def test_existing_branch(self): - repo_utils.checkout_repo(self.empty_repo, self.local_dir, 'master') + repo_utils.enforce_repo_state(self.empty_repo, self.local_dir, + 'master') assert os.path.exists(self.local_dir) def test_non_existing_branch(self): with raises(repo_utils.BranchNotFoundError): - repo_utils.checkout_repo(self.empty_repo, self.local_dir, 'blah') + repo_utils.enforce_repo_state(self.empty_repo, self.local_dir, + 'blah') assert not os.path.exists(self.local_dir) def test_multiple_calls_same_branch(self): - repo_utils.checkout_repo(self.empty_repo, self.local_dir, 'master') + repo_utils.enforce_repo_state(self.empty_repo, self.local_dir, + 'master') assert os.path.exists(self.local_dir) - repo_utils.checkout_repo(self.empty_repo, self.local_dir, 'master') + repo_utils.enforce_repo_state(self.empty_repo, self.local_dir, + 'master') assert os.path.exists(self.local_dir) - repo_utils.checkout_repo(self.empty_repo, self.local_dir, 'master') + repo_utils.enforce_repo_state(self.empty_repo, self.local_dir, + 'master') assert os.path.exists(self.local_dir) def test_multiple_calls_different_branches(self): with raises(repo_utils.BranchNotFoundError): - repo_utils.checkout_repo(self.empty_repo, self.local_dir, 'blah') + repo_utils.enforce_repo_state(self.empty_repo, self.local_dir, + 'blah1') assert not os.path.exists(self.local_dir) - repo_utils.checkout_repo(self.empty_repo, self.local_dir, 'master') + repo_utils.enforce_repo_state(self.empty_repo, self.local_dir, + 'master') assert os.path.exists(self.local_dir) - repo_utils.checkout_repo(self.empty_repo, self.local_dir, 'master') + repo_utils.enforce_repo_state(self.empty_repo, self.local_dir, + 'master') assert os.path.exists(self.local_dir) with raises(repo_utils.BranchNotFoundError): - repo_utils.checkout_repo(self.empty_repo, self.local_dir, 'blah') + repo_utils.enforce_repo_state(self.empty_repo, self.local_dir, + 'blah2') assert not os.path.exists(self.local_dir) - repo_utils.checkout_repo(self.empty_repo, self.local_dir, 'master') + repo_utils.enforce_repo_state(self.empty_repo, self.local_dir, + 'master') assert os.path.exists(self.local_dir) diff --git a/teuthology/worker.py b/teuthology/worker.py index 1a44480a5c..53da3b7d1c 100644 --- a/teuthology/worker.py +++ b/teuthology/worker.py @@ -17,7 +17,7 @@ from . import safepath from .config import config as teuth_config from .kill import kill_job from .misc import read_config -from .repo_utils import checkout_repo, BranchNotFoundError +from .repo_utils import enforce_repo_state, BranchNotFoundError log = logging.getLogger(__name__) start_time = datetime.utcnow() @@ -85,7 +85,7 @@ def fetch_teuthology_branch(dest_path, branch='master'): try: teuthology_git_upstream = teuth_config.ceph_git_base_url + \ 'teuthology.git' - checkout_repo(teuthology_git_upstream, dest_path, branch) + enforce_repo_state(teuthology_git_upstream, dest_path, branch) log.debug("Bootstrapping %s", dest_path) # This magic makes the bootstrap script not attempt to clobber an -- 2.39.5