Split up repo helper into separate functions

Signed-off-by: Zack Cerza <zack.cerza@inktank.com>

diff --git a/teuthology/repo_utils.py b/teuthology/repo_utils.py
index 1cebd695ffa992ced4050ad17a49f1d57716fd19..7c5bd43cb802940aec5a7cb6f16a04014e16e0d3 100644 (file)
--- a/teuthology/repo_utils.py
+++ b/teuthology/repo_utils.py
@@ -7,7 +7,7 @@ import time
 log = logging.getLogger(__name__)
 
 
-def checkout_repo(repo_url, dest_path, branch):
+def enforce_repo_state(repo_url, dest_path, branch):
     """
     Use git to either clone or update a given repo, forcing it to switch to the
     specified branch.
@@ -18,34 +18,84 @@ def checkout_repo(repo_url, dest_path, branch):
     :raises:          BranchNotFoundError if the branch is not found;
                       RuntimeError for other errors
     """
-    if not os.path.isdir(dest_path):
-        log.info("Cloning %s %s from upstream", repo_url, branch)
-        proc = subprocess.Popen(
-            ('git', 'clone', '--branch', branch, repo_url, dest_path),
-            cwd=os.path.dirname(dest_path),
-            stdout=subprocess.PIPE,
-            stderr=subprocess.STDOUT)
+    try:
+        if not os.path.isdir(dest_path):
+            clone_repo(repo_url, dest_path, branch)
+        elif time.time() - os.stat('/etc/passwd').st_mtime > 60:
+            # only do this at most once per minute
+            fetch_branch(dest_path, branch)
+            out = subprocess.check_output(('touch', dest_path))
+            if out:
+                log.info(out)
+        else:
+            log.info("%s was just updated; assuming it is current", branch)
+
+        reset_repo(repo_url, dest_path, branch)
+    except BranchNotFoundError:
+        shutil.rmtree(dest_path, ignore_errors=True)
+        raise
+
+
+def clone_repo(repo_url, dest_path, branch):
+    """
+    Clone a repo into a path
+
+    :param repo_url:  The full URL to the repo (not including the branch)
+    :param dest_path: The full path to the destination directory
+    :param branch:    The branch.
+    :raises:          BranchNotFoundError if the branch is not found;
+                      RuntimeError for other errors
+    """
+    log.info("Cloning %s %s from upstream", repo_url, branch)
+    proc = subprocess.Popen(
+        ('git', 'clone', '--branch', branch, repo_url, dest_path),
+        cwd=os.path.dirname(dest_path),
+        stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT)
+    if proc.wait() != 0:
         not_found_str = "Remote branch %s not found" % branch
         out = proc.stdout.read()
-        if proc.wait() != 0:
-            log.error(out)
-            if not_found_str in out:
-                raise BranchNotFoundError(branch, repo_url)
-            else:
-                raise RuntimeError("git clone failed!")
-    elif time.time() - os.stat('/etc/passwd').st_mtime > 60:
-        # only do this at most once per minute
-        log.info("Fetching %s from upstream", branch)
-        out = subprocess.check_output(('git', 'fetch', '-p', 'origin'),
-                                      cwd=dest_path)
-        if out:
-            log.info(out)
-        out = subprocess.check_output(('touch', dest_path))
-        if out:
-            log.info(out)
-    else:
-        log.info("%s was just updated; assuming it is current", branch)
+        log.error(out)
+        if not_found_str in out:
+            raise BranchNotFoundError(branch, repo_url)
+        else:
+            raise RuntimeError("git clone failed!")
 
+
+def fetch_branch(dest_path, branch):
+    """
+    Call "git fetch -p origin <branch>"
+
+    :param dest_path: The full path to the destination directory
+    :param branch:    The branch.
+    :raises:          BranchNotFoundError if the branch is not found;
+                      RuntimeError for other errors
+    """
+    log.info("Fetching %s from upstream", branch)
+    proc = subprocess.Popen(
+        ('git', 'fetch', '-p', 'origin', branch),
+        cwd=dest_path,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT)
+    if proc.wait() != 0:
+        not_found_str = "fatal: Couldn't find remote ref %s" % branch
+        out = proc.stdout.read()
+        log.error(out)
+        if not_found_str in out:
+            raise BranchNotFoundError(branch)
+        else:
+            raise RuntimeError("git fetch failed!")
+
+
+def reset_repo(repo_url, dest_path, branch):
+    """
+
+    :param repo_url:  The full URL to the repo (not including the branch)
+    :param dest_path: The full path to the destination directory
+    :param branch:    The branch.
+    :raises:          BranchNotFoundError if the branch is not found;
+                      RuntimeError for other errors
+    """
     # This try/except block will notice if the requested branch doesn't
     # exist, whether it was cloned or fetched.
     try:
@@ -54,15 +104,18 @@ def checkout_repo(repo_url, dest_path, branch):
             cwd=dest_path,
         )
     except subprocess.CalledProcessError:
-        shutil.rmtree(dest_path)
         raise BranchNotFoundError(branch, repo_url)
 
 
 class BranchNotFoundError(ValueError):
-    def __init__(self, branch, repo):
+    def __init__(self, branch, repo=None):
         self.branch = branch
         self.repo = repo
 
     def __str__(self):
-        return "Branch {branch} not found in repo: {repo}".format(
-            branch=self.branch, repo=self.repo)
+        if self.repo:
+            repo_str = " in repo: %s" % self.repo
+        else:
+            repo_str = ""
+        return "Branch {branch} not found{repo_str}!".format(
+            branch=self.branch, repo_str=repo_str)

diff --git a/teuthology/suite.py b/teuthology/suite.py
index 9d5f83f722388f6c46e3155db314bf8fa0faf379..5fd22be12ba9b0ba59e1f6a32c03ecf4a8f4a548 100644 (file)
--- a/teuthology/suite.py
+++ b/teuthology/suite.py
@@ -19,7 +19,7 @@ from tempfile import NamedTemporaryFile
 import teuthology
 from . import lock
 from .config import config
-from .repo_utils import checkout_repo, BranchNotFoundError
+from .repo_utils import enforce_repo_state, BranchNotFoundError
 
 log = logging.getLogger(__name__)
 
@@ -119,11 +119,11 @@ def fetch_suite_repo(branch, test_name):
     suite_repo_path = os.path.join(src_base_path,
                                    'ceph-qa-suite_' + branch)
     try:
-        checkout_repo(
+        enforce_repo_state(
             repo_url=os.path.join(config.ceph_git_base_url, 'teuthology.git'),
             dest_path=os.path.join(src_base_path, 'teuthology'),
             branch='master')
-        checkout_repo(
+        enforce_repo_state(
             repo_url=os.path.join(config.ceph_git_base_url,
                                   'ceph-qa-suite.git'),
             dest_path=suite_repo_path,

diff --git a/teuthology/test/test_repo_utils.py b/teuthology/test/test_repo_utils.py
index fdb822783436dbf697532d9880234bbb87447dc8..9152b125b63152fbf7487715259e805625c0db5b 100644 (file)
--- a/teuthology/test/test_repo_utils.py
+++ b/teuthology/test/test_repo_utils.py
@@ -18,32 +18,42 @@ class TestRepoUtils(object):
         shutil.rmtree(self.local_dir, ignore_errors=True)
 
     def test_existing_branch(self):
-        repo_utils.checkout_repo(self.empty_repo, self.local_dir, 'master')
+        repo_utils.enforce_repo_state(self.empty_repo, self.local_dir,
+                                      'master')
         assert os.path.exists(self.local_dir)
 
     def test_non_existing_branch(self):
         with raises(repo_utils.BranchNotFoundError):
-            repo_utils.checkout_repo(self.empty_repo, self.local_dir, 'blah')
+            repo_utils.enforce_repo_state(self.empty_repo, self.local_dir,
+                                          'blah')
         assert not os.path.exists(self.local_dir)
 
     def test_multiple_calls_same_branch(self):
-        repo_utils.checkout_repo(self.empty_repo, self.local_dir, 'master')
+        repo_utils.enforce_repo_state(self.empty_repo, self.local_dir,
+                                      'master')
         assert os.path.exists(self.local_dir)
-        repo_utils.checkout_repo(self.empty_repo, self.local_dir, 'master')
+        repo_utils.enforce_repo_state(self.empty_repo, self.local_dir,
+                                      'master')
         assert os.path.exists(self.local_dir)
-        repo_utils.checkout_repo(self.empty_repo, self.local_dir, 'master')
+        repo_utils.enforce_repo_state(self.empty_repo, self.local_dir,
+                                      'master')
         assert os.path.exists(self.local_dir)
 
     def test_multiple_calls_different_branches(self):
         with raises(repo_utils.BranchNotFoundError):
-            repo_utils.checkout_repo(self.empty_repo, self.local_dir, 'blah')
+            repo_utils.enforce_repo_state(self.empty_repo, self.local_dir,
+                                          'blah1')
         assert not os.path.exists(self.local_dir)
-        repo_utils.checkout_repo(self.empty_repo, self.local_dir, 'master')
+        repo_utils.enforce_repo_state(self.empty_repo, self.local_dir,
+                                      'master')
         assert os.path.exists(self.local_dir)
-        repo_utils.checkout_repo(self.empty_repo, self.local_dir, 'master')
+        repo_utils.enforce_repo_state(self.empty_repo, self.local_dir,
+                                      'master')
         assert os.path.exists(self.local_dir)
         with raises(repo_utils.BranchNotFoundError):
-            repo_utils.checkout_repo(self.empty_repo, self.local_dir, 'blah')
+            repo_utils.enforce_repo_state(self.empty_repo, self.local_dir,
+                                          'blah2')
         assert not os.path.exists(self.local_dir)
-        repo_utils.checkout_repo(self.empty_repo, self.local_dir, 'master')
+        repo_utils.enforce_repo_state(self.empty_repo, self.local_dir,
+                                      'master')
         assert os.path.exists(self.local_dir)

diff --git a/teuthology/worker.py b/teuthology/worker.py
index 1a44480a5ce400807b97d5b695139796e322f7fc..53da3b7d1c6d12a8557cfbaef1aa8c27df3cb76e 100644 (file)
--- a/teuthology/worker.py
+++ b/teuthology/worker.py
@@ -17,7 +17,7 @@ from . import safepath
 from .config import config as teuth_config
 from .kill import kill_job
 from .misc import read_config
-from .repo_utils import checkout_repo, BranchNotFoundError
+from .repo_utils import enforce_repo_state, BranchNotFoundError
 
 log = logging.getLogger(__name__)
 start_time = datetime.utcnow()
@@ -85,7 +85,7 @@ def fetch_teuthology_branch(dest_path, branch='master'):
     try:
         teuthology_git_upstream = teuth_config.ceph_git_base_url + \
             'teuthology.git'
-        checkout_repo(teuthology_git_upstream, dest_path, branch)
+        enforce_repo_state(teuthology_git_upstream, dest_path, branch)
 
         log.debug("Bootstrapping %s", dest_path)
         # This magic makes the bootstrap script not attempt to clobber an