Make mirroring single entry; mirror is now self-correcting, update_mirror is gone.lc/staging/fixes2

1 files changed, 28 insertions, 50 deletions

diff --git a/ybd/repos.py b/ybd/repos.py
index 3f74994..aabfd52 100644
--- a/ybd/repos.py
+++ b/ybd/repos.py
@@ -104,15 +104,9 @@ def get_tree(dn):
             if app.config.get('tree-server'):
                 app.log(dn, 'WARNING: no tree from tree-server for', ref)
 
-        mirror(dn['name'], dn['repo'])
+    mirror(dn['name'], dn['repo'])
 
     with app.chdir(info['dir']), open(os.devnull, "w") as fnull:
-        if call(['git', 'rev-parse', ref + '^{object}'], stdout=fnull,
-                stderr=fnull):
-            # can't resolve ref. is it upstream?
-            app.log(dn, 'Fetching from upstream to resolve %s' % ref)
-            update_mirror(dn['name'], dn['repo'], info['dir'])
-
         try:
             tree = check_output(['git', 'rev-parse', ref + '^{tree}'],
                                 universal_newlines=True)[0:-1]
@@ -126,34 +120,33 @@ def get_tree(dn):
 
 def mirror(name, repo):
     tempfile.tempdir = app.config['tmp']
-    tmpdir = tempfile.mkdtemp()
     info = get_transport_info(repo)
-    try:
-        tar_file = info['name'] + '.tar'
-        app.log(name, 'Try fetching tarball %s' % tar_file)
-        # try tarball first
-        with app.chdir(tmpdir), open(os.devnull, "w") as fnull:
-            call(['wget', os.path.join(app.config['tar-url'], tar_file)],
-                 stdout=fnull, stderr=fnull)
-            call(['tar', 'xf', tar_file], stderr=fnull)
-            call(['git', 'config', 'gc.autodetach', 'false'], stderr=fnull)
-            os.remove(tar_file)
-            update_mirror(name, repo, tmpdir)
-    except:
-        app.log(name, 'Try git clone from', info['url'])
-        with open(os.devnull, "w") as fnull:
-            if call(['git', 'clone', '--mirror', '-n', info['url'], tmpdir]):
-                app.log(name, 'Failed to clone', repo, exit=True)
-
-    with app.chdir(tmpdir):
-        if call(['git', 'rev-parse']):
-            app.log(name, 'Problem mirroring git repo at', tmpdir, exit=True)
-
-    try:
+    if not os.path.exists(info['dir']) and \
+       app.config.get('tar-url', None) is not None:
+        try:
+            tmpdir = tempfile.mkdtemp()
+            with app.chdir(tmpdir), open(os.devnull, "w") as fnull:
+                t = os.path.join(app.config['tar-url'], tar_file)
+                print "Getting %s" % t
+                call(['wget', t], stdout=fnull, stderr=fnull)
+                call(['tar', 'xf', tar_file], stderr=fnull)
+                call(['git', 'config', 'gc.autodetach', 'false'], stderr=fnull)
+                os.remove(tar_file)
+            shutil.move(tmpdir, info['dir'])
+        except:
+            pass
+
+    if os.path.exists(info['dir']):
+        with app.chdir(info['dir']):
+            if call(['git', 'fetch', info['url'],
+                     '+refs/*:refs/*', '--prune']):
+                raise Exception("Failed to update mirror for %s" % info['url'])
+    else:
+        tmpdir = tempfile.mkdtemp()
+        print("Cloning %s" % info['url'])
+        if call(['git', 'clone', '--mirror', '-n', info['url'], tmpdir]):
+            raise Exception("Failed to clone %s" % info['url'])
         shutil.move(tmpdir, info['dir'])
-        app.log(name, 'Git repo is mirrored at', info['dir'])
-    except:
-        pass
 
 
 def fetch(repo):
@@ -167,15 +160,6 @@ def mirror_has_ref(gitdir, ref):
         return out == 0
 
 
-def update_mirror(name, repo, gitdir):
-    with app.chdir(gitdir), open(os.devnull, "w") as fnull:
-        app.log(name, 'Refreshing mirror for %s' % repo)
-        repo_url = get_repo_url(repo)
-        if call(['git', 'fetch', repo_url, '+refs/*:refs/*', '--prune'],
-                stdout=fnull, stderr=fnull):
-            app.log(name, 'Git update mirror failed', repo, exit=True)
-
-
 def checkout(dn):
     _checkout(dn['name'], dn['repo'], dn['ref'], dn['checkout'])
 
@@ -186,10 +170,7 @@ def checkout(dn):
 
 def _checkout(name, repo, ref, checkout):
     gitdir = get_transport_info(repo)['dir']
-    if not os.path.exists(gitdir):
-        mirror(name, repo)
-    elif not mirror_has_ref(gitdir, ref):
-        update_mirror(name, repo, gitdir)
+    mirror(name, repo)
     # checkout the required version from git
     with open(os.devnull, "w") as fnull:
         # We need to pass '--no-hardlinks' because right now there's nothing to
@@ -222,10 +203,7 @@ def extract_commit(name, repo, ref, target_dir):
     target_dir.
     '''
     gitdir = get_transport_info(repo)['dir']
-    if not os.path.exists(gitdir):
-        mirror(name, repo)
-    elif not mirror_has_ref(gitdir, ref):
-        update_mirror(name, repo, gitdir)
+    mirror(name, repo)
 
     with tempfile.NamedTemporaryFile() as git_index_file:
         git_env = os.environ.copy()