1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
|
# Copyright 2014 Antoine "hashar" Musso
# Copyright 2014 Wikimedia Foundation Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import git
import logging
import os
import re
import yaml
from git import GitCommandError
from zuul.lib.clonemapper import CloneMapper
from zuul.merger.merger import Repo
class Cloner(object):
log = logging.getLogger("zuul.Cloner")
def __init__(self, git_base_url, projects, workspace, zuul_branch,
zuul_ref, zuul_url, branch=None, clone_map_file=None,
project_branches=None, cache_dir=None):
self.clone_map = []
self.dests = None
self.branch = branch
self.git_url = git_base_url
self.cache_dir = cache_dir
self.projects = projects
self.workspace = workspace
self.zuul_branch = zuul_branch
self.zuul_ref = zuul_ref
self.zuul_url = zuul_url
self.project_branches = project_branches or {}
if clone_map_file:
self.readCloneMap(clone_map_file)
def readCloneMap(self, clone_map_file):
clone_map_file = os.path.expanduser(clone_map_file)
if not os.path.exists(clone_map_file):
raise Exception("Unable to read clone map file at %s." %
clone_map_file)
clone_map_file = open(clone_map_file)
self.clone_map = yaml.load(clone_map_file).get('clonemap')
self.log.info("Loaded map containing %s rules", len(self.clone_map))
return self.clone_map
def execute(self):
mapper = CloneMapper(self.clone_map, self.projects)
dests = mapper.expand(workspace=self.workspace)
self.log.info("Preparing %s repositories", len(dests))
for project, dest in dests.iteritems():
self.prepareRepo(project, dest)
self.log.info("Prepared all repositories")
def cloneUpstream(self, project, dest):
# Check for a cached git repo first
git_cache = '%s/%s' % (self.cache_dir, project)
git_upstream = '%s/%s' % (self.git_url, project)
if (self.cache_dir and
os.path.exists(git_cache) and
not os.path.exists(dest)):
# file:// tells git not to hard-link across repos
git_cache = 'file://%s' % git_cache
self.log.info("Creating repo %s from cache %s",
project, git_cache)
new_repo = git.Repo.clone_from(git_cache, dest)
self.log.info("Updating origin remote in repo %s to %s",
project, git_upstream)
origin = new_repo.remotes.origin.config_writer.set(
'url', git_upstream)
else:
self.log.info("Creating repo %s from upstream %s",
project, git_upstream)
repo = Repo(
remote=git_upstream,
local=dest,
email=None,
username=None)
if not repo.isInitialized():
raise Exception("Error cloning %s to %s" % (git_upstream, dest))
return repo
def fetchFromZuul(self, repo, project, ref):
zuul_remote = '%s/%s' % (self.zuul_url, project)
try:
repo.fetchFrom(zuul_remote, ref)
self.log.debug("Fetched ref %s from %s", ref, project)
return True
except (ValueError, GitCommandError):
self.log.debug("Project %s in Zuul does not have ref %s",
project, ref)
return False
def prepareRepo(self, project, dest):
"""Clone a repository for project at dest and apply a reference
suitable for testing. The reference lookup is attempted in this order:
1) Zuul reference for the indicated branch
2) Zuul reference for the master branch
3) The tip of the indicated branch
4) The tip of the master branch
The "indicated branch" is one of the following:
A) The project-specific override branch (from project_branches arg)
B) The user specified branch (from the branch arg)
C) ZUUL_BRANCH (from the zuul_branch arg)
"""
repo = self.cloneUpstream(project, dest)
repo.update()
# Ensure that we don't have stale remotes around
repo.prune()
indicated_branch = self.branch or self.zuul_branch
if project in self.project_branches:
indicated_branch = self.project_branches[project]
override_zuul_ref = re.sub(self.zuul_branch, indicated_branch,
self.zuul_ref)
if repo.hasBranch(indicated_branch):
self.log.debug("upstream repo has branch %s", indicated_branch)
fallback_branch = indicated_branch
else:
self.log.debug("upstream repo is missing branch %s",
self.branch)
# FIXME should be origin HEAD branch which might not be 'master'
fallback_branch = 'master'
fallback_zuul_ref = re.sub(self.zuul_branch, fallback_branch,
self.zuul_ref)
if (self.fetchFromZuul(repo, project, override_zuul_ref)
or (fallback_zuul_ref != override_zuul_ref and
self.fetchFromZuul(repo, project, fallback_zuul_ref))
):
# Work around a bug in GitPython which can not parse FETCH_HEAD
gitcmd = git.Git(dest)
fetch_head = gitcmd.rev_parse('FETCH_HEAD')
repo.checkout(fetch_head)
self.log.info("Prepared %s repo with commit %s",
project, fetch_head)
else:
# Checkout branch
self.log.debug("Falling back to branch %s", fallback_branch)
try:
repo.checkout('remotes/origin/%s' % fallback_branch)
except (ValueError, GitCommandError):
self.log.exception("Fallback branch not found: %s",
fallback_branch)
self.log.info("Prepared %s repo with branch %s",
project, fallback_branch)
|
