From c80f4c77633df44c5c8418056d2266f35978d172 Mon Sep 17 00:00:00 2001 From: Felipe Contreras Date: Tue, 30 Apr 2013 20:09:53 -0500 Subject: remote-bzr: cleanup CustomTree This code was not used at all. Signed-off-by: Felipe Contreras Signed-off-by: Junio C Hamano --- contrib/remote-helpers/git-remote-bzr | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) (limited to 'contrib') diff --git a/contrib/remote-helpers/git-remote-bzr b/contrib/remote-helpers/git-remote-bzr index c19ed0e26b..7452a572c1 100755 --- a/contrib/remote-helpers/git-remote-bzr +++ b/contrib/remote-helpers/git-remote-bzr @@ -385,9 +385,6 @@ class CustomTree(): def __init__(self, repo, revid, parents, files): global files_cache - self.repo = repo - self.revid = revid - self.parents = parents self.updates = {} def copy_tree(revid): @@ -435,7 +432,7 @@ class CustomTree(): if basename == '': return None fid = bzrlib.generate_ids.gen_file_id(path) - d = add_entry(fid, dirname, 'directory') + add_entry(fid, dirname, 'directory') return fid def add_entry(fid, path, kind, mode = None): @@ -458,7 +455,6 @@ class CustomTree(): (None, executable)) self.files[path] = change[0] changes.append(change) - return change def update_entry(fid, path, kind, mode = None): dirname, basename = os.path.split(path) @@ -480,7 +476,6 @@ class CustomTree(): (None, executable)) self.files[path] = change[0] changes.append(change) - return change def remove_entry(fid, path, kind): dirname, basename = os.path.split(path) @@ -495,7 +490,6 @@ class CustomTree(): (None, None)) del self.files[path] changes.append(change) - return change for fid, f in self.updates.iteritems(): path = f['path'] -- cgit v1.2.1 From 181662080098e77a0fd814ffe84be605a058b5d1 Mon Sep 17 00:00:00 2001 From: Felipe Contreras Date: Tue, 30 Apr 2013 20:09:54 -0500 Subject: remote-bzr: delay blob fetching until the very end Might be more efficient, but the real reason to use the marks will be revealed in upcoming patches. Signed-off-by: Felipe Contreras Signed-off-by: Junio C Hamano --- contrib/remote-helpers/git-remote-bzr | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'contrib') diff --git a/contrib/remote-helpers/git-remote-bzr b/contrib/remote-helpers/git-remote-bzr index 7452a572c1..91b5cda767 100755 --- a/contrib/remote-helpers/git-remote-bzr +++ b/contrib/remote-helpers/git-remote-bzr @@ -506,10 +506,12 @@ class CustomTree(): return changes def get_file_with_stat(self, file_id, path=None): - return (StringIO.StringIO(self.updates[file_id]['data']), None) + mark = self.updates[file_id]['mark'] + return (StringIO.StringIO(blob_marks[mark]), None) def get_symlink_target(self, file_id): - return self.updates[file_id]['data'] + mark = self.updates[file_id]['mark'] + return blob_marks[mark] def c_style_unescape(string): if string[0] == string[-1] == '"': @@ -553,7 +555,7 @@ def parse_commit(parser): if parser.check('M'): t, m, mark_ref, path = line.split(' ', 3) mark = int(mark_ref[1:]) - f = { 'mode' : m, 'data' : blob_marks[mark] } + f = { 'mode' : m, 'mark' : mark } elif parser.check('D'): t, path = line.split(' ') f = { 'deleted' : True } -- cgit v1.2.1 From aa12a431f3bb96fa01cd1efa97eb1273d9a00c0e Mon Sep 17 00:00:00 2001 From: Felipe Contreras Date: Tue, 30 Apr 2013 20:09:55 -0500 Subject: remote-bzr: fix order of locking in CustomTree It doesn't seem to make any difference, but revision_tree() requires a lock. Signed-off-by: Felipe Contreras Signed-off-by: Junio C Hamano --- contrib/remote-helpers/git-remote-bzr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'contrib') diff --git a/contrib/remote-helpers/git-remote-bzr b/contrib/remote-helpers/git-remote-bzr index 91b5cda767..b428244cac 100755 --- a/contrib/remote-helpers/git-remote-bzr +++ b/contrib/remote-helpers/git-remote-bzr @@ -389,8 +389,8 @@ class CustomTree(): def copy_tree(revid): files = files_cache[revid] = {} - tree = repo.repository.revision_tree(revid) repo.lock_read() + tree = repo.repository.revision_tree(revid) try: for path, entry in tree.iter_entries_by_dir(): files[path] = entry.file_id -- cgit v1.2.1 From 715d64fe99885a37c38aff75eebfaf42b669d914 Mon Sep 17 00:00:00 2001 From: Felipe Contreras Date: Tue, 30 Apr 2013 20:09:56 -0500 Subject: remote-bzr: always try to update the worktree And fail properly when we can't. Signed-off-by: Felipe Contreras Signed-off-by: Junio C Hamano --- contrib/remote-helpers/git-remote-bzr | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'contrib') diff --git a/contrib/remote-helpers/git-remote-bzr b/contrib/remote-helpers/git-remote-bzr index b428244cac..eb91d2814b 100755 --- a/contrib/remote-helpers/git-remote-bzr +++ b/contrib/remote-helpers/git-remote-bzr @@ -641,9 +641,13 @@ def do_export(parser): except bzrlib.errors.DivergedBranches: print "error %s non-fast forward" % ref continue - else: + + try: wt = repo.bzrdir.open_workingtree() wt.update() + except bzrlib.errors.NoWorkingTree: + pass + print "ok %s" % ref print -- cgit v1.2.1 From f38dfc4c32507de2f7cf05350fe8b34fd4edd45d Mon Sep 17 00:00:00 2001 From: Felipe Contreras Date: Tue, 30 Apr 2013 20:09:57 -0500 Subject: remote-bzr: add support to push merges In order to do that, we need to store the marks of every file, so that they can be fetched when needed. Unfortunately we can't tell bazaar that nothing changed, we need to send the data so that it can figure it out by itself. And since it will be requesting a bunch of information by the file_id, it's better to have a helper dict (rev_files), so that we can fetch it quickly. Signed-off-by: Felipe Contreras Signed-off-by: Junio C Hamano --- contrib/remote-helpers/git-remote-bzr | 29 ++++++++++++++++++++-------- contrib/remote-helpers/test-bzr.sh | 36 +++++++++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+), 8 deletions(-) (limited to 'contrib') diff --git a/contrib/remote-helpers/git-remote-bzr b/contrib/remote-helpers/git-remote-bzr index eb91d2814b..6a7f836ea6 100755 --- a/contrib/remote-helpers/git-remote-bzr +++ b/contrib/remote-helpers/git-remote-bzr @@ -393,7 +393,7 @@ class CustomTree(): tree = repo.repository.revision_tree(revid) try: for path, entry in tree.iter_entries_by_dir(): - files[path] = entry.file_id + files[path] = [entry.file_id, None] finally: repo.unlock() return files @@ -408,12 +408,18 @@ class CustomTree(): self.base_files = copy_tree(self.base_id) self.files = files_cache[revid] = self.base_files.copy() + self.rev_files = {} + + for path, data in self.files.iteritems(): + fid, mark = data + self.rev_files[fid] = [path, mark] for path, f in files.iteritems(): - fid = self.files.get(path, None) + fid, mark = self.files.get(path, [None, None]) if not fid: fid = bzrlib.generate_ids.gen_file_id(path) f['path'] = path + self.rev_files[fid] = [path, mark] self.updates[fid] = f def last_revision(self): @@ -423,10 +429,10 @@ class CustomTree(): changes = [] def get_parent(dirname, basename): - parent_fid = self.base_files.get(dirname, None) + parent_fid, mark = self.base_files.get(dirname, [None, None]) if parent_fid: return parent_fid - parent_fid = self.files.get(dirname, None) + parent_fid, mark = self.files.get(dirname, [None, None]) if parent_fid: return parent_fid if basename == '': @@ -453,7 +459,7 @@ class CustomTree(): (None, basename), (None, kind), (None, executable)) - self.files[path] = change[0] + self.files[path] = [change[0], None] changes.append(change) def update_entry(fid, path, kind, mode = None): @@ -474,7 +480,7 @@ class CustomTree(): (None, basename), (None, kind), (None, executable)) - self.files[path] = change[0] + self.files[path] = [change[0], None] changes.append(change) def remove_entry(fid, path, kind): @@ -503,16 +509,23 @@ class CustomTree(): else: add_entry(fid, path, 'file', f['mode']) + self.files[path][1] = f['mark'] + self.rev_files[fid][1] = f['mark'] + return changes def get_file_with_stat(self, file_id, path=None): - mark = self.updates[file_id]['mark'] + path, mark = self.rev_files[file_id] return (StringIO.StringIO(blob_marks[mark]), None) def get_symlink_target(self, file_id): - mark = self.updates[file_id]['mark'] + path, mark = self.rev_files[file_id] return blob_marks[mark] + def id2path(self, file_id): + path, mark = self.rev_files[file_id] + return path + def c_style_unescape(string): if string[0] == string[-1] == '"': return string.decode('string-escape')[1:-1] diff --git a/contrib/remote-helpers/test-bzr.sh b/contrib/remote-helpers/test-bzr.sh index 34666e1d0f..eca347634e 100755 --- a/contrib/remote-helpers/test-bzr.sh +++ b/contrib/remote-helpers/test-bzr.sh @@ -228,4 +228,40 @@ test_expect_success 'push utf-8 filenames' ' test_cmp expected actual ' +test_expect_success 'pushing a merge' ' + mkdir -p tmp && cd tmp && + test_when_finished "cd .. && rm -rf tmp" && + + ( + bzr init bzrrepo && + cd bzrrepo && + echo one > content && + bzr add content && + bzr commit -m one + ) && + + git clone "bzr::$PWD/bzrrepo" gitrepo && + + ( + cd bzrrepo && + echo two > content && + bzr commit -m two + ) && + + ( + cd gitrepo && + echo three > content && + git commit -a -m three && + git fetch && + git merge origin/master || true && + echo three > content && + git commit -a --no-edit && + git push + ) && + + echo three > expected && + cat bzrrepo/content > actual && + test_cmp expected actual +' + test_done -- cgit v1.2.1 From 38e7167e9bdc57262c7298912c4bf9df4d68f6fe Mon Sep 17 00:00:00 2001 From: Felipe Contreras Date: Tue, 30 Apr 2013 20:09:58 -0500 Subject: remote-bzr: fixes for branch diverge If the branches diverge we want to reset the pointer to where the remote actually is. Since we can access remote branches just as easily as local ones, let's do so. Signed-off-by: Felipe Contreras Signed-off-by: Junio C Hamano --- contrib/remote-helpers/git-remote-bzr | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) (limited to 'contrib') diff --git a/contrib/remote-helpers/git-remote-bzr b/contrib/remote-helpers/git-remote-bzr index 6a7f836ea6..bf254a0787 100755 --- a/contrib/remote-helpers/git-remote-bzr +++ b/contrib/remote-helpers/git-remote-bzr @@ -354,6 +354,7 @@ def do_import(parser): if os.path.exists(path): print "feature import-marks=%s" % path print "feature export-marks=%s" % path + print "feature force" sys.stdout.flush() while parser.check('import'): @@ -716,7 +717,12 @@ def get_repo(url, alias): # pull d = bzrlib.bzrdir.BzrDir.open(clone_path) branch = d.open_branch() - result = branch.pull(remote_branch, [], None, False) + try: + result = branch.pull(remote_branch, [], None, False) + except bzrlib.errors.DivergedBranches: + # use remote branch for now + peer = None + return remote_branch else: # clone d = origin.sprout(clone_path, None, -- cgit v1.2.1 From b25df87fad2b12349a55146d5c87625109a58221 Mon Sep 17 00:00:00 2001 From: Felipe Contreras Date: Tue, 30 Apr 2013 20:09:59 -0500 Subject: remote-bzr: fix partially pushed merge If part of the merge was already pushed, we don't have the blob_marks available, however, the commits are already stored in bazaar, so we can use the revision_tree to fetch the contents. We want to do this only when there's no other option. There's no easy way to test this. Signed-off-by: Felipe Contreras Signed-off-by: Junio C Hamano --- contrib/remote-helpers/git-remote-bzr | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) (limited to 'contrib') diff --git a/contrib/remote-helpers/git-remote-bzr b/contrib/remote-helpers/git-remote-bzr index bf254a0787..fdead31a63 100755 --- a/contrib/remote-helpers/git-remote-bzr +++ b/contrib/remote-helpers/git-remote-bzr @@ -387,6 +387,7 @@ class CustomTree(): global files_cache self.updates = {} + self.branch = repo def copy_tree(revid): files = files_cache[revid] = {} @@ -515,13 +516,21 @@ class CustomTree(): return changes - def get_file_with_stat(self, file_id, path=None): + def get_content(self, file_id): path, mark = self.rev_files[file_id] - return (StringIO.StringIO(blob_marks[mark]), None) + if mark: + return blob_marks[mark] + + # last resort + tree = self.branch.repository.revision_tree(self.base_id) + return tree.get_file_text(file_id) + + def get_file_with_stat(self, file_id, path=None): + content = self.get_content(file_id) + return (StringIO.StringIO(content), None) def get_symlink_target(self, file_id): - path, mark = self.rev_files[file_id] - return blob_marks[mark] + return self.get_content(file_id) def id2path(self, file_id): path, mark = self.rev_files[file_id] -- cgit v1.2.1 From 5df4fad31902ba449e25c22f490526e8022a692c Mon Sep 17 00:00:00 2001 From: Felipe Contreras Date: Tue, 30 Apr 2013 20:10:00 -0500 Subject: remote-bzr: use branch variable when appropriate There should be no functional changes. Basically we want to reserve the 'repo' variable. Signed-off-by: Felipe Contreras Signed-off-by: Junio C Hamano --- contrib/remote-helpers/git-remote-bzr | 35 ++++++++++++++++++----------------- 1 file changed, 18 insertions(+), 17 deletions(-) (limited to 'contrib') diff --git a/contrib/remote-helpers/git-remote-bzr b/contrib/remote-helpers/git-remote-bzr index fdead31a63..0c67236487 100755 --- a/contrib/remote-helpers/git-remote-bzr +++ b/contrib/remote-helpers/git-remote-bzr @@ -257,7 +257,8 @@ def export_branch(branch, name): tip = marks.get_tip(name) repo = branch.repository - repo.lock_read() + + branch.lock_read() revs = branch.iter_merge_sorted_revisions(None, tip, 'exclude', 'forward') count = 0 @@ -325,7 +326,7 @@ def export_branch(branch, name): print "progress revision %s (%d/%d)" % (revid, count, len(revs)) print "#############################################################" - repo.unlock() + branch.unlock() revid = branch.last_revision() @@ -383,21 +384,21 @@ def parse_blob(parser): class CustomTree(): - def __init__(self, repo, revid, parents, files): + def __init__(self, branch, revid, parents, files): global files_cache self.updates = {} - self.branch = repo + self.branch = branch def copy_tree(revid): files = files_cache[revid] = {} - repo.lock_read() - tree = repo.repository.revision_tree(revid) + branch.lock_read() + tree = branch.repository.revision_tree(revid) try: for path, entry in tree.iter_entries_by_dir(): files[path] = [entry.file_id, None] finally: - repo.unlock() + branch.unlock() return files if len(parents) == 0: @@ -587,20 +588,20 @@ def parse_commit(parser): path = c_style_unescape(path).decode('utf-8') files[path] = f - repo = parser.repo + branch = parser.repo committer, date, tz = committer parents = [str(mark_to_rev(p)) for p in parents] revid = bzrlib.generate_ids.gen_revision_id(committer, date) props = {} - props['branch-nick'] = repo.nick + props['branch-nick'] = branch.nick - mtree = CustomTree(repo, revid, parents, files) + mtree = CustomTree(branch, revid, parents, files) changes = mtree.iter_changes() - repo.lock_write() + branch.lock_write() try: - builder = repo.get_commit_builder(parents, None, date, tz, committer, props, revid) + builder = branch.get_commit_builder(parents, None, date, tz, committer, props, revid) try: list(builder.record_iter_changes(mtree, mtree.last_revision(), changes)) builder.finish_inventory() @@ -609,7 +610,7 @@ def parse_commit(parser): builder.abort() raise finally: - repo.unlock() + branch.unlock() parsed_refs[ref] = revid marks.new_mark(revid, commit_mark) @@ -653,20 +654,20 @@ def do_export(parser): else: die('unhandled export command: %s' % line) - repo = parser.repo + branch = parser.repo for ref, revid in parsed_refs.iteritems(): if ref == 'refs/heads/master': - repo.generate_revision_history(revid, marks.get_tip('master')) + branch.generate_revision_history(revid, marks.get_tip('master')) if peer: try: - repo.push(peer, stop_revision=revid) + branch.push(peer, stop_revision=revid) except bzrlib.errors.DivergedBranches: print "error %s non-fast forward" % ref continue try: - wt = repo.bzrdir.open_workingtree() + wt = branch.bzrdir.open_workingtree() wt.update() except bzrlib.errors.NoWorkingTree: pass -- cgit v1.2.1 From 95b0c60831148ca929e31b214ab3d29537ba20c9 Mon Sep 17 00:00:00 2001 From: Felipe Contreras Date: Tue, 30 Apr 2013 20:10:01 -0500 Subject: remote-bzr: add support for bzr repos In bazaar, a repository can contain multiple branches, and previously we were supporting only one branch at a time. Now we fetch them all. Signed-off-by: Felipe Contreras Signed-off-by: Junio C Hamano --- contrib/remote-helpers/git-remote-bzr | 158 ++++++++++++++++++++++------------ contrib/remote-helpers/test-bzr.sh | 36 ++++++++ 2 files changed, 138 insertions(+), 56 deletions(-) (limited to 'contrib') diff --git a/contrib/remote-helpers/git-remote-bzr b/contrib/remote-helpers/git-remote-bzr index 0c67236487..3b3306181e 100755 --- a/contrib/remote-helpers/git-remote-bzr +++ b/contrib/remote-helpers/git-remote-bzr @@ -27,6 +27,7 @@ import bzrlib.generate_ids import bzrlib.transport import bzrlib.errors import bzrlib.ui +import bzrlib.urlutils import sys import os @@ -250,12 +251,13 @@ def export_files(tree, files): return final -def export_branch(branch, name): +def export_branch(repo, name): global prefix ref = '%s/heads/%s' % (prefix, name) tip = marks.get_tip(name) + branch = branches[name] repo = branch.repository branch.lock_read() @@ -323,7 +325,7 @@ def export_branch(branch, name): count += 1 if (count % 100 == 0): - print "progress revision %s (%d/%d)" % (revid, count, len(revs)) + print "progress revision %s '%s' (%d/%d)" % (revid, name, count, len(revs)) print "#############################################################" branch.unlock() @@ -348,7 +350,7 @@ def export_tag(repo, name): def do_import(parser): global dirname - branch = parser.repo + repo = parser.repo path = os.path.join(dirname, 'marks-git') print "feature done" @@ -362,10 +364,10 @@ def do_import(parser): ref = parser[1] if ref.startswith('refs/heads/'): name = ref[len('refs/heads/'):] - export_branch(branch, name) + export_branch(repo, name) if ref.startswith('refs/tags/'): name = ref[len('refs/tags/'):] - export_tag(branch, name) + export_tag(repo, name) parser.next() print 'done' @@ -551,8 +553,11 @@ def parse_commit(parser): ref = parser[1] parser.next() - if ref != 'refs/heads/master': - die("bzr doesn't support multiple branches; use 'master'") + if ref.startswith('refs/heads/'): + name = ref[len('refs/heads/'):] + branch = branches[name] + else: + die('unknown ref') commit_mark = parser.get_mark() parser.next() @@ -588,8 +593,6 @@ def parse_commit(parser): path = c_style_unescape(path).decode('utf-8') files[path] = f - branch = parser.repo - committer, date, tz = committer parents = [str(mark_to_rev(p)) for p in parents] revid = bzrlib.generate_ids.gen_revision_id(committer, date) @@ -621,9 +624,6 @@ def parse_reset(parser): ref = parser[1] parser.next() - if ref != 'refs/heads/master': - die("bzr doesn't support multiple branches; use 'master'") - # ugh if parser.check('commit'): parse_commit(parser) @@ -636,7 +636,7 @@ def parse_reset(parser): parsed_refs[ref] = mark_to_rev(from_mark) def do_export(parser): - global parsed_refs, dirname, peer + global parsed_refs, dirname parser.next() @@ -654,23 +654,24 @@ def do_export(parser): else: die('unhandled export command: %s' % line) - branch = parser.repo - for ref, revid in parsed_refs.iteritems(): - if ref == 'refs/heads/master': - branch.generate_revision_history(revid, marks.get_tip('master')) - if peer: - try: - branch.push(peer, stop_revision=revid) - except bzrlib.errors.DivergedBranches: - print "error %s non-fast forward" % ref - continue + name = ref[len('refs/heads/'):] + branch = branches[name] + branch.generate_revision_history(revid, marks.get_tip(name)) + if name in peers: + peer = peers[name] try: - wt = branch.bzrdir.open_workingtree() - wt.update() - except bzrlib.errors.NoWorkingTree: - pass + peer.bzrdir.push_branch(branch, revision_id=revid) + except bzrlib.errors.DivergedBranches: + print "error %s non-fast forward" % ref + continue + + try: + wt = branch.bzrdir.open_workingtree() + wt.update() + except bzrlib.errors.NoWorkingTree: + pass print "ok %s" % ref @@ -697,9 +698,15 @@ def ref_is_valid(name): def do_list(parser): global tags - print "? refs/heads/%s" % 'master' - branch = parser.repo + master_branch = None + + for name in branches: + if not master_branch: + master_branch = name + print "? refs/heads/%s" % name + + branch = branches[master_branch] branch.lock_read() for tag, revid in branch.tags.get_tag_dict().items(): try: @@ -711,41 +718,77 @@ def do_list(parser): print "? refs/tags/%s" % tag tags[tag] = revid branch.unlock() - print "@refs/heads/%s HEAD" % 'master' + + print "@refs/heads/%s HEAD" % master_branch print +def get_remote_branch(origin, remote_branch, name): + global dirname, peers + + branch_path = os.path.join(dirname, 'clone', name) + if os.path.exists(branch_path): + # pull + d = bzrlib.bzrdir.BzrDir.open(branch_path) + branch = d.open_branch() + try: + branch.pull(remote_branch, [], None, False) + except bzrlib.errors.DivergedBranches: + # use remote branch for now + return remote_branch + else: + # clone + d = origin.sprout(branch_path, None, + hardlink=True, create_tree_if_local=False, + force_new_repo=False, + source_branch=remote_branch) + branch = d.open_branch() + + return branch + def get_repo(url, alias): - global dirname, peer + global dirname, peer, branches + normal_url = bzrlib.urlutils.normalize_url(url) origin = bzrlib.bzrdir.BzrDir.open(url) - branch = origin.open_branch() - - if not isinstance(origin.transport, bzrlib.transport.local.LocalTransport): - clone_path = os.path.join(dirname, 'clone') - remote_branch = branch - if os.path.exists(clone_path): - # pull - d = bzrlib.bzrdir.BzrDir.open(clone_path) - branch = d.open_branch() - try: - result = branch.pull(remote_branch, [], None, False) - except bzrlib.errors.DivergedBranches: - # use remote branch for now - peer = None - return remote_branch + is_local = isinstance(origin.transport, bzrlib.transport.local.LocalTransport) + + clone_path = os.path.join(dirname, 'clone') + + try: + repo = origin.open_repository() + except bzrlib.errors.NoRepositoryPresent: + # branch + + name = 'master' + branch = origin.open_branch() + + if not is_local: + if not os.path.exists(clone_path): + os.mkdir(clone_path) + peers[name] = branch + branches[name] = get_remote_branch(origin, branch, name) else: - # clone - d = origin.sprout(clone_path, None, - hardlink=True, create_tree_if_local=False, - source_branch=remote_branch) - branch = d.open_branch() - branch.bind(remote_branch) - - peer = remote_branch + branches[name] = branch + + return branch.repository else: - peer = None + # repository - return branch + if not is_local and not os.path.exists(clone_path): + clonedir = bzrlib.bzrdir.BzrDir.create(clone_path) + + for branch in repo.find_branches(): + + name = repo.user_transport.relpath(branch.base) + name = name if name != '' else 'master' + + if not is_local: + peers[name] = branch + branches[name] = get_remote_branch(origin, branch, name) + else: + branches[name] = branch + + return repo def fix_path(alias, orig_url): url = urlparse.urlparse(orig_url, 'file') @@ -762,6 +805,7 @@ def main(args): global parsed_refs global files_cache global is_tmp + global branches, peers alias = args[1] url = args[2] @@ -772,6 +816,8 @@ def main(args): parsed_refs = {} files_cache = {} marks = None + branches = {} + peers = {} if alias[5:] == url: is_tmp = True diff --git a/contrib/remote-helpers/test-bzr.sh b/contrib/remote-helpers/test-bzr.sh index eca347634e..cec55f132e 100755 --- a/contrib/remote-helpers/test-bzr.sh +++ b/contrib/remote-helpers/test-bzr.sh @@ -264,4 +264,40 @@ test_expect_success 'pushing a merge' ' test_cmp expected actual ' +cat > expected <> content && + bzr add content && + bzr commit -m one + ) && + + bzr branch bzrrepo/trunk bzrrepo/branch && + ( + cd bzrrepo/branch && + echo two >> content && + bzr commit -m one + ) && + + git clone "bzr::$PWD/bzrrepo" gitrepo && + ( + cd gitrepo && + git for-each-ref --format "%(refname:short)" refs/remotes/origin > ../actual + ) && + + test_cmp ../expected actual +' + test_done -- cgit v1.2.1 From 42b48ef25d8c77f839d4a2d4cec9b65379ece63e Mon Sep 17 00:00:00 2001 From: Felipe Contreras Date: Tue, 30 Apr 2013 20:10:02 -0500 Subject: remote-bzr: fix branch names When branches have '/' in their name (aka. sub-branches), bazaar seems to choke while creating the new directory. Also, git cannot have both 'foo' and 'foo/bar'. So let's replace slashes with a plus sign. Signed-off-by: Felipe Contreras Signed-off-by: Junio C Hamano --- contrib/remote-helpers/git-remote-bzr | 1 + 1 file changed, 1 insertion(+) (limited to 'contrib') diff --git a/contrib/remote-helpers/git-remote-bzr b/contrib/remote-helpers/git-remote-bzr index 3b3306181e..38cec07ed8 100755 --- a/contrib/remote-helpers/git-remote-bzr +++ b/contrib/remote-helpers/git-remote-bzr @@ -781,6 +781,7 @@ def get_repo(url, alias): name = repo.user_transport.relpath(branch.base) name = name if name != '' else 'master' + name = name.replace('/', '+') if not is_local: peers[name] = branch -- cgit v1.2.1 From c95c35f4b8908fcda713c09eba4e55aac2c39be6 Mon Sep 17 00:00:00 2001 From: Felipe Contreras Date: Tue, 30 Apr 2013 20:10:03 -0500 Subject: remote-bzr: add support for shared repo This way all the remotes share the same data, so adding multiple remotes, or renaming them doesn't create extra overhead. Signed-off-by: Felipe Contreras Signed-off-by: Junio C Hamano --- contrib/remote-helpers/git-remote-bzr | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) (limited to 'contrib') diff --git a/contrib/remote-helpers/git-remote-bzr b/contrib/remote-helpers/git-remote-bzr index 38cec07ed8..9fe830e3cd 100755 --- a/contrib/remote-helpers/git-remote-bzr +++ b/contrib/remote-helpers/git-remote-bzr @@ -752,7 +752,20 @@ def get_repo(url, alias): origin = bzrlib.bzrdir.BzrDir.open(url) is_local = isinstance(origin.transport, bzrlib.transport.local.LocalTransport) - clone_path = os.path.join(dirname, 'clone') + shared_path = os.path.join(gitdir, 'bzr') + try: + shared_dir = bzrlib.bzrdir.BzrDir.open(shared_path) + except bzrlib.errors.NotBranchError: + shared_dir = bzrlib.bzrdir.BzrDir.create(shared_path) + try: + shared_repo = shared_dir.open_repository() + except bzrlib.errors.NoRepositoryPresent: + shared_repo = shared_dir.create_repository(shared=True) + + if not is_local: + clone_path = os.path.join(dirname, 'clone') + if not os.path.exists(clone_path): + os.mkdir(clone_path) try: repo = origin.open_repository() @@ -763,8 +776,6 @@ def get_repo(url, alias): branch = origin.open_branch() if not is_local: - if not os.path.exists(clone_path): - os.mkdir(clone_path) peers[name] = branch branches[name] = get_remote_branch(origin, branch, name) else: @@ -774,9 +785,6 @@ def get_repo(url, alias): else: # repository - if not is_local and not os.path.exists(clone_path): - clonedir = bzrlib.bzrdir.BzrDir.create(clone_path) - for branch in repo.find_branches(): name = repo.user_transport.relpath(branch.base) @@ -800,7 +808,7 @@ def fix_path(alias, orig_url): subprocess.call(cmd) def main(args): - global marks, prefix, dirname + global marks, prefix, gitdir, dirname global tags, filenodes global blob_marks global parsed_refs -- cgit v1.2.1 From 3f6e7c0af18b5362d5d7b847d2b92ce38e72e030 Mon Sep 17 00:00:00 2001 From: Felipe Contreras Date: Tue, 30 Apr 2013 20:10:04 -0500 Subject: remote-bzr: improve author sanitazion So that we don't end up with '', and also synchronize it with the one from remote-hg. Signed-off-by: Felipe Contreras Signed-off-by: Junio C Hamano --- contrib/remote-helpers/git-remote-bzr | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) (limited to 'contrib') diff --git a/contrib/remote-helpers/git-remote-bzr b/contrib/remote-helpers/git-remote-bzr index 9fe830e3cd..af46016dc9 100755 --- a/contrib/remote-helpers/git-remote-bzr +++ b/contrib/remote-helpers/git-remote-bzr @@ -38,6 +38,7 @@ import atexit, shutil, hashlib, urlparse, subprocess NAME_RE = re.compile('^([^<>]+)') AUTHOR_RE = re.compile('^([^<>]+?)? ?<([^<>]*)>$') +EMAIL_RE = re.compile('^([^<>]+[^ \\\t<>])?\\b(?:[ \\t<>]*?)\\b([^ \\t<>]+@[^ \\t<>]+)') RAW_AUTHOR_RE = re.compile('^(\w+) (.+)? <(.*)> (\d+) ([+-]\d+)') def die(msg, *args): @@ -175,9 +176,19 @@ def fixup_user(user): name = m.group(1) mail = m.group(2).strip() else: - m = NAME_RE.match(user) + m = EMAIL_RE.match(user) if m: - name = m.group(1).strip() + name = m.group(1) + mail = m.group(2) + else: + m = NAME_RE.match(user) + if m: + name = m.group(1).strip() + + if not name: + name = 'unknown' + if not mail: + mail = 'Unknown' return '%s <%s>' % (name, mail) -- cgit v1.2.1 From 850dd25c9a245da0e6b23c36cf9f45d6ee08b237 Mon Sep 17 00:00:00 2001 From: Felipe Contreras Date: Tue, 30 Apr 2013 20:10:05 -0500 Subject: remote-bzr: add custom method to find branches The official method is incredibly inefficient and slow. Signed-off-by: Felipe Contreras Signed-off-by: Junio C Hamano --- contrib/remote-helpers/git-remote-bzr | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) (limited to 'contrib') diff --git a/contrib/remote-helpers/git-remote-bzr b/contrib/remote-helpers/git-remote-bzr index af46016dc9..11f24152bf 100755 --- a/contrib/remote-helpers/git-remote-bzr +++ b/contrib/remote-helpers/git-remote-bzr @@ -756,6 +756,25 @@ def get_remote_branch(origin, remote_branch, name): return branch +def find_branches(repo): + transport = repo.user_transport + + for fn in transport.iter_files_recursive(): + if not fn.endswith('.bzr/branch-format'): + continue + + name = subdir = fn[:-len('/.bzr/branch-format')] + name = name if name != '' else 'master' + name = name.replace('/', '+') + + try: + cur = transport.clone(subdir) + branch = bzrlib.branch.Branch.open_from_transport(cur) + except bzrlib.errors.NotBranchError: + continue + else: + yield name, branch + def get_repo(url, alias): global dirname, peer, branches @@ -796,11 +815,7 @@ def get_repo(url, alias): else: # repository - for branch in repo.find_branches(): - - name = repo.user_transport.relpath(branch.base) - name = name if name != '' else 'master' - name = name.replace('/', '+') + for name, branch in find_branches(repo): if not is_local: peers[name] = branch -- cgit v1.2.1 From 248663c4ff931f1f42fb7f8233d0ae23f80c7835 Mon Sep 17 00:00:00 2001 From: Felipe Contreras Date: Tue, 30 Apr 2013 20:10:06 -0500 Subject: remote-bzr: add option to specify branches We might not want all the branches. And branch handling in bazaar is rather tricky, so it's safer to simply specify them. Signed-off-by: Felipe Contreras Signed-off-by: Junio C Hamano --- contrib/remote-helpers/git-remote-bzr | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) (limited to 'contrib') diff --git a/contrib/remote-helpers/git-remote-bzr b/contrib/remote-helpers/git-remote-bzr index 11f24152bf..d284afc0d3 100755 --- a/contrib/remote-helpers/git-remote-bzr +++ b/contrib/remote-helpers/git-remote-bzr @@ -13,6 +13,9 @@ # or # % git clone bzr::lp:myrepo # +# If you want to specify which branches you want track (per repo): +# git config remote-bzr.branches 'trunk, devel, test' +# import sys @@ -51,6 +54,12 @@ def warn(msg, *args): def gittz(tz): return '%+03d%02d' % (tz / 3600, tz % 3600 / 60) +def get_config(config): + cmd = ['git', 'config', '--get', config] + process = subprocess.Popen(cmd, stdout=subprocess.PIPE) + output, _ = process.communicate() + return output + class Marks: def __init__(self, path): @@ -756,7 +765,7 @@ def get_remote_branch(origin, remote_branch, name): return branch -def find_branches(repo): +def find_branches(repo, wanted): transport = repo.user_transport for fn in transport.iter_files_recursive(): @@ -767,6 +776,9 @@ def find_branches(repo): name = name if name != '' else 'master' name = name.replace('/', '+') + if wanted and not name in wanted: + continue + try: cur = transport.clone(subdir) branch = bzrlib.branch.Branch.open_from_transport(cur) @@ -815,7 +827,11 @@ def get_repo(url, alias): else: # repository - for name, branch in find_branches(repo): + wanted = get_config('remote-bzr.branches').rstrip().split(', ') + # stupid python + wanted = [e for e in wanted if e] + + for name, branch in find_branches(repo, wanted): if not is_local: peers[name] = branch -- cgit v1.2.1 From a39769995050f5fe1d4907514caf9eb32ef6185a Mon Sep 17 00:00:00 2001 From: Felipe Contreras Date: Tue, 30 Apr 2013 20:10:07 -0500 Subject: remote-bzr: improve progress reporting No need to manually count the revisions, and also, this would help to iterate more properly. Signed-off-by: Felipe Contreras Signed-off-by: Junio C Hamano --- contrib/remote-helpers/git-remote-bzr | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) (limited to 'contrib') diff --git a/contrib/remote-helpers/git-remote-bzr b/contrib/remote-helpers/git-remote-bzr index d284afc0d3..2e210089bf 100755 --- a/contrib/remote-helpers/git-remote-bzr +++ b/contrib/remote-helpers/git-remote-bzr @@ -282,13 +282,16 @@ def export_branch(repo, name): branch.lock_read() revs = branch.iter_merge_sorted_revisions(None, tip, 'exclude', 'forward') - count = 0 + tip_revno = branch.revision_id_to_revno(tip) + last_revno, _ = branch.last_revision_info() + total = last_revno - tip_revno - revs = [revid for revid, _, _, _ in revs if not marks.is_marked(revid)] + revs = [revid, seq for revid, _, seq, _ in revs if not marks.is_marked(revid)] - for revid in revs: + for revid, seq in revs: rev = repo.get_revision(revid) + revno = seq[0] parents = rev.parent_ids time = rev.timestamp @@ -343,10 +346,13 @@ def export_branch(repo, name): print "M %s :%u %s" % f print - count += 1 - if (count % 100 == 0): - print "progress revision %s '%s' (%d/%d)" % (revid, name, count, len(revs)) - print "#############################################################" + if len(seq) > 1: + # let's skip branch revisions from the progress report + continue + + progress = (revno - tip_revno) + if (progress % 100 == 0): + print "progress revision %d '%s' (%d/%d)" % (revno, name, progress, total) branch.unlock() -- cgit v1.2.1 From 38cecbdf52a081b732726180ad5e182d15117914 Mon Sep 17 00:00:00 2001 From: Felipe Contreras Date: Tue, 30 Apr 2013 20:10:08 -0500 Subject: remote-bzr: iterate revisions properly This way we don't need to store the list of all the revisions, which doesn't seem to be very memory efficient with bazaar's design, for whatever reason. Signed-off-by: Felipe Contreras Signed-off-by: Junio C Hamano --- contrib/remote-helpers/git-remote-bzr | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'contrib') diff --git a/contrib/remote-helpers/git-remote-bzr b/contrib/remote-helpers/git-remote-bzr index 2e210089bf..66e3861c21 100755 --- a/contrib/remote-helpers/git-remote-bzr +++ b/contrib/remote-helpers/git-remote-bzr @@ -286,9 +286,10 @@ def export_branch(repo, name): last_revno, _ = branch.last_revision_info() total = last_revno - tip_revno - revs = [revid, seq for revid, _, seq, _ in revs if not marks.is_marked(revid)] + for revid, _, seq, _ in revs: - for revid, seq in revs: + if marks.is_marked(revid): + continue rev = repo.get_revision(revid) revno = seq[0] -- cgit v1.2.1 From 747c9a377fb73aa30f6b4dca958e3b121f88bb66 Mon Sep 17 00:00:00 2001 From: Felipe Contreras Date: Tue, 30 Apr 2013 20:10:09 -0500 Subject: remote-bzr: delay peer branch usage So it doesn't time out. Signed-off-by: Felipe Contreras Signed-off-by: Junio C Hamano --- contrib/remote-helpers/git-remote-bzr | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'contrib') diff --git a/contrib/remote-helpers/git-remote-bzr b/contrib/remote-helpers/git-remote-bzr index 66e3861c21..b2d67f7f37 100755 --- a/contrib/remote-helpers/git-remote-bzr +++ b/contrib/remote-helpers/git-remote-bzr @@ -687,7 +687,7 @@ def do_export(parser): branch.generate_revision_history(revid, marks.get_tip(name)) if name in peers: - peer = peers[name] + peer = bzrlib.branch.Branch.open(peers[name]) try: peer.bzrdir.push_branch(branch, revision_id=revid) except bzrlib.errors.DivergedBranches: @@ -825,7 +825,7 @@ def get_repo(url, alias): branch = origin.open_branch() if not is_local: - peers[name] = branch + peers[name] = branch.base branches[name] = get_remote_branch(origin, branch, name) else: branches[name] = branch @@ -841,7 +841,7 @@ def get_repo(url, alias): for name, branch in find_branches(repo, wanted): if not is_local: - peers[name] = branch + peers[name] = branch.base branches[name] = get_remote_branch(origin, branch, name) else: branches[name] = branch -- cgit v1.2.1 From d421c02b4144df9b0448e321930ee6a604cdd3fc Mon Sep 17 00:00:00 2001 From: Felipe Contreras Date: Tue, 30 Apr 2013 20:10:10 -0500 Subject: remote-bzr: access branches only when needed Bazaar doesn't seem to be tested for multiple usage of branches, so resources seem to be leaked all over. Let's try to minimize this by accessing the Branch objects only when needed. Signed-off-by: Felipe Contreras Signed-off-by: Junio C Hamano --- contrib/remote-helpers/git-remote-bzr | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) (limited to 'contrib') diff --git a/contrib/remote-helpers/git-remote-bzr b/contrib/remote-helpers/git-remote-bzr index b2d67f7f37..161f83193c 100755 --- a/contrib/remote-helpers/git-remote-bzr +++ b/contrib/remote-helpers/git-remote-bzr @@ -277,7 +277,7 @@ def export_branch(repo, name): ref = '%s/heads/%s' % (prefix, name) tip = marks.get_tip(name) - branch = branches[name] + branch = bzrlib.branch.Branch.open(branches[name]) repo = branch.repository branch.lock_read() @@ -582,7 +582,7 @@ def parse_commit(parser): if ref.startswith('refs/heads/'): name = ref[len('refs/heads/'):] - branch = branches[name] + branch = bzrlib.branch.Branch.open(branches[name]) else: die('unknown ref') @@ -683,7 +683,7 @@ def do_export(parser): for ref, revid in parsed_refs.iteritems(): name = ref[len('refs/heads/'):] - branch = branches[name] + branch = bzrlib.branch.Branch.open(branches[name]) branch.generate_revision_history(revid, marks.get_tip(name)) if name in peers: @@ -733,7 +733,7 @@ def do_list(parser): master_branch = name print "? refs/heads/%s" % name - branch = branches[master_branch] + branch = bzrlib.branch.Branch.open(branches[master_branch]) branch.lock_read() for tag, revid in branch.tags.get_tag_dict().items(): try: @@ -822,13 +822,15 @@ def get_repo(url, alias): # branch name = 'master' - branch = origin.open_branch() + remote_branch = origin.open_branch() if not is_local: - peers[name] = branch.base - branches[name] = get_remote_branch(origin, branch, name) + peers[name] = remote_branch.base + branch = get_remote_branch(origin, remote_branch, name) else: - branches[name] = branch + branch = remote_branch + + branches[name] = branch.base return branch.repository else: @@ -838,13 +840,15 @@ def get_repo(url, alias): # stupid python wanted = [e for e in wanted if e] - for name, branch in find_branches(repo, wanted): + for name, remote_branch in find_branches(repo, wanted): if not is_local: - peers[name] = branch.base - branches[name] = get_remote_branch(origin, branch, name) + peers[name] = remote_branch.base + branch = get_remote_branch(origin, remote_branch, name) else: - branches[name] = branch + branch = remote_branch + + branches[name] = branch.base return repo -- cgit v1.2.1 From 081811216e99fe0396d74d8a2bd6dd3d119dccde Mon Sep 17 00:00:00 2001 From: Felipe Contreras Date: Fri, 3 May 2013 19:31:06 -0500 Subject: remote-bzr: convert all unicode keys to str Otherwise some versions of bazaar might barf. Signed-off-by: Felipe Contreras Signed-off-by: Junio C Hamano --- contrib/remote-helpers/git-remote-bzr | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'contrib') diff --git a/contrib/remote-helpers/git-remote-bzr b/contrib/remote-helpers/git-remote-bzr index 161f83193c..bbaaa8f6f3 100755 --- a/contrib/remote-helpers/git-remote-bzr +++ b/contrib/remote-helpers/git-remote-bzr @@ -95,7 +95,7 @@ class Marks: return self.marks[rev] def to_rev(self, mark): - return self.rev_marks[mark] + return str(self.rev_marks[mark]) def next_mark(self): self.last_mark += 1 @@ -621,7 +621,7 @@ def parse_commit(parser): files[path] = f committer, date, tz = committer - parents = [str(mark_to_rev(p)) for p in parents] + parents = [mark_to_rev(p) for p in parents] revid = bzrlib.generate_ids.gen_revision_id(committer, date) props = {} props['branch-nick'] = branch.nick -- cgit v1.2.1 From 4c00819910706aa8ae450292085e47e52c333440 Mon Sep 17 00:00:00 2001 From: Felipe Contreras Date: Fri, 3 May 2013 19:31:07 -0500 Subject: remote-bzr: avoid bad refs Versions of fast-export before v1.8.2 throws a bad 'reset' commands because of a behavior in transport-helper that is not even needed. We should ignore them, otherwise we will treat them as branches and fail. This was fixed in v1.8.2, but some people use this script in older versions of git. Also, check if the ref was a tag, and skip it for now. Signed-off-by: Felipe Contreras Signed-off-by: Junio C Hamano --- contrib/remote-helpers/git-remote-bzr | 38 +++++++++++++++++++++-------------- 1 file changed, 23 insertions(+), 15 deletions(-) (limited to 'contrib') diff --git a/contrib/remote-helpers/git-remote-bzr b/contrib/remote-helpers/git-remote-bzr index bbaaa8f6f3..0ef30f8d55 100755 --- a/contrib/remote-helpers/git-remote-bzr +++ b/contrib/remote-helpers/git-remote-bzr @@ -682,23 +682,31 @@ def do_export(parser): die('unhandled export command: %s' % line) for ref, revid in parsed_refs.iteritems(): - name = ref[len('refs/heads/'):] - branch = bzrlib.branch.Branch.open(branches[name]) - branch.generate_revision_history(revid, marks.get_tip(name)) + if ref.startswith('refs/heads/'): + name = ref[len('refs/heads/'):] + branch = bzrlib.branch.Branch.open(branches[name]) + branch.generate_revision_history(revid, marks.get_tip(name)) - if name in peers: - peer = bzrlib.branch.Branch.open(peers[name]) - try: - peer.bzrdir.push_branch(branch, revision_id=revid) - except bzrlib.errors.DivergedBranches: - print "error %s non-fast forward" % ref - continue + if name in peers: + peer = bzrlib.branch.Branch.open(peers[name]) + try: + peer.bzrdir.push_branch(branch, revision_id=revid) + except bzrlib.errors.DivergedBranches: + print "error %s non-fast forward" % ref + continue - try: - wt = branch.bzrdir.open_workingtree() - wt.update() - except bzrlib.errors.NoWorkingTree: - pass + try: + wt = branch.bzrdir.open_workingtree() + wt.update() + except bzrlib.errors.NoWorkingTree: + pass + elif ref.startswith('refs/tags/'): + # TODO: implement tag push + print "error %s pushing tags not supported" % ref + continue + else: + # transport-helper/fast-export bugs + continue print "ok %s" % ref -- cgit v1.2.1