summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--app/models/merge_request_diff.rb18
-rw-r--r--changelogs/unreleased/fix-import-fork.yml4
-rw-r--r--lib/gitlab/import_export/hash_util.rb25
-rw-r--r--lib/gitlab/import_export/import_export.yml2
-rw-r--r--lib/gitlab/import_export/importer.rb2
-rw-r--r--lib/gitlab/import_export/merge_request_parser.rb41
-rw-r--r--lib/gitlab/import_export/project_tree_restorer.rb2
-rw-r--r--lib/gitlab/import_export/relation_factory.rb17
-rw-r--r--spec/lib/gitlab/import_export/fork_spec.rb49
-rw-r--r--spec/lib/gitlab/import_export/hash_util_spec.rb28
-rw-r--r--spec/lib/gitlab/import_export/merge_request_parser_spec.rb31
-rw-r--r--spec/lib/gitlab/import_export/project_tree_restorer_spec.rb6
-rw-r--r--spec/lib/gitlab/import_export/project_tree_saver_spec.rb4
-rw-r--r--spec/lib/gitlab/import_export/relation_factory_spec.rb2
-rw-r--r--spec/lib/gitlab/import_export/repo_saver_spec.rb (renamed from spec/lib/gitlab/import_export/repo_bundler_spec.rb)0
15 files changed, 215 insertions, 16 deletions
diff --git a/app/models/merge_request_diff.rb b/app/models/merge_request_diff.rb
index baee00b8fcd..6ad56b842b2 100644
--- a/app/models/merge_request_diff.rb
+++ b/app/models/merge_request_diff.rb
@@ -177,6 +177,16 @@ class MergeRequestDiff < ActiveRecord::Base
st_commits.count
end
+ def utf8_st_diffs
+ return [] if st_diffs.blank?
+
+ st_diffs.map do |diff|
+ diff.each do |k, v|
+ diff[k] = encode_utf8(v) if v.respond_to?(:encoding)
+ end
+ end
+ end
+
private
# Old GitLab implementations may have generated diffs as ["--broken-diff"].
@@ -270,14 +280,6 @@ class MergeRequestDiff < ActiveRecord::Base
project.merge_base_commit(head_commit_sha, start_commit_sha).try(:sha)
end
- def utf8_st_diffs
- st_diffs.map do |diff|
- diff.each do |k, v|
- diff[k] = encode_utf8(v) if v.respond_to?(:encoding)
- end
- end
- end
-
#
# #save or #update_attributes providing changes on serialized attributes do a lot of
# serialization and deserialization calls resulting in bad performance.
diff --git a/changelogs/unreleased/fix-import-fork.yml b/changelogs/unreleased/fix-import-fork.yml
new file mode 100644
index 00000000000..ff8dd131995
--- /dev/null
+++ b/changelogs/unreleased/fix-import-fork.yml
@@ -0,0 +1,4 @@
+---
+title: Fix Import/Export MR diffs not showing and missing forked MRs
+merge_request:
+author:
diff --git a/lib/gitlab/import_export/hash_util.rb b/lib/gitlab/import_export/hash_util.rb
new file mode 100644
index 00000000000..d4adeeb3797
--- /dev/null
+++ b/lib/gitlab/import_export/hash_util.rb
@@ -0,0 +1,25 @@
+module Gitlab
+ module ImportExport
+ class HashUtil
+ def self.deep_symbolize_array!(array)
+ return if array.blank?
+
+ array.map! do |hash|
+ hash.deep_symbolize_keys!
+
+ yield(hash) if block_given?
+
+ hash
+ end
+ end
+
+ def self.deep_symbolize_array_with_date!(array)
+ self.deep_symbolize_array!(array) do |hash|
+ hash.select { |k, _v| k.to_s.end_with?('_date') }.each do |key, value|
+ hash[key] = Time.zone.parse(value)
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml
index ab74c8782f6..f69288f7d67 100644
--- a/lib/gitlab/import_export/import_export.yml
+++ b/lib/gitlab/import_export/import_export.yml
@@ -89,3 +89,5 @@ methods:
- :type
merge_request_diff:
- :utf8_st_diffs
+ merge_requests:
+ - :diff_head_sha
diff --git a/lib/gitlab/import_export/importer.rb b/lib/gitlab/import_export/importer.rb
index 063ce74ecad..fbdd74788bc 100644
--- a/lib/gitlab/import_export/importer.rb
+++ b/lib/gitlab/import_export/importer.rb
@@ -9,7 +9,7 @@ module Gitlab
end
def execute
- if import_file && check_version! && [project_tree, avatar_restorer, repo_restorer, wiki_restorer, uploads_restorer].all?(&:restore)
+ if import_file && check_version! && [repo_restorer, wiki_restorer, project_tree, avatar_restorer, uploads_restorer].all?(&:restore)
project_tree.restored_project
else
raise Projects::ImportService::Error.new(@shared.errors.join(', '))
diff --git a/lib/gitlab/import_export/merge_request_parser.rb b/lib/gitlab/import_export/merge_request_parser.rb
new file mode 100644
index 00000000000..c20adc20bfd
--- /dev/null
+++ b/lib/gitlab/import_export/merge_request_parser.rb
@@ -0,0 +1,41 @@
+module Gitlab
+ module ImportExport
+ class MergeRequestParser
+ FORKED_PROJECT_ID = -1
+
+ def initialize(project, diff_head_sha, merge_request, relation_hash)
+ @project = project
+ @diff_head_sha = diff_head_sha
+ @merge_request = merge_request
+ @relation_hash = relation_hash
+ end
+
+ def parse!
+ if fork_merge_request? && @diff_head_sha
+ @merge_request.source_project_id = @relation_hash['project_id']
+
+ fetch_ref unless branch_exists?(@merge_request.source_branch)
+ create_target_branch unless branch_exists?(@merge_request.target_branch)
+ end
+
+ @merge_request
+ end
+
+ def create_target_branch
+ @project.repository.create_branch(@merge_request.target_branch, @merge_request.target_branch_sha)
+ end
+
+ def fetch_ref
+ @project.repository.fetch_ref(@project.repository.path, @diff_head_sha, @merge_request.source_branch)
+ end
+
+ def branch_exists?(branch_name)
+ @project.repository.branch_exists?(branch_name)
+ end
+
+ def fork_merge_request?
+ @relation_hash['source_project_id'] == FORKED_PROJECT_ID
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb
index cda6ddf0443..df21ff22216 100644
--- a/lib/gitlab/import_export/project_tree_restorer.rb
+++ b/lib/gitlab/import_export/project_tree_restorer.rb
@@ -119,7 +119,7 @@ module Gitlab
relation_hash: parsed_relation_hash(relation_hash),
members_mapper: members_mapper,
user: @user,
- project_id: restored_project.id)
+ project: restored_project)
end.compact
relation_hash_list.is_a?(Array) ? relation_array : relation_array.first
diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb
index d44563333a5..fb43e7ccdbb 100644
--- a/lib/gitlab/import_export/relation_factory.rb
+++ b/lib/gitlab/import_export/relation_factory.rb
@@ -29,11 +29,12 @@ module Gitlab
new(*args).create
end
- def initialize(relation_sym:, relation_hash:, members_mapper:, user:, project_id:)
+ def initialize(relation_sym:, relation_hash:, members_mapper:, user:, project:)
@relation_name = OVERRIDES[relation_sym] || relation_sym
- @relation_hash = relation_hash.except('noteable_id').merge('project_id' => project_id)
+ @relation_hash = relation_hash.except('noteable_id').merge('project_id' => project.id)
@members_mapper = members_mapper
@user = user
+ @project = project
@imported_object_retries = 0
end
@@ -66,7 +67,7 @@ module Gitlab
remove_encrypted_attributes!
@relation_hash['data'].deep_symbolize_keys! if @relation_name == :events && @relation_hash['data']
- set_st_diffs if @relation_name == :merge_request_diff
+ set_st_diff_commits if @relation_name == :merge_request_diff
end
def update_user_references
@@ -105,6 +106,8 @@ module Gitlab
imported_object do |object|
object.commit_id = nil
end
+ elsif @relation_name == :merge_requests
+ MergeRequestParser.new(@project, @relation_hash.delete('diff_head_sha'), imported_object, @relation_hash).parse!
else
imported_object
end
@@ -115,7 +118,7 @@ module Gitlab
# If source and target are the same, populate them with the new project ID.
if @relation_hash['source_project_id']
- @relation_hash['source_project_id'] = same_source_and_target? ? project_id : -1
+ @relation_hash['source_project_id'] = same_source_and_target? ? project_id : MergeRequestParser::FORKED_PROJECT_ID
end
# project_id may not be part of the export, but we always need to populate it if required.
@@ -166,6 +169,7 @@ module Gitlab
def imported_object
yield(existing_or_new_object) if block_given?
existing_or_new_object.importing = true if existing_or_new_object.respond_to?(:importing)
+
existing_or_new_object
rescue ActiveRecord::RecordNotUnique
# as the operation is not atomic, retry in the unlikely scenario an INSERT is
@@ -188,8 +192,11 @@ module Gitlab
relation_class: relation_class)
end
- def set_st_diffs
+ def set_st_diff_commits
@relation_hash['st_diffs'] = @relation_hash.delete('utf8_st_diffs')
+
+ HashUtil.deep_symbolize_array!(@relation_hash['st_diffs'])
+ HashUtil.deep_symbolize_array_with_date!(@relation_hash['st_commits'])
end
def existing_or_new_object
diff --git a/spec/lib/gitlab/import_export/fork_spec.rb b/spec/lib/gitlab/import_export/fork_spec.rb
new file mode 100644
index 00000000000..c5ce06afd73
--- /dev/null
+++ b/spec/lib/gitlab/import_export/fork_spec.rb
@@ -0,0 +1,49 @@
+require 'spec_helper'
+
+describe 'forked project import', services: true do
+ let(:user) { create(:user) }
+ let!(:project_with_repo) { create(:project, :test_repo, name: 'test-repo-restorer', path: 'test-repo-restorer') }
+ let!(:project) { create(:empty_project) }
+ let(:export_path) { "#{Dir.tmpdir}/project_tree_saver_spec" }
+ let(:shared) { Gitlab::ImportExport::Shared.new(relative_path: project.path_with_namespace) }
+ let(:forked_from_project) { create(:project) }
+ let(:fork_link) { create(:forked_project_link, forked_from_project: project_with_repo) }
+ let(:repo_saver) { Gitlab::ImportExport::RepoSaver.new(project: project_with_repo, shared: shared) }
+ let(:bundle_path) { File.join(shared.export_path, Gitlab::ImportExport.project_bundle_filename) }
+
+ let(:repo_restorer) do
+ Gitlab::ImportExport::RepoRestorer.new(path_to_bundle: bundle_path, shared: shared, project: project)
+ end
+
+ let!(:merge_request) do
+ create(:merge_request, source_project: fork_link.forked_to_project, target_project: project_with_repo)
+ end
+
+ let(:saver) do
+ Gitlab::ImportExport::ProjectTreeSaver.new(project: project_with_repo, current_user: user, shared: shared)
+ end
+
+ let(:restorer) do
+ Gitlab::ImportExport::ProjectTreeRestorer.new(user: user, shared: shared, project: project)
+ end
+
+ before do
+ allow_any_instance_of(Gitlab::ImportExport).to receive(:storage_path).and_return(export_path)
+
+ saver.save
+ repo_saver.save
+
+ repo_restorer.restore
+ restorer.restore
+ end
+
+ after do
+ FileUtils.rm_rf(export_path)
+ FileUtils.rm_rf(project_with_repo.repository.path_to_repo)
+ FileUtils.rm_rf(project.repository.path_to_repo)
+ end
+
+ it 'can access the MR' do
+ expect(project.merge_requests.first.ensure_ref_fetched.first).to include('refs/merge-requests/1/head')
+ end
+end
diff --git a/spec/lib/gitlab/import_export/hash_util_spec.rb b/spec/lib/gitlab/import_export/hash_util_spec.rb
new file mode 100644
index 00000000000..1c3a0b23ece
--- /dev/null
+++ b/spec/lib/gitlab/import_export/hash_util_spec.rb
@@ -0,0 +1,28 @@
+require 'spec_helper'
+
+describe Gitlab::ImportExport::HashUtil, lib: true do
+ let(:stringified_array) { [{ 'test' => 1 }] }
+ let(:stringified_array_with_date) { [{ 'test_date' => '2016-04-06 06:17:44 +0200' }] }
+
+ describe '.deep_symbolize_array!' do
+ it 'symbolizes keys' do
+ expect { described_class.deep_symbolize_array!(stringified_array) }.to change {
+ stringified_array.first.keys.first
+ }.from('test').to(:test)
+ end
+ end
+
+ describe '.deep_symbolize_array_with_date!' do
+ it 'symbolizes keys' do
+ expect { described_class.deep_symbolize_array_with_date!(stringified_array_with_date) }.to change {
+ stringified_array_with_date.first.keys.first
+ }.from('test_date').to(:test_date)
+ end
+
+ it 'transforms date strings into Time objects' do
+ expect { described_class.deep_symbolize_array_with_date!(stringified_array_with_date) }.to change {
+ stringified_array_with_date.first.values.first.class
+ }.from(String).to(ActiveSupport::TimeWithZone)
+ end
+ end
+end
diff --git a/spec/lib/gitlab/import_export/merge_request_parser_spec.rb b/spec/lib/gitlab/import_export/merge_request_parser_spec.rb
new file mode 100644
index 00000000000..349be4596b6
--- /dev/null
+++ b/spec/lib/gitlab/import_export/merge_request_parser_spec.rb
@@ -0,0 +1,31 @@
+require 'spec_helper'
+
+describe Gitlab::ImportExport::MergeRequestParser do
+ let(:user) { create(:user) }
+ let!(:project) { create(:project, :test_repo, name: 'test-repo-restorer', path: 'test-repo-restorer') }
+ let(:forked_from_project) { create(:project) }
+ let(:fork_link) { create(:forked_project_link, forked_from_project: project) }
+
+ let!(:merge_request) do
+ create(:merge_request, source_project: fork_link.forked_to_project, target_project: project)
+ end
+
+ let(:parsed_merge_request) do
+ described_class.new(project,
+ merge_request.diff_head_sha,
+ merge_request,
+ merge_request.as_json).parse!
+ end
+
+ after do
+ FileUtils.rm_rf(project.repository.path_to_repo)
+ end
+
+ it 'has a source branch' do
+ expect(project.repository.branch_exists?(parsed_merge_request.source_branch)).to be true
+ end
+
+ it 'has a target branch' do
+ expect(project.repository.branch_exists?(parsed_merge_request.target_branch)).to be true
+ end
+end
diff --git a/spec/lib/gitlab/import_export/project_tree_restorer_spec.rb b/spec/lib/gitlab/import_export/project_tree_restorer_spec.rb
index c36f12dbd82..af9c25acb02 100644
--- a/spec/lib/gitlab/import_export/project_tree_restorer_spec.rb
+++ b/spec/lib/gitlab/import_export/project_tree_restorer_spec.rb
@@ -82,6 +82,12 @@ describe Gitlab::ImportExport::ProjectTreeRestorer, services: true do
expect(MergeRequestDiff.where.not(st_diffs: nil).count).to eq(9)
end
+ it 'has the correct time for merge request st_commits' do
+ st_commits = MergeRequestDiff.where.not(st_commits: nil).first.st_commits
+
+ expect(st_commits.first[:committed_date]).to be_kind_of(Time)
+ end
+
it 'has labels associated to label links, associated to issues' do
expect(Label.first.label_links.first.target).not_to be_nil
end
diff --git a/spec/lib/gitlab/import_export/project_tree_saver_spec.rb b/spec/lib/gitlab/import_export/project_tree_saver_spec.rb
index 012c22ec5ad..d2d89e3b019 100644
--- a/spec/lib/gitlab/import_export/project_tree_saver_spec.rb
+++ b/spec/lib/gitlab/import_export/project_tree_saver_spec.rb
@@ -79,6 +79,10 @@ describe Gitlab::ImportExport::ProjectTreeSaver, services: true do
expect(saved_project_json['merge_requests'].first['merge_request_diff']).not_to be_empty
end
+ it 'has merge requests diff st_diffs' do
+ expect(saved_project_json['merge_requests'].first['merge_request_diff']['utf8_st_diffs']).not_to be_nil
+ end
+
it 'has merge requests comments' do
expect(saved_project_json['merge_requests'].first['notes']).not_to be_empty
end
diff --git a/spec/lib/gitlab/import_export/relation_factory_spec.rb b/spec/lib/gitlab/import_export/relation_factory_spec.rb
index 57e412b0cef..fcc23a75ca1 100644
--- a/spec/lib/gitlab/import_export/relation_factory_spec.rb
+++ b/spec/lib/gitlab/import_export/relation_factory_spec.rb
@@ -9,7 +9,7 @@ describe Gitlab::ImportExport::RelationFactory, lib: true do
relation_hash: relation_hash,
members_mapper: members_mapper,
user: user,
- project_id: project.id)
+ project: project)
end
context 'hook object' do
diff --git a/spec/lib/gitlab/import_export/repo_bundler_spec.rb b/spec/lib/gitlab/import_export/repo_saver_spec.rb
index a7f4e11271e..a7f4e11271e 100644
--- a/spec/lib/gitlab/import_export/repo_bundler_spec.rb
+++ b/spec/lib/gitlab/import_export/repo_saver_spec.rb