diff options
| -rw-r--r-- | app/controllers/import/google_code_controller.rb | 52 | ||||
| -rw-r--r-- | app/views/import/google_code/new.html.haml | 13 | ||||
| -rw-r--r-- | app/views/import/google_code/new_user_map.html.haml | 20 | ||||
| -rw-r--r-- | app/views/import/google_code/status.html.haml | 1 | ||||
| -rw-r--r-- | config/routes.rb | 3 | ||||
| -rw-r--r-- | lib/gitlab/google_code_import/client.rb | 25 | ||||
| -rw-r--r-- | lib/gitlab/google_code_import/importer.rb | 32 | ||||
| -rw-r--r-- | lib/gitlab/google_code_import/project_creator.rb | 12 | ||||
| -rw-r--r-- | lib/gitlab/google_code_import/repository.rb | 4 | ||||
| -rw-r--r-- | spec/lib/gitlab/google_code_import/importer_spec.rb | 11 |
10 files changed, 150 insertions, 23 deletions
diff --git a/app/controllers/import/google_code_controller.rb b/app/controllers/import/google_code_controller.rb index 9bf29386bc9..fb4ef987367 100644 --- a/app/controllers/import/google_code_controller.rb +++ b/app/controllers/import/google_code_controller.rb @@ -1,4 +1,5 @@ class Import::GoogleCodeController < Import::BaseController + before_filter :user_map, only: [:new_user_map, :create_user_map] def new @@ -17,11 +18,46 @@ class Import::GoogleCodeController < Import::BaseController return redirect_to :back, alert: "The uploaded file is not a valid Google Takeout archive." end - unless Gitlab::GoogleCodeImport::Client.new(dump).valid? + client = Gitlab::GoogleCodeImport::Client.new(dump) + unless client.valid? return redirect_to :back, alert: "The uploaded file is not a valid Google Takeout archive." end session[:google_code_dump] = dump + + if params[:create_user_map] == "1" + redirect_to new_user_map_import_google_code_path + else + redirect_to status_import_google_code_path + end + end + + def new_user_map + + end + + def create_user_map + user_map_json = params[:user_map] + user_map_json = "{}" if user_map_json.blank? + + begin + user_map = JSON.parse(user_map_json) + rescue + flash.now[:alert] = "The entered user map is not a valid JSON user map." + + render "new_user_map" and return + end + + unless user_map.is_a?(Hash) && user_map.all? { |k, v| k.is_a?(String) && v.is_a?(String) } + flash.now[:alert] = "The entered user map is not a valid JSON user map." + + render "new_user_map" and return + end + + session[:google_code_user_map] = user_map + + flash[:notice] = "The user map has been saved. Continue by selecting the projects you want to import." + redirect_to status_import_google_code_path end @@ -51,7 +87,9 @@ class Import::GoogleCodeController < Import::BaseController namespace = @target_namespace - @project = Gitlab::GoogleCodeImport::ProjectCreator.new(repo, namespace, current_user).execute + user_map = session[:google_code_user_map] + + @project = Gitlab::GoogleCodeImport::ProjectCreator.new(repo, namespace, current_user, user_map).execute end private @@ -60,4 +98,14 @@ class Import::GoogleCodeController < Import::BaseController @client ||= Gitlab::GoogleCodeImport::Client.new(session[:google_code_dump]) end + def user_map + @user_map ||= begin + user_map = client.user_map + + stored_user_map = session[:google_code_user_map] + user_map.update(stored_user_map) if stored_user_map + + Hash[user_map.sort] + end + end end diff --git a/app/views/import/google_code/new.html.haml b/app/views/import/google_code/new.html.haml index 0b4edc68428..ce78fec205f 100644 --- a/app/views/import/google_code/new.html.haml +++ b/app/views/import/google_code/new.html.haml @@ -46,6 +46,15 @@ %input{type: "file", name: "dump_file", id: "dump_file"} %li %p - Continue to the next step: + Do you want to customize how Google Code email addresses and usernames are imported into GitLab? %p - = submit_tag 'Select projects to import', class: "btn btn-create" + = label_tag :create_user_map_0 do + = radio_button_tag :create_user_map, 0, true + No, directly import the existing email addresses and usernames. + %p + = label_tag :create_user_map_1 do + = radio_button_tag :create_user_map, 1, false + Yes, let me map Google Code users to full names or GitLab users. + %li + %p + = submit_tag 'Continue to the next step', class: "btn btn-create" diff --git a/app/views/import/google_code/new_user_map.html.haml b/app/views/import/google_code/new_user_map.html.haml new file mode 100644 index 00000000000..2996d3b659b --- /dev/null +++ b/app/views/import/google_code/new_user_map.html.haml @@ -0,0 +1,20 @@ +%h3.page-title + %i.fa.fa-google + Import projects from Google Code +%hr + += form_tag create_user_map_import_google_code_path, class: 'form-horizontal' do + %p + Customize how Google Code email addresses and usernames are imported into GitLab. + In the next step, you'll be able to select the projects you want to import. + %p + The user map is a JSON document mapping Google Code users (as keys) to the way they will be imported into GitLab (as values). By default the username is masked to ensure users' privacy. + %p + To map a Google Code user to a full name or GitLab user, simply replace the value, e.g. <code>"johnsmith@gmail.com": "John Smith"</code> or <code>"johnsmith@gmail.com": "@johnsmith"</code>. Be sure to preserve the surrounding double quotes and other interpunction. + + .form-group + .col-sm-12 + = text_area_tag :user_map, JSON.pretty_generate(@user_map), class: 'form-control', rows: 15 + + .form-actions + = submit_tag 'Continue to the next step', class: "btn btn-create" diff --git a/app/views/import/google_code/status.html.haml b/app/views/import/google_code/status.html.haml index eba9c5296bc..8f24a1ce85d 100644 --- a/app/views/import/google_code/status.html.haml +++ b/app/views/import/google_code/status.html.haml @@ -7,6 +7,7 @@ %hr %p = button_tag 'Import all projects', class: "btn btn-success js-import-all" + = link_to "Specify user map", new_user_map_import_google_code_path, class: "btn prepend-left-10" %table.table.import-jobs %thead diff --git a/config/routes.rb b/config/routes.rb index 88e0227121b..c99980f5e29 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -86,6 +86,9 @@ Gitlab::Application.routes.draw do get :status post :callback get :jobs + + get :new_user_map, path: :user_map + post :create_user_map, path: :user_map end end diff --git a/lib/gitlab/google_code_import/client.rb b/lib/gitlab/google_code_import/client.rb index 0514494d3ad..02f31e45f88 100644 --- a/lib/gitlab/google_code_import/client.rb +++ b/lib/gitlab/google_code_import/client.rb @@ -3,6 +3,12 @@ module Gitlab class Client attr_reader :raw_data + def self.mask_email(author) + parts = author.split("@", 2) + parts[0] = "#{parts[0][0...-3]}..." + parts.join("@") + end + def initialize(raw_data) @raw_data = raw_data end @@ -18,6 +24,25 @@ module Gitlab def repo(id) repos.find { |repo| repo.id == id } end + + def user_map + user_map = Hash.new { |hash, user| hash[user] = self.class.mask_email(user) } + + repos.each do |repo| + next unless repo.valid? && repo.issues + + repo.issues.each do |raw_issue| + # Touching is enough to add the entry and masked email. + user_map[raw_issue["author"]["name"]] + + raw_issue["comments"]["items"].each do |raw_comment| + user_map[raw_comment["author"]["name"]] + end + end + end + + Hash[user_map.sort] + end end end end diff --git a/lib/gitlab/google_code_import/importer.rb b/lib/gitlab/google_code_import/importer.rb index eaa8c6bc6d3..730fbb8606b 100644 --- a/lib/gitlab/google_code_import/importer.rb +++ b/lib/gitlab/google_code_import/importer.rb @@ -5,7 +5,7 @@ module Gitlab def initialize(project) @project = project - @repo = GoogleCodeImport::Repository.new(project.import_data) + @repo = GoogleCodeImport::Repository.new(project.import_data["repo"]) @closed_statuses = [] @known_labels = Set.new @@ -25,6 +25,17 @@ module Gitlab private + def user_map + @user_map ||= begin + user_map = Hash.new { |hash, user| hash[user] = Client.mask_email(user) } + + stored_user_map = project.import_data["user_map"] + user_map.update(stored_user_map) if stored_user_map + + user_map + end + end + def import_status_labels repo.raw_data["issuesConfig"]["statuses"].each do |status| closed = !status["meansOpen"] @@ -45,14 +56,13 @@ module Gitlab end def import_issues - return unless repo.raw_data["issues"] + return unless repo.issues last_id = 0 deleted_issues = [] - issues = repo.raw_data["issues"]["items"] - issues.each do |raw_issue| + repo.issues.each do |raw_issue| while raw_issue["id"] > last_id + 1 last_id += 1 @@ -66,7 +76,7 @@ module Gitlab end last_id = raw_issue["id"] - author = mask_email(raw_issue["author"]["name"]) + author = user_map[raw_issue["author"]["name"]] date = DateTime.parse(raw_issue["published"]).to_formatted_s(:long) body = [] @@ -119,7 +129,7 @@ module Gitlab comments.each do |raw_comment| next if raw_comment.has_key?("deletedBy") - author = mask_email(raw_comment["author"]["name"]) + author = user_map[raw_comment["author"]["name"]] date = DateTime.parse(raw_comment["published"]).to_formatted_s(:long) body = [] @@ -202,12 +212,6 @@ module Gitlab "Status: #{name}" end - def mask_email(author) - parts = author.split("@", 2) - parts[0] = "#{parts[0][0...-3]}..." - parts.join("@") - end - def linkify_issues(s) s.gsub(/([Ii]ssue) ([0-9]+)/, '\1 #\2') end @@ -248,14 +252,14 @@ module Gitlab end if raw_updates.has_key?("owner") - updates << "*Owner: #{mask_email(raw_updates["owner"])}*" + updates << "*Owner: #{user_map[raw_updates["owner"]]}*" end if raw_updates.has_key?("cc") cc = raw_updates["cc"].map do |l| deleted = l.start_with?("-") l = l[1..-1] if deleted - l = mask_email(l) + l = user_map[l] l = "~~#{l}~~" if deleted l end diff --git a/lib/gitlab/google_code_import/project_creator.rb b/lib/gitlab/google_code_import/project_creator.rb index 933e144d8ea..7ac4387d79d 100644 --- a/lib/gitlab/google_code_import/project_creator.rb +++ b/lib/gitlab/google_code_import/project_creator.rb @@ -1,15 +1,21 @@ module Gitlab module GoogleCodeImport class ProjectCreator - attr_reader :repo, :namespace, :current_user + attr_reader :repo, :namespace, :current_user, :user_map - def initialize(repo, namespace, current_user) + def initialize(repo, namespace, current_user, user_map = nil) @repo = repo @namespace = namespace @current_user = current_user + @user_map = user_map end def execute + import_data = { + "repo" => repo.raw_data, + "user_map" => user_map + } + @project = Project.new( name: repo.name, path: repo.name, @@ -20,7 +26,7 @@ module Gitlab import_type: "google_code", import_source: repo.name, import_url: repo.import_url, - import_data: repo.raw_data + import_data: import_data ) if @project.save! diff --git a/lib/gitlab/google_code_import/repository.rb b/lib/gitlab/google_code_import/repository.rb index 39a884d8292..ad33fc2cad2 100644 --- a/lib/gitlab/google_code_import/repository.rb +++ b/lib/gitlab/google_code_import/repository.rb @@ -34,6 +34,10 @@ module Gitlab def import_url raw_data["repositoryUrls"].first end + + def issues + raw_data["issues"] && raw_data["issues"]["items"] + end end end end diff --git a/spec/lib/gitlab/google_code_import/importer_spec.rb b/spec/lib/gitlab/google_code_import/importer_spec.rb index 370d4410104..a68f69d337c 100644 --- a/spec/lib/gitlab/google_code_import/importer_spec.rb +++ b/spec/lib/gitlab/google_code_import/importer_spec.rb @@ -3,7 +3,14 @@ require "spec_helper" describe Gitlab::GoogleCodeImport::Importer do let(:raw_data) { JSON.parse(File.read(Rails.root.join("spec/fixtures/GoogleCodeProjectHosting.json"))) } let(:client) { Gitlab::GoogleCodeImport::Client.new(raw_data) } - let(:import_data) { client.repo("tint2").raw_data } + let(:import_data) { + { + "repo" => client.repo("tint2").raw_data, + "user_map" => { + "thilo..." => "@thilo123" + } + } + } let(:project) { create(:project, import_data: import_data) } subject { described_class.new(project) } @@ -58,7 +65,7 @@ describe Gitlab::GoogleCodeImport::Importer do note = project.issues.first.notes.first expect(note).to_not be_nil expect(note.note).to include("Comment 1") - expect(note.note).to include("thilo...") + expect(note.note).to include("@thilo123") expect(note.note).to include("November 18, 2009 05:14") expect(note.note).to include("applied, thanks.") expect(note.note).to include("Status: Fixed") |