Merge branch 'master' of gitlab.com:gitlab-org/gitlab-ce

4 files changed, 502 insertions, 0 deletions

diff --git a/lib/gitlab/google_code_import/client.rb b/lib/gitlab/google_code_import/client.rb
new file mode 100644
index 00000000000..02f31e45f88
--- /dev/null
+++ b/lib/gitlab/google_code_import/client.rb
@@ -0,0 +1,48 @@
+module Gitlab
+  module GoogleCodeImport
+    class Client
+      attr_reader :raw_data
+
+      def self.mask_email(author)
+        parts = author.split("@", 2)
+        parts[0] = "#{parts[0][0...-3]}..."
+        parts.join("@")
+      end
+
+      def initialize(raw_data)
+        @raw_data = raw_data
+      end
+
+      def valid?
+        raw_data.is_a?(Hash) && raw_data["kind"] == "projecthosting#user" && raw_data.has_key?("projects")
+      end
+
+      def repos
+        @repos ||= raw_data["projects"].map { |raw_repo| GoogleCodeImport::Repository.new(raw_repo) }.select(&:git?)
+      end
+
+      def repo(id)
+        repos.find { |repo| repo.id == id }
+      end
+
+      def user_map
+        user_map = Hash.new { |hash, user| hash[user] = self.class.mask_email(user) }
+
+        repos.each do |repo|
+          next unless repo.valid? && repo.issues
+
+          repo.issues.each do |raw_issue|
+            # Touching is enough to add the entry and masked email.
+            user_map[raw_issue["author"]["name"]]
+
+            raw_issue["comments"]["items"].each do |raw_comment|
+              user_map[raw_comment["author"]["name"]]
+            end
+          end
+        end
+
+        Hash[user_map.sort]
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/google_code_import/importer.rb b/lib/gitlab/google_code_import/importer.rb
new file mode 100644
index 00000000000..777472cf3c5
--- /dev/null
+++ b/lib/gitlab/google_code_import/importer.rb
@@ -0,0 +1,365 @@
+module Gitlab
+  module GoogleCodeImport
+    class Importer
+      attr_reader :project, :repo
+
+      def initialize(project)
+        @project = project
+        @repo = GoogleCodeImport::Repository.new(project.import_data["repo"])
+
+        @closed_statuses = []
+        @known_labels = Set.new
+      end
+
+      def execute
+        return true unless repo.valid?
+
+        import_status_labels
+
+        import_labels
+
+        import_issues
+
+        true
+      end
+
+      private
+
+      def user_map
+        @user_map ||= begin
+          user_map = Hash.new { |hash, user| hash[user] = Client.mask_email(user) }
+
+          stored_user_map = project.import_data["user_map"]
+          user_map.update(stored_user_map) if stored_user_map
+
+          user_map
+        end
+      end
+
+      def import_status_labels
+        repo.raw_data["issuesConfig"]["statuses"].each do |status|
+          closed = !status["meansOpen"]
+          @closed_statuses << status["status"] if closed
+
+          name = nice_status_name(status["status"])
+          create_label(name)
+          @known_labels << name
+        end
+      end
+
+      def import_labels
+        repo.raw_data["issuesConfig"]["labels"].each do |label|
+          name = nice_label_name(label["label"])
+          create_label(name)
+          @known_labels << name
+        end
+      end
+
+      def import_issues
+        return unless repo.issues
+
+        last_id = 0
+
+        deleted_issues = []
+
+        repo.issues.each do |raw_issue|
+          while raw_issue["id"] > last_id + 1
+            last_id += 1
+
+            issue = project.issues.create!(
+              title:        "Deleted issue",
+              description:  "*This issue has been deleted*",
+              author_id:    project.creator_id,
+              state:        "closed"
+            )
+            deleted_issues << issue
+          end
+          last_id = raw_issue["id"]
+
+          author  = user_map[raw_issue["author"]["name"]]
+          date    = DateTime.parse(raw_issue["published"]).to_formatted_s(:long)
+
+          comments = raw_issue["comments"]["items"]
+          issue_comment = comments.shift
+
+          content     = format_content(issue_comment["content"])
+          attachments = format_attachments(raw_issue["id"], 0, issue_comment["attachments"])
+
+          body = format_issue_body(author, date, content, attachments)
+
+          labels = []
+          raw_issue["labels"].each do |label|
+            name = nice_label_name(label)
+            labels << name
+
+            unless @known_labels.include?(name)
+              create_label(name)
+              @known_labels << name
+            end
+          end
+          labels << nice_status_name(raw_issue["status"])
+
+          assignee_id = nil
+          if raw_issue.has_key?("owner")
+            username = user_map[raw_issue["owner"]["name"]]
+
+            if username.start_with?("@")
+              username = username[1..-1]
+
+              if user = User.find_by(username: username)
+                assignee_id = user.id
+              end
+            end
+          end
+
+          issue = project.issues.create!(
+            title:        raw_issue["title"],
+            description:  body,
+            author_id:    project.creator_id,
+            assignee_id:  assignee_id,
+            state:        raw_issue["state"] == "closed" ? "closed" : "opened"
+          )
+          issue.add_labels_by_names(labels)
+
+          import_issue_comments(issue, comments)
+        end
+
+        deleted_issues.each(&:destroy!)
+      end
+
+      def import_issue_comments(issue, comments)
+        comments.each do |raw_comment|
+          next if raw_comment.has_key?("deletedBy")
+
+          content     = format_content(raw_comment["content"])
+          updates     = format_updates(raw_comment["updates"])
+          attachments = format_attachments(issue.iid, raw_comment["id"], raw_comment["attachments"])
+
+          next if content.blank? && updates.blank? && attachments.blank?
+
+          author  = user_map[raw_comment["author"]["name"]]
+          date    = DateTime.parse(raw_comment["published"]).to_formatted_s(:long)
+
+          body = format_issue_comment_body(
+            raw_comment["id"],
+            author,
+            date,
+            content,
+            updates,
+            attachments
+          )
+
+          issue.notes.create!(
+            project_id: project.id,
+            author_id:  project.creator_id,
+            note:       body
+          )
+        end
+      end
+
+      def nice_label_color(name)
+        case name
+        when /\AComponent:/
+          "#fff39e"
+        when /\AOpSys:/
+          "#e2e2e2"
+        when /\AMilestone:/
+          "#fee3ff"
+
+        when *@closed_statuses.map { |s| nice_status_name(s) }
+          "#cfcfcf"
+        when "Status: New"
+          "#428bca"
+        when "Status: Accepted"
+          "#5cb85c"
+        when "Status: Started"
+          "#8e44ad"
+        
+        when "Priority: Critical"
+          "#ffcfcf"
+        when "Priority: High"
+          "#deffcf"
+        when "Priority: Medium"
+          "#fff5cc"
+        when "Priority: Low"
+          "#cfe9ff"
+        
+        when "Type: Defect"
+          "#d9534f"
+        when "Type: Enhancement"
+          "#44ad8e"
+        when "Type: Task"
+          "#4b6dd0"
+        when "Type: Review"
+          "#8e44ad"
+        when "Type: Other"
+          "#7f8c8d"
+        else
+          "#e2e2e2"
+        end
+      end
+
+      def nice_label_name(name)
+        name.sub("-", ": ")
+      end
+
+      def nice_status_name(name)
+        "Status: #{name}"
+      end
+
+      def linkify_issues(s)
+        s.gsub(/([Ii]ssue) ([0-9]+)/, '\1 #\2')
+      end
+
+      def escape_for_markdown(s)
+        s = s.gsub("*", "\\*")
+        s = s.gsub("#", "\\#")
+        s = s.gsub("`", "\\`")
+        s = s.gsub(":", "\\:")
+        s = s.gsub("-", "\\-")
+        s = s.gsub("+", "\\+")
+        s = s.gsub("_", "\\_")
+        s = s.gsub("(", "\\(")
+        s = s.gsub(")", "\\)")
+        s = s.gsub("[", "\\[")
+        s = s.gsub("]", "\\]")
+        s = s.gsub("<", "\\<")
+        s = s.gsub(">", "\\>")
+        s = s.gsub("\r", "")
+        s = s.gsub("\n", "  \n")
+        s
+      end
+
+      def create_label(name)
+        color = nice_label_color(name)
+        project.labels.create!(name: name, color: color)
+      end
+
+      def format_content(raw_content)
+        linkify_issues(escape_for_markdown(raw_content))
+      end
+
+      def format_updates(raw_updates)
+        updates = []
+
+        if raw_updates.has_key?("status")
+          updates << "*Status: #{raw_updates["status"]}*"
+        end
+
+        if raw_updates.has_key?("owner")
+          updates << "*Owner: #{user_map[raw_updates["owner"]]}*"
+        end
+
+        if raw_updates.has_key?("cc")
+          cc = raw_updates["cc"].map do |l| 
+            deleted = l.start_with?("-") 
+            l = l[1..-1] if deleted
+            l = user_map[l]
+            l = "~~#{l}~~" if deleted
+            l
+          end
+
+          updates << "*Cc: #{cc.join(", ")}*"
+        end
+
+        if raw_updates.has_key?("labels")
+          labels = raw_updates["labels"].map do |l| 
+            deleted = l.start_with?("-") 
+            l = l[1..-1] if deleted
+            l = nice_label_name(l)
+            l = "~~#{l}~~" if deleted
+            l
+          end
+
+          updates << "*Labels: #{labels.join(", ")}*"
+        end
+
+        if raw_updates.has_key?("mergedInto")
+          updates << "*Merged into: ##{raw_updates["mergedInto"]}*"
+        end
+
+        if raw_updates.has_key?("blockedOn")
+          blocked_ons = raw_updates["blockedOn"].map do |raw_blocked_on|
+            name, id = raw_blocked_on.split(":", 2)
+            if name == project.import_source
+              "##{id}"
+            else
+              "#{project.namespace.path}/#{name}##{id}"
+            end
+          end
+          updates << "*Blocked on: #{blocked_ons.join(", ")}*"
+        end
+
+        if raw_updates.has_key?("blocking")
+          blockings = raw_updates["blocking"].map do |raw_blocked_on|
+            name, id = raw_blocked_on.split(":", 2)
+            if name == project.import_source
+              "##{id}"
+            else
+              "#{project.namespace.path}/#{name}##{id}"
+            end
+          end
+          updates << "*Blocking: #{blockings.join(", ")}*"
+        end
+
+        updates
+      end
+
+      def format_attachments(issue_id, comment_id, raw_attachments)
+        return [] unless raw_attachments
+
+        raw_attachments.map do |attachment|
+          next if attachment["isDeleted"]
+
+          filename = attachment["fileName"]
+          link = "https://storage.googleapis.com/google-code-attachments/#{@repo.name}/issue-#{issue_id}/comment-#{comment_id}/#{filename}"
+          
+          text = "[#{filename}](#{link})"
+          text = "!#{text}" if filename =~ /\.(png|jpg|jpeg|gif|bmp|tiff)\z/
+          text
+        end.compact
+      end
+
+      def format_issue_comment_body(id, author, date, content, updates, attachments)
+        body = []
+        body << "*Comment #{id} by #{author} on #{date}*"
+        body << "---"
+
+        if content.blank?
+          content = "*(No comment has been entered for this change)*"
+        end
+        body << content
+
+        if updates.any?
+          body << "---"
+          body += updates
+        end
+
+        if attachments.any?
+          body << "---"
+          body += attachments
+        end
+
+        body.join("\n\n")
+      end
+
+      def format_issue_body(author, date, content, attachments)
+        body = []
+        body << "*By #{author} on #{date}*"
+        body << "---"
+
+        if content.blank?
+          content = "*(No description has been entered for this issue)*"
+        end
+        body << content
+
+        if attachments.any?
+          body << "---"
+          body += attachments
+        end
+
+        body.join("\n\n")
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/google_code_import/project_creator.rb b/lib/gitlab/google_code_import/project_creator.rb
new file mode 100644
index 00000000000..7ac4387d79d
--- /dev/null
+++ b/lib/gitlab/google_code_import/project_creator.rb
@@ -0,0 +1,46 @@
+module Gitlab
+  module GoogleCodeImport
+    class ProjectCreator
+      attr_reader :repo, :namespace, :current_user, :user_map
+
+      def initialize(repo, namespace, current_user, user_map = nil)
+        @repo = repo
+        @namespace = namespace
+        @current_user = current_user
+        @user_map = user_map
+      end
+
+      def execute
+        import_data = {
+          "repo"      => repo.raw_data,
+          "user_map"  => user_map
+        }
+
+        @project = Project.new(
+          name: repo.name,
+          path: repo.name,
+          description: repo.summary,
+          namespace: namespace,
+          creator: current_user,
+          visibility_level: Gitlab::VisibilityLevel::PUBLIC,
+          import_type: "google_code",
+          import_source: repo.name,
+          import_url: repo.import_url,
+          import_data: import_data
+        )
+
+        if @project.save!
+          @project.reload
+
+          if @project.import_failed?
+            @project.import_retry
+          else
+            @project.import_start
+          end
+        end
+
+        @project
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/google_code_import/repository.rb b/lib/gitlab/google_code_import/repository.rb
new file mode 100644
index 00000000000..ad33fc2cad2
--- /dev/null
+++ b/lib/gitlab/google_code_import/repository.rb
@@ -0,0 +1,43 @@
+module Gitlab
+  module GoogleCodeImport
+    class Repository
+      attr_accessor :raw_data
+
+      def initialize(raw_data)
+        @raw_data = raw_data
+      end
+
+      def valid?
+        raw_data.is_a?(Hash) && raw_data["kind"] == "projecthosting#project"
+      end
+
+      def id
+        raw_data["externalId"]
+      end
+
+      def name
+        raw_data["name"]
+      end
+
+      def summary
+        raw_data["summary"]
+      end
+
+      def description
+        raw_data["description"]
+      end
+
+      def git?
+        raw_data["versionControlSystem"] == "git"
+      end
+
+      def import_url
+        raw_data["repositoryUrls"].first
+      end
+
+      def issues
+        raw_data["issues"] && raw_data["issues"]["items"]
+      end
+    end
+  end
+end