Merge branch 'github-importer' into 'master'

Does not halt the GitHub import process when an error occurs

## What are the relevant issue numbers?

Fixes #20385 

https://gitlab.com/gitlab-org/gitlab-ce/issues/20149

## Does this MR meet the acceptance criteria?

- [x] [CHANGELOG](https://gitlab.com/gitlab-org/gitlab-ce/blob/master/CHANGELOG) entry added
- Tests
  - [x] All builds are passing
- [x] Conform by the [style guides](https://gitlab.com/gitlab-org/gitlab-ce/blob/master/CONTRIBUTING.md#style-guides)
- [x] Branch has no merge conflicts with `master` (if you do - rebase it please)
- [x] [Squashed related commits together](https://git-scm.com/book/en/Git-Tools-Rewriting-History#Squashing-Commits)

See merge request !5763

2 files changed, 58 insertions, 33 deletions

diff --git a/lib/gitlab/github_import/importer.rb b/lib/gitlab/github_import/importer.rb
index 9ddc8905bd6..02ffb43d89b 100644
--- a/lib/gitlab/github_import/importer.rb
+++ b/lib/gitlab/github_import/importer.rb
@@ -3,12 +3,13 @@ module Gitlab
     class Importer
       include Gitlab::ShellAdapter
 
-      attr_reader :client, :project, :repo, :repo_url
+      attr_reader :client, :errors, :project, :repo, :repo_url
 
       def initialize(project)
         @project  = project
         @repo     = project.import_source
         @repo_url = project.import_url
+        @errors   = []
 
         if credentials
           @client = Client.new(credentials[:user])
@@ -18,8 +19,14 @@ module Gitlab
       end
 
       def execute
-        import_labels && import_milestones && import_issues &&
-          import_pull_requests && import_wiki
+        import_labels
+        import_milestones
+        import_issues
+        import_pull_requests
+        import_wiki
+        handle_errors
+
+        true
       end
 
       private
@@ -28,22 +35,37 @@ module Gitlab
         @credentials ||= project.import_data.credentials if project.import_data
       end
 
+      def handle_errors
+        return unless errors.any?
+
+        project.update_column(:import_error, {
+          message: 'The remote data could not be fully imported.',
+          errors: errors
+        }.to_json)
+      end
+
       def import_labels
         labels = client.labels(repo, per_page: 100)
-        labels.each { |raw| LabelFormatter.new(project, raw).create! }
 
-        true
-      rescue ActiveRecord::RecordInvalid => e
-        raise Projects::ImportService::Error, e.message
+        labels.each do |raw|
+          begin
+            LabelFormatter.new(project, raw).create!
+          rescue => e
+            errors << { type: :label, url: Gitlab::UrlSanitizer.sanitize(raw.url), errors: e.message }
+          end
+        end
       end
 
       def import_milestones
         milestones = client.milestones(repo, state: :all, per_page: 100)
-        milestones.each { |raw| MilestoneFormatter.new(project, raw).create! }
 
-        true
-      rescue ActiveRecord::RecordInvalid => e
-        raise Projects::ImportService::Error, e.message
+        milestones.each do |raw|
+          begin
+            MilestoneFormatter.new(project, raw).create!
+          rescue => e
+            errors << { type: :milestone, url: Gitlab::UrlSanitizer.sanitize(raw.url), errors: e.message }
+          end
+        end
       end
 
       def import_issues
@@ -53,15 +75,15 @@ module Gitlab
           gh_issue = IssueFormatter.new(project, raw)
 
           if gh_issue.valid?
-            issue = gh_issue.create!
-            apply_labels(issue)
-            import_comments(issue) if gh_issue.has_comments?
+            begin
+              issue = gh_issue.create!
+              apply_labels(issue)
+              import_comments(issue) if gh_issue.has_comments?
+            rescue => e
+              errors << { type: :issue, url: Gitlab::UrlSanitizer.sanitize(raw.url), errors: e.message }
+            end
           end
         end
-
-        true
-      rescue ActiveRecord::RecordInvalid => e
-        raise Projects::ImportService::Error, e.message
       end
 
       def import_pull_requests
@@ -77,14 +99,12 @@ module Gitlab
             apply_labels(merge_request)
             import_comments(merge_request)
             import_comments_on_diff(merge_request)
-          rescue ActiveRecord::RecordInvalid => e
-            raise Projects::ImportService::Error, e.message
+          rescue => e
+            errors << { type: :pull_request, url: Gitlab::UrlSanitizer.sanitize(pull_request.url), errors: e.message }
           ensure
             clean_up_restored_branches(pull_request)
           end
         end
-
-        true
       end
 
       def restore_source_branch(pull_request)
@@ -98,7 +118,7 @@ module Gitlab
       def remove_branch(name)
         project.repository.delete_branch(name)
       rescue Rugged::ReferenceError
-        nil
+        errors << { type: :remove_branch, name: name }
       end
 
       def clean_up_restored_branches(pull_request)
@@ -112,9 +132,10 @@ module Gitlab
         issue = client.issue(repo, issuable.iid)
 
         if issue.labels.count > 0
-          label_ids = issue.labels.map do |raw|
-            Label.find_by(LabelFormatter.new(project, raw).attributes).try(:id)
-          end
+          label_ids = issue.labels
+            .map { |raw| LabelFormatter.new(project, raw).attributes }
+            .map { |attrs| Label.find_by(attrs).try(:id) }
+            .compact
 
           issuable.update_attribute(:label_ids, label_ids)
         end
@@ -132,8 +153,12 @@ module Gitlab
 
       def create_comments(issuable, comments)
         comments.each do |raw|
-          comment = CommentFormatter.new(project, raw)
-          issuable.notes.create!(comment.attributes)
+          begin
+            comment = CommentFormatter.new(project, raw)
+            issuable.notes.create!(comment.attributes)
+          rescue => e
+            errors << { type: :comment, url: Gitlab::UrlSanitizer.sanitize(raw.url), errors: e.message }
+          end
         end
       end
 
@@ -143,16 +168,12 @@ module Gitlab
           gitlab_shell.import_repository(project.repository_storage_path, wiki.path_with_namespace, wiki.import_url)
           project.update_attribute(:wiki_enabled, true)
         end
-
-        true
       rescue Gitlab::Shell::Error => e
         # GitHub error message when the wiki repo has not been created,
         # this means that repo has wiki enabled, but have no pages. So,
         # we can skip the import.
         if e.message !~ /repository not exported/
-          raise Projects::ImportService::Error, e.message
-        else
-          true
+          errors << { type: :wiki, errors: e.message }
         end
       end
     end
diff --git a/lib/gitlab/github_import/pull_request_formatter.rb b/lib/gitlab/github_import/pull_request_formatter.rb
index b84538a090a..04aa3664f64 100644
--- a/lib/gitlab/github_import/pull_request_formatter.rb
+++ b/lib/gitlab/github_import/pull_request_formatter.rb
@@ -56,6 +56,10 @@ module Gitlab
         end
       end
 
+      def url
+        raw_data.url
+      end
+
       private
 
       def assigned?