Merge remote-tracking branch 'origin/master' into add-sentry-js-again-with-vue

34 files changed, 1034 insertions, 158 deletions

diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb
index ddff3c8c1e8..86bf567fe69 100644
--- a/lib/api/helpers.rb
+++ b/lib/api/helpers.rb
@@ -102,7 +102,7 @@ module API
     end
 
     def authenticate!
-      unauthorized! unless current_user && can?(current_user, :access_api)
+      unauthorized! unless current_user && can?(initial_current_user, :access_api)
     end
 
     def authenticate_non_get!
diff --git a/lib/api/internal.rb b/lib/api/internal.rb
index 5b48ee8665f..ebed26dd178 100644
--- a/lib/api/internal.rb
+++ b/lib/api/internal.rb
@@ -140,7 +140,7 @@ module API
         begin
           Gitlab::GitalyClient::Notifications.new(project.repository).post_receive
         rescue GRPC::Unavailable => e
-          render_api_error(e, 500)
+          render_api_error!(e, 500)
         end
       end
     end
diff --git a/lib/api/issues.rb b/lib/api/issues.rb
index 244725bb292..522f0f3be92 100644
--- a/lib/api/issues.rb
+++ b/lib/api/issues.rb
@@ -219,6 +219,21 @@ module API
         authorize!(:destroy_issue, issue)
         issue.destroy
       end
+
+      desc 'List merge requests closing issue'  do
+        success Entities::MergeRequestBasic
+      end
+      params do
+        requires :issue_iid, type: Integer, desc: 'The internal ID of a project issue'
+      end
+      get ':id/issues/:issue_iid/closed_by' do
+        issue = find_project_issue(params[:issue_iid])
+
+        merge_request_ids = MergeRequestsClosingIssues.where(issue_id: issue).select(:merge_request_id)
+        merge_requests = MergeRequestsFinder.new(current_user, project_id: user_project.id).execute.where(id: merge_request_ids)
+
+        present paginate(merge_requests), with: Entities::MergeRequestBasic, current_user: current_user, project: user_project
+      end
     end
   end
 end
diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb
index c7dc2ea336f..e5793fbc5cb 100644
--- a/lib/api/merge_requests.rb
+++ b/lib/api/merge_requests.rb
@@ -33,6 +33,17 @@ module API
           end
         end
 
+        def find_merge_requests(args = {})
+          args = params.merge(args)
+
+          args[:milestone_title] = args.delete(:milestone)
+          args[:label_name] = args.delete(:labels)
+
+          merge_requests = MergeRequestsFinder.new(current_user, args).execute.inc_notes_with_associations
+
+          merge_requests.reorder(args[:order_by] => args[:sort])
+        end
+
         params :optional_params_ce do
           optional :description, type: String, desc: 'The description of the merge request'
           optional :assignee_id, type: Integer, desc: 'The ID of a user to assign the merge request'
@@ -57,23 +68,15 @@ module API
         optional :sort, type: String, values: %w[asc desc], default: 'desc',
                         desc: 'Return merge requests sorted in `asc` or `desc` order.'
         optional :iids, type: Array[Integer], desc: 'The IID array of merge requests'
+        optional :milestone, type: String, desc: 'Return merge requests for a specific milestone'
+        optional :labels, type: String, desc: 'Comma-separated list of label names'
         use :pagination
       end
       get ":id/merge_requests" do
         authorize! :read_merge_request, user_project
 
-        merge_requests = user_project.merge_requests.inc_notes_with_associations
-        merge_requests = filter_by_iid(merge_requests, params[:iids]) if params[:iids].present?
-
-        merge_requests =
-          case params[:state]
-          when 'opened' then merge_requests.opened
-          when 'closed' then merge_requests.closed
-          when 'merged' then merge_requests.merged
-          else merge_requests
-          end
+        merge_requests = find_merge_requests(project_id: user_project.id)
 
-        merge_requests = merge_requests.reorder(params[:order_by] => params[:sort])
         present paginate(merge_requests), with: Entities::MergeRequestBasic, current_user: current_user, project: user_project
       end
 
diff --git a/lib/backup/manager.rb b/lib/backup/manager.rb
index 8d64c82272a..330cd963626 100644
--- a/lib/backup/manager.rb
+++ b/lib/backup/manager.rb
@@ -15,7 +15,7 @@ module Backup
       s[:gitlab_version]     = Gitlab::VERSION
       s[:tar_version]        = tar_version
       s[:skipped]            = ENV["SKIP"]
-      tar_file = "#{s[:backup_created_at].strftime('%s_%Y_%m_%d')}#{FILE_NAME_SUFFIX}"
+      tar_file = "#{s[:backup_created_at].strftime('%s_%Y_%m_%d_')}#{s[:gitlab_version]}#{FILE_NAME_SUFFIX}"
 
       Dir.chdir(backup_path) do
         File.open("#{backup_path}/backup_information.yml", "w+") do |file|
diff --git a/lib/banzai/filter/emoji_filter.rb b/lib/banzai/filter/emoji_filter.rb
index d6138816e70..6255a611dbe 100644
--- a/lib/banzai/filter/emoji_filter.rb
+++ b/lib/banzai/filter/emoji_filter.rb
@@ -53,7 +53,10 @@ module Banzai
 
       # Build a regexp that matches all valid :emoji: names.
       def self.emoji_pattern
-        @emoji_pattern ||= /:(#{Gitlab::Emoji.emojis_names.map { |name| Regexp.escape(name) }.join('|')}):/
+        @emoji_pattern ||=
+          /(?<=[^[:alnum:]:]|\n|^)
+          :(#{Gitlab::Emoji.emojis_names.map { |name| Regexp.escape(name) }.join('|')}):
+          (?=[^[:alnum:]:]|$)/x
       end
 
       # Build a regexp that matches all valid unicode emojis names.
diff --git a/lib/github/client.rb b/lib/github/client.rb
new file mode 100644
index 00000000000..e65d908d232
--- /dev/null
+++ b/lib/github/client.rb
@@ -0,0 +1,23 @@
+module Github
+  class Client
+    attr_reader :connection, :rate_limit
+
+    def initialize(options)
+      @connection = Faraday.new(url: options.fetch(:url)) do |faraday|
+        faraday.options.open_timeout = options.fetch(:timeout, 60)
+        faraday.options.timeout = options.fetch(:timeout, 60)
+        faraday.authorization 'token', options.fetch(:token)
+        faraday.adapter :net_http
+      end
+
+      @rate_limit = RateLimit.new(connection)
+    end
+
+    def get(url, query = {})
+      exceed, reset_in = rate_limit.get
+      sleep reset_in if exceed
+
+      Github::Response.new(connection.get(url, query))
+    end
+  end
+end
diff --git a/lib/github/collection.rb b/lib/github/collection.rb
new file mode 100644
index 00000000000..014b2038c4b
--- /dev/null
+++ b/lib/github/collection.rb
@@ -0,0 +1,29 @@
+module Github
+  class Collection
+    attr_reader :options
+
+    def initialize(options)
+      @options = options
+    end
+
+    def fetch(url, query = {})
+      return [] if url.blank?
+
+      Enumerator.new do |yielder|
+        loop do
+          response = client.get(url, query)
+          response.body.each { |item| yielder << item }
+
+          raise StopIteration unless response.rels.key?(:next)
+          url = response.rels[:next]
+        end
+      end.lazy
+    end
+
+    private
+
+    def client
+      @client ||= Github::Client.new(options)
+    end
+  end
+end
diff --git a/lib/github/error.rb b/lib/github/error.rb
new file mode 100644
index 00000000000..66d7afaa787
--- /dev/null
+++ b/lib/github/error.rb
@@ -0,0 +1,3 @@
+module Github
+  RepositoryFetchError = Class.new(StandardError)
+end
diff --git a/lib/github/import.rb b/lib/github/import.rb
new file mode 100644
index 00000000000..d49761fd6c6
--- /dev/null
+++ b/lib/github/import.rb
@@ -0,0 +1,409 @@
+require_relative 'error'
+module Github
+  class Import
+    include Gitlab::ShellAdapter
+
+    class MergeRequest < ::MergeRequest
+      self.table_name = 'merge_requests'
+
+      self.reset_callbacks :save
+      self.reset_callbacks :commit
+      self.reset_callbacks :update
+      self.reset_callbacks :validate
+    end
+
+    class Issue < ::Issue
+      self.table_name = 'issues'
+
+      self.reset_callbacks :save
+      self.reset_callbacks :commit
+      self.reset_callbacks :update
+      self.reset_callbacks :validate
+    end
+
+    class Note < ::Note
+      self.table_name = 'notes'
+
+      self.reset_callbacks :save
+      self.reset_callbacks :commit
+      self.reset_callbacks :update
+      self.reset_callbacks :validate
+    end
+
+    class LegacyDiffNote < ::LegacyDiffNote
+      self.table_name = 'notes'
+
+      self.reset_callbacks :commit
+      self.reset_callbacks :update
+      self.reset_callbacks :validate
+    end
+
+    attr_reader :project, :repository, :repo, :options, :errors, :cached, :verbose
+
+    def initialize(project, options)
+      @project = project
+      @repository = project.repository
+      @repo = project.import_source
+      @options = options
+      @verbose = options.fetch(:verbose, false)
+      @cached  = Hash.new { |hash, key| hash[key] = Hash.new }
+      @errors  = []
+    end
+
+    # rubocop: disable Rails/Output
+    def execute
+      puts 'Fetching repository...'.color(:aqua) if verbose
+      fetch_repository
+      puts 'Fetching labels...'.color(:aqua) if verbose
+      fetch_labels
+      puts 'Fetching milestones...'.color(:aqua) if verbose
+      fetch_milestones
+      puts 'Fetching pull requests...'.color(:aqua) if verbose
+      fetch_pull_requests
+      puts 'Fetching issues...'.color(:aqua) if verbose
+      fetch_issues
+      puts 'Cloning wiki repository...'.color(:aqua) if verbose
+      fetch_wiki_repository
+      puts 'Expiring repository cache...'.color(:aqua) if verbose
+      expire_repository_cache
+
+      true
+    rescue Github::RepositoryFetchError
+      false
+    ensure
+      keep_track_of_errors
+    end
+
+    private
+
+    def fetch_repository
+      begin
+        project.create_repository unless project.repository.exists?
+        project.repository.add_remote('github', "https://{options.fetch(:token)}@github.com/#{repo}.git")
+        project.repository.set_remote_as_mirror('github')
+        project.repository.fetch_remote('github', forced: true)
+      rescue Gitlab::Shell::Error => e
+        error(:project, "https://github.com/#{repo}.git", e.message)
+        raise Github::RepositoryFetchError
+      end
+    end
+
+    def fetch_wiki_repository
+      wiki_url  = "https://{options.fetch(:token)}@github.com/#{repo}.wiki.git"
+      wiki_path = "#{project.path_with_namespace}.wiki"
+
+      unless project.wiki.repository_exists?
+        gitlab_shell.import_repository(project.repository_storage_path, wiki_path, wiki_url)
+      end
+    rescue Gitlab::Shell::Error => e
+      # GitHub error message when the wiki repo has not been created,
+      # this means that repo has wiki enabled, but have no pages. So,
+      # we can skip the import.
+      if e.message !~ /repository not exported/
+        errors(:wiki, wiki_url, e.message)
+      end
+    end
+
+    def fetch_labels
+      url = "/repos/#{repo}/labels"
+
+      while url
+        response = Github::Client.new(options).get(url)
+
+        response.body.each do |raw|
+          begin
+            representation = Github::Representation::Label.new(raw)
+
+            label = project.labels.find_or_create_by!(title: representation.title) do |label|
+              label.color = representation.color
+            end
+
+            cached[:label_ids][label.title] = label.id
+          rescue => e
+            error(:label, representation.url, e.message)
+          end
+        end
+
+        url = response.rels[:next]
+      end
+    end
+
+    def fetch_milestones
+      url = "/repos/#{repo}/milestones"
+
+      while url
+        response = Github::Client.new(options).get(url, state: :all)
+
+        response.body.each do |raw|
+          begin
+            milestone = Github::Representation::Milestone.new(raw)
+            next if project.milestones.where(iid: milestone.iid).exists?
+
+            project.milestones.create!(
+              iid: milestone.iid,
+              title: milestone.title,
+              description: milestone.description,
+              due_date: milestone.due_date,
+              state: milestone.state,
+              created_at: milestone.created_at,
+              updated_at: milestone.updated_at
+            )
+          rescue => e
+            error(:milestone, milestone.url, e.message)
+          end
+        end
+
+        url = response.rels[:next]
+      end
+    end
+
+    def fetch_pull_requests
+      url = "/repos/#{repo}/pulls"
+
+      while url
+        response = Github::Client.new(options).get(url, state: :all, sort: :created, direction: :asc)
+
+        response.body.each do |raw|
+          pull_request  = Github::Representation::PullRequest.new(raw, options.merge(project: project))
+          merge_request = MergeRequest.find_or_initialize_by(iid: pull_request.iid, source_project_id: project.id)
+          next unless merge_request.new_record? && pull_request.valid?
+
+          begin
+            restore_branches(pull_request)
+
+            author_id   = user_id(pull_request.author, project.creator_id)
+            description = format_description(pull_request.description, pull_request.author)
+
+            merge_request.attributes = {
+              iid: pull_request.iid,
+              title: pull_request.title,
+              description: description,
+              source_project: pull_request.source_project,
+              source_branch: pull_request.source_branch_name,
+              source_branch_sha: pull_request.source_branch_sha,
+              target_project: pull_request.target_project,
+              target_branch: pull_request.target_branch_name,
+              target_branch_sha: pull_request.target_branch_sha,
+              state: pull_request.state,
+              milestone_id: milestone_id(pull_request.milestone),
+              author_id: author_id,
+              assignee_id: user_id(pull_request.assignee),
+              created_at: pull_request.created_at,
+              updated_at: pull_request.updated_at
+            }
+
+            merge_request.save!(validate: false)
+            merge_request.merge_request_diffs.create
+
+            # Fetch review comments
+            review_comments_url = "/repos/#{repo}/pulls/#{pull_request.iid}/comments"
+            fetch_comments(merge_request, :review_comment, review_comments_url, LegacyDiffNote)
+
+            # Fetch comments
+            comments_url = "/repos/#{repo}/issues/#{pull_request.iid}/comments"
+            fetch_comments(merge_request, :comment, comments_url)
+          rescue => e
+            error(:pull_request, pull_request.url, e.message)
+          ensure
+            clean_up_restored_branches(pull_request)
+          end
+        end
+
+        url = response.rels[:next]
+      end
+    end
+
+    def fetch_issues
+      url = "/repos/#{repo}/issues"
+
+      while url
+        response = Github::Client.new(options).get(url, state: :all, sort: :created, direction: :asc)
+
+        response.body.each do |raw|
+          representation = Github::Representation::Issue.new(raw, options)
+
+          begin
+            # Every pull request is an issue, but not every issue
+            # is a pull request. For this reason, "shared" actions
+            # for both features, like manipulating assignees, labels
+            # and milestones, are provided within the Issues API.
+            if representation.pull_request?
+              next unless representation.has_labels?
+
+              merge_request = MergeRequest.find_by!(target_project_id: project.id, iid: representation.iid)
+              merge_request.update_attribute(:label_ids, label_ids(representation.labels))
+            else
+              next if Issue.where(iid: representation.iid, project_id: project.id).exists?
+
+              author_id          = user_id(representation.author, project.creator_id)
+              issue              = Issue.new
+              issue.iid          = representation.iid
+              issue.project_id   = project.id
+              issue.title        = representation.title
+              issue.description  = format_description(representation.description, representation.author)
+              issue.state        = representation.state
+              issue.label_ids    = label_ids(representation.labels)
+              issue.milestone_id = milestone_id(representation.milestone)
+              issue.author_id    = author_id
+              issue.assignee_id  = user_id(representation.assignee)
+              issue.created_at   = representation.created_at
+              issue.updated_at   = representation.updated_at
+              issue.save!(validate: false)
+
+              # Fetch comments
+              if representation.has_comments?
+                comments_url = "/repos/#{repo}/issues/#{issue.iid}/comments"
+                fetch_comments(issue, :comment, comments_url)
+              end
+            end
+          rescue => e
+            error(:issue, representation.url, e.message)
+          end
+        end
+
+        url = response.rels[:next]
+      end
+    end
+
+    def fetch_comments(noteable, type, url, klass = Note)
+      while url
+        comments = Github::Client.new(options).get(url)
+
+        ActiveRecord::Base.no_touching do
+          comments.body.each do |raw|
+            begin
+              representation  = Github::Representation::Comment.new(raw, options)
+              author_id       = user_id(representation.author, project.creator_id)
+
+              note            = klass.new
+              note.project_id = project.id
+              note.noteable   = noteable
+              note.note       = format_description(representation.note, representation.author)
+              note.commit_id  = representation.commit_id
+              note.line_code  = representation.line_code
+              note.author_id  = author_id
+              note.created_at = representation.created_at
+              note.updated_at = representation.updated_at
+              note.save!(validate: false)
+            rescue => e
+              error(type, representation.url, e.message)
+            end
+          end
+        end
+
+        url = comments.rels[:next]
+      end
+    end
+
+    def fetch_releases
+      url = "/repos/#{repo}/releases"
+
+      while url
+        response = Github::Client.new(options).get(url)
+
+        response.body.each do |raw|
+          representation = Github::Representation::Release.new(raw)
+          next unless representation.valid?
+
+          release = ::Release.find_or_initialize_by(project_id: project.id, tag: representation.tag)
+          next unless relese.new_record?
+
+          begin
+            release.description = representation.description
+            release.created_at  = representation.created_at
+            release.updated_at  = representation.updated_at
+            release.save!(validate: false)
+          rescue => e
+            error(:release, representation.url, e.message)
+          end
+        end
+
+        url = response.rels[:next]
+      end
+    end
+
+    def restore_branches(pull_request)
+      restore_source_branch(pull_request) unless pull_request.source_branch_exists?
+      restore_target_branch(pull_request) unless pull_request.target_branch_exists?
+    end
+
+    def restore_source_branch(pull_request)
+      repository.create_branch(pull_request.source_branch_name, pull_request.source_branch_sha)
+    end
+
+    def restore_target_branch(pull_request)
+      repository.create_branch(pull_request.target_branch_name, pull_request.target_branch_sha)
+    end
+
+    def remove_branch(name)
+      repository.delete_branch(name)
+    rescue Rugged::ReferenceError
+      errors << { type: :branch, url: nil, error: "Could not clean up restored branch: #{name}" }
+    end
+
+    def clean_up_restored_branches(pull_request)
+      return if pull_request.opened?
+
+      remove_branch(pull_request.source_branch_name) unless pull_request.source_branch_exists?
+      remove_branch(pull_request.target_branch_name) unless pull_request.target_branch_exists?
+    end
+
+    def label_ids(labels)
+      labels.map { |attrs| cached[:label_ids][attrs.fetch('name')] }.compact
+    end
+
+    def milestone_id(milestone)
+      return unless milestone.present?
+
+      project.milestones.select(:id).find_by(iid: milestone.iid)&.id
+    end
+
+    def user_id(user, fallback_id = nil)
+      return unless user.present?
+      return cached[:user_ids][user.id] if cached[:user_ids].key?(user.id)
+
+      gitlab_user_id = user_id_by_external_uid(user.id) || user_id_by_email(user.email)
+
+      cached[:gitlab_user_ids][user.id] = gitlab_user_id.present?
+      cached[:user_ids][user.id] = gitlab_user_id || fallback_id
+    end
+
+    def user_id_by_email(email)
+      return nil unless email
+
+      ::User.find_by_any_email(email)&.id
+    end
+
+    def user_id_by_external_uid(id)
+      return nil unless id
+
+      ::User.select(:id)
+            .joins(:identities)
+            .merge(::Identity.where(provider: :github, extern_uid: id))
+            .first&.id
+    end
+
+    def format_description(body, author)
+      return body if cached[:gitlab_user_ids][author.id]
+
+      "*Created by: #{author.username}*\n\n#{body}"
+    end
+
+    def expire_repository_cache
+      repository.expire_content_cache
+    end
+
+    def keep_track_of_errors
+      return unless errors.any?
+
+      project.update_column(:import_error, {
+        message: 'The remote data could not be fully imported.',
+        errors: errors
+      }.to_json)
+    end
+
+    def error(type, url, message)
+      errors << { type: type, url: Gitlab::UrlSanitizer.sanitize(url), error: message }
+    end
+  end
+end
diff --git a/lib/github/rate_limit.rb b/lib/github/rate_limit.rb
new file mode 100644
index 00000000000..884693d093c
--- /dev/null
+++ b/lib/github/rate_limit.rb
@@ -0,0 +1,27 @@
+module Github
+  class RateLimit
+    SAFE_REMAINING_REQUESTS = 100
+    SAFE_RESET_TIME         = 500
+    RATE_LIMIT_URL          = '/rate_limit'.freeze
+
+    attr_reader :connection
+
+    def initialize(connection)
+      @connection = connection
+    end
+
+    def get
+      response = connection.get(RATE_LIMIT_URL)
+
+      # GitHub Rate Limit API returns 404 when the rate limit is disabled
+      return false unless response.status != 404
+
+      body      = Oj.load(response.body, class_cache: false, mode: :compat)
+      remaining = body.dig('rate', 'remaining').to_i
+      reset_in  = body.dig('rate', 'reset').to_i
+      exceed    = remaining <= SAFE_REMAINING_REQUESTS
+
+      [exceed, reset_in]
+    end
+  end
+end
diff --git a/lib/github/repositories.rb b/lib/github/repositories.rb
new file mode 100644
index 00000000000..c1c9448f305
--- /dev/null
+++ b/lib/github/repositories.rb
@@ -0,0 +1,19 @@
+module Github
+  class Repositories
+    attr_reader :options
+
+    def initialize(options)
+      @options = options
+    end
+
+    def fetch
+      Collection.new(options).fetch(repos_url)
+    end
+
+    private
+
+    def repos_url
+      '/user/repos'
+    end
+  end
+end
diff --git a/lib/github/representation/base.rb b/lib/github/representation/base.rb
new file mode 100644
index 00000000000..f26bdbdd546
--- /dev/null
+++ b/lib/github/representation/base.rb
@@ -0,0 +1,30 @@
+module Github
+  module Representation
+    class Base
+      def initialize(raw, options = {})
+        @raw     = raw
+        @options = options
+      end
+
+      def id
+        raw['id']
+      end
+
+      def url
+        raw['url']
+      end
+
+      def created_at
+        raw['created_at']
+      end
+
+      def updated_at
+        raw['updated_at']
+      end
+
+      private
+
+      attr_reader :raw, :options
+    end
+  end
+end
diff --git a/lib/github/representation/branch.rb b/lib/github/representation/branch.rb
new file mode 100644
index 00000000000..d1dac6944f0
--- /dev/null
+++ b/lib/github/representation/branch.rb
@@ -0,0 +1,51 @@
+module Github
+  module Representation
+    class Branch < Representation::Base
+      attr_reader :repository
+
+      def user
+        raw.dig('user', 'login') || 'unknown'
+      end
+
+      def repo
+        return @repo if defined?(@repo)
+
+        @repo = Github::Representation::Repo.new(raw['repo']) if raw['repo'].present?
+      end
+
+      def ref
+        raw['ref']
+      end
+
+      def sha
+        raw['sha']
+      end
+
+      def short_sha
+        Commit.truncate_sha(sha)
+      end
+
+      def exists?
+        branch_exists? && commit_exists?
+      end
+
+      def valid?
+        sha.present? && ref.present?
+      end
+
+      private
+
+      def branch_exists?
+        repository.branch_exists?(ref)
+      end
+
+      def commit_exists?
+        repository.branch_names_contains(sha).include?(ref)
+      end
+
+      def repository
+        @repository ||= options.fetch(:repository)
+      end
+    end
+  end
+end
diff --git a/lib/github/representation/comment.rb b/lib/github/representation/comment.rb
new file mode 100644
index 00000000000..1b5be91461b
--- /dev/null
+++ b/lib/github/representation/comment.rb
@@ -0,0 +1,42 @@
+module Github
+  module Representation
+    class Comment < Representation::Base
+      def note
+        raw['body'] || ''
+      end
+
+      def author
+        @author ||= Github::Representation::User.new(raw['user'], options)
+      end
+
+      def commit_id
+        raw['commit_id']
+      end
+
+      def line_code
+        return unless on_diff?
+
+        parsed_lines = Gitlab::Diff::Parser.new.parse(diff_hunk.lines)
+        generate_line_code(parsed_lines.to_a.last)
+      end
+
+      private
+
+      def generate_line_code(line)
+        Gitlab::Diff::LineCode.generate(file_path, line.new_pos, line.old_pos)
+      end
+
+      def on_diff?
+        diff_hunk.present?
+      end
+
+      def diff_hunk
+        raw['diff_hunk']
+      end
+
+      def file_path
+        raw['path']
+      end
+    end
+  end
+end
diff --git a/lib/github/representation/issuable.rb b/lib/github/representation/issuable.rb
new file mode 100644
index 00000000000..9713b82615d
--- /dev/null
+++ b/lib/github/representation/issuable.rb
@@ -0,0 +1,37 @@
+module Github
+  module Representation
+    class Issuable < Representation::Base
+      def iid
+        raw['number']
+      end
+
+      def title
+        raw['title']
+      end
+
+      def description
+        raw['body'] || ''
+      end
+
+      def milestone
+        return unless raw['milestone'].present?
+
+        @milestone ||= Github::Representation::Milestone.new(raw['milestone'])
+      end
+
+      def author
+        @author ||= Github::Representation::User.new(raw['user'], options)
+      end
+
+      def assignee
+        return unless assigned?
+
+        @assignee ||= Github::Representation::User.new(raw['assignee'], options)
+      end
+
+      def assigned?
+        raw['assignee'].present?
+      end
+    end
+  end
+end
diff --git a/lib/github/representation/issue.rb b/lib/github/representation/issue.rb
new file mode 100644
index 00000000000..df3540a6e6c
--- /dev/null
+++ b/lib/github/representation/issue.rb
@@ -0,0 +1,25 @@
+module Github
+  module Representation
+    class Issue < Representation::Issuable
+      def labels
+        raw['labels']
+      end
+
+      def state
+        raw['state'] == 'closed' ? 'closed' : 'opened'
+      end
+
+      def has_comments?
+        raw['comments'] > 0
+      end
+
+      def has_labels?
+        labels.count > 0
+      end
+
+      def pull_request?
+        raw['pull_request'].present?
+      end
+    end
+  end
+end
diff --git a/lib/github/representation/label.rb b/lib/github/representation/label.rb
new file mode 100644
index 00000000000..60aa51f9569
--- /dev/null
+++ b/lib/github/representation/label.rb
@@ -0,0 +1,13 @@
+module Github
+  module Representation
+    class Label < Representation::Base
+      def color
+        "##{raw['color']}"
+      end
+
+      def title
+        raw['name']
+      end
+    end
+  end
+end
diff --git a/lib/github/representation/milestone.rb b/lib/github/representation/milestone.rb
new file mode 100644
index 00000000000..917e6394ad4
--- /dev/null
+++ b/lib/github/representation/milestone.rb
@@ -0,0 +1,25 @@
+module Github
+  module Representation
+    class Milestone < Representation::Base
+      def iid
+        raw['number']
+      end
+
+      def title
+        raw['title']
+      end
+
+      def description
+        raw['description']
+      end
+
+      def due_date
+        raw['due_on']
+      end
+
+      def state
+        raw['state'] == 'closed' ? 'closed' : 'active'
+      end
+    end
+  end
+end
diff --git a/lib/github/representation/pull_request.rb b/lib/github/representation/pull_request.rb
new file mode 100644
index 00000000000..ac9c8283b4b
--- /dev/null
+++ b/lib/github/representation/pull_request.rb
@@ -0,0 +1,78 @@
+module Github
+  module Representation
+    class PullRequest < Representation::Issuable
+      attr_reader :project
+
+      delegate :user, :repo, :ref, :sha, to: :source_branch, prefix: true
+      delegate :user, :exists?, :repo, :ref, :sha, :short_sha, to: :target_branch, prefix: true
+
+      def source_project
+        project
+      end
+
+      def source_branch_exists?
+        !cross_project? && source_branch.exists?
+      end
+
+      def source_branch_name
+        @source_branch_name ||=
+          if cross_project? || !source_branch_exists?
+            source_branch_name_prefixed
+          else
+            source_branch_ref
+          end
+      end
+
+      def target_project
+        project
+      end
+
+      def target_branch_name
+        @target_branch_name ||= target_branch_exists? ? target_branch_ref : target_branch_name_prefixed
+      end
+
+      def state
+        return 'merged' if raw['state'] == 'closed' && raw['merged_at'].present?
+        return 'closed' if raw['state'] == 'closed'
+
+        'opened'
+      end
+
+      def opened?
+        state == 'opened'
+      end
+
+      def valid?
+        source_branch.valid? && target_branch.valid?
+      end
+
+      private
+
+      def project
+        @project ||= options.fetch(:project)
+      end
+
+      def source_branch
+        @source_branch ||= Representation::Branch.new(raw['head'], repository: project.repository)
+      end
+
+      def source_branch_name_prefixed
+        "gh-#{target_branch_short_sha}/#{iid}/#{source_branch_user}/#{source_branch_ref}"
+      end
+
+      def target_branch
+        @target_branch ||= Representation::Branch.new(raw['base'], repository: project.repository)
+      end
+
+      def target_branch_name_prefixed
+        "gl-#{target_branch_short_sha}/#{iid}/#{target_branch_user}/#{target_branch_ref}"
+      end
+
+      def cross_project?
+        return true if source_branch_repo.nil?
+
+        source_branch_repo.id != target_branch_repo.id
+      end
+    end
+  end
+end
diff --git a/lib/github/representation/release.rb b/lib/github/representation/release.rb
new file mode 100644
index 00000000000..e7e4b428c1a
--- /dev/null
+++ b/lib/github/representation/release.rb
@@ -0,0 +1,17 @@
+module Github
+  module Representation
+    class Release < Representation::Base
+      def description
+        raw['body']
+      end
+
+      def tag
+        raw['tag_name']
+      end
+
+      def valid?
+        !raw['draft']
+      end
+    end
+  end
+end
diff --git a/lib/github/representation/repo.rb b/lib/github/representation/repo.rb
new file mode 100644
index 00000000000..6938aa7db05
--- /dev/null
+++ b/lib/github/representation/repo.rb
@@ -0,0 +1,6 @@
+module Github
+  module Representation
+    class Repo < Representation::Base
+    end
+  end
+end
diff --git a/lib/github/representation/user.rb b/lib/github/representation/user.rb
new file mode 100644
index 00000000000..18591380e25
--- /dev/null
+++ b/lib/github/representation/user.rb
@@ -0,0 +1,15 @@
+module Github
+  module Representation
+    class User < Representation::Base
+      def email
+        return @email if defined?(@email)
+
+        @email = Github::User.new(username, options).get.fetch('email', nil)
+      end
+
+      def username
+        raw['login']
+      end
+    end
+  end
+end
diff --git a/lib/github/response.rb b/lib/github/response.rb
new file mode 100644
index 00000000000..761c524b553
--- /dev/null
+++ b/lib/github/response.rb
@@ -0,0 +1,25 @@
+module Github
+  class Response
+    attr_reader :raw, :headers, :status
+
+    def initialize(response)
+      @raw     = response
+      @headers = response.headers
+      @status  = response.status
+    end
+
+    def body
+      Oj.load(raw.body, class_cache: false, mode: :compat)
+    end
+
+    def rels
+      links = headers['Link'].to_s.split(', ').map do |link|
+        href, name = link.match(/<(.*?)>; rel="(\w+)"/).captures
+
+        [name.to_sym, href]
+      end
+
+      Hash[*links.flatten]
+    end
+  end
+end
diff --git a/lib/github/user.rb b/lib/github/user.rb
new file mode 100644
index 00000000000..f88a29e590b
--- /dev/null
+++ b/lib/github/user.rb
@@ -0,0 +1,24 @@
+module Github
+  class User
+    attr_reader :username, :options
+
+    def initialize(username, options)
+      @username = username
+      @options  = options
+    end
+
+    def get
+      client.get(user_url).body
+    end
+
+    private
+
+    def client
+      @client ||= Github::Client.new(options)
+    end
+
+    def user_url
+      "/users/#{username}"
+    end
+  end
+end
diff --git a/lib/gitlab/asciidoc.rb b/lib/gitlab/asciidoc.rb
index d575367d81a..fba80c7132e 100644
--- a/lib/gitlab/asciidoc.rb
+++ b/lib/gitlab/asciidoc.rb
@@ -14,28 +14,16 @@ module Gitlab
     # Public: Converts the provided Asciidoc markup into HTML.
     #
     # input         - the source text in Asciidoc format
-    # context       - a Hash with the template context:
-    #                 :commit
-    #                 :project
-    #                 :project_wiki
-    #                 :requested_path
-    #                 :ref
-    # asciidoc_opts - a Hash of options to pass to the Asciidoctor converter
     #
-    def self.render(input, context, asciidoc_opts = {})
-      asciidoc_opts.reverse_merge!(
-        safe: :secure,
-        backend: :gitlab_html5,
-        attributes: []
-      )
-      asciidoc_opts[:attributes].unshift(*DEFAULT_ADOC_ATTRS)
+    def self.render(input)
+      asciidoc_opts = { safe: :secure,
+                        backend: :gitlab_html5,
+                        attributes: DEFAULT_ADOC_ATTRS }
 
       plantuml_setup
 
       html = ::Asciidoctor.convert(input, asciidoc_opts)
 
-      html = Banzai.post_process(html, context)
-
       filter = Banzai::Filter::SanitizationFilter.new(html)
       html = filter.call.to_s
 
diff --git a/lib/gitlab/cache/ci/project_pipeline_status.rb b/lib/gitlab/cache/ci/project_pipeline_status.rb
index b358f2efa4f..4fc9a075edc 100644
--- a/lib/gitlab/cache/ci/project_pipeline_status.rb
+++ b/lib/gitlab/cache/ci/project_pipeline_status.rb
@@ -15,18 +15,51 @@ module Gitlab
           end
         end
 
+        def self.load_in_batch_for_projects(projects)
+          cached_results_for_projects(projects).zip(projects).each do |result, project|
+            project.pipeline_status = new(project, result)
+            project.pipeline_status.load_status
+          end
+        end
+
+        def self.cached_results_for_projects(projects)
+          result = Gitlab::Redis.with do |redis|
+            redis.multi do
+              projects.each do |project|
+                cache_key = cache_key_for_project(project)
+                redis.exists(cache_key)
+                redis.hmget(cache_key, :sha, :status, :ref)
+              end
+            end
+          end
+
+          result.each_slice(2).map do |(cache_key_exists, (sha, status, ref))|
+            pipeline_info = { sha: sha, status: status, ref: ref }
+            { loaded_from_cache: cache_key_exists, pipeline_info: pipeline_info }
+          end
+        end
+
+        def self.cache_key_for_project(project)
+          "projects/#{project.id}/pipeline_status"
+        end
+
         def self.update_for_pipeline(pipeline)
-          new(pipeline.project,
-              sha: pipeline.sha,
-              status: pipeline.status,
-              ref: pipeline.ref).store_in_cache_if_needed
+          pipeline_info = {
+            sha: pipeline.sha,
+            status: pipeline.status,
+            ref: pipeline.ref
+          }
+
+          new(pipeline.project, pipeline_info: pipeline_info).
+            store_in_cache_if_needed
         end
 
-        def initialize(project, sha: nil, status: nil, ref: nil)
+        def initialize(project, pipeline_info: {}, loaded_from_cache: nil)
           @project = project
-          @sha = sha
-          @ref = ref
-          @status = status
+          @sha = pipeline_info[:sha]
+          @ref = pipeline_info[:ref]
+          @status = pipeline_info[:status]
+          @loaded = loaded_from_cache
         end
 
         def has_status?
@@ -85,6 +118,8 @@ module Gitlab
         end
 
         def has_cache?
+          return self.loaded unless self.loaded.nil?
+
           Gitlab::Redis.with do |redis|
             redis.exists(cache_key)
           end
@@ -95,7 +130,7 @@ module Gitlab
         end
 
         def cache_key
-          "projects/#{project.id}/build_status"
+          self.class.cache_key_for_project(project)
         end
       end
     end
diff --git a/lib/gitlab/data_builder/push.rb b/lib/gitlab/data_builder/push.rb
index d76aa38f741..1ff34553f0a 100644
--- a/lib/gitlab/data_builder/push.rb
+++ b/lib/gitlab/data_builder/push.rb
@@ -41,7 +41,7 @@ module Gitlab
         type = Gitlab::Git.tag_ref?(ref) ? 'tag_push' : 'push'
 
         # Hash to be passed as post_receive_data
-        data = {
+        {
           object_kind: type,
           event_name: type,
           before: oldrev,
@@ -61,16 +61,15 @@ module Gitlab
           repository: project.hook_attrs.slice(:name, :url, :description, :homepage,
                                                :git_http_url, :git_ssh_url, :visibility_level)
         }
-
-        data
       end
 
       # This method provide a sample data generated with
       # existing project and commits to test webhooks
       def build_sample(project, user)
-        commits = project.repository.commits(project.default_branch, limit: 3)
         ref = "#{Gitlab::Git::BRANCH_REF_PREFIX}#{project.default_branch}"
-        build(project, user, commits.last.id, commits.first.id, ref, commits)
+        commits = project.repository.commits(project.default_branch.to_s, limit: 3) rescue []
+
+        build(project, user, commits.last&.id, commits.first&.id, ref, commits)
       end
 
       def checkout_sha(repository, newrev, ref)
diff --git a/lib/gitlab/git/blob.rb b/lib/gitlab/git/blob.rb
index 98fd4e78126..e8bb9e1f805 100644
--- a/lib/gitlab/git/blob.rb
+++ b/lib/gitlab/git/blob.rb
@@ -109,10 +109,6 @@ module Gitlab
         @binary.nil? ? super : @binary == true
       end
 
-      def empty?
-        !data || data == ''
-      end
-
       def data
         encode! @data
       end
diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb
index d7dac9f6149..452dba7971d 100644
--- a/lib/gitlab/git/repository.rb
+++ b/lib/gitlab/git/repository.rb
@@ -494,7 +494,9 @@ module Gitlab
       #     :contains is the commit contained by the refs from which to begin (SHA1 or name)
       #     :max_count is the maximum number of commits to fetch
       #     :skip is the number of commits to skip
-      #     :order is the commits order and allowed value is :date(default) or :topo
+      #     :order is the commits order and allowed value is :none (default), :date, or :topo
+      #        commit ordering types are documented here:
+      #        http://www.rubydoc.info/github/libgit2/rugged/Rugged#SORT_NONE-constant)
       #
       def find_commits(options = {})
         actual_options = options.dup
@@ -522,11 +524,8 @@ module Gitlab
           end
         end
 
-        if actual_options[:order] == :topo
-          walker.sorting(Rugged::SORT_TOPO)
-        else
-          walker.sorting(Rugged::SORT_NONE)
-        end
+        sort_type = rugged_sort_type(actual_options[:order])
+        walker.sorting(sort_type)
 
         commits = []
         offset = actual_options[:skip]
@@ -1273,6 +1272,18 @@ module Gitlab
       def gitaly_ref_client
         @gitaly_ref_client ||= Gitlab::GitalyClient::Ref.new(self)
       end
+
+      # Returns the `Rugged` sorting type constant for a given
+      # sort type key. Valid keys are `:none`, `:topo`, and `:date`
+      def rugged_sort_type(key)
+        @rugged_sort_types ||= {
+          none: Rugged::SORT_NONE,
+          topo: Rugged::SORT_TOPO,
+          date: Rugged::SORT_DATE
+        }
+
+        @rugged_sort_types.fetch(key, Rugged::SORT_NONE)
+      end
     end
   end
 end
diff --git a/lib/gitlab/other_markup.rb b/lib/gitlab/other_markup.rb
index e67acf28c94..c2adc9aa10b 100644
--- a/lib/gitlab/other_markup.rb
+++ b/lib/gitlab/other_markup.rb
@@ -4,19 +4,11 @@ module Gitlab
     # Public: Converts the provided markup into HTML.
     #
     # input         - the source text in a markup format
-    # context       - a Hash with the template context:
-    #                 :commit
-    #                 :project
-    #                 :project_wiki
-    #                 :requested_path
-    #                 :ref
     #
-    def self.render(file_name, input, context)
+    def self.render(file_name, input)
       html = GitHub::Markup.render(file_name, input).
         force_encoding(input.encoding)
 
-      html = Banzai.post_process(html, context)
-
       filter = Banzai::Filter::SanitizationFilter.new(html)
       html = filter.call.to_s
 
diff --git a/lib/gitlab/template/dockerfile_template.rb b/lib/gitlab/template/dockerfile_template.rb
index d5d3e045a42..20b054b0bd8 100644
--- a/lib/gitlab/template/dockerfile_template.rb
+++ b/lib/gitlab/template/dockerfile_template.rb
@@ -8,7 +8,7 @@ module Gitlab
 
       class << self
         def extension
-          'Dockerfile'
+          '.Dockerfile'
         end
 
         def categories
@@ -18,7 +18,7 @@ module Gitlab
         end
 
         def base_dir
-          Rails.root.join('vendor/dockerfile')
+          Rails.root.join('vendor/Dockerfile')
         end
 
         def finder(project = nil)
diff --git a/lib/tasks/gitlab/update_templates.rake b/lib/tasks/gitlab/update_templates.rake
index cb2adc81c9d..1b04e1350ed 100644
--- a/lib/tasks/gitlab/update_templates.rake
+++ b/lib/tasks/gitlab/update_templates.rake
@@ -5,7 +5,7 @@ namespace :gitlab do
   end
 
   def update(template)
-    sub_dir = template.repo_url.match(/([a-z-]+)\.git\z/)[1]
+    sub_dir = template.repo_url.match(/([A-Za-z-]+)\.git\z/)[1]
     dir = File.join(vendor_directory, sub_dir)
 
     unless clone_repository(template.repo_url, dir)
@@ -45,7 +45,11 @@ namespace :gitlab do
     Template.new(
       "https://gitlab.com/gitlab-org/gitlab-ci-yml.git",
       /(\.{1,2}|LICENSE|CONTRIBUTING.md|Pages|autodeploy|\.gitlab-ci.yml)\z/
-    )
+    ),
+    Template.new(
+      "https://gitlab.com/gitlab-org/Dockerfile.git",
+      /(\.{1,2}|LICENSE|CONTRIBUTING.md|\.Dockerfile)\z/
+    ),
   ].freeze
 
   def vendor_directory
diff --git a/lib/tasks/import.rake b/lib/tasks/import.rake
index a9dad6a1bf0..bc76d7edc55 100644
--- a/lib/tasks/import.rake
+++ b/lib/tasks/import.rake
@@ -1,67 +1,5 @@
 require 'benchmark'
 require 'rainbow/ext/string'
-require_relative '../gitlab/shell_adapter'
-require_relative '../gitlab/github_import/importer'
-
-class NewImporter < ::Gitlab::GithubImport::Importer
-  def execute
-    # Same as ::Gitlab::GithubImport::Importer#execute, but showing some progress.
-    puts 'Importing repository...'.color(:aqua)
-    import_repository unless project.repository_exists?
-
-    puts 'Importing labels...'.color(:aqua)
-    import_labels
-
-    puts 'Importing milestones...'.color(:aqua)
-    import_milestones
-
-    puts 'Importing pull requests...'.color(:aqua)
-    import_pull_requests
-
-    puts 'Importing issues...'.color(:aqua)
-    import_issues
-
-    puts 'Importing issue comments...'.color(:aqua)
-    import_comments(:issues)
-
-    puts 'Importing pull request comments...'.color(:aqua)
-    import_comments(:pull_requests)
-
-    puts 'Importing wiki...'.color(:aqua)
-    import_wiki
-
-    # Gitea doesn't have a Release API yet
-    # See https://github.com/go-gitea/gitea/issues/330
-    unless project.gitea_import?
-      import_releases
-    end
-
-    handle_errors
-
-    project.repository.after_import
-    project.import_finish
-
-    true
-  end
-
-  def import_repository
-    begin
-      raise 'Blocked import URL.' if Gitlab::UrlBlocker.blocked_url?(project.import_url)
-
-      project.create_repository
-      project.repository.add_remote(project.import_type, project.import_url)
-      project.repository.set_remote_as_mirror(project.import_type)
-      project.repository.fetch_remote(project.import_type, forced: true)
-    rescue => e
-      # Expire cache to prevent scenarios such as:
-      # 1. First import failed, but the repo was imported successfully, so +exists?+ returns true
-      # 2. Retried import, repo is broken or not imported but +exists?+ still returns true
-      project.repository.expire_content_cache if project.repository_exists?
-
-      raise "Error importing repository #{project.import_url} into #{project.path_with_namespace} - #{e.message}"
-    end
-  end
-end
 
 class GithubImport
   def self.run!(*args)
@@ -69,14 +7,14 @@ class GithubImport
   end
 
   def initialize(token, gitlab_username, project_path, extras)
-    @token = token
+    @options = { url: 'https://api.github.com', token: token, verbose: true }
     @project_path = project_path
     @current_user = User.find_by_username(gitlab_username)
     @github_repo = extras.empty? ? nil : extras.first
   end
 
   def run!
-    @repo = GithubRepos.new(@token, @current_user, @github_repo).choose_one!
+    @repo = GithubRepos.new(@options, @current_user, @github_repo).choose_one!
 
     raise 'No repo found!' unless @repo
 
@@ -90,25 +28,24 @@ class GithubImport
   private
 
   def show_warning!
-    puts "This will import GH #{@repo.full_name.bright} into GL #{@project_path.bright} as #{@current_user.name}"
+    puts "This will import GitHub #{@repo['full_name'].bright} into GitLab #{@project_path.bright} as #{@current_user.name}"
     puts "Permission checks are ignored. Press any key to continue.".color(:red)
 
     STDIN.getch
 
-    puts 'Starting the import...'.color(:green)
+    puts 'Starting the import (this could take a while)'.color(:green)
   end
 
   def import!
-    import_url = @project.import_url.gsub(/\:\/\/(.*@)?/, "://#{@token}@")
-    @project.update(import_url: import_url)
-
     @project.import_start
 
     timings = Benchmark.measure do
-      NewImporter.new(@project).execute
+      Github::Import.new(@project, @options).execute
     end
 
     puts "Import finished. Timings: #{timings}".color(:green)
+
+    @project.import_finish
   end
 
   def new_project
@@ -116,17 +53,17 @@ class GithubImport
       namespace_path, _sep, name = @project_path.rpartition('/')
       namespace = find_or_create_namespace(namespace_path)
 
-      Project.create!(
-        import_url: "https://#{@token}@github.com/#{@repo.full_name}.git",
+      Projects::CreateService.new(
+        @current_user,
         name: name,
         path: name,
-        description: @repo.description,
-        namespace: namespace,
+        description: @repo['description'],
+        namespace_id: namespace.id,
         visibility_level: visibility_level,
         import_type: 'github',
-        import_source: @repo.full_name,
-        creator: @current_user
-      )
+        import_source: @repo['full_name'],
+        skip_wiki: @repo['has_wiki']
+      ).execute
     end
   end
 
@@ -134,7 +71,6 @@ class GithubImport
     return @current_user.namespace if names == @current_user.namespace_path
     return @current_user.namespace unless @current_user.can_create_group?
 
-    names = params[:target_namespace].presence || names
     full_path_namespace = Namespace.find_by_full_path(names)
 
     return full_path_namespace if full_path_namespace
@@ -159,13 +95,13 @@ class GithubImport
   end
 
   def visibility_level
-    @repo.private ? Gitlab::VisibilityLevel::PRIVATE : current_application_settings.default_project_visibility
+    @repo['private'] ? Gitlab::VisibilityLevel::PRIVATE : current_application_settings.default_project_visibility
   end
 end
 
 class GithubRepos
-  def initialize(token, current_user, github_repo)
-    @token = token
+  def initialize(options, current_user, github_repo)
+    @options = options
     @current_user = current_user
     @github_repo = github_repo
   end
@@ -174,17 +110,17 @@ class GithubRepos
     return found_github_repo if @github_repo
 
     repos.each do |repo|
-      print "ID: #{repo[:id].to_s.bright} ".color(:green)
-      puts "- Name: #{repo[:full_name]}".color(:green)
+      print "ID: #{repo['id'].to_s.bright}".color(:green)
+      print "\tName: #{repo['full_name']}\n".color(:green)
     end
 
     print 'ID? '.bright
 
-    repos.find { |repo| repo[:id] == repo_id }
+    repos.find { |repo| repo['id'] == repo_id }
   end
 
   def found_github_repo
-    repos.find { |repo| repo[:full_name] == @github_repo }
+    repos.find { |repo| repo['full_name'] == @github_repo }
   end
 
   def repo_id
@@ -192,11 +128,7 @@ class GithubRepos
   end
 
   def repos
-    @repos ||= client.repos
-  end
-
-  def client
-    @client ||= Gitlab::GithubImport::Client.new(@token, {})
+    Github::Repositories.new(@options).fetch
   end
 end