Merge branch 'master' into feature/runner-per-group

30 files changed, 505 insertions, 140 deletions

diff --git a/lib/api/discussions.rb b/lib/api/discussions.rb
index 7975f35ab1e..13c34e3473a 100644
--- a/lib/api/discussions.rb
+++ b/lib/api/discussions.rb
@@ -5,11 +5,12 @@ module API
 
     before { authenticate! }
 
-    NOTEABLE_TYPES = [Issue, Snippet].freeze
+    NOTEABLE_TYPES = [Issue, Snippet, MergeRequest, Commit].freeze
 
     NOTEABLE_TYPES.each do |noteable_type|
       parent_type = noteable_type.parent_class.to_s.underscore
       noteables_str = noteable_type.to_s.underscore.pluralize
+      noteables_path = noteable_type == Commit ? "repository/#{noteables_str}" : noteables_str
 
       params do
         requires :id, type: String, desc: "The ID of a #{parent_type}"
@@ -19,14 +20,12 @@ module API
           success Entities::Discussion
         end
         params do
-          requires :noteable_id, type: Integer, desc: 'The ID of the noteable'
+          requires :noteable_id, types: [Integer, String], desc: 'The ID of the noteable'
           use :pagination
         end
-        get ":id/#{noteables_str}/:noteable_id/discussions" do
+        get ":id/#{noteables_path}/:noteable_id/discussions" do
           noteable = find_noteable(parent_type, noteables_str, params[:noteable_id])
 
-          break not_found!("Discussions") unless can?(current_user, noteable_read_ability_name(noteable), noteable)
-
           notes = noteable.notes
             .inc_relations_for_view
             .includes(:noteable)
@@ -43,13 +42,13 @@ module API
         end
         params do
           requires :discussion_id, type: String, desc: 'The ID of a discussion'
-          requires :noteable_id, type: Integer, desc: 'The ID of the noteable'
+          requires :noteable_id, types: [Integer, String], desc: 'The ID of the noteable'
         end
-        get ":id/#{noteables_str}/:noteable_id/discussions/:discussion_id" do
+        get ":id/#{noteables_path}/:noteable_id/discussions/:discussion_id" do
           noteable = find_noteable(parent_type, noteables_str, params[:noteable_id])
           notes = readable_discussion_notes(noteable, params[:discussion_id])
 
-          if notes.empty? || !can?(current_user, noteable_read_ability_name(noteable), noteable)
+          if notes.empty?
             break not_found!("Discussion")
           end
 
@@ -62,19 +61,36 @@ module API
           success Entities::Discussion
         end
         params do
-          requires :noteable_id, type: Integer, desc: 'The ID of the noteable'
+          requires :noteable_id, types: [Integer, String], desc: 'The ID of the noteable'
           requires :body, type: String, desc: 'The content of a note'
           optional :created_at, type: String, desc: 'The creation date of the note'
+          optional :position, type: Hash do
+            requires :base_sha, type: String, desc: 'Base commit SHA in the source branch'
+            requires :start_sha, type: String, desc: 'SHA referencing commit in target branch'
+            requires :head_sha, type: String, desc: 'SHA referencing HEAD of this merge request'
+            requires :position_type, type: String, desc: 'Type of the position reference', values: %w(text image)
+            optional :new_path, type: String, desc: 'File path after change'
+            optional :new_line, type: Integer, desc: 'Line number after change'
+            optional :old_path, type: String, desc: 'File path before change'
+            optional :old_line, type: Integer, desc: 'Line number before change'
+            optional :width, type: Integer, desc: 'Width of the image'
+            optional :height, type: Integer, desc: 'Height of the image'
+            optional :x, type: Integer, desc: 'X coordinate in the image'
+            optional :y, type: Integer, desc: 'Y coordinate in the image'
+          end
         end
-        post ":id/#{noteables_str}/:noteable_id/discussions" do
+        post ":id/#{noteables_path}/:noteable_id/discussions" do
           noteable = find_noteable(parent_type, noteables_str, params[:noteable_id])
+          type = params[:position] ? 'DiffNote' : 'DiscussionNote'
+          id_key = noteable.is_a?(Commit) ? :commit_id : :noteable_id
 
           opts = {
             note: params[:body],
             created_at: params[:created_at],
-            type: 'DiscussionNote',
+            type: type,
             noteable_type: noteables_str.classify,
-            noteable_id: noteable.id
+            position: params[:position],
+            id_key => noteable.id
           }
 
           note = create_note(noteable, opts)
@@ -91,13 +107,13 @@ module API
         end
         params do
           requires :discussion_id, type: String, desc: 'The ID of a discussion'
-          requires :noteable_id, type: Integer, desc: 'The ID of the noteable'
+          requires :noteable_id, types: [Integer, String], desc: 'The ID of the noteable'
         end
-        get ":id/#{noteables_str}/:noteable_id/discussions/:discussion_id/notes" do
+        get ":id/#{noteables_path}/:noteable_id/discussions/:discussion_id/notes" do
           noteable = find_noteable(parent_type, noteables_str, params[:noteable_id])
           notes = readable_discussion_notes(noteable, params[:discussion_id])
 
-          if notes.empty? || !can?(current_user, noteable_read_ability_name(noteable), noteable)
+          if notes.empty?
             break not_found!("Notes")
           end
 
@@ -108,12 +124,12 @@ module API
           success Entities::Note
         end
         params do
-          requires :noteable_id, type: Integer, desc: 'The ID of the noteable'
+          requires :noteable_id, types: [Integer, String], desc: 'The ID of the noteable'
           requires :discussion_id, type: String, desc: 'The ID of a discussion'
           requires :body, type: String, desc: 'The content of a note'
           optional :created_at, type: String, desc: 'The creation date of the note'
         end
-        post ":id/#{noteables_str}/:noteable_id/discussions/:discussion_id/notes" do
+        post ":id/#{noteables_path}/:noteable_id/discussions/:discussion_id/notes" do
           noteable = find_noteable(parent_type, noteables_str, params[:noteable_id])
           notes = readable_discussion_notes(noteable, params[:discussion_id])
 
@@ -139,11 +155,11 @@ module API
           success Entities::Note
         end
         params do
-          requires :noteable_id, type: Integer, desc: 'The ID of the noteable'
+          requires :noteable_id, types: [Integer, String], desc: 'The ID of the noteable'
           requires :discussion_id, type: String, desc: 'The ID of a discussion'
           requires :note_id, type: Integer, desc: 'The ID of a note'
         end
-        get ":id/#{noteables_str}/:noteable_id/discussions/:discussion_id/notes/:note_id" do
+        get ":id/#{noteables_path}/:noteable_id/discussions/:discussion_id/notes/:note_id" do
           noteable = find_noteable(parent_type, noteables_str, params[:noteable_id])
 
           get_note(noteable, params[:note_id])
@@ -153,30 +169,52 @@ module API
           success Entities::Note
         end
         params do
-          requires :noteable_id, type: Integer, desc: 'The ID of the noteable'
+          requires :noteable_id, types: [Integer, String], desc: 'The ID of the noteable'
           requires :discussion_id, type: String, desc: 'The ID of a discussion'
           requires :note_id, type: Integer, desc: 'The ID of a note'
-          requires :body, type: String, desc: 'The content of a note'
+          optional :body, type: String, desc: 'The content of a note'
+          optional :resolved, type: Boolean, desc: 'Mark note resolved/unresolved'
+          exactly_one_of :body, :resolved
         end
-        put ":id/#{noteables_str}/:noteable_id/discussions/:discussion_id/notes/:note_id" do
+        put ":id/#{noteables_path}/:noteable_id/discussions/:discussion_id/notes/:note_id" do
           noteable = find_noteable(parent_type, noteables_str, params[:noteable_id])
 
-          update_note(noteable, params[:note_id])
+          if params[:resolved].nil?
+            update_note(noteable, params[:note_id])
+          else
+            resolve_note(noteable, params[:note_id], params[:resolved])
+          end
         end
 
         desc "Delete a comment in a #{noteable_type.to_s.downcase} discussion" do
           success Entities::Note
         end
         params do
-          requires :noteable_id, type: Integer, desc: 'The ID of the noteable'
+          requires :noteable_id, types: [Integer, String], desc: 'The ID of the noteable'
           requires :discussion_id, type: String, desc: 'The ID of a discussion'
           requires :note_id, type: Integer, desc: 'The ID of a note'
         end
-        delete ":id/#{noteables_str}/:noteable_id/discussions/:discussion_id/notes/:note_id" do
+        delete ":id/#{noteables_path}/:noteable_id/discussions/:discussion_id/notes/:note_id" do
           noteable = find_noteable(parent_type, noteables_str, params[:noteable_id])
 
           delete_note(noteable, params[:note_id])
         end
+
+        if Noteable::RESOLVABLE_TYPES.include?(noteable_type.to_s)
+          desc "Resolve/unresolve an existing #{noteable_type.to_s.downcase} discussion" do
+            success Entities::Discussion
+          end
+          params do
+            requires :noteable_id, types: [Integer, String], desc: 'The ID of the noteable'
+            requires :discussion_id, type: String, desc: 'The ID of a discussion'
+            requires :resolved, type: Boolean, desc: 'Mark discussion resolved/unresolved'
+          end
+          put ":id/#{noteables_path}/:noteable_id/discussions/:discussion_id" do
+            noteable = find_noteable(parent_type, noteables_str, params[:noteable_id])
+
+            resolve_discussion(noteable, params[:discussion_id], params[:resolved])
+          end
+        end
       end
     end
 
diff --git a/lib/api/entities.rb b/lib/api/entities.rb
index f28c4bcc784..6ff1c0624e5 100644
--- a/lib/api/entities.rb
+++ b/lib/api/entities.rb
@@ -291,6 +291,10 @@ module API
       end
     end
 
+    class DiffRefs < Grape::Entity
+      expose :base_sha, :head_sha, :start_sha
+    end
+
     class Commit < Grape::Entity
       expose :id, :short_id, :title, :created_at
       expose :parent_ids
@@ -606,6 +610,8 @@ module API
         merge_request.metrics&.pipeline
       end
 
+      expose :diff_refs, using: Entities::DiffRefs
+
       def build_available?(options)
         options[:project]&.feature_available?(:builds, options[:current_user])
       end
@@ -647,6 +653,11 @@ module API
       expose :id, :key, :created_at
     end
 
+    class DiffPosition < Grape::Entity
+      expose :base_sha, :start_sha, :head_sha, :old_path, :new_path,
+        :position_type
+    end
+
     class Note < Grape::Entity
       # Only Issue and MergeRequest have iid
       NOTEABLE_TYPES_WITH_IID = %w(Issue MergeRequest).freeze
@@ -660,6 +671,14 @@ module API
       expose :system?, as: :system
       expose :noteable_id, :noteable_type
 
+      expose :position, if: ->(note, options) { note.diff_note? } do |note|
+        note.position.to_h
+      end
+
+      expose :resolvable?, as: :resolvable
+      expose :resolved?, as: :resolved, if: ->(note, options) { note.resolvable? }
+      expose :resolved_by, using: Entities::UserBasic, if: ->(note, options) { note.resolvable? }
+
       # Avoid N+1 queries as much as possible
       expose(:noteable_iid) { |note| note.noteable.iid if NOTEABLE_TYPES_WITH_IID.include?(note.noteable_type) }
     end
diff --git a/lib/api/groups.rb b/lib/api/groups.rb
index 4a4df1b8b9e..92e3d5cc10a 100644
--- a/lib/api/groups.rb
+++ b/lib/api/groups.rb
@@ -37,13 +37,11 @@ module API
         use :pagination
       end
 
-      def find_groups(params)
-        find_params = {
-          all_available: params[:all_available],
-          custom_attributes: params[:custom_attributes],
-          owned: params[:owned]
-        }
-        find_params[:parent] = find_group!(params[:id]) if params[:id]
+      def find_groups(params, parent_id = nil)
+        find_params = params.slice(:all_available, :custom_attributes, :owned)
+        find_params[:parent] = find_group!(parent_id) if parent_id
+        find_params[:all_available] =
+          find_params.fetch(:all_available, current_user&.full_private_access?)
 
         groups = GroupsFinder.new(current_user, find_params).execute
         groups = groups.search(params[:search]) if params[:search].present?
@@ -85,7 +83,7 @@ module API
         use :with_custom_attributes
       end
       get do
-        groups = find_groups(params)
+        groups = find_groups(declared_params(include_missing: false), params[:id])
         present_groups params, groups
       end
 
@@ -213,7 +211,7 @@ module API
         use :with_custom_attributes
       end
       get ":id/subgroups" do
-        groups = find_groups(params)
+        groups = find_groups(declared_params(include_missing: false), params[:id])
         present_groups params, groups
       end
 
diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb
index b8657cd7ee4..2ed331d4fd2 100644
--- a/lib/api/helpers.rb
+++ b/lib/api/helpers.rb
@@ -171,6 +171,10 @@ module API
       MergeRequestsFinder.new(current_user, project_id: user_project.id).find_by!(iid: iid)
     end
 
+    def find_project_commit(id)
+      user_project.commit_by(oid: id)
+    end
+
     def find_project_snippet(id)
       finder_params = { project: user_project }
       SnippetsFinder.new(current_user, finder_params).find(id)
diff --git a/lib/api/helpers/custom_attributes.rb b/lib/api/helpers/custom_attributes.rb
index 70e4eda95f8..10d652e33f5 100644
--- a/lib/api/helpers/custom_attributes.rb
+++ b/lib/api/helpers/custom_attributes.rb
@@ -7,6 +7,9 @@ module API
         helpers do
           params :with_custom_attributes do
             optional :with_custom_attributes, type: Boolean, default: false, desc: 'Include custom attributes in the response'
+
+            optional :custom_attributes, type: Hash,
+                                         desc: 'Filter with custom attributes'
           end
 
           def with_custom_attributes(collection_or_resource, options = {})
diff --git a/lib/api/helpers/notes_helpers.rb b/lib/api/helpers/notes_helpers.rb
index b74b8149834..b4bfb677d72 100644
--- a/lib/api/helpers/notes_helpers.rb
+++ b/lib/api/helpers/notes_helpers.rb
@@ -21,6 +21,23 @@ module API
         end
       end
 
+      def resolve_note(noteable, note_id, resolved)
+        note = noteable.notes.find(note_id)
+
+        authorize! :resolve_note, note
+
+        bad_request!("Note is not resolvable") unless note.resolvable?
+
+        if resolved
+          parent = noteable_parent(noteable)
+          ::Notes::ResolveService.new(parent, current_user).execute(note)
+        else
+          note.unresolve!
+        end
+
+        present note, with: Entities::Note
+      end
+
       def delete_note(noteable, note_id)
         note = noteable.notes.find(note_id)
 
@@ -35,7 +52,7 @@ module API
 
       def get_note(noteable, note_id)
         note = noteable.notes.with_metadata.find(params[:note_id])
-        can_read_note = can?(current_user, noteable_read_ability_name(noteable), noteable) && !note.cross_reference_not_visible_for?(current_user)
+        can_read_note = !note.cross_reference_not_visible_for?(current_user)
 
         if can_read_note
           present note, with: Entities::Note
@@ -49,7 +66,20 @@ module API
       end
 
       def find_noteable(parent, noteables_str, noteable_id)
-        public_send("find_#{parent}_#{noteables_str.singularize}", noteable_id) # rubocop:disable GitlabSecurity/PublicSend
+        noteable = public_send("find_#{parent}_#{noteables_str.singularize}", noteable_id) # rubocop:disable GitlabSecurity/PublicSend
+
+        readable =
+          if noteable.is_a?(Commit)
+            # for commits there is not :read_commit policy, check if user
+            # has :read_note permission on the commit's project
+            can?(current_user, :read_note, user_project)
+          else
+            can?(current_user, noteable_read_ability_name(noteable), noteable)
+          end
+
+        return not_found!(noteables_str) unless readable
+
+        noteable
       end
 
       def noteable_parent(noteable)
@@ -57,11 +87,8 @@ module API
       end
 
       def create_note(noteable, opts)
-        noteables_str = noteable.model_name.to_s.underscore.pluralize
-
-        return not_found!(noteables_str) unless can?(current_user, noteable_read_ability_name(noteable), noteable)
-
-        authorize! :create_note, noteable
+        policy_object = noteable.is_a?(Commit) ? user_project : noteable
+        authorize!(:create_note, policy_object)
 
         parent = noteable_parent(noteable)
 
@@ -73,6 +100,21 @@ module API
         project = parent if parent.is_a?(Project)
         ::Notes::CreateService.new(project, current_user, opts).execute
       end
+
+      def resolve_discussion(noteable, discussion_id, resolved)
+        discussion = noteable.find_discussion(discussion_id)
+
+        forbidden! unless discussion.can_resolve?(current_user)
+
+        if resolved
+          parent = noteable_parent(noteable)
+          ::Discussions::ResolveService.new(parent, current_user, merge_request: noteable).execute(discussion)
+        else
+          discussion.unresolve!
+        end
+
+        present discussion, with: Entities::Discussion
+      end
     end
   end
 end
diff --git a/lib/api/notes.rb b/lib/api/notes.rb
index 69f1df6b341..39923e6d5b5 100644
--- a/lib/api/notes.rb
+++ b/lib/api/notes.rb
@@ -31,23 +31,19 @@ module API
         get ":id/#{noteables_str}/:noteable_id/notes" do
           noteable = find_noteable(parent_type, noteables_str, params[:noteable_id])
 
-          if can?(current_user, noteable_read_ability_name(noteable), noteable)
-            # We exclude notes that are cross-references and that cannot be viewed
-            # by the current user. By doing this exclusion at this level and not
-            # at the DB query level (which we cannot in that case), the current
-            # page can have less elements than :per_page even if
-            # there's more than one page.
-            raw_notes = noteable.notes.with_metadata.reorder(params[:order_by] => params[:sort])
-            notes =
-              # paginate() only works with a relation. This could lead to a
-              # mismatch between the pagination headers info and the actual notes
-              # array returned, but this is really a edge-case.
-              paginate(raw_notes)
-              .reject { |n| n.cross_reference_not_visible_for?(current_user) }
-            present notes, with: Entities::Note
-          else
-            not_found!("Notes")
-          end
+          # We exclude notes that are cross-references and that cannot be viewed
+          # by the current user. By doing this exclusion at this level and not
+          # at the DB query level (which we cannot in that case), the current
+          # page can have less elements than :per_page even if
+          # there's more than one page.
+          raw_notes = noteable.notes.with_metadata.reorder(params[:order_by] => params[:sort])
+          notes =
+            # paginate() only works with a relation. This could lead to a
+            # mismatch between the pagination headers info and the actual notes
+            # array returned, but this is really a edge-case.
+            paginate(raw_notes)
+            .reject { |n| n.cross_reference_not_visible_for?(current_user) }
+          present notes, with: Entities::Note
         end
 
         desc "Get a single #{noteable_type.to_s.downcase} note" do
diff --git a/lib/api/pipelines.rb b/lib/api/pipelines.rb
index d2b8b832e4e..735591fedd5 100644
--- a/lib/api/pipelines.rb
+++ b/lib/api/pipelines.rb
@@ -19,6 +19,7 @@ module API
         optional :status,   type: String, values: HasStatus::AVAILABLE_STATUSES,
                             desc: 'The status of pipelines'
         optional :ref,      type: String, desc: 'The ref of pipelines'
+        optional :sha,      type: String, desc: 'The sha of pipelines'
         optional :yaml_errors, type: Boolean, desc: 'Returns pipelines with invalid configurations'
         optional :name,     type: String, desc: 'The name of the user who triggered pipelines'
         optional :username, type: String, desc: 'The username of the user who triggered pipelines'
diff --git a/lib/banzai/filter/commit_trailers_filter.rb b/lib/banzai/filter/commit_trailers_filter.rb
index ef16df1f3ae..7b55e8b36f6 100644
--- a/lib/banzai/filter/commit_trailers_filter.rb
+++ b/lib/banzai/filter/commit_trailers_filter.rb
@@ -13,7 +13,6 @@ module Banzai
     # * https://git.wiki.kernel.org/index.php/CommitMessageConventions
     class CommitTrailersFilter < HTML::Pipeline::Filter
       include ActionView::Helpers::TagHelper
-      include ApplicationHelper
       include AvatarsHelper
 
       TRAILER_REGEXP = /(?<label>[[:alpha:]-]+-by:)/i.freeze
diff --git a/lib/gitlab/background_migration/migrate_stage_index.rb b/lib/gitlab/background_migration/migrate_stage_index.rb
new file mode 100644
index 00000000000..f90f35a913d
--- /dev/null
+++ b/lib/gitlab/background_migration/migrate_stage_index.rb
@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+# rubocop:disable Style/Documentation
+
+module Gitlab
+  module BackgroundMigration
+    class MigrateStageIndex
+      def perform(start_id, stop_id)
+        migrate_stage_index_sql(start_id.to_i, stop_id.to_i).tap do |sql|
+          ActiveRecord::Base.connection.execute(sql)
+        end
+      end
+
+      private
+
+      def migrate_stage_index_sql(start_id, stop_id)
+        if Gitlab::Database.postgresql?
+          <<~SQL
+            WITH freqs AS (
+              SELECT stage_id, stage_idx, COUNT(*) AS freq FROM ci_builds
+                WHERE stage_id BETWEEN #{start_id} AND #{stop_id}
+                  AND stage_idx IS NOT NULL
+                GROUP BY stage_id, stage_idx
+            ), indexes AS (
+              SELECT DISTINCT stage_id, last_value(stage_idx)
+                OVER (PARTITION BY stage_id ORDER BY freq ASC) AS index
+                FROM freqs
+            )
+
+            UPDATE ci_stages SET position = indexes.index
+              FROM indexes WHERE indexes.stage_id = ci_stages.id
+                AND ci_stages.position IS NULL;
+          SQL
+        else
+          <<~SQL
+            UPDATE ci_stages
+              SET position =
+                (SELECT stage_idx FROM ci_builds
+                  WHERE ci_builds.stage_id = ci_stages.id
+                  GROUP BY ci_builds.stage_idx ORDER BY COUNT(*) DESC LIMIT 1)
+            WHERE ci_stages.id BETWEEN #{start_id} AND #{stop_id}
+              AND ci_stages.position IS NULL
+          SQL
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/base_doorkeeper_controller.rb b/lib/gitlab/base_doorkeeper_controller.rb
new file mode 100644
index 00000000000..e4227af25d2
--- /dev/null
+++ b/lib/gitlab/base_doorkeeper_controller.rb
@@ -0,0 +1,8 @@
+# This is a base controller for doorkeeper.
+# It adds the `can?` helper used in the views.
+module Gitlab
+  class BaseDoorkeeperController < ActionController::Base
+    include Gitlab::Allowable
+    helper_method :can?
+  end
+end
diff --git a/lib/gitlab/ci/cron_parser.rb b/lib/gitlab/ci/cron_parser.rb
index 551483d0aaa..73f36735e35 100644
--- a/lib/gitlab/ci/cron_parser.rb
+++ b/lib/gitlab/ci/cron_parser.rb
@@ -6,7 +6,7 @@ module Gitlab
 
       def initialize(cron, cron_timezone = 'UTC')
         @cron = cron
-        @cron_timezone = ActiveSupport::TimeZone.find_tzinfo(cron_timezone).name
+        @cron_timezone = timezone_name(cron_timezone)
       end
 
       def next_time_from(time)
@@ -24,6 +24,12 @@ module Gitlab
 
       private
 
+      def timezone_name(timezone)
+        ActiveSupport::TimeZone.find_tzinfo(timezone).name
+      rescue TZInfo::InvalidTimezoneIdentifier
+        timezone
+      end
+
       # NOTE:
       # cron_timezone can only accept timezones listed in TZInfo::Timezone.
       # Aliases of Timezones from ActiveSupport::TimeZone are NOT accepted,
diff --git a/lib/gitlab/ci/pipeline/chain/populate.rb b/lib/gitlab/ci/pipeline/chain/populate.rb
index d299a5677de..69b8a8fc68f 100644
--- a/lib/gitlab/ci/pipeline/chain/populate.rb
+++ b/lib/gitlab/ci/pipeline/chain/populate.rb
@@ -14,14 +14,10 @@ module Gitlab
             @command.seeds_block&.call(pipeline)
 
             ##
-            # Populate pipeline with all stages and builds from pipeline seeds.
+            # Populate pipeline with all stages, and stages with builds.
             #
             pipeline.stage_seeds.each do |stage|
               pipeline.stages << stage.to_resource
-
-              stage.seeds.each do |build|
-                pipeline.builds << build.to_resource
-              end
             end
 
             if pipeline.stages.none?
diff --git a/lib/gitlab/ci/pipeline/seed/stage.rb b/lib/gitlab/ci/pipeline/seed/stage.rb
index c101f30d6e8..2b58d9863a0 100644
--- a/lib/gitlab/ci/pipeline/seed/stage.rb
+++ b/lib/gitlab/ci/pipeline/seed/stage.rb
@@ -19,6 +19,7 @@ module Gitlab
 
           def attributes
             { name: @attributes.fetch(:name),
+              position: @attributes.fetch(:index),
               pipeline: @pipeline,
               project: @pipeline.project }
           end
diff --git a/lib/gitlab/database/arel_methods.rb b/lib/gitlab/database/arel_methods.rb
new file mode 100644
index 00000000000..d7e3ce08b32
--- /dev/null
+++ b/lib/gitlab/database/arel_methods.rb
@@ -0,0 +1,18 @@
+module Gitlab
+  module Database
+    module ArelMethods
+      private
+
+      # In Arel 7.0.0 (Arel 7.1.4 is used in Rails 5.0) the `engine` parameter of `Arel::UpdateManager#initializer`
+      # was removed.
+      # Remove this file and inline this method when removing rails5? code.
+      def arel_update_manager
+        if Gitlab.rails5?
+          Arel::UpdateManager.new
+        else
+          Arel::UpdateManager.new(ActiveRecord::Base)
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb
index 77079e5e72b..c21bae5e16b 100644
--- a/lib/gitlab/database/migration_helpers.rb
+++ b/lib/gitlab/database/migration_helpers.rb
@@ -1,6 +1,8 @@
 module Gitlab
   module Database
     module MigrationHelpers
+      include Gitlab::Database::ArelMethods
+
       BACKGROUND_MIGRATION_BATCH_SIZE = 1000 # Number of rows to process per job
       BACKGROUND_MIGRATION_JOB_BUFFER_SIZE = 1000 # Number of jobs to bulk queue at a time
 
@@ -314,7 +316,7 @@ module Gitlab
           stop_arel = yield table, stop_arel if block_given?
           stop_row = exec_query(stop_arel.to_sql).to_hash.first
 
-          update_arel = Arel::UpdateManager.new(ActiveRecord::Base)
+          update_arel = arel_update_manager
             .table(table)
             .set([[table[column], value]])
             .where(table[:id].gteq(start_id))
diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb
index 1a697396ff1..14de28a1d08 100644
--- a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb
+++ b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb
@@ -3,6 +3,8 @@ module Gitlab
     module RenameReservedPathsMigration
       module V1
         class RenameBase
+          include Gitlab::Database::ArelMethods
+
           attr_reader :paths, :migration
 
           delegate :update_column_in_batches,
@@ -62,10 +64,10 @@ module Gitlab
                                             old_full_path,
                                             new_full_path)
 
-            update = Arel::UpdateManager.new(ActiveRecord::Base)
-                       .table(routes)
-                       .set([[routes[:path], replace_statement]])
-                       .where(Arel::Nodes::SqlLiteral.new(filter))
+            update = arel_update_manager
+              .table(routes)
+              .set([[routes[:path], replace_statement]])
+              .where(Arel::Nodes::SqlLiteral.new(filter))
 
             execute(update.to_sql)
           end
diff --git a/lib/gitlab/diff/file_collection/base.rb b/lib/gitlab/diff/file_collection/base.rb
index a6007ebf531..c79d8d3cb21 100644
--- a/lib/gitlab/diff/file_collection/base.rb
+++ b/lib/gitlab/diff/file_collection/base.rb
@@ -36,6 +36,8 @@ module Gitlab
         private
 
         def decorate_diff!(diff)
+          return diff if diff.is_a?(File)
+
           Gitlab::Diff::File.new(diff, repository: project.repository, diff_refs: diff_refs, fallback_diff_refs: fallback_diff_refs)
         end
       end
diff --git a/lib/gitlab/diff/position.rb b/lib/gitlab/diff/position.rb
index 690b27cde81..978962ab2eb 100644
--- a/lib/gitlab/diff/position.rb
+++ b/lib/gitlab/diff/position.rb
@@ -12,6 +12,10 @@ module Gitlab
                :head_sha,
                :old_line,
                :new_line,
+               :width,
+               :height,
+               :x,
+               :y,
                :position_type, to: :formatter
 
       # A position can belong to a text line or to an image coordinate
diff --git a/lib/gitlab/git/raw_diff_change.rb b/lib/gitlab/git/raw_diff_change.rb
index eb3d8819239..92f6c45ce25 100644
--- a/lib/gitlab/git/raw_diff_change.rb
+++ b/lib/gitlab/git/raw_diff_change.rb
@@ -38,7 +38,9 @@ module Gitlab
       end
 
       def extract_operation
-        case @raw_operation&.first(1)
+        return :unknown unless @raw_operation
+
+        case @raw_operation[0]
         when 'A'
           :added
         when 'C'
diff --git a/lib/gitlab/git/remote_repository.rb b/lib/gitlab/git/remote_repository.rb
index 6bd6e58feeb..f40e59a8dd0 100644
--- a/lib/gitlab/git/remote_repository.rb
+++ b/lib/gitlab/git/remote_repository.rb
@@ -12,7 +12,7 @@ module Gitlab
     # class.
     #
     class RemoteRepository
-      attr_reader :path, :relative_path, :gitaly_repository
+      attr_reader :relative_path, :gitaly_repository
 
       def initialize(repository)
         @relative_path = repository.relative_path
@@ -21,7 +21,6 @@ module Gitlab
         # These instance variables will not be available in gitaly-ruby, where
         # we have no disk access to this repository.
         @repository = repository
-        @path = repository.path
       end
 
       def empty?
@@ -69,6 +68,10 @@ module Gitlab
         env
       end
 
+      def path
+        @repository.path
+      end
+
       private
 
       # Must return an object that responds to 'address' and 'storage'.
diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb
index 5a6e2e0b937..84d37f77fbb 100644
--- a/lib/gitlab/git/repository.rb
+++ b/lib/gitlab/git/repository.rb
@@ -142,15 +142,7 @@ module Gitlab
       end
 
       def exists?
-        Gitlab::GitalyClient.migrate(:repository_exists, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |enabled|
-          if enabled
-            gitaly_repository_client.exists?
-          else
-            circuit_breaker.perform do
-              File.exist?(File.join(path, 'refs'))
-            end
-          end
-        end
+        gitaly_repository_client.exists?
       end
 
       # Returns an Array of branch names
@@ -399,18 +391,6 @@ module Gitlab
         nil
       end
 
-      def archive_prefix(ref, sha, append_sha:)
-        append_sha = (ref != sha) if append_sha.nil?
-
-        project_name = self.name.chomp('.git')
-        formatted_ref = ref.tr('/', '-')
-
-        prefix_segments = [project_name, formatted_ref]
-        prefix_segments << sha if append_sha
-
-        prefix_segments.join('-')
-      end
-
       def archive_metadata(ref, storage_path, format = "tar.gz", append_sha:)
         ref ||= root_ref
         commit = Gitlab::Git::Commit.find(self, ref)
@@ -421,12 +401,44 @@ module Gitlab
         {
           'RepoPath' => path,
           'ArchivePrefix' => prefix,
-          'ArchivePath' => archive_file_path(prefix, storage_path, format),
+          'ArchivePath' => archive_file_path(storage_path, commit.id, prefix, format),
           'CommitId' => commit.id
         }
       end
 
-      def archive_file_path(name, storage_path, format = "tar.gz")
+      # This is both the filename of the archive (missing the extension) and the
+      # name of the top-level member of the archive under which all files go
+      #
+      # FIXME: The generated prefix is incorrect for projects with hashed
+      # storage enabled
+      def archive_prefix(ref, sha, append_sha:)
+        append_sha = (ref != sha) if append_sha.nil?
+
+        project_name = self.name.chomp('.git')
+        formatted_ref = ref.tr('/', '-')
+
+        prefix_segments = [project_name, formatted_ref]
+        prefix_segments << sha if append_sha
+
+        prefix_segments.join('-')
+      end
+      private :archive_prefix
+
+      # The full path on disk where the archive should be stored. This is used
+      # to cache the archive between requests.
+      #
+      # The path is a global namespace, so needs to be globally unique. This is
+      # achieved by including `gl_repository` in the path.
+      #
+      # Archives relating to a particular ref when the SHA is not present in the
+      # filename must be invalidated when the ref is updated to point to a new
+      # SHA. This is achieved by including the SHA in the path.
+      #
+      # As this is a full path on disk, it is not "cloud native". This should
+      # be resolved by either removing the cache, or moving the implementation
+      # into Gitaly and removing the ArchivePath parameter from the git-archive
+      # senddata response.
+      def archive_file_path(storage_path, sha, name, format = "tar.gz")
         # Build file path
         return nil unless name
 
@@ -444,8 +456,9 @@ module Gitlab
           end
 
         file_name = "#{name}.#{extension}"
-        File.join(storage_path, self.name, file_name)
+        File.join(storage_path, self.gl_repository, sha, file_name)
       end
+      private :archive_file_path
 
       # Return repo size in megabytes
       def size
@@ -1187,6 +1200,8 @@ module Gitlab
           if is_enabled
             gitaly_fetch_ref(source_repository, source_ref: source_ref, target_ref: target_ref)
           else
+            # When removing this code, also remove source_repository#path
+            # to remove deprecated method calls
             local_fetch_ref(source_repository.path, source_ref: source_ref, target_ref: target_ref)
           end
         end
@@ -1554,7 +1569,8 @@ module Gitlab
       end
 
       def checksum
-        gitaly_migrate(:calculate_checksum) do |is_enabled|
+        gitaly_migrate(:calculate_checksum,
+                      status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled|
           if is_enabled
             gitaly_repository_client.calculate_checksum
           else
diff --git a/lib/gitlab/gitaly_client/repository_service.rb b/lib/gitlab/gitaly_client/repository_service.rb
index bf5a491e28d..498187997e1 100644
--- a/lib/gitlab/gitaly_client/repository_service.rb
+++ b/lib/gitlab/gitaly_client/repository_service.rb
@@ -142,7 +142,7 @@ module Gitlab
           :repository_service,
           :is_rebase_in_progress,
           request,
-          timeout: GitalyClient.default_timeout
+          timeout: GitalyClient.fast_timeout
         )
 
         response.in_progress
@@ -159,7 +159,7 @@ module Gitlab
           :repository_service,
           :is_squash_in_progress,
           request,
-          timeout: GitalyClient.default_timeout
+          timeout: GitalyClient.fast_timeout
         )
 
         response.in_progress
diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb
index a7e055ac444..c741dabe168 100644
--- a/lib/gitlab/gon_helper.rb
+++ b/lib/gitlab/gon_helper.rb
@@ -19,6 +19,7 @@ module Gitlab
       gon.gitlab_logo            = ActionController::Base.helpers.asset_path('gitlab_logo.png')
       gon.sprite_icons           = IconsHelper.sprite_icon_path
       gon.sprite_file_icons      = IconsHelper.sprite_file_icons_path
+      gon.emoji_sprites_css_path = ActionController::Base.helpers.stylesheet_path('emoji_sprites')
       gon.test_env               = Rails.env.test?
       gon.suggested_label_colors = LabelsHelper.suggested_colors
 
diff --git a/lib/gitlab/kubernetes/helm/base_command.rb b/lib/gitlab/kubernetes/helm/base_command.rb
index 6e4df05aa7e..3d778da90c7 100644
--- a/lib/gitlab/kubernetes/helm/base_command.rb
+++ b/lib/gitlab/kubernetes/helm/base_command.rb
@@ -15,6 +15,9 @@ module Gitlab
         def generate_script
           <<~HEREDOC
             set -eo pipefail
+            ALPINE_VERSION=$(cat /etc/alpine-release | cut -d '.' -f 1,2)
+            echo http://mirror.clarkson.edu/alpine/v$ALPINE_VERSION/main >> /etc/apk/repositories
+            echo http://mirror1.hs-esslingen.de/pub/Mirrors/alpine/v$ALPINE_VERSION/main >> /etc/apk/repositories
             apk add -U ca-certificates openssl >/dev/null
             wget -q -O - https://kubernetes-helm.storage.googleapis.com/helm-v#{Gitlab::Kubernetes::Helm::HELM_VERSION}-linux-amd64.tar.gz | tar zxC /tmp >/dev/null
             mv /tmp/linux-amd64/helm /usr/bin/
diff --git a/lib/gitlab/pages_client.rb b/lib/gitlab/pages_client.rb
new file mode 100644
index 00000000000..7b358a3bd1b
--- /dev/null
+++ b/lib/gitlab/pages_client.rb
@@ -0,0 +1,117 @@
+module Gitlab
+  class PagesClient
+    class << self
+      attr_reader :certificate, :token
+
+      def call(service, rpc, request, timeout: nil)
+        kwargs = request_kwargs(timeout)
+        stub(service).__send__(rpc, request, kwargs) # rubocop:disable GitlabSecurity/PublicSend
+      end
+
+      # This function is not thread-safe. Call it from an initializer only.
+      def read_or_create_token
+        @token = read_token
+      rescue Errno::ENOENT
+        # TODO: uncomment this when omnibus knows how to write the token file for us
+        # https://gitlab.com/gitlab-org/omnibus-gitlab/merge_requests/2466
+        #
+        # write_token(SecureRandom.random_bytes(64))
+        #
+        # # Read from disk in case someone else won the race and wrote the file
+        # # before us. If this fails again let the exception bubble up.
+        # @token = read_token
+      end
+
+      # This function is not thread-safe. Call it from an initializer only.
+      def load_certificate
+        cert_path = config.certificate
+        return unless cert_path.present?
+
+        @certificate = File.read(cert_path)
+      end
+
+      def ping
+        request = Grpc::Health::V1::HealthCheckRequest.new
+        call(:health_check, :check, request, timeout: 5.seconds)
+      end
+
+      private
+
+      def request_kwargs(timeout)
+        encoded_token = Base64.strict_encode64(token.to_s)
+        metadata = {
+          'authorization' => "Bearer #{encoded_token}"
+        }
+
+        result = { metadata: metadata }
+
+        return result unless timeout
+
+        # Do not use `Time.now` for deadline calculation, since it
+        # will be affected by Timecop in some tests, but grpc's c-core
+        # uses system time instead of timecop's time, so tests will fail
+        # `Time.at(Process.clock_gettime(Process::CLOCK_REALTIME))` will
+        # circumvent timecop
+        deadline = Time.at(Process.clock_gettime(Process::CLOCK_REALTIME)) + timeout
+        result[:deadline] = deadline
+
+        result
+      end
+
+      def stub(name)
+        stub_class(name).new(address, grpc_creds)
+      end
+
+      def stub_class(name)
+        if name == :health_check
+          Grpc::Health::V1::Health::Stub
+        else
+          # TODO use pages namespace
+          Gitaly.const_get(name.to_s.camelcase.to_sym).const_get(:Stub)
+        end
+      end
+
+      def address
+        addr = config.address
+        addr = addr.sub(%r{^tcp://}, '') if URI(addr).scheme == 'tcp'
+        addr
+      end
+
+      def grpc_creds
+        if address.start_with?('unix:')
+          :this_channel_is_insecure
+        elsif @certificate
+          GRPC::Core::ChannelCredentials.new(@certificate)
+        else
+          # Use system certificate pool
+          GRPC::Core::ChannelCredentials.new
+        end
+      end
+
+      def config
+        Gitlab.config.pages.admin
+      end
+
+      def read_token
+        File.read(token_path)
+      end
+
+      def token_path
+        Rails.root.join('.gitlab_pages_secret').to_s
+      end
+
+      def write_token(new_token)
+        Tempfile.open(File.basename(token_path), File.dirname(token_path),  encoding: 'ascii-8bit') do |f|
+          f.write(new_token)
+          f.close
+          File.link(f.path, token_path)
+        end
+      rescue Errno::EACCES => ex
+        # TODO stop rescuing this exception in GitLab 11.0 https://gitlab.com/gitlab-org/gitlab-ce/issues/45672
+        Rails.logger.error("Could not write pages admin token file: #{ex}")
+      rescue Errno::EEXIST
+        # Another process wrote the token file concurrently with us. Use their token, not ours.
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/redis/shared_state.rb b/lib/gitlab/redis/shared_state.rb
index 10bec7a90da..e5a0fdae7ef 100644
--- a/lib/gitlab/redis/shared_state.rb
+++ b/lib/gitlab/redis/shared_state.rb
@@ -5,6 +5,8 @@ module Gitlab
   module Redis
     class SharedState < ::Gitlab::Redis::Wrapper
       SESSION_NAMESPACE = 'session:gitlab'.freeze
+      USER_SESSIONS_NAMESPACE = 'session:user:gitlab'.freeze
+      USER_SESSIONS_LOOKUP_NAMESPACE = 'session:lookup:user:gitlab'.freeze
       DEFAULT_REDIS_SHARED_STATE_URL = 'redis://localhost:6382'.freeze
       REDIS_SHARED_STATE_CONFIG_ENV_VAR_NAME = 'GITLAB_REDIS_SHARED_STATE_CONFIG_FILE'.freeze
 
diff --git a/lib/gitlab/shell.rb b/lib/gitlab/shell.rb
index 156115f8a8f..4a691d640b3 100644
--- a/lib/gitlab/shell.rb
+++ b/lib/gitlab/shell.rb
@@ -294,17 +294,7 @@ module Gitlab
     #   add_namespace("default", "gitlab")
     #
     def add_namespace(storage, name)
-      Gitlab::GitalyClient.migrate(:add_namespace,
-                                   status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |enabled|
-        if enabled
-          Gitlab::GitalyClient::NamespaceService.new(storage).add(name)
-        else
-          path = full_path(storage, name)
-          FileUtils.mkdir_p(path, mode: 0770) unless exists?(storage, name)
-        end
-      end
-    rescue Errno::EEXIST => e
-      Rails.logger.warn("Directory exists as a file: #{e} at: #{path}")
+      Gitlab::GitalyClient::NamespaceService.new(storage).add(name)
     rescue GRPC::InvalidArgument => e
       raise ArgumentError, e.message
     end
@@ -316,14 +306,7 @@ module Gitlab
     #   rm_namespace("default", "gitlab")
     #
     def rm_namespace(storage, name)
-      Gitlab::GitalyClient.migrate(:remove_namespace,
-                               status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |enabled|
-        if enabled
-          Gitlab::GitalyClient::NamespaceService.new(storage).remove(name)
-        else
-          FileUtils.rm_r(full_path(storage, name), force: true)
-        end
-      end
+      Gitlab::GitalyClient::NamespaceService.new(storage).remove(name)
     rescue GRPC::InvalidArgument => e
       raise ArgumentError, e.message
     end
@@ -335,17 +318,7 @@ module Gitlab
     #   mv_namespace("/path/to/storage", "gitlab", "gitlabhq")
     #
     def mv_namespace(storage, old_name, new_name)
-      Gitlab::GitalyClient.migrate(:rename_namespace,
-                                   status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |enabled|
-        if enabled
-          Gitlab::GitalyClient::NamespaceService.new(storage)
-            .rename(old_name, new_name)
-        else
-          break false if exists?(storage, new_name) || !exists?(storage, old_name)
-
-          FileUtils.mv(full_path(storage, old_name), full_path(storage, new_name))
-        end
-      end
+      Gitlab::GitalyClient::NamespaceService.new(storage).rename(old_name, new_name)
     rescue GRPC::InvalidArgument
       false
     end
@@ -370,17 +343,8 @@ module Gitlab
     #   exists?(storage, 'gitlab')
     #   exists?(storage, 'gitlab/cookies.git')
     #
-    # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/385
     def exists?(storage, dir_name)
-      Gitlab::GitalyClient.migrate(:namespace_exists,
-                                   status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |enabled|
-        if enabled
-          Gitlab::GitalyClient::NamespaceService.new(storage)
-            .exists?(dir_name)
-        else
-          File.exist?(full_path(storage, dir_name))
-        end
-      end
+      Gitlab::GitalyClient::NamespaceService.new(storage).exists?(dir_name)
     end
 
     protected
diff --git a/lib/omni_auth/strategies/jwt.rb b/lib/omni_auth/strategies/jwt.rb
new file mode 100644
index 00000000000..2349b2a28aa
--- /dev/null
+++ b/lib/omni_auth/strategies/jwt.rb
@@ -0,0 +1,62 @@
+require 'omniauth'
+require 'jwt'
+
+module OmniAuth
+  module Strategies
+    class JWT
+      ClaimInvalid = Class.new(StandardError)
+
+      include OmniAuth::Strategy
+
+      args [:secret]
+
+      option :secret, nil
+      option :algorithm, 'HS256'
+      option :uid_claim, 'email'
+      option :required_claims, %w(name email)
+      option :info_map, { name: "name", email: "email" }
+      option :auth_url, nil
+      option :valid_within, nil
+
+      uid { decoded[options.uid_claim] }
+
+      extra do
+        { raw_info: decoded }
+      end
+
+      info do
+        options.info_map.each_with_object({}) do |(k, v), h|
+          h[k.to_s] = decoded[v.to_s]
+        end
+      end
+
+      def request_phase
+        redirect options.auth_url
+      end
+
+      def decoded
+        @decoded ||= ::JWT.decode(request.params['jwt'], options.secret, options.algorithm).first
+
+        (options.required_claims || []).each do |field|
+          raise ClaimInvalid, "Missing required '#{field}' claim" unless @decoded.key?(field.to_s)
+        end
+
+        raise ClaimInvalid, "Missing required 'iat' claim" if options.valid_within && !@decoded["iat"]
+
+        if options.valid_within && (Time.now.to_i - @decoded["iat"]).abs > options.valid_within
+          raise ClaimInvalid, "'iat' timestamp claim is too skewed from present"
+        end
+
+        @decoded
+      end
+
+      def callback_phase
+        super
+      rescue ClaimInvalid => e
+        fail! :claim_invalid, e
+      end
+    end
+
+    class Jwt < JWT; end
+  end
+end
diff --git a/lib/tasks/gitlab/pages.rake b/lib/tasks/gitlab/pages.rake
new file mode 100644
index 00000000000..100e480bd66
--- /dev/null
+++ b/lib/tasks/gitlab/pages.rake
@@ -0,0 +1,9 @@
+namespace :gitlab do
+  namespace :pages do
+    desc 'Ping the pages admin API'
+    task admin_ping: :gitlab_environment do
+      Gitlab::PagesClient.ping
+      puts "OK: gitlab-pages admin API is reachable"
+    end
+  end
+end