Merge branch 'master' into zj-realtime-env-list

67 files changed, 1594 insertions, 508 deletions

diff --git a/lib/api/api.rb b/lib/api/api.rb
index 52cd7cbe3db..7ae2f3cad40 100644
--- a/lib/api/api.rb
+++ b/lib/api/api.rb
@@ -45,9 +45,9 @@ module API
     end
 
     before { allow_access_with_scope :api }
-    before { Gitlab::I18n.set_locale(current_user) }
+    before { Gitlab::I18n.locale = current_user&.preferred_language }
 
-    after { Gitlab::I18n.reset_locale }
+    after { Gitlab::I18n.use_default_locale }
 
     rescue_from Gitlab::Access::AccessDeniedError do
       rack_response({ 'message' => '403 Forbidden' }.to_json, 403)
@@ -94,6 +94,7 @@ module API
     mount ::API::DeployKeys
     mount ::API::Deployments
     mount ::API::Environments
+    mount ::API::Features
     mount ::API::Files
     mount ::API::Groups
     mount ::API::Internal
@@ -110,6 +111,7 @@ module API
     mount ::API::Notes
     mount ::API::NotificationSettings
     mount ::API::Pipelines
+    mount ::API::PipelineSchedules
     mount ::API::ProjectHooks
     mount ::API::Projects
     mount ::API::ProjectSnippets
diff --git a/lib/api/commit_statuses.rb b/lib/api/commit_statuses.rb
index 827a38d33da..10f2d5ef6a3 100644
--- a/lib/api/commit_statuses.rb
+++ b/lib/api/commit_statuses.rb
@@ -68,7 +68,14 @@ module API
 
         name = params[:name] || params[:context] || 'default'
 
-        pipeline = @project.ensure_pipeline(ref, commit.sha, current_user)
+        pipeline = @project.pipeline_for(ref, commit.sha)
+        unless pipeline
+          pipeline = @project.pipelines.create!(
+            source: :external,
+            sha: commit.sha,
+            ref: ref,
+            user: current_user)
+        end
 
         status = GenericCommitStatus.running_or_pending.find_or_initialize_by(
           project: @project,
diff --git a/lib/api/entities.rb b/lib/api/entities.rb
index 01cc8e8e1ca..fc8183a62c1 100644
--- a/lib/api/entities.rb
+++ b/lib/api/entities.rb
@@ -152,7 +152,10 @@ module API
       expose :web_url
       expose :request_access_enabled
       expose :full_name, :full_path
-      expose :parent_id
+
+      if ::Group.supports_nested_groups?
+        expose :parent_id
+      end
 
       expose :statistics, if: :statistics do
         with_options format_with: -> (value) { value.to_i } do
@@ -252,7 +255,9 @@ module API
 
     class RepoDiff < Grape::Entity
       expose :old_path, :new_path, :a_mode, :b_mode, :diff
-      expose :new_file, :renamed_file, :deleted_file
+      expose :new_file?, as: :new_file
+      expose :renamed_file?, as: :renamed_file
+      expose :deleted_file?, as: :deleted_file
     end
 
     class Milestone < ProjectEntity
@@ -681,6 +686,17 @@ module API
       expose :coverage
     end
 
+    class PipelineSchedule < Grape::Entity
+      expose :id
+      expose :description, :ref, :cron, :cron_timezone, :next_run_at, :active
+      expose :created_at, :updated_at
+      expose :owner, using: Entities::UserBasic
+    end
+
+    class PipelineScheduleDetails < PipelineSchedule
+      expose :last_pipeline, using: Entities::PipelineBasic
+    end
+
     class EnvironmentBasic < Grape::Entity
       expose :id, :name, :slug, :external_url
     end
@@ -737,6 +753,28 @@ module API
       expose :impersonation
     end
 
+    class FeatureGate < Grape::Entity
+      expose :key
+      expose :value
+    end
+
+    class Feature < Grape::Entity
+      expose :name
+      expose :state
+      expose :gates, using: FeatureGate do |model|
+        model.gates.map do |gate|
+          value = model.gate_values[gate.key]
+
+          # By default all gate values are populated. Only show relevant ones.
+          if (value.is_a?(Integer) && value.zero?) || (value.is_a?(Set) && value.empty?)
+            next
+          end
+
+          { key: gate.key, value: value }
+        end.compact
+      end
+    end
+
     module JobRequest
       class JobInfo < Grape::Entity
         expose :name, :stage
diff --git a/lib/api/features.rb b/lib/api/features.rb
new file mode 100644
index 00000000000..cff0ba2ddff
--- /dev/null
+++ b/lib/api/features.rb
@@ -0,0 +1,36 @@
+module API
+  class Features < Grape::API
+    before { authenticated_as_admin! }
+
+    resource :features do
+      desc 'Get a list of all features' do
+        success Entities::Feature
+      end
+      get do
+        features = Feature.all
+
+        present features, with: Entities::Feature, current_user: current_user
+      end
+
+      desc 'Set the gate value for the given feature' do
+        success Entities::Feature
+      end
+      params do
+        requires :value, type: String, desc: '`true` or `false` to enable/disable, an integer for percentage of time'
+      end
+      post ':name' do
+        feature = Feature.get(params[:name])
+
+        if %w(0 false).include?(params[:value])
+          feature.disable
+        elsif params[:value] == 'true'
+          feature.enable
+        else
+          feature.enable_percentage_of_time(params[:value].to_i)
+        end
+
+        present feature, with: Entities::Feature, current_user: current_user
+      end
+    end
+  end
+end
diff --git a/lib/api/groups.rb b/lib/api/groups.rb
index 3da7d735da8..e14a988a153 100644
--- a/lib/api/groups.rb
+++ b/lib/api/groups.rb
@@ -70,7 +70,11 @@ module API
       params do
         requires :name, type: String, desc: 'The name of the group'
         requires :path, type: String, desc: 'The path of the group'
-        optional :parent_id, type: Integer, desc: 'The parent group id for creating nested group'
+
+        if ::Group.supports_nested_groups?
+          optional :parent_id, type: Integer, desc: 'The parent group id for creating nested group'
+        end
+
         use :optional_params
       end
       post do
@@ -147,8 +151,8 @@ module API
       end
       get ":id/projects" do
         group = find_group!(params[:id])
-        projects = GroupProjectsFinder.new(group: group, current_user: current_user).execute
-        projects = filter_projects(projects)
+        projects = GroupProjectsFinder.new(group: group, current_user: current_user, params: project_finder_params).execute
+        projects = reorder_projects(projects)
         entity = params[:simple] ? Entities::BasicProjectDetails : Entities::Project
         present paginate(projects), with: entity, current_user: current_user
       end
diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb
index 226a7ddd50e..d61450f8258 100644
--- a/lib/api/helpers.rb
+++ b/lib/api/helpers.rb
@@ -256,31 +256,21 @@ module API
 
     # project helpers
 
-    def filter_projects(projects)
-      if params[:membership]
-        projects = projects.merge(current_user.authorized_projects)
-      end
-
-      if params[:owned]
-        projects = projects.merge(current_user.owned_projects)
-      end
-
-      if params[:starred]
-        projects = projects.merge(current_user.starred_projects)
-      end
-
-      if params[:search].present?
-        projects = projects.search(params[:search])
-      end
-
-      if params[:visibility].present?
-        projects = projects.search_by_visibility(params[:visibility])
-      end
-
-      projects = projects.where(archived: params[:archived])
+    def reorder_projects(projects)
       projects.reorder(params[:order_by] => params[:sort])
     end
 
+    def project_finder_params
+      finder_params = {}
+      finder_params[:owned] = true if params[:owned].present?
+      finder_params[:non_public] = true if params[:membership].present?
+      finder_params[:starred] = true if params[:starred].present?
+      finder_params[:visibility_level] = Gitlab::VisibilityLevel.level_value(params[:visibility]) if params[:visibility]
+      finder_params[:archived] = params[:archived]
+      finder_params[:search] = params[:search] if params[:search]
+      finder_params
+    end
+
     # file helpers
 
     def uploaded_file(field, uploads_path)
diff --git a/lib/api/pipeline_schedules.rb b/lib/api/pipeline_schedules.rb
new file mode 100644
index 00000000000..93d89209934
--- /dev/null
+++ b/lib/api/pipeline_schedules.rb
@@ -0,0 +1,131 @@
+module API
+  class PipelineSchedules < Grape::API
+    include PaginationParams
+
+    before { authenticate! }
+
+    params do
+      requires :id, type: String, desc: 'The ID of a project'
+    end
+    resource :projects, requirements: { id: %r{[^/]+} } do
+      desc 'Get all pipeline schedules' do
+        success Entities::PipelineSchedule
+      end
+      params do
+        use :pagination
+        optional :scope,    type: String, values: %w[active inactive],
+                            desc: 'The scope of pipeline schedules'
+      end
+      get ':id/pipeline_schedules' do
+        authorize! :read_pipeline_schedule, user_project
+
+        schedules = PipelineSchedulesFinder.new(user_project).execute(scope: params[:scope])
+          .preload([:owner, :last_pipeline])
+        present paginate(schedules), with: Entities::PipelineSchedule
+      end
+
+      desc 'Get a single pipeline schedule' do
+        success Entities::PipelineScheduleDetails
+      end
+      params do
+        requires :pipeline_schedule_id, type: Integer,  desc: 'The pipeline schedule id'
+      end
+      get ':id/pipeline_schedules/:pipeline_schedule_id' do
+        authorize! :read_pipeline_schedule, user_project
+
+        not_found!('PipelineSchedule') unless pipeline_schedule
+
+        present pipeline_schedule, with: Entities::PipelineScheduleDetails
+      end
+
+      desc 'Create a new pipeline schedule' do
+        success Entities::PipelineScheduleDetails
+      end
+      params do
+        requires :description, type: String, desc: 'The description of pipeline schedule'
+        requires :ref, type: String, desc: 'The branch/tag name will be triggered'
+        requires :cron, type: String, desc: 'The cron'
+        optional :cron_timezone, type: String, default: 'UTC', desc: 'The timezone'
+        optional :active, type: Boolean, default: true, desc: 'The activation of pipeline schedule'
+      end
+      post ':id/pipeline_schedules' do
+        authorize! :create_pipeline_schedule, user_project
+
+        pipeline_schedule = Ci::CreatePipelineScheduleService
+          .new(user_project, current_user, declared_params(include_missing: false))
+          .execute
+
+        if pipeline_schedule.persisted?
+          present pipeline_schedule, with: Entities::PipelineScheduleDetails
+        else
+          render_validation_error!(pipeline_schedule)
+        end
+      end
+
+      desc 'Edit a pipeline schedule' do
+        success Entities::PipelineScheduleDetails
+      end
+      params do
+        requires :pipeline_schedule_id, type: Integer,  desc: 'The pipeline schedule id'
+        optional :description, type: String, desc: 'The description of pipeline schedule'
+        optional :ref, type: String, desc: 'The branch/tag name will be triggered'
+        optional :cron, type: String, desc: 'The cron'
+        optional :cron_timezone, type: String, desc: 'The timezone'
+        optional :active, type: Boolean, desc: 'The activation of pipeline schedule'
+      end
+      put ':id/pipeline_schedules/:pipeline_schedule_id' do
+        authorize! :update_pipeline_schedule, user_project
+
+        not_found!('PipelineSchedule') unless pipeline_schedule
+
+        if pipeline_schedule.update(declared_params(include_missing: false))
+          present pipeline_schedule, with: Entities::PipelineScheduleDetails
+        else
+          render_validation_error!(pipeline_schedule)
+        end
+      end
+
+      desc 'Take ownership of a pipeline schedule' do
+        success Entities::PipelineScheduleDetails
+      end
+      params do
+        requires :pipeline_schedule_id, type: Integer,  desc: 'The pipeline schedule id'
+      end
+      post ':id/pipeline_schedules/:pipeline_schedule_id/take_ownership' do
+        authorize! :update_pipeline_schedule, user_project
+
+        not_found!('PipelineSchedule') unless pipeline_schedule
+
+        if pipeline_schedule.own!(current_user)
+          present pipeline_schedule, with: Entities::PipelineScheduleDetails
+        else
+          render_validation_error!(pipeline_schedule)
+        end
+      end
+
+      desc 'Delete a pipeline schedule' do
+        success Entities::PipelineScheduleDetails
+      end
+      params do
+        requires :pipeline_schedule_id, type: Integer,  desc: 'The pipeline schedule id'
+      end
+      delete ':id/pipeline_schedules/:pipeline_schedule_id' do
+        authorize! :admin_pipeline_schedule, user_project
+
+        not_found!('PipelineSchedule') unless pipeline_schedule
+
+        status :accepted
+        present pipeline_schedule.destroy, with: Entities::PipelineScheduleDetails
+      end
+    end
+
+    helpers do
+      def pipeline_schedule
+        @pipeline_schedule ||=
+          user_project.pipeline_schedules
+                      .preload(:owner, :last_pipeline)
+                      .find_by(id: params.delete(:pipeline_schedule_id))
+      end
+    end
+  end
+end
diff --git a/lib/api/pipelines.rb b/lib/api/pipelines.rb
index 9117704aa46..e505cae3992 100644
--- a/lib/api/pipelines.rb
+++ b/lib/api/pipelines.rb
@@ -47,7 +47,7 @@ module API
         new_pipeline = Ci::CreatePipelineService.new(user_project,
                                                      current_user,
                                                      declared_params(include_missing: false))
-                           .execute(ignore_skip_ci: true, save_on_errors: false)
+                           .execute(:api, ignore_skip_ci: true, save_on_errors: false)
         if new_pipeline.persisted?
           present new_pipeline, with: Entities::Pipeline
         else
diff --git a/lib/api/projects.rb b/lib/api/projects.rb
index ed5004e8d1a..d00d4fe1737 100644
--- a/lib/api/projects.rb
+++ b/lib/api/projects.rb
@@ -21,6 +21,7 @@ module API
         optional :request_access_enabled, type: Boolean, desc: 'Allow users to request member access'
         optional :only_allow_merge_if_pipeline_succeeds, type: Boolean, desc: 'Only allow to merge if builds succeed'
         optional :only_allow_merge_if_all_discussions_are_resolved, type: Boolean, desc: 'Only allow to merge if all discussions are resolved'
+        optional :tag_list, type: Array[String], desc: 'The list of tags for a project'
       end
 
       params :optional_params do
@@ -58,6 +59,8 @@ module API
           optional :owned, type: Boolean, default: false, desc: 'Limit by owned by authenticated user'
           optional :starred, type: Boolean, default: false, desc: 'Limit by starred status'
           optional :membership, type: Boolean, default: false, desc: 'Limit by projects that the current user is a member of'
+          optional :with_issues_enabled, type: Boolean, default: false, desc: 'Limit by enabled issues feature'
+          optional :with_merge_requests_enabled, type: Boolean, default: false, desc: 'Limit by enabled merge requests feature'
         end
 
         params :create_params do
@@ -65,16 +68,19 @@ module API
           optional :import_url, type: String, desc: 'URL from which the project is imported'
         end
 
-        def present_projects(projects, options = {})
+        def present_projects(options = {})
+          projects = ProjectsFinder.new(current_user: current_user, params: project_finder_params).execute
+          projects = reorder_projects(projects)
+          projects = projects.with_statistics if params[:statistics]
+          projects = projects.with_issues_enabled if params[:with_issues_enabled]
+          projects = projects.with_merge_requests_enabled if params[:with_merge_requests_enabled]
+
           options = options.reverse_merge(
-            with: Entities::Project,
-            current_user: current_user,
-            simple: params[:simple]
+            with: current_user ? Entities::ProjectWithAccess : Entities::BasicProjectDetails,
+            statistics: params[:statistics],
+            current_user: current_user
           )
-
-          projects = filter_projects(projects)
-          projects = projects.with_statistics if options[:statistics]
-          options[:with] = Entities::BasicProjectDetails if options[:simple]
+          options[:with] = Entities::BasicProjectDetails if params[:simple]
 
           present paginate(projects), options
         end
@@ -88,8 +94,7 @@ module API
         use :statistics_params
       end
       get do
-        entity = current_user ? Entities::ProjectWithAccess : Entities::BasicProjectDetails
-        present_projects ProjectsFinder.new(current_user: current_user).execute, with: entity, statistics: params[:statistics]
+        present_projects
       end
 
       desc 'Create new project' do
@@ -225,6 +230,7 @@ module API
             :request_access_enabled,
             :shared_runners_enabled,
             :snippets_enabled,
+            :tag_list,
             :visibility,
             :wiki_enabled
           ]
diff --git a/lib/api/repositories.rb b/lib/api/repositories.rb
index 8f16e532ecb..14d2bff9cb5 100644
--- a/lib/api/repositories.rb
+++ b/lib/api/repositories.rb
@@ -85,7 +85,7 @@ module API
         optional :sha, type: String, desc: 'The commit sha of the archive to be downloaded'
         optional :format, type: String, desc: 'The archive format'
       end
-      get ':id/repository/archive', requirements: { format: Gitlab::Regex.archive_formats_regex } do
+      get ':id/repository/archive', requirements: { format: Gitlab::PathRegex.archive_formats_regex } do
         begin
           send_git_archive user_project.repository, ref: params[:sha], format: params[:format]
         rescue
diff --git a/lib/api/services.rb b/lib/api/services.rb
index cb07df9e249..47bd9940f77 100644
--- a/lib/api/services.rb
+++ b/lib/api/services.rb
@@ -304,7 +304,13 @@ module API
           required: true,
           name: :url,
           type: String,
-          desc: 'The URL to the JIRA project which is being linked to this GitLab project, e.g., https://jira.example.com'
+          desc: 'The base URL to the JIRA instance web interface which is being linked to this GitLab project. E.g., https://jira.example.com'
+        },
+        {
+          required: false,
+          name: :api_url,
+          type: String,
+          desc: 'The base URL to the JIRA instance API. Web URL value will be used if not set. E.g., https://jira-api.example.com'
         },
         {
           required: true,
diff --git a/lib/api/v3/entities.rb b/lib/api/v3/entities.rb
index 332f233bf5e..2e1b243c2db 100644
--- a/lib/api/v3/entities.rb
+++ b/lib/api/v3/entities.rb
@@ -137,7 +137,10 @@ module API
         expose :web_url
         expose :request_access_enabled
         expose :full_name, :full_path
-        expose :parent_id
+
+        if ::Group.supports_nested_groups?
+          expose :parent_id
+        end
 
         expose :statistics, if: :statistics do
           with_options format_with: -> (value) { value.to_i } do
diff --git a/lib/api/v3/groups.rb b/lib/api/v3/groups.rb
index 6187445fc8d..2c52d21fa1c 100644
--- a/lib/api/v3/groups.rb
+++ b/lib/api/v3/groups.rb
@@ -74,7 +74,11 @@ module API
         params do
           requires :name, type: String, desc: 'The name of the group'
           requires :path, type: String, desc: 'The path of the group'
-          optional :parent_id, type: Integer, desc: 'The parent group id for creating nested group'
+
+          if ::Group.supports_nested_groups?
+            optional :parent_id, type: Integer, desc: 'The parent group id for creating nested group'
+          end
+
           use :optional_params
         end
         post do
diff --git a/lib/api/v3/helpers.rb b/lib/api/v3/helpers.rb
index 0f234d4cdad..d9e76560d03 100644
--- a/lib/api/v3/helpers.rb
+++ b/lib/api/v3/helpers.rb
@@ -14,6 +14,33 @@ module API
         authorize! access_level, merge_request
         merge_request
       end
+
+      # project helpers
+
+      def filter_projects(projects)
+        if params[:membership]
+          projects = projects.merge(current_user.authorized_projects)
+        end
+
+        if params[:owned]
+          projects = projects.merge(current_user.owned_projects)
+        end
+
+        if params[:starred]
+          projects = projects.merge(current_user.starred_projects)
+        end
+
+        if params[:search].present?
+          projects = projects.search(params[:search])
+        end
+
+        if params[:visibility].present?
+          projects = projects.where(visibility_level: Gitlab::VisibilityLevel.level_value(params[:visibility]))
+        end
+
+        projects = projects.where(archived: params[:archived])
+        projects.reorder(params[:order_by] => params[:sort])
+      end
     end
   end
 end
diff --git a/lib/api/v3/projects.rb b/lib/api/v3/projects.rb
index 164612cb8dd..896c00b88e7 100644
--- a/lib/api/v3/projects.rb
+++ b/lib/api/v3/projects.rb
@@ -147,7 +147,7 @@ module API
         get '/starred' do
           authenticate!
 
-          present_projects current_user.viewable_starred_projects
+          present_projects ProjectsFinder.new(current_user: current_user, params: { starred: true }).execute
         end
 
         desc 'Get all projects for admin user' do
diff --git a/lib/api/v3/repositories.rb b/lib/api/v3/repositories.rb
index e4d14bc8168..0eaa0de2eef 100644
--- a/lib/api/v3/repositories.rb
+++ b/lib/api/v3/repositories.rb
@@ -72,7 +72,7 @@ module API
           optional :sha, type: String, desc: 'The commit sha of the archive to be downloaded'
           optional :format, type: String, desc: 'The archive format'
         end
-        get ':id/repository/archive', requirements: { format: Gitlab::Regex.archive_formats_regex } do
+        get ':id/repository/archive', requirements: { format: Gitlab::PathRegex.archive_formats_regex } do
           begin
             send_git_archive user_project.repository, ref: params[:sha], format: params[:format]
           rescue
diff --git a/lib/banzai/reference_parser/base_parser.rb b/lib/banzai/reference_parser/base_parser.rb
index c2503fa2adc..d99a3bfa625 100644
--- a/lib/banzai/reference_parser/base_parser.rb
+++ b/lib/banzai/reference_parser/base_parser.rb
@@ -163,14 +163,15 @@ module Banzai
       # been queried the object is returned from the cache.
       def collection_objects_for_ids(collection, ids)
         if RequestStore.active?
+          ids = ids.map(&:to_i)
           cache = collection_cache[collection_cache_key(collection)]
-          to_query = ids.map(&:to_i) - cache.keys
+          to_query = ids - cache.keys
 
           unless to_query.empty?
             collection.where(id: to_query).each { |row| cache[row.id] = row }
           end
 
-          cache.values
+          cache.values_at(*ids)
         else
           collection.where(id: ids)
         end
diff --git a/lib/constraints/group_url_constrainer.rb b/lib/constraints/group_url_constrainer.rb
index 0ea2f97352d..6fc1d56d7a0 100644
--- a/lib/constraints/group_url_constrainer.rb
+++ b/lib/constraints/group_url_constrainer.rb
@@ -1,9 +1,9 @@
 class GroupUrlConstrainer
   def matches?(request)
-    id = request.params[:group_id] || request.params[:id]
+    full_path = request.params[:group_id] || request.params[:id]
 
-    return false unless DynamicPathValidator.valid_namespace_path?(id)
+    return false unless DynamicPathValidator.valid_group_path?(full_path)
 
-    Group.find_by_full_path(id, follow_redirects: request.get?).present?
+    Group.find_by_full_path(full_path, follow_redirects: request.get?).present?
   end
 end
diff --git a/lib/constraints/project_url_constrainer.rb b/lib/constraints/project_url_constrainer.rb
index 4444a1abee3..4c0aee6c48f 100644
--- a/lib/constraints/project_url_constrainer.rb
+++ b/lib/constraints/project_url_constrainer.rb
@@ -2,7 +2,7 @@ class ProjectUrlConstrainer
   def matches?(request)
     namespace_path = request.params[:namespace_id]
     project_path = request.params[:project_id] || request.params[:id]
-    full_path = namespace_path + '/' + project_path
+    full_path = [namespace_path, project_path].join('/')
 
     return false unless DynamicPathValidator.valid_project_path?(full_path)
 
diff --git a/lib/constraints/user_url_constrainer.rb b/lib/constraints/user_url_constrainer.rb
index 28159dc0dec..d16ae7f3f40 100644
--- a/lib/constraints/user_url_constrainer.rb
+++ b/lib/constraints/user_url_constrainer.rb
@@ -1,5 +1,9 @@
 class UserUrlConstrainer
   def matches?(request)
-    User.find_by_full_path(request.params[:username], follow_redirects: request.get?).present?
+    full_path = request.params[:username]
+
+    return false unless DynamicPathValidator.valid_user_path?(full_path)
+
+    User.find_by_full_path(full_path, follow_redirects: request.get?).present?
   end
 end
diff --git a/lib/feature.rb b/lib/feature.rb
new file mode 100644
index 00000000000..2e2b343f82c
--- /dev/null
+++ b/lib/feature.rb
@@ -0,0 +1,41 @@
+require 'flipper/adapters/active_record'
+
+class Feature
+  # Classes to override flipper table names
+  class FlipperFeature < Flipper::Adapters::ActiveRecord::Feature
+    # Using `self.table_name` won't work. ActiveRecord bug?
+    superclass.table_name = 'features'
+  end
+
+  class FlipperGate < Flipper::Adapters::ActiveRecord::Gate
+    superclass.table_name = 'feature_gates'
+  end
+
+  class << self
+    def all
+      flipper.features.to_a
+    end
+
+    def get(key)
+      flipper.feature(key)
+    end
+
+    def persisted?(feature)
+      # Flipper creates on-memory features when asked for a not-yet-created one.
+      # If we want to check if a feature has been actually set, we look for it
+      # on the persisted features list.
+      all.map(&:name).include?(feature.name)
+    end
+
+    private
+
+    def flipper
+      @flipper ||= begin
+        adapter = Flipper::Adapters::ActiveRecord.new(
+          feature_class: FlipperFeature, gate_class: FlipperGate)
+
+        Flipper.new(adapter)
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/status/build/cancelable.rb b/lib/gitlab/ci/status/build/cancelable.rb
index 57b533bad99..439ef0ce015 100644
--- a/lib/gitlab/ci/status/build/cancelable.rb
+++ b/lib/gitlab/ci/status/build/cancelable.rb
@@ -12,7 +12,7 @@ module Gitlab
           end
 
           def action_path
-            cancel_namespace_project_build_path(subject.project.namespace,
+            cancel_namespace_project_job_path(subject.project.namespace,
                                                 subject.project,
                                                 subject)
           end
diff --git a/lib/gitlab/ci/status/build/common.rb b/lib/gitlab/ci/status/build/common.rb
index 3fec2c5d4db..b173c23fba4 100644
--- a/lib/gitlab/ci/status/build/common.rb
+++ b/lib/gitlab/ci/status/build/common.rb
@@ -8,7 +8,7 @@ module Gitlab
           end
 
           def details_path
-            namespace_project_build_path(subject.project.namespace,
+            namespace_project_job_path(subject.project.namespace,
                                          subject.project,
                                          subject)
           end
diff --git a/lib/gitlab/ci/status/build/play.rb b/lib/gitlab/ci/status/build/play.rb
index c6139f1b716..e80f3263794 100644
--- a/lib/gitlab/ci/status/build/play.rb
+++ b/lib/gitlab/ci/status/build/play.rb
@@ -20,7 +20,7 @@ module Gitlab
           end
 
           def action_path
-            play_namespace_project_build_path(subject.project.namespace,
+            play_namespace_project_job_path(subject.project.namespace,
                                               subject.project,
                                               subject)
           end
diff --git a/lib/gitlab/ci/status/build/retryable.rb b/lib/gitlab/ci/status/build/retryable.rb
index 505f80848b2..56303e4cb17 100644
--- a/lib/gitlab/ci/status/build/retryable.rb
+++ b/lib/gitlab/ci/status/build/retryable.rb
@@ -16,7 +16,7 @@ module Gitlab
           end
 
           def action_path
-            retry_namespace_project_build_path(subject.project.namespace,
+            retry_namespace_project_job_path(subject.project.namespace,
                                                subject.project,
                                                subject)
           end
diff --git a/lib/gitlab/ci/status/build/stop.rb b/lib/gitlab/ci/status/build/stop.rb
index 0b5199e5483..2778d6f3b52 100644
--- a/lib/gitlab/ci/status/build/stop.rb
+++ b/lib/gitlab/ci/status/build/stop.rb
@@ -20,7 +20,7 @@ module Gitlab
           end
 
           def action_path
-            play_namespace_project_build_path(subject.project.namespace,
+            play_namespace_project_job_path(subject.project.namespace,
                                               subject.project,
                                               subject)
           end
diff --git a/lib/gitlab/ci/trace/stream.rb b/lib/gitlab/ci/trace/stream.rb
index fa462cbe095..c4c0623df6c 100644
--- a/lib/gitlab/ci/trace/stream.rb
+++ b/lib/gitlab/ci/trace/stream.rb
@@ -73,7 +73,7 @@ module Gitlab
 
           match = ""
 
-          stream.each_line do |line|
+          reverse_line do |line|
             matches = line.scan(regex)
             next unless matches.is_a?(Array)
             next if matches.empty?
@@ -86,34 +86,39 @@ module Gitlab
           nil
         rescue
           # if bad regex or something goes wrong we dont want to interrupt transition
-          # so we just silentrly ignore error for now
+          # so we just silently ignore error for now
         end
 
         private
 
-        def read_last_lines(last_lines)
-          chunks = []
-          pos = lines = 0
-          max = stream.size
-
-          # We want an extra line to make sure fist line has full contents
-          while lines <= last_lines && pos < max
-            pos += BUFFER_SIZE
-
-            buf =
-              if pos <= max
-                stream.seek(-pos, IO::SEEK_END)
-                stream.read(BUFFER_SIZE)
-              else # Reached the head, read only left
-                stream.seek(0)
-                stream.read(BUFFER_SIZE - (pos - max))
-              end
-
-            lines += buf.count("\n")
-            chunks.unshift(buf)
+        def read_last_lines(limit)
+          to_enum(:reverse_line).first(limit).reverse.join
+        end
+
+        def reverse_line
+          stream.seek(0, IO::SEEK_END)
+          debris = ''
+
+          until (buf = read_backward(BUFFER_SIZE)).empty?
+            buf += debris
+            debris, *lines = buf.each_line.to_a
+            lines.reverse_each do |line|
+              yield(line.force_encoding('UTF-8'))
+            end
           end
 
-          chunks.join.lines.last(last_lines).join
+          yield(debris.force_encoding('UTF-8')) unless debris.empty?
+        end
+
+        def read_backward(length)
+          cur_offset = stream.tell
+          start = cur_offset - length
+          start = 0 if start < 0
+
+          stream.seek(start, IO::SEEK_SET)
+          stream.read(cur_offset - start).tap do
+            stream.seek(start, IO::SEEK_SET)
+          end
         end
       end
     end
diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb
index e76c9abbe04..a412bb6dbd2 100644
--- a/lib/gitlab/database/migration_helpers.rb
+++ b/lib/gitlab/database/migration_helpers.rb
@@ -42,7 +42,7 @@ module Gitlab
             'in the body of your migration class'
         end
 
-        if Database.postgresql?
+        if supports_drop_index_concurrently?
           options = options.merge({ algorithm: :concurrently })
           disable_statement_timeout
         end
@@ -50,6 +50,39 @@ module Gitlab
         remove_index(table_name, options.merge({ column: column_name }))
       end
 
+      # Removes an existing index, concurrently when supported
+      #
+      # On PostgreSQL this method removes an index concurrently.
+      #
+      # Example:
+      #
+      #     remove_concurrent_index :users, "index_X_by_Y"
+      #
+      # See Rails' `remove_index` for more info on the available arguments.
+      def remove_concurrent_index_by_name(table_name, index_name, options = {})
+        if transaction_open?
+          raise 'remove_concurrent_index_by_name can not be run inside a transaction, ' \
+            'you can disable transactions by calling disable_ddl_transaction! ' \
+            'in the body of your migration class'
+        end
+
+        if supports_drop_index_concurrently?
+          options = options.merge({ algorithm: :concurrently })
+          disable_statement_timeout
+        end
+
+        remove_index(table_name, options.merge({ name: index_name }))
+      end
+
+      # Only available on Postgresql >= 9.2
+      def supports_drop_index_concurrently?
+        return false unless Database.postgresql?
+
+        version = select_one("SELECT current_setting('server_version_num') AS v")['v'].to_i
+
+        version >= 90200
+      end
+
       # Adds a foreign key with only minimal locking on the tables involved.
       #
       # This method only requires minimal locking when using PostgreSQL. When
diff --git a/lib/gitlab/dependency_linker.rb b/lib/gitlab/dependency_linker.rb
index c45ae8feb2c..3192bf6f667 100644
--- a/lib/gitlab/dependency_linker.rb
+++ b/lib/gitlab/dependency_linker.rb
@@ -1,7 +1,16 @@
 module Gitlab
   module DependencyLinker
     LINKERS = [
-      GemfileLinker
+      GemfileLinker,
+      GemspecLinker,
+      PackageJsonLinker,
+      ComposerJsonLinker,
+      PodfileLinker,
+      PodspecLinker,
+      PodspecJsonLinker,
+      CartfileLinker,
+      GodepsJsonLinker,
+      RequirementsTxtLinker
     ].freeze
 
     def self.linker(blob_name)
diff --git a/lib/gitlab/dependency_linker/base_linker.rb b/lib/gitlab/dependency_linker/base_linker.rb
index 5f4027e7e81..7bbd154eb03 100644
--- a/lib/gitlab/dependency_linker/base_linker.rb
+++ b/lib/gitlab/dependency_linker/base_linker.rb
@@ -1,6 +1,9 @@
 module Gitlab
   module DependencyLinker
     class BaseLinker
+      URL_REGEX = %r{https?://[^'" ]+}.freeze
+      REPO_REGEX = %r{[^/'" ]+/[^/'" ]+}.freeze
+
       class_attribute :file_type
 
       def self.support?(blob_name)
@@ -26,59 +29,20 @@ module Gitlab
 
       private
 
-      def package_url(name)
-        raise NotImplementedError
-      end
-
       def link_dependencies
         raise NotImplementedError
       end
 
-      def package_link(name, url = package_url(name))
-        return name unless url
-
-        %{<a href="#{ERB::Util.html_escape_once(url)}" rel="noopener noreferrer" target="_blank">#{ERB::Util.html_escape_once(name)}</a>}
+      def license_url(name)
+        Licensee::License.find(name)&.url
       end
 
-      # Links package names in a method call or assignment string argument.
-      #
-      # Example:
-      #   link_method_call("gem")
-      #   # Will link `package` in `gem "package"`, `gem("package")` and `gem = "package"`
-      #
-      #   link_method_call("gem", "specific_package")
-      #   # Will link `specific_package` in `gem "specific_package"`
-      #
-      #   link_method_call("github", /[^\/]+\/[^\/]+/)
-      #   # Will link `user/repo` in `github "user/repo"`, but not `github "package"`
-      #
-      #   link_method_call(%w[add_dependency add_development_dependency])
-      #   # Will link `spec.add_dependency "package"` and `spec.add_development_dependency "package"`
-      #
-      #   link_method_call("name")
-      #   # Will link `package` in `self.name = "package"`
-      def link_method_call(method_names, value = nil, &url_proc)
-        value =
-          case value
-          when String
-            Regexp.escape(value)
-          when nil
-            /[^'"]+/
-          else
-            value
-          end
-
-        method_names = Array(method_names).map { |name| Regexp.escape(name) }
-
-        regex = %r{
-          #{Regexp.union(method_names)} # Method name
-          \s*                           # Whitespace
-          [(=]?                         # Opening brace or equals sign
-          \s*                           # Whitespace
-          ['"](?<name>#{value})['"]     # Package name in quotes
-        }x
+      def github_url(name)
+        "https://github.com/#{name}"
+      end
 
-        link_regex(regex, &url_proc)
+      def link_tag(name, url)
+        %{<a href="#{ERB::Util.html_escape_once(url)}" rel="nofollow noreferrer noopener" target="_blank">#{ERB::Util.html_escape_once(name)}</a>}
       end
 
       # Links package names based on regex.
@@ -86,13 +50,13 @@ module Gitlab
       # Example:
       #   link_regex(/(github:|:github =>)\s*['"](?<name>[^'"]+)['"]/)
       #   # Will link `user/repo` in `github: "user/repo"` or `:github => "user/repo"`
-      def link_regex(regex)
+      def link_regex(regex, &url_proc)
         highlighted_lines.map!.with_index do |rich_line, i|
           marker = StringRegexMarker.new(plain_lines[i], rich_line.html_safe)
 
           marker.mark(regex, group: :name) do |text, left:, right:|
-            url = block_given? ? yield(text) : package_url(text)
-            package_link(text, url)
+            url = yield(text)
+            url ? link_tag(text, url) : text
           end
         end
       end
@@ -104,6 +68,19 @@ module Gitlab
       def highlighted_lines
         @highlighted_lines ||= highlighted_text.lines
       end
+
+      def regexp_for_value(value, default: /[^'" ]+/)
+        case value
+        when Array
+          Regexp.union(value.map { |v| regexp_for_value(v, default: default) })
+        when String
+          Regexp.escape(value)
+        when Regexp
+          value
+        else
+          default
+        end
+      end
     end
   end
 end
diff --git a/lib/gitlab/dependency_linker/cartfile_linker.rb b/lib/gitlab/dependency_linker/cartfile_linker.rb
new file mode 100644
index 00000000000..4f69f2c4ab2
--- /dev/null
+++ b/lib/gitlab/dependency_linker/cartfile_linker.rb
@@ -0,0 +1,14 @@
+module Gitlab
+  module DependencyLinker
+    class CartfileLinker < MethodLinker
+      self.file_type = :cartfile
+
+      private
+
+      def link_dependencies
+        link_method_call('github', REPO_REGEX, &method(:github_url))
+        link_method_call(%w[github git binary], URL_REGEX, &:itself)
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/dependency_linker/cocoapods.rb b/lib/gitlab/dependency_linker/cocoapods.rb
new file mode 100644
index 00000000000..2fbde7da1b4
--- /dev/null
+++ b/lib/gitlab/dependency_linker/cocoapods.rb
@@ -0,0 +1,10 @@
+module Gitlab
+  module DependencyLinker
+    module Cocoapods
+      def package_url(name)
+        package = name.split("/", 2).first
+        "https://cocoapods.org/pods/#{package}"
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/dependency_linker/composer_json_linker.rb b/lib/gitlab/dependency_linker/composer_json_linker.rb
new file mode 100644
index 00000000000..0245bf4077a
--- /dev/null
+++ b/lib/gitlab/dependency_linker/composer_json_linker.rb
@@ -0,0 +1,18 @@
+module Gitlab
+  module DependencyLinker
+    class ComposerJsonLinker < PackageJsonLinker
+      self.file_type = :composer_json
+
+      private
+
+      def link_packages
+        link_packages_at_key("require", &method(:package_url))
+        link_packages_at_key("require-dev", &method(:package_url))
+      end
+
+      def package_url(name)
+        "https://packagist.org/packages/#{name}" if name =~ %r{\A#{REPO_REGEX}\z}
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/dependency_linker/gemfile_linker.rb b/lib/gitlab/dependency_linker/gemfile_linker.rb
index 9b82e126528..d034ea67387 100644
--- a/lib/gitlab/dependency_linker/gemfile_linker.rb
+++ b/lib/gitlab/dependency_linker/gemfile_linker.rb
@@ -1,28 +1,31 @@
 module Gitlab
   module DependencyLinker
-    class GemfileLinker < BaseLinker
+    class GemfileLinker < MethodLinker
       self.file_type = :gemfile
 
       private
 
       def link_dependencies
-        # Link `gem "package_name"` to https://rubygems.org/gems/package_name
-        link_method_call("gem")
+        link_urls
+        link_packages
+      end
 
+      def link_urls
         # Link `github: "user/repo"` to https://github.com/user/repo
-        link_regex(/(github:|:github\s*=>)\s*['"](?<name>[^'"]+)['"]/) do |name|
-          "https://github.com/#{name}"
-        end
+        link_regex(/(github:|:github\s*=>)\s*['"](?<name>[^'"]+)['"]/, &method(:github_url))
 
         # Link `git: "https://gitlab.example.com/user/repo"` to https://gitlab.example.com/user/repo
-        link_regex(%r{(git:|:git\s*=>)\s*['"](?<name>https?://[^'"]+)['"]}) { |url| url }
+        link_regex(%r{(git:|:git\s*=>)\s*['"](?<name>#{URL_REGEX})['"]}, &:itself)
 
         # Link `source "https://rubygems.org"` to https://rubygems.org
-        link_method_call("source", %r{https?://[^'"]+}) { |url| url }
+        link_method_call('source', URL_REGEX, &:itself)
       end
 
-      def package_url(name)
-        "https://rubygems.org/gems/#{name}"
+      def link_packages
+        # Link `gem "package_name"` to https://rubygems.org/gems/package_name
+        link_method_call('gem') do |name|
+          "https://rubygems.org/gems/#{name}"
+        end
       end
     end
   end
diff --git a/lib/gitlab/dependency_linker/gemspec_linker.rb b/lib/gitlab/dependency_linker/gemspec_linker.rb
new file mode 100644
index 00000000000..f1783ee2ab4
--- /dev/null
+++ b/lib/gitlab/dependency_linker/gemspec_linker.rb
@@ -0,0 +1,18 @@
+module Gitlab
+  module DependencyLinker
+    class GemspecLinker < MethodLinker
+      self.file_type = :gemspec
+
+      private
+
+      def link_dependencies
+        link_method_call('homepage', URL_REGEX, &:itself)
+        link_method_call('license', &method(:license_url))
+
+        link_method_call(%w[name add_dependency add_runtime_dependency add_development_dependency]) do |name|
+          "https://rubygems.org/gems/#{name}"
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/dependency_linker/godeps_json_linker.rb b/lib/gitlab/dependency_linker/godeps_json_linker.rb
new file mode 100644
index 00000000000..fe091baee6d
--- /dev/null
+++ b/lib/gitlab/dependency_linker/godeps_json_linker.rb
@@ -0,0 +1,26 @@
+module Gitlab
+  module DependencyLinker
+    class GodepsJsonLinker < JsonLinker
+      NESTED_REPO_REGEX = %r{([^/]+/)+[^/]+?}.freeze
+
+      self.file_type = :godeps_json
+
+      private
+
+      def link_dependencies
+        link_json('ImportPath') do |path|
+          case path
+          when %r{\A(?<repo>gitlab\.com/#{NESTED_REPO_REGEX})\.git/(?<path>.+)\z},
+              %r{\A(?<repo>git(lab|hub)\.com/#{REPO_REGEX})/(?<path>.+)\z}
+
+            "https://#{$~[:repo]}/tree/master/#{$~[:path]}"
+          when /\Agolang\.org/
+            "https://godoc.org/#{path}"
+          else
+            "https://#{path}"
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/dependency_linker/json_linker.rb b/lib/gitlab/dependency_linker/json_linker.rb
new file mode 100644
index 00000000000..a8ef25233d8
--- /dev/null
+++ b/lib/gitlab/dependency_linker/json_linker.rb
@@ -0,0 +1,44 @@
+module Gitlab
+  module DependencyLinker
+    class JsonLinker < BaseLinker
+      def link
+        return highlighted_text unless json
+
+        super
+      end
+
+      private
+
+      # Links package names in a JSON key or values.
+      #
+      # Example:
+      #   link_json('name')
+      #   # Will link `package` in `"name": "package"`
+      #
+      #   link_json('name', 'specific_package')
+      #   # Will link `specific_package` in `"name": "specific_package"`
+      #
+      #   link_json('name', /[^\/]+\/[^\/]+/)
+      #   # Will link `user/repo` in `"name": "user/repo"`, but not `"name": "package"`
+      #
+      #   link_json('specific_package', '1.0.1', link: :key)
+      #   # Will link `specific_package` in `"specific_package": "1.0.1"`
+      def link_json(key, value = nil, link: :value, &url_proc)
+        key = regexp_for_value(key, default: /[^" ]+/)
+        value = regexp_for_value(value, default: /[^" ]+/)
+
+        if link == :value
+          value = /(?<name>#{value})/
+        else
+          key = /(?<name>#{key})/
+        end
+
+        link_regex(/"#{key}":\s*"#{value}"/, &url_proc)
+      end
+
+      def json
+        @json ||= JSON.parse(plain_text) rescue nil
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/dependency_linker/method_linker.rb b/lib/gitlab/dependency_linker/method_linker.rb
new file mode 100644
index 00000000000..0ffa2a83c93
--- /dev/null
+++ b/lib/gitlab/dependency_linker/method_linker.rb
@@ -0,0 +1,39 @@
+module Gitlab
+  module DependencyLinker
+    class MethodLinker < BaseLinker
+      private
+
+      # Links package names in a method call or assignment string argument.
+      #
+      # Example:
+      #   link_method_call('gem')
+      #   # Will link `package` in `gem "package"`, `gem("package")` and `gem = "package"`
+      #
+      #   link_method_call('gem', 'specific_package')
+      #   # Will link `specific_package` in `gem "specific_package"`
+      #
+      #   link_method_call('github', /[^\/"]+\/[^\/"]+/)
+      #   # Will link `user/repo` in `github "user/repo"`, but not `github "package"`
+      #
+      #   link_method_call(%w[add_dependency add_development_dependency])
+      #   # Will link `spec.add_dependency "package"` and `spec.add_development_dependency "package"`
+      #
+      #   link_method_call('name')
+      #   # Will link `package` in `self.name = "package"`
+      def link_method_call(method_name, value = nil, &url_proc)
+        method_name = regexp_for_value(method_name)
+        value = regexp_for_value(value)
+
+        regex = %r{
+          #{method_name}            # Method name
+          \s*                       # Whitespace
+          [(=]?                     # Opening brace or equals sign
+          \s*                       # Whitespace
+          ['"](?<name>#{value})['"] # Package name in quotes
+        }x
+
+        link_regex(regex, &url_proc)
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/dependency_linker/package_json_linker.rb b/lib/gitlab/dependency_linker/package_json_linker.rb
new file mode 100644
index 00000000000..330c95f0880
--- /dev/null
+++ b/lib/gitlab/dependency_linker/package_json_linker.rb
@@ -0,0 +1,44 @@
+module Gitlab
+  module DependencyLinker
+    class PackageJsonLinker < JsonLinker
+      self.file_type = :package_json
+
+      private
+
+      def link_dependencies
+        link_json('name', json["name"], &method(:package_url))
+        link_json('license', &method(:license_url))
+        link_json(%w[homepage url], URL_REGEX, &:itself)
+
+        link_packages
+      end
+
+      def link_packages
+        link_packages_at_key("dependencies", &method(:package_url))
+        link_packages_at_key("devDependencies", &method(:package_url))
+      end
+
+      def link_packages_at_key(key, &url_proc)
+        dependencies = json[key]
+        return unless dependencies
+
+        dependencies.each do |name, version|
+          link_json(name, version, link: :key, &url_proc)
+
+          link_json(name) do |value|
+            case value
+            when /\A#{URL_REGEX}\z/
+              value
+            when /\A#{REPO_REGEX}\z/
+              github_url(value)
+            end
+          end
+        end
+      end
+
+      def package_url(name)
+        "https://npmjs.com/package/#{name}"
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/dependency_linker/podfile_linker.rb b/lib/gitlab/dependency_linker/podfile_linker.rb
new file mode 100644
index 00000000000..60ad166ea17
--- /dev/null
+++ b/lib/gitlab/dependency_linker/podfile_linker.rb
@@ -0,0 +1,15 @@
+module Gitlab
+  module DependencyLinker
+    class PodfileLinker < GemfileLinker
+      include Cocoapods
+
+      self.file_type = :podfile
+
+      private
+
+      def link_packages
+        link_method_call('pod', &method(:package_url))
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/dependency_linker/podspec_json_linker.rb b/lib/gitlab/dependency_linker/podspec_json_linker.rb
new file mode 100644
index 00000000000..d82237ed3f1
--- /dev/null
+++ b/lib/gitlab/dependency_linker/podspec_json_linker.rb
@@ -0,0 +1,32 @@
+module Gitlab
+  module DependencyLinker
+    class PodspecJsonLinker < JsonLinker
+      include Cocoapods
+
+      self.file_type = :podspec_json
+
+      private
+
+      def link_dependencies
+        link_json('name', json["name"], &method(:package_url))
+        link_json('license', &method(:license_url))
+        link_json(%w[homepage git], URL_REGEX, &:itself)
+
+        link_packages_at_key("dependencies", &method(:package_url))
+
+        json["subspecs"]&.each do |subspec|
+          link_packages_at_key("dependencies", subspec, &method(:package_url))
+        end
+      end
+
+      def link_packages_at_key(key, root = json, &url_proc)
+        dependencies = root[key]
+        return unless dependencies
+
+        dependencies.each do |name, _|
+          link_regex(/"(?<name>#{Regexp.escape(name)})":\s*\[/, &url_proc)
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/dependency_linker/podspec_linker.rb b/lib/gitlab/dependency_linker/podspec_linker.rb
new file mode 100644
index 00000000000..a52c7a02439
--- /dev/null
+++ b/lib/gitlab/dependency_linker/podspec_linker.rb
@@ -0,0 +1,24 @@
+module Gitlab
+  module DependencyLinker
+    class PodspecLinker < MethodLinker
+      include Cocoapods
+
+      STRING_REGEX = /['"](?<name>[^'"]+)['"]/.freeze
+
+      self.file_type = :podspec
+
+      private
+
+      def link_dependencies
+        link_method_call('homepage', URL_REGEX, &:itself)
+
+        link_regex(%r{(git:|:git\s*=>)\s*['"](?<name>#{URL_REGEX})['"]}, &:itself)
+
+        link_method_call('license', &method(:license_url))
+        link_regex(/license\s*=\s*\{\s*(type:|:type\s*=>)\s*#{STRING_REGEX}/, &method(:license_url))
+
+        link_method_call(%w[name dependency], &method(:package_url))
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/dependency_linker/requirements_txt_linker.rb b/lib/gitlab/dependency_linker/requirements_txt_linker.rb
new file mode 100644
index 00000000000..2e197e5cd94
--- /dev/null
+++ b/lib/gitlab/dependency_linker/requirements_txt_linker.rb
@@ -0,0 +1,17 @@
+module Gitlab
+  module DependencyLinker
+    class RequirementsTxtLinker < BaseLinker
+      self.file_type = :requirements_txt
+
+      private
+
+      def link_dependencies
+        link_regex(/^(?<name>(?![a-z+]+:)[^#.-][^ ><=;\[]+)/) do |name|
+          "https://pypi.python.org/pypi/#{name}"
+        end
+
+        link_regex(%r{^(?<name>https?://[^ ]+)}, &:itself)
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/diff/file.rb b/lib/gitlab/diff/file.rb
index c6bf25b5874..2aef7fdaa35 100644
--- a/lib/gitlab/diff/file.rb
+++ b/lib/gitlab/diff/file.rb
@@ -1,16 +1,17 @@
 module Gitlab
   module Diff
     class File
-      attr_reader :diff, :repository, :diff_refs
+      attr_reader :diff, :repository, :diff_refs, :fallback_diff_refs
 
-      delegate :new_file, :deleted_file, :renamed_file,
-        :old_path, :new_path, :a_mode, :b_mode,
+      delegate :new_file?, :deleted_file?, :renamed_file?,
+        :old_path, :new_path, :a_mode, :b_mode, :mode_changed?,
         :submodule?, :too_large?, :collapsed?, to: :diff, prefix: false
 
-      def initialize(diff, repository:, diff_refs: nil)
+      def initialize(diff, repository:, diff_refs: nil, fallback_diff_refs: nil)
         @diff = diff
         @repository = repository
         @diff_refs = diff_refs
+        @fallback_diff_refs = fallback_diff_refs
       end
 
       def position(line)
@@ -49,24 +50,60 @@ module Gitlab
         line_code(line) if line
       end
 
+      def old_sha
+        diff_refs&.base_sha
+      end
+
+      def new_sha
+        diff_refs&.head_sha
+      end
+
+      def content_sha
+        return old_content_sha if deleted_file?
+        return @content_sha if defined?(@content_sha)
+
+        refs = diff_refs || fallback_diff_refs
+        @content_sha = refs&.head_sha
+      end
+
       def content_commit
-        return unless diff_refs
+        return @content_commit if defined?(@content_commit)
+
+        sha = content_sha
+        @content_commit = repository.commit(sha) if sha
+      end
+
+      def old_content_sha
+        return if new_file?
+        return @old_content_sha if defined?(@old_content_sha)
 
-        repository.commit(deleted_file ? old_ref : new_ref)
+        refs = diff_refs || fallback_diff_refs
+        @old_content_sha = refs&.base_sha
       end
 
       def old_content_commit
-        return unless diff_refs
+        return @old_content_commit if defined?(@old_content_commit)
 
-        repository.commit(old_ref)
+        sha = old_content_sha
+        @old_content_commit = repository.commit(sha) if sha
       end
 
-      def old_ref
-        diff_refs.try(:base_sha)
+      def blob
+        return @blob if defined?(@blob)
+
+        sha = content_sha
+        return @blob = nil unless sha
+
+        repository.blob_at(sha, file_path)
       end
 
-      def new_ref
-        diff_refs.try(:head_sha)
+      def old_blob
+        return @old_blob if defined?(@old_blob)
+
+        sha = old_content_sha
+        return @old_blob = nil unless sha
+
+        @old_blob = repository.blob_at(sha, old_path)
       end
 
       attr_writer :highlighted_diff_lines
@@ -85,10 +122,6 @@ module Gitlab
         @parallel_diff_lines ||= Gitlab::Diff::ParallelDiff.new(self).parallelize
       end
 
-      def mode_changed?
-        a_mode && b_mode && a_mode != b_mode
-      end
-
       def raw_diff
         diff.diff.to_s
       end
@@ -117,20 +150,8 @@ module Gitlab
         diff_lines.count(&:removed?)
       end
 
-      def old_blob(commit = old_content_commit)
-        return unless commit
-
-        repository.blob_at(commit.id, old_path)
-      end
-
-      def blob(commit = content_commit)
-        return unless commit
-
-        repository.blob_at(commit.id, file_path)
-      end
-
       def file_identifier
-        "#{file_path}-#{new_file}-#{deleted_file}-#{renamed_file}"
+        "#{file_path}-#{new_file?}-#{deleted_file?}-#{renamed_file?}"
       end
     end
   end
diff --git a/lib/gitlab/diff/file_collection/base.rb b/lib/gitlab/diff/file_collection/base.rb
index 7c32adc6ce7..79836a2fbab 100644
--- a/lib/gitlab/diff/file_collection/base.rb
+++ b/lib/gitlab/diff/file_collection/base.rb
@@ -2,7 +2,7 @@ module Gitlab
   module Diff
     module FileCollection
       class Base
-        attr_reader :project, :diff_options, :diff_view, :diff_refs
+        attr_reader :project, :diff_options, :diff_refs, :fallback_diff_refs
 
         delegate :count, :size, :real_size, to: :diff_files
 
@@ -10,14 +10,15 @@ module Gitlab
           ::Commit.max_diff_options.merge(ignore_whitespace_change: false, no_collapse: false)
         end
 
-        def initialize(diffable, project:, diff_options: nil, diff_refs: nil)
+        def initialize(diffable, project:, diff_options: nil, diff_refs: nil, fallback_diff_refs: nil)
           diff_options = self.class.default_options.merge(diff_options || {})
 
-          @diffable     = diffable
-          @diffs        = diffable.raw_diffs(diff_options)
-          @project      = project
+          @diffable = diffable
+          @diffs = diffable.raw_diffs(diff_options)
+          @project = project
           @diff_options = diff_options
-          @diff_refs    = diff_refs
+          @diff_refs = diff_refs
+          @fallback_diff_refs = fallback_diff_refs
         end
 
         def diff_files
@@ -35,7 +36,7 @@ module Gitlab
         private
 
         def decorate_diff!(diff)
-          Gitlab::Diff::File.new(diff, repository: project.repository, diff_refs: diff_refs)
+          Gitlab::Diff::File.new(diff, repository: project.repository, diff_refs: diff_refs, fallback_diff_refs: fallback_diff_refs)
         end
       end
     end
diff --git a/lib/gitlab/diff/file_collection/merge_request_diff.rb b/lib/gitlab/diff/file_collection/merge_request_diff.rb
index 0bd226ef050..9a58b500a2c 100644
--- a/lib/gitlab/diff/file_collection/merge_request_diff.rb
+++ b/lib/gitlab/diff/file_collection/merge_request_diff.rb
@@ -8,7 +8,8 @@ module Gitlab
           super(merge_request_diff,
             project: merge_request_diff.project,
             diff_options: diff_options,
-            diff_refs: merge_request_diff.diff_refs)
+            diff_refs: merge_request_diff.diff_refs,
+            fallback_diff_refs: merge_request_diff.fallback_diff_refs)
         end
 
         def diff_files
diff --git a/lib/gitlab/diff/highlight.rb b/lib/gitlab/diff/highlight.rb
index 7db896522a9..ed2f541977a 100644
--- a/lib/gitlab/diff/highlight.rb
+++ b/lib/gitlab/diff/highlight.rb
@@ -3,7 +3,7 @@ module Gitlab
     class Highlight
       attr_reader :diff_file, :diff_lines, :raw_lines, :repository
 
-      delegate :old_path, :new_path, :old_ref, :new_ref, to: :diff_file, prefix: :diff
+      delegate :old_path, :new_path, :old_sha, :new_sha, to: :diff_file, prefix: :diff
 
       def initialize(diff_lines, repository: nil)
         @repository = repository
@@ -61,12 +61,12 @@ module Gitlab
 
       def old_lines
         return unless diff_file
-        @old_lines ||= Gitlab::Highlight.highlight_lines(self.repository, diff_old_ref, diff_old_path)
+        @old_lines ||= Gitlab::Highlight.highlight_lines(self.repository, diff_old_sha, diff_old_path)
       end
 
       def new_lines
         return unless diff_file
-        @new_lines ||= Gitlab::Highlight.highlight_lines(self.repository, diff_new_ref, diff_new_path)
+        @new_lines ||= Gitlab::Highlight.highlight_lines(self.repository, diff_new_sha, diff_new_path)
       end
     end
   end
diff --git a/lib/gitlab/etag_caching/router.rb b/lib/gitlab/etag_caching/router.rb
index cf279628beb..2f9d8bfc266 100644
--- a/lib/gitlab/etag_caching/router.rb
+++ b/lib/gitlab/etag_caching/router.rb
@@ -11,8 +11,9 @@ module Gitlab
       USED_IN_ROUTES = %w[noteable issue notes issues realtime_changes
                           commit pipelines merge_requests new
                           environments].freeze
-      RESERVED_WORDS = Gitlab::Regex::ILLEGAL_PROJECT_PATH_WORDS - USED_IN_ROUTES
-      RESERVED_WORDS_REGEX = Regexp.union(*RESERVED_WORDS)
+      RESERVED_WORDS = Gitlab::PathRegex::ILLEGAL_PROJECT_PATH_WORDS - USED_IN_ROUTES
+      RESERVED_WORDS_REGEX = Regexp.union(*RESERVED_WORDS.map(&Regexp.method(:escape)))
+
       ROUTES = [
         Gitlab::EtagCaching::Router::Route.new(
           %r(^(?!.*(#{RESERVED_WORDS_REGEX})).*/noteable/issue/\d+/notes\z),
diff --git a/lib/gitlab/git/diff.rb b/lib/gitlab/git/diff.rb
index 31d1b66b4f7..deade337354 100644
--- a/lib/gitlab/git/diff.rb
+++ b/lib/gitlab/git/diff.rb
@@ -11,6 +11,10 @@ module Gitlab
       # Stats properties
       attr_accessor :new_file, :renamed_file, :deleted_file
 
+      alias_method :new_file?, :new_file
+      alias_method :deleted_file?, :deleted_file
+      alias_method :renamed_file?, :renamed_file
+
       attr_accessor :too_large
 
       # The maximum size of a diff to display.
@@ -208,6 +212,10 @@ module Gitlab
         hash
       end
 
+      def mode_changed?
+        a_mode && b_mode && a_mode != b_mode
+      end
+
       def submodule?
         a_mode == '160000' || b_mode == '160000'
       end
diff --git a/lib/gitlab/git/diff_collection.rb b/lib/gitlab/git/diff_collection.rb
index bcbad8ec829..898a5ae15f2 100644
--- a/lib/gitlab/git/diff_collection.rb
+++ b/lib/gitlab/git/diff_collection.rb
@@ -19,22 +19,19 @@ module Gitlab
         @line_count = 0
         @byte_count = 0
         @overflow = false
+        @empty = true
         @array = Array.new
       end
 
       def each(&block)
-        if @populated
-          # @iterator.each is slower than just iterating the array in place
-          @array.each(&block)
-        else
-          Gitlab::GitalyClient.migrate(:commit_raw_diffs) do
-            each_patch(&block)
-          end
+        Gitlab::GitalyClient.migrate(:commit_raw_diffs) do
+          each_patch(&block)
         end
       end
 
       def empty?
-        !@iterator.any?
+        any? # Make sure the iterator has been exercised
+        @empty
       end
 
       def overflow?
@@ -60,17 +57,17 @@ module Gitlab
         collection = each_with_index do |element, i|
           @array[i] = yield(element)
         end
-        @populated = true
         collection
       end
 
+      alias_method :to_ary, :to_a
+
       private
 
       def populate!
         return if @populated
 
         each { nil } # force a loop through all diffs
-        @populated = true
         nil
       end
 
@@ -79,15 +76,17 @@ module Gitlab
       end
 
       def each_patch
-        @iterator.each_with_index do |raw, i|
-          # First yield cached Diff instances from @array
-          if @array[i]
-            yield @array[i]
-            next
-          end
+        i = 0
+        @array.each do |diff|
+          yield diff
+          i += 1
+        end
+
+        return if @overflow
+        return if @iterator.nil?
 
-          # We have exhausted @array, time to create new Diff instances or stop.
-          break if @overflow
+        @iterator.each do |raw|
+          @empty = false
 
           if !@all_diffs && i >= @max_files
             @overflow = true
@@ -113,7 +112,13 @@ module Gitlab
           end
 
           yield @array[i] = diff
+          i += 1
         end
+
+        @populated = true
+
+        # Allow iterator to be garbage-collected. It cannot be reused anyway.
+        @iterator = nil
       end
     end
   end
diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb
index b9f1ac144b6..9d6adbdb4ac 100644
--- a/lib/gitlab/git/repository.rb
+++ b/lib/gitlab/git/repository.rb
@@ -1006,31 +1006,39 @@ module Gitlab
       # Parses the contents of a .gitmodules file and returns a hash of
       # submodule information.
       def parse_gitmodules(commit, content)
-        results = {}
+        modules = {}
 
-        current = ""
-        content.split("\n").each do |txt|
-          if txt =~ /^\s*\[/
-            current = txt.match(/(?<=").*(?=")/)[0]
-            results[current] = {}
-          else
-            next unless results[current]
-            match_data = txt.match(/(\w+)\s*=\s*(.*)/)
-            next unless match_data
-            target = match_data[2].chomp
-            results[current][match_data[1]] = target
+        name = nil
+        content.each_line do |line|
+          case line.strip
+          when /\A\[submodule "(?<name>[^"]+)"\]\z/ # Submodule header
+            name = $~[:name]
+            modules[name] = {}
+          when /\A(?<key>\w+)\s*=\s*(?<value>.*)\z/ # Key/value pair
+            key = $~[:key]
+            value = $~[:value].chomp
+
+            next unless name && modules[name]
+
+            modules[name][key] = value
 
-            if match_data[1] == "path"
+            if key == 'path'
               begin
-                results[current]["id"] = blob_content(commit, target)
+                modules[name]['id'] = blob_content(commit, value)
               rescue InvalidBlobName
-                results.delete(current)
+                # The current entry is invalid
+                modules.delete(name)
+                name = nil
               end
             end
+          when /\A#/ # Comment
+            next
+          else # Invalid line
+            name = nil
           end
         end
 
-        results
+        modules
       end
 
       # Returns true if +commit+ introduced changes to +path+, using commit
@@ -1086,7 +1094,12 @@ module Gitlab
           elsif tmp_entry.nil?
             return nil
           else
-            tmp_entry = rugged.lookup(tmp_entry[:oid])
+            begin
+              tmp_entry = rugged.lookup(tmp_entry[:oid])
+            rescue Rugged::OdbError, Rugged::InvalidError, Rugged::ReferenceError
+              return nil
+            end
+
             return nil unless tmp_entry.type == :tree
             tmp_entry = tmp_entry[dir]
           end
diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb
index 72466700c05..2343446bf22 100644
--- a/lib/gitlab/gitaly_client.rb
+++ b/lib/gitlab/gitaly_client.rb
@@ -2,6 +2,12 @@ require 'gitaly'
 
 module Gitlab
   module GitalyClient
+    module MigrationStatus
+      DISABLED = 1
+      OPT_IN = 2
+      OPT_OUT = 3
+    end
+
     SERVER_VERSION_FILE = 'GITALY_SERVER_VERSION'.freeze
 
     MUTEX = Mutex.new
@@ -46,8 +52,20 @@ module Gitlab
       Gitlab.config.gitaly.enabled
     end
 
-    def self.feature_enabled?(feature)
-      enabled? && ENV["GITALY_#{feature.upcase}"] == '1'
+    def self.feature_enabled?(feature, status: MigrationStatus::OPT_IN)
+      return false if !enabled? || status == MigrationStatus::DISABLED
+
+      feature = Feature.get("gitaly_#{feature}")
+
+      # If the feature hasn't been set, turn it on if it's opt-out
+      return status == MigrationStatus::OPT_OUT unless Feature.persisted?(feature)
+
+      if feature.percentage_of_time_value > 0
+        # Probabilistically enable this feature
+        return Random.rand() * 100 < feature.percentage_of_time_value
+      end
+
+      feature.enabled?
     end
 
     def self.migrate(feature)
diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb
index 21f2e6b6970..319633656ff 100644
--- a/lib/gitlab/gon_helper.rb
+++ b/lib/gitlab/gon_helper.rb
@@ -1,3 +1,5 @@
+# rubocop:disable Metrics/AbcSize
+
 module Gitlab
   module GonHelper
     def add_gon_variables
@@ -13,6 +15,7 @@ module Gitlab
       gon.sentry_dsn             = current_application_settings.clientside_sentry_dsn if current_application_settings.clientside_sentry_enabled
       gon.gitlab_url             = Gitlab.config.gitlab.url
       gon.revision               = Gitlab::REVISION
+      gon.gitlab_logo            = ActionController::Base.helpers.asset_path('gitlab_logo.png')
 
       if current_user
         gon.current_user_id = current_user.id
diff --git a/lib/gitlab/group_hierarchy.rb b/lib/gitlab/group_hierarchy.rb
new file mode 100644
index 00000000000..e9d5d52cabb
--- /dev/null
+++ b/lib/gitlab/group_hierarchy.rb
@@ -0,0 +1,104 @@
+module Gitlab
+  # Retrieving of parent or child groups based on a base ActiveRecord relation.
+  #
+  # This class uses recursive CTEs and as a result will only work on PostgreSQL.
+  class GroupHierarchy
+    attr_reader :base, :model
+
+    # base - An instance of ActiveRecord::Relation for which to get parent or
+    #        child groups.
+    def initialize(base)
+      @base = base
+      @model = base.model
+    end
+
+    # Returns a relation that includes the base set of groups and all their
+    # ancestors (recursively).
+    def base_and_ancestors
+      return model.none unless Group.supports_nested_groups?
+
+      base_and_ancestors_cte.apply_to(model.all)
+    end
+
+    # Returns a relation that includes the base set of groups and all their
+    # descendants (recursively).
+    def base_and_descendants
+      return model.none unless Group.supports_nested_groups?
+
+      base_and_descendants_cte.apply_to(model.all)
+    end
+
+    # Returns a relation that includes the base groups, their ancestors, and the
+    # descendants of the base groups.
+    #
+    # The resulting query will roughly look like the following:
+    #
+    #     WITH RECURSIVE ancestors AS ( ... ),
+    #       descendants AS ( ... )
+    #     SELECT *
+    #     FROM (
+    #       SELECT *
+    #       FROM ancestors namespaces
+    #
+    #       UNION
+    #
+    #       SELECT *
+    #       FROM descendants namespaces
+    #     ) groups;
+    #
+    # Using this approach allows us to further add criteria to the relation with
+    # Rails thinking it's selecting data the usual way.
+    def all_groups
+      return base unless Group.supports_nested_groups?
+
+      ancestors = base_and_ancestors_cte
+      descendants = base_and_descendants_cte
+
+      ancestors_table = ancestors.alias_to(groups_table)
+      descendants_table = descendants.alias_to(groups_table)
+
+      union = SQL::Union.new([model.unscoped.from(ancestors_table),
+                              model.unscoped.from(descendants_table)])
+
+      model.
+        unscoped.
+        with.
+        recursive(ancestors.to_arel, descendants.to_arel).
+        from("(#{union.to_sql}) #{model.table_name}")
+    end
+
+    private
+
+    def base_and_ancestors_cte
+      cte = SQL::RecursiveCTE.new(:base_and_ancestors)
+
+      cte << base.except(:order)
+
+      # Recursively get all the ancestors of the base set.
+      cte << model.
+        from([groups_table, cte.table]).
+        where(groups_table[:id].eq(cte.table[:parent_id])).
+        except(:order)
+
+      cte
+    end
+
+    def base_and_descendants_cte
+      cte = SQL::RecursiveCTE.new(:base_and_descendants)
+
+      cte << base.except(:order)
+
+      # Recursively get all the descendants of the base set.
+      cte << model.
+        from([groups_table, cte.table]).
+        where(groups_table[:parent_id].eq(cte.table[:id])).
+        except(:order)
+
+      cte
+    end
+
+    def groups_table
+      model.arel_table
+    end
+  end
+end
diff --git a/lib/gitlab/health_checks/fs_shards_check.rb b/lib/gitlab/health_checks/fs_shards_check.rb
index df962d203b7..e78b7f22e03 100644
--- a/lib/gitlab/health_checks/fs_shards_check.rb
+++ b/lib/gitlab/health_checks/fs_shards_check.rb
@@ -2,6 +2,9 @@ module Gitlab
   module HealthChecks
     class FsShardsCheck
       extend BaseAbstractCheck
+      RANDOM_STRING = SecureRandom.hex(1000).freeze
+      COMMAND_TIMEOUT = '1'.freeze
+      TIMEOUT_EXECUTABLE = 'timeout'.freeze
 
       class << self
         def readiness
@@ -41,8 +44,6 @@ module Gitlab
 
         private
 
-        RANDOM_STRING = SecureRandom.hex(1000).freeze
-
         def operation_metrics(ok_metric, latency_metric, operation, **labels)
           with_timing operation do |result, elapsed|
             [
@@ -63,8 +64,8 @@ module Gitlab
           @storage_paths ||= Gitlab.config.repositories.storages
         end
 
-        def with_timeout(args)
-          %w{timeout 1}.concat(args)
+        def exec_with_timeout(cmd_args, *args, &block)
+          Gitlab::Popen.popen([TIMEOUT_EXECUTABLE, COMMAND_TIMEOUT].concat(cmd_args), *args, &block)
         end
 
         def tmp_file_path(storage_name)
@@ -78,7 +79,7 @@ module Gitlab
         def storage_stat_test(storage_name)
           stat_path = File.join(path(storage_name), '.')
           begin
-            _, status = Gitlab::Popen.popen(with_timeout(%W{ stat #{stat_path} }))
+            _, status = exec_with_timeout(%W{ stat #{stat_path} })
             status == 0
           rescue Errno::ENOENT
             File.exist?(stat_path) && File::Stat.new(stat_path).readable?
@@ -86,7 +87,7 @@ module Gitlab
         end
 
         def storage_write_test(tmp_path)
-          _, status = Gitlab::Popen.popen(with_timeout(%W{ tee #{tmp_path} })) do |stdin|
+          _, status = exec_with_timeout(%W{ tee #{tmp_path} }) do |stdin|
             stdin.write(RANDOM_STRING)
           end
           status == 0
@@ -96,7 +97,7 @@ module Gitlab
         end
 
         def storage_read_test(tmp_path)
-          _, status = Gitlab::Popen.popen(with_timeout(%W{ diff #{tmp_path} - })) do |stdin|
+          _, status = exec_with_timeout(%W{ diff #{tmp_path} - }) do |stdin|
             stdin.write(RANDOM_STRING)
           end
           status == 0
@@ -106,7 +107,7 @@ module Gitlab
         end
 
         def delete_test_file(tmp_path)
-          _, status = Gitlab::Popen.popen(with_timeout(%W{ rm -f #{tmp_path} }))
+          _, status = exec_with_timeout(%W{ rm -f #{tmp_path} })
           status == 0
         rescue Errno::ENOENT
           File.delete(tmp_path) rescue Errno::ENOENT
diff --git a/lib/gitlab/i18n.rb b/lib/gitlab/i18n.rb
index 3411516319f..5ab3eeb3aff 100644
--- a/lib/gitlab/i18n.rb
+++ b/lib/gitlab/i18n.rb
@@ -12,15 +12,36 @@ module Gitlab
       AVAILABLE_LANGUAGES.keys
     end
 
-    def set_locale(current_user)
-      requested_locale = current_user&.preferred_language || ::I18n.default_locale
-      locale = FastGettext.set_locale(requested_locale)
-      ::I18n.locale = locale
+    def locale
+      FastGettext.locale
     end
 
-    def reset_locale
+    def locale=(locale_string)
+      requested_locale = locale_string || ::I18n.default_locale
+      new_locale = FastGettext.set_locale(requested_locale)
+      ::I18n.locale = new_locale
+    end
+
+    def use_default_locale
       FastGettext.set_locale(::I18n.default_locale)
       ::I18n.locale = ::I18n.default_locale
     end
+
+    def with_locale(locale_string)
+      original_locale = locale
+
+      self.locale = locale_string
+      yield
+    ensure
+      self.locale = original_locale
+    end
+
+    def with_user_locale(user, &block)
+      with_locale(user&.preferred_language, &block)
+    end
+
+    def with_default_locale(&block)
+      with_locale(::I18n.default_locale, &block)
+    end
   end
 end
diff --git a/lib/gitlab/o_auth/provider.rb b/lib/gitlab/o_auth/provider.rb
index 9ad7a38d505..ac9d66c836d 100644
--- a/lib/gitlab/o_auth/provider.rb
+++ b/lib/gitlab/o_auth/provider.rb
@@ -22,7 +22,11 @@ module Gitlab
       def self.config_for(name)
         name = name.to_s
         if ldap_provider?(name)
-          Gitlab::LDAP::Config.new(name).options
+          if Gitlab::LDAP::Config.valid_provider?(name)
+            Gitlab::LDAP::Config.new(name).options
+          else
+            nil
+          end
         else
           Gitlab.config.omniauth.providers.find { |provider| provider.name == name }
         end
diff --git a/lib/gitlab/path_regex.rb b/lib/gitlab/path_regex.rb
new file mode 100644
index 00000000000..9ff6829cd49
--- /dev/null
+++ b/lib/gitlab/path_regex.rb
@@ -0,0 +1,265 @@
+module Gitlab
+  module PathRegex
+    extend self
+
+    # All routes that appear on the top level must be listed here.
+    # This will make sure that groups cannot be created with these names
+    # as these routes would be masked by the paths already in place.
+    #
+    # Example:
+    #   /api/api-project
+    #
+    #  the path `api` shouldn't be allowed because it would be masked by `api/*`
+    #
+    TOP_LEVEL_ROUTES = %w[
+      -
+      .well-known
+      abuse_reports
+      admin
+      all
+      api
+      assets
+      autocomplete
+      ci
+      dashboard
+      explore
+      files
+      groups
+      health_check
+      help
+      hooks
+      import
+      invites
+      issues
+      jwt
+      koding
+      member
+      merge_requests
+      new
+      notes
+      notification_settings
+      oauth
+      profile
+      projects
+      public
+      repository
+      robots.txt
+      s
+      search
+      sent_notifications
+      services
+      snippets
+      teams
+      u
+      unicorn_test
+      unsubscribes
+      uploads
+      users
+    ].freeze
+
+    # This list should contain all words following `/*namespace_id/:project_id` in
+    # routes that contain a second wildcard.
+    #
+    # Example:
+    #   /*namespace_id/:project_id/badges/*ref/build
+    #
+    # If `badges` was allowed as a project/group name, we would not be able to access the
+    # `badges` route for those projects:
+    #
+    # Consider a namespace with path `foo/bar` and a project called `badges`.
+    # The route to the build badge would then be `/foo/bar/badges/badges/master/build.svg`
+    #
+    # When accessing this path the route would be matched to the `badges` path
+    # with the following params:
+    #   - namespace_id: `foo`
+    #   - project_id: `bar`
+    #   - ref: `badges/master`
+    #
+    # Failing to find the project, this would result in a 404.
+    #
+    # By rejecting `badges` the router can _count_ on the fact that `badges` will
+    # be preceded by the `namespace/project`.
+    PROJECT_WILDCARD_ROUTES = %w[
+      -
+      badges
+      blame
+      blob
+      builds
+      commits
+      create
+      create_dir
+      edit
+      environments/folders
+      files
+      find_file
+      gitlab-lfs/objects
+      info/lfs/objects
+      new
+      preview
+      raw
+      refs
+      tree
+      update
+      wikis
+    ].freeze
+
+    # These are all the paths that follow `/groups/*id/ or `/groups/*group_id`
+    # We need to reject these because we have a `/groups/*id` page that is the same
+    # as the `/*id`.
+    #
+    # If we would allow a subgroup to be created with the name `activity` then
+    # this group would not be accessible through `/groups/parent/activity` since
+    # this would map to the activity-page of its parent.
+    GROUP_ROUTES = %w[
+      activity
+      analytics
+      audit_events
+      avatar
+      edit
+      group_members
+      hooks
+      issues
+      labels
+      ldap
+      ldap_group_links
+      merge_requests
+      milestones
+      notification_setting
+      pipeline_quota
+      projects
+      subgroups
+    ].freeze
+
+    ILLEGAL_PROJECT_PATH_WORDS = PROJECT_WILDCARD_ROUTES
+    ILLEGAL_GROUP_PATH_WORDS = (PROJECT_WILDCARD_ROUTES | GROUP_ROUTES).freeze
+
+    # The namespace regex is used in JavaScript to validate usernames in the "Register" form. However, Javascript
+    # does not support the negative lookbehind assertion (?<!) that disallows usernames ending in `.git` and `.atom`.
+    # Since this is a non-trivial problem to solve in Javascript (heavily complicate the regex, modify view code to
+    # allow non-regex validations, etc), `NAMESPACE_FORMAT_REGEX_JS` serves as a Javascript-compatible version of
+    # `NAMESPACE_FORMAT_REGEX`, with the negative lookbehind assertion removed. This means that the client-side validation
+    # will pass for usernames ending in `.atom` and `.git`, but will be caught by the server-side validation.
+    PATH_REGEX_STR = '[a-zA-Z0-9_\.][a-zA-Z0-9_\-\.]*'.freeze
+    NAMESPACE_FORMAT_REGEX_JS = PATH_REGEX_STR + '[a-zA-Z0-9_\-]|[a-zA-Z0-9_]'.freeze
+
+    NO_SUFFIX_REGEX = /(?<!\.git|\.atom)/.freeze
+    NAMESPACE_FORMAT_REGEX = /(?:#{NAMESPACE_FORMAT_REGEX_JS})#{NO_SUFFIX_REGEX}/.freeze
+    PROJECT_PATH_FORMAT_REGEX = /(?:#{PATH_REGEX_STR})#{NO_SUFFIX_REGEX}/.freeze
+    FULL_NAMESPACE_FORMAT_REGEX = %r{(#{NAMESPACE_FORMAT_REGEX}/)*#{NAMESPACE_FORMAT_REGEX}}.freeze
+
+    def root_namespace_route_regex
+      @root_namespace_route_regex ||= begin
+        illegal_words = Regexp.new(Regexp.union(TOP_LEVEL_ROUTES).source, Regexp::IGNORECASE)
+
+        single_line_regexp %r{
+          (?!(#{illegal_words})/)
+          #{NAMESPACE_FORMAT_REGEX}
+        }x
+      end
+    end
+
+    def full_namespace_route_regex
+      @full_namespace_route_regex ||= begin
+        illegal_words = Regexp.new(Regexp.union(ILLEGAL_GROUP_PATH_WORDS).source, Regexp::IGNORECASE)
+
+        single_line_regexp %r{
+          #{root_namespace_route_regex}
+          (?:
+            /
+            (?!#{illegal_words}/)
+            #{NAMESPACE_FORMAT_REGEX}
+          )*
+        }x
+      end
+    end
+
+    def project_route_regex
+      @project_route_regex ||= begin
+        illegal_words = Regexp.new(Regexp.union(ILLEGAL_PROJECT_PATH_WORDS).source, Regexp::IGNORECASE)
+
+        single_line_regexp %r{
+          (?!(#{illegal_words})/)
+          #{PROJECT_PATH_FORMAT_REGEX}
+        }x
+      end
+    end
+
+    def project_git_route_regex
+      @project_git_route_regex ||= /#{project_route_regex}\.git/.freeze
+    end
+
+    def root_namespace_path_regex
+      @root_namespace_path_regex ||= %r{\A#{root_namespace_route_regex}/\z}
+    end
+
+    def full_namespace_path_regex
+      @full_namespace_path_regex ||= %r{\A#{full_namespace_route_regex}/\z}
+    end
+
+    def project_path_regex
+      @project_path_regex ||= %r{\A#{project_route_regex}/\z}
+    end
+
+    def full_project_path_regex
+      @full_project_path_regex ||= %r{\A#{full_namespace_route_regex}/#{project_route_regex}/\z}
+    end
+
+    def full_namespace_format_regex
+      @namespace_format_regex ||= /A#{FULL_NAMESPACE_FORMAT_REGEX}\z/.freeze
+    end
+
+    def namespace_format_regex
+      @namespace_format_regex ||= /\A#{NAMESPACE_FORMAT_REGEX}\z/.freeze
+    end
+
+    def namespace_format_message
+      "can contain only letters, digits, '_', '-' and '.'. " \
+      "Cannot start with '-' or end in '.', '.git' or '.atom'." \
+    end
+
+    def project_path_format_regex
+      @project_path_format_regex ||= /\A#{PROJECT_PATH_FORMAT_REGEX}\z/.freeze
+    end
+
+    def project_path_format_message
+      "can contain only letters, digits, '_', '-' and '.'. " \
+      "Cannot start with '-', end in '.git' or end in '.atom'" \
+    end
+
+    def archive_formats_regex
+      #                           |zip|tar|    tar.gz    |         tar.bz2         |
+      @archive_formats_regex ||= /(zip|tar|tar\.gz|tgz|gz|tar\.bz2|tbz|tbz2|tb2|bz2)/.freeze
+    end
+
+    def git_reference_regex
+      # Valid git ref regex, see:
+      # https://www.kernel.org/pub/software/scm/git/docs/git-check-ref-format.html
+
+      @git_reference_regex ||= single_line_regexp %r{
+        (?!
+           (?# doesn't begins with)
+           \/|                    (?# rule #6)
+           (?# doesn't contain)
+           .*(?:
+              [\/.]\.|            (?# rule #1,3)
+              \/\/|               (?# rule #6)
+              @\{|                (?# rule #8)
+              \\                  (?# rule #9)
+           )
+        )
+        [^\000-\040\177~^:?*\[]+  (?# rule #4-5)
+        (?# doesn't end with)
+        (?<!\.lock)               (?# rule #1)
+        (?<![\/.])                (?# rule #6-7)
+      }x
+    end
+
+    private
+
+    def single_line_regexp(regex)
+      # Turns a multiline extended regexp into a single line one,
+      # beacuse `rake routes` breaks on multiline regexes.
+      Regexp.new(regex.source.gsub(/\(\?#.+?\)/, '').gsub(/\s*/, ''), regex.options ^ Regexp::EXTENDED).freeze
+    end
+  end
+end
diff --git a/lib/gitlab/project_authorizations/with_nested_groups.rb b/lib/gitlab/project_authorizations/with_nested_groups.rb
new file mode 100644
index 00000000000..bb0df1e3dad
--- /dev/null
+++ b/lib/gitlab/project_authorizations/with_nested_groups.rb
@@ -0,0 +1,125 @@
+module Gitlab
+  module ProjectAuthorizations
+    # Calculating new project authorizations when supporting nested groups.
+    #
+    # This class relies on Common Table Expressions to efficiently get all data,
+    # including data for nested groups. As a result this class can only be used
+    # on PostgreSQL.
+    class WithNestedGroups
+      attr_reader :user
+
+      # user - The User object for which to calculate the authorizations.
+      def initialize(user)
+        @user = user
+      end
+
+      def calculate
+        cte = recursive_cte
+        cte_alias = cte.table.alias(Group.table_name)
+        projects = Project.arel_table
+        links = ProjectGroupLink.arel_table
+
+        relations = [
+          # The project a user has direct access to.
+          user.projects.select_for_project_authorization,
+
+          # The personal projects of the user.
+          user.personal_projects.select_as_master_for_project_authorization,
+
+          # Projects that belong directly to any of the groups the user has
+          # access to.
+          Namespace.
+            unscoped.
+            select([alias_as_column(projects[:id], 'project_id'),
+                    cte_alias[:access_level]]).
+            from(cte_alias).
+            joins(:projects),
+
+          # Projects shared with any of the namespaces the user has access to.
+          Namespace.
+            unscoped.
+            select([links[:project_id],
+                    least(cte_alias[:access_level],
+                          links[:group_access],
+                          'access_level')]).
+            from(cte_alias).
+            joins('INNER JOIN project_group_links ON project_group_links.group_id = namespaces.id').
+            joins('INNER JOIN projects ON projects.id = project_group_links.project_id').
+            joins('INNER JOIN namespaces p_ns ON p_ns.id = projects.namespace_id').
+            where('p_ns.share_with_group_lock IS FALSE')
+        ]
+
+        union = Gitlab::SQL::Union.new(relations)
+
+        ProjectAuthorization.
+          unscoped.
+          with.
+          recursive(cte.to_arel).
+          select_from_union(union)
+      end
+
+      private
+
+      # Builds a recursive CTE that gets all the groups the current user has
+      # access to, including any nested groups.
+      def recursive_cte
+        cte = Gitlab::SQL::RecursiveCTE.new(:namespaces_cte)
+        members = Member.arel_table
+        namespaces = Namespace.arel_table
+
+        # Namespaces the user is a member of.
+        cte << user.groups.
+          select([namespaces[:id], members[:access_level]]).
+          except(:order)
+
+        # Sub groups of any groups the user is a member of.
+        cte << Group.select([namespaces[:id],
+                             greatest(members[:access_level],
+                                      cte.table[:access_level], 'access_level')]).
+          joins(join_cte(cte)).
+          joins(join_members).
+          except(:order)
+
+        cte
+      end
+
+      # Builds a LEFT JOIN to join optional memberships onto the CTE.
+      def join_members
+        members = Member.arel_table
+        namespaces = Namespace.arel_table
+
+        cond = members[:source_id].
+          eq(namespaces[:id]).
+          and(members[:source_type].eq('Namespace')).
+          and(members[:requested_at].eq(nil)).
+          and(members[:user_id].eq(user.id))
+
+        Arel::Nodes::OuterJoin.new(members, Arel::Nodes::On.new(cond))
+      end
+
+      # Builds an INNER JOIN to join namespaces onto the CTE.
+      def join_cte(cte)
+        namespaces = Namespace.arel_table
+        cond = cte.table[:id].eq(namespaces[:parent_id])
+
+        Arel::Nodes::InnerJoin.new(cte.table, Arel::Nodes::On.new(cond))
+      end
+
+      def greatest(left, right, column_alias)
+        sql_function('GREATEST', [left, right], column_alias)
+      end
+
+      def least(left, right, column_alias)
+        sql_function('LEAST', [left, right], column_alias)
+      end
+
+      def sql_function(name, args, column_alias)
+        alias_as_column(Arel::Nodes::NamedFunction.new(name, args), column_alias)
+      end
+
+      def alias_as_column(value, alias_to)
+        Arel::Nodes::As.new(value, Arel::Nodes::SqlLiteral.new(alias_to))
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/project_authorizations/without_nested_groups.rb b/lib/gitlab/project_authorizations/without_nested_groups.rb
new file mode 100644
index 00000000000..627e8c5fba2
--- /dev/null
+++ b/lib/gitlab/project_authorizations/without_nested_groups.rb
@@ -0,0 +1,35 @@
+module Gitlab
+  module ProjectAuthorizations
+    # Calculating new project authorizations when not supporting nested groups.
+    class WithoutNestedGroups
+      attr_reader :user
+
+      # user - The User object for which to calculate the authorizations.
+      def initialize(user)
+        @user = user
+      end
+
+      def calculate
+        relations = [
+          # Projects the user is a direct member of
+          user.projects.select_for_project_authorization,
+
+          # Personal projects
+          user.personal_projects.select_as_master_for_project_authorization,
+
+          # Projects of groups the user is a member of
+          user.groups_projects.select_for_project_authorization,
+
+          # Projects shared with groups the user is a member of
+          user.groups.joins(:shared_projects).select_for_project_authorization
+        ]
+
+        union = Gitlab::SQL::Union.new(relations)
+
+        ProjectAuthorization.
+          unscoped.
+          select_from_union(union)
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb
index f609850f8fa..e4d2a992470 100644
--- a/lib/gitlab/regex.rb
+++ b/lib/gitlab/regex.rb
@@ -2,203 +2,6 @@ module Gitlab
   module Regex
     extend self
 
-    # All routes that appear on the top level must be listed here.
-    # This will make sure that groups cannot be created with these names
-    # as these routes would be masked by the paths already in place.
-    #
-    # Example:
-    #   /api/api-project
-    #
-    #  the path `api` shouldn't be allowed because it would be masked by `api/*`
-    #
-    TOP_LEVEL_ROUTES = %w[
-      -
-      .well-known
-      abuse_reports
-      admin
-      all
-      api
-      assets
-      autocomplete
-      ci
-      dashboard
-      explore
-      files
-      groups
-      health_check
-      help
-      hooks
-      import
-      invites
-      issues
-      jwt
-      koding
-      member
-      merge_requests
-      new
-      notes
-      notification_settings
-      oauth
-      profile
-      projects
-      public
-      repository
-      robots.txt
-      s
-      search
-      sent_notifications
-      services
-      snippets
-      teams
-      u
-      unicorn_test
-      unsubscribes
-      uploads
-      users
-    ].freeze
-
-    # This list should contain all words following `/*namespace_id/:project_id` in
-    # routes that contain a second wildcard.
-    #
-    # Example:
-    #   /*namespace_id/:project_id/badges/*ref/build
-    #
-    # If `badges` was allowed as a project/group name, we would not be able to access the
-    # `badges` route for those projects:
-    #
-    # Consider a namespace with path `foo/bar` and a project called `badges`.
-    # The route to the build badge would then be `/foo/bar/badges/badges/master/build.svg`
-    #
-    # When accessing this path the route would be matched to the `badges` path
-    # with the following params:
-    #   - namespace_id: `foo`
-    #   - project_id: `bar`
-    #   - ref: `badges/master`
-    #
-    # Failing to find the project, this would result in a 404.
-    #
-    # By rejecting `badges` the router can _count_ on the fact that `badges` will
-    # be preceded by the `namespace/project`.
-    PROJECT_WILDCARD_ROUTES = %w[
-      badges
-      blame
-      blob
-      builds
-      commits
-      create
-      create_dir
-      edit
-      environments/folders
-      files
-      find_file
-      gitlab-lfs/objects
-      info/lfs/objects
-      new
-      preview
-      raw
-      refs
-      tree
-      update
-      wikis
-    ].freeze
-
-    # These are all the paths that follow `/groups/*id/ or `/groups/*group_id`
-    # We need to reject these because we have a `/groups/*id` page that is the same
-    # as the `/*id`.
-    #
-    # If we would allow a subgroup to be created with the name `activity` then
-    # this group would not be accessible through `/groups/parent/activity` since
-    # this would map to the activity-page of its parent.
-    GROUP_ROUTES = %w[
-      activity
-      analytics
-      audit_events
-      avatar
-      edit
-      group_members
-      hooks
-      issues
-      labels
-      ldap
-      ldap_group_links
-      merge_requests
-      milestones
-      notification_setting
-      pipeline_quota
-      projects
-      subgroups
-    ].freeze
-
-    ILLEGAL_PROJECT_PATH_WORDS = PROJECT_WILDCARD_ROUTES
-    ILLEGAL_GROUP_PATH_WORDS = (PROJECT_WILDCARD_ROUTES | GROUP_ROUTES).freeze
-
-    # The namespace regex is used in Javascript to validate usernames in the "Register" form. However, Javascript
-    # does not support the negative lookbehind assertion (?<!) that disallows usernames ending in `.git` and `.atom`.
-    # Since this is a non-trivial problem to solve in Javascript (heavily complicate the regex, modify view code to
-    # allow non-regex validatiions, etc), `NAMESPACE_REGEX_STR_JS` serves as a Javascript-compatible version of
-    # `NAMESPACE_REGEX_STR`, with the negative lookbehind assertion removed. This means that the client-side validation
-    # will pass for usernames ending in `.atom` and `.git`, but will be caught by the server-side validation.
-    PATH_REGEX_STR = '[a-zA-Z0-9_\.][a-zA-Z0-9_\-\.]*'.freeze
-    NAMESPACE_REGEX_STR_JS = PATH_REGEX_STR + '[a-zA-Z0-9_\-]|[a-zA-Z0-9_]'.freeze
-    NO_SUFFIX_REGEX_STR = '(?<!\.git|\.atom)'.freeze
-    NAMESPACE_REGEX_STR = "(?:#{NAMESPACE_REGEX_STR_JS})#{NO_SUFFIX_REGEX_STR}".freeze
-    PROJECT_REGEX_STR = "(?:#{PATH_REGEX_STR})#{NO_SUFFIX_REGEX_STR}".freeze
-
-    # Same as NAMESPACE_REGEX_STR but allows `/` in the path.
-    # So `group/subgroup` will match this regex but not NAMESPACE_REGEX_STR
-    FULL_NAMESPACE_REGEX_STR = "(?:#{NAMESPACE_REGEX_STR}/)*#{NAMESPACE_REGEX_STR}".freeze
-
-    def root_namespace_route_regex
-      @root_namespace_route_regex ||= begin
-        illegal_words = Regexp.new(Regexp.union(TOP_LEVEL_ROUTES).source, Regexp::IGNORECASE)
-
-        single_line_regexp %r{
-          (?!(#{illegal_words})/)
-          #{NAMESPACE_REGEX_STR}
-        }x
-      end
-    end
-
-    def root_namespace_path_regex
-      @root_namespace_path_regex ||= %r{\A#{root_namespace_route_regex}/\z}
-    end
-
-    def full_namespace_path_regex
-      @full_namespace_path_regex ||= %r{\A#{namespace_route_regex}/\z}
-    end
-
-    def full_project_path_regex
-      @full_project_path_regex ||= %r{\A#{namespace_route_regex}/#{project_route_regex}/\z}
-    end
-
-    def namespace_regex
-      @namespace_regex ||= /\A#{NAMESPACE_REGEX_STR}\z/.freeze
-    end
-
-    def full_namespace_regex
-      @full_namespace_regex ||= %r{\A#{FULL_NAMESPACE_REGEX_STR}\z}
-    end
-
-    def namespace_route_regex
-      @namespace_route_regex ||= begin
-        illegal_words = Regexp.new(Regexp.union(ILLEGAL_GROUP_PATH_WORDS).source, Regexp::IGNORECASE)
-
-        single_line_regexp %r{
-          #{root_namespace_route_regex}
-          (?:
-            /
-            (?!#{illegal_words}/)
-            #{NAMESPACE_REGEX_STR}
-          )*
-        }x
-      end
-    end
-
-    def namespace_regex_message
-      "can contain only letters, digits, '_', '-' and '.'. " \
-      "Cannot start with '-' or end in '.', '.git' or '.atom'." \
-    end
-
     def namespace_name_regex
       @namespace_name_regex ||= /\A[\p{Alnum}\p{Pd}_\. ]*\z/.freeze
     end
@@ -216,34 +19,6 @@ module Gitlab
       "It must start with letter, digit, emoji or '_'."
     end
 
-    def project_path_regex
-      @project_path_regex ||= %r{\A#{project_route_regex}/\z}
-    end
-
-    def project_route_regex
-      @project_route_regex ||= begin
-        illegal_words = Regexp.new(Regexp.union(ILLEGAL_PROJECT_PATH_WORDS).source, Regexp::IGNORECASE)
-
-        single_line_regexp %r{
-          (?!(#{illegal_words})/)
-          #{PROJECT_REGEX_STR}
-        }x
-      end
-    end
-
-    def project_git_route_regex
-      @project_git_route_regex ||= /#{project_route_regex}\.git/.freeze
-    end
-
-    def project_path_format_regex
-      @project_path_format_regex ||= /\A#{PROJECT_REGEX_STR}\z/.freeze
-    end
-
-    def project_path_regex_message
-      "can contain only letters, digits, '_', '-' and '.'. " \
-      "Cannot start with '-', end in '.git' or end in '.atom'" \
-    end
-
     def file_name_regex
       @file_name_regex ||= /\A[[[:alnum:]]_\-\.\@\+]*\z/.freeze
     end
@@ -252,36 +27,8 @@ module Gitlab
       "can contain only letters, digits, '_', '-', '@', '+' and '.'."
     end
 
-    def archive_formats_regex
-      #                           |zip|tar|    tar.gz    |         tar.bz2         |
-      @archive_formats_regex ||= /(zip|tar|tar\.gz|tgz|gz|tar\.bz2|tbz|tbz2|tb2|bz2)/.freeze
-    end
-
-    def git_reference_regex
-      # Valid git ref regex, see:
-      # https://www.kernel.org/pub/software/scm/git/docs/git-check-ref-format.html
-
-      @git_reference_regex ||= single_line_regexp %r{
-        (?!
-           (?# doesn't begins with)
-           \/|                    (?# rule #6)
-           (?# doesn't contain)
-           .*(?:
-              [\/.]\.|            (?# rule #1,3)
-              \/\/|               (?# rule #6)
-              @\{|                (?# rule #8)
-              \\                  (?# rule #9)
-           )
-        )
-        [^\000-\040\177~^:?*\[]+  (?# rule #4-5)
-        (?# doesn't end with)
-        (?<!\.lock)               (?# rule #1)
-        (?<![\/.])                (?# rule #6-7)
-      }x
-    end
-
     def container_registry_reference_regex
-      git_reference_regex
+      Gitlab::PathRegex.git_reference_regex
     end
 
     ##
@@ -315,13 +62,5 @@ module Gitlab
       "can contain only lowercase letters, digits, and '-'. " \
       "Must start with a letter, and cannot end with '-'"
     end
-
-    private
-
-    def single_line_regexp(regex)
-      # Turns a multiline extended regexp into a single line one,
-      # beacuse `rake routes` breaks on multiline regexes.
-      Regexp.new(regex.source.gsub(/\(\?#.+?\)/, '').gsub(/\s*/, ''), regex.options ^ Regexp::EXTENDED).freeze
-    end
   end
 end
diff --git a/lib/gitlab/routes/legacy_builds.rb b/lib/gitlab/routes/legacy_builds.rb
new file mode 100644
index 00000000000..36d1a8a6f64
--- /dev/null
+++ b/lib/gitlab/routes/legacy_builds.rb
@@ -0,0 +1,36 @@
+module Gitlab
+  module Routes
+    class LegacyBuilds
+      def initialize(map)
+        @map = map
+      end
+
+      def draw
+        @map.instance_eval do
+          resources :builds, only: [:index, :show], constraints: { id: /\d+/ } do
+            collection do
+              resources :artifacts, only: [], controller: 'build_artifacts' do
+                collection do
+                  get :latest_succeeded,
+                    path: '*ref_name_and_path',
+                    format: false
+                end
+              end
+            end
+
+            member do
+              get :raw
+            end
+
+            resource :artifacts, only: [], controller: 'build_artifacts' do
+              get :download
+              get :browse, path: 'browse(/*path)', format: false
+              get :file, path: 'file/*path', format: false
+              get :raw, path: 'raw/*path', format: false
+            end
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/sql/recursive_cte.rb b/lib/gitlab/sql/recursive_cte.rb
new file mode 100644
index 00000000000..5b1b03820a3
--- /dev/null
+++ b/lib/gitlab/sql/recursive_cte.rb
@@ -0,0 +1,62 @@
+module Gitlab
+  module SQL
+    # Class for easily building recursive CTE statements.
+    #
+    # Example:
+    #
+    #     cte = RecursiveCTE.new(:my_cte_name)
+    #     ns = Arel::Table.new(:namespaces)
+    #
+    #     cte << Namespace.
+    #       where(ns[:parent_id].eq(some_namespace_id))
+    #
+    #     cte << Namespace.
+    #       from([ns, cte.table]).
+    #       where(ns[:parent_id].eq(cte.table[:id]))
+    #
+    #     Namespace.with.
+    #       recursive(cte.to_arel).
+    #       from(cte.alias_to(ns))
+    class RecursiveCTE
+      attr_reader :table
+
+      # name - The name of the CTE as a String or Symbol.
+      def initialize(name)
+        @table = Arel::Table.new(name)
+        @queries = []
+      end
+
+      # Adds a query to the body of the CTE.
+      #
+      # relation - The relation object to add to the body of the CTE.
+      def <<(relation)
+        @queries << relation
+      end
+
+      # Returns the Arel relation for this CTE.
+      def to_arel
+        sql = Arel::Nodes::SqlLiteral.new(Union.new(@queries).to_sql)
+
+        Arel::Nodes::As.new(table, Arel::Nodes::Grouping.new(sql))
+      end
+
+      # Returns an "AS" statement that aliases the CTE name as the given table
+      # name. This allows one to trick ActiveRecord into thinking it's selecting
+      # from an actual table, when in reality it's selecting from a CTE.
+      #
+      # alias_table - The Arel table to use as the alias.
+      def alias_to(alias_table)
+        Arel::Nodes::As.new(table, alias_table)
+      end
+
+      # Applies the CTE to the given relation, returning a new one that will
+      # query from it.
+      def apply_to(relation)
+        relation.except(:where).
+          with.
+          recursive(to_arel).
+          from(alias_to(relation.model.arel_table))
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/url_sanitizer.rb b/lib/gitlab/url_sanitizer.rb
index 9ce13feb79a..c81dc7e30d0 100644
--- a/lib/gitlab/url_sanitizer.rb
+++ b/lib/gitlab/url_sanitizer.rb
@@ -18,12 +18,6 @@ module Gitlab
       false
     end
 
-    def self.http_credentials_for_user(user)
-      return {} unless user.respond_to?(:username)
-
-      { user: user.username }
-    end
-
     def initialize(url, credentials: nil)
       @url = Addressable::URI.parse(url.strip)
       @credentials = credentials
diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb
index fe37e4da94f..18d8b4f4744 100644
--- a/lib/gitlab/workhorse.rb
+++ b/lib/gitlab/workhorse.rb
@@ -31,8 +31,7 @@ module Gitlab
 
           feature_enabled = case action.to_s
                             when 'git_receive_pack'
-                              # Disabled for now, see https://gitlab.com/gitlab-org/gitaly/issues/172
-                              false
+                              Gitlab::GitalyClient.feature_enabled?(:post_receive_pack)
                             when 'git_upload_pack'
                               Gitlab::GitalyClient.feature_enabled?(:post_upload_pack)
                             when 'info_refs'
diff --git a/lib/support/init.d/gitlab b/lib/support/init.d/gitlab
index 6e351365de0..c5f93336346 100755
--- a/lib/support/init.d/gitlab
+++ b/lib/support/init.d/gitlab
@@ -48,7 +48,7 @@ gitlab_pages_pid_path="$pid_path/gitlab-pages.pid"
 gitlab_pages_options="-pages-domain example.com -pages-root $app_root/shared/pages -listen-proxy 127.0.0.1:8090"
 gitlab_pages_log="$app_root/log/gitlab-pages.log"
 shell_path="/bin/bash"
-gitaly_enabled=false
+gitaly_enabled=true
 gitaly_dir=$(cd $app_root/../gitaly 2> /dev/null && pwd)
 gitaly_pid_path="$pid_path/gitaly.pid"
 gitaly_log="$app_root/log/gitaly.log"
diff --git a/lib/support/init.d/gitlab.default.example b/lib/support/init.d/gitlab.default.example
index 9472c3c992f..295c79fccfc 100644
--- a/lib/support/init.d/gitlab.default.example
+++ b/lib/support/init.d/gitlab.default.example
@@ -86,5 +86,7 @@ mail_room_pid_path="$pid_path/mail_room.pid"
 shell_path="/bin/bash"
 
 # This variable controls whether the init script starts/stops Gitaly
-gitaly_enabled=false
+gitaly_enabled=true
+gitaly_dir=$(cd $app_root/../gitaly 2> /dev/null && pwd)
+gitaly_pid_path="$pid_path/gitaly.pid"
 gitaly_log="$app_root/log/gitaly.log"