Merge branch 'master' into feature/gb/manual-actions-protected-branches-permissions

* master: (641 commits)
  Revert "Fix registry for projects with uppercases in path"
  Fix registry for projects with uppercases in path
  Move event icons into events_helper
  Reset New branch button when issue state changes
  Add link to environments on kubernetes.md
  Indent system notes on desktop screens
  Improve webpack-dev-server compatibility with non-localhost setups.
  Add changelog entry
  Fix recent searches icon alignment in Safari
  Use preload to avoid Rails using JOIN
  Fix NUMBER_OF_TRUNCATED_DIFF_LINES re-definition error
  Prepare for zero downtime migrations
  Fix filtered search input width for IE
  Fix the `gitlab:gitlab_shell:check` task
  Fixed random failures with Poll spec
  Include CONTRIBUTING.md file when importing .gitlab-ci.yml templates
  Let uses hide verbose output by default
  Separate examples for each other
  Collapse similar sibling scenarios
  Use empty_project for resources that are independent of the repo
  ...

Conflicts:
	app/views/projects/ci/builds/_build.html.haml

84 files changed, 1814 insertions, 441 deletions

diff --git a/lib/api/entities.rb b/lib/api/entities.rb
index 00d44821e3f..9919762cd82 100644
--- a/lib/api/entities.rb
+++ b/lib/api/entities.rb
@@ -14,7 +14,7 @@ module API
 
     class User < UserBasic
       expose :created_at
-      expose :is_admin?, as: :is_admin
+      expose :admin?, as: :is_admin
       expose :bio, :location, :skype, :linkedin, :twitter, :website_url, :organization
     end
 
@@ -184,19 +184,15 @@ module API
       end
 
       expose :protected do |repo_branch, options|
-        options[:project].protected_branch?(repo_branch.name)
+        ProtectedBranch.protected?(options[:project], repo_branch.name)
       end
 
       expose :developers_can_push do |repo_branch, options|
-        project = options[:project]
-        access_levels = project.protected_branches.matching(repo_branch.name).map(&:push_access_levels).flatten
-        access_levels.any? { |access_level| access_level.access_level == Gitlab::Access::DEVELOPER }
+        options[:project].protected_branches.developers_can?(:push, repo_branch.name)
       end
 
       expose :developers_can_merge do |repo_branch, options|
-        project = options[:project]
-        access_levels = project.protected_branches.matching(repo_branch.name).map(&:merge_access_levels).flatten
-        access_levels.any? { |access_level| access_level.access_level == Gitlab::Access::DEVELOPER }
+        options[:project].protected_branches.developers_can?(:merge, repo_branch.name)
       end
     end
 
@@ -615,9 +611,9 @@ module API
       expose :locked
       expose :version, :revision, :platform, :architecture
       expose :contacted_at
-      expose :token, if: lambda { |runner, options| options[:current_user].is_admin? || !runner.is_shared? }
+      expose :token, if: lambda { |runner, options| options[:current_user].admin? || !runner.is_shared? }
       expose :projects, with: Entities::BasicProjectDetails do |runner, options|
-        if options[:current_user].is_admin?
+        if options[:current_user].admin?
           runner.projects
         else
           options[:current_user].authorized_projects.where(id: runner.projects)
diff --git a/lib/api/groups.rb b/lib/api/groups.rb
index 605769eddde..09d105f6b4c 100644
--- a/lib/api/groups.rb
+++ b/lib/api/groups.rb
@@ -5,11 +5,16 @@ module API
     before { authenticate! }
 
     helpers do
-      params :optional_params do
+      params :optional_params_ce do
         optional :description, type: String, desc: 'The description of the group'
         optional :visibility, type: String, values: Gitlab::VisibilityLevel.string_values, desc: 'The visibility of the group'
         optional :lfs_enabled, type: Boolean, desc: 'Enable/disable LFS for the projects in this group'
         optional :request_access_enabled, type: Boolean, desc: 'Allow users to request member access'
+        optional :share_with_group_lock, type: Boolean, desc: 'Prevent sharing a project with another group within this group'
+      end
+
+      params :optional_params do
+        use :optional_params_ce
       end
 
       params :statistics_params do
@@ -56,7 +61,7 @@ module API
         groups = groups.where.not(id: params[:skip_groups]) if params[:skip_groups].present?
         groups = groups.reorder(params[:order_by] => params[:sort])
 
-        present_groups groups, statistics: params[:statistics] && current_user.is_admin?
+        present_groups groups, statistics: params[:statistics] && current_user.admin?
       end
 
       desc 'Create a group. Available only for users who can create groups.' do
diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb
index 61527c1e20b..ddff3c8c1e8 100644
--- a/lib/api/helpers.rb
+++ b/lib/api/helpers.rb
@@ -118,7 +118,7 @@ module API
 
     def authenticated_as_admin!
       authenticate!
-      forbidden! unless current_user.is_admin?
+      forbidden! unless current_user.admin?
     end
 
     def authorize!(action, subject = :global)
@@ -358,7 +358,7 @@ module API
       return unless sudo_identifier
       return unless initial_current_user
 
-      unless initial_current_user.is_admin?
+      unless initial_current_user.admin?
         forbidden!('Must be admin to use sudo')
       end
 
diff --git a/lib/api/helpers/internal_helpers.rb b/lib/api/helpers/internal_helpers.rb
index 2135a787b11..810e5063996 100644
--- a/lib/api/helpers/internal_helpers.rb
+++ b/lib/api/helpers/internal_helpers.rb
@@ -53,12 +53,12 @@ module API
         ]
       end
 
-      def parse_allowed_environment_variables
-        return if params[:env].blank?
+      def parse_env
+        return {} if params[:env].blank?
 
         JSON.parse(params[:env])
-
       rescue JSON::ParserError
+        {}
       end
     end
   end
diff --git a/lib/api/internal.rb b/lib/api/internal.rb
index 56c597dffcb..215bc03d0e9 100644
--- a/lib/api/internal.rb
+++ b/lib/api/internal.rb
@@ -11,14 +11,16 @@ module API
       # Params:
       #   key_id - ssh key id for Git over SSH
       #   user_id - user id for Git over HTTP
+      #   protocol - Git access protocol being used, e.g. HTTP or SSH
       #   project - project path with namespace
       #   action - git action (git-upload-pack or git-receive-pack)
-      #   ref - branch name
-      #   forced_push - forced_push
-      #   protocol - Git access protocol being used, e.g. HTTP or SSH
+      #   changes - changes as "oldrev newrev ref", see Gitlab::ChangesList
       post "/allowed" do
         status 200
 
+        # Stores some Git-specific env thread-safely
+        Gitlab::Git::Env.set(parse_env)
+
         actor =
           if params[:key_id]
             Key.find_by(id: params[:key_id])
@@ -30,18 +32,10 @@ module API
 
         actor.update_last_used_at if actor.is_a?(Key)
 
-        access =
-          if wiki?
-            Gitlab::GitAccessWiki.new(actor, project, protocol, authentication_abilities: ssh_authentication_abilities)
-          else
-            Gitlab::GitAccess.new(actor,
-                                  project,
-                                  protocol,
-                                  authentication_abilities: ssh_authentication_abilities,
-                                  env: parse_allowed_environment_variables)
-          end
-
-        access_status = access.check(params[:action], params[:changes])
+        access_checker = wiki? ? Gitlab::GitAccessWiki : Gitlab::GitAccess
+        access_status = access_checker
+          .new(actor, project, protocol, authentication_abilities: ssh_authentication_abilities)
+          .check(params[:action], params[:changes])
 
         response = { status: access_status.status, message: access_status.message }
 
@@ -142,7 +136,7 @@ module API
         project = Project.find_by_full_path(relative_path.sub(/\.(git|wiki)\z/, ''))
 
         begin
-          Gitlab::GitalyClient::Notifications.new(project.repository_storage, relative_path).post_receive
+          Gitlab::GitalyClient::Notifications.new(project.repository).post_receive
         rescue GRPC::Unavailable => e
           render_api_error(e, 500)
         end
diff --git a/lib/api/issues.rb b/lib/api/issues.rb
index 09053e615cb..05423c17449 100644
--- a/lib/api/issues.rb
+++ b/lib/api/issues.rb
@@ -30,7 +30,7 @@ module API
         use :pagination
       end
 
-      params :issue_params do
+      params :issue_params_ce do
         optional :description, type: String, desc: 'The description of an issue'
         optional :assignee_id, type: Integer, desc: 'The ID of a user to assign issue'
         optional :milestone_id, type: Integer, desc: 'The ID of a milestone to assign issue'
@@ -38,6 +38,10 @@ module API
         optional :due_date, type: String, desc: 'Date time string in the format YEAR-MONTH-DAY'
         optional :confidential, type: Boolean, desc: 'Boolean parameter if the issue should be confidential'
       end
+
+      params :issue_params do
+        use :issue_params_ce
+      end
     end
 
     resource :issues do
diff --git a/lib/api/jobs.rb b/lib/api/jobs.rb
index ffab0aafe59..288b03d940c 100644
--- a/lib/api/jobs.rb
+++ b/lib/api/jobs.rb
@@ -118,7 +118,7 @@ module API
         content_type 'text/plain'
         env['api.format'] = :binary
 
-        trace = build.trace
+        trace = build.trace.raw
         body trace
       end
 
diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb
index c8033664133..cb7aec47cf0 100644
--- a/lib/api/merge_requests.rb
+++ b/lib/api/merge_requests.rb
@@ -33,13 +33,17 @@ module API
           end
         end
 
-        params :optional_params do
+        params :optional_params_ce do
           optional :description, type: String, desc: 'The description of the merge request'
           optional :assignee_id, type: Integer, desc: 'The ID of a user to assign the merge request'
           optional :milestone_id, type: Integer, desc: 'The ID of a milestone to assign the merge request'
           optional :labels, type: String, desc: 'Comma-separated list of label names'
           optional :remove_source_branch, type: Boolean, desc: 'Remove source branch when merging'
         end
+
+        params :optional_params do
+          use :optional_params_ce
+        end
       end
 
       desc 'List merge requests' do
@@ -145,14 +149,24 @@ module API
         success Entities::MergeRequest
       end
       params do
+        # CE
+        at_least_one_of_ce = [
+          :assignee_id,
+          :description,
+          :labels,
+          :milestone_id,
+          :remove_source_branch,
+          :state_event,
+          :target_branch,
+          :title
+        ]
         optional :title, type: String, allow_blank: false, desc: 'The title of the merge request'
         optional :target_branch, type: String, allow_blank: false, desc: 'The target branch'
         optional :state_event, type: String, values: %w[close reopen],
                                desc: 'Status of the merge request'
+
         use :optional_params
-        at_least_one_of :title, :target_branch, :description, :assignee_id,
-                        :milestone_id, :labels, :state_event,
-                        :remove_source_branch
+        at_least_one_of(*at_least_one_of_ce)
       end
       put ':id/merge_requests/:merge_request_iid' do
         merge_request = find_merge_request_with_access(params.delete(:merge_request_iid), :update_merge_request)
@@ -173,6 +187,7 @@ module API
         success Entities::MergeRequest
       end
       params do
+        # CE
         optional :merge_commit_message, type: String, desc: 'Custom merge commit message'
         optional :should_remove_source_branch, type: Boolean,
                                                desc: 'When true, the source branch will be deleted if possible'
diff --git a/lib/api/notes.rb b/lib/api/notes.rb
index de39e579ac3..e281e3230fd 100644
--- a/lib/api/notes.rb
+++ b/lib/api/notes.rb
@@ -78,7 +78,7 @@ module API
           }
 
           if can?(current_user, noteable_read_ability_name(noteable), noteable)
-            if params[:created_at] && (current_user.is_admin? || user_project.owner == current_user)
+            if params[:created_at] && (current_user.admin? || user_project.owner == current_user)
               opts[:created_at] = params[:created_at]
             end
 
diff --git a/lib/api/project_hooks.rb b/lib/api/project_hooks.rb
index 53791166c33..87dfd1573a4 100644
--- a/lib/api/project_hooks.rb
+++ b/lib/api/project_hooks.rb
@@ -13,7 +13,7 @@ module API
         optional :merge_requests_events, type: Boolean, desc: "Trigger hook on merge request events"
         optional :tag_push_events, type: Boolean, desc: "Trigger hook on tag push events"
         optional :note_events, type: Boolean, desc: "Trigger hook on note(comment) events"
-        optional :build_events, type: Boolean, desc: "Trigger hook on build events"
+        optional :job_events, type: Boolean, desc: "Trigger hook on job events"
         optional :pipeline_events, type: Boolean, desc: "Trigger hook on pipeline events"
         optional :wiki_page_events, type: Boolean, desc: "Trigger hook on wiki events"
         optional :enable_ssl_verification, type: Boolean, desc: "Do SSL verification when triggering the hook"
@@ -53,7 +53,10 @@ module API
         use :project_hook_properties
       end
       post ":id/hooks" do
-        hook = user_project.hooks.new(declared_params(include_missing: false))
+        hook_params = declared_params(include_missing: false)
+        hook_params[:build_events] = hook_params.delete(:job_events) { false }
+
+        hook = user_project.hooks.new(hook_params)
 
         if hook.save
           present hook, with: Entities::ProjectHook
@@ -74,7 +77,10 @@ module API
       put ":id/hooks/:hook_id" do
         hook = user_project.hooks.find(params.delete(:hook_id))
 
-        if hook.update_attributes(declared_params(include_missing: false))
+        update_params = declared_params(include_missing: false)
+        update_params[:build_events] = update_params.delete(:job_events) if update_params[:job_events]
+
+        if hook.update_attributes(update_params)
           present hook, with: Entities::ProjectHook
         else
           error!("Invalid url given", 422) if hook.errors[:url].present?
diff --git a/lib/api/projects.rb b/lib/api/projects.rb
index 766fbea53e6..50842370947 100644
--- a/lib/api/projects.rb
+++ b/lib/api/projects.rb
@@ -6,7 +6,7 @@ module API
     before { authenticate_non_get! }
 
     helpers do
-      params :optional_params do
+      params :optional_params_ce do
         optional :description, type: String, desc: 'The description of the project'
         optional :issues_enabled, type: Boolean, desc: 'Flag indication if the issue tracker is enabled'
         optional :merge_requests_enabled, type: Boolean, desc: 'Flag indication if merge requests are enabled'
@@ -22,6 +22,10 @@ module API
         optional :only_allow_merge_if_pipeline_succeeds, type: Boolean, desc: 'Only allow to merge if builds succeed'
         optional :only_allow_merge_if_all_discussions_are_resolved, type: Boolean, desc: 'Only allow to merge if all discussions are resolved'
       end
+
+      params :optional_params do
+        use :optional_params_ce
+      end
     end
 
     resource :projects do
@@ -198,17 +202,33 @@ module API
         success Entities::Project
       end
       params do
+        # CE
+        at_least_one_of_ce =
+          [
+            :builds_enabled,
+            :container_registry_enabled,
+            :default_branch,
+            :description,
+            :issues_enabled,
+            :lfs_enabled,
+            :merge_requests_enabled,
+            :name,
+            :only_allow_merge_if_all_discussions_are_resolved,
+            :only_allow_merge_if_pipeline_succeeds,
+            :path,
+            :public_builds,
+            :request_access_enabled,
+            :shared_runners_enabled,
+            :snippets_enabled,
+            :visibility,
+            :wiki_enabled,
+          ]
         optional :name, type: String, desc: 'The name of the project'
         optional :default_branch, type: String, desc: 'The default branch of the project'
         optional :path, type: String, desc: 'The path of the repository'
+
         use :optional_params
-        at_least_one_of :name, :description, :issues_enabled, :merge_requests_enabled,
-                        :wiki_enabled, :builds_enabled, :snippets_enabled,
-                        :shared_runners_enabled, :container_registry_enabled,
-                        :lfs_enabled, :visibility, :public_builds,
-                        :request_access_enabled, :only_allow_merge_if_pipeline_succeeds,
-                        :only_allow_merge_if_all_discussions_are_resolved, :path,
-                        :default_branch
+        at_least_one_of(*at_least_one_of_ce)
       end
       put ':id' do
         authorize_admin_project
diff --git a/lib/api/runner.rb b/lib/api/runner.rb
index d288369e362..6fbb02cb3aa 100644
--- a/lib/api/runner.rb
+++ b/lib/api/runner.rb
@@ -115,7 +115,7 @@ module API
       put '/:id' do
         job = authenticate_job!
 
-        job.update_attributes(trace: params[:trace]) if params[:trace]
+        job.trace.set(params[:trace]) if params[:trace]
 
         Gitlab::Metrics.add_event(:update_build,
                                   project: job.project.path_with_namespace)
@@ -145,16 +145,14 @@ module API
         content_range = request.headers['Content-Range']
         content_range = content_range.split('-')
 
-        current_length = job.trace_length
-        unless current_length == content_range[0].to_i
-          return error!('416 Range Not Satisfiable', 416, { 'Range' => "0-#{current_length}" })
+        stream_size = job.trace.append(request.body.read, content_range[0].to_i)
+        if stream_size < 0
+          return error!('416 Range Not Satisfiable', 416, { 'Range' => "0-#{-stream_size}" })
         end
 
-        job.append_trace(request.body.read, content_range[0].to_i)
-
         status 202
         header 'Job-Status', job.status
-        header 'Range', "0-#{job.trace_length}"
+        header 'Range', "0-#{stream_size}"
       end
 
       desc 'Authorize artifacts uploading for job' do
diff --git a/lib/api/runners.rb b/lib/api/runners.rb
index a77c876a749..db6c7c59092 100644
--- a/lib/api/runners.rb
+++ b/lib/api/runners.rb
@@ -161,18 +161,18 @@ module API
       end
 
       def authenticate_show_runner!(runner)
-        return if runner.is_shared || current_user.is_admin?
+        return if runner.is_shared || current_user.admin?
         forbidden!("No access granted") unless user_can_access_runner?(runner)
       end
 
       def authenticate_update_runner!(runner)
-        return if current_user.is_admin?
+        return if current_user.admin?
         forbidden!("Runner is shared") if runner.is_shared?
         forbidden!("No access granted") unless user_can_access_runner?(runner)
       end
 
       def authenticate_delete_runner!(runner)
-        return if current_user.is_admin?
+        return if current_user.admin?
         forbidden!("Runner is shared") if runner.is_shared?
         forbidden!("Runner associated with more than one project") if runner.projects.count > 1
         forbidden!("No access granted") unless user_can_access_runner?(runner)
@@ -181,7 +181,7 @@ module API
       def authenticate_enable_runner!(runner)
         forbidden!("Runner is shared") if runner.is_shared?
         forbidden!("Runner is locked") if runner.locked?
-        return if current_user.is_admin?
+        return if current_user.admin?
         forbidden!("No access granted") unless user_can_access_runner?(runner)
       end
 
diff --git a/lib/api/services.rb b/lib/api/services.rb
index 6802a99311e..23ef62c2258 100644
--- a/lib/api/services.rb
+++ b/lib/api/services.rb
@@ -488,6 +488,14 @@ module API
           desc: 'The channel name'
         }
       ],
+      'microsoft-teams' => [
+        {
+          required: true,
+          name: :webhook,
+          type: String,
+          desc: 'The Microsoft Teams webhook. e.g. https://outlook.office.com/webhook/…'
+        }
+      ],
       'mattermost' => [
         {
           required: true,
@@ -550,6 +558,7 @@ module API
       RedmineService,
       SlackService,
       MattermostService,
+      MicrosoftTeamsService,
       TeamcityService,
     ]
 
@@ -633,7 +642,7 @@ module API
           service_params = declared_params(include_missing: false).merge(active: true)
 
           if service.update_attributes(service_params)
-            present service, with: Entities::ProjectService, include_passwords: current_user.is_admin?
+            present service, with: Entities::ProjectService, include_passwords: current_user.admin?
           else
             render_api_error!('400 Bad Request', 400)
           end
@@ -664,7 +673,7 @@ module API
       end
       get ":id/services/:service_slug" do
         service = user_project.find_or_initialize_service(params[:service_slug].underscore)
-        present service, with: Entities::ProjectService, include_passwords: current_user.is_admin?
+        present service, with: Entities::ProjectService, include_passwords: current_user.admin?
       end
     end
 
diff --git a/lib/api/settings.rb b/lib/api/settings.rb
index c7f97ad2aab..d01c7f2703b 100644
--- a/lib/api/settings.rb
+++ b/lib/api/settings.rb
@@ -20,6 +20,55 @@ module API
       success Entities::ApplicationSetting
     end
     params do
+      # CE
+      at_least_one_of_ce = [
+        :admin_notification_email,
+        :after_sign_out_path,
+        :after_sign_up_text,
+        :akismet_enabled,
+        :container_registry_token_expire_delay,
+        :default_artifacts_expire_in,
+        :default_branch_protection,
+        :default_group_visibility,
+        :default_project_visibility,
+        :default_projects_limit,
+        :default_snippet_visibility,
+        :disabled_oauth_sign_in_sources,
+        :domain_blacklist_enabled,
+        :domain_whitelist,
+        :email_author_in_body,
+        :enabled_git_access_protocol,
+        :gravatar_enabled,
+        :help_page_text,
+        :home_page_url,
+        :housekeeping_enabled,
+        :html_emails_enabled,
+        :import_sources,
+        :koding_enabled,
+        :max_artifacts_size,
+        :max_attachment_size,
+        :max_pages_size,
+        :metrics_enabled,
+        :plantuml_enabled,
+        :polling_interval_multiplier,
+        :recaptcha_enabled,
+        :repository_checks_enabled,
+        :repository_storage,
+        :require_two_factor_authentication,
+        :restricted_visibility_levels,
+        :send_user_confirmation_email,
+        :sentry_enabled,
+        :session_expire_delay,
+        :shared_runners_enabled,
+        :sidekiq_throttling_enabled,
+        :sign_in_text,
+        :signin_enabled,
+        :signup_enabled,
+        :terminal_max_session_time,
+        :user_default_external,
+        :user_oauth_applications,
+        :version_check_enabled
+      ]
       optional :default_branch_protection, type: Integer, values: [0, 1, 2], desc: 'Determine if developers can push to master'
       optional :default_project_visibility, type: String, values: Gitlab::VisibilityLevel.string_values, desc: 'The default project visibility'
       optional :default_snippet_visibility, type: String, values: Gitlab::VisibilityLevel.string_values, desc: 'The default snippet visibility'
@@ -111,22 +160,8 @@ module API
       end
       optional :terminal_max_session_time, type: Integer, desc: 'Maximum time for web terminal websocket connection (in seconds). Set to 0 for unlimited time.'
       optional :polling_interval_multiplier, type: BigDecimal, desc: 'Interval multiplier used by endpoints that perform polling. Set to 0 to disable polling.'
-      at_least_one_of :default_branch_protection, :default_project_visibility, :default_snippet_visibility,
-                      :default_group_visibility, :restricted_visibility_levels, :import_sources,
-                      :enabled_git_access_protocol, :gravatar_enabled, :default_projects_limit,
-                      :max_attachment_size, :session_expire_delay, :disabled_oauth_sign_in_sources,
-                      :user_oauth_applications, :user_default_external, :signup_enabled,
-                      :send_user_confirmation_email, :domain_whitelist, :domain_blacklist_enabled,
-                      :after_sign_up_text, :signin_enabled, :require_two_factor_authentication,
-                      :home_page_url, :after_sign_out_path, :sign_in_text, :help_page_text,
-                      :shared_runners_enabled, :max_artifacts_size,
-                      :default_artifacts_expire_in, :max_pages_size,
-                      :container_registry_token_expire_delay,
-                      :metrics_enabled, :sidekiq_throttling_enabled, :recaptcha_enabled,
-                      :akismet_enabled, :admin_notification_email, :sentry_enabled,
-                      :repository_storage, :repository_checks_enabled, :koding_enabled, :plantuml_enabled,
-                      :version_check_enabled, :email_author_in_body, :html_emails_enabled,
-                      :housekeeping_enabled, :terminal_max_session_time, :polling_interval_multiplier
+
+      at_least_one_of(*at_least_one_of_ce)
     end
     put "application/settings" do
       attrs = declared_params(include_missing: false)
diff --git a/lib/api/users.rb b/lib/api/users.rb
index 992a751b37d..eedc59f8636 100644
--- a/lib/api/users.rb
+++ b/lib/api/users.rb
@@ -37,11 +37,13 @@ module API
         success Entities::UserBasic
       end
       params do
+        # CE
         optional :username, type: String, desc: 'Get a single user with a specific username'
         optional :search, type: String, desc: 'Search for a username'
         optional :active, type: Boolean, default: false, desc: 'Filters only active users'
         optional :external, type: Boolean, default: false, desc: 'Filters only external users'
         optional :blocked, type: Boolean, default: false, desc: 'Filters only blocked users'
+
         use :pagination
       end
       get do
@@ -56,10 +58,10 @@ module API
           users = users.active if params[:active]
           users = users.search(params[:search]) if params[:search].present?
           users = users.blocked if params[:blocked]
-          users = users.external if params[:external] && current_user.is_admin?
+          users = users.external if params[:external] && current_user.admin?
         end
 
-        entity = current_user.is_admin? ? Entities::UserPublic : Entities::UserBasic
+        entity = current_user.admin? ? Entities::UserPublic : Entities::UserBasic
         present paginate(users), with: entity
       end
 
@@ -73,7 +75,7 @@ module API
         user = User.find_by(id: params[:id])
         not_found!('User') unless user
 
-        if current_user && current_user.is_admin?
+        if current_user && current_user.admin?
           present user, with: Entities::UserPublic
         elsif can?(current_user, :read_user, user)
           present user, with: Entities::User
diff --git a/lib/api/v3/builds.rb b/lib/api/v3/builds.rb
index 6f97102c6ef..4dd03cdf24b 100644
--- a/lib/api/v3/builds.rb
+++ b/lib/api/v3/builds.rb
@@ -120,7 +120,7 @@ module API
           content_type 'text/plain'
           env['api.format'] = :binary
 
-          trace = build.trace
+          trace = build.trace.raw
           body trace
         end
 
diff --git a/lib/api/v3/groups.rb b/lib/api/v3/groups.rb
index 9b27411ae21..63d464b926b 100644
--- a/lib/api/v3/groups.rb
+++ b/lib/api/v3/groups.rb
@@ -54,7 +54,7 @@ module API
           groups = groups.where.not(id: params[:skip_groups]) if params[:skip_groups].present?
           groups = groups.reorder(params[:order_by] => params[:sort])
 
-          present_groups groups, statistics: params[:statistics] && current_user.is_admin?
+          present_groups groups, statistics: params[:statistics] && current_user.admin?
         end
 
         desc 'Get list of owned groups for authenticated user' do
diff --git a/lib/api/v3/notes.rb b/lib/api/v3/notes.rb
index 4f8e0eff4ff..009ec5c6bbd 100644
--- a/lib/api/v3/notes.rb
+++ b/lib/api/v3/notes.rb
@@ -79,7 +79,7 @@ module API
             noteable = user_project.send(noteables_str.to_sym).find(params[:noteable_id])
 
             if can?(current_user, noteable_read_ability_name(noteable), noteable)
-              if params[:created_at] && (current_user.is_admin? || user_project.owner == current_user)
+              if params[:created_at] && (current_user.admin? || user_project.owner == current_user)
                 opts[:created_at] = params[:created_at]
               end
 
diff --git a/lib/api/v3/runners.rb b/lib/api/v3/runners.rb
index 1934d6e578c..faa265f3314 100644
--- a/lib/api/v3/runners.rb
+++ b/lib/api/v3/runners.rb
@@ -50,7 +50,7 @@ module API
 
       helpers do
         def authenticate_delete_runner!(runner)
-          return if current_user.is_admin?
+          return if current_user.admin?
           forbidden!("Runner is shared") if runner.is_shared?
           forbidden!("Runner associated with more than one project") if runner.projects.count > 1
           forbidden!("No access granted") unless user_can_access_runner?(runner)
diff --git a/lib/api/v3/services.rb b/lib/api/v3/services.rb
index 3bacaeee032..61629a04174 100644
--- a/lib/api/v3/services.rb
+++ b/lib/api/v3/services.rb
@@ -501,6 +501,12 @@ module API
             desc: 'The channel name'
           }
         ],
+        'microsoft-teams' => [
+          required: true,
+          name: :webhook,
+          type: String,
+          desc: 'The Microsoft Teams webhook. e.g. https://outlook.office.com/webhook/…'
+        ],
         'mattermost' => [
           {
             required: true,
@@ -596,7 +602,7 @@ module API
         end
         get ":id/services/:service_slug" do
           service = user_project.find_or_initialize_service(params[:service_slug].underscore)
-          present service, with: Entities::ProjectService, include_passwords: current_user.is_admin?
+          present service, with: Entities::ProjectService, include_passwords: current_user.admin?
         end
       end
 
diff --git a/lib/banzai/filter/issuable_state_filter.rb b/lib/banzai/filter/issuable_state_filter.rb
new file mode 100644
index 00000000000..6b78aa795b4
--- /dev/null
+++ b/lib/banzai/filter/issuable_state_filter.rb
@@ -0,0 +1,35 @@
+module Banzai
+  module Filter
+    # HTML filter that appends state information to issuable links.
+    # Runs as a post-process filter as issuable state might change whilst
+    # Markdown is in the cache.
+    #
+    # This filter supports cross-project references.
+    class IssuableStateFilter < HTML::Pipeline::Filter
+      VISIBLE_STATES = %w(closed merged).freeze
+
+      def call
+        extractor = Banzai::IssuableExtractor.new(project, current_user)
+        issuables = extractor.extract([doc])
+
+        issuables.each do |node, issuable|
+          if VISIBLE_STATES.include?(issuable.state)
+            node.children.last.content += " [#{issuable.state}]"
+          end
+        end
+
+        doc
+      end
+
+      private
+
+      def current_user
+        context[:current_user]
+      end
+
+      def project
+        context[:project]
+      end
+    end
+  end
+end
diff --git a/lib/banzai/filter/redactor_filter.rb b/lib/banzai/filter/redactor_filter.rb
index c59a80dd1c7..9f9882b3b40 100644
--- a/lib/banzai/filter/redactor_filter.rb
+++ b/lib/banzai/filter/redactor_filter.rb
@@ -7,7 +7,7 @@ module Banzai
     #
     class RedactorFilter < HTML::Pipeline::Filter
       def call
-        Redactor.new(project, current_user).redact([doc])
+        Redactor.new(project, current_user).redact([doc]) unless context[:skip_redaction]
 
         doc
       end
diff --git a/lib/banzai/issuable_extractor.rb b/lib/banzai/issuable_extractor.rb
new file mode 100644
index 00000000000..c5ce360e172
--- /dev/null
+++ b/lib/banzai/issuable_extractor.rb
@@ -0,0 +1,36 @@
+module Banzai
+  # Extract references to issuables from multiple documents
+
+  # This populates RequestStore cache used in Banzai::ReferenceParser::IssueParser
+  # and Banzai::ReferenceParser::MergeRequestParser
+  # Populating the cache should happen before processing documents one-by-one
+  # so we can avoid N+1 queries problem
+
+  class IssuableExtractor
+    QUERY = %q(
+      descendant-or-self::a[contains(concat(" ", @class, " "), " gfm ")]
+      [@data-reference-type="issue" or @data-reference-type="merge_request"]
+    ).freeze
+
+    attr_reader :project, :user
+
+    def initialize(project, user)
+      @project = project
+      @user = user
+    end
+
+    # Returns Hash in the form { node => issuable_instance }
+    def extract(documents)
+      nodes = documents.flat_map do |document|
+        document.xpath(QUERY)
+      end
+
+      issue_parser = Banzai::ReferenceParser::IssueParser.new(project, user)
+      merge_request_parser = Banzai::ReferenceParser::MergeRequestParser.new(project, user)
+
+      issue_parser.issues_for_nodes(nodes).merge(
+        merge_request_parser.merge_requests_for_nodes(nodes)
+      )
+    end
+  end
+end
diff --git a/lib/banzai/object_renderer.rb b/lib/banzai/object_renderer.rb
index 9f8eb0931b8..002a3341ccd 100644
--- a/lib/banzai/object_renderer.rb
+++ b/lib/banzai/object_renderer.rb
@@ -31,7 +31,8 @@ module Banzai
     #
     # Returns the same input objects.
     def render(objects, attribute)
-      documents = render_objects(objects, attribute)
+      documents = render_documents(objects, attribute)
+      documents = post_process_documents(documents, objects, attribute)
       redacted = redact_documents(documents)
 
       objects.each_with_index do |object, index|
@@ -41,9 +42,24 @@ module Banzai
       end
     end
 
-    # Renders the attribute of every given object.
-    def render_objects(objects, attribute)
-      render_attributes(objects, attribute)
+    private
+
+    def render_documents(objects, attribute)
+      pipeline = HTML::Pipeline.new([])
+
+      objects.map do |object|
+        pipeline.to_document(Banzai.render_field(object, attribute))
+      end
+    end
+
+    def post_process_documents(documents, objects, attribute)
+      # Called here to populate cache, refer to IssuableExtractor docs
+      IssuableExtractor.new(project, user).extract(documents)
+
+      documents.zip(objects).map do |document, object|
+        context = context_for(object, attribute)
+        Banzai::Pipeline[:post_process].to_document(document, context)
+      end
     end
 
     # Redacts the list of documents.
@@ -57,25 +73,15 @@ module Banzai
 
     # Returns a Banzai context for the given object and attribute.
     def context_for(object, attribute)
-      context = base_context.dup
-      context = context.merge(object.banzai_render_context(attribute))
-      context
-    end
-
-    # Renders the attributes of a set of objects.
-    #
-    # Returns an Array of `Nokogiri::HTML::Document`.
-    def render_attributes(objects, attribute)
-      objects.map do |object|
-        string = Banzai.render_field(object, attribute)
-        context = context_for(object, attribute)
-
-        Banzai::Pipeline[:relative_link].to_document(string, context)
-      end
+      base_context.merge(object.banzai_render_context(attribute))
     end
 
     def base_context
-      @base_context ||= @redaction_context.merge(current_user: user, project: project)
+      @base_context ||= @redaction_context.merge(
+        current_user: user,
+        project: project,
+        skip_redaction: true
+      )
     end
   end
 end
diff --git a/lib/banzai/pipeline/post_process_pipeline.rb b/lib/banzai/pipeline/post_process_pipeline.rb
index ecff094b1e5..131ac3b0eec 100644
--- a/lib/banzai/pipeline/post_process_pipeline.rb
+++ b/lib/banzai/pipeline/post_process_pipeline.rb
@@ -4,6 +4,7 @@ module Banzai
       def self.filters
         FilterArray[
           Filter::RelativeLinkFilter,
+          Filter::IssuableStateFilter,
           Filter::RedactorFilter
         ]
       end
diff --git a/lib/banzai/reference_parser/base_parser.rb b/lib/banzai/reference_parser/base_parser.rb
index 52fdb9a2140..dabf71d6aeb 100644
--- a/lib/banzai/reference_parser/base_parser.rb
+++ b/lib/banzai/reference_parser/base_parser.rb
@@ -62,8 +62,7 @@ module Banzai
 
         nodes.select do |node|
           if node.has_attribute?(project_attr)
-            node_id = node.attr(project_attr).to_i
-            can_read_reference?(user, projects[node_id])
+            can_read_reference?(user, projects[node])
           else
             true
           end
@@ -112,12 +111,12 @@ module Banzai
         per_project
       end
 
-      # Returns a Hash containing objects for an attribute grouped per their
-      # IDs.
+      # Returns a Hash containing objects for an attribute grouped per the
+      # nodes that reference them.
       #
       # The returned Hash uses the following format:
       #
-      #     { id value => row }
+      #     { node => row }
       #
       # nodes - An Array of HTML nodes to process.
       #
@@ -132,9 +131,14 @@ module Banzai
         return {} if nodes.empty?
 
         ids = unique_attribute_values(nodes, attribute)
-        rows = collection_objects_for_ids(collection, ids)
+        collection_objects = collection_objects_for_ids(collection, ids)
+        objects_by_id = collection_objects.index_by(&:id)
 
-        rows.index_by(&:id)
+        nodes.each_with_object({}) do |node, hash|
+          if node.has_attribute?(attribute)
+            hash[node] = objects_by_id[node.attr(attribute).to_i]
+          end
+        end
       end
 
       # Returns an Array containing all unique values of an attribute of the
@@ -201,7 +205,7 @@ module Banzai
       #
       # The returned Hash uses the following format:
       #
-      #     { project ID => project }
+      #     { node => project }
       #
       def projects_for_nodes(nodes)
         @projects_for_nodes ||=
diff --git a/lib/banzai/reference_parser/issue_parser.rb b/lib/banzai/reference_parser/issue_parser.rb
index 6c20dec5734..e02b360924a 100644
--- a/lib/banzai/reference_parser/issue_parser.rb
+++ b/lib/banzai/reference_parser/issue_parser.rb
@@ -13,14 +13,14 @@ module Banzai
           issues_readable_by_user(issues.values, user).to_set
 
         nodes.select do |node|
-          readable_issues.include?(issue_for_node(issues, node))
+          readable_issues.include?(issues[node])
         end
       end
 
       def referenced_by(nodes)
         issues = issues_for_nodes(nodes)
 
-        nodes.map { |node| issue_for_node(issues, node) }.uniq
+        nodes.map { |node| issues[node] }.compact.uniq
       end
 
       def issues_for_nodes(nodes)
@@ -44,12 +44,6 @@ module Banzai
           self.class.data_attribute
         )
       end
-
-      private
-
-      def issue_for_node(issues, node)
-        issues[node.attr(self.class.data_attribute).to_i]
-      end
     end
   end
 end
diff --git a/lib/banzai/reference_parser/merge_request_parser.rb b/lib/banzai/reference_parser/merge_request_parser.rb
index 40451947e6c..84a28b33d7c 100644
--- a/lib/banzai/reference_parser/merge_request_parser.rb
+++ b/lib/banzai/reference_parser/merge_request_parser.rb
@@ -3,14 +3,41 @@ module Banzai
     class MergeRequestParser < BaseParser
       self.reference_type = :merge_request
 
-      def references_relation
-        MergeRequest.includes(:author, :assignee, :target_project)
+      def nodes_visible_to_user(user, nodes)
+        merge_requests = merge_requests_for_nodes(nodes)
+
+        nodes.select do |node|
+          merge_request = merge_requests[node]
+
+          merge_request && can?(user, :read_merge_request, merge_request.project)
+        end
       end
 
-      private
+      def referenced_by(nodes)
+        merge_requests = merge_requests_for_nodes(nodes)
+
+        nodes.map { |node| merge_requests[node] }.compact.uniq
+      end
 
-      def can_read_reference?(user, ref_project)
-        can?(user, :read_merge_request, ref_project)
+      def merge_requests_for_nodes(nodes)
+        @merge_requests_for_nodes ||= grouped_objects_for_nodes(
+          nodes,
+          MergeRequest.includes(
+            :author,
+            :assignee,
+            {
+              # These associations are primarily used for checking permissions.
+              # Eager loading these ensures we don't end up running dozens of
+              # queries in this process.
+              target_project: [
+                { namespace: :owner },
+                { group: [:owners, :group_members] },
+                :invited_groups,
+                :project_members
+              ]
+            }),
+          self.class.data_attribute
+        )
       end
     end
   end
diff --git a/lib/banzai/reference_parser/user_parser.rb b/lib/banzai/reference_parser/user_parser.rb
index 7adaffa19c1..09b66cbd8fb 100644
--- a/lib/banzai/reference_parser/user_parser.rb
+++ b/lib/banzai/reference_parser/user_parser.rb
@@ -49,7 +49,7 @@ module Banzai
       # Check if project belongs to a group which
       # user can read.
       def can_read_group_reference?(node, user, groups)
-        node_group = groups[node.attr('data-group').to_i]
+        node_group = groups[node]
 
         node_group && can?(user, :read_group, node_group)
       end
@@ -74,8 +74,8 @@ module Banzai
           if project && project_id && project.id == project_id.to_i
             true
           elsif project_id && user_id
-            project = projects[project_id.to_i]
-            user = users[user_id.to_i]
+            project = projects[node]
+            user = users[node]
 
             project && user ? project.team.member?(user) : false
           else
diff --git a/lib/bitbucket/representation/base.rb b/lib/bitbucket/representation/base.rb
index 94adaacc9b5..800d5a075c6 100644
--- a/lib/bitbucket/representation/base.rb
+++ b/lib/bitbucket/representation/base.rb
@@ -1,6 +1,8 @@
 module Bitbucket
   module Representation
     class Base
+      attr_reader :raw
+
       def initialize(raw)
         @raw = raw
       end
@@ -8,10 +10,6 @@ module Bitbucket
       def self.decorate(entries)
         entries.map { |entry| new(entry)}
       end
-
-      private
-
-      attr_reader :raw
     end
   end
 end
diff --git a/lib/ci/ansi2html.rb b/lib/ci/ansi2html.rb
index b3ccad7b28d..1020452480a 100644
--- a/lib/ci/ansi2html.rb
+++ b/lib/ci/ansi2html.rb
@@ -132,34 +132,54 @@ module Ci
 
       STATE_PARAMS = [:offset, :n_open_tags, :fg_color, :bg_color, :style_mask].freeze
 
-      def convert(raw, new_state)
+      def convert(stream, new_state)
         reset_state
-        restore_state(raw, new_state) if new_state.present?
-
-        start = @offset
-        ansi = raw[@offset..-1]
+        restore_state(new_state, stream) if new_state.present?
+
+        append = false
+        truncated = false
+
+        cur_offset = stream.tell
+        if cur_offset > @offset
+          @offset = cur_offset
+          truncated = true
+        else
+          stream.seek(@offset)
+          append = @offset > 0
+        end
+        start_offset = @offset
 
         open_new_tag
 
-        s = StringScanner.new(ansi)
-        until s.eos?
-          if s.scan(/\e([@-_])(.*?)([@-~])/)
-            handle_sequence(s)
-          elsif s.scan(/\e(([@-_])(.*?)?)?$/)
-            break
-          elsif s.scan(/</)
-            @out << '&lt;'
-          elsif s.scan(/\r?\n/)
-            @out << '<br>'
-          else
-            @out << s.scan(/./m)
+        stream.each_line do |line|
+          s = StringScanner.new(line)
+          until s.eos?
+            if s.scan(/\e([@-_])(.*?)([@-~])/)
+              handle_sequence(s)
+            elsif s.scan(/\e(([@-_])(.*?)?)?$/)
+              break
+            elsif s.scan(/</)
+              @out << '&lt;'
+            elsif s.scan(/\r?\n/)
+              @out << '<br>'
+            else
+              @out << s.scan(/./m)
+            end
+            @offset += s.matched_size
           end
-          @offset += s.matched_size
         end
 
         close_open_tags()
 
-        { state: state, html: @out, text: ansi[0, @offset - start], append: start > 0 }
+        OpenStruct.new(
+          html: @out,
+          state: state,
+          append: append,
+          truncated: truncated,
+          offset: start_offset,
+          size: stream.tell - start_offset,
+          total: stream.size
+        )
       end
 
       def handle_sequence(s)
@@ -240,10 +260,10 @@ module Ci
         Base64.urlsafe_encode64(state.to_json)
       end
 
-      def restore_state(raw, new_state)
+      def restore_state(new_state, stream)
         state = Base64.urlsafe_decode64(new_state)
         state = JSON.parse(state, symbolize_names: true)
-        return if state[:offset].to_i > raw.length
+        return if state[:offset].to_i > stream.size
 
         STATE_PARAMS.each do |param|
           send("#{param}=".to_sym, state[param])
diff --git a/lib/ci/api/builds.rb b/lib/ci/api/builds.rb
index 95cc6308c3b..67b269b330c 100644
--- a/lib/ci/api/builds.rb
+++ b/lib/ci/api/builds.rb
@@ -61,7 +61,7 @@ module Ci
 
           update_runner_info
 
-          build.update_attributes(trace: params[:trace]) if params[:trace]
+          build.trace.set(params[:trace]) if params[:trace]
 
           Gitlab::Metrics.add_event(:update_build,
                                     project: build.project.path_with_namespace)
@@ -92,16 +92,14 @@ module Ci
           content_range = request.headers['Content-Range']
           content_range = content_range.split('-')
 
-          current_length = build.trace_length
-          unless current_length == content_range[0].to_i
-            return error!('416 Range Not Satisfiable', 416, { 'Range' => "0-#{current_length}" })
+          stream_size = build.trace.append(request.body.read, content_range[0].to_i)
+          if stream_size < 0
+            return error!('416 Range Not Satisfiable', 416, { 'Range' => "0-#{-stream_size}" })
           end
 
-          build.append_trace(request.body.read, content_range[0].to_i)
-
           status 202
           header 'Build-Status', build.status
-          header 'Range', "0-#{build.trace_length}"
+          header 'Range', "0-#{stream_size}"
         end
 
         # Authorize artifacts uploading for build - Runners only
diff --git a/lib/container_registry/blob.rb b/lib/container_registry/blob.rb
index eb5a2596177..d5f85f9fcad 100644
--- a/lib/container_registry/blob.rb
+++ b/lib/container_registry/blob.rb
@@ -38,11 +38,11 @@ module ContainerRegistry
     end
 
     def delete
-      client.delete_blob(repository.name, digest)
+      client.delete_blob(repository.path, digest)
     end
 
     def data
-      @data ||= client.blob(repository.name, digest, type)
+      @data ||= client.blob(repository.path, digest, type)
     end
   end
 end
diff --git a/lib/container_registry/path.rb b/lib/container_registry/path.rb
new file mode 100644
index 00000000000..a4b5f2aba6c
--- /dev/null
+++ b/lib/container_registry/path.rb
@@ -0,0 +1,70 @@
+module ContainerRegistry
+  ##
+  # Class responsible for extracting project and repository name from
+  # image repository path provided by a containers registry API response.
+  #
+  # Example:
+  #
+  # some/group/my_project/my/image ->
+  #   project: some/group/my_project
+  #   repository: my/image
+  #
+  class Path
+    InvalidRegistryPathError = Class.new(StandardError)
+
+    LEVELS_SUPPORTED = 3
+
+    def initialize(path)
+      @path = path
+    end
+
+    def valid?
+      @path =~ Gitlab::Regex.container_repository_name_regex &&
+        components.size > 1 &&
+        components.size < Namespace::NUMBER_OF_ANCESTORS_ALLOWED
+    end
+
+    def components
+      @components ||= @path.to_s.split('/')
+    end
+
+    def nodes
+      raise InvalidRegistryPathError unless valid?
+
+      @nodes ||= components.size.downto(2).map do |length|
+        components.take(length).join('/')
+      end
+    end
+
+    def has_project?
+      repository_project.present?
+    end
+
+    def has_repository?
+      return false unless has_project?
+
+      repository_project.container_repositories
+        .where(name: repository_name).any?
+    end
+
+    def root_repository?
+      @path == repository_project.full_path
+    end
+
+    def repository_project
+      @project ||= Project
+        .where_full_path_in(nodes.first(LEVELS_SUPPORTED))
+        .first
+    end
+
+    def repository_name
+      return unless has_project?
+
+      @path.remove(%r(^#{Regexp.escape(repository_project.full_path)}/?))
+    end
+
+    def to_s
+      @path
+    end
+  end
+end
diff --git a/lib/container_registry/registry.rb b/lib/container_registry/registry.rb
index 0e634f6b6ef..63bce655f57 100644
--- a/lib/container_registry/registry.rb
+++ b/lib/container_registry/registry.rb
@@ -8,10 +8,6 @@ module ContainerRegistry
       @client = ContainerRegistry::Client.new(uri, options)
     end
 
-    def repository(name)
-      ContainerRegistry::Repository.new(self, name)
-    end
-
     private
 
     def default_path
diff --git a/lib/container_registry/repository.rb b/lib/container_registry/repository.rb
deleted file mode 100644
index 0e4a7cb3cc9..00000000000
--- a/lib/container_registry/repository.rb
+++ /dev/null
@@ -1,48 +0,0 @@
-module ContainerRegistry
-  class Repository
-    attr_reader :registry, :name
-
-    delegate :client, to: :registry
-
-    def initialize(registry, name)
-      @registry, @name = registry, name
-    end
-
-    def path
-      [registry.path, name].compact.join('/')
-    end
-
-    def tag(tag)
-      ContainerRegistry::Tag.new(self, tag)
-    end
-
-    def manifest
-      return @manifest if defined?(@manifest)
-
-      @manifest = client.repository_tags(name)
-    end
-
-    def valid?
-      manifest.present?
-    end
-
-    def tags
-      return @tags if defined?(@tags)
-      return [] unless manifest && manifest['tags']
-
-      @tags = manifest['tags'].map do |tag|
-        ContainerRegistry::Tag.new(self, tag)
-      end
-    end
-
-    def blob(config)
-      ContainerRegistry::Blob.new(self, config)
-    end
-
-    def delete_tags
-      return unless tags
-
-      tags.all?(&:delete)
-    end
-  end
-end
diff --git a/lib/container_registry/tag.rb b/lib/container_registry/tag.rb
index 59040199920..728deea224f 100644
--- a/lib/container_registry/tag.rb
+++ b/lib/container_registry/tag.rb
@@ -22,15 +22,17 @@ module ContainerRegistry
     end
 
     def manifest
-      return @manifest if defined?(@manifest)
-
-      @manifest = client.repository_manifest(repository.name, name)
+      @manifest ||= client.repository_manifest(repository.path, name)
     end
 
     def path
       "#{repository.path}:#{name}"
     end
 
+    def location
+      "#{repository.location}:#{name}"
+    end
+
     def [](key)
       return unless manifest
 
@@ -38,9 +40,7 @@ module ContainerRegistry
     end
 
     def digest
-      return @digest if defined?(@digest)
-
-      @digest = client.repository_tag_digest(repository.name, name)
+      @digest ||= client.repository_tag_digest(repository.path, name)
     end
 
     def config_blob
@@ -82,7 +82,7 @@ module ContainerRegistry
     def delete
       return unless digest
 
-      client.delete_repository_tag(repository.name, digest)
+      client.delete_repository_tag(repository.path, digest)
     end
   end
 end
diff --git a/lib/gitlab/bitbucket_import/importer.rb b/lib/gitlab/bitbucket_import/importer.rb
index f4efa20374a..5a6d9ae99a0 100644
--- a/lib/gitlab/bitbucket_import/importer.rb
+++ b/lib/gitlab/bitbucket_import/importer.rb
@@ -149,7 +149,7 @@ module Gitlab
             description += @formatter.author_line(pull_request.author) unless find_user_id(pull_request.author)
             description += pull_request.description
 
-            merge_request = project.merge_requests.create(
+            merge_request = project.merge_requests.create!(
               iid: pull_request.iid,
               title: pull_request.title,
               description: description,
@@ -168,7 +168,7 @@ module Gitlab
 
             import_pull_request_comments(pull_request, merge_request) if merge_request.persisted?
           rescue StandardError => e
-            errors << { type: :pull_request, iid: pull_request.iid, errors: e.message }
+            errors << { type: :pull_request, iid: pull_request.iid, errors: e.message, trace: e.backtrace.join("\n"), raw_response: pull_request.raw }
           end
         end
       end
diff --git a/lib/gitlab/cache/ci/project_pipeline_status.rb b/lib/gitlab/cache/ci/project_pipeline_status.rb
new file mode 100644
index 00000000000..b358f2efa4f
--- /dev/null
+++ b/lib/gitlab/cache/ci/project_pipeline_status.rb
@@ -0,0 +1,103 @@
+# This class is not backed by a table in the main database.
+# It loads the latest Pipeline for the HEAD of a repository, and caches that
+# in Redis.
+module Gitlab
+  module Cache
+    module Ci
+      class ProjectPipelineStatus
+        attr_accessor :sha, :status, :ref, :project, :loaded
+
+        delegate :commit, to: :project
+
+        def self.load_for_project(project)
+          new(project).tap do |status|
+            status.load_status
+          end
+        end
+
+        def self.update_for_pipeline(pipeline)
+          new(pipeline.project,
+              sha: pipeline.sha,
+              status: pipeline.status,
+              ref: pipeline.ref).store_in_cache_if_needed
+        end
+
+        def initialize(project, sha: nil, status: nil, ref: nil)
+          @project = project
+          @sha = sha
+          @ref = ref
+          @status = status
+        end
+
+        def has_status?
+          loaded? && sha.present? && status.present?
+        end
+
+        def load_status
+          return if loaded?
+
+          if has_cache?
+            load_from_cache
+          else
+            load_from_project
+            store_in_cache
+          end
+
+          self.loaded = true
+        end
+
+        def load_from_project
+          return unless commit
+
+          self.sha = commit.sha
+          self.status = commit.status
+          self.ref = project.default_branch
+        end
+
+        # We only cache the status for the HEAD commit of a project
+        # This status is rendered in project lists
+        def store_in_cache_if_needed
+          return delete_from_cache unless commit
+          return unless sha
+          return unless ref
+
+          if commit.sha == sha && project.default_branch == ref
+            store_in_cache
+          end
+        end
+
+        def load_from_cache
+          Gitlab::Redis.with do |redis|
+            self.sha, self.status, self.ref = redis.hmget(cache_key, :sha, :status, :ref)
+          end
+        end
+
+        def store_in_cache
+          Gitlab::Redis.with do |redis|
+            redis.mapped_hmset(cache_key, { sha: sha, status: status, ref: ref })
+          end
+        end
+
+        def delete_from_cache
+          Gitlab::Redis.with do |redis|
+            redis.del(cache_key)
+          end
+        end
+
+        def has_cache?
+          Gitlab::Redis.with do |redis|
+            redis.exists(cache_key)
+          end
+        end
+
+        def loaded?
+          self.loaded
+        end
+
+        def cache_key
+          "projects/#{project.id}/build_status"
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/checks/change_access.rb b/lib/gitlab/checks/change_access.rb
index c85f79127bc..8793b20aa35 100644
--- a/lib/gitlab/checks/change_access.rb
+++ b/lib/gitlab/checks/change_access.rb
@@ -5,14 +5,14 @@ module Gitlab
       attr_reader :user_access, :project, :skip_authorization, :protocol
 
       def initialize(
-        change, user_access:, project:, env: {}, skip_authorization: false,
+        change, user_access:, project:, skip_authorization: false,
         protocol:
       )
         @oldrev, @newrev, @ref = change.values_at(:oldrev, :newrev, :ref)
         @branch_name = Gitlab::Git.branch_name(@ref)
+        @tag_name = Gitlab::Git.tag_name(@ref)
         @user_access = user_access
         @project = project
-        @env = env
         @skip_authorization = skip_authorization
         @protocol = protocol
       end
@@ -32,11 +32,11 @@ module Gitlab
       def protected_branch_checks
         return if skip_authorization
         return unless @branch_name
-        return unless project.protected_branch?(@branch_name)
+        return unless ProtectedBranch.protected?(project, @branch_name)
 
         if forced_push?
           return "You are not allowed to force push code to a protected branch on this project."
-        elsif Gitlab::Git.blank_ref?(@newrev)
+        elsif deletion?
           return "You are not allowed to delete protected branches from this project."
         end
 
@@ -58,13 +58,29 @@ module Gitlab
       def tag_checks
         return if skip_authorization
 
-        tag_ref = Gitlab::Git.tag_name(@ref)
+        return unless @tag_name
 
-        if tag_ref && protected_tag?(tag_ref) && user_access.cannot_do_action?(:admin_project)
-          "You are not allowed to change existing tags on this project."
+        if tag_exists? && user_access.cannot_do_action?(:admin_project)
+          return "You are not allowed to change existing tags on this project."
+        end
+
+        protected_tag_checks
+      end
+
+      def protected_tag_checks
+        return unless tag_protected?
+        return "Protected tags cannot be updated." if update?
+        return "Protected tags cannot be deleted." if deletion?
+
+        unless user_access.can_create_tag?(@tag_name)
+          return "You are not allowed to create this tag as it is protected."
         end
       end
 
+      def tag_protected?
+        ProtectedTag.protected?(project, @tag_name)
+      end
+
       def push_checks
         return if skip_authorization
 
@@ -75,12 +91,20 @@ module Gitlab
 
       private
 
-      def protected_tag?(tag_name)
-        project.repository.tag_exists?(tag_name)
+      def tag_exists?
+        project.repository.tag_exists?(@tag_name)
       end
 
       def forced_push?
-        Gitlab::Checks::ForcePush.force_push?(@project, @oldrev, @newrev, env: @env)
+        Gitlab::Checks::ForcePush.force_push?(@project, @oldrev, @newrev)
+      end
+
+      def update?
+        !Gitlab::Git.blank_ref?(@oldrev) && !deletion?
+      end
+
+      def deletion?
+        Gitlab::Git.blank_ref?(@newrev)
       end
 
       def matching_merge_request?
diff --git a/lib/gitlab/checks/force_push.rb b/lib/gitlab/checks/force_push.rb
index de0c9049ebf..1e73f89158d 100644
--- a/lib/gitlab/checks/force_push.rb
+++ b/lib/gitlab/checks/force_push.rb
@@ -1,20 +1,16 @@
 module Gitlab
   module Checks
     class ForcePush
-      def self.force_push?(project, oldrev, newrev, env: {})
+      def self.force_push?(project, oldrev, newrev)
         return false if project.empty_repo?
 
         # Created or deleted branch
         if Gitlab::Git.blank_ref?(oldrev) || Gitlab::Git.blank_ref?(newrev)
           false
         else
-          missed_ref, exit_status = Gitlab::Git::RevList.new(oldrev, newrev, project: project, env: env).execute
-
-          if exit_status == 0
-            missed_ref.present?
-          else
-            raise "Got a non-zero exit code while calling out to `git rev-list` in the force-push check."
-          end
+          Gitlab::Git::RevList.new(
+            path_to_repo: project.repository.path_to_repo,
+            oldrev: oldrev, newrev: newrev).missed_ref.present?
         end
       end
     end
diff --git a/lib/gitlab/ci/cron_parser.rb b/lib/gitlab/ci/cron_parser.rb
new file mode 100644
index 00000000000..a3cc350ef22
--- /dev/null
+++ b/lib/gitlab/ci/cron_parser.rb
@@ -0,0 +1,34 @@
+module Gitlab
+  module Ci
+    class CronParser
+      VALID_SYNTAX_SAMPLE_TIME_ZONE = 'UTC'.freeze
+      VALID_SYNTAX_SAMPLE_CRON = '* * * * *'.freeze
+
+      def initialize(cron, cron_timezone = 'UTC')
+        @cron = cron
+        @cron_timezone = cron_timezone
+      end
+
+      def next_time_from(time)
+        @cron_line ||= try_parse_cron(@cron, @cron_timezone)
+        @cron_line.next_time(time).in_time_zone(Time.zone) if @cron_line.present?
+      end
+
+      def cron_valid?
+        try_parse_cron(@cron, VALID_SYNTAX_SAMPLE_TIME_ZONE).present?
+      end
+
+      def cron_timezone_valid?
+        try_parse_cron(VALID_SYNTAX_SAMPLE_CRON, @cron_timezone).present?
+      end
+
+      private
+
+      def try_parse_cron(cron, cron_timezone)
+        Rufus::Scheduler.parse("#{cron} #{cron_timezone}")
+      rescue
+        # noop
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/trace.rb b/lib/gitlab/ci/trace.rb
new file mode 100644
index 00000000000..5b835bb669a
--- /dev/null
+++ b/lib/gitlab/ci/trace.rb
@@ -0,0 +1,136 @@
+module Gitlab
+  module Ci
+    class Trace
+      attr_reader :job
+
+      delegate :old_trace, to: :job
+
+      def initialize(job)
+        @job = job
+      end
+
+      def html(last_lines: nil)
+        read do |stream|
+          stream.html(last_lines: last_lines)
+        end
+      end
+
+      def raw(last_lines: nil)
+        read do |stream|
+          stream.raw(last_lines: last_lines)
+        end
+      end
+
+      def extract_coverage(regex)
+        read do |stream|
+          stream.extract_coverage(regex)
+        end
+      end
+
+      def set(data)
+        write do |stream|
+          data = job.hide_secrets(data)
+          stream.set(data)
+        end
+      end
+
+      def append(data, offset)
+        write do |stream|
+          current_length = stream.size
+          return -current_length unless current_length == offset
+
+          data = job.hide_secrets(data)
+          stream.append(data, offset)
+          stream.size
+        end
+      end
+
+      def exist?
+        current_path.present? || old_trace.present?
+      end
+
+      def read
+        stream = Gitlab::Ci::Trace::Stream.new do
+          if current_path
+            File.open(current_path, "rb")
+          elsif old_trace
+            StringIO.new(old_trace)
+          end
+        end
+
+        yield stream
+      ensure
+        stream&.close
+      end
+
+      def write
+        stream = Gitlab::Ci::Trace::Stream.new do
+          File.open(ensure_path, "a+b")
+        end
+
+        yield(stream).tap do
+          job.touch if job.needs_touch?
+        end
+      ensure
+        stream&.close
+      end
+
+      def erase!
+        paths.each do |trace_path|
+          FileUtils.rm(trace_path, force: true)
+        end
+
+        job.erase_old_trace!
+      end
+
+      private
+
+      def ensure_path
+        return current_path if current_path
+
+        ensure_directory
+        default_path
+      end
+
+      def ensure_directory
+        unless Dir.exist?(default_directory)
+          FileUtils.mkdir_p(default_directory)
+        end
+      end
+
+      def current_path
+        @current_path ||= paths.find do |trace_path|
+          File.exist?(trace_path)
+        end
+      end
+
+      def paths
+        [
+          default_path,
+          deprecated_path
+        ].compact
+      end
+
+      def default_directory
+        File.join(
+          Settings.gitlab_ci.builds_path,
+          job.created_at.utc.strftime("%Y_%m"),
+          job.project_id.to_s
+        )
+      end
+
+      def default_path
+        File.join(default_directory, "#{job.id}.log")
+      end
+
+      def deprecated_path
+        File.join(
+          Settings.gitlab_ci.builds_path,
+          job.created_at.utc.strftime("%Y_%m"),
+          job.project.ci_id.to_s,
+          "#{job.id}.log"
+        ) if job.project&.ci_id
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/trace/stream.rb b/lib/gitlab/ci/trace/stream.rb
new file mode 100644
index 00000000000..41dcf846fed
--- /dev/null
+++ b/lib/gitlab/ci/trace/stream.rb
@@ -0,0 +1,122 @@
+module Gitlab
+  module Ci
+    class Trace
+      # This was inspired from: http://stackoverflow.com/a/10219411/1520132
+      class Stream
+        BUFFER_SIZE = 4096
+        LIMIT_SIZE = 50.kilobytes
+
+        attr_reader :stream
+
+        delegate :close, :tell, :seek, :size, :path, :truncate, to: :stream, allow_nil: true
+
+        delegate :valid?, to: :stream, as: :present?, allow_nil: true
+
+        def initialize
+          @stream = yield
+        end
+
+        def valid?
+          self.stream.present?
+        end
+
+        def file?
+          self.path.present?
+        end
+
+        def limit(last_bytes = LIMIT_SIZE)
+          stream_size = size
+          if stream_size < last_bytes
+            last_bytes = stream_size
+          end
+          stream.seek(-last_bytes, IO::SEEK_END)
+        end
+
+        def append(data, offset)
+          stream.truncate(offset)
+          stream.seek(0, IO::SEEK_END)
+          stream.write(data)
+          stream.flush()
+        end
+
+        def set(data)
+          truncate(0)
+          stream.write(data)
+          stream.flush()
+        end
+
+        def raw(last_lines: nil)
+          return unless valid?
+
+          if last_lines.to_i > 0
+            read_last_lines(last_lines)
+          else
+            stream.read
+          end
+        end
+
+        def html_with_state(state = nil)
+          ::Ci::Ansi2html.convert(stream, state)
+        end
+
+        def html(last_lines: nil)
+          text = raw(last_lines: last_lines)
+          stream = StringIO.new(text)
+          ::Ci::Ansi2html.convert(stream).html
+        end
+
+        def extract_coverage(regex)
+          return unless valid?
+          return unless regex
+
+          regex = Regexp.new(regex)
+
+          match = ""
+
+          stream.each_line do |line|
+            matches = line.scan(regex)
+            next unless matches.is_a?(Array)
+            next if matches.empty?
+
+            match = matches.flatten.last
+            coverage = match.gsub(/\d+(\.\d+)?/).first
+            return coverage if coverage.present?
+          end
+
+          nil
+        rescue
+          # if bad regex or something goes wrong we dont want to interrupt transition
+          # so we just silentrly ignore error for now
+        end
+
+        private
+
+        def read_last_lines(last_lines)
+          chunks = []
+          pos = lines = 0
+          max = stream.size
+
+          # We want an extra line to make sure fist line has full contents
+          while lines <= last_lines && pos < max
+            pos += BUFFER_SIZE
+
+            buf =
+              if pos <= max
+                stream.seek(-pos, IO::SEEK_END)
+                stream.read(BUFFER_SIZE)
+              else # Reached the head, read only left
+                stream.seek(0)
+                stream.read(BUFFER_SIZE - (pos - max))
+              end
+
+            lines += buf.count("\n")
+            chunks.unshift(buf)
+          end
+
+          chunks.join.lines.last(last_lines).join
+            .force_encoding(Encoding.default_external)
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/trace_reader.rb b/lib/gitlab/ci/trace_reader.rb
deleted file mode 100644
index 1d7ddeb3e0f..00000000000
--- a/lib/gitlab/ci/trace_reader.rb
+++ /dev/null
@@ -1,50 +0,0 @@
-module Gitlab
-  module Ci
-    # This was inspired from: http://stackoverflow.com/a/10219411/1520132
-    class TraceReader
-      BUFFER_SIZE = 4096
-
-      attr_accessor :path, :buffer_size
-
-      def initialize(new_path, buffer_size: BUFFER_SIZE)
-        self.path = new_path
-        self.buffer_size = Integer(buffer_size)
-      end
-
-      def read(last_lines: nil)
-        if last_lines
-          read_last_lines(last_lines)
-        else
-          File.read(path)
-        end
-      end
-
-      def read_last_lines(max_lines)
-        File.open(path) do |file|
-          chunks = []
-          pos = lines = 0
-          max = file.size
-
-          # We want an extra line to make sure fist line has full contents
-          while lines <= max_lines && pos < max
-            pos += buffer_size
-
-            buf = if pos <= max
-                    file.seek(-pos, IO::SEEK_END)
-                    file.read(buffer_size)
-                  else # Reached the head, read only left
-                    file.seek(0)
-                    file.read(buffer_size - (pos - max))
-                  end
-
-            lines += buf.count("\n")
-            chunks.unshift(buf)
-          end
-
-          chunks.join.lines.last(max_lines).join
-            .force_encoding(Encoding.default_external)
-        end
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb
index 63b8d0d3b9d..d0bd1299671 100644
--- a/lib/gitlab/database.rb
+++ b/lib/gitlab/database.rb
@@ -57,16 +57,16 @@ module Gitlab
       postgresql? ? "RANDOM()" : "RAND()"
     end
 
-    def true_value
-      if Gitlab::Database.postgresql?
+    def self.true_value
+      if postgresql?
         "'t'"
       else
         1
       end
     end
 
-    def false_value
-      if Gitlab::Database.postgresql?
+    def self.false_value
+      if postgresql?
         "'f'"
       else
         0
diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb
index 525aa920328..6dabbe0264c 100644
--- a/lib/gitlab/database/migration_helpers.rb
+++ b/lib/gitlab/database/migration_helpers.rb
@@ -89,7 +89,8 @@ module Gitlab
         ADD CONSTRAINT #{key_name}
         FOREIGN KEY (#{column})
         REFERENCES #{target} (id)
-        ON DELETE #{on_delete} NOT VALID;
+        #{on_delete ? "ON DELETE #{on_delete}" : ''}
+        NOT VALID;
         EOF
 
         # Validate the existing constraint. This can potentially take a very
@@ -114,6 +115,14 @@ module Gitlab
         execute('SET statement_timeout TO 0') if Database.postgresql?
       end
 
+      def true_value
+        Database.true_value
+      end
+
+      def false_value
+        Database.false_value
+      end
+
       # Updates the value of a column in batches.
       #
       # This method updates the table in batches of 5% of the total row count.
@@ -250,6 +259,245 @@ module Gitlab
           raise error
         end
       end
+
+      # Renames a column without requiring downtime.
+      #
+      # Concurrent renames work by using database triggers to ensure both the
+      # old and new column are in sync. However, this method will _not_ remove
+      # the triggers or the old column automatically; this needs to be done
+      # manually in a post-deployment migration. This can be done using the
+      # method `cleanup_concurrent_column_rename`.
+      #
+      # table - The name of the database table containing the column.
+      # old - The old column name.
+      # new - The new column name.
+      # type - The type of the new column. If no type is given the old column's
+      #        type is used.
+      def rename_column_concurrently(table, old, new, type: nil)
+        if transaction_open?
+          raise 'rename_column_concurrently can not be run inside a transaction'
+        end
+
+        trigger_name = rename_trigger_name(table, old, new)
+        quoted_table = quote_table_name(table)
+        quoted_old = quote_column_name(old)
+        quoted_new = quote_column_name(new)
+
+        if Database.postgresql?
+          install_rename_triggers_for_postgresql(trigger_name, quoted_table,
+                                                 quoted_old, quoted_new)
+        else
+          install_rename_triggers_for_mysql(trigger_name, quoted_table,
+                                            quoted_old, quoted_new)
+        end
+
+        old_col = column_for(table, old)
+        new_type = type || old_col.type
+
+        add_column(table, new, new_type,
+                   limit: old_col.limit,
+                   default: old_col.default,
+                   null: old_col.null,
+                   precision: old_col.precision,
+                   scale: old_col.scale)
+
+        update_column_in_batches(table, new, Arel::Table.new(table)[old])
+
+        copy_indexes(table, old, new)
+        copy_foreign_keys(table, old, new)
+      end
+
+      # Changes the type of a column concurrently.
+      #
+      # table - The table containing the column.
+      # column - The name of the column to change.
+      # new_type - The new column type.
+      def change_column_type_concurrently(table, column, new_type)
+        temp_column = "#{column}_for_type_change"
+
+        rename_column_concurrently(table, column, temp_column, type: new_type)
+      end
+
+      # Performs cleanup of a concurrent type change.
+      #
+      # table - The table containing the column.
+      # column - The name of the column to change.
+      # new_type - The new column type.
+      def cleanup_concurrent_column_type_change(table, column)
+        temp_column = "#{column}_for_type_change"
+
+        transaction do
+          # This has to be performed in a transaction as otherwise we might have
+          # inconsistent data.
+          cleanup_concurrent_column_rename(table, column, temp_column)
+          rename_column(table, temp_column, column)
+        end
+      end
+
+      # Cleans up a concurrent column name.
+      #
+      # This method takes care of removing previously installed triggers as well
+      # as removing the old column.
+      #
+      # table - The name of the database table.
+      # old - The name of the old column.
+      # new - The name of the new column.
+      def cleanup_concurrent_column_rename(table, old, new)
+        trigger_name = rename_trigger_name(table, old, new)
+
+        if Database.postgresql?
+          remove_rename_triggers_for_postgresql(table, trigger_name)
+        else
+          remove_rename_triggers_for_mysql(trigger_name)
+        end
+
+        remove_column(table, old)
+      end
+
+      # Performs a concurrent column rename when using PostgreSQL.
+      def install_rename_triggers_for_postgresql(trigger, table, old, new)
+        execute <<-EOF.strip_heredoc
+        CREATE OR REPLACE FUNCTION #{trigger}()
+        RETURNS trigger AS
+        $BODY$
+        BEGIN
+          NEW.#{new} := NEW.#{old};
+          RETURN NEW;
+        END;
+        $BODY$
+        LANGUAGE 'plpgsql'
+        VOLATILE
+        EOF
+
+        execute <<-EOF.strip_heredoc
+        CREATE TRIGGER #{trigger}
+        BEFORE INSERT OR UPDATE
+        ON #{table}
+        FOR EACH ROW
+        EXECUTE PROCEDURE #{trigger}()
+        EOF
+      end
+
+      # Installs the triggers necessary to perform a concurrent column rename on
+      # MySQL.
+      def install_rename_triggers_for_mysql(trigger, table, old, new)
+        execute <<-EOF.strip_heredoc
+        CREATE TRIGGER #{trigger}_insert
+        BEFORE INSERT
+        ON #{table}
+        FOR EACH ROW
+        SET NEW.#{new} = NEW.#{old}
+        EOF
+
+        execute <<-EOF.strip_heredoc
+        CREATE TRIGGER #{trigger}_update
+        BEFORE UPDATE
+        ON #{table}
+        FOR EACH ROW
+        SET NEW.#{new} = NEW.#{old}
+        EOF
+      end
+
+      # Removes the triggers used for renaming a PostgreSQL column concurrently.
+      def remove_rename_triggers_for_postgresql(table, trigger)
+        execute("DROP TRIGGER #{trigger} ON #{table}")
+        execute("DROP FUNCTION #{trigger}()")
+      end
+
+      # Removes the triggers used for renaming a MySQL column concurrently.
+      def remove_rename_triggers_for_mysql(trigger)
+        execute("DROP TRIGGER #{trigger}_insert")
+        execute("DROP TRIGGER #{trigger}_update")
+      end
+
+      # Returns the (base) name to use for triggers when renaming columns.
+      def rename_trigger_name(table, old, new)
+        'trigger_' + Digest::SHA256.hexdigest("#{table}_#{old}_#{new}").first(12)
+      end
+
+      # Returns an Array containing the indexes for the given column
+      def indexes_for(table, column)
+        column = column.to_s
+
+        indexes(table).select { |index| index.columns.include?(column) }
+      end
+
+      # Returns an Array containing the foreign keys for the given column.
+      def foreign_keys_for(table, column)
+        column = column.to_s
+
+        foreign_keys(table).select { |fk| fk.column == column }
+      end
+
+      # Copies all indexes for the old column to a new column.
+      #
+      # table - The table containing the columns and indexes.
+      # old - The old column.
+      # new - The new column.
+      def copy_indexes(table, old, new)
+        old = old.to_s
+        new = new.to_s
+
+        indexes_for(table, old).each do |index|
+          new_columns = index.columns.map do |column|
+            column == old ? new : column
+          end
+
+          # This is necessary as we can't properly rename indexes such as
+          # "ci_taggings_idx".
+          unless index.name.include?(old)
+            raise "The index #{index.name} can not be copied as it does not "\
+              "mention the old column. You have to rename this index manually first."
+          end
+
+          name = index.name.gsub(old, new)
+
+          options = {
+            unique: index.unique,
+            name: name,
+            length: index.lengths,
+            order: index.orders
+          }
+
+          # These options are not supported by MySQL, so we only add them if
+          # they were previously set.
+          options[:using] = index.using if index.using
+          options[:where] = index.where if index.where
+
+          unless index.opclasses.blank?
+            opclasses = index.opclasses.dup
+
+            # Copy the operator classes for the old column (if any) to the new
+            # column.
+            opclasses[new] = opclasses.delete(old) if opclasses[old]
+
+            options[:opclasses] = opclasses
+          end
+
+          add_concurrent_index(table, new_columns, options)
+        end
+      end
+
+      # Copies all foreign keys for the old column to the new column.
+      #
+      # table - The table containing the columns and indexes.
+      # old - The old column.
+      # new - The new column.
+      def copy_foreign_keys(table, old, new)
+        foreign_keys_for(table, old).each do |fk|
+          add_concurrent_foreign_key(fk.from_table,
+                                     fk.to_table,
+                                     column: new,
+                                     on_delete: fk.on_delete)
+        end
+      end
+
+      # Returns the column for the given table and column name.
+      def column_for(table, name)
+        name = name.to_s
+
+        columns(table).find { |column| column.name == name }
+      end
     end
   end
 end
diff --git a/lib/gitlab/database/multi_threaded_migration.rb b/lib/gitlab/database/multi_threaded_migration.rb
new file mode 100644
index 00000000000..7ae5a4c17c8
--- /dev/null
+++ b/lib/gitlab/database/multi_threaded_migration.rb
@@ -0,0 +1,52 @@
+module Gitlab
+  module Database
+    module MultiThreadedMigration
+      MULTI_THREAD_AR_CONNECTION = :thread_local_ar_connection
+
+      # This overwrites the default connection method so that every thread can
+      # use a thread-local connection, while still supporting all of Rails'
+      # migration methods.
+      def connection
+        Thread.current[MULTI_THREAD_AR_CONNECTION] ||
+          ActiveRecord::Base.connection
+      end
+
+      # Starts a thread-pool for N threads, along with N threads each using a
+      # single connection. The provided block is yielded from inside each
+      # thread.
+      #
+      # Example:
+      #
+      #     with_multiple_threads(4) do
+      #       execute('SELECT ...')
+      #     end
+      #
+      # thread_count - The number of threads to start.
+      #
+      # join - When set to true this method will join the threads, blocking the
+      #        caller until all threads have finished running.
+      #
+      # Returns an Array containing the started threads.
+      def with_multiple_threads(thread_count, join: true)
+        pool = Gitlab::Database.create_connection_pool(thread_count)
+
+        threads = Array.new(thread_count) do
+          Thread.new do
+            pool.with_connection do |connection|
+              begin
+                Thread.current[MULTI_THREAD_AR_CONNECTION] = connection
+                yield
+              ensure
+                Thread.current[MULTI_THREAD_AR_CONNECTION] = nil
+              end
+            end
+          end
+        end
+
+        threads.each(&:join) if join
+
+        threads
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/diff/diff_refs.rb b/lib/gitlab/diff/diff_refs.rb
index 8406ca4269c..7948782aecc 100644
--- a/lib/gitlab/diff/diff_refs.rb
+++ b/lib/gitlab/diff/diff_refs.rb
@@ -18,6 +18,12 @@ module Gitlab
           head_sha == other.head_sha
       end
 
+      alias_method :eql?, :==
+
+      def hash
+        [base_sha, start_sha, head_sha].hash
+      end
+
       # There is only one case in which we will have `start_sha` and `head_sha`,
       # but not `base_sha`, which is when a diff is generated between an
       # orphaned branch and another branch, which means there _is_ no base, but
diff --git a/lib/gitlab/diff/line.rb b/lib/gitlab/diff/line.rb
index 114656958e3..0a15c6d9358 100644
--- a/lib/gitlab/diff/line.rb
+++ b/lib/gitlab/diff/line.rb
@@ -33,6 +33,10 @@ module Gitlab
         new_pos unless removed? || meta?
       end
 
+      def line
+        new_line || old_line
+      end
+
       def unchanged?
         type.nil?
       end
diff --git a/lib/gitlab/email/handler.rb b/lib/gitlab/email/handler.rb
index 35ea2e0ef59..b07c68d1498 100644
--- a/lib/gitlab/email/handler.rb
+++ b/lib/gitlab/email/handler.rb
@@ -5,7 +5,11 @@ require 'gitlab/email/handler/unsubscribe_handler'
 module Gitlab
   module Email
     module Handler
-      HANDLERS = [UnsubscribeHandler, CreateNoteHandler, CreateIssueHandler].freeze
+      HANDLERS = [
+        UnsubscribeHandler,
+        CreateNoteHandler,
+        CreateIssueHandler
+      ].freeze
 
       def self.for(mail, mail_key)
         HANDLERS.find do |klass|
diff --git a/lib/gitlab/email/handler/create_note_handler.rb b/lib/gitlab/email/handler/create_note_handler.rb
index d87ba427f4b..0e22f2189ee 100644
--- a/lib/gitlab/email/handler/create_note_handler.rb
+++ b/lib/gitlab/email/handler/create_note_handler.rb
@@ -1,4 +1,3 @@
-
 require 'gitlab/email/handler/base_handler'
 require 'gitlab/email/handler/reply_processing'
 
@@ -42,17 +41,7 @@ module Gitlab
         end
 
         def create_note
-          Notes::CreateService.new(
-            project,
-            author,
-            note:           message,
-            noteable_type:  sent_notification.noteable_type,
-            noteable_id:    sent_notification.noteable_id,
-            commit_id:      sent_notification.commit_id,
-            line_code:      sent_notification.line_code,
-            position:       sent_notification.position,
-            type:           sent_notification.note_type
-          ).execute
+          sent_notification.create_reply(message)
         end
       end
     end
diff --git a/lib/gitlab/etag_caching/middleware.rb b/lib/gitlab/etag_caching/middleware.rb
index 630fe4fa849..270d67dd50c 100644
--- a/lib/gitlab/etag_caching/middleware.rb
+++ b/lib/gitlab/etag_caching/middleware.rb
@@ -1,24 +1,12 @@
 module Gitlab
   module EtagCaching
     class Middleware
-      RESERVED_WORDS = NamespaceValidator::WILDCARD_ROUTES.map { |word| "/#{word}/" }.join('|')
-      ROUTES = [
-        {
-          regexp: %r(^(?!.*(#{RESERVED_WORDS})).*/noteable/issue/\d+/notes\z),
-          name: 'issue_notes'
-        },
-        {
-          regexp: %r(^(?!.*(#{RESERVED_WORDS})).*/issues/\d+/rendered_title\z),
-          name: 'issue_title'
-        }
-      ].freeze
-
       def initialize(app)
         @app = app
       end
 
       def call(env)
-        route = match_current_route(env)
+        route = Gitlab::EtagCaching::Router.match(env)
         return @app.call(env) unless route
 
         track_event(:etag_caching_middleware_used, route)
@@ -39,10 +27,6 @@ module Gitlab
 
       private
 
-      def match_current_route(env)
-        ROUTES.find { |route| route[:regexp].match(env['PATH_INFO']) }
-      end
-
       def get_etag(env)
         cache_key = env['PATH_INFO']
         store = Gitlab::EtagCaching::Store.new
@@ -65,7 +49,7 @@ module Gitlab
 
         status_code = Gitlab::PollingInterval.polling_enabled? ? 304 : 429
 
-        [status_code, { 'ETag' => etag }, ['']]
+        [status_code, { 'ETag' => etag }, []]
       end
 
       def track_cache_miss(if_none_match, cached_value_present, route)
@@ -79,7 +63,7 @@ module Gitlab
       end
 
       def track_event(name, route)
-        Gitlab::Metrics.add_event(name, endpoint: route[:name])
+        Gitlab::Metrics.add_event(name, endpoint: route.name)
       end
     end
   end
diff --git a/lib/gitlab/etag_caching/router.rb b/lib/gitlab/etag_caching/router.rb
new file mode 100644
index 00000000000..f6e4f279c06
--- /dev/null
+++ b/lib/gitlab/etag_caching/router.rb
@@ -0,0 +1,39 @@
+module Gitlab
+  module EtagCaching
+    class Router
+      Route = Struct.new(:regexp, :name)
+
+      RESERVED_WORDS = NamespaceValidator::WILDCARD_ROUTES.map { |word| "/#{word}/" }.join('|')
+      ROUTES = [
+        Gitlab::EtagCaching::Router::Route.new(
+          %r(^(?!.*(#{RESERVED_WORDS})).*/noteable/issue/\d+/notes\z),
+          'issue_notes'
+        ),
+        Gitlab::EtagCaching::Router::Route.new(
+          %r(^(?!.*(#{RESERVED_WORDS})).*/issues/\d+/rendered_title\z),
+          'issue_title'
+        ),
+        Gitlab::EtagCaching::Router::Route.new(
+          %r(^(?!.*(#{RESERVED_WORDS})).*/commit/\S+/pipelines\.json\z),
+          'commit_pipelines'
+        ),
+        Gitlab::EtagCaching::Router::Route.new(
+          %r(^(?!.*(#{RESERVED_WORDS})).*/merge_requests/new\.json\z),
+          'new_merge_request_pipelines'
+        ),
+        Gitlab::EtagCaching::Router::Route.new(
+          %r(^(?!.*(#{RESERVED_WORDS})).*/merge_requests/\d+/pipelines\.json\z),
+          'merge_request_pipelines'
+        ),
+        Gitlab::EtagCaching::Router::Route.new(
+          %r(^(?!.*(#{RESERVED_WORDS})).*/pipelines\.json\z),
+          'project_pipelines'
+        )
+      ].freeze
+
+      def self.match(env)
+        ROUTES.find { |route| route.regexp.match(env['PATH_INFO']) }
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/git/env.rb b/lib/gitlab/git/env.rb
new file mode 100644
index 00000000000..0fdc57ec954
--- /dev/null
+++ b/lib/gitlab/git/env.rb
@@ -0,0 +1,38 @@
+module Gitlab
+  module Git
+    # Ephemeral (per request) storage for environment variables that some Git
+    # commands may need.
+    #
+    # For example, in pre-receive hooks, new objects are put in a temporary
+    # $GIT_OBJECT_DIRECTORY. Without it set, the new objects cannot be retrieved
+    # (this would break push rules for instance).
+    #
+    # This class is thread-safe via RequestStore.
+    class Env
+      WHITELISTED_GIT_VARIABLES = %w[
+        GIT_OBJECT_DIRECTORY
+        GIT_ALTERNATE_OBJECT_DIRECTORIES
+      ].freeze
+
+      def self.set(env)
+        return unless RequestStore.active?
+
+        RequestStore.store[:gitlab_git_env] = whitelist_git_env(env)
+      end
+
+      def self.all
+        return {} unless RequestStore.active?
+
+        RequestStore.fetch(:gitlab_git_env) { {} }
+      end
+
+      def self.[](key)
+        all[key]
+      end
+
+      def self.whitelist_git_env(env)
+        env.select { |key, _| WHITELISTED_GIT_VARIABLES.include?(key.to_s) }.with_indifferent_access
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb
index 2e4314932c8..d7dac9f6149 100644
--- a/lib/gitlab/git/repository.rb
+++ b/lib/gitlab/git/repository.rb
@@ -8,6 +8,10 @@ module Gitlab
     class Repository
       include Gitlab::Git::Popen
 
+      ALLOWED_OBJECT_DIRECTORIES_VARIABLES = %w[
+        GIT_OBJECT_DIRECTORY
+        GIT_ALTERNATE_OBJECT_DIRECTORIES
+      ].freeze
       SEARCH_CONTEXT_LINES = 3
 
       NoRepository = Class.new(StandardError)
@@ -41,13 +45,15 @@ module Gitlab
 
       # Default branch in the repository
       def root_ref
-        @root_ref ||= Gitlab::GitalyClient.migrate(:root_ref) do |is_enabled|
-          if is_enabled
-            gitaly_ref_client.default_branch_name
-          else
-            discover_default_branch
-          end
-        end
+        # NOTE: This feature is intentionally disabled until
+        # https://gitlab.com/gitlab-org/gitaly/issues/179 is resolved
+        # @root_ref ||= Gitlab::GitalyClient.migrate(:root_ref) do |is_enabled|
+        #   if is_enabled
+        #     gitaly_ref_client.default_branch_name
+        #   else
+        @root_ref ||= discover_default_branch
+        #   end
+        # end
       rescue GRPC::BadStatus => e
         raise CommandError.new(e)
       end
@@ -58,7 +64,7 @@ module Gitlab
       end
 
       def rugged
-        @rugged ||= Rugged::Repository.new(path)
+        @rugged ||= Rugged::Repository.new(path, alternates: alternate_object_directories)
       rescue Rugged::RepositoryError, Rugged::OSError
         raise NoRepository.new('no repository for such path')
       end
@@ -66,13 +72,15 @@ module Gitlab
       # Returns an Array of branch names
       # sorted by name ASC
       def branch_names
-        Gitlab::GitalyClient.migrate(:branch_names) do |is_enabled|
-          if is_enabled
-            gitaly_ref_client.branch_names
-          else
-            branches.map(&:name)
-          end
-        end
+        # Gitlab::GitalyClient.migrate(:branch_names) do |is_enabled|
+        #   NOTE: This feature is intentionally disabled until
+        #   https://gitlab.com/gitlab-org/gitaly/issues/179 is resolved
+        #   if is_enabled
+        #     gitaly_ref_client.branch_names
+        #   else
+        branches.map(&:name)
+        #   end
+        # end
       rescue GRPC::BadStatus => e
         raise CommandError.new(e)
       end
@@ -127,13 +135,15 @@ module Gitlab
 
       # Returns an Array of tag names
       def tag_names
-        Gitlab::GitalyClient.migrate(:tag_names) do |is_enabled|
-          if is_enabled
-            gitaly_ref_client.tag_names
-          else
-            rugged.tags.map { |t| t.name }
-          end
-        end
+        # Gitlab::GitalyClient.migrate(:tag_names) do |is_enabled|
+        #   NOTE: This feature is intentionally disabled until
+        #   https://gitlab.com/gitlab-org/gitaly/issues/179 is resolved
+        #   if is_enabled
+        #     gitaly_ref_client.tag_names
+        #   else
+        rugged.tags.map { |t| t.name }
+        #   end
+        # end
       rescue GRPC::BadStatus => e
         raise CommandError.new(e)
       end
@@ -452,6 +462,23 @@ module Gitlab
         Gitlab::Git::DiffCollection.new(diff_patches(from, to, options, *paths), options)
       end
 
+      # Returns a RefName for a given SHA
+      def ref_name_for_sha(ref_path, sha)
+        # NOTE: This feature is intentionally disabled until
+        # https://gitlab.com/gitlab-org/gitaly/issues/180 is resolved
+        # Gitlab::GitalyClient.migrate(:find_ref_name) do |is_enabled|
+        #   if is_enabled
+        #     gitaly_ref_client.find_ref_name(sha, ref_path)
+        #   else
+        args = %W(#{Gitlab.config.git.bin_path} for-each-ref --count=1 #{ref_path} --contains #{sha})
+
+        # Not found -> ["", 0]
+        # Found -> ["b8d95eb4969eefacb0a58f6a28f6803f8070e7b9 commit\trefs/environments/production/77\n", 0]
+        Gitlab::Popen.popen(args, @path).first.split.last
+        #   end
+        # end
+      end
+
       # Returns commits collection
       #
       # Ex.
@@ -953,8 +980,20 @@ module Gitlab
         @attributes.attributes(path)
       end
 
+      def gitaly_repository
+        Gitlab::GitalyClient::Util.repository(@repository_storage, @relative_path)
+      end
+
+      def gitaly_channel
+        Gitlab::GitalyClient.get_channel(@repository_storage)
+      end
+
       private
 
+      def alternate_object_directories
+        Gitlab::Git::Env.all.values_at(*ALLOWED_OBJECT_DIRECTORIES_VARIABLES).compact
+      end
+
       # Get the content of a blob for a given commit.  If the blob is a commit
       # (for submodules) then return the blob's OID.
       def blob_content(commit, blob_name)
@@ -1232,7 +1271,7 @@ module Gitlab
       end
 
       def gitaly_ref_client
-        @gitaly_ref_client ||= Gitlab::GitalyClient::Ref.new(@repository_storage, @relative_path)
+        @gitaly_ref_client ||= Gitlab::GitalyClient::Ref.new(self)
       end
     end
   end
diff --git a/lib/gitlab/git/rev_list.rb b/lib/gitlab/git/rev_list.rb
index 79dd0cf7df2..a16b0ed76f4 100644
--- a/lib/gitlab/git/rev_list.rb
+++ b/lib/gitlab/git/rev_list.rb
@@ -1,41 +1,42 @@
 module Gitlab
   module Git
     class RevList
-      attr_reader :project, :env
-
-      ALLOWED_VARIABLES = %w[GIT_OBJECT_DIRECTORY GIT_ALTERNATE_OBJECT_DIRECTORIES].freeze
-
-      def initialize(oldrev, newrev, project:, env: nil)
-        @project = project
-        @env = env.presence || {}
-        @args = [Gitlab.config.git.bin_path,
-                 "--git-dir=#{project.repository.path_to_repo}",
-                 "rev-list",
-                 "--max-count=1",
-                 oldrev,
-                 "^#{newrev}"]
+      attr_reader :oldrev, :newrev, :path_to_repo
+
+      def initialize(path_to_repo:, newrev:, oldrev: nil)
+        @oldrev = oldrev
+        @newrev = newrev
+        @path_to_repo = path_to_repo
       end
 
-      def execute
-        Gitlab::Popen.popen(@args, nil, parse_environment_variables)
+      # This method returns an array of new references
+      def new_refs
+        execute([*base_args, newrev, '--not', '--all'])
       end
 
-      def valid?
-        environment_variables.all? do |(name, value)|
-          value.to_s.start_with?(project.repository.path_to_repo)
-        end
+      # This methods returns an array of missed references
+      def missed_ref
+        execute([*base_args, '--max-count=1', oldrev, "^#{newrev}"])
       end
 
       private
 
-      def parse_environment_variables
-        return {} unless valid?
+      def execute(args)
+        output, status = Gitlab::Popen.popen(args, nil, Gitlab::Git::Env.all.stringify_keys)
+
+        unless status.zero?
+          raise "Got a non-zero exit code while calling out `#{args.join(' ')}`."
+        end
 
-        environment_variables
+        output.split("\n")
       end
 
-      def environment_variables
-        @environment_variables ||= env.slice(*ALLOWED_VARIABLES).compact
+      def base_args
+        [
+          Gitlab.config.git.bin_path,
+          "--git-dir=#{path_to_repo}",
+          'rev-list'
+        ]
       end
     end
   end
diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb
index eea2f206902..99724db8da2 100644
--- a/lib/gitlab/git_access.rb
+++ b/lib/gitlab/git_access.rb
@@ -18,13 +18,12 @@ module Gitlab
 
     attr_reader :actor, :project, :protocol, :user_access, :authentication_abilities
 
-    def initialize(actor, project, protocol, authentication_abilities:, env: {})
+    def initialize(actor, project, protocol, authentication_abilities:)
       @actor    = actor
       @project  = project
       @protocol = protocol
       @authentication_abilities = authentication_abilities
       @user_access = UserAccess.new(user, project: project)
-      @env = env
     end
 
     def check(cmd, changes)
@@ -152,7 +151,6 @@ module Gitlab
         change,
         user_access: user_access,
         project: project,
-        env: @env,
         skip_authorization: deploy_key?,
         protocol: protocol
       ).exec
diff --git a/lib/gitlab/gitaly_client/commit.rb b/lib/gitlab/gitaly_client/commit.rb
index f15faebe27e..b7f39f3ef0b 100644
--- a/lib/gitlab/gitaly_client/commit.rb
+++ b/lib/gitlab/gitaly_client/commit.rb
@@ -7,14 +7,13 @@ module Gitlab
 
       class << self
         def diff_from_parent(commit, options = {})
-          project   = commit.project
-          channel   = GitalyClient.get_channel(project.repository_storage)
-          stub      = Gitaly::Diff::Stub.new(nil, nil, channel_override: channel)
-          repo      = Gitaly::Repository.new(path: project.repository.path_to_repo)
-          parent    = commit.parents[0]
+          repository = commit.project.repository
+          gitaly_repo = repository.gitaly_repository
+          stub = Gitaly::Diff::Stub.new(nil, nil, channel_override: repository.gitaly_channel)
+          parent = commit.parents[0]
           parent_id = parent ? parent.id : EMPTY_TREE_ID
-          request   = Gitaly::CommitDiffRequest.new(
-            repository: repo,
+          request = Gitaly::CommitDiffRequest.new(
+            repository: gitaly_repo,
             left_commit_id: parent_id,
             right_commit_id: commit.id
           )
@@ -23,12 +22,10 @@ module Gitlab
         end
 
         def is_ancestor(repository, ancestor_id, child_id)
-          project = Project.find_by_path(repository.path)
-          channel = GitalyClient.get_channel(project.repository_storage)
-          stub = Gitaly::Commit::Stub.new(nil, nil, channel_override: channel)
-          repo = Gitaly::Repository.new(path: repository.path_to_repo)
+          gitaly_repo = repository.gitaly_repository
+          stub = Gitaly::Commit::Stub.new(nil, nil, channel_override: repository.gitaly_channel)
           request = Gitaly::CommitIsAncestorRequest.new(
-            repository: repo,
+            repository: gitaly_repo,
             ancestor_id: ancestor_id,
             child_id: child_id
           )
diff --git a/lib/gitlab/gitaly_client/notifications.rb b/lib/gitlab/gitaly_client/notifications.rb
index f0d93ded91b..a94a54883db 100644
--- a/lib/gitlab/gitaly_client/notifications.rb
+++ b/lib/gitlab/gitaly_client/notifications.rb
@@ -3,13 +3,14 @@ module Gitlab
     class Notifications
       attr_accessor :stub
 
-      def initialize(repository_storage, relative_path)
-        @channel, @repository = Util.process_path(repository_storage, relative_path)
-        @stub = Gitaly::Notifications::Stub.new(nil, nil, channel_override: @channel)
+      # 'repository' is a Gitlab::Git::Repository
+      def initialize(repository)
+        @gitaly_repo = repository.gitaly_repository
+        @stub = Gitaly::Notifications::Stub.new(nil, nil, channel_override: repository.gitaly_channel)
       end
 
       def post_receive
-        request = Gitaly::PostReceiveRequest.new(repository: @repository)
+        request = Gitaly::PostReceiveRequest.new(repository: @gitaly_repo)
         @stub.post_receive(request)
       end
     end
diff --git a/lib/gitlab/gitaly_client/ref.rb b/lib/gitlab/gitaly_client/ref.rb
index bfc5fa573c7..d3c0743db4e 100644
--- a/lib/gitlab/gitaly_client/ref.rb
+++ b/lib/gitlab/gitaly_client/ref.rb
@@ -3,26 +3,37 @@ module Gitlab
     class Ref
       attr_accessor :stub
 
-      def initialize(repository_storage, relative_path)
-        @channel, @repository = Util.process_path(repository_storage, relative_path)
-        @stub = Gitaly::Ref::Stub.new(nil, nil, channel_override: @channel)
+      # 'repository' is a Gitlab::Git::Repository
+      def initialize(repository)
+        @gitaly_repo = repository.gitaly_repository
+        @stub = Gitaly::Ref::Stub.new(nil, nil, channel_override: repository.gitaly_channel)
       end
 
       def default_branch_name
-        request = Gitaly::FindDefaultBranchNameRequest.new(repository: @repository)
+        request = Gitaly::FindDefaultBranchNameRequest.new(repository: @gitaly_repo)
         stub.find_default_branch_name(request).name.gsub(/^refs\/heads\//, '')
       end
 
       def branch_names
-        request = Gitaly::FindAllBranchNamesRequest.new(repository: @repository)
+        request = Gitaly::FindAllBranchNamesRequest.new(repository: @gitaly_repo)
         consume_refs_response(stub.find_all_branch_names(request), prefix: 'refs/heads/')
       end
 
       def tag_names
-        request = Gitaly::FindAllTagNamesRequest.new(repository: @repository)
+        request = Gitaly::FindAllTagNamesRequest.new(repository: @gitaly_repo)
         consume_refs_response(stub.find_all_tag_names(request), prefix: 'refs/tags/')
       end
 
+      def find_ref_name(commit_id, ref_prefix)
+        request = Gitaly::FindRefNameRequest.new(
+          repository: @repository,
+          commit_id: commit_id,
+          prefix: ref_prefix
+        )
+
+        stub.find_ref_name(request).name
+      end
+
       private
 
       def consume_refs_response(response, prefix:)
diff --git a/lib/gitlab/gitaly_client/util.rb b/lib/gitlab/gitaly_client/util.rb
index d272c25d1f9..4acd297f5cb 100644
--- a/lib/gitlab/gitaly_client/util.rb
+++ b/lib/gitlab/gitaly_client/util.rb
@@ -1,12 +1,14 @@
 module Gitlab
   module GitalyClient
     module Util
-      def self.process_path(repository_storage, relative_path)
-        channel = GitalyClient.get_channel(repository_storage)
-        storage_path = Gitlab.config.repositories.storages[repository_storage]['path']
-        repository = Gitaly::Repository.new(path: File.join(storage_path, relative_path))
-
-        [channel, repository]
+      class << self
+        def repository(repository_storage, relative_path)
+          Gitaly::Repository.new(
+            path: File.join(Gitlab.config.repositories.storages[repository_storage]['path'], relative_path),
+            storage_name: repository_storage,
+            relative_path: relative_path,
+          )
+        end
       end
     end
   end
diff --git a/lib/gitlab/health_checks/base_abstract_check.rb b/lib/gitlab/health_checks/base_abstract_check.rb
new file mode 100644
index 00000000000..7de6d4d9367
--- /dev/null
+++ b/lib/gitlab/health_checks/base_abstract_check.rb
@@ -0,0 +1,45 @@
+module Gitlab
+  module HealthChecks
+    module BaseAbstractCheck
+      def name
+        super.demodulize.underscore
+      end
+
+      def human_name
+        name.sub(/_check$/, '').capitalize
+      end
+
+      def readiness
+        raise NotImplementedError
+      end
+
+      def liveness
+        HealthChecks::Result.new(true)
+      end
+
+      def metrics
+        []
+      end
+
+      protected
+
+      def metric(name, value, **labels)
+        Metric.new(name, value, labels)
+      end
+
+      def with_timing(proc)
+        start = Time.now
+        result = proc.call
+        yield result, Time.now.to_f - start.to_f
+      end
+
+      def catch_timeout(seconds, &block)
+        begin
+          Timeout.timeout(seconds.to_i, &block)
+        rescue Timeout::Error => ex
+          ex
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/health_checks/db_check.rb b/lib/gitlab/health_checks/db_check.rb
new file mode 100644
index 00000000000..fd94984f8a2
--- /dev/null
+++ b/lib/gitlab/health_checks/db_check.rb
@@ -0,0 +1,29 @@
+module Gitlab
+  module HealthChecks
+    class DbCheck
+      extend SimpleAbstractCheck
+
+      class << self
+        private
+
+        def metric_prefix
+          'db_ping'
+        end
+
+        def is_successful?(result)
+          result == '1'
+        end
+
+        def check
+          catch_timeout 10.seconds do
+            if Gitlab::Database.postgresql?
+              ActiveRecord::Base.connection.execute('SELECT 1 as ping')&.first&.[]('ping')
+            else
+              ActiveRecord::Base.connection.execute('SELECT 1 as ping')&.first&.first&.to_s
+            end
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/health_checks/fs_shards_check.rb b/lib/gitlab/health_checks/fs_shards_check.rb
new file mode 100644
index 00000000000..df962d203b7
--- /dev/null
+++ b/lib/gitlab/health_checks/fs_shards_check.rb
@@ -0,0 +1,117 @@
+module Gitlab
+  module HealthChecks
+    class FsShardsCheck
+      extend BaseAbstractCheck
+
+      class << self
+        def readiness
+          repository_storages.map do |storage_name|
+            begin
+              tmp_file_path = tmp_file_path(storage_name)
+
+              if !storage_stat_test(storage_name)
+                HealthChecks::Result.new(false, 'cannot stat storage', shard: storage_name)
+              elsif !storage_write_test(tmp_file_path)
+                HealthChecks::Result.new(false, 'cannot write to storage', shard: storage_name)
+              elsif !storage_read_test(tmp_file_path)
+                HealthChecks::Result.new(false, 'cannot read from storage', shard: storage_name)
+              else
+                HealthChecks::Result.new(true, nil, shard: storage_name)
+              end
+            rescue RuntimeError => ex
+              message = "unexpected error #{ex} when checking storage #{storage_name}"
+              Rails.logger.error(message)
+              HealthChecks::Result.new(false, message, shard: storage_name)
+            ensure
+              delete_test_file(tmp_file_path)
+            end
+          end
+        end
+
+        def metrics
+          repository_storages.flat_map do |storage_name|
+            tmp_file_path = tmp_file_path(storage_name)
+            [
+              operation_metrics(:filesystem_accessible, :filesystem_access_latency, -> { storage_stat_test(storage_name) }, shard: storage_name),
+              operation_metrics(:filesystem_writable, :filesystem_write_latency, -> { storage_write_test(tmp_file_path) }, shard: storage_name),
+              operation_metrics(:filesystem_readable, :filesystem_read_latency, -> { storage_read_test(tmp_file_path) }, shard: storage_name)
+            ].flatten
+          end
+        end
+
+        private
+
+        RANDOM_STRING = SecureRandom.hex(1000).freeze
+
+        def operation_metrics(ok_metric, latency_metric, operation, **labels)
+          with_timing operation do |result, elapsed|
+            [
+              metric(latency_metric, elapsed, **labels),
+              metric(ok_metric, result ? 1 : 0, **labels)
+            ]
+          end
+        rescue RuntimeError => ex
+          Rails.logger("unexpected error #{ex} when checking #{ok_metric}")
+          [metric(ok_metric, 0, **labels)]
+        end
+
+        def repository_storages
+          @repository_storage ||= Gitlab::CurrentSettings.current_application_settings.repository_storages
+        end
+
+        def storages_paths
+          @storage_paths ||= Gitlab.config.repositories.storages
+        end
+
+        def with_timeout(args)
+          %w{timeout 1}.concat(args)
+        end
+
+        def tmp_file_path(storage_name)
+          Dir::Tmpname.create(%w(fs_shards_check +deleted), path(storage_name)) { |path| path }
+        end
+
+        def path(storage_name)
+          storages_paths&.dig(storage_name, 'path')
+        end
+
+        def storage_stat_test(storage_name)
+          stat_path = File.join(path(storage_name), '.')
+          begin
+            _, status = Gitlab::Popen.popen(with_timeout(%W{ stat #{stat_path} }))
+            status == 0
+          rescue Errno::ENOENT
+            File.exist?(stat_path) && File::Stat.new(stat_path).readable?
+          end
+        end
+
+        def storage_write_test(tmp_path)
+          _, status = Gitlab::Popen.popen(with_timeout(%W{ tee #{tmp_path} })) do |stdin|
+            stdin.write(RANDOM_STRING)
+          end
+          status == 0
+        rescue Errno::ENOENT
+          written_bytes = File.write(tmp_path, RANDOM_STRING) rescue Errno::ENOENT
+          written_bytes == RANDOM_STRING.length
+        end
+
+        def storage_read_test(tmp_path)
+          _, status = Gitlab::Popen.popen(with_timeout(%W{ diff #{tmp_path} - })) do |stdin|
+            stdin.write(RANDOM_STRING)
+          end
+          status == 0
+        rescue Errno::ENOENT
+          file_contents = File.read(tmp_path) rescue Errno::ENOENT
+          file_contents == RANDOM_STRING
+        end
+
+        def delete_test_file(tmp_path)
+          _, status = Gitlab::Popen.popen(with_timeout(%W{ rm -f #{tmp_path} }))
+          status == 0
+        rescue Errno::ENOENT
+          File.delete(tmp_path) rescue Errno::ENOENT
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/health_checks/metric.rb b/lib/gitlab/health_checks/metric.rb
new file mode 100644
index 00000000000..1a2eab0b005
--- /dev/null
+++ b/lib/gitlab/health_checks/metric.rb
@@ -0,0 +1,3 @@
+module Gitlab::HealthChecks
+  Metric = Struct.new(:name, :value, :labels)
+end
diff --git a/lib/gitlab/health_checks/redis_check.rb b/lib/gitlab/health_checks/redis_check.rb
new file mode 100644
index 00000000000..57bbe5b3ad0
--- /dev/null
+++ b/lib/gitlab/health_checks/redis_check.rb
@@ -0,0 +1,25 @@
+module Gitlab
+  module HealthChecks
+    class RedisCheck
+      extend SimpleAbstractCheck
+
+      class << self
+        private
+
+        def metric_prefix
+          'redis_ping'
+        end
+
+        def is_successful?(result)
+          result == 'PONG'
+        end
+
+        def check
+          catch_timeout 10.seconds do
+            Gitlab::Redis.with(&:ping)
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/health_checks/result.rb b/lib/gitlab/health_checks/result.rb
new file mode 100644
index 00000000000..8086760023e
--- /dev/null
+++ b/lib/gitlab/health_checks/result.rb
@@ -0,0 +1,3 @@
+module Gitlab::HealthChecks
+  Result = Struct.new(:success, :message, :labels)
+end
diff --git a/lib/gitlab/health_checks/simple_abstract_check.rb b/lib/gitlab/health_checks/simple_abstract_check.rb
new file mode 100644
index 00000000000..fbe1645c1b1
--- /dev/null
+++ b/lib/gitlab/health_checks/simple_abstract_check.rb
@@ -0,0 +1,43 @@
+module Gitlab
+  module HealthChecks
+    module SimpleAbstractCheck
+      include BaseAbstractCheck
+
+      def readiness
+        check_result = check
+        if is_successful?(check_result)
+          HealthChecks::Result.new(true)
+        elsif check_result.is_a?(Timeout::Error)
+          HealthChecks::Result.new(false, "#{human_name} check timed out")
+        else
+          HealthChecks::Result.new(false, "unexpected #{human_name} check result: #{check_result}")
+        end
+      end
+
+      def metrics
+        with_timing method(:check) do |result, elapsed|
+          Rails.logger.error("#{human_name} check returned unexpected result #{result}") unless is_successful?(result)
+          [
+            metric("#{metric_prefix}_timeout", result.is_a?(Timeout::Error) ? 1 : 0),
+            metric("#{metric_prefix}_success", is_successful?(result) ? 1 : 0),
+            metric("#{metric_prefix}_latency", elapsed)
+          ]
+        end
+      end
+
+      private
+
+      def metric_prefix
+        raise NotImplementedError
+      end
+
+      def is_successful?(result)
+        raise NotImplementedError
+      end
+
+      def check
+        raise NotImplementedError
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml
index f69288f7d67..899a6567768 100644
--- a/lib/gitlab/import_export/import_export.yml
+++ b/lib/gitlab/import_export/import_export.yml
@@ -39,13 +39,16 @@ project_tree:
       - :author
       - :events
     - :statuses
-  - :triggers
+  - triggers:
+    - :trigger_schedule
   - :deploy_keys
   - :services
   - :hooks
   - protected_branches:
     - :merge_access_levels
     - :push_access_levels
+  - protected_tags:
+    - :create_access_levels
   - :project_feature
 
 # Only include the following attributes for the models specified.
diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb
index df21ff22216..2e349b5f9a9 100644
--- a/lib/gitlab/import_export/project_tree_restorer.rb
+++ b/lib/gitlab/import_export/project_tree_restorer.rb
@@ -52,7 +52,11 @@ module Gitlab
           create_sub_relations(relation, @tree_hash) if relation.is_a?(Hash)
 
           relation_key = relation.is_a?(Hash) ? relation.keys.first : relation
-          relation_hash = create_relation(relation_key, @tree_hash[relation_key.to_s])
+          relation_hash_list = @tree_hash[relation_key.to_s]
+
+          next unless relation_hash_list
+
+          relation_hash = create_relation(relation_key, relation_hash_list)
           saved << restored_project.append_or_update_attribute(relation_key, relation_hash)
         end
         saved.all?
diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb
index fb43e7ccdbb..4a54e7ef2e7 100644
--- a/lib/gitlab/import_export/relation_factory.rb
+++ b/lib/gitlab/import_export/relation_factory.rb
@@ -5,10 +5,12 @@ module Gitlab
                     pipelines: 'Ci::Pipeline',
                     statuses: 'commit_status',
                     triggers: 'Ci::Trigger',
+                    trigger_schedule: 'Ci::TriggerSchedule',
                     builds: 'Ci::Build',
                     hooks: 'ProjectHook',
                     merge_access_levels: 'ProtectedBranch::MergeAccessLevel',
                     push_access_levels: 'ProtectedBranch::PushAccessLevel',
+                    create_access_levels: 'ProtectedTag::CreateAccessLevel',
                     labels: :project_labels,
                     priorities: :label_priorities,
                     label: :project_label }.freeze
@@ -184,7 +186,7 @@ module Gitlab
       end
 
       def admin_user?
-        @user.is_admin?
+        @user.admin?
       end
 
       def parsed_relation_hash
diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb
index 5e5f5ff1589..e599dd4a656 100644
--- a/lib/gitlab/regex.rb
+++ b/lib/gitlab/regex.rb
@@ -121,6 +121,13 @@ module Gitlab
       git_reference_regex
     end
 
+    ##
+    # Docker Distribution Registry 2.4.1 repository name rules
+    #
+    def container_repository_name_regex
+      @container_repository_regex ||= %r{\A[a-z0-9]+(?:[-._/][a-z0-9]+)*\Z}
+    end
+
     def environment_name_regex
       @environment_name_regex ||= /\A[a-zA-Z0-9_\\\/\${}. -]+\z/.freeze
     end
diff --git a/lib/gitlab/user_access.rb b/lib/gitlab/user_access.rb
index f260c0c535f..54728e5ff0e 100644
--- a/lib/gitlab/user_access.rb
+++ b/lib/gitlab/user_access.rb
@@ -28,14 +28,23 @@ module Gitlab
       true
     end
 
+    def can_create_tag?(ref)
+      return false unless can_access_git?
+
+      if ProtectedTag.protected?(project, ref)
+        project.protected_tags.protected_ref_accessible_to?(ref, user, action: :create)
+      else
+        user.can?(:push_code, project)
+      end
+    end
+
     def can_push_to_branch?(ref)
       return false unless can_access_git?
 
-      if project.protected_branch?(ref)
+      if ProtectedBranch.protected?(project, ref)
         return true if project.empty_repo? && project.user_can_push_to_empty_repo?(user)
 
-        access_levels = project.protected_branches.matching(ref).map(&:push_access_levels).flatten
-        has_access = access_levels.any? { |access_level| access_level.check_access(user) }
+        has_access = project.protected_branches.protected_ref_accessible_to?(ref, user, action: :push)
 
         has_access || !project.repository.branch_exists?(ref) && can_merge_to_branch?(ref)
       else
@@ -46,9 +55,8 @@ module Gitlab
     def can_merge_to_branch?(ref)
       return false unless can_access_git?
 
-      if project.protected_branch?(ref)
-        access_levels = project.protected_branches.matching(ref).map(&:merge_access_levels).flatten
-        access_levels.any? { |access_level| access_level.check_access(user) }
+      if ProtectedBranch.protected?(project, ref)
+        project.protected_branches.protected_ref_accessible_to?(ref, user, action: :merge)
       else
         user.can?(:push_code, project)
       end
diff --git a/lib/gitlab/visibility_level.rb b/lib/gitlab/visibility_level.rb
index 8f1d1fdc02e..2e31f4462f9 100644
--- a/lib/gitlab/visibility_level.rb
+++ b/lib/gitlab/visibility_level.rb
@@ -63,7 +63,7 @@ module Gitlab
       end
 
       def allowed_for?(user, level)
-        user.is_admin? || allowed_level?(level.to_i)
+        user.admin? || allowed_level?(level.to_i)
       end
 
       # Return true if the specified level is allowed for the current user.
diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb
index a8a7bf9bc12..e6e40f6945d 100644
--- a/lib/gitlab/workhorse.rb
+++ b/lib/gitlab/workhorse.rb
@@ -24,14 +24,8 @@ module Gitlab
         }
 
         if Gitlab.config.gitaly.enabled
-          storage = repository.project.repository_storage
-          address = Gitlab::GitalyClient.get_address(storage)
-          # TODO: use GitalyClient code to assemble the Repository message
-          params[:Repository] = Gitaly::Repository.new(
-            path: repo_path,
-            storage_name: storage,
-            relative_path: Gitlab::RepoPath.strip_storage_path(repo_path),
-          ).to_h
+          address = Gitlab::GitalyClient.get_address(repository.project.repository_storage)
+          params[:Repository] = repository.gitaly_repository.to_h
 
           feature_enabled = case action.to_s
                             when 'git_receive_pack'
diff --git a/lib/microsoft_teams/activity.rb b/lib/microsoft_teams/activity.rb
new file mode 100644
index 00000000000..d2c420efdaf
--- /dev/null
+++ b/lib/microsoft_teams/activity.rb
@@ -0,0 +1,19 @@
+module MicrosoftTeams
+  class Activity
+    def initialize(title:, subtitle:, text:, image:)
+      @title = title
+      @subtitle = subtitle
+      @text = text
+      @image = image
+    end
+
+    def prepare
+      {
+        'activityTitle' => @title,
+        'activitySubtitle' => @subtitle,
+        'activityText' => @text,
+        'activityImage' => @image
+      }
+    end
+  end
+end
diff --git a/lib/microsoft_teams/notifier.rb b/lib/microsoft_teams/notifier.rb
new file mode 100644
index 00000000000..3bef68a1bcb
--- /dev/null
+++ b/lib/microsoft_teams/notifier.rb
@@ -0,0 +1,46 @@
+module MicrosoftTeams
+  class Notifier
+    def initialize(webhook)
+      @webhook = webhook
+      @header = { 'Content-type' => 'application/json' }
+    end
+
+    def ping(options = {})
+      result = false
+
+      begin
+        response = HTTParty.post(
+          @webhook.to_str,
+          headers: @header,
+          body: body(options)
+        )
+
+        result = true if response
+      rescue HTTParty::Error, StandardError => error
+        Rails.logger.info("#{self.class.name}: Error while connecting to #{@webhook}: #{error.message}")
+      end
+
+      result
+    end
+
+    private
+
+    def body(options = {})
+      result = { 'sections' => [] }
+
+      result['title'] = options[:title]
+      result['summary'] = options[:pretext]
+      result['sections'] << MicrosoftTeams::Activity.new(options[:activity]).prepare
+
+      attachments = options[:attachments]
+      unless attachments.blank?
+        result['sections'] << {
+          'title' => 'Details',
+          'facts' => [{ 'name' => 'Attachments', 'value' => attachments }]
+        }
+      end
+
+      result.to_json
+    end
+  end
+end
diff --git a/lib/support/init.d/gitlab b/lib/support/init.d/gitlab
index 09e121e5120..6e351365de0 100755
--- a/lib/support/init.d/gitlab
+++ b/lib/support/init.d/gitlab
@@ -326,8 +326,7 @@ start_gitlab() {
       echo "Gitaly is already running with pid $gapid, not restarting"
     else
       $app_root/bin/daemon_with_pidfile $gitaly_pid_path \
-        $app_root/bin/with_env $gitaly_dir/env \
-          $gitaly_dir/gitaly >> $gitaly_log 2>&1 &
+          $gitaly_dir/gitaly $gitaly_dir/config.toml >> $gitaly_log 2>&1 &
     fi
   fi
 
diff --git a/lib/tasks/gitlab/check.rake b/lib/tasks/gitlab/check.rake
index a9a48f7188f..f41c73154f5 100644
--- a/lib/tasks/gitlab/check.rake
+++ b/lib/tasks/gitlab/check.rake
@@ -431,8 +431,7 @@ namespace :gitlab do
 
     def check_repo_base_user_and_group
       gitlab_shell_ssh_user = Gitlab.config.gitlab_shell.ssh_user
-      gitlab_shell_owner_group = Gitlab.config.gitlab_shell.owner_group
-      puts "Repo paths owned by #{gitlab_shell_ssh_user}:#{gitlab_shell_owner_group}?"
+      puts "Repo paths owned by #{gitlab_shell_ssh_user}:root, or #{gitlab_shell_ssh_user}:#{Gitlab.config.gitlab_shell.owner_group}?"
 
       Gitlab.config.repositories.storages.each do |name, repository_storage|
         repo_base_path = repository_storage['path']
@@ -443,15 +442,16 @@ namespace :gitlab do
           break
         end
 
-        uid = uid_for(gitlab_shell_ssh_user)
-        gid = gid_for(gitlab_shell_owner_group)
-        if File.stat(repo_base_path).uid == uid && File.stat(repo_base_path).gid == gid
+        user_id = uid_for(gitlab_shell_ssh_user)
+        root_group_id = gid_for('root')
+        group_ids = [root_group_id, gid_for(Gitlab.config.gitlab_shell.owner_group)]
+        if File.stat(repo_base_path).uid == user_id && group_ids.include?(File.stat(repo_base_path).gid)
           puts "yes".color(:green)
         else
           puts "no".color(:red)
-          puts "  User id for #{gitlab_shell_ssh_user}: #{uid}. Groupd id for #{gitlab_shell_owner_group}: #{gid}".color(:blue)
+          puts "  User id for #{gitlab_shell_ssh_user}: #{user_id}. Groupd id for root: #{root_group_id}".color(:blue)
           try_fixing_it(
-            "sudo chown -R #{gitlab_shell_ssh_user}:#{gitlab_shell_owner_group} #{repo_base_path}"
+            "sudo chown -R #{gitlab_shell_ssh_user}:root #{repo_base_path}"
           )
           for_more_information(
             see_installation_guide_section "GitLab Shell"
diff --git a/lib/tasks/gitlab/gitaly.rake b/lib/tasks/gitlab/gitaly.rake
index c288e17ac8d..9f6cfe3957c 100644
--- a/lib/tasks/gitlab/gitaly.rake
+++ b/lib/tasks/gitlab/gitaly.rake
@@ -19,5 +19,19 @@ namespace :gitlab do
         run_command!([command])
       end
     end
+
+    desc "GitLab | Print storage configuration in TOML format"
+    task storage_config: :environment do
+      require 'toml'
+
+      puts "# Gitaly storage configuration generated from #{Gitlab.config.source} on #{Time.current.to_s(:long)}"
+      puts "# This is in TOML format suitable for use in Gitaly's config.toml file."
+
+      config = Gitlab.config.repositories.storages.map do |key, val|
+        { name: key, path: val['path'] }
+      end
+
+      puts TOML.dump(storage: config)
+    end
   end
 end
diff --git a/lib/tasks/gitlab/update_templates.rake b/lib/tasks/gitlab/update_templates.rake
index dbdfb335a5c..cb2adc81c9d 100644
--- a/lib/tasks/gitlab/update_templates.rake
+++ b/lib/tasks/gitlab/update_templates.rake
@@ -44,7 +44,7 @@ namespace :gitlab do
     ),
     Template.new(
       "https://gitlab.com/gitlab-org/gitlab-ci-yml.git",
-      /(\.{1,2}|LICENSE|Pages|autodeploy|\.gitlab-ci.yml)\z/
+      /(\.{1,2}|LICENSE|CONTRIBUTING.md|Pages|autodeploy|\.gitlab-ci.yml)\z/
     )
   ].freeze
 
diff --git a/lib/tasks/import.rake b/lib/tasks/import.rake
index 350afeb5c0b..15131fbf755 100644
--- a/lib/tasks/import.rake
+++ b/lib/tasks/import.rake
@@ -48,9 +48,16 @@ class NewImporter < ::Gitlab::GithubImport::Importer
     begin
       raise 'Blocked import URL.' if Gitlab::UrlBlocker.blocked_url?(project.import_url)
 
-      gitlab_shell.import_repository(project.repository_storage_path, project.path_with_namespace, project.import_url)
+      project.create_repository
+      project.repository.add_remote(project.import_type, project.import_url)
+      project.repository.set_remote_as_mirror(project.import_type)
+      project.repository.fetch_remote(project.import_type, forced: true)
+      project.repository.remove_remote(project.import_type)
     rescue => e
-      project.repository.before_import if project.repository_exists?
+      # Expire cache to prevent scenarios such as:
+      # 1. First import failed, but the repo was imported successfully, so +exists?+ returns true
+      # 2. Retried import, repo is broken or not imported but +exists?+ still returns true
+      project.repository.expire_content_cache if project.repository_exists?
 
       raise "Error importing repository #{project.import_url} into #{project.path_with_namespace} - #{e.message}"
     end