Merge branch 'master' into 'feature/gb/pipeline-only-except-with-modified-paths'feature/gb/pipeline-only-except-with-modified-paths

# Conflicts:
#   app/models/ci/pipeline.rb

29 files changed, 490 insertions, 197 deletions

diff --git a/lib/gitlab/background_migration/encrypt_columns.rb b/lib/gitlab/background_migration/encrypt_columns.rb
new file mode 100644
index 00000000000..0d333e47e7b
--- /dev/null
+++ b/lib/gitlab/background_migration/encrypt_columns.rb
@@ -0,0 +1,80 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module BackgroundMigration
+    # EncryptColumn migrates data from an unencrypted column - `foo`, say - to
+    # an encrypted column - `encrypted_foo`, say.
+    #
+    # For this background migration to work, the table that is migrated _has_ to
+    # have an `id` column as the primary key. Additionally, the encrypted column
+    # should be managed by attr_encrypted, and map to an attribute with the same
+    # name as the unencrypted column (i.e., the unencrypted column should be
+    # shadowed).
+    #
+    # To avoid depending on a particular version of the model in app/, add a
+    # model to `lib/gitlab/background_migration/models/encrypt_columns` and use
+    # it in the migration that enqueues the jobs, so code can be shared.
+    class EncryptColumns
+      def perform(model, attributes, from, to)
+        model = model.constantize if model.is_a?(String)
+        attributes = expand_attributes(model, Array(attributes).map(&:to_sym))
+
+        model.transaction do
+          # Use SELECT ... FOR UPDATE to prevent the value being changed while
+          # we are encrypting it
+          relation = model.where(id: from..to).lock
+
+          relation.each do |instance|
+            encrypt!(instance, attributes)
+          end
+        end
+      end
+
+      private
+
+      # Build a hash of { attribute => encrypted column name }
+      def expand_attributes(klass, attributes)
+        expanded = attributes.flat_map do |attribute|
+          attr_config = klass.encrypted_attributes[attribute]
+          crypt_column_name = attr_config&.fetch(:attribute)
+
+          raise "Couldn't determine encrypted column for #{klass}##{attribute}" if
+            crypt_column_name.nil?
+
+          [attribute, crypt_column_name]
+        end
+
+        Hash[*expanded]
+      end
+
+      # Generate ciphertext for each column and update the database
+      def encrypt!(instance, attributes)
+        to_clear = attributes
+          .map { |plain, crypt| apply_attribute!(instance, plain, crypt) }
+          .compact
+          .flat_map { |plain| [plain, nil] }
+
+        to_clear = Hash[*to_clear]
+
+        if instance.changed?
+          instance.save!
+          instance.update_columns(to_clear)
+        end
+      end
+
+      def apply_attribute!(instance, plain_column, crypt_column)
+        plaintext = instance[plain_column]
+        ciphertext = instance[crypt_column]
+
+        # No need to do anything if the plaintext is nil, or an encrypted
+        # value already exists
+        return nil unless plaintext.present? && !ciphertext.present?
+
+        # attr_encrypted will calculate and set the expected value for us
+        instance.public_send("#{plain_column}=", plaintext) # rubocop:disable GitlabSecurity/PublicSend
+
+        plain_column
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/background_migration/models/encrypt_columns/web_hook.rb b/lib/gitlab/background_migration/models/encrypt_columns/web_hook.rb
new file mode 100644
index 00000000000..bb76eb8ed48
--- /dev/null
+++ b/lib/gitlab/background_migration/models/encrypt_columns/web_hook.rb
@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module BackgroundMigration
+    module Models
+      module EncryptColumns
+        # This model is shared between synchronous and background migrations to
+        # encrypt the `token` and `url` columns
+        class WebHook < ActiveRecord::Base
+          include ::EachBatch
+
+          self.table_name = 'web_hooks'
+          self.inheritance_column = :_type_disabled
+
+          attr_encrypted :token,
+                         mode:      :per_attribute_iv,
+                         algorithm: 'aes-256-gcm',
+                         key:       Settings.attr_encrypted_db_key_base_truncated
+
+          attr_encrypted :url,
+                         mode:      :per_attribute_iv,
+                         algorithm: 'aes-256-gcm',
+                         key:       Settings.attr_encrypted_db_key_base_truncated
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/background_migration/populate_external_pipeline_source.rb b/lib/gitlab/background_migration/populate_external_pipeline_source.rb
new file mode 100644
index 00000000000..036fe641757
--- /dev/null
+++ b/lib/gitlab/background_migration/populate_external_pipeline_source.rb
@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+# rubocop:disable Style/Documentation
+
+module Gitlab
+  module BackgroundMigration
+    class PopulateExternalPipelineSource
+      module Migratable
+        class Pipeline < ActiveRecord::Base
+          self.table_name = 'ci_pipelines'
+
+          def self.sources
+            {
+              unknown: nil,
+              push: 1,
+              web: 2,
+              trigger: 3,
+              schedule: 4,
+              api: 5,
+              external: 6
+            }
+          end
+        end
+
+        class CommitStatus < ActiveRecord::Base
+          self.table_name = 'ci_builds'
+          self.inheritance_column = :_type_disabled
+
+          scope :has_pipeline, -> { where('ci_builds.commit_id=ci_pipelines.id') }
+          scope :of_type, -> (type) { where('type=?', type) }
+        end
+      end
+
+      def perform(start_id, stop_id)
+        external_pipelines(start_id, stop_id)
+          .update_all(source: Migratable::Pipeline.sources[:external])
+      end
+
+      private
+
+      def external_pipelines(start_id, stop_id)
+        Migratable::Pipeline.where(id: (start_id..stop_id))
+          .where(
+            'EXISTS (?) AND NOT EXISTS (?)',
+            Migratable::CommitStatus.of_type('GenericCommitStatus').has_pipeline.select(1),
+            Migratable::CommitStatus.of_type('Ci::Build').has_pipeline.select(1)
+          )
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/config/entry/reports.rb b/lib/gitlab/ci/config/entry/reports.rb
index 7bc482a6f48..98f12c226b3 100644
--- a/lib/gitlab/ci/config/entry/reports.rb
+++ b/lib/gitlab/ci/config/entry/reports.rb
@@ -11,7 +11,7 @@ module Gitlab
           include Validatable
           include Attributable
 
-          ALLOWED_KEYS = %i[junit sast dependency_scanning container_scanning dast].freeze
+          ALLOWED_KEYS = %i[junit codequality sast dependency_scanning container_scanning dast].freeze
 
           attributes ALLOWED_KEYS
 
@@ -21,6 +21,7 @@ module Gitlab
 
             with_options allow_nil: true do
               validates :junit, array_of_strings_or_string: true
+              validates :codequality, array_of_strings_or_string: true
               validates :sast, array_of_strings_or_string: true
               validates :dependency_scanning, array_of_strings_or_string: true
               validates :container_scanning, array_of_strings_or_string: true
diff --git a/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml
index d8fcdfac266..a96595b33a5 100644
--- a/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml
@@ -49,7 +49,7 @@ variables:
   POSTGRES_DB: $CI_ENVIRONMENT_SLUG
 
   KUBERNETES_VERSION: 1.8.6
-  HELM_VERSION: 2.10.0
+  HELM_VERSION: 2.11.0
 
   DOCKER_DRIVER: overlay2
 
@@ -162,7 +162,10 @@ sast:
   artifacts:
     paths: [gl-sast-report.json]
   only:
-    - branches
+    refs:
+      - branches
+    variables:
+      - $GITLAB_FEATURES =~ /\bsast\b/
   except:
     variables:
       - $SAST_DISABLED
@@ -179,7 +182,10 @@ dependency_scanning:
   artifacts:
     paths: [gl-dependency-scanning-report.json]
   only:
-    - branches
+    refs:
+      - branches
+    variables:
+      - $GITLAB_FEATURES =~ /\bdependency_scanning\b/
   except:
     variables:
       - $DEPENDENCY_SCANNING_DISABLED
@@ -196,7 +202,10 @@ container_scanning:
   artifacts:
     paths: [gl-container-scanning-report.json]
   only:
-    - branches
+    refs:
+      - branches
+    variables:
+      - $GITLAB_FEATURES =~ /\bsast_container\b/
   except:
     variables:
       - $CONTAINER_SCANNING_DISABLED
@@ -215,6 +224,8 @@ dast:
     refs:
       - branches
     kubernetes: active
+    variables:
+      - $GITLAB_FEATURES =~ /\bdast\b/
   except:
     refs:
       - master
@@ -228,7 +239,7 @@ review:
     - install_dependencies
     - download_chart
     - ensure_namespace
-    - install_tiller
+    - initialize_tiller
     - create_secret
     - deploy
     - persist_environment_url
@@ -254,6 +265,7 @@ stop_review:
     GIT_STRATEGY: none
   script:
     - install_dependencies
+    - initialize_tiller
     - delete
   environment:
     name: review/$CI_COMMIT_REF_NAME
@@ -288,7 +300,7 @@ staging:
     - install_dependencies
     - download_chart
     - ensure_namespace
-    - install_tiller
+    - initialize_tiller
     - create_secret
     - deploy
   environment:
@@ -312,7 +324,7 @@ canary:
     - install_dependencies
     - download_chart
     - ensure_namespace
-    - install_tiller
+    - initialize_tiller
     - create_secret
     - deploy canary
   environment:
@@ -333,7 +345,7 @@ canary:
     - install_dependencies
     - download_chart
     - ensure_namespace
-    - install_tiller
+    - initialize_tiller
     - create_secret
     - deploy
     - delete canary
@@ -381,7 +393,7 @@ production_manual:
     - install_dependencies
     - download_chart
     - ensure_namespace
-    - install_tiller
+    - initialize_tiller
     - create_secret
     - deploy rollout $ROLLOUT_PERCENTAGE
     - scale stable $((100-ROLLOUT_PERCENTAGE))
@@ -586,26 +598,59 @@ rollout 100%:
       secret_name=''
     fi
 
-    helm upgrade --install \
-      --wait \
-      --set service.enabled="$service_enabled" \
-      --set releaseOverride="$CI_ENVIRONMENT_SLUG" \
-      --set image.repository="$CI_APPLICATION_REPOSITORY" \
-      --set image.tag="$CI_APPLICATION_TAG" \
-      --set image.pullPolicy=IfNotPresent \
-      --set image.secrets[0].name="$secret_name" \
-      --set application.track="$track" \
-      --set application.database_url="$DATABASE_URL" \
-      --set service.url="$CI_ENVIRONMENT_URL" \
-      --set replicaCount="$replicas" \
-      --set postgresql.enabled="$postgres_enabled" \
-      --set postgresql.nameOverride="postgres" \
-      --set postgresql.postgresUser="$POSTGRES_USER" \
-      --set postgresql.postgresPassword="$POSTGRES_PASSWORD" \
-      --set postgresql.postgresDatabase="$POSTGRES_DB" \
-      --namespace="$KUBE_NAMESPACE" \
-      "$name" \
-      chart/
+    if [[ -n "$DB_INITIALIZE" && -z "$(helm ls -q "^$name$")" ]]; then
+      helm upgrade --install \
+        --wait \
+        --set service.enabled="$service_enabled" \
+        --set releaseOverride="$CI_ENVIRONMENT_SLUG" \
+        --set image.repository="$CI_APPLICATION_REPOSITORY" \
+        --set image.tag="$CI_APPLICATION_TAG" \
+        --set image.pullPolicy=IfNotPresent \
+        --set image.secrets[0].name="$secret_name" \
+        --set application.track="$track" \
+        --set application.database_url="$DATABASE_URL" \
+        --set service.url="$CI_ENVIRONMENT_URL" \
+        --set replicaCount="$replicas" \
+        --set postgresql.enabled="$postgres_enabled" \
+        --set postgresql.nameOverride="postgres" \
+        --set postgresql.postgresUser="$POSTGRES_USER" \
+        --set postgresql.postgresPassword="$POSTGRES_PASSWORD" \
+        --set postgresql.postgresDatabase="$POSTGRES_DB" \
+        --set application.initializeCommand="$DB_INITIALIZE" \
+        --namespace="$KUBE_NAMESPACE" \
+        "$name" \
+        chart/
+
+      helm upgrade --reuse-values \
+        --wait \
+        --set application.initializeCommand="" \
+        --set application.migrateCommand="$DB_MIGRATE" \
+        --namespace="$KUBE_NAMESPACE" \
+        "$name" \
+        chart/
+    else
+      helm upgrade --install \
+        --wait \
+        --set service.enabled="$service_enabled" \
+        --set releaseOverride="$CI_ENVIRONMENT_SLUG" \
+        --set image.repository="$CI_APPLICATION_REPOSITORY" \
+        --set image.tag="$CI_APPLICATION_TAG" \
+        --set image.pullPolicy=IfNotPresent \
+        --set image.secrets[0].name="$secret_name" \
+        --set application.track="$track" \
+        --set application.database_url="$DATABASE_URL" \
+        --set service.url="$CI_ENVIRONMENT_URL" \
+        --set replicaCount="$replicas" \
+        --set postgresql.enabled="$postgres_enabled" \
+        --set postgresql.nameOverride="postgres" \
+        --set postgresql.postgresUser="$POSTGRES_USER" \
+        --set postgresql.postgresPassword="$POSTGRES_PASSWORD" \
+        --set postgresql.postgresDatabase="$POSTGRES_DB" \
+        --set application.migrateCommand="$DB_MIGRATE" \
+        --namespace="$KUBE_NAMESPACE" \
+        "$name" \
+        chart/
+    fi
 
     kubectl rollout status -n "$KUBE_NAMESPACE" -w "deployment/$name"
   }
@@ -640,7 +685,12 @@ rollout 100%:
 
     curl "https://kubernetes-helm.storage.googleapis.com/helm-v${HELM_VERSION}-linux-amd64.tar.gz" | tar zx
     mv linux-amd64/helm /usr/bin/
+    mv linux-amd64/tiller /usr/bin/
     helm version --client
+    tiller -version
+
+    helm init --client-only
+    helm plugin install https://github.com/adamreese/helm-local
 
     curl -L -o /usr/bin/kubectl "https://storage.googleapis.com/kubernetes-release/release/v${KUBERNETES_VERSION}/bin/linux/amd64/kubectl"
     chmod +x /usr/bin/kubectl
@@ -747,10 +797,13 @@ rollout 100%:
     echo ""
   }
 
-  function install_tiller() {
+  function initialize_tiller() {
     echo "Checking Tiller..."
-    helm init --upgrade
-    kubectl rollout status -n "$TILLER_NAMESPACE" -w "deployment/tiller-deploy"
+
+    helm local start
+    helm local status
+    export HELM_HOST=":44134"
+
     if ! helm version --debug; then
       echo "Failed to init Tiller."
       return 1
diff --git a/lib/gitlab/data_builder/pipeline.rb b/lib/gitlab/data_builder/pipeline.rb
index eb246d393a1..f382992cb0a 100644
--- a/lib/gitlab/data_builder/pipeline.rb
+++ b/lib/gitlab/data_builder/pipeline.rb
@@ -26,7 +26,8 @@ module Gitlab
           stages: pipeline.stages_names,
           created_at: pipeline.created_at,
           finished_at: pipeline.finished_at,
-          duration: pipeline.duration
+          duration: pipeline.duration,
+          variables: pipeline.variables.map(&:hook_attrs)
         }
       end
 
diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb
index 7f012312819..30541ee3553 100644
--- a/lib/gitlab/database/migration_helpers.rb
+++ b/lib/gitlab/database/migration_helpers.rb
@@ -1073,6 +1073,10 @@ into similar problems in the future (e.g. when new tables are created).
 
         connection.select_value(index_sql).to_i > 0
       end
+
+      def mysql_compatible_index_length
+        Gitlab::Database.mysql? ? 20 : nil
+      end
     end
   end
 end
diff --git a/lib/gitlab/diff/highlight.rb b/lib/gitlab/diff/highlight.rb
index 1f012043e56..a605ddb5c33 100644
--- a/lib/gitlab/diff/highlight.rb
+++ b/lib/gitlab/diff/highlight.rb
@@ -24,7 +24,7 @@ module Gitlab
           # ignore highlighting for "match" lines
           next diff_line if diff_line.meta?
 
-          rich_line = highlight_line(diff_line) || diff_line.text
+          rich_line = highlight_line(diff_line) || ERB::Util.html_escape(diff_line.text)
 
           if line_inline_diffs = inline_diffs[i]
             begin
diff --git a/lib/gitlab/diff/position.rb b/lib/gitlab/diff/position.rb
index fc280f96ec1..f967494199e 100644
--- a/lib/gitlab/diff/position.rb
+++ b/lib/gitlab/diff/position.rb
@@ -69,6 +69,10 @@ module Gitlab
         JSON.generate(formatter.to_h, opts)
       end
 
+      def as_json(opts = nil)
+        to_h.as_json(opts)
+      end
+
       def type
         formatter.line_age
       end
diff --git a/lib/gitlab/git/commit.rb b/lib/gitlab/git/commit.rb
index 5b264868af0..74cdabfed9d 100644
--- a/lib/gitlab/git/commit.rb
+++ b/lib/gitlab/git/commit.rb
@@ -53,9 +53,6 @@ module Gitlab
           # Already a commit?
           return commit_id if commit_id.is_a?(Gitlab::Git::Commit)
 
-          # A rugged reference?
-          commit_id = Gitlab::Git::Ref.dereference_object(commit_id)
-
           # Some weird thing?
           return nil unless commit_id.is_a?(String)
 
@@ -127,8 +124,6 @@ module Gitlab
         #        :topo, or any combination of them (in an array). Commit ordering types
         #        are documented here:
         #        http://www.rubydoc.info/github/libgit2/rugged/Rugged#SORT_NONE-constant)
-        #
-        # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/326
         def find_all(repo, options = {})
           repo.wrapped_gitaly_errors do
             Gitlab::GitalyClient::CommitService.new(repo).find_all_commits(options)
@@ -328,7 +323,6 @@ module Gitlab
         entry = @repository.gitaly_commit_client.tree_entry(id, path, 1)
         return unless entry
 
-        # To be compatible with the rugged format
         entry = entry.to_h
         entry.delete(:data)
         entry[:name] = File.basename(path)
@@ -346,8 +340,8 @@ module Gitlab
           subject: message_split[0] ? message_split[0].chomp.b : "",
           body: raw_commit.message.b,
           parent_ids: raw_commit.parent_ids,
-          author: gitaly_commit_author_from_rugged(raw_commit.author),
-          committer: gitaly_commit_author_from_rugged(raw_commit.committer)
+          author: gitaly_commit_author_from_raw(raw_commit.author),
+          committer: gitaly_commit_author_from_raw(raw_commit.committer)
         )
       end
 
@@ -381,7 +375,7 @@ module Gitlab
         SERIALIZE_KEYS
       end
 
-      def gitaly_commit_author_from_rugged(author_or_committer)
+      def gitaly_commit_author_from_raw(author_or_committer)
         Gitaly::CommitAuthor.new(
           name: author_or_committer[:name].b,
           email: author_or_committer[:email].b,
diff --git a/lib/gitlab/git/diff.rb b/lib/gitlab/git/diff.rb
index f6b51dc3982..0d96211f4d4 100644
--- a/lib/gitlab/git/diff.rb
+++ b/lib/gitlab/git/diff.rb
@@ -19,13 +19,17 @@ module Gitlab
 
       alias_method :expanded?, :expanded
 
-      SERIALIZE_KEYS = %i(diff new_path old_path a_mode b_mode new_file renamed_file deleted_file too_large).freeze
+      # The default maximum content size to display a diff patch.
+      #
+      # If this value ever changes, make sure to create a migration to update
+      # current records, and default of `ApplicationSettings#diff_max_patch_bytes`.
+      DEFAULT_MAX_PATCH_BYTES = 100.kilobytes
 
-      # The maximum size of a diff to display.
-      SIZE_LIMIT = 100.kilobytes
+      # This is a limitation applied on the source (Gitaly), therefore we don't allow
+      # persisting limits over that.
+      MAX_PATCH_BYTES_UPPER_BOUND = 500.kilobytes
 
-      # The maximum size before a diff is collapsed.
-      COLLAPSE_LIMIT = 10.kilobytes
+      SERIALIZE_KEYS = %i(diff new_path old_path a_mode b_mode new_file renamed_file deleted_file too_large).freeze
 
       class << self
         def between(repo, head, base, options = {}, *paths)
@@ -105,6 +109,26 @@ module Gitlab
         def binary_message(old_path, new_path)
           "Binary files #{old_path} and #{new_path} differ\n"
         end
+
+        # Returns the limit of bytes a single diff file can reach before it
+        # appears as 'collapsed' for end-users.
+        # By convention, it's 10% of the persisted `diff_max_patch_bytes`.
+        #
+        # Example: If we have 100k for the `diff_max_patch_bytes`, it will be 10k by
+        # default.
+        #
+        # Patches surpassing this limit should still be persisted in the database.
+        def patch_safe_limit_bytes
+          patch_hard_limit_bytes / 10
+        end
+
+        # Returns the limit for a single diff file (patch).
+        #
+        # Patches surpassing this limit shouldn't be persisted in the database
+        # and will be presented as 'too large' for end-users.
+        def patch_hard_limit_bytes
+          Gitlab::CurrentSettings.diff_max_patch_bytes
+        end
       end
 
       def initialize(raw_diff, expanded: true)
@@ -150,7 +174,7 @@ module Gitlab
 
       def too_large?
         if @too_large.nil?
-          @too_large = @diff.bytesize >= SIZE_LIMIT
+          @too_large = @diff.bytesize >= self.class.patch_hard_limit_bytes
         else
           @too_large
         end
@@ -168,7 +192,7 @@ module Gitlab
       def collapsed?
         return @collapsed if defined?(@collapsed)
 
-        @collapsed = !expanded && @diff.bytesize >= COLLAPSE_LIMIT
+        @collapsed = !expanded && @diff.bytesize >= self.class.patch_safe_limit_bytes
       end
 
       def collapse!
@@ -219,30 +243,6 @@ module Gitlab
           collapse!
         end
       end
-
-      # If the patch surpasses any of the diff limits it calls the appropiate
-      # prune method and returns true. Otherwise returns false.
-      def prune_large_patch(patch)
-        size = 0
-
-        patch.each_hunk do |hunk|
-          hunk.each_line do |line|
-            size += line.content.bytesize
-
-            if size >= SIZE_LIMIT
-              too_large!
-              return true # rubocop:disable Cop/AvoidReturnFromBlocks
-            end
-          end
-        end
-
-        if !expanded && size >= COLLAPSE_LIMIT
-          collapse!
-          return true
-        end
-
-        false
-      end
     end
   end
 end
diff --git a/lib/gitlab/git/diff_collection.rb b/lib/gitlab/git/diff_collection.rb
index 20dce8d0e06..47ebca7c4a2 100644
--- a/lib/gitlab/git/diff_collection.rb
+++ b/lib/gitlab/git/diff_collection.rb
@@ -19,7 +19,7 @@ module Gitlab
         limits[:safe_max_files] = [limits[:max_files], DEFAULT_LIMITS[:max_files]].min
         limits[:safe_max_lines] = [limits[:max_lines], DEFAULT_LIMITS[:max_lines]].min
         limits[:safe_max_bytes] = limits[:safe_max_files] * 5.kilobytes # Average 5 KB per file
-        limits[:max_patch_bytes] = Gitlab::Git::Diff::SIZE_LIMIT
+        limits[:max_patch_bytes] = Gitlab::Git::Diff.patch_hard_limit_bytes
 
         OpenStruct.new(limits)
       end
diff --git a/lib/gitlab/git/ref.rb b/lib/gitlab/git/ref.rb
index fa71a4e7ea7..31a280155bd 100644
--- a/lib/gitlab/git/ref.rb
+++ b/lib/gitlab/git/ref.rb
@@ -1,5 +1,3 @@
-# Gitaly note: JV: probably no RPC's here (just one interaction with Rugged).
-
 module Gitlab
   module Git
     class Ref
@@ -26,13 +24,6 @@ module Gitlab
         str.gsub(%r{\Arefs/heads/}, '')
       end
 
-      # Gitaly: this method will probably be migrated indirectly via its call sites.
-      def self.dereference_object(object)
-        object = object.target while object.is_a?(Rugged::Tag::Annotation)
-
-        object
-      end
-
       def initialize(repository, name, target, dereferenced_target)
         @name = Gitlab::Git.ref_name(name)
         @dereferenced_target = dereferenced_target
diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb
index 3d5a63bdbac..7732049b69b 100644
--- a/lib/gitlab/git/repository.rb
+++ b/lib/gitlab/git/repository.rb
@@ -9,14 +9,6 @@ module Gitlab
       include Gitlab::EncodingHelper
       include Gitlab::Utils::StrongMemoize
 
-      ALLOWED_OBJECT_DIRECTORIES_VARIABLES = %w[
-        GIT_OBJECT_DIRECTORY
-        GIT_ALTERNATE_OBJECT_DIRECTORIES
-      ].freeze
-      ALLOWED_OBJECT_RELATIVE_DIRECTORIES_VARIABLES = %w[
-        GIT_OBJECT_DIRECTORY_RELATIVE
-        GIT_ALTERNATE_OBJECT_DIRECTORIES_RELATIVE
-      ].freeze
       SEARCH_CONTEXT_LINES = 3
       REV_LIST_COMMIT_LIMIT = 2_000
       # In https://gitlab.com/gitlab-org/gitaly/merge_requests/698
@@ -104,15 +96,6 @@ module Gitlab
         raise Gitlab::Git::CommandError.new(e.message)
       end
 
-      # This method will be removed when Gitaly reaches v1.1.
-      def rugged
-        circuit_breaker.perform do
-          Rugged::Repository.new(path, alternates: alternate_object_directories)
-        end
-      rescue Rugged::RepositoryError, Rugged::OSError
-        raise NoRepository.new('no repository for such path')
-      end
-
       def circuit_breaker
         @circuit_breaker ||= Gitlab::Git::Storage::CircuitBreaker.for_storage(storage)
       end
@@ -638,20 +621,6 @@ module Gitlab
         end
       end
 
-      AUTOCRLF_VALUES = {
-        "true" => true,
-        "false" => false,
-        "input" => :input
-      }.freeze
-
-      def autocrlf
-        AUTOCRLF_VALUES[rugged.config['core.autocrlf']]
-      end
-
-      def autocrlf=(value)
-        rugged.config['core.autocrlf'] = AUTOCRLF_VALUES.invert[value]
-      end
-
       # Returns result like "git ls-files" , recursive and full file path
       #
       # Ex.
@@ -953,6 +922,12 @@ module Gitlab
         end
       end
 
+      def list_last_commits_for_tree(sha, path, offset: 0, limit: 25)
+        wrapped_gitaly_errors do
+          gitaly_commit_client.list_last_commits_for_tree(sha, path, offset: offset, limit: limit)
+        end
+      end
+
       def last_commit_for_path(sha, path)
         wrapped_gitaly_errors do
           gitaly_commit_client.last_commit_for_path(sha, path)
@@ -1018,14 +993,6 @@ module Gitlab
         found_module && found_module['url']
       end
 
-      def alternate_object_directories
-        relative_object_directories.map { |d| File.join(path, d) }
-      end
-
-      def relative_object_directories
-        Gitlab::Git::HookEnv.all(gl_repository).values_at(*ALLOWED_OBJECT_RELATIVE_DIRECTORIES_VARIABLES).flatten.compact
-      end
-
       # Returns true if the given ref name exists
       #
       # Ref names must start with `refs/`.
diff --git a/lib/gitlab/git/wiki.rb b/lib/gitlab/git/wiki.rb
index d2dc4f2e688..072019dfb0a 100644
--- a/lib/gitlab/git/wiki.rb
+++ b/lib/gitlab/git/wiki.rb
@@ -1,9 +1,16 @@
+# We only need Gollum::Page so let's not load all of gollum-lib.
+require 'gollum-lib/pagination'
+require 'gollum-lib/wiki'
+require 'gollum-lib/page'
+
 module Gitlab
   module Git
     class Wiki
       DuplicatePageError = Class.new(StandardError)
       OperationError = Class.new(StandardError)
 
+      DEFAULT_PAGINATION = Kaminari.config.default_per_page
+
       CommitDetails = Struct.new(:user_id, :username, :name, :email, :message) do
         def to_h
           { user_id: user_id, username: username, name: name, email: email, message: message }
@@ -74,7 +81,7 @@ module Gitlab
         # Gitaly uses gollum-lib to get the versions. Gollum defaults to 20
         # per page, but also fetches 20 if `limit` or `per_page` < 20.
         # Slicing returns an array with the expected number of items.
-        slice_bound = options[:limit] || options[:per_page] || Gollum::Page.per_page
+        slice_bound = options[:limit] || options[:per_page] || DEFAULT_PAGINATION
         versions[0..slice_bound]
       end
 
@@ -104,26 +111,6 @@ module Gitlab
 
       private
 
-      def new_page(gollum_page)
-        Gitlab::Git::WikiPage.new(gollum_page, new_version(gollum_page, gollum_page.version.id))
-      end
-
-      def new_version(gollum_page, commit_id)
-        Gitlab::Git::WikiPageVersion.new(version(commit_id), gollum_page&.format)
-      end
-
-      def version(commit_id)
-        commit_find_proc = -> { Gitlab::Git::Commit.find(@repository, commit_id) }
-
-        Gitlab::SafeRequestStore.fetch([:wiki_version_commit, commit_id]) { commit_find_proc.call }
-      end
-
-      def assert_type!(object, klass)
-        unless object.is_a?(klass)
-          raise ArgumentError, "expected a #{klass}, got #{object.inspect}"
-        end
-      end
-
       def gitaly_wiki_client
         @gitaly_wiki_client ||= Gitlab::GitalyClient::WikiService.new(@repository)
       end
diff --git a/lib/gitlab/git/wiki_file.rb b/lib/gitlab/git/wiki_file.rb
index 84335aca4bc..64313bb04e8 100644
--- a/lib/gitlab/git/wiki_file.rb
+++ b/lib/gitlab/git/wiki_file.rb
@@ -3,17 +3,12 @@ module Gitlab
     class WikiFile
       attr_reader :mime_type, :raw_data, :name, :path
 
-      # This class is meant to be serializable so that it can be constructed
-      # by Gitaly and sent over the network to GitLab.
-      #
-      # Because Gollum::File is not serializable we must get all the data from
-      # 'gollum_file' during initialization, and NOT store it in an instance
-      # variable.
-      def initialize(gollum_file)
-        @mime_type = gollum_file.mime_type
-        @raw_data = gollum_file.raw_data
-        @name = gollum_file.name
-        @path = gollum_file.path
+      # This class wraps Gitlab::GitalyClient::WikiFile
+      def initialize(gitaly_file)
+        @mime_type = gitaly_file.mime_type
+        @raw_data = gitaly_file.raw_data
+        @name = gitaly_file.name
+        @path = gitaly_file.path
       end
     end
   end
diff --git a/lib/gitlab/git/wiki_page.rb b/lib/gitlab/git/wiki_page.rb
index 669ae11a423..c4087c9ebdc 100644
--- a/lib/gitlab/git/wiki_page.rb
+++ b/lib/gitlab/git/wiki_page.rb
@@ -3,25 +3,15 @@ module Gitlab
     class WikiPage
       attr_reader :url_path, :title, :format, :path, :version, :raw_data, :name, :text_data, :historical, :formatted_data
 
-      # This class is meant to be serializable so that it can be constructed
-      # by Gitaly and sent over the network to GitLab.
-      #
-      # Because Gollum::Page is not serializable we must get all the data from
-      # 'gollum_page' during initialization, and NOT store it in an instance
-      # variable.
-      #
-      # Note that 'version' is a WikiPageVersion instance which it itself
-      # serializable. That means it's OK to store 'version' in an instance
-      # variable.
-      def initialize(gollum_page, version)
-        @url_path = gollum_page.url_path
-        @title = gollum_page.title
-        @format = gollum_page.format
-        @path = gollum_page.path
-        @raw_data = gollum_page.raw_data
-        @name = gollum_page.name
-        @historical = gollum_page.historical?
-        @formatted_data = gollum_page.formatted_data if gollum_page.is_a?(Gollum::Page)
+      # This class abstracts away Gitlab::GitalyClient::WikiPage
+      def initialize(gitaly_page, version)
+        @url_path = gitaly_page.url_path
+        @title = gitaly_page.title
+        @format = gitaly_page.format
+        @path = gitaly_page.path
+        @raw_data = gitaly_page.raw_data
+        @name = gitaly_page.name
+        @historical = gitaly_page.historical?
 
         @version = version
       end
diff --git a/lib/gitlab/git/wiki_page_version.rb b/lib/gitlab/git/wiki_page_version.rb
index 55f1afedcab..d5e7e70fd31 100644
--- a/lib/gitlab/git/wiki_page_version.rb
+++ b/lib/gitlab/git/wiki_page_version.rb
@@ -3,11 +3,6 @@ module Gitlab
     class WikiPageVersion
       attr_reader :commit, :format
 
-      # This class is meant to be serializable so that it can be constructed
-      # by Gitaly and sent over the network to GitLab.
-      #
-      # Both 'commit' (a Gitlab::Git::Commit) and 'format' (a string) are
-      # serializable.
       def initialize(commit, format)
         @commit = commit
         @format = format
diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb
index 30cd09a0ca7..240a0d7d1b8 100644
--- a/lib/gitlab/git_access.rb
+++ b/lib/gitlab/git_access.rb
@@ -24,8 +24,8 @@ module Gitlab
       cannot_push_to_read_only: "You can't push code to a read-only GitLab instance."
     }.freeze
 
-    DOWNLOAD_COMMANDS = %w{ git-upload-pack git-upload-archive }.freeze
-    PUSH_COMMANDS = %w{ git-receive-pack }.freeze
+    DOWNLOAD_COMMANDS = %w{git-upload-pack git-upload-archive}.freeze
+    PUSH_COMMANDS = %w{git-receive-pack}.freeze
     ALL_COMMANDS = DOWNLOAD_COMMANDS + PUSH_COMMANDS
 
     attr_reader :actor, :project, :protocol, :authentication_abilities, :namespace_path, :project_path, :redirected_path, :auth_result_type, :changes
diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb
index 07e5e204b68..085b2a127a5 100644
--- a/lib/gitlab/gitaly_client/commit_service.rb
+++ b/lib/gitlab/gitaly_client/commit_service.rb
@@ -148,6 +148,24 @@ module Gitlab
         GitalyClient.call(@repository.storage, :commit_service, :count_commits, request, timeout: GitalyClient.medium_timeout).count
       end
 
+      def list_last_commits_for_tree(revision, path, offset: 0, limit: 25)
+        request = Gitaly::ListLastCommitsForTreeRequest.new(
+          repository: @gitaly_repo,
+          revision: encode_binary(revision),
+          path: encode_binary(path.to_s),
+          offset: offset,
+          limit: limit
+        )
+
+        response = GitalyClient.call(@repository.storage, :commit_service, :list_last_commits_for_tree, request, timeout: GitalyClient.medium_timeout)
+
+        response.each_with_object({}) do |gitaly_response, hsh|
+          gitaly_response.commits.each do |commit_for_tree|
+            hsh[commit_for_tree.path] = Gitlab::Git::Commit.new(@repository, commit_for_tree.commit)
+          end
+        end
+      end
+
       def last_commit_for_path(revision, path)
         request = Gitaly::LastCommitForPathRequest.new(
           repository: @gitaly_repo,
diff --git a/lib/gitlab/gitaly_client/wiki_service.rb b/lib/gitlab/gitaly_client/wiki_service.rb
index 75be7d1f5a0..7c2c228ad01 100644
--- a/lib/gitlab/gitaly_client/wiki_service.rb
+++ b/lib/gitlab/gitaly_client/wiki_service.rb
@@ -110,7 +110,7 @@ module Gitlab
           repository: @gitaly_repo,
           page_path: encode_binary(page_path),
           page: options[:page] || 1,
-          per_page: options[:per_page] || Gollum::Page.per_page
+          per_page: options[:per_page] || Gitlab::Git::Wiki::DEFAULT_PAGINATION
         )
 
         stream = GitalyClient.call(@repository.storage, :wiki_service, :wiki_get_page_versions, request, timeout: GitalyClient.medium_timeout)
diff --git a/lib/gitlab/highlight.rb b/lib/gitlab/highlight.rb
index 5408a1a6838..0b6cc893db1 100644
--- a/lib/gitlab/highlight.rb
+++ b/lib/gitlab/highlight.rb
@@ -1,5 +1,8 @@
 module Gitlab
   class Highlight
+    TIMEOUT_BACKGROUND = 30.seconds
+    TIMEOUT_FOREGROUND = 3.seconds
+
     def self.highlight(blob_name, blob_content, repository: nil, plain: false)
       new(blob_name, blob_content, repository: repository)
         .highlight(blob_content, continue: false, plain: plain)
@@ -51,11 +54,20 @@ module Gitlab
     end
 
     def highlight_rich(text, continue: true)
-      @formatter.format(lexer.lex(text, continue: continue), tag: lexer.tag).html_safe
+      tag = lexer.tag
+      tokens = lexer.lex(text, continue: continue)
+      Timeout.timeout(timeout_time) { @formatter.format(tokens, tag: tag).html_safe }
+    rescue Timeout::Error => e
+      Gitlab::Sentry.track_exception(e)
+      highlight_plain(text)
     rescue
       highlight_plain(text)
     end
 
+    def timeout_time
+      Sidekiq.server? ? TIMEOUT_BACKGROUND : TIMEOUT_FOREGROUND
+    end
+
     def link_dependencies(text, highlighted_text)
       Gitlab::DependencyLinker.link(blob_name, text, highlighted_text)
     end
diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml
index a19b3c88627..2bed470514b 100644
--- a/lib/gitlab/import_export/import_export.yml
+++ b/lib/gitlab/import_export/import_export.yml
@@ -147,6 +147,12 @@ excluded_attributes:
     - :reference
     - :reference_html
     - :epic_id
+  hooks:
+    - :token
+    - :encrypted_token
+    - :encrypted_token_iv
+    - :encrypted_url
+    - :encrypted_url_iv
 
 methods:
   labels:
diff --git a/lib/gitlab/kubernetes/helm/api.rb b/lib/gitlab/kubernetes/helm/api.rb
index 2dd74c68075..e21bc531444 100644
--- a/lib/gitlab/kubernetes/helm/api.rb
+++ b/lib/gitlab/kubernetes/helm/api.rb
@@ -17,6 +17,12 @@ module Gitlab
           kubeclient.create_pod(command.pod_resource)
         end
 
+        def update(command)
+          namespace.ensure_exists!
+          update_config_map(command)
+          kubeclient.create_pod(command.pod_resource)
+        end
+
         ##
         # Returns Pod phase
         #
@@ -36,6 +42,12 @@ module Gitlab
           kubeclient.delete_pod(pod_name, namespace.name)
         end
 
+        def get_config_map(config_map_name)
+          namespace.ensure_exists!
+
+          kubeclient.get_config_map(config_map_name, namespace.name)
+        end
+
         private
 
         attr_reader :kubeclient, :namespace
@@ -46,6 +58,12 @@ module Gitlab
           end
         end
 
+        def update_config_map(command)
+          command.config_map_resource.tap do |config_map_resource|
+            kubeclient.update_config_map(config_map_resource)
+          end
+        end
+
         def create_service_account(command)
           command.service_account_resource.tap do |service_account_resource|
             break unless service_account_resource
diff --git a/lib/gitlab/kubernetes/helm/upgrade_command.rb b/lib/gitlab/kubernetes/helm/upgrade_command.rb
new file mode 100644
index 00000000000..74188046739
--- /dev/null
+++ b/lib/gitlab/kubernetes/helm/upgrade_command.rb
@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Kubernetes
+    module Helm
+      class UpgradeCommand
+        include BaseCommand
+
+        attr_reader :name, :chart, :version, :repository, :files
+
+        def initialize(name, chart:, files:, rbac:, version: nil, repository: nil)
+          @name = name
+          @chart = chart
+          @rbac = rbac
+          @version = version
+          @files = files
+          @repository = repository
+        end
+
+        def generate_script
+          super + [
+            init_command,
+            repository_command,
+            script_command
+          ].compact.join("\n")
+        end
+
+        def rbac?
+          @rbac
+        end
+
+        def pod_name
+          "upgrade-#{name}"
+        end
+
+        private
+
+        def init_command
+          'helm init --client-only >/dev/null'
+        end
+
+        def repository_command
+          "helm repo add #{name} #{repository}" if repository
+        end
+
+        def script_command
+          upgrade_flags = "#{optional_version_flag}#{optional_tls_flags}" \
+            " --reset-values" \
+            " --install" \
+            " --namespace #{::Gitlab::Kubernetes::Helm::NAMESPACE}" \
+            " -f /data/helm/#{name}/config/values.yaml"
+
+          "helm upgrade #{name} #{chart}#{upgrade_flags} >/dev/null\n"
+        end
+
+        def optional_version_flag
+          " --version #{version}" if version
+        end
+
+        def optional_tls_flags
+          return unless files.key?(:'ca.pem')
+
+          " --tls" \
+            " --tls-ca-cert #{files_dir}/ca.pem" \
+            " --tls-cert #{files_dir}/cert.pem" \
+            " --tls-key #{files_dir}/key.pem"
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/tree_summary.rb b/lib/gitlab/tree_summary.rb
index b05d408b1c0..c2955cd374c 100644
--- a/lib/gitlab/tree_summary.rb
+++ b/lib/gitlab/tree_summary.rb
@@ -73,25 +73,29 @@ module Gitlab
     end
 
     def fill_last_commits!(entries)
-      # n+1: https://gitlab.com/gitlab-org/gitlab-ce/issues/37433
-      Gitlab::GitalyClient.allow_n_plus_1_calls do
-        entries.each do |entry|
-          raw_commit = repository.last_commit_for_path(commit.id, entry_path(entry))
+      # Ensure the path is in "path/" format
+      ensured_path =
+        if path
+          File.join(*[path, ""])
+        end
+
+      commits_hsh = repository.list_last_commits_for_tree(commit.id, ensured_path, offset: offset, limit: limit)
 
-          if raw_commit
-            commit = resolve_commit(raw_commit)
+      entries.each do |entry|
+        path_key = entry_path(entry)
+        commit = cache_commit(commits_hsh[path_key])
 
-            entry[:commit] = commit
-            entry[:commit_path] =  commit_path(commit)
-          end
+        if commit
+          entry[:commit] = commit
+          entry[:commit_path] = commit_path(commit)
         end
       end
     end
 
-    def resolve_commit(raw_commit)
-      return nil unless raw_commit.present?
+    def cache_commit(commit)
+      return nil unless commit.present?
 
-      resolved_commits[raw_commit.id] ||= ::Commit.new(raw_commit, project)
+      resolved_commits[commit.id] ||= commit
     end
 
     def commit_path(commit)
diff --git a/lib/gitlab/url_sanitizer.rb b/lib/gitlab/url_sanitizer.rb
index 308a95d2f09..29672d68cad 100644
--- a/lib/gitlab/url_sanitizer.rb
+++ b/lib/gitlab/url_sanitizer.rb
@@ -3,7 +3,7 @@ module Gitlab
     ALLOWED_SCHEMES = %w[http https ssh git].freeze
 
     def self.sanitize(content)
-      regexp = URI::Parser.new.make_regexp(ALLOWED_SCHEMES)
+      regexp = URI::DEFAULT_PARSER.make_regexp(ALLOWED_SCHEMES)
 
       content.gsub(regexp) { |url| new(url).masked_url }
     rescue Addressable::URI::InvalidURIError
diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb
index f7d8ee571cd..5097c3253c9 100644
--- a/lib/gitlab/usage_data.rb
+++ b/lib/gitlab/usage_data.rb
@@ -10,6 +10,7 @@ module Gitlab
                           .merge(features_usage_data)
                           .merge(components_usage_data)
                           .merge(cycle_analytics_usage_data)
+                          .merge(usage_counters)
       end
 
       def to_json(force_refresh: false)
@@ -106,6 +107,12 @@ module Gitlab
         }
       end
 
+      def usage_counters
+        {
+          web_ide_commits: Gitlab::WebIdeCommitsCounter.total_count
+        }
+      end
+
       def components_usage_data
         {
           gitlab_pages: { enabled: Gitlab.config.pages.enabled, version: Gitlab::Pages::VERSION },
diff --git a/lib/gitlab/web_ide_commits_counter.rb b/lib/gitlab/web_ide_commits_counter.rb
new file mode 100644
index 00000000000..1cd9b5295b9
--- /dev/null
+++ b/lib/gitlab/web_ide_commits_counter.rb
@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module WebIdeCommitsCounter
+    WEB_IDE_COMMITS_KEY = "WEB_IDE_COMMITS_COUNT".freeze
+
+    class << self
+      def increment
+        Gitlab::Redis::SharedState.with { |redis| redis.incr(WEB_IDE_COMMITS_KEY) }
+      end
+
+      def total_count
+        Gitlab::Redis::SharedState.with { |redis| redis.get(WEB_IDE_COMMITS_KEY).to_i }
+      end
+    end
+  end
+end