Add latest changes from gitlab-org/gitlab@master

7 files changed, 152 insertions, 23 deletions

diff --git a/lib/api/internal/pages.rb b/lib/api/internal/pages.rb
index 003af7f6dd4..a2fe3e09df8 100644
--- a/lib/api/internal/pages.rb
+++ b/lib/api/internal/pages.rb
@@ -25,7 +25,7 @@ module API
           end
           get "/" do
             host = Namespace.find_by_pages_host(params[:host]) || PagesDomain.find_by_domain(params[:host])
-            not_found! unless host
+            no_content! unless host
 
             virtual_domain = host.pages_virtual_domain
             no_content! unless virtual_domain
diff --git a/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml
index ef2fc561201..f708e95c2cf 100644
--- a/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml
@@ -1,7 +1,7 @@
 # Read more about this feature here: https://docs.gitlab.com/ee/user/application_security/container_scanning/
 
 variables:
-  CS_MAJOR_VERSION: 1
+  CS_MAJOR_VERSION: 2
 
 container_scanning:
   stage: test
diff --git a/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml
index 3f9d61a79f5..a37714eeed3 100644
--- a/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml
@@ -43,7 +43,6 @@ dependency_scanning:
           DS_ANALYZER_IMAGE_TAG \
           DS_DEFAULT_ANALYZERS \
           DS_EXCLUDED_PATHS \
-          DEP_SCAN_DISABLE_REMOTE_CHECKS \
           DS_DOCKER_CLIENT_NEGOTIATION_TIMEOUT \
           DS_PULL_ANALYZER_IMAGE_TIMEOUT \
           DS_RUN_ANALYZER_TIMEOUT \
diff --git a/lib/gitlab/diff/deprecated_highlight_cache.rb b/lib/gitlab/diff/deprecated_highlight_cache.rb
new file mode 100644
index 00000000000..47347686973
--- /dev/null
+++ b/lib/gitlab/diff/deprecated_highlight_cache.rb
@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+#
+module Gitlab
+  module Diff
+    class DeprecatedHighlightCache
+      delegate :diffable, to: :@diff_collection
+      delegate :diff_options, to: :@diff_collection
+
+      def initialize(diff_collection, backend: Rails.cache)
+        @backend = backend
+        @diff_collection = diff_collection
+      end
+
+      # - Reads from cache
+      # - Assigns DiffFile#highlighted_diff_lines for cached files
+      def decorate(diff_file)
+        if content = read_file(diff_file)
+          diff_file.highlighted_diff_lines = content.map do |line|
+            Gitlab::Diff::Line.init_from_hash(line)
+          end
+        end
+      end
+
+      # It populates a Hash in order to submit a single write to the memory
+      # cache. This avoids excessive IO generated by N+1's (1 writing for
+      # each highlighted line or file).
+      def write_if_empty
+        return if cached_content.present?
+
+        @diff_collection.diff_files.each do |diff_file|
+          next unless cacheable?(diff_file)
+
+          diff_file_id = diff_file.file_identifier
+
+          cached_content[diff_file_id] = diff_file.highlighted_diff_lines.map(&:to_hash)
+        end
+
+        cache.write(key, cached_content, expires_in: 1.week)
+      end
+
+      def clear
+        cache.delete(key)
+      end
+
+      def key
+        [diffable, 'highlighted-diff-files', Gitlab::Diff::Line::SERIALIZE_KEYS, diff_options]
+      end
+
+      private
+
+      def read_file(diff_file)
+        cached_content[diff_file.file_identifier]
+      end
+
+      def cache
+        @backend
+      end
+
+      def cached_content
+        @cached_content ||= cache.read(key) || {}
+      end
+
+      def cacheable?(diff_file)
+        diffable.present? && diff_file.text? && diff_file.diffable?
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/diff/file_collection/merge_request_diff.rb b/lib/gitlab/diff/file_collection/merge_request_diff.rb
index 3d661111f13..1d99349304b 100644
--- a/lib/gitlab/diff/file_collection/merge_request_diff.rb
+++ b/lib/gitlab/diff/file_collection/merge_request_diff.rb
@@ -37,7 +37,11 @@ module Gitlab
         private
 
         def cache
-          @cache ||= Gitlab::Diff::HighlightCache.new(self)
+          @cache ||= if Feature.enabled?(:hset_redis_diff_caching, project)
+                       Gitlab::Diff::HighlightCache.new(self)
+                     else
+                       Gitlab::Diff::DeprecatedHighlightCache.new(self)
+                     end
         end
       end
     end
diff --git a/lib/gitlab/diff/highlight_cache.rb b/lib/gitlab/diff/highlight_cache.rb
index e4390771db2..12a1db70b36 100644
--- a/lib/gitlab/diff/highlight_cache.rb
+++ b/lib/gitlab/diff/highlight_cache.rb
@@ -3,16 +3,19 @@
 module Gitlab
   module Diff
     class HighlightCache
-      delegate :diffable, to: :@diff_collection
+      EXPIRATION = 1.week
+      VERSION = 1
+
+      delegate :diffable,     to: :@diff_collection
       delegate :diff_options, to: :@diff_collection
 
-      def initialize(diff_collection, backend: Rails.cache)
-        @backend = backend
+      def initialize(diff_collection)
         @diff_collection = diff_collection
       end
 
       # - Reads from cache
       # - Assigns DiffFile#highlighted_diff_lines for cached files
+      #
       def decorate(diff_file)
         if content = read_file(diff_file)
           diff_file.highlighted_diff_lines = content.map do |line|
@@ -21,43 +24,95 @@ module Gitlab
         end
       end
 
-      # It populates a Hash in order to submit a single write to the memory
-      # cache. This avoids excessive IO generated by N+1's (1 writing for
-      # each highlighted line or file).
+      # For every file that isn't already contained in the redis hash, store the
+      #   result of #highlighted_diff_lines, then submit the uncached content
+      #   to #write_to_redis_hash to submit a single write. This avoids excessive
+      #   IO generated by N+1's (1 writing for each highlighted line or file).
+      #
       def write_if_empty
-        return if cached_content.present?
+        return if uncached_files.empty?
 
-        @diff_collection.diff_files.each do |diff_file|
+        new_cache_content = {}
+        uncached_files.each do |diff_file|
           next unless cacheable?(diff_file)
 
-          diff_file_id = diff_file.file_identifier
-
-          cached_content[diff_file_id] = diff_file.highlighted_diff_lines.map(&:to_hash)
+          new_cache_content[diff_file.file_path] = diff_file.highlighted_diff_lines.map(&:to_hash)
         end
 
-        cache.write(key, cached_content, expires_in: 1.week)
+        write_to_redis_hash(new_cache_content)
       end
 
       def clear
-        cache.delete(key)
+        Gitlab::Redis::Cache.with do |redis|
+          redis.del(key)
+        end
       end
 
       def key
-        [diffable, 'highlighted-diff-files', Gitlab::Diff::Line::SERIALIZE_KEYS, diff_options]
+        @redis_key ||= ['highlighted-diff-files', diffable.cache_key, VERSION, diff_options].join(":")
       end
 
       private
 
-      def read_file(diff_file)
-        cached_content[diff_file.file_identifier]
+      def uncached_files
+        diff_files = @diff_collection.diff_files
+
+        diff_files.select { |file| read_cache[file.file_path].nil? }
       end
 
-      def cache
-        @backend
+      # Given a hash of:
+      #   { "file/to/cache" =>
+      #   [ { line_code: "a5cc2925ca8258af241be7e5b0381edf30266302_19_19",
+      #       rich_text: " <span id=\"LC19\" class=\"line\" lang=\"plaintext\">config/initializers/secret_token.rb</span>\n",
+      #       text: " config/initializers/secret_token.rb",
+      #       type: nil,
+      #       index: 3,
+      #       old_pos: 19,
+      #       new_pos: 19 }
+      #   ] }
+      #
+      #   ...it will write/update a Gitlab::Redis hash (HSET)
+      #
+      def write_to_redis_hash(hash)
+        Gitlab::Redis::Cache.with do |redis|
+          redis.pipelined do
+            hash.each do |diff_file_id, highlighted_diff_lines_hash|
+              redis.hset(key, diff_file_id, highlighted_diff_lines_hash.to_json)
+            end
+
+            # HSETs have to have their expiration date manually updated
+            #
+            redis.expire(key, EXPIRATION)
+          end
+        end
+      end
+
+      def file_paths
+        @file_paths ||= @diff_collection.diffs.collect(&:file_path)
+      end
+
+      def read_file(diff_file)
+        cached_content[diff_file.file_path]
       end
 
       def cached_content
-        @cached_content ||= cache.read(key) || {}
+        @cached_content ||= read_cache
+      end
+
+      def read_cache
+        return {} unless file_paths.any?
+
+        results = []
+
+        Gitlab::Redis::Cache.with do |redis|
+          results = redis.hmget(key, file_paths)
+        end
+
+        results.map! do |result|
+          JSON.parse(result, symbolize_names: true) unless result.nil?
+        end
+
+        file_paths.zip(results).to_h
       end
 
       def cacheable?(diff_file)
diff --git a/lib/gitlab/diff/line.rb b/lib/gitlab/diff/line.rb
index 001748afb41..28ea1921f90 100644
--- a/lib/gitlab/diff/line.rb
+++ b/lib/gitlab/diff/line.rb
@@ -3,6 +3,9 @@
 module Gitlab
   module Diff
     class Line
+      # When SERIALIZE_KEYS is updated, to reset the redis cache entries you'll
+      #   need to bump the VERSION constant on Gitlab::Diff::HighlightCache
+      #
       SERIALIZE_KEYS = %i(line_code rich_text text type index old_pos new_pos).freeze
 
       attr_reader :line_code, :type, :old_pos, :new_pos