Add latest changes from gitlab-org/gitlab@master

9 files changed, 372 insertions, 20 deletions

diff --git a/lib/gitlab/ci/reports/coverage_reports.rb b/lib/gitlab/ci/reports/coverage_report.rb
index 31afb636d2f..201863fd311 100644
--- a/lib/gitlab/ci/reports/coverage_reports.rb
+++ b/lib/gitlab/ci/reports/coverage_report.rb
@@ -3,7 +3,7 @@
 module Gitlab
   module Ci
     module Reports
-      class CoverageReports
+      class CoverageReport
         attr_reader :files
 
         def initialize
diff --git a/lib/gitlab/diff/line.rb b/lib/gitlab/diff/line.rb
index 316a0d2815a..75127098600 100644
--- a/lib/gitlab/diff/line.rb
+++ b/lib/gitlab/diff/line.rb
@@ -10,7 +10,7 @@ module Gitlab
 
       attr_reader :marker_ranges
       attr_writer :text, :rich_text
-      attr_accessor :index, :old_pos, :new_pos, :line_code, :type
+      attr_accessor :index, :old_pos, :new_pos, :line_code, :type, :embedded_image
 
       def initialize(text, type, index, old_pos, new_pos, parent_file: nil, line_code: nil, rich_text: nil)
         @text = text
diff --git a/lib/gitlab/diff/rendered/notebook/diff_file.rb b/lib/gitlab/diff/rendered/notebook/diff_file.rb
index 1f064d8af50..68011555c3c 100644
--- a/lib/gitlab/diff/rendered/notebook/diff_file.rb
+++ b/lib/gitlab/diff/rendered/notebook/diff_file.rb
@@ -14,6 +14,7 @@ module Gitlab
           LOG_IPYNBDIFF_TIMEOUT = 'IPYNB_DIFF_TIMEOUT'
           LOG_IPYNBDIFF_INVALID = 'IPYNB_DIFF_INVALID'
           LOG_IPYNBDIFF_TRUNCATED = 'IPYNB_DIFF_TRUNCATED'
+          EMBEDDED_IMAGE_PATTERN = '    ![](data:image'
 
           attr_reader :source_diff
 
@@ -69,7 +70,6 @@ module Gitlab
               Timeout.timeout(timeout_time) do
                 IpynbDiff.diff(source_diff.old_blob&.data, source_diff.new_blob&.data,
                                raise_if_invalid_nb: true,
-                               hide_images: true,
                                diffy_opts: { include_diff_info: true })&.tap do
                   log_event(LOG_IPYNBDIFF_GENERATED)
                 end
@@ -109,6 +109,9 @@ module Gitlab
             line.type = "#{line.type || 'unchanged'}-nomappinginraw" unless addition_line_maps[line.new_pos] || removal_line_maps[line.old_pos]
 
             line.line_code = line_code(line)
+
+            line.rich_text = image_as_rich_text(line)
+
             line
           end
 
@@ -152,6 +155,18 @@ module Gitlab
             Gitlab::ErrorTracking.log_exception(error) if error
             nil
           end
+
+          def image_as_rich_text(line)
+            # Strip the initial +, -, or space for the diff context
+            line_text = line.text[1..]
+
+            if line_text.starts_with?(EMBEDDED_IMAGE_PATTERN)
+              image_body = line_text.delete_prefix(EMBEDDED_IMAGE_PATTERN).delete_suffix(')')
+              "<img src=\"data:image#{CGI.escapeHTML(image_body)}\">".html_safe
+            else
+              line.rich_text
+            end
+          end
         end
       end
     end
diff --git a/lib/gitlab/git_access_project.rb b/lib/gitlab/git_access_project.rb
index 7e9bab4a8e6..732e0e14257 100644
--- a/lib/gitlab/git_access_project.rb
+++ b/lib/gitlab/git_access_project.rb
@@ -74,3 +74,5 @@ module Gitlab
     end
   end
 end
+
+Gitlab::GitAccessProject.prepend_mod_with('Gitlab::GitAccessProject')
diff --git a/lib/gitlab/redis/duplicate_jobs.rb b/lib/gitlab/redis/duplicate_jobs.rb
new file mode 100644
index 00000000000..af29bd25342
--- /dev/null
+++ b/lib/gitlab/redis/duplicate_jobs.rb
@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Redis
+    # Pseudo-store to transition `Gitlab::SidekiqMiddleware::DuplicateJobs` from
+    # using `Sidekiq.redis` to using the `SharedState` redis store.
+    class DuplicateJobs < ::Gitlab::Redis::Wrapper
+      class << self
+        def store_name
+          'SharedState'
+        end
+
+        private
+
+        def redis
+          primary_store = ::Redis.new(Gitlab::Redis::SharedState.params)
+          secondary_store = ::Redis.new(Gitlab::Redis::Queues.params)
+
+          MultiStore.new(primary_store, secondary_store, name.demodulize)
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/redis/multi_store.rb b/lib/gitlab/redis/multi_store.rb
new file mode 100644
index 00000000000..7a164f603de
--- /dev/null
+++ b/lib/gitlab/redis/multi_store.rb
@@ -0,0 +1,297 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Redis
+    class MultiStore
+      include Gitlab::Utils::StrongMemoize
+
+      class ReadFromPrimaryError < StandardError
+        def message
+          'Value not found on the redis primary store. Read from the redis secondary store successful.'
+        end
+      end
+      class PipelinedDiffError < StandardError
+        def message
+          'Pipelined command executed on both stores successfully but results differ between them.'
+        end
+      end
+      class MethodMissingError < StandardError
+        def message
+          'Method missing. Falling back to execute method on the redis secondary store.'
+        end
+      end
+
+      attr_reader :primary_store, :secondary_store, :instance_name
+
+      FAILED_TO_READ_ERROR_MESSAGE = 'Failed to read from the redis primary_store.'
+      FAILED_TO_WRITE_ERROR_MESSAGE = 'Failed to write to the redis primary_store.'
+      FAILED_TO_RUN_PIPELINE = 'Failed to execute pipeline on the redis primary_store.'
+
+      SKIP_LOG_METHOD_MISSING_FOR_COMMANDS = %i(info).freeze
+
+      READ_COMMANDS = %i(
+        get
+        mget
+        smembers
+        scard
+      ).freeze
+
+      WRITE_COMMANDS = %i(
+        set
+        setnx
+        setex
+        sadd
+        srem
+        del
+        flushdb
+        rpush
+      ).freeze
+
+      PIPELINED_COMMANDS = %i(
+        pipelined
+        multi
+      ).freeze
+
+      # To transition between two Redis store, `primary_store` should be the target store,
+      # and `secondary_store` should be the current store. Transition is controlled with feature flags:
+      #
+      # - At the default state, all read and write operations are executed in the secondary instance.
+      # - Turning use_primary_and_secondary_stores_for_<instance_name> on: The store writes to both instances.
+      #   The read commands are executed in primary, but fallback to secondary.
+      #   Other commands are executed in the the default instance (Secondary).
+      # - Turning use_primary_store_as_default_for_<instance_name> on: The behavior is the same as above,
+      #   but other commands are executed in the primary now.
+      # - Turning use_primary_and_secondary_stores_for_<instance_name> off: commands are executed in the primary store.
+      def initialize(primary_store, secondary_store, instance_name)
+        @primary_store = primary_store
+        @secondary_store = secondary_store
+        @instance_name = instance_name
+
+        validate_stores!
+      end
+
+      # rubocop:disable GitlabSecurity/PublicSend
+      READ_COMMANDS.each do |name|
+        define_method(name) do |*args, &block|
+          if use_primary_and_secondary_stores?
+            read_command(name, *args, &block)
+          else
+            default_store.send(name, *args, &block)
+          end
+        end
+      end
+
+      WRITE_COMMANDS.each do |name|
+        define_method(name) do |*args, **kwargs, &block|
+          if use_primary_and_secondary_stores?
+            write_command(name, *args, **kwargs, &block)
+          else
+            default_store.send(name, *args, **kwargs, &block)
+          end
+        end
+      end
+
+      PIPELINED_COMMANDS.each do |name|
+        define_method(name) do |*args, **kwargs, &block|
+          if use_primary_and_secondary_stores?
+            pipelined_both(name, *args, **kwargs, &block)
+          else
+            default_store.send(name, *args, **kwargs, &block)
+          end
+        end
+      end
+
+      def method_missing(...)
+        return @instance.send(...) if @instance
+
+        log_method_missing(...)
+
+        default_store.send(...)
+      end
+      # rubocop:enable GitlabSecurity/PublicSend
+
+      def respond_to_missing?(command_name, include_private = false)
+        true
+      end
+
+      # This is needed because of Redis::Rack::Connection is requiring Redis::Store
+      # https://github.com/redis-store/redis-rack/blob/a833086ba494083b6a384a1a4e58b36573a9165d/lib/redis/rack/connection.rb#L15
+      # Done similarly in https://github.com/lsegal/yard/blob/main/lib/yard/templates/template.rb#L122
+      def is_a?(klass)
+        return true if klass == default_store.class
+
+        super(klass)
+      end
+      alias_method :kind_of?, :is_a?
+
+      def to_s
+        use_primary_and_secondary_stores? ? primary_store.to_s : default_store.to_s
+      end
+
+      def use_primary_and_secondary_stores?
+        feature_enabled?("use_primary_and_secondary_stores_for")
+      end
+
+      def use_primary_store_as_default?
+        feature_enabled?("use_primary_store_as_default_for")
+      end
+
+      def increment_pipelined_command_error_count(command_name)
+        @pipelined_command_error ||= Gitlab::Metrics.counter(:gitlab_redis_multi_store_pipelined_diff_error_total,
+                                                             'Redis MultiStore pipelined command diff between stores')
+        @pipelined_command_error.increment(command: command_name, instance_name: instance_name)
+      end
+
+      def increment_read_fallback_count(command_name)
+        @read_fallback_counter ||= Gitlab::Metrics.counter(:gitlab_redis_multi_store_read_fallback_total,
+                                                           'Client side Redis MultiStore reading fallback')
+        @read_fallback_counter.increment(command: command_name, instance_name: instance_name)
+      end
+
+      def increment_method_missing_count(command_name)
+        @method_missing_counter ||= Gitlab::Metrics.counter(:gitlab_redis_multi_store_method_missing_total,
+                                                            'Client side Redis MultiStore method missing')
+        @method_missing_counter.increment(command: command_name, instance_name: instance_name)
+      end
+
+      def log_error(exception, command_name, extra = {})
+        Gitlab::ErrorTracking.log_exception(
+          exception,
+          command_name: command_name,
+          extra: extra.merge(instance_name: instance_name))
+      end
+
+      private
+
+      # @return [Boolean]
+      def feature_enabled?(prefix)
+        feature_table_exists? &&
+          Feature.enabled?("#{prefix}_#{instance_name.underscore}") &&
+          !same_redis_store?
+      end
+
+      # @return [Boolean]
+      def feature_table_exists?
+        Feature::FlipperFeature.table_exists?
+      rescue StandardError
+        false
+      end
+
+      def default_store
+        use_primary_store_as_default? ? primary_store : secondary_store
+      end
+
+      def log_method_missing(command_name, *_args)
+        return if SKIP_LOG_METHOD_MISSING_FOR_COMMANDS.include?(command_name)
+
+        log_error(MethodMissingError.new, command_name)
+        increment_method_missing_count(command_name)
+      end
+
+      def read_command(command_name, *args, &block)
+        if @instance
+          send_command(@instance, command_name, *args, &block)
+        else
+          read_one_with_fallback(command_name, *args, &block)
+        end
+      end
+
+      def write_command(command_name, *args, **kwargs, &block)
+        if @instance
+          send_command(@instance, command_name, *args, **kwargs, &block)
+        else
+          write_both(command_name, *args, **kwargs, &block)
+        end
+      end
+
+      def read_one_with_fallback(command_name, *args, &block)
+        begin
+          value = send_command(primary_store, command_name, *args, &block)
+        rescue StandardError => e
+          log_error(e, command_name,
+            multi_store_error_message: FAILED_TO_READ_ERROR_MESSAGE)
+        end
+
+        value || fallback_read(command_name, *args, &block)
+      end
+
+      def fallback_read(command_name, *args, &block)
+        value = send_command(secondary_store, command_name, *args, &block)
+
+        if value
+          log_error(ReadFromPrimaryError.new, command_name)
+          increment_read_fallback_count(command_name)
+        end
+
+        value
+      end
+
+      def write_both(command_name, *args, **kwargs, &block)
+        begin
+          send_command(primary_store, command_name, *args, **kwargs, &block)
+        rescue StandardError => e
+          log_error(e, command_name,
+            multi_store_error_message: FAILED_TO_WRITE_ERROR_MESSAGE)
+        end
+
+        send_command(secondary_store, command_name, *args, **kwargs, &block)
+      end
+
+      # Run the entire pipeline on both stores. We assume that `&block` is idempotent.
+      def pipelined_both(command_name, *args, **kwargs, &block)
+        begin
+          result_primary = send_command(primary_store, command_name, *args, **kwargs, &block)
+        rescue StandardError => e
+          log_error(e, command_name, multi_store_error_message: FAILED_TO_RUN_PIPELINE)
+        end
+
+        result_secondary = send_command(secondary_store, command_name, *args, **kwargs, &block)
+
+        # Pipelined commands return an array with all results. If they differ,
+        # log an error
+        if result_primary != result_secondary
+          log_error(PipelinedDiffError.new, command_name)
+          increment_pipelined_command_error_count(command_name)
+        end
+
+        result_secondary
+      end
+
+      def same_redis_store?
+        strong_memoize(:same_redis_store) do
+          # <Redis client v4.4.0 for redis:///path_to/redis/redis.socket/5>"
+          primary_store.inspect == secondary_store.inspect
+        end
+      end
+
+      # rubocop:disable GitlabSecurity/PublicSend
+      def send_command(redis_instance, command_name, *args, **kwargs, &block)
+        if block_given?
+          # Make sure that block is wrapped and executed only on the redis instance that is executing the block
+          redis_instance.send(command_name, *args, **kwargs) do |*params|
+            with_instance(redis_instance, *params, &block)
+          end
+        else
+          redis_instance.send(command_name, *args, **kwargs)
+        end
+      end
+      # rubocop:enable GitlabSecurity/PublicSend
+
+      def with_instance(instance, *params)
+        @instance = instance
+
+        yield(*params)
+      ensure
+        @instance = nil
+      end
+
+      def validate_stores!
+        raise ArgumentError, 'primary_store is required' unless primary_store
+        raise ArgumentError, 'secondary_store is required' unless secondary_store
+        raise ArgumentError, 'instance_name is required' unless instance_name
+        raise ArgumentError, 'invalid primary_store' unless primary_store.is_a?(::Redis)
+        raise ArgumentError, 'invalid secondary_store' unless secondary_store.is_a?(::Redis)
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/sidekiq_logging/logs_jobs.rb b/lib/gitlab/sidekiq_logging/logs_jobs.rb
index cfe91b9a266..de08de6632b 100644
--- a/lib/gitlab/sidekiq_logging/logs_jobs.rb
+++ b/lib/gitlab/sidekiq_logging/logs_jobs.rb
@@ -11,7 +11,11 @@ module Gitlab
       def parse_job(job)
         # Error information from the previous try is in the payload for
         # displaying in the Sidekiq UI, but is very confusing in logs!
-        job = job.except('error_backtrace', 'error_class', 'error_message')
+        job = job.except(
+          'error_backtrace', 'error_class', 'error_message',
+          'exception.backtrace', 'exception.class', 'exception.message', 'exception.sql'
+        )
+
         job['class'] = job.delete('wrapped') if job['wrapped'].present?
 
         job['job_size_bytes'] = Sidekiq.dump_json(job['args']).bytesize
diff --git a/lib/gitlab/sidekiq_logging/structured_logger.rb b/lib/gitlab/sidekiq_logging/structured_logger.rb
index a9bfcce2e0a..6eb39981ef4 100644
--- a/lib/gitlab/sidekiq_logging/structured_logger.rb
+++ b/lib/gitlab/sidekiq_logging/structured_logger.rb
@@ -79,9 +79,14 @@ module Gitlab
         if job_exception
           payload['message'] = "#{message}: fail: #{payload['duration_s']} sec"
           payload['job_status'] = 'fail'
-          payload['error_message'] = job_exception.message
-          payload['error_class'] = job_exception.class.name
-          add_exception_backtrace!(job_exception, payload)
+
+          Gitlab::ExceptionLogFormatter.format!(job_exception, payload)
+
+          # Deprecated fields for compatibility
+          # See https://gitlab.com/gitlab-org/gitlab/-/issues/364241
+          payload['error_class'] = payload['exception.class']
+          payload['error_message'] = payload['exception.message']
+          payload['error_backtrace'] = payload['exception.backtrace']
         else
           payload['message'] = "#{message}: done: #{payload['duration_s']} sec"
           payload['job_status'] = 'done'
@@ -98,12 +103,6 @@ module Gitlab
         payload['completed_at'] = Time.now.utc.to_f
       end
 
-      def add_exception_backtrace!(job_exception, payload)
-        return if job_exception.backtrace.blank?
-
-        payload['error_backtrace'] = Rails.backtrace_cleaner.clean(job_exception.backtrace)
-      end
-
       def elapsed(t0)
         t1 = get_time
         { duration: t1[:now] - t0[:now] }
diff --git a/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb b/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb
index 601c8d1c3cf..7533770e254 100644
--- a/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb
+++ b/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb
@@ -63,7 +63,7 @@ module Gitlab
           read_jid = nil
           read_wal_locations = {}
 
-          Sidekiq.redis do |redis|
+          with_redis do |redis|
             redis.multi do |multi|
               multi.set(idempotency_key, jid, ex: expiry, nx: true)
               read_wal_locations = check_existing_wal_locations!(multi, expiry)
@@ -81,7 +81,7 @@ module Gitlab
         def update_latest_wal_location!
           return unless job_wal_locations.present?
 
-          Sidekiq.redis do |redis|
+          with_redis do |redis|
             redis.multi do |multi|
               job_wal_locations.each do |connection_name, location|
                 multi.eval(
@@ -100,20 +100,19 @@ module Gitlab
           strong_memoize(:latest_wal_locations) do
             read_wal_locations = {}
 
-            Sidekiq.redis do |redis|
+            with_redis do |redis|
               redis.multi do |multi|
                 job_wal_locations.keys.each do |connection_name|
                   read_wal_locations[connection_name] = multi.lindex(wal_location_key(connection_name), 0)
                 end
               end
             end
-
             read_wal_locations.transform_values(&:value).compact
           end
         end
 
         def delete!
-          Sidekiq.redis do |redis|
+          with_redis do |redis|
             redis.multi do |multi|
               multi.del(idempotency_key, deduplicated_flag_key)
               delete_wal_locations!(multi)
@@ -140,7 +139,7 @@ module Gitlab
         def set_deduplicated_flag!(expiry = duplicate_key_ttl)
           return unless reschedulable?
 
-          Sidekiq.redis do |redis|
+          with_redis do |redis|
             redis.set(deduplicated_flag_key, DEDUPLICATED_FLAG_VALUE, ex: expiry, nx: true)
           end
         end
@@ -148,7 +147,7 @@ module Gitlab
         def should_reschedule?
           return false unless reschedulable?
 
-          Sidekiq.redis do |redis|
+          with_redis do |redis|
             redis.get(deduplicated_flag_key).present?
           end
         end
@@ -272,6 +271,18 @@ module Gitlab
         def reschedulable?
           !scheduled? && options[:if_deduplicated] == :reschedule_once
         end
+
+        def with_redis
+          if Feature.enabled?(:use_primary_and_secondary_stores_for_duplicate_jobs) ||
+             Feature.enabled?(:use_primary_store_as_default_for_duplicate_jobs)
+            # TODO: Swap for Gitlab::Redis::SharedState after store transition
+            # https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/923
+            Gitlab::Redis::DuplicateJobs.with { |redis| yield redis }
+          else
+            # Keep the old behavior intact if neither feature flag is turned on
+            Sidekiq.redis { |redis| yield redis }
+          end
+        end
       end
     end
   end