Add latest changes from gitlab-org/gitlab@master

7 files changed, 77 insertions, 19 deletions

diff --git a/lib/gitlab/kubernetes/helm/api.rb b/lib/gitlab/kubernetes/helm/api.rb
index 3ed07818302..3b843799d66 100644
--- a/lib/gitlab/kubernetes/helm/api.rb
+++ b/lib/gitlab/kubernetes/helm/api.rb
@@ -3,7 +3,7 @@
 module Gitlab
   module Kubernetes
     module Helm
-      class Api
+      class API
         def initialize(kubeclient)
           @kubeclient = kubeclient
           @namespace = Gitlab::Kubernetes::Namespace.new(
diff --git a/lib/gitlab/sidekiq_logging/deduplication_logger.rb b/lib/gitlab/sidekiq_logging/deduplication_logger.rb
new file mode 100644
index 00000000000..01810e474dc
--- /dev/null
+++ b/lib/gitlab/sidekiq_logging/deduplication_logger.rb
@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module SidekiqLogging
+    class DeduplicationLogger
+      include Singleton
+      include LogsJobs
+
+      def log(job, deduplication_type)
+        payload = parse_job(job)
+        payload['job_status'] = 'deduplicated'
+        payload['message'] = "#{base_message(payload)}: deduplicated: #{deduplication_type}"
+        payload['deduplication_type'] = deduplication_type
+
+        Sidekiq.logger.info payload
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/sidekiq_logging/logs_jobs.rb b/lib/gitlab/sidekiq_logging/logs_jobs.rb
new file mode 100644
index 00000000000..55d711c54ae
--- /dev/null
+++ b/lib/gitlab/sidekiq_logging/logs_jobs.rb
@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module SidekiqLogging
+    module LogsJobs
+      def base_message(payload)
+        "#{payload['class']} JID-#{payload['jid']}"
+      end
+
+      def parse_job(job)
+        # Error information from the previous try is in the payload for
+        # displaying in the Sidekiq UI, but is very confusing in logs!
+        job = job.except('error_backtrace', 'error_class', 'error_message')
+
+        # Add process id params
+        job['pid'] = ::Process.pid
+
+        job.delete('args') unless ENV['SIDEKIQ_LOG_ARGUMENTS']
+        job['args'] = Gitlab::Utils::LogLimitedArray.log_limited_array(job['args'].map(&:to_s)) if job['args']
+
+        job
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/sidekiq_logging/structured_logger.rb b/lib/gitlab/sidekiq_logging/structured_logger.rb
index 1c36e4a4a5a..af9072ea201 100644
--- a/lib/gitlab/sidekiq_logging/structured_logger.rb
+++ b/lib/gitlab/sidekiq_logging/structured_logger.rb
@@ -6,6 +6,8 @@ require 'active_record/log_subscriber'
 module Gitlab
   module SidekiqLogging
     class StructuredLogger
+      include LogsJobs
+
       def call(job, queue)
         started_time = get_time
         base_payload = parse_job(job)
@@ -24,10 +26,6 @@ module Gitlab
 
       private
 
-      def base_message(payload)
-        "#{payload['class']} JID-#{payload['jid']}"
-      end
-
       def add_instrumentation_keys!(job, output_payload)
         output_payload.merge!(job.slice(*::Gitlab::InstrumentationHelper::KEYS))
       end
@@ -76,20 +74,6 @@ module Gitlab
         payload['completed_at'] = Time.now.utc.to_f
       end
 
-      def parse_job(job)
-        # Error information from the previous try is in the payload for
-        # displaying in the Sidekiq UI, but is very confusing in logs!
-        job = job.except('error_backtrace', 'error_class', 'error_message')
-
-        # Add process id params
-        job['pid'] = ::Process.pid
-
-        job.delete('args') unless ENV['SIDEKIQ_LOG_ARGUMENTS']
-        job['args'] = Gitlab::Utils::LogLimitedArray.log_limited_array(job['args'].map(&:to_s)) if job['args']
-
-        job
-      end
-
       def elapsed(t0)
         t1 = get_time
         {
diff --git a/lib/gitlab/sidekiq_middleware/duplicate_jobs.rb b/lib/gitlab/sidekiq_middleware/duplicate_jobs.rb
new file mode 100644
index 00000000000..23222430902
--- /dev/null
+++ b/lib/gitlab/sidekiq_middleware/duplicate_jobs.rb
@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+require 'digest'
+
+module Gitlab
+  module SidekiqMiddleware
+    module DuplicateJobs
+      def self.drop_duplicates?
+        Feature.enabled?(:drop_duplicate_sidekiq_jobs)
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb b/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb
index b84673c4cee..c6fb50b4610 100644
--- a/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb
+++ b/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb
@@ -66,6 +66,10 @@ module Gitlab
           jid != existing_jid
         end
 
+        def droppable?
+          idempotent? && duplicate? && DuplicateJobs.drop_duplicates?
+        end
+
         private
 
         attr_reader :queue_name, :strategy, :job
@@ -98,6 +102,14 @@ module Gitlab
         def idempotency_string
           "#{worker_class_name}:#{arguments.join('-')}"
         end
+
+        def idempotent?
+          worker_class = worker_class_name.to_s.safe_constantize
+          return false unless worker_class
+          return false unless worker_class.respond_to?(:idempotent?)
+
+          worker_class.idempotent?
+        end
       end
     end
   end
diff --git a/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/until_executing.rb b/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/until_executing.rb
index b8f49b67a59..674e436b714 100644
--- a/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/until_executing.rb
+++ b/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/until_executing.rb
@@ -17,6 +17,11 @@ module Gitlab
               job['duplicate-of'] = duplicate_job.existing_jid
             end
 
+            if duplicate_job.droppable?
+              Gitlab::SidekiqLogging::DeduplicationLogger.instance.log(job, "dropped until executing")
+              return false
+            end
+
             yield
           end