Add latest changes from gitlab-org/gitlab@master

6 files changed, 84 insertions, 100 deletions

diff --git a/lib/gitlab/background_migration/cleanup_personal_access_tokens_with_nil_expires_at.rb b/lib/gitlab/background_migration/cleanup_personal_access_tokens_with_nil_expires_at.rb
new file mode 100644
index 00000000000..e8ee2a4c251
--- /dev/null
+++ b/lib/gitlab/background_migration/cleanup_personal_access_tokens_with_nil_expires_at.rb
@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module BackgroundMigration
+    # Clean up personal access tokens with expires_at value is nil
+    # and set the value to new default 365 days
+    class CleanupPersonalAccessTokensWithNilExpiresAt < BatchedMigrationJob
+      feature_category :system_access
+
+      EXPIRES_AT_DEFAULT = 365.days.from_now
+
+      scope_to ->(relation) { relation.where(expires_at: nil) }
+      operation_name :update_all
+
+      def perform
+        each_sub_batch do |sub_batch|
+          sub_batch.update_all(expires_at: EXPIRES_AT_DEFAULT)
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/ansi2json.rb b/lib/gitlab/ci/ansi2json.rb
index 70b68c7b821..79114d35916 100644
--- a/lib/gitlab/ci/ansi2json.rb
+++ b/lib/gitlab/ci/ansi2json.rb
@@ -4,8 +4,8 @@
 module Gitlab
   module Ci
     module Ansi2json
-      def self.convert(ansi, state = nil, verify_state: false)
-        Converter.new.convert(ansi, state, verify_state: verify_state)
+      def self.convert(ansi, state = nil)
+        Converter.new.convert(ansi, state)
       end
     end
   end
diff --git a/lib/gitlab/ci/ansi2json/converter.rb b/lib/gitlab/ci/ansi2json/converter.rb
index 84541208a2f..78f6c5bf0aa 100644
--- a/lib/gitlab/ci/ansi2json/converter.rb
+++ b/lib/gitlab/ci/ansi2json/converter.rb
@@ -4,13 +4,9 @@ module Gitlab
   module Ci
     module Ansi2json
       class Converter
-        def convert(stream, new_state, verify_state: false)
+        def convert(stream, new_state)
           @lines = []
-          @state = if verify_state
-                     SignedState.new(new_state, stream.size)
-                   else
-                     State.new(new_state, stream.size)
-                   end
+          @state = State.new(new_state, stream.size)
 
           append = false
           truncated = false
diff --git a/lib/gitlab/ci/ansi2json/signed_state.rb b/lib/gitlab/ci/ansi2json/signed_state.rb
deleted file mode 100644
index 98e2419f0a8..00000000000
--- a/lib/gitlab/ci/ansi2json/signed_state.rb
+++ /dev/null
@@ -1,74 +0,0 @@
-# frozen_string_literal: true
-
-require 'openssl'
-
-module Gitlab
-  module Ci
-    module Ansi2json
-      class SignedState < ::Gitlab::Ci::Ansi2json::State
-        include Gitlab::Utils::StrongMemoize
-
-        SIGNATURE_KEY_SALT = 'gitlab-ci-ansi2json-state'
-        SEPARATOR = '--'
-
-        def encode
-          encoded = super
-
-          encoded + SEPARATOR + sign(encoded)
-        end
-
-        private
-
-        def sign(message)
-          ::OpenSSL::HMAC.hexdigest(
-            signature_digest,
-            signature_key,
-            message
-          )
-        end
-
-        def verify(signed_message)
-          signature_length = signature_digest.digest_length * 2 # a byte is exactly two hexadecimals
-          message_length = signed_message.length - SEPARATOR.length - signature_length
-          return if message_length <= 0
-
-          signature = signed_message.last(signature_length)
-          message = signed_message.first(message_length)
-          return unless valid_signature?(message, signature)
-
-          message
-        end
-
-        def valid_signature?(message, signature)
-          expected_signature = sign(message)
-          expected_signature.bytesize == signature.bytesize &&
-            ::OpenSSL.fixed_length_secure_compare(signature, expected_signature)
-        end
-
-        def decode_state(data)
-          return if data.blank?
-
-          encoded_state = verify(data)
-          if encoded_state.blank?
-            ::Gitlab::AppLogger.warn(message: "#{self.class}: signature missing or invalid", invalid_state: data)
-            return
-          end
-
-          decoded_state = Base64.urlsafe_decode64(encoded_state)
-          return unless decoded_state.present?
-
-          ::Gitlab::Json.parse(decoded_state)
-        end
-
-        def signature_digest
-          ::OpenSSL::Digest.new('SHA256')
-        end
-
-        def signature_key
-          ::Gitlab::Application.key_generator.generate_key(SIGNATURE_KEY_SALT, signature_digest.block_length)
-        end
-        strong_memoize_attr :signature_key
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/ci/ansi2json/state.rb b/lib/gitlab/ci/ansi2json/state.rb
index 279e1929b22..3aec1cde1bc 100644
--- a/lib/gitlab/ci/ansi2json/state.rb
+++ b/lib/gitlab/ci/ansi2json/state.rb
@@ -1,11 +1,18 @@
 # frozen_string_literal: true
 
+require 'openssl'
+
 # In this class we keep track of the state changes that the
 # Converter makes as it scans through the log stream.
 module Gitlab
   module Ci
     module Ansi2json
       class State
+        include Gitlab::Utils::StrongMemoize
+
+        SIGNATURE_KEY_SALT = 'gitlab-ci-ansi2json-state'
+        SEPARATOR = '--'
+
         attr_accessor :offset, :current_line, :inherited_style, :open_sections, :last_line_offset
 
         def initialize(new_state, stream_size)
@@ -24,7 +31,9 @@ module Gitlab
             open_sections: @open_sections
           }.to_json
 
-          Base64.urlsafe_encode64(json, padding: false)
+          encoded = Base64.urlsafe_encode64(json, padding: false)
+
+          encoded + SEPARATOR + sign(encoded)
         end
 
         def open_section(section, timestamp, options)
@@ -86,27 +95,55 @@ module Gitlab
           end
         end
 
-        def decode_state(state)
-          return unless state.present?
+        def decode_state(data)
+          return if data.blank?
+
+          encoded_state = verify(data)
+          if encoded_state.blank?
+            ::Gitlab::AppLogger.warn(message: "#{self.class}: signature missing or invalid", invalid_state: data)
+            return
+          end
 
-          decoded_state = Base64.urlsafe_decode64(state)
+          decoded_state = Base64.urlsafe_decode64(encoded_state)
           return unless decoded_state.present?
 
           ::Gitlab::Json.parse(decoded_state)
-        rescue ArgumentError, JSON::ParserError => error
-          # This rescue is so that we don't break during the rollout or rollback
-          # of `sign_and_verify_ansi2json_state`, because we may receive a
-          # signed state even when the flag is disabled, and this would result
-          # in invalid Base64 (ArgumentError) or invalid JSON in case the signed
-          # state happens to decode as valid Base64 (JSON::ParserError).
-          #
-          # Once the flag has been fully rolled out this should not
-          # be possible (it would imply a backend bug) and we not rescue from
-          # this.
-          ::Gitlab::AppLogger.warn(message: "#{self.class}: decode error", invalid_state: state, error: error)
-
-          nil
         end
+
+        def sign(message)
+          ::OpenSSL::HMAC.hexdigest(
+            signature_digest,
+            signature_key,
+            message
+          )
+        end
+
+        def verify(signed_message)
+          signature_length = signature_digest.digest_length * 2 # a byte is exactly two hexadecimals
+          message_length = signed_message.length - SEPARATOR.length - signature_length
+          return if message_length <= 0
+
+          signature = signed_message.last(signature_length)
+          message = signed_message.first(message_length)
+          return unless valid_signature?(message, signature)
+
+          message
+        end
+
+        def valid_signature?(message, signature)
+          expected_signature = sign(message)
+          expected_signature.bytesize == signature.bytesize &&
+            ::OpenSSL.fixed_length_secure_compare(signature, expected_signature)
+        end
+
+        def signature_digest
+          ::OpenSSL::Digest.new('SHA256')
+        end
+
+        def signature_key
+          ::Gitlab::Application.key_generator.generate_key(SIGNATURE_KEY_SALT, signature_digest.block_length)
+        end
+        strong_memoize_attr :signature_key
       end
     end
   end
diff --git a/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb b/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb
index bb87104630c..3ed9c1743ed 100644
--- a/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb
+++ b/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb
@@ -147,7 +147,10 @@ module Gitlab
           end
           local cookie = cmsgpack.unpack(cookie_msgpack)
           cookie.deduplicated = "1"
-          redis.call("set", KEYS[1], cmsgpack.pack(cookie), "ex", redis.call("ttl", KEYS[1]))
+          local ttl = redis.call("ttl", KEYS[1])
+          if ttl > 0 then
+            redis.call("set", KEYS[1], cmsgpack.pack(cookie), "ex", ttl)
+          end
         LUA
 
         def should_reschedule?