From 5225ffb5ccfe2fe0e55a3327d43f28f4ed08ae63 Mon Sep 17 00:00:00 2001 From: GitLab Bot Date: Sun, 14 May 2023 00:07:42 +0000 Subject: Add latest changes from gitlab-org/gitlab@master --- ...p_personal_access_tokens_with_nil_expires_at.rb | 22 +++++++ lib/gitlab/ci/ansi2json.rb | 4 +- lib/gitlab/ci/ansi2json/converter.rb | 8 +-- lib/gitlab/ci/ansi2json/signed_state.rb | 74 ---------------------- lib/gitlab/ci/ansi2json/state.rb | 71 ++++++++++++++++----- .../duplicate_jobs/duplicate_job.rb | 5 +- 6 files changed, 84 insertions(+), 100 deletions(-) create mode 100644 lib/gitlab/background_migration/cleanup_personal_access_tokens_with_nil_expires_at.rb delete mode 100644 lib/gitlab/ci/ansi2json/signed_state.rb (limited to 'lib') diff --git a/lib/gitlab/background_migration/cleanup_personal_access_tokens_with_nil_expires_at.rb b/lib/gitlab/background_migration/cleanup_personal_access_tokens_with_nil_expires_at.rb new file mode 100644 index 00000000000..e8ee2a4c251 --- /dev/null +++ b/lib/gitlab/background_migration/cleanup_personal_access_tokens_with_nil_expires_at.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Clean up personal access tokens with expires_at value is nil + # and set the value to new default 365 days + class CleanupPersonalAccessTokensWithNilExpiresAt < BatchedMigrationJob + feature_category :system_access + + EXPIRES_AT_DEFAULT = 365.days.from_now + + scope_to ->(relation) { relation.where(expires_at: nil) } + operation_name :update_all + + def perform + each_sub_batch do |sub_batch| + sub_batch.update_all(expires_at: EXPIRES_AT_DEFAULT) + end + end + end + end +end diff --git a/lib/gitlab/ci/ansi2json.rb b/lib/gitlab/ci/ansi2json.rb index 70b68c7b821..79114d35916 100644 --- a/lib/gitlab/ci/ansi2json.rb +++ b/lib/gitlab/ci/ansi2json.rb @@ -4,8 +4,8 @@ module Gitlab module Ci module Ansi2json - def self.convert(ansi, state = nil, verify_state: false) - Converter.new.convert(ansi, state, verify_state: verify_state) + def self.convert(ansi, state = nil) + Converter.new.convert(ansi, state) end end end diff --git a/lib/gitlab/ci/ansi2json/converter.rb b/lib/gitlab/ci/ansi2json/converter.rb index 84541208a2f..78f6c5bf0aa 100644 --- a/lib/gitlab/ci/ansi2json/converter.rb +++ b/lib/gitlab/ci/ansi2json/converter.rb @@ -4,13 +4,9 @@ module Gitlab module Ci module Ansi2json class Converter - def convert(stream, new_state, verify_state: false) + def convert(stream, new_state) @lines = [] - @state = if verify_state - SignedState.new(new_state, stream.size) - else - State.new(new_state, stream.size) - end + @state = State.new(new_state, stream.size) append = false truncated = false diff --git a/lib/gitlab/ci/ansi2json/signed_state.rb b/lib/gitlab/ci/ansi2json/signed_state.rb deleted file mode 100644 index 98e2419f0a8..00000000000 --- a/lib/gitlab/ci/ansi2json/signed_state.rb +++ /dev/null @@ -1,74 +0,0 @@ -# frozen_string_literal: true - -require 'openssl' - -module Gitlab - module Ci - module Ansi2json - class SignedState < ::Gitlab::Ci::Ansi2json::State - include Gitlab::Utils::StrongMemoize - - SIGNATURE_KEY_SALT = 'gitlab-ci-ansi2json-state' - SEPARATOR = '--' - - def encode - encoded = super - - encoded + SEPARATOR + sign(encoded) - end - - private - - def sign(message) - ::OpenSSL::HMAC.hexdigest( - signature_digest, - signature_key, - message - ) - end - - def verify(signed_message) - signature_length = signature_digest.digest_length * 2 # a byte is exactly two hexadecimals - message_length = signed_message.length - SEPARATOR.length - signature_length - return if message_length <= 0 - - signature = signed_message.last(signature_length) - message = signed_message.first(message_length) - return unless valid_signature?(message, signature) - - message - end - - def valid_signature?(message, signature) - expected_signature = sign(message) - expected_signature.bytesize == signature.bytesize && - ::OpenSSL.fixed_length_secure_compare(signature, expected_signature) - end - - def decode_state(data) - return if data.blank? - - encoded_state = verify(data) - if encoded_state.blank? - ::Gitlab::AppLogger.warn(message: "#{self.class}: signature missing or invalid", invalid_state: data) - return - end - - decoded_state = Base64.urlsafe_decode64(encoded_state) - return unless decoded_state.present? - - ::Gitlab::Json.parse(decoded_state) - end - - def signature_digest - ::OpenSSL::Digest.new('SHA256') - end - - def signature_key - ::Gitlab::Application.key_generator.generate_key(SIGNATURE_KEY_SALT, signature_digest.block_length) - end - strong_memoize_attr :signature_key - end - end - end -end diff --git a/lib/gitlab/ci/ansi2json/state.rb b/lib/gitlab/ci/ansi2json/state.rb index 279e1929b22..3aec1cde1bc 100644 --- a/lib/gitlab/ci/ansi2json/state.rb +++ b/lib/gitlab/ci/ansi2json/state.rb @@ -1,11 +1,18 @@ # frozen_string_literal: true +require 'openssl' + # In this class we keep track of the state changes that the # Converter makes as it scans through the log stream. module Gitlab module Ci module Ansi2json class State + include Gitlab::Utils::StrongMemoize + + SIGNATURE_KEY_SALT = 'gitlab-ci-ansi2json-state' + SEPARATOR = '--' + attr_accessor :offset, :current_line, :inherited_style, :open_sections, :last_line_offset def initialize(new_state, stream_size) @@ -24,7 +31,9 @@ module Gitlab open_sections: @open_sections }.to_json - Base64.urlsafe_encode64(json, padding: false) + encoded = Base64.urlsafe_encode64(json, padding: false) + + encoded + SEPARATOR + sign(encoded) end def open_section(section, timestamp, options) @@ -86,27 +95,55 @@ module Gitlab end end - def decode_state(state) - return unless state.present? + def decode_state(data) + return if data.blank? + + encoded_state = verify(data) + if encoded_state.blank? + ::Gitlab::AppLogger.warn(message: "#{self.class}: signature missing or invalid", invalid_state: data) + return + end - decoded_state = Base64.urlsafe_decode64(state) + decoded_state = Base64.urlsafe_decode64(encoded_state) return unless decoded_state.present? ::Gitlab::Json.parse(decoded_state) - rescue ArgumentError, JSON::ParserError => error - # This rescue is so that we don't break during the rollout or rollback - # of `sign_and_verify_ansi2json_state`, because we may receive a - # signed state even when the flag is disabled, and this would result - # in invalid Base64 (ArgumentError) or invalid JSON in case the signed - # state happens to decode as valid Base64 (JSON::ParserError). - # - # Once the flag has been fully rolled out this should not - # be possible (it would imply a backend bug) and we not rescue from - # this. - ::Gitlab::AppLogger.warn(message: "#{self.class}: decode error", invalid_state: state, error: error) - - nil end + + def sign(message) + ::OpenSSL::HMAC.hexdigest( + signature_digest, + signature_key, + message + ) + end + + def verify(signed_message) + signature_length = signature_digest.digest_length * 2 # a byte is exactly two hexadecimals + message_length = signed_message.length - SEPARATOR.length - signature_length + return if message_length <= 0 + + signature = signed_message.last(signature_length) + message = signed_message.first(message_length) + return unless valid_signature?(message, signature) + + message + end + + def valid_signature?(message, signature) + expected_signature = sign(message) + expected_signature.bytesize == signature.bytesize && + ::OpenSSL.fixed_length_secure_compare(signature, expected_signature) + end + + def signature_digest + ::OpenSSL::Digest.new('SHA256') + end + + def signature_key + ::Gitlab::Application.key_generator.generate_key(SIGNATURE_KEY_SALT, signature_digest.block_length) + end + strong_memoize_attr :signature_key end end end diff --git a/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb b/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb index bb87104630c..3ed9c1743ed 100644 --- a/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb +++ b/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb @@ -147,7 +147,10 @@ module Gitlab end local cookie = cmsgpack.unpack(cookie_msgpack) cookie.deduplicated = "1" - redis.call("set", KEYS[1], cmsgpack.pack(cookie), "ex", redis.call("ttl", KEYS[1])) + local ttl = redis.call("ttl", KEYS[1]) + if ttl > 0 then + redis.call("set", KEYS[1], cmsgpack.pack(cookie), "ex", ttl) + end LUA def should_reschedule? -- cgit v1.2.1