diff options
Diffstat (limited to 'lib/gitlab')
-rw-r--r-- | lib/gitlab/background_migration/user_mentions/models/epic.rb | 1 | ||||
-rw-r--r-- | lib/gitlab/checks/snippet_check.rb | 2 | ||||
-rw-r--r-- | lib/gitlab/email/receiver.rb | 8 | ||||
-rw-r--r-- | lib/gitlab/git_access_snippet.rb | 8 | ||||
-rw-r--r-- | lib/gitlab/graphql/docs/helper.rb | 22 | ||||
-rw-r--r-- | lib/gitlab/graphql/docs/templates/default.md.haml | 5 | ||||
-rw-r--r-- | lib/gitlab/incoming_email.rb | 8 | ||||
-rw-r--r-- | lib/gitlab/sidekiq_middleware.rb | 2 | ||||
-rw-r--r-- | lib/gitlab/sidekiq_middleware/duplicate_jobs/client.rb | 13 | ||||
-rw-r--r-- | lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb | 104 | ||||
-rw-r--r-- | lib/gitlab/sidekiq_middleware/duplicate_jobs/server.rb | 13 | ||||
-rw-r--r-- | lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies.rb | 21 | ||||
-rw-r--r-- | lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/until_executing.rb | 36 |
13 files changed, 230 insertions, 13 deletions
diff --git a/lib/gitlab/background_migration/user_mentions/models/epic.rb b/lib/gitlab/background_migration/user_mentions/models/epic.rb index 9797c86478e..ad1d904cde7 100644 --- a/lib/gitlab/background_migration/user_mentions/models/epic.rb +++ b/lib/gitlab/background_migration/user_mentions/models/epic.rb @@ -18,7 +18,6 @@ module Gitlab self.table_name = 'epics' belongs_to :author, class_name: "User" - belongs_to :project belongs_to :group def self.user_mention_model diff --git a/lib/gitlab/checks/snippet_check.rb b/lib/gitlab/checks/snippet_check.rb index 26dd772764a..be25fe3e7c4 100644 --- a/lib/gitlab/checks/snippet_check.rb +++ b/lib/gitlab/checks/snippet_check.rb @@ -21,7 +21,7 @@ module Gitlab def exec if creation? || deletion? - raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:create_delete_branch] + raise GitAccess::ForbiddenError, ERROR_MESSAGES[:create_delete_branch] end # TODO: https://gitlab.com/gitlab-org/gitlab/issues/205628 diff --git a/lib/gitlab/email/receiver.rb b/lib/gitlab/email/receiver.rb index f028102da9b..b7b9288517d 100644 --- a/lib/gitlab/email/receiver.rb +++ b/lib/gitlab/email/receiver.rb @@ -34,8 +34,7 @@ module Gitlab ignore_auto_reply!(mail) - mail_key = extract_mail_key(mail) - handler = Handler.for(mail, mail_key) + handler = find_handler(mail) raise UnknownIncomingEmail unless handler @@ -46,6 +45,11 @@ module Gitlab private + def find_handler(mail) + mail_key = extract_mail_key(mail) + Handler.for(mail, mail_key) + end + def build_mail Mail::Message.new(@raw) rescue Encoding::UndefinedConversionError, diff --git a/lib/gitlab/git_access_snippet.rb b/lib/gitlab/git_access_snippet.rb index ff1af9bede4..3956fc8a483 100644 --- a/lib/gitlab/git_access_snippet.rb +++ b/lib/gitlab/git_access_snippet.rb @@ -28,7 +28,7 @@ module Gitlab # TODO: Investigate if expanding actor/authentication types are needed. # https://gitlab.com/gitlab-org/gitlab/issues/202190 if actor && !actor.is_a?(User) && !actor.instance_of?(Key) - raise UnauthorizedError, ERROR_MESSAGES[:authentication_mechanism] + raise ForbiddenError, ERROR_MESSAGES[:authentication_mechanism] end unless Feature.enabled?(:version_snippets, user) @@ -53,7 +53,7 @@ module Gitlab override :check_push_access! def check_push_access! - raise UnauthorizedError, ERROR_MESSAGES[:update_snippet] unless user + raise ForbiddenError, ERROR_MESSAGES[:update_snippet] unless user check_change_access! end @@ -74,7 +74,7 @@ module Gitlab passed = guest_can_download_code? || user_can_download_code? unless passed - raise UnauthorizedError, ERROR_MESSAGES[:read_snippet] + raise ForbiddenError, ERROR_MESSAGES[:read_snippet] end end @@ -91,7 +91,7 @@ module Gitlab override :check_change_access! def check_change_access! unless user_access.can_do_action?(:update_snippet) - raise UnauthorizedError, ERROR_MESSAGES[:update_snippet] + raise ForbiddenError, ERROR_MESSAGES[:update_snippet] end changes_list.each do |change| diff --git a/lib/gitlab/graphql/docs/helper.rb b/lib/gitlab/graphql/docs/helper.rb index 56524120ffd..0dd28b32511 100644 --- a/lib/gitlab/graphql/docs/helper.rb +++ b/lib/gitlab/graphql/docs/helper.rb @@ -25,6 +25,28 @@ module Gitlab fields.sort_by { |field| field[:name] } end + def render_field(field) + '| %s | %s | %s |' % [ + render_field_name(field), + render_field_type(field[:type][:info]), + render_field_description(field) + ] + end + + def render_field_name(field) + rendered_name = "`#{field[:name]}`" + rendered_name += ' **{warning-solid}**' if field[:is_deprecated] + rendered_name + end + + # Returns the field description. If the field has been deprecated, + # the deprecation reason will be returned in place of the description. + def render_field_description(field) + return field[:description] unless field[:is_deprecated] + + "**Deprecated:** #{field[:deprecation_reason]}" + end + # Some fields types are arrays of other types and are displayed # on docs wrapped in square brackets, for example: [String!]. # This makes GitLab docs renderer thinks they are links so here diff --git a/lib/gitlab/graphql/docs/templates/default.md.haml b/lib/gitlab/graphql/docs/templates/default.md.haml index b126a22c301..8c033526557 100644 --- a/lib/gitlab/graphql/docs/templates/default.md.haml +++ b/lib/gitlab/graphql/docs/templates/default.md.haml @@ -11,6 +11,9 @@ Each table below documents a GraphQL type. Types match loosely to models, but not all fields and methods on a model are available via GraphQL. + + CAUTION: **Caution:** + Fields that are deprecated are marked with **{warning-solid}**. \ - objects.each do |type| - unless type[:fields].empty? @@ -22,5 +25,5 @@ ~ "| Name | Type | Description |" ~ "| --- | ---- | ---------- |" - sorted_fields(type[:fields]).each do |field| - = "| `#{field[:name]}` | #{render_field_type(field[:type][:info])} | #{field[:description]} |" + = render_field(field) \ diff --git a/lib/gitlab/incoming_email.rb b/lib/gitlab/incoming_email.rb index 4547a9b0a01..2889dbc68cc 100644 --- a/lib/gitlab/incoming_email.rb +++ b/lib/gitlab/incoming_email.rb @@ -28,8 +28,9 @@ module Gitlab config.address.sub(WILDCARD_PLACEHOLDER, "#{key}#{UNSUBSCRIBE_SUFFIX}") end - def key_from_address(address) - regex = address_regex + def key_from_address(address, wildcard_address: nil) + wildcard_address ||= config.address + regex = address_regex(wildcard_address) return unless regex match = address.match(regex) @@ -55,8 +56,7 @@ module Gitlab private - def address_regex - wildcard_address = config.address + def address_regex(wildcard_address) return unless wildcard_address regex = Regexp.escape(wildcard_address) diff --git a/lib/gitlab/sidekiq_middleware.rb b/lib/gitlab/sidekiq_middleware.rb index 6c27213df49..c3a52a1986d 100644 --- a/lib/gitlab/sidekiq_middleware.rb +++ b/lib/gitlab/sidekiq_middleware.rb @@ -20,6 +20,7 @@ module Gitlab chain.add Gitlab::SidekiqMiddleware::AdminMode::Server chain.add Gitlab::SidekiqStatus::ServerMiddleware chain.add Gitlab::SidekiqMiddleware::WorkerContext::Server + chain.add Gitlab::SidekiqMiddleware::DuplicateJobs::Server end end @@ -33,6 +34,7 @@ module Gitlab chain.add Gitlab::SidekiqMiddleware::WorkerContext::Client # needs to be before the Labkit middleware chain.add Labkit::Middleware::Sidekiq::Client chain.add Gitlab::SidekiqMiddleware::AdminMode::Client + chain.add Gitlab::SidekiqMiddleware::DuplicateJobs::Client end end end diff --git a/lib/gitlab/sidekiq_middleware/duplicate_jobs/client.rb b/lib/gitlab/sidekiq_middleware/duplicate_jobs/client.rb new file mode 100644 index 00000000000..bb0c18735bb --- /dev/null +++ b/lib/gitlab/sidekiq_middleware/duplicate_jobs/client.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +module Gitlab + module SidekiqMiddleware + module DuplicateJobs + class Client + def call(worker_class, job, queue, _redis_pool, &block) + DuplicateJob.new(job, queue).schedule(&block) + end + end + end + end +end diff --git a/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb b/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb new file mode 100644 index 00000000000..b84673c4cee --- /dev/null +++ b/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb @@ -0,0 +1,104 @@ +# frozen_string_literal: true + +require 'digest' + +module Gitlab + module SidekiqMiddleware + module DuplicateJobs + # This class defines an identifier of a job in a queue + # The identifier based on a job's class and arguments. + # + # As strategy decides when to keep track of the job in redis and when to + # remove it. + # + # Storing the deduplication key in redis can be done by calling `check!` + # check returns the `jid` of the job if it was scheduled, or the `jid` of + # the duplicate job if it was already scheduled + # + # When new jobs can be scheduled again, the strategy calls `#delete`. + class DuplicateJob + DUPLICATE_KEY_TTL = 6.hours + + attr_reader :existing_jid + + def initialize(job, queue_name, strategy: :until_executing) + @job = job + @queue_name = queue_name + @strategy = strategy + end + + # This will continue the middleware chain if the job should be scheduled + # It will return false if the job needs to be cancelled + def schedule(&block) + Strategies.for(strategy).new(self).schedule(job, &block) + end + + # This will continue the server middleware chain if the job should be + # executed. + # It will return false if the job should not be executed. + def perform(&block) + Strategies.for(strategy).new(self).perform(job, &block) + end + + # This method will return the jid that was set in redis + def check! + read_jid = nil + + Sidekiq.redis do |redis| + redis.multi do |multi| + redis.set(idempotency_key, jid, ex: DUPLICATE_KEY_TTL, nx: true) + read_jid = redis.get(idempotency_key) + end + end + + self.existing_jid = read_jid.value + end + + def delete! + Sidekiq.redis do |redis| + redis.del(idempotency_key) + end + end + + def duplicate? + raise "Call `#check!` first to check for existing duplicates" unless existing_jid + + jid != existing_jid + end + + private + + attr_reader :queue_name, :strategy, :job + attr_writer :existing_jid + + def worker_class_name + job['class'] + end + + def arguments + job['args'] + end + + def jid + job['jid'] + end + + def idempotency_key + @idempotency_key ||= "#{namespace}:#{idempotency_hash}" + end + + def idempotency_hash + Digest::SHA256.hexdigest(idempotency_string) + end + + def namespace + "#{Gitlab::Redis::Queues::SIDEKIQ_NAMESPACE}:duplicate:#{queue_name}" + end + + def idempotency_string + "#{worker_class_name}:#{arguments.join('-')}" + end + end + end + end +end diff --git a/lib/gitlab/sidekiq_middleware/duplicate_jobs/server.rb b/lib/gitlab/sidekiq_middleware/duplicate_jobs/server.rb new file mode 100644 index 00000000000..a35edc5774e --- /dev/null +++ b/lib/gitlab/sidekiq_middleware/duplicate_jobs/server.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +module Gitlab + module SidekiqMiddleware + module DuplicateJobs + class Server + def call(worker, job, queue, &block) + DuplicateJob.new(job, queue).perform(&block) + end + end + end + end +end diff --git a/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies.rb b/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies.rb new file mode 100644 index 00000000000..a08310a58ff --- /dev/null +++ b/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module Gitlab + module SidekiqMiddleware + module DuplicateJobs + module Strategies + UnknownStrategyError = Class.new(StandardError) + + STRATEGIES = { + until_executing: UntilExecuting + }.freeze + + def self.for(name) + STRATEGIES.fetch(name) + rescue KeyError + raise UnknownStrategyError, "Unknown deduplication strategy #{name}" + end + end + end + end +end diff --git a/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/until_executing.rb b/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/until_executing.rb new file mode 100644 index 00000000000..b8f49b67a59 --- /dev/null +++ b/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/until_executing.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +module Gitlab + module SidekiqMiddleware + module DuplicateJobs + module Strategies + # This strategy takes a lock before scheduling the job in a queue and + # removes the lock before the job starts allowing a new job to be queued + # while a job is still executing. + class UntilExecuting + def initialize(duplicate_job) + @duplicate_job = duplicate_job + end + + def schedule(job) + if duplicate_job.check! && duplicate_job.duplicate? + job['duplicate-of'] = duplicate_job.existing_jid + end + + yield + end + + def perform(_job) + duplicate_job.delete! + + yield + end + + private + + attr_reader :duplicate_job + end + end + end + end +end |