diff options
Diffstat (limited to 'app')
22 files changed, 211 insertions, 52 deletions
diff --git a/app/assets/javascripts/clusters/stores/clusters_store.js b/app/assets/javascripts/clusters/stores/clusters_store.js index f64f0ca616f..ada5a49e246 100644 --- a/app/assets/javascripts/clusters/stores/clusters_store.js +++ b/app/assets/javascripts/clusters/stores/clusters_store.js @@ -171,6 +171,7 @@ export default class ClusterStore { this.state.applications.cert_manager.email || serverAppEntry.email; } else if (appId === JUPYTER) { this.state.applications.jupyter.hostname = + this.state.applications.jupyter.hostname || serverAppEntry.hostname || (this.state.applications.ingress.externalIp ? `jupyter.${this.state.applications.ingress.externalIp}.nip.io` diff --git a/app/controllers/autocomplete_controller.rb b/app/controllers/autocomplete_controller.rb index 091327931c2..f111c7ca8cc 100644 --- a/app/controllers/autocomplete_controller.rb +++ b/app/controllers/autocomplete_controller.rb @@ -16,7 +16,7 @@ class AutocompleteController < ApplicationController .new(params: params, current_user: current_user, project: project, group: group) .execute - render json: UserSerializer.new.represent(users) + render json: UserSerializer.new(params).represent(users, project: project) end def user diff --git a/app/controllers/boards/issues_controller.rb b/app/controllers/boards/issues_controller.rb index 90528f75ffd..1d1a72d21f1 100644 --- a/app/controllers/boards/issues_controller.rb +++ b/app/controllers/boards/issues_controller.rb @@ -26,7 +26,7 @@ module Boards list_service = Boards::Issues::ListService.new(board_parent, current_user, filter_params) issues = list_service.execute issues = issues.page(params[:page]).per(params[:per] || 20).without_count - Issue.move_to_end(issues) if Gitlab::Database.read_write? + Issue.move_nulls_to_end(issues) if Gitlab::Database.read_write? issues = issues.preload(:milestone, :assignees, project: [ diff --git a/app/controllers/concerns/issuable_actions.rb b/app/controllers/concerns/issuable_actions.rb index 6fa2f75be33..398cb728e05 100644 --- a/app/controllers/concerns/issuable_actions.rb +++ b/app/controllers/concerns/issuable_actions.rb @@ -98,13 +98,12 @@ module IssuableActions render json: { notice: "#{quantity} #{resource_name.pluralize(quantity)} updated" } end - # rubocop: disable CodeReuse/ActiveRecord + # rubocop:disable CodeReuse/ActiveRecord def discussions - notes = issuable.discussion_notes - .inc_relations_for_view - .with_notes_filter(notes_filter) - .includes(:noteable) - .fresh + notes = NotesFinder.new(current_user, finder_params_for_issuable).execute + .inc_relations_for_view + .includes(:noteable) + .fresh if notes_filter != UserPreference::NOTES_FILTERS[:only_comments] notes = ResourceEvents::MergeIntoNotesService.new(issuable, current_user).execute(notes) @@ -117,7 +116,7 @@ module IssuableActions render json: discussion_serializer.represent(discussions, context: self) end - # rubocop: enable CodeReuse/ActiveRecord + # rubocop:enable CodeReuse/ActiveRecord private @@ -222,4 +221,13 @@ module IssuableActions def parent @project || @group # rubocop:disable Gitlab/ModuleWithInstanceVariables end + + # rubocop:disable Gitlab/ModuleWithInstanceVariables + def finder_params_for_issuable + { + target: @issuable, + notes_filter: notes_filter + }.tap { |new_params| new_params[:project] = project if respond_to?(:project, true) } + end + # rubocop:enable Gitlab/ModuleWithInstanceVariables end diff --git a/app/controllers/concerns/notes_actions.rb b/app/controllers/concerns/notes_actions.rb index 0098c4cdf4c..d2a961efff7 100644 --- a/app/controllers/concerns/notes_actions.rb +++ b/app/controllers/concerns/notes_actions.rb @@ -243,7 +243,7 @@ module NotesActions end def notes_finder - @notes_finder ||= NotesFinder.new(project, current_user, finder_params) + @notes_finder ||= NotesFinder.new(current_user, finder_params) end def note_serializer diff --git a/app/controllers/projects/notes_controller.rb b/app/controllers/projects/notes_controller.rb index 3152a38fd8e..65d9b074eee 100644 --- a/app/controllers/projects/notes_controller.rb +++ b/app/controllers/projects/notes_controller.rb @@ -68,7 +68,7 @@ class Projects::NotesController < Projects::ApplicationController alias_method :awardable, :note def finder_params - params.merge(last_fetched_at: last_fetched_at, notes_filter: notes_filter) + params.merge(project: project, last_fetched_at: last_fetched_at, notes_filter: notes_filter) end def authorize_admin_note! diff --git a/app/controllers/snippets/notes_controller.rb b/app/controllers/snippets/notes_controller.rb index 612897f27e6..551b37cb3d3 100644 --- a/app/controllers/snippets/notes_controller.rb +++ b/app/controllers/snippets/notes_controller.rb @@ -27,7 +27,9 @@ class Snippets::NotesController < ApplicationController alias_method :noteable, :snippet def finder_params - params.merge(last_fetched_at: last_fetched_at, target_id: snippet.id, target_type: 'personal_snippet') + params.merge(last_fetched_at: last_fetched_at, target_id: snippet.id, target_type: 'personal_snippet').tap do |merged_params| + merged_params[:project] = project if respond_to?(:project) + end end def authorize_read_snippet! diff --git a/app/finders/notes_finder.rb b/app/finders/notes_finder.rb index 8f610d7dddb..f7d9100bb78 100644 --- a/app/finders/notes_finder.rb +++ b/app/finders/notes_finder.rb @@ -3,6 +3,8 @@ class NotesFinder FETCH_OVERLAP = 5.seconds + attr_reader :target_type + # Used to filter Notes # When used with target_type and target_id this returns notes specifically for the controller # @@ -10,15 +12,17 @@ class NotesFinder # current_user - which user check authorizations with # project - which project to look for notes on # params: + # target: noteable # target_type: string # target_id: integer # last_fetched_at: time # search: string # - def initialize(project, current_user, params = {}) - @project = project + def initialize(current_user, params = {}) + @project = params[:project] @current_user = current_user - @params = params + @params = params.dup + @target_type = @params[:target_type] end def execute @@ -32,7 +36,27 @@ class NotesFinder def target return @target if defined?(@target) - target_type = @params[:target_type] + if target_given? + use_explicit_target + else + find_target_by_type_and_ids + end + end + + private + + def target_given? + @params.key?(:target) + end + + def use_explicit_target + @target = @params[:target] + @target_type = @target.class.name.underscore + + @target + end + + def find_target_by_type_and_ids target_id = @params[:target_id] target_iid = @params[:target_iid] @@ -45,13 +69,11 @@ class NotesFinder @project.commit(target_id) end else - noteables_for_type_by_id(target_type, target_id, target_iid) + noteable_for_type_by_id(target_type, target_id, target_iid) end end - private - - def noteables_for_type_by_id(type, id, iid) + def noteable_for_type_by_id(type, id, iid) query = if id { id: id } else @@ -77,10 +99,6 @@ class NotesFinder search(notes) end - def target_type - @params[:target_type] - end - # rubocop: disable CodeReuse/ActiveRecord def notes_of_any_type types = %w(commit issue merge_request snippet) diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index 07813e03f3a..dd2bfc42af9 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -38,6 +38,7 @@ module Ci has_one :deployment, as: :deployable, class_name: 'Deployment' has_many :trace_sections, class_name: 'Ci::BuildTraceSection' has_many :trace_chunks, class_name: 'Ci::BuildTraceChunk', foreign_key: :build_id + has_many :needs, class_name: 'Ci::BuildNeed', foreign_key: :build_id, inverse_of: :build has_many :job_artifacts, class_name: 'Ci::JobArtifact', foreign_key: :job_id, dependent: :destroy, inverse_of: :job # rubocop:disable Cop/ActiveRecordDependent has_many :job_variables, class_name: 'Ci::JobVariable', foreign_key: :job_id @@ -50,6 +51,7 @@ module Ci accepts_nested_attributes_for :runner_session accepts_nested_attributes_for :job_variables + accepts_nested_attributes_for :needs delegate :url, to: :runner_session, prefix: true, allow_nil: true delegate :terminal_specification, to: :runner_session, allow_nil: true @@ -713,11 +715,21 @@ module Ci depended_jobs = depends_on_builds - return depended_jobs unless options[:dependencies].present? + # find all jobs that are dependent on + if options[:dependencies].present? + depended_jobs = depended_jobs.select do |job| + options[:dependencies].include?(job.name) + end + end - depended_jobs.select do |job| - options[:dependencies].include?(job.name) + # find all jobs that are needed by this one + if options[:needs].present? + depended_jobs = depended_jobs.select do |job| + options[:needs].include?(job.name) + end end + + depended_jobs end def empty_dependencies? diff --git a/app/models/ci/build_need.rb b/app/models/ci/build_need.rb new file mode 100644 index 00000000000..6531dfd332f --- /dev/null +++ b/app/models/ci/build_need.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +module Ci + class BuildNeed < ApplicationRecord + extend Gitlab::Ci::Model + + belongs_to :build, class_name: "Ci::Build", foreign_key: :build_id, inverse_of: :needs + + validates :build, presence: true + validates :name, presence: true, length: { maximum: 128 } + + scope :scoped_build, -> { where('ci_builds.id=ci_build_needs.build_id') } + end +end diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb index 1c76f401690..3515f0b83ee 100644 --- a/app/models/ci/pipeline.rb +++ b/app/models/ci/pipeline.rb @@ -611,8 +611,8 @@ module Ci end # rubocop: disable CodeReuse/ServiceClass - def process! - Ci::ProcessPipelineService.new(project, user).execute(self) + def process!(trigger_build_name = nil) + Ci::ProcessPipelineService.new(project, user).execute(self, trigger_build_name) end # rubocop: enable CodeReuse/ServiceClass diff --git a/app/models/commit_status.rb b/app/models/commit_status.rb index be6f3e9c5b0..d7eb78db5b8 100644 --- a/app/models/commit_status.rb +++ b/app/models/commit_status.rb @@ -43,6 +43,12 @@ class CommitStatus < ApplicationRecord scope :after_stage, -> (index) { where('stage_idx > ?', index) } scope :processables, -> { where(type: %w[Ci::Build Ci::Bridge]) } + scope :with_needs, -> (names = nil) do + needs = Ci::BuildNeed.scoped_build.select(1) + needs = needs.where(name: names) if names + where('EXISTS (?)', needs).preload(:needs) + end + # We use `CommitStatusEnums.failure_reasons` here so that EE can more easily # extend this `Hash` with new values. enum_with_nil failure_reason: ::CommitStatusEnums.failure_reasons @@ -116,7 +122,7 @@ class CommitStatus < ApplicationRecord commit_status.run_after_commit do if pipeline_id if complete? || manual? - PipelineProcessWorker.perform_async(pipeline_id) + BuildProcessWorker.perform_async(id) else PipelineUpdateWorker.perform_async(pipeline_id) end diff --git a/app/models/concerns/ci/metadatable.rb b/app/models/concerns/ci/metadatable.rb index 9eed9492b37..304cc71e9dc 100644 --- a/app/models/concerns/ci/metadatable.rb +++ b/app/models/concerns/ci/metadatable.rb @@ -29,6 +29,7 @@ module Ci def degenerate! self.class.transaction do self.update!(options: nil, yaml_variables: nil) + self.needs.all.delete_all self.metadata&.destroy end end diff --git a/app/models/concerns/relative_positioning.rb b/app/models/concerns/relative_positioning.rb index 9cd7b8d6258..4a1441805fc 100644 --- a/app/models/concerns/relative_positioning.rb +++ b/app/models/concerns/relative_positioning.rb @@ -34,7 +34,7 @@ module RelativePositioning end class_methods do - def move_to_end(objects) + def move_nulls_to_end(objects) objects = objects.reject(&:relative_position) return if objects.empty? @@ -43,7 +43,7 @@ module RelativePositioning self.transaction do objects.each do |object| - relative_position = position_between(max_relative_position, MAX_POSITION) + relative_position = position_between(max_relative_position || START_POSITION, MAX_POSITION) object.relative_position = relative_position max_relative_position = relative_position object.save(touch: false) diff --git a/app/models/merge_request.rb b/app/models/merge_request.rb index 8ade91933a4..5e8a6a7d5e5 100644 --- a/app/models/merge_request.rb +++ b/app/models/merge_request.rb @@ -752,7 +752,7 @@ class MergeRequest < ApplicationRecord end def check_mergeability - MergeRequests::MergeabilityCheckService.new(self).execute + MergeRequests::MergeabilityCheckService.new(self).execute(retry_lease: false) end # rubocop: enable CodeReuse/ServiceClass diff --git a/app/models/namespace/aggregation_schedule.rb b/app/models/namespace/aggregation_schedule.rb index 0bef352cf24..61a7eb4b576 100644 --- a/app/models/namespace/aggregation_schedule.rb +++ b/app/models/namespace/aggregation_schedule.rb @@ -6,21 +6,13 @@ class Namespace::AggregationSchedule < ApplicationRecord self.primary_key = :namespace_id - DEFAULT_LEASE_TIMEOUT = 3.hours + DEFAULT_LEASE_TIMEOUT = 1.5.hours.to_i REDIS_SHARED_KEY = 'gitlab:update_namespace_statistics_delay'.freeze belongs_to :namespace after_create :schedule_root_storage_statistics - def self.delay_timeout - redis_timeout = Gitlab::Redis::SharedState.with do |redis| - redis.get(REDIS_SHARED_KEY) - end - - redis_timeout.nil? ? DEFAULT_LEASE_TIMEOUT : redis_timeout.to_i - end - def schedule_root_storage_statistics run_after_commit_or_now do try_obtain_lease do @@ -28,7 +20,7 @@ class Namespace::AggregationSchedule < ApplicationRecord .perform_async(namespace_id) Namespaces::RootStatisticsWorker - .perform_in(self.class.delay_timeout, namespace_id) + .perform_in(DEFAULT_LEASE_TIMEOUT, namespace_id) end end end @@ -37,7 +29,7 @@ class Namespace::AggregationSchedule < ApplicationRecord # Used by ExclusiveLeaseGuard def lease_timeout - self.class.delay_timeout + DEFAULT_LEASE_TIMEOUT end # Used by ExclusiveLeaseGuard diff --git a/app/serializers/user_serializer.rb b/app/serializers/user_serializer.rb index 2111e1b5667..d988caea92d 100644 --- a/app/serializers/user_serializer.rb +++ b/app/serializers/user_serializer.rb @@ -2,4 +2,21 @@ class UserSerializer < BaseSerializer entity UserEntity + + def represent(resource, opts = {}, entity = nil) + if params[:merge_request_iid] + merge_request = opts[:project].merge_requests.find_by_iid!(params[:merge_request_iid]) + preload_max_member_access(merge_request.project, Array(resource)) + + super(resource, opts.merge(merge_request: merge_request), MergeRequestAssigneeEntity) + else + super + end + end + + private + + def preload_max_member_access(project, users) + project.team.max_member_access_for_user_ids(users.map(&:id)) + end end diff --git a/app/services/ci/process_pipeline_service.rb b/app/services/ci/process_pipeline_service.rb index 207cc5017d0..e46615bcf75 100644 --- a/app/services/ci/process_pipeline_service.rb +++ b/app/services/ci/process_pipeline_service.rb @@ -4,19 +4,23 @@ module Ci class ProcessPipelineService < BaseService attr_reader :pipeline - def execute(pipeline) + def execute(pipeline, trigger_build_name = nil) @pipeline = pipeline update_retried - new_builds = + success = stage_indexes_of_created_processables.flat_map do |index| process_stage(index) - end + end.any? + + # we evaluate dependent needs, + # only when the another job has finished + success = process_builds_with_needs(trigger_build_name) || success @pipeline.update_status - new_builds.any? + success end private @@ -36,6 +40,28 @@ module Ci end end + def process_builds_with_needs(trigger_build_name) + return false unless trigger_build_name + return false unless Feature.enabled?(:ci_dag_support, project) + + created_processables + .with_needs(trigger_build_name) + .find_each + .map(&method(:process_build_with_needs)) + .any? + end + + def process_build_with_needs(build) + current_status = status_for_build_needs(build.needs.map(&:name)) + + return unless HasStatus::COMPLETED_STATUSES.include?(current_status) + + Gitlab::OptimisticLocking.retry_lock(build) do |subject| + Ci::ProcessBuildService.new(project, @user) + .execute(subject, current_status) + end + end + # rubocop: disable CodeReuse/ActiveRecord def status_for_prior_stages(index) pipeline.builds.where('stage_idx < ?', index).latest.status || 'success' @@ -43,6 +69,12 @@ module Ci # rubocop: enable CodeReuse/ActiveRecord # rubocop: disable CodeReuse/ActiveRecord + def status_for_build_needs(needs) + pipeline.builds.where(name: needs).latest.status || 'success' + end + # rubocop: enable CodeReuse/ActiveRecord + + # rubocop: disable CodeReuse/ActiveRecord def stage_indexes_of_created_processables created_processables.order(:stage_idx).pluck(Arel.sql('DISTINCT stage_idx')) end diff --git a/app/services/ci/retry_build_service.rb b/app/services/ci/retry_build_service.rb index fab8a179843..338495ba030 100644 --- a/app/services/ci/retry_build_service.rb +++ b/app/services/ci/retry_build_service.rb @@ -5,7 +5,7 @@ module Ci CLONE_ACCESSORS = %i[pipeline project ref tag options name allow_failure stage stage_id stage_idx trigger_request yaml_variables when environment coverage_regex - description tag_list protected].freeze + description tag_list protected needs].freeze def execute(build) reprocess!(build).tap do |new_build| diff --git a/app/services/merge_requests/mergeability_check_service.rb b/app/services/merge_requests/mergeability_check_service.rb index 9fa50c9448f..962e2327b3e 100644 --- a/app/services/merge_requests/mergeability_check_service.rb +++ b/app/services/merge_requests/mergeability_check_service.rb @@ -3,6 +3,7 @@ module MergeRequests class MergeabilityCheckService < ::BaseService include Gitlab::Utils::StrongMemoize + include Gitlab::ExclusiveLeaseHelpers delegate :project, to: :@merge_request delegate :repository, to: :project @@ -21,13 +22,35 @@ module MergeRequests # where we need the current state of the merge ref in repository, the `recheck` # argument is required. # + # retry_lease - Concurrent calls wait for at least 10 seconds until the + # lease is granted (other process finishes running). Returns an error + # ServiceResponse if the lease is not granted during this time. + # # Returns a ServiceResponse indicating merge_status is/became can_be_merged # and the merge-ref is synced. Success in case of being/becoming mergeable, # error otherwise. - def execute(recheck: false) + def execute(recheck: false, retry_lease: true) return ServiceResponse.error(message: 'Invalid argument') unless merge_request return ServiceResponse.error(message: 'Unsupported operation') if Gitlab::Database.read_only? + return check_mergeability(recheck) unless merge_ref_auto_sync_lock_enabled? + + in_write_lock(retry_lease: retry_lease) do |retried| + # When multiple calls are waiting for the same lock (retry_lease), + # it's possible that when granted, the MR status was already updated for + # that object, therefore we reset if there was a lease retry. + merge_request.reset if retried + + check_mergeability(recheck) + end + rescue FailedToObtainLockError => error + ServiceResponse.error(message: error.message) + end + + private + + attr_reader :merge_request + def check_mergeability(recheck) recheck! if recheck update_merge_status @@ -46,9 +69,21 @@ module MergeRequests ServiceResponse.success(payload: payload) end - private + # It's possible for this service to send concurrent requests to Gitaly in order + # to "git update-ref" the same ref. Therefore we handle a light exclusive + # lease here. + # + def in_write_lock(retry_lease:, &block) + lease_key = "mergeability_check:#{merge_request.id}" - attr_reader :merge_request + lease_opts = { + ttl: 1.minute, + retries: retry_lease ? 10 : 0, + sleep_sec: retry_lease ? 1.second : 0 + } + + in_lock(lease_key, lease_opts, &block) + end def payload strong_memoize(:payload) do @@ -116,5 +151,9 @@ module MergeRequests def merge_ref_auto_sync_enabled? Feature.enabled?(:merge_ref_auto_sync, project, default_enabled: true) end + + def merge_ref_auto_sync_lock_enabled? + Feature.enabled?(:merge_ref_auto_sync_lock, project, default_enabled: true) + end end end diff --git a/app/workers/all_queues.yml b/app/workers/all_queues.yml index 991a177018e..400becdd023 100644 --- a/app/workers/all_queues.yml +++ b/app/workers/all_queues.yml @@ -88,6 +88,7 @@ - pipeline_processing:ci_build_prepare - pipeline_processing:build_queue - pipeline_processing:build_success +- pipeline_processing:build_process - pipeline_processing:pipeline_process - pipeline_processing:pipeline_success - pipeline_processing:pipeline_update diff --git a/app/workers/build_process_worker.rb b/app/workers/build_process_worker.rb new file mode 100644 index 00000000000..19e590ee1d7 --- /dev/null +++ b/app/workers/build_process_worker.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +class BuildProcessWorker + include ApplicationWorker + include PipelineQueue + + queue_namespace :pipeline_processing + + # rubocop: disable CodeReuse/ActiveRecord + def perform(build_id) + CommitStatus.find_by(id: build_id).try do |build| + build.pipeline.process!(build.name) + end + end + # rubocop: enable CodeReuse/ActiveRecord +end |