diff options
Diffstat (limited to 'app')
19 files changed, 170 insertions, 81 deletions
diff --git a/app/assets/javascripts/admin/application_settings/network_outbound.js b/app/assets/javascripts/admin/application_settings/network_outbound.js new file mode 100644 index 00000000000..ad7ed85131c --- /dev/null +++ b/app/assets/javascripts/admin/application_settings/network_outbound.js @@ -0,0 +1,28 @@ +export default () => { + const denyAllRequests = document.querySelector('.js-deny-all-requests'); + + if (!denyAllRequests) { + return; + } + + denyAllRequests.addEventListener('change', () => { + const denyAll = denyAllRequests.checked; + const allowLocalRequests = document.querySelectorAll('.js-allow-local-requests'); + const denyAllRequestsWarning = document.querySelector('.js-deny-all-requests-warning'); + + if (denyAll) { + denyAllRequestsWarning.classList.remove('gl-display-none'); + } else { + denyAllRequestsWarning.classList.add('gl-display-none'); + } + + allowLocalRequests.forEach((allowLocalRequest) => { + /* eslint-disable no-param-reassign */ + if (denyAll) { + allowLocalRequest.checked = false; + } + allowLocalRequest.disabled = denyAll; + /* eslint-enable no-param-reassign */ + }); + }); +}; diff --git a/app/assets/javascripts/pages/admin/application_settings/network/index.js b/app/assets/javascripts/pages/admin/application_settings/network/index.js new file mode 100644 index 00000000000..841c68c5cd0 --- /dev/null +++ b/app/assets/javascripts/pages/admin/application_settings/network/index.js @@ -0,0 +1,3 @@ +import initNetworkOutbound from '~/admin/application_settings/network_outbound'; + +initNetworkOutbound(); diff --git a/app/helpers/sidebars_helper.rb b/app/helpers/sidebars_helper.rb index 17ba56e8a67..823cb9e1de7 100644 --- a/app/helpers/sidebars_helper.rb +++ b/app/helpers/sidebars_helper.rb @@ -83,17 +83,16 @@ module SidebarsHelper end def super_sidebar_nav_panel(nav: nil, project: nil, user: nil, group: nil, current_ref: nil, ref_type: nil) + context_adds = { route_is_active: method(:active_nav_link?), is_super_sidebar: true } case nav when 'project' - context = project_sidebar_context(project, user, current_ref, ref_type: ref_type, - route_is_active: method(:active_nav_link?)) + context = project_sidebar_context(project, user, current_ref, ref_type: ref_type, **context_adds) Sidebars::Projects::SuperSidebarPanel.new(context) when 'group' - context = group_sidebar_context(group, user, route_is_active: method(:active_nav_link?)) + context = group_sidebar_context(group, user, **context_adds) Sidebars::Groups::Panel.new(context) else - Sidebars::YourWork::Panel.new(Sidebars::Context.new(current_user: user, container: nil, - route_is_active: method(:active_nav_link?))) + Sidebars::YourWork::Panel.new(Sidebars::Context.new(current_user: user, container: nil, **context_adds)) end end diff --git a/app/models/ci/catalog/listing.rb b/app/models/ci/catalog/listing.rb new file mode 100644 index 00000000000..99a5230b64e --- /dev/null +++ b/app/models/ci/catalog/listing.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +module Ci + module Catalog + class Listing + # This class is the SSoT to displaying the list of resources in the + # CI/CD Catalog given a namespace as a scope. + # This model is not directly backed by a table and joins catalog resources + # with projects to return relevant data. + def initialize(namespace) + raise ArgumentError, 'Namespace is not a root namespace' unless namespace.root? + + @namespace = namespace + end + + def resources + Ci::Catalog::Resource + .joins(:project).includes(:project) + .merge(Project.in_namespace(namespace.self_and_descendant_ids)) + end + + private + + attr_reader :namespace + end + end +end diff --git a/app/models/ci/catalog/resource.rb b/app/models/ci/catalog/resource.rb new file mode 100644 index 00000000000..1b3dec5f54d --- /dev/null +++ b/app/models/ci/catalog/resource.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +module Ci + module Catalog + # This class represents a CI/CD Catalog resource. + # A Catalog resource is normally associated to a project. + # This model connects to the `main` database because of its + # dependency on the Project model and its need to join with that table + # in order to generate the CI/CD catalog. + class Resource < ::ApplicationRecord + self.table_name = 'catalog_resources' + + belongs_to :project + end + end +end diff --git a/app/models/ci/runner_machine_build.rb b/app/models/ci/runner_machine_build.rb index ac2b258557a..95418db3619 100644 --- a/app/models/ci/runner_machine_build.rb +++ b/app/models/ci/runner_machine_build.rb @@ -3,16 +3,11 @@ module Ci class RunnerMachineBuild < Ci::ApplicationRecord include Ci::Partitionable - include PartitionedTable self.table_name = :p_ci_runner_machine_builds self.primary_key = :build_id - partitionable scope: :build - partitioned_by :partition_id, - strategy: :ci_sliding_list, - next_partition_if: proc { false }, - detach_partition_if: proc { false } + partitionable scope: :build, partitioned: true belongs_to :build, inverse_of: :runner_machine_build, class_name: 'Ci::Build' belongs_to :runner_machine, inverse_of: :runner_machine_builds, class_name: 'Ci::RunnerMachine' diff --git a/app/models/ci/stage.rb b/app/models/ci/stage.rb index 5df6f774767..02093bdf153 100644 --- a/app/models/ci/stage.rb +++ b/app/models/ci/stage.rb @@ -118,6 +118,7 @@ module Ci end end + # This will be removed with ci_remove_ensure_stage_service def update_legacy_status set_status(latest_stage_status.to_s) end @@ -151,6 +152,7 @@ module Ci blocked? || skipped? end + # This will be removed with ci_remove_ensure_stage_service def latest_stage_status statuses.latest.composite_status || 'skipped' end diff --git a/app/models/concerns/ci/has_status.rb b/app/models/concerns/ci/has_status.rb index c10a9221efb..2971ecb04b8 100644 --- a/app/models/concerns/ci/has_status.rb +++ b/app/models/concerns/ci/has_status.rb @@ -23,6 +23,7 @@ module Ci UnknownStatusError = Class.new(StandardError) class_methods do + # This will be removed with ci_remove_ensure_stage_service def composite_status Gitlab::Ci::Status::Composite .new(all, with_allow_failure: columns_hash.key?('allow_failure')) diff --git a/app/models/concerns/ci/partitionable.rb b/app/models/concerns/ci/partitionable.rb index fa0c2221968..28cc17432bc 100644 --- a/app/models/concerns/ci/partitionable.rb +++ b/app/models/concerns/ci/partitionable.rb @@ -69,9 +69,10 @@ module Ci end class_methods do - def partitionable(scope:, through: nil) + def partitionable(scope:, through: nil, partitioned: false) handle_partitionable_through(through) handle_partitionable_scope(scope) + handle_partitionable_ddl(partitioned) end private @@ -95,6 +96,17 @@ module Ci end end end + + def handle_partitionable_ddl(partitioned) + return unless partitioned + + include ::PartitionedTable + + partitioned_by :partition_id, + strategy: :ci_sliding_list, + next_partition_if: proc { false }, + detach_partition_if: proc { false } + end end end end diff --git a/app/models/integrations/base_slash_commands.rb b/app/models/integrations/base_slash_commands.rb index 619579a543a..eece67b86d4 100644 --- a/app/models/integrations/base_slash_commands.rb +++ b/app/models/integrations/base_slash_commands.rb @@ -6,8 +6,6 @@ module Integrations class BaseSlashCommands < Integration attribute :category, default: 'chat' - prop_accessor :token - has_many :chat_names, foreign_key: :integration_id, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent def valid_token?(token) @@ -24,18 +22,6 @@ module Integrations false end - def fields - [ - { - type: 'password', - name: 'token', - non_empty_password_title: s_('ProjectService|Enter new token'), - non_empty_password_help: s_('ProjectService|Leave blank to use your current token.'), - placeholder: 'XXxxXXxxXXxxXXxxXXxxXXxx' - } - ] - end - def trigger(params) return unless valid_token?(params[:token]) diff --git a/app/models/integrations/mattermost_slash_commands.rb b/app/models/integrations/mattermost_slash_commands.rb index 62fe4820e55..f5079b9b907 100644 --- a/app/models/integrations/mattermost_slash_commands.rb +++ b/app/models/integrations/mattermost_slash_commands.rb @@ -4,7 +4,11 @@ module Integrations class MattermostSlashCommands < BaseSlashCommands include Ci::TriggersHelper - prop_accessor :token + field :token, + type: 'password', + non_empty_password_title: -> { s_('ProjectService|Enter new token') }, + non_empty_password_help: -> { s_('ProjectService|Leave blank to use your current token.') }, + placeholder: '' def testable? false diff --git a/app/models/integrations/slack_slash_commands.rb b/app/models/integrations/slack_slash_commands.rb index 01a87e8e25e..343c8d68166 100644 --- a/app/models/integrations/slack_slash_commands.rb +++ b/app/models/integrations/slack_slash_commands.rb @@ -4,6 +4,12 @@ module Integrations class SlackSlashCommands < BaseSlashCommands include Ci::TriggersHelper + field :token, + type: 'password', + non_empty_password_title: -> { s_('ProjectService|Enter new token') }, + non_empty_password_help: -> { s_('ProjectService|Leave blank to use your current token.') }, + placeholder: '' + def title 'Slack slash commands' end diff --git a/app/models/project.rb b/app/models/project.rb index a26116ee830..e2f5e51453d 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -373,7 +373,6 @@ class Project < ApplicationRecord inverse_of: :project has_many :stages, class_name: 'Ci::Stage', inverse_of: :project has_many :ci_refs, class_name: 'Ci::Ref', inverse_of: :project - has_many :pipeline_metadata, class_name: 'Ci::PipelineMetadata', inverse_of: :project has_many :pending_builds, class_name: 'Ci::PendingBuild' has_many :builds, class_name: 'Ci::Build', inverse_of: :project diff --git a/app/services/ci/pipeline_processing/atomic_processing_service.rb b/app/services/ci/pipeline_processing/atomic_processing_service.rb index 2b8eb104be5..4f2230ea1fc 100644 --- a/app/services/ci/pipeline_processing/atomic_processing_service.rb +++ b/app/services/ci/pipeline_processing/atomic_processing_service.rb @@ -48,10 +48,10 @@ module Ci def update_stage!(stage) # Update processables for a given stage in bulk/slices @collection - .created_processable_ids_for_stage_position(stage.position) + .created_processable_ids_in_stage(stage.position) .in_groups_of(BATCH_SIZE, false) { |ids| update_processables!(ids) } - status = @collection.status_for_stage_position(stage.position) + status = @collection.status_of_stage(stage.position) stage.set_status(status) end @@ -79,29 +79,27 @@ module Ci end def update_processable!(processable) - status = processable_status(processable) - return unless Ci::HasStatus::COMPLETED_STATUSES.include?(status) + previous_status = status_of_previous_processables(processable) + # We do not continue to process the processable if the previous status is not completed + return unless Ci::HasStatus::COMPLETED_STATUSES.include?(previous_status) - # transition status if possible Gitlab::OptimisticLocking.retry_lock(processable, name: 'atomic_processing_update_processable') do |subject| Ci::ProcessBuildService.new(project, subject.user) - .execute(subject, status) + .execute(subject, previous_status) # update internal representation of status - # to make the status change of processable - # to be taken into account during further processing - @collection.set_processable_status( - processable.id, processable.status, processable.lock_version) + # to make the status change of processable to be taken into account during further processing + @collection.set_processable_status(processable.id, processable.status, processable.lock_version) end end - def processable_status(processable) + def status_of_previous_processables(processable) if processable.scheduling_type_dag? # Processable uses DAG, get status of all dependent needs - @collection.status_for_names(processable.aggregated_needs_names.to_a, dag: true) + @collection.status_of_processables(processable.aggregated_needs_names.to_a, dag: true) else # Processable uses Stages, get status of prior stage - @collection.status_for_prior_stage_position(processable.stage_idx.to_i) + @collection.status_of_processables_prior_to_stage(processable.stage_idx.to_i) end end diff --git a/app/services/ci/pipeline_processing/atomic_processing_service/status_collection.rb b/app/services/ci/pipeline_processing/atomic_processing_service/status_collection.rb index 676c2ecb257..9738e4e65b7 100644 --- a/app/services/ci/pipeline_processing/atomic_processing_service/status_collection.rb +++ b/app/services/ci/pipeline_processing/atomic_processing_service/status_collection.rb @@ -35,40 +35,40 @@ module Ci status_for_array(all_statuses, dag: false) end + # This methods gets composite status for processables at a given stage + def status_of_stage(stage_position) + strong_memoize("status_of_stage_#{stage_position}") do + stage_statuses = all_statuses_grouped_by_stage_position[stage_position].to_a + + status_for_array(stage_statuses.flatten, dag: false) + end + end + # This methods gets composite status for processables with given names - def status_for_names(names, dag:) + def status_of_processables(names, dag:) name_statuses = all_statuses_by_name.slice(*names) status_for_array(name_statuses.values, dag: dag) end # This methods gets composite status for processables before given stage - def status_for_prior_stage_position(position) - strong_memoize("status_for_prior_stage_position_#{position}") do + def status_of_processables_prior_to_stage(stage_position) + strong_memoize("status_of_processables_prior_to_stage_#{stage_position}") do stage_statuses = all_statuses_grouped_by_stage_position - .select { |stage_position, _| stage_position < position } + .select { |position, _| position < stage_position } status_for_array(stage_statuses.values.flatten, dag: false) end end # This methods gets a list of processables for a given stage - def created_processable_ids_for_stage_position(current_position) - all_statuses_grouped_by_stage_position[current_position] + def created_processable_ids_in_stage(stage_position) + all_statuses_grouped_by_stage_position[stage_position] .to_a .select { |processable| processable[:status] == 'created' } .map { |processable| processable[:id] } end - # This methods gets composite status for processables at a given stage - def status_for_stage_position(current_position) - strong_memoize("status_for_stage_position_#{current_position}") do - stage_statuses = all_statuses_grouped_by_stage_position[current_position].to_a - - status_for_array(stage_statuses.flatten, dag: false) - end - end - # This method returns a list of all processable, that are to be processed def processing_processables all_statuses.lazy.reject { |status| status[:processed] } diff --git a/app/services/ci/process_build_service.rb b/app/services/ci/process_build_service.rb index a5300cfd29f..afaf18a4de2 100644 --- a/app/services/ci/process_build_service.rb +++ b/app/services/ci/process_build_service.rb @@ -2,40 +2,40 @@ module Ci class ProcessBuildService < BaseService - def execute(build, current_status) - if valid_statuses_for_build(build).include?(current_status) - process(build) + def execute(processable, current_status) + if valid_statuses_for_processable(processable).include?(current_status) + process(processable) true else - build.skip + processable.skip false end end private - def process(build) - return enqueue(build) if build.enqueue_immediately? + def process(processable) + return enqueue(processable) if processable.enqueue_immediately? - if build.schedulable? - build.schedule - elsif build.action? - build.actionize + if processable.schedulable? + processable.schedule + elsif processable.action? + processable.actionize else - enqueue(build) + enqueue(processable) end end - def enqueue(build) - return build.drop!(:failed_outdated_deployment_job) if build.outdated_deployment? + def enqueue(processable) + return processable.drop!(:failed_outdated_deployment_job) if processable.outdated_deployment? - build.enqueue + processable.enqueue end - def valid_statuses_for_build(build) - case build.when + def valid_statuses_for_processable(processable) + case processable.when when 'on_success', 'manual', 'delayed' - build.scheduling_type_dag? ? %w[success] : %w[success skipped] + processable.scheduling_type_dag? ? %w[success] : %w[success skipped] when 'on_failure' %w[failed] when 'always' diff --git a/app/views/admin/application_settings/_outbound.html.haml b/app/views/admin/application_settings/_outbound.html.haml index 1821c8ef4bb..566bcc5b8e0 100644 --- a/app/views/admin/application_settings/_outbound.html.haml +++ b/app/views/admin/application_settings/_outbound.html.haml @@ -1,25 +1,37 @@ +- deny_all_requests = Feature.enabled?(:deny_all_requests) && @application_setting.deny_all_requests + = gitlab_ui_form_for @application_setting, url: network_admin_application_settings_path(anchor: 'js-outbound-settings'), html: { class: 'fieldset-form' } do |f| = form_errors(@application_setting) %fieldset .form-group + - if Feature.enabled?(:deny_all_requests) + = f.gitlab_ui_checkbox_component :deny_all_requests, + s_('OutboundRequests|Block all requests, except for IP addresses, IP ranges, and domain names defined in the allowlist'), + checkbox_options: { class: 'js-deny-all-requests' } + = render Pajamas::AlertComponent.new(variant: :warning, + dismissible: false, + alert_options: { class: "gl-mb-3 js-deny-all-requests-warning #{'gl-display-none' unless deny_all_requests}" }) do |c| + = c.body do + = s_('OutboundRequests|Webhooks and integrations might not work properly.') = f.gitlab_ui_checkbox_component :allow_local_requests_from_web_hooks_and_services, - s_('OutboundRequests|Allow requests to the local network from web hooks and services'), - checkbox_options: { data: { qa_selector: 'allow_requests_from_services_checkbox' } } + s_('OutboundRequests|Allow requests to the local network from webhooks and integrations'), + checkbox_options: { disabled: deny_all_requests, class: 'js-allow-local-requests', data: { qa_selector: 'allow_requests_from_services_checkbox' } } = f.gitlab_ui_checkbox_component :allow_local_requests_from_system_hooks, - s_('OutboundRequests|Allow requests to the local network from system hooks') + s_('OutboundRequests|Allow requests to the local network from system hooks'), + checkbox_options: { disabled: deny_all_requests, class: 'js-allow-local-requests' } .form-group = f.label :outbound_local_requests_allowlist_raw, class: 'label-bold' do - = s_('OutboundRequests|Local IP addresses and domain names that hooks and services may access') + = s_('OutboundRequests|Local IP addresses and domain names that hooks and integrations can access') = f.text_area :outbound_local_requests_allowlist_raw, placeholder: "example.com, 192.168.1.1, xn--itlab-j1a.com", class: 'form-control gl-form-input', rows: 8 %span.form-text.text-muted - = s_('OutboundRequests|Requests to these domains and IP addresses are accessible to both system hooks and web hooks even when local requests are not allowed. IP ranges such as 1:0:0:0:0:0:0:0/124 and 127.0.0.0/28 are supported. Domain wildcards are not supported. To separate entries use commas, semicolons, or newlines. The allowlist can hold a maximum of 1000 entries. Domains must be IDNA encoded.') + = s_('OutboundRequests|Requests can be made to these IP addresses and domains even when local requests are not allowed. IP ranges such as %{code_start}1:0:0:0:0:0:0:0/124%{code_end} and %{code_start}127.0.0.0/28%{code_end} are supported. Domain wildcards are not supported. To separate entries, use commas, semicolons, or newlines. The allowlist can have a maximum of 1000 entries. Domains must be IDNA-encoded.').html_safe % { code_start: '<code>'.html_safe, code_end: '</code>'.html_safe } = link_to _('Learn more.'), help_page_path('security/webhooks.md', anchor: 'create-an-allowlist-for-local-requests'), target: '_blank', rel: 'noopener noreferrer' .form-group = f.gitlab_ui_checkbox_component :dns_rebinding_protection_enabled, - s_('OutboundRequests|Enforce DNS rebinding attack protection'), - help_text: s_('OutboundRequests|Resolve IP addresses once and uses them to submit requests.') + s_('OutboundRequests|Enforce DNS-rebinding attack protection'), + help_text: s_('OutboundRequests|Resolve IP addresses for outbound requests to prevent DNS-rebinding attacks.') = f.submit _('Save changes'), pajamas_button: true, data: { qa_selector: 'save_changes_button' } diff --git a/app/views/admin/application_settings/network.html.haml b/app/views/admin/application_settings/network.html.haml index fe5f0960d01..b20fc703f18 100644 --- a/app/views/admin/application_settings/network.html.haml +++ b/app/views/admin/application_settings/network.html.haml @@ -92,7 +92,7 @@ = render Pajamas::ButtonComponent.new(button_options: { class: 'js-settings-toggle' }) do = expanded_by_default? ? _('Collapse') : _('Expand') %p - = s_('OutboundRequests|Allow requests to the local network from hooks and services.') + = s_('OutboundRequests|Allow requests to the local network from hooks and integrations.') = link_to _('Learn more.'), help_page_path('security/webhooks.md'), target: '_blank', rel: 'noopener noreferrer' .settings-content = render 'outbound' diff --git a/app/workers/stage_update_worker.rb b/app/workers/stage_update_worker.rb index e0d8958fc80..97da76346b6 100644 --- a/app/workers/stage_update_worker.rb +++ b/app/workers/stage_update_worker.rb @@ -1,5 +1,6 @@ # frozen_string_literal: true +# This will be scheduled to be removed after removing the FF ci_remove_ensure_stage_service class StageUpdateWorker include ApplicationWorker |