diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2021-02-05 09:09:10 +0000 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2021-02-05 09:09:10 +0000 |
commit | 315243f87739dd1edda2b75361f826abc91d4069 (patch) | |
tree | f72b9e0c3e79e263687ea8d127a3e78bde9d8760 | |
parent | 1c4f17276df2af5a6c205c2a3ceec907751edf68 (diff) | |
download | gitlab-ce-315243f87739dd1edda2b75361f826abc91d4069.tar.gz |
Add latest changes from gitlab-org/gitlab@master
63 files changed, 633 insertions, 257 deletions
diff --git a/.rubocop_manual_todo.yml b/.rubocop_manual_todo.yml index 15a61e68c06..9517b932b86 100644 --- a/.rubocop_manual_todo.yml +++ b/.rubocop_manual_todo.yml @@ -2648,55 +2648,6 @@ Style/FrozenStringLiteralComment: - 'bin/secpick' - 'danger/changes_size/Dangerfile' - 'danger/metadata/Dangerfile' - - 'db/migrate/20190325080727_truncate_user_fullname.rb' - - 'db/migrate/20190828110802_add_not_null_constraints_to_prometheus_metrics_y_label_and_unit.rb' - - 'db/migrate/20190828172831_create_package_tag.rb' - - 'db/migrate/20191003195218_add_pendo_enabled_to_application_settings.rb' - - 'db/migrate/20191003195620_add_pendo_url_to_application_settings.rb' - - 'db/migrate/20191120200015_add_index_to_grafana_integrations.rb' - - 'db/migrate/20200229171700_create_custom_emojis.rb' - - 'db/migrate/20201004163918_remove_project_id_and_id_index_from_vulnerabilities_table.rb' - - 'db/migrate/20201214000000_change_mr_allow_maintainer_to_push_default.rb' - - 'db/optional_migrations/composite_primary_keys.rb' - - 'db/post_migrate/20190214112022_schedule_sync_issuables_state_id.rb' - - 'db/post_migrate/20201030121314_schedule_update_existing_users_that_require_two_factor_auth.rb' - - 'ee/db/fixtures/development/20_burndown.rb' - - 'ee/db/fixtures/development/20_vulnerabilities.rb' - - 'ee/db/fixtures/development/22_epics.rb' - - 'ee/db/geo/migrate/20170206203234_create_project_registry.rb' - - 'ee/db/geo/migrate/20170223033541_create_file_registry.rb' - - 'ee/db/geo/migrate/20170302005747_add_index_to_project_id_on_project_registry.rb' - - 'ee/db/geo/migrate/20170526214010_convert_file_bytes_to_int64.rb' - - 'ee/db/geo/migrate/20170605154253_create_event_log_state.rb' - - 'ee/db/geo/migrate/20170606155045_add_needs_resync_to_project_registry.rb' - - 'ee/db/geo/migrate/20170614201943_add_last_wiki_synced_at_to_project_registry.rb' - - 'ee/db/geo/migrate/20170627195211_add_index_to_project_registry.rb' - - 'ee/db/geo/migrate/20170906174622_remove_duplicates_from_project_registry.rb' - - 'ee/db/geo/migrate/20170906182752_add_unique_index_to_project_id_on_project_registry.rb' - - 'ee/db/geo/migrate/20171005045404_remove_file_uploads_from_registry.rb' - - 'ee/db/geo/migrate/20171009162208_add_file_registry_success.rb' - - 'ee/db/geo/migrate/20171009162209_add_file_registry_success_index.rb' - - 'ee/db/geo/migrate/20171101105200_add_retry_count_fields_to_registries.rb' - - 'ee/db/geo/migrate/20171115143841_add_last_sync_failure_to_project_registry.rb' - - 'ee/db/geo/migrate/20180201154345_add_repository_verification_to_project_registry.rb' - - 'ee/db/geo/migrate/20180314175612_add_partial_index_to_project_registy_verification_failure_columns.rb' - - 'ee/db/geo/migrate/20180315222132_add_partial_index_to_project_registy_checksum_columns.rb' - - 'ee/db/geo/migrate/20180321144947_change_repository_verification_checksum_to_sha.rb' - - 'ee/db/geo/migrate/20180322062741_migrate_ci_job_artifacts_to_separate_registry.rb' - - 'ee/db/geo/migrate/20180323182105_add_missing_on_primary_to_file_registry.rb' - - 'ee/db/geo/migrate/20180327071612_add_partial_index_to_project_registy_checksum_sha_columns.rb' - - 'ee/db/geo/migrate/20180402170913_add_missing_on_primary_to_job_artifact_registry..rb' - - 'ee/db/geo/migrate/20180405074130_add_partial_index_project_repository_verification.rb' - - 'ee/db/geo/migrate/20180412213305_add_index_to_artifact_id_on_job_artifact_registry.rb' - - 'ee/db/geo/migrate/20180419174834_add_checksum_mismatch_fields_to_project_registry.rb' - - 'ee/db/geo/migrate/20180419192603_add_indexes_to_checksum_mismatch_fields_on_project_registry.rb' - - 'ee/db/geo/migrate/20180427114641_add_repository_check_to_geo_project_registry.rb' - - 'ee/db/geo/migrate/20180510223634_set_resync_flag_for_retried_projects.rb' - - 'ee/db/geo/migrate/20180613184349_add_resync_was_scheduled_at_to_project_registry.rb' - - 'ee/db/geo/post_migrate/20180320011914_remove_last_verification_failed_columns_from_geo_project_registry.rb' - - 'ee/db/geo/post_migrate/20180320013929_remove_last_verification_at_columns_from_geo_project_registry.rb' - - 'ee/db/geo/post_migrate/20180326171626_remove_old_repository_verification_checksum_from_geo_project_registry.rb' - - 'ee/db/geo/post_migrate/20180331055706_delete_job_artifacts_from_file_registry.rb' - 'ee/lib/tasks/geo.rake' - 'ee/lib/tasks/geo/git.rake' - 'ee/lib/tasks/geo/replication.rake' diff --git a/CHANGELOG.md b/CHANGELOG.md index b5f1235d615..d0688ebf570 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,14 @@ documentation](doc/development/changelog.md) for instructions on adding your own entry. +## 13.8.3 (2021-02-05) + +### Fixed (2 changes) + +- Revert multipart URL optimization for AWS S3. !52561 +- Fix regression with old wiki image uploads. !52656 + + ## 13.8.2 (2021-02-01) ### Security (5 changes) diff --git a/GITALY_SERVER_VERSION b/GITALY_SERVER_VERSION index 5ad3363b080..22c0679fc86 100644 --- a/GITALY_SERVER_VERSION +++ b/GITALY_SERVER_VERSION @@ -1 +1 @@ -0da32707243b9c339f75cd5a1c03b00c88502f86 +a3568f3bf5dbe84c238d2666af4ec7cc438faa31 diff --git a/app/assets/stylesheets/framework/buttons.scss b/app/assets/stylesheets/framework/buttons.scss index 050c9039b2e..e1cde8594fd 100644 --- a/app/assets/stylesheets/framework/buttons.scss +++ b/app/assets/stylesheets/framework/buttons.scss @@ -348,14 +348,6 @@ } } -.btn-build { - margin-left: 10px; - - svg { - fill: $gl-text-color-secondary; - } -} - .clone-dropdown-btn a { color: $gray-700; diff --git a/app/graphql/resolvers/board_lists_resolver.rb b/app/graphql/resolvers/board_lists_resolver.rb index 7bdff8ba61f..a97ac3220d5 100644 --- a/app/graphql/resolvers/board_lists_resolver.rb +++ b/app/graphql/resolvers/board_lists_resolver.rb @@ -28,7 +28,7 @@ module Resolvers context.scoped_set!(:issue_filters, issue_filters(issue_filters)) if load_preferences?(lookahead) - List.preload_preferences_for_user(lists, context[:current_user]) + List.preload_preferences_for_user(lists, current_user) end offset_pagination(lists) @@ -39,7 +39,7 @@ module Resolvers def board_lists(id) service = ::Boards::Lists::ListService.new( board.resource_parent, - context[:current_user], + current_user, list_id: extract_list_id(id) ) @@ -47,7 +47,8 @@ module Resolvers end def load_preferences?(lookahead) - lookahead&.selection(:edges)&.selection(:node)&.selects?(:collapsed) + lookahead&.selection(:edges)&.selection(:node)&.selects?(:collapsed) || + lookahead&.selection(:nodes)&.selects?(:collapsed) end def extract_list_id(gid) diff --git a/app/graphql/resolvers/snippets/blobs_resolver.rb b/app/graphql/resolvers/snippets/blobs_resolver.rb index 868d34ae7ad..672214df7d5 100644 --- a/app/graphql/resolvers/snippets/blobs_resolver.rb +++ b/app/graphql/resolvers/snippets/blobs_resolver.rb @@ -15,13 +15,10 @@ module Resolvers required: false, description: 'Paths of the blobs.' - def resolve(**args) + def resolve(paths: []) authorize!(snippet) - return [snippet.blob] if snippet.empty_repo? - paths = Array(args.fetch(:paths, [])) - if paths.empty? snippet.blobs else diff --git a/app/graphql/types/base_field.rb b/app/graphql/types/base_field.rb index c4ce2cecd8b..f145b9d45c3 100644 --- a/app/graphql/types/base_field.rb +++ b/app/graphql/types/base_field.rb @@ -30,7 +30,7 @@ module Types def resolve_field(obj, args, ctx) ctx.schema.after_lazy(obj) do |after_obj| query_ctx = ctx.query.context - inner_obj = after_obj && after_obj.object + inner_obj = after_obj&.object ctx.schema.after_lazy(to_ruby_args(after_obj, args, ctx)) do |ruby_args| if authorized?(inner_obj, ruby_args, query_ctx) diff --git a/app/models/ci/bridge.rb b/app/models/ci/bridge.rb index ef3891908f7..ca400cebe4e 100644 --- a/app/models/ci/bridge.rb +++ b/app/models/ci/bridge.rb @@ -27,7 +27,7 @@ module Ci # rubocop:enable Cop/ActiveRecordSerialize state_machine :status do - after_transition [:created, :manual] => :pending do |bridge| + after_transition [:created, :manual, :waiting_for_resource] => :pending do |bridge| next unless bridge.downstream_project bridge.run_after_commit do @@ -156,6 +156,10 @@ module Ci false end + def any_unmet_prerequisites? + false + end + def expanded_environment_name end diff --git a/app/views/projects/artifacts/_artifact.html.haml b/app/views/projects/artifacts/_artifact.html.haml index 6c9b6ec164e..229de2f759c 100644 --- a/app/views/projects/artifacts/_artifact.html.haml +++ b/app/views/projects/artifacts/_artifact.html.haml @@ -50,12 +50,12 @@ .table-action-buttons .btn-group - if can?(current_user, :read_build, @project) - = link_to download_project_job_artifacts_path(@project, artifact.job), rel: 'nofollow', download: '', title: _('Download artifacts'), data: { placement: 'top', container: 'body' }, ref: 'tooltip', aria: { label: _('Download artifacts') }, class: 'gl-button btn btn-default btn-build has-tooltip ml-0' do - = sprite_icon('download') + = link_to download_project_job_artifacts_path(@project, artifact.job), rel: 'nofollow', download: '', title: _('Download artifacts'), data: { placement: 'top', container: 'body' }, ref: 'tooltip', aria: { label: _('Download artifacts') }, class: 'gl-button btn btn-default btn-icon has-tooltip' do + = sprite_icon('download', css_class: 'gl-icon') - = link_to browse_project_job_artifacts_path(@project, artifact.job), rel: 'nofollow', title: _('Browse artifacts'), data: { placement: 'top', container: 'body' }, ref: 'tooltip', aria: { label: _('Browse artifacts') }, class: 'gl-button btn btn-default btn-build has-tooltip' do - = sprite_icon('folder-open') + = link_to browse_project_job_artifacts_path(@project, artifact.job), rel: 'nofollow', title: _('Browse artifacts'), data: { placement: 'top', container: 'body' }, ref: 'tooltip', aria: { label: _('Browse artifacts') }, class: 'gl-button btn btn-default btn-icon has-tooltip' do + = sprite_icon('folder-open', css_class: 'gl-icon') - if can?(current_user, :destroy_artifacts, @project) - = link_to project_artifact_path(@project, artifact), data: { placement: 'top', container: 'body', confirm: _('Are you sure you want to delete these artifacts?') }, method: :delete, title: _('Delete artifacts'), ref: 'tooltip', aria: { label: _('Delete artifacts') }, class: 'gl-button btn btn-danger has-tooltip' do - = sprite_icon('remove') + = link_to project_artifact_path(@project, artifact), data: { placement: 'top', container: 'body', confirm: _('Are you sure you want to delete these artifacts?') }, method: :delete, title: _('Delete artifacts'), ref: 'tooltip', aria: { label: _('Delete artifacts') }, class: 'gl-button btn btn-danger btn-icon has-tooltip' do + = sprite_icon('remove', css_class: 'gl-icon') diff --git a/app/views/projects/ci/builds/_build.html.haml b/app/views/projects/ci/builds/_build.html.haml index b7a265c0b17..0cc595de7be 100644 --- a/app/views/projects/ci/builds/_build.html.haml +++ b/app/views/projects/ci/builds/_build.html.haml @@ -98,35 +98,35 @@ %td .gl-display-flex - - if can?(current_user, :read_job_artifacts, job) && job.artifacts? - = link_to download_project_job_artifacts_path(job.project, job), rel: 'nofollow', download: '', title: _('Download artifacts'), class: 'btn btn-build btn-default gl-button btn-icon btn-svg' do - = sprite_icon('download') - - if can?(current_user, :update_build, job) - - if job.active? - = link_to cancel_project_job_path(job.project, job, continue: { to: request.fullpath }), method: :post, title: _('Cancel'), class: 'btn gl-button btn-build btn-default' do - = sprite_icon('close') - - elsif job.scheduled? - .btn-group - .btn.gl-button.btn-default{ disabled: true } - = sprite_icon('planning') + .btn-group + - if can?(current_user, :read_job_artifacts, job) && job.artifacts? + = link_to download_project_job_artifacts_path(job.project, job), rel: 'nofollow', download: '', title: _('Download artifacts'), class: 'gl-button btn btn-default btn-icon' do + = sprite_icon('download', css_class: 'gl-icon') + - if can?(current_user, :update_build, job) + - if job.active? + = link_to cancel_project_job_path(job.project, job, continue: { to: request.fullpath }), method: :post, title: _('Cancel'), class: 'gl-button btn btn-default btn-icon' do + = sprite_icon('close', css_class: 'gl-icon') + - elsif job.scheduled? + .gl-button.btn.btn-default.btn-icon.disabled{ disabled: true } + = sprite_icon('planning', css_class: 'gl-icon') %time.js-remaining-time{ datetime: job.scheduled_at.utc.iso8601 } = duration_in_numbers(job.execute_in) - confirmation_message = s_("DelayedJobs|Are you sure you want to run %{job_name} immediately? This job will run automatically after it's timer finishes.") % { job_name: job.name } = link_to play_project_job_path(job.project, job, return_to: request.original_url), method: :post, title: s_('DelayedJobs|Start now'), - class: 'btn gl-button btn-default btn-build has-tooltip', + class: 'gl-button btn btn-default btn-icon has-tooltip', data: { confirm: confirmation_message } do - = sprite_icon('play') + = sprite_icon('play', css_class: 'gl-icon') = link_to unschedule_project_job_path(job.project, job, return_to: request.original_url), method: :post, title: s_('DelayedJobs|Unschedule'), - class: 'btn gl-button btn-default btn-build has-tooltip' do - = sprite_icon('time-out') - - elsif allow_retry - - if job.playable? && !admin && can?(current_user, :update_build, job) - = link_to play_project_job_path(job.project, job, return_to: request.original_url), method: :post, title: _('Play'), class: 'btn gl-button btn-build btn-default' do - = custom_icon('icon_play') - - elsif job.retryable? - = link_to retry_project_job_path(job.project, job, return_to: request.original_url), method: :post, title: _('Retry'), class: 'btn btn-build gl-button btn-icon btn-default' do - = sprite_icon('repeat', css_class: 'gl-icon') + class: 'gl-button btn btn-default btn-icon has-tooltip' do + = sprite_icon('time-out', css_class: 'gl-icon') + - elsif allow_retry + - if job.playable? && !admin && can?(current_user, :update_build, job) + = link_to play_project_job_path(job.project, job, return_to: request.original_url), method: :post, title: _('Play'), class: 'gl-button btn btn-default btn-icon' do + = sprite_icon('play', css_class: 'gl-icon') + - elsif job.retryable? + = link_to retry_project_job_path(job.project, job, return_to: request.original_url), method: :post, title: _('Retry'), class: 'gl-button btn btn-default btn-icon' do + = sprite_icon('repeat', css_class: 'gl-icon') diff --git a/app/views/projects/deployments/_rollback.haml b/app/views/projects/deployments/_rollback.haml index c5e884473ff..78972a5b7b9 100644 --- a/app/views/projects/deployments/_rollback.haml +++ b/app/views/projects/deployments/_rollback.haml @@ -1,8 +1,8 @@ - if deployment.deployable && can?(current_user, :create_deployment, deployment) - tooltip = deployment.last? ? s_('Environments|Re-deploy to environment') : s_('Environments|Rollback environment') - = button_tag class: 'btn gl-button btn-default btn-build has-tooltip', type: 'button', data: { toggle: 'modal', target: "#confirm-rollback-modal-#{deployment.id}" }, title: tooltip do + = button_tag class: 'gl-button btn btn-default btn-icon has-tooltip', type: 'button', data: { toggle: 'modal', target: "#confirm-rollback-modal-#{deployment.id}" }, title: tooltip do - if deployment.last? - = sprite_icon('repeat') + = sprite_icon('repeat', css_class: 'gl-icon') - else - = sprite_icon('redo') + = sprite_icon('redo', css_class: 'gl-icon') = render 'projects/deployments/confirm_rollback_modal', deployment: deployment diff --git a/app/views/projects/pipelines/_with_tabs.html.haml b/app/views/projects/pipelines/_with_tabs.html.haml index 5f99396d744..0787cf5b735 100644 --- a/app/views/projects/pipelines/_with_tabs.html.haml +++ b/app/views/projects/pipelines/_with_tabs.html.haml @@ -67,7 +67,7 @@ = build.present.callout_failure_message %td.responsive-table-cell.build-actions - if can?(current_user, :update_build, job) && job.retryable? - = link_to retry_project_job_path(build.project, build, return_to: request.original_url), method: :post, title: _('Retry'), class: 'btn btn-build gl-button btn-icon btn-default' do + = link_to retry_project_job_path(build.project, build, return_to: request.original_url), method: :post, title: _('Retry'), class: 'gl-button btn btn-default btn-icon' do = sprite_icon('repeat', css_class: 'gl-icon') - if can?(current_user, :read_build, job) %tr.build-trace-row.responsive-table-border-end diff --git a/changelogs/unreleased/300263-remove-btn-build-class.yml b/changelogs/unreleased/300263-remove-btn-build-class.yml new file mode 100644 index 00000000000..d952957e639 --- /dev/null +++ b/changelogs/unreleased/300263-remove-btn-build-class.yml @@ -0,0 +1,6 @@ +--- +title: Align action buttons in jobs page. Reduce icon buttons width in jobs, artifacts + and environment pages. +merge_request: 53155 +author: +type: changed diff --git a/changelogs/unreleased/fj-fix-regression-old-uploads.yml b/changelogs/unreleased/fj-fix-regression-old-uploads.yml deleted file mode 100644 index 5e2cc5fcf9d..00000000000 --- a/changelogs/unreleased/fj-fix-regression-old-uploads.yml +++ /dev/null @@ -1,5 +0,0 @@ ---- -title: Fix regression with old wiki image uploads -merge_request: 52656 -author: -type: fixed diff --git a/changelogs/unreleased/qmnguyen0711-640-sidekiq-jobretry-skip-should-not-mask-the-underlying-sid.yml b/changelogs/unreleased/qmnguyen0711-640-sidekiq-jobretry-skip-should-not-mask-the-underlying-sid.yml new file mode 100644 index 00000000000..21c991359ae --- /dev/null +++ b/changelogs/unreleased/qmnguyen0711-640-sidekiq-jobretry-skip-should-not-mask-the-underlying-sid.yml @@ -0,0 +1,5 @@ +--- +title: Unwrap Sidekiq exceptions and jobs in the structured logs and metrics +merge_request: 53006 +author: +type: changed diff --git a/changelogs/unreleased/sh-revert-multipart-upload-optimization.yml b/changelogs/unreleased/sh-revert-multipart-upload-optimization.yml deleted file mode 100644 index ff3fa4f8919..00000000000 --- a/changelogs/unreleased/sh-revert-multipart-upload-optimization.yml +++ /dev/null @@ -1,5 +0,0 @@ ---- -title: Revert multipart URL optimization for AWS S3 -merge_request: 52561 -author: -type: fixed diff --git a/changelogs/unreleased/support-ci-resource-group-in-cross-project-pipeline.yml b/changelogs/unreleased/support-ci-resource-group-in-cross-project-pipeline.yml new file mode 100644 index 00000000000..b34a6ce32bc --- /dev/null +++ b/changelogs/unreleased/support-ci-resource-group-in-cross-project-pipeline.yml @@ -0,0 +1,5 @@ +--- +title: Pipeline-level concurrency control with Cross-Project/Parent-Child pipelines +merge_request: 53007 +author: +type: added diff --git a/config/initializers/sidekiq.rb b/config/initializers/sidekiq.rb index 43beae3f50d..19c5e4df854 100644 --- a/config/initializers/sidekiq.rb +++ b/config/initializers/sidekiq.rb @@ -31,9 +31,9 @@ Sidekiq.configure_server do |config| Sidekiq.logger.formatter = Gitlab::SidekiqLogging::JSONFormatter.new config.options[:job_logger] = Gitlab::SidekiqLogging::StructuredLogger - # Remove the default-provided handler + # Remove the default-provided handler. The exception is logged inside + # Gitlab::SidekiqLogging::StructuredLogger config.error_handlers.reject! { |handler| handler.is_a?(Sidekiq::ExceptionHandler::Logger) } - config.error_handlers << Gitlab::SidekiqLogging::ExceptionHandler.new end config.redis = queues_config_hash diff --git a/db/migrate/20190325080727_truncate_user_fullname.rb b/db/migrate/20190325080727_truncate_user_fullname.rb index e5f88671eef..29255d173d8 100644 --- a/db/migrate/20190325080727_truncate_user_fullname.rb +++ b/db/migrate/20190325080727_truncate_user_fullname.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + # rubocop:disable Migration/UpdateLargeTable class TruncateUserFullname < ActiveRecord::Migration[5.0] include Gitlab::Database::MigrationHelpers diff --git a/db/migrate/20190828110802_add_not_null_constraints_to_prometheus_metrics_y_label_and_unit.rb b/db/migrate/20190828110802_add_not_null_constraints_to_prometheus_metrics_y_label_and_unit.rb index 6f3650ca966..1b238907af6 100644 --- a/db/migrate/20190828110802_add_not_null_constraints_to_prometheus_metrics_y_label_and_unit.rb +++ b/db/migrate/20190828110802_add_not_null_constraints_to_prometheus_metrics_y_label_and_unit.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + class AddNotNullConstraintsToPrometheusMetricsYLabelAndUnit < ActiveRecord::Migration[5.2] DOWNTIME = false diff --git a/db/migrate/20190828172831_create_package_tag.rb b/db/migrate/20190828172831_create_package_tag.rb index 3d26b7ce602..a70b71db51d 100644 --- a/db/migrate/20190828172831_create_package_tag.rb +++ b/db/migrate/20190828172831_create_package_tag.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + # frozen_string_literal: true . class CreatePackageTag < ActiveRecord::Migration[5.2] diff --git a/db/migrate/20191003195218_add_pendo_enabled_to_application_settings.rb b/db/migrate/20191003195218_add_pendo_enabled_to_application_settings.rb index 29ae831d4f4..2d937327ca1 100644 --- a/db/migrate/20191003195218_add_pendo_enabled_to_application_settings.rb +++ b/db/migrate/20191003195218_add_pendo_enabled_to_application_settings.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + class AddPendoEnabledToApplicationSettings < ActiveRecord::Migration[5.2] include Gitlab::Database::MigrationHelpers diff --git a/db/migrate/20191003195620_add_pendo_url_to_application_settings.rb b/db/migrate/20191003195620_add_pendo_url_to_application_settings.rb index 6763cb5544c..cbeb1ef5186 100644 --- a/db/migrate/20191003195620_add_pendo_url_to_application_settings.rb +++ b/db/migrate/20191003195620_add_pendo_url_to_application_settings.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + class AddPendoUrlToApplicationSettings < ActiveRecord::Migration[5.2] include Gitlab::Database::MigrationHelpers diff --git a/db/migrate/20191120200015_add_index_to_grafana_integrations.rb b/db/migrate/20191120200015_add_index_to_grafana_integrations.rb index 87292c86e97..c67f6850baf 100644 --- a/db/migrate/20191120200015_add_index_to_grafana_integrations.rb +++ b/db/migrate/20191120200015_add_index_to_grafana_integrations.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + class AddIndexToGrafanaIntegrations < ActiveRecord::Migration[5.2] include Gitlab::Database::MigrationHelpers diff --git a/db/migrate/20200229171700_create_custom_emojis.rb b/db/migrate/20200229171700_create_custom_emojis.rb index 1a60d7c8a63..f0574831f33 100644 --- a/db/migrate/20200229171700_create_custom_emojis.rb +++ b/db/migrate/20200229171700_create_custom_emojis.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + class CreateCustomEmojis < ActiveRecord::Migration[5.2] include Gitlab::Database::MigrationHelpers DOWNTIME = false diff --git a/db/migrate/20201004163918_remove_project_id_and_id_index_from_vulnerabilities_table.rb b/db/migrate/20201004163918_remove_project_id_and_id_index_from_vulnerabilities_table.rb index 7a59e706bf3..73028b6b795 100644 --- a/db/migrate/20201004163918_remove_project_id_and_id_index_from_vulnerabilities_table.rb +++ b/db/migrate/20201004163918_remove_project_id_and_id_index_from_vulnerabilities_table.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + class RemoveProjectIdAndIdIndexFromVulnerabilitiesTable < ActiveRecord::Migration[6.0] include Gitlab::Database::MigrationHelpers diff --git a/db/migrate/20201214000000_change_mr_allow_maintainer_to_push_default.rb b/db/migrate/20201214000000_change_mr_allow_maintainer_to_push_default.rb index 0d97d54f3e4..47eec16807b 100644 --- a/db/migrate/20201214000000_change_mr_allow_maintainer_to_push_default.rb +++ b/db/migrate/20201214000000_change_mr_allow_maintainer_to_push_default.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + class ChangeMrAllowMaintainerToPushDefault < ActiveRecord::Migration[6.0] include Gitlab::Database::MigrationHelpers diff --git a/db/optional_migrations/composite_primary_keys.rb b/db/optional_migrations/composite_primary_keys.rb index 1fcb9664ff6..13bc58b8692 100644 --- a/db/optional_migrations/composite_primary_keys.rb +++ b/db/optional_migrations/composite_primary_keys.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + # This migration adds a primary key constraint to tables # that only have a composite unique key. # diff --git a/db/post_migrate/20190214112022_schedule_sync_issuables_state_id.rb b/db/post_migrate/20190214112022_schedule_sync_issuables_state_id.rb index c05ad73d616..34536e22bbf 100644 --- a/db/post_migrate/20190214112022_schedule_sync_issuables_state_id.rb +++ b/db/post_migrate/20190214112022_schedule_sync_issuables_state_id.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + class ScheduleSyncIssuablesStateId < ActiveRecord::Migration[5.0] # This migration schedules the sync of state_id for issues and merge requests # which are converting the state column from string to integer. diff --git a/db/post_migrate/20201030121314_schedule_update_existing_users_that_require_two_factor_auth.rb b/db/post_migrate/20201030121314_schedule_update_existing_users_that_require_two_factor_auth.rb index 2cb7c9c5250..6a031c28ed7 100644 --- a/db/post_migrate/20201030121314_schedule_update_existing_users_that_require_two_factor_auth.rb +++ b/db/post_migrate/20201030121314_schedule_update_existing_users_that_require_two_factor_auth.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + # # frozen_string_literal: true class ScheduleUpdateExistingUsersThatRequireTwoFactorAuth < ActiveRecord::Migration[6.0] diff --git a/doc/ci/lint.md b/doc/ci/lint.md index 969f1fbc47b..3888a750d6a 100644 --- a/doc/ci/lint.md +++ b/doc/ci/lint.md @@ -1,6 +1,6 @@ --- -stage: none -group: unassigned +stage: Verify +group: Pipeline Authoring info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- <!-- markdownlint-disable MD044 --> diff --git a/doc/ci/yaml/README.md b/doc/ci/yaml/README.md index 750ecd91085..f3ad2932e9a 100644 --- a/doc/ci/yaml/README.md +++ b/doc/ci/yaml/README.md @@ -3924,6 +3924,60 @@ It can't start or end with `/`. For more information, see [Deployments Safety](../environments/deployment_safety.md). +#### Pipeline-level concurrency control with Cross-Project/Parent-Child pipelines + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/39057) in GitLab 13.9. + +You can define `resource_group` for downstream pipelines that are sensitive to concurrent +executions. The [`trigger` keyword](#trigger) can trigger downstream pipelines. The +[`resource_group` keyword](#resource_group) can co-exist with it. This is useful to control the +concurrency for deployment pipelines, while running non-sensitive jobs concurrently. + +This example has two pipeline configurations in a project. When a pipeline starts running, +non-sensitive jobs are executed first and aren't affected by concurrent executions in other +pipelines. However, GitLab ensures that there are no other deployment pipelines running before +triggering a deployment (child) pipeline. If other deployment pipelines are running, GitLab waits +until those pipelines finish before running another one. + +```yaml +# .gitlab-ci.yml (parent pipeline) + +build: + stage: build + script: echo "Building..." + +test: + stage: test + script: echo "Testing..." + +deploy: + stage: deploy + trigger: + include: deploy.gitlab-ci.yml + strategy: depend + resource_group: AWS-production +``` + +```yaml +# deploy.gitlab-ci.yml (child pipeline) + +stages: + - provision + - deploy + +provision: + stage: provision + script: echo "Provisioning..." + +deployment: + stage: deploy + script: echo "Deploying..." +``` + +Note that you must define [`strategy: depend`](#linking-pipelines-with-triggerstrategy) +with the `trigger` keyword. This ensures that the lock isn't released until the downstream pipeline +finishes. + ### `release` > [Introduced](https://gitlab.com/gitlab-org/gitlab/merge_requests/19298) in GitLab 13.2. diff --git a/lib/gitlab/ci/pipeline/seed/build.rb b/lib/gitlab/ci/pipeline/seed/build.rb index 48411af6f38..3770bb4b328 100644 --- a/lib/gitlab/ci/pipeline/seed/build.rb +++ b/lib/gitlab/ci/pipeline/seed/build.rb @@ -73,17 +73,28 @@ module Gitlab def to_resource strong_memoize(:resource) do - if bridge? - ::Ci::Bridge.new(attributes) - else - ::Ci::Build.new(attributes).tap do |build| - build.assign_attributes(self.class.environment_attributes_for(build)) - build.resource_group = Seed::Build::ResourceGroup.new(build, @resource_group_key).to_resource - end + processable = initialize_processable + assign_resource_group(processable) + processable + end + end + + def initialize_processable + if bridge? + ::Ci::Bridge.new(attributes) + else + ::Ci::Build.new(attributes).tap do |build| + build.assign_attributes(self.class.environment_attributes_for(build)) end end end + def assign_resource_group(processable) + processable.resource_group = + Seed::Processable::ResourceGroup.new(processable, @resource_group_key) + .to_resource + end + def self.environment_attributes_for(build) return {} unless build.has_environment? diff --git a/lib/gitlab/ci/pipeline/seed/build/resource_group.rb b/lib/gitlab/ci/pipeline/seed/processable/resource_group.rb index 794bd06be25..f8ea6d4184c 100644 --- a/lib/gitlab/ci/pipeline/seed/build/resource_group.rb +++ b/lib/gitlab/ci/pipeline/seed/processable/resource_group.rb @@ -4,7 +4,7 @@ module Gitlab module Ci module Pipeline module Seed - class Build + module Processable class ResourceGroup < Seed::Base include Gitlab::Utils::StrongMemoize diff --git a/lib/gitlab/ci/status/bridge/factory.rb b/lib/gitlab/ci/status/bridge/factory.rb index b9bd66cee71..4d5a94a3beb 100644 --- a/lib/gitlab/ci/status/bridge/factory.rb +++ b/lib/gitlab/ci/status/bridge/factory.rb @@ -8,6 +8,7 @@ module Gitlab def self.extended_statuses [[Status::Bridge::Failed], [Status::Bridge::Manual], + [Status::Bridge::WaitingForResource], [Status::Bridge::Play], [Status::Bridge::Action]] end diff --git a/lib/gitlab/ci/status/bridge/waiting_for_resource.rb b/lib/gitlab/ci/status/bridge/waiting_for_resource.rb new file mode 100644 index 00000000000..d2c8f71a609 --- /dev/null +++ b/lib/gitlab/ci/status/bridge/waiting_for_resource.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Status + module Bridge + class WaitingForResource < Status::Processable::WaitingForResource + end + end + end + end +end diff --git a/lib/gitlab/ci/status/build/waiting_for_resource.rb b/lib/gitlab/ci/status/build/waiting_for_resource.rb index 008e6a17bdd..5dcc060a990 100644 --- a/lib/gitlab/ci/status/build/waiting_for_resource.rb +++ b/lib/gitlab/ci/status/build/waiting_for_resource.rb @@ -4,22 +4,7 @@ module Gitlab module Ci module Status module Build - class WaitingForResource < Status::Extended - ## - # TODO: image is shared with 'pending' - # until we get a dedicated one - # - def illustration - { - image: 'illustrations/pending_job_empty.svg', - size: 'svg-430', - title: _('This job is waiting for resource: ') + subject.resource_group.key - } - end - - def self.matches?(build, _) - build.waiting_for_resource? - end + class WaitingForResource < Status::Processable::WaitingForResource end end end diff --git a/lib/gitlab/ci/status/processable/waiting_for_resource.rb b/lib/gitlab/ci/status/processable/waiting_for_resource.rb new file mode 100644 index 00000000000..c9b1dd795d0 --- /dev/null +++ b/lib/gitlab/ci/status/processable/waiting_for_resource.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Status + module Processable + class WaitingForResource < Status::Extended + ## + # TODO: image is shared with 'pending' + # until we get a dedicated one + # + def illustration + { + image: 'illustrations/pending_job_empty.svg', + size: 'svg-430', + title: _('This job is waiting for resource: ') + subject.resource_group.key + } + end + + def self.matches?(processable, _) + processable.waiting_for_resource? + end + end + end + end + end +end diff --git a/lib/gitlab/sidekiq_death_handler.rb b/lib/gitlab/sidekiq_death_handler.rb index f86d9f17b5f..91bfc7dca80 100644 --- a/lib/gitlab/sidekiq_death_handler.rb +++ b/lib/gitlab/sidekiq_death_handler.rb @@ -6,7 +6,7 @@ module Gitlab include ::Gitlab::SidekiqMiddleware::MetricsHelper def handler(job, _exception) - labels = create_labels(job['class'].constantize, job['queue']) + labels = create_labels(job['class'].constantize, job['queue'], job) counter.increment(labels) end diff --git a/lib/gitlab/sidekiq_logging/exception_handler.rb b/lib/gitlab/sidekiq_logging/exception_handler.rb deleted file mode 100644 index 8ae6addc2c6..00000000000 --- a/lib/gitlab/sidekiq_logging/exception_handler.rb +++ /dev/null @@ -1,27 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module SidekiqLogging - class ExceptionHandler - def call(job_exception, context) - data = { - error_class: job_exception.class.name, - error_message: job_exception.message - } - - if context.is_a?(Hash) - data.merge!(context) - # correlation_id, jid, and class are available inside the job - # Hash, so promote these arguments to the root tree so that - # can be searched alongside other Sidekiq log messages. - job_data = data.delete(:job) - data.merge!(job_data) if job_data.present? - end - - data[:error_backtrace] = Rails.backtrace_cleaner.clean(job_exception.backtrace) if job_exception.backtrace.present? - - Sidekiq.logger.warn(data) - end - end - end -end diff --git a/lib/gitlab/sidekiq_logging/logs_jobs.rb b/lib/gitlab/sidekiq_logging/logs_jobs.rb index dc81c34c4d0..6f8cc1c60e9 100644 --- a/lib/gitlab/sidekiq_logging/logs_jobs.rb +++ b/lib/gitlab/sidekiq_logging/logs_jobs.rb @@ -12,6 +12,7 @@ module Gitlab # Error information from the previous try is in the payload for # displaying in the Sidekiq UI, but is very confusing in logs! job = job.except('error_backtrace', 'error_class', 'error_message') + job['class'] = job.delete('wrapped') if job['wrapped'].present? # Add process id params job['pid'] = ::Process.pid diff --git a/lib/gitlab/sidekiq_logging/structured_logger.rb b/lib/gitlab/sidekiq_logging/structured_logger.rb index f7b826ba648..741ebd2ae7f 100644 --- a/lib/gitlab/sidekiq_logging/structured_logger.rb +++ b/lib/gitlab/sidekiq_logging/structured_logger.rb @@ -18,6 +18,17 @@ module Gitlab yield Sidekiq.logger.info log_job_done(job, started_time, base_payload) + rescue Sidekiq::JobRetry::Handled => job_exception + # Sidekiq::JobRetry::Handled is raised by the internal Sidekiq + # processor. It is a wrapper around real exception indicating an + # exception is already handled by the Job retrier. The real exception + # should be unwrapped before being logged. + # + # For more information: + # https://github.com/mperham/sidekiq/blob/v5.2.7/lib/sidekiq/processor.rb#L173 + Sidekiq.logger.warn log_job_done(job, started_time, base_payload, job_exception.cause || job_exception) + + raise rescue => job_exception Sidekiq.logger.warn log_job_done(job, started_time, base_payload, job_exception) @@ -68,6 +79,7 @@ module Gitlab payload['job_status'] = 'fail' payload['error_message'] = job_exception.message payload['error_class'] = job_exception.class.name + add_exception_backtrace!(job_exception, payload) else payload['message'] = "#{message}: done: #{payload['duration_s']} sec" payload['job_status'] = 'done' @@ -88,6 +100,12 @@ module Gitlab payload['completed_at'] = Time.now.utc.to_f end + def add_exception_backtrace!(job_exception, payload) + return if job_exception.backtrace.blank? + + payload['error_backtrace'] = Rails.backtrace_cleaner.clean(job_exception.backtrace) + end + def elapsed(t0) t1 = get_time { diff --git a/lib/gitlab/sidekiq_middleware/client_metrics.rb b/lib/gitlab/sidekiq_middleware/client_metrics.rb index 7ee8a623d30..6bc08a97c07 100644 --- a/lib/gitlab/sidekiq_middleware/client_metrics.rb +++ b/lib/gitlab/sidekiq_middleware/client_metrics.rb @@ -11,10 +11,10 @@ module Gitlab @metrics = init_metrics end - def call(worker_class, _job, queue, _redis_pool) + def call(worker_class, job, queue, _redis_pool) # worker_class can either be the string or class of the worker being enqueued. worker_class = worker_class.safe_constantize if worker_class.respond_to?(:safe_constantize) - labels = create_labels(worker_class, queue) + labels = create_labels(worker_class, queue, job) @metrics.fetch(ENQUEUED).increment(labels, 1) diff --git a/lib/gitlab/sidekiq_middleware/metrics_helper.rb b/lib/gitlab/sidekiq_middleware/metrics_helper.rb index 5c1ce2b98e8..60e79ee1188 100644 --- a/lib/gitlab/sidekiq_middleware/metrics_helper.rb +++ b/lib/gitlab/sidekiq_middleware/metrics_helper.rb @@ -8,9 +8,11 @@ module Gitlab private - def create_labels(worker_class, queue) + def create_labels(worker_class, queue, job) + worker_name = (job['wrapped'].presence || worker_class).to_s + labels = { queue: queue.to_s, - worker: worker_class.to_s, + worker: worker_name, urgency: "", external_dependencies: FALSE_LABEL, feature_category: "", diff --git a/lib/gitlab/sidekiq_middleware/server_metrics.rb b/lib/gitlab/sidekiq_middleware/server_metrics.rb index 7f3048f4c6e..4ab8d313ad8 100644 --- a/lib/gitlab/sidekiq_middleware/server_metrics.rb +++ b/lib/gitlab/sidekiq_middleware/server_metrics.rb @@ -20,7 +20,7 @@ module Gitlab # in metrics and can use them in the `ThreadsSampler` for setting a label Thread.current.name ||= Gitlab::Metrics::Samplers::ThreadsSampler::SIDEKIQ_WORKER_THREAD_NAME - labels = create_labels(worker.class, queue) + labels = create_labels(worker.class, queue, job) queue_duration = ::Gitlab::InstrumentationHelper.queue_duration_for_job(job) @metrics[:sidekiq_jobs_queue_duration_seconds].observe(labels, queue_duration) if queue_duration diff --git a/spec/factories/ci/bridge.rb b/spec/factories/ci/bridge.rb index 7258b3367d3..6cbcabca7ab 100644 --- a/spec/factories/ci/bridge.rb +++ b/spec/factories/ci/bridge.rb @@ -1,17 +1,10 @@ # frozen_string_literal: true FactoryBot.define do - factory :ci_bridge, class: 'Ci::Bridge' do + factory :ci_bridge, class: 'Ci::Bridge', parent: :ci_processable do name { 'bridge' } - stage { 'test' } - stage_idx { 0 } - ref { 'master' } - tag { false } created_at { '2013-10-29 09:50:00 CET' } status { :created } - scheduling_type { 'stage' } - - pipeline factory: :ci_pipeline trait :variables do yaml_variables do diff --git a/spec/factories/ci/builds.rb b/spec/factories/ci/builds.rb index c85918a3187..0f4216708d2 100644 --- a/spec/factories/ci/builds.rb +++ b/spec/factories/ci/builds.rb @@ -3,15 +3,10 @@ include ActionDispatch::TestProcess FactoryBot.define do - factory :ci_build, class: 'Ci::Build' do + factory :ci_build, class: 'Ci::Build', parent: :ci_processable do name { 'test' } - stage { 'test' } - stage_idx { 0 } - ref { 'master' } - tag { false } add_attribute(:protected) { false } created_at { 'Di 29. Okt 09:50:00 CET 2013' } - scheduling_type { 'stage' } pending options do @@ -28,7 +23,6 @@ FactoryBot.define do ] end - pipeline factory: :ci_pipeline project { pipeline.project } trait :degenerated do @@ -79,10 +73,6 @@ FactoryBot.define do status { 'created' } end - trait :waiting_for_resource do - status { 'waiting_for_resource' } - end - trait :preparing do status { 'preparing' } end @@ -213,14 +203,6 @@ FactoryBot.define do trigger_request factory: :ci_trigger_request end - trait :resource_group do - waiting_for_resource_at { 5.minutes.ago } - - after(:build) do |build, evaluator| - build.resource_group = create(:ci_resource_group, project: build.project) - end - end - trait :with_deployment do after(:build) do |build, evaluator| ## diff --git a/spec/factories/ci/processable.rb b/spec/factories/ci/processable.rb new file mode 100644 index 00000000000..0550f4c23fa --- /dev/null +++ b/spec/factories/ci/processable.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +FactoryBot.define do + factory :ci_processable, class: 'Ci::Processable' do + name { 'processable' } + stage { 'test' } + stage_idx { 0 } + ref { 'master' } + tag { false } + pipeline factory: :ci_pipeline + project { pipeline.project } + scheduling_type { 'stage' } + + trait :waiting_for_resource do + status { 'waiting_for_resource' } + end + + trait :resource_group do + waiting_for_resource_at { 5.minutes.ago } + + after(:build) do |processable, evaluator| + processable.resource_group = create(:ci_resource_group, project: processable.project) + end + end + end +end diff --git a/spec/features/projects/pipelines/pipeline_spec.rb b/spec/features/projects/pipelines/pipeline_spec.rb index e1d68a3f12e..a72762e5f7b 100644 --- a/spec/features/projects/pipelines/pipeline_spec.rb +++ b/spec/features/projects/pipelines/pipeline_spec.rb @@ -846,6 +846,28 @@ RSpec.describe 'Pipeline', :js do end end end + + context 'when deploy job is a bridge to trigger a downstream pipeline' do + let!(:deploy_job) do + create(:ci_bridge, :created, stage: 'deploy', name: 'deploy', + stage_idx: 2, pipeline: pipeline, project: project, resource_group: resource_group) + end + + it 'shows deploy job as waiting for resource' do + subject + + within('.pipeline-header-container') do + expect(page).to have_content('waiting') + end + + within('.pipeline-graph') do + within '.stage-column:nth-child(2)' do + expect(page).to have_content('deploy') + expect(page).to have_css('.ci-status-icon-waiting-for-resource') + end + end + end + end end end end diff --git a/spec/lib/gitlab/ci/pipeline/seed/build_spec.rb b/spec/lib/gitlab/ci/pipeline/seed/build_spec.rb index cf020fc343c..0efc7484699 100644 --- a/spec/lib/gitlab/ci/pipeline/seed/build_spec.rb +++ b/spec/lib/gitlab/ci/pipeline/seed/build_spec.rb @@ -383,14 +383,25 @@ RSpec.describe Gitlab::Ci::Pipeline::Seed::Build do end context 'when job is a bridge' do - let(:attributes) do + let(:base_attributes) do { name: 'rspec', ref: 'master', options: { trigger: 'my/project' }, scheduling_type: :stage } end + let(:attributes) { base_attributes } + it { is_expected.to be_a(::Ci::Bridge) } it { is_expected.to be_valid } + + context 'when job belongs to a resource group' do + let(:attributes) { base_attributes.merge(resource_group_key: 'iOS') } + + it 'returns a job with resource group' do + expect(subject.resource_group).not_to be_nil + expect(subject.resource_group.key).to eq('iOS') + end + end end it 'memoizes a resource object' do diff --git a/spec/lib/gitlab/ci/pipeline/seed/build/resource_group_spec.rb b/spec/lib/gitlab/ci/pipeline/seed/processable/resource_group_spec.rb index 8fcc242ba5f..b7260599de2 100644 --- a/spec/lib/gitlab/ci/pipeline/seed/build/resource_group_spec.rb +++ b/spec/lib/gitlab/ci/pipeline/seed/processable/resource_group_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Ci::Pipeline::Seed::Build::ResourceGroup do +RSpec.describe Gitlab::Ci::Pipeline::Seed::Processable::ResourceGroup do let_it_be(:project) { create(:project) } let(:job) { build(:ci_build, project: project) } let(:seed) { described_class.new(job, resource_group_key) } diff --git a/spec/lib/gitlab/ci/status/bridge/factory_spec.rb b/spec/lib/gitlab/ci/status/bridge/factory_spec.rb index d27bb98ba9a..6081f104e42 100644 --- a/spec/lib/gitlab/ci/status/bridge/factory_spec.rb +++ b/spec/lib/gitlab/ci/status/bridge/factory_spec.rb @@ -117,14 +117,31 @@ RSpec.describe Gitlab::Ci::Status::Bridge::Factory do end end + context 'when bridge is waiting for resource' do + let(:bridge) { create_bridge(:waiting_for_resource, :resource_group) } + + it 'matches correct core status' do + expect(factory.core_status).to be_a Gitlab::Ci::Status::WaitingForResource + end + + it 'fabricates status with correct details' do + expect(status.text).to eq 'waiting' + expect(status.group).to eq 'waiting-for-resource' + expect(status.icon).to eq 'status_pending' + expect(status.favicon).to eq 'favicon_pending' + expect(status.illustration).to include(:image, :size, :title) + expect(status).not_to have_details + end + end + private - def create_bridge(trait) + def create_bridge(*traits) upstream_project = create(:project, :repository) downstream_project = create(:project, :repository) upstream_pipeline = create(:ci_pipeline, :running, project: upstream_project) trigger = { trigger: { project: downstream_project.full_path, branch: 'feature' } } - create(:ci_bridge, trait, options: trigger, pipeline: upstream_pipeline) + create(:ci_bridge, *traits, options: trigger, pipeline: upstream_pipeline) end end diff --git a/spec/lib/gitlab/ci/status/bridge/waiting_for_resource_spec.rb b/spec/lib/gitlab/ci/status/bridge/waiting_for_resource_spec.rb new file mode 100644 index 00000000000..3e19df28d83 --- /dev/null +++ b/spec/lib/gitlab/ci/status/bridge/waiting_for_resource_spec.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +require 'fast_spec_helper' + +RSpec.describe Gitlab::Ci::Status::Bridge::WaitingForResource do + it { expect(described_class).to be < Gitlab::Ci::Status::Processable::WaitingForResource } +end diff --git a/spec/lib/gitlab/ci/status/build/waiting_for_resource_spec.rb b/spec/lib/gitlab/ci/status/build/waiting_for_resource_spec.rb new file mode 100644 index 00000000000..44bd5a8611a --- /dev/null +++ b/spec/lib/gitlab/ci/status/build/waiting_for_resource_spec.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +require 'fast_spec_helper' + +RSpec.describe Gitlab::Ci::Status::Build::WaitingForResource do + it { expect(described_class).to be < Gitlab::Ci::Status::Processable::WaitingForResource } +end diff --git a/spec/lib/gitlab/ci/status/processable/waiting_for_resource_spec.rb b/spec/lib/gitlab/ci/status/processable/waiting_for_resource_spec.rb new file mode 100644 index 00000000000..91a9724d043 --- /dev/null +++ b/spec/lib/gitlab/ci/status/processable/waiting_for_resource_spec.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Ci::Status::Processable::WaitingForResource do + let(:user) { create(:user) } + + subject do + processable = create(:ci_build, :waiting_for_resource, :resource_group) + described_class.new(Gitlab::Ci::Status::Core.new(processable, user)) + end + + describe '#illustration' do + it { expect(subject.illustration).to include(:image, :size, :title) } + end + + describe '.matches?' do + subject {described_class.matches?(processable, user) } + + context 'when processable is waiting for resource' do + let(:processable) { create(:ci_build, :waiting_for_resource) } + + it 'is a correct match' do + expect(subject).to be true + end + end + + context 'when processable is not waiting for resource' do + let(:processable) { create(:ci_build) } + + it 'does not match' do + expect(subject).to be false + end + end + end +end diff --git a/spec/lib/gitlab/sidekiq_logging/exception_handler_spec.rb b/spec/lib/gitlab/sidekiq_logging/exception_handler_spec.rb deleted file mode 100644 index 94dcf6f9b9a..00000000000 --- a/spec/lib/gitlab/sidekiq_logging/exception_handler_spec.rb +++ /dev/null @@ -1,44 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::SidekiqLogging::ExceptionHandler do - describe '#call' do - let(:job) do - { - "class" => "TestWorker", - "args" => [1234, 'hello'], - "retry" => false, - "queue" => "cronjob:test_queue", - "queue_namespace" => "cronjob", - "jid" => "da883554ee4fe414012f5f42", - "correlation_id" => 'cid' - } - end - - let(:exception_message) { 'An error was thrown' } - let(:backtrace) { caller } - let(:exception) { RuntimeError.new(exception_message) } - let(:logger) { double } - - before do - allow(Sidekiq).to receive(:logger).and_return(logger) - allow(exception).to receive(:backtrace).and_return(backtrace) - end - - subject { described_class.new.call(exception, { context: 'Test', job: job }) } - - it 'logs job data into root tree' do - expected_data = job.merge( - error_class: 'RuntimeError', - error_message: exception_message, - context: 'Test', - error_backtrace: Rails.backtrace_cleaner.clean(backtrace) - ) - - expect(logger).to receive(:warn).with(expected_data) - - subject - end - end -end diff --git a/spec/lib/gitlab/sidekiq_logging/structured_logger_spec.rb b/spec/lib/gitlab/sidekiq_logging/structured_logger_spec.rb index 856ae87c5bf..3291d06a2ac 100644 --- a/spec/lib/gitlab/sidekiq_logging/structured_logger_spec.rb +++ b/spec/lib/gitlab/sidekiq_logging/structured_logger_spec.rb @@ -59,7 +59,8 @@ RSpec.describe Gitlab::SidekiqLogging::StructuredLogger do 'message' => 'TestWorker JID-da883554ee4fe414012f5f42: fail: 0.0 sec', 'job_status' => 'fail', 'error_class' => 'ArgumentError', - 'error_message' => 'some exception' + 'error_message' => 'Something went wrong', + 'error_backtrace' => be_a(Array).and(be_present) ) end @@ -89,21 +90,92 @@ RSpec.describe Gitlab::SidekiqLogging::StructuredLogger do end end + it 'logs real job wrapped by active job worker' do + wrapped_job = job.merge( + "class" => "ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper", + "wrapped" => "TestWorker" + ) + Timecop.freeze(timestamp) do + expect(logger).to receive(:info).with(start_payload).ordered + expect(logger).to receive(:info).with(end_payload).ordered + expect(subject).to receive(:log_job_start).and_call_original + expect(subject).to receive(:log_job_done).and_call_original + + subject.call(wrapped_job, 'test_queue') {} + end + end + it 'logs an exception in job' do Timecop.freeze(timestamp) do expect(logger).to receive(:info).with(start_payload) - expect(logger).to receive(:warn).with(hash_including(exception_payload)) + expect(logger).to receive(:warn).with(include(exception_payload)) expect(subject).to receive(:log_job_start).and_call_original expect(subject).to receive(:log_job_done).and_call_original expect do subject.call(job, 'test_queue') do - raise ArgumentError, 'some exception' + raise ArgumentError, 'Something went wrong' end end.to raise_error(ArgumentError) end end + it 'logs the root cause of an Sidekiq::JobRetry::Skip exception in the job' do + Timecop.freeze(timestamp) do + expect(logger).to receive(:info).with(start_payload) + expect(logger).to receive(:warn).with(include(exception_payload)) + expect(subject).to receive(:log_job_start).and_call_original + expect(subject).to receive(:log_job_done).and_call_original + + expect do + subject.call(job, 'test_queue') do + raise ArgumentError, 'Something went wrong' + rescue + raise Sidekiq::JobRetry::Skip + end + end.to raise_error(Sidekiq::JobRetry::Skip) + end + end + + it 'logs the root cause of an Sidekiq::JobRetry::Handled exception in the job' do + Timecop.freeze(timestamp) do + expect(logger).to receive(:info).with(start_payload) + expect(logger).to receive(:warn).with(include(exception_payload)) + expect(subject).to receive(:log_job_start).and_call_original + expect(subject).to receive(:log_job_done).and_call_original + + expect do + subject.call(job, 'test_queue') do + raise ArgumentError, 'Something went wrong' + rescue + raise Sidekiq::JobRetry::Handled + end + end.to raise_error(Sidekiq::JobRetry::Handled) + end + end + + it 'keeps Sidekiq::JobRetry::Handled exception if the cause does not exist' do + Timecop.freeze(timestamp) do + expect(logger).to receive(:info).with(start_payload) + expect(logger).to receive(:warn).with( + include( + 'message' => 'TestWorker JID-da883554ee4fe414012f5f42: fail: 0.0 sec', + 'job_status' => 'fail', + 'error_class' => 'Sidekiq::JobRetry::Skip', + 'error_message' => 'Sidekiq::JobRetry::Skip' + ) + ) + expect(subject).to receive(:log_job_start).and_call_original + expect(subject).to receive(:log_job_done).and_call_original + + expect do + subject.call(job, 'test_queue') do + raise Sidekiq::JobRetry::Skip + end + end.to raise_error(Sidekiq::JobRetry::Skip) + end + end + it 'does not modify the job' do Timecop.freeze(timestamp) do job_copy = job.deep_dup @@ -117,6 +189,24 @@ RSpec.describe Gitlab::SidekiqLogging::StructuredLogger do end end end + + it 'does not modify the wrapped job' do + Timecop.freeze(timestamp) do + wrapped_job = job.merge( + "class" => "ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper", + "wrapped" => "TestWorker" + ) + job_copy = wrapped_job.deep_dup + + allow(logger).to receive(:info) + allow(subject).to receive(:log_job_start).and_call_original + allow(subject).to receive(:log_job_done).and_call_original + + subject.call(wrapped_job, 'test_queue') do + expect(wrapped_job).to eq(job_copy) + end + end + end end context 'with SIDEKIQ_LOG_ARGUMENTS disabled' do diff --git a/spec/lib/gitlab/sidekiq_middleware/client_metrics_spec.rb b/spec/lib/gitlab/sidekiq_middleware/client_metrics_spec.rb index f7010b2001a..e2b36125b4e 100644 --- a/spec/lib/gitlab/sidekiq_middleware/client_metrics_spec.rb +++ b/spec/lib/gitlab/sidekiq_middleware/client_metrics_spec.rb @@ -60,6 +60,27 @@ RSpec.describe Gitlab::SidekiqMiddleware::ClientMetrics do end end + context "when a worker is wrapped into ActiveJob" do + before do + stub_const('TestWrappedWorker', Class.new) + TestWrappedWorker.class_eval do + include Sidekiq::Worker + end + end + + it_behaves_like "a metrics client middleware" do + let(:job) do + { + "class" => ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper, + "wrapped" => TestWrappedWorker + } + end + + let(:worker) { TestWrappedWorker.new } + let(:labels) { default_labels.merge(urgency: "") } + end + end + context "when workers are attributed" do def create_attributed_worker_class(urgency, external_dependencies, resource_boundary, category) klass = Class.new do diff --git a/spec/lib/gitlab/sidekiq_middleware/server_metrics_spec.rb b/spec/lib/gitlab/sidekiq_middleware/server_metrics_spec.rb index 44bfaf4cc3c..e58e41d3e4f 100644 --- a/spec/lib/gitlab/sidekiq_middleware/server_metrics_spec.rb +++ b/spec/lib/gitlab/sidekiq_middleware/server_metrics_spec.rb @@ -198,6 +198,28 @@ RSpec.describe Gitlab::SidekiqMiddleware::ServerMetrics do it_behaves_like "a metrics middleware" end + context "when a worker is wrapped into ActiveJob" do + before do + stub_const('TestWrappedWorker', Class.new) + TestWrappedWorker.class_eval do + include Sidekiq::Worker + end + end + + let(:job) do + { + "class" => ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper, + "wrapped" => TestWrappedWorker + } + end + + let(:worker) { TestWrappedWorker.new } + let(:worker_class) { TestWrappedWorker } + let(:labels) { default_labels.merge(urgency: "") } + + it_behaves_like "a metrics middleware" + end + context "when workers are attributed" do def create_attributed_worker_class(urgency, external_dependencies, resource_boundary, category) Class.new do diff --git a/spec/models/ci/bridge_spec.rb b/spec/models/ci/bridge_spec.rb index 4f09f6f1da4..b50e4204e0a 100644 --- a/spec/models/ci/bridge_spec.rb +++ b/spec/models/ci/bridge_spec.rb @@ -80,6 +80,14 @@ RSpec.describe Ci::Bridge do end end + it "schedules downstream pipeline creation when the status is waiting for resource" do + bridge.status = :waiting_for_resource + + expect(bridge).to receive(:schedule_downstream_pipeline!) + + bridge.enqueue_waiting_for_resource! + end + it 'raises error when the status is failed' do bridge.status = :failed diff --git a/spec/services/ci/create_pipeline_service/cross_project_pipeline_spec.rb b/spec/services/ci/create_pipeline_service/cross_project_pipeline_spec.rb new file mode 100644 index 00000000000..9cf66dfceb0 --- /dev/null +++ b/spec/services/ci/create_pipeline_service/cross_project_pipeline_spec.rb @@ -0,0 +1,86 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Ci::CreatePipelineService, '#execute' do + let_it_be(:group) { create(:group, name: 'my-organization') } + let(:upstream_project) { create(:project, :repository, name: 'upstream', group: group) } + let(:downstram_project) { create(:project, :repository, name: 'downstream', group: group) } + let(:user) { create(:user) } + + let(:service) do + described_class.new(upstream_project, user, ref: 'master') + end + + before do + upstream_project.add_developer(user) + downstram_project.add_developer(user) + create_gitlab_ci_yml(upstream_project, upstream_config) + create_gitlab_ci_yml(downstram_project, downstream_config) + end + + context 'with resource group', :aggregate_failures do + let(:upstream_config) do + <<~YAML + instrumentation_test: + stage: test + resource_group: iOS + trigger: + project: my-organization/downstream + strategy: depend + YAML + end + + let(:downstream_config) do + <<~YAML + test: + script: echo "Testing..." + YAML + end + + it 'creates bridge job with resource group' do + pipeline = create_pipeline! + + test = pipeline.statuses.find_by(name: 'instrumentation_test') + expect(pipeline).to be_created_successfully + expect(pipeline.triggered_pipelines).not_to be_exist + expect(upstream_project.resource_groups.count).to eq(1) + expect(test).to be_a Ci::Bridge + expect(test).to be_waiting_for_resource + expect(test.resource_group.key).to eq('iOS') + end + + context 'when sidekiq processes the job', :sidekiq_inline do + it 'transitions to pending status and triggers a downstream pipeline' do + pipeline = create_pipeline! + + test = pipeline.statuses.find_by(name: 'instrumentation_test') + expect(test).to be_pending + expect(pipeline.triggered_pipelines.count).to eq(1) + end + + context 'when the resource is occupied by the other bridge' do + before do + resource_group = create(:ci_resource_group, project: upstream_project, key: 'iOS') + resource_group.assign_resource_to(create(:ci_build, project: upstream_project)) + end + + it 'stays waiting for resource' do + pipeline = create_pipeline! + + test = pipeline.statuses.find_by(name: 'instrumentation_test') + expect(test).to be_waiting_for_resource + expect(pipeline.triggered_pipelines.count).to eq(0) + end + end + end + end + + def create_pipeline! + service.execute(:push) + end + + def create_gitlab_ci_yml(project, content) + project.repository.create_file(user, '.gitlab-ci.yml', content, branch_name: 'master', message: 'test') + end +end diff --git a/spec/services/ci/create_pipeline_service/parent_child_pipeline_spec.rb b/spec/services/ci/create_pipeline_service/parent_child_pipeline_spec.rb index a3a616f0f64..a3818937113 100644 --- a/spec/services/ci/create_pipeline_service/parent_child_pipeline_spec.rb +++ b/spec/services/ci/create_pipeline_service/parent_child_pipeline_spec.rb @@ -84,21 +84,46 @@ RSpec.describe Ci::CreatePipelineService, '#execute' do stage: test resource_group: iOS trigger: - include: - - local: path/to/child.yml + include: path/to/child.yml + strategy: depend YAML end - # TODO: This test will be properly implemented in the next MR - # for https://gitlab.com/gitlab-org/gitlab/-/issues/39057. - it 'creates bridge job but still resource group is no-op', :aggregate_failures do + it 'creates bridge job with resource group', :aggregate_failures do pipeline = create_pipeline! test = pipeline.statuses.find_by(name: 'instrumentation_test') - - expect(pipeline).to be_persisted + expect(pipeline).to be_created_successfully + expect(pipeline.triggered_pipelines).not_to be_exist + expect(project.resource_groups.count).to eq(1) expect(test).to be_a Ci::Bridge - expect(project.resource_groups.count).to eq(0) + expect(test).to be_waiting_for_resource + expect(test.resource_group.key).to eq('iOS') + end + + context 'when sidekiq processes the job', :sidekiq_inline do + it 'transitions to pending status and triggers a downstream pipeline' do + pipeline = create_pipeline! + + test = pipeline.statuses.find_by(name: 'instrumentation_test') + expect(test).to be_pending + expect(pipeline.triggered_pipelines.count).to eq(1) + end + + context 'when the resource is occupied by the other bridge' do + before do + resource_group = create(:ci_resource_group, project: project, key: 'iOS') + resource_group.assign_resource_to(create(:ci_build, project: project)) + end + + it 'stays waiting for resource' do + pipeline = create_pipeline! + + test = pipeline.statuses.find_by(name: 'instrumentation_test') + expect(test).to be_waiting_for_resource + expect(pipeline.triggered_pipelines.count).to eq(0) + end + end end end end diff --git a/spec/support/shared_examples/graphql/label_fields.rb b/spec/support/shared_examples/graphql/label_fields.rb index caf5dae409a..0d09ab391f1 100644 --- a/spec/support/shared_examples/graphql/label_fields.rb +++ b/spec/support/shared_examples/graphql/label_fields.rb @@ -105,7 +105,7 @@ RSpec.shared_examples 'querying a GraphQL type with labels' do run_query(query_for(label_a)) end - it 'batches queries for labels by title' do + it 'batches queries for labels by title', :request_store do multi_selection = query_for(label_b, label_c) single_selection = query_for(label_d) |