From 039ba7ce2dc10889bbdb4abf4c398701a87a1f58 Mon Sep 17 00:00:00 2001 From: GitLab Bot Date: Thu, 5 May 2022 00:09:01 +0000 Subject: Add latest changes from gitlab-org/gitlab@master --- .rubocop_todo/gitlab/namespaced_class.yml | 1 - .rubocop_todo/layout/line_length.yml | 1 - .rubocop_todo/layout/space_inside_block_braces.yml | 3 - .../performance/method_object_as_block.yml | 1 - .rubocop_todo/style/percent_literal_delimiters.yml | 2 - GITALY_SERVER_VERSION | 2 +- GITLAB_SHELL_VERSION | 2 +- app/graphql/types/project_statistics_type.rb | 2 + app/models/application_setting.rb | 1 + .../metrics_and_profiling.html.haml | 2 - config/gitlab.yml.example | 29 -- config/initializers/1_settings.rb | 13 - config/pseudonymizer.yml | 471 --------------------- doc/administration/gitaly/index.md | 7 +- doc/api/graphql/reference/index.md | 1 + locale/gitlab.pot | 15 - package.json | 2 +- qa/qa/resource/pipeline.rb | 50 ++- ...ritance_with_forward_pipeline_variables_spec.rb | 119 ++++++ .../variable_inheritance_shared_context.rb | 8 +- qa/qa/support/matchers/have_matcher.rb | 1 + spec/graphql/types/project_statistics_type_spec.rb | 3 +- spec/support/helpers/gitaly_setup.rb | 7 +- spec/support/helpers/stub_object_storage.rb | 5 - yarn.lock | 31 +- 25 files changed, 196 insertions(+), 583 deletions(-) delete mode 100644 config/pseudonymizer.yml create mode 100644 qa/qa/specs/features/api/4_verify/api_variable_inheritance_with_forward_pipeline_variables_spec.rb diff --git a/.rubocop_todo/gitlab/namespaced_class.yml b/.rubocop_todo/gitlab/namespaced_class.yml index b076db1f6c0..97e6a8c9c3a 100644 --- a/.rubocop_todo/gitlab/namespaced_class.yml +++ b/.rubocop_todo/gitlab/namespaced_class.yml @@ -1136,7 +1136,6 @@ Gitlab/NamespacedClass: - ee/app/workers/new_epic_worker.rb - ee/app/workers/project_import_schedule_worker.rb - ee/app/workers/project_template_export_worker.rb - - ee/app/workers/pseudonymizer_worker.rb - ee/app/workers/refresh_license_compliance_checks_worker.rb - ee/app/workers/repository_push_audit_event_worker.rb - ee/app/workers/repository_update_mirror_worker.rb diff --git a/.rubocop_todo/layout/line_length.yml b/.rubocop_todo/layout/line_length.yml index 63a6cd29441..d46c7a6da62 100644 --- a/.rubocop_todo/layout/line_length.yml +++ b/.rubocop_todo/layout/line_length.yml @@ -1849,7 +1849,6 @@ Layout/LineLength: - 'ee/lib/gitlab/usage/metrics/instrumentations/count_users_creating_ci_builds_metric.rb' - 'ee/lib/gitlab/usage/metrics/instrumentations/license_metric.rb' - 'ee/lib/gitlab/usage_data_counters/license_testing_counter.rb' - - 'ee/lib/pseudonymizer/filter.rb' - 'ee/lib/system_check/app/elasticsearch_check.rb' - 'ee/lib/system_check/geo/authorized_keys_check.rb' - 'ee/lib/system_check/geo/current_node_check.rb' diff --git a/.rubocop_todo/layout/space_inside_block_braces.yml b/.rubocop_todo/layout/space_inside_block_braces.yml index 807827d2075..b3830147692 100644 --- a/.rubocop_todo/layout/space_inside_block_braces.yml +++ b/.rubocop_todo/layout/space_inside_block_braces.yml @@ -51,7 +51,6 @@ Layout/SpaceInsideBlockBraces: - 'ee/lib/ee/gitlab/background_migration/drop_invalid_remediations.rb' - 'ee/lib/elastic/latest/git_class_proxy.rb' - 'ee/lib/gitlab/auth/smartcard/san_extension.rb' - - 'ee/lib/pseudonymizer/dumper.rb' - 'ee/lib/world.rb' - 'ee/spec/controllers/autocomplete_controller_spec.rb' - 'ee/spec/controllers/countries_controller_spec.rb' @@ -134,8 +133,6 @@ Layout/SpaceInsideBlockBraces: - 'ee/spec/lib/gitlab/search_context/builder_spec.rb' - 'ee/spec/lib/gitlab/status_page/filter/image_filter_spec.rb' - 'ee/spec/lib/gitlab/usage/metrics/instrumentations/approval_project_rules_with_user_metric_spec.rb' - - 'ee/spec/lib/pseudonymizer/pager_spec.rb' - - 'ee/spec/lib/pseudonymizer/uploader_spec.rb' - 'ee/spec/mailers/credentials_inventory_mailer_spec.rb' - 'ee/spec/migrations/geo/migrate_job_artifact_registry_spec.rb' - 'ee/spec/models/analytics/devops_adoption/enabled_namespace_spec.rb' diff --git a/.rubocop_todo/performance/method_object_as_block.yml b/.rubocop_todo/performance/method_object_as_block.yml index 07ceaa93acd..265e03073db 100644 --- a/.rubocop_todo/performance/method_object_as_block.yml +++ b/.rubocop_todo/performance/method_object_as_block.yml @@ -38,7 +38,6 @@ Performance/MethodObjectAsBlock: - 'ee/lib/ee/container_registry/client.rb' - 'ee/lib/ee/gitlab/ci/config_ee.rb' - 'ee/lib/ee/gitlab/etag_caching/router/rails.rb' - - 'ee/lib/pseudonymizer/dumper.rb' - 'ee/spec/services/groups/participants_service_spec.rb' - 'lib/api/helpers/packages/conan/api_helpers.rb' - 'lib/bulk_imports/pipeline.rb' diff --git a/.rubocop_todo/style/percent_literal_delimiters.yml b/.rubocop_todo/style/percent_literal_delimiters.yml index d3f4b427a26..4673f0d71d2 100644 --- a/.rubocop_todo/style/percent_literal_delimiters.yml +++ b/.rubocop_todo/style/percent_literal_delimiters.yml @@ -413,8 +413,6 @@ Style/PercentLiteralDelimiters: - 'ee/spec/lib/gitlab/status_page/storage/s3_client_spec.rb' - 'ee/spec/lib/gitlab/status_page/storage/s3_multipart_upload_spec.rb' - 'ee/spec/lib/gitlab/vulnerabilities/parser_spec.rb' - - 'ee/spec/lib/pseudonymizer/dumper_spec.rb' - - 'ee/spec/lib/pseudonymizer/pager_spec.rb' - 'ee/spec/mailers/ci_minutes_usage_mailer_spec.rb' - 'ee/spec/mailers/credentials_inventory_mailer_spec.rb' - 'ee/spec/mailers/ee/emails/profile_spec.rb' diff --git a/GITALY_SERVER_VERSION b/GITALY_SERVER_VERSION index 292efacb872..7c709c6d081 100644 --- a/GITALY_SERVER_VERSION +++ b/GITALY_SERVER_VERSION @@ -1 +1 @@ -86aa7ee82a5dd241fd7d4b33435da0a7ecad12b0 +372599313791cb92e579e0ff02279f33cbcd71b5 diff --git a/GITLAB_SHELL_VERSION b/GITLAB_SHELL_VERSION index fe160c8eb4b..4b964e96540 100644 --- a/GITLAB_SHELL_VERSION +++ b/GITLAB_SHELL_VERSION @@ -1 +1 @@ -13.25.2 +14.0.0 diff --git a/app/graphql/types/project_statistics_type.rb b/app/graphql/types/project_statistics_type.rb index 1146774b43c..5ab3cc33e85 100644 --- a/app/graphql/types/project_statistics_type.rb +++ b/app/graphql/types/project_statistics_type.rb @@ -27,5 +27,7 @@ module Types description: 'Uploads size of the project in bytes.' field :wiki_size, GraphQL::Types::Float, null: true, description: 'Wiki size of the project in bytes.' + field :container_registry_size, GraphQL::Types::Float, null: true, + description: 'Container Registry size of the project in bytes.' end end diff --git a/app/models/application_setting.rb b/app/models/application_setting.rb index 7cd2fe705e3..b2943d61216 100644 --- a/app/models/application_setting.rb +++ b/app/models/application_setting.rb @@ -12,6 +12,7 @@ class ApplicationSetting < ApplicationRecord ignore_columns %i[static_objects_external_storage_auth_token], remove_with: '14.9', remove_after: '2022-03-22' ignore_column %i[max_package_files_for_package_destruction], remove_with: '14.9', remove_after: '2022-03-22' ignore_column :user_email_lookup_limit, remove_with: '15.0', remove_after: '2022-04-18' + ignore_column :pseudonymizer_enabled, remove_with: '15.1', remove_after: '2022-06-22' INSTANCE_REVIEW_MIN_USERS = 50 GRAFANA_URL_ERROR_MESSAGE = 'Please check your Grafana URL setting in ' \ diff --git a/app/views/admin/application_settings/metrics_and_profiling.html.haml b/app/views/admin/application_settings/metrics_and_profiling.html.haml index 7cb5760f62a..346be6ef44c 100644 --- a/app/views/admin/application_settings/metrics_and_profiling.html.haml +++ b/app/views/admin/application_settings/metrics_and_profiling.html.haml @@ -53,8 +53,6 @@ .settings-content = render 'usage' -= render_if_exists 'admin/application_settings/pseudonymizer_settings', expanded: expanded_by_default? - - if Feature.enabled?(:configure_sentry_in_application_settings, default_enabled: :yaml) %section.settings.as-sentry.no-animate#js-sentry-settings{ class: ('expanded' if expanded_by_default?), data: { qa_selector: 'sentry_settings_content' } } .settings-header diff --git a/config/gitlab.yml.example b/config/gitlab.yml.example index 57106120fb2..a2224a13d19 100644 --- a/config/gitlab.yml.example +++ b/config/gitlab.yml.example @@ -602,10 +602,6 @@ production: &base geo_registry_sync_worker: cron: "*/1 * * * *" - # Export pseudonymized data in CSV format for analysis - pseudonymizer_worker: - cron: "0 * * * *" - # Elasticsearch bulk updater for incremental updates. # NOTE: This will only take effect if elasticsearch is enabled. elastic_index_bulk_cron_worker: @@ -1215,20 +1211,6 @@ production: &base # # server_side_encryption: 'aws:kms' # # server_side_encryption_kms_key_id: 'arn:aws:kms:YOUR-KEY-ID-HERE' - ## Pseudonymizer exporter - pseudonymizer: - # Tables manifest that specifies the fields to extract and pseudonymize. - manifest: config/pseudonymizer.yml - upload: - remote_directory: 'gitlab-elt' - # Fog storage connection settings, see http://fog.io/storage/ . - connection: - # provider: AWS - # region: eu-west-1 - # aws_access_key_id: AKIAKIAKI - # aws_secret_access_key: 'secret123' - # # The remote 'directory' to store the CSV files. For S3, this would be the bucket name. - ## GitLab Shell settings gitlab_shell: path: /home/git/gitlab-shell/ @@ -1534,17 +1516,6 @@ test: backup: path: tmp/tests/backups gitaly_backup_path: tmp/tests/gitaly/_build/bin/gitaly-backup - pseudonymizer: - manifest: config/pseudonymizer.yml - upload: - # The remote 'directory' to store the CSV files. For S3, this would be the bucket name. - remote_directory: gitlab-elt.test - # Fog storage connection settings, see http://fog.io/storage/ - connection: - provider: AWS # Only AWS supported at the moment - aws_access_key_id: AWS_ACCESS_KEY_ID - aws_secret_access_key: AWS_SECRET_ACCESS_KEY - region: us-east-1 gitlab_shell: path: tmp/tests/gitlab-shell/ authorized_keys_file: tmp/tests/authorized_keys diff --git a/config/initializers/1_settings.rb b/config/initializers/1_settings.rb index 41795d60cdd..65b33d8444b 100644 --- a/config/initializers/1_settings.rb +++ b/config/initializers/1_settings.rb @@ -709,9 +709,6 @@ Gitlab.ee do Settings.cron_jobs['ldap_sync_worker'] ||= Settingslogic.new({}) Settings.cron_jobs['ldap_sync_worker']['cron'] ||= '30 1 * * *' Settings.cron_jobs['ldap_sync_worker']['job_class'] = 'LdapSyncWorker' - Settings.cron_jobs['pseudonymizer_worker'] ||= Settingslogic.new({}) - Settings.cron_jobs['pseudonymizer_worker']['cron'] ||= '0 23 * * *' - Settings.cron_jobs['pseudonymizer_worker']['job_class'] ||= 'PseudonymizerWorker' Settings.cron_jobs['update_max_seats_used_for_gitlab_com_subscriptions_worker'] ||= Settingslogic.new({}) Settings.cron_jobs['update_max_seats_used_for_gitlab_com_subscriptions_worker']['cron'] ||= '0 12 * * *' Settings.cron_jobs['update_max_seats_used_for_gitlab_com_subscriptions_worker']['job_class'] = 'UpdateMaxSeatsUsedForGitlabComSubscriptionsWorker' @@ -863,16 +860,6 @@ Settings.backup['upload']['encryption_key'] ||= ENV['GITLAB_BACKUP_ENCRYPTION_KE Settings.backup['upload']['storage_class'] ||= nil Settings.backup['gitaly_backup_path'] ||= Gitlab::Utils.which('gitaly-backup') -# -# Pseudonymizer -# -Gitlab.ee do - Settings['pseudonymizer'] ||= Settingslogic.new({}) - Settings.pseudonymizer['manifest'] = Settings.absolute(Settings.pseudonymizer['manifest'] || Rails.root.join("config/pseudonymizer.yml")) - Settings.pseudonymizer['upload'] ||= Settingslogic.new({ 'remote_directory' => nil, 'connection' => nil }) - # Settings.pseudonymizer['upload']['multipart_chunk_size'] ||= 104857600 -end - # # Git # diff --git a/config/pseudonymizer.yml b/config/pseudonymizer.yml deleted file mode 100644 index 4b4b0dfa584..00000000000 --- a/config/pseudonymizer.yml +++ /dev/null @@ -1,471 +0,0 @@ -tables: - approvals: - whitelist: - - id - - merge_request_id - - user_id - - created_at - - updated_at - approver_groups: - whitelist: - - id - - target_type - - group_id - - created_at - - updated_at - board_assignees: - whitelist: - - id - - board_id - - assignee_id - board_labels: - whitelist: - - id - - board_id - - label_id - boards: - whitelist: - - id - - project_id - - created_at - - updated_at - - milestone_id - - group_id - - weight - epic_issues: - whitelist: - - id - - epic_id - - issue_id - - relative_position - epic_metrics: - whitelist: - - id - - epic_id - - created_at - - updated_at - epics: - whitelist: - - id - - group_id - - author_id - - assignee_id - - iid - - updated_by_id - - last_edited_by_id - - lock_version - - start_date - - end_date - - last_edited_at - - created_at - - updated_at - - title - - description - issue_assignees: - whitelist: - - user_id - - issue_id - issue_links: - whitelist: - - id - - source_id - - target_id - - created_at - - updated_at - issue_metrics: - whitelist: - - id - - issue_id - - first_mentioned_in_commit_at - - first_associated_with_milestone_at - - first_added_to_board_at - - created_at - - updated_at - issues: - whitelist: - - id - - title - - author_id - - project_id - - created_at - - confidential - - updated_at - - description - - milestone_id - - state_id - - updated_by_id - - weight - - due_date - - moved_to_id - - duplicated_to_id - - lock_version - - time_estimate - - last_edited_at - - last_edited_by_id - - discussion_locked - - closed_at - label_links: - whitelist: - - id - - label_id - - target_id - - target_type - - created_at - - updated_at - label_priorities: - whitelist: - - id - - project_id - - label_id - - priority - - created_at - - updated_at - labels: - whitelist: - - id - - title - - color - - project_id - - created_at - - updated_at - - template - - type - - group_id - licenses: - whitelist: - - id - - created_at - - updated_at - merge_request_diffs: - whitelist: - - id - - state - - merge_request_id - - created_at - - updated_at - - base_commit_sha - - real_size - - head_commit_sha - - start_commit_sha - - commits_count - merge_request_metrics: - whitelist: - - id - - merge_request_id - - latest_build_started_at - - latest_build_finished_at - - first_deployed_to_production_at - - merged_at - - created_at - - updated_at - - pipeline_id - - merged_by_id - - latest_closed_by_id - - latest_closed_at - merge_requests: - whitelist: - - id - - target_branch - - source_branch - - source_project_id - - author_id - - assignee_id - - created_at - - updated_at - - milestone_id - - state_id - - merge_status - - target_project_id - - updated_by_id - - merge_error - - merge_params - - merge_when_pipeline_succeeds - - merge_user_id - - approvals_before_merge - - lock_version - - time_estimate - - squash - - last_edited_at - - last_edited_by_id - - head_pipeline_id - - discussion_locked - - latest_merge_request_diff_id - - allow_maintainer_to_push - merge_requests_closing_issues: - whitelist: - - id - - merge_request_id - - issue_id - - created_at - - updated_at - milestones: - whitelist: - - id - - project_id - - due_date - - created_at - - updated_at - - state - - start_date - - group_id - namespace_statistics: - whitelist: - - id - - namespace_id - - shared_runners_seconds - - shared_runners_seconds_last_reset - namespaces: - whitelist: - - id - - name - - path - - owner_id - - created_at - - updated_at - - type - - avatar - - membership_lock - - share_with_group_lock - - visibility_level - - request_access_enabled - - ldap_sync_status - - ldap_sync_error - - ldap_sync_last_update_at - - ldap_sync_last_successful_update_at - - ldap_sync_last_sync_at - - lfs_enabled - - parent_id - - shared_runners_minutes_limit - - repository_size_limit - - require_two_factor_authentication - - two_factor_grace_period - - project_creation_level - members: - whitelist: - - id - - access_level - - source_id - - source_type - - user_id - - notification_level - - type - - created_by_id - - invite_email - - invite_accepted_at - - requested_at - - expires_at - - ldap - - override - notification_settings: - whitelist: - - id - - user_id - - source_id - - source_type - - level - - created_at - - updated_at - - new_note - - new_issue - - reopen_issue - - close_issue - - reassign_issue - - new_merge_request - - reopen_merge_request - - close_merge_request - - reassign_merge_request - - merge_merge_request - - failed_pipeline - - success_pipeline - project_authorizations: - whitelist: - - user_id - - project_id - - access_level - project_auto_devops: - whitelist: - - id - - project_id - - created_at - - updated_at - - enabled - project_custom_attributes: - whitelist: - - id - - created_at - - updated_at - - project_id - - key - - value - project_features: - whitelist: - - id - - project_id - - merge_requests_access_level - - issues_access_level - - wiki_access_level - - snippets_access_level - - builds_access_level - - created_at - - updated_at - - repository_access_level - project_group_links: - whitelist: - - id - - project_id - - group_id - - created_at - - updated_at - - group_access - - expires_at - project_import_data: - whitelist: - - id - - project_id - project_mirror_data: - whitelist: - - id - - project_id - - retry_count - - last_update_started_at - - last_update_scheduled_at - - next_execution_timestamp - project_repository_states: - whitelist: - - id - - project_id - - repository_verification_checksum - - wiki_verification_checksum - - last_repository_verification_failure - - last_wiki_verification_failure - project_statistics: - whitelist: - - id - - project_id - - namespace_id - - commit_count - - storage_size - - repository_size - - lfs_objects_size - - build_artifacts_size - - shared_runners_seconds - - shared_runners_seconds_last_reset - projects: - whitelist: - - id - - name - - path - - description - - created_at - - updated_at - - creator_id - - namespace_id - - last_activity_at - - import_url - - visibility_level - - archived - - avatar - - merge_requests_template - - star_count - - merge_requests_rebase_enabled - - import_type - - import_source - - approvals_before_merge - - reset_approvals_on_push - - merge_requests_ff_only_enabled - - issues_template - - mirror - - mirror_user_id - - shared_runners_enabled - - build_allow_git_fetch - - build_timeout - - mirror_trigger_builds - - pending_delete - - public_builds - - last_repository_check_failed - - last_repository_check_at - - only_allow_merge_if_pipeline_succeeds - - has_external_issue_tracker - - repository_storage - - repository_read_only - - request_access_enabled - - has_external_wiki - - ci_config_path - - lfs_enabled - - only_allow_merge_if_all_discussions_are_resolved - - repository_size_limit - - printing_merge_request_link_enabled - - auto_cancel_pending_pipelines - - service_desk_enabled - - delete_error - - last_repository_updated_at - - disable_overriding_approvers_per_merge_request - - storage_version - - resolve_outdated_diff_discussions - - remote_mirror_available_overridden - - only_mirror_protected_branches - - pull_mirror_available_overridden - - mirror_overwrites_diverged_branches - - external_authorization_classification_label - subscriptions: - whitelist: - - id - - user_id - - subscribable_id - - subscribable_type - - subscribed - - created_at - - updated_at - - project_id - users: - whitelist: - - id - - remember_created_at - - sign_in_count - - current_sign_in_at - - last_sign_in_at - - current_sign_in_ip - - last_sign_in_ip - - created_at - - updated_at - - admin - - projects_limit - - failed_attempts - - locked_at - - can_create_group - - can_create_team - - state - - color_scheme_id - - password_expires_at - - created_by_id - - last_credential_check_at - - avatar - - confirmed_at - - confirmation_sent_at - - unconfirmed_email - - hide_no_ssh_key - - website_url - - admin_email_unsubscribed_at - - notification_email - - hide_no_password - - password_automatically_set - - location - - public_email - - dashboard - - project_view - - consumed_timestep - - layout - - hide_project_limit - - note - - otp_grace_period_started_at - - external - - organization - - auditor - - require_two_factor_authentication_from_group - - two_factor_grace_period - - last_activity_on - - notified_of_own_activity - - user_type - - preferred_language - - theme_id - diff --git a/doc/administration/gitaly/index.md b/doc/administration/gitaly/index.md index 97c0450de14..0316efa92bf 100644 --- a/doc/administration/gitaly/index.md +++ b/doc/administration/gitaly/index.md @@ -57,8 +57,7 @@ If you have: - Not yet migrated to Gitaly Cluster and want to continue using NFS, remain on the service you are using. NFS is supported in 14.x releases but is [deprecated](../../update/deprecations.md#nfs-for-git-repository-storage). - Support for storing Git repository data on NFS is scheduled to end for all versions of GitLab with the release of GitLab - 15.6. + Support for storing Git repository data on NFS is scheduled to end for all versions of GitLab on November 22, 2022. - Not yet migrated to Gitaly Cluster but want to migrate away from NFS, you have two options: - A sharded Gitaly instance. - Gitaly Cluster. @@ -237,7 +236,7 @@ As with normal Gitaly storages, virtual storages can be sharded. ### Moving beyond NFS Engineering support for NFS for Git repositories is deprecated. Technical support is planned to be unavailable starting -GitLab 15.6. Please see our [statement of support](https://about.gitlab.com/support/statement-of-support.html#gitaly-and-nfs) +November 22, 2022. Please see our [statement of support](https://about.gitlab.com/support/statement-of-support.html#gitaly-and-nfs) for more details. [Network File System (NFS)](https://en.wikipedia.org/wiki/Network_File_System) @@ -692,4 +691,4 @@ The second facet presents the only real solution. For this, we developed ## NFS deprecation notice Engineering support for NFS for Git repositories is deprecated. Technical support is planned to be -unavailable from GitLab 15.6. For further information, please see our [NFS Deprecation](../nfs.md#gitaly-and-nfs-deprecation) documentation. +unavailable beginning November 22, 2022. For further information, please see our [NFS Deprecation](../nfs.md#gitaly-and-nfs-deprecation) documentation. diff --git a/doc/api/graphql/reference/index.md b/doc/api/graphql/reference/index.md index 7927be6d75b..0981d1e45d9 100644 --- a/doc/api/graphql/reference/index.md +++ b/doc/api/graphql/reference/index.md @@ -15629,6 +15629,7 @@ Represents the source of a security policy belonging to a project. | ---- | ---- | ----------- | | `buildArtifactsSize` | [`Float!`](#float) | Build artifacts size of the project in bytes. | | `commitCount` | [`Float!`](#float) | Commit count of the project. | +| `containerRegistrySize` | [`Float`](#float) | Container Registry size of the project in bytes. | | `lfsObjectsSize` | [`Float!`](#float) | Large File Storage (LFS) object size of the project in bytes. | | `packagesSize` | [`Float!`](#float) | Packages size of the project in bytes. | | `pipelineArtifactsSize` | [`Float`](#float) | CI Pipeline artifacts size in bytes. | diff --git a/locale/gitlab.pot b/locale/gitlab.pot index 36ca4f8f69f..8a2f5120111 100644 --- a/locale/gitlab.pot +++ b/locale/gitlab.pot @@ -14115,9 +14115,6 @@ msgstr "" msgid "Enable PlantUML" msgstr "" -msgid "Enable Pseudonymizer data collection" -msgstr "" - msgid "Enable SSL verification" msgstr "" @@ -14217,9 +14214,6 @@ msgstr "" msgid "Enable only for confidential applications exclusively used by a trusted backend server that can securely store the client secret. Do not enable for native-mobile, single-page, or other JavaScript applications because they cannot keep the client secret confidential." msgstr "" -msgid "Enable or disable the Pseudonymizer data collection." -msgstr "" - msgid "Enable or disable version check and Service Ping." msgstr "" @@ -17377,9 +17371,6 @@ msgstr "" msgid "GitLab project export" msgstr "" -msgid "GitLab runs a background job to export pseudonymized CSVs of the GitLab database. The CSV files are then uploaded to your configured object storage directory." -msgstr "" - msgid "GitLab single sign-on URL" msgstr "" @@ -30947,12 +30938,6 @@ msgstr "" msgid "Proxy support for this API is not available currently" msgstr "" -msgid "Pseudonymized data collection is disabled. When enabled, GitLab runs a background job to export pseudonymized CSVs of the GitLab database. The CSV files are then uploaded to your configured object storage directory." -msgstr "" - -msgid "Pseudonymizer data collection" -msgstr "" - msgid "Public" msgstr "" diff --git a/package.json b/package.json index 00421ca30b0..8908958f32e 100644 --- a/package.json +++ b/package.json @@ -258,7 +258,7 @@ "stylelint": "^14.3.0", "timezone-mock": "^1.0.8", "vue-jest": "4.0.1", - "webpack-dev-server": "4.8.1", + "webpack-dev-server": "4.9.0", "xhr-mock": "^2.5.1", "yarn-check-webpack-plugin": "^1.2.0", "yarn-deduplicate": "^5.0.0" diff --git a/qa/qa/resource/pipeline.rb b/qa/qa/resource/pipeline.rb index 907a6cb8558..910065d76a8 100644 --- a/qa/qa/resource/pipeline.rb +++ b/qa/qa/resource/pipeline.rb @@ -9,10 +9,10 @@ module QA end end - attribute :id - attribute :status - attribute :ref - attribute :sha + attributes :id, + :status, + :ref, + :sha # array in form # [ @@ -33,6 +33,14 @@ module QA Page::Project::Pipeline::New.perform(&:click_run_pipeline_button) end + def fabricate_via_api! + resource_web_url(api_get) + rescue ResourceNotFoundError + super + rescue NoValueError + super + end + def ref project.default_branch end @@ -51,6 +59,40 @@ module QA variables: variables } end + + def pipeline_variables + response = get(request_url("#{api_get_path}/variables")) + + unless response.code == HTTP_STATUS_OK + raise ResourceQueryError, "Could not get variables. Request returned (#{response.code}): `#{response}`." + end + + parse_body(response) + end + + def has_variable?(key:, value:) + pipeline_variables.any? { |var| var[:key] == key && var[:value] == value } + end + + def has_no_variable?(key:, value:) + !pipeline_variables.any? { |var| var[:key] == key && var[:value] == value } + end + + def pipeline_bridges + response = get(request_url("#{api_get_path}/bridges")) + + unless response.code == HTTP_STATUS_OK + raise ResourceQueryError, "Could not get bridges. Request returned (#{response.code}): `#{response}`." + end + + parse_body(response) + end + + def downstream_pipeline_id(bridge_name:) + result = pipeline_bridges.find { |bridge| bridge[:name] == bridge_name } + + result[:downstream_pipeline][:id] + end end end end diff --git a/qa/qa/specs/features/api/4_verify/api_variable_inheritance_with_forward_pipeline_variables_spec.rb b/qa/qa/specs/features/api/4_verify/api_variable_inheritance_with_forward_pipeline_variables_spec.rb new file mode 100644 index 00000000000..2434fecf51a --- /dev/null +++ b/qa/qa/specs/features/api/4_verify/api_variable_inheritance_with_forward_pipeline_variables_spec.rb @@ -0,0 +1,119 @@ +# frozen_string_literal: true + +module QA + # TODO: + # Remove FF :ci_trigger_forward_variables + # when https://gitlab.com/gitlab-org/gitlab/-/issues/355572 is closed + RSpec.describe 'Verify', :runner, feature_flag: { + name: 'ci_trigger_forward_variables', + scope: :global + } do + describe 'Pipeline API defined variable inheritance' do + include_context 'variable inheritance test prep' + + before do + add_ci_file(downstream1_project, [downstream1_ci_file]) + add_ci_file(upstream_project, [upstream_ci_file, upstream_child1_ci_file, upstream_child2_ci_file]) + + start_pipeline_via_api_with_variable + + Support::Waiter.wait_until(max_duration: 180, sleep_interval: 5) do + upstream_pipeline.status == 'success' + end + + Support::Waiter.wait_until(max_duration: 180, sleep_interval: 5) do + downstream1_pipeline.pipeline_variables && child1_pipeline.pipeline_variables + end + end + + it( + 'is determined based on forward:pipeline_variables condition', + :aggregate_failures, + testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/360745' + ) do + # Is inheritable when true + expect(child1_pipeline).to have_variable(key: key, value: value), + "Expected to find `{key: 'TEST_VAR', value: 'This is great!'}`" \ + " but got #{child1_pipeline.pipeline_variables}" + + # Is not inheritable when false + expect(child2_pipeline).not_to have_variable(key: key, value: value), + "Did not expect to find `{key: 'TEST_VAR', value: 'This is great!'}`" \ + " but got #{child2_pipeline.pipeline_variables}" + + # Is not inheritable by default + expect(downstream1_pipeline).not_to have_variable(key: key, value: value), + "Did not expect to find `{key: 'TEST_VAR', value: 'This is great!'}`" \ + " but got #{downstream1_pipeline.pipeline_variables}" + end + + def start_pipeline_via_api_with_variable + Resource::Pipeline.fabricate_via_api! do |pipeline| + pipeline.project = upstream_project + pipeline.variables = [{ key: key, value: value, variable_type: 'env_var' }] + end + + Support::Waiter.wait_until { upstream_project.pipelines.size > 1 } + end + + def upstream_pipeline + Resource::Pipeline.fabricate_via_api! do |pipeline| + pipeline.project = upstream_project + pipeline.id = upstream_project.pipelines.first[:id] + end + end + + def child1_pipeline + Resource::Pipeline.fabricate_via_api! do |pipeline| + pipeline.project = upstream_project + pipeline.id = upstream_pipeline.downstream_pipeline_id(bridge_name: 'child1_trigger') + end + end + + def child2_pipeline + Resource::Pipeline.fabricate_via_api! do |pipeline| + pipeline.project = upstream_project + pipeline.id = upstream_pipeline.downstream_pipeline_id(bridge_name: 'child2_trigger') + end + end + + def downstream1_pipeline + Resource::Pipeline.fabricate_via_api! do |pipeline| + pipeline.project = downstream1_project + pipeline.id = upstream_pipeline.downstream_pipeline_id(bridge_name: 'downstream1_trigger') + end + end + + def upstream_ci_file + { + file_path: '.gitlab-ci.yml', + content: <<~YAML + stages: + - test + - deploy + + child1_trigger: + stage: test + trigger: + include: .child1-ci.yml + forward: + pipeline_variables: true + + child2_trigger: + stage: test + trigger: + include: .child2-ci.yml + forward: + pipeline_variables: false + + # default behavior + downstream1_trigger: + stage: deploy + trigger: + project: #{downstream1_project.full_path} + YAML + } + end + end + end +end diff --git a/qa/qa/specs/features/shared_contexts/variable_inheritance_shared_context.rb b/qa/qa/specs/features/shared_contexts/variable_inheritance_shared_context.rb index 1dc8870d4d9..12869d517b1 100644 --- a/qa/qa/specs/features/shared_contexts/variable_inheritance_shared_context.rb +++ b/qa/qa/specs/features/shared_contexts/variable_inheritance_shared_context.rb @@ -2,6 +2,8 @@ module QA RSpec.shared_context 'variable inheritance test prep' do + let(:key) { 'TEST_VAR' } + let(:value) { 'This is great!' } let(:random_string) { Faker::Alphanumeric.alphanumeric(number: 8) } let(:group) do @@ -57,7 +59,7 @@ module QA Flow::Pipeline.wait_for_latest_pipeline Page::Project::Pipeline::Index.perform(&:click_run_pipeline_button) Page::Project::Pipeline::New.perform do |new| - new.add_variable('TEST_VAR', 'This is great!') + new.add_variable(key, value) new.click_run_pipeline_button end end @@ -80,14 +82,14 @@ module QA def verify_job_log_shows_variable_value Page::Project::Job::Show.perform do |show| show.wait_until { show.successful? } - expect(show.output).to have_content('This is great!') + expect(show.output).to have_content(value) end end def verify_job_log_does_not_show_variable_value Page::Project::Job::Show.perform do |show| show.wait_until { show.successful? } - expect(show.output).to have_no_content('This is great!') + expect(show.output).to have_no_content(value) end end diff --git a/qa/qa/support/matchers/have_matcher.rb b/qa/qa/support/matchers/have_matcher.rb index b96566a9e5d..734a8890536 100644 --- a/qa/qa/support/matchers/have_matcher.rb +++ b/qa/qa/support/matchers/have_matcher.rb @@ -24,6 +24,7 @@ module QA snippet_description tag label + variable ].each do |predicate| RSpec::Matchers.define "have_#{predicate}" do |*args, **kwargs| match do |page_object| diff --git a/spec/graphql/types/project_statistics_type_spec.rb b/spec/graphql/types/project_statistics_type_spec.rb index f515907b6a8..a958a5150aa 100644 --- a/spec/graphql/types/project_statistics_type_spec.rb +++ b/spec/graphql/types/project_statistics_type_spec.rb @@ -6,6 +6,7 @@ RSpec.describe GitlabSchema.types['ProjectStatistics'] do it 'has all the required fields' do expect(described_class).to have_graphql_fields(:storage_size, :repository_size, :lfs_objects_size, :build_artifacts_size, :packages_size, :commit_count, - :wiki_size, :snippets_size, :pipeline_artifacts_size, :uploads_size) + :wiki_size, :snippets_size, :pipeline_artifacts_size, + :uploads_size, :container_registry_size) end end diff --git a/spec/support/helpers/gitaly_setup.rb b/spec/support/helpers/gitaly_setup.rb index 0ad83bdeeb2..3bec83ad0d1 100644 --- a/spec/support/helpers/gitaly_setup.rb +++ b/spec/support/helpers/gitaly_setup.rb @@ -31,6 +31,10 @@ module GitalySetup expand_path('tmp/tests/gitaly') end + def runtime_dir + expand_path('tmp/run') + end + def tmp_tests_gitaly_bin_dir File.join(tmp_tests_gitaly_dir, '_build', 'bin') end @@ -259,6 +263,7 @@ module GitalySetup { 'default' => repos_path }, force: true, options: { + runtime_dir: runtime_dir, prometheus_listen_addr: 'localhost:9236' } ) @@ -267,7 +272,7 @@ module GitalySetup { 'default' => repos_path }, force: true, options: { - runtime_dir: File.join(gitaly_dir, "run2"), + runtime_dir: runtime_dir, gitaly_socket: "gitaly2.socket", config_filename: "gitaly2.config.toml" } diff --git a/spec/support/helpers/stub_object_storage.rb b/spec/support/helpers/stub_object_storage.rb index d49a14f7f5b..024f06cae1b 100644 --- a/spec/support/helpers/stub_object_storage.rb +++ b/spec/support/helpers/stub_object_storage.rb @@ -7,11 +7,6 @@ module StubObjectStorage **params) end - def stub_object_storage_pseudonymizer - stub_object_storage(connection_params: Pseudonymizer::Uploader.object_store_credentials, - remote_directory: Pseudonymizer::Uploader.remote_directory) - end - def stub_object_storage_uploader( config:, uploader:, diff --git a/yarn.lock b/yarn.lock index bffac85940a..a221ba6e91f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2785,13 +2785,6 @@ astral-regex@^2.0.0: resolved "https://registry.yarnpkg.com/astral-regex/-/astral-regex-2.0.0.tgz#483143c567aeed4785759c0865786dc77d7d2e31" integrity sha512-Z7tMw1ytTXt5jqMcOP+OQteU1VuNK9Y02uuJtKQ1Sv69jXQKKg5cibLwGJow8yzZP+eAc18EmLGPal0bp36rvQ== -async@^2.6.2: - version "2.6.3" - resolved "https://registry.yarnpkg.com/async/-/async-2.6.3.tgz#d72625e2344a3656e3a3ad4fa749fa83299d82ff" - integrity sha512-zflvls11DCy+dQWzTW2dzuilv8Z5X/pjfmZOWba6TNIVDm+2UDaJmXSOXlasHKfNBs8oo3M0aT50fDEWfKZjXg== - dependencies: - lodash "^4.17.14" - asynckit@^0.4.0: version "0.4.0" resolved "https://registry.yarnpkg.com/asynckit/-/asynckit-0.4.0.tgz#c79ed97f7f34cb8f2ba1bc9790bcc366474b4b79" @@ -4735,7 +4728,7 @@ debug@4.3.4, debug@^4.0.0, debug@^4.0.1, debug@^4.1.0, debug@^4.1.1, debug@^4.3. dependencies: ms "2.1.2" -debug@^3.1.0, debug@^3.1.1, debug@^3.2.6, debug@^3.2.7: +debug@^3.1.0, debug@^3.2.6, debug@^3.2.7: version "3.2.7" resolved "https://registry.yarnpkg.com/debug/-/debug-3.2.7.tgz#72580b7e9145fb39b6676f9c5e5fb100b934179a" integrity sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ== @@ -8339,7 +8332,7 @@ lodash.values@^4.3.0: resolved "https://registry.yarnpkg.com/lodash.values/-/lodash.values-4.3.0.tgz#a3a6c2b0ebecc5c2cba1c17e6e620fe81b53d347" integrity sha1-o6bCsOvsxcLLocF+bmIP6BtT00c= -lodash@^4.17.10, lodash@^4.17.13, lodash@^4.17.14, lodash@^4.17.15, lodash@^4.17.19, lodash@^4.17.20, lodash@^4.17.21, lodash@^4.17.4: +lodash@^4.17.10, lodash@^4.17.13, lodash@^4.17.15, lodash@^4.17.19, lodash@^4.17.20, lodash@^4.17.21, lodash@^4.17.4: version "4.17.21" resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.21.tgz#679591c564c3bffaae8454cf0b3df370c3d6911c" integrity sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg== @@ -9081,7 +9074,7 @@ mixin-deep@^1.2.0: for-in "^1.0.2" is-extendable "^1.0.1" -mkdirp@^0.5.1, mkdirp@^0.5.3, mkdirp@^0.5.5, mkdirp@~0.5.1: +mkdirp@^0.5.1, mkdirp@^0.5.3, mkdirp@~0.5.1: version "0.5.5" resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-0.5.5.tgz#d91cefd62d1436ca0f41620e251288d420099def" integrity sha512-NKmAlESf6jMGym1++R0Ra7wvhV+wFW63FaSOFPwRahvea0gMUcGUhVeAg/0BC0wiv9ih5NYPB1Wn1UEI1/L+xQ== @@ -9923,15 +9916,6 @@ portal-vue@^2.1.6, portal-vue@^2.1.7: resolved "https://registry.yarnpkg.com/portal-vue/-/portal-vue-2.1.7.tgz#ea08069b25b640ca08a5b86f67c612f15f4e4ad4" integrity sha512-+yCno2oB3xA7irTt0EU5Ezw22L2J51uKAacE/6hMPMoO/mx3h4rXFkkBkT4GFsMDv/vEe8TNKC3ujJJ0PTwb6g== -portfinder@^1.0.28: - version "1.0.28" - resolved "https://registry.yarnpkg.com/portfinder/-/portfinder-1.0.28.tgz#67c4622852bd5374dd1dd900f779f53462fac778" - integrity sha512-Se+2isanIcEqf2XMHjyUKskczxbPH7dQnlMjXX6+dybayyHvAf/TCgyMRlzf/B6QDhAEFOGes0pzRo3by4AbMA== - dependencies: - async "^2.6.2" - debug "^3.1.1" - mkdirp "^0.5.5" - posix-character-classes@^0.1.0: version "0.1.1" resolved "https://registry.yarnpkg.com/posix-character-classes/-/posix-character-classes-0.1.1.tgz#01eac0fe3b5af71a2a6c02feabb8c1fef7e00eab" @@ -12923,10 +12907,10 @@ webpack-dev-middleware@^5.3.1: range-parser "^1.2.1" schema-utils "^4.0.0" -webpack-dev-server@4.8.1: - version "4.8.1" - resolved "https://registry.yarnpkg.com/webpack-dev-server/-/webpack-dev-server-4.8.1.tgz#58f9d797710d6e25fa17d6afab8708f958c11a29" - integrity sha512-dwld70gkgNJa33czmcj/PlKY/nOy/BimbrgZRaR9vDATBQAYgLzggR0nxDtPLJiLrMgZwbE6RRfJ5vnBBasTyg== +webpack-dev-server@4.9.0: + version "4.9.0" + resolved "https://registry.yarnpkg.com/webpack-dev-server/-/webpack-dev-server-4.9.0.tgz#737dbf44335bb8bde68f8f39127fc401c97a1557" + integrity sha512-+Nlb39iQSOSsFv0lWUuUTim3jDQO8nhK3E68f//J2r5rIcp4lULHXz2oZ0UVdEeWXEh5lSzYUlzarZhDAeAVQw== dependencies: "@types/bonjour" "^3.5.9" "@types/connect-history-api-fallback" "^1.3.5" @@ -12948,7 +12932,6 @@ webpack-dev-server@4.8.1: ipaddr.js "^2.0.1" open "^8.0.9" p-retry "^4.5.0" - portfinder "^1.0.28" rimraf "^3.0.2" schema-utils "^4.0.0" selfsigned "^2.0.1" -- cgit v1.2.1