summaryrefslogtreecommitdiff
path: root/lib
diff options
context:
space:
mode:
Diffstat (limited to 'lib')
-rw-r--r--lib/api/access_requests.rb2
-rw-r--r--lib/api/admin/ci/variables.rb2
-rw-r--r--lib/api/admin/instance_clusters.rb2
-rw-r--r--lib/api/admin/sidekiq.rb2
-rw-r--r--lib/api/api.rb29
-rw-r--r--lib/api/appearance.rb2
-rw-r--r--lib/api/applications.rb2
-rw-r--r--lib/api/avatar.rb2
-rw-r--r--lib/api/award_emoji.rb14
-rw-r--r--lib/api/badges.rb2
-rw-r--r--lib/api/base.rb25
-rw-r--r--lib/api/boards.rb39
-rw-r--r--lib/api/boards_responses.rb35
-rw-r--r--lib/api/branches.rb2
-rw-r--r--lib/api/broadcast_messages.rb2
-rw-r--r--lib/api/ci/pipeline_schedules.rb2
-rw-r--r--lib/api/ci/pipelines.rb42
-rw-r--r--lib/api/ci/runner.rb31
-rw-r--r--lib/api/ci/runners.rb2
-rw-r--r--lib/api/commit_statuses.rb2
-rw-r--r--lib/api/commits.rb2
-rw-r--r--lib/api/composer_packages.rb2
-rw-r--r--lib/api/conan_package_endpoints.rb2
-rw-r--r--lib/api/concerns/packages/npm_endpoints.rb134
-rw-r--r--lib/api/container_registry_event.rb2
-rw-r--r--lib/api/container_repositories.rb41
-rw-r--r--lib/api/debian_package_endpoints.rb2
-rw-r--r--lib/api/dependency_proxy.rb41
-rw-r--r--lib/api/deploy_keys.rb2
-rw-r--r--lib/api/deploy_tokens.rb2
-rw-r--r--lib/api/deployments.rb2
-rw-r--r--lib/api/discussions.rb20
-rw-r--r--lib/api/entities/board.rb1
-rw-r--r--lib/api/entities/commit_signature.rb19
-rw-r--r--lib/api/entities/container_registry.rb9
-rw-r--r--lib/api/entities/invitation.rb15
-rw-r--r--lib/api/entities/merge_request_changes.rb22
-rw-r--r--lib/api/entities/package.rb3
-rw-r--r--lib/api/entities/package_file.rb1
-rw-r--r--lib/api/entities/package_version.rb2
-rw-r--r--lib/api/entities/project_hook.rb2
-rw-r--r--lib/api/entities/release.rb2
-rw-r--r--lib/api/entities/releases/link.rb6
-rw-r--r--lib/api/entities/user_status.rb1
-rw-r--r--lib/api/environments.rb2
-rw-r--r--lib/api/error_tracking.rb2
-rw-r--r--lib/api/events.rb2
-rw-r--r--lib/api/feature_flag_scopes.rb2
-rw-r--r--lib/api/feature_flags.rb2
-rw-r--r--lib/api/feature_flags_user_lists.rb7
-rw-r--r--lib/api/features.rb12
-rw-r--r--lib/api/files.rb2
-rw-r--r--lib/api/freeze_periods.rb2
-rw-r--r--lib/api/generic_packages.rb2
-rwxr-xr-xlib/api/go_proxy.rb4
-rw-r--r--lib/api/group_boards.rb2
-rw-r--r--lib/api/group_clusters.rb2
-rw-r--r--lib/api/group_container_repositories.rb2
-rw-r--r--lib/api/group_export.rb2
-rw-r--r--lib/api/group_import.rb2
-rw-r--r--lib/api/group_labels.rb16
-rw-r--r--lib/api/group_milestones.rb2
-rw-r--r--lib/api/group_packages.rb2
-rw-r--r--lib/api/group_variables.rb2
-rw-r--r--lib/api/groups.rb4
-rw-r--r--lib/api/helpers.rb15
-rw-r--r--lib/api/helpers/discussions_helpers.rb9
-rw-r--r--lib/api/helpers/issues_helpers.rb3
-rw-r--r--lib/api/helpers/label_helpers.rb12
-rw-r--r--lib/api/helpers/members_helpers.rb17
-rw-r--r--lib/api/helpers/notes_helpers.rb10
-rw-r--r--lib/api/helpers/packages/npm.rb62
-rw-r--r--lib/api/helpers/resource_label_events_helpers.rb7
-rw-r--r--lib/api/import_bitbucket_server.rb2
-rw-r--r--lib/api/import_github.rb7
-rw-r--r--lib/api/internal/base.rb40
-rw-r--r--lib/api/internal/kubernetes.rb2
-rw-r--r--lib/api/internal/lfs.rb2
-rw-r--r--lib/api/internal/pages.rb2
-rw-r--r--lib/api/invitations.rb54
-rw-r--r--lib/api/issue_links.rb2
-rw-r--r--lib/api/issues.rb4
-rw-r--r--lib/api/job_artifacts.rb2
-rw-r--r--lib/api/jobs.rb2
-rw-r--r--lib/api/keys.rb2
-rw-r--r--lib/api/labels.rb8
-rw-r--r--lib/api/lint.rb26
-rw-r--r--lib/api/markdown.rb2
-rw-r--r--lib/api/maven_packages.rb4
-rw-r--r--lib/api/members.rb6
-rw-r--r--lib/api/merge_request_approvals.rb2
-rw-r--r--lib/api/merge_request_diffs.rb2
-rw-r--r--lib/api/merge_requests.rb8
-rw-r--r--lib/api/metrics/dashboard/annotations.rb2
-rw-r--r--lib/api/metrics/user_starred_dashboards.rb2
-rw-r--r--lib/api/namespaces.rb2
-rw-r--r--lib/api/notes.rb10
-rw-r--r--lib/api/notification_settings.rb2
-rw-r--r--lib/api/npm_instance_packages.rb16
-rw-r--r--lib/api/npm_packages.rb173
-rw-r--r--lib/api/npm_project_packages.rb66
-rw-r--r--lib/api/nuget_packages.rb2
-rw-r--r--lib/api/package_files.rb2
-rw-r--r--lib/api/pages.rb2
-rw-r--r--lib/api/pages_domains.rb2
-rw-r--r--lib/api/personal_access_tokens.rb61
-rw-r--r--lib/api/project_clusters.rb2
-rw-r--r--lib/api/project_container_repositories.rb2
-rw-r--r--lib/api/project_events.rb2
-rw-r--r--lib/api/project_export.rb2
-rw-r--r--lib/api/project_hooks.rb3
-rw-r--r--lib/api/project_import.rb2
-rw-r--r--lib/api/project_milestones.rb2
-rw-r--r--lib/api/project_packages.rb2
-rw-r--r--lib/api/project_repository_storage_moves.rb2
-rw-r--r--lib/api/project_snapshots.rb2
-rw-r--r--lib/api/project_snippets.rb2
-rw-r--r--lib/api/project_statistics.rb2
-rw-r--r--lib/api/project_templates.rb2
-rw-r--r--lib/api/projects.rb50
-rw-r--r--lib/api/protected_branches.rb2
-rw-r--r--lib/api/protected_tags.rb2
-rw-r--r--lib/api/pypi_packages.rb2
-rw-r--r--lib/api/release/links.rb2
-rw-r--r--lib/api/releases.rb4
-rw-r--r--lib/api/remote_mirrors.rb2
-rw-r--r--lib/api/repositories.rb4
-rw-r--r--lib/api/resource_label_events.rb6
-rw-r--r--lib/api/resource_milestone_events.rb9
-rw-r--r--lib/api/resource_state_events.rb9
-rw-r--r--lib/api/search.rb7
-rw-r--r--lib/api/services.rb2
-rw-r--r--lib/api/settings.rb9
-rw-r--r--lib/api/sidekiq_metrics.rb2
-rw-r--r--lib/api/snippets.rb2
-rw-r--r--lib/api/statistics.rb2
-rw-r--r--lib/api/submodules.rb2
-rw-r--r--lib/api/subscriptions.rb16
-rw-r--r--lib/api/suggestions.rb2
-rw-r--r--lib/api/system_hooks.rb2
-rw-r--r--lib/api/tags.rb12
-rw-r--r--lib/api/templates.rb2
-rw-r--r--lib/api/terraform/state.rb4
-rw-r--r--lib/api/terraform/state_version.rb2
-rw-r--r--lib/api/todos.rb2
-rw-r--r--lib/api/triggers.rb2
-rw-r--r--lib/api/unleash.rb2
-rw-r--r--lib/api/usage_data.rb2
-rw-r--r--lib/api/user_counts.rb2
-rw-r--r--lib/api/users.rb131
-rw-r--r--lib/api/v3/github.rb2
-rw-r--r--lib/api/validations/validators/email_or_email_list.rb21
-rw-r--r--lib/api/variables.rb2
-rw-r--r--lib/api/version.rb2
-rw-r--r--lib/api/wikis.rb2
-rw-r--r--lib/atlassian/jira_connect/client.rb18
-rw-r--r--lib/atlassian/jira_connect/serializers/base_entity.rb6
-rw-r--r--lib/atlassian/jira_connect/serializers/pull_request_entity.rb8
-rw-r--r--lib/atlassian/jira_connect/serializers/repository_entity.rb10
-rw-r--r--lib/backup/files.rb61
-rw-r--r--lib/banzai/filter/emoji_filter.rb7
-rw-r--r--lib/banzai/filter/normalize_source_filter.rb14
-rw-r--r--lib/banzai/filter/reference_filter.rb6
-rw-r--r--lib/banzai/filter/vulnerability_reference_filter.rb22
-rw-r--r--lib/banzai/pipeline/pre_process_pipeline.rb1
-rw-r--r--lib/banzai/reference_parser/base_parser.rb5
-rw-r--r--lib/banzai/reference_parser/vulnerability_parser.rb16
-rw-r--r--lib/bitbucket_server/client.rb4
-rw-r--r--lib/bulk_imports/clients/graphql.rb49
-rw-r--r--lib/bulk_imports/clients/http.rb (renamed from lib/gitlab/bulk_import/client.rb)24
-rw-r--r--lib/bulk_imports/common/extractors/graphql_extractor.rb46
-rw-r--r--lib/bulk_imports/common/loaders/entity_loader.rb15
-rw-r--r--lib/bulk_imports/common/transformers/graphql_cleaner_transformer.rb54
-rw-r--r--lib/bulk_imports/common/transformers/underscorify_keys_transformer.rb19
-rw-r--r--lib/bulk_imports/groups/extractors/subgroups_extractor.rb29
-rw-r--r--lib/bulk_imports/groups/graphql/get_group_query.rb38
-rw-r--r--lib/bulk_imports/groups/loaders/group_loader.rb35
-rw-r--r--lib/bulk_imports/groups/pipelines/group_pipeline.rb19
-rw-r--r--lib/bulk_imports/groups/pipelines/subgroup_entities_pipeline.rb15
-rw-r--r--lib/bulk_imports/groups/transformers/group_attributes_transformer.rb81
-rw-r--r--lib/bulk_imports/groups/transformers/subgroup_to_entity_transformer.rb21
-rw-r--r--lib/bulk_imports/importers/group_importer.rb32
-rw-r--r--lib/bulk_imports/importers/groups_importer.rb36
-rw-r--r--lib/bulk_imports/pipeline.rb12
-rw-r--r--lib/bulk_imports/pipeline/attributes.rb41
-rw-r--r--lib/bulk_imports/pipeline/context.rb33
-rw-r--r--lib/bulk_imports/pipeline/runner.rb66
-rw-r--r--lib/container_registry/client.rb6
-rw-r--r--lib/csv_builders/stream.rb17
-rw-r--r--lib/expand_variables.rb28
-rw-r--r--lib/extracts_ref.rb2
-rw-r--r--lib/feature/shared.rb4
-rw-r--r--lib/gitlab/analytics/cycle_analytics/stage_events/code_stage_start.rb34
-rw-r--r--lib/gitlab/analytics/instance_statistics/workers_argument_builder.rb27
-rw-r--r--lib/gitlab/application_rate_limiter.rb3
-rw-r--r--lib/gitlab/auth.rb2
-rw-r--r--lib/gitlab/auth/auth_finders.rb2
-rw-r--r--lib/gitlab/background_migration/backfill_design_internal_ids.rb130
-rw-r--r--lib/gitlab/background_migration/backfill_jira_tracker_deployment_type2.rb86
-rw-r--r--lib/gitlab/background_migration/backfill_merge_request_cleanup_schedules.rb40
-rw-r--r--lib/gitlab/background_migration/populate_has_vulnerabilities.rb62
-rw-r--r--lib/gitlab/background_migration/populate_missing_vulnerability_dismissal_information.rb86
-rw-r--r--lib/gitlab/background_migration/populate_vulnerability_feedback_pipeline_id.rb13
-rw-r--r--lib/gitlab/background_migration/replace_blocked_by_links.rb19
-rw-r--r--lib/gitlab/badge/coverage/report.rb38
-rw-r--r--lib/gitlab/bitbucket_server_import/importer.rb71
-rw-r--r--lib/gitlab/chat/output.rb44
-rw-r--r--lib/gitlab/ci/build/rules/rule/clause/changes.rb12
-rw-r--r--lib/gitlab/ci/charts.rb81
-rw-r--r--lib/gitlab/ci/config/entry/bridge.rb11
-rw-r--r--lib/gitlab/ci/config/entry/job.rb11
-rw-r--r--lib/gitlab/ci/config/entry/processable.rb11
-rw-r--r--lib/gitlab/ci/config/entry/product/variables.rb6
-rw-r--r--lib/gitlab/ci/config/external/mapper.rb10
-rw-r--r--lib/gitlab/ci/features.rb22
-rw-r--r--lib/gitlab/ci/jwt.rb16
-rw-r--r--lib/gitlab/ci/lint.rb2
-rw-r--r--lib/gitlab/ci/pipeline/chain/cancel_pending_pipelines.rb10
-rw-r--r--lib/gitlab/ci/pipeline/chain/seed.rb10
-rw-r--r--lib/gitlab/ci/pipeline/chain/seed_block.rb31
-rw-r--r--lib/gitlab/ci/pipeline/seed/environment.rb15
-rw-r--r--lib/gitlab/ci/reports/test_case.rb8
-rw-r--r--lib/gitlab/ci/reports/test_failure_history.rb43
-rw-r--r--lib/gitlab/ci/reports/test_suite_comparer.rb26
-rw-r--r--lib/gitlab/ci/runner_instructions.rb2
-rw-r--r--lib/gitlab/ci/templates/AWS/Deploy-ECS.gitlab-ci.yml1
-rw-r--r--lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml5
-rw-r--r--lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml10
-rw-r--r--lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml3
-rw-r--r--lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml2
-rw-r--r--lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml2
-rw-r--r--lib/gitlab/ci/templates/Jobs/Deploy/ECS.gitlab-ci.yml56
-rw-r--r--lib/gitlab/ci/templates/Managed-Cluster-Applications.gitlab-ci.yml2
-rw-r--r--lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml167
-rw-r--r--lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml3
-rw-r--r--lib/gitlab/ci/templates/Security/Coverage-Fuzzing.gitlab-ci.yml8
-rw-r--r--lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml2
-rw-r--r--lib/gitlab/ci/templates/Security/Secure-Binaries.gitlab-ci.yml4
-rw-r--r--lib/gitlab/ci/templates/Terraform.latest.gitlab-ci.yml3
-rw-r--r--lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml13
-rw-r--r--lib/gitlab/ci/trace/chunked_io.rb4
-rw-r--r--lib/gitlab/conflict/file.rb47
-rw-r--r--lib/gitlab/current_settings.rb4
-rw-r--r--lib/gitlab/danger/commit_linter.rb2
-rw-r--r--lib/gitlab/danger/helper.rb7
-rw-r--r--lib/gitlab/data_builder/feature_flag.rb19
-rw-r--r--lib/gitlab/database/batch_count.rb4
-rw-r--r--lib/gitlab/database/partitioning/monthly_strategy.rb19
-rw-r--r--lib/gitlab/database/partitioning/replace_table.rb114
-rw-r--r--lib/gitlab/database/partitioning_migration_helpers.rb1
-rw-r--r--lib/gitlab/database/partitioning_migration_helpers/index_helpers.rb90
-rw-r--r--lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb93
-rw-r--r--lib/gitlab/database/postgres_partition.rb23
-rw-r--r--lib/gitlab/database/postgres_partitioned_table.rb35
-rw-r--r--lib/gitlab/database/reindexing.rb1
-rw-r--r--lib/gitlab/dependency_linker/base_linker.rb5
-rw-r--r--lib/gitlab/design_management/copy_design_collection_model_attributes.yml1
-rw-r--r--lib/gitlab/diff/file_collection/merge_request_diff_batch.rb16
-rw-r--r--lib/gitlab/diff/line.rb4
-rw-r--r--lib/gitlab/error_tracking.rb7
-rw-r--r--lib/gitlab/etag_caching/middleware.rb27
-rw-r--r--lib/gitlab/etag_caching/router.rb44
-rw-r--r--lib/gitlab/experimentation.rb180
-rw-r--r--lib/gitlab/experimentation/controller_concern.rb127
-rw-r--r--lib/gitlab/experimentation/group_types.rb10
-rw-r--r--lib/gitlab/git/diff.rb2
-rw-r--r--lib/gitlab/git/repository.rb2
-rw-r--r--lib/gitlab/gitaly_client/repository_service.rb4
-rw-r--r--lib/gitlab/github_import.rb21
-rw-r--r--lib/gitlab/github_import/client.rb53
-rw-r--r--lib/gitlab/github_import/sequential_importer.rb5
-rw-r--r--lib/gitlab/gon_helper.rb7
-rw-r--r--lib/gitlab/grape_logging/formatters/lograge_with_timestamp.rb2
-rw-r--r--lib/gitlab/grape_logging/loggers/content_logger.rb16
-rw-r--r--lib/gitlab/graphql/authorize/authorize_field_service.rb48
-rw-r--r--lib/gitlab/graphql/docs/helper.rb6
-rw-r--r--lib/gitlab/graphql/docs/templates/default.md.haml2
-rw-r--r--lib/gitlab/graphql/lazy.rb26
-rw-r--r--lib/gitlab/graphql/loaders/batch_model_loader.rb11
-rw-r--r--lib/gitlab/graphql/present/instrumentation.rb11
-rw-r--r--lib/gitlab/group_search_results.rb4
-rw-r--r--lib/gitlab/hook_data/release_builder.rb45
-rw-r--r--lib/gitlab/i18n/po_linter.rb18
-rw-r--r--lib/gitlab/import_export/importer.rb14
-rw-r--r--lib/gitlab/import_export/json/ndjson_reader.rb9
-rw-r--r--lib/gitlab/import_export/project/import_export.yml18
-rw-r--r--lib/gitlab/import_export/project/sample/date_calculator.rb1
-rw-r--r--lib/gitlab/import_export/project/sample/relation_factory.rb42
-rw-r--r--lib/gitlab/import_export/project/sample/relation_tree_restorer.rb33
-rw-r--r--lib/gitlab/import_export/project/sample/sample_data_relation_tree_restorer.rb51
-rw-r--r--lib/gitlab/import_export/project/sample/tree_restorer.rb19
-rw-r--r--lib/gitlab/import_export/project/tree_restorer.rb6
-rw-r--r--lib/gitlab/import_export/uploads_manager.rb4
-rw-r--r--lib/gitlab/instrumentation/throttle.rb17
-rw-r--r--lib/gitlab/instrumentation_helper.rb6
-rw-r--r--lib/gitlab/json.rb36
-rw-r--r--lib/gitlab/kubernetes/helm/base_command.rb85
-rw-r--r--lib/gitlab/kubernetes/helm/certificate.rb73
-rw-r--r--lib/gitlab/kubernetes/helm/client_command.rb38
-rw-r--r--lib/gitlab/kubernetes/helm/delete_command.rb36
-rw-r--r--lib/gitlab/kubernetes/helm/init_command.rb43
-rw-r--r--lib/gitlab/kubernetes/helm/install_command.rb85
-rw-r--r--lib/gitlab/kubernetes/helm/patch_command.rb65
-rw-r--r--lib/gitlab/kubernetes/helm/pod.rb9
-rw-r--r--lib/gitlab/kubernetes/helm/reset_command.rb48
-rw-r--r--lib/gitlab/kubernetes/helm/v2/base_command.rb93
-rw-r--r--lib/gitlab/kubernetes/helm/v2/certificate.rb75
-rw-r--r--lib/gitlab/kubernetes/helm/v2/client_command.rb40
-rw-r--r--lib/gitlab/kubernetes/helm/v2/delete_command.rb38
-rw-r--r--lib/gitlab/kubernetes/helm/v2/init_command.rb45
-rw-r--r--lib/gitlab/kubernetes/helm/v2/install_command.rb87
-rw-r--r--lib/gitlab/kubernetes/helm/v2/patch_command.rb67
-rw-r--r--lib/gitlab/kubernetes/helm/v2/reset_command.rb50
-rw-r--r--lib/gitlab/kubernetes/helm/v3/base_command.rb101
-rw-r--r--lib/gitlab/kubernetes/helm/v3/delete_command.rb35
-rw-r--r--lib/gitlab/kubernetes/helm/v3/install_command.rb80
-rw-r--r--lib/gitlab/kubernetes/helm/v3/patch_command.rb60
-rw-r--r--lib/gitlab/kubernetes/kube_client.rb28
-rw-r--r--lib/gitlab/legacy_github_import/client.rb5
-rw-r--r--lib/gitlab/legacy_github_import/importer.rb4
-rw-r--r--lib/gitlab/metrics/requests_rack_middleware.rb55
-rw-r--r--lib/gitlab/middleware/handle_malformed_strings.rb103
-rw-r--r--lib/gitlab/middleware/handle_null_bytes.rb61
-rw-r--r--lib/gitlab/middleware/read_only/controller.rb39
-rw-r--r--lib/gitlab/octokit/middleware.rb2
-rw-r--r--lib/gitlab/omniauth_initializer.rb10
-rw-r--r--lib/gitlab/path_regex.rb11
-rw-r--r--lib/gitlab/project_search_results.rb13
-rw-r--r--lib/gitlab/quick_actions/extractor.rb4
-rw-r--r--lib/gitlab/quick_actions/merge_request_actions.rb12
-rw-r--r--lib/gitlab/redis/wrapper.rb4
-rw-r--r--lib/gitlab/reference_extractor.rb4
-rw-r--r--lib/gitlab/regex.rb4
-rw-r--r--lib/gitlab/repository_size_checker.rb10
-rw-r--r--lib/gitlab/repository_size_error_message.rb8
-rw-r--r--lib/gitlab/repository_url_builder.rb3
-rw-r--r--lib/gitlab/robots_txt/parser.rb60
-rw-r--r--lib/gitlab/search/found_blob.rb3
-rw-r--r--lib/gitlab/search/sort_options.rb21
-rw-r--r--lib/gitlab/search_results.rb21
-rw-r--r--lib/gitlab/setup_helper.rb1
-rw-r--r--lib/gitlab/sidekiq_cluster/cli.rb23
-rw-r--r--lib/gitlab/sidekiq_logging/logs_jobs.rb2
-rw-r--r--lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies.rb1
-rw-r--r--lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/base.rb37
-rw-r--r--lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/deduplicates_when_scheduling.rb47
-rw-r--r--lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/none.rb5
-rw-r--r--lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/until_executed.rb22
-rw-r--r--lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/until_executing.rb40
-rw-r--r--lib/gitlab/static_site_editor/config/generated_config.rb4
-rw-r--r--lib/gitlab/template/base_template.rb14
-rw-r--r--lib/gitlab/tracking.rb25
-rw-r--r--lib/gitlab/tracking/destinations/base.rb13
-rw-r--r--lib/gitlab/tracking/destinations/snowplow.rb49
-rw-r--r--lib/gitlab/url_blocker.rb18
-rw-r--r--lib/gitlab/url_blockers/domain_allowlist_entry.rb (renamed from lib/gitlab/url_blockers/domain_whitelist_entry.rb)2
-rw-r--r--lib/gitlab/url_blockers/ip_allowlist_entry.rb (renamed from lib/gitlab/url_blockers/ip_whitelist_entry.rb)2
-rw-r--r--lib/gitlab/url_blockers/url_allowlist.rb (renamed from lib/gitlab/url_blockers/url_whitelist.rb)24
-rw-r--r--lib/gitlab/url_builder.rb2
-rw-r--r--lib/gitlab/usage_data.rb69
-rw-r--r--lib/gitlab/usage_data_counters/aggregated_metrics/common.yml17
-rw-r--r--lib/gitlab/usage_data_counters/designs_counter.rb38
-rw-r--r--lib/gitlab/usage_data_counters/hll_redis_counter.rb244
-rw-r--r--lib/gitlab/usage_data_counters/issue_activity_unique_counter.rb42
-rw-r--r--lib/gitlab/usage_data_counters/known_events/common.yml (renamed from lib/gitlab/usage_data_counters/known_events.yml)99
-rw-r--r--lib/gitlab/usage_data_counters/known_events/package_events.yml265
-rw-r--r--lib/gitlab/usage_data_counters/static_site_editor_counter.rb2
-rw-r--r--lib/gitlab/usage_data_counters/track_unique_events.rb11
-rw-r--r--lib/gitlab/usage_data_counters/web_ide_counter.rb31
-rw-r--r--lib/gitlab/user_access.rb6
-rw-r--r--lib/gitlab/webpack/dev_server_middleware.rb12
-rw-r--r--lib/gitlab/whats_new.rb26
-rw-r--r--lib/gitlab/with_feature_category.rb50
-rw-r--r--lib/gitlab/workhorse.rb2
-rw-r--r--lib/quality/test_level.rb17
-rw-r--r--lib/rouge/formatters/html_gitlab.rb2
-rw-r--r--lib/rspec_flaky/flaky_examples_collection.rb1
-rw-r--r--lib/support/nginx/gitlab2
-rw-r--r--lib/support/nginx/gitlab-ssl2
-rw-r--r--lib/system_check/app/git_version_check.rb2
-rw-r--r--lib/tasks/gitlab/gitaly.rake2
-rw-r--r--lib/tasks/gitlab/packages/events.rake40
382 files changed, 6389 insertions, 1912 deletions
diff --git a/lib/api/access_requests.rb b/lib/api/access_requests.rb
index 7e3d70a210a..e6ce62a1c6e 100644
--- a/lib/api/access_requests.rb
+++ b/lib/api/access_requests.rb
@@ -8,6 +8,8 @@ module API
helpers ::API::Helpers::MembersHelpers
+ feature_category :authentication_and_authorization
+
%w[group project].each do |source_type|
params do
requires :id, type: String, desc: "The #{source_type} ID"
diff --git a/lib/api/admin/ci/variables.rb b/lib/api/admin/ci/variables.rb
index 44c389d6f94..654d3a48162 100644
--- a/lib/api/admin/ci/variables.rb
+++ b/lib/api/admin/ci/variables.rb
@@ -8,6 +8,8 @@ module API
before { authenticated_as_admin! }
+ feature_category :continuous_integration
+
namespace 'admin' do
namespace 'ci' do
namespace 'variables' do
diff --git a/lib/api/admin/instance_clusters.rb b/lib/api/admin/instance_clusters.rb
index ce1bdd65eff..679e231b283 100644
--- a/lib/api/admin/instance_clusters.rb
+++ b/lib/api/admin/instance_clusters.rb
@@ -5,6 +5,8 @@ module API
class InstanceClusters < ::API::Base
include PaginationParams
+ feature_category :kubernetes_management
+
before do
authenticated_as_admin!
end
diff --git a/lib/api/admin/sidekiq.rb b/lib/api/admin/sidekiq.rb
index c2e9de5fb4e..7e561783685 100644
--- a/lib/api/admin/sidekiq.rb
+++ b/lib/api/admin/sidekiq.rb
@@ -5,6 +5,8 @@ module API
class Sidekiq < ::API::Base
before { authenticated_as_admin! }
+ feature_category :not_owned
+
namespace 'admin' do
namespace 'sidekiq' do
namespace 'queues' do
diff --git a/lib/api/api.rb b/lib/api/api.rb
index 84b4d5a5835..ea149f25584 100644
--- a/lib/api/api.rb
+++ b/lib/api/api.rb
@@ -25,7 +25,8 @@ module API
Gitlab::GrapeLogging::Loggers::QueueDurationLogger.new,
Gitlab::GrapeLogging::Loggers::PerfLogger.new,
Gitlab::GrapeLogging::Loggers::CorrelationIdLogger.new,
- Gitlab::GrapeLogging::Loggers::ContextLogger.new
+ Gitlab::GrapeLogging::Loggers::ContextLogger.new,
+ Gitlab::GrapeLogging::Loggers::ContentLogger.new
]
allow_access_with_scope :api
@@ -48,11 +49,17 @@ module API
before do
coerce_nil_params_to_array!
+ api_endpoint = env['api.endpoint']
+ feature_category = api_endpoint.options[:for].try(:feature_category_for_app, api_endpoint).to_s
+
+ header[Gitlab::Metrics::RequestsRackMiddleware::FEATURE_CATEGORY_HEADER] = feature_category
+
Gitlab::ApplicationContext.push(
user: -> { @current_user },
project: -> { @project },
namespace: -> { @group },
- caller_id: route.origin
+ caller_id: route.origin,
+ feature_category: feature_category
)
end
@@ -115,7 +122,14 @@ module API
format :json
formatter :json, Gitlab::Json::GrapeFormatter
- content_type :txt, "text/plain"
+
+ # There is a small chance some users depend on the old behavior.
+ # We this change under a feature flag to see if affects GitLab.com users.
+ if Gitlab::Database.cached_table_exists?('features') && Feature.enabled?(:api_json_content_type)
+ content_type :json, 'application/json'
+ else
+ content_type :txt, 'text/plain'
+ end
# Ensure the namespace is right, otherwise we might load Grape::API::Helpers
helpers ::API::Helpers
@@ -147,6 +161,8 @@ module API
mount ::API::Commits
mount ::API::CommitStatuses
mount ::API::ContainerRegistryEvent
+ mount ::API::ContainerRepositories
+ mount ::API::DependencyProxy
mount ::API::DeployKeys
mount ::API::DeployTokens
mount ::API::Deployments
@@ -171,6 +187,7 @@ module API
mount ::API::ImportBitbucketServer
mount ::API::ImportGithub
mount ::API::IssueLinks
+ mount ::API::Invitations
mount ::API::Issues
mount ::API::JobArtifacts
mount ::API::Jobs
@@ -202,7 +219,8 @@ module API
mount ::API::DebianGroupPackages
mount ::API::DebianProjectPackages
mount ::API::MavenPackages
- mount ::API::NpmPackages
+ mount ::API::NpmProjectPackages
+ mount ::API::NpmInstancePackages
mount ::API::GenericPackages
mount ::API::GoProxy
mount ::API::Pages
@@ -222,6 +240,7 @@ module API
mount ::API::ProjectTemplates
mount ::API::Terraform::State
mount ::API::Terraform::StateVersion
+ mount ::API::PersonalAccessTokens
mount ::API::ProtectedBranches
mount ::API::ProtectedTags
mount ::API::Releases
@@ -266,7 +285,7 @@ module API
end
end
- route :any, '*path' do
+ route :any, '*path', feature_category: :not_owned do
error!('404 Not Found', 404)
end
end
diff --git a/lib/api/appearance.rb b/lib/api/appearance.rb
index 00b495bbc1e..fe498bf611b 100644
--- a/lib/api/appearance.rb
+++ b/lib/api/appearance.rb
@@ -4,6 +4,8 @@ module API
class Appearance < ::API::Base
before { authenticated_as_admin! }
+ feature_category :navigation
+
helpers do
def current_appearance
@current_appearance ||= (::Appearance.current || ::Appearance.new)
diff --git a/lib/api/applications.rb b/lib/api/applications.rb
index 2afe8763d9d..8b14e16b495 100644
--- a/lib/api/applications.rb
+++ b/lib/api/applications.rb
@@ -5,6 +5,8 @@ module API
class Applications < ::API::Base
before { authenticated_as_admin! }
+ feature_category :authentication_and_authorization
+
resource :applications do
helpers do
def validate_redirect_uri(value)
diff --git a/lib/api/avatar.rb b/lib/api/avatar.rb
index 5a9b9940fcf..a42d89ddf83 100644
--- a/lib/api/avatar.rb
+++ b/lib/api/avatar.rb
@@ -2,6 +2,8 @@
module API
class Avatar < ::API::Base
+ feature_category :users
+
resource :avatar do
desc 'Return avatar url for a user' do
success Entities::Avatar
diff --git a/lib/api/award_emoji.rb b/lib/api/award_emoji.rb
index 6d40ae8f5ff..8ea4f32d3eb 100644
--- a/lib/api/award_emoji.rb
+++ b/lib/api/award_emoji.rb
@@ -6,9 +6,9 @@ module API
before { authenticate! }
AWARDABLES = [
- { type: 'issue', find_by: :iid },
- { type: 'merge_request', find_by: :iid },
- { type: 'snippet', find_by: :id }
+ { type: 'issue', find_by: :iid, feature_category: :issue_tracking },
+ { type: 'merge_request', find_by: :iid, feature_category: :code_review },
+ { type: 'snippet', find_by: :id, feature_category: :snippets }
].freeze
params do
@@ -34,7 +34,7 @@ module API
params do
use :pagination
end
- get endpoint do
+ get endpoint, feature_category: awardable_params[:feature_category] do
if can_read_awardable?
awards = awardable.award_emoji
present paginate(awards), with: Entities::AwardEmoji
@@ -50,7 +50,7 @@ module API
params do
requires :award_id, type: Integer, desc: 'The ID of the award'
end
- get "#{endpoint}/:award_id" do
+ get "#{endpoint}/:award_id", feature_category: awardable_params[:feature_category] do
if can_read_awardable?
present awardable.award_emoji.find(params[:award_id]), with: Entities::AwardEmoji
else
@@ -65,7 +65,7 @@ module API
params do
requires :name, type: String, desc: 'The name of a award_emoji (without colons)'
end
- post endpoint do
+ post endpoint, feature_category: awardable_params[:feature_category] do
not_found!('Award Emoji') unless can_read_awardable? && can_award_awardable?
service = AwardEmojis::AddService.new(awardable, params[:name], current_user).execute
@@ -84,7 +84,7 @@ module API
params do
requires :award_id, type: Integer, desc: 'The ID of an award emoji'
end
- delete "#{endpoint}/:award_id" do
+ delete "#{endpoint}/:award_id", feature_category: awardable_params[:feature_category] do
award = awardable.award_emoji.find(params[:award_id])
unauthorized! unless award.user == current_user || current_user.admin?
diff --git a/lib/api/badges.rb b/lib/api/badges.rb
index fc00594c9ec..04f155be4e1 100644
--- a/lib/api/badges.rb
+++ b/lib/api/badges.rb
@@ -8,6 +8,8 @@ module API
helpers ::API::Helpers::BadgesHelpers
+ feature_category :continuous_integration
+
helpers do
def find_source_if_admin(source_type)
source = find_source(source_type, params[:id])
diff --git a/lib/api/base.rb b/lib/api/base.rb
index e174cef3bad..33e47c18fcd 100644
--- a/lib/api/base.rb
+++ b/lib/api/base.rb
@@ -2,5 +2,30 @@
module API
class Base < Grape::API::Instance # rubocop:disable API/Base
+ include ::Gitlab::WithFeatureCategory
+
+ class << self
+ def feature_category_for_app(app)
+ feature_category_for_action(path_for_app(app))
+ end
+
+ def path_for_app(app)
+ normalize_path(app.namespace, app.options[:path].first)
+ end
+
+ def route(methods, paths = ['/'], route_options = {}, &block)
+ if category = route_options.delete(:feature_category)
+ feature_category(category, Array(paths).map { |path| normalize_path(namespace, path) })
+ end
+
+ super
+ end
+
+ private
+
+ def normalize_path(namespace, path)
+ [namespace.presence, path.to_s.chomp('/').presence].compact.join('/')
+ end
+ end
end
end
diff --git a/lib/api/boards.rb b/lib/api/boards.rb
index d2d1628aff4..f4b23c507f4 100644
--- a/lib/api/boards.rb
+++ b/lib/api/boards.rb
@@ -9,6 +9,8 @@ module API
before { authenticate! }
+ feature_category :boards
+
helpers do
def board_parent
user_project
@@ -40,6 +42,43 @@ module API
authorize!(:read_board, user_project)
present board, with: Entities::Board
end
+
+ desc 'Create a project board' do
+ detail 'This feature was introduced in 10.4'
+ success Entities::Board
+ end
+ params do
+ requires :name, type: String, desc: 'The board name'
+ end
+ post '/' do
+ authorize!(:admin_board, board_parent)
+
+ create_board
+ end
+
+ desc 'Update a project board' do
+ detail 'This feature was introduced in 11.0'
+ success Entities::Board
+ end
+ params do
+ use :update_params
+ end
+ put '/:board_id' do
+ authorize!(:admin_board, board_parent)
+
+ update_board
+ end
+
+ desc 'Delete a project board' do
+ detail 'This feature was introduced in 10.4'
+ success Entities::Board
+ end
+
+ delete '/:board_id' do
+ authorize!(:admin_board, board_parent)
+
+ delete_board
+ end
end
params do
diff --git a/lib/api/boards_responses.rb b/lib/api/boards_responses.rb
index 6a86c02bf4a..2ae82f78e01 100644
--- a/lib/api/boards_responses.rb
+++ b/lib/api/boards_responses.rb
@@ -10,6 +10,35 @@ module API
board_parent.boards.find(params[:board_id])
end
+ def create_board
+ forbidden! unless board_parent.multiple_issue_boards_available?
+
+ response =
+ ::Boards::CreateService.new(board_parent, current_user, { name: params[:name] }).execute
+
+ present response.payload, with: Entities::Board
+ end
+
+ def update_board
+ service = ::Boards::UpdateService.new(board_parent, current_user, declared_params(include_missing: false))
+ service.execute(board)
+
+ if board.valid?
+ present board, with: Entities::Board
+ else
+ bad_request!("Failed to save board #{board.errors.messages}")
+ end
+ end
+
+ def delete_board
+ forbidden! unless board_parent.multiple_issue_boards_available?
+
+ destroy_conditionally!(board) do |board|
+ service = ::Boards::DestroyService.new(board_parent, current_user)
+ service.execute(board)
+ end
+ end
+
def board_lists
board.destroyable_lists
end
@@ -62,6 +91,12 @@ module API
params :list_creation_params do
requires :label_id, type: Integer, desc: 'The ID of an existing label'
end
+
+ params :update_params do
+ # Configurable issue boards are not available in CE/EE Core.
+ # https://docs.gitlab.com/ee/user/project/issue_board.html#configurable-issue-boards
+ optional :name, type: String, desc: 'The board name'
+ end
end
end
end
diff --git a/lib/api/branches.rb b/lib/api/branches.rb
index 37cce6eafba..6842e93a4de 100644
--- a/lib/api/branches.rb
+++ b/lib/api/branches.rb
@@ -10,6 +10,8 @@ module API
after_validation { content_type "application/json" }
+ feature_category :source_code_management
+
before do
require_repository_enabled!
authorize! :download_code, user_project
diff --git a/lib/api/broadcast_messages.rb b/lib/api/broadcast_messages.rb
index 8ce7694bbfd..0762c276aad 100644
--- a/lib/api/broadcast_messages.rb
+++ b/lib/api/broadcast_messages.rb
@@ -4,6 +4,8 @@ module API
class BroadcastMessages < ::API::Base
include PaginationParams
+ feature_category :navigation
+
resource :broadcast_messages do
helpers do
def find_message
diff --git a/lib/api/ci/pipeline_schedules.rb b/lib/api/ci/pipeline_schedules.rb
index b669acf668c..8a9ba2cbe0f 100644
--- a/lib/api/ci/pipeline_schedules.rb
+++ b/lib/api/ci/pipeline_schedules.rb
@@ -7,6 +7,8 @@ module API
before { authenticate! }
+ feature_category :continuous_integration
+
params do
requires :id, type: String, desc: 'The ID of a project'
end
diff --git a/lib/api/ci/pipelines.rb b/lib/api/ci/pipelines.rb
index 61e03ed1a95..1b36e27f6c9 100644
--- a/lib/api/ci/pipelines.rb
+++ b/lib/api/ci/pipelines.rb
@@ -7,6 +7,8 @@ module API
before { authenticate_non_get! }
+ feature_category :continuous_integration
+
params do
requires :id, type: String, desc: 'The project ID'
end
@@ -128,15 +130,9 @@ module API
pipeline = user_project.all_pipelines.find(params[:pipeline_id])
- if Feature.enabled?(:ci_jobs_finder_refactor, default_enabled: true)
- builds = ::Ci::JobsFinder
- .new(current_user: current_user, pipeline: pipeline, params: params)
- .execute
- else
- authorize!(:read_build, pipeline)
- builds = pipeline.builds
- builds = filter_builds(builds, params[:scope])
- end
+ builds = ::Ci::JobsFinder
+ .new(current_user: current_user, pipeline: pipeline, params: params)
+ .execute
builds = builds.with_preloads
@@ -157,16 +153,9 @@ module API
pipeline = user_project.all_pipelines.find(params[:pipeline_id])
- if Feature.enabled?(:ci_jobs_finder_refactor, default_enabled: true)
- bridges = ::Ci::JobsFinder
- .new(current_user: current_user, pipeline: pipeline, params: params, type: ::Ci::Bridge)
- .execute
- else
- authorize!(:read_pipeline, pipeline)
- bridges = pipeline.bridges
- bridges = filter_builds(bridges, params[:scope])
- end
-
+ bridges = ::Ci::JobsFinder
+ .new(current_user: current_user, pipeline: pipeline, params: params, type: ::Ci::Bridge)
+ .execute
bridges = bridges.with_preloads
present paginate(bridges), with: Entities::Ci::Bridge
@@ -246,21 +235,6 @@ module API
end
helpers do
- # NOTE: This method should be removed once the ci_jobs_finder_refactor FF is
- # removed. https://gitlab.com/gitlab-org/gitlab/-/issues/245183
- # rubocop: disable CodeReuse/ActiveRecord
- def filter_builds(builds, scope)
- return builds if scope.nil? || scope.empty?
-
- available_statuses = ::CommitStatus::AVAILABLE_STATUSES
-
- unknown = scope - available_statuses
- render_api_error!('Scope contains invalid value(s)', 400) unless unknown.empty?
-
- builds.where(status: scope)
- end
- # rubocop: enable CodeReuse/ActiveRecord
-
def pipeline
strong_memoize(:pipeline) do
user_project.all_pipelines.find(params[:pipeline_id])
diff --git a/lib/api/ci/runner.rb b/lib/api/ci/runner.rb
index ef679147c9f..85232b4ae1b 100644
--- a/lib/api/ci/runner.rb
+++ b/lib/api/ci/runner.rb
@@ -5,6 +5,10 @@ module API
class Runner < ::API::Base
helpers ::API::Helpers::Runner
+ content_type :txt, 'text/plain'
+
+ feature_category :continuous_integration
+
resource :runners do
desc 'Registers a new Runner' do
success Entities::RunnerRegistrationDetails
@@ -203,27 +207,18 @@ module API
error!('400 Missing header Content-Range', 400) unless request.headers.key?('Content-Range')
content_range = request.headers['Content-Range']
- content_range = content_range.split('-')
-
- # TODO:
- # it seems that `Content-Range` as formatted by runner is wrong,
- # the `byte_end` should point to final byte, but it points byte+1
- # that means that we have to calculate end of body,
- # as we cannot use `content_length[1]`
- # Issue: https://gitlab.com/gitlab-org/gitlab-runner/issues/3275
-
- body_data = request.body.read
- body_start = content_range[0].to_i
- body_end = body_start + body_data.bytesize
-
- stream_size = job.trace.append(body_data, body_start)
- unless stream_size == body_end
- break error!('416 Range Not Satisfiable', 416, { 'Range' => "0-#{stream_size}" })
+
+ result = ::Ci::AppendBuildTraceService
+ .new(job, content_range: content_range)
+ .execute(request.body.read)
+
+ if result.status == 416
+ break error!('416 Range Not Satisfiable', 416, { 'Range' => "0-#{result.stream_size}" })
end
- status 202
+ status result.status
header 'Job-Status', job.status
- header 'Range', "0-#{stream_size}"
+ header 'Range', "0-#{result.stream_size}"
header 'X-GitLab-Trace-Update-Interval', job.trace.update_interval.to_s
end
diff --git a/lib/api/ci/runners.rb b/lib/api/ci/runners.rb
index d37f10fe631..44ffc941cfa 100644
--- a/lib/api/ci/runners.rb
+++ b/lib/api/ci/runners.rb
@@ -7,6 +7,8 @@ module API
before { authenticate! }
+ feature_category :continuous_integration
+
resource :runners do
desc 'Get runners available for user' do
success Entities::Runner
diff --git a/lib/api/commit_statuses.rb b/lib/api/commit_statuses.rb
index af103b8c1f8..26af921432c 100644
--- a/lib/api/commit_statuses.rb
+++ b/lib/api/commit_statuses.rb
@@ -4,6 +4,8 @@ require 'mime/types'
module API
class CommitStatuses < ::API::Base
+ feature_category :continuous_integration
+
params do
requires :id, type: String, desc: 'The ID of a project'
end
diff --git a/lib/api/commits.rb b/lib/api/commits.rb
index 582ccd41847..a24848082a9 100644
--- a/lib/api/commits.rb
+++ b/lib/api/commits.rb
@@ -6,6 +6,8 @@ module API
class Commits < ::API::Base
include PaginationParams
+ feature_category :source_code_management
+
before do
require_repository_enabled!
authorize! :download_code, user_project
diff --git a/lib/api/composer_packages.rb b/lib/api/composer_packages.rb
index 1becbd668a3..0ac5cc45ccf 100644
--- a/lib/api/composer_packages.rb
+++ b/lib/api/composer_packages.rb
@@ -9,6 +9,8 @@ module API
include ::API::Helpers::Packages::BasicAuthHelpers::Constants
include ::Gitlab::Utils::StrongMemoize
+ feature_category :package_registry
+
content_type :json, 'application/json'
default_format :json
diff --git a/lib/api/conan_package_endpoints.rb b/lib/api/conan_package_endpoints.rb
index 9b6867a328b..188a42f26f8 100644
--- a/lib/api/conan_package_endpoints.rb
+++ b/lib/api/conan_package_endpoints.rb
@@ -29,6 +29,8 @@ module API
CONAN_FILES = (Gitlab::Regex::Packages::CONAN_RECIPE_FILES + Gitlab::Regex::Packages::CONAN_PACKAGE_FILES).freeze
included do
+ feature_category :package_registry
+
helpers ::API::Helpers::PackagesManagerClientsHelpers
helpers ::API::Helpers::Packages::Conan::ApiHelpers
helpers ::API::Helpers::RelatedResourcesHelpers
diff --git a/lib/api/concerns/packages/npm_endpoints.rb b/lib/api/concerns/packages/npm_endpoints.rb
new file mode 100644
index 00000000000..a91db93b182
--- /dev/null
+++ b/lib/api/concerns/packages/npm_endpoints.rb
@@ -0,0 +1,134 @@
+# frozen_string_literal: true
+
+# NPM Package Manager Client API
+#
+# These API endpoints are not consumed directly by users, so there is no documentation for the
+# individual endpoints. They are called by the NPM package manager client when users run commands
+# like `npm install` or `npm publish`. The usage of the GitLab NPM registry is documented here:
+# https://docs.gitlab.com/ee/user/packages/npm_registry/
+#
+# Technical debt: https://gitlab.com/gitlab-org/gitlab/issues/35798
+#
+# Caution: This Concern has to be included at the end of the API class
+# The last route of this Concern has a globbing wildcard that will match all urls.
+# As such, routes declared after the last route of this Concern will not match any url.
+module API
+ module Concerns
+ module Packages
+ module NpmEndpoints
+ extend ActiveSupport::Concern
+
+ included do
+ helpers ::API::Helpers::Packages::DependencyProxyHelpers
+
+ before do
+ require_packages_enabled!
+ authenticate_non_get!
+ end
+
+ params do
+ requires :package_name, type: String, desc: 'Package name'
+ end
+ namespace '-/package/*package_name' do
+ desc 'Get all tags for a given an NPM package' do
+ detail 'This feature was introduced in GitLab 12.7'
+ success ::API::Entities::NpmPackageTag
+ end
+ get 'dist-tags', format: false, requirements: ::API::Helpers::Packages::Npm::NPM_ENDPOINT_REQUIREMENTS do
+ package_name = params[:package_name]
+
+ bad_request!('Package Name') if package_name.blank?
+
+ authorize_read_package!(project)
+
+ packages = ::Packages::Npm::PackageFinder.new(project, package_name)
+ .execute
+
+ not_found! if packages.empty?
+
+ present ::Packages::Npm::PackagePresenter.new(package_name, packages),
+ with: ::API::Entities::NpmPackageTag
+ end
+
+ params do
+ requires :tag, type: String, desc: "Package dist-tag"
+ end
+ namespace 'dist-tags/:tag', requirements: ::API::Helpers::Packages::Npm::NPM_ENDPOINT_REQUIREMENTS do
+ desc 'Create or Update the given tag for the given NPM package and version' do
+ detail 'This feature was introduced in GitLab 12.7'
+ end
+ put format: false do
+ package_name = params[:package_name]
+ version = env['api.request.body']
+ tag = params[:tag]
+
+ bad_request!('Package Name') if package_name.blank?
+ bad_request!('Version') if version.blank?
+ bad_request!('Tag') if tag.blank?
+
+ authorize_create_package!(project)
+
+ package = ::Packages::Npm::PackageFinder
+ .new(project, package_name)
+ .find_by_version(version)
+ not_found!('Package') unless package
+
+ ::Packages::Npm::CreateTagService.new(package, tag).execute
+
+ no_content!
+ end
+
+ desc 'Deletes the given tag' do
+ detail 'This feature was introduced in GitLab 12.7'
+ end
+ delete format: false do
+ package_name = params[:package_name]
+ tag = params[:tag]
+
+ bad_request!('Package Name') if package_name.blank?
+ bad_request!('Tag') if tag.blank?
+
+ authorize_destroy_package!(project)
+
+ package_tag = ::Packages::TagsFinder
+ .new(project, package_name, package_type: :npm)
+ .find_by_name(tag)
+
+ not_found!('Package tag') unless package_tag
+
+ ::Packages::RemoveTagService.new(package_tag).execute
+
+ no_content!
+ end
+ end
+ end
+
+ desc 'NPM registry metadata endpoint' do
+ detail 'This feature was introduced in GitLab 11.8'
+ end
+ params do
+ requires :package_name, type: String, desc: 'Package name'
+ end
+ route_setting :authentication, job_token_allowed: true, deploy_token_allowed: true
+ get '*package_name', format: false, requirements: ::API::Helpers::Packages::Npm::NPM_ENDPOINT_REQUIREMENTS do
+ package_name = params[:package_name]
+
+ packages = ::Packages::Npm::PackageFinder.new(project_or_nil, package_name)
+ .execute
+
+ redirect_request = project_or_nil.blank? || packages.empty?
+
+ redirect_registry_request(redirect_request, :npm, package_name: package_name) do
+ authorize_read_package!(project)
+
+ not_found!('Packages') if packages.empty?
+
+ present ::Packages::Npm::PackagePresenter.new(package_name, packages),
+ with: ::API::Entities::NpmPackage
+ end
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/api/container_registry_event.rb b/lib/api/container_registry_event.rb
index 6c4b80b612a..9bad31f6661 100644
--- a/lib/api/container_registry_event.rb
+++ b/lib/api/container_registry_event.rb
@@ -4,6 +4,8 @@ module API
class ContainerRegistryEvent < ::API::Base
DOCKER_DISTRIBUTION_EVENTS_V1_JSON = 'application/vnd.docker.distribution.events.v1+json'
+ feature_category :package_registry
+
before { authenticate_registry_notification! }
resource :container_registry_event do
diff --git a/lib/api/container_repositories.rb b/lib/api/container_repositories.rb
new file mode 100644
index 00000000000..c84527f26e7
--- /dev/null
+++ b/lib/api/container_repositories.rb
@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module API
+ class ContainerRepositories < ::API::Base
+ include Gitlab::Utils::StrongMemoize
+ helpers ::API::Helpers::PackagesHelpers
+
+ before { authenticate! }
+
+ feature_category :container_registry
+
+ namespace 'registry' do
+ params do
+ requires :id, type: String, desc: 'The ID of a project'
+ end
+ resource :repositories, requirements: { id: /[0-9]*/ } do
+ desc 'Get a container repository' do
+ detail 'This feature was introduced in GitLab 13.6.'
+ success Entities::ContainerRegistry::Repository
+ end
+ params do
+ optional :tags, type: Boolean, default: false, desc: 'Determines if tags should be included'
+ optional :tags_count, type: Boolean, default: false, desc: 'Determines if the tags count should be included'
+ end
+ get ':id' do
+ authorize!(:read_container_image, repository)
+
+ present repository, with: Entities::ContainerRegistry::Repository, tags: params[:tags], tags_count: params[:tags_count], user: current_user
+ end
+ end
+ end
+
+ helpers do
+ def repository
+ strong_memoize(:repository) do
+ ContainerRepository.find(params[:id])
+ end
+ end
+ end
+ end
+end
diff --git a/lib/api/debian_package_endpoints.rb b/lib/api/debian_package_endpoints.rb
index 168b3ca7a4f..c95c75b7e5c 100644
--- a/lib/api/debian_package_endpoints.rb
+++ b/lib/api/debian_package_endpoints.rb
@@ -26,6 +26,8 @@ module API
}.freeze
included do
+ feature_category :package_registry
+
helpers ::API::Helpers::PackagesHelpers
helpers ::API::Helpers::Packages::BasicAuthHelpers
diff --git a/lib/api/dependency_proxy.rb b/lib/api/dependency_proxy.rb
new file mode 100644
index 00000000000..3379bb2f029
--- /dev/null
+++ b/lib/api/dependency_proxy.rb
@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module API
+ class DependencyProxy < ::API::Base
+ helpers ::API::Helpers::PackagesHelpers
+
+ feature_category :dependency_proxy
+
+ helpers do
+ def obtain_new_purge_cache_lease
+ Gitlab::ExclusiveLease
+ .new("dependency_proxy:delete_group_blobs:#{user_group.id}",
+ timeout: 1.hour)
+ .try_obtain
+ end
+ end
+
+ before do
+ authorize! :admin_group, user_group
+ end
+
+ params do
+ requires :id, type: String, desc: 'The ID of a group'
+ end
+ resource :groups, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do
+ desc 'Deletes all dependency_proxy_blobs for a group' do
+ detail 'This feature was introduced in GitLab 12.10'
+ end
+ delete ':id/dependency_proxy/cache' do
+ not_found! unless user_group.dependency_proxy_feature_available?
+
+ message = 'This request has already been made. It may take some time to purge the cache. You can run this at most once an hour for a given group'
+ render_api_error!(message, 409) unless obtain_new_purge_cache_lease
+
+ # rubocop:disable CodeReuse/Worker
+ PurgeDependencyProxyCacheWorker.perform_async(current_user.id, user_group.id)
+ # rubocop:enable CodeReuse/Worker
+ end
+ end
+ end
+end
diff --git a/lib/api/deploy_keys.rb b/lib/api/deploy_keys.rb
index 314f5b6ee1d..0a541620c3a 100644
--- a/lib/api/deploy_keys.rb
+++ b/lib/api/deploy_keys.rb
@@ -6,6 +6,8 @@ module API
before { authenticate! }
+ feature_category :continuous_delivery
+
helpers do
def add_deploy_keys_project(project, attrs = {})
project.deploy_keys_projects.create(attrs)
diff --git a/lib/api/deploy_tokens.rb b/lib/api/deploy_tokens.rb
index 1c156b8b3bb..5fab590eb4e 100644
--- a/lib/api/deploy_tokens.rb
+++ b/lib/api/deploy_tokens.rb
@@ -4,6 +4,8 @@ module API
class DeployTokens < ::API::Base
include PaginationParams
+ feature_category :continuous_delivery
+
helpers do
def scope_params
scopes = params.delete(:scopes)
diff --git a/lib/api/deployments.rb b/lib/api/deployments.rb
index ff06bdbae16..5346fcf03c9 100644
--- a/lib/api/deployments.rb
+++ b/lib/api/deployments.rb
@@ -7,6 +7,8 @@ module API
before { authenticate! }
+ feature_category :continuous_delivery
+
params do
requires :id, type: String, desc: 'The project ID'
end
diff --git a/lib/api/discussions.rb b/lib/api/discussions.rb
index 3d2608c8c5a..4c4ec200060 100644
--- a/lib/api/discussions.rb
+++ b/lib/api/discussions.rb
@@ -8,7 +8,7 @@ module API
before { authenticate! }
- Helpers::DiscussionsHelpers.noteable_types.each do |noteable_type|
+ Helpers::DiscussionsHelpers.feature_category_per_noteable_type.each do |noteable_type, feature_category|
parent_type = noteable_type.parent_class.to_s.underscore
noteables_str = noteable_type.to_s.underscore.pluralize
noteables_path = noteable_type == Commit ? "repository/#{noteables_str}" : noteables_str
@@ -25,7 +25,7 @@ module API
use :pagination
end
- get ":id/#{noteables_path}/:noteable_id/discussions" do
+ get ":id/#{noteables_path}/:noteable_id/discussions", feature_category: feature_category do
noteable = find_noteable(noteable_type, params[:noteable_id])
discussion_ids = paginate(noteable.discussion_ids_relation)
@@ -41,7 +41,7 @@ module API
requires :discussion_id, type: String, desc: 'The ID of a discussion'
requires :noteable_id, types: [Integer, String], desc: 'The ID of the noteable'
end
- get ":id/#{noteables_path}/:noteable_id/discussions/:discussion_id" do
+ get ":id/#{noteables_path}/:noteable_id/discussions/:discussion_id", feature_category: feature_category do
noteable = find_noteable(noteable_type, params[:noteable_id])
notes = readable_discussion_notes(noteable, params[:discussion_id])
@@ -91,7 +91,7 @@ module API
end
end
end
- post ":id/#{noteables_path}/:noteable_id/discussions" do
+ post ":id/#{noteables_path}/:noteable_id/discussions", feature_category: feature_category do
noteable = find_noteable(noteable_type, params[:noteable_id])
type = params[:position] ? 'DiffNote' : 'DiscussionNote'
id_key = noteable.is_a?(Commit) ? :commit_id : :noteable_id
@@ -121,7 +121,7 @@ module API
requires :discussion_id, type: String, desc: 'The ID of a discussion'
requires :noteable_id, types: [Integer, String], desc: 'The ID of the noteable'
end
- get ":id/#{noteables_path}/:noteable_id/discussions/:discussion_id/notes" do
+ get ":id/#{noteables_path}/:noteable_id/discussions/:discussion_id/notes", feature_category: feature_category do
noteable = find_noteable(noteable_type, params[:noteable_id])
notes = readable_discussion_notes(noteable, params[:discussion_id])
@@ -141,7 +141,7 @@ module API
requires :body, type: String, desc: 'The content of a note'
optional :created_at, type: String, desc: 'The creation date of the note'
end
- post ":id/#{noteables_path}/:noteable_id/discussions/:discussion_id/notes" do
+ post ":id/#{noteables_path}/:noteable_id/discussions/:discussion_id/notes", feature_category: feature_category do
noteable = find_noteable(noteable_type, params[:noteable_id])
notes = readable_discussion_notes(noteable, params[:discussion_id])
first_note = notes.first
@@ -175,7 +175,7 @@ module API
requires :discussion_id, type: String, desc: 'The ID of a discussion'
requires :note_id, type: Integer, desc: 'The ID of a note'
end
- get ":id/#{noteables_path}/:noteable_id/discussions/:discussion_id/notes/:note_id" do
+ get ":id/#{noteables_path}/:noteable_id/discussions/:discussion_id/notes/:note_id", feature_category: feature_category do
noteable = find_noteable(noteable_type, params[:noteable_id])
get_note(noteable, params[:note_id])
@@ -192,7 +192,7 @@ module API
optional :resolved, type: Boolean, desc: 'Mark note resolved/unresolved'
exactly_one_of :body, :resolved
end
- put ":id/#{noteables_path}/:noteable_id/discussions/:discussion_id/notes/:note_id" do
+ put ":id/#{noteables_path}/:noteable_id/discussions/:discussion_id/notes/:note_id", feature_category: feature_category do
noteable = find_noteable(noteable_type, params[:noteable_id])
if params[:resolved].nil?
@@ -210,7 +210,7 @@ module API
requires :discussion_id, type: String, desc: 'The ID of a discussion'
requires :note_id, type: Integer, desc: 'The ID of a note'
end
- delete ":id/#{noteables_path}/:noteable_id/discussions/:discussion_id/notes/:note_id" do
+ delete ":id/#{noteables_path}/:noteable_id/discussions/:discussion_id/notes/:note_id", feature_category: feature_category do
noteable = find_noteable(noteable_type, params[:noteable_id])
delete_note(noteable, params[:note_id])
@@ -225,7 +225,7 @@ module API
requires :discussion_id, type: String, desc: 'The ID of a discussion'
requires :resolved, type: Boolean, desc: 'Mark discussion resolved/unresolved'
end
- put ":id/#{noteables_path}/:noteable_id/discussions/:discussion_id" do
+ put ":id/#{noteables_path}/:noteable_id/discussions/:discussion_id", feature_category: feature_category do
noteable = find_noteable(noteable_type, params[:noteable_id])
resolve_discussion(noteable, params[:discussion_id], params[:resolved])
diff --git a/lib/api/entities/board.rb b/lib/api/entities/board.rb
index 5bb1cde0fa9..b7a50408313 100644
--- a/lib/api/entities/board.rb
+++ b/lib/api/entities/board.rb
@@ -4,6 +4,7 @@ module API
module Entities
class Board < Grape::Entity
expose :id
+ expose :name
expose :project, using: Entities::BasicProjectDetails
expose :lists, using: Entities::List do |board|
diff --git a/lib/api/entities/commit_signature.rb b/lib/api/entities/commit_signature.rb
index b5232273521..505ce462edf 100644
--- a/lib/api/entities/commit_signature.rb
+++ b/lib/api/entities/commit_signature.rb
@@ -4,13 +4,28 @@ module API
module Entities
class CommitSignature < Grape::Entity
expose :signature_type
+
expose :signature, merge: true do |commit, options|
- if commit.signature.is_a?(GpgSignature)
- ::API::Entities::GpgCommitSignature.represent commit.signature, options
+ if commit.signature.is_a?(GpgSignature) || commit.raw_commit_from_rugged?
+ ::API::Entities::GpgCommitSignature.represent commit_signature(commit), options
elsif commit.signature.is_a?(X509CommitSignature)
::API::Entities::X509Signature.represent commit.signature, options
end
end
+
+ expose :commit_source do |commit, _|
+ commit.raw_commit_from_rugged? ? "rugged" : "gitaly"
+ end
+
+ private
+
+ def commit_signature(commit)
+ if commit.raw_commit_from_rugged?
+ commit.gpg_commit.signature
+ else
+ commit.signature
+ end
+ end
end
end
end
diff --git a/lib/api/entities/container_registry.rb b/lib/api/entities/container_registry.rb
index c430b73580b..c9c2c5156cc 100644
--- a/lib/api/entities/container_registry.rb
+++ b/lib/api/entities/container_registry.rb
@@ -10,6 +10,8 @@ module API
end
class Repository < Grape::Entity
+ include ::API::Helpers::RelatedResourcesHelpers
+
expose :id
expose :name
expose :path
@@ -19,6 +21,13 @@ module API
expose :expiration_policy_started_at, as: :cleanup_policy_started_at
expose :tags_count, if: -> (_, options) { options[:tags_count] }
expose :tags, using: Tag, if: -> (_, options) { options[:tags] }
+ expose :delete_api_path, if: ->(object, options) { Ability.allowed?(options[:user], :admin_container_image, object) }
+
+ private
+
+ def delete_api_path
+ expose_url api_v4_projects_registry_repositories_path(repository_id: object.id, id: object.project_id)
+ end
end
class TagDetails < Tag
diff --git a/lib/api/entities/invitation.rb b/lib/api/entities/invitation.rb
new file mode 100644
index 00000000000..342f4804cf3
--- /dev/null
+++ b/lib/api/entities/invitation.rb
@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module API
+ module Entities
+ class Invitation < Grape::Entity
+ expose :access_level
+ expose :requested_at
+ expose :expires_at
+ expose :invite_email
+ expose :invite_token
+ expose :user_name, if: -> (member, _) { member.user.present? }
+ expose :created_by_name
+ end
+ end
+end
diff --git a/lib/api/entities/merge_request_changes.rb b/lib/api/entities/merge_request_changes.rb
index a835d119736..488f33dfb93 100644
--- a/lib/api/entities/merge_request_changes.rb
+++ b/lib/api/entities/merge_request_changes.rb
@@ -4,7 +4,27 @@ module API
module Entities
class MergeRequestChanges < MergeRequest
expose :diffs, as: :changes, using: Entities::Diff do |compare, _|
- compare.raw_diffs(limits: false).to_a
+ Array(diff_collection(compare))
+ end
+
+ expose :overflow?, as: :overflow
+
+ private
+
+ def overflow?
+ expose_raw_diffs? ? false : diff_collection(object).overflow?
+ end
+
+ def diff_collection(compare)
+ @diffs ||= if expose_raw_diffs?
+ compare.raw_diffs(limits: false)
+ else
+ compare.diffs.diffs
+ end
+ end
+
+ def expose_raw_diffs?
+ options[:access_raw_diffs] || ::Feature.enabled?(:mrc_api_use_raw_diffs_from_gitaly, options[:project])
end
end
end
diff --git a/lib/api/entities/package.rb b/lib/api/entities/package.rb
index b54f0e04a9d..e7153f9bebb 100644
--- a/lib/api/entities/package.rb
+++ b/lib/api/entities/package.rb
@@ -38,7 +38,8 @@ module API
expose :project_path, if: ->(obj, opts) { opts[:group] && Ability.allowed?(opts[:user], :read_project, obj.project) }
expose :tags
- expose :pipeline, if: ->(package) { package.build_info }, using: Package::Pipeline
+ expose :pipeline, if: ->(package) { package.original_build_info }, using: Package::Pipeline
+ expose :pipelines, if: ->(package) { package.pipelines.present? }, using: Package::Pipeline
expose :versions, using: ::API::Entities::PackageVersion, unless: ->(_, opts) { opts[:collection] }
diff --git a/lib/api/entities/package_file.rb b/lib/api/entities/package_file.rb
index 8be4e5a4316..2cc2f62a948 100644
--- a/lib/api/entities/package_file.rb
+++ b/lib/api/entities/package_file.rb
@@ -6,6 +6,7 @@ module API
expose :id, :package_id, :created_at
expose :file_name, :size
expose :file_md5, :file_sha1
+ expose :pipelines, if: ->(package_file) { package_file.pipelines.present? }, using: Package::Pipeline
end
end
end
diff --git a/lib/api/entities/package_version.rb b/lib/api/entities/package_version.rb
index 5f3e86c3229..82522d3f423 100644
--- a/lib/api/entities/package_version.rb
+++ b/lib/api/entities/package_version.rb
@@ -8,7 +8,7 @@ module API
expose :created_at
expose :tags
- expose :pipeline, if: ->(package) { package.build_info }, using: Package::Pipeline
+ expose :pipeline, if: ->(package) { package.original_build_info }, using: Package::Pipeline
end
end
end
diff --git a/lib/api/entities/project_hook.rb b/lib/api/entities/project_hook.rb
index 751f9500252..6c71e5d317c 100644
--- a/lib/api/entities/project_hook.rb
+++ b/lib/api/entities/project_hook.rb
@@ -5,7 +5,7 @@ module API
class ProjectHook < Hook
expose :project_id, :issues_events, :confidential_issues_events
expose :note_events, :confidential_note_events, :pipeline_events, :wiki_page_events, :deployment_events
- expose :job_events
+ expose :job_events, :releases_events
expose :push_events_branch_filter
end
end
diff --git a/lib/api/entities/release.rb b/lib/api/entities/release.rb
index afe14cf33cf..44a46c5861e 100644
--- a/lib/api/entities/release.rb
+++ b/lib/api/entities/release.rb
@@ -30,8 +30,6 @@ module API
expose :evidences, using: Entities::Releases::Evidence, expose_nil: false, if: ->(_, _) { can_download_code? }
expose :_links do
expose :self_url, as: :self, expose_nil: false
- expose :merge_requests_url, expose_nil: false
- expose :issues_url, expose_nil: false
expose :edit_url, expose_nil: false
end
diff --git a/lib/api/entities/releases/link.rb b/lib/api/entities/releases/link.rb
index 654df2e2caf..c1d83a8924f 100644
--- a/lib/api/entities/releases/link.rb
+++ b/lib/api/entities/releases/link.rb
@@ -14,10 +14,8 @@ module API
def direct_asset_url
return object.url unless object.filepath
- release = object.release
- project = release.project
-
- Gitlab::Routing.url_helpers.project_release_url(project, release) << object.filepath
+ release = object.release.present
+ release.download_url(object.filepath)
end
end
end
diff --git a/lib/api/entities/user_status.rb b/lib/api/entities/user_status.rb
index 9bc4cbf240f..1d5cc27e5ef 100644
--- a/lib/api/entities/user_status.rb
+++ b/lib/api/entities/user_status.rb
@@ -5,6 +5,7 @@ module API
class UserStatus < Grape::Entity
expose :emoji
expose :message
+ expose :availability
expose :message_html do |entity|
MarkupHelper.markdown_field(entity, :message)
end
diff --git a/lib/api/environments.rb b/lib/api/environments.rb
index 0e780d4ef36..5dd2fa22690 100644
--- a/lib/api/environments.rb
+++ b/lib/api/environments.rb
@@ -7,6 +7,8 @@ module API
before { authenticate! }
+ feature_category :continuous_delivery
+
params do
requires :id, type: String, desc: 'The project ID'
end
diff --git a/lib/api/error_tracking.rb b/lib/api/error_tracking.rb
index 03f83477954..0e44c8b1081 100644
--- a/lib/api/error_tracking.rb
+++ b/lib/api/error_tracking.rb
@@ -4,6 +4,8 @@ module API
class ErrorTracking < ::API::Base
before { authenticate! }
+ feature_category :error_tracking
+
params do
requires :id, type: String, desc: 'The ID of a project'
end
diff --git a/lib/api/events.rb b/lib/api/events.rb
index 43efacf9c0b..233c62b5389 100644
--- a/lib/api/events.rb
+++ b/lib/api/events.rb
@@ -8,6 +8,8 @@ module API
allow_access_with_scope :read_user, if: -> (request) { request.get? }
+ feature_category :users
+
resource :events do
desc "List currently authenticated user's events" do
detail 'This feature was introduced in GitLab 9.3.'
diff --git a/lib/api/feature_flag_scopes.rb b/lib/api/feature_flag_scopes.rb
index d77e243aa88..3f3bf4d9f42 100644
--- a/lib/api/feature_flag_scopes.rb
+++ b/lib/api/feature_flag_scopes.rb
@@ -7,6 +7,8 @@ module API
ENVIRONMENT_SCOPE_ENDPOINT_REQUIREMENTS = FeatureFlags::FEATURE_FLAG_ENDPOINT_REQUIREMENTS
.merge(environment_scope: API::NO_SLASH_URL_PART_REGEX)
+ feature_category :feature_flags
+
before do
authorize_read_feature_flags!
end
diff --git a/lib/api/feature_flags.rb b/lib/api/feature_flags.rb
index 613c3fb0f5b..67168ba9be6 100644
--- a/lib/api/feature_flags.rb
+++ b/lib/api/feature_flags.rb
@@ -7,6 +7,8 @@ module API
FEATURE_FLAG_ENDPOINT_REQUIREMENTS = API::NAMESPACE_OR_PROJECT_REQUIREMENTS
.merge(name: API::NO_SLASH_URL_PART_REGEX)
+ feature_category :feature_flags
+
before do
authorize_read_feature_flags!
end
diff --git a/lib/api/feature_flags_user_lists.rb b/lib/api/feature_flags_user_lists.rb
index e5218cfd7f1..086bcbcdc89 100644
--- a/lib/api/feature_flags_user_lists.rb
+++ b/lib/api/feature_flags_user_lists.rb
@@ -8,6 +8,8 @@ module API
message.is_a?(String) ? { message: message }.to_json : message.to_json
}
+ feature_category :feature_flags
+
before do
authorize_admin_feature_flags_user_lists!
end
@@ -22,10 +24,13 @@ module API
success ::API::Entities::FeatureFlag::UserList
end
params do
+ optional :search, type: String, desc: 'Returns the list of user lists matching the search critiera'
+
use :pagination
end
get do
- present paginate(user_project.operations_feature_flags_user_lists),
+ user_lists = ::FeatureFlagsUserListsFinder.new(user_project, current_user, params).execute
+ present paginate(user_lists),
with: ::API::Entities::FeatureFlag::UserList
end
diff --git a/lib/api/features.rb b/lib/api/features.rb
index 5d2e545abd6..2c2e3e3d0c9 100644
--- a/lib/api/features.rb
+++ b/lib/api/features.rb
@@ -4,6 +4,8 @@ module API
class Features < ::API::Base
before { authenticated_as_admin! }
+ feature_category :feature_flags
+
helpers do
def gate_value(params)
case params[:value]
@@ -61,6 +63,8 @@ module API
mutually_exclusive :key, :project
end
post ':name' do
+ validate_feature_flag_name!(params[:name])
+
feature = Feature.get(params[:name]) # rubocop:disable Gitlab/AvoidFeatureGet
targets = gate_targets(params)
value = gate_value(params)
@@ -97,5 +101,13 @@ module API
no_content!
end
end
+
+ helpers do
+ def validate_feature_flag_name!(name)
+ # no-op
+ end
+ end
end
end
+
+API::Features.prepend_if_ee('EE::API::Features')
diff --git a/lib/api/files.rb b/lib/api/files.rb
index 6833fc429e2..cb73bde73f5 100644
--- a/lib/api/files.rb
+++ b/lib/api/files.rb
@@ -9,6 +9,8 @@ module API
# Prevents returning plain/text responses for files with .txt extension
after_validation { content_type "application/json" }
+ feature_category :source_code_management
+
helpers ::API::Helpers::HeadersHelpers
helpers do
diff --git a/lib/api/freeze_periods.rb b/lib/api/freeze_periods.rb
index a83e36165a2..d001ced8581 100644
--- a/lib/api/freeze_periods.rb
+++ b/lib/api/freeze_periods.rb
@@ -6,6 +6,8 @@ module API
before { authenticate! }
+ feature_category :continuous_delivery
+
params do
requires :id, type: String, desc: 'The ID of a project'
end
diff --git a/lib/api/generic_packages.rb b/lib/api/generic_packages.rb
index a0c33ab65b9..3e1dd044c8d 100644
--- a/lib/api/generic_packages.rb
+++ b/lib/api/generic_packages.rb
@@ -7,6 +7,8 @@ module API
file_name: API::NO_SLASH_URL_PART_REGEX
}.freeze
+ feature_category :package_registry
+
before do
require_packages_enabled!
authenticate!
diff --git a/lib/api/go_proxy.rb b/lib/api/go_proxy.rb
index 30f0cfb4dfd..8fb4c561c40 100755
--- a/lib/api/go_proxy.rb
+++ b/lib/api/go_proxy.rb
@@ -4,11 +4,15 @@ module API
helpers Gitlab::Golang
helpers ::API::Helpers::PackagesHelpers
+ feature_category :package_registry
+
# basic semver, except case encoded (A => !a)
MODULE_VERSION_REGEX = /v(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-([-.!a-z0-9]+))?(?:\+([-.!a-z0-9]+))?/.freeze
MODULE_VERSION_REQUIREMENTS = { module_version: MODULE_VERSION_REGEX }.freeze
+ content_type :txt, 'text/plain'
+
before { require_packages_enabled! }
helpers do
diff --git a/lib/api/group_boards.rb b/lib/api/group_boards.rb
index d4574b22d99..ac5a1a2ce94 100644
--- a/lib/api/group_boards.rb
+++ b/lib/api/group_boards.rb
@@ -7,6 +7,8 @@ module API
prepend_if_ee('EE::API::BoardsResponses') # rubocop: disable Cop/InjectEnterpriseEditionModule
+ feature_category :boards
+
before do
authenticate!
end
diff --git a/lib/api/group_clusters.rb b/lib/api/group_clusters.rb
index 75429cf7a5c..a435b050042 100644
--- a/lib/api/group_clusters.rb
+++ b/lib/api/group_clusters.rb
@@ -6,6 +6,8 @@ module API
before { authenticate! }
+ feature_category :kubernetes_management
+
params do
requires :id, type: String, desc: 'The ID of the group'
end
diff --git a/lib/api/group_container_repositories.rb b/lib/api/group_container_repositories.rb
index 1bb26b3931c..4fede0ad583 100644
--- a/lib/api/group_container_repositories.rb
+++ b/lib/api/group_container_repositories.rb
@@ -8,6 +8,8 @@ module API
before { authorize_read_group_container_images! }
+ feature_category :package_registry
+
REPOSITORY_ENDPOINT_REQUIREMENTS = API::NAMESPACE_OR_PROJECT_REQUIREMENTS.merge(
tag_name: API::NO_SLASH_URL_PART_REGEX)
diff --git a/lib/api/group_export.rb b/lib/api/group_export.rb
index 6ebaa8de185..29ffbea687a 100644
--- a/lib/api/group_export.rb
+++ b/lib/api/group_export.rb
@@ -10,6 +10,8 @@ module API
authorize! :admin_group, user_group
end
+ feature_category :importers
+
params do
requires :id, type: String, desc: 'The ID of a group'
end
diff --git a/lib/api/group_import.rb b/lib/api/group_import.rb
index e703a217fd5..4a752732652 100644
--- a/lib/api/group_import.rb
+++ b/lib/api/group_import.rb
@@ -2,6 +2,8 @@
module API
class GroupImport < ::API::Base
+ feature_category :importers
+
helpers Helpers::FileUploadHelpers
helpers do
diff --git a/lib/api/group_labels.rb b/lib/api/group_labels.rb
index 8443ddf10ce..bf3ac8800b7 100644
--- a/lib/api/group_labels.rb
+++ b/lib/api/group_labels.rb
@@ -7,6 +7,8 @@ module API
before { authenticate! }
+ feature_category :issue_tracking
+
params do
requires :id, type: String, desc: 'The ID of a group'
end
@@ -20,10 +22,16 @@ module API
desc: 'Include issue and merge request counts'
optional :include_ancestor_groups, type: Boolean, default: true,
desc: 'Include ancestor groups'
+ optional :include_descendant_groups, type: Boolean, default: false,
+ desc: 'Include descendant groups. This feature was added in GitLab 13.6'
+ optional :only_group_labels, type: Boolean, default: true,
+ desc: 'Toggle to include only group labels or also project labels. This feature was added in GitLab 13.6'
+ optional :search, type: String,
+ desc: 'Keyword to filter labels by. This feature was added in GitLab 13.6'
use :pagination
end
get ':id/labels' do
- get_labels(user_group, Entities::GroupLabel, include_ancestor_groups: params[:include_ancestor_groups])
+ get_labels(user_group, Entities::GroupLabel, declared_params)
end
desc 'Get a single label' do
@@ -33,9 +41,13 @@ module API
params do
optional :include_ancestor_groups, type: Boolean, default: true,
desc: 'Include ancestor groups'
+ optional :include_descendant_groups, type: Boolean, default: false,
+ desc: 'Include descendant groups. This feature was added in GitLab 13.6'
+ optional :only_group_labels, type: Boolean, default: true,
+ desc: 'Toggle to include only group labels or also project labels. This feature was added in GitLab 13.6'
end
get ':id/labels/:name' do
- get_label(user_group, Entities::GroupLabel, include_ancestor_groups: params[:include_ancestor_groups])
+ get_label(user_group, Entities::GroupLabel, declared_params)
end
desc 'Create a new label' do
diff --git a/lib/api/group_milestones.rb b/lib/api/group_milestones.rb
index aef9877b84c..dfffd3b1209 100644
--- a/lib/api/group_milestones.rb
+++ b/lib/api/group_milestones.rb
@@ -7,6 +7,8 @@ module API
before { authenticate! }
+ feature_category :issue_tracking
+
params do
requires :id, type: String, desc: 'The ID of a group'
end
diff --git a/lib/api/group_packages.rb b/lib/api/group_packages.rb
index 5b6290df0dd..31b28c3990f 100644
--- a/lib/api/group_packages.rb
+++ b/lib/api/group_packages.rb
@@ -8,6 +8,8 @@ module API
authorize_packages_access!(user_group)
end
+ feature_category :package_registry
+
helpers ::API::Helpers::PackagesHelpers
params do
diff --git a/lib/api/group_variables.rb b/lib/api/group_variables.rb
index ee110d67fa5..0c40db02eb5 100644
--- a/lib/api/group_variables.rb
+++ b/lib/api/group_variables.rb
@@ -7,6 +7,8 @@ module API
before { authenticate! }
before { authorize! :admin_build, user_group }
+ feature_category :continuous_integration
+
params do
requires :id, type: String, desc: 'The ID of a group'
end
diff --git a/lib/api/groups.rb b/lib/api/groups.rb
index bf3d6c3c7e0..a8b1cdab021 100644
--- a/lib/api/groups.rb
+++ b/lib/api/groups.rb
@@ -7,6 +7,8 @@ module API
before { authenticate_non_get! }
+ feature_category :subgroups
+
helpers Helpers::GroupsHelpers
helpers do
@@ -46,7 +48,7 @@ module API
find_params.fetch(:all_available, current_user&.can_read_all_resources?)
groups = GroupsFinder.new(current_user, find_params).execute
- groups = groups.search(params[:search]) if params[:search].present?
+ groups = groups.search(params[:search], include_parents: true) if params[:search].present?
groups = groups.where.not(id: params[:skip_groups]) if params[:skip_groups].present?
order_options = { params[:order_by] => params[:sort] }
order_options["id"] ||= "asc"
diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb
index c8aee1f3479..147d8407142 100644
--- a/lib/api/helpers.rb
+++ b/lib/api/helpers.rb
@@ -89,16 +89,15 @@ module API
@project ||= find_project!(params[:id])
end
- def available_labels_for(label_parent, include_ancestor_groups: true)
- search_params = { include_ancestor_groups: include_ancestor_groups }
-
+ def available_labels_for(label_parent, params = { include_ancestor_groups: true, only_group_labels: true })
if label_parent.is_a?(Project)
- search_params[:project_id] = label_parent.id
+ params.delete(:only_group_labels)
+ params[:project_id] = label_parent.id
else
- search_params.merge!(group_id: label_parent.id, only_group_labels: true)
+ params[:group_id] = label_parent.id
end
- LabelsFinder.new(current_user, search_params).execute
+ LabelsFinder.new(current_user, params).execute
end
def find_user(id)
@@ -388,8 +387,8 @@ module API
render_api_error!('401 Unauthorized', 401)
end
- def not_allowed!
- render_api_error!('405 Method Not Allowed', 405)
+ def not_allowed!(message = nil)
+ render_api_error!(message || '405 Method Not Allowed', :method_not_allowed)
end
def not_acceptable!
diff --git a/lib/api/helpers/discussions_helpers.rb b/lib/api/helpers/discussions_helpers.rb
index 799d5582b38..3c0db1d0ea9 100644
--- a/lib/api/helpers/discussions_helpers.rb
+++ b/lib/api/helpers/discussions_helpers.rb
@@ -3,10 +3,15 @@
module API
module Helpers
module DiscussionsHelpers
- def self.noteable_types
+ def self.feature_category_per_noteable_type
# This is a method instead of a constant, allowing EE to more easily
# extend it.
- [Issue, Snippet, MergeRequest, Commit]
+ {
+ Issue => :issue_tracking,
+ Snippet => :snippets,
+ MergeRequest => :code_review,
+ Commit => :code_review
+ }
end
end
end
diff --git a/lib/api/helpers/issues_helpers.rb b/lib/api/helpers/issues_helpers.rb
index 638b31cc7ba..b303f1f845d 100644
--- a/lib/api/helpers/issues_helpers.rb
+++ b/lib/api/helpers/issues_helpers.rb
@@ -5,6 +5,9 @@ module API
module IssuesHelpers
extend Grape::API::Helpers
+ params :negatable_issue_filter_params_ee do
+ end
+
params :optional_issue_params_ee do
end
diff --git a/lib/api/helpers/label_helpers.rb b/lib/api/helpers/label_helpers.rb
index 2fb2d9b79cf..4018f2dec21 100644
--- a/lib/api/helpers/label_helpers.rb
+++ b/lib/api/helpers/label_helpers.rb
@@ -28,23 +28,23 @@ module API
at_least_one_of :new_name, :color, :description
end
- def find_label(parent, id_or_title, include_ancestor_groups: true)
- labels = available_labels_for(parent, include_ancestor_groups: include_ancestor_groups)
+ def find_label(parent, id_or_title, params = { include_ancestor_groups: true })
+ labels = available_labels_for(parent, params)
label = labels.find_by_id(id_or_title) || labels.find_by_title(id_or_title)
label || not_found!('Label')
end
- def get_labels(parent, entity, include_ancestor_groups: true)
- present paginate(available_labels_for(parent, include_ancestor_groups: include_ancestor_groups)),
+ def get_labels(parent, entity, params = {})
+ present paginate(available_labels_for(parent, params)),
with: entity,
current_user: current_user,
parent: parent,
with_counts: params[:with_counts]
end
- def get_label(parent, entity, include_ancestor_groups: true)
- label = find_label(parent, params_id_or_title, include_ancestor_groups: include_ancestor_groups)
+ def get_label(parent, entity, params = {})
+ label = find_label(parent, params_id_or_title, params)
present label, with: entity, current_user: current_user, parent: parent
end
diff --git a/lib/api/helpers/members_helpers.rb b/lib/api/helpers/members_helpers.rb
index 5cc435e6801..431001c227d 100644
--- a/lib/api/helpers/members_helpers.rb
+++ b/lib/api/helpers/members_helpers.rb
@@ -20,12 +20,23 @@ module API
# rubocop: disable CodeReuse/ActiveRecord
def retrieve_members(source, params:, deep: false)
- members = deep ? find_all_members(source) : source.members.where.not(user_id: nil)
+ members = deep ? find_all_members(source) : source_members(source).where.not(user_id: nil)
members = members.includes(:user)
members = members.references(:user).merge(User.search(params[:query])) if params[:query].present?
members = members.where(user_id: params[:user_ids]) if params[:user_ids].present?
members
end
+
+ def retrieve_member_invitations(source, query = nil)
+ members = source_members(source).where.not(invite_token: nil)
+ members = members.includes(:user)
+ members = members.where(invite_email: query) if query.present?
+ members
+ end
+
+ def source_members(source)
+ source.members
+ end
# rubocop: enable CodeReuse/ActiveRecord
def find_all_members(source)
@@ -48,6 +59,10 @@ module API
def present_members(members)
present members, with: Entities::Member, current_user: current_user, show_seat_info: params[:show_seat_info]
end
+
+ def present_member_invitations(invitations)
+ present invitations, with: Entities::Invitation, current_user: current_user
+ end
end
end
end
diff --git a/lib/api/helpers/notes_helpers.rb b/lib/api/helpers/notes_helpers.rb
index f61bcfe963e..6798c4d284b 100644
--- a/lib/api/helpers/notes_helpers.rb
+++ b/lib/api/helpers/notes_helpers.rb
@@ -5,10 +5,12 @@ module API
module NotesHelpers
include ::RendersNotes
- def self.noteable_types
- # This is a method instead of a constant, allowing EE to more easily
- # extend it.
- [Issue, MergeRequest, Snippet]
+ def self.feature_category_per_noteable_type
+ {
+ Issue => :issue_tracking,
+ MergeRequest => :code_review,
+ Snippet => :snippets
+ }
end
def update_note(noteable, note_id)
diff --git a/lib/api/helpers/packages/npm.rb b/lib/api/helpers/packages/npm.rb
new file mode 100644
index 00000000000..c1f6a001201
--- /dev/null
+++ b/lib/api/helpers/packages/npm.rb
@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+module API
+ module Helpers
+ module Packages
+ module Npm
+ include Gitlab::Utils::StrongMemoize
+ include ::API::Helpers::PackagesHelpers
+
+ NPM_ENDPOINT_REQUIREMENTS = {
+ package_name: API::NO_SLASH_URL_PART_REGEX
+ }.freeze
+
+ def endpoint_scope
+ params[:id].present? ? :project : :instance
+ end
+
+ def project
+ strong_memoize(:project) do
+ case endpoint_scope
+ when :project
+ user_project
+ when :instance
+ # Simulate the same behavior as #user_project by re-using #find_project!
+ # but take care if the project_id is nil as #find_project! is not designed
+ # to handle it.
+ project_id = project_id_or_nil
+
+ not_found!('Project') unless project_id
+
+ find_project!(project_id)
+ end
+ end
+ end
+
+ def project_or_nil
+ # mainly used by the metadata endpoint where we need to get a project
+ # and return nil if not found (no errors should be raised)
+ strong_memoize(:project_or_nil) do
+ next unless project_id_or_nil
+
+ find_project(project_id_or_nil)
+ end
+ end
+
+ def project_id_or_nil
+ strong_memoize(:project_id_or_nil) do
+ case endpoint_scope
+ when :project
+ params[:id]
+ when :instance
+ ::Packages::Package.npm
+ .with_name(params[:package_name])
+ .first
+ &.project_id
+ end
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/api/helpers/resource_label_events_helpers.rb b/lib/api/helpers/resource_label_events_helpers.rb
index 423bd4e704b..ad2733baffc 100644
--- a/lib/api/helpers/resource_label_events_helpers.rb
+++ b/lib/api/helpers/resource_label_events_helpers.rb
@@ -3,10 +3,13 @@
module API
module Helpers
module ResourceLabelEventsHelpers
- def self.eventable_types
+ def self.feature_category_per_eventable_type
# This is a method instead of a constant, allowing EE to more easily
# extend it.
- [Issue, MergeRequest]
+ {
+ Issue => :issue_tracking,
+ MergeRequest => :code_review
+ }
end
end
end
diff --git a/lib/api/import_bitbucket_server.rb b/lib/api/import_bitbucket_server.rb
index a0238c24f3b..ecd78c6e6db 100644
--- a/lib/api/import_bitbucket_server.rb
+++ b/lib/api/import_bitbucket_server.rb
@@ -2,6 +2,8 @@
module API
class ImportBitbucketServer < ::API::Base
+ feature_category :importers
+
helpers do
def client
@client ||= BitbucketServer::Client.new(credentials)
diff --git a/lib/api/import_github.rb b/lib/api/import_github.rb
index 61fce7a2c1b..c91a7700f58 100644
--- a/lib/api/import_github.rb
+++ b/lib/api/import_github.rb
@@ -2,6 +2,8 @@
module API
class ImportGithub < ::API::Base
+ feature_category :importers
+
rescue_from Octokit::Unauthorized, with: :provider_unauthorized
before do
@@ -11,7 +13,7 @@ module API
helpers do
def client
@client ||= if Feature.enabled?(:remove_legacy_github_client)
- Gitlab::GithubImport::Client.new(params[:personal_access_token])
+ Gitlab::GithubImport::Client.new(params[:personal_access_token], host: params[:github_hostname])
else
Gitlab::LegacyGithubImport::Client.new(params[:personal_access_token], client_options)
end
@@ -22,7 +24,7 @@ module API
end
def client_options
- {}
+ { host: params[:github_hostname] }
end
def provider
@@ -43,6 +45,7 @@ module API
requires :repo_id, type: Integer, desc: 'GitHub repository ID'
optional :new_name, type: String, desc: 'New repo name'
requires :target_namespace, type: String, desc: 'Namespace to import repo into'
+ optional :github_hostname, type: String, desc: 'Custom GitHub enterprise hostname'
end
post 'import/github' do
result = Import::GithubService.new(client, current_user, params).execute(access_params, provider)
diff --git a/lib/api/internal/base.rb b/lib/api/internal/base.rb
index 6d8f13c36e6..61ef1d5bde0 100644
--- a/lib/api/internal/base.rb
+++ b/lib/api/internal/base.rb
@@ -7,10 +7,16 @@ module API
before { authenticate_by_gitlab_shell_token! }
before do
+ api_endpoint = env['api.endpoint']
+ feature_category = api_endpoint.options[:for].try(:feature_category_for_app, api_endpoint).to_s
+
+ header[Gitlab::Metrics::RequestsRackMiddleware::FEATURE_CATEGORY_HEADER] = feature_category
+
Gitlab::ApplicationContext.push(
user: -> { actor&.user },
project: -> { project },
- caller_id: route.origin
+ caller_id: route.origin,
+ feature_category: feature_category
)
end
@@ -28,10 +34,10 @@ module API
{ status: success, message: message }.merge(extra_options).compact
end
- def lfs_authentication_url(project)
+ def lfs_authentication_url(container)
# This is a separate method so that EE can alter its behaviour more
# easily.
- project.http_url_to_repo
+ container.lfs_http_url_to_repo
end
def check_allowed(params)
@@ -122,13 +128,15 @@ module API
# changes - changes as "oldrev newrev ref", see Gitlab::ChangesList
# check_ip - optional, only in EE version, may limit access to
# group resources based on its IP restrictions
- post "/allowed" do
+ post "/allowed", feature_category: :source_code_management do
# It was moved to a separate method so that EE can alter its behaviour more
# easily.
check_allowed(params)
end
- post "/lfs_authenticate" do
+ post "/lfs_authenticate", feature_category: :source_code_management do
+ not_found! unless container&.lfs_enabled?
+
status 200
unless actor.key_or_user
@@ -139,14 +147,14 @@ module API
Gitlab::LfsToken
.new(actor.key_or_user)
- .authentication_payload(lfs_authentication_url(project))
+ .authentication_payload(lfs_authentication_url(container))
end
#
# Get a ssh key using the fingerprint
#
# rubocop: disable CodeReuse/ActiveRecord
- get '/authorized_keys' do
+ get '/authorized_keys', feature_category: :source_code_management do
fingerprint = params.fetch(:fingerprint) do
Gitlab::InsecureKeyFingerprint.new(params.fetch(:key)).fingerprint
end
@@ -159,11 +167,11 @@ module API
#
# Discover user by ssh key, user id or username
#
- get '/discover' do
+ get '/discover', feature_category: :authentication_and_authorization do
present actor.user, with: Entities::UserSafe
end
- get '/check' do
+ get '/check', feature_category: :not_owned do
{
api_version: API.version,
gitlab_version: Gitlab::VERSION,
@@ -172,7 +180,7 @@ module API
}
end
- post '/two_factor_recovery_codes' do
+ post '/two_factor_recovery_codes', feature_category: :authentication_and_authorization do
status 200
actor.update_last_used_at!
@@ -201,7 +209,7 @@ module API
{ success: true, recovery_codes: codes }
end
- post '/personal_access_token' do
+ post '/personal_access_token', feature_category: :authentication_and_authorization do
status 200
actor.update_last_used_at!
@@ -239,7 +247,7 @@ module API
end
result = ::PersonalAccessTokens::CreateService.new(
- user, name: params[:name], scopes: params[:scopes], expires_at: expires_at
+ current_user: user, target_user: user, params: { name: params[:name], scopes: params[:scopes], expires_at: expires_at }
).execute
unless result.status == :success
@@ -251,7 +259,7 @@ module API
{ success: true, token: access_token.token, scopes: access_token.scopes, expires_at: access_token.expires_at }
end
- post '/pre_receive' do
+ post '/pre_receive', feature_category: :source_code_management do
status 200
reference_counter_increased = Gitlab::ReferenceCounter.new(params[:gl_repository]).increase
@@ -259,7 +267,7 @@ module API
{ reference_counter_increased: reference_counter_increased }
end
- post '/post_receive' do
+ post '/post_receive', feature_category: :source_code_management do
status 200
response = PostReceiveService.new(actor.user, repository, project, params).execute
@@ -267,7 +275,7 @@ module API
present response, with: Entities::InternalPostReceive::Response
end
- post '/two_factor_config' do
+ post '/two_factor_config', feature_category: :authentication_and_authorization do
status 200
break { success: false } unless Feature.enabled?(:two_factor_for_cli)
@@ -289,7 +297,7 @@ module API
end
end
- post '/two_factor_otp_check' do
+ post '/two_factor_otp_check', feature_category: :authentication_and_authorization do
status 200
break { success: false } unless Feature.enabled?(:two_factor_for_cli)
diff --git a/lib/api/internal/kubernetes.rb b/lib/api/internal/kubernetes.rb
index 90e224b2ccb..d4690709de4 100644
--- a/lib/api/internal/kubernetes.rb
+++ b/lib/api/internal/kubernetes.rb
@@ -4,6 +4,8 @@ module API
# Kubernetes Internal API
module Internal
class Kubernetes < ::API::Base
+ feature_category :kubernetes_management
+
before do
check_feature_enabled
authenticate_gitlab_kas_request!
diff --git a/lib/api/internal/lfs.rb b/lib/api/internal/lfs.rb
index 630f0ec77a8..66baa4f1034 100644
--- a/lib/api/internal/lfs.rb
+++ b/lib/api/internal/lfs.rb
@@ -7,6 +7,8 @@ module API
before { authenticate_by_gitlab_shell_token! }
+ feature_category :source_code_management
+
helpers do
def find_lfs_object(lfs_oid)
LfsObject.find_by_oid(lfs_oid)
diff --git a/lib/api/internal/pages.rb b/lib/api/internal/pages.rb
index 51136144c19..690f52d89f3 100644
--- a/lib/api/internal/pages.rb
+++ b/lib/api/internal/pages.rb
@@ -4,6 +4,8 @@ module API
# Pages Internal API
module Internal
class Pages < ::API::Base
+ feature_category :pages
+
before do
authenticate_gitlab_pages_request!
end
diff --git a/lib/api/invitations.rb b/lib/api/invitations.rb
new file mode 100644
index 00000000000..be8147908e9
--- /dev/null
+++ b/lib/api/invitations.rb
@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+module API
+ class Invitations < ::API::Base
+ include PaginationParams
+
+ feature_category :users
+
+ before { authenticate! }
+
+ helpers ::API::Helpers::MembersHelpers
+
+ %w[group project].each do |source_type|
+ params do
+ requires :id, type: String, desc: "The #{source_type} ID"
+ end
+ resource source_type.pluralize, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do
+ desc 'Invite non-members by email address to a group or project.' do
+ detail 'This feature was introduced in GitLab 13.6'
+ success Entities::Invitation
+ end
+ params do
+ requires :email, types: [String, Array[String]], email_or_email_list: true, desc: 'The email address to invite, or multiple emails separated by comma'
+ requires :access_level, type: Integer, values: Gitlab::Access.all_values, desc: 'A valid access level (defaults: `30`, developer access level)'
+ optional :expires_at, type: DateTime, desc: 'Date string in the format YEAR-MONTH-DAY'
+ end
+ post ":id/invitations" do
+ source = find_source(source_type, params[:id])
+
+ authorize_admin_source!(source_type, source)
+
+ ::Members::InviteService.new(current_user, params).execute(source)
+ end
+
+ desc 'Get a list of group or project invitations viewable by the authenticated user' do
+ detail 'This feature was introduced in GitLab 13.6'
+ success Entities::Invitation
+ end
+ params do
+ optional :query, type: String, desc: 'A query string to search for members'
+ use :pagination
+ end
+ get ":id/invitations" do
+ source = find_source(source_type, params[:id])
+ query = params[:query]
+
+ invitations = paginate(retrieve_member_invitations(source, query))
+
+ present_member_invitations invitations
+ end
+ end
+ end
+ end
+end
diff --git a/lib/api/issue_links.rb b/lib/api/issue_links.rb
index db4979c9052..e938dbbae87 100644
--- a/lib/api/issue_links.rb
+++ b/lib/api/issue_links.rb
@@ -6,6 +6,8 @@ module API
before { authenticate! }
+ feature_category :issue_tracking
+
params do
requires :id, type: String, desc: 'The ID of a project'
requires :issue_iid, type: Integer, desc: 'The internal ID of a project issue'
diff --git a/lib/api/issues.rb b/lib/api/issues.rb
index 143f9e40736..6a6ee7a4e1c 100644
--- a/lib/api/issues.rb
+++ b/lib/api/issues.rb
@@ -8,6 +8,8 @@ module API
before { authenticate_non_get! }
+ feature_category :issue_tracking
+
helpers do
params :negatable_issue_filter_params do
optional :labels, type: Array[String], coerce_with: ::API::Validations::Types::CommaSeparatedToArray.coerce, desc: 'Comma-separated list of label names'
@@ -26,6 +28,8 @@ module API
coerce_with: Validations::Validators::CheckAssigneesCount.coerce,
desc: 'Return issues which are assigned to the user with the given username'
mutually_exclusive :assignee_id, :assignee_username
+
+ use :negatable_issue_filter_params_ee
end
params :issues_stats_params do
diff --git a/lib/api/job_artifacts.rb b/lib/api/job_artifacts.rb
index 536b361b308..1faa28d6f07 100644
--- a/lib/api/job_artifacts.rb
+++ b/lib/api/job_artifacts.rb
@@ -4,6 +4,8 @@ module API
class JobArtifacts < ::API::Base
before { authenticate_non_get! }
+ feature_category :continuous_integration
+
# EE::API::JobArtifacts would override the following helpers
helpers do
def authorize_download_artifacts!
diff --git a/lib/api/jobs.rb b/lib/api/jobs.rb
index bdb23b4a9be..51659c2e8a1 100644
--- a/lib/api/jobs.rb
+++ b/lib/api/jobs.rb
@@ -6,6 +6,8 @@ module API
before { authenticate! }
+ feature_category :continuous_integration
+
params do
requires :id, type: String, desc: 'The ID of a project'
end
diff --git a/lib/api/keys.rb b/lib/api/keys.rb
index 2e4568029b5..fb1bedd5e92 100644
--- a/lib/api/keys.rb
+++ b/lib/api/keys.rb
@@ -5,6 +5,8 @@ module API
class Keys < ::API::Base
before { authenticate! }
+ feature_category :authentication_and_authorization
+
resource :keys do
desc 'Get single ssh key by id. Only available to admin users' do
success Entities::SSHKeyWithUser
diff --git a/lib/api/labels.rb b/lib/api/labels.rb
index 0cc9f33bd07..a8fc277989e 100644
--- a/lib/api/labels.rb
+++ b/lib/api/labels.rb
@@ -7,6 +7,8 @@ module API
before { authenticate! }
+ feature_category :issue_tracking
+
params do
requires :id, type: String, desc: 'The ID of a project'
end
@@ -19,10 +21,12 @@ module API
desc: 'Include issue and merge request counts'
optional :include_ancestor_groups, type: Boolean, default: true,
desc: 'Include ancestor groups'
+ optional :search, type: String,
+ desc: 'Keyword to filter labels by. This feature was added in GitLab 13.6'
use :pagination
end
get ':id/labels' do
- get_labels(user_project, Entities::ProjectLabel, include_ancestor_groups: params[:include_ancestor_groups])
+ get_labels(user_project, Entities::ProjectLabel, declared_params)
end
desc 'Get a single label' do
@@ -34,7 +38,7 @@ module API
desc: 'Include ancestor groups'
end
get ':id/labels/:name' do
- get_label(user_project, Entities::ProjectLabel, include_ancestor_groups: params[:include_ancestor_groups])
+ get_label(user_project, Entities::ProjectLabel, declared_params)
end
desc 'Create a new label' do
diff --git a/lib/api/lint.rb b/lib/api/lint.rb
index bfd152f70b1..58181adaa93 100644
--- a/lib/api/lint.rb
+++ b/lib/api/lint.rb
@@ -2,6 +2,8 @@
module API
class Lint < ::API::Base
+ feature_category :pipeline_authoring
+
namespace :ci do
desc 'Validation of .gitlab-ci.yml content'
params do
@@ -15,9 +17,9 @@ module API
status 200
response = if error.blank?
- { status: 'valid', errors: [] }
+ { status: 'valid', errors: [], warnings: result.warnings }
else
- { status: 'invalid', errors: [error] }
+ { status: 'invalid', errors: [error], warnings: result.warnings }
end
response.tap do |response|
@@ -44,5 +46,25 @@ module API
present result, with: Entities::Ci::Lint::Result, current_user: current_user
end
end
+
+ resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do
+ desc 'Validation of .gitlab-ci.yml content' do
+ detail 'This feature was introduced in GitLab 13.6.'
+ end
+ params do
+ requires :content, type: String, desc: 'Content of .gitlab-ci.yml'
+ optional :dry_run, type: Boolean, default: false, desc: 'Run pipeline creation simulation, or only do static check.'
+ end
+ post ':id/ci/lint' do
+ authorize! :download_code, user_project
+
+ result = Gitlab::Ci::Lint
+ .new(project: user_project, current_user: current_user)
+ .validate(params[:content], dry_run: params[:dry_run])
+
+ status 200
+ present result, with: Entities::Ci::Lint::Result, current_user: current_user
+ end
+ end
end
end
diff --git a/lib/api/markdown.rb b/lib/api/markdown.rb
index 97549abd273..de612ff8321 100644
--- a/lib/api/markdown.rb
+++ b/lib/api/markdown.rb
@@ -2,6 +2,8 @@
module API
class Markdown < ::API::Base
+ feature_category :not_owned
+
params do
requires :text, type: String, desc: "The markdown text to render"
optional :gfm, type: Boolean, desc: "Render text using GitLab Flavored Markdown"
diff --git a/lib/api/maven_packages.rb b/lib/api/maven_packages.rb
index a3e2fa84c32..7b4e52d18e8 100644
--- a/lib/api/maven_packages.rb
+++ b/lib/api/maven_packages.rb
@@ -5,6 +5,8 @@ module API
file_name: API::NO_SLASH_URL_PART_REGEX
}.freeze
+ feature_category :package_registry
+
content_type :md5, 'text/plain'
content_type :sha1, 'text/plain'
content_type :binary, 'application/octet-stream'
@@ -244,7 +246,7 @@ module API
file_md5: params['file.md5']
}
- ::Packages::CreatePackageFileService.new(package, file_params).execute
+ ::Packages::CreatePackageFileService.new(package, file_params.merge(build: current_authenticated_job)).execute
end
end
end
diff --git a/lib/api/members.rb b/lib/api/members.rb
index c28b3b1cc7c..803de51651a 100644
--- a/lib/api/members.rb
+++ b/lib/api/members.rb
@@ -6,6 +6,8 @@ module API
before { authenticate! }
+ feature_category :authentication_and_authorization
+
helpers ::API::Helpers::MembersHelpers
%w[group project].each do |source_type|
@@ -134,7 +136,7 @@ module API
source = find_source(source_type, params.delete(:id))
authorize_admin_source!(source_type, source)
- member = source.members.find_by!(user_id: params[:user_id])
+ member = source_members(source).find_by!(user_id: params[:user_id])
updated_member =
::Members::UpdateService
.new(current_user, declared_params(include_missing: false))
@@ -157,7 +159,7 @@ module API
# rubocop: disable CodeReuse/ActiveRecord
delete ":id/members/:user_id" do
source = find_source(source_type, params[:id])
- member = source.members.find_by!(user_id: params[:user_id])
+ member = source_members(source).find_by!(user_id: params[:user_id])
destroy_conditionally!(member) do
::Members::DestroyService.new(current_user).execute(member, unassign_issuables: params[:unassign_issuables])
diff --git a/lib/api/merge_request_approvals.rb b/lib/api/merge_request_approvals.rb
index 14d6e3995ea..27ef0b9c7cd 100644
--- a/lib/api/merge_request_approvals.rb
+++ b/lib/api/merge_request_approvals.rb
@@ -4,6 +4,8 @@ module API
class MergeRequestApprovals < ::API::Base
before { authenticate_non_get! }
+ feature_category :code_review
+
helpers do
params :ee_approval_params do
end
diff --git a/lib/api/merge_request_diffs.rb b/lib/api/merge_request_diffs.rb
index 22023888bbd..0ffb38438eb 100644
--- a/lib/api/merge_request_diffs.rb
+++ b/lib/api/merge_request_diffs.rb
@@ -7,6 +7,8 @@ module API
before { authenticate! }
+ feature_category :code_review
+
params do
requires :id, type: String, desc: 'The ID of a project'
end
diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb
index b24dd870c8b..d17e451093b 100644
--- a/lib/api/merge_requests.rb
+++ b/lib/api/merge_requests.rb
@@ -8,6 +8,8 @@ module API
before { authenticate_non_get! }
+ feature_category :code_review
+
helpers Helpers::MergeRequestsHelpers
# EE::API::MergeRequests would override the following helpers
@@ -350,7 +352,11 @@ module API
get ':id/merge_requests/:merge_request_iid/changes' do
merge_request = find_merge_request_with_access(params[:merge_request_iid])
- present merge_request, with: Entities::MergeRequestChanges, current_user: current_user, project: user_project
+ present merge_request,
+ with: Entities::MergeRequestChanges,
+ current_user: current_user,
+ project: user_project,
+ access_raw_diffs: params.fetch(:access_raw_diffs, false)
end
desc 'Get the merge request pipelines' do
diff --git a/lib/api/metrics/dashboard/annotations.rb b/lib/api/metrics/dashboard/annotations.rb
index b6bc0af2202..0989340b3ea 100644
--- a/lib/api/metrics/dashboard/annotations.rb
+++ b/lib/api/metrics/dashboard/annotations.rb
@@ -4,6 +4,8 @@ module API
module Metrics
module Dashboard
class Annotations < ::API::Base
+ feature_category :metrics
+
desc 'Create a new monitoring dashboard annotation' do
success Entities::Metrics::Dashboard::Annotation
end
diff --git a/lib/api/metrics/user_starred_dashboards.rb b/lib/api/metrics/user_starred_dashboards.rb
index cb6e7099247..909f7f0405d 100644
--- a/lib/api/metrics/user_starred_dashboards.rb
+++ b/lib/api/metrics/user_starred_dashboards.rb
@@ -3,6 +3,8 @@
module API
module Metrics
class UserStarredDashboards < ::API::Base
+ feature_category :metrics
+
resource :projects do
desc 'Marks selected metrics dashboard as starred' do
success Entities::Metrics::UserStarredDashboard
diff --git a/lib/api/namespaces.rb b/lib/api/namespaces.rb
index f98a1f6dd1d..25a901c18b6 100644
--- a/lib/api/namespaces.rb
+++ b/lib/api/namespaces.rb
@@ -6,6 +6,8 @@ module API
before { authenticate! }
+ feature_category :subgroups
+
helpers do
params :optional_list_params_ee do
# EE::API::Namespaces would override this helper
diff --git a/lib/api/notes.rb b/lib/api/notes.rb
index 0db537ca616..d249431b2f8 100644
--- a/lib/api/notes.rb
+++ b/lib/api/notes.rb
@@ -7,7 +7,7 @@ module API
before { authenticate! }
- Helpers::NotesHelpers.noteable_types.each do |noteable_type|
+ Helpers::NotesHelpers.feature_category_per_noteable_type.each do |noteable_type, feature_category|
parent_type = noteable_type.parent_class.to_s.underscore
noteables_str = noteable_type.to_s.underscore.pluralize
@@ -29,7 +29,7 @@ module API
use :pagination
end
# rubocop: disable CodeReuse/ActiveRecord
- get ":id/#{noteables_str}/:noteable_id/notes" do
+ get ":id/#{noteables_str}/:noteable_id/notes", feature_category: feature_category do
noteable = find_noteable(noteable_type, params[:noteable_id])
# We exclude notes that are cross-references and that cannot be viewed
@@ -57,7 +57,7 @@ module API
requires :note_id, type: Integer, desc: 'The ID of a note'
requires :noteable_id, type: Integer, desc: 'The ID of the noteable'
end
- get ":id/#{noteables_str}/:noteable_id/notes/:note_id" do
+ get ":id/#{noteables_str}/:noteable_id/notes/:note_id", feature_category: feature_category do
noteable = find_noteable(noteable_type, params[:noteable_id])
get_note(noteable, params[:note_id])
end
@@ -71,7 +71,7 @@ module API
optional :confidential, type: Boolean, desc: 'Confidentiality note flag, default is false'
optional :created_at, type: String, desc: 'The creation date of the note'
end
- post ":id/#{noteables_str}/:noteable_id/notes" do
+ post ":id/#{noteables_str}/:noteable_id/notes", feature_category: feature_category do
noteable = find_noteable(noteable_type, params[:noteable_id])
opts = {
@@ -104,7 +104,7 @@ module API
optional :body, type: String, allow_blank: false, desc: 'The content of a note'
optional :confidential, type: Boolean, desc: 'Confidentiality note flag'
end
- put ":id/#{noteables_str}/:noteable_id/notes/:note_id" do
+ put ":id/#{noteables_str}/:noteable_id/notes/:note_id", feature_category: feature_category do
noteable = find_noteable(noteable_type, params[:noteable_id])
update_note(noteable, params[:note_id])
diff --git a/lib/api/notification_settings.rb b/lib/api/notification_settings.rb
index bad3f5ead7a..7d28394e034 100644
--- a/lib/api/notification_settings.rb
+++ b/lib/api/notification_settings.rb
@@ -5,6 +5,8 @@ module API
class NotificationSettings < ::API::Base
before { authenticate! }
+ feature_category :users
+
helpers ::API::Helpers::MembersHelpers
resource :notification_settings do
diff --git a/lib/api/npm_instance_packages.rb b/lib/api/npm_instance_packages.rb
new file mode 100644
index 00000000000..12fc008e00f
--- /dev/null
+++ b/lib/api/npm_instance_packages.rb
@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+module API
+ class NpmInstancePackages < ::API::Base
+ helpers ::API::Helpers::Packages::Npm
+
+ feature_category :package_registry
+
+ rescue_from ActiveRecord::RecordInvalid do |e|
+ render_api_error!(e.message, 400)
+ end
+
+ namespace 'packages/npm' do
+ include ::API::Concerns::Packages::NpmEndpoints
+ end
+ end
+end
diff --git a/lib/api/npm_packages.rb b/lib/api/npm_packages.rb
deleted file mode 100644
index 1443b28c1ee..00000000000
--- a/lib/api/npm_packages.rb
+++ /dev/null
@@ -1,173 +0,0 @@
-# frozen_string_literal: true
-module API
- class NpmPackages < ::API::Base
- helpers ::API::Helpers::PackagesHelpers
- helpers ::API::Helpers::Packages::DependencyProxyHelpers
-
- NPM_ENDPOINT_REQUIREMENTS = {
- package_name: API::NO_SLASH_URL_PART_REGEX
- }.freeze
-
- rescue_from ActiveRecord::RecordInvalid do |e|
- render_api_error!(e.message, 400)
- end
-
- before do
- require_packages_enabled!
- authenticate_non_get!
- end
-
- helpers do
- def project_by_package_name
- strong_memoize(:project_by_package_name) do
- ::Packages::Package.npm.with_name(params[:package_name]).first&.project
- end
- end
- end
-
- desc 'Get all tags for a given an NPM package' do
- detail 'This feature was introduced in GitLab 12.7'
- success ::API::Entities::NpmPackageTag
- end
- params do
- requires :package_name, type: String, desc: 'Package name'
- end
- get 'packages/npm/-/package/*package_name/dist-tags', format: false, requirements: NPM_ENDPOINT_REQUIREMENTS do
- package_name = params[:package_name]
-
- bad_request!('Package Name') if package_name.blank?
-
- authorize_read_package!(project_by_package_name)
-
- packages = ::Packages::Npm::PackageFinder.new(project_by_package_name, package_name)
- .execute
-
- present ::Packages::Npm::PackagePresenter.new(package_name, packages),
- with: ::API::Entities::NpmPackageTag
- end
-
- params do
- requires :package_name, type: String, desc: 'Package name'
- requires :tag, type: String, desc: "Package dist-tag"
- end
- namespace 'packages/npm/-/package/*package_name/dist-tags/:tag', requirements: NPM_ENDPOINT_REQUIREMENTS do
- desc 'Create or Update the given tag for the given NPM package and version' do
- detail 'This feature was introduced in GitLab 12.7'
- end
- put format: false do
- package_name = params[:package_name]
- version = env['api.request.body']
- tag = params[:tag]
-
- bad_request!('Package Name') if package_name.blank?
- bad_request!('Version') if version.blank?
- bad_request!('Tag') if tag.blank?
-
- authorize_create_package!(project_by_package_name)
-
- package = ::Packages::Npm::PackageFinder
- .new(project_by_package_name, package_name)
- .find_by_version(version)
- not_found!('Package') unless package
-
- ::Packages::Npm::CreateTagService.new(package, tag).execute
-
- no_content!
- end
-
- desc 'Deletes the given tag' do
- detail 'This feature was introduced in GitLab 12.7'
- end
- delete format: false do
- package_name = params[:package_name]
- tag = params[:tag]
-
- bad_request!('Package Name') if package_name.blank?
- bad_request!('Tag') if tag.blank?
-
- authorize_destroy_package!(project_by_package_name)
-
- package_tag = ::Packages::TagsFinder
- .new(project_by_package_name, package_name, package_type: :npm)
- .find_by_name(tag)
-
- not_found!('Package tag') unless package_tag
-
- ::Packages::RemoveTagService.new(package_tag).execute
-
- no_content!
- end
- end
-
- desc 'NPM registry endpoint at instance level' do
- detail 'This feature was introduced in GitLab 11.8'
- end
- params do
- requires :package_name, type: String, desc: 'Package name'
- end
- route_setting :authentication, job_token_allowed: true, deploy_token_allowed: true
- get 'packages/npm/*package_name', format: false, requirements: NPM_ENDPOINT_REQUIREMENTS do
- package_name = params[:package_name]
-
- redirect_registry_request(project_by_package_name.blank?, :npm, package_name: package_name) do
- authorize_read_package!(project_by_package_name)
-
- packages = ::Packages::Npm::PackageFinder
- .new(project_by_package_name, package_name).execute
-
- present ::Packages::Npm::PackagePresenter.new(package_name, packages),
- with: ::API::Entities::NpmPackage
- end
- end
-
- params do
- requires :id, type: String, desc: 'The ID of a project'
- end
- resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do
- desc 'Download the NPM tarball' do
- detail 'This feature was introduced in GitLab 11.8'
- end
- params do
- requires :package_name, type: String, desc: 'Package name'
- requires :file_name, type: String, desc: 'Package file name'
- end
- route_setting :authentication, job_token_allowed: true, deploy_token_allowed: true
- get ':id/packages/npm/*package_name/-/*file_name', format: false do
- authorize_read_package!(user_project)
-
- package = user_project.packages.npm
- .by_name_and_file_name(params[:package_name], params[:file_name])
-
- package_file = ::Packages::PackageFileFinder
- .new(package, params[:file_name]).execute!
-
- track_package_event('pull_package', package)
-
- present_carrierwave_file!(package_file.file)
- end
-
- desc 'Create NPM package' do
- detail 'This feature was introduced in GitLab 11.8'
- end
- params do
- requires :package_name, type: String, desc: 'Package name'
- requires :versions, type: Hash, desc: 'Package version info'
- end
- route_setting :authentication, job_token_allowed: true, deploy_token_allowed: true
- put ':id/packages/npm/:package_name', requirements: NPM_ENDPOINT_REQUIREMENTS do
- authorize_create_package!(user_project)
-
- track_package_event('push_package', :npm)
-
- created_package = ::Packages::Npm::CreatePackageService
- .new(user_project, current_user, params.merge(build: current_authenticated_job)).execute
-
- if created_package[:status] == :error
- render_api_error!(created_package[:message], created_package[:http_status])
- else
- created_package
- end
- end
- end
- end
-end
diff --git a/lib/api/npm_project_packages.rb b/lib/api/npm_project_packages.rb
new file mode 100644
index 00000000000..887084dc9ae
--- /dev/null
+++ b/lib/api/npm_project_packages.rb
@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+module API
+ class NpmProjectPackages < ::API::Base
+ helpers ::API::Helpers::Packages::Npm
+
+ feature_category :package_registry
+
+ rescue_from ActiveRecord::RecordInvalid do |e|
+ render_api_error!(e.message, 400)
+ end
+
+ params do
+ requires :id, type: String, desc: 'The ID of a project'
+ end
+ namespace 'projects/:id/packages/npm' do
+ desc 'Download the NPM tarball' do
+ detail 'This feature was introduced in GitLab 11.8'
+ end
+ params do
+ requires :package_name, type: String, desc: 'Package name'
+ requires :file_name, type: String, desc: 'Package file name'
+ end
+ route_setting :authentication, job_token_allowed: true, deploy_token_allowed: true
+ get '*package_name/-/*file_name', format: false do
+ authorize_read_package!(project)
+
+ package = project.packages.npm
+ .by_name_and_file_name(params[:package_name], params[:file_name])
+
+ not_found!('Package') unless package
+
+ package_file = ::Packages::PackageFileFinder
+ .new(package, params[:file_name]).execute!
+
+ track_package_event('pull_package', package, category: 'API::NpmPackages')
+
+ present_carrierwave_file!(package_file.file)
+ end
+
+ desc 'Create NPM package' do
+ detail 'This feature was introduced in GitLab 11.8'
+ end
+ params do
+ requires :package_name, type: String, desc: 'Package name'
+ requires :versions, type: Hash, desc: 'Package version info'
+ end
+ route_setting :authentication, job_token_allowed: true, deploy_token_allowed: true
+ put ':package_name', requirements: ::API::Helpers::Packages::Npm::NPM_ENDPOINT_REQUIREMENTS do
+ authorize_create_package!(project)
+
+ track_package_event('push_package', :npm, category: 'API::NpmPackages')
+
+ created_package = ::Packages::Npm::CreatePackageService
+ .new(project, current_user, params.merge(build: current_authenticated_job)).execute
+
+ if created_package[:status] == :error
+ render_api_error!(created_package[:message], created_package[:http_status])
+ else
+ created_package
+ end
+ end
+
+ include ::API::Concerns::Packages::NpmEndpoints
+ end
+ end
+end
diff --git a/lib/api/nuget_packages.rb b/lib/api/nuget_packages.rb
index 0f2c956a9df..65a85f3c930 100644
--- a/lib/api/nuget_packages.rb
+++ b/lib/api/nuget_packages.rb
@@ -10,6 +10,8 @@ module API
helpers ::API::Helpers::PackagesManagerClientsHelpers
helpers ::API::Helpers::Packages::BasicAuthHelpers
+ feature_category :package_registry
+
POSITIVE_INTEGER_REGEX = %r{\A[1-9]\d*\z}.freeze
NON_NEGATIVE_INTEGER_REGEX = %r{\A0|[1-9]\d*\z}.freeze
diff --git a/lib/api/package_files.rb b/lib/api/package_files.rb
index c1fc9a6e4d8..4a33f3e8af2 100644
--- a/lib/api/package_files.rb
+++ b/lib/api/package_files.rb
@@ -8,6 +8,8 @@ module API
authorize_packages_access!(user_project)
end
+ feature_category :package_registry
+
helpers ::API::Helpers::PackagesHelpers
params do
diff --git a/lib/api/pages.rb b/lib/api/pages.rb
index 813307c498f..5f695f3853d 100644
--- a/lib/api/pages.rb
+++ b/lib/api/pages.rb
@@ -2,6 +2,8 @@
module API
class Pages < ::API::Base
+ feature_category :pages
+
before do
require_pages_config_enabled!
authenticated_with_can_read_all_resources!
diff --git a/lib/api/pages_domains.rb b/lib/api/pages_domains.rb
index 00c51298c45..2e7f8475509 100644
--- a/lib/api/pages_domains.rb
+++ b/lib/api/pages_domains.rb
@@ -4,6 +4,8 @@ module API
class PagesDomains < ::API::Base
include PaginationParams
+ feature_category :pages
+
PAGES_DOMAINS_ENDPOINT_REQUIREMENTS = API::NAMESPACE_OR_PROJECT_REQUIREMENTS.merge(domain: API::NO_SLASH_URL_PART_REGEX)
before do
diff --git a/lib/api/personal_access_tokens.rb b/lib/api/personal_access_tokens.rb
new file mode 100644
index 00000000000..2c60938b75a
--- /dev/null
+++ b/lib/api/personal_access_tokens.rb
@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+module API
+ class PersonalAccessTokens < ::API::Base
+ include ::API::PaginationParams
+
+ feature_category :authentication_and_authorization
+
+ desc 'Get all Personal Access Tokens' do
+ detail 'This feature was added in GitLab 13.3'
+ success Entities::PersonalAccessToken
+ end
+ params do
+ optional :user_id, type: Integer, desc: 'User ID'
+
+ use :pagination
+ end
+
+ before do
+ authenticate!
+ restrict_non_admins! unless current_user.admin?
+ end
+
+ helpers do
+ def finder_params(current_user)
+ current_user.admin? ? { user: user(params[:user_id]) } : { user: current_user }
+ end
+
+ def user(user_id)
+ UserFinder.new(user_id).find_by_id
+ end
+
+ def restrict_non_admins!
+ return if params[:user_id].blank?
+
+ unauthorized! unless Ability.allowed?(current_user, :read_user_personal_access_tokens, user(params[:user_id]))
+ end
+
+ def find_token(id)
+ PersonalAccessToken.find(id) || not_found!
+ end
+ end
+
+ resources :personal_access_tokens do
+ get do
+ tokens = PersonalAccessTokensFinder.new(finder_params(current_user), current_user).execute
+
+ present paginate(tokens), with: Entities::PersonalAccessToken
+ end
+
+ delete ':id' do
+ service = ::PersonalAccessTokens::RevokeService.new(
+ current_user,
+ token: find_token(params[:id])
+ ).execute
+
+ service.success? ? no_content! : bad_request!(nil)
+ end
+ end
+ end
+end
diff --git a/lib/api/project_clusters.rb b/lib/api/project_clusters.rb
index 46ccb4ba1a0..cfb0c5fd705 100644
--- a/lib/api/project_clusters.rb
+++ b/lib/api/project_clusters.rb
@@ -6,6 +6,8 @@ module API
before { authenticate! }
+ feature_category :kubernetes_management
+
params do
requires :id, type: String, desc: 'The ID of the project'
end
diff --git a/lib/api/project_container_repositories.rb b/lib/api/project_container_repositories.rb
index d565531d372..3125de88de5 100644
--- a/lib/api/project_container_repositories.rb
+++ b/lib/api/project_container_repositories.rb
@@ -10,6 +10,8 @@ module API
before { authorize_read_container_images! }
+ feature_category :package_registry
+
params do
requires :id, type: String, desc: 'The ID of a project'
end
diff --git a/lib/api/project_events.rb b/lib/api/project_events.rb
index 3765473bc0e..69b47f9420d 100644
--- a/lib/api/project_events.rb
+++ b/lib/api/project_events.rb
@@ -6,6 +6,8 @@ module API
include APIGuard
helpers ::API::Helpers::EventsHelpers
+ feature_category :users
+
params do
requires :id, type: String, desc: 'The ID of a project'
end
diff --git a/lib/api/project_export.rb b/lib/api/project_export.rb
index 184f89200ab..76b3dea723a 100644
--- a/lib/api/project_export.rb
+++ b/lib/api/project_export.rb
@@ -4,6 +4,8 @@ module API
class ProjectExport < ::API::Base
helpers Helpers::RateLimiter
+ feature_category :importers
+
before do
not_found! unless Gitlab::CurrentSettings.project_export_enabled?
authorize_admin_project
diff --git a/lib/api/project_hooks.rb b/lib/api/project_hooks.rb
index bc2d8c816a8..431ba199131 100644
--- a/lib/api/project_hooks.rb
+++ b/lib/api/project_hooks.rb
@@ -7,6 +7,8 @@ module API
before { authenticate! }
before { authorize_admin_project }
+ feature_category :integrations
+
helpers do
params :project_hook_properties do
requires :url, type: String, desc: "The URL to send the request to"
@@ -21,6 +23,7 @@ module API
optional :pipeline_events, type: Boolean, desc: "Trigger hook on pipeline events"
optional :wiki_page_events, type: Boolean, desc: "Trigger hook on wiki events"
optional :deployment_events, type: Boolean, desc: "Trigger hook on deployment events"
+ optional :releases_events, type: Boolean, desc: "Trigger hook on release events"
optional :enable_ssl_verification, type: Boolean, desc: "Do SSL verification when triggering the hook"
optional :token, type: String, desc: "Secret token to validate received payloads; this will not be returned in the response"
optional :push_events_branch_filter, type: String, desc: "Trigger hook on specified branch only"
diff --git a/lib/api/project_import.rb b/lib/api/project_import.rb
index 5c4e1d73ee1..15b06cea385 100644
--- a/lib/api/project_import.rb
+++ b/lib/api/project_import.rb
@@ -8,6 +8,8 @@ module API
helpers Helpers::FileUploadHelpers
helpers Helpers::RateLimiter
+ feature_category :importers
+
helpers do
def import_params
declared_params(include_missing: false)
diff --git a/lib/api/project_milestones.rb b/lib/api/project_milestones.rb
index a81118f44bd..8675de33923 100644
--- a/lib/api/project_milestones.rb
+++ b/lib/api/project_milestones.rb
@@ -7,6 +7,8 @@ module API
before { authenticate! }
+ feature_category :issue_tracking
+
params do
requires :id, type: String, desc: 'The ID of a project'
end
diff --git a/lib/api/project_packages.rb b/lib/api/project_packages.rb
index b8d97b1243a..56e94333433 100644
--- a/lib/api/project_packages.rb
+++ b/lib/api/project_packages.rb
@@ -8,6 +8,8 @@ module API
authorize_packages_access!(user_project)
end
+ feature_category :package_registry
+
helpers ::API::Helpers::PackagesHelpers
params do
diff --git a/lib/api/project_repository_storage_moves.rb b/lib/api/project_repository_storage_moves.rb
index 38eb74663d3..fe6de3ea385 100644
--- a/lib/api/project_repository_storage_moves.rb
+++ b/lib/api/project_repository_storage_moves.rb
@@ -6,6 +6,8 @@ module API
before { authenticated_as_admin! }
+ feature_category :gitaly
+
resource :project_repository_storage_moves do
desc 'Get a list of all project repository storage moves' do
detail 'This feature was introduced in GitLab 13.0.'
diff --git a/lib/api/project_snapshots.rb b/lib/api/project_snapshots.rb
index e19afb6e8e4..d33d2976b1c 100644
--- a/lib/api/project_snapshots.rb
+++ b/lib/api/project_snapshots.rb
@@ -6,6 +6,8 @@ module API
before { authorize_read_git_snapshot! }
+ feature_category :source_code_management
+
resource :projects do
desc 'Download a (possibly inconsistent) snapshot of a repository' do
detail 'This feature was introduced in GitLab 10.7'
diff --git a/lib/api/project_snippets.rb b/lib/api/project_snippets.rb
index b4de260fe49..899984fe0ba 100644
--- a/lib/api/project_snippets.rb
+++ b/lib/api/project_snippets.rb
@@ -6,6 +6,8 @@ module API
before { check_snippets_enabled }
+ feature_category :snippets
+
params do
requires :id, type: String, desc: 'The ID of a project'
end
diff --git a/lib/api/project_statistics.rb b/lib/api/project_statistics.rb
index 1ead969fc81..3db8d20ebac 100644
--- a/lib/api/project_statistics.rb
+++ b/lib/api/project_statistics.rb
@@ -2,6 +2,8 @@
module API
class ProjectStatistics < ::API::Base
+ feature_category :source_code_management
+
before do
authenticate!
authorize! :daily_statistics, user_project
diff --git a/lib/api/project_templates.rb b/lib/api/project_templates.rb
index 7d851de0237..af5d96969ef 100644
--- a/lib/api/project_templates.rb
+++ b/lib/api/project_templates.rb
@@ -12,6 +12,8 @@ module API
before { authenticate_non_get! }
+ feature_category :templates
+
params do
requires :id, type: String, desc: 'The ID of a project'
requires :type, type: String, values: TEMPLATE_TYPES, desc: 'The type (dockerfiles|gitignores|gitlab_ci_ymls|licenses|metrics_dashboard_ymls|issues|merge_requests) of the template'
diff --git a/lib/api/projects.rb b/lib/api/projects.rb
index ecee76ae60c..2012c348cd1 100644
--- a/lib/api/projects.rb
+++ b/lib/api/projects.rb
@@ -11,6 +11,8 @@ module API
before { authenticate_non_get! }
+ feature_category :projects, ['/projects/:id/custom_attributes', '/projects/:id/custom_attributes/:key']
+
helpers do
# EE::API::Projects would override this method
def apply_filters(projects)
@@ -150,7 +152,7 @@ module API
use :statistics_params
use :with_custom_attributes
end
- get ":user_id/projects" do
+ get ":user_id/projects", feature_category: :projects do
user = find_user(params[:user_id])
not_found!('User') unless user
@@ -167,7 +169,7 @@ module API
use :collection_params
use :statistics_params
end
- get ":user_id/starred_projects" do
+ get ":user_id/starred_projects", feature_category: :projects do
user = find_user(params[:user_id])
not_found!('User') unless user
@@ -187,7 +189,7 @@ module API
use :statistics_params
use :with_custom_attributes
end
- get do
+ get feature_category: :projects do
present_projects load_projects
end
@@ -234,7 +236,7 @@ module API
use :create_params
end
# rubocop: disable CodeReuse/ActiveRecord
- post "user/:user_id" do
+ post "user/:user_id", feature_category: :projects do
Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab/issues/21139')
authenticated_as_admin!
user = User.find_by(id: params.delete(:user_id))
@@ -270,7 +272,7 @@ module API
optional :license, type: Boolean, default: false,
desc: 'Include project license data'
end
- get ":id" do
+ get ":id", feature_category: :projects do
options = {
with: current_user ? Entities::ProjectWithAccess : Entities::BasicProjectDetails,
current_user: current_user,
@@ -294,7 +296,7 @@ module API
optional :path, type: String, desc: 'The path that will be assigned to the fork'
optional :name, type: String, desc: 'The name that will be assigned to the fork'
end
- post ':id/fork' do
+ post ':id/fork', feature_category: :source_code_management do
Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-foss/issues/42284')
not_found! unless can?(current_user, :fork_project, user_project)
@@ -332,14 +334,14 @@ module API
use :collection_params
use :with_custom_attributes
end
- get ':id/forks' do
+ get ':id/forks', feature_category: :source_code_management do
forks = ForkProjectsFinder.new(user_project, params: project_finder_params, current_user: current_user).execute
present_projects forks, request_scope: user_project
end
desc 'Check pages access of this project'
- get ':id/pages_access' do
+ get ':id/pages_access', feature_category: :pages do
authorize! :read_pages_content, user_project unless user_project.public_pages?
status 200
end
@@ -357,7 +359,7 @@ module API
at_least_one_of(*Helpers::ProjectsHelpers.update_params_at_least_one_of)
end
- put ':id' do
+ put ':id', feature_category: :projects do
authorize_admin_project
attrs = declared_params(include_missing: false)
authorize! :rename_project, user_project if attrs[:name].present?
@@ -381,7 +383,7 @@ module API
desc 'Archive a project' do
success Entities::Project
end
- post ':id/archive' do
+ post ':id/archive', feature_category: :projects do
authorize!(:archive_project, user_project)
::Projects::UpdateService.new(user_project, current_user, archived: true).execute
@@ -392,7 +394,7 @@ module API
desc 'Unarchive a project' do
success Entities::Project
end
- post ':id/unarchive' do
+ post ':id/unarchive', feature_category: :projects do
authorize!(:archive_project, user_project)
::Projects::UpdateService.new(user_project, current_user, archived: false).execute
@@ -403,7 +405,7 @@ module API
desc 'Star a project' do
success Entities::Project
end
- post ':id/star' do
+ post ':id/star', feature_category: :projects do
if current_user.starred?(user_project)
not_modified!
else
@@ -417,7 +419,7 @@ module API
desc 'Unstar a project' do
success Entities::Project
end
- post ':id/unstar' do
+ post ':id/unstar', feature_category: :projects do
if current_user.starred?(user_project)
current_user.toggle_star(user_project)
user_project.reset
@@ -435,21 +437,21 @@ module API
optional :search, type: String, desc: 'Return list of users matching the search criteria'
use :pagination
end
- get ':id/starrers' do
+ get ':id/starrers', feature_category: :projects do
starrers = UsersStarProjectsFinder.new(user_project, params, current_user: current_user).execute
present paginate(starrers), with: Entities::UserStarsProject
end
desc 'Get languages in project repository'
- get ':id/languages' do
+ get ':id/languages', feature_category: :source_code_management do
::Projects::RepositoryLanguagesService
.new(user_project, current_user)
.execute.map { |lang| [lang.name, lang.share] }.to_h
end
desc 'Delete a project'
- delete ":id" do
+ delete ":id", feature_category: :projects do
authorize! :remove_project, user_project
delete_project(user_project)
@@ -459,7 +461,7 @@ module API
params do
requires :forked_from_id, type: String, desc: 'The ID of the project it was forked from'
end
- post ":id/fork/:forked_from_id" do
+ post ":id/fork/:forked_from_id", feature_category: :source_code_management do
authorize! :admin_project, user_project
fork_from_project = find_project!(params[:forked_from_id])
@@ -478,7 +480,7 @@ module API
end
desc 'Remove a forked_from relationship'
- delete ":id/fork" do
+ delete ":id/fork", feature_category: :source_code_management do
authorize! :remove_fork_project, user_project
result = destroy_conditionally!(user_project) do
@@ -496,7 +498,7 @@ module API
requires :group_access, type: Integer, values: Gitlab::Access.values, as: :link_group_access, desc: 'The group access level'
optional :expires_at, type: Date, desc: 'Share expiration date'
end
- post ":id/share" do
+ post ":id/share", feature_category: :authentication_and_authorization do
authorize! :admin_project, user_project
group = Group.find_by_id(params[:group_id])
@@ -518,7 +520,7 @@ module API
requires :group_id, type: Integer, desc: 'The ID of the group'
end
# rubocop: disable CodeReuse/ActiveRecord
- delete ":id/share/:group_id" do
+ delete ":id/share/:group_id", feature_category: :authentication_and_authorization do
authorize! :admin_project, user_project
link = user_project.project_group_links.find_by(group_id: params[:group_id])
@@ -535,7 +537,7 @@ module API
# TODO: remove rubocop disable - https://gitlab.com/gitlab-org/gitlab/issues/14960
requires :file, type: File, desc: 'The file to be uploaded' # rubocop:disable Scalability/FileUploads
end
- post ":id/uploads" do
+ post ":id/uploads", feature_category: :not_owned do
upload = UploadService.new(user_project, params[:file]).execute
present upload, with: Entities::ProjectUpload
@@ -549,7 +551,7 @@ module API
optional :skip_users, type: Array[Integer], coerce_with: ::API::Validations::Types::CommaSeparatedToIntegerArray.coerce, desc: 'Filter out users with the specified IDs'
use :pagination
end
- get ':id/users' do
+ get ':id/users', feature_category: :authentication_and_authorization do
users = DeclarativePolicy.subject_scope { user_project.team.users }
users = users.search(params[:search]) if params[:search].present?
users = users.where_not_in(params[:skip_users]) if params[:skip_users].present?
@@ -560,7 +562,7 @@ module API
desc 'Start the housekeeping task for a project' do
detail 'This feature was introduced in GitLab 9.0.'
end
- post ':id/housekeeping' do
+ post ':id/housekeeping', feature_category: :source_code_management do
authorize_admin_project
begin
@@ -574,7 +576,7 @@ module API
params do
requires :namespace, type: String, desc: 'The ID or path of the new namespace'
end
- put ":id/transfer" do
+ put ":id/transfer", feature_category: :projects do
authorize! :change_namespace, user_project
namespace = find_namespace!(params[:namespace])
diff --git a/lib/api/protected_branches.rb b/lib/api/protected_branches.rb
index a448682d8bd..17574739a7c 100644
--- a/lib/api/protected_branches.rb
+++ b/lib/api/protected_branches.rb
@@ -8,6 +8,8 @@ module API
before { authorize_admin_project }
+ feature_category :source_code_management
+
helpers Helpers::ProtectedBranchesHelpers
params do
diff --git a/lib/api/protected_tags.rb b/lib/api/protected_tags.rb
index dd3e407ffc9..b9385df1f8d 100644
--- a/lib/api/protected_tags.rb
+++ b/lib/api/protected_tags.rb
@@ -8,6 +8,8 @@ module API
before { authorize_admin_project }
+ feature_category :source_code_management
+
params do
requires :id, type: String, desc: 'The ID of a project'
end
diff --git a/lib/api/pypi_packages.rb b/lib/api/pypi_packages.rb
index 5622bc6e42d..7104fb8d999 100644
--- a/lib/api/pypi_packages.rb
+++ b/lib/api/pypi_packages.rb
@@ -12,6 +12,8 @@ module API
helpers ::API::Helpers::Packages::BasicAuthHelpers
include ::API::Helpers::Packages::BasicAuthHelpers::Constants
+ feature_category :package_registry
+
default_format :json
rescue_from ArgumentError do |e|
diff --git a/lib/api/release/links.rb b/lib/api/release/links.rb
index 23de9f9fc9f..d3a185a51c8 100644
--- a/lib/api/release/links.rb
+++ b/lib/api/release/links.rb
@@ -10,6 +10,8 @@ module API
before { authorize! :read_release, user_project }
+ feature_category :release_orchestration
+
params do
requires :id, type: String, desc: 'The ID of a project'
end
diff --git a/lib/api/releases.rb b/lib/api/releases.rb
index 3bd6ea77403..c20e618efd1 100644
--- a/lib/api/releases.rb
+++ b/lib/api/releases.rb
@@ -9,6 +9,8 @@ module API
before { authorize_read_releases! }
+ feature_category :release_orchestration
+
params do
requires :id, type: String, desc: 'The ID of a project'
end
@@ -89,7 +91,7 @@ module API
optional :name, type: String, desc: 'The name of the release'
optional :description, type: String, desc: 'Release notes with markdown support'
optional :released_at, type: DateTime, desc: 'The date when the release will be/was ready.'
- optional :milestones, type: Array, desc: 'The titles of the related milestones'
+ optional :milestones, type: Array[String], coerce_with: ::API::Validations::Types::CommaSeparatedToArray.coerce, desc: 'The titles of the related milestones'
end
put ':id/releases/:tag_name', requirements: RELEASE_ENDPOINT_REQUIREMENTS do
authorize_update_release!
diff --git a/lib/api/remote_mirrors.rb b/lib/api/remote_mirrors.rb
index f63ea04a529..83096772d32 100644
--- a/lib/api/remote_mirrors.rb
+++ b/lib/api/remote_mirrors.rb
@@ -4,6 +4,8 @@ module API
class RemoteMirrors < ::API::Base
include PaginationParams
+ feature_category :source_code_management
+
before do
unauthorized! unless can?(current_user, :admin_remote_mirror, user_project)
end
diff --git a/lib/api/repositories.rb b/lib/api/repositories.rb
index 38ac1f22a48..8af8ffc3b63 100644
--- a/lib/api/repositories.rb
+++ b/lib/api/repositories.rb
@@ -6,10 +6,14 @@ module API
class Repositories < ::API::Base
include PaginationParams
+ content_type :txt, 'text/plain'
+
helpers ::API::Helpers::HeadersHelpers
before { authorize! :download_code, user_project }
+ feature_category :source_code_management
+
params do
requires :id, type: String, desc: 'The ID of a project'
end
diff --git a/lib/api/resource_label_events.rb b/lib/api/resource_label_events.rb
index d3a219f0810..33589f6c393 100644
--- a/lib/api/resource_label_events.rb
+++ b/lib/api/resource_label_events.rb
@@ -7,7 +7,7 @@ module API
before { authenticate! }
- Helpers::ResourceLabelEventsHelpers.eventable_types.each do |eventable_type|
+ Helpers::ResourceLabelEventsHelpers.feature_category_per_eventable_type.each do |eventable_type, feature_category|
parent_type = eventable_type.parent_class.to_s.underscore
eventables_str = eventable_type.to_s.underscore.pluralize
@@ -24,7 +24,7 @@ module API
use :pagination
end
- get ":id/#{eventables_str}/:eventable_id/resource_label_events" do
+ get ":id/#{eventables_str}/:eventable_id/resource_label_events", feature_category: feature_category do
eventable = find_noteable(eventable_type, params[:eventable_id])
events = eventable.resource_label_events.inc_relations
@@ -40,7 +40,7 @@ module API
requires :event_id, type: String, desc: 'The ID of a resource label event'
requires :eventable_id, types: [Integer, String], desc: 'The ID of the eventable'
end
- get ":id/#{eventables_str}/:eventable_id/resource_label_events/:event_id" do
+ get ":id/#{eventables_str}/:eventable_id/resource_label_events/:event_id", feature_category: feature_category do
eventable = find_noteable(eventable_type, params[:eventable_id])
event = eventable.resource_label_events.find(params[:event_id])
diff --git a/lib/api/resource_milestone_events.rb b/lib/api/resource_milestone_events.rb
index 21411f68dd5..aeedd7ad109 100644
--- a/lib/api/resource_milestone_events.rb
+++ b/lib/api/resource_milestone_events.rb
@@ -7,7 +7,10 @@ module API
before { authenticate! }
- [Issue, MergeRequest].each do |eventable_type|
+ {
+ Issue => :issue_tracking,
+ MergeRequest => :code_review
+ }.each do |eventable_type, feature_category|
parent_type = eventable_type.parent_class.to_s.underscore
eventables_str = eventable_type.to_s.underscore.pluralize
@@ -23,7 +26,7 @@ module API
use :pagination
end
- get ":id/#{eventables_str}/:eventable_id/resource_milestone_events" do
+ get ":id/#{eventables_str}/:eventable_id/resource_milestone_events", feature_category: feature_category do
eventable = find_noteable(eventable_type, params[:eventable_id])
events = ResourceMilestoneEventFinder.new(current_user, eventable).execute
@@ -38,7 +41,7 @@ module API
requires :event_id, type: String, desc: 'The ID of a resource milestone event'
requires :eventable_id, types: [Integer, String], desc: 'The ID of the eventable'
end
- get ":id/#{eventables_str}/:eventable_id/resource_milestone_events/:event_id" do
+ get ":id/#{eventables_str}/:eventable_id/resource_milestone_events/:event_id", feature_category: feature_category do
eventable = find_noteable(eventable_type, params[:eventable_id])
event = eventable.resource_milestone_events.find(params[:event_id])
diff --git a/lib/api/resource_state_events.rb b/lib/api/resource_state_events.rb
index 9bfda39be90..3460aa2c00e 100644
--- a/lib/api/resource_state_events.rb
+++ b/lib/api/resource_state_events.rb
@@ -7,7 +7,10 @@ module API
before { authenticate! }
- [Issue, MergeRequest].each do |eventable_class|
+ {
+ Issue => :issue_tracking,
+ MergeRequest => :code_review
+ }.each do |eventable_class, feature_category|
eventable_name = eventable_class.to_s.underscore
params do
@@ -22,7 +25,7 @@ module API
use :pagination
end
- get ":id/#{eventable_name.pluralize}/:eventable_iid/resource_state_events" do
+ get ":id/#{eventable_name.pluralize}/:eventable_iid/resource_state_events", feature_category: feature_category do
eventable = find_noteable(eventable_class, params[:eventable_iid])
events = ResourceStateEventFinder.new(current_user, eventable).execute
@@ -37,7 +40,7 @@ module API
requires :eventable_iid, types: Integer, desc: "The IID of the #{eventable_name}"
requires :event_id, type: Integer, desc: 'The ID of a resource state event'
end
- get ":id/#{eventable_name.pluralize}/:eventable_iid/resource_state_events/:event_id" do
+ get ":id/#{eventable_name.pluralize}/:eventable_iid/resource_state_events/:event_id", feature_category: feature_category do
eventable = find_noteable(eventable_class, params[:eventable_iid])
event = ResourceStateEventFinder.new(current_user, eventable).find(params[:event_id])
diff --git a/lib/api/search.rb b/lib/api/search.rb
index 85f0a8e2e60..f0ffe6ba443 100644
--- a/lib/api/search.rb
+++ b/lib/api/search.rb
@@ -6,6 +6,8 @@ module API
before { authenticate! }
+ feature_category :global_search
+
helpers do
SCOPE_ENTITY = {
merge_requests: Entities::MergeRequestBasic,
@@ -35,8 +37,11 @@ module API
state: params[:state],
confidential: params[:confidential],
snippets: snippets?,
+ basic_search: params[:basic_search],
page: params[:page],
- per_page: params[:per_page]
+ per_page: params[:per_page],
+ order_by: params[:order_by],
+ sort: params[:sort]
}.merge(additional_params)
results = SearchService.new(current_user, search_params).search_objects(preload_method)
diff --git a/lib/api/services.rb b/lib/api/services.rb
index 5f3d14010a8..cfcae13e518 100644
--- a/lib/api/services.rb
+++ b/lib/api/services.rb
@@ -1,6 +1,8 @@
# frozen_string_literal: true
module API
class Services < ::API::Base
+ feature_category :integrations
+
services = Helpers::ServicesHelpers.services
service_classes = Helpers::ServicesHelpers.service_classes
diff --git a/lib/api/settings.rb b/lib/api/settings.rb
index dc917d9c529..b95856d99d1 100644
--- a/lib/api/settings.rb
+++ b/lib/api/settings.rb
@@ -4,6 +4,8 @@ module API
class Settings < ::API::Base
before { authenticated_as_admin! }
+ feature_category :not_owned
+
helpers Helpers::SettingsHelpers
helpers do
@@ -51,9 +53,9 @@ module API
optional :default_projects_limit, type: Integer, desc: 'The maximum number of personal projects'
optional :default_snippet_visibility, type: String, values: Gitlab::VisibilityLevel.string_values, desc: 'The default snippet visibility'
optional :disabled_oauth_sign_in_sources, type: Array[String], coerce_with: Validations::Types::CommaSeparatedToArray.coerce, desc: 'Disable certain OAuth sign-in sources'
- optional :domain_blacklist_enabled, type: Boolean, desc: 'Enable domain blacklist for sign ups'
- optional :domain_blacklist, type: Array[String], coerce_with: Validations::Types::CommaSeparatedToArray.coerce, desc: 'Users with e-mail addresses that match these domain(s) will NOT be able to sign-up. Wildcards allowed. Use separate lines for multiple entries. Ex: domain.com, *.domain.com'
- optional :domain_whitelist, type: Array[String], coerce_with: Validations::Types::CommaSeparatedToArray.coerce, desc: 'ONLY users with e-mail addresses that match these domain(s) will be able to sign-up. Wildcards allowed. Use separate lines for multiple entries. Ex: domain.com, *.domain.com'
+ optional :domain_denylist_enabled, type: Boolean, desc: 'Enable domain denylist for sign ups'
+ optional :domain_denylist, type: Array[String], coerce_with: Validations::Types::CommaSeparatedToArray.coerce, desc: 'Users with e-mail addresses that match these domain(s) will NOT be able to sign-up. Wildcards allowed. Use separate lines for multiple entries. Ex: domain.com, *.domain.com'
+ optional :domain_allowlist, type: Array[String], coerce_with: Validations::Types::CommaSeparatedToArray.coerce, desc: 'ONLY users with e-mail addresses that match these domain(s) will be able to sign-up. Wildcards allowed. Use separate lines for multiple entries. Ex: domain.com, *.domain.com'
optional :eks_integration_enabled, type: Boolean, desc: 'Enable integration with Amazon EKS'
given eks_integration_enabled: -> (val) { val } do
requires :eks_account_id, type: String, desc: 'Amazon account ID for EKS integration'
@@ -157,6 +159,7 @@ module API
optional :issues_create_limit, type: Integer, desc: "Maximum number of issue creation requests allowed per minute per user. Set to 0 for unlimited requests per minute."
optional :raw_blob_request_limit, type: Integer, desc: "Maximum number of requests per minute for each raw path. Set to 0 for unlimited requests per minute."
optional :wiki_page_max_content_bytes, type: Integer, desc: "Maximum wiki page content size in bytes"
+ optional :require_admin_approval_after_user_signup, type: Boolean, desc: 'Require explicit admin approval for new signups'
ApplicationSetting::SUPPORTED_KEY_TYPES.each do |type|
optional :"#{type}_key_restriction",
diff --git a/lib/api/sidekiq_metrics.rb b/lib/api/sidekiq_metrics.rb
index b025dbfab37..680363d036e 100644
--- a/lib/api/sidekiq_metrics.rb
+++ b/lib/api/sidekiq_metrics.rb
@@ -6,6 +6,8 @@ module API
class SidekiqMetrics < ::API::Base
before { authenticated_as_admin! }
+ feature_category :not_owned
+
helpers do
def queue_metrics
Sidekiq::Queue.all.each_with_object({}) do |queue, hash|
diff --git a/lib/api/snippets.rb b/lib/api/snippets.rb
index 2e67b9649bc..52b597fb788 100644
--- a/lib/api/snippets.rb
+++ b/lib/api/snippets.rb
@@ -5,6 +5,8 @@ module API
class Snippets < ::API::Base
include PaginationParams
+ feature_category :snippets
+
resource :snippets do
helpers Helpers::SnippetsHelpers
helpers do
diff --git a/lib/api/statistics.rb b/lib/api/statistics.rb
index fa7176491ba..1814e1a6782 100644
--- a/lib/api/statistics.rb
+++ b/lib/api/statistics.rb
@@ -4,6 +4,8 @@ module API
class Statistics < ::API::Base
before { authenticated_as_admin! }
+ feature_category :instance_statistics
+
COUNTED_ITEMS = [Project, User, Group, ForkNetworkMember, ForkNetwork, Issue,
MergeRequest, Note, Snippet, Key, Milestone].freeze
diff --git a/lib/api/submodules.rb b/lib/api/submodules.rb
index e2ceb49c119..5c71a18c6d0 100644
--- a/lib/api/submodules.rb
+++ b/lib/api/submodules.rb
@@ -4,6 +4,8 @@ module API
class Submodules < ::API::Base
before { authenticate! }
+ feature_category :source_code_management
+
helpers do
def commit_params(attrs)
{
diff --git a/lib/api/subscriptions.rb b/lib/api/subscriptions.rb
index 35a28da4736..914bab52929 100644
--- a/lib/api/subscriptions.rb
+++ b/lib/api/subscriptions.rb
@@ -11,25 +11,29 @@ module API
type: 'merge_requests',
entity: Entities::MergeRequest,
source: Project,
- finder: ->(id) { find_merge_request_with_access(id, :update_merge_request) }
+ finder: ->(id) { find_merge_request_with_access(id, :update_merge_request) },
+ feature_category: :code_review
},
{
type: 'issues',
entity: Entities::Issue,
source: Project,
- finder: ->(id) { find_project_issue(id) }
+ finder: ->(id) { find_project_issue(id) },
+ feature_category: :issue_tracking
},
{
type: 'labels',
entity: Entities::ProjectLabel,
source: Project,
- finder: ->(id) { find_label(user_project, id) }
+ finder: ->(id) { find_label(user_project, id) },
+ feature_category: :issue_tracking
},
{
type: 'labels',
entity: Entities::GroupLabel,
source: Group,
- finder: ->(id) { find_label(user_group, id) }
+ finder: ->(id) { find_label(user_group, id) },
+ feature_category: :issue_tracking
}
]
@@ -44,7 +48,7 @@ module API
desc 'Subscribe to a resource' do
success subscribable[:entity]
end
- post ":id/#{subscribable[:type]}/:subscribable_id/subscribe" do
+ post ":id/#{subscribable[:type]}/:subscribable_id/subscribe", subscribable.slice(:feature_category) do
parent = parent_resource(source_type)
resource = instance_exec(params[:subscribable_id], &subscribable[:finder])
@@ -59,7 +63,7 @@ module API
desc 'Unsubscribe from a resource' do
success subscribable[:entity]
end
- post ":id/#{subscribable[:type]}/:subscribable_id/unsubscribe" do
+ post ":id/#{subscribable[:type]}/:subscribable_id/unsubscribe", subscribable.slice(:feature_category) do
parent = parent_resource(source_type)
resource = instance_exec(params[:subscribable_id], &subscribable[:finder])
diff --git a/lib/api/suggestions.rb b/lib/api/suggestions.rb
index f23d279c3f4..a024d6de874 100644
--- a/lib/api/suggestions.rb
+++ b/lib/api/suggestions.rb
@@ -4,6 +4,8 @@ module API
class Suggestions < ::API::Base
before { authenticate! }
+ feature_category :code_review
+
resource :suggestions do
desc 'Apply suggestion patch in the Merge Request it was created' do
success Entities::Suggestion
diff --git a/lib/api/system_hooks.rb b/lib/api/system_hooks.rb
index 2820d305d0f..42e16d47a0b 100644
--- a/lib/api/system_hooks.rb
+++ b/lib/api/system_hooks.rb
@@ -4,6 +4,8 @@ module API
class SystemHooks < ::API::Base
include PaginationParams
+ feature_category :integrations
+
before do
authenticate!
authenticated_as_admin!
diff --git a/lib/api/tags.rb b/lib/api/tags.rb
index b969394ec47..7636c45bdac 100644
--- a/lib/api/tags.rb
+++ b/lib/api/tags.rb
@@ -23,7 +23,7 @@ module API
optional :search, type: String, desc: 'Return list of tags matching the search criteria'
use :pagination
end
- get ':id/repository/tags' do
+ get ':id/repository/tags', feature_category: :source_code_management do
tags = ::TagsFinder.new(user_project.repository,
sort: "#{params[:order_by]}_#{params[:sort]}",
search: params[:search]).execute
@@ -37,7 +37,7 @@ module API
params do
requires :tag_name, type: String, desc: 'The name of the tag'
end
- get ':id/repository/tags/:tag_name', requirements: TAG_ENDPOINT_REQUIREMENTS do
+ get ':id/repository/tags/:tag_name', requirements: TAG_ENDPOINT_REQUIREMENTS, feature_category: :source_code_management do
tag = user_project.repository.find_tag(params[:tag_name])
not_found!('Tag') unless tag
@@ -54,7 +54,7 @@ module API
optional :message, type: String, desc: 'Specifying a message creates an annotated tag'
optional :release_description, type: String, desc: 'Specifying release notes stored in the GitLab database (deprecated in GitLab 11.7)'
end
- post ':id/repository/tags' do
+ post ':id/repository/tags', :release_orchestration do
authorize_admin_tag
result = ::Tags::CreateService.new(user_project, current_user)
@@ -86,7 +86,7 @@ module API
params do
requires :tag_name, type: String, desc: 'The name of the tag'
end
- delete ':id/repository/tags/:tag_name', requirements: TAG_ENDPOINT_REQUIREMENTS do
+ delete ':id/repository/tags/:tag_name', requirements: TAG_ENDPOINT_REQUIREMENTS, feature_category: :source_code_management do
authorize_admin_tag
tag = user_project.repository.find_tag(params[:tag_name])
@@ -112,7 +112,7 @@ module API
requires :tag_name, type: String, desc: 'The name of the tag', as: :tag
requires :description, type: String, desc: 'Release notes with markdown support'
end
- post ':id/repository/tags/:tag_name/release', requirements: TAG_ENDPOINT_REQUIREMENTS do
+ post ':id/repository/tags/:tag_name/release', requirements: TAG_ENDPOINT_REQUIREMENTS, feature_category: :release_orchestration do
authorize_create_release!
##
@@ -144,7 +144,7 @@ module API
requires :tag_name, type: String, desc: 'The name of the tag', as: :tag
requires :description, type: String, desc: 'Release notes with markdown support'
end
- put ':id/repository/tags/:tag_name/release', requirements: TAG_ENDPOINT_REQUIREMENTS do
+ put ':id/repository/tags/:tag_name/release', requirements: TAG_ENDPOINT_REQUIREMENTS, feature_category: :release_orchestration do
authorize_update_release!
result = ::Releases::UpdateService
diff --git a/lib/api/templates.rb b/lib/api/templates.rb
index 0b427bbf5b9..b7fb35eac03 100644
--- a/lib/api/templates.rb
+++ b/lib/api/templates.rb
@@ -4,6 +4,8 @@ module API
class Templates < ::API::Base
include PaginationParams
+ feature_category :templates
+
GLOBAL_TEMPLATE_TYPES = {
gitignores: {
gitlab_version: 8.8
diff --git a/lib/api/terraform/state.rb b/lib/api/terraform/state.rb
index 3dbde4639ca..c664c0a4590 100644
--- a/lib/api/terraform/state.rb
+++ b/lib/api/terraform/state.rb
@@ -7,6 +7,8 @@ module API
class State < ::API::Base
include ::Gitlab::Utils::StrongMemoize
+ feature_category :infrastructure_as_code
+
default_format :json
before do
@@ -51,7 +53,7 @@ module API
no_content! if data.empty?
remote_state_handler.handle_with_lock do |state|
- state.update_file!(CarrierWaveStringFile.new(data), version: params[:serial])
+ state.update_file!(CarrierWaveStringFile.new(data), version: params[:serial], build: current_authenticated_job)
end
body false
diff --git a/lib/api/terraform/state_version.rb b/lib/api/terraform/state_version.rb
index b4a0efd7a2b..d3680323b9f 100644
--- a/lib/api/terraform/state_version.rb
+++ b/lib/api/terraform/state_version.rb
@@ -5,6 +5,8 @@ module API
class StateVersion < ::API::Base
default_format :json
+ feature_category :infrastructure_as_code
+
before do
authenticate!
authorize! :read_terraform_state, user_project
diff --git a/lib/api/todos.rb b/lib/api/todos.rb
index ce07d13cc9a..03850ba1c4e 100644
--- a/lib/api/todos.rb
+++ b/lib/api/todos.rb
@@ -6,6 +6,8 @@ module API
before { authenticate! }
+ feature_category :issue_tracking
+
ISSUABLE_TYPES = {
'merge_requests' => ->(iid) { find_merge_request_with_access(iid) },
'issues' => ->(iid) { find_project_issue(iid) }
diff --git a/lib/api/triggers.rb b/lib/api/triggers.rb
index 960d004a04c..aebbc95cbea 100644
--- a/lib/api/triggers.rb
+++ b/lib/api/triggers.rb
@@ -6,6 +6,8 @@ module API
HTTP_GITLAB_EVENT_HEADER = "HTTP_#{WebHookService::GITLAB_EVENT_HEADER}".underscore.upcase
+ feature_category :continuous_integration
+
params do
requires :id, type: String, desc: 'The ID of a project'
end
diff --git a/lib/api/unleash.rb b/lib/api/unleash.rb
index 907422118f1..3148c56339a 100644
--- a/lib/api/unleash.rb
+++ b/lib/api/unleash.rb
@@ -4,6 +4,8 @@ module API
class Unleash < ::API::Base
include PaginationParams
+ feature_category :feature_flags
+
namespace :feature_flags do
resource :unleash, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do
params do
diff --git a/lib/api/usage_data.rb b/lib/api/usage_data.rb
index fa5bfc1cbe9..7b038ec74bb 100644
--- a/lib/api/usage_data.rb
+++ b/lib/api/usage_data.rb
@@ -4,6 +4,8 @@ module API
class UsageData < ::API::Base
before { authenticate! }
+ feature_category :collection
+
namespace 'usage_data' do
before do
not_found! unless Feature.enabled?(:usage_data_api, default_enabled: true)
diff --git a/lib/api/user_counts.rb b/lib/api/user_counts.rb
index 6d9db53fec8..3071f08e1de 100644
--- a/lib/api/user_counts.rb
+++ b/lib/api/user_counts.rb
@@ -2,6 +2,8 @@
module API
class UserCounts < ::API::Base
+ feature_category :navigation
+
resource :user_counts do
desc 'Return the user specific counts' do
detail 'Open MR Count'
diff --git a/lib/api/users.rb b/lib/api/users.rb
index e7c1d644324..501ed629c7e 100644
--- a/lib/api/users.rb
+++ b/lib/api/users.rb
@@ -8,6 +8,8 @@ module API
allow_access_with_scope :read_user, if: -> (request) { request.get? }
+ feature_category :users, ['/users/:id/custom_attributes', '/users/:id/custom_attributes/:key']
+
resource :users, requirements: { uid: /[0-9]*/, id: /[0-9]*/ } do
include CustomAttributesEndpoints
@@ -63,9 +65,9 @@ module API
params :sort_params do
optional :order_by, type: String, values: %w[id name username created_at updated_at],
- default: 'id', desc: 'Return users ordered by a field'
+ default: 'id', desc: 'Return users ordered by a field'
optional :sort, type: String, values: %w[asc desc], default: 'desc',
- desc: 'Return users sorted in ascending and descending order'
+ desc: 'Return users sorted in ascending and descending order'
end
end
@@ -93,7 +95,7 @@ module API
use :optional_index_params_ee
end
# rubocop: disable CodeReuse/ActiveRecord
- get do
+ get feature_category: :users do
authenticated_as_admin! if params[:external].present? || (params[:extern_uid].present? && params[:provider].present?)
unless current_user&.admin?
@@ -134,7 +136,7 @@ module API
use :with_custom_attributes
end
# rubocop: disable CodeReuse/ActiveRecord
- get ":id" do
+ get ":id", feature_category: :users do
user = User.find_by(id: params[:id])
not_found!('User') unless user && can?(current_user, :read_user, user)
@@ -149,7 +151,7 @@ module API
params do
requires :user_id, type: String, desc: 'The ID or username of the user'
end
- get ":user_id/status", requirements: API::USER_REQUIREMENTS do
+ get ":user_id/status", requirements: API::USER_REQUIREMENTS, feature_category: :users do
user = find_user(params[:user_id])
not_found!('User') unless user && can?(current_user, :read_user, user)
@@ -170,7 +172,7 @@ module API
optional :force_random_password, type: Boolean, desc: 'Flag indicating a random password will be set'
use :optional_attributes
end
- post do
+ post feature_category: :users do
authenticated_as_admin!
params = declared_params(include_missing: false)
@@ -204,7 +206,7 @@ module API
use :optional_attributes
end
# rubocop: disable CodeReuse/ActiveRecord
- put ":id" do
+ put ":id", feature_category: :users do
authenticated_as_admin!
user = User.find_by(id: params.delete(:id))
@@ -245,7 +247,7 @@ module API
requires :provider, type: String, desc: 'The external provider'
end
# rubocop: disable CodeReuse/ActiveRecord
- delete ":id/identities/:provider" do
+ delete ":id/identities/:provider", feature_category: :authentication_and_authorization do
authenticated_as_admin!
user = User.find_by(id: params[:id])
@@ -268,7 +270,7 @@ module API
optional :expires_at, type: DateTime, desc: 'The expiration date of the SSH key in ISO 8601 format (YYYY-MM-DDTHH:MM:SSZ)'
end
# rubocop: disable CodeReuse/ActiveRecord
- post ":id/keys" do
+ post ":id/keys", feature_category: :authentication_and_authorization do
authenticated_as_admin!
user = User.find_by(id: params.delete(:id))
@@ -291,7 +293,7 @@ module API
requires :user_id, type: String, desc: 'The ID or username of the user'
use :pagination
end
- get ':user_id/keys', requirements: API::USER_REQUIREMENTS do
+ get ':user_id/keys', requirements: API::USER_REQUIREMENTS, feature_category: :authentication_and_authorization do
user = find_user(params[:user_id])
not_found!('User') unless user && can?(current_user, :read_user, user)
@@ -307,7 +309,7 @@ module API
requires :key_id, type: Integer, desc: 'The ID of the SSH key'
end
# rubocop: disable CodeReuse/ActiveRecord
- delete ':id/keys/:key_id' do
+ delete ':id/keys/:key_id', feature_category: :authentication_and_authorization do
authenticated_as_admin!
user = User.find_by(id: params[:id])
@@ -332,7 +334,7 @@ module API
requires :key, type: String, desc: 'The new GPG key'
end
# rubocop: disable CodeReuse/ActiveRecord
- post ':id/gpg_keys' do
+ post ':id/gpg_keys', feature_category: :authentication_and_authorization do
authenticated_as_admin!
user = User.find_by(id: params.delete(:id))
@@ -357,7 +359,7 @@ module API
use :pagination
end
# rubocop: disable CodeReuse/ActiveRecord
- get ':id/gpg_keys' do
+ get ':id/gpg_keys', feature_category: :authentication_and_authorization do
user = User.find_by(id: params[:id])
not_found!('User') unless user
@@ -374,7 +376,7 @@ module API
requires :key_id, type: Integer, desc: 'The ID of the GPG key'
end
# rubocop: disable CodeReuse/ActiveRecord
- get ':id/gpg_keys/:key_id' do
+ get ':id/gpg_keys/:key_id', feature_category: :authentication_and_authorization do
user = User.find_by(id: params[:id])
not_found!('User') unless user
@@ -393,7 +395,7 @@ module API
requires :key_id, type: Integer, desc: 'The ID of the GPG key'
end
# rubocop: disable CodeReuse/ActiveRecord
- delete ':id/gpg_keys/:key_id' do
+ delete ':id/gpg_keys/:key_id', feature_category: :authentication_and_authorization do
authenticated_as_admin!
user = User.find_by(id: params[:id])
@@ -417,7 +419,7 @@ module API
requires :key_id, type: Integer, desc: 'The ID of the GPG key'
end
# rubocop: disable CodeReuse/ActiveRecord
- post ':id/gpg_keys/:key_id/revoke' do
+ post ':id/gpg_keys/:key_id/revoke', feature_category: :authentication_and_authorization do
authenticated_as_admin!
user = User.find_by(id: params[:id])
@@ -440,7 +442,7 @@ module API
optional :skip_confirmation, type: Boolean, desc: 'Skip confirmation of email and assume it is verified'
end
# rubocop: disable CodeReuse/ActiveRecord
- post ":id/emails" do
+ post ":id/emails", feature_category: :users do
authenticated_as_admin!
user = User.find_by(id: params.delete(:id))
@@ -464,7 +466,7 @@ module API
use :pagination
end
# rubocop: disable CodeReuse/ActiveRecord
- get ':id/emails' do
+ get ':id/emails', feature_category: :users do
authenticated_as_admin!
user = User.find_by(id: params[:id])
not_found!('User') unless user
@@ -481,7 +483,7 @@ module API
requires :email_id, type: Integer, desc: 'The ID of the email'
end
# rubocop: disable CodeReuse/ActiveRecord
- delete ':id/emails/:email_id' do
+ delete ':id/emails/:email_id', feature_category: :users do
authenticated_as_admin!
user = User.find_by(id: params[:id])
not_found!('User') unless user
@@ -503,7 +505,7 @@ module API
optional :hard_delete, type: Boolean, desc: "Whether to remove a user's contributions"
end
# rubocop: disable CodeReuse/ActiveRecord
- delete ":id" do
+ delete ":id", feature_category: :users do
Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab/issues/20757')
authenticated_as_admin!
@@ -523,7 +525,7 @@ module API
requires :id, type: Integer, desc: 'The ID of the user'
end
# rubocop: disable CodeReuse/ActiveRecord
- post ':id/activate' do
+ post ':id/activate', feature_category: :authentication_and_authorization do
authenticated_as_admin!
user = User.find_by(id: params[:id])
@@ -538,7 +540,7 @@ module API
requires :id, type: Integer, desc: 'The ID of the user'
end
# rubocop: disable CodeReuse/ActiveRecord
- post ':id/deactivate' do
+ post ':id/deactivate', feature_category: :authentication_and_authorization do
authenticated_as_admin!
user = User.find_by(id: params[:id])
not_found!('User') unless user
@@ -564,7 +566,7 @@ module API
requires :id, type: Integer, desc: 'The ID of the user'
end
# rubocop: disable CodeReuse/ActiveRecord
- post ':id/block' do
+ post ':id/block', feature_category: :authentication_and_authorization do
authenticated_as_admin!
user = User.find_by(id: params[:id])
not_found!('User') unless user
@@ -589,7 +591,7 @@ module API
requires :id, type: Integer, desc: 'The ID of the user'
end
# rubocop: disable CodeReuse/ActiveRecord
- post ':id/unblock' do
+ post ':id/unblock', feature_category: :authentication_and_authorization do
authenticated_as_admin!
user = User.find_by(id: params[:id])
not_found!('User') unless user
@@ -612,7 +614,7 @@ module API
optional :type, type: String, values: %w[Project Namespace]
use :pagination
end
- get ":user_id/memberships" do
+ get ":user_id/memberships", feature_category: :users do
authenticated_as_admin!
user = find_user_by_id(params)
@@ -656,7 +658,9 @@ module API
use :pagination
optional :state, type: String, default: 'all', values: %w[all active inactive], desc: 'Filters (all|active|inactive) impersonation_tokens'
end
- get { present paginate(finder(declared_params(include_missing: false)).execute), with: Entities::ImpersonationToken }
+ get feature_category :authentication_and_authorization do
+ present paginate(finder(declared_params(include_missing: false)).execute), with: Entities::ImpersonationToken
+ end
desc 'Create a impersonation token. Available only for admins.' do
detail 'This feature was introduced in GitLab 9.0'
@@ -667,7 +671,7 @@ module API
optional :expires_at, type: Date, desc: 'The expiration date in the format YEAR-MONTH-DAY of the impersonation token'
optional :scopes, type: Array, desc: 'The array of scopes of the impersonation token'
end
- post do
+ post feature_category: :authentication_and_authorization do
impersonation_token = finder.build(declared_params(include_missing: false))
if impersonation_token.save
@@ -684,7 +688,7 @@ module API
params do
requires :impersonation_token_id, type: Integer, desc: 'The ID of the impersonation token'
end
- get ':impersonation_token_id' do
+ get ':impersonation_token_id', feature_category: :authentication_and_authorization do
present find_impersonation_token, with: Entities::ImpersonationToken
end
@@ -694,7 +698,7 @@ module API
params do
requires :impersonation_token_id, type: Integer, desc: 'The ID of the impersonation token'
end
- delete ':impersonation_token_id' do
+ delete ':impersonation_token_id', feature_category: :authentication_and_authorization do
token = find_impersonation_token
destroy_conditionally!(token) do
@@ -702,6 +706,40 @@ module API
end
end
end
+
+ resource :personal_access_tokens do
+ helpers do
+ def target_user
+ find_user_by_id(params)
+ end
+ end
+
+ before { authenticated_as_admin! }
+
+ desc 'Create a personal access token. Available only for admins.' do
+ detail 'This feature was introduced in GitLab 13.6'
+ success Entities::PersonalAccessTokenWithToken
+ end
+ params do
+ requires :name, type: String, desc: 'The name of the personal access token'
+ requires :scopes, type: Array[String], coerce_with: ::API::Validations::Types::CommaSeparatedToArray.coerce, values: ::Gitlab::Auth.all_available_scopes.map(&:to_s),
+ desc: 'The array of scopes of the personal access token'
+ optional :expires_at, type: Date, desc: 'The expiration date in the format YEAR-MONTH-DAY of the personal access token'
+ end
+ post feature_category: :authentication_and_authorization do
+ not_found! unless Feature.enabled?(:pat_creation_api_for_admin)
+
+ response = ::PersonalAccessTokens::CreateService.new(
+ current_user: current_user, target_user: target_user, params: declared_params(include_missing: false)
+ ).execute
+
+ if response.success?
+ present response.payload[:personal_access_token], with: Entities::PersonalAccessTokenWithToken
+ else
+ render_api_error!(response.message, response.http_status || :unprocessable_entity)
+ end
+ end
+ end
end
end
@@ -716,7 +754,7 @@ module API
desc 'Get the currently authenticated user' do
success Entities::UserPublic
end
- get do
+ get feature_category: :users do
entity =
if current_user.admin?
Entities::UserWithAdmin
@@ -734,7 +772,7 @@ module API
params do
use :pagination
end
- get "keys" do
+ get "keys", feature_category: :authentication_and_authorization do
keys = current_user.keys.preload_users
present paginate(keys), with: Entities::SSHKey
@@ -747,7 +785,7 @@ module API
requires :key_id, type: Integer, desc: 'The ID of the SSH key'
end
# rubocop: disable CodeReuse/ActiveRecord
- get "keys/:key_id" do
+ get "keys/:key_id", feature_category: :authentication_and_authorization do
key = current_user.keys.find_by(id: params[:key_id])
not_found!('Key') unless key
@@ -763,7 +801,7 @@ module API
requires :title, type: String, desc: 'The title of the new SSH key'
optional :expires_at, type: DateTime, desc: 'The expiration date of the SSH key in ISO 8601 format (YYYY-MM-DDTHH:MM:SSZ)'
end
- post "keys" do
+ post "keys", feature_category: :authentication_and_authorization do
key = ::Keys::CreateService.new(current_user, declared_params(include_missing: false)).execute
if key.persisted?
@@ -780,7 +818,7 @@ module API
requires :key_id, type: Integer, desc: 'The ID of the SSH key'
end
# rubocop: disable CodeReuse/ActiveRecord
- delete "keys/:key_id" do
+ delete "keys/:key_id", feature_category: :authentication_and_authorization do
key = current_user.keys.find_by(id: params[:key_id])
not_found!('Key') unless key
@@ -798,7 +836,7 @@ module API
params do
use :pagination
end
- get 'gpg_keys' do
+ get 'gpg_keys', feature_category: :authentication_and_authorization do
present paginate(current_user.gpg_keys), with: Entities::GpgKey
end
@@ -810,7 +848,7 @@ module API
requires :key_id, type: Integer, desc: 'The ID of the GPG key'
end
# rubocop: disable CodeReuse/ActiveRecord
- get 'gpg_keys/:key_id' do
+ get 'gpg_keys/:key_id', feature_category: :authentication_and_authorization do
key = current_user.gpg_keys.find_by(id: params[:key_id])
not_found!('GPG Key') unless key
@@ -825,7 +863,7 @@ module API
params do
requires :key, type: String, desc: 'The new GPG key'
end
- post 'gpg_keys' do
+ post 'gpg_keys', feature_category: :authentication_and_authorization do
key = ::GpgKeys::CreateService.new(current_user, declared_params(include_missing: false)).execute
if key.persisted?
@@ -842,7 +880,7 @@ module API
requires :key_id, type: Integer, desc: 'The ID of the GPG key'
end
# rubocop: disable CodeReuse/ActiveRecord
- post 'gpg_keys/:key_id/revoke' do
+ post 'gpg_keys/:key_id/revoke', feature_category: :authentication_and_authorization do
key = current_user.gpg_keys.find_by(id: params[:key_id])
not_found!('GPG Key') unless key
@@ -858,7 +896,7 @@ module API
requires :key_id, type: Integer, desc: 'The ID of the SSH key'
end
# rubocop: disable CodeReuse/ActiveRecord
- delete 'gpg_keys/:key_id' do
+ delete 'gpg_keys/:key_id', feature_category: :authentication_and_authorization do
key = current_user.gpg_keys.find_by(id: params[:key_id])
not_found!('GPG Key') unless key
@@ -875,7 +913,7 @@ module API
params do
use :pagination
end
- get "emails" do
+ get "emails", feature_category: :users do
present paginate(current_user.emails), with: Entities::Email
end
@@ -886,7 +924,7 @@ module API
requires :email_id, type: Integer, desc: 'The ID of the email'
end
# rubocop: disable CodeReuse/ActiveRecord
- get "emails/:email_id" do
+ get "emails/:email_id", feature_category: :users do
email = current_user.emails.find_by(id: params[:email_id])
not_found!('Email') unless email
@@ -900,7 +938,7 @@ module API
params do
requires :email, type: String, desc: 'The new email'
end
- post "emails" do
+ post "emails", feature_category: :users do
email = Emails::CreateService.new(current_user, declared_params.merge(user: current_user)).execute
if email.errors.blank?
@@ -915,7 +953,7 @@ module API
requires :email_id, type: Integer, desc: 'The ID of the email'
end
# rubocop: disable CodeReuse/ActiveRecord
- delete "emails/:email_id" do
+ delete "emails/:email_id", feature_category: :users do
email = current_user.emails.find_by(id: params[:email_id])
not_found!('Email') unless email
@@ -931,7 +969,7 @@ module API
use :pagination
end
# rubocop: disable CodeReuse/ActiveRecord
- get "activities" do
+ get "activities", feature_category: :users do
authenticated_as_admin!
activities = User
@@ -948,8 +986,9 @@ module API
params do
optional :emoji, type: String, desc: "The emoji to set on the status"
optional :message, type: String, desc: "The status message to set"
+ optional :availability, type: String, desc: "The availability of user to set"
end
- put "status" do
+ put "status", feature_category: :users do
forbidden! unless can?(current_user, :update_user_status, current_user)
if ::Users::SetStatusService.new(current_user, declared_params).execute
@@ -962,7 +1001,7 @@ module API
desc 'get the status of the current user' do
success Entities::UserStatus
end
- get 'status' do
+ get 'status', feature_category: :users do
present current_user.status || {}, with: Entities::UserStatus
end
end
diff --git a/lib/api/v3/github.rb b/lib/api/v3/github.rb
index aed88e6091c..327335aec2d 100644
--- a/lib/api/v3/github.rb
+++ b/lib/api/v3/github.rb
@@ -22,6 +22,8 @@ module API
include PaginationParams
+ feature_category :integrations
+
before do
authorize_jira_user_agent!(request)
authenticate!
diff --git a/lib/api/validations/validators/email_or_email_list.rb b/lib/api/validations/validators/email_or_email_list.rb
new file mode 100644
index 00000000000..b7f2a0cd443
--- /dev/null
+++ b/lib/api/validations/validators/email_or_email_list.rb
@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module API
+ module Validations
+ module Validators
+ class EmailOrEmailList < Grape::Validations::Base
+ def validate_param!(attr_name, params)
+ value = params[attr_name]
+
+ return unless value
+
+ return if value.split(',').map { |v| ValidateEmail.valid?(v) }.all?
+
+ raise Grape::Exceptions::Validation,
+ params: [@scope.full_name(attr_name)],
+ message: "contains an invalid email address"
+ end
+ end
+ end
+ end
+end
diff --git a/lib/api/variables.rb b/lib/api/variables.rb
index f5de3d844e6..94fa98b7a14 100644
--- a/lib/api/variables.rb
+++ b/lib/api/variables.rb
@@ -7,6 +7,8 @@ module API
before { authenticate! }
before { authorize! :admin_build, user_project }
+ feature_category :continuous_integration
+
helpers do
def filter_variable_parameters(params)
# This method exists so that EE can more easily filter out certain
diff --git a/lib/api/version.rb b/lib/api/version.rb
index 841b55f8d6c..f8072658cc6 100644
--- a/lib/api/version.rb
+++ b/lib/api/version.rb
@@ -9,6 +9,8 @@ module API
before { authenticate! }
+ feature_category :not_owned
+
METADATA_QUERY = <<~EOF
{
metadata {
diff --git a/lib/api/wikis.rb b/lib/api/wikis.rb
index 21f457046f1..3fa42be47a9 100644
--- a/lib/api/wikis.rb
+++ b/lib/api/wikis.rb
@@ -4,6 +4,8 @@ module API
class Wikis < ::API::Base
helpers ::API::Helpers::WikisHelpers
+ feature_category :wiki
+
helpers do
attr_reader :container
diff --git a/lib/atlassian/jira_connect/client.rb b/lib/atlassian/jira_connect/client.rb
index 0b578c03782..f81ed462174 100644
--- a/lib/atlassian/jira_connect/client.rb
+++ b/lib/atlassian/jira_connect/client.rb
@@ -3,19 +3,25 @@
module Atlassian
module JiraConnect
class Client < Gitlab::HTTP
+ def self.generate_update_sequence_id
+ Gitlab::Metrics::System.monotonic_time.to_i
+ end
+
def initialize(base_uri, shared_secret)
@base_uri = base_uri
@shared_secret = shared_secret
end
- def store_dev_info(project:, commits: nil, branches: nil, merge_requests: nil)
+ def store_dev_info(project:, commits: nil, branches: nil, merge_requests: nil, update_sequence_id: nil)
dev_info_json = {
repositories: [
Serializers::RepositoryEntity.represent(
project,
commits: commits,
branches: branches,
- merge_requests: merge_requests
+ merge_requests: merge_requests,
+ user_notes_count: user_notes_count(merge_requests),
+ update_sequence_id: update_sequence_id
)
]
}.to_json
@@ -32,6 +38,14 @@ module Atlassian
private
+ def user_notes_count(merge_requests)
+ return unless merge_requests
+
+ Note.count_for_collection(merge_requests.map(&:id), 'MergeRequest').map do |count_group|
+ [count_group.noteable_id, count_group.count]
+ end.to_h
+ end
+
def jwt_token(http_method, uri)
claims = Atlassian::Jwt.build_claims(
Atlassian::JiraConnect.app_key,
diff --git a/lib/atlassian/jira_connect/serializers/base_entity.rb b/lib/atlassian/jira_connect/serializers/base_entity.rb
index c5490aa3f54..94deb174a45 100644
--- a/lib/atlassian/jira_connect/serializers/base_entity.rb
+++ b/lib/atlassian/jira_connect/serializers/base_entity.rb
@@ -9,12 +9,12 @@ module Atlassian
format_with(:string) { |value| value.to_s }
- expose :monotonic_time, as: :updateSequenceId
+ expose :update_sequence_id, as: :updateSequenceId
private
- def monotonic_time
- Gitlab::Metrics::System.monotonic_time.to_i
+ def update_sequence_id
+ options[:update_sequence_id] || Client.generate_update_sequence_id
end
end
end
diff --git a/lib/atlassian/jira_connect/serializers/pull_request_entity.rb b/lib/atlassian/jira_connect/serializers/pull_request_entity.rb
index 0ddfcbf52ea..e2dc197969b 100644
--- a/lib/atlassian/jira_connect/serializers/pull_request_entity.rb
+++ b/lib/atlassian/jira_connect/serializers/pull_request_entity.rb
@@ -20,7 +20,13 @@ module Atlassian
end
expose :title
expose :author, using: JiraConnect::Serializers::AuthorEntity
- expose :user_notes_count, as: :commentCount
+ expose :commentCount do |mr|
+ if options[:user_notes_count]
+ options[:user_notes_count].fetch(mr.id, 0)
+ else
+ mr.user_notes_count
+ end
+ end
expose :source_branch, as: :sourceBranch
expose :target_branch, as: :destinationBranch
expose :lastUpdate do |mr|
diff --git a/lib/atlassian/jira_connect/serializers/repository_entity.rb b/lib/atlassian/jira_connect/serializers/repository_entity.rb
index 819ca2b62e0..616bbc85bfe 100644
--- a/lib/atlassian/jira_connect/serializers/repository_entity.rb
+++ b/lib/atlassian/jira_connect/serializers/repository_entity.rb
@@ -15,13 +15,17 @@ module Atlassian
end
expose :commits do |project, options|
- JiraConnect::Serializers::CommitEntity.represent options[:commits], project: project
+ JiraConnect::Serializers::CommitEntity.represent options[:commits], project: project, update_sequence_id: options[:update_sequence_id]
end
expose :branches do |project, options|
- JiraConnect::Serializers::BranchEntity.represent options[:branches], project: project
+ JiraConnect::Serializers::BranchEntity.represent options[:branches], project: project, update_sequence_id: options[:update_sequence_id]
end
expose :pullRequests do |project, options|
- JiraConnect::Serializers::PullRequestEntity.represent options[:merge_requests], project: project
+ JiraConnect::Serializers::PullRequestEntity.represent(
+ options[:merge_requests],
+ update_sequence_id: options[:update_sequence_id],
+ user_notes_count: options[:user_notes_count]
+ )
end
end
end
diff --git a/lib/backup/files.rb b/lib/backup/files.rb
index 619a62fd6f6..a0948f8c0f5 100644
--- a/lib/backup/files.rb
+++ b/lib/backup/files.rb
@@ -26,7 +26,7 @@ module Backup
FileUtils.rm_f(backup_tarball)
if ENV['STRATEGY'] == 'copy'
- cmd = [%w(rsync -a), exclude_dirs(:rsync), %W(#{app_files_dir} #{Gitlab.config.backup.path})].flatten
+ cmd = [%w[rsync -a], exclude_dirs(:rsync), %W[#{app_files_dir} #{Gitlab.config.backup.path}]].flatten
output, status = Gitlab::Popen.popen(cmd)
unless status == 0
@@ -34,19 +34,27 @@ module Backup
raise Backup::Error, 'Backup failed'
end
- tar_cmd = [tar, exclude_dirs(:tar), %W(-C #{@backup_files_dir} -cf - .)].flatten
- run_pipeline!([tar_cmd, gzip_cmd], out: [backup_tarball, 'w', 0600])
+ tar_cmd = [tar, exclude_dirs(:tar), %W[-C #{@backup_files_dir} -cf - .]].flatten
+ status_list, output = run_pipeline!([tar_cmd, gzip_cmd], out: [backup_tarball, 'w', 0600])
FileUtils.rm_rf(@backup_files_dir)
else
- tar_cmd = [tar, exclude_dirs(:tar), %W(-C #{app_files_dir} -cf - .)].flatten
- run_pipeline!([tar_cmd, gzip_cmd], out: [backup_tarball, 'w', 0600])
+ tar_cmd = [tar, exclude_dirs(:tar), %W[-C #{app_files_dir} -cf - .]].flatten
+ status_list, output = run_pipeline!([tar_cmd, gzip_cmd], out: [backup_tarball, 'w', 0600])
+ end
+
+ unless pipeline_succeeded?(tar_status: status_list[0], gzip_status: status_list[1], output: output)
+ raise Backup::Error, "Backup operation failed: #{output}"
end
end
def restore
backup_existing_files_dir
- run_pipeline!([%w(gzip -cd), %W(#{tar} --unlink-first --recursive-unlink -C #{app_files_dir} -xf -)], in: backup_tarball)
+ cmd_list = [%w[gzip -cd], %W[#{tar} --unlink-first --recursive-unlink -C #{app_files_dir} -xf -]]
+ status_list, output = run_pipeline!(cmd_list, in: backup_tarball)
+ unless pipeline_succeeded?(gzip_status: status_list[0], tar_status: status_list[1], output: output)
+ raise Backup::Error, "Restore operation failed: #{output}"
+ end
end
def tar
@@ -78,13 +86,44 @@ module Backup
def run_pipeline!(cmd_list, options = {})
err_r, err_w = IO.pipe
options[:err] = err_w
- status = Open3.pipeline(*cmd_list, options)
+ status_list = Open3.pipeline(*cmd_list, options)
err_w.close
- return if status.compact.all?(&:success?)
- regex = /^g?tar: \.: Cannot mkdir: No such file or directory$/
- error = err_r.read
- raise Backup::Error, "Backup failed. #{error}" unless error =~ regex
+ [status_list, err_r.read]
+ end
+
+ def noncritical_warning?(warning)
+ noncritical_warnings = [
+ /^g?tar: \.: Cannot mkdir: No such file or directory$/
+ ]
+
+ noncritical_warnings.map { |w| warning =~ w }.any?
+ end
+
+ def pipeline_succeeded?(tar_status:, gzip_status:, output:)
+ return false unless gzip_status&.success?
+
+ tar_status&.success? || tar_ignore_non_success?(tar_status.exitstatus, output)
+ end
+
+ def tar_ignore_non_success?(exitstatus, output)
+ # tar can exit with nonzero code:
+ # 1 - if some files changed (i.e. a CI job is currently writes to log)
+ # 2 - if it cannot create `.` directory (see issue https://gitlab.com/gitlab-org/gitlab/-/issues/22442)
+ # http://www.gnu.org/software/tar/manual/html_section/tar_19.html#Synopsis
+ # so check tar status 1 or stderr output against some non-critical warnings
+ if exitstatus == 1
+ $stdout.puts "Ignoring tar exit status 1 'Some files differ': #{output}"
+ return true
+ end
+
+ # allow tar to fail with other non-success status if output contain non-critical warning
+ if noncritical_warning?(output)
+ $stdout.puts "Ignoring non-success exit status #{exitstatus} due to output of non-critical warning(s): #{output}"
+ return true
+ end
+
+ false
end
def exclude_dirs(fmt)
diff --git a/lib/banzai/filter/emoji_filter.rb b/lib/banzai/filter/emoji_filter.rb
index b32fe5e8301..8952a3ff6b4 100644
--- a/lib/banzai/filter/emoji_filter.rb
+++ b/lib/banzai/filter/emoji_filter.rb
@@ -8,6 +8,7 @@ module Banzai
# Based on HTML::Pipeline::EmojiFilter
class EmojiFilter < HTML::Pipeline::Filter
IGNORED_ANCESTOR_TAGS = %w(pre code tt).to_set
+ IGNORE_UNICODE_EMOJIS = %w(™ © ®).freeze
def call
doc.search(".//text()").each do |node|
@@ -60,7 +61,11 @@ module Banzai
# Build a regexp that matches all valid unicode emojis names.
def self.emoji_unicode_pattern
- @emoji_unicode_pattern ||= /(#{Gitlab::Emoji.emojis_unicodes.map { |moji| Regexp.escape(moji) }.join('|')})/
+ @emoji_unicode_pattern ||=
+ begin
+ filtered_emojis = Gitlab::Emoji.emojis_unicodes - IGNORE_UNICODE_EMOJIS
+ /(#{filtered_emojis.map { |moji| Regexp.escape(moji) }.join('|')})/
+ end
end
private
diff --git a/lib/banzai/filter/normalize_source_filter.rb b/lib/banzai/filter/normalize_source_filter.rb
new file mode 100644
index 00000000000..975cd540873
--- /dev/null
+++ b/lib/banzai/filter/normalize_source_filter.rb
@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module Banzai
+ module Filter
+ class NormalizeSourceFilter < HTML::Pipeline::Filter
+ UTF8_BOM = "\xEF\xBB\xBF"
+
+ def call
+ # Remove UTF8_BOM from beginning of source text
+ html.delete_prefix(UTF8_BOM)
+ end
+ end
+ end
+end
diff --git a/lib/banzai/filter/reference_filter.rb b/lib/banzai/filter/reference_filter.rb
index cfd4b932568..d22a0e0b504 100644
--- a/lib/banzai/filter/reference_filter.rb
+++ b/lib/banzai/filter/reference_filter.rb
@@ -119,7 +119,7 @@ module Banzai
# Yields the link's URL and inner HTML whenever the node is a valid <a> tag.
def yield_valid_link(node)
- link = CGI.unescape(node.attr('href').to_s)
+ link = unescape_link(node.attr('href').to_s)
inner_html = node.inner_html
return unless link.force_encoding('UTF-8').valid_encoding?
@@ -127,6 +127,10 @@ module Banzai
yield link, inner_html
end
+ def unescape_link(href)
+ CGI.unescape(href)
+ end
+
def replace_text_when_pattern_matches(node, index, pattern)
return unless node.text =~ pattern
diff --git a/lib/banzai/filter/vulnerability_reference_filter.rb b/lib/banzai/filter/vulnerability_reference_filter.rb
new file mode 100644
index 00000000000..a59e9836d69
--- /dev/null
+++ b/lib/banzai/filter/vulnerability_reference_filter.rb
@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module Banzai
+ module Filter
+ # The actual filter is implemented in the EE mixin
+ class VulnerabilityReferenceFilter < IssuableReferenceFilter
+ self.reference_type = :vulnerability
+
+ def self.object_class
+ Vulnerability
+ end
+
+ private
+
+ def project
+ context[:project]
+ end
+ end
+ end
+end
+
+Banzai::Filter::VulnerabilityReferenceFilter.prepend_if_ee('EE::Banzai::Filter::VulnerabilityReferenceFilter')
diff --git a/lib/banzai/pipeline/pre_process_pipeline.rb b/lib/banzai/pipeline/pre_process_pipeline.rb
index 4c2b4ca1665..1f7cb437fcd 100644
--- a/lib/banzai/pipeline/pre_process_pipeline.rb
+++ b/lib/banzai/pipeline/pre_process_pipeline.rb
@@ -5,6 +5,7 @@ module Banzai
class PreProcessPipeline < BasePipeline
def self.filters
FilterArray[
+ Filter::NormalizeSourceFilter,
Filter::FrontMatterFilter,
Filter::BlockquoteFenceFilter,
]
diff --git a/lib/banzai/reference_parser/base_parser.rb b/lib/banzai/reference_parser/base_parser.rb
index c4d7e40b46c..3dfea8ee895 100644
--- a/lib/banzai/reference_parser/base_parser.rb
+++ b/lib/banzai/reference_parser/base_parser.rb
@@ -178,7 +178,10 @@ module Banzai
collection.where(id: to_query).each { |row| cache[row.id] = row }
end
- ids.uniq.map { |id| cache[id] }.compact
+ ids.each_with_object([]) do |id, array|
+ row = cache[id]
+ array << row if row
+ end
else
collection.where(id: ids)
end
diff --git a/lib/banzai/reference_parser/vulnerability_parser.rb b/lib/banzai/reference_parser/vulnerability_parser.rb
new file mode 100644
index 00000000000..143f2605927
--- /dev/null
+++ b/lib/banzai/reference_parser/vulnerability_parser.rb
@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module Banzai
+ module ReferenceParser
+ # The actual parser is implemented in the EE mixin
+ class VulnerabilityParser < IssuableParser
+ self.reference_type = :vulnerability
+
+ def records_for_nodes(_nodes)
+ {}
+ end
+ end
+ end
+end
+
+Banzai::ReferenceParser::VulnerabilityParser.prepend_if_ee('::EE::Banzai::ReferenceParser::VulnerabilityParser')
diff --git a/lib/bitbucket_server/client.rb b/lib/bitbucket_server/client.rb
index cf55c692271..8e84afe51d7 100644
--- a/lib/bitbucket_server/client.rb
+++ b/lib/bitbucket_server/client.rb
@@ -8,9 +8,9 @@ module BitbucketServer
@connection = Connection.new(options)
end
- def pull_requests(project_key, repo)
+ def pull_requests(project_key, repo, page_offset: 0, limit: nil)
path = "/projects/#{project_key}/repos/#{repo}/pull-requests?state=ALL"
- get_collection(path, :pull_request)
+ get_collection(path, :pull_request, page_offset: page_offset, limit: limit)
end
def activities(project_key, repo, pull_request_id)
diff --git a/lib/bulk_imports/clients/graphql.rb b/lib/bulk_imports/clients/graphql.rb
new file mode 100644
index 00000000000..b067431aeae
--- /dev/null
+++ b/lib/bulk_imports/clients/graphql.rb
@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+module BulkImports
+ module Clients
+ class Graphql
+ class HTTP < Graphlient::Adapters::HTTP::Adapter
+ def execute(document:, operation_name: nil, variables: {}, context: {})
+ response = ::Gitlab::HTTP.post(
+ url,
+ headers: headers,
+ follow_redirects: false,
+ body: {
+ query: document.to_query_string,
+ operationName: operation_name,
+ variables: variables
+ }.to_json
+ )
+
+ ::Gitlab::Json.parse(response.body)
+ end
+ end
+ private_constant :HTTP
+
+ attr_reader :client
+
+ delegate :query, :parse, :execute, to: :client
+
+ def initialize(url: Gitlab::COM_URL, token: nil)
+ @url = Gitlab::Utils.append_path(url, '/api/graphql')
+ @token = token
+ @client = Graphlient::Client.new(
+ @url,
+ options(http: HTTP)
+ )
+ end
+
+ def options(extra = {})
+ return extra unless @token
+
+ {
+ headers: {
+ 'Content-Type' => 'application/json',
+ 'Authorization' => "Bearer #{@token}"
+ }
+ }.merge(extra)
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/bulk_import/client.rb b/lib/bulk_imports/clients/http.rb
index c6e77a158cd..2e81863e53a 100644
--- a/lib/gitlab/bulk_import/client.rb
+++ b/lib/bulk_imports/clients/http.rb
@@ -1,8 +1,8 @@
# frozen_string_literal: true
-module Gitlab
- module BulkImport
- class Client
+module BulkImports
+ module Clients
+ class Http
API_VERSION = 'v4'.freeze
DEFAULT_PAGE = 1.freeze
DEFAULT_PER_PAGE = 30.freeze
@@ -18,7 +18,7 @@ module Gitlab
end
def get(resource, query = {})
- response = with_error_handling do
+ with_error_handling do
Gitlab::HTTP.get(
resource_url(resource),
headers: request_headers,
@@ -26,8 +26,22 @@ module Gitlab
query: query.merge(request_query)
)
end
+ end
+
+ def each_page(method, resource, query = {}, &block)
+ return to_enum(__method__, method, resource, query) unless block_given?
+
+ next_page = @page
- response.parsed_response
+ while next_page
+ @page = next_page.to_i
+
+ response = self.public_send(method, resource, query) # rubocop: disable GitlabSecurity/PublicSend
+ collection = response.parsed_response
+ next_page = response.headers['x-next-page'].presence
+
+ yield collection
+ end
end
private
diff --git a/lib/bulk_imports/common/extractors/graphql_extractor.rb b/lib/bulk_imports/common/extractors/graphql_extractor.rb
new file mode 100644
index 00000000000..7d58032cfcc
--- /dev/null
+++ b/lib/bulk_imports/common/extractors/graphql_extractor.rb
@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module BulkImports
+ module Common
+ module Extractors
+ class GraphqlExtractor
+ def initialize(query)
+ @query = query[:query]
+ @query_string = @query.to_s
+ @variables = @query.variables
+ end
+
+ def extract(context)
+ @context = context
+
+ Enumerator.new do |yielder|
+ result = graphql_client.execute(parsed_query, query_variables(context.entity))
+
+ yielder << result.original_hash.deep_dup
+ end
+ end
+
+ private
+
+ def graphql_client
+ @graphql_client ||= BulkImports::Clients::Graphql.new(
+ url: @context.configuration.url,
+ token: @context.configuration.access_token
+ )
+ end
+
+ def parsed_query
+ @parsed_query ||= graphql_client.parse(@query.to_s)
+ end
+
+ def query_variables(entity)
+ return unless @variables
+
+ @variables.transform_values do |entity_attribute|
+ entity.public_send(entity_attribute) # rubocop:disable GitlabSecurity/PublicSend
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/bulk_imports/common/loaders/entity_loader.rb b/lib/bulk_imports/common/loaders/entity_loader.rb
new file mode 100644
index 00000000000..4540b892c88
--- /dev/null
+++ b/lib/bulk_imports/common/loaders/entity_loader.rb
@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module BulkImports
+ module Common
+ module Loaders
+ class EntityLoader
+ def initialize(*args); end
+
+ def load(context, entity)
+ context.entity.bulk_import.entities.create!(entity)
+ end
+ end
+ end
+ end
+end
diff --git a/lib/bulk_imports/common/transformers/graphql_cleaner_transformer.rb b/lib/bulk_imports/common/transformers/graphql_cleaner_transformer.rb
new file mode 100644
index 00000000000..dce0fac6999
--- /dev/null
+++ b/lib/bulk_imports/common/transformers/graphql_cleaner_transformer.rb
@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+# Cleanup GraphQL original response hash from unnecessary nesting
+# 1. Remove ['data']['group'] or ['data']['project'] hash nesting
+# 2. Remove ['edges'] & ['nodes'] array wrappings
+# 3. Remove ['node'] hash wrapping
+#
+# @example
+# data = {"data"=>{"group"=> {
+# "name"=>"test",
+# "fullName"=>"test",
+# "description"=>"test",
+# "labels"=>{"edges"=>[{"node"=>{"title"=>"label1"}}, {"node"=>{"title"=>"label2"}}, {"node"=>{"title"=>"label3"}}]}}}}
+#
+# BulkImports::Common::Transformers::GraphqlCleanerTransformer.new.transform(nil, data)
+#
+# {"name"=>"test", "fullName"=>"test", "description"=>"test", "labels"=>[{"title"=>"label1"}, {"title"=>"label2"}, {"title"=>"label3"}]}
+module BulkImports
+ module Common
+ module Transformers
+ class GraphqlCleanerTransformer
+ EDGES = 'edges'
+ NODE = 'node'
+
+ def initialize(options = {})
+ @options = options
+ end
+
+ def transform(_, data)
+ return data unless data.is_a?(Hash)
+
+ data = data.dig('data', 'group') || data.dig('data', 'project') || data
+
+ clean_edges_and_nodes(data)
+ end
+
+ def clean_edges_and_nodes(data)
+ case data
+ when Array
+ data.map(&method(:clean_edges_and_nodes))
+ when Hash
+ if data.key?(NODE)
+ clean_edges_and_nodes(data[NODE])
+ else
+ data.transform_values { |value| clean_edges_and_nodes(value.try(:fetch, EDGES, value) || value) }
+ end
+ else
+ data
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/bulk_imports/common/transformers/underscorify_keys_transformer.rb b/lib/bulk_imports/common/transformers/underscorify_keys_transformer.rb
new file mode 100644
index 00000000000..b32ab28fdbb
--- /dev/null
+++ b/lib/bulk_imports/common/transformers/underscorify_keys_transformer.rb
@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module BulkImports
+ module Common
+ module Transformers
+ class UnderscorifyKeysTransformer
+ def initialize(options = {})
+ @options = options
+ end
+
+ def transform(_, data)
+ data.deep_transform_keys do |key|
+ key.to_s.underscore
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/bulk_imports/groups/extractors/subgroups_extractor.rb b/lib/bulk_imports/groups/extractors/subgroups_extractor.rb
new file mode 100644
index 00000000000..5c5e686cec5
--- /dev/null
+++ b/lib/bulk_imports/groups/extractors/subgroups_extractor.rb
@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module BulkImports
+ module Groups
+ module Extractors
+ class SubgroupsExtractor
+ def initialize(*args); end
+
+ def extract(context)
+ encoded_parent_path = ERB::Util.url_encode(context.entity.source_full_path)
+
+ http_client(context.entity.bulk_import.configuration)
+ .each_page(:get, "groups/#{encoded_parent_path}/subgroups")
+ .flat_map(&:itself)
+ end
+
+ private
+
+ def http_client(configuration)
+ @http_client ||= BulkImports::Clients::Http.new(
+ uri: configuration.url,
+ token: configuration.access_token,
+ per_page: 100
+ )
+ end
+ end
+ end
+ end
+end
diff --git a/lib/bulk_imports/groups/graphql/get_group_query.rb b/lib/bulk_imports/groups/graphql/get_group_query.rb
new file mode 100644
index 00000000000..c50b99aae4e
--- /dev/null
+++ b/lib/bulk_imports/groups/graphql/get_group_query.rb
@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+module BulkImports
+ module Groups
+ module Graphql
+ module GetGroupQuery
+ extend self
+
+ def to_s
+ <<-'GRAPHQL'
+ query($full_path: ID!) {
+ group(fullPath: $full_path) {
+ name
+ path
+ fullPath
+ description
+ visibility
+ emailsDisabled
+ lfsEnabled
+ mentionsDisabled
+ projectCreationLevel
+ requestAccessEnabled
+ requireTwoFactorAuthentication
+ shareWithGroupLock
+ subgroupCreationLevel
+ twoFactorGracePeriod
+ }
+ }
+ GRAPHQL
+ end
+
+ def variables
+ { full_path: :source_full_path }
+ end
+ end
+ end
+ end
+end
diff --git a/lib/bulk_imports/groups/loaders/group_loader.rb b/lib/bulk_imports/groups/loaders/group_loader.rb
new file mode 100644
index 00000000000..386fc695182
--- /dev/null
+++ b/lib/bulk_imports/groups/loaders/group_loader.rb
@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module BulkImports
+ module Groups
+ module Loaders
+ class GroupLoader
+ def initialize(options = {})
+ @options = options
+ end
+
+ def load(context, data)
+ return unless user_can_create_group?(context.current_user, data)
+
+ group = ::Groups::CreateService.new(context.current_user, data).execute
+
+ context.entity.update!(group: group)
+
+ group
+ end
+
+ private
+
+ def user_can_create_group?(current_user, data)
+ if data['parent_id']
+ parent = Namespace.find_by_id(data['parent_id'])
+
+ Ability.allowed?(current_user, :create_subgroup, parent)
+ else
+ Ability.allowed?(current_user, :create_group)
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/bulk_imports/groups/pipelines/group_pipeline.rb b/lib/bulk_imports/groups/pipelines/group_pipeline.rb
new file mode 100644
index 00000000000..2b7d0ef7658
--- /dev/null
+++ b/lib/bulk_imports/groups/pipelines/group_pipeline.rb
@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module BulkImports
+ module Groups
+ module Pipelines
+ class GroupPipeline
+ include Pipeline
+
+ extractor Common::Extractors::GraphqlExtractor, query: Graphql::GetGroupQuery
+
+ transformer Common::Transformers::GraphqlCleanerTransformer
+ transformer Common::Transformers::UnderscorifyKeysTransformer
+ transformer Groups::Transformers::GroupAttributesTransformer
+
+ loader Groups::Loaders::GroupLoader
+ end
+ end
+ end
+end
diff --git a/lib/bulk_imports/groups/pipelines/subgroup_entities_pipeline.rb b/lib/bulk_imports/groups/pipelines/subgroup_entities_pipeline.rb
new file mode 100644
index 00000000000..6384e9d5972
--- /dev/null
+++ b/lib/bulk_imports/groups/pipelines/subgroup_entities_pipeline.rb
@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module BulkImports
+ module Groups
+ module Pipelines
+ class SubgroupEntitiesPipeline
+ include Pipeline
+
+ extractor BulkImports::Groups::Extractors::SubgroupsExtractor
+ transformer BulkImports::Groups::Transformers::SubgroupToEntityTransformer
+ loader BulkImports::Common::Loaders::EntityLoader
+ end
+ end
+ end
+end
diff --git a/lib/bulk_imports/groups/transformers/group_attributes_transformer.rb b/lib/bulk_imports/groups/transformers/group_attributes_transformer.rb
new file mode 100644
index 00000000000..7de9a430421
--- /dev/null
+++ b/lib/bulk_imports/groups/transformers/group_attributes_transformer.rb
@@ -0,0 +1,81 @@
+# frozen_string_literal: true
+
+module BulkImports
+ module Groups
+ module Transformers
+ class GroupAttributesTransformer
+ def initialize(options = {})
+ @options = options
+ end
+
+ def transform(context, data)
+ import_entity = context.entity
+
+ data
+ .then { |data| transform_name(import_entity, data) }
+ .then { |data| transform_path(import_entity, data) }
+ .then { |data| transform_full_path(data) }
+ .then { |data| transform_parent(context, import_entity, data) }
+ .then { |data| transform_visibility_level(data) }
+ .then { |data| transform_project_creation_level(data) }
+ .then { |data| transform_subgroup_creation_level(data) }
+ end
+
+ private
+
+ def transform_name(import_entity, data)
+ data['name'] = import_entity.destination_name
+ data
+ end
+
+ def transform_path(import_entity, data)
+ data['path'] = import_entity.destination_name.parameterize
+ data
+ end
+
+ def transform_full_path(data)
+ data.delete('full_path')
+ data
+ end
+
+ def transform_parent(context, import_entity, data)
+ current_user = context.current_user
+ namespace = Namespace.find_by_full_path(import_entity.destination_namespace)
+
+ return data if namespace == current_user.namespace
+
+ data['parent_id'] = namespace.id
+ data
+ end
+
+ def transform_visibility_level(data)
+ visibility = data['visibility']
+
+ return data unless visibility.present?
+
+ data['visibility_level'] = Gitlab::VisibilityLevel.string_options[visibility]
+ data.delete('visibility')
+ data
+ end
+
+ def transform_project_creation_level(data)
+ project_creation_level = data['project_creation_level']
+
+ return data unless project_creation_level.present?
+
+ data['project_creation_level'] = Gitlab::Access.project_creation_string_options[project_creation_level]
+ data
+ end
+
+ def transform_subgroup_creation_level(data)
+ subgroup_creation_level = data['subgroup_creation_level']
+
+ return data unless subgroup_creation_level.present?
+
+ data['subgroup_creation_level'] = Gitlab::Access.subgroup_creation_string_options[subgroup_creation_level]
+ data
+ end
+ end
+ end
+ end
+end
diff --git a/lib/bulk_imports/groups/transformers/subgroup_to_entity_transformer.rb b/lib/bulk_imports/groups/transformers/subgroup_to_entity_transformer.rb
new file mode 100644
index 00000000000..6c3c299c2d2
--- /dev/null
+++ b/lib/bulk_imports/groups/transformers/subgroup_to_entity_transformer.rb
@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module BulkImports
+ module Groups
+ module Transformers
+ class SubgroupToEntityTransformer
+ def initialize(*args); end
+
+ def transform(context, entry)
+ {
+ source_type: :group_entity,
+ source_full_path: entry['full_path'],
+ destination_name: entry['name'],
+ destination_namespace: context.entity.group.full_path,
+ parent_id: context.entity.id
+ }
+ end
+ end
+ end
+ end
+end
diff --git a/lib/bulk_imports/importers/group_importer.rb b/lib/bulk_imports/importers/group_importer.rb
new file mode 100644
index 00000000000..c7253590c87
--- /dev/null
+++ b/lib/bulk_imports/importers/group_importer.rb
@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module BulkImports
+ module Importers
+ class GroupImporter
+ def initialize(entity)
+ @entity = entity
+ end
+
+ def execute
+ entity.start!
+ bulk_import = entity.bulk_import
+ configuration = bulk_import.configuration
+
+ context = BulkImports::Pipeline::Context.new(
+ current_user: bulk_import.user,
+ entity: entity,
+ configuration: configuration
+ )
+
+ BulkImports::Groups::Pipelines::GroupPipeline.new.run(context)
+ BulkImports::Groups::Pipelines::SubgroupEntitiesPipeline.new.run(context)
+
+ entity.finish!
+ end
+
+ private
+
+ attr_reader :entity
+ end
+ end
+end
diff --git a/lib/bulk_imports/importers/groups_importer.rb b/lib/bulk_imports/importers/groups_importer.rb
new file mode 100644
index 00000000000..8641577ff47
--- /dev/null
+++ b/lib/bulk_imports/importers/groups_importer.rb
@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module BulkImports
+ module Importers
+ class GroupsImporter
+ def initialize(bulk_import_id)
+ @bulk_import = BulkImport.find(bulk_import_id)
+ end
+
+ def execute
+ bulk_import.start! unless bulk_import.started?
+
+ if entities_to_import.empty?
+ bulk_import.finish!
+ else
+ entities_to_import.each do |entity|
+ BulkImports::Importers::GroupImporter.new(entity).execute
+ end
+
+ # A new BulkImportWorker job is enqueued to either
+ # - Process the new BulkImports::Entity created for the subgroups
+ # - Or to mark the `bulk_import` as finished.
+ BulkImportWorker.perform_async(bulk_import.id)
+ end
+ end
+
+ private
+
+ attr_reader :bulk_import
+
+ def entities_to_import
+ @entities_to_import ||= bulk_import.entities.with_status(:created)
+ end
+ end
+ end
+end
diff --git a/lib/bulk_imports/pipeline.rb b/lib/bulk_imports/pipeline.rb
new file mode 100644
index 00000000000..70e6030ea2c
--- /dev/null
+++ b/lib/bulk_imports/pipeline.rb
@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+module BulkImports
+ module Pipeline
+ extend ActiveSupport::Concern
+
+ included do
+ include Attributes
+ include Runner
+ end
+ end
+end
diff --git a/lib/bulk_imports/pipeline/attributes.rb b/lib/bulk_imports/pipeline/attributes.rb
new file mode 100644
index 00000000000..ebfbaf6f6ba
--- /dev/null
+++ b/lib/bulk_imports/pipeline/attributes.rb
@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module BulkImports
+ module Pipeline
+ module Attributes
+ extend ActiveSupport::Concern
+ include Gitlab::ClassAttributes
+
+ class_methods do
+ def extractor(klass, options = nil)
+ add_attribute(:extractors, klass, options)
+ end
+
+ def transformer(klass, options = nil)
+ add_attribute(:transformers, klass, options)
+ end
+
+ def loader(klass, options = nil)
+ add_attribute(:loaders, klass, options)
+ end
+
+ def add_attribute(sym, klass, options)
+ class_attributes[sym] ||= []
+ class_attributes[sym] << { klass: klass, options: options }
+ end
+
+ def extractors
+ class_attributes[:extractors]
+ end
+
+ def transformers
+ class_attributes[:transformers]
+ end
+
+ def loaders
+ class_attributes[:loaders]
+ end
+ end
+ end
+ end
+end
diff --git a/lib/bulk_imports/pipeline/context.rb b/lib/bulk_imports/pipeline/context.rb
new file mode 100644
index 00000000000..ad19f5cad7d
--- /dev/null
+++ b/lib/bulk_imports/pipeline/context.rb
@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module BulkImports
+ module Pipeline
+ class Context
+ include Gitlab::Utils::LazyAttributes
+
+ Attribute = Struct.new(:name, :type)
+
+ PIPELINE_ATTRIBUTES = [
+ Attribute.new(:current_user, User),
+ Attribute.new(:entity, ::BulkImports::Entity),
+ Attribute.new(:configuration, ::BulkImports::Configuration)
+ ].freeze
+
+ def initialize(args)
+ assign_attributes(args)
+ end
+
+ private
+
+ PIPELINE_ATTRIBUTES.each do |attr|
+ lazy_attr_reader attr.name, type: attr.type
+ end
+
+ def assign_attributes(values)
+ values.slice(*PIPELINE_ATTRIBUTES.map(&:name)).each do |name, value|
+ instance_variable_set("@#{name}", value)
+ end
+ end
+ end
+ end
+end
diff --git a/lib/bulk_imports/pipeline/runner.rb b/lib/bulk_imports/pipeline/runner.rb
new file mode 100644
index 00000000000..04038e50399
--- /dev/null
+++ b/lib/bulk_imports/pipeline/runner.rb
@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+module BulkImports
+ module Pipeline
+ module Runner
+ extend ActiveSupport::Concern
+
+ included do
+ private
+
+ def extractors
+ @extractors ||= self.class.extractors.map(&method(:instantiate))
+ end
+
+ def transformers
+ @transformers ||= self.class.transformers.map(&method(:instantiate))
+ end
+
+ def loaders
+ @loaders ||= self.class.loaders.map(&method(:instantiate))
+ end
+
+ def pipeline_name
+ @pipeline ||= self.class.name
+ end
+
+ def instantiate(class_config)
+ class_config[:klass].new(class_config[:options])
+ end
+ end
+
+ def run(context)
+ info(context, message: "Pipeline started", pipeline: pipeline_name)
+
+ extractors.each do |extractor|
+ extractor.extract(context).each do |entry|
+ info(context, extractor: extractor.class.name)
+
+ transformers.each do |transformer|
+ info(context, transformer: transformer.class.name)
+ entry = transformer.transform(context, entry)
+ end
+
+ loaders.each do |loader|
+ info(context, loader: loader.class.name)
+ loader.load(context, entry)
+ end
+ end
+ end
+ end
+
+ private # rubocop:disable Lint/UselessAccessModifier
+
+ def info(context, extra = {})
+ logger.info({
+ entity: context.entity.id,
+ entity_type: context.entity.source_type
+ }.merge(extra))
+ end
+
+ def logger
+ @logger ||= Gitlab::Import::Logger.build
+ end
+ end
+ end
+end
diff --git a/lib/container_registry/client.rb b/lib/container_registry/client.rb
index e6ca33d749b..35f299c17e4 100644
--- a/lib/container_registry/client.rb
+++ b/lib/container_registry/client.rb
@@ -15,6 +15,7 @@ module ContainerRegistry
CONTAINER_IMAGE_V1_TYPE = 'application/vnd.docker.container.image.v1+json'
REGISTRY_VERSION_HEADER = 'gitlab-container-registry-version'
REGISTRY_FEATURES_HEADER = 'gitlab-container-registry-features'
+ REGISTRY_TAG_DELETE_FEATURE = 'tag_delete'
ACCEPTED_TYPES = [DOCKER_DISTRIBUTION_MANIFEST_V2_TYPE, OCI_MANIFEST_V1_TYPE].freeze
@@ -25,8 +26,6 @@ module ContainerRegistry
registry_config = Gitlab.config.registry
return false unless registry_config.enabled && registry_config.api_url.present?
- return true if ::Gitlab.com?
-
token = Auth::ContainerRegistryAuthenticationService.access_token([], [])
client = new(registry_config.api_url, token: token)
client.supports_tag_delete?
@@ -81,6 +80,9 @@ module ContainerRegistry
# the DELETE method in the Allow header. Others reply with an 404 Not Found.
def supports_tag_delete?
strong_memoize(:supports_tag_delete) do
+ registry_features = Gitlab::CurrentSettings.container_registry_features || []
+ next true if ::Gitlab.com? && registry_features.include?(REGISTRY_TAG_DELETE_FEATURE)
+
response = faraday.run_request(:options, '/v2/name/tags/reference/tag', '', {})
response.success? && response.headers['allow']&.include?('DELETE')
end
diff --git a/lib/csv_builders/stream.rb b/lib/csv_builders/stream.rb
new file mode 100644
index 00000000000..a2b9fca84cb
--- /dev/null
+++ b/lib/csv_builders/stream.rb
@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module CsvBuilders
+ class Stream < CsvBuilder
+ def render(max_rows = 100_000)
+ max_rows_including_header = max_rows + 1
+
+ Enumerator.new do |csv|
+ csv << CSV.generate_line(headers)
+
+ each do |object|
+ csv << CSV.generate_line(row(object))
+ end
+ end.lazy.take(max_rows_including_header) # rubocop: disable CodeReuse/ActiveRecord
+ end
+ end
+end
diff --git a/lib/expand_variables.rb b/lib/expand_variables.rb
index 3a50925d628..dc8f9d0c970 100644
--- a/lib/expand_variables.rb
+++ b/lib/expand_variables.rb
@@ -1,17 +1,39 @@
# frozen_string_literal: true
module ExpandVariables
+ VARIABLES_REGEXP = /\$([a-zA-Z_][a-zA-Z0-9_]*)|\${\g<1>}|%\g<1>%/.freeze
+
class << self
def expand(value, variables)
+ replace_with(value, variables) do |vars_hash, last_match|
+ match_or_blank_value(vars_hash, last_match)
+ end
+ end
+
+ def expand_existing(value, variables)
+ replace_with(value, variables) do |vars_hash, last_match|
+ match_or_original_value(vars_hash, last_match)
+ end
+ end
+
+ private
+
+ def replace_with(value, variables)
variables_hash = nil
- value.gsub(/\$([a-zA-Z_][a-zA-Z0-9_]*)|\${\g<1>}|%\g<1>%/) do
+ value.gsub(VARIABLES_REGEXP) do
variables_hash ||= transform_variables(variables)
- variables_hash[Regexp.last_match(1) || Regexp.last_match(2)]
+ yield(variables_hash, Regexp.last_match)
end
end
- private
+ def match_or_blank_value(variables, last_match)
+ variables[last_match[1] || last_match[2]]
+ end
+
+ def match_or_original_value(variables, last_match)
+ match_or_blank_value(variables, last_match) || last_match[0]
+ end
def transform_variables(variables)
# Lazily initialise variables
diff --git a/lib/extracts_ref.rb b/lib/extracts_ref.rb
index adbe93cfa3a..34511423d4a 100644
--- a/lib/extracts_ref.rb
+++ b/lib/extracts_ref.rb
@@ -68,7 +68,7 @@ module ExtractsRef
raise InvalidPathError if @ref.match?(/\s/)
- @commit = @repo.commit(@ref)
+ @commit = @repo.commit(@ref) if @ref.present?
end
# rubocop:enable Gitlab/ModuleWithInstanceVariables
diff --git a/lib/feature/shared.rb b/lib/feature/shared.rb
index 9ec56ee6b52..1fcbc8fa173 100644
--- a/lib/feature/shared.rb
+++ b/lib/feature/shared.rb
@@ -10,6 +10,8 @@ class Feature
# rollout_issue: defines if `bin/feature-flag` asks for rollout issue
# default_enabled: defines a default state of a feature flag when created by `bin/feature-flag`
# ee_only: defines that a feature flag can only be created in a context of EE
+ # deprecated: defines if a feature flag type that is deprecated and to be removed,
+ # the deprecated types are hidden from all interfaces
# example: usage being shown when exception is raised
TYPES = {
development: {
@@ -37,6 +39,7 @@ class Feature
},
licensed: {
description: 'Permanent feature flags used to temporarily disable licensed features.',
+ deprecated: true,
optional: true,
rollout_issue: false,
ee_only: true,
@@ -54,6 +57,7 @@ class Feature
name
introduced_by_url
rollout_issue_url
+ milestone
type
group
default_enabled
diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/code_stage_start.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/code_stage_start.rb
index c5f843d5f1a..4bb225b63f1 100644
--- a/lib/gitlab/analytics/cycle_analytics/stage_events/code_stage_start.rb
+++ b/lib/gitlab/analytics/cycle_analytics/stage_events/code_stage_start.rb
@@ -18,24 +18,46 @@ module Gitlab
end
def timestamp_projection
- issue_metrics_table[:first_mentioned_in_commit_at]
+ Arel::Nodes::NamedFunction.new('COALESCE', column_list)
end
override :column_list
def column_list
- [timestamp_projection]
+ [
+ issue_metrics_table[:first_mentioned_in_commit_at],
+ mr_metrics_table[:first_commit_at]
+ ]
end
# rubocop: disable CodeReuse/ActiveRecord
def apply_query_customization(query)
- issue_metrics_join = mr_closing_issues_table
- .join(issue_metrics_table)
+ query
+ .joins(merge_requests_closing_issues_join)
+ .joins(issue_metrics_join)
+ .joins(mr_metrics_join)
+ end
+ # rubocop: enable CodeReuse/ActiveRecord
+
+ def issue_metrics_join
+ mr_closing_issues_table
+ .join(issue_metrics_table, Arel::Nodes::OuterJoin)
.on(mr_closing_issues_table[:issue_id].eq(issue_metrics_table[:issue_id]))
.join_sources
+ end
- query.joins(:merge_requests_closing_issues).joins(issue_metrics_join)
+ def merge_requests_closing_issues_join
+ mr_table
+ .join(mr_closing_issues_table, Arel::Nodes::OuterJoin)
+ .on(mr_table[:id].eq(mr_closing_issues_table[:merge_request_id]))
+ .join_sources
+ end
+
+ def mr_metrics_join
+ mr_metrics_table
+ .join(mr_metrics_table, Arel::Nodes::OuterJoin)
+ .on(mr_metrics_table[:merge_request_id].eq(mr_table[:id]))
+ .join_sources
end
- # rubocop: enable CodeReuse/ActiveRecord
end
end
end
diff --git a/lib/gitlab/analytics/instance_statistics/workers_argument_builder.rb b/lib/gitlab/analytics/instance_statistics/workers_argument_builder.rb
index 636bba22c23..54b3bbb3ce6 100644
--- a/lib/gitlab/analytics/instance_statistics/workers_argument_builder.rb
+++ b/lib/gitlab/analytics/instance_statistics/workers_argument_builder.rb
@@ -11,21 +11,36 @@ module Gitlab
def execute
measurement_identifiers.map do |measurement_identifier|
- query_scope = ::Analytics::InstanceStatistics::Measurement::IDENTIFIER_QUERY_MAPPING[measurement_identifier]&.call
+ query_scope = query_mappings[measurement_identifier]&.call
next if query_scope.nil?
- # Determining the query range (id range) as early as possible in order to get more accurate counts.
- start = query_scope.minimum(:id)
- finish = query_scope.maximum(:id)
-
- [measurement_identifier, start, finish, recorded_at]
+ [measurement_identifier, *determine_start_and_finish(measurement_identifier, query_scope), recorded_at]
end.compact
end
private
attr_reader :measurement_identifiers, :recorded_at
+
+ # Determining the query range (id range) as early as possible in order to get more accurate counts.
+ def determine_start_and_finish(measurement_identifier, query_scope)
+ queries = custom_min_max_queries[measurement_identifier]
+
+ if queries
+ [queries[:minimum_query].call, queries[:maximum_query].call]
+ else
+ [query_scope.minimum(:id), query_scope.maximum(:id)]
+ end
+ end
+
+ def custom_min_max_queries
+ ::Analytics::InstanceStatistics::Measurement.identifier_min_max_queries
+ end
+
+ def query_mappings
+ ::Analytics::InstanceStatistics::Measurement.identifier_query_mapping
+ end
end
end
end
diff --git a/lib/gitlab/application_rate_limiter.rb b/lib/gitlab/application_rate_limiter.rb
index 6173918b453..e92bbe4f529 100644
--- a/lib/gitlab/application_rate_limiter.rb
+++ b/lib/gitlab/application_rate_limiter.rb
@@ -33,7 +33,8 @@ module Gitlab
group_import: { threshold: -> { application_settings.group_import_limit }, interval: 1.minute },
group_testing_hook: { threshold: 5, interval: 1.minute },
profile_add_new_email: { threshold: 5, interval: 1.minute },
- profile_resend_email_confirmation: { threshold: 5, interval: 1.minute }
+ profile_resend_email_confirmation: { threshold: 5, interval: 1.minute },
+ update_environment_canary_ingress: { threshold: 1, interval: 1.minute }
}.freeze
end
diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb
index 001c083c778..fadd6eb848d 100644
--- a/lib/gitlab/auth.rb
+++ b/lib/gitlab/auth.rb
@@ -158,7 +158,7 @@ module Gitlab
if Service.available_services_names.include?(underscored_service)
# We treat underscored_service as a trusted input because it is included
- # in the Service.available_services_names whitelist.
+ # in the Service.available_services_names allowlist.
service = project.public_send("#{underscored_service}_service") # rubocop:disable GitlabSecurity/PublicSend
if service && service.activated? && service.valid_token?(password)
diff --git a/lib/gitlab/auth/auth_finders.rb b/lib/gitlab/auth/auth_finders.rb
index 3d3f7212053..f3975fe219a 100644
--- a/lib/gitlab/auth/auth_finders.rb
+++ b/lib/gitlab/auth/auth_finders.rb
@@ -83,6 +83,8 @@ module Gitlab
return unless ::Gitlab::Auth::CI_JOB_USER == login
job = find_valid_running_job_by_token!(password)
+ @current_authenticated_job = job # rubocop:disable Gitlab/ModuleWithInstanceVariables
+
job.user
end
diff --git a/lib/gitlab/background_migration/backfill_design_internal_ids.rb b/lib/gitlab/background_migration/backfill_design_internal_ids.rb
new file mode 100644
index 00000000000..553571d5d00
--- /dev/null
+++ b/lib/gitlab/background_migration/backfill_design_internal_ids.rb
@@ -0,0 +1,130 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module BackgroundMigration
+ # Backfill design.iid for a range of projects
+ class BackfillDesignInternalIds
+ # See app/models/internal_id
+ # This is a direct copy of the application code with the following changes:
+ # - usage enum is hard-coded to the value for design_management_designs
+ # - init is not passed around, but ignored
+ class InternalId < ActiveRecord::Base
+ def self.track_greatest(subject, scope, new_value)
+ InternalIdGenerator.new(subject, scope).track_greatest(new_value)
+ end
+
+ # Increments #last_value with new_value if it is greater than the current,
+ # and saves the record
+ #
+ # The operation locks the record and gathers a `ROW SHARE` lock (in PostgreSQL).
+ # As such, the increment is atomic and safe to be called concurrently.
+ def track_greatest_and_save!(new_value)
+ update_and_save { self.last_value = [last_value || 0, new_value].max }
+ end
+
+ private
+
+ def update_and_save(&block)
+ lock!
+ yield
+ # update_and_save_counter.increment(usage: usage, changed: last_value_changed?)
+ save!
+ last_value
+ end
+ end
+
+ # See app/models/internal_id
+ class InternalIdGenerator
+ attr_reader :subject, :scope, :scope_attrs
+
+ def initialize(subject, scope)
+ @subject = subject
+ @scope = scope
+
+ raise ArgumentError, 'Scope is not well-defined, need at least one column for scope (given: 0)' if scope.empty?
+ end
+
+ # Create a record in internal_ids if one does not yet exist
+ # and set its new_value if it is higher than the current last_value
+ #
+ # Note this will acquire a ROW SHARE lock on the InternalId record
+ def track_greatest(new_value)
+ subject.transaction do
+ record.track_greatest_and_save!(new_value)
+ end
+ end
+
+ def record
+ @record ||= (lookup || create_record)
+ end
+
+ def lookup
+ InternalId.find_by(**scope, usage: usage_value)
+ end
+
+ def usage_value
+ 10 # see Enums::InternalId - this is the value for design_management_designs
+ end
+
+ # Create InternalId record for (scope, usage) combination, if it doesn't exist
+ #
+ # We blindly insert without synchronization. If another process
+ # was faster in doing this, we'll realize once we hit the unique key constraint
+ # violation. We can safely roll-back the nested transaction and perform
+ # a lookup instead to retrieve the record.
+ def create_record
+ subject.transaction(requires_new: true) do
+ InternalId.create!(
+ **scope,
+ usage: usage_value,
+ last_value: 0
+ )
+ end
+ rescue ActiveRecord::RecordNotUnique
+ lookup
+ end
+ end
+
+ attr_reader :design_class
+
+ def initialize(design_class)
+ @design_class = design_class
+ end
+
+ def perform(relation)
+ start_id, end_id = relation.pluck("min(project_id), max(project_id)").flatten
+ table = 'design_management_designs'
+
+ ActiveRecord::Base.connection.execute <<~SQL
+ WITH
+ starting_iids(project_id, iid) as (
+ SELECT project_id, MAX(COALESCE(iid, 0))
+ FROM #{table}
+ WHERE project_id BETWEEN #{start_id} AND #{end_id}
+ GROUP BY project_id
+ ),
+ with_calculated_iid(id, iid) as (
+ SELECT design.id,
+ init.iid + ROW_NUMBER() OVER (PARTITION BY design.project_id ORDER BY design.id ASC)
+ FROM #{table} as design, starting_iids as init
+ WHERE design.project_id BETWEEN #{start_id} AND #{end_id}
+ AND design.iid IS NULL
+ AND init.project_id = design.project_id
+ )
+
+ UPDATE #{table}
+ SET iid = with_calculated_iid.iid
+ FROM with_calculated_iid
+ WHERE #{table}.id = with_calculated_iid.id
+ SQL
+
+ # track the new greatest IID value
+ relation.each do |design|
+ current_max = design_class.where(project_id: design.project_id).maximum(:iid)
+ scope = { project_id: design.project_id }
+ InternalId.track_greatest(design, scope, current_max)
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/background_migration/backfill_jira_tracker_deployment_type2.rb b/lib/gitlab/background_migration/backfill_jira_tracker_deployment_type2.rb
new file mode 100644
index 00000000000..61145f6a445
--- /dev/null
+++ b/lib/gitlab/background_migration/backfill_jira_tracker_deployment_type2.rb
@@ -0,0 +1,86 @@
+# frozen_string_literal: true
+
+# Based on https://community.developer.atlassian.com/t/get-rest-api-3-filter-search/29459/2,
+# it's enough at the moment to simply notice if the url is from `atlassian.net`
+module Gitlab
+ module BackgroundMigration
+ # Backfill the deployment_type in jira_tracker_data table
+ class BackfillJiraTrackerDeploymentType2
+ # Migration only version of jira_tracker_data table
+ class JiraTrackerDataTemp < ApplicationRecord
+ self.table_name = 'jira_tracker_data'
+
+ def self.encryption_options
+ {
+ key: Settings.attr_encrypted_db_key_base_32,
+ encode: true,
+ mode: :per_attribute_iv,
+ algorithm: 'aes-256-gcm'
+ }
+ end
+
+ attr_encrypted :url, encryption_options
+ attr_encrypted :api_url, encryption_options
+
+ enum deployment_type: { unknown: 0, server: 1, cloud: 2 }, _prefix: :deployment
+ end
+
+ # Migration only version of services table
+ class JiraServiceTemp < ApplicationRecord
+ self.table_name = 'services'
+ self.inheritance_column = :_type_disabled
+ end
+
+ def perform(start_id, stop_id)
+ @server_ids = []
+ @cloud_ids = []
+
+ JiraTrackerDataTemp
+ .where(id: start_id..stop_id, deployment_type: 0)
+ .each do |jira_tracker_data|
+ collect_deployment_type(jira_tracker_data)
+ end
+
+ unless cloud_ids.empty?
+ JiraTrackerDataTemp.where(id: cloud_ids)
+ .update_all(deployment_type: JiraTrackerDataTemp.deployment_types[:cloud])
+ end
+
+ unless server_ids.empty?
+ JiraTrackerDataTemp.where(id: server_ids)
+ .update_all(deployment_type: JiraTrackerDataTemp.deployment_types[:server])
+ end
+
+ mark_jobs_as_succeeded(start_id, stop_id)
+ end
+
+ private
+
+ attr_reader :server_ids, :cloud_ids
+
+ def client_url(jira_tracker_data)
+ jira_tracker_data.api_url.presence || jira_tracker_data.url.presence
+ end
+
+ def server_type(url)
+ url.downcase.include?('.atlassian.net') ? :cloud : :server
+ end
+
+ def collect_deployment_type(jira_tracker_data)
+ url = client_url(jira_tracker_data)
+ return unless url
+
+ case server_type(url)
+ when :cloud
+ cloud_ids << jira_tracker_data.id
+ else
+ server_ids << jira_tracker_data.id
+ end
+ end
+
+ def mark_jobs_as_succeeded(*arguments)
+ Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded(self.class.name, arguments)
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/background_migration/backfill_merge_request_cleanup_schedules.rb b/lib/gitlab/background_migration/backfill_merge_request_cleanup_schedules.rb
new file mode 100644
index 00000000000..8a58cf9b302
--- /dev/null
+++ b/lib/gitlab/background_migration/backfill_merge_request_cleanup_schedules.rb
@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module BackgroundMigration
+ # Backfill merge request cleanup schedules of closed/merged merge requests
+ # without any corresponding records.
+ class BackfillMergeRequestCleanupSchedules
+ # Model used for migration added in https://gitlab.com/gitlab-org/gitlab/-/merge_requests/46782.
+ class MergeRequest < ActiveRecord::Base
+ include EachBatch
+
+ self.table_name = 'merge_requests'
+
+ def self.eligible
+ where('merge_requests.state_id IN (2, 3)')
+ end
+ end
+
+ def perform(start_id, end_id)
+ eligible_mrs = MergeRequest.eligible.where(id: start_id..end_id)
+ scheduled_at_column = "COALESCE(metrics.merged_at, COALESCE(metrics.latest_closed_at, merge_requests.updated_at)) + interval '14 days'"
+ query =
+ eligible_mrs
+ .select("merge_requests.id, #{scheduled_at_column}, NOW(), NOW()")
+ .joins('LEFT JOIN merge_request_metrics metrics ON metrics.merge_request_id = merge_requests.id')
+
+ result = ActiveRecord::Base.connection.execute <<~SQL
+ INSERT INTO merge_request_cleanup_schedules (merge_request_id, scheduled_at, created_at, updated_at)
+ #{query.to_sql}
+ ON CONFLICT (merge_request_id) DO NOTHING;
+ SQL
+
+ ::Gitlab::BackgroundMigration::Logger.info(
+ message: 'Backfilled merge_request_cleanup_schedules records',
+ count: result.cmd_tuples
+ )
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/background_migration/populate_has_vulnerabilities.rb b/lib/gitlab/background_migration/populate_has_vulnerabilities.rb
new file mode 100644
index 00000000000..78140b768fc
--- /dev/null
+++ b/lib/gitlab/background_migration/populate_has_vulnerabilities.rb
@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module BackgroundMigration
+ # This class populates missing dismissal information for
+ # vulnerability entries.
+ class PopulateHasVulnerabilities
+ class ProjectSetting < ActiveRecord::Base # rubocop:disable Style/Documentation
+ self.table_name = 'project_settings'
+
+ UPSERT_SQL = <<~SQL
+ WITH upsert_data (project_id, has_vulnerabilities, created_at, updated_at) AS (
+ SELECT projects.id, true, current_timestamp, current_timestamp FROM projects WHERE projects.id IN (%{project_ids})
+ )
+ INSERT INTO project_settings
+ (project_id, has_vulnerabilities, created_at, updated_at)
+ (SELECT * FROM upsert_data)
+ ON CONFLICT (project_id)
+ DO UPDATE SET
+ has_vulnerabilities = true,
+ updated_at = EXCLUDED.updated_at
+ SQL
+
+ def self.upsert_for(project_ids)
+ connection.execute(UPSERT_SQL % { project_ids: project_ids.join(', ') })
+ end
+ end
+
+ class Vulnerability < ActiveRecord::Base # rubocop:disable Style/Documentation
+ include EachBatch
+
+ self.table_name = 'vulnerabilities'
+ end
+
+ def perform(*project_ids)
+ ProjectSetting.upsert_for(project_ids)
+ rescue StandardError => e
+ log_error(e, project_ids)
+ ensure
+ log_info(project_ids)
+ end
+
+ private
+
+ def log_error(error, project_ids)
+ ::Gitlab::BackgroundMigration::Logger.error(
+ migrator: self.class.name,
+ message: error.message,
+ project_ids: project_ids
+ )
+ end
+
+ def log_info(project_ids)
+ ::Gitlab::BackgroundMigration::Logger.info(
+ migrator: self.class.name,
+ message: 'Projects has been processed to populate `has_vulnerabilities` information',
+ count: project_ids.length
+ )
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/background_migration/populate_missing_vulnerability_dismissal_information.rb b/lib/gitlab/background_migration/populate_missing_vulnerability_dismissal_information.rb
new file mode 100644
index 00000000000..bc0a181a06c
--- /dev/null
+++ b/lib/gitlab/background_migration/populate_missing_vulnerability_dismissal_information.rb
@@ -0,0 +1,86 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module BackgroundMigration
+ # This class populates missing dismissal information for
+ # vulnerability entries.
+ class PopulateMissingVulnerabilityDismissalInformation
+ class Vulnerability < ActiveRecord::Base # rubocop:disable Style/Documentation
+ include EachBatch
+
+ self.table_name = 'vulnerabilities'
+
+ has_one :finding, class_name: '::Gitlab::BackgroundMigration::PopulateMissingVulnerabilityDismissalInformation::Finding'
+
+ scope :broken, -> { where('state = 2 AND (dismissed_at IS NULL OR dismissed_by_id IS NULL)') }
+
+ def copy_dismissal_information
+ return unless finding&.dismissal_feedback
+
+ update_columns(
+ dismissed_at: finding.dismissal_feedback.created_at,
+ dismissed_by_id: finding.dismissal_feedback.author_id
+ )
+ end
+ end
+
+ class Finding < ActiveRecord::Base # rubocop:disable Style/Documentation
+ include ShaAttribute
+
+ self.table_name = 'vulnerability_occurrences'
+
+ sha_attribute :project_fingerprint
+
+ def dismissal_feedback
+ Feedback.dismissal.where(category: report_type, project_fingerprint: project_fingerprint, project_id: project_id).first
+ end
+ end
+
+ class Feedback < ActiveRecord::Base # rubocop:disable Style/Documentation
+ DISMISSAL_TYPE = 0
+
+ self.table_name = 'vulnerability_feedback'
+
+ scope :dismissal, -> { where(feedback_type: DISMISSAL_TYPE) }
+ end
+
+ def perform(*vulnerability_ids)
+ Vulnerability.includes(:finding).where(id: vulnerability_ids).each { |vulnerability| populate_for(vulnerability) }
+
+ log_info(vulnerability_ids)
+ end
+
+ private
+
+ def populate_for(vulnerability)
+ log_warning(vulnerability) unless vulnerability.copy_dismissal_information
+ rescue StandardError => error
+ log_error(error, vulnerability)
+ end
+
+ def log_info(vulnerability_ids)
+ ::Gitlab::BackgroundMigration::Logger.info(
+ migrator: self.class.name,
+ message: 'Dismissal information has been copied',
+ count: vulnerability_ids.length
+ )
+ end
+
+ def log_warning(vulnerability)
+ ::Gitlab::BackgroundMigration::Logger.warn(
+ migrator: self.class.name,
+ message: 'Could not update vulnerability!',
+ vulnerability_id: vulnerability.id
+ )
+ end
+
+ def log_error(error, vulnerability)
+ ::Gitlab::BackgroundMigration::Logger.error(
+ migrator: self.class.name,
+ message: error.message,
+ vulnerability_id: vulnerability.id
+ )
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/background_migration/populate_vulnerability_feedback_pipeline_id.rb b/lib/gitlab/background_migration/populate_vulnerability_feedback_pipeline_id.rb
new file mode 100644
index 00000000000..fc79f7125e3
--- /dev/null
+++ b/lib/gitlab/background_migration/populate_vulnerability_feedback_pipeline_id.rb
@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module BackgroundMigration
+ # This class updates vulnerability feedback entities with no pipeline id assigned.
+ class PopulateVulnerabilityFeedbackPipelineId
+ def perform(project_ids)
+ end
+ end
+ end
+end
+
+Gitlab::BackgroundMigration::PopulateVulnerabilityFeedbackPipelineId.prepend_if_ee('EE::Gitlab::BackgroundMigration::PopulateVulnerabilityFeedbackPipelineId')
diff --git a/lib/gitlab/background_migration/replace_blocked_by_links.rb b/lib/gitlab/background_migration/replace_blocked_by_links.rb
index 26626aaef79..0c29887bb00 100644
--- a/lib/gitlab/background_migration/replace_blocked_by_links.rb
+++ b/lib/gitlab/background_migration/replace_blocked_by_links.rb
@@ -12,14 +12,19 @@ module Gitlab
blocked_by_links = IssueLink.where(id: start_id..stop_id).where(link_type: 2)
ActiveRecord::Base.transaction do
- # if there is duplicit bi-directional relation (issue2 is blocked by issue1
- # and issue1 already links issue2), then we can just delete 'blocked by'.
- # This should be rare as we have a pre-create check which checks if issues are
- # already linked
- blocked_by_links
+ # There could be two edge cases:
+ # 1) issue1 is blocked by issue2 AND issue2 blocks issue1 (type 1)
+ # 2) issue1 is blocked by issue2 AND issue2 is related to issue1 (type 0)
+ # In both cases cases we couldn't convert blocked by relation to
+ # `issue2 blocks issue` because there is already a link with the same
+ # source/target id. To avoid these conflicts, we first delete any
+ # "opposite" links before we update `blocked by` relation. This
+ # should be rare as we have a pre-create check which checks if issues
+ # are already linked
+ opposite_ids = blocked_by_links
+ .select('opposite_links.id')
.joins('INNER JOIN issue_links as opposite_links ON issue_links.source_id = opposite_links.target_id AND issue_links.target_id = opposite_links.source_id')
- .where('opposite_links.link_type': 1)
- .delete_all
+ IssueLink.where(id: opposite_ids).delete_all
blocked_by_links.update_all('source_id=target_id,target_id=source_id,link_type=1')
end
diff --git a/lib/gitlab/badge/coverage/report.rb b/lib/gitlab/badge/coverage/report.rb
index 0df6e858bf4..390da014a5a 100644
--- a/lib/gitlab/badge/coverage/report.rb
+++ b/lib/gitlab/badge/coverage/report.rb
@@ -17,8 +17,6 @@ module Gitlab
key_width: opts[:key_width].to_i,
key_text: opts[:key_text]
}
-
- @pipeline = @project.ci_pipelines.latest_successful_for_ref(@ref)
end
def entity
@@ -42,19 +40,35 @@ module Gitlab
private
- # rubocop: disable CodeReuse/ActiveRecord
+ def successful_pipeline
+ @successful_pipeline ||= @project.ci_pipelines.latest_successful_for_ref(@ref)
+ end
+
+ def failed_pipeline
+ @failed_pipeline ||= @project.ci_pipelines.latest_failed_for_ref(@ref)
+ end
+
+ def running_pipeline
+ @running_pipeline ||= @project.ci_pipelines.latest_running_for_ref(@ref)
+ end
+
def raw_coverage
- return unless @pipeline
+ latest =
+ if @job.present?
+ builds = ::Ci::Build
+ .in_pipelines([successful_pipeline, running_pipeline, failed_pipeline])
+ .latest
+ .success
+ .for_ref(@ref)
+ .by_name(@job)
+
+ builds.max_by(&:created_at)
+ else
+ successful_pipeline
+ end
- if @job.blank?
- @pipeline.coverage
- else
- @pipeline.builds
- .find_by(name: @job)
- .try(:coverage)
- end
+ latest&.coverage
end
- # rubocop: enable CodeReuse/ActiveRecord
end
end
end
diff --git a/lib/gitlab/bitbucket_server_import/importer.rb b/lib/gitlab/bitbucket_server_import/importer.rb
index aca5a63a424..d29799f1029 100644
--- a/lib/gitlab/bitbucket_server_import/importer.rb
+++ b/lib/gitlab/bitbucket_server_import/importer.rb
@@ -4,11 +4,14 @@ module Gitlab
module BitbucketServerImport
class Importer
attr_reader :recover_missing_commits
- attr_reader :project, :project_key, :repository_slug, :client, :errors, :users
+ attr_reader :project, :project_key, :repository_slug, :client, :errors, :users, :already_imported_cache_key
attr_accessor :logger
REMOTE_NAME = 'bitbucket_server'
BATCH_SIZE = 100
+ # The base cache key to use for tracking already imported objects.
+ ALREADY_IMPORTED_CACHE_KEY =
+ 'bitbucket_server-importer/already-imported/%{project}/%{collection}'
TempBranch = Struct.new(:name, :sha)
@@ -36,17 +39,25 @@ module Gitlab
@users = {}
@temp_branches = []
@logger = Gitlab::Import::Logger.build
+ @already_imported_cache_key = ALREADY_IMPORTED_CACHE_KEY %
+ { project: project.id, collection: collection_method }
+ end
+
+ def collection_method
+ :pull_requests
end
def execute
import_repository
import_pull_requests
+ download_lfs_objects
delete_temp_branches
handle_errors
metrics.track_finished_import
log_info(stage: "complete")
+ Gitlab::Cache::Import::Caching.expire(already_imported_cache_key, 15.minutes.to_i)
true
end
@@ -148,6 +159,14 @@ module Gitlab
raise
end
+ def download_lfs_objects
+ result = Projects::LfsPointers::LfsImportService.new(project).execute
+
+ if result[:status] == :error
+ errors << { type: :lfs_objects, errors: "The Lfs import process failed. #{result[:message]}" }
+ end
+ end
+
# Bitbucket Server keeps tracks of references for open pull requests in
# refs/heads/pull-requests, but closed and merged requests get moved
# into hidden internal refs under stash-refs/pull-requests. Unless the
@@ -158,16 +177,29 @@ module Gitlab
# on the remote server. Then we have to issue a `git fetch` to download these
# branches.
def import_pull_requests
- pull_requests = client.pull_requests(project_key, repository_slug).to_a
+ page = 0
+
+ log_info(stage: 'import_pull_requests', message: "starting")
+
+ loop do
+ log_debug(stage: 'import_pull_requests', message: "importing page #{page} and batch-size #{BATCH_SIZE} from #{page * BATCH_SIZE} to #{(page + 1) * BATCH_SIZE}")
+
+ pull_requests = client.pull_requests(project_key, repository_slug, page_offset: page, limit: BATCH_SIZE).to_a
- # Creating branches on the server and fetching the newly-created branches
- # may take a number of network round-trips. Do this in batches so that we can
- # avoid doing a git fetch for every new branch.
- pull_requests.each_slice(BATCH_SIZE) do |batch|
- restore_branches(batch) if recover_missing_commits
+ break if pull_requests.empty?
- batch.each do |pull_request|
- import_bitbucket_pull_request(pull_request)
+ # Creating branches on the server and fetching the newly-created branches
+ # may take a number of network round-trips. This used to be done in batches to
+ # avoid doing a git fetch for every new branch, as the whole process is now
+ # batched, we do not need to separately do this in batches.
+ restore_branches(pull_requests) if recover_missing_commits
+
+ pull_requests.each do |pull_request|
+ if already_imported?(pull_request)
+ log_info(stage: 'import_pull_requests', message: 'already imported', iid: pull_request.iid)
+ else
+ import_bitbucket_pull_request(pull_request)
+ end
rescue StandardError => e
Gitlab::ErrorTracking.log_exception(
e,
@@ -177,9 +209,25 @@ module Gitlab
backtrace = Gitlab::BacktraceCleaner.clean_backtrace(e.backtrace)
errors << { type: :pull_request, iid: pull_request.iid, errors: e.message, backtrace: backtrace.join("\n"), raw_response: pull_request.raw }
end
+
+ log_debug(stage: 'import_pull_requests', message: "finished page #{page} and batch-size #{BATCH_SIZE}")
+ page += 1
end
end
+ # Returns true if the given object has already been imported, false
+ # otherwise.
+ #
+ # object - The object to check.
+ def already_imported?(pull_request)
+ Gitlab::Cache::Import::Caching.set_includes?(already_imported_cache_key, pull_request.iid)
+ end
+
+ # Marks the given object as "already imported".
+ def mark_as_imported(pull_request)
+ Gitlab::Cache::Import::Caching.set_add(already_imported_cache_key, pull_request.iid)
+ end
+
def delete_temp_branches
@temp_branches.each do |branch|
client.delete_branch(project_key, repository_slug, branch.name, branch.sha)
@@ -227,6 +275,7 @@ module Gitlab
end
log_info(stage: 'import_bitbucket_pull_requests', message: 'finished', iid: pull_request.iid)
+ mark_as_imported(pull_request)
end
def import_pull_request_comments(pull_request, merge_request)
@@ -378,6 +427,10 @@ module Gitlab
}
end
+ def log_debug(details)
+ logger.debug(log_base_data.merge(details))
+ end
+
def log_info(details)
logger.info(log_base_data.merge(details))
end
diff --git a/lib/gitlab/chat/output.rb b/lib/gitlab/chat/output.rb
index 411b1555a7d..4a55b81a9eb 100644
--- a/lib/gitlab/chat/output.rb
+++ b/lib/gitlab/chat/output.rb
@@ -12,7 +12,10 @@ module Gitlab
PRIMARY_SECTION = 'chat_reply'
# The backup trace section in case the primary one could not be found.
- FALLBACK_SECTION = 'build_script'
+ FALLBACK_SECTION = 'step_script'
+
+ # `step_script` used to be `build_script` before runner 13.1
+ LEGACY_SECTION = 'build_script'
# build - The `Ci::Build` to obtain the output from.
def initialize(build)
@@ -37,24 +40,6 @@ module Gitlab
end
end
- # Returns the offset to seek to and the number of bytes to read relative
- # to the offset.
- def read_offset_and_length
- section = find_build_trace_section(PRIMARY_SECTION) ||
- find_build_trace_section(FALLBACK_SECTION)
-
- unless section
- raise(
- MissingBuildSectionError,
- "The build_script trace section could not be found for build #{build.id}"
- )
- end
-
- length = section[:byte_end] - section[:byte_start]
-
- [section[:byte_start], length]
- end
-
# Removes the line containing the executed command from the build output.
#
# output - A `String` containing the output of a trace section.
@@ -88,6 +73,27 @@ module Gitlab
def trace
@trace ||= build.trace
end
+
+ private
+
+ # Returns the offset to seek to and the number of bytes to read relative
+ # to the offset.
+ def read_offset_and_length
+ section = find_build_trace_section(PRIMARY_SECTION) ||
+ find_build_trace_section(FALLBACK_SECTION) ||
+ find_build_trace_section(LEGACY_SECTION)
+
+ unless section
+ raise(
+ MissingBuildSectionError,
+ "The build_script trace section could not be found for build #{build.id}"
+ )
+ end
+
+ length = section[:byte_end] - section[:byte_start]
+
+ [section[:byte_start], length]
+ end
end
end
end
diff --git a/lib/gitlab/ci/build/rules/rule/clause/changes.rb b/lib/gitlab/ci/build/rules/rule/clause/changes.rb
index 728a66ca87f..cbecce57163 100644
--- a/lib/gitlab/ci/build/rules/rule/clause/changes.rb
+++ b/lib/gitlab/ci/build/rules/rule/clause/changes.rb
@@ -11,12 +11,22 @@ module Gitlab
def satisfied_by?(pipeline, context)
return true if pipeline.modified_paths.nil?
+ expanded_globs = expand_globs(pipeline, context)
pipeline.modified_paths.any? do |path|
- @globs.any? do |glob|
+ expanded_globs.any? do |glob|
File.fnmatch?(glob, path, File::FNM_PATHNAME | File::FNM_DOTMATCH | File::FNM_EXTGLOB)
end
end
end
+
+ def expand_globs(pipeline, context)
+ return @globs unless ::Feature.enabled?(:ci_variable_expansion_in_rules_changes, pipeline.project, default_enabled: true)
+ return @globs unless context
+
+ @globs.map do |glob|
+ ExpandVariables.expand_existing(glob, context.variables)
+ end
+ end
end
end
end
diff --git a/lib/gitlab/ci/charts.rb b/lib/gitlab/ci/charts.rb
index 3fbfdffe277..25fb9c0ca97 100644
--- a/lib/gitlab/ci/charts.rb
+++ b/lib/gitlab/ci/charts.rb
@@ -3,38 +3,8 @@
module Gitlab
module Ci
module Charts
- module DailyInterval
- # rubocop: disable CodeReuse/ActiveRecord
- def grouped_count(query)
- query
- .group("DATE(#{::Ci::Pipeline.table_name}.created_at)")
- .count(:created_at)
- .transform_keys { |date| date.strftime(@format) } # rubocop:disable Gitlab/ModuleWithInstanceVariables
- end
- # rubocop: enable CodeReuse/ActiveRecord
-
- def interval_step
- @interval_step ||= 1.day
- end
- end
-
- module MonthlyInterval
- # rubocop: disable CodeReuse/ActiveRecord
- def grouped_count(query)
- query
- .group("to_char(#{::Ci::Pipeline.table_name}.created_at, '01 Month YYYY')")
- .count(:created_at)
- .transform_keys(&:squish)
- end
- # rubocop: enable CodeReuse/ActiveRecord
-
- def interval_step
- @interval_step ||= 1.month
- end
- end
-
class Chart
- attr_reader :labels, :total, :success, :project, :pipeline_times
+ attr_reader :from, :to, :labels, :total, :success, :project, :pipeline_times
def initialize(project)
@labels = []
@@ -46,48 +16,59 @@ module Gitlab
collect
end
+ private
+
+ attr_reader :interval
+
# rubocop: disable CodeReuse/ActiveRecord
def collect
query = project.all_pipelines
- .where("? > #{::Ci::Pipeline.table_name}.created_at AND #{::Ci::Pipeline.table_name}.created_at > ?", @to, @from) # rubocop:disable GitlabSecurity/SqlInjection
+ .where(::Ci::Pipeline.arel_table['created_at'].gteq(@from))
+ .where(::Ci::Pipeline.arel_table['created_at'].lteq(@to))
totals_count = grouped_count(query)
success_count = grouped_count(query.success)
current = @from
- while current < @to
- label = current.strftime(@format)
-
- @labels << label
- @total << (totals_count[label] || 0)
- @success << (success_count[label] || 0)
+ while current <= @to
+ @labels << current.strftime(@format)
+ @total << (totals_count[current] || 0)
+ @success << (success_count[current] || 0)
current += interval_step
end
end
# rubocop: enable CodeReuse/ActiveRecord
+
+ # rubocop: disable CodeReuse/ActiveRecord
+ def grouped_count(query)
+ query
+ .group("date_trunc('#{interval}', #{::Ci::Pipeline.table_name}.created_at)")
+ .count(:created_at)
+ end
+ # rubocop: enable CodeReuse/ActiveRecord
+
+ def interval_step
+ @interval_step ||= 1.public_send(interval) # rubocop: disable GitlabSecurity/PublicSend
+ end
end
class YearChart < Chart
- include MonthlyInterval
- attr_reader :to, :from
-
def initialize(*)
@to = Date.today.end_of_month.end_of_day
- @from = @to.years_ago(1).beginning_of_month.beginning_of_day
- @format = '%d %B %Y'
+ @from = (@to - 1.year).beginning_of_month.beginning_of_day
+ @interval = :month
+ @format = '%B %Y'
super
end
end
class MonthChart < Chart
- include DailyInterval
- attr_reader :to, :from
-
def initialize(*)
@to = Date.today.end_of_day
- @from = 1.month.ago.beginning_of_day
+ @from = (@to - 1.month).beginning_of_day
+ @interval = :day
@format = '%d %B'
super
@@ -95,12 +76,10 @@ module Gitlab
end
class WeekChart < Chart
- include DailyInterval
- attr_reader :to, :from
-
def initialize(*)
@to = Date.today.end_of_day
- @from = 1.week.ago.beginning_of_day
+ @from = (@to - 1.week).beginning_of_day
+ @interval = :day
@format = '%d %B'
super
diff --git a/lib/gitlab/ci/config/entry/bridge.rb b/lib/gitlab/ci/config/entry/bridge.rb
index 1740032e5c7..70fcc1d586a 100644
--- a/lib/gitlab/ci/config/entry/bridge.rb
+++ b/lib/gitlab/ci/config/entry/bridge.rb
@@ -18,7 +18,6 @@ module Gitlab
validates :config, allowed_keys: ALLOWED_KEYS + PROCESSABLE_ALLOWED_KEYS
with_options allow_nil: true do
- validates :allow_failure, boolean: true
validates :when, inclusion: {
in: ALLOWED_WHEN,
message: "should be one of: #{ALLOWED_WHEN.join(', ')}"
@@ -48,7 +47,7 @@ module Gitlab
inherit: false,
metadata: { allowed_needs: %i[job bridge] }
- attributes :when, :allow_failure
+ attributes :when
def self.matching?(name, config)
!name.to_s.start_with?('.') &&
@@ -60,14 +59,6 @@ module Gitlab
true
end
- def manual_action?
- self.when == 'manual'
- end
-
- def ignored?
- allow_failure.nil? ? manual_action? : allow_failure
- end
-
def value
super.merge(
trigger: (trigger_value if trigger_defined?),
diff --git a/lib/gitlab/ci/config/entry/job.rb b/lib/gitlab/ci/config/entry/job.rb
index ecc2c5cb729..1ce7060df22 100644
--- a/lib/gitlab/ci/config/entry/job.rb
+++ b/lib/gitlab/ci/config/entry/job.rb
@@ -24,7 +24,6 @@ module Gitlab
validates :script, presence: true
with_options allow_nil: true do
- validates :allow_failure, boolean: true
validates :when, inclusion: {
in: ALLOWED_WHEN,
message: "should be one of: #{ALLOWED_WHEN.join(', ')}"
@@ -118,7 +117,7 @@ module Gitlab
description: 'Parallel configuration for this job.',
inherit: false
- attributes :script, :tags, :allow_failure, :when, :dependencies,
+ attributes :script, :tags, :when, :dependencies,
:needs, :retry, :parallel, :start_in,
:interruptible, :timeout, :resource_group, :release
@@ -141,18 +140,10 @@ module Gitlab
end
end
- def manual_action?
- self.when == 'manual'
- end
-
def delayed?
self.when == 'delayed'
end
- def ignored?
- allow_failure.nil? ? manual_action? : allow_failure
- end
-
def value
super.merge(
before_script: before_script_value,
diff --git a/lib/gitlab/ci/config/entry/processable.rb b/lib/gitlab/ci/config/entry/processable.rb
index f10c509d0cc..c0315e5f901 100644
--- a/lib/gitlab/ci/config/entry/processable.rb
+++ b/lib/gitlab/ci/config/entry/processable.rb
@@ -32,6 +32,7 @@ module Gitlab
with_options allow_nil: true do
validates :extends, array_of_strings_or_string: true
validates :rules, array_of_hashes: true
+ validates :allow_failure, boolean: true
end
end
@@ -64,7 +65,7 @@ module Gitlab
inherit: false,
default: {}
- attributes :extends, :rules
+ attributes :extends, :rules, :allow_failure
end
def compose!(deps = nil)
@@ -136,6 +137,14 @@ module Gitlab
root_variables.merge(variables_value.to_h)
end
+
+ def manual_action?
+ self.when == 'manual'
+ end
+
+ def ignored?
+ allow_failure.nil? ? manual_action? : allow_failure
+ end
end
end
end
diff --git a/lib/gitlab/ci/config/entry/product/variables.rb b/lib/gitlab/ci/config/entry/product/variables.rb
index 2481989060e..aa34cfb3acc 100644
--- a/lib/gitlab/ci/config/entry/product/variables.rb
+++ b/lib/gitlab/ci/config/entry/product/variables.rb
@@ -14,7 +14,7 @@ module Gitlab
validations do
validates :config, variables: { array_values: true }
validates :config, length: {
- minimum: :minimum,
+ minimum: 1,
too_short: 'requires at least %{count} items'
}
end
@@ -28,10 +28,6 @@ module Gitlab
.map { |key, value| [key.to_s, Array(value).map(&:to_s)] }
.to_h
end
-
- def minimum
- ::Gitlab::Ci::Features.one_dimensional_matrix_enabled? ? 1 : 2
- end
end
end
end
diff --git a/lib/gitlab/ci/config/external/mapper.rb b/lib/gitlab/ci/config/external/mapper.rb
index 97ae6c4ceba..90692eafc3f 100644
--- a/lib/gitlab/ci/config/external/mapper.rb
+++ b/lib/gitlab/ci/config/external/mapper.rb
@@ -33,6 +33,7 @@ module Gitlab
locations
.compact
.map(&method(:normalize_location))
+ .flat_map(&method(:expand_project_files))
.each(&method(:verify_duplicates!))
.map(&method(:select_first_matching))
end
@@ -52,6 +53,15 @@ module Gitlab
end
end
+ def expand_project_files(location)
+ return location unless ::Feature.enabled?(:ci_include_multiple_files_from_project, context.project, default_enabled: true)
+ return location unless location[:project]
+
+ Array.wrap(location[:file]).map do |file|
+ location.merge(file: file)
+ end
+ end
+
def normalize_location_string(location)
if ::Gitlab::UrlSanitizer.valid?(location)
{ remote: location }
diff --git a/lib/gitlab/ci/features.rb b/lib/gitlab/ci/features.rb
index 1b58e3ec71a..661189eea50 100644
--- a/lib/gitlab/ci/features.rb
+++ b/lib/gitlab/ci/features.rb
@@ -32,16 +32,12 @@ module Gitlab
end
# NOTE: The feature flag `disallow_to_create_merge_request_pipelines_in_target_project`
- # is a safe switch to disable the feature for a parituclar project when something went wrong,
+ # is a safe switch to disable the feature for a particular project when something went wrong,
# therefore it's not supposed to be enabled by default.
def self.disallow_to_create_merge_request_pipelines_in_target_project?(target_project)
::Feature.enabled?(:ci_disallow_to_create_merge_request_pipelines_in_target_project, target_project)
end
- def self.lint_creates_pipeline_with_dry_run?(project)
- ::Feature.enabled?(:ci_lint_creates_pipeline_with_dry_run, project, default_enabled: true)
- end
-
def self.project_transactionless_destroy?(project)
Feature.enabled?(:project_transactionless_destroy, project, default_enabled: false)
end
@@ -59,13 +55,21 @@ module Gitlab
::Feature.enabled?(:ci_trace_log_invalid_chunks, project, type: :ops, default_enabled: false)
end
- def self.one_dimensional_matrix_enabled?
- ::Feature.enabled?(:one_dimensional_matrix, default_enabled: true)
- end
-
def self.manual_bridges_enabled?(project)
::Feature.enabled?(:ci_manual_bridges, project, default_enabled: true)
end
+
+ def self.auto_rollback_available?(project)
+ ::Feature.enabled?(:cd_auto_rollback, project) && project&.feature_available?(:auto_rollback)
+ end
+
+ def self.seed_block_run_before_workflow_rules_enabled?(project)
+ ::Feature.enabled?(:ci_seed_block_run_before_workflow_rules, project, default_enabled: true)
+ end
+
+ def self.ci_pipeline_editor_page_enabled?(project)
+ ::Feature.enabled?(:ci_pipeline_editor_page, project, default_enabled: false)
+ end
end
end
end
diff --git a/lib/gitlab/ci/jwt.rb b/lib/gitlab/ci/jwt.rb
index 491facd0a43..a8943eadf4f 100644
--- a/lib/gitlab/ci/jwt.rb
+++ b/lib/gitlab/ci/jwt.rb
@@ -6,6 +6,8 @@ module Gitlab
NOT_BEFORE_TIME = 5
DEFAULT_EXPIRE_TIME = 60 * 5
+ NoSigningKeyError = Class.new(StandardError)
+
def self.for_build(build)
self.new(build, ttl: build.metadata_timeout).encoded
end
@@ -27,7 +29,7 @@ module Gitlab
private
- attr_reader :build, :ttl, :key_data
+ attr_reader :build, :ttl
def reserved_claims
now = Time.now.to_i
@@ -60,7 +62,17 @@ module Gitlab
end
def key
- @key ||= OpenSSL::PKey::RSA.new(Rails.application.secrets.openid_connect_signing_key)
+ @key ||= begin
+ key_data = if Feature.enabled?(:ci_jwt_signing_key, build.project, default_enabled: true)
+ Gitlab::CurrentSettings.ci_jwt_signing_key
+ else
+ Rails.application.secrets.openid_connect_signing_key
+ end
+
+ raise NoSigningKeyError unless key_data
+
+ OpenSSL::PKey::RSA.new(key_data)
+ end
end
def public_key
diff --git a/lib/gitlab/ci/lint.rb b/lib/gitlab/ci/lint.rb
index 44f2ac23ce3..fb795152abe 100644
--- a/lib/gitlab/ci/lint.rb
+++ b/lib/gitlab/ci/lint.rb
@@ -24,7 +24,7 @@ module Gitlab
end
def validate(content, dry_run: false)
- if dry_run && Gitlab::Ci::Features.lint_creates_pipeline_with_dry_run?(@project)
+ if dry_run
simulate_pipeline_creation(content)
else
static_validation(content)
diff --git a/lib/gitlab/ci/pipeline/chain/cancel_pending_pipelines.rb b/lib/gitlab/ci/pipeline/chain/cancel_pending_pipelines.rb
index 468f3bc4689..a864c843dd8 100644
--- a/lib/gitlab/ci/pipeline/chain/cancel_pending_pipelines.rb
+++ b/lib/gitlab/ci/pipeline/chain/cancel_pending_pipelines.rb
@@ -25,7 +25,7 @@ module Gitlab
# rubocop: disable CodeReuse/ActiveRecord
def auto_cancelable_pipelines
- project.ci_pipelines
+ pipelines
.where(ref: pipeline.ref)
.where.not(id: pipeline.same_family_pipeline_ids)
.where.not(sha: project.commit(pipeline.ref).try(:id))
@@ -33,6 +33,14 @@ module Gitlab
.with_only_interruptible_builds
end
# rubocop: enable CodeReuse/ActiveRecord
+
+ def pipelines
+ if ::Feature.enabled?(:ci_auto_cancel_all_pipelines, project, default_enabled: false)
+ project.all_pipelines.ci_and_parent_sources
+ else
+ project.ci_pipelines
+ end
+ end
end
end
end
diff --git a/lib/gitlab/ci/pipeline/chain/seed.rb b/lib/gitlab/ci/pipeline/chain/seed.rb
index e10a0bc3718..ba86b08d209 100644
--- a/lib/gitlab/ci/pipeline/chain/seed.rb
+++ b/lib/gitlab/ci/pipeline/chain/seed.rb
@@ -19,10 +19,12 @@ module Gitlab
# Build to prevent erroring out on ambiguous refs.
pipeline.protected = @command.protected_ref?
- ##
- # Populate pipeline with block argument of CreatePipelineService#execute.
- #
- @command.seeds_block&.call(pipeline)
+ unless ::Gitlab::Ci::Features.seed_block_run_before_workflow_rules_enabled?(project)
+ ##
+ # Populate pipeline with block argument of CreatePipelineService#execute.
+ #
+ @command.seeds_block&.call(pipeline)
+ end
##
# Gather all runtime build/stage errors
diff --git a/lib/gitlab/ci/pipeline/chain/seed_block.rb b/lib/gitlab/ci/pipeline/chain/seed_block.rb
new file mode 100644
index 00000000000..f8e62949bea
--- /dev/null
+++ b/lib/gitlab/ci/pipeline/chain/seed_block.rb
@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Ci
+ module Pipeline
+ module Chain
+ class SeedBlock < Chain::Base
+ include Chain::Helpers
+ include Gitlab::Utils::StrongMemoize
+
+ def perform!
+ return unless ::Gitlab::Ci::Features.seed_block_run_before_workflow_rules_enabled?(project)
+
+ ##
+ # Populate pipeline with block argument of CreatePipelineService#execute.
+ #
+ @command.seeds_block&.call(pipeline)
+
+ raise "Pipeline cannot be persisted by `seeds_block`" if pipeline.persisted?
+ end
+
+ def break?
+ return false unless ::Gitlab::Ci::Features.seed_block_run_before_workflow_rules_enabled?(project)
+
+ pipeline.errors.any?
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/ci/pipeline/seed/environment.rb b/lib/gitlab/ci/pipeline/seed/environment.rb
index 42e8c365824..b20dc383419 100644
--- a/lib/gitlab/ci/pipeline/seed/environment.rb
+++ b/lib/gitlab/ci/pipeline/seed/environment.rb
@@ -12,12 +12,23 @@ module Gitlab
end
def to_resource
- job.project.environments
- .safe_find_or_create_by(name: expanded_environment_name)
+ environments.safe_find_or_create_by(name: expanded_environment_name) do |environment|
+ environment.auto_stop_in = auto_stop_in
+ end
end
private
+ def environments
+ job.project.environments
+ end
+
+ def auto_stop_in
+ if Feature.enabled?(:environment_auto_stop_start_on_create)
+ job.environment_auto_stop_in
+ end
+ end
+
def expanded_environment_name
job.expanded_environment_name
end
diff --git a/lib/gitlab/ci/reports/test_case.rb b/lib/gitlab/ci/reports/test_case.rb
index 8c70dbb6931..09121191047 100644
--- a/lib/gitlab/ci/reports/test_case.rb
+++ b/lib/gitlab/ci/reports/test_case.rb
@@ -10,7 +10,7 @@ module Gitlab
STATUS_ERROR = 'error'
STATUS_TYPES = [STATUS_ERROR, STATUS_FAILED, STATUS_SUCCESS, STATUS_SKIPPED].freeze
- attr_reader :suite_name, :name, :classname, :execution_time, :status, :file, :system_output, :stack_trace, :key, :attachment, :job
+ attr_reader :suite_name, :name, :classname, :execution_time, :status, :file, :system_output, :stack_trace, :key, :attachment, :job, :recent_failures
def initialize(params)
@suite_name = params.fetch(:suite_name)
@@ -24,9 +24,15 @@ module Gitlab
@attachment = params.fetch(:attachment, nil)
@job = params.fetch(:job, nil)
+ @recent_failures = nil
+
@key = hash_key("#{suite_name}_#{classname}_#{name}")
end
+ def set_recent_failures(count, base_branch)
+ @recent_failures = { count: count, base_branch: base_branch }
+ end
+
def has_attachment?
attachment.present?
end
diff --git a/lib/gitlab/ci/reports/test_failure_history.rb b/lib/gitlab/ci/reports/test_failure_history.rb
new file mode 100644
index 00000000000..beceac5423a
--- /dev/null
+++ b/lib/gitlab/ci/reports/test_failure_history.rb
@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Ci
+ module Reports
+ class TestFailureHistory
+ include Gitlab::Utils::StrongMemoize
+
+ def initialize(failed_test_cases, project)
+ @failed_test_cases = build_map(failed_test_cases)
+ @project = project
+ end
+
+ def load!
+ return unless Feature.enabled?(:test_failure_history, project)
+
+ recent_failures_count.each do |key_hash, count|
+ failed_test_cases[key_hash].set_recent_failures(count, project.default_branch_or_master)
+ end
+ end
+
+ private
+
+ attr_reader :report, :project, :failed_test_cases
+
+ def recent_failures_count
+ ::Ci::TestCaseFailure.recent_failures_count(
+ project: project,
+ test_case_keys: failed_test_cases.keys
+ )
+ end
+
+ def build_map(test_cases)
+ {}.tap do |hash|
+ test_cases.each do |test_case|
+ hash[test_case.key] = test_case
+ end
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/ci/reports/test_suite_comparer.rb b/lib/gitlab/ci/reports/test_suite_comparer.rb
index a58de43e55e..239fc3b15e7 100644
--- a/lib/gitlab/ci/reports/test_suite_comparer.rb
+++ b/lib/gitlab/ci/reports/test_suite_comparer.rb
@@ -6,6 +6,9 @@ module Gitlab
class TestSuiteComparer
include Gitlab::Utils::StrongMemoize
+ DEFAULT_MAX_TESTS = 100
+ DEFAULT_MIN_TESTS = 10
+
attr_reader :name, :base_suite, :head_suite
def initialize(name, base_suite, head_suite)
@@ -81,6 +84,29 @@ module Gitlab
def error_count
new_errors.count + existing_errors.count
end
+
+ # This is used to limit the presented test cases but does not affect
+ # total count of tests in the summary
+ def limited_tests
+ strong_memoize(:limited_tests) do
+ # rubocop: disable CodeReuse/ActiveRecord
+ OpenStruct.new(
+ new_failures: new_failures.take(max_tests),
+ existing_failures: existing_failures.take(max_tests(new_failures)),
+ resolved_failures: resolved_failures.take(max_tests(new_failures, existing_failures)),
+ new_errors: new_errors.take(max_tests),
+ existing_errors: existing_errors.take(max_tests(new_errors)),
+ resolved_errors: resolved_errors.take(max_tests(new_errors, existing_errors))
+ )
+ # rubocop: enable CodeReuse/ActiveRecord
+ end
+ end
+
+ private
+
+ def max_tests(*used)
+ [DEFAULT_MAX_TESTS - used.map(&:count).sum, DEFAULT_MIN_TESTS].max
+ end
end
end
end
diff --git a/lib/gitlab/ci/runner_instructions.rb b/lib/gitlab/ci/runner_instructions.rb
index 2171637687f..dd0bfa768a8 100644
--- a/lib/gitlab/ci/runner_instructions.rb
+++ b/lib/gitlab/ci/runner_instructions.rb
@@ -106,7 +106,7 @@ module Gitlab
end
def get_file(path)
- File.read(path)
+ File.read(Rails.root.join(path).to_s)
end
def registration_token
diff --git a/lib/gitlab/ci/templates/AWS/Deploy-ECS.gitlab-ci.yml b/lib/gitlab/ci/templates/AWS/Deploy-ECS.gitlab-ci.yml
index 82b2f5c035e..453803a6f7e 100644
--- a/lib/gitlab/ci/templates/AWS/Deploy-ECS.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/AWS/Deploy-ECS.gitlab-ci.yml
@@ -4,6 +4,7 @@ stages:
- review
- deploy
- production
+ - cleanup
variables:
AUTO_DEVOPS_PLATFORM_TARGET: ECS
diff --git a/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml
index cba13f374f4..a13f2046291 100644
--- a/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml
@@ -160,9 +160,10 @@ include:
- template: Jobs/Build.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml
- template: Jobs/Test.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Jobs/Test.gitlab-ci.yml
- template: Jobs/Code-Quality.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml
- - template: Jobs/Code-Intelligence.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Jobs/Code-Intelligence.gitlab-ci.yml
+ - template: Jobs/Code-Intelligence.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Jobs/Code-Intelligence.gitlab-ci.yml
- template: Jobs/Deploy.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml
- - template: Jobs/Deploy/ECS.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Jobs/Deploy/ECS.gitlab-ci.yml
+ - template: Jobs/Deploy/ECS.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Jobs/Deploy/ECS.gitlab-ci.yml
+ - template: Jobs/Deploy/EC2.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Jobs/Deploy/EC2.gitlab-ci.yml
- template: Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml
- template: Jobs/Browser-Performance-Testing.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml
- template: Security/DAST.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml
diff --git a/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml
index 0c3598a61a7..1c25d9d583b 100644
--- a/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml
@@ -16,4 +16,14 @@ build:
fi
- /build/build.sh
rules:
+ - if: '$AUTO_DEVOPS_PLATFORM_TARGET == "EC2"'
+ when: never
- if: '$CI_COMMIT_TAG || $CI_COMMIT_BRANCH'
+
+build_artifact:
+ stage: build
+ script:
+ - printf "To build your project, please create a build_artifact job into your .gitlab-ci.yml file.\nMore information at https://docs.gitlab.com/ee/ci/cloud_deployment\n"
+ - exit 1
+ rules:
+ - if: '$AUTO_DEVOPS_PLATFORM_TARGET == "EC2"'
diff --git a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml
index ec33020205b..fe23641802b 100644
--- a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml
@@ -7,7 +7,7 @@ code_quality:
variables:
DOCKER_DRIVER: overlay2
DOCKER_TLS_CERTDIR: ""
- CODE_QUALITY_IMAGE: "registry.gitlab.com/gitlab-org/ci-cd/codequality:0.85.10-gitlab.1"
+ CODE_QUALITY_IMAGE: "registry.gitlab.com/gitlab-org/ci-cd/codequality:0.85.18"
needs: []
script:
- export SOURCE_CODE=$PWD
@@ -34,6 +34,7 @@ code_quality:
CODECLIMATE_DEBUG \
CODECLIMATE_DEV \
REPORT_STDOUT \
+ REPORT_FORMAT \
ENGINE_MEMORY_LIMIT_BYTES \
) \
--volume "$PWD":/code \
diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml
index 33d77e39bc9..c4e194bd658 100644
--- a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml
@@ -186,7 +186,7 @@ production_manual:
when: never
- if: '$CI_COMMIT_BRANCH != "master"'
when: never
- # $INCREMENTAL_ROLLOUT_ENABLED is for compamtibilty with pre-GitLab 11.4 syntax
+ # $INCREMENTAL_ROLLOUT_ENABLED is for compatibility with pre-GitLab 11.4 syntax
- if: '$INCREMENTAL_ROLLOUT_MODE == "manual" || $INCREMENTAL_ROLLOUT_ENABLED'
when: manual
diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml
index 8b921305c11..385959389de 100644
--- a/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml
@@ -183,7 +183,7 @@ production_manual:
when: never
- if: '$CI_COMMIT_BRANCH != "master"'
when: never
- # $INCREMENTAL_ROLLOUT_ENABLED is for compamtibilty with pre-GitLab 11.4 syntax
+ # $INCREMENTAL_ROLLOUT_ENABLED is for compatibility with pre-GitLab 11.4 syntax
- if: '$INCREMENTAL_ROLLOUT_MODE == "manual" || $INCREMENTAL_ROLLOUT_ENABLED'
when: manual
diff --git a/lib/gitlab/ci/templates/Jobs/Deploy/ECS.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy/ECS.gitlab-ci.yml
index 317e8bfab0e..0289ba1c473 100644
--- a/lib/gitlab/ci/templates/Jobs/Deploy/ECS.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Jobs/Deploy/ECS.gitlab-ci.yml
@@ -8,8 +8,11 @@
#
# More about including CI templates: https://docs.gitlab.com/ee/ci/yaml/#includetemplate
-.deploy_to_ecs:
+.ecs_image:
image: 'registry.gitlab.com/gitlab-org/cloud-deploy/aws-ecs:latest'
+
+.deploy_to_ecs:
+ extends: .ecs_image
dependencies: []
script:
- ecs update-task-definition
@@ -17,8 +20,6 @@
.review_ecs_base:
stage: review
extends: .deploy_to_ecs
- environment:
- name: review/$CI_COMMIT_REF_NAME
.production_ecs_base:
stage: production
@@ -26,8 +27,18 @@
environment:
name: production
+.stop_review_ecs_base:
+ extends: .ecs_image
+ stage: cleanup
+ allow_failure: true
+ script:
+ - ecs stop-task
+
review_ecs:
extends: .review_ecs_base
+ environment:
+ name: review/$CI_COMMIT_REF_NAME
+ on_stop: stop_review_ecs
rules:
- if: '$AUTO_DEVOPS_PLATFORM_TARGET != "ECS"'
when: never
@@ -39,8 +50,46 @@ review_ecs:
when: never
- if: '$CI_COMMIT_TAG || $CI_COMMIT_BRANCH'
+stop_review_ecs:
+ extends: .stop_review_ecs_base
+ environment:
+ name: review/$CI_COMMIT_REF_NAME
+ action: stop
+ rules:
+ - if: '$AUTO_DEVOPS_PLATFORM_TARGET != "ECS"'
+ when: never
+ - if: '$CI_KUBERNETES_ACTIVE'
+ when: never
+ - if: '$REVIEW_DISABLED'
+ when: never
+ - if: '$CI_COMMIT_BRANCH == "master"'
+ when: never
+ - if: '$CI_COMMIT_TAG || $CI_COMMIT_BRANCH'
+ when: manual
+
review_fargate:
extends: .review_ecs_base
+ environment:
+ name: review/$CI_COMMIT_REF_NAME
+ on_stop: stop_review_fargate
+ script:
+ - ecs update-task-definition
+ rules:
+ - if: '$AUTO_DEVOPS_PLATFORM_TARGET != "FARGATE"'
+ when: never
+ - if: '$CI_KUBERNETES_ACTIVE'
+ when: never
+ - if: '$REVIEW_DISABLED'
+ when: never
+ - if: '$CI_COMMIT_BRANCH == "master"'
+ when: never
+ - if: '$CI_COMMIT_TAG || $CI_COMMIT_BRANCH'
+
+stop_review_fargate:
+ extends: .stop_review_ecs_base
+ environment:
+ name: review/$CI_COMMIT_REF_NAME
+ action: stop
rules:
- if: '$AUTO_DEVOPS_PLATFORM_TARGET != "FARGATE"'
when: never
@@ -51,6 +100,7 @@ review_fargate:
- if: '$CI_COMMIT_BRANCH == "master"'
when: never
- if: '$CI_COMMIT_TAG || $CI_COMMIT_BRANCH'
+ when: manual
production_ecs:
extends: .production_ecs_base
diff --git a/lib/gitlab/ci/templates/Managed-Cluster-Applications.gitlab-ci.yml b/lib/gitlab/ci/templates/Managed-Cluster-Applications.gitlab-ci.yml
index a9638f564f3..3f62d92ad13 100644
--- a/lib/gitlab/ci/templates/Managed-Cluster-Applications.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Managed-Cluster-Applications.gitlab-ci.yml
@@ -1,6 +1,6 @@
apply:
stage: deploy
- image: "registry.gitlab.com/gitlab-org/cluster-integration/cluster-applications:v0.33.0"
+ image: "registry.gitlab.com/gitlab-org/cluster-integration/cluster-applications:v0.34.1"
environment:
name: production
variables:
diff --git a/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml
index c3a92b67a8b..0ae8fd833c4 100644
--- a/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml
@@ -1,3 +1,9 @@
+# Read more about this feature here: https://docs.gitlab.com/ee/user/application_security/api_fuzzing/
+
+# Configure the scanning tool through the environment variables.
+# List of the variables: https://docs.gitlab.com/ee/user/application_security/api_fuzzing/#available-variables
+# How to set: https://docs.gitlab.com/ee/ci/yaml/#variables
+
stages:
- build
- test
@@ -7,7 +13,7 @@ stages:
variables:
FUZZAPI_PROFILE: Quick
FUZZAPI_VERSION: latest
- FUZZAPI_CONFIG: "/app/.gitlab-api-fuzzing.yml"
+ FUZZAPI_CONFIG: .gitlab-api-fuzzing.yml
FUZZAPI_TIMEOUT: 30
FUZZAPI_REPORT: gl-api-fuzzing-report.xml
#
@@ -17,9 +23,70 @@ variables:
# available (non 500 response to HTTP(s))
FUZZAPI_SERVICE_START_TIMEOUT: "300"
#
+ FUZZAPI_IMAGE: registry.gitlab.com/gitlab-org/security-products/analyzers/api-fuzzing:${FUZZAPI_VERSION}-engine
+ #
+
+apifuzzer_fuzz_unlicensed:
+ stage: fuzz
+ allow_failure: true
+ rules:
+ - if: '$GITLAB_FEATURES !~ /\bapi_fuzzing\b/ && $API_FUZZING_DISABLED == null'
+ - when: never
+ script:
+ - |
+ echo "Error: Your GitLab project is not licensed for API Fuzzing."
+ - exit 1
apifuzzer_fuzz:
stage: fuzz
+ image:
+ name: $FUZZAPI_IMAGE
+ entrypoint: ["/bin/bash", "-l", "-c"]
+ variables:
+ FUZZAPI_PROJECT: $CI_PROJECT_PATH
+ FUZZAPI_API: http://apifuzzer:80
+ TZ: America/Los_Angeles
+ services:
+ - name: $FUZZAPI_IMAGE
+ alias: apifuzzer
+ entrypoint: ["dotnet", "/peach/Peach.Web.dll"]
+ allow_failure: true
+ rules:
+ - if: $FUZZAPI_D_TARGET_IMAGE
+ when: never
+ - if: $FUZZAPI_D_WORKER_IMAGE
+ when: never
+ - if: $API_FUZZING_DISABLED
+ when: never
+ - if: $API_FUZZING_DISABLED_FOR_DEFAULT_BRANCH &&
+ $CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME
+ when: never
+ - if: $GITLAB_FEATURES =~ /\bapi_fuzzing\b/
+ script:
+ #
+ # Validate options
+ - |
+ if [ "$FUZZAPI_HAR$FUZZAPI_OPENAPI$FUZZAPI_POSTMAN_COLLECTION" == "" ]; then \
+ echo "Error: One of FUZZAPI_HAR, FUZZAPI_OPENAPI, or FUZZAPI_POSTMAN_COLLECTION must be provided."; \
+ echo "See https://docs.gitlab.com/ee/user/application_security/api_fuzzing/ for information on how to configure API Fuzzing."; \
+ exit 1; \
+ fi
+ #
+ # Run user provided pre-script
+ - sh -c "$FUZZAPI_PRE_SCRIPT"
+ #
+ # Start scanning
+ - worker-entry
+ #
+ # Run user provided post-script
+ - sh -c "$FUZZAPI_POST_SCRIPT"
+ #
+ artifacts:
+ reports:
+ junit: $FUZZAPI_REPORT
+
+apifuzzer_fuzz_dnd:
+ stage: fuzz
image: docker:19.03.12
variables:
DOCKER_DRIVER: overlay2
@@ -28,20 +95,19 @@ apifuzzer_fuzz:
FUZZAPI_API: http://apifuzzer:80
allow_failure: true
rules:
+ - if: $FUZZAPI_D_TARGET_IMAGE == null && $FUZZAPI_D_WORKER_IMAGE == null
+ when: never
- if: $API_FUZZING_DISABLED
when: never
- if: $API_FUZZING_DISABLED_FOR_DEFAULT_BRANCH &&
- $CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME
- when: never
- - if: $FUZZAPI_HAR == null &&
- $FUZZAPI_OPENAPI == null &&
- $FUZZAPI_D_WORKER_IMAGE == null
+ $CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME
when: never
- if: $GITLAB_FEATURES =~ /\bapi_fuzzing\b/
services:
- docker:19.03.12-dind
script:
#
+ #
- docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
#
- docker network create --driver bridge $FUZZAPI_D_NETWORK
@@ -56,30 +122,13 @@ apifuzzer_fuzz:
--network $FUZZAPI_D_NETWORK \
-e Proxy:Port=8000 \
-e TZ=America/Los_Angeles \
- -e FUZZAPI_API=http://127.0.0.1:80 \
- -e FUZZAPI_PROJECT \
- -e FUZZAPI_PROFILE \
- -e FUZZAPI_CONFIG \
- -e FUZZAPI_REPORT \
- -e FUZZAPI_HAR \
- -e FUZZAPI_OPENAPI \
- -e FUZZAPI_TARGET_URL \
- -e FUZZAPI_OVERRIDES_FILE \
- -e FUZZAPI_OVERRIDES_ENV \
- -e FUZZAPI_OVERRIDES_CMD \
- -e FUZZAPI_OVERRIDES_INTERVAL \
- -e FUZZAPI_TIMEOUT \
- -e FUZZAPI_VERBOSE \
- -e FUZZAPI_SERVICE_START_TIMEOUT \
- -e FUZZAPI_HTTP_USERNAME \
- -e FUZZAPI_HTTP_PASSWORD \
-e GITLAB_FEATURES \
- -v $CI_PROJECT_DIR:/app \
-p 80:80 \
-p 8000:8000 \
-p 514:514 \
--restart=no \
- registry.gitlab.com/gitlab-org/security-products/analyzers/api-fuzzing:${FUZZAPI_VERSION}-engine
+ $FUZZAPI_IMAGE \
+ dotnet /peach/Peach.Web.dll
#
# Start target container
- |
@@ -94,19 +143,31 @@ apifuzzer_fuzz:
$FUZZAPI_D_TARGET_IMAGE \
; fi
#
- # Start worker container
+ # Start worker container if provided
- |
if [ "$FUZZAPI_D_WORKER_IMAGE" != "" ]; then \
- echo "Starting worker image $FUZZAPI_D_WORKER_IMAGE" \
+ echo "Starting worker image $FUZZAPI_D_WORKER_IMAGE"; \
docker run \
--name worker \
--network $FUZZAPI_D_NETWORK \
-e FUZZAPI_API=http://apifuzzer:80 \
-e FUZZAPI_PROJECT \
-e FUZZAPI_PROFILE \
- -e FUZZAPI_AUTOMATION_CMD \
-e FUZZAPI_CONFIG \
-e FUZZAPI_REPORT \
+ -e FUZZAPI_HAR \
+ -e FUZZAPI_OPENAPI \
+ -e FUZZAPI_POSTMAN_COLLECTION \
+ -e FUZZAPI_TARGET_URL \
+ -e FUZZAPI_OVERRIDES_FILE \
+ -e FUZZAPI_OVERRIDES_ENV \
+ -e FUZZAPI_OVERRIDES_CMD \
+ -e FUZZAPI_OVERRIDES_INTERVAL \
+ -e FUZZAPI_TIMEOUT \
+ -e FUZZAPI_VERBOSE \
+ -e FUZZAPI_SERVICE_START_TIMEOUT \
+ -e FUZZAPI_HTTP_USERNAME \
+ -e FUZZAPI_HTTP_PASSWORD \
-e CI_COMMIT_BRANCH=${CI_COMMIT_BRANCH} \
$FUZZAPI_D_WORKER_ENV \
$FUZZAPI_D_WORKER_PORTS \
@@ -115,13 +176,49 @@ apifuzzer_fuzz:
$FUZZAPI_D_WORKER_IMAGE \
; fi
#
- # Wait for testing to complete if api fuzzer is scanning
- - if [ "$FUZZAPI_HAR$FUZZAPI_OPENAPI" != "" ]; then echo "Waiting for API Fuzzer to exit"; docker wait apifuzzer; fi
+ # Start API Fuzzing provided worker if no other worker present
+ - |
+ if [ "$FUZZAPI_D_WORKER_IMAGE" == "" ]; then \
+ if [ "$FUZZAPI_HAR$FUZZAPI_OPENAPI$FUZZAPI_POSTMAN_COLLECTION" == "" ]; then \
+ echo "Error: One of FUZZAPI_HAR, FUZZAPI_OPENAPI, or FUZZAPI_POSTMAN_COLLECTION must be provided."; \
+ echo "See https://docs.gitlab.com/ee/user/application_security/api_fuzzing/ for information on how to configure API Fuzzing."; \
+ exit 1; \
+ fi; \
+ docker run \
+ --name worker \
+ --network $FUZZAPI_D_NETWORK \
+ -e TZ=America/Los_Angeles \
+ -e FUZZAPI_API=http://apifuzzer:80 \
+ -e FUZZAPI_PROJECT \
+ -e FUZZAPI_PROFILE \
+ -e FUZZAPI_CONFIG \
+ -e FUZZAPI_REPORT \
+ -e FUZZAPI_HAR \
+ -e FUZZAPI_OPENAPI \
+ -e FUZZAPI_POSTMAN_COLLECTION \
+ -e FUZZAPI_TARGET_URL \
+ -e FUZZAPI_OVERRIDES_FILE \
+ -e FUZZAPI_OVERRIDES_ENV \
+ -e FUZZAPI_OVERRIDES_CMD \
+ -e FUZZAPI_OVERRIDES_INTERVAL \
+ -e FUZZAPI_TIMEOUT \
+ -e FUZZAPI_VERBOSE \
+ -e FUZZAPI_SERVICE_START_TIMEOUT \
+ -e FUZZAPI_HTTP_USERNAME \
+ -e FUZZAPI_HTTP_PASSWORD \
+ -v $CI_PROJECT_DIR:/app \
+ -p 81:80 \
+ -p 8001:8000 \
+ -p 515:514 \
+ --restart=no \
+ $FUZZAPI_IMAGE \
+ worker-entry \
+ ; fi
#
- # Propagate exit code from api fuzzer (if any)
- - if [[ $(docker inspect apifuzzer --format='{{.State.ExitCode}}') != "0" ]]; then echo "API Fuzzing exited with an error. Logs are available as job artifacts."; docker logs apifuzzer; exit 1; fi
+ # Propagate exit code from api fuzzing scanner (if any)
+ - if [[ $(docker inspect apifuzzer --format='{{.State.ExitCode}}') != "0" ]]; then echo "API Fuzzing scanner exited with an error. Logs are available as job artifacts."; exit 1; fi
#
- # Run user provided pre-script
+ # Run user provided post-script
- sh -c "$FUZZAPI_POST_SCRIPT"
#
after_script:
@@ -129,13 +226,13 @@ apifuzzer_fuzz:
# Shutdown all containers
- echo "Stopping all containers"
- if [ "$FUZZAPI_D_TARGET_IMAGE" != "" ]; then docker stop target; fi
- - if [ "$FUZZAPI_D_WORKER_IMAGE" != "" ]; then docker stop worker; fi
+ - docker stop worker
- docker stop apifuzzer
#
# Save docker logs
- docker logs apifuzzer &> gl-api_fuzzing-logs.log
- if [ "$FUZZAPI_D_TARGET_IMAGE" != "" ]; then docker logs target &> gl-api_fuzzing-target-logs.log; fi
- - if [ "$FUZZAPI_D_WORKER_IMAGE" != "" ]; then docker logs worker &> gl-api_fuzzing-worker-logs.log; fi
+ - docker logs worker &> gl-api_fuzzing-worker-logs.log
#
artifacts:
when: always
diff --git a/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml
index 21bcdd8d9b5..3cbde9d30c8 100644
--- a/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml
@@ -4,8 +4,7 @@ variables:
# Setting this variable will affect all Security templates
# (SAST, Dependency Scanning, ...)
SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers"
-
- CS_MAJOR_VERSION: 2
+ CS_MAJOR_VERSION: 3
container_scanning:
stage: test
diff --git a/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.gitlab-ci.yml
index e268b48d133..a1b6dc2cc1b 100644
--- a/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.gitlab-ci.yml
@@ -11,6 +11,14 @@ variables:
COVFUZZ_URL_PREFIX: "https://gitlab.com/gitlab-org/security-products/analyzers/gitlab-cov-fuzz/-/raw"
+coverage_fuzzing_unlicensed:
+ stage: test
+ allow_failure: true
+ rules:
+ - if: $GITLAB_FEATURES !~ /\bcoverage_fuzzing\b/ && $COVFUZZ_DISABLED == null
+ script:
+ - echo "ERROR Your GitLab project is missing licensing for Coverage Fuzzing" && exit 1
+
.fuzz_base:
stage: fuzz
allow_failure: true
diff --git a/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml
index 4418ff18d73..a51cb61da6d 100644
--- a/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml
@@ -134,6 +134,7 @@ mobsf-android-sast:
name: "$SAST_ANALYZER_IMAGE"
variables:
SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/mobsf:$SAST_ANALYZER_IMAGE_TAG"
+ MOBSF_API_KEY: key
rules:
- if: $SAST_DISABLED
when: never
@@ -152,6 +153,7 @@ mobsf-ios-sast:
name: "$SAST_ANALYZER_IMAGE"
variables:
SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/mobsf:$SAST_ANALYZER_IMAGE_TAG"
+ MOBSF_API_KEY: key
rules:
- if: $SAST_DISABLED
when: never
diff --git a/lib/gitlab/ci/templates/Security/Secure-Binaries.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Secure-Binaries.gitlab-ci.yml
index 2d2e0859373..232c320562b 100644
--- a/lib/gitlab/ci/templates/Security/Secure-Binaries.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Security/Secure-Binaries.gitlab-ci.yml
@@ -131,6 +131,8 @@ secrets:
variables:
- $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" &&
$SECURE_BINARIES_ANALYZERS =~ /\bsecrets\b/
+ variables:
+ SECURE_BINARIES_ANALYZER_VERSION: "3"
sobelow:
extends: .download_images
@@ -162,6 +164,8 @@ klar:
variables:
- $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" &&
$SECURE_BINARIES_ANALYZERS =~ /\bklar\b/
+ variables:
+ SECURE_BINARIES_ANALYZER_VERSION: "3"
clair-vulnerabilities-db:
extends: .download_images
diff --git a/lib/gitlab/ci/templates/Terraform.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Terraform.latest.gitlab-ci.yml
index b08ccf18b58..5963d7138c5 100644
--- a/lib/gitlab/ci/templates/Terraform.latest.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Terraform.latest.gitlab-ci.yml
@@ -1,11 +1,12 @@
include:
- - template: Terraform/Base.latest.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Terraform/Base.gitlab-ci.yml
+ - template: Terraform/Base.latest.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml
stages:
- init
- validate
- build
- deploy
+ - cleanup
init:
extends: .init
diff --git a/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml
index 000a1a7f580..e455bfac9de 100644
--- a/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml
@@ -12,9 +12,6 @@
image:
name: registry.gitlab.com/gitlab-org/terraform-images/stable:latest
-before_script:
- - cd ${TF_ROOT}
-
variables:
TF_ROOT: ${CI_PROJECT_DIR}
@@ -26,16 +23,19 @@ cache:
.init: &init
stage: init
script:
+ - cd ${TF_ROOT}
- gitlab-terraform init
.validate: &validate
stage: validate
script:
+ - cd ${TF_ROOT}
- gitlab-terraform validate
.build: &build
stage: build
script:
+ - cd ${TF_ROOT}
- gitlab-terraform plan
- gitlab-terraform plan-json
artifacts:
@@ -47,7 +47,14 @@ cache:
.deploy: &deploy
stage: deploy
script:
+ - cd ${TF_ROOT}
- gitlab-terraform apply
when: manual
only:
- master
+
+.destroy: &destroy
+ stage: cleanup
+ script:
+ - gitlab-terraform destroy
+ when: manual
diff --git a/lib/gitlab/ci/trace/chunked_io.rb b/lib/gitlab/ci/trace/chunked_io.rb
index e99889f4a25..6f3e4ccf48d 100644
--- a/lib/gitlab/ci/trace/chunked_io.rb
+++ b/lib/gitlab/ci/trace/chunked_io.rb
@@ -75,7 +75,7 @@ module Gitlab
until length <= 0 || eof?
data = chunk_slice_from_offset
- raise FailedToGetChunkError if data.empty?
+ raise FailedToGetChunkError if data.to_s.empty?
chunk_bytes = [CHUNK_SIZE - chunk_offset, length].min
chunk_data_slice = data.byteslice(0, chunk_bytes)
@@ -100,7 +100,7 @@ module Gitlab
until eof?
data = chunk_slice_from_offset
- raise FailedToGetChunkError if data.empty?
+ raise FailedToGetChunkError if data.to_s.empty?
new_line = data.index("\n")
diff --git a/lib/gitlab/conflict/file.rb b/lib/gitlab/conflict/file.rb
index 0ca99506311..4d7590a8e38 100644
--- a/lib/gitlab/conflict/file.rb
+++ b/lib/gitlab/conflict/file.rb
@@ -9,6 +9,11 @@ module Gitlab
CONTEXT_LINES = 3
+ CONFLICT_TYPES = {
+ "old" => "conflict_marker_their",
+ "new" => "conflict_marker_our"
+ }.freeze
+
attr_reader :merge_request
# 'raw' holds the Gitlab::Git::Conflict::File that this instance wraps
@@ -46,6 +51,34 @@ module Gitlab
end
end
+ def diff_lines_for_serializer
+ # calculate sections and highlight lines before changing types
+ sections && highlight_lines!
+
+ sections.flat_map do |section|
+ if section[:conflict]
+ lines = []
+
+ initial_type = nil
+ section[:lines].each do |line|
+ if line.type != initial_type
+ lines << create_separator_line(line)
+ initial_type = line.type
+ end
+
+ line.type = CONFLICT_TYPES[line.type]
+ lines << line
+ end
+
+ lines << create_separator_line(lines.last)
+
+ lines
+ else
+ section[:lines]
+ end
+ end
+ end
+
def sections
return @sections if @sections
@@ -93,9 +126,15 @@ module Gitlab
lines = tail_lines
elsif conflict_before
- # We're at the end of the file (no conflicts after), so just remove extra
- # trailing lines.
+ # We're at the end of the file (no conflicts after)
+ number_of_trailing_lines = lines.size
+
+ # Remove extra trailing lines
lines = lines.first(CONTEXT_LINES)
+
+ if number_of_trailing_lines > CONTEXT_LINES
+ lines << create_match_line(lines.last)
+ end
end
end
@@ -117,6 +156,10 @@ module Gitlab
Gitlab::Diff::Line.new('', 'match', line.index, line.old_pos, line.new_pos)
end
+ def create_separator_line(line)
+ Gitlab::Diff::Line.new('', 'conflict_marker', line.index, nil, nil)
+ end
+
# Any line beginning with a letter, an underscore, or a dollar can be used in a
# match line header. Only context sections can contain match lines, as match lines
# have to exist in both versions of the file.
diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb
index 2b08d3c63bb..d0579a44219 100644
--- a/lib/gitlab/current_settings.rb
+++ b/lib/gitlab/current_settings.rb
@@ -16,8 +16,8 @@ module Gitlab
@in_memory_application_settings = nil
end
- def method_missing(name, *args, &block)
- current_application_settings.send(name, *args, &block) # rubocop:disable GitlabSecurity/PublicSend
+ def method_missing(name, *args, **kwargs, &block)
+ current_application_settings.send(name, *args, **kwargs, &block) # rubocop:disable GitlabSecurity/PublicSend
end
def respond_to_missing?(name, include_private = false)
diff --git a/lib/gitlab/danger/commit_linter.rb b/lib/gitlab/danger/commit_linter.rb
index 7b01db125a9..2e469359bdc 100644
--- a/lib/gitlab/danger/commit_linter.rb
+++ b/lib/gitlab/danger/commit_linter.rb
@@ -191,7 +191,7 @@ module Gitlab
end
def subject_starts_with_lowercase?
- first_char = subject.sub(/\A\[.+\]\s/, '')[0]
+ first_char = subject.sub(/\A(\[.+\]|\w+:)\s/, '')[0]
first_char_downcased = first_char.downcase
return true unless ('a'..'z').cover?(first_char_downcased)
diff --git a/lib/gitlab/danger/helper.rb b/lib/gitlab/danger/helper.rb
index 783a5f1715c..89f21e8bd23 100644
--- a/lib/gitlab/danger/helper.rb
+++ b/lib/gitlab/danger/helper.rb
@@ -168,7 +168,7 @@ module Gitlab
%r{\A(\.gitlab-ci\.yml\z|\.gitlab\/ci)} => :engineering_productivity,
%r{\A\.codeclimate\.yml\z} => :engineering_productivity,
- %r{\A\.overcommit\.yml\.example\z} => :engineering_productivity,
+ %r{\Alefthook.yml\z} => :engineering_productivity,
%r{\A\.editorconfig\z} => :engineering_productivity,
%r{Dangerfile\z} => :engineering_productivity,
%r{\A(ee/)?(danger/|lib/gitlab/danger/)} => :engineering_productivity,
@@ -190,7 +190,7 @@ module Gitlab
%r{\A(ee/)?vendor/} => :backend,
%r{\A(Gemfile|Gemfile.lock|Rakefile)\z} => :backend,
%r{\A[A-Z_]+_VERSION\z} => :backend,
- %r{\A\.rubocop(_todo)?\.yml\z} => :backend,
+ %r{\A\.rubocop((_manual)?_todo)?\.yml\z} => :backend,
%r{\Afile_hooks/} => :backend,
%r{\A(ee/)?qa/} => :qa,
@@ -200,6 +200,9 @@ module Gitlab
%r{\Alocale/gitlab\.pot\z} => :none,
%r{\Adata/whats_new/} => :none,
+ # GraphQL auto generated doc files and schema
+ %r{\Adoc/api/graphql/reference/} => :backend,
+
# Fallbacks in case the above patterns miss anything
%r{\.rb\z} => :backend,
%r{(
diff --git a/lib/gitlab/data_builder/feature_flag.rb b/lib/gitlab/data_builder/feature_flag.rb
new file mode 100644
index 00000000000..2f675ace7e1
--- /dev/null
+++ b/lib/gitlab/data_builder/feature_flag.rb
@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module DataBuilder
+ module FeatureFlag
+ extend self
+
+ def build(feature_flag, user)
+ {
+ object_kind: 'feature_flag',
+ project: feature_flag.project.hook_attrs,
+ user: user.hook_attrs,
+ user_url: Gitlab::UrlBuilder.build(user),
+ object_attributes: feature_flag.hook_attrs
+ }
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/database/batch_count.rb b/lib/gitlab/database/batch_count.rb
index 11d9881aac2..6f79e965cd5 100644
--- a/lib/gitlab/database/batch_count.rb
+++ b/lib/gitlab/database/batch_count.rb
@@ -128,9 +128,9 @@ module Gitlab
end
def between_condition(start, finish)
- return @column.between(start..(finish - 1)) if @column.is_a?(Arel::Attributes::Attribute)
+ return @column.between(start...finish) if @column.is_a?(Arel::Attributes::Attribute)
- { @column => start..(finish - 1) }
+ { @column => start...finish }
end
def actual_start(start)
diff --git a/lib/gitlab/database/partitioning/monthly_strategy.rb b/lib/gitlab/database/partitioning/monthly_strategy.rb
index ecc05d9654a..82ea1ce26fb 100644
--- a/lib/gitlab/database/partitioning/monthly_strategy.rb
+++ b/lib/gitlab/database/partitioning/monthly_strategy.rb
@@ -17,23 +17,8 @@ module Gitlab
end
def current_partitions
- result = connection.select_all(<<~SQL)
- select
- pg_class.relname,
- parent_class.relname as base_table,
- pg_get_expr(pg_class.relpartbound, inhrelid) as condition
- from pg_class
- inner join pg_inherits i on pg_class.oid = inhrelid
- inner join pg_class parent_class on parent_class.oid = inhparent
- inner join pg_namespace ON pg_namespace.oid = pg_class.relnamespace
- where pg_namespace.nspname = #{connection.quote(Gitlab::Database::DYNAMIC_PARTITIONS_SCHEMA)}
- and parent_class.relname = #{connection.quote(table_name)}
- and pg_class.relispartition
- order by pg_class.relname
- SQL
-
- result.map do |record|
- TimePartition.from_sql(table_name, record['relname'], record['condition'])
+ Gitlab::Database::PostgresPartition.for_parent_table(table_name).map do |partition|
+ TimePartition.from_sql(table_name, partition.name, partition.condition)
end
end
diff --git a/lib/gitlab/database/partitioning/replace_table.rb b/lib/gitlab/database/partitioning/replace_table.rb
new file mode 100644
index 00000000000..6f6af223fa2
--- /dev/null
+++ b/lib/gitlab/database/partitioning/replace_table.rb
@@ -0,0 +1,114 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Database
+ module Partitioning
+ class ReplaceTable
+ DELIMITER = ";\n\n"
+
+ attr_reader :original_table, :replacement_table, :replaced_table, :primary_key_column,
+ :sequence, :original_primary_key, :replacement_primary_key, :replaced_primary_key
+
+ def initialize(original_table, replacement_table, replaced_table, primary_key_column)
+ @original_table = original_table
+ @replacement_table = replacement_table
+ @replaced_table = replaced_table
+ @primary_key_column = primary_key_column
+
+ @sequence = default_sequence(original_table, primary_key_column)
+ @original_primary_key = default_primary_key(original_table)
+ @replacement_primary_key = default_primary_key(replacement_table)
+ @replaced_primary_key = default_primary_key(replaced_table)
+ end
+
+ def perform
+ yield sql_to_replace_table if block_given?
+
+ execute(sql_to_replace_table)
+ end
+
+ private
+
+ delegate :execute, :quote_table_name, :quote_column_name, to: :connection
+ def connection
+ @connection ||= ActiveRecord::Base.connection
+ end
+
+ def default_sequence(table, column)
+ "#{table}_#{column}_seq"
+ end
+
+ def default_primary_key(table)
+ "#{table}_pkey"
+ end
+
+ def sql_to_replace_table
+ @sql_to_replace_table ||= combined_sql_statements.map(&:chomp).join(DELIMITER)
+ end
+
+ def combined_sql_statements
+ statements = []
+
+ statements << alter_column_default(original_table, primary_key_column, expression: nil)
+ statements << alter_column_default(replacement_table, primary_key_column,
+ expression: "nextval('#{quote_table_name(sequence)}'::regclass)")
+
+ statements << alter_sequence_owned_by(sequence, replacement_table, primary_key_column)
+
+ rename_table_objects(statements, original_table, replaced_table, original_primary_key, replaced_primary_key)
+ rename_table_objects(statements, replacement_table, original_table, replacement_primary_key, original_primary_key)
+
+ statements
+ end
+
+ def rename_table_objects(statements, old_table, new_table, old_primary_key, new_primary_key)
+ statements << rename_table(old_table, new_table)
+ statements << rename_constraint(new_table, old_primary_key, new_primary_key)
+
+ rename_partitions(statements, old_table, new_table)
+ end
+
+ def rename_partitions(statements, old_table_name, new_table_name)
+ Gitlab::Database::PostgresPartition.for_parent_table(old_table_name).each do |partition|
+ new_partition_name = partition.name.sub(/#{old_table_name}/, new_table_name)
+ old_primary_key = default_primary_key(partition.name)
+ new_primary_key = default_primary_key(new_partition_name)
+
+ statements << rename_constraint(partition.identifier, old_primary_key, new_primary_key)
+ statements << rename_table(partition.identifier, new_partition_name)
+ end
+ end
+
+ def alter_column_default(table_name, column_name, expression:)
+ default_clause = expression.nil? ? 'DROP DEFAULT' : "SET DEFAULT #{expression}"
+
+ <<~SQL
+ ALTER TABLE #{quote_table_name(table_name)}
+ ALTER COLUMN #{quote_column_name(column_name)} #{default_clause}
+ SQL
+ end
+
+ def alter_sequence_owned_by(sequence_name, table_name, column_name)
+ <<~SQL
+ ALTER SEQUENCE #{quote_table_name(sequence_name)}
+ OWNED BY #{quote_table_name(table_name)}.#{quote_column_name(column_name)}
+ SQL
+ end
+
+ def rename_table(old_name, new_name)
+ <<~SQL
+ ALTER TABLE #{quote_table_name(old_name)}
+ RENAME TO #{quote_table_name(new_name)}
+ SQL
+ end
+
+ def rename_constraint(table_name, old_name, new_name)
+ <<~SQL
+ ALTER TABLE #{quote_table_name(table_name)}
+ RENAME CONSTRAINT #{quote_column_name(old_name)} TO #{quote_column_name(new_name)}
+ SQL
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/database/partitioning_migration_helpers.rb b/lib/gitlab/database/partitioning_migration_helpers.rb
index 881177a195e..3196dd20356 100644
--- a/lib/gitlab/database/partitioning_migration_helpers.rb
+++ b/lib/gitlab/database/partitioning_migration_helpers.rb
@@ -5,6 +5,7 @@ module Gitlab
module PartitioningMigrationHelpers
include ForeignKeyHelpers
include TableManagementHelpers
+ include IndexHelpers
end
end
end
diff --git a/lib/gitlab/database/partitioning_migration_helpers/index_helpers.rb b/lib/gitlab/database/partitioning_migration_helpers/index_helpers.rb
new file mode 100644
index 00000000000..f367292f4b0
--- /dev/null
+++ b/lib/gitlab/database/partitioning_migration_helpers/index_helpers.rb
@@ -0,0 +1,90 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Database
+ module PartitioningMigrationHelpers
+ module IndexHelpers
+ include Gitlab::Database::MigrationHelpers
+ include Gitlab::Database::SchemaHelpers
+
+ # Concurrently creates a new index on a partitioned table. In concept this works similarly to
+ # `add_concurrent_index`, and won't block reads or writes on the table while the index is being built.
+ #
+ # A special helper is required for partitioning because Postgres does not support concurrently building indexes
+ # on partitioned tables. This helper concurrently adds the same index to each partition, and creates the final
+ # index on the parent table once all of the partitions are indexed. This is the recommended safe way to add
+ # indexes to partitioned tables.
+ #
+ # Example:
+ #
+ # add_concurrent_partitioned_index :users, :some_column
+ #
+ # See Rails' `add_index` for more info on the available arguments.
+ def add_concurrent_partitioned_index(table_name, column_names, options = {})
+ raise ArgumentError, 'A name is required for indexes added to partitioned tables' unless options[:name]
+
+ partitioned_table = find_partitioned_table(table_name)
+
+ if index_name_exists?(table_name, options[:name])
+ Gitlab::AppLogger.warn "Index not created because it already exists (this may be due to an aborted" \
+ " migration or similar): table_name: #{table_name}, index_name: #{options[:name]}"
+
+ return
+ end
+
+ partitioned_table.postgres_partitions.each do |partition|
+ partition_index_name = generated_index_name(partition.identifier, options[:name])
+ partition_options = options.merge(name: partition_index_name)
+
+ add_concurrent_index(partition.identifier, column_names, partition_options)
+ end
+
+ with_lock_retries do
+ add_index(table_name, column_names, options)
+ end
+ end
+
+ # Safely removes an existing index from a partitioned table. The method name is a bit inaccurate as it does not
+ # drop the index concurrently, but it's named as such to maintain consistency with other similar helpers, and
+ # indicate that this should be safe to use in a production environment.
+ #
+ # In current versions of Postgres it's impossible to drop an index concurrently, or drop an index from an
+ # individual partition that exists across the entire partitioned table. As a result this helper drops the index
+ # from the parent table, which automatically cascades to all partitions. While this does require an exclusive
+ # lock, dropping an index is a fast operation that won't block the table for a significant period of time.
+ #
+ # Example:
+ #
+ # remove_concurrent_partitioned_index_by_name :users, 'index_name_goes_here'
+ def remove_concurrent_partitioned_index_by_name(table_name, index_name)
+ find_partitioned_table(table_name)
+
+ unless index_name_exists?(table_name, index_name)
+ Gitlab::AppLogger.warn "Index not removed because it does not exist (this may be due to an aborted " \
+ "migration or similar): table_name: #{table_name}, index_name: #{index_name}"
+
+ return
+ end
+
+ with_lock_retries do
+ remove_index(table_name, name: index_name)
+ end
+ end
+
+ private
+
+ def find_partitioned_table(table_name)
+ partitioned_table = Gitlab::Database::PostgresPartitionedTable.find_by_name_in_current_schema(table_name)
+
+ raise ArgumentError, "#{table_name} is not a partitioned table" unless partitioned_table
+
+ partitioned_table
+ end
+
+ def generated_index_name(partition_name, index_name)
+ object_name("#{partition_name}_#{index_name}", 'index')
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb b/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb
index f7b0306b769..686dda80207 100644
--- a/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb
+++ b/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb
@@ -66,7 +66,10 @@ module Gitlab
create_range_partitioned_copy(table_name, partitioned_table_name, partition_column, primary_key)
create_daterange_partitions(partitioned_table_name, partition_column.name, min_date, max_date)
end
- create_trigger_to_sync_tables(table_name, partitioned_table_name, primary_key)
+
+ with_lock_retries do
+ create_trigger_to_sync_tables(table_name, partitioned_table_name, primary_key)
+ end
end
# Clean up a partitioned copy of an existing table. First, deletes the database function and trigger that were
@@ -81,13 +84,9 @@ module Gitlab
assert_not_in_transaction_block(scope: ERROR_SCOPE)
with_lock_retries do
- trigger_name = make_sync_trigger_name(table_name)
- drop_trigger(table_name, trigger_name)
+ drop_sync_trigger(table_name)
end
- function_name = make_sync_function_name(table_name)
- drop_function(function_name)
-
partitioned_table_name = make_partitioned_table_name(table_name)
drop_table(partitioned_table_name)
end
@@ -177,6 +176,53 @@ module Gitlab
end
end
+ # Replaces a non-partitioned table with its partitioned copy. This is the final step in a partitioning
+ # migration, which makes the partitioned table ready for use by the application. The partitioned copy should be
+ # replaced with the original table in such a way that it appears seamless to any database clients. The replaced
+ # table will be renamed to "#{replaced_table}_archived". Partitions and primary key constraints will also be
+ # renamed to match the naming scheme of the parent table.
+ #
+ # **NOTE** This method should only be used after all other migration steps have completed successfully.
+ # There are several limitations to this method that MUST be handled before, or during, the swap migration:
+ #
+ # - Secondary indexes and foreign keys are not automatically recreated on the partitioned table.
+ # - Some types of constraints (UNIQUE and EXCLUDE) which rely on indexes, will not automatically be recreated
+ # on the partitioned table, since the underlying index will not be present.
+ # - Foreign keys referencing the original non-partitioned table, would also need to be updated to reference the
+ # partitioned table, but unfortunately this is not supported in PG11.
+ # - Views referencing the original table will not be automatically updated to reference the partitioned table.
+ #
+ # Example:
+ #
+ # replace_with_partitioned_table :audit_events
+ #
+ def replace_with_partitioned_table(table_name)
+ assert_table_is_allowed(table_name)
+
+ partitioned_table_name = make_partitioned_table_name(table_name)
+ archived_table_name = make_archived_table_name(table_name)
+ primary_key_name = connection.primary_key(table_name)
+
+ replace_table(table_name, partitioned_table_name, archived_table_name, primary_key_name)
+ end
+
+ # Rolls back a migration that replaced a non-partitioned table with its partitioned copy. This can be used to
+ # restore the original non-partitioned table in the event of an unexpected issue.
+ #
+ # Example:
+ #
+ # rollback_replace_with_partitioned_table :audit_events
+ #
+ def rollback_replace_with_partitioned_table(table_name)
+ assert_table_is_allowed(table_name)
+
+ partitioned_table_name = make_partitioned_table_name(table_name)
+ archived_table_name = make_archived_table_name(table_name)
+ primary_key_name = connection.primary_key(archived_table_name)
+
+ replace_table(table_name, archived_table_name, partitioned_table_name, primary_key_name)
+ end
+
private
def assert_table_is_allowed(table_name)
@@ -190,6 +236,10 @@ module Gitlab
tmp_table_name("#{table}_part")
end
+ def make_archived_table_name(table)
+ "#{table}_archived"
+ end
+
def make_sync_function_name(table)
object_name(table, 'table_sync_function')
end
@@ -270,12 +320,18 @@ module Gitlab
function_name = make_sync_function_name(source_table_name)
trigger_name = make_sync_trigger_name(source_table_name)
- with_lock_retries do
- create_sync_function(function_name, partitioned_table_name, unique_key)
- create_comment('FUNCTION', function_name, "Partitioning migration: table sync for #{source_table_name} table")
+ create_sync_function(function_name, partitioned_table_name, unique_key)
+ create_comment('FUNCTION', function_name, "Partitioning migration: table sync for #{source_table_name} table")
- create_sync_trigger(source_table_name, trigger_name, function_name)
- end
+ create_sync_trigger(source_table_name, trigger_name, function_name)
+ end
+
+ def drop_sync_trigger(source_table_name)
+ trigger_name = make_sync_trigger_name(source_table_name)
+ drop_trigger(source_table_name, trigger_name)
+
+ function_name = make_sync_function_name(source_table_name)
+ drop_function(function_name)
end
def create_sync_function(name, partitioned_table_name, unique_key)
@@ -358,6 +414,21 @@ module Gitlab
end
end
end
+
+ def replace_table(original_table_name, replacement_table_name, replaced_table_name, primary_key_name)
+ replace_table = Gitlab::Database::Partitioning::ReplaceTable.new(original_table_name.to_s,
+ replacement_table_name, replaced_table_name, primary_key_name)
+
+ with_lock_retries do
+ drop_sync_trigger(original_table_name)
+
+ replace_table.perform do |sql|
+ say("replace_table(\"#{sql}\")")
+ end
+
+ create_trigger_to_sync_tables(original_table_name, replaced_table_name, primary_key_name)
+ end
+ end
end
end
end
diff --git a/lib/gitlab/database/postgres_partition.rb b/lib/gitlab/database/postgres_partition.rb
new file mode 100644
index 00000000000..0986372586b
--- /dev/null
+++ b/lib/gitlab/database/postgres_partition.rb
@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Database
+ class PostgresPartition < ActiveRecord::Base
+ self.primary_key = :identifier
+
+ belongs_to :postgres_partitioned_table, foreign_key: 'parent_identifier', primary_key: 'identifier'
+
+ scope :by_identifier, ->(identifier) do
+ raise ArgumentError, "Partition name is not fully qualified with a schema: #{identifier}" unless identifier =~ /^\w+\.\w+$/
+
+ find(identifier)
+ end
+
+ scope :for_parent_table, ->(name) { where("parent_identifier = concat(current_schema(), '.', ?)", name).order(:name) }
+
+ def to_s
+ name
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/database/postgres_partitioned_table.rb b/lib/gitlab/database/postgres_partitioned_table.rb
new file mode 100644
index 00000000000..5d2eaa22ee4
--- /dev/null
+++ b/lib/gitlab/database/postgres_partitioned_table.rb
@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Database
+ class PostgresPartitionedTable < ActiveRecord::Base
+ DYNAMIC_PARTITION_STRATEGIES = %w[range list].freeze
+
+ self.primary_key = :identifier
+
+ has_many :postgres_partitions, foreign_key: 'parent_identifier', primary_key: 'identifier'
+
+ scope :by_identifier, ->(identifier) do
+ raise ArgumentError, "Table name is not fully qualified with a schema: #{identifier}" unless identifier =~ /^\w+\.\w+$/
+
+ find(identifier)
+ end
+
+ def self.find_by_name_in_current_schema(name)
+ find_by("identifier = concat(current_schema(), '.', ?)", name)
+ end
+
+ def dynamic?
+ DYNAMIC_PARTITION_STRATEGIES.include?(strategy)
+ end
+
+ def static?
+ !dynamic?
+ end
+
+ def to_s
+ name
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/database/reindexing.rb b/lib/gitlab/database/reindexing.rb
index 074752fe75b..c77e000254f 100644
--- a/lib/gitlab/database/reindexing.rb
+++ b/lib/gitlab/database/reindexing.rb
@@ -10,6 +10,7 @@ module Gitlab
def self.candidate_indexes
Gitlab::Database::PostgresIndex
.regular
+ .where('NOT expression')
.not_match("^#{ConcurrentReindex::TEMPORARY_INDEX_PREFIX}")
.not_match("^#{ConcurrentReindex::REPLACED_INDEX_PREFIX}")
end
diff --git a/lib/gitlab/dependency_linker/base_linker.rb b/lib/gitlab/dependency_linker/base_linker.rb
index a4e265eba88..d735fb55652 100644
--- a/lib/gitlab/dependency_linker/base_linker.rb
+++ b/lib/gitlab/dependency_linker/base_linker.rb
@@ -6,6 +6,7 @@ module Gitlab
URL_REGEX = %r{https?://[^'" ]+}.freeze
GIT_INVALID_URL_REGEX = /^git\+#{URL_REGEX}/.freeze
REPO_REGEX = %r{[^/'" ]+/[^/'" ]+}.freeze
+ VALID_LINK_ATTRIBUTES = %w[href rel target].freeze
include ActionView::Helpers::SanitizeHelper
@@ -66,7 +67,7 @@ module Gitlab
def link_tag(name, url)
sanitize(
%{<a href="#{ERB::Util.html_escape_once(url)}" rel="nofollow noreferrer noopener" target="_blank">#{ERB::Util.html_escape_once(name)}</a>},
- attributes: %w[href rel target]
+ attributes: VALID_LINK_ATTRIBUTES
)
end
@@ -77,7 +78,7 @@ module Gitlab
# # Will link `user/repo` in `github: "user/repo"` or `:github => "user/repo"`
def link_regex(regex, &url_proc)
highlighted_lines.map!.with_index do |rich_line, i|
- marker = StringRegexMarker.new(plain_lines[i].chomp, rich_line.html_safe)
+ marker = StringRegexMarker.new((plain_lines[i].chomp! || plain_lines[i]), rich_line.html_safe)
marker.mark(regex, group: :name) do |text, left:, right:|
url = yield(text)
diff --git a/lib/gitlab/design_management/copy_design_collection_model_attributes.yml b/lib/gitlab/design_management/copy_design_collection_model_attributes.yml
index 1d341e6520e..95f15bd6dee 100644
--- a/lib/gitlab/design_management/copy_design_collection_model_attributes.yml
+++ b/lib/gitlab/design_management/copy_design_collection_model_attributes.yml
@@ -29,6 +29,7 @@ ignore_design_attributes:
- id
- issue_id
- project_id
+ - iid
ignore_version_attributes:
- id
diff --git a/lib/gitlab/diff/file_collection/merge_request_diff_batch.rb b/lib/gitlab/diff/file_collection/merge_request_diff_batch.rb
index c6d1e0b93a7..9af66318b89 100644
--- a/lib/gitlab/diff/file_collection/merge_request_diff_batch.rb
+++ b/lib/gitlab/diff/file_collection/merge_request_diff_batch.rb
@@ -18,10 +18,8 @@ module Gitlab
def initialize(merge_request_diff, batch_page, batch_size, diff_options:)
super(merge_request_diff, diff_options: diff_options)
- batch_page ||= DEFAULT_BATCH_PAGE
- batch_size ||= DEFAULT_BATCH_SIZE
+ @paginated_collection = load_paginated_collection(batch_page, batch_size, diff_options)
- @paginated_collection = relation.page(batch_page).per(batch_size)
@pagination_data = {
current_page: @paginated_collection.current_page,
next_page: @paginated_collection.next_page,
@@ -63,6 +61,18 @@ module Gitlab
def relation
@merge_request_diff.merge_request_diff_files
end
+
+ def load_paginated_collection(batch_page, batch_size, diff_options)
+ batch_page ||= DEFAULT_BATCH_PAGE
+ batch_size ||= DEFAULT_BATCH_SIZE
+
+ paths = diff_options&.fetch(:paths, nil)
+
+ paginated_collection = relation.page(batch_page).per(batch_size)
+ paginated_collection = paginated_collection.by_paths(paths) if paths
+
+ paginated_collection
+ end
end
end
end
diff --git a/lib/gitlab/diff/line.rb b/lib/gitlab/diff/line.rb
index 379fc6af875..af9140215f0 100644
--- a/lib/gitlab/diff/line.rb
+++ b/lib/gitlab/diff/line.rb
@@ -8,9 +8,9 @@ module Gitlab
#
SERIALIZE_KEYS = %i(line_code rich_text text type index old_pos new_pos).freeze
- attr_reader :line_code, :type, :old_pos, :new_pos
+ attr_reader :line_code, :old_pos, :new_pos
attr_writer :rich_text
- attr_accessor :text, :index
+ attr_accessor :text, :index, :type
def initialize(text, type, index, old_pos, new_pos, parent_file: nil, line_code: nil, rich_text: nil)
@text, @type, @index = text, type, index
diff --git a/lib/gitlab/error_tracking.rb b/lib/gitlab/error_tracking.rb
index 803acef9a40..a5ace2be773 100644
--- a/lib/gitlab/error_tracking.rb
+++ b/lib/gitlab/error_tracking.rb
@@ -123,6 +123,7 @@ module Gitlab
end
extra = sanitize_request_parameters(extra)
+ inject_sql_query_into_extra(exception, extra)
if sentry && Raven.configuration.server
Raven.capture_exception(exception, tags: default_tags, extra: extra)
@@ -149,6 +150,12 @@ module Gitlab
filter.filter(parameters)
end
+ def inject_sql_query_into_extra(exception, extra)
+ return unless exception.is_a?(ActiveRecord::StatementInvalid)
+
+ extra[:sql] = PgQuery.normalize(exception.sql.to_s)
+ end
+
def sentry_dsn
return unless Rails.env.production? || Rails.env.development?
return unless Gitlab.config.sentry.enabled
diff --git a/lib/gitlab/etag_caching/middleware.rb b/lib/gitlab/etag_caching/middleware.rb
index 303e1a23e6b..fc3c05c57b2 100644
--- a/lib/gitlab/etag_caching/middleware.rb
+++ b/lib/gitlab/etag_caching/middleware.rb
@@ -3,6 +3,14 @@
module Gitlab
module EtagCaching
class Middleware
+ SKIP_HEADER_KEY = 'X-Gitlab-Skip-Etag'
+
+ class << self
+ def skip!(response)
+ response.set_header(SKIP_HEADER_KEY, '1')
+ end
+ end
+
def initialize(app)
@app = app
end
@@ -22,9 +30,7 @@ module Gitlab
else
track_cache_miss(if_none_match, cached_value_present, route)
- status, headers, body = @app.call(env)
- headers['ETag'] = etag
- [status, headers, body]
+ maybe_apply_etag(etag, *@app.call(env))
end
end
@@ -43,6 +49,13 @@ module Gitlab
[weak_etag_format(current_value), cached_value_present]
end
+ def maybe_apply_etag(etag, status, headers, body)
+ headers['ETag'] = etag unless
+ Gitlab::Utils.to_boolean(headers.delete(SKIP_HEADER_KEY))
+
+ [status, headers, body]
+ end
+
def weak_etag_format(value)
%Q{W/"#{value}"}
end
@@ -54,7 +67,13 @@ module Gitlab
add_instrument_for_cache_hit(status_code, route, request)
- [status_code, { 'ETag' => etag, 'X-Gitlab-From-Cache' => 'true' }, []]
+ new_headers = {
+ 'ETag' => etag,
+ 'X-Gitlab-From-Cache' => 'true',
+ ::Gitlab::Metrics::RequestsRackMiddleware::FEATURE_CATEGORY_HEADER => route.feature_category
+ }
+
+ [status_code, new_headers, []]
end
def track_cache_miss(if_none_match, cached_value_present, route)
diff --git a/lib/gitlab/etag_caching/router.rb b/lib/gitlab/etag_caching/router.rb
index 17d9cf08367..769ac2784d1 100644
--- a/lib/gitlab/etag_caching/router.rb
+++ b/lib/gitlab/etag_caching/router.rb
@@ -3,7 +3,7 @@
module Gitlab
module EtagCaching
class Router
- Route = Struct.new(:regexp, :name)
+ Route = Struct.new(:regexp, :name, :feature_category)
# We enable an ETag for every request matching the regex.
# To match a regex the path needs to match the following:
# - Don't contain a reserved word (expect for the words used in the
@@ -20,59 +20,73 @@ module Gitlab
ROUTES = [
Gitlab::EtagCaching::Router::Route.new(
%r(#{RESERVED_WORDS_PREFIX}/noteable/issue/\d+/notes\z),
- 'issue_notes'
+ 'issue_notes',
+ 'issue_tracking'
),
Gitlab::EtagCaching::Router::Route.new(
%r(#{RESERVED_WORDS_PREFIX}/noteable/merge_request/\d+/notes\z),
- 'merge_request_notes'
+ 'merge_request_notes',
+ 'code_review'
),
Gitlab::EtagCaching::Router::Route.new(
%r(#{RESERVED_WORDS_PREFIX}/issues/\d+/realtime_changes\z),
- 'issue_title'
+ 'issue_title',
+ 'issue_tracking'
),
Gitlab::EtagCaching::Router::Route.new(
%r(#{RESERVED_WORDS_PREFIX}/commit/\S+/pipelines\.json\z),
- 'commit_pipelines'
+ 'commit_pipelines',
+ 'continuous_integration'
),
Gitlab::EtagCaching::Router::Route.new(
%r(#{RESERVED_WORDS_PREFIX}/merge_requests/new\.json\z),
- 'new_merge_request_pipelines'
+ 'new_merge_request_pipelines',
+ 'continuous_integration'
),
Gitlab::EtagCaching::Router::Route.new(
%r(#{RESERVED_WORDS_PREFIX}/merge_requests/\d+/pipelines\.json\z),
- 'merge_request_pipelines'
+ 'merge_request_pipelines',
+ 'continuous_integration'
),
Gitlab::EtagCaching::Router::Route.new(
%r(#{RESERVED_WORDS_PREFIX}/pipelines\.json\z),
- 'project_pipelines'
+ 'project_pipelines',
+ 'continuous_integration'
),
Gitlab::EtagCaching::Router::Route.new(
%r(#{RESERVED_WORDS_PREFIX}/pipelines/\d+\.json\z),
- 'project_pipeline'
+ 'project_pipeline',
+ 'continuous_integration'
),
Gitlab::EtagCaching::Router::Route.new(
%r(#{RESERVED_WORDS_PREFIX}/builds/\d+\.json\z),
- 'project_build'
+ 'project_build',
+ 'continuous_integration'
),
Gitlab::EtagCaching::Router::Route.new(
%r(#{RESERVED_WORDS_PREFIX}/clusters/\d+/environments\z),
- 'cluster_environments'
+ 'cluster_environments',
+ 'continuous_delivery'
),
Gitlab::EtagCaching::Router::Route.new(
%r(#{RESERVED_WORDS_PREFIX}/environments\.json\z),
- 'environments'
+ 'environments',
+ 'continuous_delivery'
),
Gitlab::EtagCaching::Router::Route.new(
%r(#{RESERVED_WORDS_PREFIX}/import/github/realtime_changes\.json\z),
- 'realtime_changes_import_github'
+ 'realtime_changes_import_github',
+ 'importers'
),
Gitlab::EtagCaching::Router::Route.new(
%r(#{RESERVED_WORDS_PREFIX}/import/gitea/realtime_changes\.json\z),
- 'realtime_changes_import_gitea'
+ 'realtime_changes_import_gitea',
+ 'importers'
),
Gitlab::EtagCaching::Router::Route.new(
%r(#{RESERVED_WORDS_PREFIX}/merge_requests/\d+/cached_widget\.json\z),
- 'merge_request_widget'
+ 'merge_request_widget',
+ 'code_review'
)
].freeze
diff --git a/lib/gitlab/experimentation.rb b/lib/gitlab/experimentation.rb
index 1ce3ffe4c86..6e39776bbd4 100644
--- a/lib/gitlab/experimentation.rb
+++ b/lib/gitlab/experimentation.rb
@@ -6,6 +6,7 @@
# Experiment options:
# - environment (optional, defaults to enabled for development and GitLab.com)
# - tracking_category (optional, used to set the category when tracking an experiment event)
+# - use_backwards_compatible_subject_index (optional, set this to true if you need backwards compatibility)
#
# The experiment is controlled by a Feature Flag (https://docs.gitlab.com/ee/development/feature_flags/controls.html),
# which is named "#{experiment_key}_experiment_percentage" and *must* be set with a percentage and not be used for other purposes.
@@ -30,168 +31,60 @@
module Gitlab
module Experimentation
EXPERIMENTS = {
- signup_flow: {
- tracking_category: 'Growth::Acquisition::Experiment::SignUpFlow'
- },
onboarding_issues: {
- tracking_category: 'Growth::Conversion::Experiment::OnboardingIssues'
- },
- suggest_pipeline: {
- tracking_category: 'Growth::Expansion::Experiment::SuggestPipeline'
+ tracking_category: 'Growth::Conversion::Experiment::OnboardingIssues',
+ use_backwards_compatible_subject_index: true
},
ci_notification_dot: {
- tracking_category: 'Growth::Expansion::Experiment::CiNotificationDot'
+ tracking_category: 'Growth::Expansion::Experiment::CiNotificationDot',
+ use_backwards_compatible_subject_index: true
},
upgrade_link_in_user_menu_a: {
- tracking_category: 'Growth::Expansion::Experiment::UpgradeLinkInUserMenuA'
+ tracking_category: 'Growth::Expansion::Experiment::UpgradeLinkInUserMenuA',
+ use_backwards_compatible_subject_index: true
},
invite_members_version_a: {
- tracking_category: 'Growth::Expansion::Experiment::InviteMembersVersionA'
+ tracking_category: 'Growth::Expansion::Experiment::InviteMembersVersionA',
+ use_backwards_compatible_subject_index: true
},
invite_members_version_b: {
- tracking_category: 'Growth::Expansion::Experiment::InviteMembersVersionB'
+ tracking_category: 'Growth::Expansion::Experiment::InviteMembersVersionB',
+ use_backwards_compatible_subject_index: true
+ },
+ invite_members_empty_group_version_a: {
+ tracking_category: 'Growth::Expansion::Experiment::InviteMembersEmptyGroupVersionA',
+ use_backwards_compatible_subject_index: true
},
new_create_project_ui: {
- tracking_category: 'Manage::Import::Experiment::NewCreateProjectUi'
+ tracking_category: 'Manage::Import::Experiment::NewCreateProjectUi',
+ use_backwards_compatible_subject_index: true
},
contact_sales_btn_in_app: {
- tracking_category: 'Growth::Conversion::Experiment::ContactSalesInApp'
+ tracking_category: 'Growth::Conversion::Experiment::ContactSalesInApp',
+ use_backwards_compatible_subject_index: true
},
customize_homepage: {
- tracking_category: 'Growth::Expansion::Experiment::CustomizeHomepage'
+ tracking_category: 'Growth::Expansion::Experiment::CustomizeHomepage',
+ use_backwards_compatible_subject_index: true
},
invite_email: {
- tracking_category: 'Growth::Acquisition::Experiment::InviteEmail'
+ tracking_category: 'Growth::Acquisition::Experiment::InviteEmail',
+ use_backwards_compatible_subject_index: true
},
invitation_reminders: {
- tracking_category: 'Growth::Acquisition::Experiment::InvitationReminders'
+ tracking_category: 'Growth::Acquisition::Experiment::InvitationReminders',
+ use_backwards_compatible_subject_index: true
},
group_only_trials: {
- tracking_category: 'Growth::Conversion::Experiment::GroupOnlyTrials'
+ tracking_category: 'Growth::Conversion::Experiment::GroupOnlyTrials',
+ use_backwards_compatible_subject_index: true
},
default_to_issues_board: {
- tracking_category: 'Growth::Conversion::Experiment::DefaultToIssuesBoard'
+ tracking_category: 'Growth::Conversion::Experiment::DefaultToIssuesBoard',
+ use_backwards_compatible_subject_index: true
}
}.freeze
- GROUP_CONTROL = :control
- GROUP_EXPERIMENTAL = :experimental
-
- # Controller concern that checks if an `experimentation_subject_id cookie` is present and sets it if absent.
- # Used for A/B testing of experimental features. Exposes the `experiment_enabled?(experiment_name)` method
- # to controllers and views. It returns true when the experiment is enabled and the user is selected as part
- # of the experimental group.
- #
- module ControllerConcern
- extend ActiveSupport::Concern
-
- included do
- before_action :set_experimentation_subject_id_cookie, unless: :dnt_enabled?
- helper_method :experiment_enabled?, :experiment_tracking_category_and_group
- end
-
- def set_experimentation_subject_id_cookie
- return if cookies[:experimentation_subject_id].present?
-
- cookies.permanent.signed[:experimentation_subject_id] = {
- value: SecureRandom.uuid,
- secure: ::Gitlab.config.gitlab.https,
- httponly: true
- }
- end
-
- def push_frontend_experiment(experiment_key)
- var_name = experiment_key.to_s.camelize(:lower)
- enabled = experiment_enabled?(experiment_key)
-
- gon.push({ experiments: { var_name => enabled } }, true)
- end
-
- def experiment_enabled?(experiment_key)
- return false if dnt_enabled?
-
- return true if Experimentation.enabled_for_value?(experiment_key, experimentation_subject_index)
- return true if forced_enabled?(experiment_key)
-
- false
- end
-
- def track_experiment_event(experiment_key, action, value = nil)
- return if dnt_enabled?
-
- track_experiment_event_for(experiment_key, action, value) do |tracking_data|
- ::Gitlab::Tracking.event(tracking_data.delete(:category), tracking_data.delete(:action), **tracking_data)
- end
- end
-
- def frontend_experimentation_tracking_data(experiment_key, action, value = nil)
- return if dnt_enabled?
-
- track_experiment_event_for(experiment_key, action, value) do |tracking_data|
- gon.push(tracking_data: tracking_data)
- end
- end
-
- def record_experiment_user(experiment_key)
- return if dnt_enabled?
- return unless Experimentation.enabled?(experiment_key) && current_user
-
- ::Experiment.add_user(experiment_key, tracking_group(experiment_key), current_user)
- end
-
- def experiment_tracking_category_and_group(experiment_key)
- "#{tracking_category(experiment_key)}:#{tracking_group(experiment_key, '_group')}"
- end
-
- private
-
- def dnt_enabled?
- Gitlab::Utils.to_boolean(request.headers['DNT'])
- end
-
- def experimentation_subject_id
- cookies.signed[:experimentation_subject_id]
- end
-
- def experimentation_subject_index
- return if experimentation_subject_id.blank?
-
- experimentation_subject_id.delete('-').hex % 100
- end
-
- def track_experiment_event_for(experiment_key, action, value)
- return unless Experimentation.enabled?(experiment_key)
-
- yield experimentation_tracking_data(experiment_key, action, value)
- end
-
- def experimentation_tracking_data(experiment_key, action, value)
- {
- category: tracking_category(experiment_key),
- action: action,
- property: tracking_group(experiment_key, "_group"),
- label: experimentation_subject_id,
- value: value
- }.compact
- end
-
- def tracking_category(experiment_key)
- Experimentation.experiment(experiment_key).tracking_category
- end
-
- def tracking_group(experiment_key, suffix = nil)
- return unless Experimentation.enabled?(experiment_key)
-
- group = experiment_enabled?(experiment_key) ? GROUP_EXPERIMENTAL : GROUP_CONTROL
-
- suffix ? "#{group}#{suffix}" : group
- end
-
- def forced_enabled?(experiment_key)
- params.has_key?(:force_experiment) && params[:force_experiment] == experiment_key.to_s
- end
- end
-
class << self
def experiment(key)
Experiment.new(EXPERIMENTS[key].merge(key: key))
@@ -201,7 +94,7 @@ module Gitlab
return false unless EXPERIMENTS.key?(experiment_key)
experiment = experiment(experiment_key)
- experiment.enabled? && experiment.enabled_for_environment?
+ experiment.enabled_for_environment? && experiment.enabled?
end
def enabled_for_attribute?(experiment_key, attribute)
@@ -209,13 +102,18 @@ module Gitlab
enabled_for_value?(experiment_key, index)
end
- def enabled_for_value?(experiment_key, experimentation_subject_index)
- enabled?(experiment_key) &&
- experiment(experiment_key).enabled_for_index?(experimentation_subject_index)
+ def enabled_for_value?(experiment_key, value)
+ enabled?(experiment_key) && experiment(experiment_key).enabled_for_index?(value)
end
end
- Experiment = Struct.new(:key, :environment, :tracking_category, keyword_init: true) do
+ Experiment = Struct.new(
+ :key,
+ :environment,
+ :tracking_category,
+ :use_backwards_compatible_subject_index,
+ keyword_init: true
+ ) do
def enabled?
experiment_percentage > 0
end
diff --git a/lib/gitlab/experimentation/controller_concern.rb b/lib/gitlab/experimentation/controller_concern.rb
new file mode 100644
index 00000000000..c6d15d7d82d
--- /dev/null
+++ b/lib/gitlab/experimentation/controller_concern.rb
@@ -0,0 +1,127 @@
+# frozen_string_literal: true
+
+require 'zlib'
+
+# Controller concern that checks if an `experimentation_subject_id cookie` is present and sets it if absent.
+# Used for A/B testing of experimental features. Exposes the `experiment_enabled?(experiment_name)` method
+# to controllers and views. It returns true when the experiment is enabled and the user is selected as part
+# of the experimental group.
+#
+module Gitlab
+ module Experimentation
+ module ControllerConcern
+ include ::Gitlab::Experimentation::GroupTypes
+ extend ActiveSupport::Concern
+
+ included do
+ before_action :set_experimentation_subject_id_cookie, unless: :dnt_enabled?
+ helper_method :experiment_enabled?, :experiment_tracking_category_and_group
+ end
+
+ def set_experimentation_subject_id_cookie
+ return if cookies[:experimentation_subject_id].present?
+
+ cookies.permanent.signed[:experimentation_subject_id] = {
+ value: SecureRandom.uuid,
+ secure: ::Gitlab.config.gitlab.https,
+ httponly: true
+ }
+ end
+
+ def push_frontend_experiment(experiment_key)
+ var_name = experiment_key.to_s.camelize(:lower)
+ enabled = experiment_enabled?(experiment_key)
+
+ gon.push({ experiments: { var_name => enabled } }, true)
+ end
+
+ def experiment_enabled?(experiment_key)
+ return false if dnt_enabled?
+
+ return true if Experimentation.enabled_for_value?(experiment_key, experimentation_subject_index(experiment_key))
+ return true if forced_enabled?(experiment_key)
+
+ false
+ end
+
+ def track_experiment_event(experiment_key, action, value = nil)
+ return if dnt_enabled?
+
+ track_experiment_event_for(experiment_key, action, value) do |tracking_data|
+ ::Gitlab::Tracking.event(tracking_data.delete(:category), tracking_data.delete(:action), **tracking_data)
+ end
+ end
+
+ def frontend_experimentation_tracking_data(experiment_key, action, value = nil)
+ return if dnt_enabled?
+
+ track_experiment_event_for(experiment_key, action, value) do |tracking_data|
+ gon.push(tracking_data: tracking_data)
+ end
+ end
+
+ def record_experiment_user(experiment_key)
+ return if dnt_enabled?
+ return unless Experimentation.enabled?(experiment_key) && current_user
+
+ ::Experiment.add_user(experiment_key, tracking_group(experiment_key), current_user)
+ end
+
+ def experiment_tracking_category_and_group(experiment_key)
+ "#{tracking_category(experiment_key)}:#{tracking_group(experiment_key, '_group')}"
+ end
+
+ private
+
+ def dnt_enabled?
+ Gitlab::Utils.to_boolean(request.headers['DNT'])
+ end
+
+ def experimentation_subject_id
+ cookies.signed[:experimentation_subject_id]
+ end
+
+ def experimentation_subject_index(experiment_key)
+ return if experimentation_subject_id.blank?
+
+ if Experimentation.experiment(experiment_key).use_backwards_compatible_subject_index
+ experimentation_subject_id.delete('-').hex % 100
+ else
+ Zlib.crc32("#{experiment_key}#{experimentation_subject_id}") % 100
+ end
+ end
+
+ def track_experiment_event_for(experiment_key, action, value)
+ return unless Experimentation.enabled?(experiment_key)
+
+ yield experimentation_tracking_data(experiment_key, action, value)
+ end
+
+ def experimentation_tracking_data(experiment_key, action, value)
+ {
+ category: tracking_category(experiment_key),
+ action: action,
+ property: tracking_group(experiment_key, "_group"),
+ label: experimentation_subject_id,
+ value: value
+ }.compact
+ end
+
+ def tracking_category(experiment_key)
+ Experimentation.experiment(experiment_key).tracking_category
+ end
+
+ def tracking_group(experiment_key, suffix = nil)
+ return unless Experimentation.enabled?(experiment_key)
+
+ group = experiment_enabled?(experiment_key) ? GROUP_EXPERIMENTAL : GROUP_CONTROL
+
+ suffix ? "#{group}#{suffix}" : group
+ end
+
+ def forced_enabled?(experiment_key)
+ params.has_key?(:force_experiment) && params[:force_experiment] == experiment_key.to_s
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/experimentation/group_types.rb b/lib/gitlab/experimentation/group_types.rb
new file mode 100644
index 00000000000..8e8f7284b99
--- /dev/null
+++ b/lib/gitlab/experimentation/group_types.rb
@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Experimentation
+ module GroupTypes
+ GROUP_CONTROL = :control
+ GROUP_EXPERIMENTAL = :experimental
+ end
+ end
+end
diff --git a/lib/gitlab/git/diff.rb b/lib/gitlab/git/diff.rb
index 78c47023c08..209917073c7 100644
--- a/lib/gitlab/git/diff.rb
+++ b/lib/gitlab/git/diff.rb
@@ -25,7 +25,7 @@ module Gitlab
#
# If this value ever changes, make sure to create a migration to update
# current records, and default of `ApplicationSettings#diff_max_patch_bytes`.
- DEFAULT_MAX_PATCH_BYTES = 100.kilobytes
+ DEFAULT_MAX_PATCH_BYTES = 200.kilobytes
# This is a limitation applied on the source (Gitaly), therefore we don't allow
# persisting limits over that.
diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb
index 1a3409c1f84..bc712e87e99 100644
--- a/lib/gitlab/git/repository.rb
+++ b/lib/gitlab/git/repository.rb
@@ -302,7 +302,7 @@ module Gitlab
private :archive_file_path
def archive_version_path
- return '' unless Feature.enabled?(:include_lfs_blobs_in_archive)
+ return '' unless Feature.enabled?(:include_lfs_blobs_in_archive, default_enabled: true)
'@v2'
end
diff --git a/lib/gitlab/gitaly_client/repository_service.rb b/lib/gitlab/gitaly_client/repository_service.rb
index 20ad6d0184b..e41a406ebd3 100644
--- a/lib/gitlab/gitaly_client/repository_service.rb
+++ b/lib/gitlab/gitaly_client/repository_service.rb
@@ -26,8 +26,8 @@ module Gitlab
GitalyClient.call(@storage, :repository_service, :cleanup, request, timeout: GitalyClient.fast_timeout)
end
- def garbage_collect(create_bitmap)
- request = Gitaly::GarbageCollectRequest.new(repository: @gitaly_repo, create_bitmap: create_bitmap)
+ def garbage_collect(create_bitmap, prune:)
+ request = Gitaly::GarbageCollectRequest.new(repository: @gitaly_repo, create_bitmap: create_bitmap, prune: prune)
GitalyClient.call(@storage, :repository_service, :garbage_collect, request, timeout: GitalyClient.long_timeout)
end
diff --git a/lib/gitlab/github_import.rb b/lib/gitlab/github_import.rb
index 9a7c406d981..c3cc15e10f7 100644
--- a/lib/gitlab/github_import.rb
+++ b/lib/gitlab/github_import.rb
@@ -6,10 +6,13 @@ module Gitlab
[:heads, :tags, '+refs/pull/*/head:refs/merge-requests/*/head']
end
- def self.new_client_for(project, token: nil, parallel: true)
+ def self.new_client_for(project, token: nil, host: nil, parallel: true)
token_to_use = token || project.import_data&.credentials&.fetch(:user)
-
- Client.new(token_to_use, parallel: parallel)
+ Client.new(
+ token_to_use,
+ host: host.presence || self.formatted_import_url(project),
+ parallel: parallel
+ )
end
# Returns the ID of the ghost user.
@@ -18,5 +21,17 @@ module Gitlab
Gitlab::Cache::Import::Caching.read_integer(key) || Gitlab::Cache::Import::Caching.write(key, User.select(:id).ghost.id)
end
+
+ # Get formatted GitHub import URL. If github.com is in the import URL, this will return nil and octokit will use the default github.com API URL
+ def self.formatted_import_url(project)
+ url = URI.parse(project.import_url)
+
+ unless url.host == 'github.com'
+ url.user = nil
+ url.password = nil
+ url.path = "/api/v3"
+ url.to_s
+ end
+ end
end
end
diff --git a/lib/gitlab/github_import/client.rb b/lib/gitlab/github_import/client.rb
index 22803c5cd71..dfe60fb5a03 100644
--- a/lib/gitlab/github_import/client.rb
+++ b/lib/gitlab/github_import/client.rb
@@ -18,6 +18,8 @@ module Gitlab
attr_reader :octokit
+ SEARCH_MAX_REQUESTS_PER_MINUTE = 30
+
# A single page of data and the corresponding page number.
Page = Struct.new(:objects, :number)
@@ -28,9 +30,12 @@ module Gitlab
# rate limit at once. The threshold is put in place to not hit the limit
# in most cases.
RATE_LIMIT_THRESHOLD = 50
+ SEARCH_RATE_LIMIT_THRESHOLD = 3
# token - The GitHub API token to use.
#
+ # host - The GitHub hostname. If nil, github.com will be used.
+ #
# per_page - The number of objects that should be displayed per page.
#
# parallel - When set to true hitting the rate limit will result in a
@@ -39,11 +44,13 @@ module Gitlab
# this value to `true` for parallel importing is crucial as
# otherwise hitting the rate limit will result in a thread
# being blocked in a `sleep()` call for up to an hour.
- def initialize(token, per_page: 100, parallel: true)
+ def initialize(token, host: nil, per_page: 100, parallel: true)
+ @host = host
@octokit = ::Octokit::Client.new(
access_token: token,
per_page: per_page,
- api_endpoint: api_endpoint
+ api_endpoint: api_endpoint,
+ web_endpoint: web_endpoint
)
@octokit.connection_options[:ssl] = { verify: verify_ssl }
@@ -148,8 +155,26 @@ module Gitlab
end
end
+ def search_repos_by_name(name)
+ each_page(:search_repositories, search_query(str: name, type: :name))
+ end
+
+ def search_query(str:, type:, include_collaborations: true, include_orgs: true)
+ query = "#{str} in:#{type} is:public,private user:#{octokit.user.login}"
+
+ query = [query, collaborations_subquery].join(' ') if include_collaborations
+ query = [query, organizations_subquery].join(' ') if include_orgs
+
+ query
+ end
+
# Returns `true` if we're still allowed to perform API calls.
+ # Search API has rate limit of 30, use lowered threshold when search is used.
def requests_remaining?
+ if requests_limit == SEARCH_MAX_REQUESTS_PER_MINUTE
+ return remaining_requests > SEARCH_RATE_LIMIT_THRESHOLD
+ end
+
remaining_requests > RATE_LIMIT_THRESHOLD
end
@@ -157,6 +182,10 @@ module Gitlab
octokit.rate_limit.remaining
end
+ def requests_limit
+ octokit.rate_limit.limit
+ end
+
def raise_or_wait_for_rate_limit
rate_limit_counter.increment
@@ -181,7 +210,11 @@ module Gitlab
end
def api_endpoint
- custom_api_endpoint || default_api_endpoint
+ @host || custom_api_endpoint || default_api_endpoint
+ end
+
+ def web_endpoint
+ @host || custom_api_endpoint || ::Octokit::Default.web_endpoint
end
def custom_api_endpoint
@@ -213,6 +246,20 @@ module Gitlab
'The number of GitHub API calls performed when importing projects'
)
end
+
+ private
+
+ def collaborations_subquery
+ each_object(:repos, nil, { affiliation: 'collaborator' })
+ .map { |repo| "repo:#{repo.full_name}" }
+ .join(' ')
+ end
+
+ def organizations_subquery
+ each_object(:organizations)
+ .map { |org| "org:#{org.login}" }
+ .join(' ')
+ end
end
end
end
diff --git a/lib/gitlab/github_import/sequential_importer.rb b/lib/gitlab/github_import/sequential_importer.rb
index 6a181caf65d..cb6b2017208 100644
--- a/lib/gitlab/github_import/sequential_importer.rb
+++ b/lib/gitlab/github_import/sequential_importer.rb
@@ -25,10 +25,11 @@ module Gitlab
# project - The project to import the data into.
# token - The token to use for the GitHub API.
- def initialize(project, token: nil)
+ # host - The GitHub hostname. If nil, github.com will be used.
+ def initialize(project, token: nil, host: nil)
@project = project
@client = GithubImport
- .new_client_for(project, token: token, parallel: false)
+ .new_client_for(project, token: token, host: host, parallel: false)
end
def execute
diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb
index 10660649623..2d41ad76618 100644
--- a/lib/gitlab/gon_helper.rb
+++ b/lib/gitlab/gon_helper.rb
@@ -28,6 +28,7 @@ module Gitlab
gon.sprite_icons = IconsHelper.sprite_icon_path
gon.sprite_file_icons = IconsHelper.sprite_file_icons_path
gon.emoji_sprites_css_path = ActionController::Base.helpers.stylesheet_path('emoji_sprites')
+ gon.select2_css_path = ActionController::Base.helpers.stylesheet_path('lazy_bundles/select2.css')
gon.test_env = Rails.env.test?
gon.disable_animations = Gitlab.config.gitlab['disable_animations']
gon.suggested_label_colors = LabelsHelper.suggested_colors
@@ -58,9 +59,13 @@ module Gitlab
# args - Any additional arguments to pass to `Feature.enabled?`. This allows
# you to check if a flag is enabled for a particular user.
def push_frontend_feature_flag(name, *args, **kwargs)
- var_name = name.to_s.camelize(:lower)
enabled = Feature.enabled?(name, *args, **kwargs)
+ push_to_gon_features(name, enabled)
+ end
+
+ def push_to_gon_features(name, enabled)
+ var_name = name.to_s.camelize(:lower)
# Here the `true` argument signals gon that the value should be merged
# into any existing ones, instead of overwriting them. This allows you to
# use this method to push multiple feature flags.
diff --git a/lib/gitlab/grape_logging/formatters/lograge_with_timestamp.rb b/lib/gitlab/grape_logging/formatters/lograge_with_timestamp.rb
index ac149cadb5b..a0dccbcdab3 100644
--- a/lib/gitlab/grape_logging/formatters/lograge_with_timestamp.rb
+++ b/lib/gitlab/grape_logging/formatters/lograge_with_timestamp.rb
@@ -41,6 +41,8 @@ module Gitlab
data.map! { |v| utf8_encode_values(v) }
when String
encode_utf8(data)
+ when Integer
+ data
end
end
end
diff --git a/lib/gitlab/grape_logging/loggers/content_logger.rb b/lib/gitlab/grape_logging/loggers/content_logger.rb
new file mode 100644
index 00000000000..658953adc80
--- /dev/null
+++ b/lib/gitlab/grape_logging/loggers/content_logger.rb
@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module GrapeLogging
+ module Loggers
+ class ContentLogger < ::GrapeLogging::Loggers::Base
+ def parameters(request, _)
+ {
+ content_length: request.env['CONTENT_LENGTH'],
+ content_range: request.env['HTTP_CONTENT_RANGE']
+ }.compact
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/graphql/authorize/authorize_field_service.rb b/lib/gitlab/graphql/authorize/authorize_field_service.rb
index cbf3e7b8429..e8db619f88a 100644
--- a/lib/gitlab/graphql/authorize/authorize_field_service.rb
+++ b/lib/gitlab/graphql/authorize/authorize_field_service.rb
@@ -46,6 +46,8 @@ module Gitlab
# Returns any authorize metadata from @field
def field_authorizations
+ return [] if @field.metadata[:authorize] == true
+
Array.wrap(@field.metadata[:authorize])
end
@@ -54,7 +56,7 @@ module Gitlab
# The field is a built-in/scalar type, or a list of scalars
# authorize using the parent's object
parent_typed_object.object
- elsif @field.connection? || resolved_type.is_a?(Array)
+ elsif @field.connection? || @field.type.list? || resolved_type.is_a?(Array)
# The field is a connection or a list of non-built-in types, we'll
# authorize each element when rendering
nil
@@ -75,16 +77,25 @@ module Gitlab
# no need to do anything
elsif authorizing_object
# Authorizing fields representing scalars, or a simple field with an object
- resolved_type if allowed_access?(current_user, authorizing_object)
+ ::Gitlab::Graphql::Lazy.with_value(authorizing_object) do |object|
+ resolved_type if allowed_access?(current_user, object)
+ end
elsif @field.connection?
- # A connection with pagination, modify the visible nodes on the
- # connection type in place
- resolved_type.object.edge_nodes.to_a.keep_if { |node| allowed_access?(current_user, node) }
- resolved_type
- elsif resolved_type.is_a? Array
+ ::Gitlab::Graphql::Lazy.with_value(resolved_type) do |type|
+ # A connection with pagination, modify the visible nodes on the
+ # connection type in place
+ nodes = to_nodes(type)
+ nodes.keep_if { |node| allowed_access?(current_user, node) } if nodes
+ type
+ end
+ elsif @field.type.list? || resolved_type.is_a?(Array)
# A simple list of rendered types each object being an object to authorize
- resolved_type.select do |single_object_type|
- allowed_access?(current_user, realized(single_object_type).object)
+ ::Gitlab::Graphql::Lazy.with_value(resolved_type) do |items|
+ items.select do |single_object_type|
+ object_type = realized(single_object_type)
+ object = object_type.try(:object) || object_type
+ allowed_access?(current_user, object)
+ end
end
else
raise "Can't authorize #{@field}"
@@ -93,18 +104,23 @@ module Gitlab
# Ensure that we are dealing with realized objects, not delayed promises
def realized(thing)
- case thing
- when BatchLoader::GraphQL
- thing.sync
- when GraphQL::Execution::Lazy
- thing.value # part of the private api, but we need to unwrap it here.
+ ::Gitlab::Graphql::Lazy.force(thing)
+ end
+
+ # Try to get the connection
+ # can be at type.object or at type
+ def to_nodes(type)
+ if type.respond_to?(:nodes)
+ type.nodes
+ elsif type.respond_to?(:object)
+ to_nodes(type.object)
else
- thing
+ nil
end
end
def allowed_access?(current_user, object)
- object = object.sync if object.respond_to?(:sync)
+ object = realized(object)
authorizations.all? do |ability|
Ability.allowed?(current_user, ability, object)
diff --git a/lib/gitlab/graphql/docs/helper.rb b/lib/gitlab/graphql/docs/helper.rb
index dcd0e12cbfc..503b1064b11 100644
--- a/lib/gitlab/graphql/docs/helper.rb
+++ b/lib/gitlab/graphql/docs/helper.rb
@@ -81,11 +81,15 @@ module Gitlab
# We are ignoring connections and built in types for now,
# they should be added when queries are generated.
def objects
- graphql_object_types.select do |object_type|
+ object_types = graphql_object_types.select do |object_type|
!object_type[:name]["Connection"] &&
!object_type[:name]["Edge"] &&
!object_type[:name]["__"]
end
+
+ object_types.each do |type|
+ type[:fields] += type[:connections]
+ end
end
# We ignore the built-in enum types.
diff --git a/lib/gitlab/graphql/docs/templates/default.md.haml b/lib/gitlab/graphql/docs/templates/default.md.haml
index ec052943589..97df4233905 100644
--- a/lib/gitlab/graphql/docs/templates/default.md.haml
+++ b/lib/gitlab/graphql/docs/templates/default.md.haml
@@ -14,6 +14,8 @@
CAUTION: **Caution:**
Fields that are deprecated are marked with **{warning-solid}**.
+ Items (fields, enums, etc) that have been removed according to our [deprecation process](../index.md#deprecation-process) can be found
+ in [Removed Items](../removed_items.md).
\
:plain
diff --git a/lib/gitlab/graphql/lazy.rb b/lib/gitlab/graphql/lazy.rb
index a7f7610a041..3cc11047387 100644
--- a/lib/gitlab/graphql/lazy.rb
+++ b/lib/gitlab/graphql/lazy.rb
@@ -3,17 +3,41 @@
module Gitlab
module Graphql
class Lazy
+ include Gitlab::Utils::StrongMemoize
+
+ def initialize(&block)
+ @proc = block
+ end
+
+ def force
+ strong_memoize(:force) { self.class.force(@proc.call) }
+ end
+
+ def then(&block)
+ self.class.new { yield force }
+ end
+
# Force evaluation of a (possibly) lazy value
def self.force(value)
case value
+ when ::Gitlab::Graphql::Lazy
+ value.force
when ::BatchLoader::GraphQL
value.sync
+ when ::GraphQL::Execution::Lazy
+ value.value # part of the private api, but we can force this as well
when ::Concurrent::Promise
- value.execute.value
+ value.execute if value.state == :unscheduled
+
+ value.value # value.value(10.seconds)
else
value
end
end
+
+ def self.with_value(unforced, &block)
+ self.new { unforced }.then(&block)
+ end
end
end
end
diff --git a/lib/gitlab/graphql/loaders/batch_model_loader.rb b/lib/gitlab/graphql/loaders/batch_model_loader.rb
index 164fe74148c..9b85ba164d4 100644
--- a/lib/gitlab/graphql/loaders/batch_model_loader.rb
+++ b/lib/gitlab/graphql/loaders/batch_model_loader.rb
@@ -12,14 +12,11 @@ module Gitlab
# rubocop: disable CodeReuse/ActiveRecord
def find
- BatchLoader::GraphQL.for({ model: model_class, id: model_id.to_i }).batch do |loader_info, loader|
- per_model = loader_info.group_by { |info| info[:model] }
- per_model.each do |model, info|
- ids = info.map { |i| i[:id] }
- results = model.where(id: ids)
+ BatchLoader::GraphQL.for(model_id.to_i).batch(key: model_class) do |ids, loader, args|
+ model = args[:key]
+ results = model.where(id: ids)
- results.each { |record| loader.call({ model: model, id: record.id }, record) }
- end
+ results.each { |record| loader.call(record.id, record) }
end
end
# rubocop: enable CodeReuse/ActiveRecord
diff --git a/lib/gitlab/graphql/present/instrumentation.rb b/lib/gitlab/graphql/present/instrumentation.rb
index 941a4f434a1..b8535575da5 100644
--- a/lib/gitlab/graphql/present/instrumentation.rb
+++ b/lib/gitlab/graphql/present/instrumentation.rb
@@ -4,6 +4,8 @@ module Gitlab
module Graphql
module Present
class Instrumentation
+ SAFE_CONTEXT_KEYS = %i[current_user].freeze
+
def instrument(type, field)
return field unless field.metadata[:type_class]
@@ -22,7 +24,8 @@ module Gitlab
next old_resolver.call(presented_type, args, context)
end
- presenter = presented_in.presenter_class.new(object, **context.to_h)
+ attrs = safe_context_values(context)
+ presenter = presented_in.presenter_class.new(object, **attrs)
# we have to use the new `authorized_new` method, as `new` is protected
wrapped = presented_type.class.authorized_new(presenter, context)
@@ -34,6 +37,12 @@ module Gitlab
resolve(resolve_with_presenter)
end
end
+
+ private
+
+ def safe_context_values(context)
+ context.to_h.slice(*SAFE_CONTEXT_KEYS)
+ end
end
end
end
diff --git a/lib/gitlab/group_search_results.rb b/lib/gitlab/group_search_results.rb
index 5fec50eecd2..dd872caee0e 100644
--- a/lib/gitlab/group_search_results.rb
+++ b/lib/gitlab/group_search_results.rb
@@ -4,10 +4,10 @@ module Gitlab
class GroupSearchResults < SearchResults
attr_reader :group
- def initialize(current_user, query, limit_projects = nil, group:, default_project_filter: false, sort: nil, filters: {})
+ def initialize(current_user, query, limit_projects = nil, group:, default_project_filter: false, order_by: nil, sort: nil, filters: {})
@group = group
- super(current_user, query, limit_projects, default_project_filter: default_project_filter, sort: sort, filters: filters)
+ super(current_user, query, limit_projects, default_project_filter: default_project_filter, order_by: order_by, sort: sort, filters: filters)
end
# rubocop:disable CodeReuse/ActiveRecord
diff --git a/lib/gitlab/hook_data/release_builder.rb b/lib/gitlab/hook_data/release_builder.rb
new file mode 100644
index 00000000000..b15c260f4a8
--- /dev/null
+++ b/lib/gitlab/hook_data/release_builder.rb
@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module HookData
+ class ReleaseBuilder < BaseBuilder
+ def self.safe_hook_attributes
+ %i[
+ id
+ created_at
+ description
+ name
+ released_at
+ tag
+ ].freeze
+ end
+
+ alias_method :release, :object
+
+ def build(action)
+ attrs = {
+ object_kind: object_kind,
+ project: release.project.hook_attrs,
+ description: absolute_image_urls(release.description),
+ url: Gitlab::UrlBuilder.build(release),
+ action: action,
+ assets: {
+ count: release.assets_count,
+ links: release.links.map(&:hook_attrs),
+ sources: release.sources.map(&:hook_attrs)
+ },
+ commit: release.commit.hook_attrs
+ }
+
+ release.attributes.with_indifferent_access.slice(*self.class.safe_hook_attributes)
+ .merge!(attrs)
+ end
+
+ private
+
+ def object_kind
+ release.class.name.underscore
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/i18n/po_linter.rb b/lib/gitlab/i18n/po_linter.rb
index e56b88dfce0..33054a5b9bf 100644
--- a/lib/gitlab/i18n/po_linter.rb
+++ b/lib/gitlab/i18n/po_linter.rb
@@ -24,7 +24,9 @@ module Gitlab
return 'PO-syntax errors' => [parse_error]
end
- validate_entries
+ Gitlab::I18n.with_locale(locale) do
+ validate_entries
+ end
end
def parse_po
@@ -156,12 +158,10 @@ module Gitlab
end
def validate_translation(errors, entry)
- Gitlab::I18n.with_locale(locale) do
- if entry.has_plural?
- translate_plural(entry)
- else
- translate_singular(entry)
- end
+ if entry.has_plural?
+ translate_plural(entry)
+ else
+ translate_singular(entry)
end
# `sprintf` could raise an `ArgumentError` when invalid passing something
@@ -230,9 +230,7 @@ module Gitlab
# This calls the C function that defines the pluralization rule, it can
# return a boolean (`false` represents 0, `true` represents 1) or an integer
# that specifies the plural form to be used for the given number
- pluralization_result = Gitlab::I18n.with_locale(locale) do
- FastGettext.pluralisation_rule.call(counter)
- end
+ pluralization_result = FastGettext.pluralisation_rule.call(counter)
case pluralization_result
when false
diff --git a/lib/gitlab/import_export/importer.rb b/lib/gitlab/import_export/importer.rb
index 7b8689069d8..8e78f6e274a 100644
--- a/lib/gitlab/import_export/importer.rb
+++ b/lib/gitlab/import_export/importer.rb
@@ -55,9 +55,17 @@ module Gitlab
end
def project_tree
- @project_tree ||= Gitlab::ImportExport::Project::TreeRestorer.new(user: current_user,
- shared: shared,
- project: project)
+ @project_tree ||= project_tree_class.new(user: current_user,
+ shared: shared,
+ project: project)
+ end
+
+ def project_tree_class
+ sample_data_template? ? Gitlab::ImportExport::Project::Sample::TreeRestorer : Gitlab::ImportExport::Project::TreeRestorer
+ end
+
+ def sample_data_template?
+ project&.import_data&.data&.dig('sample_data')
end
def avatar_restorer
diff --git a/lib/gitlab/import_export/json/ndjson_reader.rb b/lib/gitlab/import_export/json/ndjson_reader.rb
index 0d9839b86cf..5c8edd485e5 100644
--- a/lib/gitlab/import_export/json/ndjson_reader.rb
+++ b/lib/gitlab/import_export/json/ndjson_reader.rb
@@ -29,9 +29,9 @@ module Gitlab
json_decode(data)
end
- def consume_relation(importable_path, key)
+ def consume_relation(importable_path, key, mark_as_consumed: true)
Enumerator.new do |documents|
- next unless @consumed_relations.add?("#{importable_path}/#{key}")
+ next if mark_as_consumed && !@consumed_relations.add?("#{importable_path}/#{key}")
# This reads from `tree/project/merge_requests.ndjson`
path = file_path(importable_path, "#{key}.ndjson")
@@ -44,11 +44,6 @@ module Gitlab
end
end
- # TODO: Move clear logic into main comsume_relation method (see https://gitlab.com/gitlab-org/gitlab/-/merge_requests/41699#note_430465330)
- def clear_consumed_relations
- @consumed_relations.clear
- end
-
private
def json_decode(string)
diff --git a/lib/gitlab/import_export/project/import_export.yml b/lib/gitlab/import_export/project/import_export.yml
index a0526ba0414..ae7ddbc5eba 100644
--- a/lib/gitlab/import_export/project/import_export.yml
+++ b/lib/gitlab/import_export/project/import_export.yml
@@ -185,6 +185,7 @@ excluded_attributes:
- :secret
- :encrypted_secret_token
- :encrypted_secret_token_iv
+ - :repository_read_only
merge_request_diff:
- :external_diff
- :stored_externally
@@ -410,8 +411,25 @@ ee:
- :deploy_access_levels
- :service_desk_setting
- :security_setting
+ - :push_rule
included_attributes:
issuable_sla:
- :issue
- :due_at
+ push_rule:
+ - :force_push_regex
+ - :delete_branch_regex
+ - :commit_message_regex
+ - :author_email_regex
+ - :file_name_regex
+ - :branch_name_regex
+ - :commit_message_negative_regex
+ - :max_file_size
+ - :deny_delete_tag
+ - :member_check
+ - :is_sample
+ - :prevent_secrets
+ - :reject_unsigned_commits
+ - :commit_committer_check
+ - :regexp_uses_re2
diff --git a/lib/gitlab/import_export/project/sample/date_calculator.rb b/lib/gitlab/import_export/project/sample/date_calculator.rb
index 2d989d21166..543fd25d883 100644
--- a/lib/gitlab/import_export/project/sample/date_calculator.rb
+++ b/lib/gitlab/import_export/project/sample/date_calculator.rb
@@ -9,7 +9,6 @@ module Gitlab
def initialize(dates)
@dates = dates.dup
- @dates.flatten!
@dates.compact!
@dates.sort!
@dates.map! { |date| date.to_time.to_f }
diff --git a/lib/gitlab/import_export/project/sample/relation_factory.rb b/lib/gitlab/import_export/project/sample/relation_factory.rb
new file mode 100644
index 00000000000..6e59174f9a3
--- /dev/null
+++ b/lib/gitlab/import_export/project/sample/relation_factory.rb
@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module ImportExport
+ module Project
+ module Sample
+ class RelationFactory < Project::RelationFactory
+ DATE_MODELS = %i[issues milestones].freeze
+
+ def initialize(date_calculator:, **args)
+ super(**args)
+
+ @date_calculator = date_calculator
+ end
+
+ private
+
+ def setup_models
+ super
+
+ # Override due date attributes in data hash for Sample Data templates
+ # Dates are moved by taking the closest one to average and moving that (and rest around it) to the date of import
+ override_date_attributes
+ end
+
+ def override_date_attributes
+ return unless DATE_MODELS.include?(@relation_name)
+
+ @relation_hash['start_date'] = calculate_by_closest_date(@relation_hash['start_date']&.to_time)
+ @relation_hash['due_date'] = calculate_by_closest_date(@relation_hash['due_date']&.to_time)
+ end
+
+ def calculate_by_closest_date(date)
+ return unless date
+
+ @date_calculator.calculate_by_closest_date_to_average(date)
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/import_export/project/sample/relation_tree_restorer.rb b/lib/gitlab/import_export/project/sample/relation_tree_restorer.rb
new file mode 100644
index 00000000000..44ccb67a531
--- /dev/null
+++ b/lib/gitlab/import_export/project/sample/relation_tree_restorer.rb
@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module ImportExport
+ module Project
+ module Sample
+ class RelationTreeRestorer < ImportExport::RelationTreeRestorer
+ def initialize(*args)
+ super
+
+ @date_calculator = Gitlab::ImportExport::Project::Sample::DateCalculator.new(dates)
+ end
+
+ private
+
+ def relation_factory_params(*args)
+ super.merge(date_calculator: @date_calculator)
+ end
+
+ def dates
+ return [] if relation_reader.legacy?
+
+ RelationFactory::DATE_MODELS.flat_map do |tag|
+ relation_reader.consume_relation(@importable_path, tag, mark_as_consumed: false).map do |model|
+ model.first['due_date']
+ end
+ end
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/import_export/project/sample/sample_data_relation_tree_restorer.rb b/lib/gitlab/import_export/project/sample/sample_data_relation_tree_restorer.rb
deleted file mode 100644
index b0c3940b5f9..00000000000
--- a/lib/gitlab/import_export/project/sample/sample_data_relation_tree_restorer.rb
+++ /dev/null
@@ -1,51 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module ImportExport
- module Project
- module Sample
- class SampleDataRelationTreeRestorer < RelationTreeRestorer
- DATE_MODELS = %i[issues milestones].freeze
-
- def initialize(*args)
- super
-
- date_calculator
- end
-
- private
-
- def build_relation(relation_key, relation_definition, data_hash)
- # Override due date attributes in data hash for Sample Data templates
- # Dates are moved by taking the closest one to average and moving that (and rest around it) to the date of import
- # TODO: To move this logic to RelationFactory (see: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/41699#note_430465333)
- override_date_attributes!(relation_key, data_hash)
- super
- end
-
- def override_date_attributes!(relation_key, data_hash)
- return unless DATE_MODELS.include?(relation_key.to_sym)
-
- data_hash['start_date'] = date_calculator.calculate_by_closest_date_to_average(data_hash['start_date'].to_time) unless data_hash['start_date'].nil?
- data_hash['due_date'] = date_calculator.calculate_by_closest_date_to_average(data_hash['due_date'].to_time) unless data_hash['due_date'].nil?
- end
-
- # TODO: Move clear logic into main comsume_relation method (see https://gitlab.com/gitlab-org/gitlab/-/merge_requests/41699#note_430465330)
- def dates
- unless relation_reader.legacy?
- DATE_MODELS.map do |tag|
- relation_reader.consume_relation(@importable_path, tag).map { |model| model.first['due_date'] }.tap do
- relation_reader.clear_consumed_relations
- end
- end
- end
- end
-
- def date_calculator
- @date_calculator ||= Gitlab::ImportExport::Project::Sample::DateCalculator.new(dates)
- end
- end
- end
- end
- end
-end
diff --git a/lib/gitlab/import_export/project/sample/tree_restorer.rb b/lib/gitlab/import_export/project/sample/tree_restorer.rb
new file mode 100644
index 00000000000..1d4b5328cb9
--- /dev/null
+++ b/lib/gitlab/import_export/project/sample/tree_restorer.rb
@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module ImportExport
+ module Project
+ module Sample
+ class TreeRestorer < Project::TreeRestorer
+ def relation_tree_restorer_class
+ RelationTreeRestorer
+ end
+
+ def relation_factory
+ RelationFactory
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/import_export/project/tree_restorer.rb b/lib/gitlab/import_export/project/tree_restorer.rb
index b1d647281ab..fb9e5be1877 100644
--- a/lib/gitlab/import_export/project/tree_restorer.rb
+++ b/lib/gitlab/import_export/project/tree_restorer.rb
@@ -85,11 +85,7 @@ module Gitlab
end
def relation_tree_restorer_class
- sample_data_template? ? Sample::SampleDataRelationTreeRestorer : RelationTreeRestorer
- end
-
- def sample_data_template?
- @project&.import_data&.data&.dig('sample_data')
+ RelationTreeRestorer
end
def members_mapper
diff --git a/lib/gitlab/import_export/uploads_manager.rb b/lib/gitlab/import_export/uploads_manager.rb
index 26e7d2cf765..428bcbe8dc5 100644
--- a/lib/gitlab/import_export/uploads_manager.rb
+++ b/lib/gitlab/import_export/uploads_manager.rb
@@ -86,6 +86,10 @@ module Gitlab
mkdir_p(File.join(uploads_export_path, secret))
download_or_copy_upload(upload, upload_path)
+ rescue Errno::ENAMETOOLONG => e
+ # Do not fail entire project export if downloaded file has filename that exceeds 255 characters.
+ # Ignore raised exception, skip such upload, log the error and keep going with the export instead.
+ Gitlab::ErrorTracking.log_exception(e, project_id: @project.id)
end
end
end
diff --git a/lib/gitlab/instrumentation/throttle.rb b/lib/gitlab/instrumentation/throttle.rb
new file mode 100644
index 00000000000..0b7e990fb2e
--- /dev/null
+++ b/lib/gitlab/instrumentation/throttle.rb
@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Instrumentation
+ class Throttle
+ KEY = :instrumentation_throttle_safelist
+
+ def self.safelist
+ Gitlab::SafeRequestStore[KEY]
+ end
+
+ def self.safelist=(name)
+ Gitlab::SafeRequestStore[KEY] = name
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/instrumentation_helper.rb b/lib/gitlab/instrumentation_helper.rb
index 3a29d2e7efa..d7228099eaf 100644
--- a/lib/gitlab/instrumentation_helper.rb
+++ b/lib/gitlab/instrumentation_helper.rb
@@ -21,6 +21,7 @@ module Gitlab
instrument_rugged(payload)
instrument_redis(payload)
instrument_elasticsearch(payload)
+ instrument_throttle(payload)
end
def instrument_gitaly(payload)
@@ -56,6 +57,11 @@ module Gitlab
payload[:elasticsearch_duration_s] = Gitlab::Instrumentation::ElasticsearchTransport.query_time
end
+ def instrument_throttle(payload)
+ safelist = Gitlab::Instrumentation::Throttle.safelist
+ payload[:throttle_safelist] = safelist if safelist.present?
+ end
+
# Returns the queuing duration for a Sidekiq job in seconds, as a float, if the
# `enqueued_at` field or `created_at` field is available.
#
diff --git a/lib/gitlab/json.rb b/lib/gitlab/json.rb
index 29cfec443e8..8565f664cd4 100644
--- a/lib/gitlab/json.rb
+++ b/lib/gitlab/json.rb
@@ -67,15 +67,6 @@ module Gitlab
::JSON.pretty_generate(object, opts)
end
- # Feature detection for using Oj instead of the `json` gem.
- #
- # @return [Boolean]
- def enable_oj?
- return false unless feature_table_exists?
-
- Feature.enabled?(:oj_json, default_enabled: true)
- end
-
private
# Convert JSON string into Ruby through toggleable adapters.
@@ -91,11 +82,7 @@ module Gitlab
def adapter_load(string, *args, **opts)
opts = standardize_opts(opts)
- if enable_oj?
- Oj.load(string, opts)
- else
- ::JSON.parse(string, opts)
- end
+ Oj.load(string, opts)
rescue Oj::ParseError, Encoding::UndefinedConversionError => ex
raise parser_error.new(ex)
end
@@ -120,11 +107,7 @@ module Gitlab
#
# @return [String]
def adapter_dump(object, *args, **opts)
- if enable_oj?
- Oj.dump(object, opts)
- else
- ::JSON.dump(object, *args)
- end
+ Oj.dump(object, opts)
end
# Generates JSON for an object but with fewer options, using toggleable adapters.
@@ -135,11 +118,7 @@ module Gitlab
def adapter_generate(object, opts = {})
opts = standardize_opts(opts)
- if enable_oj?
- Oj.generate(object, opts)
- else
- ::JSON.generate(object, opts)
- end
+ Oj.generate(object, opts)
end
# Take a JSON standard options hash and standardize it to work across adapters
@@ -149,11 +128,8 @@ module Gitlab
# @return [Hash]
def standardize_opts(opts)
opts ||= {}
-
- if enable_oj?
- opts[:mode] = :rails
- opts[:symbol_keys] = opts[:symbolize_keys] || opts[:symbolize_names]
- end
+ opts[:mode] = :rails
+ opts[:symbol_keys] = opts[:symbolize_keys] || opts[:symbolize_names]
opts
end
@@ -213,7 +189,7 @@ module Gitlab
# @param object [Object]
# @return [String]
def self.call(object, env = nil)
- if Gitlab::Json.enable_oj? && Feature.enabled?(:grape_gitlab_json, default_enabled: true)
+ if Feature.enabled?(:grape_gitlab_json, default_enabled: true)
Gitlab::Json.dump(object)
else
Grape::Formatter::Json.call(object, env)
diff --git a/lib/gitlab/kubernetes/helm/base_command.rb b/lib/gitlab/kubernetes/helm/base_command.rb
deleted file mode 100644
index 49d2969f7f3..00000000000
--- a/lib/gitlab/kubernetes/helm/base_command.rb
+++ /dev/null
@@ -1,85 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module Kubernetes
- module Helm
- class BaseCommand
- attr_reader :name, :files
-
- def initialize(rbac:, name:, files:)
- @rbac = rbac
- @name = name
- @files = files
- end
-
- def rbac?
- @rbac
- end
-
- def pod_resource
- pod_service_account_name = rbac? ? service_account_name : nil
-
- Gitlab::Kubernetes::Helm::Pod.new(self, namespace, service_account_name: pod_service_account_name).generate
- end
-
- def generate_script
- <<~HEREDOC
- set -xeo pipefail
- HEREDOC
- end
-
- def pod_name
- "install-#{name}"
- end
-
- def config_map_resource
- Gitlab::Kubernetes::ConfigMap.new(name, files).generate
- end
-
- def service_account_resource
- return unless rbac?
-
- Gitlab::Kubernetes::ServiceAccount.new(service_account_name, namespace).generate
- end
-
- def cluster_role_binding_resource
- return unless rbac?
-
- subjects = [{ kind: 'ServiceAccount', name: service_account_name, namespace: namespace }]
-
- Gitlab::Kubernetes::ClusterRoleBinding.new(
- cluster_role_binding_name,
- cluster_role_name,
- subjects
- ).generate
- end
-
- def file_names
- files.keys
- end
-
- private
-
- def files_dir
- "/data/helm/#{name}/config"
- end
-
- def namespace
- Gitlab::Kubernetes::Helm::NAMESPACE
- end
-
- def service_account_name
- Gitlab::Kubernetes::Helm::SERVICE_ACCOUNT
- end
-
- def cluster_role_binding_name
- Gitlab::Kubernetes::Helm::CLUSTER_ROLE_BINDING
- end
-
- def cluster_role_name
- Gitlab::Kubernetes::Helm::CLUSTER_ROLE
- end
- end
- end
- end
-end
diff --git a/lib/gitlab/kubernetes/helm/certificate.rb b/lib/gitlab/kubernetes/helm/certificate.rb
deleted file mode 100644
index 598714e0874..00000000000
--- a/lib/gitlab/kubernetes/helm/certificate.rb
+++ /dev/null
@@ -1,73 +0,0 @@
-# frozen_string_literal: true
-module Gitlab
- module Kubernetes
- module Helm
- class Certificate
- INFINITE_EXPIRY = 1000.years
- SHORT_EXPIRY = 30.minutes
-
- attr_reader :key, :cert
-
- def key_string
- @key.to_s
- end
-
- def cert_string
- @cert.to_pem
- end
-
- def self.from_strings(key_string, cert_string)
- key = OpenSSL::PKey::RSA.new(key_string)
- cert = OpenSSL::X509::Certificate.new(cert_string)
- new(key, cert)
- end
-
- def self.generate_root
- _issue(signed_by: nil, expires_in: INFINITE_EXPIRY, certificate_authority: true)
- end
-
- def issue(expires_in: SHORT_EXPIRY)
- self.class._issue(signed_by: self, expires_in: expires_in, certificate_authority: false)
- end
-
- private
-
- def self._issue(signed_by:, expires_in:, certificate_authority:)
- key = OpenSSL::PKey::RSA.new(4096)
- public_key = key.public_key
-
- subject = OpenSSL::X509::Name.parse("/C=US")
-
- cert = OpenSSL::X509::Certificate.new
- cert.subject = subject
-
- cert.issuer = signed_by&.cert&.subject || subject
-
- cert.not_before = Time.now
- cert.not_after = expires_in.from_now
- cert.public_key = public_key
- cert.serial = 0x0
- cert.version = 2
-
- if certificate_authority
- extension_factory = OpenSSL::X509::ExtensionFactory.new
- extension_factory.subject_certificate = cert
- extension_factory.issuer_certificate = cert
- cert.add_extension(extension_factory.create_extension('subjectKeyIdentifier', 'hash'))
- cert.add_extension(extension_factory.create_extension('basicConstraints', 'CA:TRUE', true))
- cert.add_extension(extension_factory.create_extension('keyUsage', 'cRLSign,keyCertSign', true))
- end
-
- cert.sign(signed_by&.key || key, OpenSSL::Digest::SHA256.new)
-
- new(key, cert)
- end
-
- def initialize(key, cert)
- @key = key
- @cert = cert
- end
- end
- end
- end
-end
diff --git a/lib/gitlab/kubernetes/helm/client_command.rb b/lib/gitlab/kubernetes/helm/client_command.rb
deleted file mode 100644
index a9e93c0c90e..00000000000
--- a/lib/gitlab/kubernetes/helm/client_command.rb
+++ /dev/null
@@ -1,38 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module Kubernetes
- module Helm
- module ClientCommand
- def init_command
- <<~SHELL.chomp
- export HELM_HOST="localhost:44134"
- tiller -listen ${HELM_HOST} -alsologtostderr &
- helm init --client-only
- SHELL
- end
-
- def repository_command
- ['helm', 'repo', 'add', name, repository].shelljoin if repository
- end
-
- private
-
- def repository_update_command
- 'helm repo update'
- end
-
- def optional_tls_flags
- return [] unless files.key?(:'ca.pem')
-
- [
- '--tls',
- '--tls-ca-cert', "#{files_dir}/ca.pem",
- '--tls-cert', "#{files_dir}/cert.pem",
- '--tls-key', "#{files_dir}/key.pem"
- ]
- end
- end
- end
- end
-end
diff --git a/lib/gitlab/kubernetes/helm/delete_command.rb b/lib/gitlab/kubernetes/helm/delete_command.rb
deleted file mode 100644
index f8b9601bc98..00000000000
--- a/lib/gitlab/kubernetes/helm/delete_command.rb
+++ /dev/null
@@ -1,36 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module Kubernetes
- module Helm
- class DeleteCommand < BaseCommand
- include ClientCommand
-
- attr_reader :predelete, :postdelete
-
- def initialize(predelete: nil, postdelete: nil, **args)
- super(**args)
- @predelete = predelete
- @postdelete = postdelete
- end
-
- def generate_script
- super + [
- init_command,
- predelete,
- delete_command,
- postdelete
- ].compact.join("\n")
- end
-
- def pod_name
- "uninstall-#{name}"
- end
-
- def delete_command
- ['helm', 'delete', '--purge', name].shelljoin
- end
- end
- end
- end
-end
diff --git a/lib/gitlab/kubernetes/helm/init_command.rb b/lib/gitlab/kubernetes/helm/init_command.rb
deleted file mode 100644
index e4844e255c5..00000000000
--- a/lib/gitlab/kubernetes/helm/init_command.rb
+++ /dev/null
@@ -1,43 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module Kubernetes
- module Helm
- class InitCommand < BaseCommand
- def generate_script
- super + [
- init_helm_command
- ].join("\n")
- end
-
- private
-
- def init_helm_command
- command = %w[helm init] + init_command_flags
-
- command.shelljoin
- end
-
- def init_command_flags
- tls_flags + optional_service_account_flag
- end
-
- def tls_flags
- [
- '--tiller-tls',
- '--tiller-tls-verify',
- '--tls-ca-cert', "#{files_dir}/ca.pem",
- '--tiller-tls-cert', "#{files_dir}/cert.pem",
- '--tiller-tls-key', "#{files_dir}/key.pem"
- ]
- end
-
- def optional_service_account_flag
- return [] unless rbac?
-
- ['--service-account', service_account_name]
- end
- end
- end
- end
-end
diff --git a/lib/gitlab/kubernetes/helm/install_command.rb b/lib/gitlab/kubernetes/helm/install_command.rb
deleted file mode 100644
index d166842fce6..00000000000
--- a/lib/gitlab/kubernetes/helm/install_command.rb
+++ /dev/null
@@ -1,85 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module Kubernetes
- module Helm
- class InstallCommand < BaseCommand
- include ClientCommand
-
- attr_reader :chart, :repository, :preinstall, :postinstall
- attr_accessor :version
-
- def initialize(chart:, version: nil, repository: nil, preinstall: nil, postinstall: nil, **args)
- super(**args)
- @chart = chart
- @version = version
- @repository = repository
- @preinstall = preinstall
- @postinstall = postinstall
- end
-
- def generate_script
- super + [
- init_command,
- repository_command,
- repository_update_command,
- preinstall,
- install_command,
- postinstall
- ].compact.join("\n")
- end
-
- private
-
- # Uses `helm upgrade --install` which means we can use this for both
- # installation and uprade of applications
- def install_command
- command = ['helm', 'upgrade', name, chart] +
- install_flag +
- rollback_support_flag +
- reset_values_flag +
- optional_version_flag +
- rbac_create_flag +
- namespace_flag +
- value_flag
-
- command.shelljoin
- end
-
- def install_flag
- ['--install']
- end
-
- def reset_values_flag
- ['--reset-values']
- end
-
- def value_flag
- ['-f', "/data/helm/#{name}/config/values.yaml"]
- end
-
- def namespace_flag
- ['--namespace', Gitlab::Kubernetes::Helm::NAMESPACE]
- end
-
- def rbac_create_flag
- if rbac?
- %w[--set rbac.create=true,rbac.enabled=true]
- else
- %w[--set rbac.create=false,rbac.enabled=false]
- end
- end
-
- def optional_version_flag
- return [] unless version
-
- ['--version', version]
- end
-
- def rollback_support_flag
- ['--atomic', '--cleanup-on-fail']
- end
- end
- end
- end
-end
diff --git a/lib/gitlab/kubernetes/helm/patch_command.rb b/lib/gitlab/kubernetes/helm/patch_command.rb
deleted file mode 100644
index a33dbdac134..00000000000
--- a/lib/gitlab/kubernetes/helm/patch_command.rb
+++ /dev/null
@@ -1,65 +0,0 @@
-# frozen_string_literal: true
-
-# PatchCommand is for updating values in installed charts without overwriting
-# existing values.
-module Gitlab
- module Kubernetes
- module Helm
- class PatchCommand < BaseCommand
- include ClientCommand
-
- attr_reader :chart, :repository
- attr_accessor :version
-
- def initialize(chart:, version:, repository: nil, **args)
- super(**args)
-
- # version is mandatory to prevent chart mismatches
- # we do not want our values interpreted in the context of the wrong version
- raise ArgumentError, 'version is required' if version.blank?
-
- @chart = chart
- @version = version
- @repository = repository
- end
-
- def generate_script
- super + [
- init_command,
- repository_command,
- repository_update_command,
- upgrade_command
- ].compact.join("\n")
- end
-
- private
-
- def upgrade_command
- command = ['helm', 'upgrade', name, chart] +
- reuse_values_flag +
- version_flag +
- namespace_flag +
- value_flag
-
- command.shelljoin
- end
-
- def reuse_values_flag
- ['--reuse-values']
- end
-
- def value_flag
- ['-f', "/data/helm/#{name}/config/values.yaml"]
- end
-
- def namespace_flag
- ['--namespace', Gitlab::Kubernetes::Helm::NAMESPACE]
- end
-
- def version_flag
- ['--version', version]
- end
- end
- end
- end
-end
diff --git a/lib/gitlab/kubernetes/helm/pod.rb b/lib/gitlab/kubernetes/helm/pod.rb
index 75484f80070..9d0207e6b1f 100644
--- a/lib/gitlab/kubernetes/helm/pod.rb
+++ b/lib/gitlab/kubernetes/helm/pod.rb
@@ -27,7 +27,7 @@ module Gitlab
def container_specification
{
name: 'helm',
- image: "registry.gitlab.com/gitlab-org/cluster-integration/helm-install-image/releases/#{Gitlab::Kubernetes::Helm::HELM_VERSION}-kube-#{Gitlab::Kubernetes::Helm::KUBECTL_VERSION}",
+ image: "registry.gitlab.com/gitlab-org/cluster-integration/helm-install-image/releases/#{command.class::HELM_VERSION}-kube-#{Gitlab::Kubernetes::Helm::KUBECTL_VERSION}-alpine-3.12",
env: generate_pod_env(command),
command: %w(/bin/sh),
args: %w(-c $(COMMAND_SCRIPT))
@@ -50,11 +50,10 @@ module Gitlab
end
def generate_pod_env(command)
- {
- HELM_VERSION: Gitlab::Kubernetes::Helm::HELM_VERSION,
- TILLER_NAMESPACE: namespace_name,
+ command.env.merge(
+ HELM_VERSION: command.class::HELM_VERSION,
COMMAND_SCRIPT: command.generate_script
- }.map { |key, value| { name: key, value: value } }
+ ).map { |key, value| { name: key, value: value } }
end
def volumes_specification
diff --git a/lib/gitlab/kubernetes/helm/reset_command.rb b/lib/gitlab/kubernetes/helm/reset_command.rb
deleted file mode 100644
index f1f7938039c..00000000000
--- a/lib/gitlab/kubernetes/helm/reset_command.rb
+++ /dev/null
@@ -1,48 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module Kubernetes
- module Helm
- class ResetCommand < BaseCommand
- include ClientCommand
-
- def generate_script
- super + [
- reset_helm_command,
- delete_tiller_replicaset,
- delete_tiller_clusterrolebinding
- ].join("\n")
- end
-
- def pod_name
- "uninstall-#{name}"
- end
-
- private
-
- # This method can be delete once we upgrade Helm to > 12.13.0
- # https://gitlab.com/gitlab-org/gitlab-foss/merge_requests/27096#note_159695900
- #
- # Tracking this method to be removed here:
- # https://gitlab.com/gitlab-org/gitlab-foss/issues/52791#note_199374155
- def delete_tiller_replicaset
- delete_args = %w[replicaset -n gitlab-managed-apps -l name=tiller]
-
- Gitlab::Kubernetes::KubectlCmd.delete(*delete_args)
- end
-
- def delete_tiller_clusterrolebinding
- delete_args = %w[clusterrolebinding tiller-admin]
-
- Gitlab::Kubernetes::KubectlCmd.delete(*delete_args)
- end
-
- def reset_helm_command
- command = %w[helm reset] + optional_tls_flags
-
- command.shelljoin
- end
- end
- end
- end
-end
diff --git a/lib/gitlab/kubernetes/helm/v2/base_command.rb b/lib/gitlab/kubernetes/helm/v2/base_command.rb
new file mode 100644
index 00000000000..931c2248310
--- /dev/null
+++ b/lib/gitlab/kubernetes/helm/v2/base_command.rb
@@ -0,0 +1,93 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Kubernetes
+ module Helm
+ module V2
+ class BaseCommand
+ attr_reader :name, :files
+
+ HELM_VERSION = '2.16.9'
+
+ def initialize(rbac:, name:, files:)
+ @rbac = rbac
+ @name = name
+ @files = files
+ end
+
+ def env
+ { TILLER_NAMESPACE: namespace }
+ end
+
+ def rbac?
+ @rbac
+ end
+
+ def pod_resource
+ pod_service_account_name = rbac? ? service_account_name : nil
+
+ Gitlab::Kubernetes::Helm::Pod.new(self, namespace, service_account_name: pod_service_account_name).generate
+ end
+
+ def generate_script
+ <<~HEREDOC
+ set -xeo pipefail
+ HEREDOC
+ end
+
+ def pod_name
+ "install-#{name}"
+ end
+
+ def config_map_resource
+ Gitlab::Kubernetes::ConfigMap.new(name, files).generate
+ end
+
+ def service_account_resource
+ return unless rbac?
+
+ Gitlab::Kubernetes::ServiceAccount.new(service_account_name, namespace).generate
+ end
+
+ def cluster_role_binding_resource
+ return unless rbac?
+
+ subjects = [{ kind: 'ServiceAccount', name: service_account_name, namespace: namespace }]
+
+ Gitlab::Kubernetes::ClusterRoleBinding.new(
+ cluster_role_binding_name,
+ cluster_role_name,
+ subjects
+ ).generate
+ end
+
+ def file_names
+ files.keys
+ end
+
+ private
+
+ def files_dir
+ "/data/helm/#{name}/config"
+ end
+
+ def namespace
+ Gitlab::Kubernetes::Helm::NAMESPACE
+ end
+
+ def service_account_name
+ Gitlab::Kubernetes::Helm::SERVICE_ACCOUNT
+ end
+
+ def cluster_role_binding_name
+ Gitlab::Kubernetes::Helm::CLUSTER_ROLE_BINDING
+ end
+
+ def cluster_role_name
+ Gitlab::Kubernetes::Helm::CLUSTER_ROLE
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/kubernetes/helm/v2/certificate.rb b/lib/gitlab/kubernetes/helm/v2/certificate.rb
new file mode 100644
index 00000000000..f603ff44ef3
--- /dev/null
+++ b/lib/gitlab/kubernetes/helm/v2/certificate.rb
@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+module Gitlab
+ module Kubernetes
+ module Helm
+ module V2
+ class Certificate
+ INFINITE_EXPIRY = 1000.years
+ SHORT_EXPIRY = 30.minutes
+
+ attr_reader :key, :cert
+
+ def key_string
+ @key.to_s
+ end
+
+ def cert_string
+ @cert.to_pem
+ end
+
+ def self.from_strings(key_string, cert_string)
+ key = OpenSSL::PKey::RSA.new(key_string)
+ cert = OpenSSL::X509::Certificate.new(cert_string)
+ new(key, cert)
+ end
+
+ def self.generate_root
+ _issue(signed_by: nil, expires_in: INFINITE_EXPIRY, certificate_authority: true)
+ end
+
+ def issue(expires_in: SHORT_EXPIRY)
+ self.class._issue(signed_by: self, expires_in: expires_in, certificate_authority: false)
+ end
+
+ private
+
+ def self._issue(signed_by:, expires_in:, certificate_authority:)
+ key = OpenSSL::PKey::RSA.new(4096)
+ public_key = key.public_key
+
+ subject = OpenSSL::X509::Name.parse("/C=US")
+
+ cert = OpenSSL::X509::Certificate.new
+ cert.subject = subject
+
+ cert.issuer = signed_by&.cert&.subject || subject
+
+ cert.not_before = Time.now.utc
+ cert.not_after = expires_in.from_now.utc
+ cert.public_key = public_key
+ cert.serial = 0x0
+ cert.version = 2
+
+ if certificate_authority
+ extension_factory = OpenSSL::X509::ExtensionFactory.new
+ extension_factory.subject_certificate = cert
+ extension_factory.issuer_certificate = cert
+ cert.add_extension(extension_factory.create_extension('subjectKeyIdentifier', 'hash'))
+ cert.add_extension(extension_factory.create_extension('basicConstraints', 'CA:TRUE', true))
+ cert.add_extension(extension_factory.create_extension('keyUsage', 'cRLSign,keyCertSign', true))
+ end
+
+ cert.sign(signed_by&.key || key, OpenSSL::Digest::SHA256.new)
+
+ new(key, cert)
+ end
+
+ def initialize(key, cert)
+ @key = key
+ @cert = cert
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/kubernetes/helm/v2/client_command.rb b/lib/gitlab/kubernetes/helm/v2/client_command.rb
new file mode 100644
index 00000000000..88693a28d6c
--- /dev/null
+++ b/lib/gitlab/kubernetes/helm/v2/client_command.rb
@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Kubernetes
+ module Helm
+ module V2
+ module ClientCommand
+ def init_command
+ <<~SHELL.chomp
+ export HELM_HOST="localhost:44134"
+ tiller -listen ${HELM_HOST} -alsologtostderr &
+ helm init --client-only
+ SHELL
+ end
+
+ def repository_command
+ ['helm', 'repo', 'add', name, repository].shelljoin if repository
+ end
+
+ private
+
+ def repository_update_command
+ 'helm repo update'
+ end
+
+ def optional_tls_flags
+ return [] unless files.key?(:'ca.pem')
+
+ [
+ '--tls',
+ '--tls-ca-cert', "#{files_dir}/ca.pem",
+ '--tls-cert', "#{files_dir}/cert.pem",
+ '--tls-key', "#{files_dir}/key.pem"
+ ]
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/kubernetes/helm/v2/delete_command.rb b/lib/gitlab/kubernetes/helm/v2/delete_command.rb
new file mode 100644
index 00000000000..4d52fc1398f
--- /dev/null
+++ b/lib/gitlab/kubernetes/helm/v2/delete_command.rb
@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Kubernetes
+ module Helm
+ module V2
+ class DeleteCommand < BaseCommand
+ include ClientCommand
+
+ attr_reader :predelete, :postdelete
+
+ def initialize(predelete: nil, postdelete: nil, **args)
+ super(**args)
+ @predelete = predelete
+ @postdelete = postdelete
+ end
+
+ def generate_script
+ super + [
+ init_command,
+ predelete,
+ delete_command,
+ postdelete
+ ].compact.join("\n")
+ end
+
+ def pod_name
+ "uninstall-#{name}"
+ end
+
+ def delete_command
+ ['helm', 'delete', '--purge', name].shelljoin
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/kubernetes/helm/v2/init_command.rb b/lib/gitlab/kubernetes/helm/v2/init_command.rb
new file mode 100644
index 00000000000..f8b52feb5b6
--- /dev/null
+++ b/lib/gitlab/kubernetes/helm/v2/init_command.rb
@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Kubernetes
+ module Helm
+ module V2
+ class InitCommand < BaseCommand
+ def generate_script
+ super + [
+ init_helm_command
+ ].join("\n")
+ end
+
+ private
+
+ def init_helm_command
+ command = %w[helm init] + init_command_flags
+
+ command.shelljoin
+ end
+
+ def init_command_flags
+ tls_flags + optional_service_account_flag
+ end
+
+ def tls_flags
+ [
+ '--tiller-tls',
+ '--tiller-tls-verify',
+ '--tls-ca-cert', "#{files_dir}/ca.pem",
+ '--tiller-tls-cert', "#{files_dir}/cert.pem",
+ '--tiller-tls-key', "#{files_dir}/key.pem"
+ ]
+ end
+
+ def optional_service_account_flag
+ return [] unless rbac?
+
+ ['--service-account', service_account_name]
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/kubernetes/helm/v2/install_command.rb b/lib/gitlab/kubernetes/helm/v2/install_command.rb
new file mode 100644
index 00000000000..10e16723e45
--- /dev/null
+++ b/lib/gitlab/kubernetes/helm/v2/install_command.rb
@@ -0,0 +1,87 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Kubernetes
+ module Helm
+ module V2
+ class InstallCommand < BaseCommand
+ include ClientCommand
+
+ attr_reader :chart, :repository, :preinstall, :postinstall
+ attr_accessor :version
+
+ def initialize(chart:, version: nil, repository: nil, preinstall: nil, postinstall: nil, **args)
+ super(**args)
+ @chart = chart
+ @version = version
+ @repository = repository
+ @preinstall = preinstall
+ @postinstall = postinstall
+ end
+
+ def generate_script
+ super + [
+ init_command,
+ repository_command,
+ repository_update_command,
+ preinstall,
+ install_command,
+ postinstall
+ ].compact.join("\n")
+ end
+
+ private
+
+ # Uses `helm upgrade --install` which means we can use this for both
+ # installation and uprade of applications
+ def install_command
+ command = ['helm', 'upgrade', name, chart] +
+ install_flag +
+ rollback_support_flag +
+ reset_values_flag +
+ optional_version_flag +
+ rbac_create_flag +
+ namespace_flag +
+ value_flag
+
+ command.shelljoin
+ end
+
+ def install_flag
+ ['--install']
+ end
+
+ def reset_values_flag
+ ['--reset-values']
+ end
+
+ def value_flag
+ ['-f', "/data/helm/#{name}/config/values.yaml"]
+ end
+
+ def namespace_flag
+ ['--namespace', Gitlab::Kubernetes::Helm::NAMESPACE]
+ end
+
+ def rbac_create_flag
+ if rbac?
+ %w[--set rbac.create=true,rbac.enabled=true]
+ else
+ %w[--set rbac.create=false,rbac.enabled=false]
+ end
+ end
+
+ def optional_version_flag
+ return [] unless version
+
+ ['--version', version]
+ end
+
+ def rollback_support_flag
+ ['--atomic', '--cleanup-on-fail']
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/kubernetes/helm/v2/patch_command.rb b/lib/gitlab/kubernetes/helm/v2/patch_command.rb
new file mode 100644
index 00000000000..2855e6444b1
--- /dev/null
+++ b/lib/gitlab/kubernetes/helm/v2/patch_command.rb
@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+# PatchCommand is for updating values in installed charts without overwriting
+# existing values.
+module Gitlab
+ module Kubernetes
+ module Helm
+ module V2
+ class PatchCommand < BaseCommand
+ include ClientCommand
+
+ attr_reader :chart, :repository
+ attr_accessor :version
+
+ def initialize(chart:, version:, repository: nil, **args)
+ super(**args)
+
+ # version is mandatory to prevent chart mismatches
+ # we do not want our values interpreted in the context of the wrong version
+ raise ArgumentError, 'version is required' if version.blank?
+
+ @chart = chart
+ @version = version
+ @repository = repository
+ end
+
+ def generate_script
+ super + [
+ init_command,
+ repository_command,
+ repository_update_command,
+ upgrade_command
+ ].compact.join("\n")
+ end
+
+ private
+
+ def upgrade_command
+ command = ['helm', 'upgrade', name, chart] +
+ reuse_values_flag +
+ version_flag +
+ namespace_flag +
+ value_flag
+
+ command.shelljoin
+ end
+
+ def reuse_values_flag
+ ['--reuse-values']
+ end
+
+ def value_flag
+ ['-f', "/data/helm/#{name}/config/values.yaml"]
+ end
+
+ def namespace_flag
+ ['--namespace', Gitlab::Kubernetes::Helm::NAMESPACE]
+ end
+
+ def version_flag
+ ['--version', version]
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/kubernetes/helm/v2/reset_command.rb b/lib/gitlab/kubernetes/helm/v2/reset_command.rb
new file mode 100644
index 00000000000..172a0884c49
--- /dev/null
+++ b/lib/gitlab/kubernetes/helm/v2/reset_command.rb
@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Kubernetes
+ module Helm
+ module V2
+ class ResetCommand < BaseCommand
+ include ClientCommand
+
+ def generate_script
+ super + [
+ reset_helm_command,
+ delete_tiller_replicaset,
+ delete_tiller_clusterrolebinding
+ ].join("\n")
+ end
+
+ def pod_name
+ "uninstall-#{name}"
+ end
+
+ private
+
+ # This method can be delete once we upgrade Helm to > 12.13.0
+ # https://gitlab.com/gitlab-org/gitlab-foss/merge_requests/27096#note_159695900
+ #
+ # Tracking this method to be removed here:
+ # https://gitlab.com/gitlab-org/gitlab-foss/issues/52791#note_199374155
+ def delete_tiller_replicaset
+ delete_args = %w[replicaset -n gitlab-managed-apps -l name=tiller]
+
+ Gitlab::Kubernetes::KubectlCmd.delete(*delete_args)
+ end
+
+ def delete_tiller_clusterrolebinding
+ delete_args = %w[clusterrolebinding tiller-admin]
+
+ Gitlab::Kubernetes::KubectlCmd.delete(*delete_args)
+ end
+
+ def reset_helm_command
+ command = %w[helm reset] + optional_tls_flags
+
+ command.shelljoin
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/kubernetes/helm/v3/base_command.rb b/lib/gitlab/kubernetes/helm/v3/base_command.rb
new file mode 100644
index 00000000000..ca1bf5462f0
--- /dev/null
+++ b/lib/gitlab/kubernetes/helm/v3/base_command.rb
@@ -0,0 +1,101 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Kubernetes
+ module Helm
+ module V3
+ class BaseCommand
+ attr_reader :name, :files
+
+ HELM_VERSION = '3.2.4'
+
+ def initialize(rbac:, name:, files:)
+ @rbac = rbac
+ @name = name
+ @files = files
+ end
+
+ def env
+ {}
+ end
+
+ def rbac?
+ @rbac
+ end
+
+ def pod_resource
+ pod_service_account_name = rbac? ? service_account_name : nil
+
+ Gitlab::Kubernetes::Helm::Pod.new(self, namespace, service_account_name: pod_service_account_name).generate
+ end
+
+ def generate_script
+ <<~HEREDOC
+ set -xeo pipefail
+ HEREDOC
+ end
+
+ def pod_name
+ "install-#{name}"
+ end
+
+ def config_map_resource
+ Gitlab::Kubernetes::ConfigMap.new(name, files).generate
+ end
+
+ def service_account_resource
+ return unless rbac?
+
+ Gitlab::Kubernetes::ServiceAccount.new(service_account_name, namespace).generate
+ end
+
+ def cluster_role_binding_resource
+ return unless rbac?
+
+ subjects = [{ kind: 'ServiceAccount', name: service_account_name, namespace: namespace }]
+
+ Gitlab::Kubernetes::ClusterRoleBinding.new(
+ cluster_role_binding_name,
+ cluster_role_name,
+ subjects
+ ).generate
+ end
+
+ def file_names
+ files.keys
+ end
+
+ def repository_command
+ ['helm', 'repo', 'add', name, repository].shelljoin if repository
+ end
+
+ private
+
+ def repository_update_command
+ 'helm repo update'
+ end
+
+ def namespace_flag
+ ['--namespace', Gitlab::Kubernetes::Helm::NAMESPACE]
+ end
+
+ def namespace
+ Gitlab::Kubernetes::Helm::NAMESPACE
+ end
+
+ def service_account_name
+ Gitlab::Kubernetes::Helm::SERVICE_ACCOUNT
+ end
+
+ def cluster_role_binding_name
+ Gitlab::Kubernetes::Helm::CLUSTER_ROLE_BINDING
+ end
+
+ def cluster_role_name
+ Gitlab::Kubernetes::Helm::CLUSTER_ROLE
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/kubernetes/helm/v3/delete_command.rb b/lib/gitlab/kubernetes/helm/v3/delete_command.rb
new file mode 100644
index 00000000000..f628e852f54
--- /dev/null
+++ b/lib/gitlab/kubernetes/helm/v3/delete_command.rb
@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Kubernetes
+ module Helm
+ module V3
+ class DeleteCommand < BaseCommand
+ attr_reader :predelete, :postdelete
+
+ def initialize(predelete: nil, postdelete: nil, **args)
+ super(**args)
+ @predelete = predelete
+ @postdelete = postdelete
+ end
+
+ def generate_script
+ super + [
+ predelete,
+ delete_command,
+ postdelete
+ ].compact.join("\n")
+ end
+
+ def pod_name
+ "uninstall-#{name}"
+ end
+
+ def delete_command
+ ['helm', 'uninstall', name, *namespace_flag].shelljoin
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/kubernetes/helm/v3/install_command.rb b/lib/gitlab/kubernetes/helm/v3/install_command.rb
new file mode 100644
index 00000000000..20d17f49115
--- /dev/null
+++ b/lib/gitlab/kubernetes/helm/v3/install_command.rb
@@ -0,0 +1,80 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Kubernetes
+ module Helm
+ module V3
+ class InstallCommand < BaseCommand
+ attr_reader :chart, :repository, :preinstall, :postinstall
+ attr_accessor :version
+
+ def initialize(chart:, version: nil, repository: nil, preinstall: nil, postinstall: nil, **args)
+ super(**args)
+ @chart = chart
+ @version = version
+ @repository = repository
+ @preinstall = preinstall
+ @postinstall = postinstall
+ end
+
+ def generate_script
+ super + [
+ repository_command,
+ repository_update_command,
+ preinstall,
+ install_command,
+ postinstall
+ ].compact.join("\n")
+ end
+
+ private
+
+ # Uses `helm upgrade --install` which means we can use this for both
+ # installation and uprade of applications
+ def install_command
+ command = ['helm', 'upgrade', name, chart] +
+ install_flag +
+ rollback_support_flag +
+ reset_values_flag +
+ optional_version_flag +
+ rbac_create_flag +
+ namespace_flag +
+ value_flag
+
+ command.shelljoin
+ end
+
+ def install_flag
+ ['--install']
+ end
+
+ def reset_values_flag
+ ['--reset-values']
+ end
+
+ def value_flag
+ ['-f', "/data/helm/#{name}/config/values.yaml"]
+ end
+
+ def rbac_create_flag
+ if rbac?
+ %w[--set rbac.create=true,rbac.enabled=true]
+ else
+ %w[--set rbac.create=false,rbac.enabled=false]
+ end
+ end
+
+ def optional_version_flag
+ return [] unless version
+
+ ['--version', version]
+ end
+
+ def rollback_support_flag
+ ['--atomic', '--cleanup-on-fail']
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/kubernetes/helm/v3/patch_command.rb b/lib/gitlab/kubernetes/helm/v3/patch_command.rb
new file mode 100644
index 00000000000..00f340591e7
--- /dev/null
+++ b/lib/gitlab/kubernetes/helm/v3/patch_command.rb
@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+# PatchCommand is for updating values in installed charts without overwriting
+# existing values.
+module Gitlab
+ module Kubernetes
+ module Helm
+ module V3
+ class PatchCommand < BaseCommand
+ attr_reader :chart, :repository
+ attr_accessor :version
+
+ def initialize(chart:, version:, repository: nil, **args)
+ super(**args)
+
+ # version is mandatory to prevent chart mismatches
+ # we do not want our values interpreted in the context of the wrong version
+ raise ArgumentError, 'version is required' if version.blank?
+
+ @chart = chart
+ @version = version
+ @repository = repository
+ end
+
+ def generate_script
+ super + [
+ repository_command,
+ repository_update_command,
+ upgrade_command
+ ].compact.join("\n")
+ end
+
+ private
+
+ def upgrade_command
+ command = ['helm', 'upgrade', name, chart] +
+ reuse_values_flag +
+ version_flag +
+ namespace_flag +
+ value_flag
+
+ command.shelljoin
+ end
+
+ def reuse_values_flag
+ ['--reuse-values']
+ end
+
+ def value_flag
+ ['-f', "/data/helm/#{name}/config/values.yaml"]
+ end
+
+ def version_flag
+ ['--version', version]
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/kubernetes/kube_client.rb b/lib/gitlab/kubernetes/kube_client.rb
index 13cd6dcad3f..a25f005d81e 100644
--- a/lib/gitlab/kubernetes/kube_client.rb
+++ b/lib/gitlab/kubernetes/kube_client.rb
@@ -61,18 +61,11 @@ module Gitlab
# RBAC methods delegates to the apis/rbac.authorization.k8s.io api
# group client
delegate :update_cluster_role_binding,
- to: :rbac_client
-
- # RBAC methods delegates to the apis/rbac.authorization.k8s.io api
- # group client
- delegate :create_role,
- :get_role,
- :update_role,
- to: :rbac_client
-
- # RBAC methods delegates to the apis/rbac.authorization.k8s.io api
- # group client
- delegate :update_role_binding,
+ :create_role,
+ :get_role,
+ :update_role,
+ :delete_role_binding,
+ :update_role_binding,
to: :rbac_client
# non-entity methods that can only work with the core client
@@ -182,10 +175,21 @@ module Gitlab
end
end
+ def patch_ingress(*args)
+ extensions_client.discover unless extensions_client.discovered
+
+ if extensions_client.respond_to?(:patch_ingress)
+ extensions_client.patch_ingress(*args)
+ else
+ networking_client.patch_ingress(*args)
+ end
+ end
+
def create_or_update_cluster_role_binding(resource)
update_cluster_role_binding(resource)
end
+ # Note that we cannot update roleRef as that is immutable
def create_or_update_role_binding(resource)
update_role_binding(resource)
end
diff --git a/lib/gitlab/legacy_github_import/client.rb b/lib/gitlab/legacy_github_import/client.rb
index f7eaafeb446..4482610523e 100644
--- a/lib/gitlab/legacy_github_import/client.rb
+++ b/lib/gitlab/legacy_github_import/client.rb
@@ -24,6 +24,7 @@ module Gitlab
@api ||= ::Octokit::Client.new(
access_token: access_token,
api_endpoint: api_endpoint,
+ web_endpoint: web_endpoint,
# If there is no config, we're connecting to github.com and we
# should verify ssl.
connection_options: {
@@ -85,6 +86,10 @@ module Gitlab
end
end
+ def web_endpoint
+ host.presence || ::Octokit::Default.web_endpoint
+ end
+
def config
Gitlab::Auth::OAuth::Provider.config_for('github')
end
diff --git a/lib/gitlab/legacy_github_import/importer.rb b/lib/gitlab/legacy_github_import/importer.rb
index 3f9fd1b1a19..a17e3b1ad5c 100644
--- a/lib/gitlab/legacy_github_import/importer.rb
+++ b/lib/gitlab/legacy_github_import/importer.rb
@@ -36,7 +36,7 @@ module Gitlab
}
end
- @client = Client.new(credentials[:user], opts)
+ @client = Client.new(credentials[:user], **opts)
end
def execute
@@ -303,6 +303,8 @@ module Gitlab
end
imported!(resource_type)
+ rescue ::Octokit::NotFound => e
+ errors << { type: resource_type, errors: e.message }
end
def imported?(resource_type)
diff --git a/lib/gitlab/metrics/requests_rack_middleware.rb b/lib/gitlab/metrics/requests_rack_middleware.rb
index f6bda0dbea4..23d7eb67312 100644
--- a/lib/gitlab/metrics/requests_rack_middleware.rb
+++ b/lib/gitlab/metrics/requests_rack_middleware.rb
@@ -3,43 +3,70 @@
module Gitlab
module Metrics
class RequestsRackMiddleware
- HTTP_METHODS = %w(delete get head options patch post put).to_set.freeze
+ HTTP_METHODS = {
+ "delete" => %w(200 202 204 303 400 401 403 404 500 503),
+ "get" => %w(200 204 301 302 303 304 307 400 401 403 404 410 422 429 500 503),
+ "head" => %w(200 204 301 302 303 401 403 404 410 500),
+ "options" => %w(200 404),
+ "patch" => %w(200 202 204 400 403 404 409 416 500),
+ "post" => %w(200 201 202 204 301 302 303 304 400 401 403 404 406 409 410 412 422 429 500 503),
+ "put" => %w(200 202 204 400 401 403 404 405 406 409 410 422 500)
+ }.freeze
HEALTH_ENDPOINT = /^\/-\/(liveness|readiness|health|metrics)\/?$/.freeze
FEATURE_CATEGORY_HEADER = 'X-Gitlab-Feature-Category'
FEATURE_CATEGORY_DEFAULT = 'unknown'
+ # These were the top 5 categories at a point in time, chosen as a
+ # reasonable default. If we initialize every category we'll end up
+ # with an explosion in unused metric combinations, but we want the
+ # most common ones to be always present.
+ FEATURE_CATEGORIES_TO_INITIALIZE = ['authentication_and_authorization',
+ 'code_review', 'continuous_integration',
+ 'not_owned', 'source_code_management',
+ FEATURE_CATEGORY_DEFAULT].freeze
+
def initialize(app)
@app = app
end
- def self.http_request_total
- @http_request_total ||= ::Gitlab::Metrics.counter(:http_requests_total, 'Request count')
+ def self.http_requests_total
+ ::Gitlab::Metrics.counter(:http_requests_total, 'Request count')
end
def self.rack_uncaught_errors_count
- @rack_uncaught_errors_count ||= ::Gitlab::Metrics.counter(:rack_uncaught_errors_total, 'Request handling uncaught errors count')
+ ::Gitlab::Metrics.counter(:rack_uncaught_errors_total, 'Request handling uncaught errors count')
end
def self.http_request_duration_seconds
- @http_request_duration_seconds ||= ::Gitlab::Metrics.histogram(:http_request_duration_seconds, 'Request handling execution time',
- {}, [0.05, 0.1, 0.25, 0.5, 0.7, 1, 2.5, 5, 10, 25])
+ ::Gitlab::Metrics.histogram(:http_request_duration_seconds, 'Request handling execution time',
+ {}, [0.05, 0.1, 0.25, 0.5, 0.7, 1, 2.5, 5, 10, 25])
end
def self.http_health_requests_total
- @http_health_requests_total ||= ::Gitlab::Metrics.counter(:http_health_requests_total, 'Health endpoint request count')
+ ::Gitlab::Metrics.counter(:http_health_requests_total, 'Health endpoint request count')
end
- def self.initialize_http_request_duration_seconds
- HTTP_METHODS.each do |method|
+ def self.initialize_metrics
+ # This initialization is done to avoid gaps in scraped metrics after
+ # restarts. It makes sure all counters/histograms are available at
+ # process start.
+ #
+ # For example `rate(http_requests_total{status="500"}[1m])` would return
+ # no data until the first 500 error would occur.
+ HTTP_METHODS.each do |method, statuses|
http_request_duration_seconds.get({ method: method })
+
+ statuses.product(FEATURE_CATEGORIES_TO_INITIALIZE) do |status, feature_category|
+ http_requests_total.get({ method: method, status: status, feature_category: feature_category })
+ end
end
end
def call(env)
method = env['REQUEST_METHOD'].downcase
- method = 'INVALID' unless HTTP_METHODS.include?(method)
+ method = 'INVALID' unless HTTP_METHODS.key?(method)
started = Time.now.to_f
health_endpoint = health_endpoint?(env['PATH_INFO'])
status = 'undefined'
@@ -61,9 +88,13 @@ module Gitlab
raise
ensure
if health_endpoint
- RequestsRackMiddleware.http_health_requests_total.increment(status: status, method: method)
+ RequestsRackMiddleware.http_health_requests_total.increment(status: status.to_s, method: method)
else
- RequestsRackMiddleware.http_request_total.increment(status: status, method: method, feature_category: feature_category || FEATURE_CATEGORY_DEFAULT)
+ RequestsRackMiddleware.http_requests_total.increment(
+ status: status.to_s,
+ method: method,
+ feature_category: feature_category.presence || FEATURE_CATEGORY_DEFAULT
+ )
end
end
end
diff --git a/lib/gitlab/middleware/handle_malformed_strings.rb b/lib/gitlab/middleware/handle_malformed_strings.rb
new file mode 100644
index 00000000000..84f7e2e1b14
--- /dev/null
+++ b/lib/gitlab/middleware/handle_malformed_strings.rb
@@ -0,0 +1,103 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Middleware
+ # There is no valid reason for a request to contain a malformed string
+ # so just return HTTP 400 (Bad Request) if we receive one
+ class HandleMalformedStrings
+ include ActionController::HttpAuthentication::Basic
+
+ NULL_BYTE_REGEX = Regexp.new(Regexp.escape("\u0000")).freeze
+
+ attr_reader :app
+
+ def initialize(app)
+ @app = app
+ end
+
+ def call(env)
+ return [400, { 'Content-Type' => 'text/plain' }, ['Bad Request']] if request_contains_malformed_string?(env)
+
+ app.call(env)
+ end
+
+ private
+
+ def request_contains_malformed_string?(env)
+ return false if ENV['DISABLE_REQUEST_VALIDATION'] == '1'
+
+ # Duplicate the env, so it is not modified when accessing the parameters
+ # https://github.com/rails/rails/blob/34991a6ae2fc68347c01ea7382fa89004159e019/actionpack/lib/action_dispatch/http/parameters.rb#L59
+ # The modification causes problems with our multipart middleware
+ request = ActionDispatch::Request.new(env.dup)
+
+ return true if malformed_path?(request.path)
+ return true if credentials_malformed?(request)
+
+ request.params.values.any? do |value|
+ param_has_null_byte?(value)
+ end
+ rescue ActionController::BadRequest
+ # If we can't build an ActionDispatch::Request something's wrong
+ # This would also happen if `#params` contains invalid UTF-8
+ # in this case we'll return a 400
+ #
+ true
+ end
+
+ def malformed_path?(path)
+ string_malformed?(Rack::Utils.unescape(path))
+ rescue ArgumentError
+ # Rack::Utils.unescape raised this, path is malformed.
+ true
+ end
+
+ def credentials_malformed?(request)
+ credentials = if has_basic_credentials?(request)
+ decode_credentials(request).presence
+ else
+ request.authorization.presence
+ end
+
+ return false unless credentials
+
+ string_malformed?(credentials)
+ end
+
+ def param_has_null_byte?(value, depth = 0)
+ # Guard against possible attack sending large amounts of nested params
+ # Should be safe as deeply nested params are highly uncommon.
+ return false if depth > 2
+
+ depth += 1
+
+ if value.respond_to?(:match)
+ string_malformed?(value)
+ elsif value.respond_to?(:values)
+ value.values.any? do |hash_value|
+ param_has_null_byte?(hash_value, depth)
+ end
+ elsif value.is_a?(Array)
+ value.any? do |array_value|
+ param_has_null_byte?(array_value, depth)
+ end
+ else
+ false
+ end
+ end
+
+ def string_malformed?(string)
+ # We're using match rather than include, because that will raise an ArgumentError
+ # when the string contains invalid UTF8
+ #
+ # We try to encode the string from ASCII-8BIT to UTF8. If we failed to do
+ # so for certain characters in the string, those chars are probably incomplete
+ # multibyte characters.
+ string.encode(Encoding::UTF_8).match?(NULL_BYTE_REGEX)
+ rescue ArgumentError, Encoding::UndefinedConversionError
+ # If we're here, we caught a malformed string. Return true
+ true
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/middleware/handle_null_bytes.rb b/lib/gitlab/middleware/handle_null_bytes.rb
deleted file mode 100644
index c88dfb6ee0b..00000000000
--- a/lib/gitlab/middleware/handle_null_bytes.rb
+++ /dev/null
@@ -1,61 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module Middleware
- # There is no valid reason for a request to contain a null byte (U+0000)
- # so just return HTTP 400 (Bad Request) if we receive one
- class HandleNullBytes
- NULL_BYTE_REGEX = Regexp.new(Regexp.escape("\u0000")).freeze
-
- attr_reader :app
-
- def initialize(app)
- @app = app
- end
-
- def call(env)
- return [400, {}, ["Bad Request"]] if request_has_null_byte?(env)
-
- app.call(env)
- end
-
- private
-
- def request_has_null_byte?(request)
- return false if ENV['REJECT_NULL_BYTES'] == "1"
-
- request = Rack::Request.new(request)
-
- request.params.values.any? do |value|
- param_has_null_byte?(value)
- end
- end
-
- def param_has_null_byte?(value, depth = 0)
- # Guard against possible attack sending large amounts of nested params
- # Should be safe as deeply nested params are highly uncommon.
- return false if depth > 2
-
- depth += 1
-
- if value.respond_to?(:match)
- string_contains_null_byte?(value)
- elsif value.respond_to?(:values)
- value.values.any? do |hash_value|
- param_has_null_byte?(hash_value, depth)
- end
- elsif value.is_a?(Array)
- value.any? do |array_value|
- param_has_null_byte?(array_value, depth)
- end
- else
- false
- end
- end
-
- def string_contains_null_byte?(string)
- string.match?(NULL_BYTE_REGEX)
- end
- end
- end
-end
diff --git a/lib/gitlab/middleware/read_only/controller.rb b/lib/gitlab/middleware/read_only/controller.rb
index cfea4aaddf3..101172cdfcc 100644
--- a/lib/gitlab/middleware/read_only/controller.rb
+++ b/lib/gitlab/middleware/read_only/controller.rb
@@ -9,20 +9,19 @@ module Gitlab
APPLICATION_JSON_TYPES = %W{#{APPLICATION_JSON} application/vnd.git-lfs+json}.freeze
ERROR_MESSAGE = 'You cannot perform write operations on a read-only instance'
- WHITELISTED_GIT_ROUTES = {
- 'repositories/git_http' => %w{git_upload_pack git_receive_pack}
+ ALLOWLISTED_GIT_ROUTES = {
+ 'repositories/git_http' => %w{git_upload_pack}
}.freeze
- WHITELISTED_GIT_LFS_ROUTES = {
- 'repositories/lfs_api' => %w{batch},
- 'repositories/lfs_locks_api' => %w{verify create unlock}
+ ALLOWLISTED_GIT_LFS_BATCH_ROUTES = {
+ 'repositories/lfs_api' => %w{batch}
}.freeze
- WHITELISTED_GIT_REVISION_ROUTES = {
+ ALLOWLISTED_GIT_REVISION_ROUTES = {
'projects/compare' => %w{create}
}.freeze
- WHITELISTED_SESSION_ROUTES = {
+ ALLOWLISTED_SESSION_ROUTES = {
'sessions' => %w{destroy},
'admin/sessions' => %w{create destroy}
}.freeze
@@ -55,7 +54,7 @@ module Gitlab
def disallowed_request?
DISALLOWED_METHODS.include?(@env['REQUEST_METHOD']) &&
- !whitelisted_routes
+ !allowlisted_routes
end
def json_request?
@@ -87,8 +86,8 @@ module Gitlab
end
# Overridden in EE module
- def whitelisted_routes
- workhorse_passthrough_route? || internal_route? || lfs_route? || compare_git_revisions_route? || sidekiq_route? || session_route? || graphql_query?
+ def allowlisted_routes
+ workhorse_passthrough_route? || internal_route? || lfs_batch_route? || compare_git_revisions_route? || sidekiq_route? || session_route? || graphql_query?
end
# URL for requests passed through gitlab-workhorse to rails-web
@@ -96,9 +95,9 @@ module Gitlab
def workhorse_passthrough_route?
# Calling route_hash may be expensive. Only do it if we think there's a possible match
return false unless request.post? &&
- request.path.end_with?('.git/git-upload-pack', '.git/git-receive-pack')
+ request.path.end_with?('.git/git-upload-pack')
- WHITELISTED_GIT_ROUTES[route_hash[:controller]]&.include?(route_hash[:action])
+ ALLOWLISTED_GIT_ROUTES[route_hash[:controller]]&.include?(route_hash[:action])
end
def internal_route?
@@ -109,18 +108,16 @@ module Gitlab
# Calling route_hash may be expensive. Only do it if we think there's a possible match
return false unless request.post? && request.path.end_with?('compare')
- WHITELISTED_GIT_REVISION_ROUTES[route_hash[:controller]]&.include?(route_hash[:action])
+ ALLOWLISTED_GIT_REVISION_ROUTES[route_hash[:controller]]&.include?(route_hash[:action])
end
- def lfs_route?
+ # Batch upload requests are blocked in:
+ # https://gitlab.com/gitlab-org/gitlab/blob/master/app/controllers/repositories/lfs_api_controller.rb#L106
+ def lfs_batch_route?
# Calling route_hash may be expensive. Only do it if we think there's a possible match
- unless request.path.end_with?('/info/lfs/objects/batch',
- '/info/lfs/locks', '/info/lfs/locks/verify') ||
- %r{/info/lfs/locks/\d+/unlock\z}.match?(request.path)
- return false
- end
+ return unless request.path.end_with?('/info/lfs/objects/batch')
- WHITELISTED_GIT_LFS_ROUTES[route_hash[:controller]]&.include?(route_hash[:action])
+ ALLOWLISTED_GIT_LFS_BATCH_ROUTES[route_hash[:controller]]&.include?(route_hash[:action])
end
def session_route?
@@ -128,7 +125,7 @@ module Gitlab
return false unless request.post? && request.path.end_with?('/users/sign_out',
'/admin/session', '/admin/session/destroy')
- WHITELISTED_SESSION_ROUTES[route_hash[:controller]]&.include?(route_hash[:action])
+ ALLOWLISTED_SESSION_ROUTES[route_hash[:controller]]&.include?(route_hash[:action])
end
def sidekiq_route?
diff --git a/lib/gitlab/octokit/middleware.rb b/lib/gitlab/octokit/middleware.rb
index 2dd7d08a58b..a3c0fdcf467 100644
--- a/lib/gitlab/octokit/middleware.rb
+++ b/lib/gitlab/octokit/middleware.rb
@@ -8,7 +8,7 @@ module Gitlab
end
def call(env)
- Gitlab::UrlBlocker.validate!(env[:url], { allow_localhost: allow_local_requests?, allow_local_network: allow_local_requests? })
+ Gitlab::UrlBlocker.validate!(env[:url], allow_localhost: allow_local_requests?, allow_local_network: allow_local_requests?)
@app.call(env)
end
diff --git a/lib/gitlab/omniauth_initializer.rb b/lib/gitlab/omniauth_initializer.rb
index b60ecb6631b..541f9b06842 100644
--- a/lib/gitlab/omniauth_initializer.rb
+++ b/lib/gitlab/omniauth_initializer.rb
@@ -96,16 +96,6 @@ module Gitlab
args[:strategy_class] = args[:strategy_class].constantize
end
- # Providers that are known to depend on rack-oauth2, like those using
- # Omniauth::Strategies::OpenIDConnect, need to be quirked so the
- # client_auth_method argument value is passed as a symbol.
- if (args[:strategy_class] == OmniAuth::Strategies::OpenIDConnect ||
- args[:name] == 'openid_connect') &&
- args[:client_auth_method].is_a?(String)
-
- args[:client_auth_method] = args[:client_auth_method].to_sym
- end
-
args
end
diff --git a/lib/gitlab/path_regex.rb b/lib/gitlab/path_regex.rb
index 706c16f6149..ad0a5c80604 100644
--- a/lib/gitlab/path_regex.rb
+++ b/lib/gitlab/path_regex.rb
@@ -49,6 +49,9 @@ module Gitlab
s
search
sent_notifications
+ sitemap
+ sitemap.xml
+ sitemap.xml.gz
slash-command-logo.png
snippets
unsubscribes
@@ -251,6 +254,14 @@ module Gitlab
%r{#{personal_snippet_path_regex}|#{project_snippet_path_regex}}
end
+ def container_image_regex
+ @container_image_regex ||= %r{([\w\.-]+\/){0,1}[\w\.-]+}.freeze
+ end
+
+ def container_image_blob_sha_regex
+ @container_image_blob_sha_regex ||= %r{[\w+.-]+:?\w+}.freeze
+ end
+
private
def personal_snippet_path_regex
diff --git a/lib/gitlab/project_search_results.rb b/lib/gitlab/project_search_results.rb
index 333564bee01..6719dc8362b 100644
--- a/lib/gitlab/project_search_results.rb
+++ b/lib/gitlab/project_search_results.rb
@@ -4,11 +4,11 @@ module Gitlab
class ProjectSearchResults < SearchResults
attr_reader :project, :repository_ref
- def initialize(current_user, query, project:, repository_ref: nil, sort: nil, filters: {})
+ def initialize(current_user, query, project:, repository_ref: nil, order_by: nil, sort: nil, filters: {})
@project = project
@repository_ref = repository_ref.presence
- super(current_user, query, [project], sort: sort, filters: filters)
+ super(current_user, query, [project], order_by: order_by, sort: sort, filters: filters)
end
def objects(scope, page: nil, per_page: DEFAULT_PER_PAGE, preload_method: nil)
@@ -75,15 +75,6 @@ module Gitlab
@commits_count ||= commits(limit: count_limit).count
end
- def single_commit_result?
- return false if commits_count != 1
-
- counts = %i(limited_milestones_count limited_notes_count
- limited_merge_requests_count limited_issues_count
- limited_blobs_count wiki_blobs_count)
- counts.all? { |count_method| public_send(count_method) == 0 } # rubocop:disable GitlabSecurity/PublicSend
- end
-
private
def paginated_commits(page, per_page)
diff --git a/lib/gitlab/quick_actions/extractor.rb b/lib/gitlab/quick_actions/extractor.rb
index dd7a27ead01..1294e475145 100644
--- a/lib/gitlab/quick_actions/extractor.rb
+++ b/lib/gitlab/quick_actions/extractor.rb
@@ -29,9 +29,9 @@ module Gitlab
# Anything, including `/cmd arg` which are ignored by this filter
# `
- ^.*`\n*
+ `\n*
.+?
- \n*`$
+ \n*`
)
}mix.freeze
diff --git a/lib/gitlab/quick_actions/merge_request_actions.rb b/lib/gitlab/quick_actions/merge_request_actions.rb
index c8c949a9363..1986b7a1789 100644
--- a/lib/gitlab/quick_actions/merge_request_actions.rb
+++ b/lib/gitlab/quick_actions/merge_request_actions.rb
@@ -56,21 +56,21 @@ module Gitlab
@updates[:merge] = params[:merge_request_diff_head_sha]
end
- desc 'Toggle the Work In Progress status'
+ desc 'Toggle the Draft status'
explanation do
noun = quick_action_target.to_ability_name.humanize(capitalize: false)
if quick_action_target.work_in_progress?
- _("Unmarks this %{noun} as Work In Progress.")
+ _("Unmarks this %{noun} as a draft.")
else
- _("Marks this %{noun} as Work In Progress.")
+ _("Marks this %{noun} as a draft.")
end % { noun: noun }
end
execution_message do
noun = quick_action_target.to_ability_name.humanize(capitalize: false)
if quick_action_target.work_in_progress?
- _("Unmarked this %{noun} as Work In Progress.")
+ _("Unmarked this %{noun} as a draft.")
else
- _("Marked this %{noun} as Work In Progress.")
+ _("Marked this %{noun} as a draft.")
end % { noun: noun }
end
@@ -80,7 +80,7 @@ module Gitlab
# Allow it to mark as WIP on MR creation page _or_ through MR notes.
(quick_action_target.new_record? || current_user.can?(:"update_#{quick_action_target.to_ability_name}", quick_action_target))
end
- command :wip do
+ command :draft, :wip do
@updates[:wip_event] = quick_action_target.work_in_progress? ? 'unwip' : 'wip'
end
diff --git a/lib/gitlab/redis/wrapper.rb b/lib/gitlab/redis/wrapper.rb
index 5584323789b..6f80c7d439f 100644
--- a/lib/gitlab/redis/wrapper.rb
+++ b/lib/gitlab/redis/wrapper.rb
@@ -18,6 +18,10 @@ module Gitlab
pool.with { |redis| yield redis }
end
+ def version
+ with { |redis| redis.info['redis_version'] }
+ end
+
def pool
@pool ||= ConnectionPool.new(size: pool_size) { ::Redis.new(params) }
end
diff --git a/lib/gitlab/reference_extractor.rb b/lib/gitlab/reference_extractor.rb
index 01aff48b08b..d7501fc7068 100644
--- a/lib/gitlab/reference_extractor.rb
+++ b/lib/gitlab/reference_extractor.rb
@@ -4,7 +4,7 @@ module Gitlab
# Extract possible GFM references from an arbitrary String for further processing.
class ReferenceExtractor < Banzai::ReferenceExtractor
REFERABLES = %i(user issue label milestone mentioned_user mentioned_group mentioned_project
- merge_request snippet commit commit_range directly_addressed_user epic iteration).freeze
+ merge_request snippet commit commit_range directly_addressed_user epic iteration vulnerability).freeze
attr_accessor :project, :current_user, :author
# This counter is increased by a number of references filtered out by
# banzai reference exctractor. Note that this counter is stateful and
@@ -38,7 +38,7 @@ module Gitlab
end
REFERABLES.each do |type|
- define_method("#{type}s") do
+ define_method(type.to_s.pluralize) do
@references[type] ||= references(type)
end
end
diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb
index 1b169b6186b..4ae6297f6f5 100644
--- a/lib/gitlab/regex.rb
+++ b/lib/gitlab/regex.rb
@@ -22,6 +22,10 @@ module Gitlab
@composer_package_version_regex ||= %r{^v?(\d+(\.(\d+|x))*(-.+)?)}.freeze
end
+ def composer_dev_version_regex
+ @composer_dev_version_regex ||= %r{(^dev-)|(-dev$)}.freeze
+ end
+
def package_name_regex
@package_name_regex ||= %r{\A\@?(([\w\-\.\+]*)\/)*([\w\-\.]+)@?(([\w\-\.\+]*)\/)*([\w\-\.]*)\z}.freeze
end
diff --git a/lib/gitlab/repository_size_checker.rb b/lib/gitlab/repository_size_checker.rb
index 03d9f961dd9..0ed31176dd8 100644
--- a/lib/gitlab/repository_size_checker.rb
+++ b/lib/gitlab/repository_size_checker.rb
@@ -32,18 +32,24 @@ module Gitlab
def changes_will_exceed_size_limit?(change_size)
return false unless enabled?
- change_size > limit || exceeded_size(change_size) > 0
+ above_size_limit? || exceeded_size(change_size) > 0
end
# @param change_size [int] in bytes
def exceeded_size(change_size = 0)
- current_size + change_size - limit
+ size = current_size + change_size - limit
+
+ [size, 0].max
end
def error_message
@error_message_object ||= ::Gitlab::RepositorySizeErrorMessage.new(self)
end
+ def additional_repo_storage_available?
+ false
+ end
+
private
attr_reader :namespace
diff --git a/lib/gitlab/repository_size_error_message.rb b/lib/gitlab/repository_size_error_message.rb
index 556190453de..8da840779c9 100644
--- a/lib/gitlab/repository_size_error_message.rb
+++ b/lib/gitlab/repository_size_error_message.rb
@@ -4,7 +4,7 @@ module Gitlab
class RepositorySizeErrorMessage
include ActiveSupport::NumberHelper
- delegate :current_size, :limit, :exceeded_size, to: :@checker
+ delegate :current_size, :limit, :exceeded_size, :additional_repo_storage_available?, to: :@checker
# @param checher [RepositorySizeChecker]
def initialize(checker)
@@ -24,7 +24,11 @@ module Gitlab
end
def new_changes_error
- "Your push to this repository would cause it to exceed the size limit of #{formatted(limit)} so it has been rejected. #{more_info_message}"
+ if additional_repo_storage_available?
+ "Your push to this repository has been rejected because it would exceed storage limits. Please contact your GitLab administrator for more information."
+ else
+ "Your push to this repository would cause it to exceed the size limit of #{formatted(limit)} so it has been rejected. #{more_info_message}"
+ end
end
def more_info_message
diff --git a/lib/gitlab/repository_url_builder.rb b/lib/gitlab/repository_url_builder.rb
index 2b88af1f77c..a2d0d50d20b 100644
--- a/lib/gitlab/repository_url_builder.rb
+++ b/lib/gitlab/repository_url_builder.rb
@@ -4,9 +4,6 @@ module Gitlab
module RepositoryUrlBuilder
class << self
def build(path, protocol: :ssh)
- # TODO: See https://gitlab.com/gitlab-org/gitlab/-/issues/213021
- path = path.sub('@snippets', 'snippets')
-
case protocol
when :ssh
ssh_url(path)
diff --git a/lib/gitlab/robots_txt/parser.rb b/lib/gitlab/robots_txt/parser.rb
index b9a3837e468..604d2f9b35b 100644
--- a/lib/gitlab/robots_txt/parser.rb
+++ b/lib/gitlab/robots_txt/parser.rb
@@ -3,34 +3,68 @@
module Gitlab
module RobotsTxt
class Parser
- attr_reader :disallow_rules
+ DISALLOW_REGEX = /^disallow: /i.freeze
+ ALLOW_REGEX = /^allow: /i.freeze
+
+ attr_reader :disallow_rules, :allow_rules
def initialize(content)
@raw_content = content
- @disallow_rules = parse_raw_content!
+ @disallow_rules, @allow_rules = parse_raw_content!
end
def disallowed?(path)
+ return false if allow_rules.any? { |rule| path =~ rule }
+
disallow_rules.any? { |rule| path =~ rule }
end
private
- # This parser is very basic as it only knows about `Disallow:` lines,
- # and simply ignores all other lines.
+ # This parser is very basic as it only knows about `Disallow:`
+ # and `Allow:` lines, and simply ignores all other lines.
#
- # Order of predecence, 'Allow:`, etc are ignored for now.
+ # Patterns ending in `$`, and `*` for 0 or more characters are recognized.
+ #
+ # It is case insensitive and `Allow` rules takes precedence
+ # over `Disallow`.
def parse_raw_content!
- @raw_content.each_line.map do |line|
- if line.start_with?('Disallow:')
- value = line.sub('Disallow:', '').strip
- value = Regexp.escape(value).gsub('\*', '.*')
- Regexp.new("^#{value}")
- else
- nil
+ disallowed = []
+ allowed = []
+
+ @raw_content.each_line.each do |line|
+ if disallow_rule?(line)
+ disallowed << get_disallow_pattern(line)
+ elsif allow_rule?(line)
+ allowed << get_allow_pattern(line)
end
- end.compact
+ end
+
+ [disallowed, allowed]
+ end
+
+ def disallow_rule?(line)
+ line =~ DISALLOW_REGEX
+ end
+
+ def get_disallow_pattern(line)
+ get_pattern(line, DISALLOW_REGEX)
+ end
+
+ def allow_rule?(line)
+ line =~ ALLOW_REGEX
+ end
+
+ def get_allow_pattern(line)
+ get_pattern(line, ALLOW_REGEX)
+ end
+
+ def get_pattern(line, rule_regex)
+ value = line.sub(rule_regex, '').strip
+ value = Regexp.escape(value).gsub('\*', '.*')
+ value = value.sub(/\\\$$/, '$')
+ Regexp.new("^#{value}")
end
end
end
diff --git a/lib/gitlab/search/found_blob.rb b/lib/gitlab/search/found_blob.rb
index fc1abc064c7..183e582925d 100644
--- a/lib/gitlab/search/found_blob.rb
+++ b/lib/gitlab/search/found_blob.rb
@@ -9,7 +9,7 @@ module Gitlab
include Gitlab::Utils::StrongMemoize
include BlobActiveModel
- attr_reader :project, :content_match, :blob_path
+ attr_reader :project, :content_match, :blob_path, :highlight_line
PATH_REGEXP = /\A(?<ref>[^:]*):(?<path>[^\x00]*)\x00/.freeze
CONTENT_REGEXP = /^(?<ref>[^:]*):(?<path>[^\x00]*)\x00(?<startline>\d+)\x00/.freeze
@@ -26,6 +26,7 @@ module Gitlab
@binary_basename = opts.fetch(:basename, nil)
@ref = opts.fetch(:ref, nil)
@startline = opts.fetch(:startline, nil)
+ @highlight_line = opts.fetch(:highlight_line, nil)
@binary_data = opts.fetch(:data, nil)
@per_page = opts.fetch(:per_page, 20)
@project = opts.fetch(:project, nil)
diff --git a/lib/gitlab/search/sort_options.rb b/lib/gitlab/search/sort_options.rb
new file mode 100644
index 00000000000..3395c34d171
--- /dev/null
+++ b/lib/gitlab/search/sort_options.rb
@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Search
+ module SortOptions
+ def sort_and_direction(order_by, sort)
+ # Due to different uses of sort param in web vs. API requests we prefer
+ # order_by when present
+ case [order_by, sort]
+ when %w[created_at asc], [nil, 'created_asc']
+ :created_at_asc
+ when %w[created_at desc], [nil, 'created_desc']
+ :created_at_desc
+ else
+ :unknown
+ end
+ end
+ module_function :sort_and_direction # rubocop: disable Style/AccessModifierDeclarations
+ end
+ end
+end
diff --git a/lib/gitlab/search_results.rb b/lib/gitlab/search_results.rb
index b81264c5d0c..0091ae1e8ce 100644
--- a/lib/gitlab/search_results.rb
+++ b/lib/gitlab/search_results.rb
@@ -7,7 +7,7 @@ module Gitlab
DEFAULT_PAGE = 1
DEFAULT_PER_PAGE = 20
- attr_reader :current_user, :query, :sort, :filters
+ attr_reader :current_user, :query, :order_by, :sort, :filters
# Limit search results by passed projects
# It allows us to search only for projects user has access to
@@ -19,11 +19,12 @@ module Gitlab
# query
attr_reader :default_project_filter
- def initialize(current_user, query, limit_projects = nil, sort: nil, default_project_filter: false, filters: {})
+ def initialize(current_user, query, limit_projects = nil, order_by: nil, sort: nil, default_project_filter: false, filters: {})
@current_user = current_user
@query = query
@limit_projects = limit_projects || Project.all
@default_project_filter = default_project_filter
+ @order_by = order_by
@sort = sort
@filters = filters
end
@@ -94,10 +95,6 @@ module Gitlab
@limited_users_count ||= limited_count(users)
end
- def single_commit_result?
- false
- end
-
def count_limit
COUNT_LIMIT
end
@@ -132,13 +129,15 @@ module Gitlab
# rubocop: disable CodeReuse/ActiveRecord
def apply_sort(scope)
- case sort
- when 'oldest'
+ # Due to different uses of sort param we prefer order_by when
+ # present
+ case ::Gitlab::Search::SortOptions.sort_and_direction(order_by, sort)
+ when :created_at_asc
scope.reorder('created_at ASC')
- when 'newest'
+ when :created_at_desc
scope.reorder('created_at DESC')
else
- scope
+ scope.reorder('created_at DESC')
end
end
# rubocop: enable CodeReuse/ActiveRecord
@@ -219,7 +218,7 @@ module Gitlab
params[:state] = filters[:state] if filters.key?(:state)
- if [true, false].include?(filters[:confidential]) && Feature.enabled?(:search_filter_by_confidential)
+ if [true, false].include?(filters[:confidential])
params[:confidential] = filters[:confidential]
end
end
diff --git a/lib/gitlab/setup_helper.rb b/lib/gitlab/setup_helper.rb
index 4df6a50c8dd..259d3e300b6 100644
--- a/lib/gitlab/setup_helper.rb
+++ b/lib/gitlab/setup_helper.rb
@@ -99,6 +99,7 @@ module Gitlab
config[:'gitlab-shell'] = { dir: Gitlab.config.gitlab_shell.path }
config[:bin_dir] = Gitlab.config.gitaly.client_path
config[:gitlab] = { url: Gitlab.config.gitlab.url }
+ config[:logging] = { dir: Rails.root.join('log').to_s }
TomlRB.dump(config)
end
diff --git a/lib/gitlab/sidekiq_cluster/cli.rb b/lib/gitlab/sidekiq_cluster/cli.rb
index 1e5d23a8405..e471517c50a 100644
--- a/lib/gitlab/sidekiq_cluster/cli.rb
+++ b/lib/gitlab/sidekiq_cluster/cli.rb
@@ -47,16 +47,24 @@ module Gitlab
option_parser.parse!(argv)
+ # Remove with https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/646
+ if @queue_selector && @experimental_queue_selector
+ raise CommandError,
+ 'You cannot specify --queue-selector and --experimental-queue-selector together'
+ end
+
all_queues = SidekiqConfig::CliMethods.all_queues(@rails_path)
queue_names = SidekiqConfig::CliMethods.worker_queues(@rails_path)
queue_groups = argv.map do |queues|
next queue_names if queues == '*'
- # When using the experimental queue query syntax, we treat
- # each queue group as a worker attribute query, and resolve
- # the queues for the queue group using this query.
- if @experimental_queue_selector
+ # When using the queue query syntax, we treat each queue group
+ # as a worker attribute query, and resolve the queues for the
+ # queue group using this query.
+
+ # Simplify with https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/646
+ if @queue_selector || @experimental_queue_selector
SidekiqConfig::CliMethods.query_workers(queues, all_queues)
else
SidekiqConfig::CliMethods.expand_queues(queues.split(','), queue_names)
@@ -182,7 +190,12 @@ module Gitlab
@rails_path = path
end
- opt.on('--experimental-queue-selector', 'EXPERIMENTAL: Run workers based on the provided selector') do |experimental_queue_selector|
+ opt.on('--queue-selector', 'Run workers based on the provided selector') do |queue_selector|
+ @queue_selector = queue_selector
+ end
+
+ # Remove with https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/646
+ opt.on('--experimental-queue-selector', 'DEPRECATED: use --queue-selector-instead') do |experimental_queue_selector|
@experimental_queue_selector = experimental_queue_selector
end
diff --git a/lib/gitlab/sidekiq_logging/logs_jobs.rb b/lib/gitlab/sidekiq_logging/logs_jobs.rb
index 326dfdae661..dc81c34c4d0 100644
--- a/lib/gitlab/sidekiq_logging/logs_jobs.rb
+++ b/lib/gitlab/sidekiq_logging/logs_jobs.rb
@@ -16,7 +16,7 @@ module Gitlab
# Add process id params
job['pid'] = ::Process.pid
- job.delete('args') unless ENV['SIDEKIQ_LOG_ARGUMENTS']
+ job.delete('args') unless SidekiqLogArguments.enabled?
job
end
diff --git a/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies.rb b/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies.rb
index 6fdef4c354e..63e8bee4443 100644
--- a/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies.rb
+++ b/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies.rb
@@ -8,6 +8,7 @@ module Gitlab
STRATEGIES = {
until_executing: UntilExecuting,
+ until_executed: UntilExecuted,
none: None
}.freeze
diff --git a/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/base.rb b/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/base.rb
new file mode 100644
index 00000000000..df5df590281
--- /dev/null
+++ b/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/base.rb
@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module SidekiqMiddleware
+ module DuplicateJobs
+ module Strategies
+ class Base
+ def initialize(duplicate_job)
+ @duplicate_job = duplicate_job
+ end
+
+ def schedule(job)
+ raise NotImplementedError
+ end
+
+ def perform(_job)
+ raise NotImplementedError
+ end
+
+ private
+
+ attr_reader :duplicate_job
+
+ def strategy_name
+ self.class.name.to_s.demodulize.underscore.humanize.downcase
+ end
+
+ def check!
+ # The default expiry time is the DuplicateJob::DUPLICATE_KEY_TTL already
+ # Only the strategies de-duplicating when scheduling
+ duplicate_job.check!
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/deduplicates_when_scheduling.rb b/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/deduplicates_when_scheduling.rb
new file mode 100644
index 00000000000..59b0e7e29da
--- /dev/null
+++ b/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/deduplicates_when_scheduling.rb
@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module SidekiqMiddleware
+ module DuplicateJobs
+ module Strategies
+ module DeduplicatesWhenScheduling
+ def initialize(duplicate_job)
+ @duplicate_job = duplicate_job
+ end
+
+ def schedule(job)
+ if deduplicatable_job? && check! && duplicate_job.duplicate?
+ job['duplicate-of'] = duplicate_job.existing_jid
+
+ if duplicate_job.droppable?
+ Gitlab::SidekiqLogging::DeduplicationLogger.instance.log(
+ job, "dropped #{strategy_name}", duplicate_job.options)
+ return false
+ end
+ end
+
+ yield
+ end
+
+ private
+
+ def deduplicatable_job?
+ !duplicate_job.scheduled? || duplicate_job.options[:including_scheduled]
+ end
+
+ def check!
+ duplicate_job.check!(expiry)
+ end
+
+ def expiry
+ return DuplicateJob::DUPLICATE_KEY_TTL unless duplicate_job.scheduled?
+
+ time_diff = duplicate_job.scheduled_at.to_i - Time.now.to_i
+
+ time_diff > 0 ? time_diff : DuplicateJob::DUPLICATE_KEY_TTL
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/none.rb b/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/none.rb
index cd101cd16b6..acbe0efaafa 100644
--- a/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/none.rb
+++ b/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/none.rb
@@ -5,10 +5,7 @@ module Gitlab
module DuplicateJobs
module Strategies
# This strategy will never deduplicate a job
- class None
- def initialize(_duplicate_job)
- end
-
+ class None < Base
def schedule(_job)
yield
end
diff --git a/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/until_executed.rb b/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/until_executed.rb
new file mode 100644
index 00000000000..738efa36fc8
--- /dev/null
+++ b/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/until_executed.rb
@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module SidekiqMiddleware
+ module DuplicateJobs
+ module Strategies
+ # This strategy takes a lock before scheduling the job in a queue and
+ # removes the lock after the job has executed preventing a new job to be queued
+ # while a job is still executing.
+ class UntilExecuted < Base
+ include DeduplicatesWhenScheduling
+
+ def perform(_job)
+ yield
+
+ duplicate_job.delete!
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/until_executing.rb b/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/until_executing.rb
index 46ce0eb4a91..68d66383b2b 100644
--- a/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/until_executing.rb
+++ b/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/until_executing.rb
@@ -7,50 +7,14 @@ module Gitlab
# This strategy takes a lock before scheduling the job in a queue and
# removes the lock before the job starts allowing a new job to be queued
# while a job is still executing.
- class UntilExecuting
- def initialize(duplicate_job)
- @duplicate_job = duplicate_job
- end
-
- def schedule(job)
- if deduplicatable_job? && check! && duplicate_job.duplicate?
- job['duplicate-of'] = duplicate_job.existing_jid
-
- if duplicate_job.droppable?
- Gitlab::SidekiqLogging::DeduplicationLogger.instance.log(
- job, "dropped until executing", duplicate_job.options)
- return false
- end
- end
-
- yield
- end
+ class UntilExecuting < Base
+ include DeduplicatesWhenScheduling
def perform(_job)
duplicate_job.delete!
yield
end
-
- private
-
- attr_reader :duplicate_job
-
- def deduplicatable_job?
- !duplicate_job.scheduled? || duplicate_job.options[:including_scheduled]
- end
-
- def check!
- duplicate_job.check!(expiry)
- end
-
- def expiry
- return DuplicateJob::DUPLICATE_KEY_TTL unless duplicate_job.scheduled?
-
- time_diff = duplicate_job.scheduled_at.to_i - Time.now.to_i
-
- time_diff > 0 ? time_diff : DuplicateJob::DUPLICATE_KEY_TTL
- end
end
end
end
diff --git a/lib/gitlab/static_site_editor/config/generated_config.rb b/lib/gitlab/static_site_editor/config/generated_config.rb
index ff24ec69ab0..0a2cee75af7 100644
--- a/lib/gitlab/static_site_editor/config/generated_config.rb
+++ b/lib/gitlab/static_site_editor/config/generated_config.rb
@@ -34,7 +34,7 @@ module Gitlab
delegate :project, to: :repository
def supported_extensions
- %w[.md].freeze
+ %w[.md .md.erb].freeze
end
def commit_id
@@ -50,8 +50,6 @@ module Gitlab
end
def extension_supported?
- return true if path.end_with?('.md.erb') && Feature.enabled?(:sse_erb_support, project)
-
supported_extensions.any? { |ext| path.end_with?(ext) }
end
diff --git a/lib/gitlab/template/base_template.rb b/lib/gitlab/template/base_template.rb
index 50e09bdcdd6..e84937ec4ad 100644
--- a/lib/gitlab/template/base_template.rb
+++ b/lib/gitlab/template/base_template.rb
@@ -105,6 +105,20 @@ module Gitlab
files.map { |t| { name: t.name } }
end
end
+
+ def template_subsets(project = nil)
+ return [] if project && !project.repository.exists?
+
+ if categories.any?
+ categories.keys.map do |category|
+ files = self.by_category(category, project)
+ [category, files.map { |t| { key: t.key, name: t.name, content: t.content } }]
+ end.to_h
+ else
+ files = self.all(project)
+ files.map { |t| { key: t.key, name: t.name, content: t.content } }
+ end
+ end
end
end
end
diff --git a/lib/gitlab/tracking.rb b/lib/gitlab/tracking.rb
index 02d354ec43a..19be468e3d5 100644
--- a/lib/gitlab/tracking.rb
+++ b/lib/gitlab/tracking.rb
@@ -1,7 +1,5 @@
# frozen_string_literal: true
-require 'snowplow-tracker'
-
module Gitlab
module Tracking
SNOWPLOW_NAMESPACE = 'gl'
@@ -27,16 +25,11 @@ module Gitlab
end
def event(category, action, label: nil, property: nil, value: nil, context: nil)
- return unless enabled?
-
- snowplow.track_struct_event(category, action, label, property, value, context, (Time.now.to_f * 1000).to_i)
+ snowplow.event(category, action, label: label, property: property, value: value, context: context)
end
def self_describing_event(schema_url, event_data_json, context: nil)
- return unless enabled?
-
- event_json = SnowplowTracker::SelfDescribingJson.new(schema_url, event_data_json)
- snowplow.track_self_describing_event(event_json, context, (Time.now.to_f * 1000).to_i)
+ snowplow.self_describing_event(schema_url, event_data_json, context: context)
end
def snowplow_options(group)
@@ -54,19 +47,7 @@ module Gitlab
private
def snowplow
- @snowplow ||= SnowplowTracker::Tracker.new(
- emitter,
- SnowplowTracker::Subject.new,
- SNOWPLOW_NAMESPACE,
- Gitlab::CurrentSettings.snowplow_app_id
- )
- end
-
- def emitter
- SnowplowTracker::AsyncEmitter.new(
- Gitlab::CurrentSettings.snowplow_collector_hostname,
- protocol: 'https'
- )
+ @snowplow ||= Gitlab::Tracking::Destinations::Snowplow.new
end
end
end
diff --git a/lib/gitlab/tracking/destinations/base.rb b/lib/gitlab/tracking/destinations/base.rb
new file mode 100644
index 00000000000..00e92e0bd57
--- /dev/null
+++ b/lib/gitlab/tracking/destinations/base.rb
@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Tracking
+ module Destinations
+ class Base
+ def event(category, action, label: nil, property: nil, value: nil, context: nil)
+ raise NotImplementedError, "#{self} does not implement #{__method__}"
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/tracking/destinations/snowplow.rb b/lib/gitlab/tracking/destinations/snowplow.rb
new file mode 100644
index 00000000000..9cebcfe5ee1
--- /dev/null
+++ b/lib/gitlab/tracking/destinations/snowplow.rb
@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+require 'snowplow-tracker'
+
+module Gitlab
+ module Tracking
+ module Destinations
+ class Snowplow < Base
+ extend ::Gitlab::Utils::Override
+
+ override :event
+ def event(category, action, label: nil, property: nil, value: nil, context: nil)
+ return unless enabled?
+
+ tracker.track_struct_event(category, action, label, property, value, context, (Time.now.to_f * 1000).to_i)
+ end
+
+ def self_describing_event(schema_url, event_data_json, context: nil)
+ return unless enabled?
+
+ event_json = SnowplowTracker::SelfDescribingJson.new(schema_url, event_data_json)
+ tracker.track_self_describing_event(event_json, context, (Time.now.to_f * 1000).to_i)
+ end
+
+ private
+
+ def enabled?
+ Gitlab::CurrentSettings.snowplow_enabled?
+ end
+
+ def tracker
+ @tracker ||= SnowplowTracker::Tracker.new(
+ emitter,
+ SnowplowTracker::Subject.new,
+ Gitlab::Tracking::SNOWPLOW_NAMESPACE,
+ Gitlab::CurrentSettings.snowplow_app_id
+ )
+ end
+
+ def emitter
+ SnowplowTracker::AsyncEmitter.new(
+ Gitlab::CurrentSettings.snowplow_collector_hostname,
+ protocol: 'https'
+ )
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/url_blocker.rb b/lib/gitlab/url_blocker.rb
index 9213b5ebab2..eece2c343d2 100644
--- a/lib/gitlab/url_blocker.rb
+++ b/lib/gitlab/url_blocker.rb
@@ -49,7 +49,7 @@ module Gitlab
return [uri, nil] unless address_info
ip_address = ip_address(address_info)
- return [uri, nil] if domain_whitelisted?(uri) || ip_whitelisted?(ip_address, port: get_port(uri))
+ return [uri, nil] if domain_allowed?(uri) || ip_allowed?(ip_address, port: get_port(uri))
protected_uri_with_hostname = enforce_uri_hostname(ip_address, uri, dns_rebind_protection)
@@ -65,8 +65,8 @@ module Gitlab
protected_uri_with_hostname
end
- def blocked_url?(*args)
- validate!(*args)
+ def blocked_url?(url, **kwargs)
+ validate!(url, **kwargs)
false
rescue BlockedUrlError
@@ -113,8 +113,8 @@ module Gitlab
end
rescue SocketError
# If the dns rebinding protection is not enabled or the domain
- # is whitelisted we avoid the dns rebinding checks
- return if domain_whitelisted?(uri) || !dns_rebind_protection
+ # is allowed we avoid the dns rebinding checks
+ return if domain_allowed?(uri) || !dns_rebind_protection
# In the test suite we use a lot of mocked urls that are either invalid or
# don't exist. In order to avoid modifying a ton of tests and factories
@@ -253,12 +253,12 @@ module Gitlab
(uri.port.blank? || uri.port == config.gitlab_shell.ssh_port)
end
- def domain_whitelisted?(uri)
- Gitlab::UrlBlockers::UrlWhitelist.domain_whitelisted?(uri.normalized_host, port: get_port(uri))
+ def domain_allowed?(uri)
+ Gitlab::UrlBlockers::UrlAllowlist.domain_allowed?(uri.normalized_host, port: get_port(uri))
end
- def ip_whitelisted?(ip_address, port: nil)
- Gitlab::UrlBlockers::UrlWhitelist.ip_whitelisted?(ip_address, port: port)
+ def ip_allowed?(ip_address, port: nil)
+ Gitlab::UrlBlockers::UrlAllowlist.ip_allowed?(ip_address, port: port)
end
def config
diff --git a/lib/gitlab/url_blockers/domain_whitelist_entry.rb b/lib/gitlab/url_blockers/domain_allowlist_entry.rb
index b94e8ee3f69..b65bd9e1a92 100644
--- a/lib/gitlab/url_blockers/domain_whitelist_entry.rb
+++ b/lib/gitlab/url_blockers/domain_allowlist_entry.rb
@@ -2,7 +2,7 @@
module Gitlab
module UrlBlockers
- class DomainWhitelistEntry
+ class DomainAllowlistEntry
attr_reader :domain, :port
def initialize(domain, port: nil)
diff --git a/lib/gitlab/url_blockers/ip_whitelist_entry.rb b/lib/gitlab/url_blockers/ip_allowlist_entry.rb
index 88c76574d3d..b293afe166c 100644
--- a/lib/gitlab/url_blockers/ip_whitelist_entry.rb
+++ b/lib/gitlab/url_blockers/ip_allowlist_entry.rb
@@ -2,7 +2,7 @@
module Gitlab
module UrlBlockers
- class IpWhitelistEntry
+ class IpAllowlistEntry
attr_reader :ip, :port
# Argument ip should be an IPAddr object
diff --git a/lib/gitlab/url_blockers/url_whitelist.rb b/lib/gitlab/url_blockers/url_allowlist.rb
index 59f74dde7fc..60238bea75a 100644
--- a/lib/gitlab/url_blockers/url_whitelist.rb
+++ b/lib/gitlab/url_blockers/url_allowlist.rb
@@ -2,43 +2,41 @@
module Gitlab
module UrlBlockers
- class UrlWhitelist
+ class UrlAllowlist
class << self
- def ip_whitelisted?(ip_string, port: nil)
+ def ip_allowed?(ip_string, port: nil)
return false if ip_string.blank?
- ip_whitelist, _ = outbound_local_requests_whitelist_arrays
+ ip_allowlist, _ = outbound_local_requests_allowlist_arrays
ip_obj = Gitlab::Utils.string_to_ip_object(ip_string)
- ip_whitelist.any? do |ip_whitelist_entry|
- ip_whitelist_entry.match?(ip_obj, port)
+ ip_allowlist.any? do |ip_allowlist_entry|
+ ip_allowlist_entry.match?(ip_obj, port)
end
end
- def domain_whitelisted?(domain_string, port: nil)
+ def domain_allowed?(domain_string, port: nil)
return false if domain_string.blank?
- _, domain_whitelist = outbound_local_requests_whitelist_arrays
+ _, domain_allowlist = outbound_local_requests_allowlist_arrays
- domain_whitelist.any? do |domain_whitelist_entry|
- domain_whitelist_entry.match?(domain_string, port)
+ domain_allowlist.any? do |domain_allowlist_entry|
+ domain_allowlist_entry.match?(domain_string, port)
end
end
private
- attr_reader :ip_whitelist, :domain_whitelist
-
# We cannot use Gitlab::CurrentSettings as ApplicationSetting itself
# calls this class. This ends up in a cycle where
# Gitlab::CurrentSettings creates an ApplicationSetting which then
# calls this method.
#
# See https://gitlab.com/gitlab-org/gitlab/issues/9833
- def outbound_local_requests_whitelist_arrays
+ def outbound_local_requests_allowlist_arrays
return [[], []] unless ApplicationSetting.current
- ApplicationSetting.current.outbound_local_requests_whitelist_arrays
+ ApplicationSetting.current.outbound_local_requests_allowlist_arrays
end
end
end
diff --git a/lib/gitlab/url_builder.rb b/lib/gitlab/url_builder.rb
index 1e522ae63b6..ce59e10241e 100644
--- a/lib/gitlab/url_builder.rb
+++ b/lib/gitlab/url_builder.rb
@@ -32,6 +32,8 @@ module Gitlab
instance.milestone_url(object, **options)
when Note
note_url(object, **options)
+ when Release
+ instance.release_url(object, **options)
when Project
instance.project_url(object, **options)
when Snippet
diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb
index 68f24559b1f..4b0dd54683b 100644
--- a/lib/gitlab/usage_data.rb
+++ b/lib/gitlab/usage_data.rb
@@ -40,8 +40,11 @@ module Gitlab
with_finished_at(:recording_ce_finished_at) do
license_usage_data
+ .merge(system_usage_data_license)
+ .merge(system_usage_data_settings)
.merge(system_usage_data)
.merge(system_usage_data_monthly)
+ .merge(system_usage_data_weekly)
.merge(features_usage_data)
.merge(components_usage_data)
.merge(cycle_analytics_usage_data)
@@ -157,6 +160,8 @@ module Gitlab
projects_with_tracing_enabled: count(ProjectTracingSetting),
projects_with_error_tracking_enabled: count(::ErrorTracking::ProjectErrorTrackingSetting.where(enabled: true)),
projects_with_alerts_service_enabled: count(AlertsService.active),
+ projects_with_alerts_created: distinct_count(::AlertManagement::Alert, :project_id),
+ projects_with_enabled_alert_integrations: distinct_count(::AlertManagement::HttpIntegration.active, :project_id),
projects_with_prometheus_alerts: distinct_count(PrometheusAlert, :project_id),
projects_with_terraform_reports: distinct_count(::Ci::JobArtifact.terraform_reports, :project_id),
projects_with_terraform_states: distinct_count(::Terraform::State, :project_id),
@@ -212,9 +217,11 @@ module Gitlab
# rubocop: enable UsageData/LargeTable:
packages: count(::Packages::Package.where(last_28_days_time_period)),
personal_snippets: count(PersonalSnippet.where(last_28_days_time_period)),
- project_snippets: count(ProjectSnippet.where(last_28_days_time_period))
+ project_snippets: count(ProjectSnippet.where(last_28_days_time_period)),
+ projects_with_alerts_created: distinct_count(::AlertManagement::Alert.where(last_28_days_time_period), :project_id)
}.merge(
- snowplow_event_counts(last_28_days_time_period(column: :collector_tstamp))
+ snowplow_event_counts(last_28_days_time_period(column: :collector_tstamp)),
+ aggregated_metrics_monthly
).tap do |data|
data[:snippets] = data[:personal_snippets] + data[:project_snippets]
end
@@ -222,6 +229,27 @@ module Gitlab
end
# rubocop: enable CodeReuse/ActiveRecord
+ def system_usage_data_license
+ {
+ license: {}
+ }
+ end
+
+ def system_usage_data_settings
+ {
+ settings: {}
+ }
+ end
+
+ def system_usage_data_weekly
+ {
+ counts_weekly: {
+ }.merge(
+ aggregated_metrics_weekly
+ )
+ }
+ end
+
def cycle_analytics_usage_data
Gitlab::CycleAnalytics::UsageData.new.to_json
rescue ActiveRecord::StatementInvalid
@@ -500,6 +528,7 @@ module Gitlab
key => {
configure: usage_activity_by_stage_configure(time_period),
create: usage_activity_by_stage_create(time_period),
+ enablement: usage_activity_by_stage_enablement(time_period),
manage: usage_activity_by_stage_manage(time_period),
monitor: usage_activity_by_stage_monitor(time_period),
package: usage_activity_by_stage_package(time_period),
@@ -555,6 +584,11 @@ module Gitlab
end
# rubocop: enable CodeReuse/ActiveRecord
+ # Empty placeholder allows this to match the pattern used by other sections
+ def usage_activity_by_stage_enablement(time_period)
+ {}
+ end
+
# Omitted because no user, creator or author associated: `campaigns_imported_from_github`, `ldap_group_links`
# rubocop: disable CodeReuse/ActiveRecord
def usage_activity_by_stage_manage(time_period)
@@ -564,7 +598,11 @@ module Gitlab
users_created: count(::User.where(time_period), start: user_minimum_id, finish: user_maximum_id),
omniauth_providers: filtered_omniauth_provider_names.reject { |name| name == 'group_saml' },
user_auth_by_provider: distinct_count_user_auth_by_provider(time_period),
+ bulk_imports: {
+ gitlab: distinct_count(::BulkImport.where(time_period, source_type: :gitlab), :user_id)
+ },
projects_imported: {
+ total: count(Project.where(time_period).where.not(import_type: nil)),
gitlab_project: projects_imported_count('gitlab_project', time_period),
gitlab: projects_imported_count('gitlab', time_period),
github: projects_imported_count('github', time_period),
@@ -577,7 +615,8 @@ module Gitlab
issues_imported: {
jira: distinct_count(::JiraImportState.where(time_period), :user_id),
fogbugz: projects_imported_count('fogbugz', time_period),
- phabricator: projects_imported_count('phabricator', time_period)
+ phabricator: projects_imported_count('phabricator', time_period),
+ csv: distinct_count(Issues::CsvImport.where(time_period), :user_id)
},
groups_imported: distinct_count(::GroupImportState.where(time_period), :user_id)
}
@@ -592,7 +631,10 @@ module Gitlab
operations_dashboard_default_dashboard: count(::User.active.with_dashboard('operations').where(time_period),
start: user_minimum_id,
finish: user_maximum_id),
- projects_with_tracing_enabled: distinct_count(::Project.with_tracing_enabled.where(time_period), :creator_id)
+ projects_with_tracing_enabled: distinct_count(::Project.with_tracing_enabled.where(time_period), :creator_id),
+ projects_with_error_tracking_enabled: distinct_count(::Project.with_enabled_error_tracking.where(time_period), :creator_id),
+ projects_with_incidents: distinct_count(::Issue.incident.where(time_period), :project_id),
+ projects_with_alert_incidents: distinct_count(::Issue.incident.with_alert_management_alerts.where(time_period), :project_id)
}
end
# rubocop: enable CodeReuse/ActiveRecord
@@ -664,6 +706,22 @@ module Gitlab
{ redis_hll_counters: ::Gitlab::UsageDataCounters::HLLRedisCounter.unique_events_data }
end
+ def aggregated_metrics_monthly
+ return {} unless Feature.enabled?(:product_analytics_aggregated_metrics)
+
+ {
+ aggregated_metrics: ::Gitlab::UsageDataCounters::HLLRedisCounter.aggregated_metrics_monthly_data
+ }
+ end
+
+ def aggregated_metrics_weekly
+ return {} unless Feature.enabled?(:product_analytics_aggregated_metrics)
+
+ {
+ aggregated_metrics: ::Gitlab::UsageDataCounters::HLLRedisCounter.aggregated_metrics_weekly_data
+ }
+ end
+
def analytics_unique_visits_data
results = ::Gitlab::Analytics::UniqueVisits.analytics_events.each_with_object({}) do |target, hash|
hash[target] = redis_usage_data { unique_visit_service.unique_visits_for(targets: target) }
@@ -709,7 +767,8 @@ module Gitlab
data = {
action_monthly_active_users_project_repo: Gitlab::UsageDataCounters::TrackUniqueEvents::PUSH_ACTION,
action_monthly_active_users_design_management: Gitlab::UsageDataCounters::TrackUniqueEvents::DESIGN_ACTION,
- action_monthly_active_users_wiki_repo: Gitlab::UsageDataCounters::TrackUniqueEvents::WIKI_ACTION
+ action_monthly_active_users_wiki_repo: Gitlab::UsageDataCounters::TrackUniqueEvents::WIKI_ACTION,
+ action_monthly_active_users_git_write: Gitlab::UsageDataCounters::TrackUniqueEvents::GIT_WRITE_ACTION
}
data.each do |key, event|
diff --git a/lib/gitlab/usage_data_counters/aggregated_metrics/common.yml b/lib/gitlab/usage_data_counters/aggregated_metrics/common.yml
new file mode 100644
index 00000000000..97ec8423b95
--- /dev/null
+++ b/lib/gitlab/usage_data_counters/aggregated_metrics/common.yml
@@ -0,0 +1,17 @@
+#- name: unique name of aggregated metric
+# operator: aggregation operator. Valid values are:
+# - "OR": counts unique elements that were observed triggering any of following events
+# - "AND": counts unique elements that were observed triggering all of following events
+# events: list of events names to aggregate into metric. All events in this list must have the same 'redis_slot' and 'aggregation' attributes
+# see from lib/gitlab/usage_data_counters/known_events/ for the list of valid events.
+# feature_flag: name of development feature flag that will be checked before metrics aggregation is performed.
+# Corresponding feature flag should have `default_enabled` attribute set to `false`.
+# This attribute is OPTIONAL and can be omitted, when `feature_flag` is missing no feature flag will be checked.
+---
+- name: product_analytics_test_metrics_union
+ operator: OR
+ events: ['i_search_total', 'i_search_advanced', 'i_search_paid']
+ feature_flag: product_analytics_aggregated_metrics
+- name: product_analytics_test_metrics_intersection
+ operator: AND
+ events: ['i_search_total', 'i_search_advanced', 'i_search_paid']
diff --git a/lib/gitlab/usage_data_counters/designs_counter.rb b/lib/gitlab/usage_data_counters/designs_counter.rb
index 22188b555d2..07e1963f9fb 100644
--- a/lib/gitlab/usage_data_counters/designs_counter.rb
+++ b/lib/gitlab/usage_data_counters/designs_counter.rb
@@ -1,42 +1,8 @@
# frozen_string_literal: true
module Gitlab::UsageDataCounters
- class DesignsCounter
- extend Gitlab::UsageDataCounters::RedisCounter
-
+ class DesignsCounter < BaseCounter
KNOWN_EVENTS = %w[create update delete].freeze
-
- UnknownEvent = Class.new(StandardError)
-
- class << self
- # Each event gets a unique Redis key
- def redis_key(event)
- raise UnknownEvent, event unless KNOWN_EVENTS.include?(event.to_s)
-
- "USAGE_DESIGN_MANAGEMENT_DESIGNS_#{event}".upcase
- end
-
- def count(event)
- increment(redis_key(event))
- end
-
- def read(event)
- total_count(redis_key(event))
- end
-
- def totals
- KNOWN_EVENTS.map { |event| [counter_key(event), read(event)] }.to_h
- end
-
- def fallback_totals
- KNOWN_EVENTS.map { |event| [counter_key(event), -1] }.to_h
- end
-
- private
-
- def counter_key(event)
- "design_management_designs_#{event}".to_sym
- end
- end
+ PREFIX = 'design_management_designs'
end
end
diff --git a/lib/gitlab/usage_data_counters/hll_redis_counter.rb b/lib/gitlab/usage_data_counters/hll_redis_counter.rb
index eb132ef0967..573ad1dce35 100644
--- a/lib/gitlab/usage_data_counters/hll_redis_counter.rb
+++ b/lib/gitlab/usage_data_counters/hll_redis_counter.rb
@@ -5,18 +5,28 @@ module Gitlab
module HLLRedisCounter
DEFAULT_WEEKLY_KEY_EXPIRY_LENGTH = 6.weeks
DEFAULT_DAILY_KEY_EXPIRY_LENGTH = 29.days
- DEFAULT_REDIS_SLOT = ''.freeze
-
- UnknownEvent = Class.new(StandardError)
- UnknownAggregation = Class.new(StandardError)
-
- KNOWN_EVENTS_PATH = 'lib/gitlab/usage_data_counters/known_events.yml'.freeze
+ DEFAULT_REDIS_SLOT = ''
+
+ EventError = Class.new(StandardError)
+ UnknownEvent = Class.new(EventError)
+ UnknownAggregation = Class.new(EventError)
+ AggregationMismatch = Class.new(EventError)
+ SlotMismatch = Class.new(EventError)
+ CategoryMismatch = Class.new(EventError)
+ UnknownAggregationOperator = Class.new(EventError)
+ InvalidContext = Class.new(EventError)
+
+ KNOWN_EVENTS_PATH = File.expand_path('known_events/*.yml', __dir__)
ALLOWED_AGGREGATIONS = %i(daily weekly).freeze
+ UNION_OF_AGGREGATED_METRICS = 'OR'
+ INTERSECTION_OF_AGGREGATED_METRICS = 'AND'
+ ALLOWED_METRICS_AGGREGATIONS = [UNION_OF_AGGREGATED_METRICS, INTERSECTION_OF_AGGREGATED_METRICS].freeze
+ AGGREGATED_METRICS_PATH = File.expand_path('aggregated_metrics/*.yml', __dir__)
# Track event on entity_id
# Increment a Redis HLL counter for unique event_name and entity_id
#
- # All events should be added to know_events file lib/gitlab/usage_data_counters/known_events.yml
+ # All events should be added to known_events yml files lib/gitlab/usage_data_counters/known_events/
#
# Event example:
#
@@ -25,6 +35,7 @@ module Gitlab
# category: compliance # Group events in categories
# expiry: 29 # Optional expiration time in days, default value 29 days for daily and 6.weeks for weekly
# aggregation: daily # Aggregation level, keys are stored daily or weekly
+ # feature_flag: # The event feature flag
#
# Usage:
#
@@ -33,28 +44,24 @@ module Gitlab
class << self
include Gitlab::Utils::UsageData
- def track_event(entity_id, event_name, time = Time.zone.now)
- return unless Gitlab::CurrentSettings.usage_ping_enabled?
-
- event = event_for(event_name)
-
- raise UnknownEvent.new("Unknown event #{event_name}") unless event.present?
-
- Gitlab::Redis::HLL.add(key: redis_key(event, time), value: entity_id, expiry: expiry(event))
+ def track_event(value, event_name, time = Time.zone.now)
+ track(value, event_name, time: time)
end
- def unique_events(event_names:, start_date:, end_date:)
- events = events_for(Array(event_names))
-
- raise 'Events should be in same slot' unless events_in_same_slot?(events)
- raise 'Events should be in same category' unless events_in_same_category?(events)
- raise 'Events should have same aggregation level' unless events_same_aggregation?(events)
-
- aggregation = events.first[:aggregation]
+ def track_event_in_context(value, event_name, context, time = Time.zone.now)
+ return if context.blank?
+ return unless context.in?(valid_context_list)
- keys = keys_for_aggregation(aggregation, events: events, start_date: start_date, end_date: end_date)
+ track(value, event_name, context: context, time: time)
+ end
- redis_usage_data { Gitlab::Redis::HLL.count(keys: keys) }
+ def unique_events(event_names:, start_date:, end_date:, context: '')
+ count_unique_events(event_names: event_names, start_date: start_date, end_date: end_date, context: context) do |events|
+ raise SlotMismatch, events unless events_in_same_slot?(events)
+ raise CategoryMismatch, events unless events_in_same_category?(events)
+ raise AggregationMismatch, events unless events_same_aggregation?(events)
+ raise InvalidContext if context.present? && !context.in?(valid_context_list)
+ end
end
def categories
@@ -72,8 +79,8 @@ module Gitlab
events_names = events_for_category(category)
event_results = events_names.each_with_object({}) do |event, hash|
- hash["#{event}_weekly"] = unique_events(event_names: event, start_date: 7.days.ago.to_date, end_date: Date.current)
- hash["#{event}_monthly"] = unique_events(event_names: event, start_date: 4.weeks.ago.to_date, end_date: Date.current)
+ hash["#{event}_weekly"] = unique_events(event_names: [event], start_date: 7.days.ago.to_date, end_date: Date.current)
+ hash["#{event}_monthly"] = unique_events(event_names: [event], start_date: 4.weeks.ago.to_date, end_date: Date.current)
end
if eligible_for_totals?(events_names)
@@ -89,8 +96,136 @@ module Gitlab
event_for(event_name).present?
end
+ def aggregated_metrics_monthly_data
+ aggregated_metrics_data(4.weeks.ago.to_date)
+ end
+
+ def aggregated_metrics_weekly_data
+ aggregated_metrics_data(7.days.ago.to_date)
+ end
+
+ def known_events
+ @known_events ||= load_events(KNOWN_EVENTS_PATH)
+ end
+
+ def aggregated_metrics
+ @aggregated_metrics ||= load_events(AGGREGATED_METRICS_PATH)
+ end
+
private
+ def track(value, event_name, context: '', time: Time.zone.now)
+ return unless Gitlab::CurrentSettings.usage_ping_enabled?
+
+ event = event_for(event_name)
+ raise UnknownEvent, "Unknown event #{event_name}" unless event.present?
+
+ Gitlab::Redis::HLL.add(key: redis_key(event, time, context), value: value, expiry: expiry(event))
+ end
+
+ # The aray of valid context on which we allow tracking
+ def valid_context_list
+ Plan.all_plans
+ end
+
+ def aggregated_metrics_data(start_date)
+ aggregated_metrics.each_with_object({}) do |aggregation, weekly_data|
+ next if aggregation[:feature_flag] && Feature.disabled?(aggregation[:feature_flag], default_enabled: false, type: :development)
+
+ weekly_data[aggregation[:name]] = calculate_count_for_aggregation(aggregation, start_date: start_date, end_date: Date.current)
+ end
+ end
+
+ def calculate_count_for_aggregation(aggregation, start_date:, end_date:)
+ case aggregation[:operator]
+ when UNION_OF_AGGREGATED_METRICS
+ calculate_events_union(event_names: aggregation[:events], start_date: start_date, end_date: end_date)
+ when INTERSECTION_OF_AGGREGATED_METRICS
+ calculate_events_intersections(event_names: aggregation[:events], start_date: start_date, end_date: end_date)
+ else
+ raise UnknownAggregationOperator, "Events should be aggregated with one of operators #{ALLOWED_METRICS_AGGREGATIONS}"
+ end
+ end
+
+ # calculate intersection of 'n' sets based on inclusion exclusion principle https://en.wikipedia.org/wiki/Inclusion%E2%80%93exclusion_principle
+ # this method will be extracted to dedicated module with https://gitlab.com/gitlab-org/gitlab/-/issues/273391
+ def calculate_events_intersections(event_names:, start_date:, end_date:, subset_powers_cache: Hash.new({}))
+ # calculate power of intersection of all given metrics from inclusion exclusion principle
+ # |A + B + C| = (|A| + |B| + |C|) - (|A & B| + |A & C| + .. + |C & D|) + (|A & B & C|) =>
+ # |A & B & C| = - (|A| + |B| + |C|) + (|A & B| + |A & C| + .. + |C & D|) + |A + B + C|
+ # |A + B + C + D| = (|A| + |B| + |C| + |D|) - (|A & B| + |A & C| + .. + |C & D|) + (|A & B & C| + |B & C & D|) - |A & B & C & D| =>
+ # |A & B & C & D| = (|A| + |B| + |C| + |D|) - (|A & B| + |A & C| + .. + |C & D|) + (|A & B & C| + |B & C & D|) - |A + B + C + D|
+
+ # calculate each components of equation except for the last one |A & B & C & D| = (|A| + |B| + |C| + |D|) - (|A & B| + |A & C| + .. + |C & D|) + (|A & B & C| + |B & C & D|) - ...
+ subset_powers_data = subsets_intersection_powers(event_names, start_date, end_date, subset_powers_cache)
+
+ # calculate last component of the equation |A & B & C & D| = .... - |A + B + C + D|
+ power_of_union_of_all_events = begin
+ subset_powers_cache[event_names.size][event_names.join('_+_')] ||= \
+ calculate_events_union(event_names: event_names, start_date: start_date, end_date: end_date)
+ end
+
+ # in order to determine if part of equation (|A & B & C|, |A & B & C & D|), that represents the intersection that we need to calculate,
+ # is positive or negative in particular equation we need to determine if number of subsets is even or odd. Please take a look at two examples below
+ # |A + B + C| = (|A| + |B| + |C|) - (|A & B| + |A & C| + .. + |C & D|) + |A & B & C| =>
+ # |A & B & C| = - (|A| + |B| + |C|) + (|A & B| + |A & C| + .. + |C & D|) + |A + B + C|
+ # |A + B + C + D| = (|A| + |B| + |C| + |D|) - (|A & B| + |A & C| + .. + |C & D|) + (|A & B & C| + |B & C & D|) - |A & B & C & D| =>
+ # |A & B & C & D| = (|A| + |B| + |C| + |D|) - (|A & B| + |A & C| + .. + |C & D|) + (|A & B & C| + |B & C & D|) - |A + B + C + D|
+ subset_powers_size_even = subset_powers_data.size.even?
+
+ # sum all components of equation except for the last one |A & B & C & D| = (|A| + |B| + |C| + |D|) - (|A & B| + |A & C| + .. + |C & D|) + (|A & B & C| + |B & C & D|) - ... =>
+ sum_of_all_subset_powers = sum_subset_powers(subset_powers_data, subset_powers_size_even)
+
+ # add last component of the equation |A & B & C & D| = sum_of_all_subset_powers - |A + B + C + D|
+ sum_of_all_subset_powers + (subset_powers_size_even ? power_of_union_of_all_events : -power_of_union_of_all_events)
+ end
+
+ def sum_subset_powers(subset_powers_data, subset_powers_size_even)
+ sum_without_sign = subset_powers_data.to_enum.with_index.sum do |value, index|
+ (index + 1).odd? ? value : -value
+ end
+
+ (subset_powers_size_even ? -1 : 1) * sum_without_sign
+ end
+
+ def subsets_intersection_powers(event_names, start_date, end_date, subset_powers_cache)
+ subset_sizes = (1..(event_names.size - 1))
+
+ subset_sizes.map do |subset_size|
+ if subset_size > 1
+ # calculate sum of powers of intersection between each subset (with given size) of metrics: #|A + B + C + D| = ... - (|A & B| + |A & C| + .. + |C & D|)
+ event_names.combination(subset_size).sum do |events_subset|
+ subset_powers_cache[subset_size][events_subset.join('_&_')] ||= \
+ calculate_events_intersections(event_names: events_subset, start_date: start_date, end_date: end_date, subset_powers_cache: subset_powers_cache)
+ end
+ else
+ # calculate sum of powers of each set (metric) alone #|A + B + C + D| = (|A| + |B| + |C| + |D|) - ...
+ event_names.sum do |event|
+ subset_powers_cache[subset_size][event] ||= \
+ unique_events(event_names: event, start_date: start_date, end_date: end_date)
+ end
+ end
+ end
+ end
+
+ def calculate_events_union(event_names:, start_date:, end_date:)
+ count_unique_events(event_names: event_names, start_date: start_date, end_date: end_date) do |events|
+ raise SlotMismatch, events unless events_in_same_slot?(events)
+ raise AggregationMismatch, events unless events_same_aggregation?(events)
+ end
+ end
+
+ def count_unique_events(event_names:, start_date:, end_date:, context: '')
+ events = events_for(Array(event_names).map(&:to_s))
+
+ yield events if block_given?
+
+ aggregation = events.first[:aggregation]
+
+ keys = keys_for_aggregation(aggregation, events: events, start_date: start_date, end_date: end_date, context: context)
+ redis_usage_data { Gitlab::Redis::HLL.count(keys: keys) }
+ end
+
# Allow to add totals for events that are in the same redis slot, category and have the same aggregation level
# and if there are more than 1 event
def eligible_for_totals?(events_names)
@@ -100,16 +235,22 @@ module Gitlab
events_in_same_slot?(events) && events_in_same_category?(events) && events_same_aggregation?(events)
end
- def keys_for_aggregation(aggregation, events:, start_date:, end_date:)
+ def keys_for_aggregation(aggregation, events:, start_date:, end_date:, context: '')
if aggregation.to_sym == :daily
- daily_redis_keys(events: events, start_date: start_date, end_date: end_date)
+ daily_redis_keys(events: events, start_date: start_date, end_date: end_date, context: context)
else
- weekly_redis_keys(events: events, start_date: start_date, end_date: end_date)
+ weekly_redis_keys(events: events, start_date: start_date, end_date: end_date, context: context)
end
end
- def known_events
- @known_events ||= YAML.load_file(Rails.root.join(KNOWN_EVENTS_PATH)).map(&:with_indifferent_access)
+ def load_events(wildcard)
+ Dir[wildcard].each_with_object([]) do |path, events|
+ events.push(*load_yaml_from_path(path))
+ end
+ end
+
+ def load_yaml_from_path(path)
+ YAML.safe_load(File.read(path))&.map(&:with_indifferent_access)
end
def known_events_names
@@ -141,7 +282,7 @@ module Gitlab
end
def event_for(event_name)
- known_events.find { |event| event[:name] == event_name }
+ known_events.find { |event| event[:name] == event_name.to_s }
end
def events_for(event_names)
@@ -153,17 +294,26 @@ module Gitlab
end
# Compose the key in order to store events daily or weekly
- def redis_key(event, time)
+ def redis_key(event, time, context = '')
raise UnknownEvent.new("Unknown event #{event[:name]}") unless known_events_names.include?(event[:name].to_s)
raise UnknownAggregation.new("Use :daily or :weekly aggregation") unless ALLOWED_AGGREGATIONS.include?(event[:aggregation].to_sym)
+ key = apply_slot(event)
+ key = apply_time_aggregation(key, time, event)
+ key = "#{context}_#{key}" if context.present?
+ key
+ end
+
+ def apply_slot(event)
slot = redis_slot(event)
- key = if slot.present?
- event[:name].to_s.gsub(slot, "{#{slot}}")
- else
- "{#{event[:name]}}"
- end
+ if slot.present?
+ event[:name].to_s.gsub(slot, "{#{slot}}")
+ else
+ "{#{event[:name]}}"
+ end
+ end
+ def apply_time_aggregation(key, time, event)
if event[:aggregation].to_sym == :daily
year_day = time.strftime('%G-%j')
"#{year_day}-#{key}"
@@ -173,21 +323,29 @@ module Gitlab
end
end
- def daily_redis_keys(events:, start_date:, end_date:)
+ def daily_redis_keys(events:, start_date:, end_date:, context: '')
(start_date.to_date..end_date.to_date).map do |date|
- events.map { |event| redis_key(event, date) }
+ events.map { |event| redis_key(event, date, context) }
end.flatten
end
- def weekly_redis_keys(events:, start_date:, end_date:)
+ def validate_aggregation_operator!(operator)
+ return true if ALLOWED_METRICS_AGGREGATIONS.include?(operator)
+
+ raise UnknownAggregationOperator.new("Events should be aggregated with one of operators #{ALLOWED_METRICS_AGGREGATIONS}")
+ end
+
+ def weekly_redis_keys(events:, start_date:, end_date:, context: '')
weeks = end_date.to_date.cweek - start_date.to_date.cweek
weeks = 1 if weeks == 0
(0..(weeks - 1)).map do |week_increment|
- events.map { |event| redis_key(event, start_date + week_increment * 7.days) }
+ events.map { |event| redis_key(event, start_date + week_increment * 7.days, context) }
end.flatten
end
end
end
end
end
+
+Gitlab::UsageDataCounters::HLLRedisCounter.prepend_if_ee('EE::Gitlab::UsageDataCounters::HLLRedisCounter')
diff --git a/lib/gitlab/usage_data_counters/issue_activity_unique_counter.rb b/lib/gitlab/usage_data_counters/issue_activity_unique_counter.rb
index e8839875109..da013a06777 100644
--- a/lib/gitlab/usage_data_counters/issue_activity_unique_counter.rb
+++ b/lib/gitlab/usage_data_counters/issue_activity_unique_counter.rb
@@ -9,14 +9,12 @@ module Gitlab
ISSUE_CREATED = 'g_project_management_issue_created'
ISSUE_CLOSED = 'g_project_management_issue_closed'
ISSUE_DESCRIPTION_CHANGED = 'g_project_management_issue_description_changed'
- ISSUE_ITERATION_CHANGED = 'g_project_management_issue_iteration_changed'
ISSUE_LABEL_CHANGED = 'g_project_management_issue_label_changed'
ISSUE_MADE_CONFIDENTIAL = 'g_project_management_issue_made_confidential'
ISSUE_MADE_VISIBLE = 'g_project_management_issue_made_visible'
ISSUE_MILESTONE_CHANGED = 'g_project_management_issue_milestone_changed'
ISSUE_REOPENED = 'g_project_management_issue_reopened'
ISSUE_TITLE_CHANGED = 'g_project_management_issue_title_changed'
- ISSUE_WEIGHT_CHANGED = 'g_project_management_issue_weight_changed'
ISSUE_CROSS_REFERENCED = 'g_project_management_issue_cross_referenced'
ISSUE_MOVED = 'g_project_management_issue_moved'
ISSUE_RELATED = 'g_project_management_issue_related'
@@ -24,15 +22,15 @@ module Gitlab
ISSUE_MARKED_AS_DUPLICATE = 'g_project_management_issue_marked_as_duplicate'
ISSUE_LOCKED = 'g_project_management_issue_locked'
ISSUE_UNLOCKED = 'g_project_management_issue_unlocked'
- ISSUE_ADDED_TO_EPIC = 'g_project_management_issue_added_to_epic'
- ISSUE_REMOVED_FROM_EPIC = 'g_project_management_issue_removed_from_epic'
- ISSUE_CHANGED_EPIC = 'g_project_management_issue_changed_epic'
ISSUE_DESIGNS_ADDED = 'g_project_management_issue_designs_added'
ISSUE_DESIGNS_MODIFIED = 'g_project_management_issue_designs_modified'
ISSUE_DESIGNS_REMOVED = 'g_project_management_issue_designs_removed'
ISSUE_DUE_DATE_CHANGED = 'g_project_management_issue_due_date_changed'
ISSUE_TIME_ESTIMATE_CHANGED = 'g_project_management_issue_time_estimate_changed'
ISSUE_TIME_SPENT_CHANGED = 'g_project_management_issue_time_spent_changed'
+ ISSUE_COMMENT_ADDED = 'g_project_management_issue_comment_added'
+ ISSUE_COMMENT_EDITED = 'g_project_management_issue_comment_edited'
+ ISSUE_COMMENT_REMOVED = 'g_project_management_issue_comment_removed'
class << self
def track_issue_created_action(author:, time: Time.zone.now)
@@ -75,14 +73,6 @@ module Gitlab
track_unique_action(ISSUE_MILESTONE_CHANGED, author, time)
end
- def track_issue_iteration_changed_action(author:, time: Time.zone.now)
- track_unique_action(ISSUE_ITERATION_CHANGED, author, time)
- end
-
- def track_issue_weight_changed_action(author:, time: Time.zone.now)
- track_unique_action(ISSUE_WEIGHT_CHANGED, author, time)
- end
-
def track_issue_cross_referenced_action(author:, time: Time.zone.now)
track_unique_action(ISSUE_CROSS_REFERENCED, author, time)
end
@@ -111,18 +101,6 @@ module Gitlab
track_unique_action(ISSUE_UNLOCKED, author, time)
end
- def track_issue_added_to_epic_action(author:, time: Time.zone.now)
- track_unique_action(ISSUE_ADDED_TO_EPIC, author, time)
- end
-
- def track_issue_removed_from_epic_action(author:, time: Time.zone.now)
- track_unique_action(ISSUE_REMOVED_FROM_EPIC, author, time)
- end
-
- def track_issue_changed_epic_action(author:, time: Time.zone.now)
- track_unique_action(ISSUE_CHANGED_EPIC, author, time)
- end
-
def track_issue_designs_added_action(author:, time: Time.zone.now)
track_unique_action(ISSUE_DESIGNS_ADDED, author, time)
end
@@ -147,6 +125,18 @@ module Gitlab
track_unique_action(ISSUE_TIME_SPENT_CHANGED, author, time)
end
+ def track_issue_comment_added_action(author:, time: Time.zone.now)
+ track_unique_action(ISSUE_COMMENT_ADDED, author, time)
+ end
+
+ def track_issue_comment_edited_action(author:, time: Time.zone.now)
+ track_unique_action(ISSUE_COMMENT_EDITED, author, time)
+ end
+
+ def track_issue_comment_removed_action(author:, time: Time.zone.now)
+ track_unique_action(ISSUE_COMMENT_REMOVED, author, time)
+ end
+
private
def track_unique_action(action, author, time)
@@ -159,3 +149,5 @@ module Gitlab
end
end
end
+
+Gitlab::UsageDataCounters::IssueActivityUniqueCounter.prepend_if_ee('EE::Gitlab::UsageDataCounters::IssueActivityUniqueCounter')
diff --git a/lib/gitlab/usage_data_counters/known_events.yml b/lib/gitlab/usage_data_counters/known_events/common.yml
index bc56c5d6d9b..85f16ea807b 100644
--- a/lib/gitlab/usage_data_counters/known_events.yml
+++ b/lib/gitlab/usage_data_counters/known_events/common.yml
@@ -4,114 +4,141 @@
redis_slot: compliance
category: compliance
aggregation: weekly
+ feature_flag: track_unique_visits
- name: g_compliance_audit_events
category: compliance
redis_slot: compliance
aggregation: weekly
+ feature_flag: track_unique_visits
- name: i_compliance_audit_events
category: compliance
redis_slot: compliance
aggregation: weekly
+ feature_flag: track_unique_visits
- name: i_compliance_credential_inventory
category: compliance
redis_slot: compliance
aggregation: weekly
+ feature_flag: track_unique_visits
- name: a_compliance_audit_events_api
category: compliance
redis_slot: compliance
aggregation: weekly
+ feature_flag: usage_data_a_compliance_audit_events_api
# Analytics category
- name: g_analytics_contribution
category: analytics
redis_slot: analytics
aggregation: weekly
+ feature_flag: track_unique_visits
- name: g_analytics_insights
category: analytics
redis_slot: analytics
aggregation: weekly
+ feature_flag: track_unique_visits
- name: g_analytics_issues
category: analytics
redis_slot: analytics
aggregation: weekly
+ feature_flag: track_unique_visits
- name: g_analytics_productivity
category: analytics
redis_slot: analytics
aggregation: weekly
+ feature_flag: track_unique_visits
- name: g_analytics_valuestream
category: analytics
redis_slot: analytics
aggregation: weekly
+ feature_flag: track_unique_visits
- name: p_analytics_pipelines
category: analytics
redis_slot: analytics
aggregation: weekly
+ feature_flag: track_unique_visits
- name: p_analytics_code_reviews
category: analytics
redis_slot: analytics
aggregation: weekly
+ feature_flag: track_unique_visits
- name: p_analytics_valuestream
category: analytics
redis_slot: analytics
aggregation: weekly
+ feature_flag: track_unique_visits
- name: p_analytics_insights
category: analytics
redis_slot: analytics
aggregation: weekly
+ feature_flag: track_unique_visits
- name: p_analytics_issues
category: analytics
redis_slot: analytics
aggregation: weekly
+ feature_flag: track_unique_visits
- name: p_analytics_repo
category: analytics
redis_slot: analytics
aggregation: weekly
+ feature_flag: track_unique_visits
- name: i_analytics_cohorts
category: analytics
redis_slot: analytics
aggregation: weekly
+ feature_flag: track_unique_visits
- name: i_analytics_dev_ops_score
category: analytics
redis_slot: analytics
aggregation: weekly
+ feature_flag: track_unique_visits
- name: g_analytics_merge_request
category: analytics
redis_slot: analytics
aggregation: weekly
+ feature_flag: track_unique_visits
- name: p_analytics_merge_request
category: analytics
redis_slot: analytics
aggregation: weekly
+ feature_flag: track_unique_visits
- name: i_analytics_instance_statistics
category: analytics
redis_slot: analytics
aggregation: weekly
+ feature_flag: track_unique_visits
- name: g_edit_by_web_ide
category: ide_edit
redis_slot: edit
expiry: 29
aggregation: daily
+ feature_flag: track_editor_edit_actions
- name: g_edit_by_sfe
category: ide_edit
redis_slot: edit
expiry: 29
aggregation: daily
+ feature_flag: track_editor_edit_actions
- name: g_edit_by_snippet_ide
category: ide_edit
redis_slot: edit
expiry: 29
aggregation: daily
+ feature_flag: track_editor_edit_actions
- name: i_search_total
category: search
redis_slot: search
aggregation: weekly
+ feature_flag: search_track_unique_users
- name: i_search_advanced
category: search
redis_slot: search
aggregation: weekly
+ feature_flag: search_track_unique_users
- name: i_search_paid
category: search
redis_slot: search
aggregation: weekly
+ feature_flag: search_track_unique_users
- name: wiki_action
category: source_code
aggregation: daily
@@ -121,6 +148,9 @@
- name: project_action
category: source_code
aggregation: daily
+- name: git_write_action
+ category: source_code
+ aggregation: daily
- name: merge_request_action
category: source_code
aggregation: daily
@@ -133,173 +163,242 @@
redis_slot: incident_management
category: incident_management
aggregation: weekly
+ feature_flag: usage_data_incident_management_alert_status_changed
- name: incident_management_alert_assigned
redis_slot: incident_management
category: incident_management
aggregation: weekly
+ feature_flag: usage_data_incident_management_alert_assigned
- name: incident_management_alert_todo
redis_slot: incident_management
category: incident_management
aggregation: weekly
+ feature_flag: usage_data_incident_management_alert_todo
- name: incident_management_incident_created
redis_slot: incident_management
category: incident_management
aggregation: weekly
+ feature_flag: usage_data_incident_management_incident_created
- name: incident_management_incident_reopened
redis_slot: incident_management
category: incident_management
aggregation: weekly
+ feature_flag: usage_data_incident_management_incident_reopened
- name: incident_management_incident_closed
redis_slot: incident_management
category: incident_management
aggregation: weekly
+ feature_flag: usage_data_incident_management_incident_closed
- name: incident_management_incident_assigned
redis_slot: incident_management
category: incident_management
aggregation: weekly
+ feature_flag: usage_data_incident_management_incident_assigned
- name: incident_management_incident_todo
redis_slot: incident_management
category: incident_management
aggregation: weekly
+ feature_flag: usage_data_incident_management_incident_todo
- name: incident_management_incident_comment
redis_slot: incident_management
category: incident_management
aggregation: weekly
+ feature_flag: usage_data_incident_management_incident_comment
- name: incident_management_incident_zoom_meeting
redis_slot: incident_management
category: incident_management
aggregation: weekly
+ feature_flag: usage_data_incident_management_incident_zoom_meeting
- name: incident_management_incident_published
redis_slot: incident_management
category: incident_management
aggregation: weekly
+ feature_flag: usage_data_incident_management_incident_published
- name: incident_management_incident_relate
redis_slot: incident_management
category: incident_management
aggregation: weekly
+ feature_flag: usage_data_incident_management_incident_relate
- name: incident_management_incident_unrelate
redis_slot: incident_management
category: incident_management
aggregation: weekly
+ feature_flag: usage_data_incident_management_incident_unrelate
- name: incident_management_incident_change_confidential
redis_slot: incident_management
category: incident_management
aggregation: weekly
+ feature_flag: usage_data_incident_management_incident_change_confidential
# Testing category
- name: i_testing_test_case_parsed
category: testing
redis_slot: testing
aggregation: weekly
+ feature_flag: usage_data_i_testing_test_case_parsed
# Project Management group
- name: g_project_management_issue_title_changed
category: issues_edit
redis_slot: project_management
aggregation: daily
+ feature_flag: track_issue_activity_actions
- name: g_project_management_issue_description_changed
category: issues_edit
redis_slot: project_management
aggregation: daily
+ feature_flag: track_issue_activity_actions
- name: g_project_management_issue_assignee_changed
category: issues_edit
redis_slot: project_management
aggregation: daily
+ feature_flag: track_issue_activity_actions
- name: g_project_management_issue_made_confidential
category: issues_edit
redis_slot: project_management
aggregation: daily
+ feature_flag: track_issue_activity_actions
- name: g_project_management_issue_made_visible
category: issues_edit
redis_slot: project_management
aggregation: daily
+ feature_flag: track_issue_activity_actions
- name: g_project_management_issue_created
category: issues_edit
redis_slot: project_management
aggregation: daily
+ feature_flag: track_issue_activity_actions
- name: g_project_management_issue_closed
category: issues_edit
redis_slot: project_management
aggregation: daily
+ feature_flag: track_issue_activity_actions
- name: g_project_management_issue_reopened
category: issues_edit
redis_slot: project_management
aggregation: daily
+ feature_flag: track_issue_activity_actions
- name: g_project_management_issue_label_changed
category: issues_edit
redis_slot: project_management
aggregation: daily
+ feature_flag: track_issue_activity_actions
- name: g_project_management_issue_milestone_changed
category: issues_edit
redis_slot: project_management
aggregation: daily
+ feature_flag: track_issue_activity_actions
- name: g_project_management_issue_iteration_changed
category: issues_edit
redis_slot: project_management
aggregation: daily
+ feature_flag: track_issue_activity_actions
- name: g_project_management_issue_weight_changed
category: issues_edit
redis_slot: project_management
aggregation: daily
+ feature_flag: track_issue_activity_actions
- name: g_project_management_issue_cross_referenced
category: issues_edit
redis_slot: project_management
aggregation: daily
+ feature_flag: track_issue_activity_actions
- name: g_project_management_issue_moved
category: issues_edit
redis_slot: project_management
aggregation: daily
+ feature_flag: track_issue_activity_actions
- name: g_project_management_issue_related
category: issues_edit
redis_slot: project_management
aggregation: daily
+ feature_flag: track_issue_activity_actions
- name: g_project_management_issue_unrelated
category: issues_edit
redis_slot: project_management
aggregation: daily
+ feature_flag: track_issue_activity_actions
- name: g_project_management_issue_marked_as_duplicate
category: issues_edit
redis_slot: project_management
aggregation: daily
+ feature_flag: track_issue_activity_actions
- name: g_project_management_issue_locked
category: issues_edit
redis_slot: project_management
aggregation: daily
+ feature_flag: track_issue_activity_actions
- name: g_project_management_issue_unlocked
category: issues_edit
redis_slot: project_management
aggregation: daily
+ feature_flag: track_issue_activity_actions
- name: g_project_management_issue_added_to_epic
category: issues_edit
redis_slot: project_management
aggregation: daily
+ feature_flag: track_issue_activity_actions
- name: g_project_management_issue_removed_from_epic
category: issues_edit
redis_slot: project_management
aggregation: daily
+ feature_flag: track_issue_activity_actions
- name: g_project_management_issue_changed_epic
category: issues_edit
redis_slot: project_management
aggregation: daily
+ feature_flag: track_issue_activity_actions
- name: g_project_management_issue_designs_added
category: issues_edit
redis_slot: project_management
aggregation: daily
+ feature_flag: track_issue_activity_actions
- name: g_project_management_issue_designs_modified
category: issues_edit
redis_slot: project_management
aggregation: daily
+ feature_flag: track_issue_activity_actions
- name: g_project_management_issue_designs_removed
category: issues_edit
redis_slot: project_management
aggregation: daily
+ feature_flag: track_issue_activity_actions
- name: g_project_management_issue_due_date_changed
category: issues_edit
redis_slot: project_management
aggregation: daily
+ feature_flag: track_issue_activity_actions
- name: g_project_management_issue_time_estimate_changed
category: issues_edit
redis_slot: project_management
aggregation: daily
+ feature_flag: track_issue_activity_actions
- name: g_project_management_issue_time_spent_changed
category: issues_edit
redis_slot: project_management
aggregation: daily
+ feature_flag: track_issue_activity_actions
+- name: g_project_management_issue_comment_added
+ category: issues_edit
+ redis_slot: project_management
+ aggregation: daily
+ feature_flag: track_issue_activity_actions
+- name: g_project_management_issue_comment_edited
+ category: issues_edit
+ redis_slot: project_management
+ aggregation: daily
+ feature_flag: track_issue_activity_actions
+- name: g_project_management_issue_comment_removed
+ category: issues_edit
+ redis_slot: project_management
+ aggregation: daily
+ feature_flag: track_issue_activity_actions
+- name: g_project_management_issue_health_status_changed
+ category: issues_edit
+ redis_slot: project_management
+ aggregation: daily
+ feature_flag: track_issue_activity_actions
+# Secrets Management
+- name: i_ci_secrets_management_vault_build_created
+ category: ci_secrets_management
+ redis_slot: ci_secrets_management
+ aggregation: weekly
+ feature_flag: usage_data_i_ci_secrets_management_vault_build_created
diff --git a/lib/gitlab/usage_data_counters/known_events/package_events.yml b/lib/gitlab/usage_data_counters/known_events/package_events.yml
new file mode 100644
index 00000000000..7ed02aa2a85
--- /dev/null
+++ b/lib/gitlab/usage_data_counters/known_events/package_events.yml
@@ -0,0 +1,265 @@
+---
+- name: i_package_maven_user_push
+ category: maven_packages
+ aggregation: weekly
+ redis_slot: package
+- name: i_package_maven_deploy_token_push
+ category: maven_packages
+ aggregation: weekly
+ redis_slot: package
+- name: i_package_maven_user_delete
+ category: maven_packages
+ aggregation: weekly
+ redis_slot: package
+- name: i_package_maven_deploy_token_delete
+ category: maven_packages
+ aggregation: weekly
+ redis_slot: package
+- name: i_package_maven_user_pull
+ category: maven_packages
+ aggregation: weekly
+ redis_slot: package
+- name: i_package_maven_deploy_token_pull
+ category: maven_packages
+ aggregation: weekly
+ redis_slot: package
+- name: i_package_npm_user_push
+ category: npm_packages
+ aggregation: weekly
+ redis_slot: package
+- name: i_package_npm_deploy_token_push
+ category: npm_packages
+ aggregation: weekly
+ redis_slot: package
+- name: i_package_npm_user_delete
+ category: npm_packages
+ aggregation: weekly
+ redis_slot: package
+- name: i_package_npm_deploy_token_delete
+ category: npm_packages
+ aggregation: weekly
+ redis_slot: package
+- name: i_package_npm_user_pull
+ category: npm_packages
+ aggregation: weekly
+ redis_slot: package
+- name: i_package_npm_deploy_token_pull
+ category: npm_packages
+ aggregation: weekly
+ redis_slot: package
+- name: i_package_conan_user_push
+ category: conan_packages
+ aggregation: weekly
+ redis_slot: package
+- name: i_package_conan_deploy_token_push
+ category: conan_packages
+ aggregation: weekly
+ redis_slot: package
+- name: i_package_conan_user_delete
+ category: conan_packages
+ aggregation: weekly
+ redis_slot: package
+- name: i_package_conan_deploy_token_delete
+ category: conan_packages
+ aggregation: weekly
+ redis_slot: package
+- name: i_package_conan_user_pull
+ category: conan_packages
+ aggregation: weekly
+ redis_slot: package
+- name: i_package_conan_deploy_token_pull
+ category: conan_packages
+ aggregation: weekly
+ redis_slot: package
+- name: i_package_nuget_user_push
+ category: nuget_packages
+ aggregation: weekly
+ redis_slot: package
+- name: i_package_nuget_deploy_token_push
+ category: nuget_packages
+ aggregation: weekly
+ redis_slot: package
+- name: i_package_nuget_user_delete
+ category: nuget_packages
+ aggregation: weekly
+ redis_slot: package
+- name: i_package_nuget_deploy_token_delete
+ category: nuget_packages
+ aggregation: weekly
+ redis_slot: package
+- name: i_package_nuget_user_pull
+ category: nuget_packages
+ aggregation: weekly
+ redis_slot: package
+- name: i_package_nuget_deploy_token_pull
+ category: nuget_packages
+ aggregation: weekly
+ redis_slot: package
+- name: i_package_pypi_user_push
+ category: pypi_packages
+ aggregation: weekly
+ redis_slot: package
+- name: i_package_pypi_deploy_token_push
+ category: pypi_packages
+ aggregation: weekly
+ redis_slot: package
+- name: i_package_pypi_user_delete
+ category: pypi_packages
+ aggregation: weekly
+ redis_slot: package
+- name: i_package_pypi_deploy_token_delete
+ category: pypi_packages
+ aggregation: weekly
+ redis_slot: package
+- name: i_package_pypi_user_pull
+ category: pypi_packages
+ aggregation: weekly
+ redis_slot: package
+- name: i_package_pypi_deploy_token_pull
+ category: pypi_packages
+ aggregation: weekly
+ redis_slot: package
+- name: i_package_composer_user_push
+ category: composer_packages
+ aggregation: weekly
+ redis_slot: package
+- name: i_package_composer_deploy_token_push
+ category: composer_packages
+ aggregation: weekly
+ redis_slot: package
+- name: i_package_composer_user_delete
+ category: composer_packages
+ aggregation: weekly
+ redis_slot: package
+- name: i_package_composer_deploy_token_delete
+ category: composer_packages
+ aggregation: weekly
+ redis_slot: package
+- name: i_package_composer_user_pull
+ category: composer_packages
+ aggregation: weekly
+ redis_slot: package
+- name: i_package_composer_deploy_token_pull
+ category: composer_packages
+ aggregation: weekly
+ redis_slot: package
+- name: i_package_generic_user_push
+ category: generic_packages
+ aggregation: weekly
+ redis_slot: package
+- name: i_package_generic_deploy_token_push
+ category: generic_packages
+ aggregation: weekly
+ redis_slot: package
+- name: i_package_generic_user_delete
+ category: generic_packages
+ aggregation: weekly
+ redis_slot: package
+- name: i_package_generic_deploy_token_delete
+ category: generic_packages
+ aggregation: weekly
+ redis_slot: package
+- name: i_package_generic_user_pull
+ category: generic_packages
+ aggregation: weekly
+ redis_slot: package
+- name: i_package_generic_deploy_token_pull
+ category: generic_packages
+ aggregation: weekly
+ redis_slot: package
+- name: i_package_golang_user_push
+ category: golang_packages
+ aggregation: weekly
+ redis_slot: package
+- name: i_package_golang_deploy_token_push
+ category: golang_packages
+ aggregation: weekly
+ redis_slot: package
+- name: i_package_golang_user_delete
+ category: golang_packages
+ aggregation: weekly
+ redis_slot: package
+- name: i_package_golang_deploy_token_delete
+ category: golang_packages
+ aggregation: weekly
+ redis_slot: package
+- name: i_package_golang_user_pull
+ category: golang_packages
+ aggregation: weekly
+ redis_slot: package
+- name: i_package_golang_deploy_token_pull
+ category: golang_packages
+ aggregation: weekly
+ redis_slot: package
+- name: i_package_debian_user_push
+ category: debian_packages
+ aggregation: weekly
+ redis_slot: package
+- name: i_package_debian_deploy_token_push
+ category: debian_packages
+ aggregation: weekly
+ redis_slot: package
+- name: i_package_debian_user_delete
+ category: debian_packages
+ aggregation: weekly
+ redis_slot: package
+- name: i_package_debian_deploy_token_delete
+ category: debian_packages
+ aggregation: weekly
+ redis_slot: package
+- name: i_package_debian_user_pull
+ category: debian_packages
+ aggregation: weekly
+ redis_slot: package
+- name: i_package_debian_deploy_token_pull
+ category: debian_packages
+ aggregation: weekly
+ redis_slot: package
+- name: i_package_container_user_push
+ category: container_packages
+ aggregation: weekly
+ redis_slot: package
+- name: i_package_container_deploy_token_push
+ category: container_packages
+ aggregation: weekly
+ redis_slot: package
+- name: i_package_container_user_delete
+ category: container_packages
+ aggregation: weekly
+ redis_slot: package
+- name: i_package_container_deploy_token_delete
+ category: container_packages
+ aggregation: weekly
+ redis_slot: package
+- name: i_package_container_user_pull
+ category: container_packages
+ aggregation: weekly
+ redis_slot: package
+- name: i_package_container_deploy_token_pull
+ category: container_packages
+ aggregation: weekly
+ redis_slot: package
+- name: i_package_tag_user_push
+ category: tag_packages
+ aggregation: weekly
+ redis_slot: package
+- name: i_package_tag_deploy_token_push
+ category: tag_packages
+ aggregation: weekly
+ redis_slot: package
+- name: i_package_tag_user_delete
+ category: tag_packages
+ aggregation: weekly
+ redis_slot: package
+- name: i_package_tag_deploy_token_delete
+ category: tag_packages
+ aggregation: weekly
+ redis_slot: package
+- name: i_package_tag_user_pull
+ category: tag_packages
+ aggregation: weekly
+ redis_slot: package
+- name: i_package_tag_deploy_token_pull
+ category: tag_packages
+ aggregation: weekly
+ redis_slot: package
diff --git a/lib/gitlab/usage_data_counters/static_site_editor_counter.rb b/lib/gitlab/usage_data_counters/static_site_editor_counter.rb
index 8886a106da8..3c5989d1e11 100644
--- a/lib/gitlab/usage_data_counters/static_site_editor_counter.rb
+++ b/lib/gitlab/usage_data_counters/static_site_editor_counter.rb
@@ -3,7 +3,7 @@
module Gitlab
module UsageDataCounters
class StaticSiteEditorCounter < BaseCounter
- KNOWN_EVENTS = %w[views].freeze
+ KNOWN_EVENTS = %w[views commits merge_requests].freeze
PREFIX = 'static_site_editor'
class << self
diff --git a/lib/gitlab/usage_data_counters/track_unique_events.rb b/lib/gitlab/usage_data_counters/track_unique_events.rb
index 7053744b665..95380ae0b1d 100644
--- a/lib/gitlab/usage_data_counters/track_unique_events.rb
+++ b/lib/gitlab/usage_data_counters/track_unique_events.rb
@@ -8,6 +8,9 @@ module Gitlab
PUSH_ACTION = :project_action
MERGE_REQUEST_ACTION = :merge_request_action
+ GIT_WRITE_ACTIONS = [WIKI_ACTION, DESIGN_ACTION, PUSH_ACTION].freeze
+ GIT_WRITE_ACTION = :git_write_action
+
ACTION_TRANSFORMATIONS = HashWithIndifferentAccess.new({
wiki: {
created: WIKI_ACTION,
@@ -41,6 +44,8 @@ module Gitlab
return unless Gitlab::UsageDataCounters::HLLRedisCounter.known_event?(transformed_action.to_s)
Gitlab::UsageDataCounters::HLLRedisCounter.track_event(author_id, transformed_action.to_s, time)
+
+ track_git_write_action(author_id, transformed_action, time)
end
def count_unique_events(event_action:, date_from:, date_to:)
@@ -64,6 +69,12 @@ module Gitlab
def valid_action?(action)
Event.actions.key?(action)
end
+
+ def track_git_write_action(author_id, transformed_action, time)
+ return unless GIT_WRITE_ACTIONS.include?(transformed_action)
+
+ Gitlab::UsageDataCounters::HLLRedisCounter.track_event(author_id, GIT_WRITE_ACTION, time)
+ end
end
end
end
diff --git a/lib/gitlab/usage_data_counters/web_ide_counter.rb b/lib/gitlab/usage_data_counters/web_ide_counter.rb
index 00fcd42a9af..9f2f4ac3971 100644
--- a/lib/gitlab/usage_data_counters/web_ide_counter.rb
+++ b/lib/gitlab/usage_data_counters/web_ide_counter.rb
@@ -2,54 +2,43 @@
module Gitlab
module UsageDataCounters
- class WebIdeCounter
- extend RedisCounter
- KNOWN_EVENTS = %i[commits views merge_requests previews terminals pipelines].freeze
+ class WebIdeCounter < BaseCounter
+ KNOWN_EVENTS = %w[commits views merge_requests previews terminals pipelines].freeze
PREFIX = 'web_ide'
class << self
def increment_commits_count
- increment(redis_key('commits'))
+ count('commits')
end
def increment_merge_requests_count
- increment(redis_key('merge_requests'))
+ count('merge_requests')
end
def increment_views_count
- increment(redis_key('views'))
+ count('views')
end
def increment_terminals_count
- increment(redis_key('terminals'))
+ count('terminals')
end
def increment_pipelines_count
- increment(redis_key('pipelines'))
+ count('pipelines')
end
def increment_previews_count
return unless Gitlab::CurrentSettings.web_ide_clientside_preview_enabled?
- increment(redis_key('previews'))
- end
-
- def totals
- KNOWN_EVENTS.map { |event| [counter_key(event), total_count(redis_key(event))] }.to_h
- end
-
- def fallback_totals
- KNOWN_EVENTS.map { |event| [counter_key(event), -1] }.to_h
+ count('previews')
end
private
def redis_key(event)
- "#{PREFIX}_#{event}_count".upcase
- end
+ require_known_event(event)
- def counter_key(event)
- "#{PREFIX}_#{event}".to_sym
+ "#{prefix}_#{event}_count".upcase
end
end
end
diff --git a/lib/gitlab/user_access.rb b/lib/gitlab/user_access.rb
index 1c6ddc2e70f..eec89e1ab72 100644
--- a/lib/gitlab/user_access.rb
+++ b/lib/gitlab/user_access.rb
@@ -90,9 +90,13 @@ module Gitlab
def can_collaborate?(ref)
assert_project!
+ can_push? || branch_allows_collaboration_for?(ref)
+ end
+
+ def branch_allows_collaboration_for?(ref)
# Checking for an internal project or group to prevent an infinite loop:
# https://gitlab.com/gitlab-org/gitlab/issues/36805
- can_push? || (!project.internal? && project.branch_allows_collaboration?(user, ref))
+ (!project.internal? && project.branch_allows_collaboration?(user, ref))
end
def permission_cache
diff --git a/lib/gitlab/webpack/dev_server_middleware.rb b/lib/gitlab/webpack/dev_server_middleware.rb
index 069e68e8d29..88f2a4455c6 100644
--- a/lib/gitlab/webpack/dev_server_middleware.rb
+++ b/lib/gitlab/webpack/dev_server_middleware.rb
@@ -16,14 +16,14 @@ module Gitlab
super(app, backend: "#{@proxy_scheme}://#{@proxy_host}:#{@proxy_port}", **opts)
end
- # disable SSL check since any cert used here will likely be self-signed
- def rewrite_env(env)
- env["rack.ssl_verify_none"] = true
- env
- end
-
def perform_request(env)
if @proxy_path && env['PATH_INFO'].start_with?("/#{@proxy_path}")
+ # disable SSL check since any cert used here will likely be self-signed
+ env['rack.ssl_verify_none'] = true
+
+ # ensure we pass the expected Host header so webpack-dev-server doesn't complain
+ env['HTTP_HOST'] = "#{@proxy_host}:#{@proxy_port}"
+
if relative_url_root = Rails.application.config.relative_url_root
env['SCRIPT_NAME'] = ""
env['REQUEST_PATH'].sub!(/\A#{Regexp.escape(relative_url_root)}/, '')
diff --git a/lib/gitlab/whats_new.rb b/lib/gitlab/whats_new.rb
index e95ace2c475..69ccb48c544 100644
--- a/lib/gitlab/whats_new.rb
+++ b/lib/gitlab/whats_new.rb
@@ -2,27 +2,39 @@
module Gitlab
module WhatsNew
- CACHE_DURATION = 1.day
+ CACHE_DURATION = 1.hour
WHATS_NEW_FILES_PATH = Rails.root.join('data', 'whats_new', '*.yml')
private
- def whats_new_most_recent_release_items
- Rails.cache.fetch('whats_new:release_items', expires_in: CACHE_DURATION) do
- file = File.read(most_recent_release_file_path)
+ def whats_new_release_items(page: 1)
+ Rails.cache.fetch(whats_new_items_cache_key(page), expires_in: CACHE_DURATION) do
+ index = page - 1
+ file_path = whats_new_file_paths[index]
+
+ next if file_path.nil?
+
+ file = File.read(file_path)
items = YAML.safe_load(file, permitted_classes: [Date])
items if items.is_a?(Array)
end
rescue => e
- Gitlab::ErrorTracking.track_exception(e, yaml_file_path: most_recent_release_file_path)
+ Gitlab::ErrorTracking.track_exception(e, page: page)
nil
end
- def most_recent_release_file_path
- @most_recent_release_file_path ||= Dir.glob(WHATS_NEW_FILES_PATH).max
+ def whats_new_file_paths
+ @whats_new_file_paths ||= Rails.cache.fetch('whats_new:file_paths', expires_in: CACHE_DURATION) do
+ Dir.glob(WHATS_NEW_FILES_PATH).sort.reverse
+ end
+ end
+
+ def whats_new_items_cache_key(page)
+ filename = /\d*\_\d*\_\d*/.match(whats_new_file_paths&.first)
+ "whats_new:release_items:file-#{filename}:page-#{page}"
end
end
end
diff --git a/lib/gitlab/with_feature_category.rb b/lib/gitlab/with_feature_category.rb
new file mode 100644
index 00000000000..65d21daf78a
--- /dev/null
+++ b/lib/gitlab/with_feature_category.rb
@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module WithFeatureCategory
+ extend ActiveSupport::Concern
+ include Gitlab::ClassAttributes
+
+ class_methods do
+ def feature_category(category, actions = [])
+ feature_category_configuration[category] ||= []
+ feature_category_configuration[category] += actions.map(&:to_s)
+
+ validate_config!(feature_category_configuration)
+ end
+
+ def feature_category_for_action(action)
+ category_config = feature_category_configuration.find do |_, actions|
+ actions.empty? || actions.include?(action)
+ end
+
+ category_config&.first || superclass_feature_category_for_action(action)
+ end
+
+ private
+
+ def validate_config!(config)
+ empty = config.find { |_, actions| actions.empty? }
+ duplicate_actions = config.values.map(&:uniq).flatten.group_by(&:itself).select { |_, v| v.count > 1 }.keys
+
+ if config.length > 1 && empty
+ raise ArgumentError, "#{empty.first} is defined for all actions, but other categories are set"
+ end
+
+ if duplicate_actions.any?
+ raise ArgumentError, "Actions have multiple feature categories: #{duplicate_actions.join(', ')}"
+ end
+ end
+
+ def feature_category_configuration
+ class_attributes[:feature_category_config] ||= {}
+ end
+
+ def superclass_feature_category_for_action(action)
+ return unless superclass.respond_to?(:feature_category_for_action)
+
+ superclass.feature_category_for_action(action)
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb
index eb780a2f7f6..8e7af8876a4 100644
--- a/lib/gitlab/workhorse.rb
+++ b/lib/gitlab/workhorse.rb
@@ -270,7 +270,7 @@ module Gitlab
prefix: metadata['ArchivePrefix'],
format: format,
path: path.presence || "",
- include_lfs_blobs: Feature.enabled?(:include_lfs_blobs_in_archive)
+ include_lfs_blobs: Feature.enabled?(:include_lfs_blobs_in_archive, default_enabled: true)
).to_proto
)
}
diff --git a/lib/quality/test_level.rb b/lib/quality/test_level.rb
index cd94efddc1e..b239b6812ca 100644
--- a/lib/quality/test_level.rb
+++ b/lib/quality/test_level.rb
@@ -12,6 +12,9 @@ module Quality
lib/gitlab/background_migration
lib/ee/gitlab/background_migration
],
+ frontend_fixture: %w[
+ frontend/fixtures
+ ],
unit: %w[
bin
channels
@@ -63,7 +66,7 @@ module Quality
end
def pattern(level)
- @patterns[level] ||= "#{prefix}spec/#{folders_pattern(level)}{,/**/}*_spec.rb"
+ @patterns[level] ||= "#{prefix}spec/#{folders_pattern(level)}{,/**/}*#{suffix(level)}"
end
def regexp(level)
@@ -76,6 +79,9 @@ module Quality
# spec/lib/gitlab/background_migration and tests under spec/lib are unit by default
when regexp(:migration), regexp(:background_migration)
:migration
+ # Detect frontend fixture before matching other unit tests
+ when regexp(:frontend_fixture)
+ :frontend_fixture
when regexp(:unit)
:unit
when regexp(:integration)
@@ -93,6 +99,15 @@ module Quality
private
+ def suffix(level)
+ case level
+ when :frontend_fixture
+ ".rb"
+ else
+ "_spec.rb"
+ end
+ end
+
def migration_and_background_migration_folders
TEST_LEVEL_FOLDERS.fetch(:migration) + TEST_LEVEL_FOLDERS.fetch(:background_migration)
end
diff --git a/lib/rouge/formatters/html_gitlab.rb b/lib/rouge/formatters/html_gitlab.rb
index e2a7d3ef5ba..e7e0d4e471f 100644
--- a/lib/rouge/formatters/html_gitlab.rb
+++ b/lib/rouge/formatters/html_gitlab.rb
@@ -20,7 +20,7 @@ module Rouge
is_first = false
yield %(<span id="LC#{@line_number}" class="line" lang="#{@tag}">)
- line.each { |token, value| yield span(token, value.chomp) }
+ line.each { |token, value| yield span(token, value.chomp! || value) }
yield %(</span>)
@line_number += 1
diff --git a/lib/rspec_flaky/flaky_examples_collection.rb b/lib/rspec_flaky/flaky_examples_collection.rb
index b86ec82bde6..acbfb411873 100644
--- a/lib/rspec_flaky/flaky_examples_collection.rb
+++ b/lib/rspec_flaky/flaky_examples_collection.rb
@@ -1,6 +1,7 @@
# frozen_string_literal: true
require 'active_support/hash_with_indifferent_access'
+require 'delegate'
require_relative 'flaky_example'
diff --git a/lib/support/nginx/gitlab b/lib/support/nginx/gitlab
index fc984d737d5..9eafa5ef008 100644
--- a/lib/support/nginx/gitlab
+++ b/lib/support/nginx/gitlab
@@ -18,7 +18,7 @@
upstream gitlab-workhorse {
# GitLab socket file,
- # for Omnibus this would be: unix:/var/opt/gitlab/gitlab-workhorse/socket
+ # for Omnibus this would be: unix:/var/opt/gitlab/gitlab-workhorse/sockets/socket
server unix:/home/git/gitlab/tmp/sockets/gitlab-workhorse.socket fail_timeout=0;
}
diff --git a/lib/support/nginx/gitlab-ssl b/lib/support/nginx/gitlab-ssl
index ba01e250bbb..ae5c88455e4 100644
--- a/lib/support/nginx/gitlab-ssl
+++ b/lib/support/nginx/gitlab-ssl
@@ -22,7 +22,7 @@
upstream gitlab-workhorse {
# GitLab socket file,
- # for Omnibus this would be: unix:/var/opt/gitlab/gitlab-workhorse/socket
+ # for Omnibus this would be: unix:/var/opt/gitlab/gitlab-workhorse/sockets/socket
server unix:/home/git/gitlab/tmp/sockets/gitlab-workhorse.socket fail_timeout=0;
}
diff --git a/lib/system_check/app/git_version_check.rb b/lib/system_check/app/git_version_check.rb
index 0709f783f09..0f4fbe4fba5 100644
--- a/lib/system_check/app/git_version_check.rb
+++ b/lib/system_check/app/git_version_check.rb
@@ -7,7 +7,7 @@ module SystemCheck
set_check_pass -> { "yes (#{self.current_version})" }
def self.required_version
- @required_version ||= Gitlab::VersionInfo.parse('2.24.0')
+ @required_version ||= Gitlab::VersionInfo.parse('2.29.0')
end
def self.current_version
diff --git a/lib/tasks/gitlab/gitaly.rake b/lib/tasks/gitlab/gitaly.rake
index 74cf3aad951..49d2d3d62a2 100644
--- a/lib/tasks/gitlab/gitaly.rake
+++ b/lib/tasks/gitlab/gitaly.rake
@@ -22,7 +22,7 @@ Usage: rake "gitlab:gitaly:install[/installation/dir,/storage/path]")
if Rails.env.test?
command.push(
'BUNDLE_FLAGS=--no-deployment',
- "BUNDLE_PATH=#{Bundler.bundle_path}")
+ "GEM_HOME=#{Bundler.bundle_path}")
end
storage_paths = { 'default' => args.storage_path }
diff --git a/lib/tasks/gitlab/packages/events.rake b/lib/tasks/gitlab/packages/events.rake
new file mode 100644
index 00000000000..3484b9b6072
--- /dev/null
+++ b/lib/tasks/gitlab/packages/events.rake
@@ -0,0 +1,40 @@
+require 'logger'
+
+desc "GitLab | Packages | Events | Generate hll counter events file for packages"
+namespace :gitlab do
+ namespace :packages do
+ namespace :events do
+ task generate: :environment do
+ logger = Logger.new(STDOUT)
+ logger.info('Building list of package events...')
+
+ path = File.join(File.dirname(::Gitlab::UsageDataCounters::HLLRedisCounter::KNOWN_EVENTS_PATH), 'package_events.yml')
+
+ File.open(path, "w") { |file| file << generate_unique_events_list.to_yaml }
+
+ logger.info("Events file `#{path}` generated successfully")
+ rescue => e
+ logger.error("Error building events list: #{e}")
+ end
+
+ def event_pairs
+ ::Packages::Event.event_types.keys.product(::Packages::Event.originator_types.keys)
+ end
+
+ def generate_unique_events_list
+ ::Packages::Event::EVENT_SCOPES.keys.each_with_object([]) do |event_scope, events|
+ event_pairs.each do |event_type, originator|
+ if name = ::Packages::Event.allowed_event_name(event_scope, event_type, originator)
+ events << {
+ "name" => name,
+ "category" => "#{event_scope}_packages",
+ "aggregation" => "weekly",
+ "redis_slot" => "package"
+ }
+ end
+ end
+ end
+ end
+ end
+ end
+end